def setUp(self):
self.scrapy = ScrapyCtl(accounts={}, loglevel='ERROR')
self.command = Command()
self.command.stdout = MagicMock()
self.all_spiders = ['batoto', 'kissmanga', 'mangadex', 'mangafox',
'mangahere', 'mangareader', 'mangasee',
'unionmangas']
def handle(self, *args, **options):
command = options['command']
loglevel = options['loglevel']
logger.setLevel(loglevel)
if command == 'list':
scrapy = ScrapyCtl(None, loglevel)
self._list_spiders(scrapy)
elif command == 'update-proxy':
clean = options['clean']
self._update_proxy(clean)
else:
raise CommandError('Not valid command value.')
def handle(self, *args, **options):
command = options['command']
accounts = self._get_accounts(options['accounts'])
loglevel = options['loglevel']
dry_run = options['dry_run']
# Create the ScrapyCtl object to store the CrawlerProcess.
scrapy = ScrapyCtl(accounts, loglevel)
# Get the list of spiders names that we are going to work with
spiders = self._get_spiders(scrapy, options['spiders'])
if command == 'list':
self.list_spiders(spiders)
elif command == 'update-genres':
scrapy.update_genres(spiders, dry_run)
elif command == 'update-catalog':
scrapy.update_catalog(spiders, dry_run)
elif command == 'update-collection':
manga = options['manga']
url = options['url']
manga = self._get_manga(spiders, manga, url)
scrapy.update_collection(spiders, manga.name, manga.url, dry_run)
elif command == 'update-latest':
until = options['until']
if isinstance(until, basestring):
until = datetime.datetime.strptime(until, '%d-%m-%Y').date()
scrapy.update_latest(spiders, until, dry_run)
elif command == 'search':
manga = options['manga']
lang = options['lang']
details = options['details']
self.search(spiders, manga, lang, details)
elif command == 'subscribe':
user = options['user']
manga = options['manga']
url = options['url']
lang = options['lang']
issues_per_day = options['issues-per-day']
manga = self._get_manga(spiders, manga, url)
self.subscribe(user, manga, lang, issues_per_day)
elif command == 'send':
issues = options['issues']
manga = options['manga']
url = options['url']
lang = options['lang']
username = options['user']
do_not_send = options['do-not-send']
# The URL can point to a manga or an issue, so we can use
# safely in both calls
manga = self._get_manga(spiders, manga, url)
issues = self._get_issues(manga, issues, url, lang)
user_profile = self._get_user_profile(username)
self.send(issues, user_profile, accounts, loglevel, do_not_send)
elif command == 'sendsub':
utc_hour = timezone.now().hour
user_profiles = []
ignore_time = options['ignore-time']
if options['user']:
username = options['user']
user_profile = self._get_user_profile(username)
hour = (user_profile.send_at - user_profile.time_zone) % 24
if ignore_time or utc_hour == hour:
user_profiles = [user_profile]
else:
user_profiles = UserProfile.objects.filter(
user__subscription__id__gt=0).distinct()
if not ignore_time:
user_profiles = user_profiles.annotate(
hour=(24 + F('send_at') - F('time_zone')) %
24).filter(hour=utc_hour)
do_not_send = options['do-not-send']
for user_profile in user_profiles:
self.sendsub(user_profile, accounts, loglevel, do_not_send)
elif command == 'retry':
user_profiles = []
if options['user']:
username = options['user']
user_profiles = [self._get_user_profile(username)]
else:
user_profiles = UserProfile.objects.filter(
user__subscription__id__gt=0).distinct()
do_not_send = options['do-not-send']
for user_profile in user_profiles:
self.retry(user_profile, accounts, loglevel, do_not_send)
else:
raise CommandError('Not valid command value.')
# Refresh the MATERIALIZED VIEW for full text search
if command.startswith('update'):
Manga.objects.refresh()
# Print the SQL statistics in DEBUG mode
if loglevel == 'DEBUG':
queries = [
'[%s]: %s' % (q['time'], q['sql']) for q in connection.queries
]
logger.debug('\n'.join(queries))
def setUp(self):
self.scrapy = ScrapyCtl(accounts={}, loglevel='ERROR')
self.command = Command()
self.command.stdout = mock.MagicMock()
self.all_spiders = ['mangareader', 'kissmanga', 'submangaorg',
'batoto', 'mangahere', 'mangafox']