def setUp(self): self.scrapy = ScrapyCtl(accounts={}, loglevel='ERROR') self.command = Command() self.command.stdout = MagicMock() self.all_spiders = ['batoto', 'kissmanga', 'mangadex', 'mangafox', 'mangahere', 'mangareader', 'mangasee', 'unionmangas']
def handle(self, *args, **options): command = options['command'] loglevel = options['loglevel'] logger.setLevel(loglevel) if command == 'list': scrapy = ScrapyCtl(None, loglevel) self._list_spiders(scrapy) elif command == 'update-proxy': clean = options['clean'] self._update_proxy(clean) else: raise CommandError('Not valid command value.')
def handle(self, *args, **options): command = options['command'] accounts = self._get_accounts(options['accounts']) loglevel = options['loglevel'] dry_run = options['dry_run'] # Create the ScrapyCtl object to store the CrawlerProcess. scrapy = ScrapyCtl(accounts, loglevel) # Get the list of spiders names that we are going to work with spiders = self._get_spiders(scrapy, options['spiders']) if command == 'list': self.list_spiders(spiders) elif command == 'update-genres': scrapy.update_genres(spiders, dry_run) elif command == 'update-catalog': scrapy.update_catalog(spiders, dry_run) elif command == 'update-collection': manga = options['manga'] url = options['url'] manga = self._get_manga(spiders, manga, url) scrapy.update_collection(spiders, manga.name, manga.url, dry_run) elif command == 'update-latest': until = options['until'] if isinstance(until, basestring): until = datetime.datetime.strptime(until, '%d-%m-%Y').date() scrapy.update_latest(spiders, until, dry_run) elif command == 'search': manga = options['manga'] lang = options['lang'] details = options['details'] self.search(spiders, manga, lang, details) elif command == 'subscribe': user = options['user'] manga = options['manga'] url = options['url'] lang = options['lang'] issues_per_day = options['issues-per-day'] manga = self._get_manga(spiders, manga, url) self.subscribe(user, manga, lang, issues_per_day) elif command == 'send': issues = options['issues'] manga = options['manga'] url = options['url'] lang = options['lang'] username = options['user'] do_not_send = options['do-not-send'] # The URL can point to a manga or an issue, so we can use # safely in both calls manga = self._get_manga(spiders, manga, url) issues = self._get_issues(manga, issues, url, lang) user_profile = self._get_user_profile(username) self.send(issues, user_profile, accounts, loglevel, do_not_send) elif command == 'sendsub': utc_hour = timezone.now().hour user_profiles = [] ignore_time = options['ignore-time'] if options['user']: username = options['user'] user_profile = self._get_user_profile(username) hour = (user_profile.send_at - user_profile.time_zone) % 24 if ignore_time or utc_hour == hour: user_profiles = [user_profile] else: user_profiles = UserProfile.objects.filter( user__subscription__id__gt=0).distinct() if not ignore_time: user_profiles = user_profiles.annotate( hour=(24 + F('send_at') - F('time_zone')) % 24).filter(hour=utc_hour) do_not_send = options['do-not-send'] for user_profile in user_profiles: self.sendsub(user_profile, accounts, loglevel, do_not_send) elif command == 'retry': user_profiles = [] if options['user']: username = options['user'] user_profiles = [self._get_user_profile(username)] else: user_profiles = UserProfile.objects.filter( user__subscription__id__gt=0).distinct() do_not_send = options['do-not-send'] for user_profile in user_profiles: self.retry(user_profile, accounts, loglevel, do_not_send) else: raise CommandError('Not valid command value.') # Refresh the MATERIALIZED VIEW for full text search if command.startswith('update'): Manga.objects.refresh() # Print the SQL statistics in DEBUG mode if loglevel == 'DEBUG': queries = [ '[%s]: %s' % (q['time'], q['sql']) for q in connection.queries ] logger.debug('\n'.join(queries))
def setUp(self): self.scrapy = ScrapyCtl(accounts={}, loglevel='ERROR') self.command = Command() self.command.stdout = mock.MagicMock() self.all_spiders = ['mangareader', 'kissmanga', 'submangaorg', 'batoto', 'mangahere', 'mangafox']