def run_checkers(): t = TaskUtils() kwargs = { 'check_me':True, } args=(Q(id__gt = 100),) t.run_checkers(BBCNews, 'news_website__scraper', checker_runtime, 'news_checker', *args, **kwargs)
def run_checkers(): t = TaskUtils() kwargs = { 'check_me':True, } args=(Q(id__gt = 100),) t.run_checkers(JobAd, 'job_website__scraper', checker_runtime, 'job_checker', *args, **kwargs)
def run_spiders(): t = TaskUtils() # Django field lookup keyword arguments to specify which ref objects(JobWebsite) # to run kwargs = { 'scrape_me':True, } args=(Q(name = 'BBC'),) t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'news_spider', *args, **kwargs)
def run_spiders(): t = TaskUtils() #Optional: Django field lookup keyword arguments to specify which reference objects (NewsWebsite) #to use for spider runs, e.g.: kwargs = {'status': '1'} #Optional as well: For more complex lookups you can pass Q objects vi args argument args = () t.run_spiders(Source, 'scraper', 'scraper_runtime', 'article_spider', *args, **kwargs)
def run_checkers(): t = TaskUtils() #Optional: Django field lookup keyword arguments to specify which reference objects (Article) #to use for checker runs, e.g.: kwargs = { 'check_me': True, #imaginary, model Article hat no attribute 'check_me' in example } #Optional as well: For more complex lookups you can pass Q objects vi args argument args = (Q(id__gt=100),) t.run_checkers(Article, 'news_website__scraper', 'checker_runtime', 'article_checker', *args, **kwargs)
def run_spiders(): t = TaskUtils() #Optional as well: For more complex lookups you can pass Q objects vi args argument kwargs = { 'id': 2, #imaginary, model NewsWebsite hat no attribute 'scrape_me' in example 'do_action': 'yes', 'class': 'googlespider', 'itemc': 'GoogleItem' } t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'article_spider')
def run_spiders(): t = TaskUtils() #Optional: Django field lookup keyword arguments to specify which reference objects (NewsWebsite) #to use for spider runs, e.g.: kwargs = { 'status' : '1' } #Optional as well: For more complex lookups you can pass Q objects vi args argument args = () t.run_spiders(Source, 'scraper', 'scraper_runtime', 'article_spider', *args, **kwargs)
def run_spiders(): t = TaskUtils() #Optional: Django field lookup keyword arguments to specify which reference objects (NewsWebsite) #to use for spider runs, e.g.: #kwargs = { # 'scrape_me': True, #imaginary, model NewsWebsite hat no attribute 'scrape_me' in example #} #Optional as well: For more complex lookups you can pass Q objects vi args argument args = (Q(name='Wikinews'),) t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'article_spider')
def run_spiders(): t = TaskUtils() #Optional: Django field lookup keyword arguments to specify which reference objects (NewsWebsite) #to use for spider runs, e.g.: #kwargs = { # 'scrape_me': True, #imaginary, model NewsWebsite hat no attribute 'scrape_me' in example #} #Optional as well: For more complex lookups you can pass Q objects vi args argument args = (Q(name='GlobalBank'), ) t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'FirstSpider', *args) #, **kwargs)
def run_checkers(): t = TaskUtils() #Optional: Django field lookup keyword arguments to specify which reference objects (Article) #to use for checker runs, e.g.: kwargs = { 'check_me': True, #imaginary, model Article hat no attribute 'check_me' in example } #Optional as well: For more complex lookups you can pass Q objects vi args argument args = (Q(id__gt=100), ) t.run_checkers(Article, 'news_website__scraper', 'checker_runtime', 'article_checker', *args, **kwargs)
def search_it(request, template_name='core/home.html'): search = '' #add in the slug maker form = SearchForm(request.POST) # A form bound to the POST data if form.is_valid(): # All validation rules pass form.process() search = form.cleaned_data['title'] slug = slugify(search) try: e = Den.objects.get(slug=slug) except ObjectDoesNotExist: den = form.save(commit=False) den.save() #Spider Tasks t = TaskUtils() t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'article_spider', search) return redirect('den/' + slug) return render(request, template_name, {'form': form,})
def run_spiders(): t = TaskUtils() #Optional: Django field lookup keyword arguments to specify which reference objects (Source) #to use for spider runs, e.g.: kwargs = { } #Optional as well: For more complex lookups you can pass Q objects vi args argument args = () util.fetch_token() t.run_spiders(Source, 'scraper', 'scraper_runtime', 'ii_spider', *args, **kwargs) # @task() # def run_checkers(): # t = TaskUtils() # #Optional: Django field lookup keyword arguments to specify which reference objects (Article) # #to use for checker runs, e.g.: # kwargs = { # 'check_me': True, #imaginary, model Article hat no attribute 'check_me' in example # } # #Optional as well: For more complex lookups you can pass Q objects vi args argument # args = (Q(id__gt=100),) # t.run_checkers(Article, 'news_website__scraper', 'checker_runtime', 'article_checker', *args, **kwargs)
def run_checkers(): t = TaskUtils() t.run_checkers(Product, 'source__scraper', 'checker_runtime', 'product_checker')
def run_checker_tests(): t = TaskUtils() t.run_checker_tests()
def run_spiders(): t = TaskUtils() t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'article_spider')
def run_checkers(): t = TaskUtils() t.run_checkers(Article, 'news_website__scraper', 'checker_runtime', 'article_checker')
def run_checkers(): t = TaskUtils() t.run_checkers(Article, 'checker_runtime', 'article_checker')
def run_spiders(): t = TaskUtils() t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'product_spider')
def run_checkers(): t = TaskUtils() args = (Q(id__gt=100),) t.run_checkers(Data, 'news_website__scraper', 'checker_runtime', 'whale_checker', *args, **kwargs)
def run_spiders(): t = TaskUtils() args = (Q(name='egg'),) t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'whale', *args, **kwargs)
def run_spiders(): print('running spiders') t = TaskUtils() t.run_spiders(Casino, 'scraper', 'scraper_runtime', 'promotion_spider')
def run_spiders(): t = TaskUtils() t.run_spiders(InsuranceWebsite, 'scraper', 'scraper_runtime', 'loan_spider')
def run_spiders(): t = TaskUtils() t.run_spiders(LoanScraper, 'scraper', 'scraper_runtime', 'loan_spider')
def run_checkers(): t = TaskUtils() t.run_checkers(VisaInformation, 'checker_runtime', 'article_checker')
def run_checkers(): t = TaskUtils() t.run_checkers(main_models.News, 'news_website__scraper', 'checker_runtime', 'news_checker', *args, **kwargs)
def run_spiders(): t = TaskUtils() t.run_spiders(main_models.NewsWebsite, 'scraper', 'scraper_runtime', 'news_spider')
def run_spiders(): t = TaskUtils() t.run_spiders(Basicinfo, 'scraper_runtime', 'article_spider')
def run_checkers(): t = TaskUtils() t.run_checkers(Item, 'website__scraper', 'checker_runtime', 'fly_checker')
def run_checkers(): t = TaskUtils() t.run_checkers(Loan, 'loan_website__scraper', 'checker_runtime', 'loan_checker')
def run_checkers(): t = TaskUtils() kwargs = {} args = (Q(), ) t.run_checkers(Address, 'source__scraper', 'checker_runtime', 'hydro_quebec_checker', *args, **kwargs)
def run_spiders(): t = TaskUtils() t.run_spiders(Source, 'scraper', 'scraper_runtime', 'product_spider')
def run_spiders(): t = TaskUtils() kwargs = {} args = (Q(), ) t.run_spiders(AddressWebsite, 'scraper', 'scraper_runtime', 'hydro_quebec', *args, **kwargs)
def run_spiders(): t = TaskUtils() t.run_spiders(Website, 'scraper', 'scraper_runtime', 'fly_spider')
def run_spiders(): t = TaskUtils() t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'dmoz')
def run_spiders(): t = TaskUtils() t.run_spiders(Website, 'scraper', 'scraper_runtime', 'article_spider')