def run_spiders():
t = TaskUtils()
#Optional: Django field lookup keyword arguments to specify which reference objects (NewsWebsite)
#to use for spider runs, e.g.:
kwargs = {'status': '1'}
#Optional as well: For more complex lookups you can pass Q objects vi args argument
args = ()
t.run_spiders(Source, 'scraper', 'scraper_runtime', 'article_spider',
*args, **kwargs)
def run_spiders():
t = TaskUtils()
# Django field lookup keyword arguments to specify which ref objects(JobWebsite)
# to run
kwargs = {
'scrape_me':True,
}
args=(Q(name = 'BBC'),)
t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'news_spider', *args, **kwargs)
def run_spiders():
t = TaskUtils()
#Optional: Django field lookup keyword arguments to specify which reference objects (NewsWebsite)
#to use for spider runs, e.g.:
#kwargs = {
# 'scrape_me': True, #imaginary, model NewsWebsite hat no attribute 'scrape_me' in example
#}
#Optional as well: For more complex lookups you can pass Q objects vi args argument
args = (Q(name='Wikinews'),)
t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'article_spider')
def run_spiders():
t = TaskUtils()
#Optional as well: For more complex lookups you can pass Q objects vi args argument
kwargs = {
'id': 2, #imaginary, model NewsWebsite hat no attribute 'scrape_me' in example
'do_action': 'yes',
'class': 'googlespider',
'itemc': 'GoogleItem'
}
t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'article_spider')
def run_spiders():
t = TaskUtils()
#Optional: Django field lookup keyword arguments to specify which reference objects (NewsWebsite)
#to use for spider runs, e.g.:
kwargs = {
'status' : '1'
}
#Optional as well: For more complex lookups you can pass Q objects vi args argument
args = ()
t.run_spiders(Source, 'scraper', 'scraper_runtime', 'article_spider', *args, **kwargs)
def run_spiders():
t = TaskUtils()
#Optional: Django field lookup keyword arguments to specify which reference objects (NewsWebsite)
#to use for spider runs, e.g.:
#kwargs = {
# 'scrape_me': True, #imaginary, model NewsWebsite hat no attribute 'scrape_me' in example
#}
#Optional as well: For more complex lookups you can pass Q objects vi args argument
args = (Q(name='GlobalBank'), )
t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'FirstSpider',
*args) #, **kwargs)
def search_it(request, template_name='core/home.html'):
search = '' #add in the slug maker
form = SearchForm(request.POST) # A form bound to the POST data
if form.is_valid(): # All validation rules pass
form.process()
search = form.cleaned_data['title']
slug = slugify(search)
try:
e = Den.objects.get(slug=slug)
except ObjectDoesNotExist:
den = form.save(commit=False)
den.save()
#Spider Tasks
t = TaskUtils()
t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'article_spider', search)
return redirect('den/' + slug)
return render(request, template_name, {'form': form,})
def run_spiders():
t = TaskUtils()
#Optional: Django field lookup keyword arguments to specify which reference objects (Source)
#to use for spider runs, e.g.:
kwargs = {
}
#Optional as well: For more complex lookups you can pass Q objects vi args argument
args = ()
util.fetch_token()
t.run_spiders(Source, 'scraper', 'scraper_runtime', 'ii_spider', *args, **kwargs)
# @task()
# def run_checkers():
# t = TaskUtils()
# #Optional: Django field lookup keyword arguments to specify which reference objects (Article)
# #to use for checker runs, e.g.:
# kwargs = {
# 'check_me': True, #imaginary, model Article hat no attribute 'check_me' in example
# }
# #Optional as well: For more complex lookups you can pass Q objects vi args argument
# args = (Q(id__gt=100),)
# t.run_checkers(Article, 'news_website__scraper', 'checker_runtime', 'article_checker', *args, **kwargs)
def run_spiders():
t = TaskUtils()
t.run_spiders(main_models.NewsWebsite, 'scraper',
'scraper_runtime', 'news_spider')
def run_spiders():
t = TaskUtils()
t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'article_spider')
def run_spiders():
t = TaskUtils()
t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'dmoz')
def run_spiders():
t = TaskUtils()
t.run_spiders(Website, 'scraper', 'scraper_runtime', 'fly_spider')
def run_spiders():
t = TaskUtils()
t.run_spiders(LoanScraper, 'scraper', 'scraper_runtime', 'loan_spider')
def run_spiders():
print('running spiders')
t = TaskUtils()
t.run_spiders(Casino, 'scraper', 'scraper_runtime', 'promotion_spider')
def run_spiders():
t = TaskUtils()
args = (Q(name='egg'),)
t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'whale', *args, **kwargs)
def run_spiders():
t = TaskUtils()
t.run_spiders(NewsWebsite, 'scraper', 'scraper_runtime', 'product_spider')
def run_spiders():
t = TaskUtils()
t.run_spiders(InsuranceWebsite, 'scraper', 'scraper_runtime', 'loan_spider')
def run_spiders():
t = TaskUtils()
t.run_spiders(Source, 'scraper', 'scraper_runtime', 'product_spider')
def run_spiders():
t = TaskUtils()
t.run_spiders(Basicinfo, 'scraper_runtime', 'article_spider')
def run_spiders():
t = TaskUtils()
t.run_spiders(Website, 'scraper', 'scraper_runtime', 'article_spider')
def run_spiders():
t = TaskUtils()
kwargs = {}
args = (Q(), )
t.run_spiders(AddressWebsite, 'scraper', 'scraper_runtime', 'hydro_quebec',
*args, **kwargs)
