def city_spider(request):
response = HttpResponse()
response['Cache-Control'] = 'no-cache'
html_header = loader.get_template('header.html').render(Context())
html_footer = loader.get_template('footer.html').render(Context())
tpl_fetch = Template('<p>Fetching from {{site}} : {{url}} ... ')
tpl_store = Template('{{n}} cities.</p>')
tpl_total = Template('<p>总共抓取 {{total}} 条记录</p>')
response.write(html_header)
query=models.Site.objects.all()
total=0
for site in query:
factory=SpiderFactory()
engine = CitySpider()
spider = factory.new_city_spider(site.site)
engine.fetch_and_parse(spider, site.url)
response.write(tpl_fetch.render(Context({'url':site.url, 'site':site.site})))
n=engine.store_result(spider, site.site)
response.write(tpl_store.render(Context({'n':n})))
total += n
print n
response.write(tpl_total.render(Context({'total':total})))
print total
response.write(html_footer)
return response
def deal_spider_site_city(request, site, city):
"""
"""
response = HttpResponse()
response['Cache-Control'] = 'no-cache'
html_header = loader.get_template('header.html').render(Context())
html_footer = loader.get_template('footer.html').render(Context())
tpl_fetch = Template('<p>Fetching from {{site}} @ {{city}} : {{url}} ... ')
tpl_store = Template('{{n}} deals.</p>')
tpl_fetch_fail = Template(' fail.</p>')
tpl_total = Template('<p>总共抓取 {{total}} 条记录</p>')
response.write(html_header)
query=None
if (site != None) and (city != None):
query = models.SiteCity.objects.filter(site=site, city=city)
elif (site != None) and (city is None):
query = models.SiteCity.objects.filter(site=site)
elif (site is None) and (city != None):
query = models.SiteCity.objects.filter(city=city)
else:
query = models.SiteCity.objects.all()
total = 0
for sc in query:
factory=SpiderFactory()
engine = DealSpider()
spider = factory.new_deal_spider(sc.site)
response.write(tpl_fetch.render(Context({'url': sc.url, 'site':sc.site, 'city':sc.city})));
ok = engine.fetch_and_parse(spider, sc.url)
if ok:
n = engine.store_result(spider, sc.site, sc.city)
response.write(tpl_store.render(Context({'n': n})))
total += n
else:
response.write(tpl_fetch_fail.render(Context({})))
response.write(tpl_total.render(Context({'total':total})))
response.write(html_footer)
return response
