def populate_one_search(search):
import logging
logging.basicConfig(filename='single_search.log',
format='%(asctime)s %(message)s',
filemode='w', # overwrite log file each run
level=logging.INFO)
logging.info('search model is %s' %search)
for region in search.regions.all():
logging.info('region: %s' %region)
search_url = make_url(search, region)
print search_url
if search_url:
logging.info('URL: %s' % search_url)
else:
logging.info('failed to build search_url')
postings = [] # a list of postings retrieved for this search_url
doc = feedparser.parse(search_url)
if doc:
logging.info('Document retrieved from craigslist with %s entries' % len(doc['entries']))
else:
logging.info('No document retrieved from craigslist')
for entry in doc.entries:
e = Entry(entry)
logging.info(e)
if not ('wanted' in e.title or 'wtb' in e.title):
try:
# see if posting exists already, and if so, update it
posting = Posting.objects.get(posting_url=e.posting_url)
posting.vehicle_year=e.vehicle_year
posting.vehicle_price=e.vehicle_price
posting.title=e.title
posting.body=e.body
posting.save()
logging.info('Posting found and updated')
except Posting.DoesNotExist:
posting = Posting(region=region,
posting_url=e.posting_url,
vehicle_year=e.vehicle_year,
vehicle_price=e.vehicle_price,
title=e.title,
body=e.body)
posting.save()
logging.info('New posting created')
finally:
if not posting:
logging.warning('Error: Posting not created')
posting.search.add(search)
logging.info(Posting.objects.count())
for obj in Posting.objects.all():
logging.info(obj)
posting = Posting.objects.get(posting_url=e.posting_url)
posting.vehicle_year=e.vehicle_year
posting.vehicle_price=e.vehicle_price
posting.title=e.title
posting.body=e.body
posting.save()
logging.debug('Posting found and updated')
postings_updated += 1
except Posting.DoesNotExist:
posting = Posting(region=region,
posting_url=e.posting_url,
vehicle_year=e.vehicle_year,
vehicle_price=e.vehicle_price,
title=e.title,
body=e.body)
posting.save()
logging.debug('New posting created')
new_postings += 1
finally:
posting.search.add(search)
for obj in Posting.objects.all():
logging.debug(obj)
if not posting:
logging.error('Error: Posting not created')
logging.info('Searches processed: {}'.format(searches_processed))
logging.info('Postings updated: {}'.format(postings_updated))
logging.info('New postings added: {}'.format(new_postings))
logging.info('Craigslist requests submitted: {}'.format(craigslist_requests))
logging.info('Run time: {}'.format(datetime.datetime.now() - start_time))