예제 #1
0
def populate_one_search(search):
    
    import logging
    logging.basicConfig(filename='single_search.log',
                    format='%(asctime)s %(message)s',
                    filemode='w',  # overwrite log file each run 
                    level=logging.INFO)
    
    logging.info('search model is %s' %search)
    for region in search.regions.all():
        logging.info('region: %s' %region)
        search_url = make_url(search, region)
        print search_url
        if search_url:
            logging.info('URL: %s' % search_url)
        else:
            logging.info('failed to build search_url')
        postings = []  # a list of postings retrieved for this search_url
        doc = feedparser.parse(search_url)
        if doc:
            logging.info('Document retrieved from craigslist with %s entries' % len(doc['entries']))
        else:
            logging.info('No document retrieved from craigslist')
        for entry in doc.entries:
            e = Entry(entry)
            logging.info(e)
            if not ('wanted' in e.title or 'wtb' in e.title):
                try:
                    # see if posting exists already, and if so, update it
                    posting = Posting.objects.get(posting_url=e.posting_url)
                    posting.vehicle_year=e.vehicle_year
                    posting.vehicle_price=e.vehicle_price
                    posting.title=e.title
                    posting.body=e.body
                    posting.save()
                    logging.info('Posting found and updated')
                except Posting.DoesNotExist:
                    posting = Posting(region=region,
                                      posting_url=e.posting_url,
                                      vehicle_year=e.vehicle_year,
                                      vehicle_price=e.vehicle_price,
                                      title=e.title,
                                    body=e.body)
                    posting.save()
                    logging.info('New posting created')
                finally:
                    if not posting:
                        logging.warning('Error: Posting not created')
                    posting.search.add(search)
                    logging.info(Posting.objects.count())
                    for obj in Posting.objects.all():
                        logging.info(obj)
예제 #2
0
                logging.debug(e)
                if not ('wanted' in e.title or 'wtb' in e.title):
                    try:
                        # see if posting exists already, and if so, update it
                        posting = Posting.objects.get(posting_url=e.posting_url)
                        posting.vehicle_year=e.vehicle_year
                        posting.vehicle_price=e.vehicle_price
                        posting.title=e.title
                        posting.body=e.body
                        posting.save()
                        logging.debug('Posting found and updated')
                        postings_updated += 1
                    except Posting.DoesNotExist:
                        posting = Posting(region=region,
                                          posting_url=e.posting_url,
                                          vehicle_year=e.vehicle_year,
                                          vehicle_price=e.vehicle_price,
                                          title=e.title,
                                          body=e.body)
                        posting.save()
                        logging.debug('New posting created')
                        new_postings += 1
                    finally:
                        posting.search.add(search)
                        for obj in Posting.objects.all():
                            logging.debug(obj)
                    if not posting:
                        logging.error('Error: Posting not created')

    logging.info('Searches processed: {}'.format(searches_processed))
    logging.info('Postings updated: {}'.format(postings_updated))
    logging.info('New postings added: {}'.format(new_postings))