def scrape_facility(facility_url):
    try:
        facility = {}
        facility['url'] = facility_url
        facility['_id'] = facility['url']
        facility_resp = requests.get(facility['url'])
        # easier to get name, location, type from the inspection page
        # TODO: refactor to pull from facility page to avoid skipping
        # facilities with no inspections
        inspection_links = pq(facility_resp.content).find(
            'div#inspectionHistory a')
        inspection_url = THD_ROOT + '/' + pq(inspection_links[0]).attr('href')
        time.sleep(SECONDS_THROTTLE)
        inspection_resp = requests.get(inspection_url)
        doc = pq(inspection_resp.content)
        facility['name'] = doc.find(
            'div#inspectionDetail h3').text()
        m = re.search('Location: (?P<location>.*)<br/>',
                     doc.find('div#inspectionDetail').html())
        facility['location'] = m.group('location').strip()
        info = doc.find('div#inspectionInfo tr td')
        for (counter, pair) in enumerate(grouper(info, 2)):
            value = pq(pair[1]).text()
            if counter == 1:
                facility['type'] = value
        if 'MAPQUEST_API_KEY' in os.environ:
            mq = geocoders.MapQuest(os.environ['MAPQUEST_API_KEY'])
            try:
                (place, (lat, long)) = mq.geocode(facility.get('location', ''))
                facility['latitude'] = lat
                facility['longitude'] = long
            except:
                logger.exception("Could not geocode location '%s' for %s" %
                                 (facility.get('location', ''),
                                  facility.get('name', '')))
        print "facility: %s" % facility
        facility['id'] = save_facility(facility)
        return facility, facility_resp
    except:
            logger.exception("Could not scrape facility %s" %
                             facility.get('url', ''))
def scrape_inspections(startrow):
    try:
        SEARCH_PARAMS.update({'startrow': startrow})
        search_resp = requests.post(THD_ROOT + '/index.cfm', data=SEARCH_PARAMS)
        facility_links = pq(search_resp.content).find(
            'div#searchResults a.resultMore')
        for f_link in facility_links:
            facility_url = THD_ROOT + '/' + pq(f_link).attr('href')
            time.sleep(SECONDS_THROTTLE)
            facility, facility_resp = scrape_facility(facility_url)

            inspection_links = pq(facility_resp.content).find(
                'div#inspectionHistory a')
            for i_link in inspection_links:
                inspection_url = THD_ROOT + '/' + pq(i_link).attr('href')
                inspection, inspection_resp = scrape_inspection(inspection_url,
                                                                facility)

                scrape_violations(inspection_resp.content, inspection)
                time.sleep(SECONDS_THROTTLE)

            if 'MAPQUEST_API_KEY' in os.environ:
                mq = geocoders.MapQuest(os.environ['MAPQUEST_API_KEY'])
                try:
                    (place, (lat, long)) = mq.geocode(facility.get('location', ''))
                    facility['latitude'] = lat
                    facility['longitude'] = long
                except:
                    logger.exception("Could not geocode location '%s' for %s" %
                                     (facility.get('location', ''),
                                      facility.get('name', '')))

            print "facility: %s" % facility
            facility['id'] = save_facility(facility)
    except:
        logger.exception("Could not scrape at startrow: %s" % startrow)