def get_cities():
localities = []
cities = []
locality_list = scrapertools.get_content(BASE_URL + LOCALITY_LIST_URL)
locality_a_tags = locality_list.body.div.img.find_all_next('a')
locality_urls = list(set(locality['href'][1:] for locality in locality_a_tags))
for locality_url in locality_urls:
locality_main = scrapertools.get_content(BASE_URL + locality_url + LOCALITY_MAIN_URL)
locality_name = locality_main.find('h3').string
localities.append({
'name': locality_name,
'url': locality_url
})
print 'Locality: ' + locality_name
for locality in localities:
city_list = scrapertools.get_content(BASE_URL + locality['url'] + CITY_LIST_URL)
city_a_tags = city_list.find_all('a')
for city in city_a_tags:
name = str(city.string).strip()
print 'Loading {0} ({1})'.format(name, locality['name'])
cities.append({
'name': name,
'locality': locality['name'],
'baseUrl': city['href'][:city['href'].find('Food-List-ByName')],
'establishmentUrl': city['href'].replace('Count=30', '')
})
return cities
def get_establishments(city):
establishments_found = []
start = 1
count = 1000
more = True
while more:
establishment_list = scrapertools.get_content(BASE_URL + city['establishmentUrl']+'&start='+str(start)+'&count='+str(count))
if establishment_list.find(text='No documents found') is not None:
more = False
continue
start += count
establishments = establishment_list.find_all('tr')
for establishment in establishments:
details = establishment.find_all('td')
if len(details) == 4 and details[0] is not None and details[0].a is not None:
date = (None if scrapertools.get_text(details[3]) is None
else datetime.strptime(scrapertools.get_text(details[3]), '%d-%b-%Y'))
# Removes establishment IDs and newlines from the establishment name
name = re.sub('(#|\()(\s)*([0-9][0-9]-[0-9][0-9][0-9][0-9])(\))?', '', scrapertools.get_text(details[0]))
name = re.sub('(\n)+', ' ', name)
address = scrapertools.get_text(details[2])
slug_id = slugify(name.strip() + ' ' + address)
establishments_found.append({
'slug': slug_id,
'name': name.strip(),
'url': details[0].a['href'],
'address': address,
'locality': city['locality'],
'last_inspection_date': date,
'baseUrl': city['baseUrl'],
'inserted': datetime.now()
})
return establishments_found
def get_establishments(city):
establishments_found = []
start = 1
count = 1000
more = True
while more:
establishment_list = scrapertools.get_content(BASE_URL + city['establishmentUrl']+'&start='+str(start)+'&count='+str(count))
if establishment_list.find(text='No documents found') is not None:
more = False
continue
start += count
establishments = establishment_list.find_all('tr')
for establishment in establishments:
details = establishment.find_all('td')
if len(details) == 4 and details[0] is not None and details[0].a is not None:
date = (None if scrapertools.get_text(details[3]) is None
else datetime.strptime(scrapertools.get_text(details[3]), '%d-%b-%Y'))
establishments_found.append({
'name': scrapertools.get_text(details[0]),
'url': details[0].a['href'],
'address': scrapertools.get_text(details[2]),
'locality': city['locality'],
'last_inspection_date': date
})
return establishments_found
def get_establishment_details(establishment):
establishment_details = scrapertools.get_content(BASE_URL + establishment['url'])
establishment['city'] = re.sub('(<(/)?br>)|(\r)|(\n)',
'',
str(establishment_details.find(text=re.compile('Facility Location')).parent.next_sibling.find('br')))
establishment['type'] = establishment_details.find(text=re.compile("Facility Type")).parent.next_sibling.string
return establishment
def get_cities():
cities_found = []
city_names = []
locality_list = scrapertools.get_content(BASE_URL + LOCALITY_LIST_URL)
localities = locality_list.body.div.img.find_all_next('a')
for locality in localities:
city_list = scrapertools.get_content(BASE_URL + locality['href'] + CITY_LIST_URL)
cities = city_list.find_all('a')
for city in cities:
name = str(city.string).strip()
if name not in city_names:
print 'Loading ' + name
city_names.append(name)
cities_found.append({
'name': name,
'locality': str(locality.string).strip(),
'baseUrl': city['href'][:city['href'].find('Food-List-ByName')],
'establishmentUrl': city['href'].replace('Count=30', '')
})
return cities_found
def get_establishment_details(establishment):
establishment_details = scrapertools.get_content(BASE_URL + establishment['url'])
for linebreak in establishment_details.find_all('br'):
linebreak.extract()
try:
establishment['city'] = establishment_details.find(text=re.compile('^Facility Location')).parent.next_sibling.next_sibling.string
except:
establishment['city'] = 'Unknown'
try:
establishment['type'] = establishment_details.find(text=re.compile('^Facility Type')).parent.next_sibling.string
except:
establishment['type'] = 'Unknown'
return establishment
def get_inspections(establishment, city_url):
inspections_found = []
establishment_details = scrapertools.get_content(BASE_URL + establishment['url'])
inspections = establishment_details.find_all(text='Inspection Type')[0].find_parent('tr').find_all_next('tr')
for inspection in inspections:
details = inspection.find_all('td')
if details[0].a is None:
continue
violations = get_violations(BASE_URL + city_url + '/' + details[0].a['href'])
inspections_found.append({
'type': scrapertools.get_text(details[0]),
'date': datetime.strptime(scrapertools.get_text(details[1]), '%d-%b-%Y'),
'violations': violations
})
return inspections_found
def get_violations(inspection_details_url):
violations_found = []
inspection_details = scrapertools.get_content(inspection_details_url)
violations = inspection_details.find(text='Violations:').find_next('table')
if violations is None:
return []
violations = violations.find('tr').find_next_siblings()
for violation in violations:
details = violation.find_all('td')
violations_found.append({
'code': scrapertools.get_all_text(details[0])[0],
'repeat': any(['Repeat' in tag.string for tag in details[1].contents if tag.name == 'b']),
'critical': any(['Critical' in tag.string for tag in details[1].contents if tag.name == 'b']),
'corrected': any(['Corrected' in tag.string for tag in details[1].contents if tag.name == 'b']),
'correction': ' '.join([tag.string for tag in details[1].contents if tag.name == 'font']).strip(),
'observation': ' '.join([tag.string for tag in details[1].contents if tag.name == None]).strip()
})
return violations_found