def add_item(result):
if result['target']:
target = result['target']['url']
else:
target = "No target"
e = Muni.create(
zip = result['zip_code']
)
return e
def scrape():
# Where did we leave off?
last_muni = Muni.select().order_by(Muni.zip_code.asc())
last_muni = last_muni.execute()
last_zip = 210
try:
for l in last_muni:
last_zip = l.zip_code
except Exception, e:
print e
last_zip = 210 #00210 is the lowest zip code. portsmouth, new-hampshire
def setup_db():
database.connect()
try:
Muni.create_table() # only needed once
except Exception, e:
pass
def get_all_items():
for m in Muni.select().where(Muni.success == True):
print m.title + " " + str(m.tel)
def item_in_db(zip_code, org):
try:
Muni.get(Muni.zip_code == id, Muni.org == org)
return True
except Exception, e:
return False
def scrape_item_by_city_state_and_org(city, state, org, zipcode=None):
time.sleep(3)
try:
print "Scraping " + org + " in " + city + ", " + state
google_places = GooglePlaces(random.choice(GOOGLE_KEYS))
results = google_places.query(
location= city + ", " + state,
keyword=org,
)
place = results.places[0]
place.get_details()
country = 'USA'
state_code = state
state_full = code_to_state[state]
county = ''
address = place.formatted_address
url = place.website
tel = place.local_phone_number
international_tel = place.international_phone_number
name = place.name
lat = place.geo_location['lat']
lon = place.geo_location['lng']
print name
print country
print state_code
print state_full
print address
print url
print tel
print lat
print lon
print '\n\n'
m = Muni.create(
zip_code = zipcode,
zip_code_string = zipcode,
org = org,
city = city,
address = address,
county = county,
country=country,
url = url,
tel=tel,
international_tel=international_tel,
state_code = state_code,
state = state_full,
success = True,
title = name,
lat = lat,
lon = lon,
email = '',
date_created = datetime.datetime.utcnow(),
last_modified = datetime.datetime.utcnow()
)
return m
except Exception, e:
print e
# SAVE FAILURE
m = Muni.create(
zip_code = zipcode,
zip_code_string = zipcode,
org = org,
city = city,
state_code = state,
state = code_to_state[state],
success = False,
country='USA',
email = '',
lat = '0',
lon = '0',
address = '',
title = '',
tel = '',
international_tel = '',
url = '',
date_created = datetime.datetime.utcnow(),
last_modified = datetime.datetime.utcnow(),
)
return m