def test_load_items():
database = Database()
database.delete_item('members', 'testid1')
database.delete_item('members', 'testid2')
columns = database.get_columns('members')
item1 = {x: None for x in columns}
item1['id'] = 'testid1'
item2 = {x: None for x in columns}
item2['id'] = 'testid2'
items = [item1, item2]
database.load_items(items, 'members')
item1_ = database.get_item('members', 'testid1')
assert item1_['id'] == 'testid1'
item2_ = database.get_item('members', 'testid2')
assert item2_['id'] == 'testid2'
database.delete_item('members', 'testid1')
item1_ = database.get_item('members', 'testid1')
assert item1_ == None
database.delete_item('members', 'testid2')
item2_ = database.get_item('members', 'testid2')
assert item2_ == None
class EventbriteLoader(object):
"""Loads data from Eventbrite into Postgres """
def __init__(self, eventbrite_org, database=None):
daiquiri.setup(level=logging.INFO)
self.logger = daiquiri.getLogger(__name__)
self.database = Database(database=database)
self.eventbrite = Eventbrite()
self.eventbrite_org = eventbrite_org
def run(self, test=False):
""" Runs the data load process """
last_load_date = self.database.last_event_load_date()
if last_load_date:
look_back = datetime.datetime.now() - datetime.timedelta(days=1)
first_event = min(look_back, last_load_date)
start = str(first_event)[:10]
self.logger.info('Loading events starting at %s' % (start))
else:
self.logger.info('Loading events from the first available event')
start = None
events = self.get_events(start=start, page=1)
num_events = events['pagination']['object_count']
if num_events > 0:
self.logger.info('There are %s events to process' % (num_events))
else:
self.logger.info('There are not next events. Exiting')
return
more_events = True
while more_events:
for event in events['events']:
if not event:
continue
msg = "Loading information for %s" % (event['name']['text'])
self.logger.info(msg)
# Load the event into the database. Delete the current
# entry in order to maintain the unique index
event_id = event['id']
if not test:
self.database.delete_item('events', event_id)
self.load_event(event)
# Load the venue, if it does not already
# appear in the database
venue_id = event['venue_id']
venue_ = self.database.get_item('venues', venue_id)
if venue_id and not venue_:
venue = self.get_venue(venue_id)
if not test:
self.load_venue(venue)
attendees = self.get_attendees(event_id, page=1)
more_attendees = True
while more_attendees:
if not attendees:
break
for attendee in attendees['attendees']:
if not attendee:
continue
if not test:
self.database.delete_item('attendees',
attendee['id'],
{'event_id': event_id})
self.load_attendee(attendee)
if test or not attendees['pagination']['has_more_items']:
more_attendees = False
break
else:
page = attendees['pagination']['page_number'] + 1
attendees = self.get_attendees(event_id, page)
# Sleep to avoid the Eventbrite rate limit
if test:
return
else:
time.sleep(60)
if not events['pagination']['has_more_items']:
more_events = False
break
else:
page = events['pagination']['page_number'] + 1
msg = 'Pulling events on page %s' % (page)
self.logger.info(msg)
events = self.get_events(start, page)
def get_events(self, start, page=1):
"""
Pulls events from eventbrite and sleeps if the rate limit
has been exceeded
"""
org_id = self.eventbrite_org
events = self.eventbrite.get_events(org_id=org_id,
start=start,
page=page)
if not events:
# Sleep until eventbrite resets
self.logger.info('Rate limit exceed. Sleeping 30 mins')
time.sleep(3600)
events = self.eventbrite.get_events(start=start, page=page)
return events
def get_attendees(self, event_id, page=1):
"""
Pulls attendees from eventbrite and sleeps if the rate limit
has been exceeded
"""
attendees = self.eventbrite.get_attendees(event_id, page)
if not attendees:
# If events comes back as none, sleep until the
# Eventbrite rate limit resets
self.logger.info('Rate limit exceed. Sleeping 30 mins')
time.sleep(3600)
attendees = self.eventbrite.get_attendees(event_id, page)
return attendees
def get_venue(self, venue_id, page=1):
"""
Pull a venue and sleeps if the rate limit
has been exceeded
"""
venue = self.eventbrite.get_venue(venue_id, page)
if not venue:
self.logger.info('Rate limit exceed. Sleeping 30 mins')
time.sleep(3600)
venue = self.eventbrite.get_venue(event_id, page)
return venue
def load_event(self, event):
""" Loads an event into the database """
event_ = deepcopy(event)
start = arrow.get(event_['start']['utc']).datetime
event_['start_datetime'] = start
end = arrow.get(event_['end']['utc']).datetime
event_['end_datetime'] = end
description = event_['description']['text']
event_['description'] = description
name = event_['name']['text']
event_['name'] = name
event_['load_datetime'] = datetime.datetime.utcnow()
self.database.load_item(event_, 'events')
def load_attendee(self, attendee):
""" Loads an attendee into the database """
attendee_ = deepcopy(attendee)
profile = attendee_['profile']
if 'name' in profile:
attendee_['name'] = profile['name']
if 'first_name' in profile:
attendee_['first_name'] = profile['first_name']
if 'last_name' in profile:
attendee_['last_name'] = profile['last_name']
if 'email' in profile:
attendee_['email'] = profile['email']
cost = attendee_['costs']['gross']['major_value']
attendee_['cost'] = float(cost)
attendee_['load_datetime'] = datetime.datetime.utcnow()
self.database.load_item(attendee_, 'attendees')
def load_order(self, order):
""" Loads an order into the database """
order_ = deepcopy(order)
cost = order_['costs']['gross']['major_value']
order_['cost'] = float(cost)
order_['load_datetime'] = datetime.datetime.utcnow()
self.database.load_item(order_, 'orders')
def load_venue(self, venue):
""" Loads a venue into the database """
venue_ = deepcopy(venue)
for key in venue_['address']:
val = venue_['address'][key]
venue_[key] = val
venue_['latitude'] = float(venue_['latitude'])
venue_['longitude'] = float(venue_['longitude'])
self.database.load_item(venue_, 'venues')
class Geometries(object):
""" Class for parsing and loading geojson files """
def __init__(self, database=None):
daiquiri.setup(level=logging.INFO)
self.logger = daiquiri.getLogger(__name__)
self.database = Database(database=database)
self.path = os.path.dirname(os.path.realpath(__file__))
self.url = 'https://www.zip-codes.com/cache/kml-zip/'
self.search = SearchEngine(simple_zipcode=True)
self.zip_code_cache = {}
def load_all_zip_codes(self):
"""Loads all zipcodes geometries into the database. Pauses five
seconds betwen loading each zipcode to avoid overwhelming the
site we download the geometries from."""
valid_zip_codes = self.get_all_zip_codes()
for code in valid_zip_codes:
try:
self.logger.info("Loading geometry for {}".format(code))
self.load_zip_code(code)
except:
self.logger.warning('Geojson load failed for {}'.format(code))
time.sleep(5)
def get_kml(self, zip_code):
""" Pulls the KML file for a zip code """
url = self.url + '%s.kml' % (zip_code)
response = requests.get(url)
if response.status_code == 200:
return response.text
else:
return None
def get_all_zip_codes(self):
"""Creates a list of valid zip codes by checking them
against the US zip code search object."""
possible_zip_codes = [str(i).rjust(5, '0') for i in range(99999)]
valid_zip_codes = []
for code in possible_zip_codes:
results = self.get_zipcode_data(code)
if results and results['zipcode'] != None:
valid_zip_codes.append(code)
return valid_zip_codes
def load_zip_code(self, zip_code):
""" Pulls a zip code and loads it into the database """
# Fetch the KML file from the resource
kml = self.get_kml(zip_code)
if not kml:
return
filename = self.path + '/temp/%s.kml' % (zip_code)
with open(filename, 'w') as f:
f.write(kml)
# Convert the KML file to GeoJSON
kml2geojson.main.convert(filename, self.path + '/temp')
# Load the file into the database
geo_filename = self.path + '/temp/%s.geojson' % (zip_code)
with open(geo_filename, 'r') as f:
geo_json = json.load(f)
zipcode_data = self.get_zipcode_data(zip_code)
row = {
'id': zip_code,
'geometry': json.dumps(geo_json),
'city': zipcode_data['major_city'],
'county': zipcode_data['county'],
'region': zipcode_data['state']
}
self.database.delete_item('geometries', zip_code)
self.database.load_item(row, 'geometries')
# Delete the temporary files
os.remove(filename)
os.remove(geo_filename)
def get_zipcode_data(self, zipcode):
"""Pulls the city and county name for the specified zipcode."""
if zipcode in self.zip_code_cache:
return self.zip_code_cache[zipcode]
else:
results = self.search.by_zipcode(zipcode)
if results.zipcode:
zipcode_data = results.to_dict()
else:
zipcode_data = None
# Cache city and state information for later so we don't
# have to search against the search engine again
self.zip_code_cache[zipcode] = zipcode_data
return zipcode_data