def __init__(self):
self.tm = ticketmaster.TicketMaster()
self.sk = songkick.SongKick()
self.demo = demographics.Demographics(
city_to_city=self.tm.city_to_city_map,
country_to_country=self.tm.country_to_country_map)
self.trends = google_trends.Trends(use_proxy=False)
def add_demographics(df, basedate, thread_name=''):
print('I am', thread_name)
try:
existing_df = pd.read_pickle('../../data/demographics_temp' +
basedate + thread_name + '.pkl')
already_done = df.index.isin(existing_df.index)
not_done_df = df.loc[~already_done, :]
except FileNotFoundError:
print('No data already done')
not_done_df = df
existing_df = pd.DataFrame()
tm = ticketmaster.TicketMaster()
dg = demographics.Demographics(tm.city_to_city_map,
tm.country_to_country_map)
dg_cont = []
print('running demo for ', df.shape[0], 'records')
real_idx = 0
for idx, row in not_done_df.iterrows():
real_idx += 1
if np.mod(real_idx, np.floor(not_done_df.shape[0] / 100)) == 0:
print('Demo ' + thread_name + 'is',
100 * real_idx / not_done_df.shape[0], 'percent complete')
dg_df = pd.concat(dg_cont, axis=1).T
dg_df = pd.concat([dg_df, existing_df])
dg_df = dg_df.set_index('idx')
dg_df.to_pickle('../../data/demographics_temp' + basedate +
thread_name + '.pkl')
try:
row_out = dg.get_pop(row['city'], row['country'])
row_out['idx'] = idx
dg_cont.append(row_out)
except KeyboardInterrupt:
return
except BaseException as e:
if not debug:
print(e)
continue
else:
raise
dg_df = pd.concat(dg_cont, axis=1).T
dg_df = pd.concat([dg_df, existing_df])
dg_df = dg_df.set_index('idx')
return dg_df
def add_ticket_levels(df):
tm = ticketmaster.TicketMaster()
tm_cont = []
print('running avail for ', df.shape[0], 'records')
for idx, row in df.iterrows():
if np.mod(idx, np.floor(df.shape[0] / 10)) == 0:
print('TM is', 100 * idx / df.shape[0], 'percent complete')
try:
row_out = tm.add_availability_info(row)
row_out['idx'] = idx
tm_cont.append(row_out)
except KeyboardInterrupt:
return
except BaseException as e:
if not debug:
print(e)
continue
else:
raise
tm_df = pd.concat(tm_cont, axis=1).T
tm_df = tm_df.set_index('idx')
return tm_df
def add_availability(df):
tm = ticketmaster.TicketMaster()
tm_cont = []
for idx, row in df.iterrows():
if np.mod(idx, np.floor(df.shape[0] / 10)) == 0:
print('Avail is', 100 * idx / df.shape[0], 'percent complete')
try:
if 'avail_checked' in row.index and row['avail_checked']:
continue
row_out = tm.add_availability_info(row)
row_out['idx'] = idx
tm_cont.append(pd.Series(row_out))
except KeyboardInterrupt:
return
except BaseException as e:
if not debug:
print(e)
continue
else:
raise
tm_df = pd.concat(tm_cont, axis=1).T
tm_df = tm_df.set_index('idx')
return tm_df
pop_city = city_req['fields']['population']
iso_region = city_req['fields']['region']
iso_country = city_req['fields']['country']
break
if country in self.country_pop:
pop_count = self.country_pop[country]
else:
ret = json.loads(self.req.get(self.base_country + 'q=' + country+self.apikey).content)
if 'records' not in ret:
print('country', ret)
pop_count = ret['records'][0]['fields']['value']
self.country_pop[country] = pop_count
pickle.dump(self.country_pop, open('country_pop.pkl','wb'))
if country in self.country_gdp:
gdp = self.country_gdp[country]
else:
ret = json.loads(self.req.get(self.base_gdp + 'q=' + country+self.apikey).content)
if 'records' not in ret:
print('gdp', ret)
gdp = ret['records'][0]['fields']['gdp_per_capita']
self.country_gdp[country] = gdp
pickle.dump(self.country_gdp, open('country_gdp.pkl','wb'))
return pd.Series({'population_city':pop_city, 'population_country':pop_count, 'gdp_country': gdp,'iso_region':iso_region, 'iso_country':iso_country})
if __name__=='__main__':
tm = ticketmaster.TicketMaster()
dg = Demographics(city_to_city=tm.city_to_city_map, country_to_country=tm.country_to_country_map)
print(dg.get_pop('christchurch','new zealand'))
def parse_events(events):
tm = ticketmaster.TicketMaster()
events = tm.parse_events_to_df(events)
print('Got', events.shape[0], 'records')
return events