def __init__(self): self.tm = ticketmaster.TicketMaster() self.sk = songkick.SongKick() self.demo = demographics.Demographics( city_to_city=self.tm.city_to_city_map, country_to_country=self.tm.country_to_country_map) self.trends = google_trends.Trends(use_proxy=False)
def add_demographics(df, basedate, thread_name=''): print('I am', thread_name) try: existing_df = pd.read_pickle('../../data/demographics_temp' + basedate + thread_name + '.pkl') already_done = df.index.isin(existing_df.index) not_done_df = df.loc[~already_done, :] except FileNotFoundError: print('No data already done') not_done_df = df existing_df = pd.DataFrame() tm = ticketmaster.TicketMaster() dg = demographics.Demographics(tm.city_to_city_map, tm.country_to_country_map) dg_cont = [] print('running demo for ', df.shape[0], 'records') real_idx = 0 for idx, row in not_done_df.iterrows(): real_idx += 1 if np.mod(real_idx, np.floor(not_done_df.shape[0] / 100)) == 0: print('Demo ' + thread_name + 'is', 100 * real_idx / not_done_df.shape[0], 'percent complete') dg_df = pd.concat(dg_cont, axis=1).T dg_df = pd.concat([dg_df, existing_df]) dg_df = dg_df.set_index('idx') dg_df.to_pickle('../../data/demographics_temp' + basedate + thread_name + '.pkl') try: row_out = dg.get_pop(row['city'], row['country']) row_out['idx'] = idx dg_cont.append(row_out) except KeyboardInterrupt: return except BaseException as e: if not debug: print(e) continue else: raise dg_df = pd.concat(dg_cont, axis=1).T dg_df = pd.concat([dg_df, existing_df]) dg_df = dg_df.set_index('idx') return dg_df
def add_ticket_levels(df): tm = ticketmaster.TicketMaster() tm_cont = [] print('running avail for ', df.shape[0], 'records') for idx, row in df.iterrows(): if np.mod(idx, np.floor(df.shape[0] / 10)) == 0: print('TM is', 100 * idx / df.shape[0], 'percent complete') try: row_out = tm.add_availability_info(row) row_out['idx'] = idx tm_cont.append(row_out) except KeyboardInterrupt: return except BaseException as e: if not debug: print(e) continue else: raise tm_df = pd.concat(tm_cont, axis=1).T tm_df = tm_df.set_index('idx') return tm_df
def add_availability(df): tm = ticketmaster.TicketMaster() tm_cont = [] for idx, row in df.iterrows(): if np.mod(idx, np.floor(df.shape[0] / 10)) == 0: print('Avail is', 100 * idx / df.shape[0], 'percent complete') try: if 'avail_checked' in row.index and row['avail_checked']: continue row_out = tm.add_availability_info(row) row_out['idx'] = idx tm_cont.append(pd.Series(row_out)) except KeyboardInterrupt: return except BaseException as e: if not debug: print(e) continue else: raise tm_df = pd.concat(tm_cont, axis=1).T tm_df = tm_df.set_index('idx') return tm_df
pop_city = city_req['fields']['population'] iso_region = city_req['fields']['region'] iso_country = city_req['fields']['country'] break if country in self.country_pop: pop_count = self.country_pop[country] else: ret = json.loads(self.req.get(self.base_country + 'q=' + country+self.apikey).content) if 'records' not in ret: print('country', ret) pop_count = ret['records'][0]['fields']['value'] self.country_pop[country] = pop_count pickle.dump(self.country_pop, open('country_pop.pkl','wb')) if country in self.country_gdp: gdp = self.country_gdp[country] else: ret = json.loads(self.req.get(self.base_gdp + 'q=' + country+self.apikey).content) if 'records' not in ret: print('gdp', ret) gdp = ret['records'][0]['fields']['gdp_per_capita'] self.country_gdp[country] = gdp pickle.dump(self.country_gdp, open('country_gdp.pkl','wb')) return pd.Series({'population_city':pop_city, 'population_country':pop_count, 'gdp_country': gdp,'iso_region':iso_region, 'iso_country':iso_country}) if __name__=='__main__': tm = ticketmaster.TicketMaster() dg = Demographics(city_to_city=tm.city_to_city_map, country_to_country=tm.country_to_country_map) print(dg.get_pop('christchurch','new zealand'))
def parse_events(events): tm = ticketmaster.TicketMaster() events = tm.parse_events_to_df(events) print('Got', events.shape[0], 'records') return events