def get_data(self, url, object_name):
self.data = CM.get_api_data(url.format('1'))
if self.data.ok:
total_items = self.data.json()[CBDict.data.value][CBDict.paging.value][CBDict.total_items.value]
number_of_pages = self.data.json()[CBDict.data.value][CBDict.paging.value][CBDict.number_of_pages.value]
cols = self.data.json()[CBDict.data.value][CBDict.items.value][0][CBDict.properties.value].keys()
cols = list(cols)
cols.append('uuid')
print(cols)
print('Total items: {}\nTotal Pages: {}'.format(total_items, number_of_pages))
data_list = []
for j in range(0, number_of_pages):
self.data = CM.get_api_data(url.format(j + 1))
data = self.data.json()[CBDict.data.value][CBDict.items.value]
print(j, '*' * j, len(data))
for i in range(0, len(data)):
dt = data[i][CBDict.properties.value]
dt[CBDict.uuid.value] = data[i][CBDict.uuid.value]
data_list.append(dt)
df = pd.DataFrame(data_list, columns=cols)
df.to_csv(self.file_name.format(object_name, str(time.time())), sep=',', columns=cols, index=False)
print('File saved successfully!')
else:
print('SNAP! Something goes wrong.\nSTATUS: {}\nMESSAGE: {}'.format(self.data.json()[0]['status'], self.data.json()[0]['message']))
def save_orgs_relationship(self, api_url):
url = api_url + self.api_org_token
print(url)
orgs = CM.get_api_data(url)
if orgs.ok:
self.org_uuid = orgs.json()[CBDict.data.value][CBDict.uuid.value]
df = self.db.pandas_read(self.enum.SQL.sql_org_detail_exists.value.format(self.org_uuid))
if len(df) == 0:#self.db.entity_exists('MDCRaw.CRUNCHBASE.Organization', 'org_uuid', self.org_uuid):
self.save_organization_detail(self.org_uuid, orgs.json()[CBDict.data.value][CBDict.properties.value])
rs_json = orgs.json()[CBDict.data.value][CBDict.relationships.value]
self.save_funding_rounds(rs_json['funding_rounds'], self.org_uuid)
self.save_relational_entity(rs_json[self.enum.CBDict.headquarters.value], self.org_uuid,
self.enum.SQL.sql_offices_exists.value, self.enum.SQL.sql_offices_insert.value,
self.office_col)
self.save_relational_entity(rs_json['categories'], self.org_uuid,
self.enum.SQL.sql_org_category_exists.value,
self.enum.SQL.sql_org_category_insert.value, self.category_col)
# save all the related entities
# self.save_teams(rs_json['featured_team'], self.org_uuid, TeamStatus.Featured.value)
# self.save_teams(rs_json['current_team'], self.org_uuid, TeamStatus.Current.value)
# self.save_teams(rs_json['past_team'], self.org_uuid, TeamStatus.Past.value)
# self.save_teams(rs_json['board_members_and_advisors'], self.org_uuid, TeamStatus.Board.value)
# self.save_investments_invested_in(rs_json['investments'])
# self.save_relational_entity(rs_json['sub_organizations'], self.org_uuid, self.sql_sub_organization_insert)
# if rs_json[self.enum.CBDict.offices.value][self.enum.CBDict.items] is not None:
# self.save_relational_entity(rs_json[self.enum.CBDict.offices.value], self.org_uuid, self.enum.SQL.sql_offices_exists, self.sql_offices_insert, self.office_col)
# self.save_relational_entity(rs_json['founders'], self.org_uuid, self.sql_founders_insert)
# self.save_relational_entity(rs_json['acquisitions'], self.org_uuid, self.sql_acquisition_insert)
# self.save_relational_entity(rs_json['acquired_by'], self.org_uuid, self.sql_acquired_insert)
# self.save_relational_entity(rs_json['ipo'], self.org_uuid, self.sql_ipo_insert)
# self.save_relational_entity(rs_json['funds'], self.org_uuid, self.sql_funds_insert)
# self.save_relational_entity(rs_json['websites'], self.org_uuid, self.sql_websites_insert)
# self.save_relational_entity(rs_json['images'], self.org_uuid, self.sql_image_insert)
# self.save_relational_entity(rs_json['news'], self.org_uuid, self.sql_news_insert)
db.execute(self.orgs_summary_update.format(self.org_uuid))
else:
print('organization already exists.')