def fetchDataFromAPI(table, query):
StartTime = time.time()
apikeyurl='https://api.sendgrid.com/v3/api_keys'
access_attempts_url= 'https://api.sendgrid.com/v3/access_settings/activity?limit=20'
api_key_scopes= 'https://api.sendgrid.com/v3/scopes'
enforced_tls = 'https://api.sendgrid.com/v3/user/settings/enforced_tls'
whitelist_access= 'https://api.sendgrid.com/v3/access_settings/whitelist'
mail_settings ='https://api.sendgrid.com/v3/mail_settings'
partner_s_setting= 'https://api.sendgrid.com/v3/partner_settings'
design= 'https://api.sendgrid.com/v3/designs?page_size=100&summary=true'
access_requests= 'https://api.sendgrid.com/v3/scopes/requests?limit=50&offset=0'
alerts= 'https://api.sendgrid.com/v3/alerts'
user_profile= 'https://api.sendgrid.com/v3/user/profile'
subuser_profile= 'https://api.sendgrid.com/v3/subusers'
sender_verification= 'https://api.sendgrid.com/v3/verified_senders/domains'
if table == 'apikey':
api_keys=req(apikeyurl)
return dataToTable(api_keys,table,query )
elif table=='access_attempt':
access_attempt= req(access_attempts_url)
return dataToTable(access_attempt,table,query)
elif table=='key_scopes':
scopes= req(api_key_scopes)
return dataToTable(scopes,table,query)
elif table=='enforced_tls':
a= req(enforced_tls)
return dataToTable(a,table,query)
elif table=='whitelist_access':
a= req(whitelist_access)
return dataToTable(a,table,query)
elif table=='mail_settings':
a= req(mail_settings)
return dataToTable(a,table,query)
elif table=='partner_mail_settings':
a= req(partner_s_setting)
return dataToTable(a,table,query)
elif table=='design':
a= req(design)
return dataToTable(a,table,query)
elif table=='access_requests':
a= req(access_requests)
return dataToTable(a,table,query)
elif table=='alerts':
a= req(alerts)
return dataToTable(a,table,query)
elif table=='user_profile':
a= req(user_profile)
return dataToTable(a,table,query)
elif table=='subuser_profile':
a= req(subuser_profile)
return dataToTable(a,table,query)
elif table=='sender_verification':
a= req(sender_verification)
return dataToTable(a,table,query)
else:
pass
EndTime = time.time()
pg.measure_execution_time(LicenseKey,fetchDataFromAPI.__name__,StartTime,EndTime,Connector_name)
def Pagination(data, number_of_page=0):
StartTime = time.time()
if number_of_page > 0:
per_page = math.ceil(len(data) / number_of_page)
total_page = number_of_page
# else:
# total_page = len(data)
count = 1
print("Enter page number to show its data. \n")
while True:
print(data.iloc[((count - 1) * per_page): (count * per_page)])
print("Showing page : ", count, " / ", total_page,"\n")
key = input("Press 'A' or 'D' to navigate or jump to page no: ")
if (key.isdigit()):
page = int(key)
if page > total_page:
print('Page doesnot exist')
else:
count = page
elif(key.lower() in ['a','d']):
lower = key.lower()
if lower == 'a':
if count > 1:
count -= 1
elif lower == 'd':
if count < total_page:
count += 1
else:
print("Invalid page number")
pass
EndTime = time.time()
pg.measure_execution_time(LicenseKey,Pagination.__name__,StartTime,EndTime,Connector_name)
def dataToTable(results,table,query):
StartTime = time.time()
# print(results)
if table== 'apikey' or table== 'access_attempt' or table== 'whitelist_access' or table== 'mail_settings' or table=='partner_mail_settings' or table== 'design' or table== 'access_requests' :
if results['result']:
t=pd.DataFrame(results['result'])
else:
print('no data in table')
sys.exit(0)
# os._exit(0)
elif table=='sender_verification':
a=schema_parse(results['results'])
t=pd.DataFrame(a, index=[0])
elif table=='subuser_profile':
t=pd.DataFrame(results)
elif table == 'key_scopes':
t=pd.DataFrame(results['scopes'])
elif table == 'enforced_tls':
t=pd.DataFrame(results, index=[0])
elif table == 'alerts':
t=pd.DataFrame(results, index=[0])
elif table == 'user_profile':
t=pd.DataFrame(results, index=[0])
else:
pass
# print(results)
t.to_sql(table, con=engine, if_exists='replace')
r = engine.execute(query)
tble = pd.DataFrame(r, index=None,columns=r.keys())
print(tble)
return tble
EndTime = time.time()
pg.measure_execution_time(LicenseKey,dataToTable.__name__,StartTime,EndTime,Connector_name)
def initializeAPI(api_key):
StartTime = time.time()
global token
token= api_key
EndTime = time.time()
pg.measure_execution_time(LicenseKey,initializeAPI.__name__,StartTime,EndTime,Connector_name)
return token
def schedule_data(sec, rows, df,filename):
StartTime = time.time()
print("Total rows : ", len(df))
schedule.every(sec).seconds.do(
lambda: scheduler_task(data=df, rows=rows, file_name=filename)) # file_name is optional
while True:
schedule.run_pending()
if STOP:
schedule.clear()
break
EndTime = time.time()
pg.measure_execution_time(LicenseKey,schedule_data.__name__,StartTime,EndTime,Connector_name)
def connectEngine(username, password, server, database, tableName, df):
StartTime = time.time()
alchemyEngine = create_engine(
f"postgresql+psycopg2://"+username+":"+password+"@"+server+"/"+database+", pool_recycle=3600")
try:
con = alchemyEngine.connect()
df.to_sql(
tableName, con, if_exists='replace')
con.close()
print("Data uploaded to table "+tableName)
except Exception as e:
print (getattr(e, 'message', repr(e)))
EndTime = time.time()
pg.measure_execution_time(LicenseKey,ConnectEngine.__name__,StartTime,EndTime,Connector_name)
def scheduler_task(data, rows, file_name=""):
StartTime = time.time()
# assuming.......data's type is pandas framework.......
global END, START, STOP
END = START + rows
if END > len(data):
END = len(data)
STOP = 1
print(data.iloc[START: END])
START += rows
if len(file_name) > 1:
data.to_csv(file_name)
else:
print('')
print(data)
EndTime = time.time()
pg.measure_execution_time(LicenseKey,scheduler_task.__name__,StartTime,EndTime,Connector_name)
def fetchDataFromAPI(table):
StartTime = time.time()
bg_url = 'https://base.zenkit.com/api/v1/backgrounds'
bg_df_url = 'https://base.zenkit.com/api/v1/backgrounds/default'
ele_cat_url = 'https://base.zenkit.com/api/v1/elementcategories'
user_access = 'https://base.zenkit.com/api/v1/users/me/accesses'
l_w_ws_access = 'https://base.zenkit.com/api/v1/users/me/lists-without-workspace-access'
new_not_min_for = 'https://base.zenkit.com/api/v1/users/me/new-notifications-in-minimal-format'
noti = 'https://base.zenkit.com/api/v1/users/me/notifications'
current_user = '******'
ws_list = 'https://base.zenkit.com/api/v1/users/me/workspacesWithLists'
if table == 'background':
background = req(bg_url)
if not background:
pass
else:
background = parse(background)
a = background['main']
b = background['resourceTags']
send_data = {'background': a, 'background_resourceTags': b}
return send_data
elif table == 'default_background':
background_def = req(bg_df_url)
if not background_def:
pass
else:
background_def = parse(background_def)
b = background_def['main']
a = background_def['resourceTags']
send_data = {
'default_background': b,
'def_background_resourceTags': a
}
return send_data
elif table == 'element_categories':
ele_cat = req(ele_cat_url)
if not ele_cat:
pass
else:
ele_cat = parse(ele_cat)
a5 = ele_cat['filterKeys']
a4 = ele_cat['businessDataDefinition']
a1 = ele_cat['businessDataDefaults']
a2 = ele_cat['elementDataDefinition']
a3 = ele_cat['elementDataDefaults']
a = ele_cat['main']
send_data = {
'element_categories': a,
'businessDataDefaults': a1,
'elementDataDefinition': a2,
'elementDataDefaults': a3,
'businessDataDefinition': a4,
'filterKeys': a5
}
return send_data
elif table == 'user_access':
user_access = req(user_access)
if not user_access:
pass
else:
user_access = parse(user_access)
a = user_access['main']
send_data = {'user_access': a}
return send_data
elif table == 'lists-without-workspace-access':
a = req(l_w_ws_access)
if not a:
pass
else:
a = parse(a)
a1 = a['main']
a2 = a['settings']
a3 = a['emails']
send_data = {
'lists_without_workspace_access': a1,
'l_w_ws_access_settings': a2,
'l_w_ws_access_emails': a3
}
return send_data
elif table == 'new-notifications-in-minimal-format':
b = req(new_not_min_for)
if not b:
pass
else:
b = parse(b)
a = b['main']
send_data = {table: a}
return send_data
elif table == 'notifications':
c = req(noti)
print(c)
if not c:
null = dict()
return null
else:
c = parse(c)
a = c['main']
a1 = c['notifications']
send_data = {'notifications': a, "notifications1": a1}
return send_data
elif table == 'current_user':
d = req(current_user)
if not d:
pass
else:
d = parse(d)
a1 = d['main']
a2 = d['settings']
a3 = d['emails']
send_data = {
'current_user': a1,
'current_user_settings': a2,
'current_user_emails': a3
}
return send_data
elif table == 'workspacesWithLists':
e = req(ws_list)
if not e:
pass
else:
e = parse(e)
a = e['main']
send_data = {'workspacesWithLists': a}
return send_data
else:
print("please provide the right table name")
exit(0)
EndTime = time.time()
pg.measure_execution_time(LicenseKey, fetchDataFromAPI.__name__, StartTime,
EndTime, Connector_name)
def dataToTable(results, table):
StartTime = time.time()
d = results.to_sql(table, con=engine, if_exists='replace')
EndTime = time.time()
pg.measure_execution_time(LicenseKey, dataToTable.__name__, StartTime,
EndTime, Connector_name)