def returnActivitiesPageToken(token, appName, dDay, SAJson, SADelegated):
startTime = dDay + 'T00:00:00.000Z'
endTime = dDay + 'T23:59:59.999Z'
page_token = token
activities = []
try:
reports = createReportObject(cfg['scopes']['audit_log'], 'admin', 'reports_v1', SAJson, SADelegated)
except Exception as err:
bvi_log(date=dDay, resource='activities', message_id='invalid_credential', message=err, regenerate=True)
raise err
# AVAILABLE FIELDS: https://developers.google.com/admin-sdk/reports/v1/reference/activity-ref-appendix-a/admin-event-names and more.
fields = 'items(actor,events,id),nextPageToken'
try:
if page_token == '':
request = reports.activities().list(userKey='all', applicationName=appName, startTime=startTime,
endTime=endTime, fields=fields, maxResults=maxResultsPage)
else:
request = reports.activities().list(userKey='all', applicationName=appName, startTime=startTime,
endTime=endTime, fields=fields, pageToken=page_token,
maxResults=maxResultsPage)
results = execute_request_with_retries(request)
logging.info(results)
activities = results.get('items', [])
except Exception as err:
logging.error(err)
logging.error("Error Found, ending returnActivitiesPageToken!")
bvi_log(date=dDay, resource='activities', message_id='activities_api', message=err, regenerate=True)
logging.info("We have {} activities rows in the end".format(len(activities)))
yield activities
def exec_historical(mgr, step, SdDate, EdDate):
logging.info('Create Historical data')
start_date = date(int(SdDate.split('-')[0]), int(SdDate.split('-')[1]), int(SdDate.split('-')[2]))
end_date = date(int(EdDate.split('-')[0]), int(EdDate.split('-')[1]), int(EdDate.split('-')[2]))
today = date.today()
today_4 = today - timedelta(days=4)
if start_date > today_4:
logging.info('Error: Start Date > Today - 4 days, try another Start Date')
return
if end_date > today_4:
logging.info('Error: End Date > Today - 4 days, try another End Date')
return
Number_days = abs((start_date - end_date).days)
dDate = SdDate
iterating_day = start_date
n = 0
while n <= int(Number_days):
bvi_log(date=dDate, resource=get_log_step(step), message_id='start',
message='Start of {} step'.format(step))
endpoint = mgr[step]['endpoint'].replace('from_cron', dDate)
logging.info(endpoint)
taskqueue.add(queue_name=cfg['queues']['slow'], url=endpoint, method='GET')
iterating_day = iterating_day + timedelta(days=1)
dDate = iterating_day.strftime("%Y-%m-%d")
n += 1
logging.info('Sent ' + str(n) + ' days request for ' + step)
def returnUserUsagePageToken(token, dDay, SAJson, SADelegated):
dDate = dDay
page_token = token
user_usage = []
try:
reports = createReportObject(cfg['scopes']['admin_report'], 'admin', 'reports_v1', SAJson, SADelegated)
except Exception as err:
bvi_log(date=dDay, resource='user_usage', message_id='invalid_credential', message=err, regenerate=True)
raise err
# AVAILABLE FIELDS: https://developers.google.com/admin-sdk/reports/v1/reference/usage-ref-appendix-a/users
fields = 'nextPageToken,usageReports(date,entity,parameters)'
try:
if page_token == '':
request = reports.userUsageReport().get(userKey='all', date=dDay, fields=fields,
maxResults=maxResultsPage_UserUsage)
else:
request = reports.userUsageReport().get(userKey='all', date=dDay, fields=fields,
pageToken=page_token, maxResults=maxResultsPage_UserUsage)
results = execute_request_with_retries(request)
user_usage = results.get('usageReports', [])
except Exception as err:
logging.error(err)
logging.error("Error Found, ending returnUserUsagePageToken!")
bvi_log(date=dDay, resource='user_usage', message_id='user_usage_api', message=err, regenerate=True)
logging.info("We have {} user_usage rows in the end".format(len(user_usage)))
yield user_usage
def returnUserListPageToken(token, dDay, dDomain, SAJson, SADelegated):
dDate = dDay
page_token = token
users = []
try:
reports = createReportObject(cfg['scopes']['admin_directory'], 'admin', 'directory_v1', SAJson, SADelegated)
except Exception as err:
bvi_log(date=dDate, resource='users_list', message_id='invalid_credential', message=err, regenerate=True)
raise err
# AVAILABLE FIELDS: https://developers.google.com/admin-sdk/directory/v1/reference/users#resource
fields = 'nextPageToken,users(creationTime,customerId,emails,lastLoginTime,orgUnitPath,primaryEmail)'
try:
if page_token == '':
request = reports.users().list(domain=dDomain, maxResults=maxResultsPage, fields=fields)
else:
request = reports.users().list(domain=dDomain, pageToken=page_token, maxResults=maxResultsPage,
fields=fields)
results = execute_request_with_retries(request)
if 'users' in results:
users = results['users']
for user_item in users:
user_item[u'date'] = dDate
except DeadlineExceededError as err:
logging.error(err)
logging.error("Retrying!")
except Exception as err:
logging.error(err)
logging.error(dDomain)
logging.error("Error Found, ending returnUserListPageToken!")
bvi_log(date=dDate, resource='users_list', message_id='users_list_api', message=err, regenerate=True)
logging.info("We have {} user rows in the end".format(len(users)))
yield users
def get(self):
# User List today
today = date.today()
dDate = today.strftime("%Y-%m-%d")
pages = 0
page_token = self.request.get('token')
domain = self.request.get('domain')
maxPages = self.request.get('maxPages')
maxPages = int(maxPages)
numQueues = 3
queue_name = []
for x in range(1, numQueues + 1):
queue_name.append(cfg['queues']['user'] + str(x))
for pages_token in returnUsersListToken(
domain, cfg['credentials']['general'],
cfg['super_admin']['delegated'], page_token):
if not pages_token:
break
pages += 1
logging.info(
'User list pages for {} - {} / page {} so far until now'.
format(dDate, domain, pages))
tNow = datetime.now()
aNumber = tNow.strftime("%S")
aRandomNumber = random.randint(0, sys.maxsize)
aNumber = "{aNumber}_{aRandomNumber}".format(
aNumber=aNumber, aRandomNumber=aRandomNumber)
qA = random.randint(0, numQueues - 1)
taskqueue.add(
queue_name=queue_name[qA],
name='opul' + dDate + '_' + str(pages) + '_' + aNumber,
url='/one_page_user_list?token=' + pages_token + '&date=' +
dDate,
method='GET')
if pages >= maxPages:
taskqueue.add(queue_name=queue_name[qA],
name='ud' + dDate + '_' + aNumber,
url='/user_delay?token=' + pages_token +
'&maxPages=' + str(maxPages),
method='GET')
break
logging.info('User list pages for {} - {} / finally {} pages '.format(
dDate, domain, pages))
self.response.write("User Usage for {} - {} / {} pages".format(
dDate, domain, pages))
bvi_log(date=dDate,
resource='users_list',
message_id='end',
message='End of /user_delay call')
def returnActivitiesToken(dDay, appName, SAJson, SADelegated, page_token):
tokens = []
startTime = dDay + 'T00:00:00.000Z'
endTime = dDay + 'T23:59:59.999Z'
try:
reports = createReportObject(cfg['scopes']['audit_log'], 'admin',
'reports_v1', SAJson, SADelegated)
except Exception as err:
bvi_log(date=dDay,
resource='activities',
message_id='invalid_credential',
message=err,
regenerate=True)
raise err
fields = 'nextPageToken'
while True:
try:
if page_token == '':
request = reports.activities().list(userKey='all',
applicationName=appName,
startTime=startTime,
endTime=endTime,
fields=fields,
maxResults=maxResultsPage)
else:
request = reports.activities().list(userKey='all',
applicationName=appName,
startTime=startTime,
endTime=endTime,
fields=fields,
pageToken=page_token,
maxResults=maxResultsPage)
results = execute_request_with_retries(request)
if 'nextPageToken' in results:
tokens = results['nextPageToken']
page_token = results['nextPageToken']
yield tokens
else:
break
except DeadlineExceededError as err:
logging.error(err)
logging.error("Retrying!")
except Exception as err:
logging.error(err)
logging.error("Error Found, ending returnActivitiesToken !")
tokens = []
bvi_log(date=dDay,
resource='activities',
message_id='activities_api',
message=err,
regenerate=True)
break
yield tokens
def returnUserUsageToken(dDay, SAJson, SADelegated, page_token):
tokens = []
try:
reports = createReportObject(cfg['scopes']['admin_report'], 'admin',
'reports_v1', SAJson, SADelegated)
except Exception as err:
bvi_log(date=dDay,
resource='user_usage',
message_id='invalid_credential',
message=err,
regenerate=True)
raise err
# AVAILABLE FIELDS: https://developers.google.com/admin-sdk/reports/v1/reference/usage-ref-appendix-a/users
fields = 'nextPageToken'
pages = 0
while True:
try:
if page_token == '':
request = reports.userUsageReport().get(
userKey='all',
date=dDay,
fields=fields,
maxResults=maxResultsPage_UserUsage)
else:
request = reports.userUsageReport().get(
userKey='all',
date=dDay,
fields=fields,
pageToken=page_token,
maxResults=maxResultsPage_UserUsage)
results = request.execute()
pages += 1
if 'nextPageToken' in results:
tokens = results['nextPageToken']
page_token = results['nextPageToken']
yield tokens
else:
break
except DeadlineExceededError as err:
logging.error(err)
logging.error("Retrying!")
except Exception as err:
logging.error(err)
logging.error("Error Found, ending returnUserUsageToken !")
tokens = []
bvi_log(date=dDay,
resource='user_usage',
message_id='user_usage_api',
message=err,
regenerate=True)
break
logging.info("We have {} user_usage pages in the end".format(pages))
logging.info(tokens)
yield tokens
def get(self):
logging.info('Activities')
dateref = self.request.get('date', "from_cron")
if len(dateref) > 0:
try:
if dateref == "from_cron":
# Customer Usage day -4 from February 2018
today = date.today()
today_4 = today - timedelta(days=4)
dateref = today_4.strftime("%Y-%m-%d")
except ValueError:
logging.error(
"Wrong updating date = {dateref}".format(dateref=dateref))
self.response.write("Wrong updating date = {}".format(dateref))
return
bvi_log(date=dateref,
resource='activities',
message_id='start',
message='Start of /activities call')
collection = 'admin|drive|calendar|gplus'
collection = list(set(collection.split("|")))
queue_name = cfg['queues']['slow']
tNow = datetime.now()
aNumber = tNow.strftime("%S")
aRandomNumber = random.randint(0, sys.maxsize)
aNumber = "{aNumber}_{aRandomNumber}".format(
aNumber=aNumber, aRandomNumber=aRandomNumber)
processed = 0
for app in collection:
taskqueue.add(queue_name=queue_name,
name='act' + dateref + '_' + app + '_' + aNumber,
url='/activities_app?app=' + app + '&date=' +
dateref,
method='GET')
processed += 1
logging.info('Activities for {} - {} / finally {} apps '.format(
dateref, cfg['domains'], processed))
self.response.write("Activities for {} - {} / finally {} apps".format(
dateref, cfg['domains'], processed))
bvi_log(date=dateref,
resource='activities',
message_id='end',
message='End of /activities call')
def get(self):
# User Usage for a date One page at a time
token = self.request.get('token')
dDate = self.request.get('date')
decoratorDate = "".join(dDate.split("-"))
for report_items in returnUserUsagePageToken(token, dDate, cfg['credentials']['general'],
cfg['super_admin']['delegated']):
try:
writeDatainBigQuery(report_items,
'user_usage${decoratorDate}'.format(decoratorDate=decoratorDate))
except Exception as err:
bvi_log(date=dDate, resource='user_usage', message_id='bigquery_error', message=err,
regenerate=True)
raise err
def get(self):
logging.info('Customer Usage')
dateref = self.request.get('date', "from_cron")
if len(dateref) > 0:
try:
if dateref == "from_cron":
# Customer Usage day -4 from February 2018
today = date.today()
today_4 = today - timedelta(days=4)
dateref = today_4.strftime("%Y-%m-%d")
yyyy, mm, dd = dateref.split("-")
timestamp = datetime(int(yyyy), int(mm), int(dd))
dDate = dateref
except ValueError:
logging.error(
"Wrong updating date = {dateref}".format(dateref=dateref))
self.response.write("Wrong updating date = {}".format(dateref))
return
try:
bvi_log(date=dDate,
resource='exec',
message_id='start',
message='Start of BVI daily execution')
bvi_log(date=dDate,
resource='customer_usage',
message_id='start',
message='Start of /customer call')
decoratorDate = ("").join(dDate.split("-"))
for report_items in returnCustomerUsageReport(
dDate, cfg['credentials']['general'],
cfg['super_admin']['delegated']):
try:
bq_answer = writeDatainBigQuery(
report_items, 'customer_usage${decoratorDate}'.format(
decoratorDate=decoratorDate))
logging.info('Customer Usage for ' + dDate + ' - ' +
cfg['domains'] + ' / ')
self.response.write('Customer Usage for ' + dDate + ' - ' +
cfg['domains'] + ' / ')
except Exception as err:
bvi_log(date=dDate,
resource='customer_usage',
message_id='bigquery_error',
message=err,
regenerate=True)
raise err
except Exception as err:
logging.error(err)
self.response.write('Customer Usage for ' + dDate + ' - ' +
cfg['domains'] + ': ERROR')
raise err
bvi_log(date=dDate,
resource='customer_usage',
message_id='end',
message='End of /customer call')
def returnUsersListToken(dDomain, SAJson, SADelegated, page_token):
dDate = date.today().strftime("%Y-%m-%d")
try:
reports = createReportObject(cfg['scopes']['admin_directory'], 'admin',
'directory_v1', SAJson, SADelegated)
except Exception as err:
bvi_log(date=dDate,
resource='users_list',
message_id='invalid_credential',
message=err,
regenerate=True)
raise err
# AVAILABLE FIELDS: https://developers.google.com/admin-sdk/directory/v1/reference/users#resource
fields = 'nextPageToken'
tokens = []
while True:
try:
if page_token == '':
request = reports.users().list(domain=dDomain,
maxResults=maxResultsPage,
fields=fields)
else:
request = reports.users().list(domain=dDomain,
pageToken=page_token,
maxResults=maxResultsPage,
fields=fields)
results = execute_request_with_retries(request)
if 'nextPageToken' in results:
tokens = results['nextPageToken']
page_token = results['nextPageToken']
yield tokens
else:
break
except DeadlineExceededError as err:
logging.error(err)
logging.error("Retrying!")
except Exception as err:
logging.error(err)
logging.error("Error Found, ending returnUsersListToken!")
bvi_log(date=dDate,
resource='users_list',
message_id='users_list_api_token',
message=err,
regenerate=True)
tokens = []
break
yield tokens
def returnCustomerUsageReport(dDay, SAJson, SADelegated):
usage = []
page_token = None
try:
reports = createReportObject(cfg['scopes']['admin_report'], 'admin', 'reports_v1', SAJson, SADelegated)
except Exception as err:
bvi_log(date=dDay, resource='customer_usage', message_id='invalid_credential', message=err, regenerate=True)
raise err
fields = 'nextPageToken,usageReports(date,entity,parameters)'
parameters = 'accounts:gsuite_enterprise_used_licenses,accounts:gsuite_enterprise_total_licenses,accounts:gsuite_unlimited_used_licenses,accounts:gsuite_unlimited_total_licenses,accounts:gsuite_basic_used_licenses,accounts:gsuite_basic_total_licenses,accounts:num_disabled_accounts,accounts:num_suspended_users,accounts:num_users,accounts:apps_total_licenses,accounts:apps_used_licenses,accounts:vault_total_licenses,accounts:gsuite_enterprise_used_licenses,accounts:gsuite_basic_used_licenses,accounts:gsuite_unlimited_used_licenses,accounts:gsuite_enterprise_total_licenses,accounts:gsuite_unlimited_total_licenses,accounts:gsuite_basic_total_licenses,accounts:num_1day_logins,accounts:num_7day_logins,accounts:num_30day_logins,' \
'calendar:num_1day_active_users,calendar:num_7day_active_users,calendar:num_30day_active_users,' \
'cros:num_7day_active_devices,cros:num_30day_active_devices,' \
'classroom:num_1day_users,classroom:num_7day_users,classroom:num_30day_users,' \
'drive:num_1day_active_users,drive:num_7day_active_users,drive:num_30day_active_users,' \
'docs:num_docs,drive:num_1day_active_users,drive:num_7day_active_users,drive:num_30day_active_users,drive:num_30day_google_documents_active_users,drive:num_30day_google_spreadsheets_active_users,drive:num_30day_google_presentations_active_users,drive:num_30day_google_forms_active_users,drive:num_30day_google_drawings_active_users,drive:num_30day_other_types_active_users,drive:num_creators,drive:num_collaborators,drive:num_consumers,drive:num_sharers,' \
'gmail:num_1day_active_users,gmail:num_7day_active_users,gmail:num_30day_active_users,' \
'gplus:num_1day_active_users,gplus:num_7day_active_users,gplus:num_30day_active_users,' \
'device_management:num_30day_google_sync_managed_devices,device_management:num_30day_android_managed_devices,device_management:num_30day_ios_managed_devices,device_management:num_30day_total_managed_devices,device_management:num_30day_google_sync_managed_users,device_management:num_30day_android_managed_users,device_management:num_30day_ios_managed_users,device_management:num_30day_total_managed_users,' \
'sites:num_sites,sites:num_sites_created,' \
'meet:average_meeting_minutes,meet:average_meeting_minutes_with_11_to_15_calls,meet:average_meeting_minutes_with_16_to_25_calls,meet:average_meeting_minutes_with_26_to_50_calls,meet:average_meeting_minutes_with_2_calls,meet:average_meeting_minutes_with_3_to_5_calls,meet:average_meeting_minutes_with_6_to_10_calls,meet:lonely_meetings,meet:max_concurrent_usage_chromebase,meet:max_concurrent_usage_chromebox,meet:num_1day_active_users,meet:num_30day_active_users,meet:num_7day_active_users,' \
'meet:num_calls,meet:num_calls_android,meet:num_calls_by_external_users,meet:num_calls_by_internal_users,meet:num_calls_by_pstn_in_users,meet:num_calls_chromebase,meet:num_calls_chromebox,meet:num_calls_ios,meet:num_calls_jamboard,meet:num_calls_unknown_client,meet:num_calls_web,meet:num_meetings,meet:num_meetings_android,meet:num_meetings_chromebase,meet:num_meetings_chromebox,meet:num_meetings_ios,meet:num_meetings_jamboard,meet:num_meetings_unknown_client,meet:num_meetings_web,' \
'meet:num_meetings_with_11_to_15_calls,meet:num_meetings_with_16_to_25_calls,meet:num_meetings_with_26_to_50_calls,meet:num_meetings_with_2_calls,meet:num_meetings_with_3_to_5_calls,meet:num_meetings_with_6_to_10_calls,meet:num_meetings_with_external_users,meet:num_meetings_with_pstn_in_users,meet:total_call_minutes,meet:total_call_minutes_android,meet:total_call_minutes_by_external_users,meet:total_call_minutes_by_internal_users,meet:total_call_minutes_by_pstn_in_users,' \
'meet:total_call_minutes_chromebase,meet:total_call_minutes_chromebox,meet:total_call_minutes_ios,meet:total_call_minutes_jamboard,meet:total_call_minutes_unknown_client,meet:total_call_minutes_web,meet:total_meeting_minutes,' \
'gplus:num_1day_active_users,gplus:num_7day_active_users,gplus:num_30day_active_users,gplus:num_shares,gplus:num_stream_items_read,gplus:num_plusones,gplus:num_replies,gplus:num_reshares,gplus:num_communities,gplus:num_communities_public,gplus:num_communities_private,gplus:num_communities_organization_wide,gplus:num_communities_organization_private'
while True:
try:
request = reports.customerUsageReports().get(date=dDay, fields=fields, pageToken=page_token,
parameters=parameters)
results = execute_request_with_retries(request)
usage = results.get('usageReports', [])
if 'nextPageToken' in results:
page_token = results['nextPageToken']
logging.info("We have {} customer_usage rows, and more to come".format(len(usage)))
yield usage
else:
break
except DeadlineExceededError as err:
logging.error(err)
logging.error("Retrying!")
except Exception as err:
logging.error(err)
logging.error("Error Found, ending returnCustomerUsageReport!")
bvi_log(date=dDay, resource='customer_usage', message_id='customer_usage_api', message=err,
regenerate=True)
break
logging.info("We have {} customer_usage rows in the end".format(len(usage)))
yield usage
def get(self):
logging.info('Customer Usage')
dateref = self.request.get('date', "from_cron")
if len(dateref) > 0:
try:
dateref = get_dateref_or_from_cron(dateref)
dDate = dateref
except ValueError:
logging.error(
"Wrong updating date = {dateref}".format(dateref=dateref))
self.response.write("Wrong updating date = {}".format(dateref))
return
try:
decoratorDate = "".join(dDate.split("-"))
table_name = 'customer_usage${decoratorDate}'.format(
decoratorDate=decoratorDate)
# delete table if it exists to avoid data duplication
delete_table_big_query(table_name)
for report_items in returnCustomerUsageReport(
dDate, cfg['credentials']['general'],
cfg['super_admin']['delegated']):
try:
writeDatainBigQuery(report_items, table_name)
logging.info('Customer Usage for ' + dDate + ' - ' +
cfg['domains'] + ' / ')
self.response.write('Customer Usage for ' + dDate + ' - ' +
cfg['domains'] + ' / ')
except Exception as err:
bvi_log(date=dDate,
resource='customer_usage',
message_id='bigquery_error',
message=err,
regenerate=True)
raise err
except Exception as err:
logging.error(err)
self.response.write('Customer Usage for ' + dDate + ' - ' +
cfg['domains'] + ': ERROR')
raise err
def get(self):
# User Activities for a date and app One page at a time
token = self.request.get('token')
app = self.request.get('app')
dDate = self.request.get('date')
decoratorDate = ("").join(dDate.split("-"))
for report_items in returnActivitiesPageToken(
token, app, dDate, cfg['credentials']['general'],
cfg['super_admin']['delegated']):
try:
bq_answer = writeDatainBigQuery(
report_items, 'audit_log${decoratorDate}'.format(
decoratorDate=decoratorDate))
except Exception as err:
bvi_log(date=dDate,
resource='activities',
message_id='bigquery_error',
message=err,
regenerate=True)
raise err
def get(self):
# User List for a date One page at a time
token = self.request.get('token')
domain = self.request.get('domain')
dDate = self.request.get('date')
decoratorDate = ("").join(dDate.split("-"))
try:
for report_items in returnUserListPageToken(token, dDate, domain, cfg['credentials']['general'],
cfg['super_admin']['delegated']):
try:
bq_answer = writeDatainBigQuery(report_items, 'users_list_date${decoratorDate}'.format(
decoratorDate=decoratorDate))
except Exception as err:
bvi_log(date=dDate, resource='users_list', message_id='bigquery_error', message=err,
regenerate=True)
raise err
except Exception as err:
logging.error(err)
self.response.write('Users List for ' + dDate + ' - ' + domain + ': ERROR')
raise err
bvi_log(date=dDate, resource='users_list', message_id='end', message='End of /one_page_user_list call')
def get(self):
# User List today
today = date.today()
dDate = today.strftime("%Y-%m-%d")
if cfg['plan'] == 'Enterprise':
bvi_log(date=dDate,
resource='exec',
message_id='start',
message='Start of BVI daily execution')
bvi_log(date=dDate,
resource='users_list',
message_id='start',
message='Start of /user call')
page_token = ''
maxPages = cfg['task_management']['max_pages']
queue_name = cfg['queues']['user'] + str(1)
domains = list(set(cfg['domains'].split(";")))
for domain in domains:
domain = domain.strip()
tNow = datetime.now()
aNumber = tNow.strftime("%S")
aRandomNumber = random.randint(0, sys.maxsize)
aNumber = "{aNumber}_{aRandomNumber}".format(
aNumber=aNumber, aRandomNumber=aRandomNumber)
#1st page for user
taskqueue.add(
queue_name=queue_name,
name='opul' + dDate + '_' + str(maxPages) + '_' + aNumber,
url='/one_page_user_list?token=&date=' + dDate + '&domain=' +
domain,
method='GET')
#Starts user delay with token empty for 2nd page
taskqueue.add(queue_name=queue_name,
name='ud' + dDate + '_' + aNumber,
url='/user_delay?token=' + page_token + '&domain=' +
domain + '&maxPages=' + str(maxPages),
method='GET')
logging.info('User list main for {} - {} / max pages {}'.format(
dDate, cfg['domains'], maxPages))
self.response.write('User list main for {} - {} / max pages {}'.format(
dDate, cfg['domains'], maxPages))
bvi_log(date=dDate,
resource='users_list',
message_id='end',
message='end of /user call')
def get(self):
project_id = cfg['ids']['project_id']
# getting request parameters
exec_type = self.request.get('type', 'daily')
step = self.request.get('step')
begin_step = self.request.get('begin_step')
dateref = self.request.get('dateref')
start_date = self.request.get('Sdate')
end_date = self.request.get('Edate')
auto_recover = self.request.get('auto_recover', False)
if auto_recover == 'False' or not auto_recover:
auto_recover = False
enable_auto_recover = self.request.get('enable_auto_recover', True)
if enable_auto_recover == 'False' or not enable_auto_recover:
enable_auto_recover = False
if should_check_for_auto_recover() and enable_auto_recover and exec_type == 'daily' \
and step == 'first' and begin_step:
# verifying if an error occurred in the last days, only in every 'frequency' days
logging.info("[auto-recover] Verifying need to execute auto-recover...")
bigquery = createBigQueryService(cfg['scopes']['big_query'], 'bigquery', 'v2')
query = "SELECT MIN(min_date) as first_fail FROM [{}:logs.errors_dashboard]".format(project_id)
result = fetch_big_query_data(bigquery, project_id, query, 10)
rows = convert_big_query_result(result, ERROR_BEGIN)
if len(rows) == 1 and rows[0]['first_fail']:
exec_type = 'historical'
start_date = rows[0]['first_fail']
end_date = dateref
auto_recover = True
logging.info("[auto-recover] Error occurred in a previous day, moving to historical execution to run "
+ "again since the first failed execution date. "
+ "auto-recover starting from {}".format(start_date))
else:
logging.info("[auto-recover] Not needed, no errors found in last {} days.".format(
cfg['auto_recover']['days_lookback']))
log_date = dateref
if exec_type == 'daily':
date_params = '&dateref={}&enable_auto_recover={}'.format(dateref, str(enable_auto_recover))
elif exec_type == 'historical':
date_params = '&Sdate={}&Edate={}&enable_auto_recover={}'.format(start_date,
end_date, str(enable_auto_recover))
log_date = start_date
if step == 'first' and begin_step:
bvi_log(date=log_date, resource='exec_manager', message_id='start',
message='Start of BVI {} execution'.format(exec_type))
with open('manager.yaml', 'r') as mgrymlfile:
mgr = yaml.load(mgrymlfile)
exec_manager_queue = cfg['queues']['exec_manager']
if begin_step:
if exec_type == 'daily':
bvi_log(date=log_date, resource=get_log_step(step), message_id='start',
message='Start of {} step'.format(step))
endpoint = mgr[step]['endpoint'].replace('from_cron', dateref)
taskqueue.add(queue_name=exec_manager_queue, url=endpoint, method='GET')
elif exec_type == 'historical':
exec_historical(mgr, step, start_date, end_date)
# wait for tasks to be created in the queue
time.sleep(15)
count_tasks = count_queued_tasks(mgr[step].get('queues'))
if count_tasks == 0 and 'next_step' in mgr[step] and mgr[step]['next_step']:
# Finished all tasks from this step
if auto_recover and 'missing_data_table' in mgr[step]:
# Check if the auto-recover was successful
logging.info("[auto-recover] Checking for effectiveness...")
lookback_date_obj = date.today() - timedelta(days=cfg['auto_recover']['days_lookback'])
lookback_date = lookback_date_obj.strftime("%Y-%m-%d")
bigquery = createBigQueryService(cfg['scopes']['big_query'], 'bigquery', 'v2')
check_query = "SELECT MIN(report_date) AS report_date FROM [{}:{}] " \
"WHERE report_date > \'{}\'".format(project_id,
mgr[step]['missing_data_table'],
lookback_date)
check_result = fetch_big_query_data(bigquery, project_id, check_query, 10)
if 'rows' in check_result:
check_rows = convert_big_query_result(check_result, CHECK_ROW)
if len(check_rows) == 1 and check_rows[0]['report_date']:
min_error_date = check_rows[0]['report_date']
min_error_date_obj = datetime.strptime(min_error_date, "%Y-%m-%d").date()
start_date_obj = datetime.strptime(start_date, "%Y-%m-%d").date()
if min_error_date_obj > start_date_obj:
logging.info(
"[auto-recover] Min error date for '{}' ".format(mgr[step]['missing_data_table'])
+ "is greater than start_date, auto-recover should proceed.")
else:
logging.info("[auto-recover] Could not fix any missing data for '{}'. ".format(
mgr[step]['missing_data_table'])
+ "Reverting to daily ({}) execution.".format(end_date))
exec_type = 'daily'
date_params = '&dateref={}&enable_auto_recover=False'.format(end_date)
auto_recover = False
else:
logging.info(
"[auto-recover] No missing data for '{}', auto-recover should proceed.".format(
mgr[step]['missing_data_table']))
else:
logging.info(
"[auto-recover] No missing data for '{}', auto-recover should proceed.".format(
mgr[step]['missing_data_table']))
logging.info("[auto-recover] Finished checking for effectiveness.")
# Execute next step
bvi_log(date=log_date, resource=get_log_step(step), message_id='end',
message='End of {} step'.format(get_log_step(step)))
taskqueue.add(queue_name=exec_manager_queue,
url='/exec_manager?type={}{}&step={}&begin_step=True&auto_recover={}'.format(
exec_type, date_params, mgr[step]['next_step'], auto_recover),
method='GET')
elif count_tasks > 0:
# Still executing tasks, just schedule to monitor task queues again 10 seconds later
logging.info("Waiting for tasks to finish...")
taskqueue.add(queue_name=exec_manager_queue,
url='/exec_manager?type={}{}&step={}&auto_recover={}'.format(
exec_type, date_params, step, auto_recover),
method='GET', countdown=10)
else:
# Finished ALL tasks
bvi_log(date=log_date, resource='exec_manager', message_id='end',
message='End of BVI {} execution'.format(exec_type))
def update_data_level(self, bigquery, op, bigquery_setup):
level = self.request.get('level')
target = self.request.get('target')
dateref = self.request.get('dateref')
logging.info("update {} {} {}".format(level, target, dateref))
try:
if dateref == "from_cron":
today = datetime.date.today()
today_4 = today - datetime.timedelta(days=4)
dateref = today_4.strftime("%Y-%m-%d")
yyyy, mm, dd = dateref.split("-")
timestamp = datetime.datetime(int(yyyy), int(mm), int(dd))
except ValueError:
logging.error("Wrong updating date = {dateref}".format(dateref=dateref))
self.response.write("Wrong updating date = {}".format(dateref))
return
if level_is_valid(level) or len(target) > 0:
if level_is_valid(level) and not target:
# only first time
bvi_log(date=dateref, resource='bq_{op}_level{level}'.format(op=op, level=level), message_id='start',
message='Start of /bq_api call')
for index, table_def in enumerate(bigquery_setup['tables']):
if target:
target_dataset, target_name = target.split(".")
if target_dataset is None or target_name is None:
logging.error("Wrong target {target}".format(target=target))
self.response.write("Wrong target {target}".format(target=target))
if level_is_valid(level):
bvi_log(date=dateref, resource='bq_{op}_level{level}'.format(op=op, level=level),
message_id='bq_api_wrong_target', message="Wrong dataset/table target", regenerate=True)
return
elif table_def.get('dataset') == target_dataset and table_def.get('name') == target_name:
logging.info("Request for updating {}".format(target_name))
decoratorDate = None
if table_def.get('timePartitioning'):
decoratorDate = "{yyyy}{mm}{dd}".format(yyyy=yyyy, mm=mm, dd=dd)
try:
do_create_view(self, bigquery, bigquery_setup['folder'], table_def['dataset'],
"{table}".format(table=table_def['name']), True, overwrite=True,
timestamp=timestamp, decoratorDate=decoratorDate)
except Exception as err:
bvi_log(date=dateref, resource='bq_{op}_level{level}'.format(op=op, level=level),
message_id='bigquery_error', message=err, regenerate=True)
raise err
elif table_def.get('level') and str(table_def.get('level')) == str(level):
logging.info("Request for updating {} (massive level {} update)".format(table_def.get('name'), level))
queue_name = "bqUpdateByLevel{level}".format(level=level)
tNow = datetime.datetime.now()
aNumber = tNow.strftime("%S")
aRandomNumber = random.randint(0, sys.maxsize)
aNumber = "{aNumber}_{aRandomNumber}".format(aNumber=aNumber, aRandomNumber=aRandomNumber)
task = taskqueue.add(queue_name=queue_name,
name='bqupdate_level_' + level + '_' + aNumber + '-' + table_def.get(
'dataset') + 'DOT' + table_def.get('name'),
url="/bq_api?op={op}&level={level}&target={dataset}.{table}&dateref={dateref}".format(
op=op, level=level, dataset=table_def.get('dataset'), table=table_def.get('name'),
dateref=dateref), method='GET')
self.response.write("Task {}<br/> for table {}.{} on {} enqueued,<br/>ETA {}<hr/>".format(task.name, table_def.get('dataset'), table_def.get('name'), dateref, task.eta))
bvi_log(date=dateref, resource='bq_{op}_level{level}'.format(op=op, level=level), message_id='end',
message='End of /bq_api call')
else:
logging.info(
"Nothing to do with table {index} = {name}".format(index=index, name=table_def.get("name", "name")))
if level_is_valid(level):
bvi_log(date=dateref, resource='bq_{op}_level{level}'.format(op=op, level=level), message_id='end',
message='End of /bq_api call')
else:
logging.error("Wrong parameter level={level}, target={target}".format(level=level, target=target))
self.response.write("Wrong parameter level={level}, target={target}".format(level=level, target=target))
def get(self):
logging.info('User Usage')
dateref = self.request.get('date', "from_cron")
if len(dateref) > 0:
try:
if dateref == "from_cron":
# Customer Usage day -4 from February 2018
today = date.today()
today_4 = today - timedelta(days=4)
dateref = today_4.strftime("%Y-%m-%d")
yyyy, mm, dd = dateref.split("-")
timestamp = datetime(int(yyyy), int(mm), int(dd))
except ValueError:
logging.error(
"Wrong updating date = {dateref}".format(dateref=dateref))
self.response.write("Wrong updating date = {}".format(dateref))
return
try:
pages = 0
page_token = self.request.get('token')
if not page_token:
bvi_log(date=dateref,
resource='user_usage',
message_id='start',
message='Start of /user_usage call')
maxPages = cfg['task_management']['max_pages']
maxPages = int(maxPages)
numQueues = 5
queue_name = []
for x in range(1, numQueues + 1):
queue_name.append(cfg['queues']['user_usage'] + str(x))
tNow = datetime.now()
aNumber = tNow.strftime("%S")
aRandomNumber = random.randint(0, sys.maxsize)
aNumber = "{aNumber}_{aRandomNumber}".format(
aNumber=aNumber, aRandomNumber=aRandomNumber)
# 1st page for user_usage
if not page_token:
taskqueue.add(queue_name=queue_name[1],
name='opuu_1_' + dateref + '_' + aNumber,
url='/one_page_user_usage?token=&date=' +
dateref,
method='GET')
for pages_token in returnUserUsageToken(
dateref, cfg['credentials']['general'],
cfg['super_admin']['delegated'], page_token):
if not pages_token:
break
pages += 1
tNow = datetime.now()
aNumber = tNow.strftime("%S")
aRandomNumber = random.randint(0, sys.maxsize)
aNumber = "{aNumber}_{aRandomNumber}".format(
aNumber=aNumber, aRandomNumber=aRandomNumber)
qA = random.randint(0, numQueues - 1)
taskqueue.add(queue_name=queue_name[qA],
name='opuu' + dateref + '_' + str(pages) + '_' +
aNumber,
url='/one_page_user_usage?token=' + pages_token +
'&date=' + dateref,
method='GET')
if pages >= maxPages:
taskqueue.add(queue_name=queue_name[qA],
name='uu' + dateref + '_' + aNumber,
url='/user_usage?token=' + pages_token +
'&date=' + dateref,
method='GET')
break
logging.info(
'User Usage pages for {} - {} / finally {} pages '.format(
dateref, cfg['domains'], pages))
self.response.write("User Usage for {} - {} / {} pages".format(
dateref, cfg['domains'], pages))
except Exception as err:
logging.error(err)
self.response.write('User Usage for ' + dateref + ' - ' +
cfg['domains'] + ': ERROR')
raise err
bvi_log(date=dateref,
resource='user_usage',
message_id='end',
message='End of /user_usage call')
def get(self):
logging.info('Activities')
appName = self.request.get('app')
dateref = self.request.get('date')
bvi_log(date=dateref, resource='activities', message_id='start_app', message='Start of /activities_app call')
try:
pages = 0
page_token = self.request.get('token')
maxPages = cfg['task_management']['max_pages']
maxPages = int(maxPages)
numQueues = 5
queue_name = []
for x in range(1, numQueues + 1):
queue_name.append(cfg['queues']['activities'] + str(x))
tNow = datetime.now()
aNumber = tNow.strftime("%S")
aRandomNumber = random.randint(0, sys.maxsize)
aNumber = "{aNumber}_{aRandomNumber}".format(aNumber=aNumber, aRandomNumber=aRandomNumber)
# 1st page for activities
if not page_token:
taskqueue.add(queue_name=queue_name[1],
name='apua_1_' + appName + '_' + dateref + '_' + aNumber,
url='/one_page_user_activities?token=&date=' + dateref + '&app=' + appName,
method='GET')
for pages_token in returnActivitiesToken(dateref, appName, cfg['credentials']['general'],
cfg['super_admin']['delegated'], page_token):
if not pages_token:
break
pages += 1
logging.info('Pages {} of maxPages {}'.format)
logging.info('Activities pages for {} - {} / page {} so far until now'.format(dateref, appName, pages))
tNow = datetime.now()
aNumber = tNow.strftime("%S")
aRandomNumber = random.randint(0, sys.maxsize)
aNumber = "{aNumber}_{aRandomNumber}".format(aNumber=aNumber, aRandomNumber=aRandomNumber)
qA = random.randint(0, numQueues - 1)
taskqueue.add(queue_name=queue_name[qA],
name='apua_' + appName + '_' + dateref + '_' + str(pages) + '_' + aNumber,
url='/one_page_user_activities?token=' + pages_token + '&date=' + dateref + '&app=' + appName,
method='GET')
if pages >= maxPages:
taskqueue.add(queue_name=queue_name[qA],
name='act' + dateref + '_' + appName + '_' + aNumber,
url='/activities_app?token=' + pages_token + '&app=' + appName + '&date=' + dateref,
method='GET')
break
logging.info('User Usage pages for {} - {} / finally {} pages '.format(dateref, appName, pages))
self.response.write("User Usage for {} - {} / {} pages".format(dateref, appName, pages))
except Exception as err:
logging.error(err)
self.response.write('Activities for ' + dateref + ' - ' + cfg['domains'] + ': ERROR')
raise err
bvi_log(date=dateref, resource='activities', message_id='end_app', message='End of /activities_app call')
