def returnActivitiesPageToken(token, appName, dDay, SAJson, SADelegated): startTime = dDay + 'T00:00:00.000Z' endTime = dDay + 'T23:59:59.999Z' page_token = token activities = [] try: reports = createReportObject(cfg['scopes']['audit_log'], 'admin', 'reports_v1', SAJson, SADelegated) except Exception as err: bvi_log(date=dDay, resource='activities', message_id='invalid_credential', message=err, regenerate=True) raise err # AVAILABLE FIELDS: https://developers.google.com/admin-sdk/reports/v1/reference/activity-ref-appendix-a/admin-event-names and more. fields = 'items(actor,events,id),nextPageToken' try: if page_token == '': request = reports.activities().list(userKey='all', applicationName=appName, startTime=startTime, endTime=endTime, fields=fields, maxResults=maxResultsPage) else: request = reports.activities().list(userKey='all', applicationName=appName, startTime=startTime, endTime=endTime, fields=fields, pageToken=page_token, maxResults=maxResultsPage) results = execute_request_with_retries(request) logging.info(results) activities = results.get('items', []) except Exception as err: logging.error(err) logging.error("Error Found, ending returnActivitiesPageToken!") bvi_log(date=dDay, resource='activities', message_id='activities_api', message=err, regenerate=True) logging.info("We have {} activities rows in the end".format(len(activities))) yield activities
def exec_historical(mgr, step, SdDate, EdDate): logging.info('Create Historical data') start_date = date(int(SdDate.split('-')[0]), int(SdDate.split('-')[1]), int(SdDate.split('-')[2])) end_date = date(int(EdDate.split('-')[0]), int(EdDate.split('-')[1]), int(EdDate.split('-')[2])) today = date.today() today_4 = today - timedelta(days=4) if start_date > today_4: logging.info('Error: Start Date > Today - 4 days, try another Start Date') return if end_date > today_4: logging.info('Error: End Date > Today - 4 days, try another End Date') return Number_days = abs((start_date - end_date).days) dDate = SdDate iterating_day = start_date n = 0 while n <= int(Number_days): bvi_log(date=dDate, resource=get_log_step(step), message_id='start', message='Start of {} step'.format(step)) endpoint = mgr[step]['endpoint'].replace('from_cron', dDate) logging.info(endpoint) taskqueue.add(queue_name=cfg['queues']['slow'], url=endpoint, method='GET') iterating_day = iterating_day + timedelta(days=1) dDate = iterating_day.strftime("%Y-%m-%d") n += 1 logging.info('Sent ' + str(n) + ' days request for ' + step)
def returnUserUsagePageToken(token, dDay, SAJson, SADelegated): dDate = dDay page_token = token user_usage = [] try: reports = createReportObject(cfg['scopes']['admin_report'], 'admin', 'reports_v1', SAJson, SADelegated) except Exception as err: bvi_log(date=dDay, resource='user_usage', message_id='invalid_credential', message=err, regenerate=True) raise err # AVAILABLE FIELDS: https://developers.google.com/admin-sdk/reports/v1/reference/usage-ref-appendix-a/users fields = 'nextPageToken,usageReports(date,entity,parameters)' try: if page_token == '': request = reports.userUsageReport().get(userKey='all', date=dDay, fields=fields, maxResults=maxResultsPage_UserUsage) else: request = reports.userUsageReport().get(userKey='all', date=dDay, fields=fields, pageToken=page_token, maxResults=maxResultsPage_UserUsage) results = execute_request_with_retries(request) user_usage = results.get('usageReports', []) except Exception as err: logging.error(err) logging.error("Error Found, ending returnUserUsagePageToken!") bvi_log(date=dDay, resource='user_usage', message_id='user_usage_api', message=err, regenerate=True) logging.info("We have {} user_usage rows in the end".format(len(user_usage))) yield user_usage
def returnUserListPageToken(token, dDay, dDomain, SAJson, SADelegated): dDate = dDay page_token = token users = [] try: reports = createReportObject(cfg['scopes']['admin_directory'], 'admin', 'directory_v1', SAJson, SADelegated) except Exception as err: bvi_log(date=dDate, resource='users_list', message_id='invalid_credential', message=err, regenerate=True) raise err # AVAILABLE FIELDS: https://developers.google.com/admin-sdk/directory/v1/reference/users#resource fields = 'nextPageToken,users(creationTime,customerId,emails,lastLoginTime,orgUnitPath,primaryEmail)' try: if page_token == '': request = reports.users().list(domain=dDomain, maxResults=maxResultsPage, fields=fields) else: request = reports.users().list(domain=dDomain, pageToken=page_token, maxResults=maxResultsPage, fields=fields) results = execute_request_with_retries(request) if 'users' in results: users = results['users'] for user_item in users: user_item[u'date'] = dDate except DeadlineExceededError as err: logging.error(err) logging.error("Retrying!") except Exception as err: logging.error(err) logging.error(dDomain) logging.error("Error Found, ending returnUserListPageToken!") bvi_log(date=dDate, resource='users_list', message_id='users_list_api', message=err, regenerate=True) logging.info("We have {} user rows in the end".format(len(users))) yield users
def get(self): # User List today today = date.today() dDate = today.strftime("%Y-%m-%d") pages = 0 page_token = self.request.get('token') domain = self.request.get('domain') maxPages = self.request.get('maxPages') maxPages = int(maxPages) numQueues = 3 queue_name = [] for x in range(1, numQueues + 1): queue_name.append(cfg['queues']['user'] + str(x)) for pages_token in returnUsersListToken( domain, cfg['credentials']['general'], cfg['super_admin']['delegated'], page_token): if not pages_token: break pages += 1 logging.info( 'User list pages for {} - {} / page {} so far until now'. format(dDate, domain, pages)) tNow = datetime.now() aNumber = tNow.strftime("%S") aRandomNumber = random.randint(0, sys.maxsize) aNumber = "{aNumber}_{aRandomNumber}".format( aNumber=aNumber, aRandomNumber=aRandomNumber) qA = random.randint(0, numQueues - 1) taskqueue.add( queue_name=queue_name[qA], name='opul' + dDate + '_' + str(pages) + '_' + aNumber, url='/one_page_user_list?token=' + pages_token + '&date=' + dDate, method='GET') if pages >= maxPages: taskqueue.add(queue_name=queue_name[qA], name='ud' + dDate + '_' + aNumber, url='/user_delay?token=' + pages_token + '&maxPages=' + str(maxPages), method='GET') break logging.info('User list pages for {} - {} / finally {} pages '.format( dDate, domain, pages)) self.response.write("User Usage for {} - {} / {} pages".format( dDate, domain, pages)) bvi_log(date=dDate, resource='users_list', message_id='end', message='End of /user_delay call')
def returnActivitiesToken(dDay, appName, SAJson, SADelegated, page_token): tokens = [] startTime = dDay + 'T00:00:00.000Z' endTime = dDay + 'T23:59:59.999Z' try: reports = createReportObject(cfg['scopes']['audit_log'], 'admin', 'reports_v1', SAJson, SADelegated) except Exception as err: bvi_log(date=dDay, resource='activities', message_id='invalid_credential', message=err, regenerate=True) raise err fields = 'nextPageToken' while True: try: if page_token == '': request = reports.activities().list(userKey='all', applicationName=appName, startTime=startTime, endTime=endTime, fields=fields, maxResults=maxResultsPage) else: request = reports.activities().list(userKey='all', applicationName=appName, startTime=startTime, endTime=endTime, fields=fields, pageToken=page_token, maxResults=maxResultsPage) results = execute_request_with_retries(request) if 'nextPageToken' in results: tokens = results['nextPageToken'] page_token = results['nextPageToken'] yield tokens else: break except DeadlineExceededError as err: logging.error(err) logging.error("Retrying!") except Exception as err: logging.error(err) logging.error("Error Found, ending returnActivitiesToken !") tokens = [] bvi_log(date=dDay, resource='activities', message_id='activities_api', message=err, regenerate=True) break yield tokens
def returnUserUsageToken(dDay, SAJson, SADelegated, page_token): tokens = [] try: reports = createReportObject(cfg['scopes']['admin_report'], 'admin', 'reports_v1', SAJson, SADelegated) except Exception as err: bvi_log(date=dDay, resource='user_usage', message_id='invalid_credential', message=err, regenerate=True) raise err # AVAILABLE FIELDS: https://developers.google.com/admin-sdk/reports/v1/reference/usage-ref-appendix-a/users fields = 'nextPageToken' pages = 0 while True: try: if page_token == '': request = reports.userUsageReport().get( userKey='all', date=dDay, fields=fields, maxResults=maxResultsPage_UserUsage) else: request = reports.userUsageReport().get( userKey='all', date=dDay, fields=fields, pageToken=page_token, maxResults=maxResultsPage_UserUsage) results = request.execute() pages += 1 if 'nextPageToken' in results: tokens = results['nextPageToken'] page_token = results['nextPageToken'] yield tokens else: break except DeadlineExceededError as err: logging.error(err) logging.error("Retrying!") except Exception as err: logging.error(err) logging.error("Error Found, ending returnUserUsageToken !") tokens = [] bvi_log(date=dDay, resource='user_usage', message_id='user_usage_api', message=err, regenerate=True) break logging.info("We have {} user_usage pages in the end".format(pages)) logging.info(tokens) yield tokens
def get(self): logging.info('Activities') dateref = self.request.get('date', "from_cron") if len(dateref) > 0: try: if dateref == "from_cron": # Customer Usage day -4 from February 2018 today = date.today() today_4 = today - timedelta(days=4) dateref = today_4.strftime("%Y-%m-%d") except ValueError: logging.error( "Wrong updating date = {dateref}".format(dateref=dateref)) self.response.write("Wrong updating date = {}".format(dateref)) return bvi_log(date=dateref, resource='activities', message_id='start', message='Start of /activities call') collection = 'admin|drive|calendar|gplus' collection = list(set(collection.split("|"))) queue_name = cfg['queues']['slow'] tNow = datetime.now() aNumber = tNow.strftime("%S") aRandomNumber = random.randint(0, sys.maxsize) aNumber = "{aNumber}_{aRandomNumber}".format( aNumber=aNumber, aRandomNumber=aRandomNumber) processed = 0 for app in collection: taskqueue.add(queue_name=queue_name, name='act' + dateref + '_' + app + '_' + aNumber, url='/activities_app?app=' + app + '&date=' + dateref, method='GET') processed += 1 logging.info('Activities for {} - {} / finally {} apps '.format( dateref, cfg['domains'], processed)) self.response.write("Activities for {} - {} / finally {} apps".format( dateref, cfg['domains'], processed)) bvi_log(date=dateref, resource='activities', message_id='end', message='End of /activities call')
def get(self): # User Usage for a date One page at a time token = self.request.get('token') dDate = self.request.get('date') decoratorDate = "".join(dDate.split("-")) for report_items in returnUserUsagePageToken(token, dDate, cfg['credentials']['general'], cfg['super_admin']['delegated']): try: writeDatainBigQuery(report_items, 'user_usage${decoratorDate}'.format(decoratorDate=decoratorDate)) except Exception as err: bvi_log(date=dDate, resource='user_usage', message_id='bigquery_error', message=err, regenerate=True) raise err
def get(self): logging.info('Customer Usage') dateref = self.request.get('date', "from_cron") if len(dateref) > 0: try: if dateref == "from_cron": # Customer Usage day -4 from February 2018 today = date.today() today_4 = today - timedelta(days=4) dateref = today_4.strftime("%Y-%m-%d") yyyy, mm, dd = dateref.split("-") timestamp = datetime(int(yyyy), int(mm), int(dd)) dDate = dateref except ValueError: logging.error( "Wrong updating date = {dateref}".format(dateref=dateref)) self.response.write("Wrong updating date = {}".format(dateref)) return try: bvi_log(date=dDate, resource='exec', message_id='start', message='Start of BVI daily execution') bvi_log(date=dDate, resource='customer_usage', message_id='start', message='Start of /customer call') decoratorDate = ("").join(dDate.split("-")) for report_items in returnCustomerUsageReport( dDate, cfg['credentials']['general'], cfg['super_admin']['delegated']): try: bq_answer = writeDatainBigQuery( report_items, 'customer_usage${decoratorDate}'.format( decoratorDate=decoratorDate)) logging.info('Customer Usage for ' + dDate + ' - ' + cfg['domains'] + ' / ') self.response.write('Customer Usage for ' + dDate + ' - ' + cfg['domains'] + ' / ') except Exception as err: bvi_log(date=dDate, resource='customer_usage', message_id='bigquery_error', message=err, regenerate=True) raise err except Exception as err: logging.error(err) self.response.write('Customer Usage for ' + dDate + ' - ' + cfg['domains'] + ': ERROR') raise err bvi_log(date=dDate, resource='customer_usage', message_id='end', message='End of /customer call')
def returnUsersListToken(dDomain, SAJson, SADelegated, page_token): dDate = date.today().strftime("%Y-%m-%d") try: reports = createReportObject(cfg['scopes']['admin_directory'], 'admin', 'directory_v1', SAJson, SADelegated) except Exception as err: bvi_log(date=dDate, resource='users_list', message_id='invalid_credential', message=err, regenerate=True) raise err # AVAILABLE FIELDS: https://developers.google.com/admin-sdk/directory/v1/reference/users#resource fields = 'nextPageToken' tokens = [] while True: try: if page_token == '': request = reports.users().list(domain=dDomain, maxResults=maxResultsPage, fields=fields) else: request = reports.users().list(domain=dDomain, pageToken=page_token, maxResults=maxResultsPage, fields=fields) results = execute_request_with_retries(request) if 'nextPageToken' in results: tokens = results['nextPageToken'] page_token = results['nextPageToken'] yield tokens else: break except DeadlineExceededError as err: logging.error(err) logging.error("Retrying!") except Exception as err: logging.error(err) logging.error("Error Found, ending returnUsersListToken!") bvi_log(date=dDate, resource='users_list', message_id='users_list_api_token', message=err, regenerate=True) tokens = [] break yield tokens
def returnCustomerUsageReport(dDay, SAJson, SADelegated): usage = [] page_token = None try: reports = createReportObject(cfg['scopes']['admin_report'], 'admin', 'reports_v1', SAJson, SADelegated) except Exception as err: bvi_log(date=dDay, resource='customer_usage', message_id='invalid_credential', message=err, regenerate=True) raise err fields = 'nextPageToken,usageReports(date,entity,parameters)' parameters = 'accounts:gsuite_enterprise_used_licenses,accounts:gsuite_enterprise_total_licenses,accounts:gsuite_unlimited_used_licenses,accounts:gsuite_unlimited_total_licenses,accounts:gsuite_basic_used_licenses,accounts:gsuite_basic_total_licenses,accounts:num_disabled_accounts,accounts:num_suspended_users,accounts:num_users,accounts:apps_total_licenses,accounts:apps_used_licenses,accounts:vault_total_licenses,accounts:gsuite_enterprise_used_licenses,accounts:gsuite_basic_used_licenses,accounts:gsuite_unlimited_used_licenses,accounts:gsuite_enterprise_total_licenses,accounts:gsuite_unlimited_total_licenses,accounts:gsuite_basic_total_licenses,accounts:num_1day_logins,accounts:num_7day_logins,accounts:num_30day_logins,' \ 'calendar:num_1day_active_users,calendar:num_7day_active_users,calendar:num_30day_active_users,' \ 'cros:num_7day_active_devices,cros:num_30day_active_devices,' \ 'classroom:num_1day_users,classroom:num_7day_users,classroom:num_30day_users,' \ 'drive:num_1day_active_users,drive:num_7day_active_users,drive:num_30day_active_users,' \ 'docs:num_docs,drive:num_1day_active_users,drive:num_7day_active_users,drive:num_30day_active_users,drive:num_30day_google_documents_active_users,drive:num_30day_google_spreadsheets_active_users,drive:num_30day_google_presentations_active_users,drive:num_30day_google_forms_active_users,drive:num_30day_google_drawings_active_users,drive:num_30day_other_types_active_users,drive:num_creators,drive:num_collaborators,drive:num_consumers,drive:num_sharers,' \ 'gmail:num_1day_active_users,gmail:num_7day_active_users,gmail:num_30day_active_users,' \ 'gplus:num_1day_active_users,gplus:num_7day_active_users,gplus:num_30day_active_users,' \ 'device_management:num_30day_google_sync_managed_devices,device_management:num_30day_android_managed_devices,device_management:num_30day_ios_managed_devices,device_management:num_30day_total_managed_devices,device_management:num_30day_google_sync_managed_users,device_management:num_30day_android_managed_users,device_management:num_30day_ios_managed_users,device_management:num_30day_total_managed_users,' \ 'sites:num_sites,sites:num_sites_created,' \ 'meet:average_meeting_minutes,meet:average_meeting_minutes_with_11_to_15_calls,meet:average_meeting_minutes_with_16_to_25_calls,meet:average_meeting_minutes_with_26_to_50_calls,meet:average_meeting_minutes_with_2_calls,meet:average_meeting_minutes_with_3_to_5_calls,meet:average_meeting_minutes_with_6_to_10_calls,meet:lonely_meetings,meet:max_concurrent_usage_chromebase,meet:max_concurrent_usage_chromebox,meet:num_1day_active_users,meet:num_30day_active_users,meet:num_7day_active_users,' \ 'meet:num_calls,meet:num_calls_android,meet:num_calls_by_external_users,meet:num_calls_by_internal_users,meet:num_calls_by_pstn_in_users,meet:num_calls_chromebase,meet:num_calls_chromebox,meet:num_calls_ios,meet:num_calls_jamboard,meet:num_calls_unknown_client,meet:num_calls_web,meet:num_meetings,meet:num_meetings_android,meet:num_meetings_chromebase,meet:num_meetings_chromebox,meet:num_meetings_ios,meet:num_meetings_jamboard,meet:num_meetings_unknown_client,meet:num_meetings_web,' \ 'meet:num_meetings_with_11_to_15_calls,meet:num_meetings_with_16_to_25_calls,meet:num_meetings_with_26_to_50_calls,meet:num_meetings_with_2_calls,meet:num_meetings_with_3_to_5_calls,meet:num_meetings_with_6_to_10_calls,meet:num_meetings_with_external_users,meet:num_meetings_with_pstn_in_users,meet:total_call_minutes,meet:total_call_minutes_android,meet:total_call_minutes_by_external_users,meet:total_call_minutes_by_internal_users,meet:total_call_minutes_by_pstn_in_users,' \ 'meet:total_call_minutes_chromebase,meet:total_call_minutes_chromebox,meet:total_call_minutes_ios,meet:total_call_minutes_jamboard,meet:total_call_minutes_unknown_client,meet:total_call_minutes_web,meet:total_meeting_minutes,' \ 'gplus:num_1day_active_users,gplus:num_7day_active_users,gplus:num_30day_active_users,gplus:num_shares,gplus:num_stream_items_read,gplus:num_plusones,gplus:num_replies,gplus:num_reshares,gplus:num_communities,gplus:num_communities_public,gplus:num_communities_private,gplus:num_communities_organization_wide,gplus:num_communities_organization_private' while True: try: request = reports.customerUsageReports().get(date=dDay, fields=fields, pageToken=page_token, parameters=parameters) results = execute_request_with_retries(request) usage = results.get('usageReports', []) if 'nextPageToken' in results: page_token = results['nextPageToken'] logging.info("We have {} customer_usage rows, and more to come".format(len(usage))) yield usage else: break except DeadlineExceededError as err: logging.error(err) logging.error("Retrying!") except Exception as err: logging.error(err) logging.error("Error Found, ending returnCustomerUsageReport!") bvi_log(date=dDay, resource='customer_usage', message_id='customer_usage_api', message=err, regenerate=True) break logging.info("We have {} customer_usage rows in the end".format(len(usage))) yield usage
def get(self): logging.info('Customer Usage') dateref = self.request.get('date', "from_cron") if len(dateref) > 0: try: dateref = get_dateref_or_from_cron(dateref) dDate = dateref except ValueError: logging.error( "Wrong updating date = {dateref}".format(dateref=dateref)) self.response.write("Wrong updating date = {}".format(dateref)) return try: decoratorDate = "".join(dDate.split("-")) table_name = 'customer_usage${decoratorDate}'.format( decoratorDate=decoratorDate) # delete table if it exists to avoid data duplication delete_table_big_query(table_name) for report_items in returnCustomerUsageReport( dDate, cfg['credentials']['general'], cfg['super_admin']['delegated']): try: writeDatainBigQuery(report_items, table_name) logging.info('Customer Usage for ' + dDate + ' - ' + cfg['domains'] + ' / ') self.response.write('Customer Usage for ' + dDate + ' - ' + cfg['domains'] + ' / ') except Exception as err: bvi_log(date=dDate, resource='customer_usage', message_id='bigquery_error', message=err, regenerate=True) raise err except Exception as err: logging.error(err) self.response.write('Customer Usage for ' + dDate + ' - ' + cfg['domains'] + ': ERROR') raise err
def get(self): # User Activities for a date and app One page at a time token = self.request.get('token') app = self.request.get('app') dDate = self.request.get('date') decoratorDate = ("").join(dDate.split("-")) for report_items in returnActivitiesPageToken( token, app, dDate, cfg['credentials']['general'], cfg['super_admin']['delegated']): try: bq_answer = writeDatainBigQuery( report_items, 'audit_log${decoratorDate}'.format( decoratorDate=decoratorDate)) except Exception as err: bvi_log(date=dDate, resource='activities', message_id='bigquery_error', message=err, regenerate=True) raise err
def get(self): # User List for a date One page at a time token = self.request.get('token') domain = self.request.get('domain') dDate = self.request.get('date') decoratorDate = ("").join(dDate.split("-")) try: for report_items in returnUserListPageToken(token, dDate, domain, cfg['credentials']['general'], cfg['super_admin']['delegated']): try: bq_answer = writeDatainBigQuery(report_items, 'users_list_date${decoratorDate}'.format( decoratorDate=decoratorDate)) except Exception as err: bvi_log(date=dDate, resource='users_list', message_id='bigquery_error', message=err, regenerate=True) raise err except Exception as err: logging.error(err) self.response.write('Users List for ' + dDate + ' - ' + domain + ': ERROR') raise err bvi_log(date=dDate, resource='users_list', message_id='end', message='End of /one_page_user_list call')
def get(self): # User List today today = date.today() dDate = today.strftime("%Y-%m-%d") if cfg['plan'] == 'Enterprise': bvi_log(date=dDate, resource='exec', message_id='start', message='Start of BVI daily execution') bvi_log(date=dDate, resource='users_list', message_id='start', message='Start of /user call') page_token = '' maxPages = cfg['task_management']['max_pages'] queue_name = cfg['queues']['user'] + str(1) domains = list(set(cfg['domains'].split(";"))) for domain in domains: domain = domain.strip() tNow = datetime.now() aNumber = tNow.strftime("%S") aRandomNumber = random.randint(0, sys.maxsize) aNumber = "{aNumber}_{aRandomNumber}".format( aNumber=aNumber, aRandomNumber=aRandomNumber) #1st page for user taskqueue.add( queue_name=queue_name, name='opul' + dDate + '_' + str(maxPages) + '_' + aNumber, url='/one_page_user_list?token=&date=' + dDate + '&domain=' + domain, method='GET') #Starts user delay with token empty for 2nd page taskqueue.add(queue_name=queue_name, name='ud' + dDate + '_' + aNumber, url='/user_delay?token=' + page_token + '&domain=' + domain + '&maxPages=' + str(maxPages), method='GET') logging.info('User list main for {} - {} / max pages {}'.format( dDate, cfg['domains'], maxPages)) self.response.write('User list main for {} - {} / max pages {}'.format( dDate, cfg['domains'], maxPages)) bvi_log(date=dDate, resource='users_list', message_id='end', message='end of /user call')
def get(self): project_id = cfg['ids']['project_id'] # getting request parameters exec_type = self.request.get('type', 'daily') step = self.request.get('step') begin_step = self.request.get('begin_step') dateref = self.request.get('dateref') start_date = self.request.get('Sdate') end_date = self.request.get('Edate') auto_recover = self.request.get('auto_recover', False) if auto_recover == 'False' or not auto_recover: auto_recover = False enable_auto_recover = self.request.get('enable_auto_recover', True) if enable_auto_recover == 'False' or not enable_auto_recover: enable_auto_recover = False if should_check_for_auto_recover() and enable_auto_recover and exec_type == 'daily' \ and step == 'first' and begin_step: # verifying if an error occurred in the last days, only in every 'frequency' days logging.info("[auto-recover] Verifying need to execute auto-recover...") bigquery = createBigQueryService(cfg['scopes']['big_query'], 'bigquery', 'v2') query = "SELECT MIN(min_date) as first_fail FROM [{}:logs.errors_dashboard]".format(project_id) result = fetch_big_query_data(bigquery, project_id, query, 10) rows = convert_big_query_result(result, ERROR_BEGIN) if len(rows) == 1 and rows[0]['first_fail']: exec_type = 'historical' start_date = rows[0]['first_fail'] end_date = dateref auto_recover = True logging.info("[auto-recover] Error occurred in a previous day, moving to historical execution to run " + "again since the first failed execution date. " + "auto-recover starting from {}".format(start_date)) else: logging.info("[auto-recover] Not needed, no errors found in last {} days.".format( cfg['auto_recover']['days_lookback'])) log_date = dateref if exec_type == 'daily': date_params = '&dateref={}&enable_auto_recover={}'.format(dateref, str(enable_auto_recover)) elif exec_type == 'historical': date_params = '&Sdate={}&Edate={}&enable_auto_recover={}'.format(start_date, end_date, str(enable_auto_recover)) log_date = start_date if step == 'first' and begin_step: bvi_log(date=log_date, resource='exec_manager', message_id='start', message='Start of BVI {} execution'.format(exec_type)) with open('manager.yaml', 'r') as mgrymlfile: mgr = yaml.load(mgrymlfile) exec_manager_queue = cfg['queues']['exec_manager'] if begin_step: if exec_type == 'daily': bvi_log(date=log_date, resource=get_log_step(step), message_id='start', message='Start of {} step'.format(step)) endpoint = mgr[step]['endpoint'].replace('from_cron', dateref) taskqueue.add(queue_name=exec_manager_queue, url=endpoint, method='GET') elif exec_type == 'historical': exec_historical(mgr, step, start_date, end_date) # wait for tasks to be created in the queue time.sleep(15) count_tasks = count_queued_tasks(mgr[step].get('queues')) if count_tasks == 0 and 'next_step' in mgr[step] and mgr[step]['next_step']: # Finished all tasks from this step if auto_recover and 'missing_data_table' in mgr[step]: # Check if the auto-recover was successful logging.info("[auto-recover] Checking for effectiveness...") lookback_date_obj = date.today() - timedelta(days=cfg['auto_recover']['days_lookback']) lookback_date = lookback_date_obj.strftime("%Y-%m-%d") bigquery = createBigQueryService(cfg['scopes']['big_query'], 'bigquery', 'v2') check_query = "SELECT MIN(report_date) AS report_date FROM [{}:{}] " \ "WHERE report_date > \'{}\'".format(project_id, mgr[step]['missing_data_table'], lookback_date) check_result = fetch_big_query_data(bigquery, project_id, check_query, 10) if 'rows' in check_result: check_rows = convert_big_query_result(check_result, CHECK_ROW) if len(check_rows) == 1 and check_rows[0]['report_date']: min_error_date = check_rows[0]['report_date'] min_error_date_obj = datetime.strptime(min_error_date, "%Y-%m-%d").date() start_date_obj = datetime.strptime(start_date, "%Y-%m-%d").date() if min_error_date_obj > start_date_obj: logging.info( "[auto-recover] Min error date for '{}' ".format(mgr[step]['missing_data_table']) + "is greater than start_date, auto-recover should proceed.") else: logging.info("[auto-recover] Could not fix any missing data for '{}'. ".format( mgr[step]['missing_data_table']) + "Reverting to daily ({}) execution.".format(end_date)) exec_type = 'daily' date_params = '&dateref={}&enable_auto_recover=False'.format(end_date) auto_recover = False else: logging.info( "[auto-recover] No missing data for '{}', auto-recover should proceed.".format( mgr[step]['missing_data_table'])) else: logging.info( "[auto-recover] No missing data for '{}', auto-recover should proceed.".format( mgr[step]['missing_data_table'])) logging.info("[auto-recover] Finished checking for effectiveness.") # Execute next step bvi_log(date=log_date, resource=get_log_step(step), message_id='end', message='End of {} step'.format(get_log_step(step))) taskqueue.add(queue_name=exec_manager_queue, url='/exec_manager?type={}{}&step={}&begin_step=True&auto_recover={}'.format( exec_type, date_params, mgr[step]['next_step'], auto_recover), method='GET') elif count_tasks > 0: # Still executing tasks, just schedule to monitor task queues again 10 seconds later logging.info("Waiting for tasks to finish...") taskqueue.add(queue_name=exec_manager_queue, url='/exec_manager?type={}{}&step={}&auto_recover={}'.format( exec_type, date_params, step, auto_recover), method='GET', countdown=10) else: # Finished ALL tasks bvi_log(date=log_date, resource='exec_manager', message_id='end', message='End of BVI {} execution'.format(exec_type))
def update_data_level(self, bigquery, op, bigquery_setup): level = self.request.get('level') target = self.request.get('target') dateref = self.request.get('dateref') logging.info("update {} {} {}".format(level, target, dateref)) try: if dateref == "from_cron": today = datetime.date.today() today_4 = today - datetime.timedelta(days=4) dateref = today_4.strftime("%Y-%m-%d") yyyy, mm, dd = dateref.split("-") timestamp = datetime.datetime(int(yyyy), int(mm), int(dd)) except ValueError: logging.error("Wrong updating date = {dateref}".format(dateref=dateref)) self.response.write("Wrong updating date = {}".format(dateref)) return if level_is_valid(level) or len(target) > 0: if level_is_valid(level) and not target: # only first time bvi_log(date=dateref, resource='bq_{op}_level{level}'.format(op=op, level=level), message_id='start', message='Start of /bq_api call') for index, table_def in enumerate(bigquery_setup['tables']): if target: target_dataset, target_name = target.split(".") if target_dataset is None or target_name is None: logging.error("Wrong target {target}".format(target=target)) self.response.write("Wrong target {target}".format(target=target)) if level_is_valid(level): bvi_log(date=dateref, resource='bq_{op}_level{level}'.format(op=op, level=level), message_id='bq_api_wrong_target', message="Wrong dataset/table target", regenerate=True) return elif table_def.get('dataset') == target_dataset and table_def.get('name') == target_name: logging.info("Request for updating {}".format(target_name)) decoratorDate = None if table_def.get('timePartitioning'): decoratorDate = "{yyyy}{mm}{dd}".format(yyyy=yyyy, mm=mm, dd=dd) try: do_create_view(self, bigquery, bigquery_setup['folder'], table_def['dataset'], "{table}".format(table=table_def['name']), True, overwrite=True, timestamp=timestamp, decoratorDate=decoratorDate) except Exception as err: bvi_log(date=dateref, resource='bq_{op}_level{level}'.format(op=op, level=level), message_id='bigquery_error', message=err, regenerate=True) raise err elif table_def.get('level') and str(table_def.get('level')) == str(level): logging.info("Request for updating {} (massive level {} update)".format(table_def.get('name'), level)) queue_name = "bqUpdateByLevel{level}".format(level=level) tNow = datetime.datetime.now() aNumber = tNow.strftime("%S") aRandomNumber = random.randint(0, sys.maxsize) aNumber = "{aNumber}_{aRandomNumber}".format(aNumber=aNumber, aRandomNumber=aRandomNumber) task = taskqueue.add(queue_name=queue_name, name='bqupdate_level_' + level + '_' + aNumber + '-' + table_def.get( 'dataset') + 'DOT' + table_def.get('name'), url="/bq_api?op={op}&level={level}&target={dataset}.{table}&dateref={dateref}".format( op=op, level=level, dataset=table_def.get('dataset'), table=table_def.get('name'), dateref=dateref), method='GET') self.response.write("Task {}<br/> for table {}.{} on {} enqueued,<br/>ETA {}<hr/>".format(task.name, table_def.get('dataset'), table_def.get('name'), dateref, task.eta)) bvi_log(date=dateref, resource='bq_{op}_level{level}'.format(op=op, level=level), message_id='end', message='End of /bq_api call') else: logging.info( "Nothing to do with table {index} = {name}".format(index=index, name=table_def.get("name", "name"))) if level_is_valid(level): bvi_log(date=dateref, resource='bq_{op}_level{level}'.format(op=op, level=level), message_id='end', message='End of /bq_api call') else: logging.error("Wrong parameter level={level}, target={target}".format(level=level, target=target)) self.response.write("Wrong parameter level={level}, target={target}".format(level=level, target=target))
def get(self): logging.info('User Usage') dateref = self.request.get('date', "from_cron") if len(dateref) > 0: try: if dateref == "from_cron": # Customer Usage day -4 from February 2018 today = date.today() today_4 = today - timedelta(days=4) dateref = today_4.strftime("%Y-%m-%d") yyyy, mm, dd = dateref.split("-") timestamp = datetime(int(yyyy), int(mm), int(dd)) except ValueError: logging.error( "Wrong updating date = {dateref}".format(dateref=dateref)) self.response.write("Wrong updating date = {}".format(dateref)) return try: pages = 0 page_token = self.request.get('token') if not page_token: bvi_log(date=dateref, resource='user_usage', message_id='start', message='Start of /user_usage call') maxPages = cfg['task_management']['max_pages'] maxPages = int(maxPages) numQueues = 5 queue_name = [] for x in range(1, numQueues + 1): queue_name.append(cfg['queues']['user_usage'] + str(x)) tNow = datetime.now() aNumber = tNow.strftime("%S") aRandomNumber = random.randint(0, sys.maxsize) aNumber = "{aNumber}_{aRandomNumber}".format( aNumber=aNumber, aRandomNumber=aRandomNumber) # 1st page for user_usage if not page_token: taskqueue.add(queue_name=queue_name[1], name='opuu_1_' + dateref + '_' + aNumber, url='/one_page_user_usage?token=&date=' + dateref, method='GET') for pages_token in returnUserUsageToken( dateref, cfg['credentials']['general'], cfg['super_admin']['delegated'], page_token): if not pages_token: break pages += 1 tNow = datetime.now() aNumber = tNow.strftime("%S") aRandomNumber = random.randint(0, sys.maxsize) aNumber = "{aNumber}_{aRandomNumber}".format( aNumber=aNumber, aRandomNumber=aRandomNumber) qA = random.randint(0, numQueues - 1) taskqueue.add(queue_name=queue_name[qA], name='opuu' + dateref + '_' + str(pages) + '_' + aNumber, url='/one_page_user_usage?token=' + pages_token + '&date=' + dateref, method='GET') if pages >= maxPages: taskqueue.add(queue_name=queue_name[qA], name='uu' + dateref + '_' + aNumber, url='/user_usage?token=' + pages_token + '&date=' + dateref, method='GET') break logging.info( 'User Usage pages for {} - {} / finally {} pages '.format( dateref, cfg['domains'], pages)) self.response.write("User Usage for {} - {} / {} pages".format( dateref, cfg['domains'], pages)) except Exception as err: logging.error(err) self.response.write('User Usage for ' + dateref + ' - ' + cfg['domains'] + ': ERROR') raise err bvi_log(date=dateref, resource='user_usage', message_id='end', message='End of /user_usage call')
def get(self): logging.info('Activities') appName = self.request.get('app') dateref = self.request.get('date') bvi_log(date=dateref, resource='activities', message_id='start_app', message='Start of /activities_app call') try: pages = 0 page_token = self.request.get('token') maxPages = cfg['task_management']['max_pages'] maxPages = int(maxPages) numQueues = 5 queue_name = [] for x in range(1, numQueues + 1): queue_name.append(cfg['queues']['activities'] + str(x)) tNow = datetime.now() aNumber = tNow.strftime("%S") aRandomNumber = random.randint(0, sys.maxsize) aNumber = "{aNumber}_{aRandomNumber}".format(aNumber=aNumber, aRandomNumber=aRandomNumber) # 1st page for activities if not page_token: taskqueue.add(queue_name=queue_name[1], name='apua_1_' + appName + '_' + dateref + '_' + aNumber, url='/one_page_user_activities?token=&date=' + dateref + '&app=' + appName, method='GET') for pages_token in returnActivitiesToken(dateref, appName, cfg['credentials']['general'], cfg['super_admin']['delegated'], page_token): if not pages_token: break pages += 1 logging.info('Pages {} of maxPages {}'.format) logging.info('Activities pages for {} - {} / page {} so far until now'.format(dateref, appName, pages)) tNow = datetime.now() aNumber = tNow.strftime("%S") aRandomNumber = random.randint(0, sys.maxsize) aNumber = "{aNumber}_{aRandomNumber}".format(aNumber=aNumber, aRandomNumber=aRandomNumber) qA = random.randint(0, numQueues - 1) taskqueue.add(queue_name=queue_name[qA], name='apua_' + appName + '_' + dateref + '_' + str(pages) + '_' + aNumber, url='/one_page_user_activities?token=' + pages_token + '&date=' + dateref + '&app=' + appName, method='GET') if pages >= maxPages: taskqueue.add(queue_name=queue_name[qA], name='act' + dateref + '_' + appName + '_' + aNumber, url='/activities_app?token=' + pages_token + '&app=' + appName + '&date=' + dateref, method='GET') break logging.info('User Usage pages for {} - {} / finally {} pages '.format(dateref, appName, pages)) self.response.write("User Usage for {} - {} / {} pages".format(dateref, appName, pages)) except Exception as err: logging.error(err) self.response.write('Activities for ' + dateref + ' - ' + cfg['domains'] + ': ERROR') raise err bvi_log(date=dateref, resource='activities', message_id='end_app', message='End of /activities_app call')