def get(self):
# User List today
today = date.today()
dDate = today.strftime("%Y-%m-%d")
decoratorDate = today.strftime("%Y%m%d")
users = 0
domains = list(set(cfg['domains'].split(";")))
for domain in domains:
domain = domain.strip()
for user_list in returnUsersListGeneratorExtended(
domain, cfg['credentials']['general'],
cfg['super_admin']['delegated']):
users += len(user_list)
logging.info("User List === {}".format(user_list))
logging.info(
'User list for {} - {} / {} so far until now'.format(
dDate, domain, users))
writeDatainBigQuery(
user_list,
'users_list_date_extended${decoratorDate}'.format(
decoratorDate=decoratorDate))
logging.info('User list for {} - {} / finally ... {} '.format(
dDate, domain, users))
self.response.write(
"User Extended list for {dDate} - {domain} / ... {length}".
format(dDate=dDate, domain=domain, length=users))
def get(self):
dateref = self.request.get('dateref')
def prepare_and_send_email():
to = cfg['notification_email']
if not to:
logging.info("No notification email has been setup.")
return
sender = 'noreply@{}.appspotmail.com'.format(project_id)
subject = 'A BVI process has failed.'
image_link = "https://{}.appspot.com/images/google-cloud.png".format(project_id)
template_values = {
'project': project_id,
'day': dateref,
'image_link': image_link
}
template_path = 'email_templates/processfailure.html'
send_email(template_values, template_path, subject, sender, to)
scopes = cfg['scopes']['big_query']
project_id = cfg['ids']['project_id']
bigquery = createBigQueryService(scopes, 'bigquery', 'v2')
logging.info('Checking the processes for [{}]'.format(dateref))
query = "SELECT report_date, status FROM logs.status_board WHERE report_date = \'" + dateref + "\'"
result = fetch_big_query_data(bigquery, project_id, query, 10)
if 'rows' in result:
rows = convert_big_query_result(result, DAILY_STATUS)
if 'rows' not in result or len(rows) == 0:
result = None
logging.info("There is no result for daily status so there is nothing to do.")
elif dateref == rows[0]['report_date'] and rows[0]['status'] == '0':
result = True
logging.info("All processes for the day[{}] went well, so no email to be sent.".format(dateref))
else:
result = False
logging.info("There is something wrong so an email will be sent to the admin.")
prepare_and_send_email()
try:
logging.info("Writing daily run results to BQ for the day[{}]...".format(dateref))
row = [{u'executionTime': datetime.utcnow().strftime("%s"), u'report_date': dateref, u'result': result}]
writeDatainBigQuery(row, 'daily_report_status')
except Exception:
logging.info("Error writing daily run results to BQ for the day[{}].".format(dateref))
def get(self):
# User Usage for a date One page at a time
token = self.request.get('token')
dDate = self.request.get('date')
decoratorDate = "".join(dDate.split("-"))
for report_items in returnUserUsagePageToken(token, dDate, cfg['credentials']['general'],
cfg['super_admin']['delegated']):
try:
writeDatainBigQuery(report_items,
'user_usage${decoratorDate}'.format(decoratorDate=decoratorDate))
except Exception as err:
bvi_log(date=dDate, resource='user_usage', message_id='bigquery_error', message=err,
regenerate=True)
raise err
def get(self):
logging.info('Customer Usage')
dateref = self.request.get('date', "from_cron")
if len(dateref) > 0:
try:
if dateref == "from_cron":
# Customer Usage day -4 from February 2018
today = date.today()
today_4 = today - timedelta(days=4)
dateref = today_4.strftime("%Y-%m-%d")
yyyy, mm, dd = dateref.split("-")
timestamp = datetime(int(yyyy), int(mm), int(dd))
dDate = dateref
except ValueError:
logging.error(
"Wrong updating date = {dateref}".format(dateref=dateref))
self.response.write("Wrong updating date = {}".format(dateref))
return
try:
bvi_log(date=dDate,
resource='exec',
message_id='start',
message='Start of BVI daily execution')
bvi_log(date=dDate,
resource='customer_usage',
message_id='start',
message='Start of /customer call')
decoratorDate = ("").join(dDate.split("-"))
for report_items in returnCustomerUsageReport(
dDate, cfg['credentials']['general'],
cfg['super_admin']['delegated']):
try:
bq_answer = writeDatainBigQuery(
report_items, 'customer_usage${decoratorDate}'.format(
decoratorDate=decoratorDate))
logging.info('Customer Usage for ' + dDate + ' - ' +
cfg['domains'] + ' / ')
self.response.write('Customer Usage for ' + dDate + ' - ' +
cfg['domains'] + ' / ')
except Exception as err:
bvi_log(date=dDate,
resource='customer_usage',
message_id='bigquery_error',
message=err,
regenerate=True)
raise err
except Exception as err:
logging.error(err)
self.response.write('Customer Usage for ' + dDate + ' - ' +
cfg['domains'] + ': ERROR')
raise err
bvi_log(date=dDate,
resource='customer_usage',
message_id='end',
message='End of /customer call')
def get(self):
logging.info('Customer Usage')
dateref = self.request.get('date', "from_cron")
if len(dateref) > 0:
try:
dateref = get_dateref_or_from_cron(dateref)
dDate = dateref
except ValueError:
logging.error(
"Wrong updating date = {dateref}".format(dateref=dateref))
self.response.write("Wrong updating date = {}".format(dateref))
return
try:
decoratorDate = "".join(dDate.split("-"))
table_name = 'customer_usage${decoratorDate}'.format(
decoratorDate=decoratorDate)
# delete table if it exists to avoid data duplication
delete_table_big_query(table_name)
for report_items in returnCustomerUsageReport(
dDate, cfg['credentials']['general'],
cfg['super_admin']['delegated']):
try:
writeDatainBigQuery(report_items, table_name)
logging.info('Customer Usage for ' + dDate + ' - ' +
cfg['domains'] + ' / ')
self.response.write('Customer Usage for ' + dDate + ' - ' +
cfg['domains'] + ' / ')
except Exception as err:
bvi_log(date=dDate,
resource='customer_usage',
message_id='bigquery_error',
message=err,
regenerate=True)
raise err
except Exception as err:
logging.error(err)
self.response.write('Customer Usage for ' + dDate + ' - ' +
cfg['domains'] + ': ERROR')
raise err
def get(self):
# User Activities for a date and app One page at a time
token = self.request.get('token')
app = self.request.get('app')
dDate = self.request.get('date')
decoratorDate = ("").join(dDate.split("-"))
for report_items in returnActivitiesPageToken(
token, app, dDate, cfg['credentials']['general'],
cfg['super_admin']['delegated']):
try:
bq_answer = writeDatainBigQuery(
report_items, 'audit_log${decoratorDate}'.format(
decoratorDate=decoratorDate))
except Exception as err:
bvi_log(date=dDate,
resource='activities',
message_id='bigquery_error',
message=err,
regenerate=True)
raise err
def get(self):
# User List for a date One page at a time
token = self.request.get('token')
domain = self.request.get('domain')
dDate = self.request.get('date')
decoratorDate = ("").join(dDate.split("-"))
try:
for report_items in returnUserListPageToken(token, dDate, domain, cfg['credentials']['general'],
cfg['super_admin']['delegated']):
try:
bq_answer = writeDatainBigQuery(report_items, 'users_list_date${decoratorDate}'.format(
decoratorDate=decoratorDate))
except Exception as err:
bvi_log(date=dDate, resource='users_list', message_id='bigquery_error', message=err,
regenerate=True)
raise err
except Exception as err:
logging.error(err)
self.response.write('Users List for ' + dDate + ' - ' + domain + ': ERROR')
raise err
bvi_log(date=dDate, resource='users_list', message_id='end', message='End of /one_page_user_list call')