def init(self):
start_time_sec = int(round(time.time()))
analysis_date = datetime.strptime(self.execution_date, "%d/%m/%Y")
result_loc = self.data_store_location.joinpath('district_reports')
result_loc.mkdir(exist_ok=True)
result_loc.joinpath(analysis_date.strftime("%Y-%m-%d")).mkdir(exist_ok=True)
self.data_store_location.joinpath('config').mkdir(exist_ok=True)
get_data_from_blob(result_loc.joinpath('slug_state_mapping.csv'))
tenant_info = pd.read_csv(result_loc.joinpath('slug_state_mapping.csv'))
get_data_from_blob(self.data_store_location.joinpath('config', 'diksha_config.json'))
with open(self.data_store_location.joinpath('config', 'diksha_config.json'), 'r') as f:
self.config = json.loads(f.read())
for ind, row in tenant_info.iterrows():
print(row['state'])
result_loc.joinpath(row["slug"]).mkdir(exist_ok=True)
if isinstance(row['state'], str):
self.unique_users(result_loc_=result_loc.joinpath(row["slug"]), date_=analysis_date,
state_=row['state'])
end_time_sec = int(round(time.time()))
time_taken = end_time_sec - start_time_sec
metrics = [
{
"metric": "timeTakenSecs",
"value": time_taken
},
{
"metric": "date",
"value": analysis_date.strftime("%Y-%m-%d")
}
]
push_metric_event(metrics, "District Monthly Report")
def init(self):
start_time_sec = int(round(time.time()))
self.write_path = Path(
self.data_store_location).joinpath('ecg_learning_reports')
self.write_path.mkdir(exist_ok=True)
print("ECG::Start")
blob_file_name = "ecg_nation_learning"
self.csv_file_name = "{}.csv".format(blob_file_name)
self.json_file_name = "{}.json".format(blob_file_name)
self.current_time = datetime.now()
if self.is_bootstrap:
# For last 7 days
from_day = (datetime.today() + timedelta(days=-7))
from_time = int(
datetime.combine(from_day, datetime.min.time()).timestamp())
else:
# For Last 1 hour
from_day = (datetime.today() + timedelta(hours=-1))
from_time = int(
from_day.replace(minute=15, second=0,
microsecond=0).timestamp())
to_time = int(self.current_time.timestamp())
self.generate_reports(from_time, to_time)
print("ECG::Completed")
end_time_sec = int(round(time.time()))
time_taken = end_time_sec - start_time_sec
metrics = [{
"metric": "timeTakenSecs",
"value": time_taken
}, {
"metric": "date",
"value": self.current_time.strftime("%Y-%m-%d %H:%m")
}]
push_metric_event(metrics, "ECG Learning")
def init(self):
start_time_sec = int(round(time.time()))
print('Content Creation Status Report::Start')
result_loc = self.data_store_location.joinpath('content_creation')
result_loc.mkdir(exist_ok=True)
execution_date = datetime.strptime(self.execution_date, "%d/%m/%Y")
get_tenant_info(result_loc_=result_loc,
org_search_=self.org_search,
date_=execution_date)
get_content_model(result_loc_=result_loc,
druid_=self.druid_hostname,
date_=execution_date,
status_=['Live', 'Review', 'Draft', 'Unlisted'])
self.generate_report(result_loc, execution_date)
print('Content Creation Status Report::Completed')
end_time_sec = int(round(time.time()))
time_taken = end_time_sec - start_time_sec
metrics = [{
'metric': 'timeTakenSecs',
'value': time_taken
}, {
'metric': 'date',
'value': execution_date.strftime('%Y-%m-%d')
}]
push_metric_event(metrics, 'Content Creation Status')
def init(self):
start_time_sec = int(round(time.time()))
print("GPS::Start")
execution_date_ = datetime.strptime(self.execution_date, "%d/%m/%Y")
self.data_store_location.joinpath(
'textbook_reports',
execution_date_.strftime('%Y-%m-%d')).mkdir(exist_ok=True)
get_scan_counts(
result_loc_=self.data_store_location.joinpath('textbook_reports'),
druid_=self.druid_hostname,
date_=execution_date_)
get_tenant_info(
result_loc_=self.data_store_location.joinpath('textbook_reports'),
org_search_=self.org_search,
date_=execution_date_)
self.get_tbs()
self.generate_report()
print("GPS::End")
end_time_sec = int(round(time.time()))
time_taken = end_time_sec - start_time_sec
metrics = [{
"metric": "timeTakenSecs",
"value": time_taken
}, {
"metric": "date",
"value": execution_date_.strftime("%Y-%m-%d")
}]
push_metric_event(metrics, "ECG Learning")
def init(self):
start_time_sec = int(round(time.time()))
print("Content Consumption Report::Start")
self.data_store_location = Path(self.data_store_location)
org_search = self.org_search
druid = self.druid_hostname
druid_rollup = self.druid_rollup_hostname
content_search = self.content_search
execution_date = datetime.strptime(self.execution_date, "%d/%m/%Y")
result_loc = self.data_store_location.joinpath('content_plays')
result_loc.parent.joinpath('portal_dashboards').mkdir(exist_ok=True)
result_loc.parent.joinpath('config').mkdir(exist_ok=True)
get_data_from_blob(
result_loc.parent.joinpath('config', 'diksha_config.json'))
with open(result_loc.parent.joinpath('config', 'diksha_config.json'),
'r') as f:
self.config = json.loads(f.read())
get_tenant_info(result_loc_=result_loc,
org_search_=org_search,
date_=execution_date)
print("Success::Tenant info")
get_content_model(result_loc_=result_loc,
druid_=druid_rollup,
date_=execution_date,
config_=self.config,
version_='v2')
print("Success::content model snapshot")
get_tb_content_mapping(result_loc_=result_loc,
date_=execution_date,
content_search_=content_search)
print("Success::TB Content Map")
self.get_weekly_report(result_loc_=result_loc,
druid_rollup_=druid_rollup,
date_=execution_date,
config=self.config)
print("Success::Weekly Conent Consumption")
self.get_overall_report(result_loc_=result_loc,
druid_rollup_=druid_rollup,
date_=execution_date,
config=self.config)
print("Success::Overall Conent Consumption")
self.get_last_week_report(result_loc_=result_loc,
date_=execution_date,
num_weeks=6)
print("Success:::Last 6 weeks")
shutil.rmtree(result_loc)
print("Content Consumption Report::Completed")
end_time_sec = int(round(time.time()))
time_taken = end_time_sec - start_time_sec
metrics = [{
"metric": "timeTakenSecs",
"value": time_taken
}, {
"metric": "date",
"value": execution_date.strftime("%Y-%m-%d")
}]
push_metric_event(metrics, "Content Consumption Metrics")
def init(self):
start_time_sec = int(round(time.time()))
file_path = Path(__file__)
result_loc = self.data_store_location.joinpath('district_reports')
result_loc.mkdir(exist_ok=True)
result_loc.parent.joinpath('config').mkdir(exist_ok=True)
analysis_date = datetime.strptime(self.execution_date, "%d/%m/%Y")
get_data_from_blob(result_loc.joinpath('slug_state_mapping.csv'))
get_location_info(result_loc, self.location_search, analysis_date)
tenant_info = pd.read_csv(
result_loc.joinpath('slug_state_mapping.csv'))
self.druid_url = "{}druid/v2/".format(self.druid_hostname)
self.headers = {'Content-Type': "application/json"}
result_loc.parent.joinpath('config').mkdir(exist_ok=True)
get_data_from_blob(
result_loc.parent.joinpath('config', 'diksha_config.json'))
with open(result_loc.parent.joinpath('config', 'diksha_config.json'),
'r') as f:
self.config = json.loads(f.read())
for ind, row in tenant_info.iterrows():
state = row['state']
print(state)
result_loc.joinpath(
analysis_date.strftime('%Y-%m-%d')).mkdir(exist_ok=True)
result_loc.joinpath(analysis_date.strftime('%Y-%m-%d'),
row['slug']).mkdir(exist_ok=True)
path = result_loc.joinpath(analysis_date.strftime('%Y-%m-%d'),
row['slug'])
if isinstance(state, str):
self.district_devices(result_loc_=path,
date_=analysis_date,
state_=state)
self.district_plays(result_loc_=path,
date_=analysis_date,
state_=state)
self.district_scans(result_loc_=path,
date_=analysis_date,
state_=state)
self.merge_metrics(result_loc_=path, date_=analysis_date)
end_time_sec = int(round(time.time()))
time_taken = end_time_sec - start_time_sec
metrics = [{
"metric": "timeTakenSecs",
"value": time_taken
}, {
"metric": "date",
"value": analysis_date.strftime("%Y-%m-%d")
}]
push_metric_event(metrics, "District Weekly Report")
def init(self):
start_time_sec = int(round(time.time()))
print("Content Consumption Report::Start")
self.data_store_location = Path(self.data_store_location)
org_search = self.org_search
druid = self.druid_hostname
cassandra = self.cassandra_host
keyspace = self.keyspace_prefix + 'content_db'
execution_date = datetime.strptime(self.execution_date, "%d/%m/%Y")
result_loc = self.data_store_location.joinpath('content_plays')
result_loc.parent.joinpath('portal_dashboards').mkdir(exist_ok=True)
result_loc.parent.joinpath('config').mkdir(exist_ok=True)
get_data_from_blob(
result_loc.parent.joinpath('config', 'diksha_config.json'))
with open(result_loc.parent.joinpath('config', 'diksha_config.json'),
'r') as f:
self.config = json.loads(f.read())
get_tenant_info(result_loc_=result_loc,
org_search_=org_search,
date_=execution_date)
get_content_model(result_loc_=result_loc,
druid_=druid,
date_=execution_date)
self.define_keyspace(cassandra_=cassandra, keyspace_=keyspace)
for i in range(7):
analysis_date = execution_date - timedelta(days=i)
get_content_plays(result_loc_=result_loc,
date_=analysis_date,
druid_=druid)
self.insert_data_to_cassandra(result_loc_=result_loc,
date_=analysis_date,
cassandra_=cassandra,
keyspace_=keyspace)
self.get_weekly_plays(result_loc_=result_loc,
date_=execution_date,
cassandra_=cassandra,
keyspace_=keyspace)
print("Content Consumption Report::Completed")
end_time_sec = int(round(time.time()))
time_taken = end_time_sec - start_time_sec
metrics = [{
"metric": "timeTakenSecs",
"value": time_taken
}, {
"metric": "date",
"value": execution_date.strftime("%Y-%m-%d")
}]
push_metric_event(metrics, "Content Consumption Metrics")
def init(self):
start_time_sec = int(round(time.time()))
self.start_time = datetime.now()
print("Started at: ", self.start_time.strftime('%Y-%m-%d %H:%M:%S'))
end_date_ = datetime.strptime(self.execution_date, "%d/%m/%Y")
self.data_store_location.joinpath('textbook_reports').mkdir(
exist_ok=True)
self.data_store_location.joinpath(
'textbook_reports',
end_date_.strftime('%Y-%m-%d')).mkdir(exist_ok=True)
self.data_store_location.joinpath('portal_dashboards').mkdir(
exist_ok=True)
get_tenant_info(
result_loc_=self.data_store_location.joinpath('textbook_reports'),
org_search_=self.org_search,
date_=end_date_)
# # TODO: SB-15177 store scan counts in cassandra
get_scan_counts(
result_loc_=self.data_store_location.joinpath('textbook_reports'),
druid_=self.druid_hostname,
date_=end_date_)
self.backend_grade = pd.DataFrame(
sorted_grades.init()).set_index('grade')
self.generate_reports(result_loc_=self.data_store_location,
content_search_=self.content_search,
content_hierarchy_=self.content_hierarchy,
date_=end_date_)
end_time = datetime.now()
print("Ended at: ", end_time.strftime('%Y-%m-%d %H:%M:%S'))
print("Time taken: ", str(end_time - self.start_time))
end_time_sec = int(round(time.time()))
time_taken = end_time_sec - start_time_sec
metrics = [{
"metric": "timeTakenSecs",
"value": time_taken
}, {
"metric": "date",
"value": end_date_.strftime("%Y-%m-%d")
}]
push_metric_event(metrics, "ETB Creation Metrics")
def init(self):
start_time_sec = int(round(time.time()))
print('[START] Landing Page!')
self.current_time = date.today()
get_tenant_info(
result_loc_=self.data_store_location.joinpath('textbook_reports'),
org_search_=self.org_search,
date_=self.current_time)
self.generate_report()
print('[SUCCESS] Landing Page!')
end_time_sec = int(round(time.time()))
time_taken = end_time_sec - start_time_sec
metrics = [{
"metric": "timeTakenSecs",
"value": time_taken
}, {
"metric": "date",
"value": self.current_time.strftime("%Y-%m-%d")
}]
push_metric_event(metrics, "Landing Page")
def init(self):
start_time_sec = int(round(time.time()))
print("START:CMO Dashboard")
data_store_location = self.data_store_location.joinpath('portal_dashboards')
data_store_location.mkdir(exist_ok=True)
analysis_date = datetime.strptime(self.execution_date, "%d/%m/%Y")
data_store_location.joinpath('public').mkdir(exist_ok=True)
get_data_from_blob(data_store_location.joinpath('overall', 'daily_metrics.csv'))
self.data_wrangling(result_loc_=data_store_location.joinpath('overall', 'daily_metrics.csv'), date_=analysis_date)
create_json(data_store_location.joinpath('public', 'cmo_dashboard.csv'), last_update=True)
post_data_to_blob(data_store_location.joinpath('public', 'cmo_dashboard.csv'))
get_tenant_info(result_loc_=data_store_location.parent.joinpath('textbook_reports'), org_search_=self.org_search,
date_=analysis_date)
board_slug = pd.read_csv(
data_store_location.parent.joinpath('textbook_reports', analysis_date.strftime('%Y-%m-%d'), 'tenant_info.csv'))
slug_list = board_slug['slug'].unique().tolist()
for slug in slug_list:
try:
get_data_from_blob(result_loc_=data_store_location.joinpath(slug, 'daily_metrics.csv'))
self.data_wrangling(result_loc_=data_store_location.joinpath(slug, 'daily_metrics.csv'), date_=analysis_date)
create_json(read_loc_=data_store_location.joinpath(slug, 'cmo_dashboard.csv'), last_update=True)
post_data_to_blob(result_loc_=data_store_location.joinpath(slug, 'cmo_dashboard.csv'))
except:
pass
print("END:CMO Dashboard")
end_time_sec = int(round(time.time()))
time_taken = end_time_sec - start_time_sec
metrics = [
{
"metric": "timeTakenSecs",
"value": time_taken
},
{
"metric": "date",
"value": analysis_date.strftime("%Y-%m-%d")
}
]
push_metric_event(metrics, "CMO Dashboard")
def init(self):
start_time_sec = int(round(time.time()))
print("START:Content Creation")
result_loc = self.data_store_location.joinpath('content_creation')
execution_date = datetime.strptime(self.execution_date, "%d/%m/%Y")
get_tenant_info(result_loc_=result_loc, org_search_=self.org_search, date_=execution_date)
self.weekly_creation(result_loc_=result_loc, date_=execution_date, content_search_=self.content_search)
self.overall_creation(result_loc_=result_loc, date_=execution_date, content_search_=self.content_search)
self.combine_creation_reports(result_loc_=result_loc, date_=execution_date)
print("END:Content Creation")
end_time_sec = int(round(time.time()))
time_taken = end_time_sec - start_time_sec
metrics = [
{
"metric": "timeTakenSecs",
"value": time_taken
},
{
"metric": "date",
"value": execution_date.strftime("%Y-%m-%d")
}
]
push_metric_event(metrics, "Content Creation")
def init(self):
start_time_sec = int(round(time.time()))
start_time = datetime.now()
print("Started at: ", start_time.strftime('%Y-%m-%d %H:%M:%S'))
findspark.init()
execution_date = datetime.strptime(self.execution_date, "%d/%m/%Y")
analysis_date = execution_date - timedelta(1)
self.data_store_location.joinpath('tb_metadata').mkdir(exist_ok=True)
self.data_store_location.joinpath('play').mkdir(exist_ok=True)
self.data_store_location.joinpath('downloads').mkdir(exist_ok=True)
self.data_store_location.joinpath('dialcode_scans').mkdir(
exist_ok=True)
self.data_store_location.joinpath('portal_dashboards').mkdir(
exist_ok=True)
self.data_store_location.joinpath('config').mkdir(exist_ok=True)
get_data_from_blob(
self.data_store_location.joinpath('config', 'diksha_config.json'))
with open(
self.data_store_location.joinpath('config',
'diksha_config.json'),
'r') as f:
self.config = json.loads(f.read())
get_textbook_snapshot(
result_loc_=self.data_store_location.joinpath('tb_metadata'),
content_search_=self.content_search,
content_hierarchy_=self.content_hierarchy,
date_=analysis_date)
print('[Success] Textbook Snapshot')
get_tenant_info(
result_loc_=self.data_store_location.joinpath('textbook_reports'),
org_search_=self.org_search,
date_=analysis_date)
print('[Success] Tenant Info')
self.app_and_plays(
result_loc_=self.data_store_location.joinpath('play'),
date_=analysis_date)
print('[Success] App and Plays')
self.dialscans(
result_loc_=self.data_store_location.joinpath('dialcode_scans'),
date_=analysis_date)
print('[Success] DIAL Scans')
self.downloads(
result_loc_=self.data_store_location.joinpath('downloads'),
date_=analysis_date)
print('[Success] Downloads')
self.daily_metrics(read_loc_=self.data_store_location,
date_=analysis_date)
print('[Success] Daily metrics')
end_time = datetime.now()
print("Ended at: ", end_time.strftime('%Y-%m-%d %H:%M:%S'))
print("Time taken: ", str(end_time - start_time))
end_time_sec = int(round(time.time()))
time_taken = end_time_sec - start_time_sec
metrics = [{
"metric": "timeTakenSecs",
"value": time_taken
}, {
"metric": "date",
"value": execution_date.strftime("%Y-%m-%d")
}]
push_metric_event(metrics, "Consumption Metrics")