def init(self): start_time_sec = int(round(time.time())) analysis_date = datetime.strptime(self.execution_date, "%d/%m/%Y") result_loc = self.data_store_location.joinpath('district_reports') result_loc.mkdir(exist_ok=True) result_loc.joinpath(analysis_date.strftime("%Y-%m-%d")).mkdir(exist_ok=True) self.data_store_location.joinpath('config').mkdir(exist_ok=True) get_data_from_blob(result_loc.joinpath('slug_state_mapping.csv')) tenant_info = pd.read_csv(result_loc.joinpath('slug_state_mapping.csv')) get_data_from_blob(self.data_store_location.joinpath('config', 'diksha_config.json')) with open(self.data_store_location.joinpath('config', 'diksha_config.json'), 'r') as f: self.config = json.loads(f.read()) for ind, row in tenant_info.iterrows(): print(row['state']) result_loc.joinpath(row["slug"]).mkdir(exist_ok=True) if isinstance(row['state'], str): self.unique_users(result_loc_=result_loc.joinpath(row["slug"]), date_=analysis_date, state_=row['state']) end_time_sec = int(round(time.time())) time_taken = end_time_sec - start_time_sec metrics = [ { "metric": "timeTakenSecs", "value": time_taken }, { "metric": "date", "value": analysis_date.strftime("%Y-%m-%d") } ] push_metric_event(metrics, "District Monthly Report")
def init(self): start_time_sec = int(round(time.time())) self.write_path = Path( self.data_store_location).joinpath('ecg_learning_reports') self.write_path.mkdir(exist_ok=True) print("ECG::Start") blob_file_name = "ecg_nation_learning" self.csv_file_name = "{}.csv".format(blob_file_name) self.json_file_name = "{}.json".format(blob_file_name) self.current_time = datetime.now() if self.is_bootstrap: # For last 7 days from_day = (datetime.today() + timedelta(days=-7)) from_time = int( datetime.combine(from_day, datetime.min.time()).timestamp()) else: # For Last 1 hour from_day = (datetime.today() + timedelta(hours=-1)) from_time = int( from_day.replace(minute=15, second=0, microsecond=0).timestamp()) to_time = int(self.current_time.timestamp()) self.generate_reports(from_time, to_time) print("ECG::Completed") end_time_sec = int(round(time.time())) time_taken = end_time_sec - start_time_sec metrics = [{ "metric": "timeTakenSecs", "value": time_taken }, { "metric": "date", "value": self.current_time.strftime("%Y-%m-%d %H:%m") }] push_metric_event(metrics, "ECG Learning")
def init(self): start_time_sec = int(round(time.time())) print('Content Creation Status Report::Start') result_loc = self.data_store_location.joinpath('content_creation') result_loc.mkdir(exist_ok=True) execution_date = datetime.strptime(self.execution_date, "%d/%m/%Y") get_tenant_info(result_loc_=result_loc, org_search_=self.org_search, date_=execution_date) get_content_model(result_loc_=result_loc, druid_=self.druid_hostname, date_=execution_date, status_=['Live', 'Review', 'Draft', 'Unlisted']) self.generate_report(result_loc, execution_date) print('Content Creation Status Report::Completed') end_time_sec = int(round(time.time())) time_taken = end_time_sec - start_time_sec metrics = [{ 'metric': 'timeTakenSecs', 'value': time_taken }, { 'metric': 'date', 'value': execution_date.strftime('%Y-%m-%d') }] push_metric_event(metrics, 'Content Creation Status')
def init(self): start_time_sec = int(round(time.time())) print("GPS::Start") execution_date_ = datetime.strptime(self.execution_date, "%d/%m/%Y") self.data_store_location.joinpath( 'textbook_reports', execution_date_.strftime('%Y-%m-%d')).mkdir(exist_ok=True) get_scan_counts( result_loc_=self.data_store_location.joinpath('textbook_reports'), druid_=self.druid_hostname, date_=execution_date_) get_tenant_info( result_loc_=self.data_store_location.joinpath('textbook_reports'), org_search_=self.org_search, date_=execution_date_) self.get_tbs() self.generate_report() print("GPS::End") end_time_sec = int(round(time.time())) time_taken = end_time_sec - start_time_sec metrics = [{ "metric": "timeTakenSecs", "value": time_taken }, { "metric": "date", "value": execution_date_.strftime("%Y-%m-%d") }] push_metric_event(metrics, "ECG Learning")
def init(self): start_time_sec = int(round(time.time())) print("Content Consumption Report::Start") self.data_store_location = Path(self.data_store_location) org_search = self.org_search druid = self.druid_hostname druid_rollup = self.druid_rollup_hostname content_search = self.content_search execution_date = datetime.strptime(self.execution_date, "%d/%m/%Y") result_loc = self.data_store_location.joinpath('content_plays') result_loc.parent.joinpath('portal_dashboards').mkdir(exist_ok=True) result_loc.parent.joinpath('config').mkdir(exist_ok=True) get_data_from_blob( result_loc.parent.joinpath('config', 'diksha_config.json')) with open(result_loc.parent.joinpath('config', 'diksha_config.json'), 'r') as f: self.config = json.loads(f.read()) get_tenant_info(result_loc_=result_loc, org_search_=org_search, date_=execution_date) print("Success::Tenant info") get_content_model(result_loc_=result_loc, druid_=druid_rollup, date_=execution_date, config_=self.config, version_='v2') print("Success::content model snapshot") get_tb_content_mapping(result_loc_=result_loc, date_=execution_date, content_search_=content_search) print("Success::TB Content Map") self.get_weekly_report(result_loc_=result_loc, druid_rollup_=druid_rollup, date_=execution_date, config=self.config) print("Success::Weekly Conent Consumption") self.get_overall_report(result_loc_=result_loc, druid_rollup_=druid_rollup, date_=execution_date, config=self.config) print("Success::Overall Conent Consumption") self.get_last_week_report(result_loc_=result_loc, date_=execution_date, num_weeks=6) print("Success:::Last 6 weeks") shutil.rmtree(result_loc) print("Content Consumption Report::Completed") end_time_sec = int(round(time.time())) time_taken = end_time_sec - start_time_sec metrics = [{ "metric": "timeTakenSecs", "value": time_taken }, { "metric": "date", "value": execution_date.strftime("%Y-%m-%d") }] push_metric_event(metrics, "Content Consumption Metrics")
def init(self): start_time_sec = int(round(time.time())) file_path = Path(__file__) result_loc = self.data_store_location.joinpath('district_reports') result_loc.mkdir(exist_ok=True) result_loc.parent.joinpath('config').mkdir(exist_ok=True) analysis_date = datetime.strptime(self.execution_date, "%d/%m/%Y") get_data_from_blob(result_loc.joinpath('slug_state_mapping.csv')) get_location_info(result_loc, self.location_search, analysis_date) tenant_info = pd.read_csv( result_loc.joinpath('slug_state_mapping.csv')) self.druid_url = "{}druid/v2/".format(self.druid_hostname) self.headers = {'Content-Type': "application/json"} result_loc.parent.joinpath('config').mkdir(exist_ok=True) get_data_from_blob( result_loc.parent.joinpath('config', 'diksha_config.json')) with open(result_loc.parent.joinpath('config', 'diksha_config.json'), 'r') as f: self.config = json.loads(f.read()) for ind, row in tenant_info.iterrows(): state = row['state'] print(state) result_loc.joinpath( analysis_date.strftime('%Y-%m-%d')).mkdir(exist_ok=True) result_loc.joinpath(analysis_date.strftime('%Y-%m-%d'), row['slug']).mkdir(exist_ok=True) path = result_loc.joinpath(analysis_date.strftime('%Y-%m-%d'), row['slug']) if isinstance(state, str): self.district_devices(result_loc_=path, date_=analysis_date, state_=state) self.district_plays(result_loc_=path, date_=analysis_date, state_=state) self.district_scans(result_loc_=path, date_=analysis_date, state_=state) self.merge_metrics(result_loc_=path, date_=analysis_date) end_time_sec = int(round(time.time())) time_taken = end_time_sec - start_time_sec metrics = [{ "metric": "timeTakenSecs", "value": time_taken }, { "metric": "date", "value": analysis_date.strftime("%Y-%m-%d") }] push_metric_event(metrics, "District Weekly Report")
def init(self): start_time_sec = int(round(time.time())) print("Content Consumption Report::Start") self.data_store_location = Path(self.data_store_location) org_search = self.org_search druid = self.druid_hostname cassandra = self.cassandra_host keyspace = self.keyspace_prefix + 'content_db' execution_date = datetime.strptime(self.execution_date, "%d/%m/%Y") result_loc = self.data_store_location.joinpath('content_plays') result_loc.parent.joinpath('portal_dashboards').mkdir(exist_ok=True) result_loc.parent.joinpath('config').mkdir(exist_ok=True) get_data_from_blob( result_loc.parent.joinpath('config', 'diksha_config.json')) with open(result_loc.parent.joinpath('config', 'diksha_config.json'), 'r') as f: self.config = json.loads(f.read()) get_tenant_info(result_loc_=result_loc, org_search_=org_search, date_=execution_date) get_content_model(result_loc_=result_loc, druid_=druid, date_=execution_date) self.define_keyspace(cassandra_=cassandra, keyspace_=keyspace) for i in range(7): analysis_date = execution_date - timedelta(days=i) get_content_plays(result_loc_=result_loc, date_=analysis_date, druid_=druid) self.insert_data_to_cassandra(result_loc_=result_loc, date_=analysis_date, cassandra_=cassandra, keyspace_=keyspace) self.get_weekly_plays(result_loc_=result_loc, date_=execution_date, cassandra_=cassandra, keyspace_=keyspace) print("Content Consumption Report::Completed") end_time_sec = int(round(time.time())) time_taken = end_time_sec - start_time_sec metrics = [{ "metric": "timeTakenSecs", "value": time_taken }, { "metric": "date", "value": execution_date.strftime("%Y-%m-%d") }] push_metric_event(metrics, "Content Consumption Metrics")
def init(self): start_time_sec = int(round(time.time())) self.start_time = datetime.now() print("Started at: ", self.start_time.strftime('%Y-%m-%d %H:%M:%S')) end_date_ = datetime.strptime(self.execution_date, "%d/%m/%Y") self.data_store_location.joinpath('textbook_reports').mkdir( exist_ok=True) self.data_store_location.joinpath( 'textbook_reports', end_date_.strftime('%Y-%m-%d')).mkdir(exist_ok=True) self.data_store_location.joinpath('portal_dashboards').mkdir( exist_ok=True) get_tenant_info( result_loc_=self.data_store_location.joinpath('textbook_reports'), org_search_=self.org_search, date_=end_date_) # # TODO: SB-15177 store scan counts in cassandra get_scan_counts( result_loc_=self.data_store_location.joinpath('textbook_reports'), druid_=self.druid_hostname, date_=end_date_) self.backend_grade = pd.DataFrame( sorted_grades.init()).set_index('grade') self.generate_reports(result_loc_=self.data_store_location, content_search_=self.content_search, content_hierarchy_=self.content_hierarchy, date_=end_date_) end_time = datetime.now() print("Ended at: ", end_time.strftime('%Y-%m-%d %H:%M:%S')) print("Time taken: ", str(end_time - self.start_time)) end_time_sec = int(round(time.time())) time_taken = end_time_sec - start_time_sec metrics = [{ "metric": "timeTakenSecs", "value": time_taken }, { "metric": "date", "value": end_date_.strftime("%Y-%m-%d") }] push_metric_event(metrics, "ETB Creation Metrics")
def init(self): start_time_sec = int(round(time.time())) print('[START] Landing Page!') self.current_time = date.today() get_tenant_info( result_loc_=self.data_store_location.joinpath('textbook_reports'), org_search_=self.org_search, date_=self.current_time) self.generate_report() print('[SUCCESS] Landing Page!') end_time_sec = int(round(time.time())) time_taken = end_time_sec - start_time_sec metrics = [{ "metric": "timeTakenSecs", "value": time_taken }, { "metric": "date", "value": self.current_time.strftime("%Y-%m-%d") }] push_metric_event(metrics, "Landing Page")
def init(self): start_time_sec = int(round(time.time())) print("START:CMO Dashboard") data_store_location = self.data_store_location.joinpath('portal_dashboards') data_store_location.mkdir(exist_ok=True) analysis_date = datetime.strptime(self.execution_date, "%d/%m/%Y") data_store_location.joinpath('public').mkdir(exist_ok=True) get_data_from_blob(data_store_location.joinpath('overall', 'daily_metrics.csv')) self.data_wrangling(result_loc_=data_store_location.joinpath('overall', 'daily_metrics.csv'), date_=analysis_date) create_json(data_store_location.joinpath('public', 'cmo_dashboard.csv'), last_update=True) post_data_to_blob(data_store_location.joinpath('public', 'cmo_dashboard.csv')) get_tenant_info(result_loc_=data_store_location.parent.joinpath('textbook_reports'), org_search_=self.org_search, date_=analysis_date) board_slug = pd.read_csv( data_store_location.parent.joinpath('textbook_reports', analysis_date.strftime('%Y-%m-%d'), 'tenant_info.csv')) slug_list = board_slug['slug'].unique().tolist() for slug in slug_list: try: get_data_from_blob(result_loc_=data_store_location.joinpath(slug, 'daily_metrics.csv')) self.data_wrangling(result_loc_=data_store_location.joinpath(slug, 'daily_metrics.csv'), date_=analysis_date) create_json(read_loc_=data_store_location.joinpath(slug, 'cmo_dashboard.csv'), last_update=True) post_data_to_blob(result_loc_=data_store_location.joinpath(slug, 'cmo_dashboard.csv')) except: pass print("END:CMO Dashboard") end_time_sec = int(round(time.time())) time_taken = end_time_sec - start_time_sec metrics = [ { "metric": "timeTakenSecs", "value": time_taken }, { "metric": "date", "value": analysis_date.strftime("%Y-%m-%d") } ] push_metric_event(metrics, "CMO Dashboard")
def init(self): start_time_sec = int(round(time.time())) print("START:Content Creation") result_loc = self.data_store_location.joinpath('content_creation') execution_date = datetime.strptime(self.execution_date, "%d/%m/%Y") get_tenant_info(result_loc_=result_loc, org_search_=self.org_search, date_=execution_date) self.weekly_creation(result_loc_=result_loc, date_=execution_date, content_search_=self.content_search) self.overall_creation(result_loc_=result_loc, date_=execution_date, content_search_=self.content_search) self.combine_creation_reports(result_loc_=result_loc, date_=execution_date) print("END:Content Creation") end_time_sec = int(round(time.time())) time_taken = end_time_sec - start_time_sec metrics = [ { "metric": "timeTakenSecs", "value": time_taken }, { "metric": "date", "value": execution_date.strftime("%Y-%m-%d") } ] push_metric_event(metrics, "Content Creation")
def init(self): start_time_sec = int(round(time.time())) start_time = datetime.now() print("Started at: ", start_time.strftime('%Y-%m-%d %H:%M:%S')) findspark.init() execution_date = datetime.strptime(self.execution_date, "%d/%m/%Y") analysis_date = execution_date - timedelta(1) self.data_store_location.joinpath('tb_metadata').mkdir(exist_ok=True) self.data_store_location.joinpath('play').mkdir(exist_ok=True) self.data_store_location.joinpath('downloads').mkdir(exist_ok=True) self.data_store_location.joinpath('dialcode_scans').mkdir( exist_ok=True) self.data_store_location.joinpath('portal_dashboards').mkdir( exist_ok=True) self.data_store_location.joinpath('config').mkdir(exist_ok=True) get_data_from_blob( self.data_store_location.joinpath('config', 'diksha_config.json')) with open( self.data_store_location.joinpath('config', 'diksha_config.json'), 'r') as f: self.config = json.loads(f.read()) get_textbook_snapshot( result_loc_=self.data_store_location.joinpath('tb_metadata'), content_search_=self.content_search, content_hierarchy_=self.content_hierarchy, date_=analysis_date) print('[Success] Textbook Snapshot') get_tenant_info( result_loc_=self.data_store_location.joinpath('textbook_reports'), org_search_=self.org_search, date_=analysis_date) print('[Success] Tenant Info') self.app_and_plays( result_loc_=self.data_store_location.joinpath('play'), date_=analysis_date) print('[Success] App and Plays') self.dialscans( result_loc_=self.data_store_location.joinpath('dialcode_scans'), date_=analysis_date) print('[Success] DIAL Scans') self.downloads( result_loc_=self.data_store_location.joinpath('downloads'), date_=analysis_date) print('[Success] Downloads') self.daily_metrics(read_loc_=self.data_store_location, date_=analysis_date) print('[Success] Daily metrics') end_time = datetime.now() print("Ended at: ", end_time.strftime('%Y-%m-%d %H:%M:%S')) print("Time taken: ", str(end_time - start_time)) end_time_sec = int(round(time.time())) time_taken = end_time_sec - start_time_sec metrics = [{ "metric": "timeTakenSecs", "value": time_taken }, { "metric": "date", "value": execution_date.strftime("%Y-%m-%d") }] push_metric_event(metrics, "Consumption Metrics")