def update_final_output_count(process_id):
table_name = JobInfo.get_process_name(process_id)
process_name = JobInfo.get_process_name(process_id)
date = '2020-05-17' #str(datetime.datetime.now().date())
output_count = StatusInfo.get_output_count(process_name, date)
MongoOperations.update_final_output_count(table_name, output_count,
date)
def update_first_start_time(process_id):
date_now = '2020-05-17' # str(datetime.datetime.now().date())
table_name = JobInfo.get_process_name(process_id)
first_start_time = MongoOperations.get_first_start_time(
table_name, date_now)
MongoOperations.update_first_start_time(process_id, date_now,
first_start_time)
def update_application_id(process_id):
table_name = JobInfo.get_process_name(process_id)
yarn_process_name = JobInfo.get_yarn_process_name(process_id)
appl_id = StatusInfo.get_application_id(yarn_process_name)
MongoOperations.update_application_id(
table_name, "2020-05-17",
appl_id) #str(datetime.datetime.now().date())
def update_be_trends_for_after_failed(process_id):
table_name = JobInfo.get_process_name(process_id)
appl_id = Processes.get_updated_application_id(table_name)
# if appl_id is not None:
# appl_id = "application_id_01" #delete thi after testing.
be_trends_dict = StatusInfo.get_be_trends(appl_id)
for be_column_name, be_column_value in be_trends_dict.items():
MongoOperations.update_be_trends_for_after_failed(
table_name, appl_id, be_column_name, be_column_value)
def get_running_processes(process_id):
process_having_updated_flag_N = MongoOperations.get_updated_flag_N_list(
)
process_having_updated_flag_Y = MongoOperations.get_updated_flag_Y_list(
)
process_having_time_lesser_than_timenow = MongoOperations.get_process_id_lesser_current_time(
)
combined_running_process = RealTimeMonitoringStatus.remove_duplicate(
process_having_time_lesser_than_timenow +
process_having_updated_flag_N)
running_process = []
for i in combined_running_process:
if i not in process_having_updated_flag_Y:
running_process.append(i)
return running_process
def insert_initial_process_detailss(process_id, jc_status):
insert_document = {}
insert_document[
"date"] = "2020-05-17" #str(datetime.datetime.now().date())
insert_document["process_id"] = process_id
insert_document["process_name"] = JobInfo.get_process_name(process_id)
insert_document["status"] = jc_status
for i in StatusInfo.get_be_trends_columns(process_id):
insert_document[i] = None
insert_document["output_count"] = None
insert_document["application_id"] = None
insert_document["start_time"] = str(datetime.datetime.now().time())
insert_document["end_time"] = None
insert_document["number_of_retries"] = None
insert_document["list_of_alerts"] = None
insert_document["failed_reasons"] = None
MongoOperations.insert_record(DEVICE_RECON, insert_document)
return None
def get_jc_name(process_id):
return MongoOperations.get_record_from_db(JOB_INFO, "process_id", process_id, "job_control_name")
def get_yarn_process_name(process_id):
return MongoOperations.get_record_from_db(JOB_INFO, "process_id", process_id, "yarn_process_name")
def get_scheduled_time(process_id):
return MongoOperations.get_record_from_db(JOB_INFO, "process_id", process_id, "start_time")
def check_for_todays_date_if_present(process_id):
table_name = JobInfo.get_process_name(process_id)
date = "2020-05-17" #str(datetime.datetime.now().date()
return MongoOperations.check_for_todays_date_if_present(
table_name, date)
def return_count_of_records(process_id):
table_name = JobInfo.get_process_name(process_id)
MongoOperations.return_count_of_records(table_name)
def update_status(process_id):
table_name = JobInfo.get_process_name(process_id)
jc_name = JobInfo.get_jc_name(process_id)
curr_status = StatusInfo.get_jc_status(
"2020-05-17", jc_name) #str(datetime.datetime.now().date()
MongoOperations.update_status(table_name, curr_status)
def update_end_time(process_id):
table_name = JobInfo.get_process_name(process_id)
MongoOperations.update_end_time(
table_name,
'2020-05-17', #str(datetime.datetime.now().date()),
str(datetime.datetime.now().time()))
def update_status_RealTimeMonitoringStatus(process_id):
date_now = '2020-05-17' #str(datetime.datetime.now().date())
table_name = JobInfo.get_process_name(process_id)
MongoOperations.update_status_RealTimeMonitoringStatus(
table_name, process_id, date_now)
def update_updated_flag_Y(process_id):
date_now = '2020-05-17' #str(datetime.datetime.now().date())
MongoOperations.update_updated_flag_Y(process_id, date_now)
def insert_record_into_RealTimeMonitoringStatus(self):
MongoOperations.get_all_from_job_info_and_insert_RealTimeMonitoringStatus(
)
def get_updated_application_id(table_name):
curr_date = '2020-05-17' #str(datetime.datetime.now().date())
return MongoOperations.get_latest_app_id(table_name, curr_date)
def get_current_status(process_id):
table_name = JobInfo.get_process_name(process_id)
return str(MongoOperations.get_max_record_from_db(table_name))
def update_expected_end_time(process_id):
date_now = '2020-05-17' # str(datetime.datetime.now().date())
table_name = JobInfo.get_process_name(process_id)
MongoOperations.update_expected_end_time(table_name, process_id,
date_now)
def update_last_end_time(process_id):
date_now = '2020-05-17' # str(datetime.datetime.now().date())
table_name = JobInfo.get_process_name(process_id)
last_end_time = MongoOperations.get_last_end_time(table_name, date_now)
MongoOperations.update_last_end_time(process_id, date_now,
last_end_time)