def insert_et(self, input_dict, database): """ insert_et inserts a record into the status table to show an et job has started Args: input_dict -- a dictionary with the following keys: data_date -- the date of the data for the et job table_versions -- a string of '<table_name>: <table_version>' pairs Returns: Boolean of whether the insert worked """ insert_record = StatusRecord( data_date=input_dict['data_date'], et_status="started", table_versions=input_dict['table_versions'] ) insert_record.set_updated_at() return self.psql.run_sql( get_insert_string(insert_record.minimized_record()), database, 'insert', params=insert_record.minimized_record() )
def log_status_result(self, conditions, work_time_secs, database, failed=False, err_msg=None): """ log_status_result updates a record in the status table based on what happened with a job. Args: conditions -- a dict with keys 'data_date' and 'table_versions' work_time_secs -- the integer amount of time the job took in seconds failed -- whether the job succeeded err_msg -- an error message to go in the status table Return: --- """ status = "error" if failed else "complete" if err_msg: err_msg = "{0}".format(err_msg)[:ERROR_MSG_COL_SIZE] finish_time = datetime.utcnow() update = StatusRecord( et_status=status, et_runtime=work_time_secs, et_finished=finish_time, error_message=err_msg, updated_at=finish_time ) update_string = get_update_string(update.get_set_clause()) update.record['data_date'] = conditions['data_date'] update.record['table_versions'] = conditions['table_versions'] params_dict = update.minimized_record() self.psql.run_sql( update_string, database, 'update', params=params_dict )
def test_status_record_get_insert_string(): staticconf.YamlConfiguration("config.yaml") sr = StatusRecord(data_date="2014/10/13", et_status="wip", table_versions="search: 1") output_under_test = get_insert_string(sr.minimized_record()) expected_value = "INSERT INTO sdw_pipeline_status (data_date, et_status, "\ "table_versions) VALUES (%(data_date)s, %(et_status)s, "\ "%(table_versions)s)" assert expected_value == output_under_test
def insert_et(self, input_dict, database): """ insert_et inserts a record into the status table to show an et job has started Args: input_dict -- a dictionary with the following keys: data_date -- the date of the data for the et job table_versions -- a string of '<table_name>: <table_version>' pairs Returns: Boolean of whether the insert worked """ insert_record = StatusRecord( data_date=input_dict['data_date'], et_status="started", table_versions=input_dict['table_versions']) insert_record.set_updated_at() return self.psql.run_sql(get_insert_string( insert_record.minimized_record()), database, 'insert', params=insert_record.minimized_record())
def log_status_result(self, conditions, work_time_secs, database, failed=False, err_msg=None): """ log_status_result updates a record in the status table based on what happened with a job. Args: conditions -- a dict with keys 'data_date' and 'table_versions' work_time_secs -- the integer amount of time the job took in seconds failed -- whether the job succeeded err_msg -- an error message to go in the status table Return: --- """ status = "error" if failed else "complete" if err_msg: err_msg = "{0}".format(err_msg)[:ERROR_MSG_COL_SIZE] finish_time = datetime.utcnow() update = StatusRecord(et_status=status, et_runtime=work_time_secs, et_finished=finish_time, error_message=err_msg, updated_at=finish_time) update_string = get_update_string(update.get_set_clause()) update.record['data_date'] = conditions['data_date'] update.record['table_versions'] = conditions['table_versions'] params_dict = update.minimized_record() self.psql.run_sql(update_string, database, 'update', params=params_dict)
def test_status_record_minimized_record(): expected_value = {'data_date': "2014/12/01"} sr = StatusRecord(data_date="2014/12/01") assert expected_value == sr.minimized_record()