def key_exists_in_dict(key, section): fl_cfg_dict = ConfigData(gc.CONFIG_FILE_DICTIONARY) key = replace_unacceptable_chars(key, gc.ASSAY_CHARS_TO_REPLACE) try: v = fl_cfg_dict.get_item_by_key(section + "/" + key) if v is not None: return True else: return False except Exception: return False
def get_dict_value(key, section): fl_cfg_dict = ConfigData(gc.CONFIG_FILE_DICTIONARY) # replace spaces and slashes with "_" key = replace_unacceptable_chars(key, gc.ASSAY_CHARS_TO_REPLACE) try: v = fl_cfg_dict.get_item_by_key(section + "/" + key) if v is not None: return v else: return key except Exception: return key
def send_yagmail(emails_to, subject, message, main_conf=None, email_from=None, attachment_path=None, smtp_server=None, smtp_server_port=None): if main_conf: m_cfg = main_conf else: root_dir = cm.get_project_root() cnf_path = str(root_dir.joinpath(gc.CONFIG_FILE_MAIN)) m_cfg = ConfigData(cnf_path) if not email_from: email_from = m_cfg.get_value('Email/default_from_email') if not smtp_server: smtp_server = cm.get_environment_variable( m_cfg.get_item_by_key('Email/smtp_server_env_name')) if not smtp_server_port: smtp_server_port = cm.get_environment_variable( m_cfg.get_item_by_key('Email/smtp_server_port_env_name')) # receiver = emails_to # '[email protected], [email protected], [email protected]' body = message filename = attachment_path # 'test.png' yag = yagmail.SMTP(email_from, host=smtp_server, smtp_skip_login=True, smtp_ssl=False, soft_email_validation=False, port=smtp_server_port) yag.send( to=emails_to, subject=subject, contents=body, attachments=filename, )
class MetadataDB: # CFG_DB_CONN = 'DB/mdb_conn_str' # name of the config parameter storing DB connection string # CFG_DB_SQL_PROC = 'DB/mdb_sql_proc_load_sample' # name of the config parameter storing DB name of the stored proc # CFG_DB_STUDY_ID = 'DB/mdb_study_id' # name of the config parameter storing key of the MDB study id # CFG_DICT_PATH = 'DB/dict_tmpl_fields_node' # name of the config parameter storing key of dictionary path # to list of fields # CFG_DB_ALLOW_DICT_UPDATE = 'DB/mdb_allow_dict_update' # name of the config parameter storing values # for "allow dict updates" # CFG_DB_ALLOW_SAMPLE_UPDATE = 'DB/mdb_allow_sample_update' # name of the config parameter storing values # for "allow sample updates" s_conn = '' conn = None def __init__(self, study_cfg): self.cfg = ConfigData(gc.CONFIG_FILE_MAIN) # obj_cfg self.s_conn = self.cfg.get_item_by_key(gc.CFG_DB_CONN).strip() self.study_cfg = study_cfg def open_connection(self): self.conn = pyodbc.connect(self.s_conn, autocommit=True) def submit_row(self, row, file): # sample_id, row_json, dict_json, filepath): dict_json = file.get_file_dictionary_json(True) filepath = str(file.filepath) sample_id = row.sample_id row_json = row.to_json() if not self.conn: self.open_connection() str_proc = self.cfg.get_item_by_key(gc.CFG_DB_SQL_PROC).strip() study_id = self.study_cfg.get_item_by_key(gc.CFG_DB_STUDY_ID).strip() dict_path = '$.' + self.study_cfg.get_item_by_key( gc.CFG_DICT_PATH).strip() dict_upd = self.study_cfg.get_item_by_key( gc.CFG_DB_ALLOW_DICT_UPDATE).strip() sample_upd = self.study_cfg.get_item_by_key( gc.CFG_DB_ALLOW_SAMPLE_UPDATE).strip() # prepare stored proc string to be executed str_proc = str_proc.replace( self.cfg.get_item_by_key(gc.CFG_FLD_TMPL_STUDY_ID), study_id) # '{study_id}' str_proc = str_proc.replace( self.cfg.get_item_by_key(gc.CFG_FLD_TMPL_SAMPLE_ID), sample_id) # '{sample_id}' str_proc = str_proc.replace( self.cfg.get_item_by_key(gc.CFG_FLD_TMPL_ROW_JSON), row_json) # '{smpl_json}' str_proc = str_proc.replace( self.cfg.get_item_by_key(gc.CFG_FLD_TMPL_DICT_JSON), dict_json) # '{dict_json}' str_proc = str_proc.replace( self.cfg.get_item_by_key(gc.CFG_FLD_TMPL_DICT_PATH), dict_path) # '{dict_path}' str_proc = str_proc.replace( self.cfg.get_item_by_key(gc.CFG_FLD_TMPL_FILEPATH), filepath) # '{filepath}' str_proc = str_proc.replace( self.cfg.get_item_by_key(gc.CFG_FLD_TMPL_DICT_UPD), dict_upd) # '{dict_update}' str_proc = str_proc.replace( self.cfg.get_item_by_key(gc.CFG_FLD_TMPL_SAMPLE_UPD), sample_upd) # '{samlpe_update}' # get currrent file_processing_log file.logger.debug('SQL Procedure call = {}'.format(str_proc)) # print ('procedure (str_proc) = {}'.format(str_proc)) try: cursor = self.conn.cursor() cursor.execute(str_proc) # returned recordsets rs_out = [] rows = cursor.fetchall() columns = [column[0] for column in cursor.description] results = [] for row in rows: results.append(dict(zip(columns, row))) rs_out.append(results) return rs_out except Exception as ex: # report an error if DB call has failed. _str = 'Error "{}" occurred during submitting a row (sample_id = "{}") to database; ' \ 'used SQL script "{}". Here is the traceback: \n{} '.format( ex, sample_id, str_proc, traceback.format_exc()) row.error.add_error(_str) file.logger.error(_str)
class Request(File): def __init__(self, filepath, main_cfg, file_type=2, sheet_name=''): # load_configuration (main_cfg_obj) # load global and local configureations File.__init__(self, filepath, file_type) if main_cfg: self.conf_main = main_cfg else: self.conf_main = ConfigData(gc.CONFIG_FILE_MAIN) # if cfg_path=='': # self.conf_main = ConfigData(gc.CONFIG_FILE_MAIN) # else: # self.conf_main = ConfigData(cfg_path) self.error = RequestError(self) self.log_handler = None self.logger = self.setup_logger(self.wrkdir, self.filename) self.logger.info( 'Start working with Submission request file {}'.format(filepath)) # self.file_dict = OrderedDict() # self.rows = OrderedDict() self.columnlist = [] self.samples = [] self.sub_aliquots = [] self.disqualified_sub_aliquots = {} self.aliquots_to_subaliquots_map = { } # holds the map of aliquots to sub-aliquots for interpreting DB responses self.disqualified_request_path = '' # will store path to a request file with disqualified sub-aliquots self.project = '' self.bulk_location = '' self.assay = '' self.center = '' self.center_id = None self.center_code = None self.experiment_id = '' self.data_source_names = '' self.data_source_objects = { } # dictionary to store all collected data sources for the request self.aliquots = None self.qualified_aliquots = None self.raw_data = None self.assay_data = None self.attachments = None self.submission_forms = None self.submission_package = None self.data_source_names = None # will hold value corresponding to the type of data source being used (attachments are not ignored) # possible value 'db' and 'file'. The value of the variable being set based on the first data source being used self.data_source_forms_assignment = None # self.sheet_name = '' self.sheet_name = sheet_name.strip() if not self.sheet_name or len(self.sheet_name) == 0: # if sheet name was not passed as a parameter, try to get it from config file self.sheet_name = gc.REQUEST_EXCEL_WK_SHEET_NAME # 'wk_sheet_name' # print (self.sheet_name) self.logger.info('Data will be loaded from worksheet: "{}"'.format( self.sheet_name)) self.conf_assay = None self.get_file_content() def get_file_content(self): if not self.columnlist: if cm.file_exists(self.filepath): self.logger.debug('Loading file content of "{}"'.format( self.filepath)) with xlrd.open_workbook(self.filepath) as wb: if not self.sheet_name or len(self.sheet_name) == 0: # by default retrieve the first sheet in the excel file sheet = wb.sheet_by_index(0) else: # if sheet name was provided sheets = wb.sheet_names() # get list of all sheets if self.sheet_name in sheets: # if given sheet name in the list of available sheets, load the sheet sheet = wb.sheet_by_name(self.sheet_name) else: # report an error if given sheet name not in the list of available sheets _str = ( 'Given worksheet name "{}" was not found in the file "{}". ' 'Verify that the worksheet name exists in the file.' ).format(self.sheet_name, self.filepath) self.error.add_error(_str) self.logger.error(_str) self.lineList = None self.loaded = False return self.lineList sheet.cell_value(0, 0) lines = [ ] # will hold content of the request file as an array of arrays (rows) for i in range(sheet.ncols): column = [] for j in range(sheet.nrows): if i == 0: lines.append( [] ) # adds an array for each new row in the request file # print(sheet.cell_value(i, j)) cell = sheet.cell(j, i) cell_value = cell.value # take care of number and dates received from Excel and converted to float by default if cell.ctype == 2 and int(cell_value) == cell_value: # the key is integer cell_value = str(int(cell_value)) elif cell.ctype == 2: # the key is float cell_value = str(cell_value) # convert date back to human readable date format # print ('cell_value = {}'.format(cell_value)) if cell.ctype == 3: cell_value_date = xlrd.xldate_as_datetime( cell_value, wb.datemode) cell_value = cell_value_date.strftime( "%Y-%m-%directory") column.append( cell_value ) # adds value to the current column array lines[j].append( '"' + str(cell_value) + '"' ) # adds value in "csv" format for a current row # self.columnlist.append(','.join(column)) self.columnlist.append( column) # adds a column to a list of columns # populate lineList property self.lineList = [] for ln in lines: self.lineList.append(','.join(ln)) wb.unload_sheet(sheet.name) # load passed request parameters (by columns) self.get_request_parameters() # validate provided information self.logger.info( 'Validating provided request parameters. project: "{}", bulk location: "{}", ' 'assay: "{}", db_center_code_or_id: "{}",' 'Sub-Aliquots: "{}"'.format(self.project, self.bulk_location, self.assay, self.center, self.sub_aliquots)) self.validate_request_params() if self.error.exist(): # report that errors exist self.loaded = False # print(self.error.count) # print(self.error.get_errors_to_str()) _str = 'Errors ({}) were identified during validating of the request. \nError(s): {}'.format( self.error.count, self.error.get_errors_to_str()) else: self.loaded = True _str = 'Request parameters were successfully validated - no errors found.' self.logger.info(_str) # combine Experiment_id out of request parameters if self.center_code and len(self.center_code.strip()) > 0: # use center code if available self.experiment_id = "_".join( [self.project, self.center_code, self.assay]) else: # use provided value for the center column from request, if center_code is not available self.experiment_id = "_".join( [self.project, self.center, self.assay]) else: _str = 'Loading content of the file "{}" failed since the file does not appear to exist".'.format( self.filepath) self.error.add_error(_str) self.logger.error(_str) self.columnlist = None self.lineList = None self.loaded = False return self.lineList # get all values provided in the request file def get_request_parameters(self): self.project = self.columnlist[0][1] self.bulk_location = self.columnlist[1][1] self.assay = self.columnlist[2][1].lower() self.center = self.columnlist[3][ 1] # center code (if alpha numeric) or center id (if numeric) self.sub_aliquots = self.columnlist[4] if self.sub_aliquots and len(self.sub_aliquots) > 0: self.sub_aliquots.pop(0) # get rid of the column header # self.samples = self.columnlist[5] # if self.samples and len(self.samples) > 0: # self.samples.pop(0) # get rid of the column header # validates provided parameters (loaded from the submission request file) def validate_request_params(self): _str_err = '' _str_warn = '' if len(self.sub_aliquots) == 0: _str_err = '\n'.join([ _str_err, 'List of provided sub-samples is empty. ' 'Aborting processing of the submission request.' ]) # Check if empty sub-samples were provided if '' in self.sub_aliquots: i = 0 cleaned_cnt = 0 for s in self.sub_aliquots: # check for any empty sub-aliquot values and remove them if len(s.strip()) == 0: self.sub_aliquots.pop(i) cleaned_cnt += 1 else: i += 1 if cleaned_cnt > 0: _str_warn = '\n'.join([ _str_warn, 'Empty sub-aliqouts (count = {}) were removed from the list. ' 'Here is the list of sub-aliqouts after cleaning (count = {}): "{}" ' .format(cleaned_cnt, len(self.sub_aliquots), self.sub_aliquots) ]) # check for empty values if len(self.project) == 0: _str_err = '\n'.join([ _str_err, 'No Program name was provided. Aborting processing of the submission request.' ]) if len(self.bulk_location) == 0: _str_err = '\n'.join([ _str_err, 'No Bulk Location was provided. Aborting processing of the submission request.' ]) if len(self.assay) == 0: _str_err = '\n'.join([ _str_err, 'No Assay was provided. Aborting processing of the submission request.' ]) if len(self.center) == 0: _str_err = '\n'.join([ _str_err, 'No DB Center information was provided. Aborting processing of the submission request.' ]) # check for values that should match some predefined values from a dictionary # check assay value if not cm2.key_exists_in_dict(self.assay, 'assay'): _str_err = '\n'.join([ _str_err, 'Provided Assay name "{}" is not matching a list of expected assay names ' '(as stored in "{}" dictionary file). ' 'Aborting processing of the submission request.'.format( self.assay, gc.CONFIG_FILE_DICTIONARY) ]) else: # if provided assay name is expected, convert it to the name expected by the Submission logic self.assay = cm2.get_dict_value(self.assay, 'assay') # check project value if not cm2.key_exists_in_dict(self.project.lower(), 'project'): _str_err = '\n'.join([ _str_err, 'Provided Program name "{}" is not matching a list of expected names ' '(as stored in "{}" dictionary file). ' 'Aborting processing of the submission request.'.format( self.project, gc.CONFIG_FILE_DICTIONARY) ]) else: # if provided assay name is expected, convert it to the name expected by the Submission logic self.project = cm2.get_dict_value(self.project.lower(), 'project') # validate center_code or center_id value self.logger.info( 'Start validation of center value "{}" provided in the request'. format(self.center)) db = DBAccess(self.logger, self.error, self.conf_main) # create DBAccess object db.open_connection() # test center value assuming center code was provided dataset = db.validate_center_code(self.center, self.project, 'code', 'code') _str_err_out1, center_id_out1 = self.check_validation_dataset_outcome( dataset, 'center_id', 'center_code') if center_id_out1: # center id was returned, meaning center was validated fine self.center_id = center_id_out1 # get center code value from the current DB dataset _str_err_out3, center_code = self.get_field_value_from_dataset( dataset, 'center_code') if center_code: # center code retrieved OK self.center_code = center_code else: # report an error during retrieving center_code _str_err = '\n'.join([_str_err, _str_err_out3]) else: # if center code was not validated at first attempt, validate it assuming the center id was given dataset = db.validate_center_code(self.center, self.project, 'id', 'code') _str_err_out2, center_id_out2 = self.check_validation_dataset_outcome( dataset, 'center_id', 'center_id') if center_id_out2: # center id was validated at the 2nd attempt, ignore the 1st failed center code validation self.center_id = center_id_out2 # get center code value from the current DB dataset _str_err_out3, center_code = self.get_field_value_from_dataset( dataset, 'center_code') if center_code: # center code retrieved OK self.center_code = center_code else: # report an error during retrieving center_code _str_err = '\n'.join([_str_err, _str_err_out3]) else: # center validation attempts failed, report both failures _str_err = '\n'.join([_str_err, _str_err_out1, _str_err_out2]) # get list of aliquots from list of sub-aliquots self.aliquots = [ cm2.convert_sub_aliq_to_aliquot(al, self.assay) for al in self.sub_aliquots ] # create a map to convert aliquot value to sub_aliquot value (for processing DB responses given for aliquots) for sa, a in zip(self.sub_aliquots, self.aliquots): self.aliquots_to_subaliquots_map[a] = sa if self.center_id: self.logger.info('Start validation of aliquot ids vs DB') # if center id was validated in the above code, validate received aliquots vs manifest dataset in DB dataset = db.validate_aliquot_ids(self.center_id, self.aliquots) if dataset: # create dictionary of received aliquots/sample ids aliquots_to_samples_map = {} for row in dataset: if '_aliquot_id' in row and '_sample_id' in row: aliquots_to_samples_map[ row['_aliquot_id']] = row['_sample_id'] # check if each aliquot id was returned from a database and get the sample id from the dataset for sa, a in zip(self.sub_aliquots, self.aliquots): if a in aliquots_to_samples_map: if len(str(aliquots_to_samples_map[a]).strip()) > 0: self.samples.append(aliquots_to_samples_map[a]) else: _str = 'Blank Sample Id value was returned from DB for the sub-aliquot id "{}". ' \ 'The sub-aliquot was disqualified'.format(sa) self.disqualify_sub_aliquot(sa, _str) _str_warn = '\n'.join([_str_warn, _str]) else: _str = 'Sub-aliquot id "{}" was not found in the database and was disqualified'.format( sa) self.disqualify_sub_aliquot(sa, _str) _str_warn = '\n'.join([_str_warn, _str]) else: _str_err = '\n'.join([ _str_err, 'Aliquot ids cannot be validated since no data was returned from DB for ' 'center_id = "{}" and aliquot ids as following: {} '. format(self.center_id, self.aliquots) ]) db = None # report any collected errors if len(_str_err) > 0: _str_err = 'Validation of request parameters:' + _str_err self.error.add_error(_str_err) self.logger.error(_str_err) # report any collected warnings if len(_str_warn) > 0: _str_warn = 'Validation of request parameters:' + _str_warn self.logger.warning(_str_warn) def check_validation_dataset_outcome(self, dataset, validation_id_column, validation_id_name): _str_err = '' row_num = 1 validation_id_out = None if dataset: if len(dataset) >= row_num: row = dataset[row_num - 1] # get the first row of the dataset if 'status' in row: status = row['status'] if 'description' in row: description = row['description'] if validation_id_column in row: # center_id validation_id = row[validation_id_column] if status == 'OK': # validation was successful validation_id_out = validation_id elif status == 'Failed': # validation has failed _str_err = '\n'.join([ _str_err, 'Validation of the provided {} value vs DB has Failed, description: {}' .format(validation_id_name, description) ]) else: # unexpected status value was returned _str_err = '\n'.join([ _str_err, 'Validation of the provided {} value vs DB returned unexpected status {}' .format(validation_id_name, status) ]) else: _str_err = '\n'.join([ _str_err, 'Unexpected error was reported during validating {} in the DB. ' 'Check earlier entries in the log file.'.format( validation_id_name) ]) return _str_err, validation_id_out def get_field_value_from_dataset(self, dataset, field_name, row_num=None): # set default values if row_num is None: row_num = 1 # default row is #1 _str_err = '' value_out = None if dataset: if len(dataset) >= row_num: row = dataset[row_num - 1] if field_name in row: value_out = row[field_name] else: _str_err = '\n'.join([ _str_err, 'Unexpected error was reported during retrieving value of "{}" (row #{})from the dataset. ' .format(field_name, row_num) ]) return _str_err, value_out def setup_logger(self, wrkdir, filename): # m_cfg = ConfigData(gc.CONFIG_FILE_MAIN) log_folder_name = gc.REQ_LOG_DIR # gc.LOG_FOLDER_NAME # m_logger_name = gc.MAIN_LOG_NAME # m_logger = logging.getLogger(m_logger_name) logger_name = gc.REQUEST_LOG_NAME logging_level = self.conf_main.get_value('Logging/request_log_level') # if a relative path provided, convert it to the absolute address based on the application working dir if not os.path.isabs(log_folder_name): log_folder_path = Path(wrkdir) / log_folder_name else: log_folder_path = Path(log_folder_name) lg = setup_logger_common( logger_name, logging_level, log_folder_path, # Path(wrkdir) / log_folder_name, str(filename) + '_' + time.strftime("%Y%m%d_%H%M%S", time.localtime()) + '.log') self.log_handler = lg['handler'] return lg['logger'] def load_request_configuration(self): # update main config file with the project/environmetn specific details from additional config files self.load_project_config_into_main( self.project ) # loads project specific config and merges it into main config # load project specific assay config file self.conf_assay = self.load_assay_conf(self.assay, self.project) if self.conf_assay: # update loaded assay config file with project/environment specific config assay_locatoin_config.yaml self.conf_assay = self.update_cfg_dictionary_with_location_details( gc.CONFIG_FILE_ASSAY_LOCATION, self.project, self.conf_assay) def process_request(self): self.data_source_names = cm.get_value_from_dictionary( 'data_sources', self.conf_assay) # self.conf_assay['data_sources'] # path to the folder where created submission packages will be located. # since this location can be provided in the project config file, this assignment is happening # after loading the project config gc.OUTPUT_PACKAGES_DIR = self.conf_main.get_value( 'Submission_location/output_packages') for data_source_name in self.data_source_names: # if isinstance(data_source_name, tuple) if isinstance(data_source_name, str): if data_source_name == 'attachment': self.attachments = Attachment(self) elif data_source_name[-3:] == "_db": self.data_source_objects[data_source_name] = DataSourceDB( self, data_source_name, data_source_name) if not self.data_source_forms_assignment: self.data_source_forms_assignment = 'db' else: self.data_source_objects[data_source_name] = DataSource( self, data_source_name, data_source_name) if not self.data_source_forms_assignment: self.data_source_forms_assignment = 'file' elif isinstance(data_source_name, tuple): if data_source_name[0][-3:] == "_db": self.data_source_objects[ data_source_name[0]] = DataSourceDB( self, data_source_name[0], data_source_name[1]) else: self.data_source_objects[data_source_name[0]] = DataSource( self, data_source_name[0], data_source_name[1]) else: self.logger.error( 'Provided data source name ({}) is of unexpected format and cannot be processed.' .format(data_source_name)) # if data_source_forms_assignment was not assigned with any value in code before, assign a default to it # this a case when an assay submits only attachments and do not use any assay or QC data if not self.data_source_forms_assignment: self.data_source_forms_assignment = gc.DEFAULT_DATA_SOURCE_FORMS_ASSIGNMENT self.submission_package = SubmissionPackage(self) self.create_request_for_disqualified_sub_aliquots() self.create_trasfer_script_file() # check for errors and put final log entry for the request. if self.error.exist(): _str = 'Processing of the current request was finished with the following errors: {}\n'.format( self.error.get_errors_to_str()) self.logger.error(_str) else: _str = 'Processing of the current request was finished successfully.\n' self.logger.info(_str) def load_assay_conf(self, assay, project): assay_cfg_path = gc.CONFIG_FILE_ASSAY.replace('{project}', project) cfg_assay = ConfigData(assay_cfg_path) assay_config = cfg_assay.get_value(assay.upper()) if assay_config: self.logger.info( "Configuration for the {} assay was loaded from the assay config file: {}. " .format(assay.upper(), assay_cfg_path)) else: _str = "Configuration for the {} assay CANNOT be loaded from the assay config file: {}. " \ "Aborting execution.".format(assay.upper(), assay_cfg_path) self.logger.error(_str) self.error.add_error(_str) return assay_config # def update_cfg_assay_with_location_details(self, project, cfg_assay): # cfg_assay_location = ConfigData(gc.CONFIG_FILE_ASSAY_LOCATION.replace('{project}', project)) # if cfg_assay_location.loaded: # self.logger.info('Local config file "{}" was loaded and being used.'.format(cfg_assay_location.cfg_path)) # cfg_assay = cm.update_dictionary_matching_keys(cfg_assay, cfg_assay_location.get_whole_dictionary()) # else: # _str = 'Local config file "{}" was NOT loaded. Aborting processing of the current request file.'\ # .format(cfg_assay_location.cfg_path) # self.logger.error(_str) # self.error.add_error(_str) # return cfg_assay def update_cfg_dictionary_with_location_details(self, location_path, project, cfg_to_update): cfg_location = ConfigData(location_path.replace('{project}', project)) if cfg_location.loaded: self.logger.info( 'Local config file "{}" was loaded and being used.'.format( cfg_location.cfg_path)) cfg_to_update = cm.update_dictionary_matching_keys( cfg_to_update, cfg_location.get_whole_dictionary()) else: _str = 'Local config file "{}" was NOT loaded. Aborting processing of the current request file.'\ .format(cfg_location.cfg_path) self.logger.error(_str) self.error.add_error(_str) return cfg_to_update def load_project_config_into_main(self, project): # load project specific "project_config" config file cfg_project = ConfigData( gc.CONFIG_FILE_PROJECT.replace('{project}', project)) if cfg_project.loaded: # if cfg_project was loaded, update it with the environment specific settings (from project_location config) cfg_project_updated = self.update_cfg_dictionary_with_location_details( gc.CONFIG_FILE_PROJECT_LOCATION, self.project, cfg_project.get_whole_dictionary()) # update main config with the outcome of the previous updates self.conf_main.update(cfg_project_updated) def create_trasfer_script_file(self): self.logger.info("Start preparing transfer_script.sh file.") # path for the script file being created sf_path = Path(self.submission_package.submission_dir + "/transfer_script.sh") # get script file template with open('scripts/' + self.project + '/transfer_script.sh', 'r') as ft: scr_tmpl = ft.read() # update placeholders in the script with the actual values smtp_server = cm.get_environment_variable( self.conf_main.get_item_by_key('Email/smtp_server_env_name')) smtp_port = cm.get_environment_variable( self.conf_main.get_item_by_key('Email/smtp_server_port_env_name')) scr_tmpl = cm.replace_value_in_string( scr_tmpl, "{!smtp!}", smtp_server + ":" + str(smtp_port)) scr_tmpl = cm.replace_value_in_string( scr_tmpl, "{!to_email!}", ','.join(self.conf_main.get_value("Email/sent_to_emails"))) scr_tmpl = cm.replace_value_in_string( scr_tmpl, "{!from_email!}", self.conf_main.get_value("Email/default_from_email")) scr_tmpl = cm.replace_value_in_string( scr_tmpl, "{!send_email_flag!}", str(self.conf_main.get_value("Email/send_emails"))) scr_tmpl = cm.replace_value_in_string( scr_tmpl, "{!cmd!}", self.conf_main.get_value("DataTransfer/transfer_command")) # the following will be utilized if mount point is being used by the transfer script (i.e. for Peerless) scr_tmpl = cm.replace_value_in_string( scr_tmpl, "{!mp_cmd!}", self.conf_main.get_value("DataTransfer/mount_point_command")) scr_tmpl = cm.replace_value_in_string( scr_tmpl, "{!mount_local_dir!}", self.conf_main.get_value("DataTransfer/mount_local_dir")) scr_tmpl = cm.replace_value_in_string( scr_tmpl, "{!mount_remote_dir!}", self.conf_main.get_value("DataTransfer/mount_remote_dir")) scr_tmpl = cm.replace_value_in_string( scr_tmpl, "{!source_dir!}", self.submission_package.submission_dir) scr_tmpl = cm.replace_value_in_string( scr_tmpl, "{!target_dir!}", self.conf_main.get_value("DataTransfer/remote_target_dir")) ssh_server = cm.get_environment_variable( self.conf_main.get_item_by_key('DataTransfer/ssh_server_env_name')) scr_tmpl = cm.replace_value_in_string(scr_tmpl, "{!ssh_server!}", str(ssh_server)) # apply user name as the very last replacement statement, since it can be used as part of previous replacements ssh_user = cm.get_environment_variable( self.conf_main.get_item_by_key('DataTransfer/ssh_user_env_name')) scr_tmpl = cm.replace_value_in_string(scr_tmpl, "{!ssh_user!}", str(ssh_user)) set_permissions = False set_perm_value = self.conf_main.get_value("DataTransfer/exec_permis") if set_perm_value: try: exec_permission = eval(set_perm_value.strip()) set_permissions = True except Exception as ex: _str = 'Unexpected error Error "{}" occurred during evaluating of "DataTransfer/exec_permis" value ' \ '"{}" retrieved from the main config file. Permission setup operation will be skipped. \n{} '\ .format(ex, set_perm_value, traceback.format_exc()) self.logger.warning(_str) # self.error.add_error(_str) set_permissions = False with open(sf_path, "w") as sf: sf.write(scr_tmpl) if set_permissions: try: # if permissions to be set were retrieved from config file, set them here st = os.stat(sf_path) os.chmod(sf_path, st.st_mode | exec_permission) #stat.S_IXUSR except Exception as ex: _str = 'Unexpected error Error "{}" occurred during setting up permissions "{}" for the script file ' \ '"{}". \n{} '\ .format(ex, set_perm_value, sf_path, traceback.format_exc()) self.logger.warning(_str) self.error.add_error(_str) else: _str = 'Permission setup was skipped for the transfer script file. ' \ 'Note: value of "DataTransfer/exec_permis" from main config was set to "{}".'\ .format(set_perm_value) self.logger.warning(_str) self.logger.info("Finish preparing '{}' file.".format(sf_path)) def disqualify_sub_aliquot(self, sa, details): # adds a sub aliquots to the disctionary of disqualified sub_aliquots # key = sub-aliquot, value = array of details for disqualification; 1 entry can have multiple detail reasons if sa in self.disqualified_sub_aliquots.keys(): self.disqualified_sub_aliquots[sa].append(details) else: arr_details = [details] self.disqualified_sub_aliquots[sa] = arr_details self.logger.warning( 'Sub-aliquot "{}" was disqualified with the following details: "{}"' .format(sa, details)) def populate_qualified_aliquots(self): # reset self.qualified_aliquots array self.qualified_aliquots = [] #select only aliquots that were not disqualified for sa, a in zip(self.sub_aliquots, self.aliquots): if not sa in self.disqualified_sub_aliquots.keys(): self.qualified_aliquots.append(a) def create_request_for_disqualified_sub_aliquots(self): # proceed only if some disqualified sub-aliquots are present if self.disqualified_sub_aliquots: self.logger.info( "Start preparing a request file for disqualified sub-aliquots '{}'." .format([val for val in self.disqualified_sub_aliquots.keys()])) wb = xlwt.Workbook() # create empty workbook object sh = wb.add_sheet( 'Submission_Request' ) # sheet name can not be longer than 32 characters cur_row = 0 # first row for 0-based array cur_col = 0 # first col for 0-based array #write headers to the file headers = self.get_headers() for val in headers: sh.write(cur_row, cur_col, val) cur_col += 1 cur_row += 1 for sa in self.sub_aliquots: if sa in self.disqualified_sub_aliquots.keys(): sh.write(cur_row, 0, self.project) sh.write(cur_row, 1, self.bulk_location) sh.write(cur_row, 2, self.assay) sh.write(cur_row, 3, self.center) sh.write(cur_row, 4, sa) cur_row += 1 self.disqualified_request_path = Path( gc.DISQUALIFIED_REQUESTS + '/' + time.strftime("%Y%m%d_%H%M%S", time.localtime()) + '_reprocess_disqualified _' + Path(self.filename).stem + '.xls') # if DISQUALIFIED_REQUESTS folder does not exist, it will be created os.makedirs(gc.DISQUALIFIED_REQUESTS, exist_ok=True) wb.save(str(self.disqualified_request_path)) self.logger.info( "Successfully prepared the request file for disqualified sub-aliquots and saved in '{}'." .format(str(self.disqualified_request_path)))