def replace_placeholders(self): tmp_plcholders_map=self.get_placeholders_mapping().items() tmp_script_map=self.get_script_order() for key, value in tmp_plcholders_map: for script in tmp_script_map: current_script = self.run_base_dir + path_sep + script.script_file.replace("/",path_sep) if key == '<dbkit_path>': x = 1 # Populate the current dir inside the SCRIPT_DIR env. if os.path.isdir(current_script): os.environ["SCRIPT_DIR"] = current_script elif os.path.isfile(current_script): os.environ["SCRIPT_DIR"] = os.path.dirname(current_script) replaced_value = os.environ.get(value) if not replaced_value: continue while len(config.rec_param_holders(replaced_value)) > 0: replaced_value = config.replace_string_params(replaced_value,config.rec_param_holders(replaced_value)) os.environ[value] = replaced_value replaced_key = key logging.debug("replacing {} with {} in file {} ".format(replaced_key,replaced_value,current_script)) self.replace_string_in_path(current_script,replaced_key,replaced_value) logging.debug("replacing {} with {} in file {} ".format(replaced_key,replaced_value,self.activation_dir)) self.replace_string_in_path(self.activation_dir,replaced_key,replaced_value)
def replace_placeholders(self): tmp_plcholders_map = self.get_placeholders_mapping().items() tmp_script_map = self.get_script_order() for key, value in tmp_plcholders_map: for script in tmp_script_map: current_script = self.run_base_dir + path_sep + script.script_file.replace( "/", path_sep) if key == '<dbkit_path>': x = 1 # Populate the current dir inside the SCRIPT_DIR env. if os.path.isdir(current_script): os.environ["SCRIPT_DIR"] = current_script elif os.path.isfile(current_script): os.environ["SCRIPT_DIR"] = os.path.dirname(current_script) replaced_value = os.environ.get(value) if not replaced_value: continue while len(config.rec_param_holders(replaced_value)) > 0: replaced_value = config.replace_string_params( replaced_value, config.rec_param_holders(replaced_value)) os.environ[value] = replaced_value replaced_key = key logging.debug("replacing {} with {} in file {} ".format( replaced_key, replaced_value, current_script)) self.replace_string_in_path(current_script, replaced_key, replaced_value) logging.debug("replacing {} with {} in file {} ".format( replaced_key, replaced_value, self.activation_dir)) self.replace_string_in_path(self.activation_dir, replaced_key, replaced_value)
def run_single_file (self,current_script): for util in self.utility: log_number=0 is_success=False out_file = None while not is_success and log_number < int(self.error_handling.num_of_retries): os.environ["SQL_FILE"] = current_script os.environ["LOG_FILE"] = current_script + ".log_" + str(log_number) for param in self.util_params: if not os.environ.get(param): logging.error("No parameter {} is set in environment but it is requested by the db_component utility {} ".format(param,self.utility)) raise DbException("No parameter {} is set in environment but it is requested by the db_component utility {} ".format(param,self.utility)) run_util = config.replace_string_params(util,self.util_params) logging.info("running script : {} on cloud : {} ".format(os.environ.get("SQL_FILE"),os.environ.get("DB_NAME"))) logging.debug("running statement (password masked XXXX) : {}".format(run_util)) logpipe = LogPipe(logging.INFO) #errpipe = LogPipe(logging.ERROR) process = subprocess.Popen(run_util, shell=True, stdout=logpipe, stderr=logpipe) # wait for the process to terminate out, err = process.communicate() ret_code = process.returncode logpipe.close() if ret_code != 0 : if not self.error_handling.ignored_error_list: out_file = config.read_file_content(os.environ.get("LOG_FILE")) raise DbException("script failed : {} please see log {} ".format(os.environ.get("SQL_FILE"),os.environ.get("LOG_FILE"))) if "all" in self.error_handling.ignored_error_list: logging.warning("Script {} failed with errors , skipping termination due to {} setting on 'ignored_err_list' property.".format(os.environ.get("SQL_FILE"),self.error_handling.ignored_error_list)) is_success = True break # Check for ignored errors and regular erros if os.path.exists(os.environ.get("LOG_FILE")): if self.search_for_errors(os.environ.get("LOG_FILE"), self.error_handling.error_list,self.error_handling.ignored_error_list): if not out_file: out_file = config.read_file_content(os.environ.get("LOG_FILE")) raise DbException("script failed : {} please check log {} content : {}".format(os.environ.get("SQL_FILE"),os.environ.get("LOG_FILE"),os.linesep + out_file)) else: is_success = True else: is_success = True out_file = config.read_file_content(os.environ.get("LOG_FILE")) logging.warning("Retry [ {} ] script failed : {} please check log {}".format(log_number,os.environ.get("SQL_FILE"),os.environ.get("LOG_FILE"))) if not out_file and os.path.exists(os.environ["LOG_FILE"]): out_file = config.read_file_content(os.environ.get("LOG_FILE")) log_number = log_number + 1 else: if os.path.exists(os.environ.get("LOG_FILE")): if self.search_for_errors(os.environ.get("LOG_FILE"), self.error_handling.error_list,self.error_handling.ignored_error_list): if not out_file: out_file = config.read_file_content(os.environ.get("LOG_FILE")) raise DbException("script failed : {} please check log {} content : {}".format(os.environ.get("SQL_FILE"),os.environ.get("LOG_FILE"),os.linesep + out_file)) else: is_success = True else: is_success = True if log_number == int(self.error_handling.num_of_retries) and os.path.exists(os.environ.get("LOG_FILE")): raise DbException("script failed : {} please check log {} content : {}".format(os.environ.get("SQL_FILE"),os.environ.get("LOG_FILE"),os.linesep + out_file)) else: if log_number == int(self.error_handling.num_of_retries): raise DbException("script failed : {} ".format(os.environ.get("SQL_FILE")))
def run_single_file(self, current_script): for util in self.utility: log_number = 0 is_success = False out_file = None while not is_success and log_number < int( self.error_handling.num_of_retries): os.environ["SQL_FILE"] = current_script os.environ["LOG_FILE"] = current_script + ".log_" + str( log_number) for param in self.util_params: if not os.environ.get(param): logging.error( "No parameter {} is set in environment but it is requested by the db_component utility {} " .format(param, self.utility)) raise DbException( "No parameter {} is set in environment but it is requested by the db_component utility {} " .format(param, self.utility)) run_util = config.replace_string_params(util, self.util_params) logging.info("running script : {} on cloud : {} ".format( os.environ.get("SQL_FILE"), os.environ.get("DB_NAME"))) logging.debug( "running statement (password masked XXXX) : {}".format( run_util)) logpipe = LogPipe(logging.INFO) #errpipe = LogPipe(logging.ERROR) process = subprocess.Popen(run_util, shell=True, stdout=logpipe, stderr=logpipe) # wait for the process to terminate out, err = process.communicate() ret_code = process.returncode logpipe.close() if ret_code != 0: if not self.error_handling.ignored_error_list: out_file = config.read_file_content( os.environ.get("LOG_FILE")) raise DbException( "script failed : {} please see log {} ".format( os.environ.get("SQL_FILE"), os.environ.get("LOG_FILE"))) if "all" in self.error_handling.ignored_error_list: logging.warning( "Script {} failed with errors , skipping termination due to {} setting on 'ignored_err_list' property." .format(os.environ.get("SQL_FILE"), self.error_handling.ignored_error_list)) is_success = True break # Check for ignored errors and regular erros if os.path.exists(os.environ.get("LOG_FILE")): if self.search_for_errors( os.environ.get("LOG_FILE"), self.error_handling.error_list, self.error_handling.ignored_error_list): if not out_file: out_file = config.read_file_content( os.environ.get("LOG_FILE")) raise DbException( "script failed : {} please check log {} content : {}" .format(os.environ.get("SQL_FILE"), os.environ.get("LOG_FILE"), os.linesep + out_file)) else: is_success = True else: is_success = True out_file = config.read_file_content( os.environ.get("LOG_FILE")) logging.warning( "Retry [ {} ] script failed : {} please check log {}". format(log_number, os.environ.get("SQL_FILE"), os.environ.get("LOG_FILE"))) if not out_file and os.path.exists(os.environ["LOG_FILE"]): out_file = config.read_file_content( os.environ.get("LOG_FILE")) log_number = log_number + 1 else: if os.path.exists(os.environ.get("LOG_FILE")): if self.search_for_errors( os.environ.get("LOG_FILE"), self.error_handling.error_list, self.error_handling.ignored_error_list): if not out_file: out_file = config.read_file_content( os.environ.get("LOG_FILE")) raise DbException( "script failed : {} please check log {} content : {}" .format(os.environ.get("SQL_FILE"), os.environ.get("LOG_FILE"), os.linesep + out_file)) else: is_success = True else: is_success = True if log_number == int(self.error_handling.num_of_retries ) and os.path.exists( os.environ.get("LOG_FILE")): raise DbException( "script failed : {} please check log {} content : {}" .format(os.environ.get("SQL_FILE"), os.environ.get("LOG_FILE"), os.linesep + out_file)) else: if log_number == int( self.error_handling.num_of_retries): raise DbException("script failed : {} ".format( os.environ.get("SQL_FILE")))