def create_venv(self): self.logger.info("{} - Create Python Virtual Environment".format( self.blueprint_name_version_uuid), extra=self.extra) try: bin_dir = self.blueprint_dir + "/bin" # check if CREATE_VENV_DISABLE_SITE_PACKAGES is set venv_system_site_packages_disabled = 'CREATE_VENV_DISABLE_SITE_PACKAGES' in os.environ if (venv_system_site_packages_disabled): self.logger.info( "Note: CREATE_VENV_DISABLE_SITE_PACKAGES env var is set - environment creation will have site packages disabled.", extra=self.extra) venv.create( self.blueprint_dir, with_pip=True, system_site_packages=not venv_system_site_packages_disabled) self.logger.info( "{} - Creation of Python Virtual Environment finished.".format( self.blueprint_name_version_uuid), extra=self.extra) return utils.build_ret_data(True) except Exception as err: err_msg = "{} - Failed to provision Python Virtual Environment. Error: {}".format( self.blueprint_name_version_uuid, err) self.logger.info(err_msg, extra=self.extra) return utils.build_ret_data(False, error=err_msg)
def create_venv(self): self.logger.info("{} - Create Python Virtual Environment".format( self.blueprint_name_version_uuid), extra=self.extra) try: bin_dir = self.blueprint_dir + "/bin" # venv doesn't populate the activate_this.py script, hence we use from virtualenv venv.create(self.blueprint_dir, with_pip=True, system_site_packages=True) virtualenv.writefile(os.path.join(bin_dir, "activate_this.py"), virtualenv.ACTIVATE_THIS) self.logger.info( "{} - Creation of Python Virtual Environment finished.".format( self.blueprint_name_version_uuid), extra=self.extra) return utils.build_ret_data(True) except Exception as err: err_msg = "{} - Failed to provision Python Virtual Environment. Error: {}".format( self.blueprint_name_version_uuid, err) self.logger.info(err_msg, extra=self.extra) return utils.build_ret_data(False, error=err_msg)
def activate_venv(self): self.logger.info("{} - Activate Python Virtual Environment".format( self.blueprint_name_version_uuid), extra=self.extra) # Fix: The python generated activate_this.py script concatenates the env bin dir to PATH on every call # eventually this process PATH variable was so big (128Kb) that no child process could be spawn # This script will remove all duplicates; while keeping the order of the PATH folders fixpathenvvar = "os.environ['PATH']=os.pathsep.join(list(dict.fromkeys(os.environ['PATH'].split(':'))))" path = "%s/bin/activate_this.py" % self.blueprint_dir try: with open(path) as activate_this_script: exec(activate_this_script.read(), {'__file__': path}) exec(fixpathenvvar) self.logger.info("Running with PATH : {}".format( os.environ['PATH']), extra=self.extra) return utils.build_ret_data(True) except Exception as err: err_msg = "{} - Failed to activate Python Virtual Environment. Error: {}".format( self.blueprint_name_version_uuid, err) self.logger.info(err_msg, extra=self.extra) return utils.build_ret_data(False, error=err_msg)
def start_prometheus_server(self): self.logger.info("PROMETHEUS_METRICS_ENABLED: {}".format( os.environ.get('PROMETHEUS_METRICS_ENABLED')), extra=self.extra) if (os.environ.get('PROMETHEUS_METRICS_ENABLED')): if not "PROMETHEUS_PORT" in os.environ: err_msg = "ERROR: failed to start prometheus server, PROMETHEUS_PORT env variable is not found." self.logger.error(err_msg, extra=self.extra) return utils.build_ret_data(False, results_log=[], error=err_msg) server_started = getattr( prometheus.REGISTRY, '_command_executor_prometheus_server_started', None) if not server_started: self.logger.info("PROMETHEUS_PORT: {}".format( os.environ.get('PROMETHEUS_PORT')), extra=self.extra) prometheus.start_http_server( int(os.environ.get('PROMETHEUS_PORT'))) prometheus.REGISTRY._command_executor_prometheus_server_started = True
def err_exit(self, msg): self.logger.error(msg, extra=self.extra) return utils.build_ret_data(False, error=msg)
def execute_command(self, request): # STDOUT/STDERR output of the process results_log = [] # encoded payload returned by the process result = {} # workaround for when packages are not specified, we may not want to go through the install step # can just call create_venv from here. if not self.is_installed(): self.create_venv() try: if not self.is_installed(): create_venv_status = self.create_venv if not create_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]: err_msg = "{} - Failed to execute command during venv creation. Original error: {}".format( self.blueprint_name_version_uuid, create_venv_status[utils.ERR_MSG_KEY]) return utils.build_ret_data(False, error=err_msg) activate_response = self.activate_venv() if not activate_response[utils.CDS_IS_SUCCESSFUL_KEY]: orig_error = activate_response[utils.ERR_MSG_KEY] err_msg = "{} - Failed to execute command during environment activation. Original error: {}".format( self.blueprint_name_version_uuid, orig_error) return utils.build_ret_data(False, error=err_msg) # touch blueprint dir to indicate this CBA was used recently os.utime(self.blueprint_dir) cmd = "cd " + self.blueprint_dir ### if properties are defined we add them to the command properties = "" if request.properties is not None and len(request.properties) > 0: properties = " " + re.escape(MessageToJson(request.properties)) ### TODO: replace with os.environ['VIRTUAL_ENV']? if "ansible-playbook" in request.command: cmd = cmd + "; " + request.command + " -e 'ansible_python_interpreter=" + self.blueprint_dir + "/bin/python'" else: cmd = cmd + "; " + request.command + properties ### extract the original header request into sys-env variables ### OriginatorID originator_id = request.originatorId ### CorrelationID correlation_id = request.correlationId request_id_map = { 'CDS_REQUEST_ID': self.request_id, 'CDS_SUBREQUEST_ID': self.sub_request_id, 'CDS_ORIGINATOR_ID': originator_id, 'CDS_CORRELATION_ID': correlation_id } updated_env = {**os.environ, **request_id_map} self.logger.info("Running blueprint {} with timeout: {}".format( self.blueprint_name_version_uuid, self.execution_timeout), extra=self.extra) with tempfile.TemporaryFile(mode="w+") as tmp: try: completed_subprocess = subprocess.run( cmd, stdout=tmp, stderr=subprocess.STDOUT, shell=True, env=updated_env, timeout=self.execution_timeout) except TimeoutExpired: timeout_err_msg = "Running command {} failed due to timeout of {} seconds.".format( self.blueprint_name_version_uuid, self.execution_timeout) self.logger.error(timeout_err_msg, extra=self.extra) utils.parse_cmd_exec_output(tmp, self.logger, result, results_log, self.extra) return utils.build_ret_data(False, results_log=results_log, error=timeout_err_msg) utils.parse_cmd_exec_output(tmp, self.logger, result, results_log, self.extra) rc = completed_subprocess.returncode except Exception as e: err_msg = "{} - Failed to execute command. Error: {}".format( self.blueprint_name_version_uuid, e) result.update( utils.build_ret_data(False, results_log=results_log, error=err_msg)) return result # deactivate_venv(blueprint_id) #Since return code is only used to check if it's zero (success), we can just return success flag instead. self.logger.debug("python return_code : {}".format(rc), extra=self.extra) is_execution_successful = rc == 0 result.update( utils.build_ret_data(is_execution_successful, results_log=results_log)) return result
def prepare_env(self, request): results_log = [] # validate that the blueprint name in the request exists, if not, notify the caller if not self.blueprint_dir_exists(): err_msg = "CBA directory {} not found on cmd-exec. CBA will be uploaded by BP proc.".format( self.blueprint_name_version_uuid) self.logger.info(err_msg, extra=self.extra) return utils.build_ret_data(False, results_log=results_log, error=err_msg, reupload_cba=True) if not self.blueprint_tosca_meta_file_exists(): err_msg = "CBA directory {} exists on cmd-exec, but TOSCA meta file is not found!!! Returning (null) as UUID. CBA will be uploaded by BP proc.".format( self.blueprint_name_version_uuid) self.logger.info(err_msg, extra=self.extra) return utils.build_ret_data(False, results_log=results_log, error=err_msg, reupload_cba=True) self.logger.info("CBA directory {} exists on cmd-exec.".format( self.blueprint_name_version_uuid), extra=self.extra) if not self.is_installed(): create_venv_status = self.create_venv() if not create_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]: return self.err_exit( "ERROR: failed to prepare environment for request {} due to error in creating virtual Python env. Original error {}" .format(self.blueprint_name_version_uuid, create_venv_status[utils.ERR_MSG_KEY])) activate_venv_status = self.activate_venv() if not activate_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]: return self.err_exit( "ERROR: failed to prepare environment for request {} due Python venv_activation. Original error {}" .format(self.blueprint_name_version_uuid, activate_venv_status[utils.ERR_MSG_KEY])) try: with open(self.installed, "w+") as f: if not self.install_packages( request, CommandExecutor_pb2.pip, f, results_log): err_msg = "ERROR: failed to prepare environment for request {} during pip package install.".format( self.blueprint_name_version_uuid) return utils.build_ret_data(False, results_log=results_log, error=err_msg) f.write("\r\n") # TODO: is \r needed? results_log.append("\n") if not self.install_packages( request, CommandExecutor_pb2.ansible_galaxy, f, results_log): err_msg = "ERROR: failed to prepare environment for request {} during Ansible install.".format( self.blueprint_name_version_uuid) return utils.build_ret_data(False, results_log=results_log, error=err_msg) except Exception as ex: err_msg = "ERROR: failed to prepare environment for request {} during installing packages. Exception: {}".format( self.blueprint_name_version_uuid, ex) self.logger.error(err_msg, extra=self.extra) return utils.build_ret_data(False, error=err_msg) else: try: with open(self.installed, "r") as f: results_log.append(f.read()) except Exception as ex: err_msg = "ERROR: failed to prepare environment during reading 'installed' file {}. Exception: {}".format( self.installed, ex) return utils.build_ret_data(False, error=err_msg) # deactivate_venv(blueprint_id) return utils.build_ret_data(True, results_log=results_log)
def execute_command(self, request): start_time = time.time() # STDOUT/STDERR output of the process results_log = [] # encoded payload returned by the process result = {} script_err_msg = [] self.logger.info("execute_command request {}".format(request), extra=self.extra) # workaround for when packages are not specified, we may not want to go through the install step # can just call create_venv from here. if not self.is_installed(): self.create_venv() try: if not self.is_installed(): create_venv_status = self.create_venv if not create_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]: self.prometheus_counter.labels( self.PROMETHEUS_METRICS_EXEC_COMMAND_LABEL, self.blueprint_name, self.blueprint_version, request.command).inc() err_msg = "{} - Failed to execute command during venv creation. Original error: {}".format( self.blueprint_name_version_uuid, create_venv_status[utils.ERR_MSG_KEY]) return utils.build_ret_data(False, error=err_msg) # touch blueprint dir to indicate this CBA was used recently os.utime(self.blueprint_dir) cmd = "cd " + self.blueprint_dir ### if properties are defined we add them to the command properties = "" if request.properties is not None and len(request.properties) > 0: properties = " " + re.escape(MessageToJson( request.properties)).replace('"', '\\"') # SR7/SR10 compatibility hack # check if the path for the request.command does not contain UUID, then add it after cba_name/cba_version path. updated_request_command = request.command if self.blueprint_name_version in updated_request_command and self.blueprint_name_version_uuid not in updated_request_command: updated_request_command = updated_request_command.replace( self.blueprint_name_version, self.blueprint_name_version_uuid) if "ansible-playbook" in updated_request_command: cmd = cmd + "; " + updated_request_command + " -e 'ansible_python_interpreter=" + self.blueprint_dir + "/bin/python'" else: cmd = cmd + "; " + updated_request_command + properties ### extract the original header request into sys-env variables # OriginatorID originator_id = request.originatorId # CorrelationID correlation_id = request.correlationId request_id_map = { 'CDS_REQUEST_ID': self.request_id, 'CDS_SUBREQUEST_ID': self.sub_request_id, 'CDS_ORIGINATOR_ID': originator_id, 'CDS_CORRELATION_ID': correlation_id } updated_env = {**os.environ, **request_id_map} # Prepare PATH and VENV_HOME updated_env[ 'PATH'] = self.blueprint_dir + "/bin/:" + os.environ['PATH'] updated_env['VIRTUAL_ENV'] = self.blueprint_dir self.logger.info("Running blueprint {} with timeout: {}".format( self.blueprint_name_version_uuid, self.execution_timeout), extra=self.extra) with tempfile.TemporaryFile(mode="w+") as tmp: try: completed_subprocess = subprocess.run( cmd, stdout=tmp, stderr=subprocess.STDOUT, shell=True, env=updated_env, timeout=self.execution_timeout) except TimeoutExpired as timeout_ex: self.prometheus_counter.labels( self.PROMETHEUS_METRICS_EXEC_COMMAND_LABEL, self.blueprint_name, self.blueprint_version, request.command).inc() timeout_err_msg = "Running command {} failed due to timeout of {} seconds.".format( self.blueprint_name_version_uuid, self.execution_timeout) self.logger.error(timeout_err_msg, extra=self.extra) # In the time-out case, we will never get CBA's script err msg string. utils.parse_cmd_exec_output(outputfile=tmp, logger=self.logger, payload_result=result, err_msg_result=script_err_msg, results_log=results_log, extra=self.extra) return utils.build_ret_data(False, results_log=results_log, error=timeout_err_msg) utils.parse_cmd_exec_output(outputfile=tmp, logger=self.logger, payload_result=result, err_msg_result=script_err_msg, results_log=results_log, extra=self.extra) rc = completed_subprocess.returncode except Exception as e: self.prometheus_counter.labels( self.PROMETHEUS_METRICS_EXEC_COMMAND_LABEL, self.blueprint_name, self.blueprint_version, request.command).inc() err_msg = "{} - Failed to execute command. Error: {}".format( self.blueprint_name_version_uuid, e) result.update( utils.build_ret_data(False, results_log=results_log, error=err_msg)) return result # Since return code is only used to check if it's zero (success), we can just return success flag instead. is_execution_successful = rc == 0 # Propagate error message in case rc is not 0 ret_err_msg = None if is_execution_successful or not script_err_msg else script_err_msg result.update( utils.build_ret_data(is_execution_successful, results_log=results_log, error=ret_err_msg)) self.prometheus_histogram.labels( self.PROMETHEUS_METRICS_EXEC_COMMAND_LABEL, self.blueprint_name, self.blueprint_version, request.command).observe(time.time() - start_time) return result
def prepare_env(self, request): results_log = [] start_time = time.time() self.logger.info("prepare_env request {}".format(request), extra=self.extra) # validate that the blueprint name in the request exists, if not, notify the caller if not self.blueprint_dir_exists(): self.prometheus_counter.labels( self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc() err_msg = "CBA directory {} not found on cmd-exec. CBA will be uploaded by BP proc.".format( self.blueprint_name_version_uuid) self.logger.info(err_msg, extra=self.extra) return utils.build_ret_data(False, results_log=results_log, error=err_msg, reupload_cba=True) if not self.blueprint_tosca_meta_file_exists(): self.prometheus_counter.labels( self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc() err_msg = "CBA directory {} exists on cmd-exec, but TOSCA meta file is not found!!! Returning (null) as UUID. CBA will be uploaded by BP proc.".format( self.blueprint_name_version_uuid) self.logger.info(err_msg, extra=self.extra) return utils.build_ret_data(False, results_log=results_log, error=err_msg, reupload_cba=True) self.logger.info("CBA directory {} exists on cmd-exec.".format( self.blueprint_name_version_uuid), extra=self.extra) if not self.is_installed(): create_venv_status = self.create_venv() if not create_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]: self.prometheus_counter.labels( self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc() return self.err_exit( "ERROR: failed to prepare environment for request {} due to error in creating virtual Python env. Original error {}" .format(self.blueprint_name_version_uuid, create_venv_status[utils.ERR_MSG_KEY])) # Upgrade pip - venv comes with PIP 18.1, which is too old. if not self.upgrade_pip(results_log): self.prometheus_counter.labels( self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc() err_msg = "ERROR: failed to prepare environment for request {} due to error in upgrading pip.".format( self.blueprint_name_version_uuid) return utils.build_ret_data(False, results_log=results_log, error=err_msg) try: # NOTE: pip or ansible selection is done later inside 'install_packages' with open(self.installed, "w+") as f: if not self.install_packages( request, CommandExecutor_pb2.pip, f, results_log): self.prometheus_counter.labels( self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc() err_msg = "ERROR: failed to prepare environment for request {} during pip package install.".format( self.blueprint_name_version_uuid) return utils.build_ret_data(False, results_log=results_log, error=err_msg) f.write("\r\n") # TODO: is \r needed? results_log.append("\n") if not self.install_packages( request, CommandExecutor_pb2.ansible_galaxy, f, results_log): self.prometheus_counter.labels( self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc() err_msg = "ERROR: failed to prepare environment for request {} during Ansible install.".format( self.blueprint_name_version_uuid) return utils.build_ret_data(False, results_log=results_log, error=err_msg) except Exception as ex: self.prometheus_counter.labels( self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc() err_msg = "ERROR: failed to prepare environment for request {} during installing packages. Exception: {}".format( self.blueprint_name_version_uuid, ex) self.logger.error(err_msg, extra=self.extra) return utils.build_ret_data(False, error=err_msg) else: try: self.logger.info( ".installed file was found for request {}".format( self.blueprint_name_version_uuid), extra=self.extra) with open(self.installed, "r") as f: results_log.append(f.read()) except Exception as ex: self.prometheus_counter.labels( self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).inc() err_msg = "ERROR: failed to prepare environment during reading 'installed' file {}. Exception: {}".format( self.installed, ex) return utils.build_ret_data(False, error=err_msg) # deactivate_venv(blueprint_id) self.prometheus_histogram.labels( self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name, self.blueprint_version, None).observe(time.time() - start_time) return utils.build_ret_data(True, results_log=results_log)