def create_venv(self):
self.logger.info("{} - Create Python Virtual Environment".format(
self.blueprint_name_version_uuid),
extra=self.extra)
try:
bin_dir = self.blueprint_dir + "/bin"
# check if CREATE_VENV_DISABLE_SITE_PACKAGES is set
venv_system_site_packages_disabled = 'CREATE_VENV_DISABLE_SITE_PACKAGES' in os.environ
if (venv_system_site_packages_disabled):
self.logger.info(
"Note: CREATE_VENV_DISABLE_SITE_PACKAGES env var is set - environment creation will have site packages disabled.",
extra=self.extra)
venv.create(
self.blueprint_dir,
with_pip=True,
system_site_packages=not venv_system_site_packages_disabled)
self.logger.info(
"{} - Creation of Python Virtual Environment finished.".format(
self.blueprint_name_version_uuid),
extra=self.extra)
return utils.build_ret_data(True)
except Exception as err:
err_msg = "{} - Failed to provision Python Virtual Environment. Error: {}".format(
self.blueprint_name_version_uuid, err)
self.logger.info(err_msg, extra=self.extra)
return utils.build_ret_data(False, error=err_msg)
def create_venv(self):
self.logger.info("{} - Create Python Virtual Environment".format(
self.blueprint_name_version_uuid),
extra=self.extra)
try:
bin_dir = self.blueprint_dir + "/bin"
# venv doesn't populate the activate_this.py script, hence we use from virtualenv
venv.create(self.blueprint_dir,
with_pip=True,
system_site_packages=True)
virtualenv.writefile(os.path.join(bin_dir, "activate_this.py"),
virtualenv.ACTIVATE_THIS)
self.logger.info(
"{} - Creation of Python Virtual Environment finished.".format(
self.blueprint_name_version_uuid),
extra=self.extra)
return utils.build_ret_data(True)
except Exception as err:
err_msg = "{} - Failed to provision Python Virtual Environment. Error: {}".format(
self.blueprint_name_version_uuid, err)
self.logger.info(err_msg, extra=self.extra)
return utils.build_ret_data(False, error=err_msg)
def activate_venv(self):
self.logger.info("{} - Activate Python Virtual Environment".format(
self.blueprint_name_version_uuid),
extra=self.extra)
# Fix: The python generated activate_this.py script concatenates the env bin dir to PATH on every call
# eventually this process PATH variable was so big (128Kb) that no child process could be spawn
# This script will remove all duplicates; while keeping the order of the PATH folders
fixpathenvvar = "os.environ['PATH']=os.pathsep.join(list(dict.fromkeys(os.environ['PATH'].split(':'))))"
path = "%s/bin/activate_this.py" % self.blueprint_dir
try:
with open(path) as activate_this_script:
exec(activate_this_script.read(), {'__file__': path})
exec(fixpathenvvar)
self.logger.info("Running with PATH : {}".format(
os.environ['PATH']),
extra=self.extra)
return utils.build_ret_data(True)
except Exception as err:
err_msg = "{} - Failed to activate Python Virtual Environment. Error: {}".format(
self.blueprint_name_version_uuid, err)
self.logger.info(err_msg, extra=self.extra)
return utils.build_ret_data(False, error=err_msg)
def start_prometheus_server(self):
self.logger.info("PROMETHEUS_METRICS_ENABLED: {}".format(
os.environ.get('PROMETHEUS_METRICS_ENABLED')),
extra=self.extra)
if (os.environ.get('PROMETHEUS_METRICS_ENABLED')):
if not "PROMETHEUS_PORT" in os.environ:
err_msg = "ERROR: failed to start prometheus server, PROMETHEUS_PORT env variable is not found."
self.logger.error(err_msg, extra=self.extra)
return utils.build_ret_data(False,
results_log=[],
error=err_msg)
server_started = getattr(
prometheus.REGISTRY,
'_command_executor_prometheus_server_started', None)
if not server_started:
self.logger.info("PROMETHEUS_PORT: {}".format(
os.environ.get('PROMETHEUS_PORT')),
extra=self.extra)
prometheus.start_http_server(
int(os.environ.get('PROMETHEUS_PORT')))
prometheus.REGISTRY._command_executor_prometheus_server_started = True
def err_exit(self, msg):
self.logger.error(msg, extra=self.extra)
return utils.build_ret_data(False, error=msg)
def execute_command(self, request):
# STDOUT/STDERR output of the process
results_log = []
# encoded payload returned by the process
result = {}
# workaround for when packages are not specified, we may not want to go through the install step
# can just call create_venv from here.
if not self.is_installed():
self.create_venv()
try:
if not self.is_installed():
create_venv_status = self.create_venv
if not create_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]:
err_msg = "{} - Failed to execute command during venv creation. Original error: {}".format(
self.blueprint_name_version_uuid,
create_venv_status[utils.ERR_MSG_KEY])
return utils.build_ret_data(False, error=err_msg)
activate_response = self.activate_venv()
if not activate_response[utils.CDS_IS_SUCCESSFUL_KEY]:
orig_error = activate_response[utils.ERR_MSG_KEY]
err_msg = "{} - Failed to execute command during environment activation. Original error: {}".format(
self.blueprint_name_version_uuid, orig_error)
return utils.build_ret_data(False, error=err_msg)
# touch blueprint dir to indicate this CBA was used recently
os.utime(self.blueprint_dir)
cmd = "cd " + self.blueprint_dir
### if properties are defined we add them to the command
properties = ""
if request.properties is not None and len(request.properties) > 0:
properties = " " + re.escape(MessageToJson(request.properties))
### TODO: replace with os.environ['VIRTUAL_ENV']?
if "ansible-playbook" in request.command:
cmd = cmd + "; " + request.command + " -e 'ansible_python_interpreter=" + self.blueprint_dir + "/bin/python'"
else:
cmd = cmd + "; " + request.command + properties
### extract the original header request into sys-env variables
### OriginatorID
originator_id = request.originatorId
### CorrelationID
correlation_id = request.correlationId
request_id_map = {
'CDS_REQUEST_ID': self.request_id,
'CDS_SUBREQUEST_ID': self.sub_request_id,
'CDS_ORIGINATOR_ID': originator_id,
'CDS_CORRELATION_ID': correlation_id
}
updated_env = {**os.environ, **request_id_map}
self.logger.info("Running blueprint {} with timeout: {}".format(
self.blueprint_name_version_uuid, self.execution_timeout),
extra=self.extra)
with tempfile.TemporaryFile(mode="w+") as tmp:
try:
completed_subprocess = subprocess.run(
cmd,
stdout=tmp,
stderr=subprocess.STDOUT,
shell=True,
env=updated_env,
timeout=self.execution_timeout)
except TimeoutExpired:
timeout_err_msg = "Running command {} failed due to timeout of {} seconds.".format(
self.blueprint_name_version_uuid,
self.execution_timeout)
self.logger.error(timeout_err_msg, extra=self.extra)
utils.parse_cmd_exec_output(tmp, self.logger, result,
results_log, self.extra)
return utils.build_ret_data(False,
results_log=results_log,
error=timeout_err_msg)
utils.parse_cmd_exec_output(tmp, self.logger, result,
results_log, self.extra)
rc = completed_subprocess.returncode
except Exception as e:
err_msg = "{} - Failed to execute command. Error: {}".format(
self.blueprint_name_version_uuid, e)
result.update(
utils.build_ret_data(False,
results_log=results_log,
error=err_msg))
return result
# deactivate_venv(blueprint_id)
#Since return code is only used to check if it's zero (success), we can just return success flag instead.
self.logger.debug("python return_code : {}".format(rc),
extra=self.extra)
is_execution_successful = rc == 0
result.update(
utils.build_ret_data(is_execution_successful,
results_log=results_log))
return result
def prepare_env(self, request):
results_log = []
# validate that the blueprint name in the request exists, if not, notify the caller
if not self.blueprint_dir_exists():
err_msg = "CBA directory {} not found on cmd-exec. CBA will be uploaded by BP proc.".format(
self.blueprint_name_version_uuid)
self.logger.info(err_msg, extra=self.extra)
return utils.build_ret_data(False,
results_log=results_log,
error=err_msg,
reupload_cba=True)
if not self.blueprint_tosca_meta_file_exists():
err_msg = "CBA directory {} exists on cmd-exec, but TOSCA meta file is not found!!! Returning (null) as UUID. CBA will be uploaded by BP proc.".format(
self.blueprint_name_version_uuid)
self.logger.info(err_msg, extra=self.extra)
return utils.build_ret_data(False,
results_log=results_log,
error=err_msg,
reupload_cba=True)
self.logger.info("CBA directory {} exists on cmd-exec.".format(
self.blueprint_name_version_uuid),
extra=self.extra)
if not self.is_installed():
create_venv_status = self.create_venv()
if not create_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]:
return self.err_exit(
"ERROR: failed to prepare environment for request {} due to error in creating virtual Python env. Original error {}"
.format(self.blueprint_name_version_uuid,
create_venv_status[utils.ERR_MSG_KEY]))
activate_venv_status = self.activate_venv()
if not activate_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]:
return self.err_exit(
"ERROR: failed to prepare environment for request {} due Python venv_activation. Original error {}"
.format(self.blueprint_name_version_uuid,
activate_venv_status[utils.ERR_MSG_KEY]))
try:
with open(self.installed, "w+") as f:
if not self.install_packages(
request, CommandExecutor_pb2.pip, f, results_log):
err_msg = "ERROR: failed to prepare environment for request {} during pip package install.".format(
self.blueprint_name_version_uuid)
return utils.build_ret_data(False,
results_log=results_log,
error=err_msg)
f.write("\r\n") # TODO: is \r needed?
results_log.append("\n")
if not self.install_packages(
request, CommandExecutor_pb2.ansible_galaxy, f,
results_log):
err_msg = "ERROR: failed to prepare environment for request {} during Ansible install.".format(
self.blueprint_name_version_uuid)
return utils.build_ret_data(False,
results_log=results_log,
error=err_msg)
except Exception as ex:
err_msg = "ERROR: failed to prepare environment for request {} during installing packages. Exception: {}".format(
self.blueprint_name_version_uuid, ex)
self.logger.error(err_msg, extra=self.extra)
return utils.build_ret_data(False, error=err_msg)
else:
try:
with open(self.installed, "r") as f:
results_log.append(f.read())
except Exception as ex:
err_msg = "ERROR: failed to prepare environment during reading 'installed' file {}. Exception: {}".format(
self.installed, ex)
return utils.build_ret_data(False, error=err_msg)
# deactivate_venv(blueprint_id)
return utils.build_ret_data(True, results_log=results_log)
def execute_command(self, request):
start_time = time.time()
# STDOUT/STDERR output of the process
results_log = []
# encoded payload returned by the process
result = {}
script_err_msg = []
self.logger.info("execute_command request {}".format(request),
extra=self.extra)
# workaround for when packages are not specified, we may not want to go through the install step
# can just call create_venv from here.
if not self.is_installed():
self.create_venv()
try:
if not self.is_installed():
create_venv_status = self.create_venv
if not create_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]:
self.prometheus_counter.labels(
self.PROMETHEUS_METRICS_EXEC_COMMAND_LABEL,
self.blueprint_name, self.blueprint_version,
request.command).inc()
err_msg = "{} - Failed to execute command during venv creation. Original error: {}".format(
self.blueprint_name_version_uuid,
create_venv_status[utils.ERR_MSG_KEY])
return utils.build_ret_data(False, error=err_msg)
# touch blueprint dir to indicate this CBA was used recently
os.utime(self.blueprint_dir)
cmd = "cd " + self.blueprint_dir
### if properties are defined we add them to the command
properties = ""
if request.properties is not None and len(request.properties) > 0:
properties = " " + re.escape(MessageToJson(
request.properties)).replace('"', '\\"')
# SR7/SR10 compatibility hack
# check if the path for the request.command does not contain UUID, then add it after cba_name/cba_version path.
updated_request_command = request.command
if self.blueprint_name_version in updated_request_command and self.blueprint_name_version_uuid not in updated_request_command:
updated_request_command = updated_request_command.replace(
self.blueprint_name_version,
self.blueprint_name_version_uuid)
if "ansible-playbook" in updated_request_command:
cmd = cmd + "; " + updated_request_command + " -e 'ansible_python_interpreter=" + self.blueprint_dir + "/bin/python'"
else:
cmd = cmd + "; " + updated_request_command + properties
### extract the original header request into sys-env variables
# OriginatorID
originator_id = request.originatorId
# CorrelationID
correlation_id = request.correlationId
request_id_map = {
'CDS_REQUEST_ID': self.request_id,
'CDS_SUBREQUEST_ID': self.sub_request_id,
'CDS_ORIGINATOR_ID': originator_id,
'CDS_CORRELATION_ID': correlation_id
}
updated_env = {**os.environ, **request_id_map}
# Prepare PATH and VENV_HOME
updated_env[
'PATH'] = self.blueprint_dir + "/bin/:" + os.environ['PATH']
updated_env['VIRTUAL_ENV'] = self.blueprint_dir
self.logger.info("Running blueprint {} with timeout: {}".format(
self.blueprint_name_version_uuid, self.execution_timeout),
extra=self.extra)
with tempfile.TemporaryFile(mode="w+") as tmp:
try:
completed_subprocess = subprocess.run(
cmd,
stdout=tmp,
stderr=subprocess.STDOUT,
shell=True,
env=updated_env,
timeout=self.execution_timeout)
except TimeoutExpired as timeout_ex:
self.prometheus_counter.labels(
self.PROMETHEUS_METRICS_EXEC_COMMAND_LABEL,
self.blueprint_name, self.blueprint_version,
request.command).inc()
timeout_err_msg = "Running command {} failed due to timeout of {} seconds.".format(
self.blueprint_name_version_uuid,
self.execution_timeout)
self.logger.error(timeout_err_msg, extra=self.extra)
# In the time-out case, we will never get CBA's script err msg string.
utils.parse_cmd_exec_output(outputfile=tmp,
logger=self.logger,
payload_result=result,
err_msg_result=script_err_msg,
results_log=results_log,
extra=self.extra)
return utils.build_ret_data(False,
results_log=results_log,
error=timeout_err_msg)
utils.parse_cmd_exec_output(outputfile=tmp,
logger=self.logger,
payload_result=result,
err_msg_result=script_err_msg,
results_log=results_log,
extra=self.extra)
rc = completed_subprocess.returncode
except Exception as e:
self.prometheus_counter.labels(
self.PROMETHEUS_METRICS_EXEC_COMMAND_LABEL,
self.blueprint_name, self.blueprint_version,
request.command).inc()
err_msg = "{} - Failed to execute command. Error: {}".format(
self.blueprint_name_version_uuid, e)
result.update(
utils.build_ret_data(False,
results_log=results_log,
error=err_msg))
return result
# Since return code is only used to check if it's zero (success), we can just return success flag instead.
is_execution_successful = rc == 0
# Propagate error message in case rc is not 0
ret_err_msg = None if is_execution_successful or not script_err_msg else script_err_msg
result.update(
utils.build_ret_data(is_execution_successful,
results_log=results_log,
error=ret_err_msg))
self.prometheus_histogram.labels(
self.PROMETHEUS_METRICS_EXEC_COMMAND_LABEL, self.blueprint_name,
self.blueprint_version,
request.command).observe(time.time() - start_time)
return result
def prepare_env(self, request):
results_log = []
start_time = time.time()
self.logger.info("prepare_env request {}".format(request),
extra=self.extra)
# validate that the blueprint name in the request exists, if not, notify the caller
if not self.blueprint_dir_exists():
self.prometheus_counter.labels(
self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name,
self.blueprint_version, None).inc()
err_msg = "CBA directory {} not found on cmd-exec. CBA will be uploaded by BP proc.".format(
self.blueprint_name_version_uuid)
self.logger.info(err_msg, extra=self.extra)
return utils.build_ret_data(False,
results_log=results_log,
error=err_msg,
reupload_cba=True)
if not self.blueprint_tosca_meta_file_exists():
self.prometheus_counter.labels(
self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name,
self.blueprint_version, None).inc()
err_msg = "CBA directory {} exists on cmd-exec, but TOSCA meta file is not found!!! Returning (null) as UUID. CBA will be uploaded by BP proc.".format(
self.blueprint_name_version_uuid)
self.logger.info(err_msg, extra=self.extra)
return utils.build_ret_data(False,
results_log=results_log,
error=err_msg,
reupload_cba=True)
self.logger.info("CBA directory {} exists on cmd-exec.".format(
self.blueprint_name_version_uuid),
extra=self.extra)
if not self.is_installed():
create_venv_status = self.create_venv()
if not create_venv_status[utils.CDS_IS_SUCCESSFUL_KEY]:
self.prometheus_counter.labels(
self.PROMETHEUS_METRICS_PREP_ENV_LABEL,
self.blueprint_name, self.blueprint_version, None).inc()
return self.err_exit(
"ERROR: failed to prepare environment for request {} due to error in creating virtual Python env. Original error {}"
.format(self.blueprint_name_version_uuid,
create_venv_status[utils.ERR_MSG_KEY]))
# Upgrade pip - venv comes with PIP 18.1, which is too old.
if not self.upgrade_pip(results_log):
self.prometheus_counter.labels(
self.PROMETHEUS_METRICS_PREP_ENV_LABEL,
self.blueprint_name, self.blueprint_version, None).inc()
err_msg = "ERROR: failed to prepare environment for request {} due to error in upgrading pip.".format(
self.blueprint_name_version_uuid)
return utils.build_ret_data(False,
results_log=results_log,
error=err_msg)
try:
# NOTE: pip or ansible selection is done later inside 'install_packages'
with open(self.installed, "w+") as f:
if not self.install_packages(
request, CommandExecutor_pb2.pip, f, results_log):
self.prometheus_counter.labels(
self.PROMETHEUS_METRICS_PREP_ENV_LABEL,
self.blueprint_name, self.blueprint_version,
None).inc()
err_msg = "ERROR: failed to prepare environment for request {} during pip package install.".format(
self.blueprint_name_version_uuid)
return utils.build_ret_data(False,
results_log=results_log,
error=err_msg)
f.write("\r\n") # TODO: is \r needed?
results_log.append("\n")
if not self.install_packages(
request, CommandExecutor_pb2.ansible_galaxy, f,
results_log):
self.prometheus_counter.labels(
self.PROMETHEUS_METRICS_PREP_ENV_LABEL,
self.blueprint_name, self.blueprint_version,
None).inc()
err_msg = "ERROR: failed to prepare environment for request {} during Ansible install.".format(
self.blueprint_name_version_uuid)
return utils.build_ret_data(False,
results_log=results_log,
error=err_msg)
except Exception as ex:
self.prometheus_counter.labels(
self.PROMETHEUS_METRICS_PREP_ENV_LABEL,
self.blueprint_name, self.blueprint_version, None).inc()
err_msg = "ERROR: failed to prepare environment for request {} during installing packages. Exception: {}".format(
self.blueprint_name_version_uuid, ex)
self.logger.error(err_msg, extra=self.extra)
return utils.build_ret_data(False, error=err_msg)
else:
try:
self.logger.info(
".installed file was found for request {}".format(
self.blueprint_name_version_uuid),
extra=self.extra)
with open(self.installed, "r") as f:
results_log.append(f.read())
except Exception as ex:
self.prometheus_counter.labels(
self.PROMETHEUS_METRICS_PREP_ENV_LABEL,
self.blueprint_name, self.blueprint_version, None).inc()
err_msg = "ERROR: failed to prepare environment during reading 'installed' file {}. Exception: {}".format(
self.installed, ex)
return utils.build_ret_data(False, error=err_msg)
# deactivate_venv(blueprint_id)
self.prometheus_histogram.labels(
self.PROMETHEUS_METRICS_PREP_ENV_LABEL, self.blueprint_name,
self.blueprint_version, None).observe(time.time() - start_time)
return utils.build_ret_data(True, results_log=results_log)
