def verify_signature(signed_file_path, output_file_path):
"""Verifies the signed file's signature.
Returns:
True : If the signature is valid.
False : If the signature is invalid.
"""
cmd = ["gpg", "-d"]
keyring_path = configuration.get_gpg_public_keyring_path()
# if a keyring is specified in the conf, used it, else use default one
if keyring_path != "":
cmd += [GPG_NO_DEFAULT_KEYRING_OPTION, GPG_KEYRING_ARG, keyring_path]
cmd += ["--output", output_file_path, signed_file_path]
# temporary workaround for the omi/gpg bug causing gpg to create a .gpg folder in the wrong home dir
# only apply the workaround for oms installation
env = None
if "nxOMSAutomationWorkerResource" in os.path.abspath(__file__):
env = os.environ.copy()
env["HOME"] = "/var/opt/microsoft/omsagent/run"
proc = subprocessfactory.create_subprocess(cmd=cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = proc.communicate()
if proc.poll() == 0:
tracer.log_debug_trace("Signature is valid.")
return True
tracer.log_sandbox_job_runbook_signature_validation_failed(stderr)
return False
def unload_job(self):
"""Unloads the job."""
self.jrds_client.unload_job(self.job_data["subscriptionId"],
self.sandbox_id, self.job_id,
self.job_updatable_data["isDraft"],
datetime.now(), 2)
tracer.log_debug_trace("Unloading job.")
def routine(self):
self.stop_tracking_terminated_sandbox()
sandbox_actions = self.jrds_client.get_sandbox_actions()
tracer.log_debug_trace("Get sandbox action. Found " + str(len(sandbox_actions)) + " action(s).")
for action in sandbox_actions:
tracer.log_worker_sandbox_action_found(len(sandbox_actions))
sandbox_id = str(action["SandboxId"])
# prevent duplicate sandbox from running
if sandbox_id in self.running_sandboxes:
return
# create sandboxes folder if needed
sandboxes_base_path = "sandboxes"
sandbox_working_dir = os.path.join(configuration.get_working_directory_path(), sandboxes_base_path,
sandbox_id)
try:
iohelper.assert_or_create_path(sandbox_working_dir)
except OSError:
tracer.log_debug_trace("Failed to create sandbox folder.")
pass
cmd = ["python", os.path.join(configuration.get_source_directory_path(), "sandbox.py")]
process_env_variables = {"sandbox_id": sandbox_id}
sandbox_process = subprocessfactory.create_subprocess(cmd=cmd,
env=process_env_variables,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=sandbox_working_dir)
self.running_sandboxes[sandbox_id] = sandbox_process
self.monitor_sandbox_process_outputs(sandbox_id, sandbox_process)
tracer.log_worker_starting_sandbox(sandbox_id, str(sandbox_process.pid))
def generate_state_file():
state_file_name = "state.conf"
if configuration.get_state_directory_path() == configuration.DEFAULT_STATE_DIRECTORY_PATH:
state_file_path = os.path.join(configuration.get_working_directory_path(), state_file_name)
else:
state_file_path = os.path.join(configuration.get_state_directory_path(), state_file_name)
tracer.log_debug_trace("State file path : " + str(state_file_path))
if os.path.isfile(state_file_path):
os.remove(state_file_path)
section = "state"
conf_file = open(state_file_path, 'wb')
config = ConfigParser.ConfigParser()
config.add_section(section)
config.set(section, configuration.STATE_PID, str(os.getpid()))
config.set(section, configuration.WORKER_VERSION, str(configuration.get_worker_version()))
# for OMS scenarios, optional for DIY
if len(sys.argv) >= 3:
config.set(section, configuration.STATE_WORKSPACE_ID, str(sys.argv[2]))
if len(sys.argv) >= 4:
config.set(section, configuration.STATE_RESOURCE_VERSION, str(sys.argv[3]))
config.write(conf_file)
conf_file.close()
def verify_signature(signed_file_path, output_file_path):
"""Verifies the signed file's signature.
Returns:
True : If the signature is valid.
False : If the signature is invalid.
"""
cmd = GPG_DECRYPT_BASE_CMD
keyring_path = configuration.get_gpg_public_keyring_path()
# if a keyring is specified in the conf, used it, else use default one
if keyring_path != "":
cmd += [GPG_NO_DEFAULT_KEYRING_OPTION, GPG_KEYRING_ARG, keyring_path]
cmd += ["--output", output_file_path, signed_file_path]
proc = subprocessfactory.create_subprocess(cmd=cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = proc.communicate()
if proc.poll() == 0:
tracer.log_debug_trace("Signature is valid.")
return True
tracer.log_debug_trace("Signature is invalid.[exception=" + str(stderr) +
"]")
return False
def start_runbook_subprocess(self):
"""Creates the runbook subprocess based on the script language and using properties set by the derived class.
Requires self.base_cmd & self.runbook_file_path to be set by derived class.
"""
cmd = self.base_cmd + [self.runbook.runbook_file_path]
job_parameters = self.job_data.parameters
if job_parameters is not None and len(job_parameters) > 0:
for parameter in job_parameters:
tracer.log_debug_trace("Parameter is: \n" + str(parameter))
if self.runbook.definition_kind_str == "PowerShell" and parameter["Name"]:
# Handle named parameters for PowerShell arriving out of order
cmd += ["-%s" % parameter["Name"]]
cmd += [str(json.loads(parameter["Value"]))]
# Do not copy current process env var to the sandbox process
env = os.environ.copy()
env.update({"AUTOMATION_JOB_ID": str(self.job_data.job_id),
"AUTOMATION_ACTIVITY_ID": str(tracer.u_activity_id),
"PYTHONPATH": str(
configuration.get_source_directory_path())}) # windows env have to be str (not unicode)
self.runbook_subprocess = subprocessfactory.create_subprocess(cmd=cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
def start_runbook_subprocess(self):
"""Creates the runbook subprocess based on the script language and using properties set by the derived class.
Requires self.base_cmd & self.runbook_file_path to be set by derived class.
"""
cmd = self.base_cmd + [self.runbook.runbook_file_path]
job_parameters = self.job_data.parameters
if job_parameters is not None and len(job_parameters) > 0:
for parameter in job_parameters:
tracer.log_debug_trace("Parameter is: \n" + str(parameter))
if (self.runbook.definition_kind_str == "PowerShell" or self.runbook.definition_kind_str == "PowerShell7") and parameter["Name"]:
# Handle named parameters for PowerShell arriving out of order
cmd += ["-%s" % parameter["Name"]]
try:
cmd += [str(json.loads(parameter["Value"]))]
except:
cmd += [str(parameter["Value"])]
# Do not copy current process env var to the sandbox process
env = os.environ.copy()
env.update({"AUTOMATION_JOB_ID": str(self.job_data.job_id),
"AUTOMATION_ACTIVITY_ID": str(tracer.u_activity_id),
"PYTHONPATH": str(configuration.get_source_directory_path()),
"HOME": str(os.getcwd())}) # windows env have to be str (not unicode)
self.runbook_subprocess = subprocessfactory.create_subprocess(cmd=cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
def generate_state_file():
state_file_name = "state.conf"
if configuration.get_state_directory_path(
) == configuration.DEFAULT_STATE_DIRECTORY_PATH:
state_file_path = os.path.join(
configuration.get_working_directory_path(), state_file_name)
else:
state_file_path = os.path.join(
configuration.get_state_directory_path(), state_file_name)
tracer.log_debug_trace("State file path : " + str(state_file_path))
if os.path.isfile(state_file_path):
os.remove(state_file_path)
section = "state"
conf_file = open(state_file_path, 'wb')
config = ConfigParser.ConfigParser()
config.add_section(section)
config.set(section, configuration.STATE_PID, str(os.getpid()))
config.set(section, configuration.WORKER_VERSION,
str(configuration.get_worker_version()))
# for OMS scenarios, optional for DIY
if len(sys.argv) >= 3:
config.set(section, configuration.STATE_WORKSPACE_ID, str(sys.argv[2]))
if len(sys.argv) >= 4:
config.set(section, configuration.STATE_RESOURCE_VERSION,
str(sys.argv[3]))
config.write(conf_file)
conf_file.close()
def execute_runbook(self):
"""Executes the job runtime and performs runtime operation (stream upload / status change)."""
# set status to running
tracer.log_debug_trace("Starting runbook.")
self.jrds_client.set_job_status(self.sandbox_id, self.job_id,
jobstatus.RUNNING, False)
# create runbook subprocess
self.runtime.start_runbook_subprocess()
# monitor runbook output for streams
stream_handler = StreamHandler(self.job_data,
self.runtime.runbook_subprocess,
self.jrds_client)
stream_handler.daemon = True
stream_handler.start()
# wait for runbook execution to complete
pending_action = None
while stream_handler.isAlive(
) or self.runtime.runbook_subprocess.poll() is None:
try:
pending_action = self.msg_queue.get(block=False)
tracer.log_debug_trace("Pending action detected. " +
str(pending_action))
if pending_action == pendingactions.STOP_ENUM_INDEX:
self.jrds_client.set_job_status(self.sandbox_id,
self.job_id,
jobstatus.STOPPING, False)
self.runtime.kill_runbook_subprocess()
break
except Queue.Empty:
pass
time.sleep(0.2)
# handle terminal state changes
if pending_action == pendingactions.STOP_ENUM_INDEX:
self.jrds_client.set_job_status(self.sandbox_id, self.job_id,
jobstatus.STOPPED, True)
tracer.log_debug_trace("Completed - Stopped")
elif self.runtime.runbook_subprocess.poll(
) is not None and self.runtime.runbook_subprocess.poll(
) == EXIT_SUCCESS:
self.jrds_client.set_job_status(self.sandbox_id, self.job_id,
jobstatus.COMPLETED, True)
tracer.log_debug_trace("Completed - Without error")
else:
full_error_output = self.get_full_stderr_content(
self.runtime.runbook_subprocess.stderr)
self.jrds_client.set_job_status(self.sandbox_id,
self.job_id,
jobstatus.FAILED,
True,
exception=full_error_output)
tracer.log_debug_trace("Completed - With error")
def run(self):
"""Monitor the job's subprocess for output (which will be uploaded as streams).
Notes:
PowerShell stdout : http://stackoverflow.com/questions/22349139/utf-8-output-from-powershell
IMPORTANT: Do not log streams to cloud.
"""
stream_count = 0
while True:
try:
output = codecs.getwriter('utf8')(
self.runtime_process.stdout).readline().decode()
error_output = codecs.getwriter('utf8')(
self.runtime_process.stderr).readline().decode()
if output == '' and error_output == '' and self.runtime_process.poll(
) is not None:
break
if output:
if output.startswith(PREFIX_DEBUG.lower()) or \
output.startswith(PREFIX_DEBUG.upper()) or \
output.startswith(PREFIX_DEBUG.capitalize()):
self.process_debug_stream(stream_count, output)
elif output.startswith(PREFIX_ERROR.lower()) or \
output.startswith(PREFIX_ERROR.upper()) or \
output.startswith(PREFIX_ERROR.capitalize()):
self.process_error_stream(stream_count, output)
elif output.startswith(PREFIX_VERBOSE.lower()) or \
output.startswith(PREFIX_VERBOSE.upper()) or \
output.startswith(PREFIX_VERBOSE.capitalize()):
self.process_verbose_stream(stream_count, output)
elif output.startswith(PREFIX_WARNING.lower()) or \
output.startswith(PREFIX_WARNING.upper()) or \
output.startswith(PREFIX_WARNING.capitalize()):
self.process_warning_stream(stream_count, output)
else:
self.process_output_stream(stream_count, output)
stream_count += 1
# leave trace at the end to prevent encoding issue from pushing streams to cloud
# leave this as debug trace to prevent logging customer streams to automation logs
tracer.log_debug_trace("STDOUT : " + str(output.strip()))
if error_output:
self.process_error_stream(stream_count, error_output)
stream_count += 1
tracer.log_debug_trace("STDERR : " +
str(error_output.strip()))
except:
tracer.log_sandbox_job_streamhandler_unhandled_exception(
self.job_data.job_id, traceback.format_exc())
continue
tracer.log_sandbox_job_streamhandler_processing_complete(
self.job_data.job_id)
def stop_tracking_terminated_sandbox(self):
terminated_sandbox_ids = []
# detect terminated sandboxes
for sandbox_id, sandbox_process in self.running_sandboxes.items():
if sandbox_process.poll() is not None:
terminated_sandbox_ids.append(sandbox_id)
# clean-up terminated sandboxes
for sandbox_id in terminated_sandbox_ids:
removal = self.running_sandboxes.pop(sandbox_id, None)
if removal is not None:
tracer.log_debug_trace("Worker stopped tracking sandbox : " + str(sandbox_id))
def routine(self):
# clean up finished jobs
self.stop_tracking_terminated_jobs()
# get job actions
try:
job_actions = self.jrds_client.get_job_actions(self.sandbox_id)
except JrdsSandboxTerminated:
tracer.log_debug_trace("Terminating sandbox.")
global routine_loop
routine_loop = False
return
for job_action in job_actions:
job_id = job_action["JobId"]
job_data = self.jrds_client.get_job_data(job_id)
job_pending_action = job_data["pendingAction"]
job_status = job_data["jobStatus"]
# issue pending action
if job_pending_action == pendingactions.ACTIVATE_ENUM_INDEX or \
(job_pending_action is None and job_status == jobstatus.ACTIVATING_ENUM_INDEX) or \
(job_pending_action is None and job_status == jobstatus.RUNNING_ENUM_INDEX):
# check if the specified job is already running to prevent duplicate
if job_id in job_map:
continue
# create and start the new job
job_message_queue = Queue()
job_thread_exception_queue = Queue()
job = Job(self.sandbox_id, job_id, job_message_queue,
self.jrds_client, job_thread_exception_queue)
job_map[job_id] = (job, job_message_queue,
job_thread_exception_queue)
job.start()
tracer.log_debug_trace(
"Pending action activate detected.[pendingaction=" +
str(job_status) + "]")
elif job_pending_action == pendingactions.STOP_ENUM_INDEX:
# check if the specified job is already running before issuing pending action
if job_id not in job_map:
continue
# propagate pending action to job thread
job_map[job_id][1].put(job_pending_action)
tracer.log_debug_trace("Pending action detected")
elif job_pending_action is None:
tracer.log_debug_trace("No pending action detected")
else:
tracer.log_debug_trace(
"Unsupported pending action / job action")
def stop_tracking_terminated_sandbox(self):
terminated_sandbox_ids = []
# detect terminated sandboxes
for sandbox_id, sandbox_process in self.running_sandboxes.items():
if sandbox_process.poll() is not None:
terminated_sandbox_ids.append(sandbox_id)
# clean-up terminated sandboxes
for sandbox_id in terminated_sandbox_ids:
removal = self.running_sandboxes.pop(sandbox_id, None)
if removal is not None:
tracer.log_debug_trace("Worker stopped tracking sandbox : " +
str(sandbox_id))
def routine(self):
self.stop_tracking_terminated_sandbox()
sandbox_actions = self.jrds_client.get_sandbox_actions()
tracer.log_debug_trace("Get sandbox action. Found " +
str(len(sandbox_actions)) + " action(s).")
for action in sandbox_actions:
tracer.log_worker_sandbox_action_found(len(sandbox_actions))
sandbox_id = str(action["SandboxId"])
# prevent duplicate sandbox from running
if sandbox_id in self.running_sandboxes:
return
# create sandboxes folder if needed
sandbox_working_dir = os.path.join(
configuration.get_working_directory_path(),
sandboxes_root_folder_name, sandbox_id)
try:
iohelper.assert_or_create_path(sandbox_working_dir)
except OSError, exception:
tracer.log_worker_failed_to_create_sandbox_root_folder(
sandbox_id, exception)
pass
# copy current process env variable (contains configuration) and add the sanbox_id key
process_env_variables = os.environ.copy()
process_env_variables["sandbox_id"] = sandbox_id
cmd = [
"python",
os.path.join(configuration.get_source_directory_path(),
"sandbox.py")
]
tracer.log_worker_starting_sandbox(sandbox_id)
sandbox_process = subprocessfactory.create_subprocess(
cmd=cmd,
env=process_env_variables,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=sandbox_working_dir)
self.running_sandboxes[sandbox_id] = sandbox_process
tracer.log_worker_started_tracking_sandbox(sandbox_id)
self.monitor_sandbox_process_outputs(sandbox_id, sandbox_process)
tracer.log_worker_sandbox_process_started(sandbox_id,
str(sandbox_process.pid))
def routine(self):
# clean up finished jobs
self.stop_tracking_terminated_jobs()
# get job actions
try:
job_actions = self.jrds_client.get_job_actions(self.sandbox_id)
except JrdsSandboxTerminated:
tracer.log_debug_trace("Terminating sandbox.")
global routine_loop
routine_loop = False
return
for job_action in job_actions:
job_id = job_action["JobId"]
job_data = self.jrds_client.get_job_data(job_id)
job_pending_action = job_data["pendingAction"]
job_status = job_data["jobStatus"]
# issue pending action
if job_pending_action == pendingactions.ACTIVATE_ENUM_INDEX or \
(job_pending_action is None and job_status == jobstatus.ACTIVATING_ENUM_INDEX) or \
(job_pending_action is None and job_status == jobstatus.RUNNING_ENUM_INDEX):
# check if the specified job is already running to prevent duplicate
if job_id in job_map:
continue
# create and start the new job
job_message_queue = Queue()
job_thread_exception_queue = Queue()
job = Job(self.sandbox_id, job_id, job_message_queue, self.jrds_client, job_thread_exception_queue)
job_map[job_id] = (job, job_message_queue, job_thread_exception_queue)
job.start()
tracer.log_debug_trace("Pending action activate detected.[pendingaction=" +
str(job_status) + "]")
elif job_pending_action == pendingactions.STOP_ENUM_INDEX:
# check if the specified job is already running before issuing pending action
if job_id not in job_map:
continue
# propagate pending action to job thread
job_map[job_id][1].put(job_pending_action)
tracer.log_debug_trace("Pending action detected")
elif job_pending_action is None:
tracer.log_debug_trace("No pending action detected")
else:
tracer.log_debug_trace("Unsupported pending action / job action")
def run(self):
"""Main job execution logic. This methods returns when the job execution is completed.
Throws:
WorkerUnsupportedRunbookType : If the language isn't supported by by the worker.
OSUnsupportedRunbookType : If the language isn't supported by by the host.
Exception : Any unhandled exception.
"""
try:
self.load_job()
self.initialize_runtime()
self.execute_runbook()
self.unload_job()
except (WorkerUnsupportedRunbookType, OSUnsupportedRunbookType), e:
tracer.log_debug_trace("Runbook type not supported.")
self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.FAILED, True, exception=e.message)
self.unload_job()
def stop_tracking_terminated_jobs():
terminated_job_ids = []
# clean up finished jobs
for job_id, job_tuple in job_map.items():
if job_tuple[0].isAlive() is False:
try:
job_tuple[2].get(block=False)
raise SandboxRuntimeException()
except Empty:
pass
terminated_job_ids.append(job_id)
for job_id in terminated_job_ids:
removal = job_map.pop(job_id, None)
if removal is not None:
tracer.log_debug_trace("Sandbox stopped tracking job : " + str(job_id))
def stop_tracking_terminated_jobs():
terminated_job_ids = []
# clean up finished jobs
for job_id, job_tuple in job_map.items():
if job_tuple[0].isAlive() is False:
try:
job_tuple[2].get(block=False)
raise SandboxRuntimeException()
except Empty:
pass
terminated_job_ids.append(job_id)
for job_id in terminated_job_ids:
removal = job_map.pop(job_id, None)
if removal is not None:
tracer.log_debug_trace("Sandbox stopped tracking job : " +
str(job_id))
def monitor_sandbox_process_outputs(self, sandbox_id, process):
while process.poll() is None:
output = process.stdout.readline().replace("\n", "")
if output == '':
continue
if output != '':
tracer.log_sandbox_stdout(output)
if process.poll() != 0:
full_error_output = ""
while True:
error_output = process.stderr.readline()
if error_output is None or error_output == '':
break
full_error_output += error_output
tracer.log_debug_trace("Sandbox crashed : " + str(full_error_output))
tracer.log_worker_sandbox_process_exited(sandbox_id, str(process.pid), process.poll())
def routine(self):
self.stop_tracking_terminated_sandbox()
sandbox_actions = self.jrds_client.get_sandbox_actions()
tracer.log_debug_trace("Get sandbox action. Found " +
str(len(sandbox_actions)) + " action(s).")
for action in sandbox_actions:
tracer.log_worker_sandbox_action_found(len(sandbox_actions))
sandbox_id = str(action["SandboxId"])
# prevent duplicate sandbox from running
if sandbox_id in self.running_sandboxes:
return
# create sandboxes folder if needed
sandboxes_base_path = "sandboxes"
sandbox_working_dir = os.path.join(
configuration.get_working_directory_path(),
sandboxes_base_path, sandbox_id)
try:
iohelper.assert_or_create_path(sandbox_working_dir)
except OSError:
tracer.log_debug_trace("Failed to create sandbox folder.")
pass
cmd = [
"python",
os.path.join(configuration.get_source_directory_path(),
"sandbox.py")
]
process_env_variables = {"sandbox_id": sandbox_id}
sandbox_process = subprocessfactory.create_subprocess(
cmd=cmd,
env=process_env_variables,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=sandbox_working_dir)
self.running_sandboxes[sandbox_id] = sandbox_process
self.monitor_sandbox_process_outputs(sandbox_id, sandbox_process)
tracer.log_worker_starting_sandbox(sandbox_id,
str(sandbox_process.pid))
def run(self):
"""Monitor the job's subprocess for output (which will be uploaded as streams).
Notes:
PowerShell stdout : http://stackoverflow.com/questions/22349139/utf-8-output-from-powershell
IMPORTANT: Do not log streams to cloud.
"""
stream_count = 0
while True:
try:
output = codecs.getwriter('utf8')(self.runtime_process.stdout).readline()
if output == '' and self.runtime_process.poll() is not None:
break
elif output:
if output.startswith(PREFIX_DEBUG.lower()) or \
output.startswith(PREFIX_DEBUG.upper()) or \
output.startswith(PREFIX_DEBUG.capitalize()):
self.process_debug_stream(stream_count, output)
elif output.startswith(PREFIX_ERROR.lower()) or \
output.startswith(PREFIX_ERROR.upper()) or \
output.startswith(PREFIX_ERROR.capitalize()):
self.process_error_stream(stream_count, output)
elif output.startswith(PREFIX_VERBOSE.lower()) or \
output.startswith(PREFIX_VERBOSE.upper()) or \
output.startswith(PREFIX_VERBOSE.capitalize()):
self.process_verbose_stream(stream_count, output)
elif output.startswith(PREFIX_WARNING.lower()) or \
output.startswith(PREFIX_WARNING.upper()) or \
output.startswith(PREFIX_WARNING.capitalize()):
self.process_warning_stream(stream_count, output)
else:
self.process_output_stream(stream_count, output)
stream_count += 1
# leave trace at the end to prevent encoding issue from pushing streams to cloud
# leave this as debug trace to prevent logging customer streams to automation logs
tracer.log_debug_trace("STDOUT : " + str(output.strip()))
except:
tracer.log_exception_trace(traceback.format_exc())
continue
tracer.log_debug_trace("Stream processing complete.")
def monitor_sandbox_process_outputs(self, sandbox_id, process):
while process.poll() is None:
output = process.stdout.readline().replace("\n", "")
if output == '':
continue
if output != '':
tracer.log_sandbox_stdout(output)
if process.poll() != 0:
full_error_output = ""
while True:
error_output = process.stderr.readline()
if error_output is None or error_output == '':
break
full_error_output += error_output
tracer.log_debug_trace("Sandbox crashed : " +
str(full_error_output))
tracer.log_worker_sandbox_process_exited(sandbox_id, str(process.pid),
process.poll())
def run(self):
"""Main job execution logic. This methods returns when the job execution is completed.
Throws:
WorkerUnsupportedRunbookType : If the language isn't supported by by the worker.
OSUnsupportedRunbookType : If the language isn't supported by by the host.
Exception : Any unhandled exception.
"""
try:
self.load_job()
self.initialize_runtime()
self.execute_runbook()
self.unload_job()
except (WorkerUnsupportedRunbookType, OSUnsupportedRunbookType), e:
tracer.log_debug_trace("Runbook type not supported.")
self.jrds_client.set_job_status(self.sandbox_id,
self.job_id,
jobstatus.FAILED,
True,
exception=e.message)
self.unload_job()
def execute_runbook(self):
"""Executes the job runtime and performs runtime operation (stream upload / status change)."""
# set status to running
tracer.log_debug_trace("Starting runbook.")
self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.RUNNING, False)
# create runbook subprocess
self.runtime.start_runbook_subprocess()
# monitor runbook output for streams
stream_handler = StreamHandler(self.job_data, self.runtime.runbook_subprocess, self.jrds_client)
stream_handler.daemon = True
stream_handler.start()
# wait for runbook execution to complete
pending_action = None
while stream_handler.isAlive() or self.runtime.runbook_subprocess.poll() is None:
try:
pending_action = self.msg_queue.get(block=False)
tracer.log_debug_trace("Pending action detected. " + str(pending_action))
if pending_action == pendingactions.STOP_ENUM_INDEX:
self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.STOPPING, False)
self.runtime.kill_runbook_subprocess()
break
except Queue.Empty:
pass
time.sleep(0.2)
# handle terminal state changes
if pending_action == pendingactions.STOP_ENUM_INDEX:
self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.STOPPED, True)
tracer.log_debug_trace("Completed - Stopped")
elif self.runtime.runbook_subprocess.poll() is not None and self.runtime.runbook_subprocess.poll() == EXIT_SUCCESS:
self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.COMPLETED, True)
tracer.log_debug_trace("Completed - Without error")
else:
full_error_output = self.get_full_stderr_content(self.runtime.runbook_subprocess.stderr)
self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.FAILED, True,
exception=full_error_output)
tracer.log_debug_trace("Completed - With error")
def verify_signature(signed_file_path, output_file_path):
"""Verifies the signed file's signature.
Returns:
True : If the signature is valid.
False : If the signature is invalid.
"""
cmd = GPG_DECRYPT_BASE_CMD
keyring_path = configuration.get_gpg_public_keyring_path()
# if a keyring is specified in the conf, used it, else use default one
if keyring_path != "":
cmd += [GPG_NO_DEFAULT_KEYRING_OPTION, GPG_KEYRING_ARG, keyring_path]
cmd += ["--output", output_file_path, signed_file_path]
proc = subprocessfactory.create_subprocess(cmd=cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = proc.communicate()
if proc.poll() == 0:
tracer.log_debug_trace("Signature is valid.")
return True
tracer.log_debug_trace("Signature is invalid.[exception=" + str(stderr) + "]")
return False
def unload_job(self):
"""Unloads the job."""
self.jrds_client.unload_job(self.job_data["subscriptionId"], self.sandbox_id, self.job_id,
self.job_updatable_data["isDraft"], datetime.now(), 2)
tracer.log_debug_trace("Unloading job.")
self.jrds_client.set_job_status(self.sandbox_id,
self.job_id,
jobstatus.FAILED,
True,
exception=e.message)
self.unload_job()
except InvalidRunbookSignature, e:
tracer.log_debug_trace("Runbook signature is invalid.")
self.jrds_client.set_job_status(self.sandbox_id,
self.job_id,
jobstatus.FAILED,
True,
exception=e.message)
self.unload_job()
except Exception:
tracer.log_debug_trace("Job runtime unhandled exception.")
tracer.log_exception_trace(traceback.format_exc())
self.job_thread_exception_queue.put(sys.exc_info())
def execute_runbook(self):
"""Executes the job runtime and performs runtime operation (stream upload / status change)."""
# set status to running
tracer.log_debug_trace("Starting runbook.")
self.jrds_client.set_job_status(self.sandbox_id, self.job_id,
jobstatus.RUNNING, False)
# create runbook subprocess
self.runtime.start_runbook_subprocess()
# monitor runbook output for streams
stream_handler = StreamHandler(self.job_data,
class Job(Thread):
"""Job class."""
def __init__(self, sandbox_id, job_id, msg_queue, jrds_client,
job_thread_exception_queue):
Thread.__init__(self)
self.daemon = True
self.sandbox_id = sandbox_id
self.job_id = job_id
self.msg_queue = msg_queue
self.jrds_client = jrds_client
self.job_thread_exception_queue = job_thread_exception_queue
# values populated in load_job()
self.runtime = None
self.runbook = None
self.job_data = None
self.job_updatable_data = None
self.runbook_data = None
def load_job(self):
"""Load all required artifact for the job to be executed."""
self.jrds_client.set_job_status(self.sandbox_id, self.job_id,
jobstatus.ACTIVATING, False)
self.job_data = self.jrds_client.get_job_data(self.job_id)
self.job_updatable_data = self.jrds_client.get_job_updatable_data(
self.job_id)
self.runbook_data = self.jrds_client.get_runbook_data(
self.job_data["runbookVersionId"])
def initialize_runtime(self):
"""Initializes the runtime component for the job. The runtime component is language specific."""
self.runbook, self.runtime = runtimefactory.create_runtime(
self.job_data, self.runbook_data)
def run(self):
"""Main job execution logic. This methods returns when the job execution is completed.
Throws:
WorkerUnsupportedRunbookType : If the language isn't supported by by the worker.
OSUnsupportedRunbookType : If the language isn't supported by by the host.
Exception : Any unhandled exception.
"""
try:
self.load_job()
self.initialize_runtime()
self.execute_runbook()
self.unload_job()
except (WorkerUnsupportedRunbookType, OSUnsupportedRunbookType), e:
tracer.log_debug_trace("Runbook type not supported.")
self.jrds_client.set_job_status(self.sandbox_id,
self.job_id,
jobstatus.FAILED,
True,
exception=e.message)
self.unload_job()
except InvalidRunbookSignature, e:
tracer.log_debug_trace("Runbook signature is invalid.")
self.jrds_client.set_job_status(self.sandbox_id,
self.job_id,
jobstatus.FAILED,
True,
exception=e.message)
self.unload_job()
def generate_state_file():
state_file_name = "state.conf"
if configuration.get_state_directory_path(
) == configuration.DEFAULT_STATE_DIRECTORY_PATH:
state_file_path = os.path.join(
configuration.get_working_directory_path(), state_file_name)
else:
state_file_path = os.path.join(
configuration.get_state_directory_path(), state_file_name)
tracer.log_debug_trace("State file path : " + str(state_file_path))
if os.path.isfile(state_file_path):
os.remove(state_file_path)
section = "state"
conf_file = open(state_file_path, 'wb')
config = ConfigParser.ConfigParser()
config.add_section(section)
config.set(section, configuration.STATE_PID, str(os.getpid()))
config.set(section, configuration.WORKER_VERSION,
str(configuration.get_worker_version()))
# for OMS scenarios, optional for DIY
if len(sys.argv) >= 3:
config.set(section, configuration.STATE_WORKSPACE_ID, str(sys.argv[2]))
if len(sys.argv) >= 4:
config.set(section, configuration.STATE_RESOURCE_VERSION,
str(sys.argv[3]))
config.write(conf_file)
conf_file.close()
# OMS integration
# set the ownership of the state file to nxautomation:omiusers
# set the permission of the state file to 660
if os.name.lower() != "nt":
import pwd
try:
nxautomation_uid = int(pwd.getpwnam('nxautomation').pw_uid)
if os.getuid() == nxautomation_uid:
retval = subprocess.call([
"sudo", "chown", "nxautomation:omiusers", state_file_path
])
if retval != 0:
exit_on_error(
"Could not change owner of state file %s to nxautomation:omiusers"
% (state_file_path))
retval = subprocess.call(
["sudo", "chmod", "660", state_file_path])
if retval != 0:
exit_on_error(
"Could not change permission of state file %s " %
(state_file_path))
except KeyError:
# nxautomation user was not found on the system, skip this step
tracer.log_debug_trace(
"State file permission change skipped. nxautomation user not found."
)
pass
"""
try:
self.load_job()
self.initialize_runtime()
self.execute_runbook()
self.unload_job()
except (WorkerUnsupportedRunbookType, OSUnsupportedRunbookType), e:
tracer.log_debug_trace("Runbook type not supported.")
self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.FAILED, True, exception=e.message)
self.unload_job()
except InvalidRunbookSignature, e:
tracer.log_debug_trace("Runbook signature is invalid.")
self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.FAILED, True, exception=e.message)
self.unload_job()
except Exception:
tracer.log_debug_trace("Job runtime unhandled exception.")
tracer.log_exception_trace(traceback.format_exc())
self.job_thread_exception_queue.put(sys.exc_info())
def execute_runbook(self):
"""Executes the job runtime and performs runtime operation (stream upload / status change)."""
# set status to running
tracer.log_debug_trace("Starting runbook.")
self.jrds_client.set_job_status(self.sandbox_id, self.job_id, jobstatus.RUNNING, False)
# create runbook subprocess
self.runtime.start_runbook_subprocess()
# monitor runbook output for streams
stream_handler = StreamHandler(self.job_data, self.runtime.runbook_subprocess, self.jrds_client)
stream_handler.daemon = True
def validate_and_setup_path():
# default to user dir for exception logs to be writen to disk
test_file_name = "test_file"
# test certificate and key path
if not os.path.isfile(
configuration.get_jrds_cert_path()) or not os.path.isfile(
configuration.get_jrds_key_path()):
exit_on_error(
"Invalid certificate of key file path (absolute path is required)."
)
# test working directory for existence and permissions
working_directory_path = configuration.get_working_directory_path()
if not os.path.exists(working_directory_path):
exit_on_error(
"Invalid working directory path (absolute path is required).")
file_creation = test_file_creation(
os.path.join(working_directory_path, test_file_name))
if file_creation is False:
exit_on_error(
"Invalid working directory permission (read/write permissions are required)."
)
# test keyring paths
keyrings = configuration.get_gpg_public_keyrings_path()
for keyring_path in keyrings:
if keyring_path != configuration.DEFAULT_GPG_PUBLIC_KEYRING_PATH and not os.path.isfile(
keyring_path):
exit_on_error(
"Invalid gpg public keyring path (absolute path is required).")
# test state file path
if configuration.get_state_directory_path(
) != configuration.DEFAULT_STATE_DIRECTORY_PATH:
if not os.path.exists(configuration.get_state_directory_path()):
exit_on_error(
"Invalid state directory path (absolute path is required).")
file_creation = test_file_creation(
os.path.join(configuration.get_state_directory_path(),
test_file_name))
if file_creation is False:
exit_on_error(
"Invalid state directory permission (read/write permissions are required)."
)
# OMS integration
# set the working directory owner to be nxautomation:omiusers
if os.name.lower() != "nt":
import pwd
try:
nxautomation_uid = int(pwd.getpwnam('nxautomation').pw_uid)
if os.getuid() == nxautomation_uid:
retval = subprocess.call([
"sudo", "chown", "-R", "nxautomation:omiusers",
working_directory_path
])
if retval != 0:
exit_on_error(
"Could not change owner of working directory %s to nxautomation:omiusers"
% (working_directory_path))
except KeyError:
# nxautomation user was not found on the system, skip this step
tracer.log_debug_trace(
"Ownership change of working directory skipped. nxautomation user not found."
)
pass
