def __init__(self):
tracer.log_worker_starting(configuration.get_worker_version())
http_client_factory = HttpClientFactory(configuration.get_jrds_cert_path(), configuration.get_jrds_key_path(),
configuration.get_verify_certificates())
http_client = http_client_factory.create_http_client(sys.version_info)
self.jrds_client = JRDSClient(http_client)
self.running_sandboxes = {}
def __init__(self):
self.sandbox_id = os.environ["sandbox_id"]
tracer.log_sandbox_starting(self.sandbox_id, os.getpid())
http_client_factory = HttpClientFactory(
configuration.get_jrds_cert_path(),
configuration.get_jrds_key_path())
http_client = http_client_factory.create_http_client(
sys.version_info, configuration.get_verify_certificates())
self.jrds_client = JRDSClient(http_client)
def __init__(self):
self.sandbox_id = os.environ["sandbox_id"]
tracer.log_sandbox_starting(sandbox_id=self.sandbox_id, pid=os.getpid(),
worker_type=configuration.get_worker_type())
tracer.log_sandbox_configuration(sandbox_id=self.sandbox_id,
enforce_runbook_signature_validation=configuration.get_enforce_runbook_signature_validation(),
gpg_public_keyring_paths=configuration.get_gpg_public_keyrings_path(),
working_directory=os.getcwd())
http_client_factory = HttpClientFactory(configuration.get_jrds_cert_path(), configuration.get_jrds_key_path(),
configuration.get_verify_certificates())
http_client = http_client_factory.create_http_client(sys.version_info)
self.jrds_client = JRDSClient(http_client)
def __init__(self):
tracer.log_worker_starting(configuration.get_worker_version())
http_client_factory = HttpClientFactory(configuration.get_jrds_cert_path(), configuration.get_jrds_key_path(),
configuration.get_verify_certificates())
http_client = http_client_factory.create_http_client(sys.version_info)
self.jrds_client = JRDSClient(http_client)
self.running_sandboxes = {}
def init():
"""Initializes all required variable for the tracer."""
global jrds_client, jrds_cert_path, jrds_key_path, jrds_base_uri, subscription_id, \
account_id, machine_id, hybrid_worker_group_name, worker_version, activity_id, sandbox_id
# Create the http client
http_client_factory = HttpClientFactory(
configuration.get_jrds_cert_path(), configuration.get_jrds_key_path(),
configuration.get_verify_certificates())
http_client = http_client_factory.create_http_client(sys.version_info)
jrds_client = JRDSClient(http_client)
# Populate global configuration values
jrds_cert_path = configuration.get_jrds_cert_path()
jrds_key_path = configuration.get_jrds_key_path()
jrds_base_uri = configuration.get_jrds_base_uri()
subscription_id = "00000000-0000-0000-0000-000000000000" # temporary place holder
account_id = configuration.get_account_id()
machine_id = configuration.get_machine_id()
hybrid_worker_group_name = configuration.get_hybrid_worker_name()
worker_version = configuration.get_worker_version()
sandbox_id = None
try:
sandbox_id = os.environ["sandbox_id"]
except KeyError:
pass
# initialize the loggers for for all components except runbook
if configuration.get_component() != "runbook":
init_logger()
class Sandbox:
def __init__(self):
self.sandbox_id = os.environ["sandbox_id"]
tracer.log_sandbox_starting(self.sandbox_id, os.getpid())
http_client_factory = HttpClientFactory(
configuration.get_jrds_cert_path(),
configuration.get_jrds_key_path())
http_client = http_client_factory.create_http_client(
sys.version_info, configuration.get_verify_certificates())
self.jrds_client = JRDSClient(http_client)
@staticmethod
def stop_tracking_terminated_jobs():
terminated_job_ids = []
# clean up finished jobs
for job_id, job_tuple in job_map.items():
if job_tuple[0].isAlive() is False:
try:
job_tuple[2].get(block=False)
raise SandboxRuntimeException()
except Empty:
pass
terminated_job_ids.append(job_id)
for job_id in terminated_job_ids:
removal = job_map.pop(job_id, None)
if removal is not None:
tracer.log_debug_trace("Sandbox stopped tracking job : " +
str(job_id))
@safe_loop
def routine(self):
# clean up finished jobs
self.stop_tracking_terminated_jobs()
# get job actions
try:
job_actions = self.jrds_client.get_job_actions(self.sandbox_id)
except JrdsSandboxTerminated:
tracer.log_debug_trace("Terminating sandbox.")
global routine_loop
routine_loop = False
return
for job_action in job_actions:
job_id = job_action["JobId"]
job_data = self.jrds_client.get_job_data(job_id)
job_pending_action = job_data["pendingAction"]
job_status = job_data["jobStatus"]
# issue pending action
if job_pending_action == pendingactions.ACTIVATE_ENUM_INDEX or \
(job_pending_action is None and job_status == jobstatus.ACTIVATING_ENUM_INDEX) or \
(job_pending_action is None and job_status == jobstatus.RUNNING_ENUM_INDEX):
# check if the specified job is already running to prevent duplicate
if job_id in job_map:
continue
# create and start the new job
job_message_queue = Queue()
job_thread_exception_queue = Queue()
job = Job(self.sandbox_id, job_id, job_message_queue,
self.jrds_client, job_thread_exception_queue)
job_map[job_id] = (job, job_message_queue,
job_thread_exception_queue)
job.start()
tracer.log_debug_trace(
"Pending action activate detected.[pendingaction=" +
str(job_status) + "]")
elif job_pending_action == pendingactions.STOP_ENUM_INDEX:
# check if the specified job is already running before issuing pending action
if job_id not in job_map:
continue
# propagate pending action to job thread
job_map[job_id][1].put(job_pending_action)
tracer.log_debug_trace("Pending action detected")
elif job_pending_action is None:
tracer.log_debug_trace("No pending action detected")
else:
tracer.log_debug_trace(
"Unsupported pending action / job action")
def init():
"""Initializes all required variable for the tracer."""
global default_logger, sandbox_stdout, jrds_client, jrds_cert_path, jrds_key_path, jrds_base_uri, subscription_id, \
account_id, machine_id, hybrid_worker_group_name, worker_version, activity_id, sandbox_id
# Create the http client
http_client_factory = HttpClientFactory(
configuration.get_jrds_cert_path(), configuration.get_jrds_key_path(),
configuration.get_verify_certificates())
http_client = http_client_factory.create_http_client(sys.version_info)
jrds_client = JRDSClient(http_client)
# Populate global configuration values
jrds_cert_path = configuration.get_jrds_cert_path()
jrds_key_path = configuration.get_jrds_key_path()
jrds_base_uri = configuration.get_jrds_base_uri()
subscription_id = "00000000-0000-0000-0000-000000000000" # temporary place holder
account_id = configuration.get_account_id()
machine_id = configuration.get_machine_id()
hybrid_worker_group_name = configuration.get_hybrid_worker_name()
worker_version = configuration.get_worker_version()
sandbox_id = None
try:
sandbox_id = os.environ["sandbox_id"]
except KeyError:
pass
if sandbox_id is not None:
log_file_name = configuration.get_component() + sandbox_id
else:
log_file_name = configuration.get_component()
file_name = os.path.join(configuration.get_working_directory_path(),
log_file_name + '.log')
logging.Formatter.converter = time.gmtime
# Default logger
default_logger = logging.getLogger("default_logger")
default_logger.setLevel(logging.INFO)
# Logger for the sandbox traces coming back to worker
sandbox_stdout = logging.getLogger("sandbox_stdout_logger")
sandbox_stdout.setLevel(logging.INFO)
# Default rotating file handler write traces with the specified format to disk.
default_rf_handler = logging.handlers.RotatingFileHandler(
file_name, maxBytes=10485760, backupCount=5)
formatter = logging.Formatter('%(asctime)s (' + str(os.getpid()) + ')' +
configuration.get_component() +
' : %(message)s',
datefmt="%Y-%m-%d %H:%M:%S")
default_rf_handler.setFormatter(formatter)
default_logger.addHandler(default_rf_handler)
# Traces coming from sandbox child process and collected by the worker are already formatted, hence no formatter
# needed.
worker_sandbox_rf_handler = logging.handlers.RotatingFileHandler(
file_name, maxBytes=10485760, backupCount=5)
sandbox_stdout.addHandler(worker_sandbox_rf_handler)
# Stdout handler (Worker traces have to be formatted).
log_stream = logging.StreamHandler(sys.stdout)
log_stream.setFormatter(formatter)
default_logger.addHandler(log_stream)
# Stdout handler (traces coming from child process are already formatted).
sandbox_log_stream = logging.StreamHandler(sys.stdout)
sandbox_stdout.addHandler(sandbox_log_stream)
class Worker:
def __init__(self):
tracer.log_worker_starting(configuration.get_worker_version())
http_client_factory = HttpClientFactory(configuration.get_jrds_cert_path(), configuration.get_jrds_key_path(),
configuration.get_verify_certificates())
http_client = http_client_factory.create_http_client(sys.version_info)
self.jrds_client = JRDSClient(http_client)
self.running_sandboxes = {}
@staticmethod
def assert_environment_prerequisite():
jrds_cert_path = configuration.get_jrds_cert_path()
if util.assert_file_read_permission(jrds_cert_path) is False:
raise InvalidFilePermissionException(jrds_cert_path)
jrds_key_path = configuration.get_jrds_key_path()
if util.assert_file_read_permission(jrds_key_path) is False:
raise InvalidFilePermissionException(jrds_key_path)
worker_conf_path = configuration.get_worker_configuration_file_path()
if util.assert_file_read_permission(worker_conf_path) is False:
raise InvalidFilePermissionException(worker_conf_path)
proxy_conf_path = configuration.get_proxy_configuration_path()
if proxy_conf_path != configuration.DEFAULT_PROXY_CONFIGURATION_PATH and os.path.isfile(proxy_conf_path):
if util.assert_file_read_permission(proxy_conf_path) is False:
raise InvalidFilePermissionException(proxy_conf_path)
@safe_loop
def routine(self):
self.assert_environment_prerequisite()
self.stop_tracking_terminated_sandbox()
sandbox_actions = self.jrds_client.get_sandbox_actions()
if sandbox_actions is None:
tracer.log_get_sandbox_action_returned_null_data()
return
tracer.log_debug_trace("Get sandbox action. Found " + str(len(sandbox_actions)) + " action(s).")
for action in sandbox_actions:
tracer.log_worker_sandbox_action_found(len(sandbox_actions))
sandbox_id = str(action["SandboxId"])
# prevent duplicate sandbox from running
if sandbox_id in self.running_sandboxes:
continue
# create sandboxes folder if needed
sandbox_working_dir = os.path.join(configuration.get_working_directory_path(), sandboxes_root_folder_name,
sandbox_id)
try:
iohelper.assert_or_create_path(sandbox_working_dir)
except OSError, exception:
tracer.log_worker_failed_to_create_sandbox_root_folder(sandbox_id, exception)
raise SystemExit("Sandbox folder creation failed.")
# copy current process env variable (contains configuration) and add the sanbox_id key
process_env_variables = os.environ.copy()
process_env_variables["sandbox_id"] = sandbox_id
cmd = ["python", os.path.join(configuration.get_source_directory_path(), "sandbox.py"),
configuration.get_worker_configuration_file_path()]
tracer.log_worker_starting_sandbox(sandbox_id)
sandbox_process = subprocessfactory.create_subprocess(cmd=cmd,
env=process_env_variables,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=sandbox_working_dir)
self.running_sandboxes[sandbox_id] = sandbox_process
tracer.log_worker_started_tracking_sandbox(sandbox_id)
self.monitor_sandbox_process_outputs(sandbox_id, sandbox_process)
tracer.log_worker_sandbox_process_started(sandbox_id, str(sandbox_process.pid))
class Worker:
def __init__(self):
tracer.log_worker_starting(configuration.get_worker_version())
http_client_factory = HttpClientFactory(
configuration.get_jrds_cert_path(),
configuration.get_jrds_key_path(),
configuration.get_verify_certificates())
http_client = http_client_factory.create_http_client(sys.version_info)
self.jrds_client = JRDSClient(http_client)
self.running_sandboxes = {}
@safe_loop
def routine(self):
# die if pre-reqs are not met
if os.path.exists(configuration.get_jrds_cert_path()) is False or \
os.path.exists(configuration.get_jrds_key_path()) is False or \
os.path.exists(configuration.get_worker_configuration_file_path()) is False:
raise SystemExit()
self.stop_tracking_terminated_sandbox()
sandbox_actions = self.jrds_client.get_sandbox_actions()
tracer.log_debug_trace("Get sandbox action. Found " +
str(len(sandbox_actions)) + " action(s).")
for action in sandbox_actions:
tracer.log_worker_sandbox_action_found(len(sandbox_actions))
sandbox_id = str(action["SandboxId"])
# prevent duplicate sandbox from running
if sandbox_id in self.running_sandboxes:
continue
# create sandboxes folder if needed
sandbox_working_dir = os.path.join(
configuration.get_working_directory_path(),
sandboxes_root_folder_name, sandbox_id)
try:
iohelper.assert_or_create_path(sandbox_working_dir)
except OSError, exception:
tracer.log_worker_failed_to_create_sandbox_root_folder(
sandbox_id, exception)
pass
# copy current process env variable (contains configuration) and add the sanbox_id key
process_env_variables = os.environ.copy()
process_env_variables["sandbox_id"] = sandbox_id
cmd = [
"python",
os.path.join(configuration.get_source_directory_path(),
"sandbox.py"),
configuration.get_worker_configuration_file_path()
]
tracer.log_worker_starting_sandbox(sandbox_id)
sandbox_process = subprocessfactory.create_subprocess(
cmd=cmd,
env=process_env_variables,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=sandbox_working_dir)
self.running_sandboxes[sandbox_id] = sandbox_process
tracer.log_worker_started_tracking_sandbox(sandbox_id)
self.monitor_sandbox_process_outputs(sandbox_id, sandbox_process)
tracer.log_worker_sandbox_process_started(sandbox_id,
str(sandbox_process.pid))
class Worker:
def __init__(self):
tracer.log_worker_starting(configuration.get_worker_version())
http_client_factory = HttpClientFactory(configuration.get_jrds_cert_path(), configuration.get_jrds_key_path())
http_client = http_client_factory.create_http_client(sys.version_info, configuration.get_verify_certificates())
self.jrds_client = JRDSClient(http_client)
self.running_sandboxes = {}
@safe_loop
def routine(self):
self.stop_tracking_terminated_sandbox()
sandbox_actions = self.jrds_client.get_sandbox_actions()
tracer.log_debug_trace("Get sandbox action. Found " + str(len(sandbox_actions)) + " action(s).")
for action in sandbox_actions:
tracer.log_worker_sandbox_action_found(len(sandbox_actions))
sandbox_id = str(action["SandboxId"])
# prevent duplicate sandbox from running
if sandbox_id in self.running_sandboxes:
return
# create sandboxes folder if needed
sandboxes_base_path = "sandboxes"
sandbox_working_dir = os.path.join(configuration.get_working_directory_path(), sandboxes_base_path,
sandbox_id)
try:
iohelper.assert_or_create_path(sandbox_working_dir)
except OSError:
tracer.log_debug_trace("Failed to create sandbox folder.")
pass
cmd = ["python", os.path.join(configuration.get_source_directory_path(), "sandbox.py")]
process_env_variables = {"sandbox_id": sandbox_id}
sandbox_process = subprocessfactory.create_subprocess(cmd=cmd,
env=process_env_variables,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=sandbox_working_dir)
self.running_sandboxes[sandbox_id] = sandbox_process
self.monitor_sandbox_process_outputs(sandbox_id, sandbox_process)
tracer.log_worker_starting_sandbox(sandbox_id, str(sandbox_process.pid))
@background_thread
def monitor_sandbox_process_outputs(self, sandbox_id, process):
while process.poll() is None:
output = process.stdout.readline().replace("\n", "")
if output == '':
continue
if output != '':
tracer.log_sandbox_stdout(output)
if process.poll() != 0:
full_error_output = ""
while True:
error_output = process.stderr.readline()
if error_output is None or error_output == '':
break
full_error_output += error_output
tracer.log_debug_trace("Sandbox crashed : " + str(full_error_output))
tracer.log_worker_sandbox_process_exited(sandbox_id, str(process.pid), process.poll())
def stop_tracking_terminated_sandbox(self):
terminated_sandbox_ids = []
# detect terminated sandboxes
for sandbox_id, sandbox_process in self.running_sandboxes.items():
if sandbox_process.poll() is not None:
terminated_sandbox_ids.append(sandbox_id)
# clean-up terminated sandboxes
for sandbox_id in terminated_sandbox_ids:
removal = self.running_sandboxes.pop(sandbox_id, None)
if removal is not None:
tracer.log_debug_trace("Worker stopped tracking sandbox : " + str(sandbox_id))
def __init__(self):
self.sandbox_id = os.environ["sandbox_id"]
tracer.log_sandbox_starting(self.sandbox_id, os.getpid())
http_client_factory = HttpClientFactory(configuration.get_jrds_cert_path(), configuration.get_jrds_key_path())
http_client = http_client_factory.create_http_client(sys.version_info, configuration.get_verify_certificates())
self.jrds_client = JRDSClient(http_client)
class Sandbox:
def __init__(self):
self.sandbox_id = os.environ["sandbox_id"]
tracer.log_sandbox_starting(sandbox_id=self.sandbox_id, pid=os.getpid(),
worker_type=configuration.get_worker_type())
tracer.log_sandbox_configuration(sandbox_id=self.sandbox_id,
enforce_runbook_signature_validation=configuration.get_enforce_runbook_signature_validation(),
gpg_public_keyring_paths=configuration.get_gpg_public_keyrings_path(),
working_directory=os.getcwd())
http_client_factory = HttpClientFactory(configuration.get_jrds_cert_path(), configuration.get_jrds_key_path(),
configuration.get_verify_certificates())
http_client = http_client_factory.create_http_client(sys.version_info)
self.jrds_client = JRDSClient(http_client)
@staticmethod
def assert_environment_prerequisite():
jrds_cert_path = configuration.get_jrds_cert_path()
if util.assert_file_read_permission(jrds_cert_path) is False:
raise InvalidFilePermissionException(jrds_cert_path)
jrds_key_path = configuration.get_jrds_key_path()
if util.assert_file_read_permission(jrds_key_path) is False:
raise InvalidFilePermissionException(jrds_key_path)
worker_conf_path = configuration.get_worker_configuration_file_path()
if util.assert_file_read_permission(worker_conf_path) is False:
raise InvalidFilePermissionException(worker_conf_path)
proxy_conf_path = configuration.get_proxy_configuration_path()
if proxy_conf_path != configuration.DEFAULT_PROXY_CONFIGURATION_PATH and os.path.isfile(proxy_conf_path):
if util.assert_file_read_permission(proxy_conf_path) is False:
raise InvalidFilePermissionException(proxy_conf_path)
@staticmethod
def stop_tracking_terminated_jobs():
terminated_job_ids = []
# clean up finished jobs
for job_id, job_tuple in job_map.items():
if job_tuple[0].isAlive() is False:
try:
job_tuple[2].get(block=False)
raise SandboxRuntimeException()
except Empty:
pass
terminated_job_ids.append(job_id)
for job_id in terminated_job_ids:
removal = job_map.pop(job_id, None)
if removal is not None:
tracer.log_sandbox_stopped_tracking_job(job_id)
@safe_loop
def routine(self):
self.assert_environment_prerequisite()
# clean up finished jobs
self.stop_tracking_terminated_jobs()
# get job actions
try:
job_actions = self.jrds_client.get_job_actions(self.sandbox_id)
except JrdsSandboxTerminated:
tracer.log_sandbox_jrds_closure_request()
global routine_loop
routine_loop = False
return
for job_action in job_actions:
job_id = job_action["JobId"]
job_data = self.jrds_client.get_job_data(job_id)
job_pending_action = job_data.pending_action
job_status = job_data.job_status
# issue pending action
if job_pending_action == pendingactions.ACTIVATE_ENUM_INDEX or \
(job_pending_action is None and job_status == jobstatus.ACTIVATING_ENUM_INDEX) or \
(job_pending_action is None and job_status == jobstatus.RUNNING_ENUM_INDEX):
tracer.log_sandbox_pending_action_activate_detected(job_id, job_status, job_pending_action)
# check if the specified job is already running to prevent duplicate
if job_id in job_map:
continue
# create and start the new job
job_message_queue = Queue()
job_thread_exception_queue = Queue()
job = Job(self.sandbox_id, job_id, job_message_queue, self.jrds_client, job_thread_exception_queue)
job_map[job_id] = (job, job_message_queue, job_thread_exception_queue)
tracer.log_sandbox_started_tracking_job(job_id)
job.start()
elif job_pending_action == pendingactions.STOP_ENUM_INDEX:
tracer.log_sandbox_pending_action_stop_detected(job_id, job_status, job_pending_action)
# check if the specified job is already running before issuing pending action
if job_id not in job_map:
continue
# propagate pending action to job thread
job_map[job_id][1].put(job_pending_action)
elif job_pending_action is None:
tracer.log_sandbox_no_pending_action_detected(job_id, job_status)
else:
tracer.log_sandbox_unsupported_pending_action_detected(job_id, job_status, job_pending_action)
class Worker:
def __init__(self):
tracer.log_worker_starting(configuration.get_worker_version())
http_client_factory = HttpClientFactory(configuration.get_jrds_cert_path(), configuration.get_jrds_key_path(),
configuration.get_verify_certificates())
http_client = http_client_factory.create_http_client(sys.version_info)
self.jrds_client = JRDSClient(http_client)
self.running_sandboxes = {}
@staticmethod
def assert_environment_prerequisite():
jrds_cert_path = configuration.get_jrds_cert_path()
if util.assert_file_read_permission(jrds_cert_path) is False:
raise InvalidFilePermissionException(jrds_cert_path)
jrds_key_path = configuration.get_jrds_key_path()
if util.assert_file_read_permission(jrds_key_path) is False:
raise InvalidFilePermissionException(jrds_key_path)
worker_conf_path = configuration.get_worker_configuration_file_path()
if util.assert_file_read_permission(worker_conf_path) is False:
raise InvalidFilePermissionException(worker_conf_path)
proxy_conf_path = configuration.get_proxy_configuration_path()
if proxy_conf_path != configuration.DEFAULT_PROXY_CONFIGURATION_PATH and os.path.isfile(proxy_conf_path):
if util.assert_file_read_permission(proxy_conf_path) is False:
raise InvalidFilePermissionException(proxy_conf_path)
@staticmethod
def construct_jrds_msi_endpoint(sandbox_id):
url = configuration.get_jrds_base_uri() + "/automationAccounts/" + configuration.get_account_id() + \
"/Sandboxes/" + sandbox_id + "/metadata/identity/oauth2/token"
return url
@safe_loop
def routine(self):
self.assert_environment_prerequisite()
self.stop_tracking_terminated_sandbox()
sandbox_actions = self.jrds_client.get_sandbox_actions()
if sandbox_actions is None:
tracer.log_get_sandbox_action_returned_null_data()
return
tracer.log_debug_trace("Get sandbox action. Found " + str(len(sandbox_actions)) + " action(s).")
for action in sandbox_actions:
tracer.log_worker_sandbox_action_found(len(sandbox_actions))
sandbox_id = str(action["SandboxId"])
# prevent duplicate sandbox from running
if sandbox_id in self.running_sandboxes:
continue
# create sandboxes folder if needed
sandbox_working_dir = os.path.join(configuration.get_working_directory_path(), sandboxes_root_folder_name,
sandbox_id)
try:
iohelper.assert_or_create_path(sandbox_working_dir)
except OSError, exception:
tracer.log_worker_failed_to_create_sandbox_root_folder(sandbox_id, exception)
raise SystemExit("Sandbox folder creation failed.")
# copy current process env variable (contains configuration) and add the sanbox_id key
process_env_variables = os.environ.copy()
process_env_variables["sandbox_id"] = sandbox_id
msi_secret = str(action["MSISecret"])
if (msi_secret and msi_secret != "None"):
process_env_variables["MSI_SECRET"] = msi_secret
process_env_variables["MSI_ENDPOINT"] = self.construct_jrds_msi_endpoint(sandbox_id)
cmd = ["python", os.path.join(configuration.get_source_directory_path(), "sandbox.py"),
configuration.get_worker_configuration_file_path()]
tracer.log_worker_starting_sandbox(sandbox_id)
sandbox_process = subprocessfactory.create_subprocess(cmd=cmd,
env=process_env_variables,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=sandbox_working_dir)
self.running_sandboxes[sandbox_id] = sandbox_process
tracer.log_worker_started_tracking_sandbox(sandbox_id)
self.monitor_sandbox_process_outputs(sandbox_id, sandbox_process)
tracer.log_worker_sandbox_process_started(sandbox_id, str(sandbox_process.pid))
class Worker(object):
def __init__(self):
tracer.log_worker_starting(configuration.get_worker_version())
http_client_factory = HttpClientFactory(
configuration.get_jrds_cert_path(),
configuration.get_jrds_key_path(),
configuration.get_verify_certificates())
http_client = http_client_factory.create_http_client(sys.version_info)
self.jrds_client = JRDSClient(http_client)
self.running_sandboxes = {}
@staticmethod
def assert_environment_prerequisite():
jrds_cert_path = configuration.get_jrds_cert_path()
if util.assert_file_read_permission(jrds_cert_path) is False:
raise InvalidFilePermissionException(jrds_cert_path)
jrds_key_path = configuration.get_jrds_key_path()
if util.assert_file_read_permission(jrds_key_path) is False:
raise InvalidFilePermissionException(jrds_key_path)
worker_conf_path = configuration.get_worker_configuration_file_path()
if util.assert_file_read_permission(worker_conf_path) is False:
raise InvalidFilePermissionException(worker_conf_path)
proxy_conf_path = configuration.get_proxy_configuration_path()
if proxy_conf_path != configuration.DEFAULT_PROXY_CONFIGURATION_PATH and os.path.isfile(
proxy_conf_path):
if util.assert_file_read_permission(proxy_conf_path) is False:
raise InvalidFilePermissionException(proxy_conf_path)
@safe_loop
def routine(self):
self.assert_environment_prerequisite()
self.stop_tracking_terminated_sandbox()
sandbox_actions = self.jrds_client.get_sandbox_actions()
if sandbox_actions is None:
tracer.log_get_sandbox_action_returned_null_data()
return
tracer.log_debug_trace("Get sandbox action. Found " +
str(len(sandbox_actions)) + " action(s).")
for action in sandbox_actions:
tracer.log_worker_sandbox_action_found(len(sandbox_actions))
sandbox_id = str(action["SandboxId"])
# prevent duplicate sandbox from running
if sandbox_id in self.running_sandboxes:
continue
# create sandboxes folder if needed
sandbox_working_dir = os.path.join(
configuration.get_working_directory_path(),
sandboxes_root_folder_name, sandbox_id)
try:
iohelper.assert_or_create_path(sandbox_working_dir)
except OSError as exception:
tracer.log_worker_failed_to_create_sandbox_root_folder(
sandbox_id, exception)
raise SystemExit("Sandbox folder creation failed.")
# copy current process env variable (contains configuration) and add the sanbox_id key
process_env_variables = os.environ.copy()
process_env_variables["sandbox_id"] = sandbox_id
python_to_be_used = util.get_python_to_be_used()
print("Using %s" % (python_to_be_used))
cmd = [
python_to_be_used,
os.path.join(configuration.get_source_directory_path(),
"sandbox.py"),
configuration.get_worker_configuration_file_path()
]
tracer.log_worker_starting_sandbox(sandbox_id)
sandbox_process = subprocessfactory.create_subprocess(
cmd=cmd,
env=process_env_variables,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=sandbox_working_dir)
self.running_sandboxes[sandbox_id] = sandbox_process
tracer.log_worker_started_tracking_sandbox(sandbox_id)
self.monitor_sandbox_process_outputs(sandbox_id, sandbox_process)
tracer.log_worker_sandbox_process_started(sandbox_id,
str(sandbox_process.pid))
@background_thread
def monitor_sandbox_process_outputs(self, sandbox_id, process):
while process.poll() is None:
output = process.stdout.readline().decode().replace("\n", "")
if output == '':
continue
if output != '':
tracer.log_sandbox_stdout(output)
if process.poll() != 0:
full_error_output = ""
while True:
error_output = process.stderr.readline().decode()
if error_output is None or error_output == '':
break
full_error_output += error_output
tracer.log_worker_sandbox_process_crashed(sandbox_id, process.pid,
process.poll(),
full_error_output)
tracer.log_worker_sandbox_process_exited(sandbox_id, str(process.pid),
process.poll())
# cleanup sandbox directory
sandbox_working_dir = os.path.join(
configuration.get_working_directory_path(),
sandboxes_root_folder_name, sandbox_id)
shutil.rmtree(sandbox_working_dir, ignore_errors=True)
@background_thread
def telemetry_routine(self):
while True:
tracer.log_worker_general_telemetry(
configuration.get_worker_version(),
configuration.get_worker_type(),
linuxutil.get_current_username(), linuxutil.get_oms_agent_id())
tracer.log_worker_python_telemetry(platform.python_version(),
platform.python_build(),
platform.python_compiler())
tracer.log_worker_system_telemetry(platform.system(),
platform.node(),
platform.version(),
platform.machine(),
platform.processor())
try:
distributor_id, description, release, codename = linuxutil.get_lsb_release(
)
tracer.log_worker_lsb_release_telemetry(
distributor_id, description, release, codename)
except:
pass
# sleep for 6 hours, this allows us to gather daily telemetry
time.sleep(60 * 60 * 6)
def stop_tracking_terminated_sandbox(self):
terminated_sandbox_ids = []
# detect terminated sandboxes
for sandbox_id, sandbox_process in list(
self.running_sandboxes.items()):
if sandbox_process.poll() is not None:
terminated_sandbox_ids.append(sandbox_id)
# clean-up terminated sandboxes
for sandbox_id in terminated_sandbox_ids:
removal = self.running_sandboxes.pop(sandbox_id, None)
if removal is not None:
tracer.log_worker_stopped_tracking_sandbox(sandbox_id)
class Worker:
def __init__(self):
tracer.log_worker_starting(configuration.get_worker_version())
http_client_factory = HttpClientFactory(
configuration.get_jrds_cert_path(),
configuration.get_jrds_key_path())
http_client = http_client_factory.create_http_client(
sys.version_info, configuration.get_verify_certificates())
self.jrds_client = JRDSClient(http_client)
self.running_sandboxes = {}
@safe_loop
def routine(self):
self.stop_tracking_terminated_sandbox()
sandbox_actions = self.jrds_client.get_sandbox_actions()
tracer.log_debug_trace("Get sandbox action. Found " +
str(len(sandbox_actions)) + " action(s).")
for action in sandbox_actions:
tracer.log_worker_sandbox_action_found(len(sandbox_actions))
sandbox_id = str(action["SandboxId"])
# prevent duplicate sandbox from running
if sandbox_id in self.running_sandboxes:
return
# create sandboxes folder if needed
sandboxes_base_path = "sandboxes"
sandbox_working_dir = os.path.join(
configuration.get_working_directory_path(),
sandboxes_base_path, sandbox_id)
try:
iohelper.assert_or_create_path(sandbox_working_dir)
except OSError:
tracer.log_debug_trace("Failed to create sandbox folder.")
pass
cmd = [
"python",
os.path.join(configuration.get_source_directory_path(),
"sandbox.py")
]
process_env_variables = {"sandbox_id": sandbox_id}
sandbox_process = subprocessfactory.create_subprocess(
cmd=cmd,
env=process_env_variables,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=sandbox_working_dir)
self.running_sandboxes[sandbox_id] = sandbox_process
self.monitor_sandbox_process_outputs(sandbox_id, sandbox_process)
tracer.log_worker_starting_sandbox(sandbox_id,
str(sandbox_process.pid))
@background_thread
def monitor_sandbox_process_outputs(self, sandbox_id, process):
while process.poll() is None:
output = process.stdout.readline().replace("\n", "")
if output == '':
continue
if output != '':
tracer.log_sandbox_stdout(output)
if process.poll() != 0:
full_error_output = ""
while True:
error_output = process.stderr.readline()
if error_output is None or error_output == '':
break
full_error_output += error_output
tracer.log_debug_trace("Sandbox crashed : " +
str(full_error_output))
tracer.log_worker_sandbox_process_exited(sandbox_id, str(process.pid),
process.poll())
def stop_tracking_terminated_sandbox(self):
terminated_sandbox_ids = []
# detect terminated sandboxes
for sandbox_id, sandbox_process in self.running_sandboxes.items():
if sandbox_process.poll() is not None:
terminated_sandbox_ids.append(sandbox_id)
# clean-up terminated sandboxes
for sandbox_id in terminated_sandbox_ids:
removal = self.running_sandboxes.pop(sandbox_id, None)
if removal is not None:
tracer.log_debug_trace("Worker stopped tracking sandbox : " +
str(sandbox_id))
class Sandbox:
def __init__(self):
self.sandbox_id = os.environ["sandbox_id"]
tracer.log_sandbox_starting(
sandbox_id=self.sandbox_id,
pid=os.getpid(),
worker_type=configuration.get_worker_type())
tracer.log_sandbox_configuration(
sandbox_id=self.sandbox_id,
enforce_runbook_signature_validation=configuration.
get_enforce_runbook_signature_validation(),
gpg_public_keyring_paths=configuration.
get_gpg_public_keyrings_path(),
working_directory=os.getcwd())
http_client_factory = HttpClientFactory(
configuration.get_jrds_cert_path(),
configuration.get_jrds_key_path(),
configuration.get_verify_certificates())
http_client = http_client_factory.create_http_client(sys.version_info)
self.jrds_client = JRDSClient(http_client)
@staticmethod
def stop_tracking_terminated_jobs():
terminated_job_ids = []
# clean up finished jobs
for job_id, job_tuple in job_map.items():
if job_tuple[0].isAlive() is False:
try:
job_tuple[2].get(block=False)
raise SandboxRuntimeException()
except Empty:
pass
terminated_job_ids.append(job_id)
for job_id in terminated_job_ids:
removal = job_map.pop(job_id, None)
if removal is not None:
tracer.log_sandbox_stopped_tracking_job(job_id)
@safe_loop
def routine(self):
# die if pre-reqs are not met
if os.path.exists(configuration.get_jrds_cert_path()) is False or \
os.path.exists(configuration.get_jrds_key_path()) is False or \
os.path.exists(configuration.get_worker_configuration_file_path()) is False:
raise SystemExit()
# clean up finished jobs
self.stop_tracking_terminated_jobs()
# get job actions
try:
job_actions = self.jrds_client.get_job_actions(self.sandbox_id)
except JrdsSandboxTerminated:
tracer.log_sandbox_jrds_closure_request()
global routine_loop
routine_loop = False
return
for job_action in job_actions:
job_id = job_action["JobId"]
job_data = self.jrds_client.get_job_data(job_id)
job_pending_action = job_data.pending_action
job_status = job_data.job_status
# issue pending action
if job_pending_action == pendingactions.ACTIVATE_ENUM_INDEX or \
(job_pending_action is None and job_status == jobstatus.ACTIVATING_ENUM_INDEX) or \
(job_pending_action is None and job_status == jobstatus.RUNNING_ENUM_INDEX):
tracer.log_sandbox_pending_action_activate_detected(
job_id, job_status, job_pending_action)
# check if the specified job is already running to prevent duplicate
if job_id in job_map:
continue
# create and start the new job
job_message_queue = Queue()
job_thread_exception_queue = Queue()
job = Job(self.sandbox_id, job_id, job_message_queue,
self.jrds_client, job_thread_exception_queue)
job_map[job_id] = (job, job_message_queue,
job_thread_exception_queue)
tracer.log_sandbox_started_tracking_job(job_id)
job.start()
elif job_pending_action == pendingactions.STOP_ENUM_INDEX:
tracer.log_sandbox_pending_action_stop_detected(
job_id, job_status, job_pending_action)
# check if the specified job is already running before issuing pending action
if job_id not in job_map:
continue
# propagate pending action to job thread
job_map[job_id][1].put(job_pending_action)
elif job_pending_action is None:
tracer.log_sandbox_no_pending_action_detected(
job_id, job_status)
else:
tracer.log_sandbox_unsupported_pending_action_detected(
job_id, job_status, job_pending_action)
log_warning("Retrieval of automation certificate failed for " +
str(name) + ". Retrying ..")
certificate = jrds_client.get_certificate_asset(name)
if certificate is not None:
return binascii.a2b_base64(certificate[KEY_VALUE])
time_to_wait = 3 * (2**tries)
if time_to_wait > 60:
time_to_wait = 60
time.sleep(time_to_wait)
tries = tries + 1
# log not able to retrieve data
log_error("Retrieval of automation certificate failed for " + str(name) +
".")
def get_automation_certificate(name):
certificate = jrds_client.get_certificate_asset(name)
if certificate is None:
return get_automation_certificate_with_retry(name, 2)
else:
return binascii.a2b_base64(certificate[KEY_VALUE])
configuration.set_config({configuration.COMPONENT: "assets"})
json = serializerfactory.get_serializer(sys.version_info)
http_client_factory = HttpClientFactory(
configuration.get_jrds_cert_path(), configuration.get_jrds_key_path(),
configuration.get_verify_certificates())
http_client = http_client_factory.create_http_client(sys.version_info)
jrds_client = JRDSClient(http_client)
class Sandbox:
def __init__(self):
self.sandbox_id = os.environ["sandbox_id"]
tracer.log_sandbox_starting(self.sandbox_id, os.getpid())
http_client_factory = HttpClientFactory(configuration.get_jrds_cert_path(), configuration.get_jrds_key_path())
http_client = http_client_factory.create_http_client(sys.version_info, configuration.get_verify_certificates())
self.jrds_client = JRDSClient(http_client)
@staticmethod
def stop_tracking_terminated_jobs():
terminated_job_ids = []
# clean up finished jobs
for job_id, job_tuple in job_map.items():
if job_tuple[0].isAlive() is False:
try:
job_tuple[2].get(block=False)
raise SandboxRuntimeException()
except Empty:
pass
terminated_job_ids.append(job_id)
for job_id in terminated_job_ids:
removal = job_map.pop(job_id, None)
if removal is not None:
tracer.log_debug_trace("Sandbox stopped tracking job : " + str(job_id))
@safe_loop
def routine(self):
# clean up finished jobs
self.stop_tracking_terminated_jobs()
# get job actions
try:
job_actions = self.jrds_client.get_job_actions(self.sandbox_id)
except JrdsSandboxTerminated:
tracer.log_debug_trace("Terminating sandbox.")
global routine_loop
routine_loop = False
return
for job_action in job_actions:
job_id = job_action["JobId"]
job_data = self.jrds_client.get_job_data(job_id)
job_pending_action = job_data["pendingAction"]
job_status = job_data["jobStatus"]
# issue pending action
if job_pending_action == pendingactions.ACTIVATE_ENUM_INDEX or \
(job_pending_action is None and job_status == jobstatus.ACTIVATING_ENUM_INDEX) or \
(job_pending_action is None and job_status == jobstatus.RUNNING_ENUM_INDEX):
# check if the specified job is already running to prevent duplicate
if job_id in job_map:
continue
# create and start the new job
job_message_queue = Queue()
job_thread_exception_queue = Queue()
job = Job(self.sandbox_id, job_id, job_message_queue, self.jrds_client, job_thread_exception_queue)
job_map[job_id] = (job, job_message_queue, job_thread_exception_queue)
job.start()
tracer.log_debug_trace("Pending action activate detected.[pendingaction=" +
str(job_status) + "]")
elif job_pending_action == pendingactions.STOP_ENUM_INDEX:
# check if the specified job is already running before issuing pending action
if job_id not in job_map:
continue
# propagate pending action to job thread
job_map[job_id][1].put(job_pending_action)
tracer.log_debug_trace("Pending action detected")
elif job_pending_action is None:
tracer.log_debug_trace("No pending action detected")
else:
tracer.log_debug_trace("Unsupported pending action / job action")