def check_docker_dependencies(dependencies):
"""Checks docker dependencies.
Args:
dependencies(dict): dictionary of dependencies to check for.
Raises:
TurbiniaException: If dependency is not met.
"""
#TODO(wyassine): may run into issues down the line when a docker image
# does not have bash or which installed. (no linux fs layer).
log.info('Performing docker dependency check.')
job_names = list(job_manager.JobsManager.GetJobNames())
images = docker_manager.DockerManager().list_images(
return_filter='short_id')
# Iterate through list of jobs
for job, values in dependencies.items():
if job not in job_names:
log.warning(
'The job {0:s} was not found or has been disabled. Skipping '
'dependency check...'.format(job))
continue
docker_image = values.get('docker_image')
# short id only pulls the first 10 characters of image id.
if docker_image and len(docker_image) > 10:
docker_image = docker_image[0:10]
if docker_image in images:
for program in values['programs']:
cmd = 'type {0:s}'.format(program)
stdout, stderr, ret = docker_manager.ContainerManager(
values['docker_image']).execute_container(cmd, shell=True)
if ret != 0:
raise TurbiniaException(
'Job dependency {0:s} not found for job {1:s}. Please install '
'the dependency for the container or disable the job.'.
format(program, job))
job_manager.JobsManager.RegisterDockerImage(
job, values['docker_image'])
elif docker_image:
raise TurbiniaException(
'Docker image {0:s} was not found for the job {1:s}. Please '
'update the config with the correct image id'.format(
values['docker_image'], job))
def execute(
self, cmd, result, save_files=None, log_files=None, new_evidence=None,
close=False, shell=False, stderr_file=None, stdout_file=None,
success_codes=None):
"""Executes a given binary and saves output.
Args:
cmd (list|string): Command arguments to run
result (TurbiniaTaskResult): The result object to put data into.
save_files (list): A list of files to save (files referenced by Evidence
objects are automatically saved, so no need to include them).
log_files (list): A list of files to save even if execution fails.
new_evidence (list): These are new evidence objects created by the task.
If the task is successful, they will be added to the result.
close (bool): Whether to close out the result.
shell (bool): Whether the cmd is in the form of a string or a list.
success_codes (list(int)): Which return codes are considered successful.
stderr_file (str): Path to location to save stderr.
stdout_file (str): Path to location to save stdout.
Returns:
Tuple of the return code, and the TurbiniaTaskResult object
"""
# Avoid circular dependency.
from turbinia.jobs import manager as job_manager
save_files = save_files if save_files else []
log_files = log_files if log_files else []
new_evidence = new_evidence if new_evidence else []
success_codes = success_codes if success_codes else [0]
stdout = None
stderr = None
# Get timeout value.
timeout_limit = job_manager.JobsManager.GetTimeoutValue(self.job_name)
# Execute the job via docker.
docker_image = job_manager.JobsManager.GetDockerImage(self.job_name)
if docker_image:
ro_paths = [
result.input_evidence.local_path, result.input_evidence.source_path,
result.input_evidence.device_path, result.input_evidence.mount_path
]
rw_paths = [self.output_dir, self.tmp_dir]
container_manager = docker_manager.ContainerManager(docker_image)
stdout, stderr, ret = container_manager.execute_container(
cmd, shell, ro_paths=ro_paths, rw_paths=rw_paths,
timeout_limit=timeout_limit)
# Execute the job on the host system.
else:
try:
if shell:
proc = subprocess.Popen(
cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
proc.wait(timeout_limit)
else:
proc = subprocess.Popen(
cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
proc.wait(timeout_limit)
except subprocess.TimeoutExpired as exception:
# Log error and close result.
message = (
'Execution of [{0!s}] failed due to job timeout of '
'{1:d} seconds has been reached.'.format(cmd, timeout_limit))
result.log(message)
result.close(self, success=False, status=message)
# Increase timeout metric and raise exception
turbinia_worker_tasks_timeout_total.inc()
raise TurbiniaException(message)
stdout, stderr = proc.communicate()
ret = proc.returncode
result.error['stdout'] = str(stdout)
result.error['stderr'] = str(stderr)
if stderr_file and not stderr:
result.log(
'Attempting to save stderr to {0:s}, but no stderr found during '
'execution'.format(stderr_file))
elif stderr:
if not stderr_file:
_, stderr_file = tempfile.mkstemp(
suffix='.txt', prefix='stderr-', dir=self.output_dir)
result.log(
'Writing stderr to {0:s}'.format(stderr_file), level=logging.DEBUG)
with open(stderr_file, 'wb') as fh:
fh.write(stderr)
log_files.append(stderr_file)
if stdout_file and not stdout:
result.log(
'Attempting to save stdout to {0:s}, but no stdout found during '
'execution'.format(stdout_file))
elif stdout:
if not stdout_file:
_, stdout_file = tempfile.mkstemp(
suffix='.txt', prefix='stdout-', dir=self.output_dir)
result.log(
'Writing stdout to {0:s}'.format(stdout_file), level=logging.DEBUG)
with open(stdout_file, 'wb') as fh:
fh.write(stdout)
log_files.append(stdout_file)
log_files = list(set(log_files))
for file_ in log_files:
if not os.path.exists(file_):
result.log(
'Log file {0:s} does not exist to save'.format(file_),
level=logging.DEBUG)
continue
if os.path.getsize(file_) == 0:
result.log(
'Log file {0:s} is empty. Not saving'.format(file_),
level=logging.DEBUG)
continue
result.log('Output log file found at {0:s}'.format(file_))
if not self.run_local:
self.output_manager.save_local_file(file_, result)
if ret not in success_codes:
message = 'Execution of [{0!s}] failed with status {1:d}'.format(cmd, ret)
result.log(message)
if close:
result.close(self, success=False, status=message)
else:
result.log('Execution of [{0!s}] succeeded'.format(cmd))
for file_ in save_files:
if os.path.getsize(file_) == 0:
result.log(
'Output file {0:s} is empty. Not saving'.format(file_),
level=logging.DEBUG)
continue
result.log('Output save file at {0:s}'.format(file_))
if not self.run_local:
self.output_manager.save_local_file(file_, result)
for evidence in new_evidence:
# If the local path is set in the Evidence, we check to make sure that
# the path exists and is not empty before adding it.
if evidence.source_path and not os.path.exists(evidence.source_path):
message = (
'Evidence {0:s} source_path {1:s} does not exist. Not returning '
'empty Evidence.'.format(evidence.name, evidence.source_path))
result.log(message, level=logging.WARN)
elif (evidence.source_path and os.path.exists(evidence.source_path) and
os.path.getsize(evidence.source_path) == 0):
message = (
'Evidence {0:s} source_path {1:s} is empty. Not returning '
'empty new Evidence.'.format(evidence.name, evidence.source_path))
result.log(message, level=logging.WARN)
else:
result.add_evidence(evidence, self._evidence_config)
if close:
result.close(self, success=True)
return ret, result
def execute(self,
cmd,
result,
save_files=None,
log_files=None,
new_evidence=None,
close=False,
shell=False,
success_codes=None):
"""Executes a given binary and saves output.
Args:
cmd (list|string): Command arguments to run
result (TurbiniaTaskResult): The result object to put data into.
save_files (list): A list of files to save (files referenced by Evidence
objects are automatically saved, so no need to include them).
log_files (list): A list of files to save even if execution fails.
new_evidence (list): These are new evidence objects created by the task.
If the task is successful, they will be added to the result.
close (bool): Whether to close out the result.
shell (bool): Whether the cmd is in the form of a string or a list.
success_codes (list(int)): Which return codes are considered successful.
Returns:
Tuple of the return code, and the TurbiniaTaskResult object
"""
# Avoid circular dependency.
from turbinia.jobs import manager as job_manager
save_files = save_files if save_files else []
log_files = log_files if log_files else []
new_evidence = new_evidence if new_evidence else []
success_codes = success_codes if success_codes else [0]
# Execute the job via docker.
docker_image = job_manager.JobsManager.GetDockerImage(self.job_name)
if docker_image:
ro_paths = [
result.input_evidence.local_path,
result.input_evidence.source_path,
result.input_evidence.device_path,
result.input_evidence.mount_path
]
rw_paths = [self.output_dir, self.tmp_dir]
container_manager = docker_manager.ContainerManager(docker_image)
stdout, stderr, ret = container_manager.execute_container(
cmd, shell, ro_paths=ro_paths, rw_paths=rw_paths)
# Execute the job on the host system.
else:
if shell:
proc = subprocess.Popen(cmd, shell=True)
else:
proc = subprocess.Popen(cmd)
stdout, stderr = proc.communicate()
ret = proc.returncode
result.error['stdout'] = stdout
result.error['stderr'] = stderr
for file_ in log_files:
if not os.path.exists(file_):
result.log(
'Log file {0:s} does not exist to save'.format(file_),
level=logging.DEBUG)
continue
if os.path.getsize(file_) == 0:
result.log('Log file {0:s} is empty. Not saving'.format(file_),
level=logging.DEBUG)
continue
result.log('Output log file found at {0:s}'.format(file_))
if not self.run_local:
self.output_manager.save_local_file(file_, result)
if ret not in success_codes:
message = 'Execution of [{0!s}] failed with status {1:d}'.format(
cmd, ret)
result.log(message)
if close:
result.close(self, success=False, status=message)
else:
result.log('Execution of [{0!s}] succeeded'.format(cmd))
for file_ in save_files:
if os.path.getsize(file_) == 0:
result.log(
'Output file {0:s} is empty. Not saving'.format(file_),
level=logging.DEBUG)
continue
result.log('Output save file at {0:s}'.format(file_))
if not self.run_local:
self.output_manager.save_local_file(file_, result)
for evidence in new_evidence:
# If the local path is set in the Evidence, we check to make sure that
# the path exists and is not empty before adding it.
if evidence.source_path and not os.path.exists(
evidence.source_path):
message = (
'Evidence {0:s} source_path {1:s} does not exist. Not returning '
'empty Evidence.'.format(evidence.name,
evidence.source_path))
result.log(message, level=logging.WARN)
elif (evidence.source_path
and os.path.exists(evidence.source_path)
and os.path.getsize(evidence.source_path) == 0):
message = (
'Evidence {0:s} source_path {1:s} is empty. Not returning '
'empty new Evidence.'.format(evidence.name,
evidence.source_path))
result.log(message, level=logging.WARN)
else:
result.add_evidence(evidence, self._evidence_config)
if close:
result.close(self, success=True)
return ret, result
def setUp(self, mock_get, mock_docker):
self.test_img = '1234'
mock_docker.from_env.return_value = docker.client.DockerClient
mock_get.return_value = self.test_img
self.container_mgr = docker_manager.ContainerManager(self.test_img)
