def get_task_manager():
"""Return task manager object based on config.
Returns
Initialized TaskManager object.
"""
config.LoadConfig()
if config.TASK_MANAGER == u'PSQ':
return PSQTaskManager()
else:
msg = u'Task Manager type "{0:s}" not implemented'.format(
config.TASK_MANAGER)
raise turbinia.TurbiniaException(msg)
def get_state_manager():
"""Return state manager object based on config.
Returns
Initialized StateManager object.
"""
config.LoadConfig()
if config.STATE_MANAGER == 'Datastore':
return DatastoreStateManager()
else:
msg = 'State Manager type "{0:s}" not implemented'.format(
config.STATE_MANAGER)
raise TurbiniaException(msg)
def run(self, evidence, result):
"""Extracts artifacts using Plaso image_export.py.
Args:
evidence: evidence to be processed.
result: A TurbiniaTaskResult object to place task results into.
Returns:
TurbiniaTaskResult object.
"""
config.LoadConfig()
export_directory = os.path.join(self.output_dir, 'export')
image_export_log = os.path.join(self.output_dir,
'{0:s}.log'.format(self.id))
cmd = [
'image_export.py',
'--logfile',
image_export_log,
'-w',
export_directory,
'--artifact_filters',
self.artifact_name,
]
if config.DEBUG_TASKS:
cmd.append('-d')
# Path to the source image/directory.
cmd.append(evidence.local_path)
result.log('Running image_export as [{0:s}]'.format(' '.join(cmd)))
ret, _ = self.execute(cmd, result, save_files=[image_export_log])
if ret:
result.close(self, False, 'image_export.py failed.')
return result
for dirpath, _, filenames in os.walk(export_directory):
for filename in filenames:
exported_artifact = ExportedFileArtifact(
artifact_name=self.artifact_name)
exported_artifact.local_path = os.path.join(dirpath, filename)
result.log('Adding artifact {0:s}'.format(filename))
result.add_evidence(exported_artifact, evidence.config)
result.close(
self, True, 'Extracted {0:d} new {1:s} artifacts'.format(
len(result.evidence), self.artifact_name))
return result
def setup():
"""Set up logging parameters.
This will also set the root logger, which is the default logger when a named
logger is not specified. We currently use 'turbinia' as the named logger,
however some external modules that are called by Turbinia can use the root
logger, so we want to be able to optionally configure that as well.
"""
# TODO(aarontp): Add a config option to set the log level
config.LoadConfig()
logger = logging.getLogger('turbinia')
# Eliminate double logging from root logger
logger.propagate = False
need_file_handler = True
need_stream_handler = True
# We only need a handler if one of that type doesn't exist already
if logger.handlers:
for handler in logger.handlers:
# Want to do strict type-checking here because is instance will include
# subclasses and so won't distinguish between StreamHandlers and
# FileHandlers.
# pylint: disable=unidiomatic-typecheck
if type(handler) == logging.FileHandler:
need_file_handler = False
# pylint: disable=unidiomatic-typecheck
if type(handler) == logging.StreamHandler:
need_stream_handler = False
file_handler = logging.FileHandler(config.LOG_FILE)
formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s')
file_handler.setFormatter(formatter)
file_handler.setLevel(logging.DEBUG)
if need_file_handler:
logger.addHandler(file_handler)
console_handler = logging.StreamHandler()
formatter = logging.Formatter('[%(levelname)s] %(message)s')
console_handler.setFormatter(formatter)
if need_stream_handler:
logger.addHandler(console_handler)
# Configure the root logger to use exactly our handlers because other modules
# like PSQ use this, and we want to see log messages from it when executing
# from CLI.
root_log = logging.getLogger()
for handler in root_log.handlers:
root_log.removeHandler(handler)
root_log.addHandler(console_handler)
root_log.addHandler(file_handler)
def PreprocessMountDisk(loopdevice_path, partition_number):
"""Locally mounts disk in an instance.
Args:
loopdevice_path(str): The path to the block device to mount.
partition_number(int): The partition number.
Raises:
TurbiniaException: if the mount command failed to run.
Returns:
str: the path to the mounted filesystem.
"""
config.LoadConfig()
mount_prefix = config.MOUNT_DIR_PREFIX
if os.path.exists(mount_prefix) and not os.path.isdir(mount_prefix):
raise TurbiniaException(
'Mount dir {0:s} exists, but is not a directory'.format(mount_prefix))
if not os.path.exists(mount_prefix):
log.info('Creating local mount parent directory {0:s}'.format(mount_prefix))
try:
os.makedirs(mount_prefix)
except OSError as e:
raise TurbiniaException(
'Could not create mount directory {0:s}: {1!s}'.format(
mount_prefix, e))
mount_path = tempfile.mkdtemp(prefix='turbinia', dir=mount_prefix)
if not partition_number:
# The first partition loop-device made by losetup is loopXp1
partition_number = 1
path_to_partition = '{0:s}p{1:d}'.format(loopdevice_path, partition_number)
if not os.path.exists(path_to_partition):
log.info(
'Could not find {0:s}, trying {1:s}'.format(
path_to_partition, loopdevice_path))
# Else, the partition's block device is actually /dev/loopX
path_to_partition = loopdevice_path
mount_cmd = ['sudo', 'mount', path_to_partition, mount_path]
log.info('Running: {0:s}'.format(' '.join(mount_cmd)))
try:
subprocess.check_call(mount_cmd)
except subprocess.CalledProcessError as e:
raise TurbiniaException('Could not mount directory {0!s}'.format(e))
return mount_path
def PreprocessMountPartition(partition_path, filesystem_type):
"""Locally mounts disk partition in an instance.
Args:
partition_path(str): A path to a partition block device
filesystem_type(str): Filesystem of the partition to be mounted
Raises:
TurbiniaException: if the mount command failed to run.
Returns:
str: the path to the mounted filesystem.
"""
config.LoadConfig()
mount_prefix = config.MOUNT_DIR_PREFIX
if not os.path.exists(partition_path):
raise TurbiniaException(
'Could not mount partition {0:s}, the path does not exist'.format(
partition_path))
if os.path.exists(mount_prefix) and not os.path.isdir(mount_prefix):
raise TurbiniaException(
'Mount dir {0:s} exists, but is not a directory'.format(mount_prefix))
if not os.path.exists(mount_prefix):
log.info('Creating local mount parent directory {0:s}'.format(mount_prefix))
try:
os.makedirs(mount_prefix)
except OSError as e:
raise TurbiniaException(
'Could not create mount directory {0:s}: {1!s}'.format(
mount_prefix, e))
mount_path = tempfile.mkdtemp(prefix='turbinia', dir=mount_prefix)
mount_cmd = ['sudo', 'mount', '-o', 'ro']
if filesystem_type == 'EXT':
# This is in case the underlying filesystem is dirty, as we want to mount
# everything read-only.
mount_cmd.extend(['-o', 'noload'])
elif filesystem_type == 'XFS':
mount_cmd.extend(['-o', 'norecovery'])
mount_cmd.extend([partition_path, mount_path])
log.info('Running: {0:s}'.format(' '.join(mount_cmd)))
try:
subprocess.check_call(mount_cmd)
except subprocess.CalledProcessError as e:
raise TurbiniaException('Could not mount directory {0!s}'.format(e))
return mount_path
def __init__(self, jobs_denylist=None, jobs_allowlist=None):
"""Initialization for PSQ Worker.
Args:
jobs_denylist (Optional[list[str]]): Jobs we will exclude from running
jobs_allowlist (Optional[list[str]]): The only Jobs we will include to run
"""
config.LoadConfig()
psq_publisher = pubsub.PublisherClient()
psq_subscriber = pubsub.SubscriberClient()
datastore_client = datastore.Client(project=config.TURBINIA_PROJECT)
try:
self.psq = psq.Queue(
psq_publisher,
psq_subscriber,
config.TURBINIA_PROJECT,
name=config.PSQ_TOPIC,
storage=psq.DatastoreStorage(datastore_client))
except exceptions.GoogleCloudError as e:
msg = 'Error creating PSQ Queue: {0:s}'.format(str(e))
log.error(msg)
raise TurbiniaException(msg)
# Deregister jobs from denylist/allowlist.
job_manager.JobsManager.DeregisterJobs(jobs_denylist, jobs_allowlist)
disabled_jobs = list(
config.DISABLED_JOBS) if config.DISABLED_JOBS else []
disabled_jobs = [j.lower() for j in disabled_jobs]
# Only actually disable jobs that have not been allowlisted.
if jobs_allowlist:
disabled_jobs = list(set(disabled_jobs) - set(jobs_allowlist))
if disabled_jobs:
log.info(
'Disabling non-allowlisted jobs configured to be disabled in the '
'config file: {0:s}'.format(', '.join(disabled_jobs)))
job_manager.JobsManager.DeregisterJobs(jobs_denylist=disabled_jobs)
# Check for valid dependencies/directories.
dependencies = config.ParseDependencies()
if config.DOCKER_ENABLED:
check_docker_dependencies(dependencies)
check_system_dependencies(dependencies)
check_directory(config.MOUNT_DIR_PREFIX)
check_directory(config.OUTPUT_DIR)
check_directory(config.TMP_DIR)
jobs = job_manager.JobsManager.GetJobNames()
log.info('Dependency check complete. The following jobs are enabled '
'for this worker: {0:s}'.format(','.join(jobs)))
log.info('Starting PSQ listener on queue {0:s}'.format(self.psq.name))
self.worker = psq.Worker(queue=self.psq)
def setup_publisher(self):
"""Set up the pubsub publisher."""
config.LoadConfig()
self.publisher = pubsub.PublisherClient()
self.topic_path = self.publisher.topic_path(config.PROJECT,
self.topic_name)
try:
log.debug('Trying to create pubsub topic {0:s}'.format(
self.topic_path))
self.publisher.create_topic(self.topic_path)
except exceptions.Conflict:
log.debug('PubSub topic {0:s} already exists.'.format(
self.topic_path))
log.debug('Setup PubSub publisher at {0:s}'.format(self.topic_path))
def run(self, evidence, result):
"""Task that process data with Plaso.
Args:
evidence: Path to data to process.
result: A TurbiniaTaskResult object to place task results into.
Returns:
TurbiniaTaskResult object.
"""
config.LoadConfig()
plaso_evidence = PlasoFile()
# Write plaso file into tmp_dir because sqlite has issues with some shared
# filesystems (e.g NFS).
plaso_file = os.path.join(self.tmp_dir, '{0:s}.plaso'.format(self.id))
plaso_evidence.local_path = plaso_file
plaso_log = os.path.join(self.output_dir, '{0:s}.log'.format(self.id))
# TODO(aarontp): Move these flags into a recipe
cmd = (
'log2timeline.py --status_view none --hashers all '
'--partition all --vss_stores all').split()
if config.DEBUG_TASKS:
cmd.append('-d')
if isinstance(evidence, (APFSEncryptedDisk, BitlockerDisk)):
if evidence.recovery_key:
cmd.extend([
'--credential', 'recovery_password:{0:s}'.format(
evidence.recovery_key)
])
elif evidence.password:
cmd.extend(['--credential', 'password:{0:s}'.format(evidence.password)])
else:
result.close(
self, False, 'No credentials were provided '
'for a bitlocker disk.')
return result
cmd.extend(['--logfile', plaso_log])
cmd.extend([plaso_file, evidence.local_path])
result.log('Running plaso as [{0:s}]'.format(' '.join(cmd)))
self.execute(
cmd, result, log_files=[plaso_log], new_evidence=[plaso_evidence],
close=True)
return result
def run(self, evidence, result):
"""Task that extracts binaries with image_export.py.
Args:
evidence (Evidence object): The evidence we will process.
result (TurbiniaTaskResult): The object to place task results into.
Returns:
TurbiniaTaskResult object.
"""
config.LoadConfig()
binary_extraction_evidence = BinaryExtraction()
binary_extraction_evidence.local_path = self.output_dir
binary_extraction_evidence.uncompressed_directory = self.output_dir
image_export_log = os.path.join(self.output_dir, 'binary_extraction.log')
self.binary_extraction_dir = os.path.join(
self.output_dir, 'extracted_binaries')
self.json_path = os.path.join(self.binary_extraction_dir, 'hashes.json')
cmd = [
'image_export.py', '--partitions', 'all', '--no_vss', '--signatures',
'elf,exe_mz', '--logfile', image_export_log
]
if config.DEBUG_TASKS or evidence.config.get('debug_tasks'):
cmd.append('-d')
cmd.extend(['-w', self.binary_extraction_dir, evidence.local_path])
result.log('Running image_export as [{0:s}]'.format(' '.join(cmd)))
self.execute(
cmd, result, log_files=[image_export_log, self.json_path],
new_evidence=[binary_extraction_evidence], close=True)
binary_cnt, hash_cnt = self.check_extraction()
result.status = (
'Extracted {0:d} hashes and {1:d} binaries from the '
'evidence.'.format(hash_cnt, binary_cnt))
if hash_cnt != binary_cnt:
result.log(
'Number of extracted binaries is not equal to the number '
'of extracted hashes. This might indicate issues with '
'image_export.py. Check binary_extraction.log for more '
'details.', logging.WARNING)
binary_extraction_evidence.compress()
return result
def test_analyse_jupyter_config(self):
"""Tests the analyze_jupyter_config method."""
config.LoadConfig()
task = jupyter.JupyterAnalysisTask()
(report, priority,
summary) = task.analyse_config(self.BAD_JUPYTER_CONFIG)
self.assertEqual(report, self.BAD_CONFIG_REPORT)
self.assertEqual(priority, 20)
self.assertEqual(summary, self.BAD_CONFIG_SUMMARY)
(report, priority,
summary) = task.analyse_config(self.GOOD_JUPYTER_CONFIG)
self.assertEqual(report, self.GOOD_CONFIG_REPORT)
def GetContainers(self, evidence):
"""Lists the containers from an input Evidence.
We use subprocess to run the DockerExplorer script, instead of using the
Python module, because we need to make sure all DockerExplorer code runs
as root.
Args:
evidence (Evidence): the input Evidence.
Returns:
a list(dict) containing information about the containers found.
Raises:
TurbiniaException: when the docker-explorer tool cannot be found or failed
to run.
"""
config.LoadConfig()
docker_dir = GetDockerPath(evidence.mount_path)
containers_info = None
# TODO(rgayon): use docker-explorer exposed constant when
# https://github.com/google/docker-explorer/issues/80 is in.
de_binary = utils.get_exe_path('de.py')
if not de_binary:
raise TurbiniaException('Cannot find de.py in path')
docker_explorer_command = ['sudo', de_binary]
if config.DEBUG_TASKS or evidence.config.get('debug_tasks'):
docker_explorer_command.append('-d')
docker_explorer_command.extend(['-r', docker_dir, 'list', 'all_containers'])
log.info('Running {0:s}'.format(' '.join(docker_explorer_command)))
try:
json_string = subprocess.check_output(docker_explorer_command).decode(
'utf-8')
except json.JSONDecodeError as e:
raise TurbiniaException(
'Error decoding JSON output from de.py: {0!s}'.format(e))
except subprocess.CalledProcessError as e:
raise TurbiniaException('de.py returned an error: {0!s}'.format(e))
containers_info = json.loads(json_string)
return containers_info
def testParseDependencies(self):
"""Tests a valid config for the ParseDependencies() method."""
smpl_depends = 'DEPENDENCIES = [{"job": "PlasoJob","programs": ["test"], \
"docker_image": "test", "timeout":30}]'
self.WriteConfig(smpl_depends)
config.LoadConfig()
smpl_out = {
'plasojob': {
'programs': ['test'],
'docker_image': 'test',
'timeout': 30
}
}
smpl_test = config.ParseDependencies()
self.assertEqual(smpl_out, smpl_test)
def test_analyze_jenkins_no_findings(self, bruteforce_mock):
"""Test that analyze_jenkins returns valid output with no findings."""
config.LoadConfig()
task = jenkins.JenkinsAnalysisTask()
bruteforce_mock.return_value = []
(report, priority,
summary) = task.analyze_jenkins(self.EXPECTED_VERSION,
self.EXPECTED_CREDENTIALS)
self.assertEqual(report, self.JENKINS_ANALYSIS_EMPTY_REPORT)
self.assertEqual(priority, 50)
self.assertEqual(
summary,
'Jenkins version 2.121.2 found with 1 credentials, but no issues '
'detected')
def test_analyze_jenkins(self, bruteforce_mock):
"""Test that analyze_jenkins returns valid output with findings."""
config.LoadConfig()
task = jenkins.JenkinsAnalysisTask()
bruteforce_mock.return_value = [
('$2a$10$DSltvO4YXZuoLuUU77R871627TEST', 'weakpassword')
]
(report, priority,
summary) = task.analyze_jenkins(self.EXPECTED_VERSION,
self.EXPECTED_CREDENTIALS)
report = report + '\n'
self.assertEqual(report, self.JENKINS_ANALYSIS_REPORT)
self.assertEqual(priority, 10)
self.assertEqual(summary, 'Jenkins analysis found potential issues')
def get_turbinia_client(run_local=False):
"""Return Turbinia client based on config.
Returns:
Initialized BaseTurbiniaClient or TurbiniaCeleryClient object.
"""
config.LoadConfig()
# pylint: disable=no-else-return
if config.TASK_MANAGER.lower() == 'psq':
return BaseTurbiniaClient(run_local=run_local)
elif config.TASK_MANAGER.lower() == 'celery':
return TurbiniaCeleryClient(run_local=run_local)
else:
msg = 'Task Manager type "{0:s}" not implemented'.format(
config.TASK_MANAGER)
raise TurbiniaException(msg)
def test_analyse_tomcat_file(self):
"""Tests the analyze_tomcat_file method."""
config.LoadConfig()
task = tomcat.TomcatAnalysisTask()
(report, priority,
summary) = task.analyse_tomcat_file(self.TOMCAT_PASSWORD_FILE)
self.assertEqual(report, self.TOMCAT_PASSWORD_FILE_REPORT)
self.assertEqual(priority, 20)
self.assertEqual(summary, self.TOMCAT_PASSWORD_FILE_REPORT_SUMMARY)
report = task.analyse_tomcat_file(self.TOMCAT_APP_DEPLOY_LOG)[0]
self.assertEqual(report, self.TOMCAT_APP_DEPLOY_LOG_REPORT)
report = task.analyse_tomcat_file(self.TOMCAT_ACCESS_LOG)[0]
self.assertEqual(report, self.TOMCAT_ACCESS_LOG_REPORT)
def get_task_manager():
"""Return task manager object based on config.
Returns
Initialized TaskManager object.
"""
config.LoadConfig()
# pylint: disable=no-else-return
if config.TASK_MANAGER.lower() == 'psq':
return PSQTaskManager()
elif config.TASK_MANAGER.lower() == 'celery':
return CeleryTaskManager()
else:
msg = 'Task Manager type "{0:s}" not implemented'.format(
config.TASK_MANAGER)
raise turbinia.TurbiniaException(msg)
def get_state_manager():
"""Return state manager object based on config.
Returns:
Initialized StateManager object.
"""
config.LoadConfig()
# pylint: disable=no-else-return
if config.STATE_MANAGER.lower() == 'datastore':
return DatastoreStateManager()
elif config.STATE_MANAGER.lower() == 'redis':
return RedisStateManager()
else:
msg = 'State Manager type "{0:s}" not implemented'.format(
config.STATE_MANAGER)
raise TurbiniaException(msg)
def log_and_report(message, trace):
"""Log an error and if enabled, send to GCP Error Reporting API.
Args:
message(str): The user defined message to log.
trace(str): The error traceback message to log.
"""
from turbinia import config
from turbinia.lib import google_cloud
log.error(message)
log.error(trace)
# If GCP Error Reporting is enabled.
config.LoadConfig()
if config.STACKDRIVER_TRACEBACK:
client = google_cloud.setup_stackdriver_traceback(config.TURBINIA_PROJECT)
client.report_exception()
def get_task_dict(self, task):
"""Creates a dict of the fields we want to persist into storage.
This combines attributes from both the Task and the TaskResult into one flat
object representing the overall state.
Args:
task: A TurbiniaTask object.
Returns:
A dict of task attributes.
"""
task_dict = {}
for attr in task.STORED_ATTRIBUTES:
if not hasattr(task, attr):
raise TurbiniaException(
'Task {0:s} does not have attribute {1:s}'.format(
task.name, attr))
task_dict[attr] = getattr(task, attr)
if isinstance(task_dict[attr], str):
task_dict[attr] = unicode(task_dict[attr])
if task.result:
for attr in task.result.STORED_ATTRIBUTES:
if not hasattr(task.result, attr):
raise TurbiniaException(
'Task {0:s} result does not have attribute {1:s}'.
format(task.name, attr))
task_dict[attr] = getattr(task.result, attr)
if isinstance(task_dict[attr], str):
task_dict[attr] = unicode(task_dict[attr])
# Set all non-existent keys to None
all_attrs = set(TurbiniaTask.STORED_ATTRIBUTES +
TurbiniaTaskResult.STORED_ATTRIBUTES)
task_dict.update(
{k: None
for k in all_attrs if not task_dict.has_key(k)})
task_dict = self._validate_data(task_dict)
# Using the pubsub topic as an instance attribute in order to have a unique
# namespace per Turbinia installation.
# TODO(aarontp): Migrate this to actual Datastore namespaces
config.LoadConfig()
task_dict.update({'instance': config.INSTANCE_ID})
return task_dict
def SetUp(self, disk_name, project, turbinia_zone, sketch_id,
run_all_jobs):
"""Sets up the object attributes.
Args:
disk_name (str): name of the disk to process.
project (str): name of the GPC project containing the disk to process.
turbinia_zone (str): GCP zone in which the Turbinia server is running.
sketch_id (int): The Timesketch sketch id
run_all_jobs (bool): Whether to run all jobs instead of a faster subset.
"""
# TODO: Consider the case when multiple disks are provided by the previous
# module or by the CLI.
if project is None or turbinia_zone is None:
self.state.AddError(
'project or turbinia_zone are not all specified, bailing out',
critical=True)
return
self.disk_name = disk_name
self.project = project
self.turbinia_zone = turbinia_zone
self.sketch_id = sketch_id
self.run_all_jobs = run_all_jobs
try:
turbinia_config.LoadConfig()
self.turbinia_region = turbinia_config.TURBINIA_REGION
self.instance = turbinia_config.PUBSUB_TOPIC
if turbinia_config.TURBINIA_PROJECT != self.project:
self.state.AddError(
'Specified project {0!s} does not match Turbinia configured '
'project {1!s}. Use gcp_turbinia_import recipe to copy the disk '
'into the same project.'.format(
self.project, turbinia_config.TURBINIA_PROJECT),
critical=True)
return
self._output_path = tempfile.mkdtemp()
self.client = turbinia_client.TurbiniaClient()
except TurbiniaException as exception:
# TODO: determine if exception should be converted into a string as
# elsewhere in the codebase.
self.state.AddError(exception, critical=True)
return
def run(self, evidence, result):
"""Task that process data with Plaso.
Args:
evidence (Evidence object): The evidence we will process.
result (TurbiniaTaskResult): The object to place task results into.
Returns:
TurbiniaTaskResult object.
"""
config.LoadConfig()
# Write plaso file into tmp_dir because sqlite has issues with some shared
# filesystems (e.g NFS).
plaso_file = os.path.join(self.tmp_dir, '{0:s}.plaso'.format(self.id))
plaso_evidence = PlasoFile(source_path=plaso_file)
plaso_log = os.path.join(self.output_dir, '{0:s}.log'.format(self.id))
cmd = self.build_plaso_command('log2timeline.py', self.task_config)
if config.DEBUG_TASKS or self.task_config.get('debug_tasks'):
cmd.append('-d')
if evidence.credentials:
for credential_type, credential_data in evidence.credentials:
cmd.extend([
'--credential',
'{0:s}:{1:s}'.format(credential_type, credential_data)
])
cmd.extend(['--temporary_directory', self.tmp_dir])
cmd.extend(['--logfile', plaso_log])
cmd.extend(['--unattended'])
cmd.extend(['--storage_file', plaso_file])
cmd.extend([evidence.local_path])
result.log('Running plaso as [{0:s}]'.format(' '.join(cmd)))
self.execute(cmd,
result,
log_files=[plaso_log],
new_evidence=[plaso_evidence],
close=True)
return result
def __init__(self, gcs_path=None, *args, **kwargs):
"""Initialization for GCSOutputWriter.
Args:
gcs_path (string): GCS path to put output results into.
"""
super(GCSOutputWriter, self).__init__(*args, **kwargs)
self.name = 'GCSWriter'
config.LoadConfig()
self.client = storage.Client(project=config.PROJECT)
match = re.search(r'gs://(.*)/(.*)', gcs_path)
if not match:
raise TurbiniaException(
'Cannot find bucket and path from GCS config {0:s}'.format(
gcs_path))
self.bucket = match.group(1)
self.base_output_dir = match.group(2)
def setup(self):
"""Set up Celery"""
config.LoadConfig()
self.app = celery.Celery('turbinia',
broker=config.CELERY_BROKER,
backend=config.CELERY_BACKEND)
self.app.conf.update(
task_default_queue=config.INSTANCE_ID,
accept_content=['json'],
# TODO(ericzinnikas): Without task_acks_late Celery workers will start
# on one task and prefetch another (i.e. can result in 1 worker getting
# 2 plaso jobs while another worker is free). But enabling this causes
# problems with certain Celery brokers (duplicated work).
task_acks_late=False,
task_track_started=True,
worker_concurrency=1,
worker_prefetch_multiplier=1,
)
def PreprocessAttachDisk(disk_name):
"""Attaches Google Cloud Disk to an instance.
Args:
disk_name(str): The name of the Cloud Disk to attach.
Returns:
(str, list(str)): a tuple consisting of the path to the 'disk' block device
and a list of paths to partition block devices. For example:
(
'/dev/disk/by-id/google-disk0',
['/dev/disk/by-id/google-disk0-part1', '/dev/disk/by-id/google-disk0-p2']
)
"""
path = '/dev/disk/by-id/google-{0:s}'.format(disk_name)
if IsBlockDevice(path):
log.info('Disk {0:s} already attached!'.format(disk_name))
return (path, glob.glob('{0:s}-part*'.format(path)))
config.LoadConfig()
instance_name = GetLocalInstanceName()
project = gcp_project.GoogleCloudProject(config.TURBINIA_PROJECT,
default_zone=config.TURBINIA_ZONE)
instance = project.compute.GetInstance(instance_name,
zone=config.TURBINIA_ZONE)
disk = instance.GetDisk(disk_name)
log.info('Attaching disk {0:s} to instance {1:s}'.format(
disk_name, instance_name))
instance.AttachDisk(disk)
# Make sure we have a proper block device
for _ in xrange(RETRY_MAX):
if IsBlockDevice(path):
log.info('Block device {0:s} successfully attached'.format(path))
break
if os.path.exists(path):
log.info('Block device {0:s} mode is {1}'.format(
path,
os.stat(path).st_mode))
time.sleep(1)
return (path, glob.glob('{0:s}-part*'.format(path)))
def setup(self, disk_name, project, turbinia_zone):
"""Sets up the object attributes.
Args:
disk_name (string): Name of the disk to process
project (string): The project containing the disk to process
turbinia_zone (string): The zone containing the disk to process
"""
# TODO: Consider the case when multiple disks are provided by the previous
# module or by the CLI.
if self.state.input and not disk_name:
_, disk = self.state.input[0]
disk_name = disk.name
print('Using disk {0:s} from previous collector'.format(disk_name))
if disk_name is None or project is None or turbinia_zone is None:
self.state.add_error(
'disk_name, project or turbinia_zone are not all specified, bailing '
'out',
critical=True)
return
self.disk_name = disk_name
self.project = project
self.turbinia_zone = turbinia_zone
try:
turbinia_config.LoadConfig()
self.turbinia_region = turbinia_config.TURBINIA_REGION
self.instance = turbinia_config.PUBSUB_TOPIC
if turbinia_config.PROJECT != self.project:
self.state.add_error(
'Specified project {0:s} does not match Turbinia configured '
'project {1:s}. Use gcp_turbinia_import recipe to copy the disk '
'into the same project.'.format(self.project,
turbinia_config.PROJECT),
critical=True)
return
self._output_path = tempfile.mkdtemp()
self.client = turbinia_client.TurbiniaClient()
except TurbiniaException as e:
self.state.add_error(e, critical=True)
return
def __init__(self, jobs_blacklist=None, jobs_whitelist=None):
"""Initialization for PSQ Worker.
Args:
jobs_blacklist (Optional[list[str]]): Jobs we will exclude from running
jobs_whitelist (Optional[list[str]]): The only Jobs we will include to run
"""
config.LoadConfig()
psq_publisher = pubsub.PublisherClient()
psq_subscriber = pubsub.SubscriberClient()
datastore_client = datastore.Client(project=config.TURBINIA_PROJECT)
try:
self.psq = psq.Queue(
psq_publisher,
psq_subscriber,
config.TURBINIA_PROJECT,
name=config.PSQ_TOPIC,
storage=psq.DatastoreStorage(datastore_client))
except exceptions.GoogleCloudError as e:
msg = 'Error creating PSQ Queue: {0:s}'.format(str(e))
log.error(msg)
raise TurbiniaException(msg)
# Deregister jobs from blacklist/whitelist.
disabled_jobs = list(
config.DISABLED_JOBS) if config.DISABLED_JOBS else []
job_manager.JobsManager.DeregisterJobs(jobs_blacklist, jobs_whitelist)
if disabled_jobs:
log.info(
'Disabling jobs that were configured to be disabled in the '
'config file: {0:s}'.format(', '.join(disabled_jobs)))
job_manager.JobsManager.DeregisterJobs(
jobs_blacklist=disabled_jobs)
# Check for valid dependencies/directories.
check_dependencies(config.DEPENDENCIES)
check_directory(config.MOUNT_DIR_PREFIX)
check_directory(config.OUTPUT_DIR)
check_directory(config.TMP_DIR)
log.info('Starting PSQ listener on queue {0:s}'.format(self.psq.name))
self.worker = psq.Worker(queue=self.psq)
def __init__(self, *_, **__):
"""Initialization for PSQ Worker."""
config.LoadConfig()
psq_publisher = pubsub.PublisherClient()
psq_subscriber = pubsub.SubscriberClient()
datastore_client = datastore.Client(project=config.TURBINIA_PROJECT)
try:
self.psq = psq.Queue(
psq_publisher, psq_subscriber, config.TURBINIA_PROJECT,
name=config.PSQ_TOPIC, storage=psq.DatastoreStorage(datastore_client))
except exceptions.GoogleCloudError as e:
msg = 'Error creating PSQ Queue: {0:s}'.format(str(e))
log.error(msg)
raise TurbiniaException(msg)
check_directory(config.MOUNT_DIR_PREFIX)
check_directory(config.OUTPUT_DIR)
check_directory(config.TMP_DIR)
log.info('Starting PSQ listener on queue {0:s}'.format(self.psq.name))
self.worker = psq.Worker(queue=self.psq)
def setup_subscriber(self):
"""Set up the pubsub subscriber."""
config.LoadConfig()
self.subscriber = pubsub.SubscriberClient()
subscription_path = self.subscriber.subscription_path(
config.PROJECT, self.topic_name)
if not self.topic_path:
self.topic_path = self.subscriber.topic_path(
config.PROJECT, self.topic_name)
try:
log.debug(
'Trying to create subscription {0:s} on topic {1:s}'.format(
subscription_path, self.topic_path))
self.subscriber.create_subscription(subscription_path,
self.topic_path)
except exceptions.Conflict:
log.debug(
'Subscription {0:s} already exists.'.format(subscription_path))
log.debug('Setup PubSub Subscription {0:s}'.format(subscription_path))
self.subscription = self.subscriber.subscribe(subscription_path,
self._callback)
