def add_evidence(self, evidence_):
"""Adds new evidence and creates tasks to process it.
This creates all tasks configured to process the given type of evidence.
Args:
evidence_: evidence object to add.
Raises:
TurbiniaException: When no Jobs are found.
"""
if not self.jobs:
raise turbinia.TurbiniaException(
'Jobs must be registered before evidence can be added')
log.info('Adding new evidence: {0:s}'.format(str(evidence_)))
self.evidence.append(evidence_)
job_count = 0
# TODO(aarontp): Add some kind of loop detection in here so that jobs can
# register for Evidence(), or or other evidence types that may be a super
# class of the output of the job itself. Short term we could potentially
# have a run time check for this upon Job instantiation to prevent it.
for job in self.jobs:
if [True for t in job.evidence_input if isinstance(evidence_, t)]:
log.info('Adding {0:s} job to process {1:s}'.format(
job.name, evidence_.name))
job_count += 1
for task in job.create_tasks([evidence_]):
task.base_output_dir = config.OUTPUT_DIR
self.add_task(task, evidence_)
if not job_count:
log.warning('No Jobs/Tasks were created for Evidence [{0:s}]. '
'Jobs may need to be configured to allow this type of '
'Evidence as input'.format(str(evidence_)))
def _backend_setup(self, server=True, *args, **kwargs):
"""
Args:
server (bool): Whether this is the client or a server
Raises:
TurbiniaException: When there are errors creating PSQ Queue
"""
log.debug(
'Setting up PSQ Task Manager requirements on project {0:s}'.format(
config.TURBINIA_PROJECT))
self.server_pubsub = turbinia_pubsub.TurbiniaPubSub(
config.PUBSUB_TOPIC)
if server:
self.server_pubsub.setup_subscriber()
else:
self.server_pubsub.setup_publisher()
psq_publisher = pubsub.PublisherClient()
psq_subscriber = pubsub.SubscriberClient()
datastore_client = datastore.Client(project=config.TURBINIA_PROJECT)
try:
self.psq = psq.Queue(
psq_publisher,
psq_subscriber,
config.TURBINIA_PROJECT,
name=config.PSQ_TOPIC,
storage=psq.DatastoreStorage(datastore_client))
except exceptions.GoogleCloudError as e:
msg = 'Error creating PSQ Queue: {0:s}'.format(str(e))
log.error(msg)
raise turbinia.TurbiniaException(msg)
def add_evidence(self, evidence_):
"""Adds new evidence and creates tasks to process it.
This creates all tasks configured to process the given type of evidence.
Args:
evidence_: evidence object to add.
Raises:
TurbiniaException: When no Jobs are found.
"""
if not self.jobs:
raise turbinia.TurbiniaException(
'Jobs must be registered before evidence can be added')
log.info('Adding new evidence: {0:s}'.format(str(evidence_)))
job_count = 0
jobs_list = []
jobs_allowlist = evidence_.config['globals'].get('jobs_allowlist', [])
jobs_denylist = evidence_.config['globals'].get('jobs_denylist', [])
if jobs_denylist or jobs_allowlist:
log.info('Filtering Jobs with allowlist {0!s} and denylist {1!s}'.
format(jobs_allowlist, jobs_denylist))
jobs_list = jobs_manager.JobsManager.FilterJobObjects(
self.jobs, jobs_denylist, jobs_allowlist)
else:
jobs_list = self.jobs
# TODO(aarontp): Add some kind of loop detection in here so that jobs can
# register for Evidence(), or or other evidence types that may be a super
# class of the output of the job itself. Short term we could potentially
# have a run time check for this upon Job instantiation to prevent it.
for job in jobs_list:
# Doing a strict type check here for now until we can get the above
# comment figured out.
# pylint: disable=unidiomatic-typecheck
job_applicable = [
True for t in job.evidence_input if type(evidence_) == t
]
if job_applicable:
job_instance = job(request_id=evidence_.request_id,
evidence_config=evidence_.config)
for task in job_instance.create_tasks([evidence_]):
self.add_task(task, job_instance, evidence_)
self.running_jobs.append(job_instance)
log.info('Adding {0:s} job to process {1:s}'.format(
job_instance.name, evidence_.name))
job_count += 1
turbinia_jobs_total.inc()
if not job_count:
log.warning(
'No Jobs/Tasks were created for Evidence [{0:s}]. '
'Request or recipe parsing may have failed, or Jobs may need to be '
'configured to allow this type of Evidence as input'.format(
str(evidence_)))
def get_task_manager():
"""Return task manager object based on config.
Returns
Initialized TaskManager object.
"""
config.LoadConfig()
if config.TASK_MANAGER == 'PSQ':
return PSQTaskManager()
else:
msg = 'Task Manager type "{0:s}" not implemented'.format(
config.TASK_MANAGER)
raise turbinia.TurbiniaException(msg)
def add_evidence(self, evidence_):
"""Adds new evidence and creates tasks to process it.
This creates all tasks configured to process the given type of evidence.
Args:
evidence_: evidence object to add.
Raises:
TurbiniaException: When no Jobs are found.
"""
if not self.jobs:
raise turbinia.TurbiniaException(
'Jobs must be registered before evidence can be added')
log.info('Adding new evidence: {0:s}'.format(str(evidence_)))
self.evidence.append(evidence_)
job_count = 0
jobs_whitelist = evidence_.config.get('jobs_whitelist', [])
jobs_blacklist = evidence_.config.get('jobs_blacklist', [])
if jobs_blacklist or jobs_whitelist:
log.info(
'Filtering Jobs with whitelist {0!s} and blacklist {1!s}'.format(
jobs_whitelist, jobs_blacklist))
jobs_list = jobs_manager.JobsManager.FilterJobObjects(
self.jobs, jobs_blacklist, jobs_whitelist)
else:
jobs_list = self.jobs
# TODO(aarontp): Add some kind of loop detection in here so that jobs can
# register for Evidence(), or or other evidence types that may be a super
# class of the output of the job itself. Short term we could potentially
# have a run time check for this upon Job instantiation to prevent it.
for job in jobs_list:
# Doing a strict type check here for now until we can get the above
# comment figured out.
# pylint: disable=unidiomatic-typecheck
if [True for t in job.evidence_input if type(evidence_) == t]:
log.info(
'Adding {0:s} job to process {1:s}'.format(
job.name, evidence_.name))
job_count += 1
for task in job.create_tasks([evidence_]):
task.base_output_dir = config.OUTPUT_DIR
self.add_task(task, evidence_)
if not job_count:
log.warning(
'No Jobs/Tasks were created for Evidence [{0:s}]. '
'Jobs may need to be configured to allow this type of '
'Evidence as input'.format(str(evidence_)))
def get_task_manager():
"""Return task manager object based on config.
Returns
Initialized TaskManager object.
"""
config.LoadConfig()
# pylint: disable=no-else-return
if config.TASK_MANAGER.lower() == 'psq':
return PSQTaskManager()
elif config.TASK_MANAGER.lower() == 'celery':
return CeleryTaskManager()
else:
msg = 'Task Manager type "{0:s}" not implemented'.format(
config.TASK_MANAGER)
raise turbinia.TurbiniaException(msg)
def _backend_setup(self):
log.debug(
'Setting up PSQ Task Manager requirements on project {0:s}'.format(
config.PROJECT))
self.server_pubsub = turbinia_pubsub.TurbiniaPubSub(config.PUBSUB_TOPIC)
self.server_pubsub.setup()
psq_pubsub_client = pubsub.Client(project=config.PROJECT)
datastore_client = datastore.Client(project=config.PROJECT)
try:
self.psq = psq.Queue(
psq_pubsub_client,
config.PSQ_TOPIC,
storage=psq.DatastoreStorage(datastore_client))
except GaxError as e:
msg = 'Error creating PSQ Queue: {0:s}'.format(str(e))
log.error(msg)
raise turbinia.TurbiniaException(msg)
