def run_service(name,
type_,
id_,
user,
obj=None,
execute='local',
custom_config={},
**kwargs):
"""
Run a service.
:param name: The name of the service to run.
:type name: str
:param type_: The type of the object.
:type type_: str
:param id_: The identifier of the object.
:type id_: str
:param user: The user running the service.
:type user: str
:param obj: The CRITs object, if given this overrides crits_type and identifier.
:type obj: CRITs object.
:param analyst: The user updating the results.
:type analyst: str
:param execute: The execution type.
:type execute: str
:param custom_config: Use a custom configuration for this run.
:type custom_config: dict
"""
result = {'success': False}
if type_ not in settings.CRITS_TYPES:
result['html'] = "Unknown CRITs type."
return result
if name not in enabled_services():
result['html'] = "Service %s is unknown or not enabled." % name
return result
service_class = crits.services.manager.get_service_class(name)
if not service_class:
result['html'] = "Unable to get service class."
return result
if not obj:
obj = class_from_id(type_, id_)
if not obj:
result['html'] = 'Could not find object.'
return result
service = CRITsService.objects(name=name).first()
if not service:
result['html'] = "Unable to find service in database."
return result
# See if the object is a supported type for the service.
if not service_class.supported_for_type(type_):
result['html'] = "Service not supported for type '%s'" % type_
return result
# When running in threaded mode, each thread needs to have its own copy of
# the object. If we do not do this then one thread may read() from the
# object (to get the binary) and then the second would would read() without
# knowing and get undefined behavior as the file pointer would be who knows
# where. By giving each thread a local copy they can operate independently.
#
# When not running in thread mode this has no effect except wasted memory.
local_obj = local()
local_obj.obj = copy.deepcopy(obj)
# Give the service a chance to check for required fields.
try:
service_class.valid_for(local_obj.obj)
if hasattr(local_obj.obj, 'filedata'):
if local_obj.obj.filedata.grid_id:
# Reset back to the start so the service gets the full file.
local_obj.obj.filedata.seek(0)
except ServiceConfigError as e:
result['html'] = str(e)
return result
# Get the config from the database and validate the submitted options
# exist.
db_config = service.config.to_dict()
try:
service_class.validate_runtime(custom_config, db_config)
except ServiceConfigError as e:
result['html'] = str(e)
return result
final_config = db_config
# Merge the submitted config with the one from the database.
# This is because not all config options may be submitted.
final_config.update(custom_config)
form = service_class.bind_runtime_form(user, final_config)
if form:
if not form.is_valid():
# TODO: return corrected form via AJAX
result['html'] = str(form.errors)
return result
# If the form is valid, create the config using the cleaned data.
final_config = db_config
final_config.update(form.cleaned_data)
logger.info("Running %s on %s, execute=%s" %
(name, local_obj.obj.id, execute))
service_instance = service_class(notify=update_analysis_results,
complete=finish_task)
# Give the service a chance to modify the config that gets saved to the DB.
saved_config = dict(final_config)
service_class.save_runtime_config(saved_config)
task = AnalysisTask(local_obj.obj, service_instance, user)
task.config = AnalysisConfig(**saved_config)
task.start()
add_task(task)
service_instance.set_task(task)
if execute == 'process':
p = Process(target=service_instance.execute, args=(final_config, ))
p.start()
elif execute == 'thread':
t = Thread(target=service_instance.execute, args=(final_config, ))
t.start()
elif execute == 'process_pool':
if __service_process_pool__ is not None and service.compatability_mode != True:
__service_process_pool__.apply_async(func=service_work_handler,
args=(
service_instance,
final_config,
))
else:
logger.warning(
"Could not run %s on %s, execute=%s, running in process mode" %
(name, local_obj.obj.id, execute))
p = Process(target=service_instance.execute, args=(final_config, ))
p.start()
elif execute == 'thread_pool':
if __service_thread_pool__ is not None and service.compatability_mode != True:
__service_thread_pool__.apply_async(func=service_work_handler,
args=(
service_instance,
final_config,
))
else:
logger.warning(
"Could not run %s on %s, execute=%s, running in thread mode" %
(name, local_obj.obj.id, execute))
t = Thread(target=service_instance.execute, args=(final_config, ))
t.start()
elif execute == 'local':
service_instance.execute(final_config)
# Return after starting thread so web request can complete.
result['success'] = True
return result
def run_service(name, type_, id_, user, obj=None,
execute='local', custom_config={}, is_triage_run=False, **kwargs):
"""
Run a service.
:param name: The name of the service to run.
:type name: str
:param type_: The type of the object.
:type type_: str
:param id_: The identifier of the object.
:type id_: str
:param user: The user running the service.
:type user: str
:param obj: The CRITs object, if given this overrides crits_type and identifier.
:type obj: CRITs object.
:param analyst: The user updating the results.
:type analyst: str
:param execute: The execution type.
:type execute: str
:param custom_config: Use a custom configuration for this run.
:type custom_config: dict
"""
result = {'success': False}
if type_ not in settings.CRITS_TYPES:
result['html'] = "Unknown CRITs type."
return result
if name not in enabled_services():
result['html'] = "Service %s is unknown or not enabled." % name
return result
service_class = crits.services.manager.get_service_class(name)
if not service_class:
result['html'] = "Unable to get service class."
return result
if not obj:
obj = class_from_id(type_, id_)
if not obj:
result['html'] = 'Could not find object.'
return result
service = CRITsService.objects(name=name).first()
if not service:
result['html'] = "Unable to find service in database."
return result
# See if the object is a supported type for the service.
if not service_class.supported_for_type(type_):
result['html'] = "Service not supported for type '%s'" % type_
return result
# When running in threaded mode, each thread needs to have its own copy of
# the object. If we do not do this then one thread may read() from the
# object (to get the binary) and then the second would would read() without
# knowing and get undefined behavior as the file pointer would be who knows
# where. By giving each thread a local copy they can operate independently.
#
# When not running in thread mode this has no effect except wasted memory.
local_obj = local()
local_obj.obj = copy.deepcopy(obj)
# Give the service a chance to check for required fields.
try:
service_class.valid_for(local_obj.obj)
if hasattr(local_obj.obj, 'filedata'):
if local_obj.obj.filedata.grid_id:
# Reset back to the start so the service gets the full file.
local_obj.obj.filedata.seek(0)
except ServiceConfigError as e:
result['html'] = str(e)
return result
# Get the config from the database and validate the submitted options
# exist.
db_config = service.config.to_dict()
try:
service_class.validate_runtime(custom_config, db_config)
except ServiceConfigError as e:
result['html'] = str(e)
return result
final_config = db_config
# Merge the submitted config with the one from the database.
# This is because not all config options may be submitted.
final_config.update(custom_config)
form = service_class.bind_runtime_form(user, final_config)
if form:
if not form.is_valid():
# TODO: return corrected form via AJAX
result['html'] = str(form.errors)
return result
# If the form is valid, create the config using the cleaned data.
final_config = db_config
final_config.update(form.cleaned_data)
logger.info("Running %s on %s, execute=%s" % (name, local_obj.obj.id, execute))
service_instance = service_class(notify=update_analysis_results,
complete=finish_task)
# Determine if this service is being run via triage
if is_triage_run:
service_instance.is_triage_run = True
# Give the service a chance to modify the config that gets saved to the DB.
saved_config = dict(final_config)
service_class.save_runtime_config(saved_config)
task = AnalysisTask(local_obj.obj, service_instance, user)
task.config = AnalysisConfig(**saved_config)
task.start()
add_task(task)
service_instance.set_task(task)
if execute == 'process':
p = Process(target=service_instance.execute, args=(final_config,))
p.start()
elif execute == 'thread':
t = Thread(target=service_instance.execute, args=(final_config,))
t.start()
elif execute == 'process_pool':
if __service_process_pool__ is not None and service.compatability_mode != True:
__service_process_pool__.apply_async(func=service_work_handler,
args=(service_instance, final_config,))
else:
logger.warning("Could not run %s on %s, execute=%s, running in process mode" % (name, local_obj.obj.id, execute))
p = Process(target=service_instance.execute, args=(final_config,))
p.start()
elif execute == 'thread_pool':
if __service_thread_pool__ is not None and service.compatability_mode != True:
__service_thread_pool__.apply_async(func=service_work_handler,
args=(service_instance, final_config,))
else:
logger.warning("Could not run %s on %s, execute=%s, running in thread mode" % (name, local_obj.obj.id, execute))
t = Thread(target=service_instance.execute, args=(final_config,))
t.start()
elif execute == 'local':
service_instance.execute(final_config)
# Return after starting thread so web request can complete.
result['success'] = True
return result
def run_service(name, crits_type, identifier, analyst, obj=None,
execute='local', custom_config={}):
"""
Run a service.
:param name: The name of the service to run.
:type name: str
:param crits_type: The type of the object.
:type name: str
:param identifier: The identifier of the object.
:type name: str
:param obj: The CRITs object, if given this overrides crits_type and identifier.
:type obj: CRITs object.
:param analyst: The user updating the results.
:type analyst: str
:param execute: The execution type.
:type execute: str
:param custom_config: Use a custom configuration for this run.
:type custom_config: dict
"""
result = {'success': False}
if crits_type not in settings.CRITS_TYPES:
result['html'] = "Unknown CRITs type."
return result
if name not in enabled_services():
result['html'] = "Service %s is unknown or not enabled." % name
return result
service_class = crits.services.manager.get_service_class(name)
if not service_class:
result['html'] = "Unable to get service class."
return result
if not obj:
obj = class_from_id(crits_type, identifier)
if not obj:
result['html'] = 'Could not find object.'
return result
service = CRITsService.objects(name=name).first()
if not service:
result['html'] = "Unable to find service in database."
return result
# See if the object is a supported type for the service.
if not service_class.supported_for_type(crits_type):
result['html'] = "Service not supported for type '%s'" % crits_type
return result
# Give the service a chance to check for required fields.
try:
service_class.valid_for(obj)
except ServiceConfigError as e:
result['html'] = str(e)
return result
# Get the config from the database and validate the submitted options
# exist.
db_config = service.config.to_dict()
try:
service_class.validate_runtime(custom_config, db_config)
except ServiceConfigError as e:
result['html'] = str(e)
return result
final_config = db_config
# Merge the submitted config with the one from the database.
# This is because not all config options may be submitted.
final_config.update(custom_config)
form = service_class.bind_runtime_form(analyst, final_config)
if form:
if not form.is_valid():
# TODO: return corrected form via AJAX
result['html'] = str(form.errors)
return result
# If the form is valid, create the config using the cleaned data.
final_config = db_config
final_config.update(form.cleaned_data)
logger.info("Running %s on %s, execute=%s" % (name, obj.id, execute))
service_instance = service_class(notify=update_analysis_results,
complete=finish_task)
# Give the service a chance to modify the config that gets saved to the DB.
saved_config = dict(final_config)
service_class.save_runtime_config(saved_config)
task = AnalysisTask(obj, service_instance, analyst)
task.config = AnalysisConfig(**saved_config)
task.start()
add_task(task)
service_instance.set_task(task)
if execute == 'process':
p = Process(target=service_instance.execute, args=(final_config,))
p.start()
elif execute == 'thread':
t = Thread(target=service_instance.execute, args=(final_config,))
t.start()
elif execute == 'local':
service_instance.execute(final_config)
# Return after starting thread so web request can complete.
result['success'] = True
return result