def migrate_analysis_results(self):
from crits.services.analysis_result import (AnalysisResult, AnalysisConfig,
EmbeddedAnalysisResultLog)
old_results = getattr(self.unsupported_attrs, 'analysis', None)
if old_results:
for result in old_results:
ar = AnalysisResult()
ar.analysis_id = result.get('id')
if ar.analysis_id:
del result['id']
config = result.get('config', {})
ar.config = AnalysisConfig(**config)
del result['config']
logs = result.get('log', None)
if logs:
for l in logs:
le = EmbeddedAnalysisResultLog(**l)
ar.log.append(le)
del result['log']
ar.merge(arg_dict=result)
ar.object_type = self._meta['crits_type']
ar.object_id = str(self.id)
ar.save()
try:
del self.unsupported_attrs['analysis']
except:
pass
def update_config(service_name, config, analyst):
"""
Update the configuration for a service.
"""
service = CRITsService.objects(name=service_name).first()
service.config = AnalysisConfig(**config)
try:
#TODO: get/validate the config from service author to set status
#update_status(service_name)
service.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': e}
def _register_services(self, klass):
"""
Create a dict with names of available services and classes that
implement them.
This is a recursive function since __subclasses__() only returns direct
subclasses. If class A(object):, class B(A):, and class C(B):, then
A.__subclasses__() doesn't contain C.
All subclasses of the Service class are saved in the `services`
dictionary. It is intended that each of these was imported by the
_import_services function, but this is not enforced. The key in the
dictionary is the `name` class-level field, and the value is the class
itself. It is recommended that the service "example" be implemented
in a class "ExampleService" defined in a module named
"example_service", but this is not enforced, and the only string
visible to the end-user/analyst is the service name.
"""
for service_class in klass.__subclasses__():
# TODO: replace this with a proper check for a valid service
if not (hasattr(service_class, "name") and
hasattr(service_class, "version")):
# If this is a subclass of Service but not an actual service
# call this function recursively.
self._register_services(service_class)
continue
service_name = service_class.name
service_version = service_class.version
service_description = service_class.description
supported_types = service_class.supported_types
compatability_mode = service_class.compatability_mode
logger.debug("Found service subclass: %s version %s" %
(service_name, service_version))
try:
StrictVersion(service_version)
except ValueError as e:
# Unable to parse the service version
msg = ("Service %s is invalid, and will not be available." %
service_name)
logger.warning(msg)
logger.warning(e)
continue
else:
# Only register the service if it is valid.
logger.debug("Registering Service %s" % service_name)
svc_obj = CRITsService.objects(name=service_class.name).first()
service = service_class()
if not svc_obj:
svc_obj = CRITsService()
svc_obj.name = service_name
try:
new_config = service.get_config({})
svc_obj.config = AnalysisConfig(**new_config)
except ServiceConfigError:
svc_obj.status = "misconfigured"
msg = ("Service %s is misconfigured." % service_name)
logger.warning(msg)
else:
svc_obj.status = "available"
else:
existing_config = svc_obj.config.to_dict()
try:
new_config = service.get_config(existing_config)
svc_obj.config = AnalysisConfig(**new_config)
except ServiceConfigError:
svc_obj.status = "misconfigured"
svc_obj.enabled = False
svc_obj.run_on_triage = False
msg = ("Service %s is misconfigured." % service_name)
logger.warning(msg)
else:
svc_obj.status = "available"
# Give the service a chance to tell us what is wrong with the
# config.
try:
service.parse_config(svc_obj.config.to_dict())
except ServiceConfigError as e:
svc_obj.status = "misconfigured"
svc_obj.enabled = False
svc_obj.run_on_triage = False
svc_obj.description = service_description
svc_obj.version = service_version
svc_obj.supported_types = supported_types
svc_obj.compatability_mode = compatability_mode
svc_obj.save()
self._services[service_class.name] = service_class
# For anything in the database that did not import properly, mark the
# status to unavailable.
svcs = CRITsService.objects()
for svc in svcs:
if svc.name not in self._services:
svc.status = 'unavailable'
svc.enabled = False
svc.run_on_triage = False
svc.save()
def run_service(name,
type_,
id_,
user,
obj=None,
execute='local',
custom_config={},
**kwargs):
"""
Run a service.
:param name: The name of the service to run.
:type name: str
:param type_: The type of the object.
:type type_: str
:param id_: The identifier of the object.
:type id_: str
:param user: The user running the service.
:type user: str
:param obj: The CRITs object, if given this overrides crits_type and identifier.
:type obj: CRITs object.
:param analyst: The user updating the results.
:type analyst: str
:param execute: The execution type.
:type execute: str
:param custom_config: Use a custom configuration for this run.
:type custom_config: dict
"""
result = {'success': False}
if type_ not in settings.CRITS_TYPES:
result['html'] = "Unknown CRITs type."
return result
if name not in enabled_services():
result['html'] = "Service %s is unknown or not enabled." % name
return result
service_class = crits.services.manager.get_service_class(name)
if not service_class:
result['html'] = "Unable to get service class."
return result
if not obj:
obj = class_from_id(type_, id_)
if not obj:
result['html'] = 'Could not find object.'
return result
service = CRITsService.objects(name=name).first()
if not service:
result['html'] = "Unable to find service in database."
return result
# See if the object is a supported type for the service.
if not service_class.supported_for_type(type_):
result['html'] = "Service not supported for type '%s'" % type_
return result
# When running in threaded mode, each thread needs to have its own copy of
# the object. If we do not do this then one thread may read() from the
# object (to get the binary) and then the second would would read() without
# knowing and get undefined behavior as the file pointer would be who knows
# where. By giving each thread a local copy they can operate independently.
#
# When not running in thread mode this has no effect except wasted memory.
local_obj = local()
local_obj.obj = copy.deepcopy(obj)
# Give the service a chance to check for required fields.
try:
service_class.valid_for(local_obj.obj)
if hasattr(local_obj.obj, 'filedata'):
if local_obj.obj.filedata.grid_id:
# Reset back to the start so the service gets the full file.
local_obj.obj.filedata.seek(0)
except ServiceConfigError as e:
result['html'] = str(e)
return result
# Get the config from the database and validate the submitted options
# exist.
db_config = service.config.to_dict()
try:
service_class.validate_runtime(custom_config, db_config)
except ServiceConfigError as e:
result['html'] = str(e)
return result
final_config = db_config
# Merge the submitted config with the one from the database.
# This is because not all config options may be submitted.
final_config.update(custom_config)
form = service_class.bind_runtime_form(user, final_config)
if form:
if not form.is_valid():
# TODO: return corrected form via AJAX
result['html'] = str(form.errors)
return result
# If the form is valid, create the config using the cleaned data.
final_config = db_config
final_config.update(form.cleaned_data)
logger.info("Running %s on %s, execute=%s" %
(name, local_obj.obj.id, execute))
service_instance = service_class(notify=update_analysis_results,
complete=finish_task)
# Give the service a chance to modify the config that gets saved to the DB.
saved_config = dict(final_config)
service_class.save_runtime_config(saved_config)
task = AnalysisTask(local_obj.obj, service_instance, user)
task.config = AnalysisConfig(**saved_config)
task.start()
add_task(task)
service_instance.set_task(task)
if execute == 'process':
p = Process(target=service_instance.execute, args=(final_config, ))
p.start()
elif execute == 'thread':
t = Thread(target=service_instance.execute, args=(final_config, ))
t.start()
elif execute == 'process_pool':
if __service_process_pool__ is not None and service.compatability_mode != True:
__service_process_pool__.apply_async(func=service_work_handler,
args=(
service_instance,
final_config,
))
else:
logger.warning(
"Could not run %s on %s, execute=%s, running in process mode" %
(name, local_obj.obj.id, execute))
p = Process(target=service_instance.execute, args=(final_config, ))
p.start()
elif execute == 'thread_pool':
if __service_thread_pool__ is not None and service.compatability_mode != True:
__service_thread_pool__.apply_async(func=service_work_handler,
args=(
service_instance,
final_config,
))
else:
logger.warning(
"Could not run %s on %s, execute=%s, running in thread mode" %
(name, local_obj.obj.id, execute))
t = Thread(target=service_instance.execute, args=(final_config, ))
t.start()
elif execute == 'local':
service_instance.execute(final_config)
# Return after starting thread so web request can complete.
result['success'] = True
return result
def run_service(name, crits_type, identifier, analyst, obj=None,
execute='local', custom_config={}):
"""
Run a service.
:param name: The name of the service to run.
:type name: str
:param crits_type: The type of the object.
:type name: str
:param identifier: The identifier of the object.
:type name: str
:param obj: The CRITs object, if given this overrides crits_type and identifier.
:type obj: CRITs object.
:param analyst: The user updating the results.
:type analyst: str
:param execute: The execution type.
:type execute: str
:param custom_config: Use a custom configuration for this run.
:type custom_config: dict
"""
result = {'success': False}
if crits_type not in settings.CRITS_TYPES:
result['html'] = "Unknown CRITs type."
return result
if name not in enabled_services():
result['html'] = "Service %s is unknown or not enabled." % name
return result
service_class = crits.services.manager.get_service_class(name)
if not service_class:
result['html'] = "Unable to get service class."
return result
if not obj:
obj = class_from_id(crits_type, identifier)
if not obj:
result['html'] = 'Could not find object.'
return result
service = CRITsService.objects(name=name).first()
if not service:
result['html'] = "Unable to find service in database."
return result
# See if the object is a supported type for the service.
if not service_class.supported_for_type(crits_type):
result['html'] = "Service not supported for type '%s'" % crits_type
return result
# Give the service a chance to check for required fields.
try:
service_class.valid_for(obj)
except ServiceConfigError as e:
result['html'] = str(e)
return result
# Get the config from the database and validate the submitted options
# exist.
db_config = service.config.to_dict()
try:
service_class.validate_runtime(custom_config, db_config)
except ServiceConfigError as e:
result['html'] = str(e)
return result
final_config = db_config
# Merge the submitted config with the one from the database.
# This is because not all config options may be submitted.
final_config.update(custom_config)
form = service_class.bind_runtime_form(analyst, final_config)
if form:
if not form.is_valid():
# TODO: return corrected form via AJAX
result['html'] = str(form.errors)
return result
# If the form is valid, create the config using the cleaned data.
final_config = db_config
final_config.update(form.cleaned_data)
logger.info("Running %s on %s, execute=%s" % (name, obj.id, execute))
service_instance = service_class(notify=update_analysis_results,
complete=finish_task)
# Give the service a chance to modify the config that gets saved to the DB.
saved_config = dict(final_config)
service_class.save_runtime_config(saved_config)
task = AnalysisTask(obj, service_instance, analyst)
task.config = AnalysisConfig(**saved_config)
task.start()
add_task(task)
service_instance.set_task(task)
if execute == 'process':
p = Process(target=service_instance.execute, args=(final_config,))
p.start()
elif execute == 'thread':
t = Thread(target=service_instance.execute, args=(final_config,))
t.start()
elif execute == 'process_pool':
if __service_process_pool__ is not None and service.compatability_mode != True:
__service_process_pool__.apply_async(func=service_work_handler,
args=(service_instance, final_config,))
else:
logger.warning("Could not run %s on %s, execute=%s, running in process mode" % (name, obj.id, execute))
p = Process(target=service_instance.execute, args=(final_config,))
p.start()
elif execute == 'thread_pool':
if __service_thread_pool__ is not None and service.compatability_mode != True:
__service_thread_pool__.apply_async(func=service_work_handler,
args=(service_instance, final_config,))
else:
logger.warning("Could not run %s on %s, execute=%s, running in thread mode" % (name, obj.id, execute))
t = Thread(target=service_instance.execute, args=(final_config,))
t.start()
elif execute == 'local':
service_instance.execute(final_config)
# Return after starting thread so web request can complete.
result['success'] = True
return result