def _add_to_collection(self, service_class):
"""
Add a service to the database.
"""
logger.info("Adding service %s to MongoDB" % service_class.name)
description = inspect.getdoc(service_class)
config = service_class.build_default_config()
config = AnalysisConfig(**config)
new_service = CRITsService()
new_service.name = service_class.name
new_service.version = service_class.version
new_service.service_type = service_class.type_
new_service.purpose = service_class.purpose
new_service.rerunnable = service_class.rerunnable
new_service.supported_types = service_class.supported_types
new_service.required_fields = service_class.required_fields
new_service.enabled = False
new_service.run_on_triage = False
new_service.description = description
new_service.config = config
try:
new_service.save()
logger.debug('Added service %s successfully.' % service_class.name)
except ValidationError, e:
logger.warning('Failed to add service %s: %s' % (service_class.name,
e))
def update_config(self, service_name, config, analyst):
"""
Update the configuration for a service.
"""
service = CRITsService.objects(name=service_name).first()
service.config = AnalysisConfig(**config)
try:
service.save(username=analyst)
self.update_status(service_name)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': e}
def _insert_analysis_results(self, task):
"""
Insert analysis results for this task.
"""
obj_class = class_from_type(task.context.crits_type)
query = self.get_db_query(task.context)
ear = EmbeddedAnalysisResult()
tdict = task.to_dict()
tdict['analysis_type'] = tdict['type']
tdict['analysis_id'] = tdict['id']
del tdict['type']
del tdict['id']
ear.merge(arg_dict=tdict)
ear.config = AnalysisConfig(**tdict['config'])
obj_class.objects(__raw__=query).update_one(push__analysis=ear)
def _update_service(self, service_class):
"""
Update a service in the database.
"""
logger.info("Updating service %s in MongoDB" % service_class.name)
new_config = service_class.build_default_config()
current = CRITsService.objects(name=service_class.name).first()
if current:
current_config = current.config.to_dict()
# Log removed keys
removed_keys = set(current_config.keys()) - set(new_config.keys())
if removed_keys:
logger.warning("Old service configuration options removed: %s" %
str(removed_keys))
# Log added keys
added_keys = set(new_config.keys()) - set(current_config.keys())
if added_keys:
logger.warning("New service configuration options added: %s" %
str(added_keys))
# All new items need to be added to the current config
for key in added_keys:
current_config[key] = new_config[key]
current.config = AnalysisConfig(**current_config)
# Update the version number
current.version = service_class.version
try:
current.save()
logger.info('Updated service %s successfully' % service_class.name)
except:
logger.warning('Failed to update service %s' % service_class.name)
def _update_analysis_results(self, task):
"""
Update analysis results for this task.
"""
# If the task does not currently exist for the given sample in the
# database, add it.
obj_class = class_from_type(task.context.crits_type)
query = self.get_db_query(task.context)
obj = obj_class.objects(__raw__=query).first()
obj_id = obj.id
found = False
c = 0
for a in obj.analysis:
if str(a.analysis_id) == task.task_id:
found = True
break
c += 1
if not found:
logger.warning("Tried to update a task that didn't exist.")
self._insert_analysis_results(task)
else:
# Otherwise, update it.
ear = EmbeddedAnalysisResult()
tdict = task.to_dict()
tdict['analysis_type'] = tdict['type']
tdict['analysis_id'] = tdict['id']
del tdict['type']
del tdict['id']
ear.merge(arg_dict=tdict)
ear.config = AnalysisConfig(**tdict['config'])
obj_class.objects(
id=obj_id,
analysis__id=task.task_id).update_one(set__analysis__S=ear)
def _register_services(self, klass):
"""
Create a dict with names of available services and classes that
implement them.
This is a recursive function since __subclasses__() only returns direct
subclasses. If class A(object):, class B(A):, and class C(B):, then
A.__subclasses__() doesn't contain C.
All subclasses of the Service class are saved in the `services`
dictionary. It is intended that each of these was imported by the
_import_services function, but this is not enforced. The key in the
dictionary is the `name` class-level field, and the value is the class
itself. It is recommended that the service "example" be implemented
in a class "ExampleService" defined in a module named
"example_service", but this is not enforced, and the only string
visible to the end-user/analyst is the service name.
"""
for service_class in klass.__subclasses__():
# TODO: replace this with a proper check for a valid service
if not (hasattr(service_class, "name") and
hasattr(service_class, "version")):
# If this is a subclass of Service but not an actual service
# call this function recursively.
self._register_services(service_class)
continue
service_name = service_class.name
service_version = service_class.version
service_description = service_class.description
supported_types = service_class.supported_types
logger.debug("Found service subclass: %s version %s" %
(service_name, service_version))
try:
StrictVersion(service_version)
except ValueError as e:
# Unable to parse the service version
msg = ("Service %s is invalid, and will not be available." %
service_name)
logger.warning(msg)
logger.warning(e)
continue
else:
# Only register the service if it is valid.
logger.debug("Registering Service %s" % service_name)
svc_obj = CRITsService.objects(name=service_class.name).first()
service = service_class()
if not svc_obj:
svc_obj = CRITsService()
svc_obj.name = service_name
try:
new_config = service.get_config({})
svc_obj.config = AnalysisConfig(**new_config)
except ServiceConfigError:
svc_obj.status = "misconfigured"
msg = ("Service %s is misconfigured." % service_name)
logger.warning(msg)
else:
svc_obj.status = "available"
else:
existing_config = svc_obj.config.to_dict()
try:
new_config = service.get_config(existing_config)
svc_obj.config = AnalysisConfig(**new_config)
except ServiceConfigError:
svc_obj.status = "misconfigured"
svc_obj.enabled = False
svc_obj.run_on_triage = False
msg = ("Service %s is misconfigured." % service_name)
logger.warning(msg)
else:
svc_obj.status = "available"
# Give the service a chance to tell us what is wrong with the
# config.
try:
service.parse_config(svc_obj.config.to_dict())
except ServiceConfigError as e:
svc_obj.status = "misconfigured"
svc_obj.enabled = False
svc_obj.run_on_triage = False
svc_obj.description = service_description
svc_obj.version = service_version
svc_obj.supported_types = supported_types
svc_obj.save()
self._services[service_class.name] = service_class
# For anything in the database that did not import properly, mark the
# status to unavailable.
svcs = CRITsService.objects()
for svc in svcs:
if svc.name not in self._services:
svc.status = 'unavailable'
svc.enabled = False
svc.run_on_triage = False
svc.save()
def run_service(name, crits_type, identifier, analyst, obj=None,
execute='local', custom_config={}):
"""
Run a service.
:param name: The name of the service to run.
:type name: str
:param crits_type: The type of the object.
:type name: str
:param identifier: The identifier of the object.
:type name: str
:param obj: The CRITs object, if given this overrides crits_type and identifier.
:type obj: CRITs object.
:param analyst: The user updating the results.
:type analyst: str
:param execute: The execution type.
:type execute: str
:param custom_config: Use a custom configuration for this run.
:type custom_config: dict
"""
result = {'success': False}
if crits_type not in settings.CRITS_TYPES:
result['html'] = "Unknown CRITs type."
return result
if name not in enabled_services():
result['html'] = "Service %s is unknown or not enabled." % name
return result
service_class = crits.services.manager.get_service_class(name)
if not service_class:
result['html'] = "Unable to get service class."
return result
if not obj:
obj = class_from_id(crits_type, identifier)
if not obj:
result['html'] = 'Could not find object.'
return result
service = CRITsService.objects(name=name).first()
if not service:
result['html'] = "Unable to find service in database."
return result
# See if the object is a supported type for the service.
if not service_class.supported_for_type(crits_type):
result['html'] = "Service not supported for type '%s'" % crits_type
return result
# Give the service a chance to check for required fields.
try:
service_class.valid_for(obj)
except ServiceConfigError as e:
result['html'] = str(e)
return result
# Get the config from the database and validate the submitted options
# exist.
db_config = service.config.to_dict()
try:
service_class.validate_runtime(custom_config, db_config)
except ServiceConfigError as e:
result['html'] = str(e)
return result
final_config = db_config
# Merge the submitted config with the one from the database.
# This is because not all config options may be submitted.
final_config.update(custom_config)
form = service_class.bind_runtime_form(analyst, final_config)
if form:
if not form.is_valid():
# TODO: return corrected form via AJAX
result['html'] = str(form.errors)
return result
# If the form is valid, create the config using the cleaned data.
final_config = db_config
final_config.update(form.cleaned_data)
logger.info("Running %s on %s, execute=%s" % (name, obj.id, execute))
service_instance = service_class(notify=update_analysis_results,
complete=finish_task)
# Give the service a chance to modify the config that gets saved to the DB.
saved_config = dict(final_config)
service_class.save_runtime_config(saved_config)
task = AnalysisTask(obj, service_instance, analyst)
task.config = AnalysisConfig(**saved_config)
task.start()
add_task(task)
service_instance.set_task(task)
if execute == 'process':
p = Process(target=service_instance.execute, args=(final_config,))
p.start()
elif execute == 'thread':
t = Thread(target=service_instance.execute, args=(final_config,))
t.start()
elif execute == 'local':
service_instance.execute(final_config)
# Return after starting thread so web request can complete.
result['success'] = True
return result