def triage_services(status=True): """ Return names of services set to run on triage. """ if status: services = CRIPTsService.objects(run_on_triage=True, status="available") else: services = CRIPTsService.objects(run_on_triage=True) return [s.name for s in services]
def enabled_services(status=True): """ Return names of services which are enabled. """ if status: services = CRIPTsService.objects(enabled=True, status="available") else: services = CRIPTsService.objects(enabled=True) return [s.name for s in services]
def get_form(request, name, cripts_type, identifier): """ Get a configuration form for a service. """ response = {} response['name'] = name analyst = request.user.username service = CRIPTsService.objects(name=name, status__ne="unavailable").first() if not service: msg = 'Service "%s" is unavailable. Please review error logs.' % name response['error'] = msg return HttpResponse(json.dumps(response), content_type="application/json") # Get the class that implements this service. service_class = cripts.services.manager.get_service_class(name) config = service.config.to_dict() form_html = service_class.generate_runtime_form(analyst, config, cripts_type, identifier) if not form_html: return service_run(request, name, cripts_type, identifier) else: response['form'] = form_html return HttpResponse(json.dumps(response), content_type="application/json")
def get_supported_services(cripts_type): """ Get the supported services for a type. """ services = CRIPTsService.objects(enabled=True) for s in sorted(services, key=lambda s: s.name.lower()): if s.supported_types == 'all' or cripts_type in s.supported_types: yield s.name
def get_config(service_name): """ Get the configuration for a service. """ service = CRIPTsService.objects(name=service_name).first() if not service: return None return service.config
def list(request): """ List all services. """ all_services = CRIPTsService.objects() if all_services: all_services = sorted(all_services, key=lambda item: item.name.lower()) return render_to_response('services_list.html', {'services': all_services}, RequestContext(request))
def update_config(service_name, config, analyst): """ Update the configuration for a service. """ service = CRIPTsService.objects(name=service_name).first() service.config = AnalysisConfig(**config) try: #TODO: get/validate the config from service author to set status #update_status(service_name) service.save(username=analyst) return {'success': True} except ValidationError, e: return {'success': False, 'message': e}
def do_edit_config(name, analyst, post_data=None): status = {'success': False} service = CRIPTsService.objects(name=name, status__ne="unavailable").first() if not service: status['config_error'] = 'Service "%s" is unavailable. Please review error logs.' % name status['form'] = '' status['service'] = '' return status # Get the class that implements this service. service_class = cripts.services.manager.get_service_class(name) config = service.config.to_dict() cfg_form, html = service_class.generate_config_form(config) # This isn't a form object. It's the HTML. status['form'] = html status['service'] = service if post_data: #Populate the form with values from the POST request form = cfg_form(post_data) if form.is_valid(): try: service_class.parse_config(form.cleaned_data) except ServiceConfigError as e: service.status = 'misconfigured' service.save() status['config_error'] = str(e) return status result = update_config(name, form.cleaned_data, analyst) if not result['success']: return status service.status = 'available' service.save() else: status['config_error'] = form.errors return status status['success'] = True return status
def set_enabled(service_name, enabled=True, analyst=None): """ Enable/disable a service in CRIPTs. """ if enabled: logger.info("Enabling: %s" % service_name) else: logger.info("Disabling: %s" % service_name) service = CRIPTsService.objects(name=service_name).first() service.enabled = enabled try: service.save(username=analyst) if enabled: url = reverse('cripts.services.views.disable', args=(service_name,)) else: url = reverse('cripts.services.views.enable', args=(service_name,)) return {'success': True, 'url': url} except ValidationError, e: return {'success': False, 'message': e}
def get_service_config(name): status = {'success': False} service = CRIPTsService.objects(name=name, status__ne="unavailable").first() if not service: status['error'] = 'Service "%s" is unavailable. Please review error logs.' % name return status config = service.config.to_dict() service_class = cripts.services.manager.get_service_class(name) if not service_class: status['error'] = 'Service "%s" is unavilable. Please review error logs.' % name return status display_config = service_class.get_config_details(config) status['config'] = display_config status['config_error'] = _get_config_error(service) # TODO: fix code so we don't have to do this status['service'] = service.to_dict() status['success'] = True return status
def _register_services(self, klass): """ Create a dict with names of available services and classes that implement them. This is a recursive function since __subclasses__() only returns direct subclasses. If class A(object):, class B(A):, and class C(B):, then A.__subclasses__() doesn't contain C. All subclasses of the Service class are saved in the `services` dictionary. It is intended that each of these was imported by the _import_services function, but this is not enforced. The key in the dictionary is the `name` class-level field, and the value is the class itself. It is recommended that the service "example" be implemented in a class "ExampleService" defined in a module named "example_service", but this is not enforced, and the only string visible to the end-user/analyst is the service name. """ for service_class in klass.__subclasses__(): # TODO: replace this with a proper check for a valid service if not (hasattr(service_class, "name") and hasattr(service_class, "version")): # If this is a subclass of Service but not an actual service # call this function recursively. self._register_services(service_class) continue service_name = service_class.name service_version = service_class.version service_description = service_class.description supported_types = service_class.supported_types compatability_mode = service_class.compatability_mode #logger.debug("Found service subclass: %s version %s" % # (service_name, service_version)) try: StrictVersion(service_version) except ValueError as e: # Unable to parse the service version msg = ("Service %s is invalid, and will not be available." % service_name) logger.warning(msg) logger.warning(e) continue else: # Only register the service if it is valid. #logger.debug("Registering Service %s" % service_name) svc_obj = CRIPTsService.objects( name=service_class.name).first() service = service_class() if not svc_obj: svc_obj = CRIPTsService() svc_obj.name = service_name try: new_config = service.get_config({}) svc_obj.config = AnalysisConfig(**new_config) except ServiceConfigError: svc_obj.status = "misconfigured" msg = ("Service %s is misconfigured." % service_name) logger.warning(msg) else: svc_obj.status = "available" else: existing_config = svc_obj.config.to_dict() try: new_config = service.get_config(existing_config) svc_obj.config = AnalysisConfig(**new_config) except ServiceConfigError: svc_obj.status = "misconfigured" svc_obj.enabled = False svc_obj.run_on_triage = False msg = ("Service %s is misconfigured." % service_name) logger.warning(msg) else: svc_obj.status = "available" # Give the service a chance to tell us what is wrong with the # config. try: service.parse_config(svc_obj.config.to_dict()) except ServiceConfigError as e: svc_obj.status = "misconfigured" svc_obj.enabled = False svc_obj.run_on_triage = False svc_obj.description = service_description svc_obj.version = service_version svc_obj.supported_types = supported_types svc_obj.compatability_mode = compatability_mode svc_obj.save() self._services[service_class.name] = service_class # For anything in the database that did not import properly, mark the # status to unavailable. svcs = CRIPTsService.objects() for svc in svcs: if svc.name not in self._services: svc.status = 'unavailable' svc.enabled = False svc.run_on_triage = False svc.save()
def run_service(name, type_, id_, user, obj=None, execute='local', custom_config={}, **kwargs): """ Run a service. :param name: The name of the service to run. :type name: str :param type_: The type of the object. :type type_: str :param id_: The identifier of the object. :type id_: str :param user: The user running the service. :type user: str :param obj: The CRIPTs object, if given this overrides cripts_type and identifier. :type obj: CRIPTs object. :param analyst: The user updating the results. :type analyst: str :param execute: The execution type. :type execute: str :param custom_config: Use a custom configuration for this run. :type custom_config: dict """ result = {'success': False} if type_ not in settings.CRIPTS_TYPES: result['html'] = "Unknown CRIPTs type." return result if name not in enabled_services(): result['html'] = "Service %s is unknown or not enabled." % name return result service_class = cripts.services.manager.get_service_class(name) if not service_class: result['html'] = "Unable to get service class." return result if not obj: obj = class_from_id(type_, id_) if not obj: result['html'] = 'Could not find object.' return result service = CRIPTsService.objects(name=name).first() if not service: result['html'] = "Unable to find service in database." return result # See if the object is a supported type for the service. if not service_class.supported_for_type(type_): result['html'] = "Service not supported for type '%s'" % type_ return result # When running in threaded mode, each thread needs to have its own copy of # the object. If we do not do this then one thread may read() from the # object (to get the binary) and then the second would would read() without # knowing and get undefined behavior as the file pointer would be who knows # where. By giving each thread a local copy they can operate independently. # # When not running in thread mode this has no effect except wasted memory. local_obj = local() local_obj.obj = copy.deepcopy(obj) # Give the service a chance to check for required fields. try: service_class.valid_for(local_obj.obj) if hasattr(local_obj.obj, 'filedata'): if local_obj.obj.filedata.grid_id: # Reset back to the start so the service gets the full file. local_obj.obj.filedata.seek(0) except ServiceConfigError as e: result['html'] = str(e) return result # Get the config from the database and validate the submitted options # exist. db_config = service.config.to_dict() try: service_class.validate_runtime(custom_config, db_config) except ServiceConfigError as e: result['html'] = str(e) return result final_config = db_config # Merge the submitted config with the one from the database. # This is because not all config options may be submitted. final_config.update(custom_config) form = service_class.bind_runtime_form(user, final_config) if form: if not form.is_valid(): # TODO: return corrected form via AJAX result['html'] = str(form.errors) return result # If the form is valid, create the config using the cleaned data. final_config = db_config final_config.update(form.cleaned_data) logger.info("Running %s on %s, execute=%s" % (name, local_obj.obj.id, execute)) service_instance = service_class(notify=update_analysis_results, complete=finish_task) # Give the service a chance to modify the config that gets saved to the DB. saved_config = dict(final_config) service_class.save_runtime_config(saved_config) task = AnalysisTask(local_obj.obj, service_instance, user) task.config = AnalysisConfig(**saved_config) task.start() add_task(task) service_instance.set_task(task) if execute == 'process': p = Process(target=service_instance.execute, args=(final_config,)) p.start() elif execute == 'thread': t = Thread(target=service_instance.execute, args=(final_config,)) t.start() elif execute == 'process_pool': if __service_process_pool__ is not None and service.compatability_mode != True: __service_process_pool__.apply_async(func=service_work_handler, args=(service_instance, final_config,)) else: logger.warning("Could not run %s on %s, execute=%s, running in process mode" % (name, local_obj.obj.id, execute)) p = Process(target=service_instance.execute, args=(final_config,)) p.start() elif execute == 'thread_pool': if __service_thread_pool__ is not None and service.compatability_mode != True: __service_thread_pool__.apply_async(func=service_work_handler, args=(service_instance, final_config,)) else: logger.warning("Could not run %s on %s, execute=%s, running in thread mode" % (name, local_obj.obj.id, execute)) t = Thread(target=service_instance.execute, args=(final_config,)) t.start() elif execute == 'local': service_instance.execute(final_config) # Return after starting thread so web request can complete. result['success'] = True return result
def _register_services(self, klass): """ Create a dict with names of available services and classes that implement them. This is a recursive function since __subclasses__() only returns direct subclasses. If class A(object):, class B(A):, and class C(B):, then A.__subclasses__() doesn't contain C. All subclasses of the Service class are saved in the `services` dictionary. It is intended that each of these was imported by the _import_services function, but this is not enforced. The key in the dictionary is the `name` class-level field, and the value is the class itself. It is recommended that the service "example" be implemented in a class "ExampleService" defined in a module named "example_service", but this is not enforced, and the only string visible to the end-user/analyst is the service name. """ for service_class in klass.__subclasses__(): # TODO: replace this with a proper check for a valid service if not (hasattr(service_class, "name") and hasattr(service_class, "version")): # If this is a subclass of Service but not an actual service # call this function recursively. self._register_services(service_class) continue service_name = service_class.name service_version = service_class.version service_description = service_class.description supported_types = service_class.supported_types compatability_mode = service_class.compatability_mode #logger.debug("Found service subclass: %s version %s" % # (service_name, service_version)) try: StrictVersion(service_version) except ValueError as e: # Unable to parse the service version msg = ("Service %s is invalid, and will not be available." % service_name) logger.warning(msg) logger.warning(e) continue else: # Only register the service if it is valid. #logger.debug("Registering Service %s" % service_name) svc_obj = CRIPTsService.objects(name=service_class.name).first() service = service_class() if not svc_obj: svc_obj = CRIPTsService() svc_obj.name = service_name try: new_config = service.get_config({}) svc_obj.config = AnalysisConfig(**new_config) except ServiceConfigError: svc_obj.status = "misconfigured" msg = ("Service %s is misconfigured." % service_name) logger.warning(msg) else: svc_obj.status = "available" else: existing_config = svc_obj.config.to_dict() try: new_config = service.get_config(existing_config) svc_obj.config = AnalysisConfig(**new_config) except ServiceConfigError: svc_obj.status = "misconfigured" svc_obj.enabled = False svc_obj.run_on_triage = False msg = ("Service %s is misconfigured." % service_name) logger.warning(msg) else: svc_obj.status = "available" # Give the service a chance to tell us what is wrong with the # config. try: service.parse_config(svc_obj.config.to_dict()) except ServiceConfigError as e: svc_obj.status = "misconfigured" svc_obj.enabled = False svc_obj.run_on_triage = False svc_obj.description = service_description svc_obj.version = service_version svc_obj.supported_types = supported_types svc_obj.compatability_mode = compatability_mode svc_obj.save() self._services[service_class.name] = service_class # For anything in the database that did not import properly, mark the # status to unavailable. svcs = CRIPTsService.objects() for svc in svcs: if svc.name not in self._services: svc.status = 'unavailable' svc.enabled = False svc.run_on_triage = False svc.save()