def reset_all(self): """ Recreate the services collection. """ logger.debug("Dropping service collection") CRITsService.drop_collection() self._instantiate_collection() self._update_status_all()
def enabled_services(status=True): """ Return names of services which are enabled. """ if status: services = CRITsService.objects(enabled=True, status="available") else: services = CRITsService.objects(enabled=True) return [s.name for s in services]
def triage_services(status=True): """ Return names of services set to run on triage. """ if status: services = CRITsService.objects(run_on_triage=True, status="available") else: services = CRITsService.objects(run_on_triage=True) return [s.name for s in services]
def enabled_services(self): """ Return names of services which are both available and enabled. """ services = CRITsService.objects(enabled=True) return [s.name for s in services if s.name in self._services]
def _instantiate_collection(self): """ Save services information in a Mongo collection. """ logger.debug("Storing service metadata") for service_class in self._services.values(): #If not already in collection current = CRITsService.objects(name=service_class.name).first() if not current: # Add the new service self._add_to_collection(service_class) else: logger.debug("Service %s already exists, checking version." \ % service_class.name) # Check the current version logger.debug('New version: %s -- Old version: %s' \ % (service_class.version, current.version)) if (StrictVersion(service_class.version) != StrictVersion(current['version'])): self._update_service(service_class)
def detail(request, name): """ List all details about a single service. """ service = CRITsService.objects(name=name, status__ne="unavailable").first() if not service: error = 'Service "%s" is unavailable. Please review error logs.' % name return render_to_response('error.html', {'error': error}, RequestContext(request)) # TODO: fix code so we don't have to do this service = service.to_dict() service_class = crits.service_env.manager.get_service_class(name) if user_is_admin(request.user): clean = False # Only show errors if the user is an admin. error = _get_config_error(service) else: # Clean all non-public values for a non-admin clean = True error = None # Replace the dictionary with a list (to preserve order the options # were defined in the Service class), and remove data from any which # are not editable) service['config_list'] = service_class.format_config(service['config'], clean=clean) del service['config'] return render_to_response('services_detail.html', { 'service': service, 'config_error': error }, RequestContext(request))
def export_config(request, name): """ Export a service's configuration. """ # TODO: Present a form to the admin to select file format # Format is either ini or json s = CRITsService.objects(name=name).first() if s: try: data = json.dumps(s.config.to_dict()) except (ValueError, TypeError) as e: error = 'Failed to export %s configuration, please check ' \ 'error logs.' % name logger.error(error) logger.error(e) return render_to_response('error.html', {'error': error}, RequestContext(request)) response = HttpResponse(data, content_type='text/plain') response['Content-Length'] = len(data) fn = name + '.conf' response['Content-Disposition'] = 'attachment; filename="%s"' % fn return response else: error = 'Service "%s" does not exist!' % name render_to_response('error.html', {'error': error}, RequestContext(request))
def detail(request, name): """ List all details about a single service. """ service = CRITsService.objects(name=name, status__ne="unavailable").first() if not service: error = 'Service "%s" is unavailable. Please review error logs.' % name return render_to_response('error.html', {'error': error}, RequestContext(request)) # TODO: fix code so we don't have to do this service = service.to_dict() service_class = crits.service_env.manager.get_service_class(name) if user_is_admin(request.user): clean = False # Only show errors if the user is an admin. error = _get_config_error(service) else: # Clean all non-public values for a non-admin clean = True error = None # Replace the dictionary with a list (to preserve order the options # were defined in the Service class), and remove data from any which # are not editable) service['config_list'] = service_class.format_config(service['config'], clean=clean) del service['config'] return render_to_response('services_detail.html', {'service': service, 'config_error': error}, RequestContext(request))
def get_form(request, name, crits_type, identifier): """ Get a configuration form for a service. """ response = {} response['name'] = name analyst = request.user.username service = CRITsService.objects(name=name, status__ne="unavailable").first() if not service: msg = 'Service "%s" is unavailable. Please review error logs.' % name response['error'] = msg return HttpResponse(json.dumps(response), content_type="application/json") # Get the class that implements this service. service_class = crits.services.manager.get_service_class(name) config = service.config.to_dict() form_html = service_class.generate_runtime_form(analyst, config, crits_type, identifier) if not form_html: return service_run(request, name, crits_type, identifier) else: response['form'] = form_html return HttpResponse(json.dumps(response), content_type="application/json")
def update_status(self, service_name): """ Look up a service, and verify it is configured correctly """ service = CRITsService.objects(name=service_name).first() self._update_status(service)
def list(request): """ List all services. """ all_services = CRITsService.objects().order_by('+name') return render_to_response('services_list.html', {'services': all_services}, RequestContext(request))
def start_pyew_shell(request, id_, token): # Make sure we can find pyew svc = CRITsService.objects(name='Pyew').first() if not svc: text = "\nPyew not found" request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) sc = svc.config pyew = str(sc['pyew']) if not os.path.exists(pyew): text = "\nPyew not found" request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) # Find CRITs user by token query = {'unsupported_attrs.pyew_token': token} user = CRITsUser.objects(__raw__=query).first() if not user: text = "\nCould not validate user" request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) # Remove this one-time use token ua = user.unsupported_attrs delattr(ua, 'pyew_token') user.unsupported_attrs = ua try: user.save() except: pass # Make sure we have a sample to work with that this user has access to sample = Sample.objects(id=id_, source__name__in=user.get_sources_list()).first() if not sample: text = "\nNo Sample found" request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) sample_data = sample.filedata.read() if not sample_data: text = "\nCould not get Sample from GridFS: %s" % id_ request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1) # write Sample to disk # temp_sample is the sample to read try: temp_sample = tempfile.NamedTemporaryFile(delete=False) sample_name = temp_sample.name temp_sample.write(sample_data) temp_sample.close() except Exception, e: text = "\nError writing file to disk: %s" % e request.ws_stream.send_message(base64.b64encode(text), binary=False) sys.exit(1)
def get_supported_services(crits_type): """ Get the supported services for a type. """ services = CRITsService.objects(enabled=True) for s in services: if s.supported_types == 'all' or crits_type in s.supported_types: yield s.name
def get_supported_services(crits_type): """ Get the supported services for a type. """ services = CRITsService.objects(enabled=True) for s in sorted(services, key=lambda s: s.name.lower()): if s.supported_types == 'all' or crits_type in s.supported_types: yield s.name
def triage_services(self): """ Return names of available services set to run on triage. """ # TODO: This doesn't care if the service is enabled, should it? # What is the correct behavior when enabled=False, run_on_triage=True? services = CRITsService.objects(run_on_triage=True) return [s.name for s in services if s.name in self._services]
def get_config(service_name): """ Get the configuration for a service. """ service = CRITsService.objects(name=service_name).first() if not service: return None return service.config
def get_config(self, service_name): """ Get the configuration for a service. """ service = CRITsService.objects(name=service_name).first() try: return service.config except Exception, e: logger.exception(e) return self.get_service_class(service_name).build_default_config()
def pyew_port(request): svc = CRITsService.objects(name='Pyew').first() if not svc: return HttpResponse(json.dumps({}), content_type="application/json") sc = svc.config port = str(sc['port']) secure = str(sc['secure']) data = {'port': port, 'secure': secure} return HttpResponse(json.dumps(data), content_type="application/json")
def list(request): """ List all services. """ all_services = CRITsService.objects() if all_services: all_services = sorted(all_services, key=lambda item: item.name.lower()) return render(request, 'services_list.html', {'services': all_services})
def pyew_port(request): svc = CRITsService.objects(name="Pyew").first() if not svc: return HttpResponse(json.dumps({}), content_type="application/json") sc = svc.config port = str(sc["port"]) secure = str(sc["secure"]) data = {"port": port, "secure": secure} return HttpResponse(json.dumps(data), content_type="application/json")
def pyew_port(request): svc = CRITsService.objects(name='Pyew').first() if not svc: return HttpResponse(json.dumps({}), mimetype="application/json") sc = svc.config port = str(sc['port']) secure = str(sc['secure']) data = {'port': port, 'secure': secure} return HttpResponse(json.dumps(data), mimetype="application/json")
def _update_status_all(self): """ Ensure services are configured properly. """ all_services = CRITsService.objects() for service in all_services: if 'name' not in service.to_dict(): logger.warning("Invalid Service in Collection") logger.debug(service) continue self._update_status(service)
def update_config(self, service_name, config, analyst): """ Update the configuration for a service. """ service = CRITsService.objects(name=service_name).first() service.config = AnalysisConfig(**config) try: service.save(username=analyst) self.update_status(service_name) return {'success': True} except ValidationError, e: return {'success': False, 'message': e}
def update_config(service_name, config, user): """ Update the configuration for a service. """ service = CRITsService.objects(name=service_name).first() service.config = AnalysisConfig(**config) try: #TODO: get/validate the config from service author to set status #update_status(service_name) service.save(username=user.username) return {'success': True} except ValidationError, e: return {'success': False, 'message': e}
def update_config(service_name, config, analyst): """ Update the configuration for a service. """ service = CRITsService.objects(name=service_name).first() service.config = AnalysisConfig(**config) try: #TODO: get/validate the config from service author to set status #update_status(service_name) service.save(username=analyst) return {'success': True} except ValidationError, e: return {'success': False, 'message': e}
def set_triage(self, service_name, enabled=True, analyst=None): """ Enable/disable a service for running on triage (upload). """ if enabled: logger.info("Enabling triage: %s" % service_name) else: logger.info("Disabling triage: %s" % service_name) service = CRITsService.objects(name=service_name).first() service.run_on_triage = enabled try: service.save(username=analyst) return {'success': True} except ValidationError, e: return {'success': False, 'message': e}
def set_enabled(self, service_name, enabled=True, analyst=None): """ Enable/disable a service in CRITs. """ if enabled: logger.info("Enabling: %s" % service_name) else: logger.info("Disabling: %s" % service_name) service = CRITsService.objects(name=service_name).first() service.enabled = enabled try: service.save(username=analyst) return {'success': True} except ValidationError, e: return {'success': False, 'message': e}
def do_edit_config(name, analyst, post_data=None): status = {'success': False} service = CRITsService.objects(name=name, status__ne="unavailable").first() if not service: status[ 'config_error'] = 'Service "%s" is unavailable. Please review error logs.' % name status['form'] = '' status['service'] = '' return status # Get the class that implements this service. service_class = crits.services.manager.get_service_class(name) config = service.config.to_dict() cfg_form, html = service_class.generate_config_form(config) # This isn't a form object. It's the HTML. status['form'] = html status['service'] = service if post_data: #Populate the form with values from the POST request form = cfg_form(post_data) if form.is_valid(): try: service_class.parse_config(form.cleaned_data) except ServiceConfigError as e: service.status = 'misconfigured' service.save() status['config_error'] = str(e) return status result = update_config(name, form.cleaned_data, analyst) if not result['success']: return status service.status = 'available' service.save() else: status['config_error'] = form.errors return status status['success'] = True return status
def do_edit_config(name, analyst, post_data=None): status = {'success': False} service = CRITsService.objects(name=name, status__ne="unavailable").first() if not service: status['config_error'] = 'Service "%s" is unavailable. Please review error logs.' % name status['form'] = '' status['service'] = '' return status # Get the class that implements this service. service_class = crits.services.manager.get_service_class(name) config = service.config.to_dict() cfg_form, html = service_class.generate_config_form(config) # This isn't a form object. It's the HTML. status['form'] = html status['service'] = service if post_data: #Populate the form with values from the POST request form = cfg_form(post_data) if form.is_valid(): try: service_class.parse_config(form.cleaned_data) except ServiceConfigError as e: service.status = 'misconfigured' service.save() status['config_error'] = str(e) return status result = update_config(name, form.cleaned_data, analyst) if not result['success']: return status service.status = 'available' service.save() else: status['config_error'] = form.errors return status status['success'] = True return status
def set_enabled(service_name, enabled=True, analyst=None): """ Enable/disable a service in CRITs. """ if enabled: logger.info("Enabling: %s" % service_name) else: logger.info("Disabling: %s" % service_name) service = CRITsService.objects(name=service_name).first() service.enabled = enabled try: service.save(username=analyst) if enabled: url = reverse('crits.services.views.disable', args=(service_name,)) else: url = reverse('crits.services.views.enable', args=(service_name,)) return {'success': True, 'url': url} except ValidationError, e: return {'success': False, 'message': e}
def get_service_config(name): status = {'success': False} service = CRITsService.objects(name=name, status__ne="unavailable").first() if not service: status['error'] = 'Service "%s" is unavailable. Please review error logs.' % name return status config = service.config.to_dict() service_class = crits.services.manager.get_service_class(name) if not service_class: status['error'] = 'Service "%s" is unavilable. Please review error logs.' % name return status display_config = service_class.get_config_details(config) status['config'] = display_config status['config_error'] = _get_config_error(service) # TODO: fix code so we don't have to do this status['service'] = service.to_dict() status['success'] = True return status
def reset_config(request, name): """ Reset a service's configuration. This uses the values from the service class's default_config variable. """ s = CRITsService.objects(name=name).first() if not s: error = 'Service "%s" does not exist or is not configured properly.' \ ' Please Review error logs.' % name return render_to_response('error.html', {'error': error}, RequestContext(request)) result = crits.service_env.manager.reset_config(name, request.user.username) if not result['success']: return render_to_response('error.html', {'error': result['message']}, RequestContext(request)) return redirect(reverse('crits.services.views.detail', args=[name]))
def get_form(request, name, crits_type, identifier): """ Get a configuration form for a service. """ response = {} response['name'] = name service = CRITsService.objects(name=name, status__ne="unavailable").first() # TODO: return an AJAX error instead if not service: msg = 'Service "%s" is unavailable. Please review error logs.' % name response['error'] = msg return HttpResponse(json.dumps(response), mimetype="application/json") # Get the class that implements this service. service_class = crits.service_env.manager.get_service_class(name) # format_config returns a list of tuples config = service_class.format_config(service.config, printable=False) ServiceRunConfigForm = make_run_config_form(service_class) if not ServiceRunConfigForm: # this should only happen if there are no config options and the # service is rerunnable. response['redirect'] = reverse('crits.services.views.service_run', args=[name, crits_type, identifier]) else: form = ServiceRunConfigForm(dict(config)) response['form'] = render_to_string( "services_run_form.html", { 'name': name, 'form': form, 'crits_type': crits_type, 'identifier': identifier }, RequestContext(request)) return HttpResponse(json.dumps(response), mimetype="application/json")
def _update_service(self, service_class): """ Update a service in the database. """ logger.info("Updating service %s in MongoDB" % service_class.name) new_config = service_class.build_default_config() current = CRITsService.objects(name=service_class.name).first() if current: current_config = current.config.to_dict() # Log removed keys removed_keys = set(current_config.keys()) - set(new_config.keys()) if removed_keys: logger.warning("Old service configuration options removed: %s" % str(removed_keys)) # Log added keys added_keys = set(new_config.keys()) - set(current_config.keys()) if added_keys: logger.warning("New service configuration options added: %s" % str(added_keys)) # All new items need to be added to the current config for key in added_keys: current_config[key] = new_config[key] current.config = AnalysisConfig(**current_config) # Update the version number current.version = service_class.version try: current.save() logger.info('Updated service %s successfully' % service_class.name) except: logger.warning('Failed to update service %s' % service_class.name)
def set_triage(service_name, enabled=True, user=None): """ Enable/disable a service for running on triage (upload). """ if enabled: logger.info("Enabling triage: %s" % service_name) else: logger.info("Disabling triage: %s" % service_name) service = CRITsService.objects(name=service_name).first() service.run_on_triage = enabled try: service.save(username=user.username) if enabled: url = reverse('crits-services-views-disable_triage', args=(service_name,)) else: url = reverse('crits-services-views-enable_triage', args=(service_name,)) return {'success': True, 'url': url} except ValidationError, e: return {'success': False, 'message': e}
def set_triage(service_name, enabled=True, user=None): """ Enable/disable a service for running on triage (upload). """ if enabled: logger.info("Enabling triage: %s" % service_name) else: logger.info("Disabling triage: %s" % service_name) service = CRITsService.objects(name=service_name).first() service.run_on_triage = enabled try: service.save(username=user.username) if enabled: url = reverse('crits.services.views.disable_triage', args=(service_name,)) else: url = reverse('crits.services.views.enable_triage', args=(service_name,)) return {'success': True, 'url': url} except ValidationError, e: return {'success': False, 'message': e}
def get_form(request, name, crits_type, identifier): """ Get a configuration form for a service. """ response = {} response['name'] = name service = CRITsService.objects(name=name, status__ne="unavailable").first() # TODO: return an AJAX error instead if not service: msg = 'Service "%s" is unavailable. Please review error logs.' % name response['error'] = msg return HttpResponse(json.dumps(response), mimetype="application/json") # Get the class that implements this service. service_class = crits.service_env.manager.get_service_class(name) # format_config returns a list of tuples config = service_class.format_config(service.config, printable=False) ServiceRunConfigForm = make_run_config_form(service_class) if not ServiceRunConfigForm: # this should only happen if there are no config options and the # service is rerunnable. response['redirect'] = reverse('crits.services.views.service_run', args=[name, crits_type, identifier]) else: form = ServiceRunConfigForm(dict(config)) response['form'] = render_to_string("services_run_form.html", {'name': name, 'form': form, 'crits_type': crits_type, 'identifier': identifier}, RequestContext(request)) return HttpResponse(json.dumps(response), mimetype="application/json")
def run_service(name, type_, id_, user, obj=None, execute='local', custom_config={}, is_triage_run=False, **kwargs): """ Run a service. :param name: The name of the service to run. :type name: str :param type_: The type of the object. :type type_: str :param id_: The identifier of the object. :type id_: str :param user: The user running the service. :type user: str :param obj: The CRITs object, if given this overrides crits_type and identifier. :type obj: CRITs object. :param analyst: The user updating the results. :type analyst: str :param execute: The execution type. :type execute: str :param custom_config: Use a custom configuration for this run. :type custom_config: dict """ result = {'success': False} if type_ not in settings.CRITS_TYPES: result['html'] = "Unknown CRITs type." return result if name not in enabled_services(): result['html'] = "Service %s is unknown or not enabled." % name return result service_class = crits.services.manager.get_service_class(name) if not service_class: result['html'] = "Unable to get service class." return result if not obj: obj = class_from_id(type_, id_) if not obj: result['html'] = 'Could not find object.' return result service = CRITsService.objects(name=name).first() if not service: result['html'] = "Unable to find service in database." return result # See if the object is a supported type for the service. if not service_class.supported_for_type(type_): result['html'] = "Service not supported for type '%s'" % type_ return result # When running in threaded mode, each thread needs to have its own copy of # the object. If we do not do this then one thread may read() from the # object (to get the binary) and then the second would would read() without # knowing and get undefined behavior as the file pointer would be who knows # where. By giving each thread a local copy they can operate independently. # # When not running in thread mode this has no effect except wasted memory. local_obj = local() local_obj.obj = copy.deepcopy(obj) # Give the service a chance to check for required fields. try: service_class.valid_for(local_obj.obj) if hasattr(local_obj.obj, 'filedata'): if local_obj.obj.filedata.grid_id: # Reset back to the start so the service gets the full file. local_obj.obj.filedata.seek(0) except ServiceConfigError as e: result['html'] = str(e) return result # Get the config from the database and validate the submitted options # exist. db_config = service.config.to_dict() try: service_class.validate_runtime(custom_config, db_config) except ServiceConfigError as e: result['html'] = str(e) return result final_config = db_config # Merge the submitted config with the one from the database. # This is because not all config options may be submitted. final_config.update(custom_config) form = service_class.bind_runtime_form(user, final_config) if form: if not form.is_valid(): # TODO: return corrected form via AJAX result['html'] = str(form.errors) return result # If the form is valid, create the config using the cleaned data. final_config = db_config final_config.update(form.cleaned_data) logger.info("Running %s on %s, execute=%s" % (name, local_obj.obj.id, execute)) service_instance = service_class(notify=update_analysis_results, complete=finish_task) # Determine if this service is being run via triage if is_triage_run: service_instance.is_triage_run = True # Give the service a chance to modify the config that gets saved to the DB. saved_config = dict(final_config) service_class.save_runtime_config(saved_config) task = AnalysisTask(local_obj.obj, service_instance, user) task.config = AnalysisConfig(**saved_config) task.start() add_task(task) service_instance.set_task(task) if execute == 'process': p = Process(target=service_instance.execute, args=(final_config,)) p.start() elif execute == 'thread': t = Thread(target=service_instance.execute, args=(final_config,)) t.start() elif execute == 'process_pool': if __service_process_pool__ is not None and service.compatability_mode != True: __service_process_pool__.apply_async(func=service_work_handler, args=(service_instance, final_config,)) else: logger.warning("Could not run %s on %s, execute=%s, running in process mode" % (name, local_obj.obj.id, execute)) p = Process(target=service_instance.execute, args=(final_config,)) p.start() elif execute == 'thread_pool': if __service_thread_pool__ is not None and service.compatability_mode != True: __service_thread_pool__.apply_async(func=service_work_handler, args=(service_instance, final_config,)) else: logger.warning("Could not run %s on %s, execute=%s, running in thread mode" % (name, local_obj.obj.id, execute)) t = Thread(target=service_instance.execute, args=(final_config,)) t.start() elif execute == 'local': service_instance.execute(final_config) # Return after starting thread so web request can complete. result['success'] = True return result