Esempio n. 1
0
def run_service(name,
                type_,
                id_,
                user,
                obj=None,
                execute='local',
                custom_config={},
                **kwargs):
    """
    Run a service.

    :param name: The name of the service to run.
    :type name: str
    :param type_: The type of the object.
    :type type_: str
    :param id_: The identifier of the object.
    :type id_: str
    :param user: The user running the service.
    :type user: str
    :param obj: The CRITs object, if given this overrides crits_type and identifier.
    :type obj: CRITs object.
    :param analyst: The user updating the results.
    :type analyst: str
    :param execute: The execution type.
    :type execute: str
    :param custom_config: Use a custom configuration for this run.
    :type custom_config: dict
    """

    result = {'success': False}
    if type_ not in settings.CRITS_TYPES:
        result['html'] = "Unknown CRITs type."
        return result

    if name not in enabled_services():
        result['html'] = "Service %s is unknown or not enabled." % name
        return result

    service_class = crits.services.manager.get_service_class(name)
    if not service_class:
        result['html'] = "Unable to get service class."
        return result

    if not obj:
        obj = class_from_id(type_, id_)
        if not obj:
            result['html'] = 'Could not find object.'
            return result

    service = CRITsService.objects(name=name).first()
    if not service:
        result['html'] = "Unable to find service in database."
        return result

    # See if the object is a supported type for the service.
    if not service_class.supported_for_type(type_):
        result['html'] = "Service not supported for type '%s'" % type_
        return result

    # When running in threaded mode, each thread needs to have its own copy of
    # the object. If we do not do this then one thread may read() from the
    # object (to get the binary) and then the second would would read() without
    # knowing and get undefined behavior as the file pointer would be who knows
    # where. By giving each thread a local copy they can operate independently.
    #
    # When not running in thread mode this has no effect except wasted memory.
    local_obj = local()
    local_obj.obj = copy.deepcopy(obj)

    # Give the service a chance to check for required fields.
    try:
        service_class.valid_for(local_obj.obj)
        if hasattr(local_obj.obj, 'filedata'):
            if local_obj.obj.filedata.grid_id:
                # Reset back to the start so the service gets the full file.
                local_obj.obj.filedata.seek(0)
    except ServiceConfigError as e:
        result['html'] = str(e)
        return result

    # Get the config from the database and validate the submitted options
    # exist.
    db_config = service.config.to_dict()
    try:
        service_class.validate_runtime(custom_config, db_config)
    except ServiceConfigError as e:
        result['html'] = str(e)
        return result

    final_config = db_config
    # Merge the submitted config with the one from the database.
    # This is because not all config options may be submitted.
    final_config.update(custom_config)

    form = service_class.bind_runtime_form(user, final_config)
    if form:
        if not form.is_valid():
            # TODO: return corrected form via AJAX
            result['html'] = str(form.errors)
            return result

        # If the form is valid, create the config using the cleaned data.
        final_config = db_config
        final_config.update(form.cleaned_data)

    logger.info("Running %s on %s, execute=%s" %
                (name, local_obj.obj.id, execute))
    service_instance = service_class(notify=update_analysis_results,
                                     complete=finish_task)

    # Give the service a chance to modify the config that gets saved to the DB.
    saved_config = dict(final_config)
    service_class.save_runtime_config(saved_config)

    task = AnalysisTask(local_obj.obj, service_instance, user)
    task.config = AnalysisConfig(**saved_config)
    task.start()
    add_task(task)

    service_instance.set_task(task)

    if execute == 'process':
        p = Process(target=service_instance.execute, args=(final_config, ))
        p.start()
    elif execute == 'thread':
        t = Thread(target=service_instance.execute, args=(final_config, ))
        t.start()
    elif execute == 'process_pool':
        if __service_process_pool__ is not None and service.compatability_mode != True:
            __service_process_pool__.apply_async(func=service_work_handler,
                                                 args=(
                                                     service_instance,
                                                     final_config,
                                                 ))
        else:
            logger.warning(
                "Could not run %s on %s, execute=%s, running in process mode" %
                (name, local_obj.obj.id, execute))
            p = Process(target=service_instance.execute, args=(final_config, ))
            p.start()
    elif execute == 'thread_pool':
        if __service_thread_pool__ is not None and service.compatability_mode != True:
            __service_thread_pool__.apply_async(func=service_work_handler,
                                                args=(
                                                    service_instance,
                                                    final_config,
                                                ))
        else:
            logger.warning(
                "Could not run %s on %s, execute=%s, running in thread mode" %
                (name, local_obj.obj.id, execute))
            t = Thread(target=service_instance.execute, args=(final_config, ))
            t.start()
    elif execute == 'local':
        service_instance.execute(final_config)

    # Return after starting thread so web request can complete.
    result['success'] = True
    return result
Esempio n. 2
0
def run_service(name, type_, id_, user, obj=None,
                execute='local', custom_config={}, is_triage_run=False, **kwargs):
    """
    Run a service.

    :param name: The name of the service to run.
    :type name: str
    :param type_: The type of the object.
    :type type_: str
    :param id_: The identifier of the object.
    :type id_: str
    :param user: The user running the service.
    :type user: str
    :param obj: The CRITs object, if given this overrides crits_type and identifier.
    :type obj: CRITs object.
    :param analyst: The user updating the results.
    :type analyst: str
    :param execute: The execution type.
    :type execute: str
    :param custom_config: Use a custom configuration for this run.
    :type custom_config: dict
    """

    result = {'success': False}
    if type_ not in settings.CRITS_TYPES:
        result['html'] = "Unknown CRITs type."
        return result

    if name not in enabled_services():
        result['html'] = "Service %s is unknown or not enabled." % name
        return result

    service_class = crits.services.manager.get_service_class(name)
    if not service_class:
        result['html'] = "Unable to get service class."
        return result

    if not obj:
        obj = class_from_id(type_, id_)
        if not obj:
            result['html'] = 'Could not find object.'
            return result

    service = CRITsService.objects(name=name).first()
    if not service:
        result['html'] = "Unable to find service in database."
        return result

    # See if the object is a supported type for the service.
    if not service_class.supported_for_type(type_):
        result['html'] = "Service not supported for type '%s'" % type_
        return result

    # When running in threaded mode, each thread needs to have its own copy of
    # the object. If we do not do this then one thread may read() from the
    # object (to get the binary) and then the second would would read() without
    # knowing and get undefined behavior as the file pointer would be who knows
    # where. By giving each thread a local copy they can operate independently.
    #
    # When not running in thread mode this has no effect except wasted memory.
    local_obj = local()
    local_obj.obj = copy.deepcopy(obj)

    # Give the service a chance to check for required fields.
    try:
        service_class.valid_for(local_obj.obj)
        if hasattr(local_obj.obj, 'filedata'):
            if local_obj.obj.filedata.grid_id:
                # Reset back to the start so the service gets the full file.
                local_obj.obj.filedata.seek(0)
    except ServiceConfigError as e:
        result['html'] = str(e)
        return result

    # Get the config from the database and validate the submitted options
    # exist.
    db_config = service.config.to_dict()
    try:
        service_class.validate_runtime(custom_config, db_config)
    except ServiceConfigError as e:
        result['html'] = str(e)
        return result

    final_config = db_config
    # Merge the submitted config with the one from the database.
    # This is because not all config options may be submitted.
    final_config.update(custom_config)

    form = service_class.bind_runtime_form(user, final_config)
    if form:
        if not form.is_valid():
            # TODO: return corrected form via AJAX
            result['html'] = str(form.errors)
            return result

        # If the form is valid, create the config using the cleaned data.
        final_config = db_config
        final_config.update(form.cleaned_data)

    logger.info("Running %s on %s, execute=%s" % (name, local_obj.obj.id, execute))
    service_instance = service_class(notify=update_analysis_results,
                                     complete=finish_task)
    # Determine if this service is being run via triage
    if is_triage_run:
        service_instance.is_triage_run = True
        
    # Give the service a chance to modify the config that gets saved to the DB.
    saved_config = dict(final_config)
    service_class.save_runtime_config(saved_config)

    task = AnalysisTask(local_obj.obj, service_instance, user)
    task.config = AnalysisConfig(**saved_config)
    task.start()
    add_task(task)

    service_instance.set_task(task)

    if execute == 'process':
        p = Process(target=service_instance.execute, args=(final_config,))
        p.start()
    elif execute == 'thread':
        t = Thread(target=service_instance.execute, args=(final_config,))
        t.start()
    elif execute == 'process_pool':
        if __service_process_pool__ is not None and service.compatability_mode != True:
            __service_process_pool__.apply_async(func=service_work_handler,
                                                 args=(service_instance, final_config,))
        else:
            logger.warning("Could not run %s on %s, execute=%s, running in process mode" % (name, local_obj.obj.id, execute))
            p = Process(target=service_instance.execute, args=(final_config,))
            p.start()
    elif execute == 'thread_pool':
        if __service_thread_pool__ is not None and service.compatability_mode != True:
            __service_thread_pool__.apply_async(func=service_work_handler,
                                                args=(service_instance, final_config,))
        else:
            logger.warning("Could not run %s on %s, execute=%s, running in thread mode" % (name, local_obj.obj.id, execute))
            t = Thread(target=service_instance.execute, args=(final_config,))
            t.start()
    elif execute == 'local':
        service_instance.execute(final_config)

    # Return after starting thread so web request can complete.
    result['success'] = True
    return result
Esempio n. 3
0
def run_service(name, crits_type, identifier, analyst, obj=None,
                execute='local', custom_config={}):
    """
    Run a service.

    :param name: The name of the service to run.
    :type name: str
    :param crits_type: The type of the object.
    :type name: str
    :param identifier: The identifier of the object.
    :type name: str
    :param obj: The CRITs object, if given this overrides crits_type and identifier.
    :type obj: CRITs object.
    :param analyst: The user updating the results.
    :type analyst: str
    :param execute: The execution type.
    :type execute: str
    :param custom_config: Use a custom configuration for this run.
    :type custom_config: dict
    """

    result = {'success': False}
    if crits_type not in settings.CRITS_TYPES:
        result['html'] = "Unknown CRITs type."
        return result

    if name not in enabled_services():
        result['html'] = "Service %s is unknown or not enabled." % name
        return result

    service_class = crits.services.manager.get_service_class(name)
    if not service_class:
        result['html'] = "Unable to get service class."
        return result

    if not obj:
        obj = class_from_id(crits_type, identifier)
        if not obj:
            result['html'] = 'Could not find object.'
            return result

    service = CRITsService.objects(name=name).first()
    if not service:
        result['html'] = "Unable to find service in database."
        return result

    # See if the object is a supported type for the service.
    if not service_class.supported_for_type(crits_type):
        result['html'] = "Service not supported for type '%s'" % crits_type
        return result

    # Give the service a chance to check for required fields.
    try:
        service_class.valid_for(obj)
    except ServiceConfigError as e:
        result['html'] = str(e)
        return result

    # Get the config from the database and validate the submitted options
    # exist.
    db_config = service.config.to_dict()
    try:
        service_class.validate_runtime(custom_config, db_config)
    except ServiceConfigError as e:
        result['html'] = str(e)
        return result

    final_config = db_config
    # Merge the submitted config with the one from the database.
    # This is because not all config options may be submitted.
    final_config.update(custom_config)

    form = service_class.bind_runtime_form(analyst, final_config)
    if form:
        if not form.is_valid():
            # TODO: return corrected form via AJAX
            result['html'] = str(form.errors)
            return result

        # If the form is valid, create the config using the cleaned data.
        final_config = db_config
        final_config.update(form.cleaned_data)

    logger.info("Running %s on %s, execute=%s" % (name, obj.id, execute))
    service_instance = service_class(notify=update_analysis_results,
                                     complete=finish_task)

    # Give the service a chance to modify the config that gets saved to the DB.
    saved_config = dict(final_config)
    service_class.save_runtime_config(saved_config)

    task = AnalysisTask(obj, service_instance, analyst)
    task.config = AnalysisConfig(**saved_config)
    task.start()
    add_task(task)

    service_instance.set_task(task)

    if execute == 'process':
        p = Process(target=service_instance.execute, args=(final_config,))
        p.start()
    elif execute == 'thread':
        t = Thread(target=service_instance.execute, args=(final_config,))
        t.start()
    elif execute == 'local':
        service_instance.execute(final_config)

    # Return after starting thread so web request can complete.
    result['success'] = True
    return result