コード例 #1
0
def create_services(ds: AssemblylineDatastore, log=None, limit=None):
    if not limit:
        limit = len(SERVICES)

    for svc_name, svc in list(SERVICES.items())[:limit]:
        service_data = {
            "name": svc_name,
            "enabled": True,
            "category": svc[0],
            "stage": svc[1],
            "version": "3.3.0",
            "docker_config": {
                "image": f"cccs/alsvc_{svc_name.lower()}:latest",
            },
        }

        if random.choice([True, False]):
            service_data['update_config'] = {
                "method": "run",
                "sources": [random_model_obj(UpdateSource)],
                "update_interval_seconds": 600,
                "generates_signatures": True
            }

        service_data = Service(service_data)
        # Save a v3 service
        ds.service.save(f"{service_data.name}_{service_data.version}",
                        service_data)

        # Save the same service as v4
        service_data.version = "4.0.0"
        ds.service.save(f"{service_data.name}_{service_data.version}",
                        service_data)

        # Save the default delta entry
        ds.service_delta.save(service_data.name,
                              {"version": service_data.version})
        if log:
            log.info(f'\t{svc_name}')

    ds.service_delta.commit()
    ds.service.commit()
コード例 #2
0
    def register_service(self, service_data, log_prefix=""):
        keep_alive = True

        try:
            # Get heuristics list
            heuristics = service_data.pop('heuristics', None)

            # Patch update_channel, registry_type before Service registration object creation
            service_data['update_channel'] = service_data.get(
                'update_channel',
                self.config.services.preferred_update_channel)
            service_data['docker_config']['registry_type'] = service_data['docker_config'] \
                .get('registry_type', self.config.services.preferred_registry_type)
            service_data['privileged'] = service_data.get(
                'privileged', self.config.services.prefer_service_privileged)
            for dep in service_data.get('dependencies', {}).values():
                dep['container']['registry_type'] = dep.get(
                    'registry_type',
                    self.config.services.preferred_registry_type)

            # Pop unused registration service_data
            for x in ['file_required', 'tool_version']:
                service_data.pop(x, None)

            # Create Service registration object
            service = Service(service_data)

            # Fix service version, we don't need to see the stable label
            service.version = service.version.replace('stable', '')

            # Save service if it doesn't already exist
            if not self.datastore.service.exists(
                    f'{service.name}_{service.version}'):
                self.datastore.service.save(
                    f'{service.name}_{service.version}', service)
                self.datastore.service.commit()
                self.log.info(f"{log_prefix}{service.name} registered")
                keep_alive = False

            # Save service delta if it doesn't already exist
            if not self.datastore.service_delta.exists(service.name):
                self.datastore.service_delta.save(service.name,
                                                  {'version': service.version})
                self.datastore.service_delta.commit()
                self.log.info(f"{log_prefix}{service.name} "
                              f"version ({service.version}) registered")

            new_heuristics = []
            if heuristics:
                plan = self.datastore.heuristic.get_bulk_plan()
                for index, heuristic in enumerate(heuristics):
                    heuristic_id = f'#{index}'  # Set heuristic id to it's position in the list for logging purposes
                    try:
                        # Append service name to heuristic ID
                        heuristic[
                            'heur_id'] = f"{service.name.upper()}.{str(heuristic['heur_id'])}"

                        # Attack_id field is now a list, make it a list if we receive otherwise
                        attack_id = heuristic.get('attack_id', None)
                        if isinstance(attack_id, str):
                            heuristic['attack_id'] = [attack_id]

                        heuristic = Heuristic(heuristic)
                        heuristic_id = heuristic.heur_id
                        plan.add_upsert_operation(heuristic_id, heuristic)
                    except Exception as e:
                        msg = f"{service.name} has an invalid heuristic ({heuristic_id}): {str(e)}"
                        self.log.exception(f"{log_prefix}{msg}")
                        raise ValueError(msg)

                for item in self.datastore.heuristic.bulk(plan)['items']:
                    if item['update']['result'] != "noop":
                        new_heuristics.append(item['update']['_id'])
                        self.log.info(
                            f"{log_prefix}{service.name} "
                            f"heuristic {item['update']['_id']}: {item['update']['result'].upper()}"
                        )

                self.datastore.heuristic.commit()

            service_config = self.datastore.get_service_with_delta(
                service.name, as_obj=False)

            # Notify components watching for service config changes
            self.event_sender.send(service.name, {
                'operation': Operation.Added,
                'name': service.name
            })

        except ValueError as e:  # Catch errors when building Service or Heuristic model(s)
            raise e

        return dict(keep_alive=keep_alive,
                    new_heuristics=new_heuristics,
                    service_config=service_config or dict())
コード例 #3
0
def add_service(**_):
    """
    Add a service using its yaml manifest

    Variables:
    None

    Arguments:
    None

    Data Block:
    <service_manifest.yml content>

    Result example:
    { "success": true }  # Return true is the service was added
    """
    data = request.data
    enable_allowed = True
    try:
        if b"$SERVICE_TAG" in data:
            tmp_service = yaml.safe_load(data)
            tmp_service.pop('tool_version', None)
            tmp_service.pop('file_required', None)
            tmp_service.pop('heuristics', [])

            # Apply global preferences, if missing, to get the appropriate container image tags
            tmp_service['update_channel'] = tmp_service.get(
                'update_channel', config.services.preferred_update_channel)
            tmp_service['docker_config']['registry_type'] = tmp_service['docker_config'] \
                .get('registry_type', config.services.preferred_registry_type)
            _, tag_name, _ = get_latest_tag_for_service(
                Service(tmp_service), config, LOGGER)
            enable_allowed = bool(tag_name)
            tag_name = tag_name.encode() if tag_name else b'latest'
            data = data.replace(b"$SERVICE_TAG", tag_name)

        service = yaml.safe_load(data)
        # Pop the data not part of service model
        service.pop('tool_version', None)
        service.pop('file_required', None)
        heuristics = service.pop('heuristics', [])

        # Validate submission params
        for sp in service.get('submission_params', []):
            if sp['type'] == 'list' and 'list' not in sp:
                return make_api_response(
                    "",
                    err=
                    f"Missing list field for submission param: {sp['name']}",
                    status_code=400)

            if sp['default'] != sp['value']:
                return make_api_response(
                    "",
                    err=
                    f"Default and value mismatch for submission param: {sp['name']}",
                    status_code=400)

        # Apply default global configurations (if absent in service configuration)
        if not service.get('update_channel'):
            service[
                'update_channel'] = config.services.preferred_update_channel
        if not service.get('docker_config', {}).get('registry_type'):
            service['docker_config'][
                'registry_type'] = config.services.preferred_registry_type

        # Privilege can be set explicitly but also granted to services that don't require the file for analysis
        service['privileged'] = service.get(
            'privileged', config.services.prefer_service_privileged)

        for dep in service.get('dependencies', {}).values():
            dep['container']['registry_type'] = dep.get(
                'registry_type', config.services.preferred_registry_type)
        service['enabled'] = service['enabled'] and enable_allowed

        # Load service info
        service = Service(service)

        # Fix service version, we don't want to see stable if it's a stable container
        service.version = service.version.replace("stable", "")

        # Save service if it doesn't already exist
        if not STORAGE.service.get_if_exists(
                f'{service.name}_{service.version}'):
            STORAGE.service.save(f'{service.name}_{service.version}', service)
            STORAGE.service.commit()

        # Save service delta if it doesn't already exist
        if not STORAGE.service_delta.get_if_exists(service.name):
            STORAGE.service_delta.save(service.name,
                                       {'version': service.version})
            STORAGE.service_delta.commit()

        # Notify components watching for service config changes
        event_sender.send(service.name, {
            'operation': Operation.Added,
            'name': service.name
        })

        new_heuristics = []
        if heuristics:
            plan = STORAGE.heuristic.get_bulk_plan()
            for _, heuristic in enumerate(heuristics):
                try:
                    # Append service name to heuristic ID
                    heuristic[
                        'heur_id'] = f"{service.name.upper()}.{str(heuristic['heur_id'])}"

                    # Attack_id field is now a list, make it a list if we receive otherwise
                    attack_id = heuristic.get('attack_id', None)
                    if isinstance(attack_id, str):
                        heuristic['attack_id'] = [attack_id]

                    heuristic = Heuristic(heuristic)
                    heuristic_id = heuristic.heur_id
                    plan.add_upsert_operation(heuristic_id, heuristic)
                except Exception:
                    raise ValueError("Error parsing heuristics")

            for item in STORAGE.heuristic.bulk(plan)['items']:
                if item['update']['result'] != "noop":
                    new_heuristics.append(item['update']['_id'])

            STORAGE.heuristic.commit()

        return make_api_response(
            dict(service_name=service.name, new_heuristics=new_heuristics))
    except ValueError as e:  # Catch errors when building Service or Heuristic model(s)
        return make_api_response("", err=str(e), status_code=400)
コード例 #4
0
def add_service(**_):
    """
    Add a service using its yaml manifest

    Variables:
    None

    Arguments:
    None

    Data Block:
    <service_manifest.yml content>

    Result example:
    { "success": true }  # Return true is the service was added
    """
    data = request.data

    try:
        if b"$SERVICE_TAG" in data:
            tmp_service = yaml.safe_load(data)
            tmp_service.pop('tool_version', None)
            tmp_service.pop('file_required', None)
            tmp_service.pop('heuristics', [])
            tmp_service[
                'update_channel'] = config.services.preferred_update_channel
            image_name, tag_name, _ = get_latest_tag_for_service(
                Service(tmp_service), config, LOGGER)
            if tag_name:
                tag_name = tag_name.encode()
            else:
                tag_name = b'latest'

            data = data.replace(b"$SERVICE_TAG", tag_name)

        service = yaml.safe_load(data)
        # Pop the data not part of service model
        service.pop('tool_version', None)
        service.pop('file_required', None)
        heuristics = service.pop('heuristics', [])

        # Validate submission params
        for sp in service.get('submission_params', []):
            if sp['type'] == 'list' and 'list' not in sp:
                return make_api_response(
                    "",
                    err=
                    f"Missing list field for submission param: {sp['name']}",
                    status_code=400)

            if sp['default'] != sp['value']:
                return make_api_response(
                    "",
                    err=
                    f"Default and value mismatch for submission param: {sp['name']}",
                    status_code=400)

        # Fix update_channel with the system default
        service['update_channel'] = config.services.preferred_update_channel

        # Load service info
        service = Service(service)

        # Fix service version, we don't want to see stable if it's a stable container
        service.version = service.version.replace("stable", "")

        # Save service if it doesn't already exist
        if not STORAGE.service.get_if_exists(
                f'{service.name}_{service.version}'):
            STORAGE.service.save(f'{service.name}_{service.version}', service)
            STORAGE.service.commit()

        # Save service delta if it doesn't already exist
        if not STORAGE.service_delta.get_if_exists(service.name):
            STORAGE.service_delta.save(service.name,
                                       {'version': service.version})
            STORAGE.service_delta.commit()

        new_heuristics = []
        if heuristics:
            plan = STORAGE.heuristic.get_bulk_plan()
            for index, heuristic in enumerate(heuristics):
                try:
                    # Append service name to heuristic ID
                    heuristic[
                        'heur_id'] = f"{service.name.upper()}.{str(heuristic['heur_id'])}"

                    # Attack_id field is now a list, make it a list if we receive otherwise
                    attack_id = heuristic.get('attack_id', None)
                    if isinstance(attack_id, str):
                        heuristic['attack_id'] = [attack_id]

                    heuristic = Heuristic(heuristic)
                    heuristic_id = heuristic.heur_id
                    plan.add_upsert_operation(heuristic_id, heuristic)
                except Exception:
                    raise ValueError("Error parsing heuristics")

            for item in STORAGE.heuristic.bulk(plan)['items']:
                if item['update']['result'] != "noop":
                    new_heuristics.append(item['update']['_id'])

            STORAGE.heuristic.commit()

        return make_api_response(
            dict(service_name=service.name, new_heuristics=new_heuristics))
    except ValueError as e:  # Catch errors when building Service or Heuristic model(s)
        return make_api_response("", err=str(e), status_code=400)