Beispiel #1
0
    def transform_to_ecs(self):
        ecs_dict = {'ecs': {'version': self.logconfig['ecs_version']}}
        if self.logconfig['cloud_provider']:
            ecs_dict['cloud'] = {'provider': self.logconfig['cloud_provider']}
        ecs_keys = self.logconfig['ecs'].split()
        for ecs_key in ecs_keys:
            original_keys = self.logconfig[ecs_key]
            v = utils.value_from_nesteddict_by_dottedkeylist(
                self.__logdata_dict, original_keys)
            if v:
                new_ecs_dict = utils.put_value_into_nesteddict(ecs_key, v)
                if '.ip' in ecs_key:
                    # IPアドレスの場合は、validation
                    try:
                        ipaddress.ip_address(v)
                    except ValueError:
                        continue
                ecs_dict = utils.merge_dicts(ecs_dict, new_ecs_dict)
        if 'cloud' in ecs_dict:
            # Set AWS Account ID
            if ('account' in ecs_dict['cloud']
                    and 'id' in ecs_dict['cloud']['account']):
                if ecs_dict['cloud']['account']['id'] in ('unknown', ):
                    # for vpcflowlogs
                    ecs_dict['cloud']['account'] = {'id': self.accountid}
            elif self.accountid:
                ecs_dict['cloud']['account'] = {'id': self.accountid}
            else:
                ecs_dict['cloud']['account'] = {'id': 'unknown'}

            # Set AWS Region
            if 'region' in ecs_dict['cloud']:
                pass
            elif self.region:
                ecs_dict['cloud']['region'] = self.region
            else:
                ecs_dict['cloud']['region'] = 'unknown'

        # get info from firelens metadata of Elastic Container Serivce
        if 'ecs_task_arn' in self.__logdata_dict:
            ecs_task_arn_taple = self.__logdata_dict['ecs_task_arn'].split(':')
            ecs_dict['cloud']['account']['id'] = ecs_task_arn_taple[4]
            ecs_dict['cloud']['region'] = ecs_task_arn_taple[3]
            if 'ec2_instance_id' in self.__logdata_dict:
                ecs_dict['cloud']['instance'] = {
                    'id': self.__logdata_dict['ec2_instance_id']
                }
            ecs_dict['container'] = {
                'id': self.__logdata_dict['container_id'],
                'name': self.__logdata_dict['container_name']
            }

        static_ecs_keys = self.logconfig['static_ecs']
        if static_ecs_keys:
            for static_ecs_key in static_ecs_keys.split():
                new_ecs_dict = utils.put_value_into_nesteddict(
                    static_ecs_key, self.logconfig[static_ecs_key])
                ecs_dict = utils.merge_dicts(ecs_dict, new_ecs_dict)
        self.__logdata_dict = utils.merge_dicts(self.__logdata_dict, ecs_dict)
Beispiel #2
0
    def transform_to_ecs(self):
        ecs_dict = {'ecs': {'version': self.logconfig['ecs_version']}}
        if self.logconfig['cloud_provider']:
            ecs_dict['cloud'] = {'provider': self.logconfig['cloud_provider']}
        ecs_dict = self.get_value_and_input_into_ecs_dict(ecs_dict)
        if 'cloud' in ecs_dict:
            # Set AWS Account ID
            if ('account' in ecs_dict['cloud']
                    and 'id' in ecs_dict['cloud']['account']):
                if ecs_dict['cloud']['account']['id'] in ('unknown', ):
                    # for vpcflowlogs
                    ecs_dict['cloud']['account'] = {'id': self.accountid}
            elif self.accountid:
                ecs_dict['cloud']['account'] = {'id': self.accountid}
            else:
                ecs_dict['cloud']['account'] = {'id': 'unknown'}

            # Set AWS Region
            if 'region' in ecs_dict['cloud']:
                pass
            elif self.region:
                ecs_dict['cloud']['region'] = self.region
            else:
                ecs_dict['cloud']['region'] = 'unknown'

        # get info from firelens metadata of Elastic Container Serivce
        if 'ecs_task_arn' in self.logmeta:
            ecs_task_arn_taple = self.logmeta['ecs_task_arn'].split(':')
            ecs_dict['cloud']['account']['id'] = ecs_task_arn_taple[4]
            ecs_dict['cloud']['region'] = ecs_task_arn_taple[3]
            if 'ec2_instance_id' in self.logmeta:
                ecs_dict['cloud']['instance'] = {
                    'id': self.logmeta['ec2_instance_id']}
            ecs_dict['container'] = {
                'id': self.logmeta['container_id'],
                'name': self.logmeta['container_name']}

        if '__error_message' in self.logmeta:
            self.__logdata_dict['error'] = {
                'message': self.logmeta['__error_message']}
            del self.logmeta['__error_message']

        static_ecs_keys = self.logconfig['static_ecs']
        for static_ecs_key in static_ecs_keys:
            v = copy.copy(self.logconfig[static_ecs_key])
            new_ecs_dict = utils.put_value_into_nesteddict(static_ecs_key, v)
            ecs_dict = utils.merge_dicts(ecs_dict, new_ecs_dict)
        self.__logdata_dict = utils.merge_dicts(self.__logdata_dict, ecs_dict)
Beispiel #3
0
 def add_basic_field(self):
     basic_dict = {}
     if self.logformat in 'json':
         basic_dict['@message'] = str(json.dumps(self.logdata))
     else:
         basic_dict['@message'] = str(self.logdata)
     basic_dict['event'] = {'module': self.logtype}
     basic_dict['@timestamp'] = self.timestamp.isoformat()
     basic_dict['event']['ingested'] = self.event_ingested.isoformat()
     basic_dict['@log_type'] = self.logtype
     if self.__skip_normalization:
         unique_text = "{0}{1}".format(basic_dict['@message'], self.s3key)
         basic_dict['@id'] = hashlib.md5(
             unique_text.encode('utf-8')).hexdigest()
     elif self.logconfig['doc_id']:
         basic_dict['@id'] = self.__logdata_dict[self.logconfig['doc_id']]
     else:
         basic_dict['@id'] = hashlib.md5(
             str(basic_dict['@message']).encode('utf-8')).hexdigest()
     if self.loggroup:
         basic_dict['@log_group'] = self.loggroup
         basic_dict['@log_stream'] = self.logstream
     basic_dict['@log_s3bucket'] = self.s3bucket
     basic_dict['@log_s3key'] = self.s3key
     self.__logdata_dict = utils.merge_dicts(self.__logdata_dict,
                                             basic_dict)
Beispiel #4
0
def transform(logdata):
    win_dict = initial_extract_action_outcome(logdata)
    win_dict = extract_instance_id(logdata, win_dict)

    logdata = utils.merge_dicts(logdata, win_dict)

    return logdata
Beispiel #5
0
 def get_value_and_input_into_ecs_dict(self, ecs_dict):
     new_ecs_dict = {}
     ecs_keys = self.logconfig['ecs']
     for ecs_key in ecs_keys:
         original_keys = self.logconfig[ecs_key]
         if isinstance(original_keys, str):
             v = utils.value_from_nesteddict_by_dottedkeylist(
                 self.__logdata_dict, original_keys)
             if isinstance(v, str):
                 v = utils.validate_ip(v, ecs_key)
             if v:
                 new_ecs_dict = utils.put_value_into_nesteddict(ecs_key, v)
         elif isinstance(original_keys, list):
             temp_list = []
             for original_key_list in original_keys:
                 v = utils.value_from_nesteddict_by_dottedkeylist(
                     self.__logdata_dict, original_key_list)
                 if isinstance(v, str):
                     v = utils.validate_ip(v, ecs_key)
                     if v:
                         temp_list.append(v)
                 elif isinstance(v, list):
                     for i in v:
                         each_v = utils.validate_ip(i, ecs_key)
                         if each_v:
                             temp_list.append(each_v)
             if temp_list:
                 new_ecs_dict = utils.put_value_into_nesteddict(
                     ecs_key, sorted(list(set(temp_list))))
         if new_ecs_dict:
             new_ecs_dict = utils.merge_dicts(ecs_dict, new_ecs_dict)
     return ecs_dict
def transform(logdata):
    # event (ecs)
    module = (logdata['ProductFields']['aws/securityhub/ProductName']).lower()
    logdata['event']['module'] = module

    if 'guardduty' in module:
        logdata['event']['category'] = 'intrusion_detection'

        m = RE_GDTYPE.search(str(logdata['rule']['name']))
        logdata['ThreatPurpose'] = m['ThreatPurpose']
        logdata['ResourceTypeAffected'] = m['ResourceTypeAffected']
        logdata['ThreatFamilyName'] = m['ThreatFamilyName']

        action_type = (logdata['ProductFields']
                       ['aws/guardduty/service/action/actionType'])
        if 'NETWORK_CONNECTION' in action_type:
            direction_key = ('aws/guardduty/service/action/'
                             'networkConnectionAction/connectionDirection')
            direction = logdata['ProductFields'][direction_key].lower()
        elif 'DNS_REQUEST' in action_type:
            direction = "outbound"
        else:
            direction = "inbound"
        if 'network' in logdata:
            logdata['network']['direction'] = direction
        else:
            logdata['network'] = {'direction': direction}
        if "outbound" in direction:
            logdata['source'], logdata['destination'] = (
                logdata.get('destination'), logdata.get('source'))
            if not logdata['source']:
                del logdata['source']
            if not logdata['destination']:
                del logdata['destination']
        # event.category
        if logdata['ThreatPurpose'] in ('Backdoor', 'CryptoCurrency',
                                        'Trojan'):
            logdata['event']['category'] = 'malware'
    elif 'iam access analyzer' in module:
        pass
    elif 'security hub' in module:
        logdata['__doc_id_suffix'] = int(
            datetime.fromisoformat(logdata['@timestamp']).timestamp())
        logdata['rule']['name'] = logdata['Title']
    elif 'inspector' in module:
        logdata['event']['category'] = 'package'
    elif 'macie' in module:
        logdata['event']['category'] = 'intrusion_detection'
        logdata['rule']['name'] = logdata['Title']

    resouce_dict = get_values_from_asff_resources(logdata['Resources'])
    logdata = utils.merge_dicts(logdata, resouce_dict)

    return logdata
Beispiel #7
0
 def clean_multi_type_field(self):
     clean_multi_type_dict = {}
     multifield_keys = self.logconfig['json_to_text'].split()
     for multifield_key in multifield_keys:
         v = utils.value_from_nesteddict_by_dottedkey(
             self.__logdata_dict, multifield_key)
         if v:
             # json obj in json obj
             if isinstance(v, int):
                 new_dict = utils.put_value_into_nesteddict(
                     multifield_key, v)
             elif '{' in v:
                 new_dict = utils.put_value_into_nesteddict(
                     multifield_key, repr(v))
             else:
                 new_dict = utils.put_value_into_nesteddict(
                     multifield_key, str(v))
             clean_multi_type_dict = utils.merge_dicts(
                 clean_multi_type_dict, new_dict)
     self.__logdata_dict = utils.merge_dicts(self.__logdata_dict,
                                             clean_multi_type_dict)
Beispiel #8
0
def transform(logdata):
    proc = logdata.get('proc', "")
    linux_dict = {}
    linux_dict = extract_instance_id(logdata, linux_dict)
    if 'sshd' in proc:
        linux_dict = extract_from_sshd(logdata, linux_dict)
    elif 'sudo' in proc:
        linux_dict = extract_from_sudo(logdata, linux_dict)

    if linux_dict:
        logdata = utils.merge_dicts(logdata, linux_dict)
    return logdata
Beispiel #9
0
 def enrich(self):
     enrich_dict = {}
     # geoip
     geoip_list = self.logconfig['geoip'].split()
     for geoip_ecs in geoip_list:
         try:
             ipaddr = self.__logdata_dict[geoip_ecs]['ip']
         except KeyError:
             continue
         geoip, asn = self.geodb_instance.check_ipaddress(ipaddr)
         if geoip:
             enrich_dict[geoip_ecs] = {'geo': geoip}
         if geoip and asn:
             enrich_dict[geoip_ecs].update({'as': asn})
         elif asn:
             enrich_dict[geoip_ecs] = {'as': asn}
     self.__logdata_dict = utils.merge_dicts(self.__logdata_dict,
                                             enrich_dict)
def transform(logdata):
    proc = logdata.get('proc', "")
    linux_dict = {}
    linux_dict = extract_instance_id(logdata, linux_dict)

    # /var/log/secure
    if 'sshd' in proc:
        logdata['__index_name'] = 'log-linux-secure'
        linux_dict = extract_from_sshd(logdata, linux_dict)
    elif 'sudo' in proc:
        logdata['__index_name'] = 'log-linux-secure'
        linux_dict = extract_from_sudo(logdata, linux_dict)
        pass
    elif 'su' == proc:
        logdata['__index_name'] = 'log-linux-secure'
    else:
        pass

    if linux_dict:
        logdata = utils.merge_dicts(logdata, linux_dict)
    return logdata
def transform(logdata):
    if logdata['severity'] <= 3.9:
        label = "low"
    elif logdata['severity'] <= 6.9:
        label = "medium"
    elif logdata['severity'] <= 8.9:
        label = "high"
    r = re.compile(r"(?P<ThreatPurpose>\w*):(?P<ResourceTypeAffected>\w*)/"
                   r"(?P<ThreatFamilyName>[\w\&]*)")
    m = r.match(logdata['type'])
    gd = {
        'severitylabel': label,
        'ThreatPurpose': m['ThreatPurpose'],
        'ResourceTypeAffected': m['ResourceTypeAffected'],
        'ThreatFamilyName': m['ThreatFamilyName']
    }
    action_type = logdata['service']['action']['actionType']
    if 'NETWORK_CONNECTION' in action_type:
        direction = (logdata['service']['action']['networkConnectionAction']
                     ['connectionDirection'])
    elif 'DNS_REQUEST' in action_type:
        direction = "OUTBOUND"
    else:
        direction = "INBOUND"
    gd['network'] = {'direction': direction}
    logdata = utils.merge_dicts(logdata, gd)
    if "OUTBOUND" in direction:
        logdata['source'], logdata['destination'] = (
            logdata.get('destination'), logdata.get('source'))
        if not logdata['source']:
            del logdata['source']
        if not logdata['destination']:
            del logdata['destination']
    # event.category
    if logdata['ThreatPurpose'] in ('Backdoor', 'CryptoCurrency', 'Trojan'):
        logdata['event']['category'] = 'malware'
    return logdata
Beispiel #12
0
    def add_basic_field(self):
        basic_dict = {}
        basic_dict['@message'] = self.lograw
        basic_dict['@timestamp'] = self.timestamp.isoformat()
        basic_dict['@log_type'] = self.logtype
        basic_dict['@log_s3bucket'] = self.s3bucket
        basic_dict['@log_s3key'] = self.s3key
        basic_dict['@log_group'] = self.loggroup
        basic_dict['@log_stream'] = self.logstream
        basic_dict['event'] = {'module': self.logtype}
        basic_dict['event']['ingested'] = self.event_ingested.isoformat()
        if self.__skip_normalization:
            unique_text = (
                f'{basic_dict["@message"]}{self.s3key}{self.additional_id}')
            basic_dict['@id'] = hashlib.md5(
                unique_text.encode('utf-8')).hexdigest()
            del unique_text
            if '__error_message' in self.__logdata_dict:
                self.__logdata_dict['error'] = {
                    'message': self.__logdata_dict['__error_message']}
                del self.__logdata_dict['__error_message']

        elif self.logconfig['doc_id']:
            basic_dict['@id'] = self.__logdata_dict[self.logconfig['doc_id']]
        elif self.additional_id:
            unique_text = f'{basic_dict["@message"]}{self.additional_id}'
            basic_dict['@id'] = hashlib.md5(
                unique_text.encode('utf-8')).hexdigest()
            del unique_text
        else:
            unique_text = f'{basic_dict["@message"]}'
            basic_dict['@id'] = hashlib.md5(
                unique_text.encode('utf-8')).hexdigest()
            del unique_text
        self.__logdata_dict = utils.merge_dicts(
            self.__logdata_dict, basic_dict)
def transform(logdata):
    # event.outcome フィールドへ情報を投入
    if logdata.get('outcome'):
        if logdata.get('outcome', {}).get('result'):
            outcome = logdata['outcome']['result'].lower()
            if outcome == 'success' or outcome == 'allow':
                logdata['event']['outcome'] = 'success'
            elif outcome == 'failure' or outcome == 'deny':
                logdata['event']['outcome'] = 'failure'
            else:
                logdata['event']['outcome'] = 'unknown'

    # User name や email などのユーザー関連の情報を ECS フィールドとしてマッピング
    if logdata.get('actor', {}).get('type'):
        if logdata['actor']['type'] == 'User':
            q = r"^(?P<name>.*)@(?P<domain>.*)$"
            n = re.match(q, logdata['actor']['alternateId'])
            if n:
                name_domain_new_dict = {
                    'user': {
                        'domain': n.group('domain'),
                        'name': n.group('name'),
                        'email': logdata['actor']['alternateId']
                    }
                }
                utils.merge_dicts(logdata, name_domain_new_dict)
            user_new_dict = {
                'client': {
                    'user': {
                        'full_name': logdata['actor']['displayName'],
                        'id': logdata['actor']['id']
                    }
                },
                'source': {
                    'user': {
                        'full_name': logdata['actor']['displayName'],
                        'id': logdata['actor']['id']
                    }
                },
                'related': {
                    'user': logdata['actor']['displayName']
                }
            }
            utils.merge_dicts(logdata, user_new_dict)

    # 独自フィールドに対して Prefix を付与
    # Okta 独自のフィールドと他ログのフィールドで Type のコンフリクト等が発生するのを防ぐ

    # logdata 直下にある Okta 独自フィールド名
    # フィールド名が camel case のものは snake case へ変換する
    okta_fields = [
        'actor', 'debugContext', 'request', 'outcome', 'transaction',
        'authenticationContext', 'securityContext', 'displayMessage', 'uuid',
        'version', 'severity', 'published', 'legacyEventType', 'eventType'
    ]
    for field in okta_fields:
        if logdata.get(field):
            snake_field_name = str_camel_to_snake(field)
            tmp_okta_dict = set_prefix(logdata[field], snake_field_name,
                                       'okta')
            utils.merge_dicts(logdata, tmp_okta_dict)
            del logdata[field]

    # target フィールドは独自フィールドだが中身が配列となっている場合があるため別処理
    if logdata.get('target'):
        target_new_dict = {'okta': {'target': [logdata.get('target')]}}
        utils.merge_dicts(logdata, target_new_dict)

        for key in logdata['target']:
            if key['type'] == 'AppInstance':
                app_new_dict = {
                    'okta': {
                        'target': {
                            'app_name': {
                                'alternate_id': key['alternateId'],
                                'display_name': key['displayName']
                            }
                        }
                    }
                }
                utils.merge_dicts(logdata, app_new_dict)

    # Client フィールドは ECS も混ざっているため、一つずつ対応
    tmp_okta_client_dict = {
        'okta': {
            'client': {
                'device': logdata['client']['device'],
                'ip': logdata['client']['ipAddress'],
                'user_agent':
                dict_camel_to_snake(logdata['client']['userAgent']),
                'id': logdata['client']['id'],
                'zone': logdata['client']['zone']
            }
        }
    }
    utils.merge_dicts(logdata, tmp_okta_client_dict)

    # 不要な独自フィールドを削除
    del logdata['target']
    del logdata['client']['device']
    del logdata['client']['ipAddress']
    del logdata['client']['userAgent']
    del logdata['client']['geographicalContext']
    del logdata['client']['id']
    del logdata['client']['zone']

    return logdata