def transform_to_ecs(self): ecs_dict = {'ecs': {'version': self.logconfig['ecs_version']}} if self.logconfig['cloud_provider']: ecs_dict['cloud'] = {'provider': self.logconfig['cloud_provider']} ecs_keys = self.logconfig['ecs'].split() for ecs_key in ecs_keys: original_keys = self.logconfig[ecs_key] v = utils.value_from_nesteddict_by_dottedkeylist( self.__logdata_dict, original_keys) if v: new_ecs_dict = utils.put_value_into_nesteddict(ecs_key, v) if '.ip' in ecs_key: # IPアドレスの場合は、validation try: ipaddress.ip_address(v) except ValueError: continue ecs_dict = utils.merge_dicts(ecs_dict, new_ecs_dict) if 'cloud' in ecs_dict: # Set AWS Account ID if ('account' in ecs_dict['cloud'] and 'id' in ecs_dict['cloud']['account']): if ecs_dict['cloud']['account']['id'] in ('unknown', ): # for vpcflowlogs ecs_dict['cloud']['account'] = {'id': self.accountid} elif self.accountid: ecs_dict['cloud']['account'] = {'id': self.accountid} else: ecs_dict['cloud']['account'] = {'id': 'unknown'} # Set AWS Region if 'region' in ecs_dict['cloud']: pass elif self.region: ecs_dict['cloud']['region'] = self.region else: ecs_dict['cloud']['region'] = 'unknown' # get info from firelens metadata of Elastic Container Serivce if 'ecs_task_arn' in self.__logdata_dict: ecs_task_arn_taple = self.__logdata_dict['ecs_task_arn'].split(':') ecs_dict['cloud']['account']['id'] = ecs_task_arn_taple[4] ecs_dict['cloud']['region'] = ecs_task_arn_taple[3] if 'ec2_instance_id' in self.__logdata_dict: ecs_dict['cloud']['instance'] = { 'id': self.__logdata_dict['ec2_instance_id'] } ecs_dict['container'] = { 'id': self.__logdata_dict['container_id'], 'name': self.__logdata_dict['container_name'] } static_ecs_keys = self.logconfig['static_ecs'] if static_ecs_keys: for static_ecs_key in static_ecs_keys.split(): new_ecs_dict = utils.put_value_into_nesteddict( static_ecs_key, self.logconfig[static_ecs_key]) ecs_dict = utils.merge_dicts(ecs_dict, new_ecs_dict) self.__logdata_dict = utils.merge_dicts(self.__logdata_dict, ecs_dict)
def transform_to_ecs(self): ecs_dict = {'ecs': {'version': self.logconfig['ecs_version']}} if self.logconfig['cloud_provider']: ecs_dict['cloud'] = {'provider': self.logconfig['cloud_provider']} ecs_dict = self.get_value_and_input_into_ecs_dict(ecs_dict) if 'cloud' in ecs_dict: # Set AWS Account ID if ('account' in ecs_dict['cloud'] and 'id' in ecs_dict['cloud']['account']): if ecs_dict['cloud']['account']['id'] in ('unknown', ): # for vpcflowlogs ecs_dict['cloud']['account'] = {'id': self.accountid} elif self.accountid: ecs_dict['cloud']['account'] = {'id': self.accountid} else: ecs_dict['cloud']['account'] = {'id': 'unknown'} # Set AWS Region if 'region' in ecs_dict['cloud']: pass elif self.region: ecs_dict['cloud']['region'] = self.region else: ecs_dict['cloud']['region'] = 'unknown' # get info from firelens metadata of Elastic Container Serivce if 'ecs_task_arn' in self.logmeta: ecs_task_arn_taple = self.logmeta['ecs_task_arn'].split(':') ecs_dict['cloud']['account']['id'] = ecs_task_arn_taple[4] ecs_dict['cloud']['region'] = ecs_task_arn_taple[3] if 'ec2_instance_id' in self.logmeta: ecs_dict['cloud']['instance'] = { 'id': self.logmeta['ec2_instance_id']} ecs_dict['container'] = { 'id': self.logmeta['container_id'], 'name': self.logmeta['container_name']} if '__error_message' in self.logmeta: self.__logdata_dict['error'] = { 'message': self.logmeta['__error_message']} del self.logmeta['__error_message'] static_ecs_keys = self.logconfig['static_ecs'] for static_ecs_key in static_ecs_keys: v = copy.copy(self.logconfig[static_ecs_key]) new_ecs_dict = utils.put_value_into_nesteddict(static_ecs_key, v) ecs_dict = utils.merge_dicts(ecs_dict, new_ecs_dict) self.__logdata_dict = utils.merge_dicts(self.__logdata_dict, ecs_dict)
def add_basic_field(self): basic_dict = {} if self.logformat in 'json': basic_dict['@message'] = str(json.dumps(self.logdata)) else: basic_dict['@message'] = str(self.logdata) basic_dict['event'] = {'module': self.logtype} basic_dict['@timestamp'] = self.timestamp.isoformat() basic_dict['event']['ingested'] = self.event_ingested.isoformat() basic_dict['@log_type'] = self.logtype if self.__skip_normalization: unique_text = "{0}{1}".format(basic_dict['@message'], self.s3key) basic_dict['@id'] = hashlib.md5( unique_text.encode('utf-8')).hexdigest() elif self.logconfig['doc_id']: basic_dict['@id'] = self.__logdata_dict[self.logconfig['doc_id']] else: basic_dict['@id'] = hashlib.md5( str(basic_dict['@message']).encode('utf-8')).hexdigest() if self.loggroup: basic_dict['@log_group'] = self.loggroup basic_dict['@log_stream'] = self.logstream basic_dict['@log_s3bucket'] = self.s3bucket basic_dict['@log_s3key'] = self.s3key self.__logdata_dict = utils.merge_dicts(self.__logdata_dict, basic_dict)
def transform(logdata): win_dict = initial_extract_action_outcome(logdata) win_dict = extract_instance_id(logdata, win_dict) logdata = utils.merge_dicts(logdata, win_dict) return logdata
def get_value_and_input_into_ecs_dict(self, ecs_dict): new_ecs_dict = {} ecs_keys = self.logconfig['ecs'] for ecs_key in ecs_keys: original_keys = self.logconfig[ecs_key] if isinstance(original_keys, str): v = utils.value_from_nesteddict_by_dottedkeylist( self.__logdata_dict, original_keys) if isinstance(v, str): v = utils.validate_ip(v, ecs_key) if v: new_ecs_dict = utils.put_value_into_nesteddict(ecs_key, v) elif isinstance(original_keys, list): temp_list = [] for original_key_list in original_keys: v = utils.value_from_nesteddict_by_dottedkeylist( self.__logdata_dict, original_key_list) if isinstance(v, str): v = utils.validate_ip(v, ecs_key) if v: temp_list.append(v) elif isinstance(v, list): for i in v: each_v = utils.validate_ip(i, ecs_key) if each_v: temp_list.append(each_v) if temp_list: new_ecs_dict = utils.put_value_into_nesteddict( ecs_key, sorted(list(set(temp_list)))) if new_ecs_dict: new_ecs_dict = utils.merge_dicts(ecs_dict, new_ecs_dict) return ecs_dict
def transform(logdata): # event (ecs) module = (logdata['ProductFields']['aws/securityhub/ProductName']).lower() logdata['event']['module'] = module if 'guardduty' in module: logdata['event']['category'] = 'intrusion_detection' m = RE_GDTYPE.search(str(logdata['rule']['name'])) logdata['ThreatPurpose'] = m['ThreatPurpose'] logdata['ResourceTypeAffected'] = m['ResourceTypeAffected'] logdata['ThreatFamilyName'] = m['ThreatFamilyName'] action_type = (logdata['ProductFields'] ['aws/guardduty/service/action/actionType']) if 'NETWORK_CONNECTION' in action_type: direction_key = ('aws/guardduty/service/action/' 'networkConnectionAction/connectionDirection') direction = logdata['ProductFields'][direction_key].lower() elif 'DNS_REQUEST' in action_type: direction = "outbound" else: direction = "inbound" if 'network' in logdata: logdata['network']['direction'] = direction else: logdata['network'] = {'direction': direction} if "outbound" in direction: logdata['source'], logdata['destination'] = ( logdata.get('destination'), logdata.get('source')) if not logdata['source']: del logdata['source'] if not logdata['destination']: del logdata['destination'] # event.category if logdata['ThreatPurpose'] in ('Backdoor', 'CryptoCurrency', 'Trojan'): logdata['event']['category'] = 'malware' elif 'iam access analyzer' in module: pass elif 'security hub' in module: logdata['__doc_id_suffix'] = int( datetime.fromisoformat(logdata['@timestamp']).timestamp()) logdata['rule']['name'] = logdata['Title'] elif 'inspector' in module: logdata['event']['category'] = 'package' elif 'macie' in module: logdata['event']['category'] = 'intrusion_detection' logdata['rule']['name'] = logdata['Title'] resouce_dict = get_values_from_asff_resources(logdata['Resources']) logdata = utils.merge_dicts(logdata, resouce_dict) return logdata
def clean_multi_type_field(self): clean_multi_type_dict = {} multifield_keys = self.logconfig['json_to_text'].split() for multifield_key in multifield_keys: v = utils.value_from_nesteddict_by_dottedkey( self.__logdata_dict, multifield_key) if v: # json obj in json obj if isinstance(v, int): new_dict = utils.put_value_into_nesteddict( multifield_key, v) elif '{' in v: new_dict = utils.put_value_into_nesteddict( multifield_key, repr(v)) else: new_dict = utils.put_value_into_nesteddict( multifield_key, str(v)) clean_multi_type_dict = utils.merge_dicts( clean_multi_type_dict, new_dict) self.__logdata_dict = utils.merge_dicts(self.__logdata_dict, clean_multi_type_dict)
def transform(logdata): proc = logdata.get('proc', "") linux_dict = {} linux_dict = extract_instance_id(logdata, linux_dict) if 'sshd' in proc: linux_dict = extract_from_sshd(logdata, linux_dict) elif 'sudo' in proc: linux_dict = extract_from_sudo(logdata, linux_dict) if linux_dict: logdata = utils.merge_dicts(logdata, linux_dict) return logdata
def enrich(self): enrich_dict = {} # geoip geoip_list = self.logconfig['geoip'].split() for geoip_ecs in geoip_list: try: ipaddr = self.__logdata_dict[geoip_ecs]['ip'] except KeyError: continue geoip, asn = self.geodb_instance.check_ipaddress(ipaddr) if geoip: enrich_dict[geoip_ecs] = {'geo': geoip} if geoip and asn: enrich_dict[geoip_ecs].update({'as': asn}) elif asn: enrich_dict[geoip_ecs] = {'as': asn} self.__logdata_dict = utils.merge_dicts(self.__logdata_dict, enrich_dict)
def transform(logdata): proc = logdata.get('proc', "") linux_dict = {} linux_dict = extract_instance_id(logdata, linux_dict) # /var/log/secure if 'sshd' in proc: logdata['__index_name'] = 'log-linux-secure' linux_dict = extract_from_sshd(logdata, linux_dict) elif 'sudo' in proc: logdata['__index_name'] = 'log-linux-secure' linux_dict = extract_from_sudo(logdata, linux_dict) pass elif 'su' == proc: logdata['__index_name'] = 'log-linux-secure' else: pass if linux_dict: logdata = utils.merge_dicts(logdata, linux_dict) return logdata
def transform(logdata): if logdata['severity'] <= 3.9: label = "low" elif logdata['severity'] <= 6.9: label = "medium" elif logdata['severity'] <= 8.9: label = "high" r = re.compile(r"(?P<ThreatPurpose>\w*):(?P<ResourceTypeAffected>\w*)/" r"(?P<ThreatFamilyName>[\w\&]*)") m = r.match(logdata['type']) gd = { 'severitylabel': label, 'ThreatPurpose': m['ThreatPurpose'], 'ResourceTypeAffected': m['ResourceTypeAffected'], 'ThreatFamilyName': m['ThreatFamilyName'] } action_type = logdata['service']['action']['actionType'] if 'NETWORK_CONNECTION' in action_type: direction = (logdata['service']['action']['networkConnectionAction'] ['connectionDirection']) elif 'DNS_REQUEST' in action_type: direction = "OUTBOUND" else: direction = "INBOUND" gd['network'] = {'direction': direction} logdata = utils.merge_dicts(logdata, gd) if "OUTBOUND" in direction: logdata['source'], logdata['destination'] = ( logdata.get('destination'), logdata.get('source')) if not logdata['source']: del logdata['source'] if not logdata['destination']: del logdata['destination'] # event.category if logdata['ThreatPurpose'] in ('Backdoor', 'CryptoCurrency', 'Trojan'): logdata['event']['category'] = 'malware' return logdata
def add_basic_field(self): basic_dict = {} basic_dict['@message'] = self.lograw basic_dict['@timestamp'] = self.timestamp.isoformat() basic_dict['@log_type'] = self.logtype basic_dict['@log_s3bucket'] = self.s3bucket basic_dict['@log_s3key'] = self.s3key basic_dict['@log_group'] = self.loggroup basic_dict['@log_stream'] = self.logstream basic_dict['event'] = {'module': self.logtype} basic_dict['event']['ingested'] = self.event_ingested.isoformat() if self.__skip_normalization: unique_text = ( f'{basic_dict["@message"]}{self.s3key}{self.additional_id}') basic_dict['@id'] = hashlib.md5( unique_text.encode('utf-8')).hexdigest() del unique_text if '__error_message' in self.__logdata_dict: self.__logdata_dict['error'] = { 'message': self.__logdata_dict['__error_message']} del self.__logdata_dict['__error_message'] elif self.logconfig['doc_id']: basic_dict['@id'] = self.__logdata_dict[self.logconfig['doc_id']] elif self.additional_id: unique_text = f'{basic_dict["@message"]}{self.additional_id}' basic_dict['@id'] = hashlib.md5( unique_text.encode('utf-8')).hexdigest() del unique_text else: unique_text = f'{basic_dict["@message"]}' basic_dict['@id'] = hashlib.md5( unique_text.encode('utf-8')).hexdigest() del unique_text self.__logdata_dict = utils.merge_dicts( self.__logdata_dict, basic_dict)
def transform(logdata): # event.outcome フィールドへ情報を投入 if logdata.get('outcome'): if logdata.get('outcome', {}).get('result'): outcome = logdata['outcome']['result'].lower() if outcome == 'success' or outcome == 'allow': logdata['event']['outcome'] = 'success' elif outcome == 'failure' or outcome == 'deny': logdata['event']['outcome'] = 'failure' else: logdata['event']['outcome'] = 'unknown' # User name や email などのユーザー関連の情報を ECS フィールドとしてマッピング if logdata.get('actor', {}).get('type'): if logdata['actor']['type'] == 'User': q = r"^(?P<name>.*)@(?P<domain>.*)$" n = re.match(q, logdata['actor']['alternateId']) if n: name_domain_new_dict = { 'user': { 'domain': n.group('domain'), 'name': n.group('name'), 'email': logdata['actor']['alternateId'] } } utils.merge_dicts(logdata, name_domain_new_dict) user_new_dict = { 'client': { 'user': { 'full_name': logdata['actor']['displayName'], 'id': logdata['actor']['id'] } }, 'source': { 'user': { 'full_name': logdata['actor']['displayName'], 'id': logdata['actor']['id'] } }, 'related': { 'user': logdata['actor']['displayName'] } } utils.merge_dicts(logdata, user_new_dict) # 独自フィールドに対して Prefix を付与 # Okta 独自のフィールドと他ログのフィールドで Type のコンフリクト等が発生するのを防ぐ # logdata 直下にある Okta 独自フィールド名 # フィールド名が camel case のものは snake case へ変換する okta_fields = [ 'actor', 'debugContext', 'request', 'outcome', 'transaction', 'authenticationContext', 'securityContext', 'displayMessage', 'uuid', 'version', 'severity', 'published', 'legacyEventType', 'eventType' ] for field in okta_fields: if logdata.get(field): snake_field_name = str_camel_to_snake(field) tmp_okta_dict = set_prefix(logdata[field], snake_field_name, 'okta') utils.merge_dicts(logdata, tmp_okta_dict) del logdata[field] # target フィールドは独自フィールドだが中身が配列となっている場合があるため別処理 if logdata.get('target'): target_new_dict = {'okta': {'target': [logdata.get('target')]}} utils.merge_dicts(logdata, target_new_dict) for key in logdata['target']: if key['type'] == 'AppInstance': app_new_dict = { 'okta': { 'target': { 'app_name': { 'alternate_id': key['alternateId'], 'display_name': key['displayName'] } } } } utils.merge_dicts(logdata, app_new_dict) # Client フィールドは ECS も混ざっているため、一つずつ対応 tmp_okta_client_dict = { 'okta': { 'client': { 'device': logdata['client']['device'], 'ip': logdata['client']['ipAddress'], 'user_agent': dict_camel_to_snake(logdata['client']['userAgent']), 'id': logdata['client']['id'], 'zone': logdata['client']['zone'] } } } utils.merge_dicts(logdata, tmp_okta_client_dict) # 不要な独自フィールドを削除 del logdata['target'] del logdata['client']['device'] del logdata['client']['ipAddress'] del logdata['client']['userAgent'] del logdata['client']['geographicalContext'] del logdata['client']['id'] del logdata['client']['zone'] return logdata