def aws_connection(): keys = Configurations() config = Config(retries={'max_attempts': 5, 'mode': 'standard'}) client = boto3.client( 'securityhub', aws_access_key_id=keys.get_configurations()['aws_access_key_id'], aws_secret_access_key=keys.get_configurations() ['aws_secret_access_key'], region_name=keys.get_configurations()['region_name'], config=config) return client
def azure_data_collector(json_records, offset_time, logger_obj): customer_id = Configurations.get_configurations()['AzureWorkspaceID'] shared_key = Configurations.get_configurations()['AzurePrimaryKey'] log_type = Configurations.get_configurations()['LogName'] numbers_deque = collections.deque(json_records) while bool(numbers_deque): if len(numbers_deque) > 50: send_list = [numbers_deque.popleft() for _i in range(49)] azure_api_call(customer_id, shared_key, log_type, send_list, offset_time, logger_obj) else: send_list = ([ numbers_deque.popleft() for _i in range(len(numbers_deque)) ]) azure_api_call(customer_id, shared_key, log_type, send_list, offset_time, logger_obj)
def run(self): while True: customer_id = Configurations.get_configurations( )['AzureWorkspaceID'] shared_key = Configurations.get_configurations()['AzurePrimaryKey'] log_type = Configurations.get_configurations()['LogName'] api = client.DataCollectorAPIClient(customer_id, shared_key) try: health = api.health_check(log_type) if health.status_code != 500: try: json_file, offset_time = Mapper.map_sql_to_azure() except Exception as e: self.logger_.error( f"Error Database May not have been initialized") json_file = None if not json_file: # self.logger_.info("No Data received, azure thread is sleeping for 5 minutes before retrying") time.sleep(300) elif (len(json_file)) >= 1: azure_data_collector(json_file, offset_time, self.logger_) else: self.logger_.error( "Azure cannot be reached, azure thread is sleeping for 5 minutes before " "retrying") time.sleep(300) except (requests.exceptions.ConnectionError, ReadTimeout) as e: self.logger_.error( f'{e}: error occurred, Azure thread is sleeping for 5 minutes before retrying' ) time.sleep(300)
def run(self): while True: location = Configurations.get_configurations()['file_location'] for file in os.listdir(location): if file.endswith(".xml") and file.startswith("event"): with open(os.path.join(location, file)) as fd: doc = parse(fd.read()) xml_file_name = location + '/event-' + doc['evt:event'][ 'evt:event_info']['evt:incidentId'] + '.xml' json_file = Mapper.map_xml_to_asff(doc) fd.close() # Dont send empty files if (len(json_file)) >= 1: amazon_security_hub_xml(json_file, xml_file_name, self.logger_obj) else: os.remove(xml_file_name) time.sleep(10)
def map_sql_to_asff(): try: event_list, partitions = get_events('AWSUpdateDate') if bool(partitions): event_list, partitions = get_events('AWSUpdateDate') findings = [] for partition in partitions: for row in event_list: policy_events = execute_policy_events(partition, row[0]) for policy in policy_events: extra_data_sql = ExtraData() event_data = query_events(partition, row[0]) service_id = event_data[0].SERVICE_ID source_id = event_data[0].SOURCE_ID user_data = query_users(source_id) domain_id = user_data[0].DOMAIN_ID services_data = query_services(service_id) destination_data = query_destinations_users( partition, row[0]) asff_date = __iso8601_format( json.dumps(event_data[0].INSERT_DATE, indent=4, sort_keys=True, default=str)[1:-4]) findings_object = Finding() findings_object.GeneratorId = str(event_data[0].ID) findings_object.AwsAccountId = Configurations.get_configurations( )['AwsAccountId'][:12] findings_object.CreatedAt = asff_date findings_object.Description = event_data[ 0].SUBJECT or "Description Not found" network_object = Network() network_object.Protocol = services_data[ 0].PROTOCOL_ID.split("_")[1] if domain_id is not None: network_object.SourceDomain = query_domains( domain_id)[0].NAME else: network_object.SourceDomain = 'none' network_object.SourceIpV4 = user_data[0].IP network_object.SourcePort = event_data[0].PORT if destination_data[0][0].DOMAIN_ID is not None: network_object.DestinationDomain = query_domains( destination_data[0][0].DOMAIN_ID)[0].NAME else: network_object.DestinationDomain = 'none' network_object.DestinationIpV4 = destination_data[0][ 0].IP network_object.DestinationPort = destination_data[0][ 0].PORT findings_object.ProductArn = Configurations.get_arn() related_findings_object = RelatedFinding() related_findings_object.Id = event_data[0].EXTERNAL_ID related_findings_object.ProductArn = Configurations.get_arn( ) findings_object.SchemaVersion = '2018-10-08' findings_object.Title = 'Forcepoint DLP Incident' findings_object.Types = [ 'Sensitive Data Identifications/Security/ForcepointDLP' ] findings_object.UpdatedAt = asff_date resource_object = Resource() details_object = Details() product_fields_object = ProductFields() product_fields_object.ForcepointDLPSourceIP = user_data[ 0].IP product_fields_object.Text = services_data[ 0].CHANNEL_NAME product_fields_object.UpdatedAt = asff_date product_fields_object.UpdatedBy = services_data[ 0].AGENT_NAME extra_data_sql.extra_data_handler([ query_policy_categories( policy.POLICY_CATEGORY_ID)[0].NAME ], 'RuleName') extra_data_sql.extra_data_handler([user_data[0].EMAIL], 'SourceEmail') extra_data_sql.extra_data_handler( [user_data[0].EXTRA_DATA], 'SourceExtraData') extra_data_sql.extra_data_handler( [user_data[0].FULL_NAME], 'SourceFullName') extra_data_sql.extra_data_handler( [user_data[0].HOSTNAME], 'SourceHostname') extra_data_sql.extra_data_handler( [user_data[0].LOGIN_NAME], 'SourceLoginName') # extra_data_sql.extra_data_handler([user_data[0].Username], 'SourceUsername') for i in range(len(destination_data)): extra_data_sql.extra_data_handler( [destination_data[i][0].COMMON_NAME], 'DestinationCommonName') # user_defined_extras['DestinationUsername.' + str(i + 1)] extra_data_sql.extra_data_handler( [destination_data[i][0].HOSTNAME], 'DestinationHostname') extra_data_sql.extra_data_handler( [destination_data[i][0].EMAIL], 'DestinationEmail') extra_data_sql.extra_data_handler( [destination_data[i][0].EXTRA_DATA], 'DestinationExtraData') if i >= 1: if destination_data[i][ 0].DOMAIN_ID is not None: extra_data_sql.extra_data_handler([ query_domains(destination_data[i] [0].DOMAIN_ID)[0].NAME ], 'DestinationDomain') else: network_object.DestinationDomain = 'none' extra_data_sql.extra_data_handler( [destination_data[i][0].IP], 'DestinationIpV4') extra_data_sql.extra_data_handler( [destination_data[i][0].PORT], 'DestinationPort') resource_object.Type = 'Forcepoint DLP' resource_object.Details = details_object.__dict__ severity_object = Severity() findings_object.Id = 'incident_Id-%s-rule_id-%s' % ( row[0], policy[0]) severity_object.Normalized = policy.SEVERITY severity_object.Product = policy.SEVERITY severity_object.Label = 'none' resource_object.Id = str( query_policy_categories( policy.POLICY_CATEGORY_ID)[0].ID) findings_object.Resources = [resource_object.__dict__] findings_object.Severity = severity_object.__dict__ findings_object.Network = network_object.__dict__ findings_object.RelatedFindings = [ related_findings_object.__dict__ ] details_object.Other = product_fields_object.__dict__ details_other = extra_data_sql.take(50) details_object.Other.update(details_other) extra_data_sql.remove_dups(details_other) product_fields = extra_data_sql.take(50) findings_object.ProductFields = product_fields findings.append(findings_object.__dict__) asff_object = JsonFormatClass() asff_object.findings = findings cleaned_findings = remove_null_items(asff_object.findings) for i in range(len(cleaned_findings)): pos = i - 1 if cleaned_findings[pos]['Severity']['Normalized']: if not (Configurations.get_configurations( )[__normalized_severity_from_db( cleaned_findings[pos]['Severity']['Normalized'])]): del cleaned_findings[pos] for i in range(len(cleaned_findings)): cleaned_findings[i]['Severity']['Label'] = __severity_label( cleaned_findings[i]['Severity']['Normalized']) cleaned_findings[i]['Severity']['Normalized'] = ( __normalized_severity( __normalized_severity_from_db( cleaned_findings[i]['Severity']['Normalized']))) if cleaned_findings.__len__() >= 1: date_time_obj = datetime.datetime.strptime( cleaned_findings[-1]["CreatedAt"], '%Y-%m-%dT%H:%M:%SZ') offset_time = date_time_obj + datetime.timedelta(seconds=1) return cleaned_findings, str(offset_time) return cleaned_findings, None else: return None, 'None' except Exception as e: raise e
def map_sql_to_azure(): try: event_list, partitions = get_events('AzureUpdateDate') if bool(partitions): event_list, partitions = get_events('AzureUpdateDate') logs = [] for partition in partitions: for row in event_list: policy_events = execute_policy_events(partition, row[0]) for policy in policy_events: try: extra_data_sql = ExtraData() event_data = query_events(partition, row[0]) service_id = event_data[0].SERVICE_ID source_id = event_data[0].SOURCE_ID user_data = query_users(source_id) domain_id = user_data[0].DOMAIN_ID if len( user_data) > 0 else None services_data = query_services(service_id) destination_data = query_destinations_users( partition, row[0]) formatted_date = __iso8601_format( json.dumps(event_data[0].INSERT_DATE, indent=4, sort_keys=True, default=str)[1:-4]) findings_object = Finding() findings_object.TransactionSizeInBytes = event_data[ 0].ATT_TOTAL_SIZE or 0 findings_object.GeneratorId = str(event_data[0].ID) findings_object.CreatedAt = formatted_date findings_object.Description = event_data[ 0].SUBJECT or "Description Not found" findings_object.Protocol = services_data[ 0].PROTOCOL_ID.split("_")[1] if domain_id is not None: findings_object.SourceDomain = query_domains( domain_id)[0].NAME else: findings_object.SourceDomain = 'none' findings_object.SourceIpV4 = user_data[ 0].IP if len(user_data) > 0 else None findings_object.SourcePort = event_data[ 0].PORT if len(event_data) > 0 else None try: if destination_data[0][ 0].DOMAIN_ID is not None: findings_object.DestinationDomain = \ query_domains(destination_data[0][0].DOMAIN_ID)[0].NAME if len( destination_data) > 0 else None else: findings_object.DestinationDomain = 'none' except IndexError: findings_object.DestinationDomain = 'none' try: findings_object.DestinationIpV4 = destination_data[ 0][0].IP if len( destination_data) > 0 else None except IndexError: findings_object.DestinationIpV4 = 'none' try: findings_object.DestinationPort = destination_data[ 0][0].PORT if len( destination_data) > 0 else None except IndexError: findings_object.DestinationPort = 'none' findings_object.ExternalId = event_data[ 0].EXTERNAL_ID if len(event_data) > 0 else None findings_object.Title = 'Forcepoint DLP Incident' findings_object.UpdatedAt = formatted_date findings_object.ForcepointDLPSourceIP = user_data[ 0].IP if len(user_data) > 0 else None findings_object.Text = services_data[ 0].CHANNEL_NAME if len( services_data) > 0 else None findings_object.UpdatedAt = formatted_date findings_object.UpdatedBy = services_data[ 0].AGENT_NAME if len( services_data) > 0 else None extra_data_sql.extra_data_handler([ query_policy_categories( policy.POLICY_CATEGORY_ID)[0].NAME ], 'RuleName') extra_data_sql.extra_data_handler([ user_data[0].EMAIL if len(user_data) > 0 else None ], 'SourceEmail') extra_data_sql.extra_data_handler([ user_data[0].EXTRA_DATA if len(user_data) > 0 else None ], 'SourceExtraData') extra_data_sql.extra_data_handler([ user_data[0].FULL_NAME if len(user_data) > 0 else None ], 'SourceFullName') extra_data_sql.extra_data_handler([ user_data[0].HOSTNAME if len(user_data) > 0 else None ], 'SourceHostname') extra_data_sql.extra_data_handler([ user_data[0].LOGIN_NAME if len(user_data) > 0 else None ], 'SourceLoginName') for i in range(len(destination_data)): if len(destination_data[i]) > 0: extra_data_sql.extra_data_handler( [destination_data[i][0].COMMON_NAME], 'DestinationCommonName') extra_data_sql.extra_data_handler( [destination_data[i][0].HOSTNAME], 'DestinationHostname') extra_data_sql.extra_data_handler( [destination_data[i][0].EMAIL], 'DestinationEmail') extra_data_sql.extra_data_handler( [destination_data[i][0].EXTRA_DATA], 'DestinationExtraData') if i >= 1: if destination_data[i][ 0].DOMAIN_ID is not None: extra_data_sql.extra_data_handler([ query_domains( destination_data[i] [0].DOMAIN_ID)[0].NAME ], 'DestinationDomain') else: findings_object.DestinationDomain = 'none' extra_data_sql.extra_data_handler( [destination_data[i][0].IP], 'DestinationIpV4') extra_data_sql.extra_data_handler( [destination_data[i][0].PORT], 'DestinationPort') else: findings_object.DestinationCommonName = 'none' findings_object.DestinationHostname = 'none' findings_object.DestinationEmail = 'none' findings_object.DestinationExtraData = 'none' findings_object.DestinationDomain = 'none' findings_object.DestinationIpV4 = 'none' findings_object.DestinationPort = 'none' findings_object.Type = 'Forcepoint DLP' findings_object.Id = 'incident_Id-%s-rule_id-%s' % ( row[0], policy[0]) findings_object.Severity = policy.SEVERITY or 'none' findings_object.PolicyCategoryId = str( query_policy_categories( policy.POLICY_CATEGORY_ID)[0].ID) findings_object.__dict__.update( extra_data_sql.extra_data_storage) logs.append(findings_object.__dict__) except IndexError: event_data = query_events(partition, row[0]) try: formatted_date = __iso8601_format( json.dumps(event_data[0].INSERT_DATE, indent=4, sort_keys=True, default=str)[1:-4]) except: formatted_date = None row_as_list = [x for x in policy] f = open("unsent-events.txt", "a") f.writelines( f'Event ID: {str(row_as_list[1])}, Created at: {str(formatted_date)}\n' ) f.close() logs_object = JsonFormatClass() logs_object.findings = logs cleaned_findings = remove_null_items(logs_object.findings) for i in range(len(cleaned_findings)): pos = i - 1 if cleaned_findings[pos]['Severity']: if not (Configurations.get_configurations()[ __normalized_severity_from_db( cleaned_findings[pos]['Severity'])]): del cleaned_findings[pos] for i in range(len(cleaned_findings)): cleaned_findings[i][ 'Severity'] = __normalized_severity_from_db( cleaned_findings[i]['Severity']) if cleaned_findings.__len__() >= 1: date_time_obj = datetime.datetime.strptime( cleaned_findings[-1]["CreatedAt"], '%Y-%m-%dT%H:%M:%SZ') offset_time = date_time_obj + datetime.timedelta(seconds=1) return cleaned_findings, str(offset_time) return cleaned_findings, None else: return None, 'None' except Exception as e: logger.error(f"Error {e} ocurred") raise e
def map_xml_to_asff(event): findings_array_length = int(event['evt:event']['evt:rules']['evt:Count']) findings = [] keys = Configurations() for i in range(findings_array_length): extra_data = ExtraData() findings_object = Finding() findings_object.GeneratorId = event['evt:event']['evt:event_info'][ 'evt:incidentId'] findings_object.AwsAccountId = keys.get_configurations( )['AwsAccountId'][:12] findings_object.CreatedAt = __iso8601_format( event['evt:event']['evt:event_info']['evt:insert_date']) findings_object.Description = event['evt:event']['evt:event_info'][ 'evt:subject'] or "Description Not found" network_object = Network() network_object.Protocol = event['evt:event']['evt:event_info'][ 'evt:channel']['evt:protocol'].split("_")[1] network_object.SourceDomain = event['evt:event']['evt:event_info'][ 'evt:source']['evt:event_user']['evt:domain'] network_object.SourceIpV4 = event['evt:event']['evt:event_info'][ 'evt:source']['evt:event_user']['evt:ip'] network_object.SourcePort = event['evt:event']['evt:event_info'][ 'evt:source']['evt:event_user']['evt:port'] findings_object.ProductArn = keys.get_arn() related_findings_object = RelatedFinding() related_findings_object.Id = event['evt:event']['evt:event_info'][ 'evt:eventId'][:512] related_findings_object.ProductArn = keys.get_arn() findings_object.RelatedFindings = [related_findings_object.__dict__] findings_object.SchemaVersion = '2018-10-08' findings_object.Title = 'Forcepoint DLP Incident' findings_object.Types = [ 'Sensitive Data Identifications/Security/ForcepointDLP' ] findings_object.UpdatedAt = __iso8601_format( event['evt:event']['evt:event_info']['evt:insert_date']) resource_object = Resource() details_object = Details() product_fields_object = ProductFields() product_fields_object.ForcepointDLPSourceIP = \ event['evt:event']['evt:event_info']['evt:source']['evt:event_user']['evt:ip'] product_fields_object.Text = event['evt:event']['evt:event_info'][ 'evt:channel']['evt:service_name'] product_fields_object.UpdatedAt = __iso8601_format( event['evt:event']['evt:event_info']['evt:insert_date']) product_fields_object.UpdatedBy = event['evt:event']['evt:event_info'][ 'evt:channel']['evt:agent_name'] extra_data.extra_data_handler([ event['evt:event']['evt:event_info']['evt:source'] ['evt:event_user']['evt:email'] ], 'SourceEmail') extra_data.extra_data_handler([ event['evt:event']['evt:event_info']['evt:source'] ['evt:event_user']['evt:extra_data'] ], 'SourceExtraData') extra_data.extra_data_handler([ event['evt:event']['evt:event_info']['evt:source'] ['evt:event_user']['evt:full_name'] ], 'SourceFullName') extra_data.extra_data_handler([ event['evt:event']['evt:event_info']['evt:source'] ['evt:event_user']['evt:hostname'] ], 'SourceHostname') extra_data.extra_data_handler([ event['evt:event']['evt:event_info']['evt:source'] ['evt:event_user']['evt:login_name'] ], 'SourceLoginName') extra_data.extra_data_handler([ event['evt:event']['evt:event_info']['evt:source'] ['evt:event_user']['evt:username'] ], 'SourceUsername') network_destinations_object, extra_data = destinations_mapping( event, extra_data) network_object.__dict__.update(network_destinations_object) details_object.Other = product_fields_object.__dict__ resource_object.Type = 'Other' resource_object.Details = details_object.__dict__ severity_object = Severity() if int((event['evt:event']['evt:rules']['evt:Count'])) == 1: findings_object.Id = 'incident_Id:' + event['evt:event']['evt:event_info']['evt:incidentId'][ :512] + '-rule_id:' + \ event['evt:event']['evt:rules']['evt:rule']['evt:rule_id'] severity_object.Normalized = event['evt:event']['evt:rules'][ 'evt:rule']['evt:severity'] or 'LOW' resource_object.Id = event['evt:event']['evt:rules']['evt:rule'][ 'evt:policy_name'] extra_data.extra_data_handler([ event['evt:event']['evt:rules']['evt:rule']['evt:rule_name'] [:512] ], 'RuleName') elif int(event['evt:event']['evt:rules']['evt:Count']) > 1: findings_object.Id = 'incident_Id:' + event['evt:event']['evt:event_info']['evt:incidentId'][ :512] + '-rule_id:' + \ event['evt:event']['evt:rules']['evt:rule'][i]['evt:rule_id'] severity_object.Normalized = event['evt:event']['evt:rules'][ 'evt:rule'][i]['evt:severity'] or 'LOW' resource_object.Id = event['evt:event']['evt:rules']['evt:rule'][ i]['evt:policy_name'] extra_data.extra_data_handler([ event['evt:event']['evt:rules']['evt:rule'][i]['evt:rule_name'] [:512] ], 'Rule_name') findings_object.Severity = severity_object.__dict__ details_object.Other.update(extra_data.extra_data_storage) findings_object.Network = network_object.__dict__ findings_object.Resources = [resource_object.__dict__] findings.append(findings_object.__dict__) asff_object = JsonFormatClass() asff_object.findings = findings cleaned_findings = remove_null_items(asff_object.findings) for finding in cleaned_findings: i = 0 if finding['Severity']['Normalized']: if not (keys.get_configurations()[finding['Severity'] ['Normalized']]): del cleaned_findings[i] i += 1 for i in range(len(cleaned_findings)): cleaned_findings[i]['Severity']['Normalized'] = (__normalized_severity( cleaned_findings[i]['Severity']['Normalized'])) return cleaned_findings
def __init__(self, logger_): Thread.__init__(self) self.logger_ = logger_ self.daemon = True self.start() def run(self): insight_creator(self.logger_) if __name__ == "__main__": parser = argparse.ArgumentParser(description='DLPExporter') parser.add_argument('--login', action="store", dest='login', default='0') args = parser.parse_args() config = Configurations.get_configurations() if args.login != '0': username = input("Please enter your Database username: "******"Please enter your Database password: "******"sink": file_path, 'format': "{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}", "level": "INFO", 'rotation': "500 MB", 'enqueue': True, 'retention': 5