Exemplo n.º 1
0
def get_mta_siem_logs(checkpoint_dir, base_url, access_key, secret_key):
    # Set checkpoint file name to store page token
    checkpoint_filename = os.path.join(checkpoint_dir, 'get_mta_siem_logs_checkpoint')

    # Build post body for request
    post_body = dict()
    post_body['data'] = [{}]
    post_body['data'][0]['type'] = 'MTA'
    if os.path.exists(checkpoint_filename):
        post_body['data'][0]['token'] = read_file(checkpoint_filename)

    # Send request to API
    resp = connection.post_request(base_url, event_type, post_body, access_key, secret_key)

    # Process response
    if resp != 'error':
        resp_body = resp[0]
        resp_headers = resp[1]
        content_type = resp_headers['Content-Type']

        # End if response is JSON as there is no log file to download
        if content_type == 'application/json':
            log.info('No more SIEM logs available - Resting for 60 seconds')
            time.sleep(60)
            return True

        # Process log file
        elif content_type == 'application/octet-stream':
            file_name = resp_headers['Content-Disposition'].split('=\"')
            file_name = file_name[1][:-1]
            # Save file to log file path
            write_file(os.path.join(configuration.logging_details['LOG_FILE_PATH'], file_name), resp_body)
            # Save mc-siem-token page token to check point directory
            write_file(checkpoint_filename, resp_headers['mc-siem-token'])
            try:
                if configuration.syslog_details['syslog_output'] is True:
                    log.info('Loading file: ' + os.path.join(configuration.logging_details['LOG_FILE_PATH'], file_name) + ' to output to ' + configuration.syslog_details['syslog_server'] + ':' + str(configuration.syslog_details['syslog_port']))
                    with open(os.path.join(configuration.logging_details['LOG_FILE_PATH'], file_name), 'r') as log_file:
                        lines = log_file.read().splitlines()
                        for line in lines:
                            syslogger.info(line)
                    log.info('Syslog output completed for file ' + file_name)
            except Exception as e:
                log.error('Unexpected error writing to syslog. Exception: ' + str(e))

            # return true to continue loop
            return True

        else:
            # Handle errors
            log.error('Unexpected response')
            for header in resp_headers:
                log.error(header)
            return False
Exemplo n.º 2
0
def Get_TTPURL_events(base_url, access_key, secret_key):
    post_body = dict()
    resp = connection.post_request(base_url, event_type, post_body, access_key,
                                   secret_key)

    # Process response
    if resp != 'error':
        resp_body = resp[0]
        resp_headers = resp[1]
        content_type = resp_headers['Content-Type']

        # No more TTP events available
        if 'application/json' not in content_type:
            log.info('No more TTP URL logs available - Resting for 60 seconds')
            time.sleep(60)
            return True

        # Process log file
        elif 'application/json' in content_type:
            file_name = 'ttp_events'  # Storing everything into one file
            rjson = json.loads(resp_body)
            resp_body = rjson['data'][0]['clickLogs']  # Get TTP urls

            # Forward each event individually
            for row in resp_body:
                row = str(row).replace(
                    "'", '"')  # Convert audit event to valid JSON

                # Save file to log file path
                append_file(
                    os.path.join(
                        configuration.logging_details['LOG_FILE_PATH'],
                        file_name), str(row))

                try:
                    if configuration.syslog_details['syslog_output'] is True:
                        log.info(
                            'Loading file: ' + os.path.join(
                                configuration.logging_details['LOG_FILE_PATH'],
                                file_name) + ' to output to ' +
                            configuration.syslog_details['syslog_server'] +
                            ':' +
                            str(configuration.syslog_details['syslog_port']))
                        with open(
                                os.path.join(
                                    configuration.
                                    logging_details['LOG_FILE_PATH'],
                                    file_name), 'r') as log_file:
                            lines = log_file.read().splitlines()
                            for line in lines:
                                hashed_event = (hashlib.md5(
                                    line.encode('utf-8')).hexdigest())
                                with open(
                                        configuration.
                                        logging_details['CHK_POINT_DIR'] +
                                        'hash_file', 'r') as hash_file:
                                    if (hashed_event +
                                            '\n') in hash_file.readlines():
                                        print("Hash %s already exists" %
                                              (hashed_event))
                                    else:
                                        syslogger.info(line)
                                        append_file(
                                            configuration.
                                            logging_details['CHK_POINT_DIR'] +
                                            'hash_file', hashed_event)

                        log.info('Syslog output completed for file ' +
                                 file_name)

                except Exception as e:
                    log.error(
                        'Unexpected error writing to syslog. Exception: ' +
                        str(e))

            # Return True to continue loop
            return True

        else:
            # Handle errors
            log.error('Unexpected response')
            for header in resp_headers:
                log.error(header)
            return False
Exemplo n.º 3
0
def Get_TTPURL_events(base_url, access_key, secret_key):
    post_body = dict()
    resp = connection.post_request(base_url, event_type, post_body, access_key,
                                   secret_key)

    # Process response
    if resp != 'error':
        resp_body = resp[0]
        resp_headers = resp[1]
        content_type = resp_headers['Content-Type']

        # No more TTP events available
        if 'application/json' not in content_type:
            log.info('No more TTP URL logs available - Resting temporarily')
            time.sleep(interval_time)
            return True

        # Process log file
        elif 'application/json' in content_type:
            file_name = 'ttp_events'  # Storing everything into one file
            rjson = json.loads(resp_body)
            resp_body = rjson['data'][0]['clickLogs']  # Get TTP urls

            # Forward each event individually
            for row in resp_body:
                row = str(row).replace(
                    "'", '"')  # Convert audit event to valid JSON

                try:
                    if configuration.syslog_details['syslog_output'] is True:
                        hashed_event = (hashlib.md5(
                            row.encode('utf-8')).hexdigest())
                        if os.path.isfile(configuration.
                                          logging_details['CHK_POINT_DIR'] +
                                          str(event_category) +
                                          str(hashed_event)):  # If true
                            print("Hash already exists in %s" % configuration.
                                  logging_details['CHK_POINT_DIR'] +
                                  str(event_category) + str(hashed_event))
                        else:
                            log.info(
                                "Creating hash %s in %s and forwarding to configured syslog"
                                %
                                (hashed_event,
                                 configuration.logging_details['CHK_POINT_DIR']
                                 + str(event_category)))
                            os.mknod(configuration.
                                     logging_details['CHK_POINT_DIR'] +
                                     str(event_category) + str(hashed_event))
                            syslogger.info(row)

                        log.info("Syslog output completed for %s" %
                                 str(event_category))

                except Exception as e:
                    log.error(
                        'Unexpected error writing to syslog. Exception: ' +
                        str(e))

            # Return True to continue loop
            return True

        else:
            # Handle errors
            log.error('Unexpected response')
            for header in resp_headers:
                log.error(header)
            return False