def publish_to_mozdef(summary='', details={}): msg = mozdef_client.MozDefEvent('') msg.summary = summary msg.tags = ['asap'] msg.details = json.loads(json.dumps(details, default=json_serial)) region, account_id, queue_name = app.config.get('SQS_QUEUE_ARN').split( ':')[3:] msg.set_send_to_sqs(True) msg.set_sqs_queue_name(queue_name) msg.set_sqs_region(region) msg.set_sqs_aws_account_id(account_id) # Note that unlike syslog this will NEVER send to MozDef HTTP (URL is # ignored) app.logger.debug( "Alerter: Sending message to SQS queue {} in account {} in region {}". format(queue_name, account_id, region)) try: msg.send() except (botocore.exceptions.ClientError, botocore.parsers.ResponseParserError) as e: app.logger.critical( "Alerter: Attempt to send message to SQS queue {} in account {} " "in region {} failed with error {}".format(queue_name, account_id, region, e))
def fetch_auth0_logs(config, headers, fromid): lastid = fromid r = requests.get( "{url}?take={reqnr}&sort=date:1&per_page={reqnr}&from={fromid}&include_totals=true".format( url=config.auth0.url, reqnr=config.auth0.reqnr, fromid=fromid ), headers=headers, ) # If we fail here, auth0 is not responding to us the way we expected it if not r.ok: raise Exception(r.url, r.reason, r.status_code, r.json()) ret = r.json() # Sometimes API give us the requested totals.. sometimes not. # To be clear; totals are now only returned when using `page=..` and not using `from=..` parameters # The issue is that when using `page`, auth0 internally splices the log by page, which is extremely slow and the # call takes 10-20s to return each time. # When using `from` auth0 queries the index for that location which is fast, so we use `from` # this means we can't properly page results, so we have to "try to fetch" until no more logs are returned # Finally note that when using `from` the `sort` ordering is not guaranteed to work according to the API docs if type(ret) is dict and "logs" in ret: have_totals = True all_msgs = ret["logs"] else: have_totals = False all_msgs = ret # Process all new auth0 log msgs, normalize and send them to mozdef for msg in all_msgs: mozmsg = mozdef.MozDefEvent(config.mozdef.url) if config.DEBUG == "True": mozmsg.set_send_to_syslog(True, only_syslog=True) mozmsg.hostname = config.auth0.url mozmsg.tags = ["auth0"] msg = byteify(msg) msg = DotDict(msg) lastid = msg._id # Fill in mozdef msg fields from the auth0 msg try: mozmsg = process_msg(mozmsg, msg) except KeyError as e: # if this happens the msg was malformed in some way mozmsg.details["error"] = "true" mozmsg.details["errormsg"] = '"' + str(e) + '"' mozmsg.summary = "Failed to parse auth0 message" traceback.print_exc() # Save raw initial message in final message # in case we ran into parsing errors mozmsg.details["raw"] = str(msg) mozmsg.send() if have_totals: return (int(ret["total"]), int(ret["start"]), int(ret["length"]), lastid) else: return (-1, -1, -1, lastid)
def test_sample_event_username_nonexistent(self): mozmsg = mozdef.MozDefEvent('http://localhost:9090') del (self.sample_event['user_name']) del (self.sample_event['user_id']) process_msg(mozmsg, self.sample_event) assert 'username' not in mozmsg.details assert 'userid' not in mozmsg.details
def send(logger, method_name, event_dict): # only send to mozdef if `mozdef` is set if event_dict.pop('mozdef', False): msg = mozdef_client.MozDefEvent(MOZDEF) msg.summary = event_dict.get('event', '') msg.tags = [ 'mozilla/release-services/' + channel, project_name, ] if set(event_dict) - {'event'}: msg.details = event_dict.copy() msg.details.pop('event', None) msg.source = logger.name msg.set_severity( sevirity_map.get( method_name, mozdef_client.MozDefEvent.SEVERITY_INFO, ), ) msg.send() return event_dict
def main(): try: state = pickle.load(open(options.statepath, 'rb')) except IOError: # Oh, you're new. state = {'administration': 0, 'authentication': 0, 'telephony': 0} duo = duo_client.Admin(ikey=options.IKEY, skey=options.SKEY, host=options.URL) mozmsg = mozdef.MozDefEvent(options.MOZDEF_URL) mozmsg.tags = ['duosecurity', 'logs'] if options.update_tags != '': mozmsg.tags.append(options.update_tags) mozmsg.category = 'Authentication' mozmsg.source = 'DuoSecurity API' if options.DEBUG: mozmsg.debug = options.DEBUG mozmsg.set_send_to_syslog(True, only_syslog=True) # This will process events for all 3 log types and send them to MozDef. the state stores the last position in the # log when this script was last called. state = process_events( mozmsg, duo.get_administrator_log(mintime=state['administration'] + 1), 'administration', state) state = process_events( mozmsg, duo.get_authentication_log(mintime=state['authentication'] + 1), 'authentication', state) state = process_events( mozmsg, duo.get_telephony_log(mintime=state['telephony'] + 1), 'telephony', state) pickle.dump(state, open(options.statepath, 'wb'))
def main(): try: state = pickle.load(open(options.statepath, "rb")) except IOError: # Oh, you're new. # Note API v2 expect full, correct and within range timestamps in millisec so we start recently # API v1 uses normal timestamps in seconds instead state = { "administration": 0, "administration_offset": None, "authentication": 1547000000000, "authentication_offset": None, "telephony": 0, "telephony_offset": None, } # Convert v1 (sec) timestamp to v2 (ms)... if state["authentication"] < 1547000000000: state["authentication"] = int(str(state["authentication"]) + "000") duo = duo_client.Admin(ikey=options.IKEY, skey=options.SKEY, host=options.URL) mozmsg = mozdef.MozDefEvent(options.MOZDEF_URL) mozmsg.tags = ["duosecurity"] if options.update_tags != "": mozmsg.tags.append(options.update_tags) mozmsg.set_category("authentication") mozmsg.source = "DuoSecurityAPI" if options.DEBUG: mozmsg.debug = options.DEBUG mozmsg.set_send_to_syslog(True, only_syslog=True) # This will process events for all 3 log types and send them to MozDef. the state stores the last position in the # log when this script was last called. # NOTE: If administration and telephone logs support a "v2" API in the future it will most likely need to have the # same code with `next_offset` as authentication uses. state = process_events( mozmsg, duo.get_administrator_log(mintime=state["administration"] + 1), "administration", state) state = process_events( mozmsg, duo.get_authentication_log( api_version=2, limit="1000", sort="ts:asc", mintime=state["authentication"] + 1, next_offset=state["authentication_offset"], ), "authentication", state, ) state = process_events( mozmsg, duo.get_telephony_log(mintime=state["telephony"] + 1), "telephony", state) pickle.dump(state, open(options.statepath, "wb"))
def fetch_auth0_logs(config, headers, fromid): lastid = fromid r = requests.get( '{url}?take={reqnr}&sort=date:1&per_page={reqnr}&include_totals=true&from={fromid}' .format(url=config.auth0.url, reqnr=config.auth0.reqnr, fromid=fromid), headers=headers) #If we fail here, auth0 is not responding to us the way we expected it if (not r.ok): raise Exception(r.url, r.reason, r.status_code, r.json()) ret = r.json() #Sometimes API give us the requested totals.. sometimes not. if (type(ret) is dict) and ('logs' in ret.keys()): have_totals = True all_msgs = ret['logs'] else: have_totals = False all_msgs = ret #Process all new auth0 log msgs, normalize and send them to mozdef for msg in all_msgs: mozmsg = mozdef.MozDefEvent(config.mozdef.url) if config.DEBUG == 'True': mozmsg.set_send_to_syslog(True, only_syslog=True) mozmsg.hostname = config.auth0.url mozmsg.tags = ['auth0'] msg = byteify(msg) msg = DotDict(msg) lastid = msg._id #Fill in mozdef msg fields from the auth0 msg try: mozmsg = process_msg(mozmsg, msg) except KeyError as e: #if this happens the msg was malformed in some way mozmsg.details['error'] = 'true' mozmsg.details['errormsg'] = '"' + str(e) + '"' mozmsg.summary = 'Failed to parse auth0 message' if config.DEBUG == 'True': traceback.print_exc() mozmsg.send() if have_totals: return (int(ret['total']), int(ret['start']), int(ret['length']), lastid) else: return (0, 0, 0, lastid)
def send(logger, method_name, event_dict): # only send to mozdef if `mozdef` is set if event_dict.pop('mozdef', False): msg = mozdef_client.MozDefEvent(target) msg.summary = event_dict.get('event', '') msg.tags = ['relengapi'] if set(event_dict) - {'event'}: msg.details = event_dict.copy() msg.details.pop('event', None) msg.source = logger.name msg.set_severity( sev_map.get(method_name, mozdef_client.MozDefEvent.SEVERITY_INFO)) msg.send() # return the message unchanged return event_dict
def main(): mozmsg = mozdef.MozDefEvent(options.mozdef_url) mozmsg.tags = ["uptycs"] mozmsg.set_category("uptycs") mozmsg.source = "UptycsAPI" if options.debug: mozmsg.debug = options.debug mozmsg.set_send_to_syslog(True, only_syslog=True) client = UptycsClient(options.uptycs_api_json_file) # Adjust the filter as needed to set proper search window uptycs_filter = UptycsFilter() utc_now = toUTC(datetime.now()) # If an existing state file exists, set our start window to last run if os.path.exists(options.statepath): state = pickle.load(open(options.statepath, 'rb')) uptycs_filter.start_time = state["last_run"] last_alert_ids = state["last_alert_ids"] # Otherwise, we pick from a boostrap time window else: last_alert_ids = [] uptycs_filter.start_time = utc_now - timedelta( days=int(options.bootstrap_search_depth)) uptycs_filter.end_time = utc_now # Query alerts that match the uptycs_filter alerts = client.alerts(uptycs_filter) alert_ids = [] if len(alerts) > 0: for alert in alerts: if alert['id'] in last_alert_ids: continue else: alert_ids.append(alert['id']) # Process all these alerts in MozDef process_alerts(mozmsg, alerts) state = { "last_run": uptycs_filter.end_time, "last_alert_ids": alert_ids, } pickle.dump(state, open(options.statepath, "wb"))
def main(): try: state = pickle.load(open(options.statepath, 'rb')) except IOError: # Oh, you're new. # Note API v2 expect full, correct and within range timestamps in millisec so we start recently # API v1 uses normal timestamps in seconds instead state = { 'administration': 0, 'authentication': 1547000000000, 'telephony': 0 } # Convert v1 (sec) timestamp to v2 (ms)... if state['authentication'] < 1547000000000: state['authentication'] = int(str(state['authentication']) + '000') duo = duo_client.Admin(ikey=options.IKEY, skey=options.SKEY, host=options.URL) mozmsg = mozdef.MozDefEvent(options.MOZDEF_URL) mozmsg.tags = ['duosecurity'] if options.update_tags != '': mozmsg.tags.append(options.update_tags) mozmsg.set_category('authentication') mozmsg.source = 'DuoSecurityAPI' if options.DEBUG: mozmsg.debug = options.DEBUG mozmsg.set_send_to_syslog(True, only_syslog=True) # This will process events for all 3 log types and send them to MozDef. the state stores the last position in the # log when this script was last called. state = process_events( mozmsg, duo.get_administrator_log(mintime=state['administration'] + 1), 'administration', state) # TODO Should use `next_offset` instead of mintime in the future (for api v2) as its more efficient state = process_events( mozmsg, duo.get_authentication_log(api_version=2, mintime=state['authentication'] + 1), 'authentication', state) state = process_events( mozmsg, duo.get_telephony_log(mintime=state['telephony'] + 1), 'telephony', state) pickle.dump(state, open(options.statepath, 'wb'))
def main(): #Configuration loading with open('auth02mozdef.json') as fd: config = DotDict(hjson.load(fd)) if config == None: print("No configuration file 'auth02mozdef.json' found.") sys.exit(1) headers = {'Authorization': 'Bearer {}'.format(config.auth0.token), 'Accept': 'application/json'} fromid = load_state(config.state_file) r = requests.get('{url}?take={reqnr}&sort=date:1&per_page={reqnr}&include_totals=true&from={fromid}'.format( url=config.auth0.url, reqnr=config.auth0.reqnr, fromid=fromid), headers=headers) #If we fail here, auth0 is not responding to us the way we expected it if (not r.ok): raise Exception(r.url, r.reason, r.status_code, r.json()) ret = r.json() #Process all new auth0 log msgs, normalize and send them to mozdef for msg in ret: mozmsg = mozdef.MozDefEvent(config.mozdef.url) if config.DEBUG: mozmsg.set_send_to_syslog(True, only_syslog=True) mozmsg.source = config.auth0.url mozmsg.tags = ['auth0'] msg = DotDict(msg) lastid = msg._id #Fill in mozdef msg fields from the auth0 msg try: mozmsg = process_msg(mozmsg, msg) except KeyError as e: #if this happens the msg was malformed in some way mozmsg.details['error'] = 'true' mozmsg.details['errormsg'] = e mozmsg.summary = 'Failed to parse auth0 message' mozmsg.send() save_state(config.state_file, lastid)
def test_sample_event_username(self): mozmsg = mozdef.MozDefEvent('http://*****:*****@mozilla.com' assert mozmsg.details.userid == 'ad|Test-Connection|ttesterson' assert mozmsg.summary == 'Success Silent Auth [email protected]'
def main(): logger.debug('started') state = State(options.state_file_name) try: # capture the time we start running so next time we catch any events # created while we run. lastrun = toUTC(datetime.now()).isoformat() scope = [ 'https://www.googleapis.com/auth/admin.reports.audit.readonly', 'https://www.googleapis.com/auth/admin.reports.usage.readonly' ] # get our credentials credentials = service_account.Credentials.from_service_account_file( options.jsoncredentialfile, scopes=scope, subject=options.impersonate) # build a request to the admin sdk api = googleapiclient.discovery.build('admin', 'reports_v1', credentials=credentials) response = api.activities().list( userKey='all', applicationName='login', startTime=toUTC( state.data['lastrun']).strftime('%Y-%m-%dT%H:%M:%S.000Z'), maxResults=options.recordlimit).execute() # fix up the event craziness to a flatter format events = [] if 'items' in response: for i in response['items']: # flatten the sub dict/lists to pull out the good parts mozmsg = mozdef.MozDefEvent(options.url) mozmsg.category = 'google' mozmsg.tags = ['google', 'authentication'] mozmsg.severity = 'INFO' mozmsg.summary = 'google authentication: ' details = dict() for keyValue in flattenDict(i): # change key/values like: # [email protected] # to actor_email=value try: key, value = keyValue.split('=') except ValueError as e: continue key = key.replace('.', '_').lower() details[key] = value # find important keys # and adjust their location/name if 'ipaddress' in details: # it's the source ip details['sourceipaddress'] = details['ipaddress'] del details['ipaddress'] if 'id_time' in details: mozmsg.timestamp = details['id_time'] mozmsg.utctimestamp = details['id_time'] if 'events_name' in details: mozmsg.summary += details['events_name'] + ' ' if 'actor_email' in details: mozmsg.summary += details['actor_email'] + ' ' mozmsg.details = details events.append(mozmsg) # post events to mozdef logger.debug('posting {0} google events to mozdef'.format(len(events))) for e in events: e.send() # record the time we started as # the start time for next time. state.data['lastrun'] = lastrun state.write_state_file() except Exception as e: logger.error("Unhandled exception, terminating: %r" % e)
parser.add_option( '--num_times', help='Number of times event is sent to loginput (default: 20)', default=20) options, arguments = parser.parse_args() # Fill in with events you want to write events = [{ "category": "testcategory", "details": { "program": "sshd", "type": "Success Login", "username": "******", "sourceipaddress": '1.2.3.4', }, "processname": "auth0_cron", "severity": "INFO", "source": "auth0", "summary": "login invalid ldap_count_entries failed", "tags": ["auth0"], }] for num in range(0, options.num_times): for event in events: mozmsg = mozdef.MozDefEvent(options.loginput_host + "/events/") for key, value in event.items(): setattr(mozmsg, key, value) mozmsg.send() print("Wrote event to loginput") time.sleep(0.2)