def get_sumologic_fields(config, q, from_time, to_time, time_zone): fields = [] sumologic_access_id = config['sumologic_access_id'] sumologic_access_key = config['sumologic_access_key'] sumologic_root_url = config['sumologic_root_url'] LOGGER.info("Run query in sumologic") sumo = SumoLogic(sumologic_access_id, sumologic_access_key, sumologic_root_url) delay = 5 search_job = sumo.search_job(q, from_time, to_time, time_zone) status = sumo.search_job_status(search_job) while status['state'] != 'DONE GATHERING RESULTS': if status['state'] == 'CANCELLED': break time.sleep(delay) status = sumo.search_job_status(search_job) LOGGER.info(status['state']) if status['state'] == 'DONE GATHERING RESULTS': response = sumo.search_job_records(search_job, limit=1) fields = response['fields'] return fields
def __init__(self, connection, configuration): self.endpoint = (PROTOCOL + URL + ENDPOINT) % (connection.get("region")) self.auth = configuration.get('auth') self.client = SumoLogic(self.auth.get("access_id"), self.auth.get("access_key"), endpoint=self.endpoint)
def pingCollectors(): config = ConfigParser.RawConfigParser() config.readfp(open('config.properties')) # SumoLogic Access ID and Access Key accessID = config.get("Config", "accessID") accessKey = config.get("Config", "accessKey") sumo = SumoLogic(accessID, accessKey) theBlob = sumo.collectors() outputMsg = "The following collectors are offline:\n" empty = True for stuff in theBlob: collectorName = '' collectorAlive = False for attribute, value in stuff.iteritems(): if attribute == 'name': collectorName = value if attribute == 'alive': collectorAlive = value if collectorAlive == False: empty = False outputMsg += collectorName + "\n" if empty: return '' else: return outputMsg
def __init__(self, config): super(BaseAction, self).__init__(config) access_id = self.config.get('sumologic_access_id', None) access_key = self.config.get('sumologic_access_key', None) sumologic_collectors_limit = self.config.get( 'sumologic_collectors_limit', 1000) self._sumologic_access_key = access_key or None self._sumologic_access_id = access_id or None self._sumologic_collectors_limit = sumologic_collectors_limit or 1000 if self._sumologic_access_key is None or self._sumologic_access_id is None: self._client = None else: self._client = SumoLogic(self._sumologic_access_id, self._sumologic_access_key)
def get_sumologic_records(config, q, from_time, to_time, time_zone, limit): records = [] sumologic_access_id = config['sumologic_access_id'] sumologic_access_key = config['sumologic_access_key'] sumologic_root_url = config['sumologic_root_url'] LOGGER.info("Run query in sumologic") sumo = SumoLogic(sumologic_access_id, sumologic_access_key, sumologic_root_url) now_datetime = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f') custom_columns = { '_SDC_EXTRACTED_AT': now_datetime, '_SDC_BATCHED_AT': now_datetime, '_SDC_DELETED_AT': None } delay = 5 search_job = sumo.search_job(q, from_time, to_time, time_zone) LOGGER.info(search_job) status = sumo.search_job_status(search_job) while status['state'] != 'DONE GATHERING RESULTS': if status['state'] == 'CANCELLED': break time.sleep(delay) LOGGER.info(":check query status") status = sumo.search_job_status(search_job) LOGGER.info(status) LOGGER.info(status['state']) if status['state'] == 'DONE GATHERING RESULTS': record_count = status['recordCount'] count = 0 while count < record_count: LOGGER.info("Get records %d of %d, limit=%d", count, record_count, limit) response = sumo.search_job_records(search_job, limit=limit, offset = count) LOGGER.info("Got records %d of %d", count, record_count) recs = response['records'] # extract the result maps to put them in the list of records for rec in recs: records.append({**rec['map'], **custom_columns}) if len(recs) > 0: count = count + len(recs) else: break # make sure we exit if nothing comes back return records
def post_to_http_source(self, joined_results): """ Posts joined_results to a collector in Sumo sumologic Input: joined_results (List) : query_results with abuseIPDB results joined into each dictionary Return (None): Posts joined_results to specified selector """ if self.postAuthorization is True: sumo_api_post = SumoLogic( self.sumo_access_id, self.sumo_access_key, endpoint= "https://endpoint1.collection.us2.sumologic.com/receiver/v1/http/" ) post_object = sumo_api_post.post(self.uniqueHTTPCollectorCode, joined_results) print('\n') print(post_object) else: print("\nPost authorization disabled.\n")
def __init__(self, access_id, access_key, deployment): self.deployment = deployment self.sumologic_cli = SumoLogic(access_id, access_key, self.api_endpoint)
def __init__(self, props, *args, **kwargs): access_id, access_key, deployment = props["SumoAccessID"], props[ "SumoAccessKey"], props["SumoDeployment"] self.deployment = deployment self.sumologic_cli = SumoLogic(access_id, access_key, self.api_endpoint)
# python usage-report.py <accessId/email> <accessKey/password> <orgId> <fromTime> <toTime> <timezone> <timeslice> <email> # # TODO per-source # TODO log hook # TODO delete jobs? from email.mime.text import MIMEText import json from smtplib import SMTP import sys from sumologic import SumoLogic args = sys.argv sumo = SumoLogic(args[1], args[2], "https://long-api.sumologic.net/api/v1") orgId = args[3] fromTime = args[4] toTime = args[5] timezone = args[6] timeslice = args[7] fromEmail = '*****@*****.**' toEmail = args[8] lookup = "lookup/collector_name" q = r""" _sourceCategory=config "Collector by name and ID" !GURR "%s" | parse "[logger=*]" as logger | where logger = "scala.config.LoggingVisitor" | parse "Collector by name and ID, id: '*', decimal: '*', name: '*', organization ID: '*', decimal: '*', organization name: '*', organization type: '*'"
traceback.print_exc() logger.exception("error generating sumo query " + str(file) + "----" + str(e)) with open( os.path.join( args.outdir, "sigma-" + file_basename + '-error-generation.txt'), "w") as f: # f.write(json.dumps(r, indent=4, sort_keys=True) + " ERROR: %s\n\nQUERY: %s" % (e, sumo_query)) f.write(" ERROR for file: %s\n\Exception:\n %s" % (file, e)) continue try: # Run query # https://github.com/SumoLogic/sumologic-python-sdk/blob/3ad8033deb028ac45ac4099f11c04785fa426f51/scripts/search-job.py sumo = SumoLogic(args.accessid, args.accesskey, args.endpoint) toTime = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") fromTime = datetime.datetime.strptime( toTime, "%Y-%m-%dT%H:%M:%S") - datetime.timedelta(hours=24) fromTime = fromTime.strftime("%Y-%m-%dT%H:%M:%S") timeZone = 'UTC' byReceiptTime = True sj = sumo.search_job(sumo_query, fromTime, toTime, timeZone, byReceiptTime) status = sumo.search_job_status(sj) while status['state'] != 'DONE GATHERING RESULTS': if status['state'] == 'CANCELLED': break time.sleep(delay)
def connect(self, params): access_id = params.get("access_id").get("secretKey") access_key = params.get("access_key").get("secretKey") self.client = SumoLogic(access_id, access_key)
#!/usr/bin/python import os import time import datetime from sumologic import SumoLogic sumo = SumoLogic(os.environ["SUMO_ACCESS_ID"], os.environ["SUMO_ACCESS_KEY"], os.environ["SUMO_ENDPOINT"]) # --------------- INPUT with open("query.txt", "r") as query_file: query = query_file.read() duration = 160 * 60 * 1000 startTimeStep = 30 * 60 * 1000 firstStartTime = 1574636400000 lastStartTime = 1575241200000 runCount = int((lastStartTime - firstStartTime) / startTimeStep) # --------------------- def startQuery(iteration): thisStartTime = firstStartTime + iteration * startTimeStep return sumo.search_job(query, fromTime = thisStartTime, toTime = thisStartTime + duration, timeZone = "CET", byReceiptTime = False) def dateForIteration(iteration): return datetime.datetime.fromtimestamp((firstStartTime + iteration * startTimeStep) / 1000).strftime("%m/%d/%Y %I:%M %p")
from sumologic import SumoLogic if path.isfile("access.key"): cf = open("access.key", "r") creds = cf.readlines() else: sys.exit( "access.key file missing. Place your accessId and accessKey on separate lines in this file." ) args = sys.argv if len(args) < 3: sys.exit("usage: " + args[0] + " fromDate toDate [outFile]") sumo = SumoLogic(creds[0].strip(), creds[1].strip()) fromTime = args[1] toTime = args[2] timeZone = 'UTC' byReceiptTime = False r_fields = ['_messagetime', 'msg'] # names of fields to include in output try: outfile = args[3] except IndexError: outfile = "sumo_results_" + fromTime + "_to_" + toTime + ".txt" delay = 5 q = ' '.join(sys.stdin.readlines()) sj = sumo.search_job(q, fromTime, toTime, timeZone, byReceiptTime)
def sumo_query(**kwargs): # create variables based on user input query_file = kwargs.pop("query", None) access_id = kwargs.pop("access_id", None) access_key = kwargs.pop("access_key", None) endpoint = kwargs.pop("endpoint", None) from_time = kwargs.pop("from_time", None) to_time = kwargs.pop("to_time", None) by_receipt_time = kwargs.pop("by_receipt_time", None) timezone = kwargs.pop("timezone", None) verbose = kwargs.pop("verbose", None) LIMIT = kwargs.pop("limit", None) delay = 5 query = query_file.read().decode() # create connection instance sumo = SumoLogic(access_id, access_key, endpoint) # create search job search_job = sumo.search_job(query, from_time, to_time, timezone, by_receipt_time) # create search job status object and check state of the search job search_job_status = sumo.search_job_status(search_job) while search_job_status["state"] != "DONE GATHERING RESULTS": if search_job_status["state"] == "CANCELLED": break time.sleep(delay) search_job_status = sumo.search_job_status(search_job) if search_job_status["state"] == "DONE GATHERING RESULTS": count = search_job_status["recordCount"] limit = count if count < LIMIT and count != 0 else LIMIT result = sumo.search_job_messages(search_job, limit=limit) messages = result["messages"] # print result of the search job num = 0 for map in messages: # map_string = map['map']['_raw'].replace("\\n","").replace("\\t","").replace("\\","") map_string = map["map"]["_raw"] try: map_dict = eval(map_string) except NameError: map_dict = eval( map_string.replace("null", "None").replace("true", "True").replace( "false", "False")) # click.secho("//////// Docker info ////////", bg='white', fg='black') try: docker_dict = map_dict["message"]["docker"] click.secho("//////// Docker info ////////", bg="white", fg="black") pprint(docker_dict) except (KeyError, TypeError): click.secho("//////// Whole log message ////////", bg="white", fg="black") pprint(map_dict) click.secho( "\\\\\\\\\\\\\\\\ Whole log message \\\\\\\\\\\\\\\\", bg="white", fg="black", ) print() print("########" * 10 + " " + str(num) + " " + "########" * 10) print() num += 1 continue click.secho("\\\\\\\\\\\\\\\\ Docker info \\\\\\\\\\\\\\\\", bg="white", fg="black") click.secho("//////// Kubernetes info ////////", bg="white", fg="black") pprint(map_dict["message"]["kubernetes"]) click.secho("\\\\\\\\\\\\\\\\ Kubernetes info \\\\\\\\\\\\\\\\", bg="white", fg="black") click.secho("//////// Log ////////", bg="white", fg="black") try: map_log_dict = eval(map_dict["message"]["log"]) except SyntaxError: pprint(map_dict["message"]["log"], width=500) click.secho("\\\\\\\\\\\\\\\\ Log \\\\\\\\\\\\\\\\", bg="white", fg="black") print() print("########" * 10 + " " + str(num) + " " + "########" * 10) print() num += 1 continue if type(map_log_dict) != dict: pprint(map_log_dict, width=500) else: if not verbose: for key, value in map_log_dict.items(): if key == "stacktrace" or key == "errorVerbose": continue else: pprint(f"{key} : {map_log_dict[key]}", width=500) else: pprint(map_log_dict, width=500) click.secho("\\\\\\\\\\\\\\\\ Log \\\\\\\\\\\\\\\\", bg="white", fg="black") print() print("########" * 10 + " " + str(num) + " " + "########" * 10) print() num += 1
def main(): timezone = 'America/New_York' # Flag to use the by receipt time (_receiptTime), instead of the by message time (_messageTime) [default: 'false'] byReceiptTime = 'false' # Getting the current timestamp, in epoch format date_to = datetime.datetime.now().strftime('%s') # Getting the current timestamp, minus the delta hours, in epoch format date_from = datetime.datetime.now() - datetime.timedelta(hours=24) date_from = date_from.strftime('%s') # Defining new Sumo Logic search job object sumo = SumoLogic(cip_access_id, cip_access_key) # Creating the search job search = sumo.search_job(cip_query, date_from, date_to, timezone, byReceiptTime) # Verifying the status of the search job status = sumo.search_job_status(search) while status['state'] != 'DONE GATHERING RESULTS': if status['state'] == 'CANCELLED': break time.sleep(delay) status = sumo.search_job_status(search) ''' Section to handle messages ''' if status['state'] == 'DONE GATHERING RESULTS': # get results of cat on lookup table count = status['messageCount'] limit = count if count < 1000 and count != 0 else 1000 # compensate bad limit check results = sumo.search_job_messages(search, limit=limit) messages = results['messages'] # set parameters for uploading to CSE batch_size = 25 expiration = ( datetime.datetime.now() + datetime.timedelta(days=30)).strftime('%Y-%m-%dT%H:%M:%SZ') # uploading to CSE in batches of 25 for i in range(0, len(messages), batch_size): batch = messages[i:i + batch_size] items = {'items': []} for message in batch: record = message['map'] items['items'].append({ 'active': True, 'description': 'Automated update.', 'expiration': expiration, 'value': record[field_for_matchlist] }) r = requests.post(url=cse_url, headers=cse_headers, json=items) if r.status_code > 201: error = r.text status_code = r.status_code logging.error(f'Status Code: {status_code}') logging.error(f'Error uploading items: {error}') else: logging.info('Submitted 25 match list values.') pass
def __init__(self): """ Checks for command line arguements Input (None): sys.argv[] : System command line arguemnts Return (None): query (String) : The SumoLogic query defaultQuery (Bool) : Indicates if default query was executed fromTime (String) : Date for query to start search toTime (String) : Date for query to end search timeZone (String) : Timezone so query knows what timezone to align its search times to current_date (String) : Today's date uniqueHTTPCollectorCode (String) : The code for the collector to post to sumo_access_id (String) : Needed to access SumoLogic through Sumo Logic Python SDK sumo_access_key (String) : Needed to access SumoLogic through Sumo Logic Python SDK abuse_apiV2_key (String) : Needed to access AbuseIPDB api abuseIPDB_days (String) : Number of days AbuseIPDB api will use when looking up ip """ try: sys.argv[1] #query assignment except: sys.exit("Status Code 1\n1 parameters is missing (query).") else: if sys.argv[1] == "default-all": self.defaultQuery = True from default_query import query if not query: sys.exit( "Status Code 1\n1 parameters is missing in defaults (query)." ) self.query = query from default_query import current_date, fromTime, toTime, timeZone if not fromTime or not toTime or not timeZone or not current_date: sys.exit( "Status Code 1\nAt least 1 time parameters is missing in defaults (fromTime, toTime, timeZone, or current_date)." ) self.fromTime = fromTime self.toTime = toTime self.timeZone = timeZone self.current_date = current_date from sumo_collector_code import uniqueHTTPCollectorCode if not uniqueHTTPCollectorCode: sys.exit( "Status Code 1\nAt least 1 parameters is missing in defaults (uniqueHTTPCollectorCode)." ) self.uniqueHTTPCollectorCode = uniqueHTTPCollectorCode self.postAuthorization = True from api_keys import sumo_access_id, sumo_access_key if not sumo_access_id or not sumo_access_key: sys.exit( "Status Code 1\nAt least 1 sumo access parameters is missing in defaults (sumo_access_id or sumo_access_key)." ) self.sumo_access_id = sumo_access_id self.sumo_access_key = sumo_access_key self.sumo_api = SumoLogic(sumo_access_id, sumo_access_key) from api_keys import abuse_apiV2_key if not abuse_apiV2_key: sys.exit( "Status Code 1\nAt least 1 parameters is missing in defaults (abuse_apiV2_key)." ) self.abuse_apiV2_key = abuse_apiV2_key from abuseipdb_parameters import abuseIPDB_days if not abuseIPDB_days: sys.exit( "Status Code 1\nAt least 1 parameters is missing in defaults (abuseIPDB_days)." ) self.abuseIPDB_days = abuseIPDB_days self.log_directory = "./logs" return elif sys.argv[1] == "default": self.defaultQuery = True from default_query import query if not query: sys.exit( "Status Code 1\n1 parameters is missing in defaults (query)." ) self.query = query else: self.defaultQuery = False with open(str(sys.argv[1])) as file: self.query = str(file.read()) try: sys.argv[2] #fromTime assignment sys.argv[3] #toTime assignment sys.argv[4] #timeZone assignment except: sys.exit( "Status Code 1\nAt least 1 time parameters is missing (fromTime, toTime, or timeZone)." ) else: if sys.argv[2] == sys.argv[3] == sys.argv[4] == "default": from default_query import current_date, fromTime, toTime, timeZone if not fromTime or not toTime or not timeZone or not current_date: sys.exit( "Status Code 1\nAt least 1 time parameters is missing in defaults (fromTime, toTime, timeZone, or current_date)." ) self.fromTime = fromTime self.toTime = toTime self.timeZone = timeZone self.current_date = current_date elif sys.argv[2] or sys.argv[ 3] or sys.argv[4] != "default" and sys.argv[2] or sys.argv[ 3] or sys.argv[4] == "default": sys.exit( "Status Code 1\nAll parameters must be default or inputted (fromTime, toTime, or timeZone)." ) else: from default_query import current_date if not current_date: sys.exit( "Status Code 1\nAt least 1 time parameters is missing in defaults (current_date)." ) self.fromTime = sys.argv[2] self.toTime = sys.argv[3] self.timeZone = sys.argv[4] self.current_date = current_date try: sys.argv[5] #uniqueHTTPCollectorCode assignment except: sys.exit( "Status Code 1\nAt least 1 parameters is missing (uniqueHTTPCollectorCode)." ) else: if sys.argv[5] == "default": from sumo_collector_code import uniqueHTTPCollectorCode if not uniqueHTTPCollectorCode: sys.exit( "Status Code 1\nAt least 1 parameters is missing in defaults (uniqueHTTPCollectorCode)." ) self.uniqueHTTPCollectorCode = uniqueHTTPCollectorCode self.postAuthorization = True elif sys.argv[5] == "False": self.uniqueHTTPCollectorCode = "" self.postAuthorization = False else: self.uniqueHTTPCollectorCode = sys.argv[5] self.postAuthorization = True try: sys.argv[6] #sumo_access_id assignment sys.argv[7] #sumo_access_key assignment except: sys.exit( "Status Code 1\nAt least 1 sumo access parameters is missing (sumo_access_id or sumo_access_key)." ) else: if sys.argv[6] == sys.argv[7] == "default": from api_keys import sumo_access_id, sumo_access_key if not sumo_access_id or not sumo_access_key: sys.exit( "Status Code 1\nAt least 1 sumo access parameters is missing in defaults (sumo_access_id or sumo_access_key)." ) self.sumo_access_id = sumo_access_id self.sumo_access_key = sumo_access_key self.sumo_api = SumoLogic(sumo_access_id, sumo_access_key) elif sys.argv[6] == "default" and sys.argv[ 7] != "default" or sys.argv[6] != "default" and sys.argv[ 7] == "default": sys.exit( "Status Code 1\nBoth parameters must be default or inputted (sumo_access_id or sumo_access_key)." ) else: self.sumo_access_id = sys.argv[6] self.sumo_access_key = sys.argv[7] self.sumo_api = SumoLogic(self.sumo_access_id, self.sumo_access_key) try: sys.argv[8] #abuse_apiV2_key assignment except: sys.exit( "Status Code 1\nAt least 1 parameters is missing (abuse_apiV2_key)." ) else: if sys.argv[8] == "default": from api_keys import abuse_apiV2_key if not abuse_apiV2_key: sys.exit( "Status Code 1\nAt least 1 parameters is missing in defaults (abuse_apiV2_key)." ) self.abuse_apiV2_key = abuse_apiV2_key else: self.abuse_apiV2_key = sys.argv[8] try: sys.argv[9] #abuseIPDB_days assignment except: sys.exit( "Status Code 1\nAt least 1 parameters is missing (abuseIPDB_days)." ) else: if sys.argv[9] == "default": from abuseipdb_parameters import abuseIPDB_days if not abuseIPDB_days: sys.exit( "Status Code 1\nAt least 1 parameters is missing in defaults (abuseIPDB_days)." ) self.abuseIPDB_days = abuseIPDB_days else: self.abuseIPDB_days = sys.argv[9] try: sys.argv[10] #log_directory assignment except: sys.exit( "Status Code 1\nAt least 1 parameters is missing (log_directory)." ) else: if sys.argv[10] == "default": self.log_directory = "./logs" else: self.log_directory = sys.argv[10]
def connect(self, params): access_id = params.get('access_id').get('secretKey') access_key = params.get('access_key').get('secretKey') self.client = SumoLogic(access_id, access_key)
# Get collectors where field contains some specified string # # python get-collectors.py <accessId> <accessKey> <field> <string> import sys from sumologic import SumoLogic args = sys.argv sumo = SumoLogic(args[1], args[2]) field, string = args[3], args[4] cs = sumo.collectors() for c in cs: if field in c and string in c[field]: print sumo.sources(c['id'])
from decouple import config from vmware.vapi.vsphere.client import create_vsphere_client from sumologic import SumoLogic import sys # Vsphere vc_username = config('VC_USERNAME') vc_userpass = config('VC_USERPASS') vsphere1_host = config('VSPHERE1_HOST') vsphere2_host = config('VSPHERE2_HOST') # Sumo sumo_access_id = config('SUMO_ACCESS_ID') sumo_access_key = config('SUMO_ACCESS_KEY') sumo = SumoLogic(sumo_access_id, sumo_access_key) def delete_collector(sumo_collector): url = f"https://api.us2.sumologic.com/api/v1/collectors/{sumo_collector['id']}" r = requests.delete(url, auth=(sumo_access_id, sumo_access_key)) print( f"I: Deleting {sumo_collector['id']} {sumo_collector['name']} rsp: {r}" ) return r def delete_powered_off_vm_sumo_collectors(sumo_collectors, vms): for sumo_collector in sumo_collectors: if sumo_collector['alive'] == False: match_ratio, match_vm = get_vm_name_in_sumo_collector_name_ratio(