def fetch(self):
current_state = self.get_state()
output_handler = OutputHandlerFactory.get_handler(self.collection_config['OUTPUT_HANDLER'], path=self.pathname, config=self.config)
url, kwargs = self.build_fetch_params()
log_type = self.get_key()
next_request = True
count = 0
sess = ClientMixin.get_new_session()
self.log.info(f'''Fetching LogType: {log_type} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
try:
while next_request:
send_success = has_next_page = False
status, data = ClientMixin.make_request(url, method="get", session=sess, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'], **kwargs)
fetch_success = status and "results" in data
if fetch_success:
has_next_page = len(data['results']) > 0
if has_next_page:
payload, updated_state = self.transform_data(data)
params = self.build_send_params()
send_success = output_handler.send(payload, **params)
if send_success:
count +=1
self.log.debug(f'''Successfully sent LogType: {log_type} Page: {kwargs['params']['pageNum']} Datalen: {len(payload)} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
kwargs['params']['pageNum'] += 1
# save and update last_time_epoch required for next invocation
current_state.update(updated_state)
# time not available save current state new page num else continue
if not self.is_time_remaining():
self.save_state({
"start_time_epoch": convert_utc_date_to_epoch(kwargs['params']['minDate']),
"end_time_epoch": convert_utc_date_to_epoch(kwargs['params']['maxDate']),
"page_num": kwargs['params']["pageNum"],
"last_time_epoch": current_state['last_time_epoch']
})
else:
# show err unable to send save current state
self.log.error(f'''Failed to send LogType: {log_type} Page: {kwargs['params']['pageNum']} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
self.save_state({
"start_time_epoch": convert_utc_date_to_epoch(kwargs['params']['minDate']),
"end_time_epoch": convert_utc_date_to_epoch(kwargs['params']['maxDate']),
"page_num": kwargs['params']["pageNum"],
"last_time_epoch": current_state['last_time_epoch']
})
else:
self.log.debug(f'''Moving starttime window LogType: {log_type} Page: {kwargs['params']['pageNum']} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
# here fetch success is false and assuming pageNum starts from 1
# genuine no result window no change
# page_num has finished increase window calc last_time_epoch and add 1
if kwargs['params']['pageNum'] > 1:
self.save_state({
"page_num": 0,
"last_time_epoch": current_state['last_time_epoch'] + self.MOVING_WINDOW_DELTA
})
else:
self.log.error(f'''Failed to fetch LogType: {log_type} Page: {kwargs['params']['pageNum']} Reason: {data} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
next_request = fetch_success and send_success and has_next_page and self.is_time_remaining()
finally:
sess.close()
self.log.info(f'''Completed LogType: {log_type} Count: {count} Page: {kwargs['params']['pageNum']} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
def fetch(self):
current_state = self.get_state()
output_handler = OutputHandlerFactory.get_handler(self.collection_config['OUTPUT_HANDLER'], path=self.pathname, config=self.config)
url, kwargs = self.build_fetch_params()
next_request = True
sess = ClientMixin.get_new_session()
log_type = self.get_key()
count = 0
self.log.info(f'''Fetching LogType: {log_type} pageNum: {kwargs["params"]["pageNum"]}''')
try:
while next_request:
send_success = has_next_page = False
status, data = ClientMixin.make_request(url, method="get", session=sess, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'], **kwargs)
fetch_success = status and "results" in data
if fetch_success:
has_next_page = len(data['results']) > 0
if has_next_page:
payload, updated_state = self.transform_data(data)
send_success = output_handler.send(payload, **self.build_send_params())
if send_success:
count += 1
self.log.debug(f'''Fetching Project: {self.api_config['PROJECT_ID']} Alerts Page: {kwargs['params']['pageNum']} Datalen: {len(payload)} ''')
current_state.update(updated_state)
if current_state['last_page_offset'] == 0:
# do not increase if num alerts < page limit
kwargs['params']['pageNum'] += 1
else:
has_next_page = False
# time not available save current state new page num else continue
if (not self.is_time_remaining()) or (not has_next_page):
self.save_state({
"page_num": kwargs['params']["pageNum"],
"last_page_offset": current_state['last_page_offset']
})
else:
# show err unable to send save current state
self.log.error(f'''Unable to send Project: {self.api_config['PROJECT_ID']} Alerts Page: {kwargs['params']['pageNum']} ''')
self.save_state({
"page_num": kwargs['params']["pageNum"],
"last_page_offset": current_state['last_page_offset']
})
else:
self.log.debug(f'''Moving starttime window Project: {self.api_config['PROJECT_ID']} Alerts Page: {kwargs['params']['pageNum']} ''')
# here send success is false
# genuine no result window no change
# page_num has finished increase window calc last_time_epoch and add 1
self.save_state({
"page_num": kwargs['params']["pageNum"],
"last_page_offset": current_state['last_page_offset']
})
else:
self.log.error(f'''Unable to fetch Project: {self.api_config['PROJECT_ID']} Alerts Page: {kwargs['params']['pageNum']} Reason: {data} ''')
next_request = fetch_success and send_success and has_next_page and self.is_time_remaining()
finally:
sess.close()
self.log.info(f'''Completed LogType: {log_type} Count: {count} Page: {kwargs['params']['pageNum']}''')
def fetch(self):
log_type = self.get_key()
output_handler = OutputHandlerFactory.get_handler(self.collection_config['OUTPUT_HANDLER'], path=self.pathname, config=self.config)
url, kwargs = self.build_fetch_params()
self.log.info(f'''Fetching LogType: {log_type} kwargs: {kwargs}''')
state = None
payload = []
try:
fetch_success, content = ClientMixin.make_request(url, method="get", logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'], **kwargs)
if fetch_success and len(content) > 0:
payload, state = self.transform_data(content)
#Todo Make this atomic if after sending -> Ctrl - C happens then it fails to save state
params = self.build_send_params()
send_success = output_handler.send(payload, **params)
if send_success:
self.save_state(**state)
self.log.info(f'''Successfully sent LogType: {self.get_key()} Data: {len(content)}''')
else:
self.log.error(f'''Failed to send LogType: {self.get_key()}''')
else:
self.log.info(f'''No results status: {fetch_success} reason: {content}''')
finally:
output_handler.close()
self.log.info(f'''Completed LogType: {log_type} curstate: {state} datasent: {len(payload)}''')
def fetch(self, url, event_type, start_time_epoch, end_time_epoch, last_record_epoch):
params = {
'token': self.api_config['TOKEN'],
'limit': self.api_config['PAGINATION_LIMIT'],
'starttime': start_time_epoch,
'endtime': end_time_epoch,
'skip': 0,
'type': event_type
}
if last_record_epoch:
params['endtime'] = last_record_epoch # logs of same timestamp may be repeated
output_handler = OutputHandlerFactory.get_handler(self.config['Collection']['OUTPUT_HANDLER'], config=self.config)
next_request = send_success = True
page_count = total_records = 0
move_window = False
sess = self.netskope_conn.get_request_session()
last_record_epoch = None
try:
while next_request:
page_count += 1
fetch_success, respjson = ClientMixin.make_request(url, method=self.api_config['FETCH_METHOD'], session=sess, params=params, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'])
fetch_success = fetch_success and respjson["status"] == "success" # netskope sends 200 for errors
if fetch_success:
data = respjson["data"]
if len(data) > 0:
data = self.transform_data(data)
send_success = output_handler.send(data)
if send_success:
params['skip'] += len(data)
total_records += len(data)
last_record_epoch = data[-1]["timestamp"]
self.log.info(f'''Successfully Sent Page: {page_count} Event Type: {event_type} Datalen: {len(
data)} starttime: {convert_epoch_to_utc_date(
params['starttime'])} endtime: {convert_epoch_to_utc_date(params['endtime'])} skip: {params['skip']} last_record_epoch: {convert_epoch_to_utc_date(last_record_epoch)}''')
else: # no data so moving window
move_window = True
else:
self.log.error("Unable to fetch Response %s" % respjson)
next_request = fetch_success and send_success and (not move_window)
if move_window:
self.log.debug(
f'''Moving starttime window for {event_type} to {convert_epoch_to_utc_date(params["endtime"] + 1)}''')
self.set_fetch_state(event_type, end_time_epoch+1, None, None)
elif not (fetch_success and send_success): # saving skip in casee of failures for restarting in future
self.set_fetch_state(event_type, start_time_epoch, end_time_epoch, last_record_epoch)
self.log.error(
f'''Failed to send Event Type: {event_type} Page: {page_count} starttime: {convert_epoch_to_utc_date(params['starttime'])} endtime: {convert_epoch_to_utc_date(params['endtime'])} fetch_success: {fetch_success} send_success: {send_success} skip: {params['skip']} last_record_epoch: {last_record_epoch}''')
except Exception as e:
self.set_fetch_state(event_type, start_time_epoch, end_time_epoch, last_record_epoch)
raise e
finally:
self.netskope_conn.close()
output_handler.close()
self.log.info(f''' Total messages fetched {total_records} for Event Type: {event_type} starttime: {convert_epoch_to_utc_date(params['starttime'])} endtime: {convert_epoch_to_utc_date(params['endtime'])}''')
def get_violations_details(self, violation):
sess = ClientMixin.get_new_session()
if "violation_details_url" in violation:
try:
fetch_success, result = ClientMixin.make_request(violation["violation_details_url"], method="get",
session=sess, logger=self.log,
TIMEOUT=self.collection_config['TIMEOUT'],
MAX_RETRY=self.collection_config['MAX_RETRY'],
BACKOFF_FACTOR=self.collection_config[
'BACKOFF_FACTOR'],
auth=self.token)
if fetch_success:
return self.transform_violations(result, violation["violation_details_url"])
except Exception as exc:
self.log.error("Error Occurred while fetching LogType detailed violation for URL %s",
violation["violation_details_url"])
finally:
sess.close()
return []
def getpaginateddata(self, url, **kwargs):
page_num = 0
all_data = []
while True:
page_num += 1
status, data = ClientMixin.make_request(url, method="get", session=self.mongosess, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'], **kwargs)
if status and "results" in data and len(data['results']) > 0:
all_data.append(data)
kwargs['params']['pageNum'] = page_num + 1
else:
break
return all_data
def _get_audit_actions(self, audit_url):
url = audit_url + "actions"
try:
sess = ClientMixin.get_new_session()
status, result = ClientMixin.make_request(
url,
method="get",
session=sess,
logger=self.log,
TIMEOUT=self.collection_config['TIMEOUT'],
MAX_RETRY=self.collection_config['MAX_RETRY'],
BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'])
if status and result is not None:
if "actions" in result:
actions = result["actions"]
for actionName, values in actions.items():
if "workspace_or_org" == actionName:
self.WorkspaceAuditActions = values
elif "user" == actionName:
self.UserAuditActions = values
elif "file" == actionName:
self.ChannelAuditActions = values
elif "channel" == actionName:
self.FileAuditActions = values
elif "app" == actionName:
self.AppAuditActions = values
else:
if hasattr(self, "OtherAuditActions"):
self.OtherAuditActions.extend(values)
else:
self.OtherAuditActions = values
except Exception as exc:
self.log.error(
"Error Occurred while fetching Audit Actions Error %s", exc)
finally:
sess.close()
def set_new_end_epoch_time(self, event_type, start_time_epoch):
params = {
'token': self.api_config['TOKEN'],
'limit': 1,
'starttime': start_time_epoch,
'endtime': get_current_timestamp(),
'skip': 0,
'type': event_type
}
url = self.get_endpoint_url(event_type)
success, respjson = ClientMixin.make_request(url, method=self.api_config['FETCH_METHOD'], session=self.netskope_session, params=params, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'])
start_date = convert_epoch_to_utc_date(params['starttime'])
end_date = convert_epoch_to_utc_date(params['endtime'])
if success and respjson["status"] == "success" and len(respjson["data"]) > 0:
obj = self.set_fetch_state(event_type, start_time_epoch, respjson["data"][0]["timestamp"], respjson["data"][0]["timestamp"])
self.log.info(f'''Creating task for {event_type} from {start_date} to {end_date}''')
return obj
else:
self.log.info(f'''No events are available for {event_type} from {start_date} to {end_date}''')
return None
def get_last_record_epoch(self, obj):
params = {
'token': self.api_config['TOKEN'],
'limit': 1,
'starttime': obj['start_time_epoch'],
'endtime': obj['end_time_epoch'],
'skip': obj['skip'],
'type': obj['event_type']
}
if params['skip'] > 0:
params['skip'] -= 1
success, respjson = ClientMixin.make_request(obj['url'], method=self.api_config['FETCH_METHOD'], session=self.netskope_session, params=params, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'])
start_date = convert_epoch_to_utc_date(params['starttime'])
end_date = convert_epoch_to_utc_date(params['endtime'])
if success and respjson["status"] == "success" and len(respjson["data"]) > 0:
last_record_epoch = respjson["data"][0]["timestamp"]
last_record_date = convert_epoch_to_utc_date(last_record_epoch)
self.log.info(f'''last record for {obj['event_type']} from {params['starttime']} to {params['endtime']} skip: {params['skip']} is {last_record_date}''')
return last_record_epoch
else:
self.log.info("Response: %s" % respjson)
self.log.info("Setting end time epoch as last_record_epoch")
last_record_epoch = obj['end_time_epoch']
return last_record_epoch
def fetch(self):
output_handler = OutputHandlerFactory.get_handler(
self.collection_config['OUTPUT_HANDLER'], config=self.config)
url, args = self.build_fetch_params()
current_state = self.get_state()
log_type = self.get_key()
next_request = True
page_counter = 0
record_counter = 0
sess = ClientMixin.get_new_session()
try:
while next_request:
send_success = has_more_data = False
status, result = ClientMixin.make_request(
url,
method="get",
session=sess,
logger=self.log,
TIMEOUT=self.collection_config['TIMEOUT'],
MAX_RETRY=self.collection_config['MAX_RETRY'],
BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'],
params=args,
headers={"Authorization": "Bearer " + self.token})
fetch_success = status and "entries" in result
if fetch_success:
data_to_be_sent = self.transform_data(result)
if len(data_to_be_sent) > 0:
send_success = output_handler.send(
data_to_be_sent, **self.build_send_params())
if send_success:
page_counter += 1
record_counter += len(data_to_be_sent)
last_record_fetched_timestamp = data_to_be_sent[
-1]["date_create"]
self.log.debug(
"Successfully sent LogType %s, oldest %s, latest %s, number of records %s",
log_type, args["latest"], args["oldest"],
len(data_to_be_sent))
args["latest"] = float(
last_record_fetched_timestamp) - 0.00001
if self._next_cursor_is_present(result):
has_more_data = True
args["latest"] = float(
last_record_fetched_timestamp) - 0.00001
self.save_state({
"fetch_oldest":
current_state["fetch_oldest"],
"fetch_latest":
current_state["fetch_latest"],
"last_record_fetched_timestamp":
last_record_fetched_timestamp
})
else:
self.log.debug(
"moving time window for LogType %s, oldest %s, latest %s",
self.get_key(), args["oldest"],
args["latest"])
self.save_state({
"fetch_oldest":
current_state["fetch_latest"],
"fetch_latest":
None,
"last_record_fetched_timestamp":
None
})
else:
self.log.warning(
"Failed to sent LogType %s, oldest %s, latest %s",
log_type, args["oldest"], args["latest"])
else:
self.log.debug(
"No Result found for %s, Oldest %s, Latest %s",
log_type, args["oldest"], args["latest"])
self.save_state({
"fetch_oldest":
current_state["fetch_oldest"],
"fetch_latest":
None,
"last_record_fetched_timestamp":
None
})
else:
self.log.warning(
"Failed to fetch LogType %s, oldest %s, latest %s, error %s",
log_type, args["oldest"], args["latest"],
result["error"])
next_request = fetch_success and send_success and has_more_data and self.is_time_remaining(
)
except Exception as exc:
self.log.error(
"Error Occurred while fetching LogType %s, Error %s", log_type,
exc)
finally:
output_handler.close()
sess.close()
self.log.info("Completed LogType %s, Pages: %s, Records %s", log_type,
page_counter, record_counter)
def fetch(self):
output_handler = OutputHandlerFactory.get_handler(self.collection_config['OUTPUT_HANDLER'],
config=self.config)
data = self.build_fetch_params()
sess = ClientMixin.get_new_session()
next_request = True
page_counter = 0
record_counter = 0
last_record_fetched_date = data["filters"]["created_from"]
try:
while next_request:
send_success = has_more_data = False
fetch_success, result = ClientMixin.make_request(self.url, method="post", session=sess, logger=self.log,
TIMEOUT=self.collection_config['TIMEOUT'],
MAX_RETRY=self.collection_config['MAX_RETRY'],
BACKOFF_FACTOR=self.collection_config[
'BACKOFF_FACTOR'],
json=data,
auth=self.token,
headers={"content-type": "application/json"})
if fetch_success:
data_to_be_sent = self.transform_data(result)
if len(data_to_be_sent) > 0:
has_more_data = self.has_more_data(result)
send_success = output_handler.send(data_to_be_sent, **self.build_send_params())
if send_success:
page_counter += 1
record_counter += len(data_to_be_sent)
last_record_fetched_date = result["violations"][-1]["created"]
self.log.debug("Successfully sent LogType Violations, Offset %s, Created_from %s",
data["pagination"]["offset"], data["filters"]["created_from"])
if has_more_data:
data["pagination"]["offset"] = data["pagination"]["offset"] + 1
self.save_state(
{"offset": data["pagination"]["offset"],
"last_fetched_created_from": data["filters"]["created_from"]})
else:
self.save_state(
{"last_fetched_created_from": self.get_window(last_record_fetched_date)})
else:
self.log.warning("Failed to sent LogType Violations, Offset %s, Created_from %s",
data["pagination"]["offset"], data["filters"]["created_from"])
else:
self.log.debug("No Result fetched for LogType Violations, Offset %s, Created_from %s",
data["pagination"]["offset"], data["filters"]["created_from"])
# If the violations is ZERO than update the window & remove offset.
if last_record_fetched_date:
self.save_state(
{"last_fetched_created_from": self.get_window(last_record_fetched_date)})
else:
self.log.warning("Fetch failed for LogType Violations, Offset %s, Created_from %s",
data["pagination"]["offset"], data["filters"]["created_from"])
# Check for next request
next_request = fetch_success and send_success and has_more_data and self.is_time_remaining()
except Exception as exc:
self.log.error("Error Occurred while fetching LogType Violations, Offset %s, Created_from %s",
data["pagination"]["offset"], data["filters"]["created_from"])
finally:
output_handler.close()
sess.close()
self.log.info("Completed LogType Violations Pages: %s, Records %s", page_counter,
record_counter)
def __init__(self):
self.project_dir = self.get_current_dir()
super(MongoDBAtlasCollector, self).__init__(self.project_dir)
self.api_config = self.config['MongoDBAtlas']
self.digestauth = HTTPDigestAuth(username=self.api_config['PUBLIC_API_KEY'], password=self.api_config['PRIVATE_API_KEY'])
self.mongosess = ClientMixin.get_new_session(MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'])