def fetch(self):
log_type = self.get_key()
output_handler = OutputHandlerFactory.get_handler(self.collection_config['OUTPUT_HANDLER'], path=self.pathname, config=self.config)
url, kwargs = self.build_fetch_params()
self.log.info(f'''Fetching LogType: {log_type} kwargs: {kwargs}''')
state = None
payload = []
try:
fetch_success, content = ClientMixin.make_request(url, method="get", logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'], **kwargs)
if fetch_success and len(content) > 0:
payload, state = self.transform_data(content)
#Todo Make this atomic if after sending -> Ctrl - C happens then it fails to save state
params = self.build_send_params()
send_success = output_handler.send(payload, **params)
if send_success:
self.save_state(**state)
self.log.info(f'''Successfully sent LogType: {self.get_key()} Data: {len(content)}''')
else:
self.log.error(f'''Failed to send LogType: {self.get_key()}''')
else:
self.log.info(f'''No results status: {fetch_success} reason: {content}''')
finally:
output_handler.close()
self.log.info(f'''Completed LogType: {log_type} curstate: {state} datasent: {len(payload)}''')
def fetch(self):
current_state = self.get_state()
output_handler = OutputHandlerFactory.get_handler(self.collection_config['OUTPUT_HANDLER'], path=self.pathname, config=self.config)
url, kwargs = self.build_fetch_params()
log_type = self.get_key()
next_request = True
count = 0
sess = ClientMixin.get_new_session()
self.log.info(f'''Fetching LogType: {log_type} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
try:
while next_request:
send_success = has_next_page = False
status, data = ClientMixin.make_request(url, method="get", session=sess, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'], **kwargs)
fetch_success = status and "results" in data
if fetch_success:
has_next_page = len(data['results']) > 0
if has_next_page:
payload, updated_state = self.transform_data(data)
params = self.build_send_params()
send_success = output_handler.send(payload, **params)
if send_success:
count +=1
self.log.debug(f'''Successfully sent LogType: {log_type} Page: {kwargs['params']['pageNum']} Datalen: {len(payload)} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
kwargs['params']['pageNum'] += 1
# save and update last_time_epoch required for next invocation
current_state.update(updated_state)
# time not available save current state new page num else continue
if not self.is_time_remaining():
self.save_state({
"start_time_epoch": convert_utc_date_to_epoch(kwargs['params']['minDate']),
"end_time_epoch": convert_utc_date_to_epoch(kwargs['params']['maxDate']),
"page_num": kwargs['params']["pageNum"],
"last_time_epoch": current_state['last_time_epoch']
})
else:
# show err unable to send save current state
self.log.error(f'''Failed to send LogType: {log_type} Page: {kwargs['params']['pageNum']} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
self.save_state({
"start_time_epoch": convert_utc_date_to_epoch(kwargs['params']['minDate']),
"end_time_epoch": convert_utc_date_to_epoch(kwargs['params']['maxDate']),
"page_num": kwargs['params']["pageNum"],
"last_time_epoch": current_state['last_time_epoch']
})
else:
self.log.debug(f'''Moving starttime window LogType: {log_type} Page: {kwargs['params']['pageNum']} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
# here fetch success is false and assuming pageNum starts from 1
# genuine no result window no change
# page_num has finished increase window calc last_time_epoch and add 1
if kwargs['params']['pageNum'] > 1:
self.save_state({
"page_num": 0,
"last_time_epoch": current_state['last_time_epoch'] + self.MOVING_WINDOW_DELTA
})
else:
self.log.error(f'''Failed to fetch LogType: {log_type} Page: {kwargs['params']['pageNum']} Reason: {data} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
next_request = fetch_success and send_success and has_next_page and self.is_time_remaining()
finally:
sess.close()
self.log.info(f'''Completed LogType: {log_type} Count: {count} Page: {kwargs['params']['pageNum']} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
def fetch(self, url, event_type, start_time_epoch, end_time_epoch, last_record_epoch):
params = {
'token': self.api_config['TOKEN'],
'limit': self.api_config['PAGINATION_LIMIT'],
'starttime': start_time_epoch,
'endtime': end_time_epoch,
'skip': 0,
'type': event_type
}
if last_record_epoch:
params['endtime'] = last_record_epoch # logs of same timestamp may be repeated
output_handler = OutputHandlerFactory.get_handler(self.config['Collection']['OUTPUT_HANDLER'], config=self.config)
next_request = send_success = True
page_count = total_records = 0
move_window = False
sess = self.netskope_conn.get_request_session()
last_record_epoch = None
try:
while next_request:
page_count += 1
fetch_success, respjson = ClientMixin.make_request(url, method=self.api_config['FETCH_METHOD'], session=sess, params=params, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'])
fetch_success = fetch_success and respjson["status"] == "success" # netskope sends 200 for errors
if fetch_success:
data = respjson["data"]
if len(data) > 0:
data = self.transform_data(data)
send_success = output_handler.send(data)
if send_success:
params['skip'] += len(data)
total_records += len(data)
last_record_epoch = data[-1]["timestamp"]
self.log.info(f'''Successfully Sent Page: {page_count} Event Type: {event_type} Datalen: {len(
data)} starttime: {convert_epoch_to_utc_date(
params['starttime'])} endtime: {convert_epoch_to_utc_date(params['endtime'])} skip: {params['skip']} last_record_epoch: {convert_epoch_to_utc_date(last_record_epoch)}''')
else: # no data so moving window
move_window = True
else:
self.log.error("Unable to fetch Response %s" % respjson)
next_request = fetch_success and send_success and (not move_window)
if move_window:
self.log.debug(
f'''Moving starttime window for {event_type} to {convert_epoch_to_utc_date(params["endtime"] + 1)}''')
self.set_fetch_state(event_type, end_time_epoch+1, None, None)
elif not (fetch_success and send_success): # saving skip in casee of failures for restarting in future
self.set_fetch_state(event_type, start_time_epoch, end_time_epoch, last_record_epoch)
self.log.error(
f'''Failed to send Event Type: {event_type} Page: {page_count} starttime: {convert_epoch_to_utc_date(params['starttime'])} endtime: {convert_epoch_to_utc_date(params['endtime'])} fetch_success: {fetch_success} send_success: {send_success} skip: {params['skip']} last_record_epoch: {last_record_epoch}''')
except Exception as e:
self.set_fetch_state(event_type, start_time_epoch, end_time_epoch, last_record_epoch)
raise e
finally:
self.netskope_conn.close()
output_handler.close()
self.log.info(f''' Total messages fetched {total_records} for Event Type: {event_type} starttime: {convert_epoch_to_utc_date(params['starttime'])} endtime: {convert_epoch_to_utc_date(params['endtime'])}''')
def fetch(self):
current_state = self.get_state()
output_handler = OutputHandlerFactory.get_handler(self.collection_config['OUTPUT_HANDLER'], path=self.pathname, config=self.config)
url, kwargs = self.build_fetch_params()
next_request = True
sess = ClientMixin.get_new_session()
log_type = self.get_key()
count = 0
self.log.info(f'''Fetching LogType: {log_type} pageNum: {kwargs["params"]["pageNum"]}''')
try:
while next_request:
send_success = has_next_page = False
status, data = ClientMixin.make_request(url, method="get", session=sess, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'], **kwargs)
fetch_success = status and "results" in data
if fetch_success:
has_next_page = len(data['results']) > 0
if has_next_page:
payload, updated_state = self.transform_data(data)
send_success = output_handler.send(payload, **self.build_send_params())
if send_success:
count += 1
self.log.debug(f'''Fetching Project: {self.api_config['PROJECT_ID']} Alerts Page: {kwargs['params']['pageNum']} Datalen: {len(payload)} ''')
current_state.update(updated_state)
if current_state['last_page_offset'] == 0:
# do not increase if num alerts < page limit
kwargs['params']['pageNum'] += 1
else:
has_next_page = False
# time not available save current state new page num else continue
if (not self.is_time_remaining()) or (not has_next_page):
self.save_state({
"page_num": kwargs['params']["pageNum"],
"last_page_offset": current_state['last_page_offset']
})
else:
# show err unable to send save current state
self.log.error(f'''Unable to send Project: {self.api_config['PROJECT_ID']} Alerts Page: {kwargs['params']['pageNum']} ''')
self.save_state({
"page_num": kwargs['params']["pageNum"],
"last_page_offset": current_state['last_page_offset']
})
else:
self.log.debug(f'''Moving starttime window Project: {self.api_config['PROJECT_ID']} Alerts Page: {kwargs['params']['pageNum']} ''')
# here send success is false
# genuine no result window no change
# page_num has finished increase window calc last_time_epoch and add 1
self.save_state({
"page_num": kwargs['params']["pageNum"],
"last_page_offset": current_state['last_page_offset']
})
else:
self.log.error(f'''Unable to fetch Project: {self.api_config['PROJECT_ID']} Alerts Page: {kwargs['params']['pageNum']} Reason: {data} ''')
next_request = fetch_success and send_success and has_next_page and self.is_time_remaining()
finally:
sess.close()
self.log.info(f'''Completed LogType: {log_type} Count: {count} Page: {kwargs['params']['pageNum']}''')
def fetch(self):
method, args = self.build_fetch_params()
output_handler = OutputHandlerFactory.get_handler(
self.collection_config['OUTPUT_HANDLER'], config=self.config)
try:
result = self.slackClient.api_call(
method, self.collection_config['TIMEOUT'], **args)
fetch_success = result["ok"]
if fetch_success:
data_to_be_sent = self.transform_data(result)
if len(data_to_be_sent) > 0:
send_success = output_handler.send(
data_to_be_sent, **self.build_send_params())
if send_success:
self.save_state(result["paging"])
self.log.debug(
"Sent successfully for LogType %s, Page %s, Before %s, Records %s",
method, self.page, args["before"],
len(data_to_be_sent))
else:
self.log.warning(
"Send failed for LogType %s, Page %s, Before %s",
method, self.page, args["before"])
else:
self.save_state(result["paging"])
self.log.debug(
"No Result fetched for LogType %s, Page %s, Before %s",
method, self.page, args["before"])
else:
self.log.warning(
"Fetch failed for LogType %s, Page %s, Before %s, Error %s",
method, self.page, args["before"], result["error"])
except Exception as exc:
self.log.error(
"Error Occurred while fetching LogType %s, Page %s, Before %s, Error %s",
method, self.page, args["before"], exc)
self.log.info("Completed LogType %s, Page %s, Before %s", method,
self.page, args["before"])
def fetch(self):
next_request = True
method, args = self.build_fetch_params()
retry_counter = 0
page_counter = 0
record_counter = 0
output_handler = OutputHandlerFactory.get_handler(
self.collection_config['OUTPUT_HANDLER'], config=self.config)
try:
while next_request:
send_success = retry_request = has_next_cursor = False
result = self.slackClient.api_call(
method, self.collection_config['TIMEOUT'], **args)
fetch_success = result["ok"]
if fetch_success:
data_to_be_sent = self.transform_data(result)
send_success = output_handler.send(
data_to_be_sent, **self.build_send_params())
if send_success:
page_counter += 1
record_counter += len(data_to_be_sent)
has_next_cursor = self._next_cursor_is_present(result)
if has_next_cursor:
args["cursor"] = result["response_metadata"][
"next_cursor"]
self.save_state(args["cursor"], data_to_be_sent)
else:
self.save_state(None, data_to_be_sent)
else:
self.save_state(args["cursor"], [])
else:
if "error" in result and result["error"].startswith(
"invalid_cursor"):
self.save_state(None, [])
elif "Retry-After" in result["headers"]:
# The `Retry-After` header will tell you how long to wait before retrying
delay = int(result["headers"]["Retry-After"])
self.log.warning(
"Rate limited. Retrying in %s seconds", str(delay))
time.sleep(delay)
# set the counter for retry
retry_counter += 1
# retry only for Max Retry times
if retry_counter <= self.collection_config["MAX_RETRY"]:
self.log.debug("Retrying the method %s for %s",
method, args["counter"])
retry_request = True
else:
retry_request = False
else:
self.log.warning(
"Failed to fetch LogType %s, Cursor %s, Error %s",
method, args["cursor"], result["error"])
if retry_request:
next_request = True
else:
next_request = fetch_success and send_success and has_next_cursor and self.is_time_remaining(
)
finally:
output_handler.close()
self.log.info("Completed LogType %s, Pages: %s, Records %s", method,
page_counter, record_counter)
def fetch(self):
output_handler = OutputHandlerFactory.get_handler(
self.collection_config['OUTPUT_HANDLER'], config=self.config)
url, args = self.build_fetch_params()
current_state = self.get_state()
log_type = self.get_key()
next_request = True
page_counter = 0
record_counter = 0
sess = ClientMixin.get_new_session()
try:
while next_request:
send_success = has_more_data = False
status, result = ClientMixin.make_request(
url,
method="get",
session=sess,
logger=self.log,
TIMEOUT=self.collection_config['TIMEOUT'],
MAX_RETRY=self.collection_config['MAX_RETRY'],
BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'],
params=args,
headers={"Authorization": "Bearer " + self.token})
fetch_success = status and "entries" in result
if fetch_success:
data_to_be_sent = self.transform_data(result)
if len(data_to_be_sent) > 0:
send_success = output_handler.send(
data_to_be_sent, **self.build_send_params())
if send_success:
page_counter += 1
record_counter += len(data_to_be_sent)
last_record_fetched_timestamp = data_to_be_sent[
-1]["date_create"]
self.log.debug(
"Successfully sent LogType %s, oldest %s, latest %s, number of records %s",
log_type, args["latest"], args["oldest"],
len(data_to_be_sent))
args["latest"] = float(
last_record_fetched_timestamp) - 0.00001
if self._next_cursor_is_present(result):
has_more_data = True
args["latest"] = float(
last_record_fetched_timestamp) - 0.00001
self.save_state({
"fetch_oldest":
current_state["fetch_oldest"],
"fetch_latest":
current_state["fetch_latest"],
"last_record_fetched_timestamp":
last_record_fetched_timestamp
})
else:
self.log.debug(
"moving time window for LogType %s, oldest %s, latest %s",
self.get_key(), args["oldest"],
args["latest"])
self.save_state({
"fetch_oldest":
current_state["fetch_latest"],
"fetch_latest":
None,
"last_record_fetched_timestamp":
None
})
else:
self.log.warning(
"Failed to sent LogType %s, oldest %s, latest %s",
log_type, args["oldest"], args["latest"])
else:
self.log.debug(
"No Result found for %s, Oldest %s, Latest %s",
log_type, args["oldest"], args["latest"])
self.save_state({
"fetch_oldest":
current_state["fetch_oldest"],
"fetch_latest":
None,
"last_record_fetched_timestamp":
None
})
else:
self.log.warning(
"Failed to fetch LogType %s, oldest %s, latest %s, error %s",
log_type, args["oldest"], args["latest"],
result["error"])
next_request = fetch_success and send_success and has_more_data and self.is_time_remaining(
)
except Exception as exc:
self.log.error(
"Error Occurred while fetching LogType %s, Error %s", log_type,
exc)
finally:
output_handler.close()
sess.close()
self.log.info("Completed LogType %s, Pages: %s, Records %s", log_type,
page_counter, record_counter)
def fetch(self):
output_handler = OutputHandlerFactory.get_handler(
self.collection_config['OUTPUT_HANDLER'], config=self.config)
method, args = self.build_fetch_params()
current_state = self.get_state()
next_request = True
page_counter = 0
record_counter = 0
try:
while next_request:
send_success = has_more_data = False
result = self.slackClient.api_call(
method, self.collection_config['TIMEOUT'], **args)
fetch_success = result["ok"]
if fetch_success:
data_to_be_sent = self.transform_data(result)
if len(data_to_be_sent) > 0:
send_success = output_handler.send(
data_to_be_sent, **self.build_send_params())
if send_success:
page_counter += 1
record_counter += len(data_to_be_sent)
last_record_fetched_timestamp = data_to_be_sent[
-1]["ts"]
self.log.debug(
"Successfully sent LogType %s, oldest %s, latest %s, number of records %s",
method, args["oldest"], args["latest"],
len(data_to_be_sent))
if "has_more" in result and result["has_more"]:
has_more_data = True
args["latest"] = float(
last_record_fetched_timestamp) - 0.00001
self.save_state({
"fetch_oldest":
current_state["fetch_oldest"],
"fetch_latest":
current_state["fetch_latest"],
"last_record_fetched_timestamp":
last_record_fetched_timestamp
})
else:
self.log.debug(
"moving time window for LogType %s, %s, oldest %s, latest %s",
method, self.channel_name, args["oldest"],
args["latest"])
self.save_state({
"fetch_oldest":
current_state["fetch_latest"],
"fetch_latest":
None,
"last_record_fetched_timestamp":
None
})
else:
self.log.warning(
"Failed to sent LogType %s, %s, oldest %s, latest %s",
method, self.channel_name, args["oldest"],
args["latest"])
else:
self.log.debug(
"No Result found for %s, Oldest %s, Latest %s",
self.channel_name, args["oldest"], args["latest"])
self.save_state({
"fetch_oldest":
current_state["fetch_oldest"],
"fetch_latest":
None,
"last_record_fetched_timestamp":
None
})
else:
self.log.warning(
"Failed to fetch LogType %s, %s, oldest %s, latest %s, error %s",
method, self.channel_name, args["oldest"],
args["latest"], result["error"])
next_request = fetch_success and send_success and has_more_data and self.is_time_remaining(
)
except Exception as exc:
self.log.error(
"Error Occurred while fetching LogType %s, %s, Error %s",
method, self.channel_name, exc)
finally:
output_handler.close()
self.log.info("Completed LogType %s, %s, Pages: %s, Records %s",
method, self.channel_name, page_counter, record_counter)
def fetch(self):
output_handler = OutputHandlerFactory.get_handler(self.collection_config['OUTPUT_HANDLER'],
config=self.config)
data = self.build_fetch_params()
sess = ClientMixin.get_new_session()
next_request = True
page_counter = 0
record_counter = 0
last_record_fetched_date = data["filters"]["created_from"]
try:
while next_request:
send_success = has_more_data = False
fetch_success, result = ClientMixin.make_request(self.url, method="post", session=sess, logger=self.log,
TIMEOUT=self.collection_config['TIMEOUT'],
MAX_RETRY=self.collection_config['MAX_RETRY'],
BACKOFF_FACTOR=self.collection_config[
'BACKOFF_FACTOR'],
json=data,
auth=self.token,
headers={"content-type": "application/json"})
if fetch_success:
data_to_be_sent = self.transform_data(result)
if len(data_to_be_sent) > 0:
has_more_data = self.has_more_data(result)
send_success = output_handler.send(data_to_be_sent, **self.build_send_params())
if send_success:
page_counter += 1
record_counter += len(data_to_be_sent)
last_record_fetched_date = result["violations"][-1]["created"]
self.log.debug("Successfully sent LogType Violations, Offset %s, Created_from %s",
data["pagination"]["offset"], data["filters"]["created_from"])
if has_more_data:
data["pagination"]["offset"] = data["pagination"]["offset"] + 1
self.save_state(
{"offset": data["pagination"]["offset"],
"last_fetched_created_from": data["filters"]["created_from"]})
else:
self.save_state(
{"last_fetched_created_from": self.get_window(last_record_fetched_date)})
else:
self.log.warning("Failed to sent LogType Violations, Offset %s, Created_from %s",
data["pagination"]["offset"], data["filters"]["created_from"])
else:
self.log.debug("No Result fetched for LogType Violations, Offset %s, Created_from %s",
data["pagination"]["offset"], data["filters"]["created_from"])
# If the violations is ZERO than update the window & remove offset.
if last_record_fetched_date:
self.save_state(
{"last_fetched_created_from": self.get_window(last_record_fetched_date)})
else:
self.log.warning("Fetch failed for LogType Violations, Offset %s, Created_from %s",
data["pagination"]["offset"], data["filters"]["created_from"])
# Check for next request
next_request = fetch_success and send_success and has_more_data and self.is_time_remaining()
except Exception as exc:
self.log.error("Error Occurred while fetching LogType Violations, Offset %s, Created_from %s",
data["pagination"]["offset"], data["filters"]["created_from"])
finally:
output_handler.close()
sess.close()
self.log.info("Completed LogType Violations Pages: %s, Records %s", page_counter,
record_counter)