def build_fetch_params(self):
start_time_epoch, end_time_epoch = self.get_window(self.get_state()['last_time_epoch'])
start_time_date = convert_epoch_to_utc_date(start_time_epoch, date_format=self.isoformat)
end_time_date = convert_epoch_to_utc_date(end_time_epoch, date_format=self.isoformat)
return f'''{self.api_config['BASE_URL']}/groups/{self.api_config['PROJECT_ID']}/processes/{self.process_id}/databases/{self.database_name}/measurements''', {
"auth": self.digestauth,
"params": {"itemsPerPage": self.api_config['PAGINATION_LIMIT'], "granularity": "PT1M",
"start": start_time_date, "end": end_time_date
# , "m": self.api_config["METRIC_TYPES"]["DATABASE_METRICS"]
}
}
def build_fetch_params(self):
state = self.get_state()
if state["page_num"] == 0:
start_time_epoch, end_time_epoch = self.get_window(state['last_time_epoch'])
page_num = 1
else:
start_time_epoch = state['start_time_epoch']
end_time_epoch = state['end_time_epoch']
page_num = state['pageNum']
start_time_date = convert_epoch_to_utc_date(start_time_epoch, date_format=self.isoformat)
end_time_date = convert_epoch_to_utc_date(end_time_epoch, date_format=self.isoformat)
return f'''{self.api_config['BASE_URL']}/orgs/{self.api_config['ORGANIZATION_ID']}/events''', {
"auth": self.digestauth,
"params": {"itemsPerPage": self.api_config['PAGINATION_LIMIT'], "minDate": start_time_date , "maxDate": end_time_date, "pageNum": page_num}
}
def get_state(self):
key = self.get_key()
if not self.kvstore.has_key(key):
self.save_state({"last_fetched_created_from": convert_epoch_to_utc_date(self.DEFAULT_START_TIME_EPOCH,
date_format=self.DATE_FORMAT)})
obj = self.kvstore.get(key)
return obj
def set_new_end_epoch_time(self, event_type, start_time_epoch):
params = {
'token': self.api_config['TOKEN'],
'limit': 1,
'starttime': start_time_epoch,
'endtime': get_current_timestamp(),
'skip': 0,
'type': event_type
}
url = self.get_endpoint_url(event_type)
success, respjson = ClientMixin.make_request(url, method=self.api_config['FETCH_METHOD'], session=self.netskope_session, params=params, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'])
start_date = convert_epoch_to_utc_date(params['starttime'])
end_date = convert_epoch_to_utc_date(params['endtime'])
if success and respjson["status"] == "success" and len(respjson["data"]) > 0:
obj = self.set_fetch_state(event_type, start_time_epoch, respjson["data"][0]["timestamp"], respjson["data"][0]["timestamp"])
self.log.info(f'''Creating task for {event_type} from {start_date} to {end_date}''')
return obj
else:
self.log.info(f'''No events are available for {event_type} from {start_date} to {end_date}''')
return None
def get_last_record_epoch(self, obj):
params = {
'token': self.api_config['TOKEN'],
'limit': 1,
'starttime': obj['start_time_epoch'],
'endtime': obj['end_time_epoch'],
'skip': obj['skip'],
'type': obj['event_type']
}
if params['skip'] > 0:
params['skip'] -= 1
success, respjson = ClientMixin.make_request(obj['url'], method=self.api_config['FETCH_METHOD'], session=self.netskope_session, params=params, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'])
start_date = convert_epoch_to_utc_date(params['starttime'])
end_date = convert_epoch_to_utc_date(params['endtime'])
if success and respjson["status"] == "success" and len(respjson["data"]) > 0:
last_record_epoch = respjson["data"][0]["timestamp"]
last_record_date = convert_epoch_to_utc_date(last_record_epoch)
self.log.info(f'''last record for {obj['event_type']} from {params['starttime']} to {params['endtime']} skip: {params['skip']} is {last_record_date}''')
return last_record_epoch
else:
self.log.info("Response: %s" % respjson)
self.log.info("Setting end time epoch as last_record_epoch")
last_record_epoch = obj['end_time_epoch']
return last_record_epoch
def convert_to_other_time_format(self, date):
epoch = convert_utc_date_to_epoch(date, self.DATE_FORMAT)
return convert_epoch_to_utc_date(epoch, self.DATE_FORMAT_SUMO_LOGIC_TO_MATCH_WEBHOOK)
def get_window(self, last_time_utc):
start_time_epoch = convert_utc_date_to_epoch(last_time_utc,
date_format=self.DATE_FORMAT) + self.MOVING_WINDOW_DELTA
return convert_epoch_to_utc_date(start_time_epoch, date_format=self.DATE_FORMAT)