def build_fetch_params(self): start_time_epoch, end_time_epoch = self.get_window(self.get_state()['last_time_epoch']) start_time_date = convert_epoch_to_utc_date(start_time_epoch, date_format=self.isoformat) end_time_date = convert_epoch_to_utc_date(end_time_epoch, date_format=self.isoformat) return f'''{self.api_config['BASE_URL']}/groups/{self.api_config['PROJECT_ID']}/processes/{self.process_id}/databases/{self.database_name}/measurements''', { "auth": self.digestauth, "params": {"itemsPerPage": self.api_config['PAGINATION_LIMIT'], "granularity": "PT1M", "start": start_time_date, "end": end_time_date # , "m": self.api_config["METRIC_TYPES"]["DATABASE_METRICS"] } }
def build_fetch_params(self): state = self.get_state() if state["page_num"] == 0: start_time_epoch, end_time_epoch = self.get_window(state['last_time_epoch']) page_num = 1 else: start_time_epoch = state['start_time_epoch'] end_time_epoch = state['end_time_epoch'] page_num = state['pageNum'] start_time_date = convert_epoch_to_utc_date(start_time_epoch, date_format=self.isoformat) end_time_date = convert_epoch_to_utc_date(end_time_epoch, date_format=self.isoformat) return f'''{self.api_config['BASE_URL']}/orgs/{self.api_config['ORGANIZATION_ID']}/events''', { "auth": self.digestauth, "params": {"itemsPerPage": self.api_config['PAGINATION_LIMIT'], "minDate": start_time_date , "maxDate": end_time_date, "pageNum": page_num} }
def get_state(self): key = self.get_key() if not self.kvstore.has_key(key): self.save_state({"last_fetched_created_from": convert_epoch_to_utc_date(self.DEFAULT_START_TIME_EPOCH, date_format=self.DATE_FORMAT)}) obj = self.kvstore.get(key) return obj
def set_new_end_epoch_time(self, event_type, start_time_epoch): params = { 'token': self.api_config['TOKEN'], 'limit': 1, 'starttime': start_time_epoch, 'endtime': get_current_timestamp(), 'skip': 0, 'type': event_type } url = self.get_endpoint_url(event_type) success, respjson = ClientMixin.make_request(url, method=self.api_config['FETCH_METHOD'], session=self.netskope_session, params=params, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR']) start_date = convert_epoch_to_utc_date(params['starttime']) end_date = convert_epoch_to_utc_date(params['endtime']) if success and respjson["status"] == "success" and len(respjson["data"]) > 0: obj = self.set_fetch_state(event_type, start_time_epoch, respjson["data"][0]["timestamp"], respjson["data"][0]["timestamp"]) self.log.info(f'''Creating task for {event_type} from {start_date} to {end_date}''') return obj else: self.log.info(f'''No events are available for {event_type} from {start_date} to {end_date}''') return None
def get_last_record_epoch(self, obj): params = { 'token': self.api_config['TOKEN'], 'limit': 1, 'starttime': obj['start_time_epoch'], 'endtime': obj['end_time_epoch'], 'skip': obj['skip'], 'type': obj['event_type'] } if params['skip'] > 0: params['skip'] -= 1 success, respjson = ClientMixin.make_request(obj['url'], method=self.api_config['FETCH_METHOD'], session=self.netskope_session, params=params, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR']) start_date = convert_epoch_to_utc_date(params['starttime']) end_date = convert_epoch_to_utc_date(params['endtime']) if success and respjson["status"] == "success" and len(respjson["data"]) > 0: last_record_epoch = respjson["data"][0]["timestamp"] last_record_date = convert_epoch_to_utc_date(last_record_epoch) self.log.info(f'''last record for {obj['event_type']} from {params['starttime']} to {params['endtime']} skip: {params['skip']} is {last_record_date}''') return last_record_epoch else: self.log.info("Response: %s" % respjson) self.log.info("Setting end time epoch as last_record_epoch") last_record_epoch = obj['end_time_epoch'] return last_record_epoch
def convert_to_other_time_format(self, date): epoch = convert_utc_date_to_epoch(date, self.DATE_FORMAT) return convert_epoch_to_utc_date(epoch, self.DATE_FORMAT_SUMO_LOGIC_TO_MATCH_WEBHOOK)
def get_window(self, last_time_utc): start_time_epoch = convert_utc_date_to_epoch(last_time_utc, date_format=self.DATE_FORMAT) + self.MOVING_WINDOW_DELTA return convert_epoch_to_utc_date(start_time_epoch, date_format=self.DATE_FORMAT)