def fetch(self):
current_state = self.get_state()
output_handler = OutputHandlerFactory.get_handler(self.collection_config['OUTPUT_HANDLER'], path=self.pathname, config=self.config)
url, kwargs = self.build_fetch_params()
log_type = self.get_key()
next_request = True
count = 0
sess = ClientMixin.get_new_session()
self.log.info(f'''Fetching LogType: {log_type} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
try:
while next_request:
send_success = has_next_page = False
status, data = ClientMixin.make_request(url, method="get", session=sess, logger=self.log, TIMEOUT=self.collection_config['TIMEOUT'], MAX_RETRY=self.collection_config['MAX_RETRY'], BACKOFF_FACTOR=self.collection_config['BACKOFF_FACTOR'], **kwargs)
fetch_success = status and "results" in data
if fetch_success:
has_next_page = len(data['results']) > 0
if has_next_page:
payload, updated_state = self.transform_data(data)
params = self.build_send_params()
send_success = output_handler.send(payload, **params)
if send_success:
count +=1
self.log.debug(f'''Successfully sent LogType: {log_type} Page: {kwargs['params']['pageNum']} Datalen: {len(payload)} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
kwargs['params']['pageNum'] += 1
# save and update last_time_epoch required for next invocation
current_state.update(updated_state)
# time not available save current state new page num else continue
if not self.is_time_remaining():
self.save_state({
"start_time_epoch": convert_utc_date_to_epoch(kwargs['params']['minDate']),
"end_time_epoch": convert_utc_date_to_epoch(kwargs['params']['maxDate']),
"page_num": kwargs['params']["pageNum"],
"last_time_epoch": current_state['last_time_epoch']
})
else:
# show err unable to send save current state
self.log.error(f'''Failed to send LogType: {log_type} Page: {kwargs['params']['pageNum']} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
self.save_state({
"start_time_epoch": convert_utc_date_to_epoch(kwargs['params']['minDate']),
"end_time_epoch": convert_utc_date_to_epoch(kwargs['params']['maxDate']),
"page_num": kwargs['params']["pageNum"],
"last_time_epoch": current_state['last_time_epoch']
})
else:
self.log.debug(f'''Moving starttime window LogType: {log_type} Page: {kwargs['params']['pageNum']} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
# here fetch success is false and assuming pageNum starts from 1
# genuine no result window no change
# page_num has finished increase window calc last_time_epoch and add 1
if kwargs['params']['pageNum'] > 1:
self.save_state({
"page_num": 0,
"last_time_epoch": current_state['last_time_epoch'] + self.MOVING_WINDOW_DELTA
})
else:
self.log.error(f'''Failed to fetch LogType: {log_type} Page: {kwargs['params']['pageNum']} Reason: {data} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
next_request = fetch_success and send_success and has_next_page and self.is_time_remaining()
finally:
sess.close()
self.log.info(f'''Completed LogType: {log_type} Count: {count} Page: {kwargs['params']['pageNum']} starttime: {kwargs['params']['minDate']} endtime: {kwargs['params']['maxDate']}''')
def transform_data(self, data):
metrics = []
last_time_epoch = self.DEFAULT_START_TIME_EPOCH
for measurement in data['measurements']:
for datapoints in measurement['dataPoints']:
if datapoints['value'] is None:
continue
current_timestamp = convert_utc_date_to_epoch(datapoints['timestamp'], date_format=self.date_format)
cluster_name = data['hostId'].split("-", 1)[0].strip()
metrics.append(f'''projectId={data['groupId']} databaseName={data['databaseName']} hostId={data['hostId']} processId={data['processId']} metric={measurement['name']} units={measurement['units']} cluster_name={cluster_name} {datapoints['value']} {current_timestamp}''')
last_time_epoch = max(current_timestamp, last_time_epoch)
return metrics, {"last_time_epoch": last_time_epoch}
def transform_data(self, data):
metrics = []
last_time_epoch = self.DEFAULT_START_TIME_EPOCH
for measurement in data['measurements']:
for datapoints in measurement['dataPoints']:
if datapoints['value'] is None:
continue
current_timestamp = convert_utc_date_to_epoch(datapoints['timestamp'], date_format=self.date_format)
host_id = self._replace_cluster_name(data['hostId'], self.cluster_mapping)
process_id = self._replace_cluster_name(data['processId'], self.cluster_mapping)
cluster_name = self._get_cluster_name(host_id)
metrics.append(f'''projectId={data['groupId']} partitionName={data['partitionName']} hostId={host_id} processId={process_id} metric={measurement['name']} units={measurement['units']} cluster_name={cluster_name} {datapoints['value']} {current_timestamp}''')
last_time_epoch = max(current_timestamp, last_time_epoch)
return metrics, {"last_time_epoch": last_time_epoch}
def convert_to_other_time_format(self, date):
epoch = convert_utc_date_to_epoch(date, self.DATE_FORMAT)
return convert_epoch_to_utc_date(epoch, self.DATE_FORMAT_SUMO_LOGIC_TO_MATCH_WEBHOOK)
def get_window(self, last_time_utc):
start_time_epoch = convert_utc_date_to_epoch(last_time_utc,
date_format=self.DATE_FORMAT) + self.MOVING_WINDOW_DELTA
return convert_epoch_to_utc_date(start_time_epoch, date_format=self.DATE_FORMAT)