def index_data(self):
start = time.time()
with logging.LogContext(start_time=start):
msg = "collecting cloudwatch namespace={}, metric_name={} datainput={}, end_time={}".format(
self._config[acc.metric_namespace],
self._config[acc.metric_configs][0]["MetricName"],
self._config[tac.datainput], self._ckpt.max_end_time())
if self._config[acc.use_metric_format]:
if not self._config[tac.use_hec]:
logger.warning(
'Enable HEC to support send event to metric index.')
return
if 'Average' not in self._config[acc.statistics]:
logger.warning(
'Only datapoint contains average statistic is able to send to metric index.'
)
return
if self._lock.locked():
logger.debug("Last round of %s is not done yet", msg)
return
logger.info("Start %s", msg)
with self._lock:
try:
self._do_index_data()
except Exception:
logger.exception("Failed of %s.", msg)
logger.info("End of %s, one_batch_cost=%s", msg,
time.time() - start)
def create(self):
endpoint_name = self.params.resource
if self.params.resource == 'all':
endpoint_name = None
hostname = None
else:
hostname = self.params.hostname
host_stanzas = config.load_configs(self.params.config_file,
self.splunks,
self.rest_api_specs,
rest_endpoint_name=endpoint_name,
hostname=hostname)
if self.params.dry_run:
logger.info('In dry run mode, only validate configurations. '
'If no error message shows, validation passes')
return
for host_stanza in host_stanzas:
endpoint = host_stanza.rest_endpoint
with logging.LogContext(hostname=host_stanza.hostname,
endpoint=endpoint,
action='create'):
try:
self.config_mgr.create(endpoint, host_stanza.hostname,
host_stanza.stanzas)
except app_config.AppConfigException:
# Swallow this exception and carry on
continue
def wrapper(*args, **kwargs):
ctx = dict()
for key, value in pairs:
if isinstance(value, property):
value = value.fget(args[0])
ctx[key] = value
with logging.LogContext(**ctx):
return func(*args, **kwargs)
def collect_data(self):
index_store = s3ckpt.S3IndexCheckpointer(self._config)
logger.info("Start processing",
last_modified=index_store.last_modified(),
latest_scanned=index_store.latest_scanned())
# check if finished in last interval
if index_store.latest_scanned() >= self._config[
asc.terminal_scan_datetime]:
logger.info(
'Data collection already finished',
latest_scanned=index_store.latest_scanned(),
terminal_scan_datetime=self._config[
asc.terminal_scan_datetime],
)
return
with logging.LogContext(phase="discover_key"):
self._discover_keys(index_store)
with logging.LogContext(phase="fetch_key"):
self._fetch_keys(index_store)
def load_splunks(splunk_info_file):
'''
Load splunk information from file and do necessary validation
:param splunk_info_file: file name contains Splunk credetial etc in
formation in a JSON array.
:return: a dict of `SplunkInfo` objects by hostname as the key when
successful, otherwise raise Exception
'''
with logging.LogContext(splunk_info=splunk_info_file):
return _do_load_splunks(splunk_info_file)
def __call__(self):
with logging.LogContext(datainput=self._config[asc.data_input],
bucket_name=self._config[asc.bucket_name],
job_uid=self._config[asc.job_uid],
start_time=self._config[asc.start_time],
key_name=self._key.name,
last_modified=self._key.last_modified,
phase="fetch_key"):
try:
self._safe_call()
except Exception:
self._logger.exception('Failed to handle key.')
increase_error_count(self._key_store,
self._config[asc.max_retries], self._key,
self._logger)
def load_configs(config_file, splunks, rest_api_specs, **kwargs):
'''
Load stanzas information information from file and do necessary validation
:param config_file: file name which contains stanzas information which are
used to create resource
:params kwargs: client can pass in 2 additional params for now
1. rest_endpoint_name
2. hostname
when client passed in these 2 params, it will override the settings
in conf.
:return: a list of `HostStanza` object when successful,
otherwise raise Exception
'''
with logging.LogContext(config_file=config_file):
return _do_load_configs(
config_file, splunks, rest_api_specs, **kwargs)
def index_data(self):
start = time.time()
with logging.LogContext(start_time=start):
msg = "collecting cloudwatch namespace={}, metric_name={} datainput={}, end_time={}".format(
self._config[acc.metric_namespace],
self._config[acc.metric_configs][0]["MetricName"],
self._config[tac.datainput], self._ckpt.max_end_time())
if self._lock.locked():
logger.debug("Last round of %s is not done yet", msg)
return
logger.info("Start %s", msg)
with self._lock:
try:
self._do_index_data()
except Exception:
logger.exception("Failed of %s.", msg)
logger.info("End of %s, one_batch_cost=%s", msg,
time.time() - start)
def _do_index_data(self):
if self._lock.locked():
logger.info(
"The last data ingestion iteration hasn't been completed yet.")
return
start = time.time()
start_time = int(time.time())
self._config[asc.start_time] = start_time
self._config[asc.job_uid] = str(uuid.uuid4())
with logging.LogContext(start_time=self._config[asc.start_time],
job_uid=self._config[asc.job_uid]):
logger.info("Start processing.")
with self._lock:
self.collect_data()
if not self._stopped:
logger.info(
"Sweep ckpt file after completion of key discovering.")
s3ckpt.S3CkptPool.sweep_all()
cost = time.time() - start
logger.info("End of processing!", time_cost=cost)
def __call__(self):
with logging.LogContext(datainput=self._config[tac.datainput]):
self.index_data()
def get_tasks(self):
with logging.LogContext(phase="prepare"):
return self._get_tasks()
def get_one_spec(spec_file):
with logging.LogContext(spec_file=spec_file):
return _do_get_spec(spec_file)
def _task_procedure(cls, identifier, master_context, callback, app, name, params):
prefix = '' if not os.name == 'nt' else master_context
logging.RootHandler.teardown()
app.setup_root_logger(identifier)
with logging.LogContext(prefix=prefix):
callback(app, name, params)
def __call__(self):
with logging.LogContext(datainput=self._config[asc.data_input],
bucket_name=self._config[asc.bucket_name]):
self.index_data()
