def index_data(self): start = time.time() with logging.LogContext(start_time=start): msg = "collecting cloudwatch namespace={}, metric_name={} datainput={}, end_time={}".format( self._config[acc.metric_namespace], self._config[acc.metric_configs][0]["MetricName"], self._config[tac.datainput], self._ckpt.max_end_time()) if self._config[acc.use_metric_format]: if not self._config[tac.use_hec]: logger.warning( 'Enable HEC to support send event to metric index.') return if 'Average' not in self._config[acc.statistics]: logger.warning( 'Only datapoint contains average statistic is able to send to metric index.' ) return if self._lock.locked(): logger.debug("Last round of %s is not done yet", msg) return logger.info("Start %s", msg) with self._lock: try: self._do_index_data() except Exception: logger.exception("Failed of %s.", msg) logger.info("End of %s, one_batch_cost=%s", msg, time.time() - start)
def create(self): endpoint_name = self.params.resource if self.params.resource == 'all': endpoint_name = None hostname = None else: hostname = self.params.hostname host_stanzas = config.load_configs(self.params.config_file, self.splunks, self.rest_api_specs, rest_endpoint_name=endpoint_name, hostname=hostname) if self.params.dry_run: logger.info('In dry run mode, only validate configurations. ' 'If no error message shows, validation passes') return for host_stanza in host_stanzas: endpoint = host_stanza.rest_endpoint with logging.LogContext(hostname=host_stanza.hostname, endpoint=endpoint, action='create'): try: self.config_mgr.create(endpoint, host_stanza.hostname, host_stanza.stanzas) except app_config.AppConfigException: # Swallow this exception and carry on continue
def wrapper(*args, **kwargs): ctx = dict() for key, value in pairs: if isinstance(value, property): value = value.fget(args[0]) ctx[key] = value with logging.LogContext(**ctx): return func(*args, **kwargs)
def collect_data(self): index_store = s3ckpt.S3IndexCheckpointer(self._config) logger.info("Start processing", last_modified=index_store.last_modified(), latest_scanned=index_store.latest_scanned()) # check if finished in last interval if index_store.latest_scanned() >= self._config[ asc.terminal_scan_datetime]: logger.info( 'Data collection already finished', latest_scanned=index_store.latest_scanned(), terminal_scan_datetime=self._config[ asc.terminal_scan_datetime], ) return with logging.LogContext(phase="discover_key"): self._discover_keys(index_store) with logging.LogContext(phase="fetch_key"): self._fetch_keys(index_store)
def load_splunks(splunk_info_file): ''' Load splunk information from file and do necessary validation :param splunk_info_file: file name contains Splunk credetial etc in formation in a JSON array. :return: a dict of `SplunkInfo` objects by hostname as the key when successful, otherwise raise Exception ''' with logging.LogContext(splunk_info=splunk_info_file): return _do_load_splunks(splunk_info_file)
def __call__(self): with logging.LogContext(datainput=self._config[asc.data_input], bucket_name=self._config[asc.bucket_name], job_uid=self._config[asc.job_uid], start_time=self._config[asc.start_time], key_name=self._key.name, last_modified=self._key.last_modified, phase="fetch_key"): try: self._safe_call() except Exception: self._logger.exception('Failed to handle key.') increase_error_count(self._key_store, self._config[asc.max_retries], self._key, self._logger)
def load_configs(config_file, splunks, rest_api_specs, **kwargs): ''' Load stanzas information information from file and do necessary validation :param config_file: file name which contains stanzas information which are used to create resource :params kwargs: client can pass in 2 additional params for now 1. rest_endpoint_name 2. hostname when client passed in these 2 params, it will override the settings in conf. :return: a list of `HostStanza` object when successful, otherwise raise Exception ''' with logging.LogContext(config_file=config_file): return _do_load_configs( config_file, splunks, rest_api_specs, **kwargs)
def index_data(self): start = time.time() with logging.LogContext(start_time=start): msg = "collecting cloudwatch namespace={}, metric_name={} datainput={}, end_time={}".format( self._config[acc.metric_namespace], self._config[acc.metric_configs][0]["MetricName"], self._config[tac.datainput], self._ckpt.max_end_time()) if self._lock.locked(): logger.debug("Last round of %s is not done yet", msg) return logger.info("Start %s", msg) with self._lock: try: self._do_index_data() except Exception: logger.exception("Failed of %s.", msg) logger.info("End of %s, one_batch_cost=%s", msg, time.time() - start)
def _do_index_data(self): if self._lock.locked(): logger.info( "The last data ingestion iteration hasn't been completed yet.") return start = time.time() start_time = int(time.time()) self._config[asc.start_time] = start_time self._config[asc.job_uid] = str(uuid.uuid4()) with logging.LogContext(start_time=self._config[asc.start_time], job_uid=self._config[asc.job_uid]): logger.info("Start processing.") with self._lock: self.collect_data() if not self._stopped: logger.info( "Sweep ckpt file after completion of key discovering.") s3ckpt.S3CkptPool.sweep_all() cost = time.time() - start logger.info("End of processing!", time_cost=cost)
def __call__(self): with logging.LogContext(datainput=self._config[tac.datainput]): self.index_data()
def get_tasks(self): with logging.LogContext(phase="prepare"): return self._get_tasks()
def get_one_spec(spec_file): with logging.LogContext(spec_file=spec_file): return _do_get_spec(spec_file)
def _task_procedure(cls, identifier, master_context, callback, app, name, params): prefix = '' if not os.name == 'nt' else master_context logging.RootHandler.teardown() app.setup_root_logger(identifier) with logging.LogContext(prefix=prefix): callback(app, name, params)
def __call__(self): with logging.LogContext(datainput=self._config[asc.data_input], bucket_name=self._config[asc.bucket_name]): self.index_data()