class CfFinisher(Filter): KEY_FOR_DATA = "data_to_insert" KEY_NAME = "measurement" DEFAULT_CORR_PREFIX = "cf_corr_" DEFAULT_SEPARATOR = "_" STATUS_CODE_SUCCESSFUL = 0 def __init__(self, conf): super(CfFinisher, self).__init__(conf) # TODO: type of database engine # should be loaded from the workflow configuration file self._dbConnector = InfluxDbConnector(conf) self._dbConnector.connect() self._result = None self.node = conf["default"]["node"] def run(self, **kwargs): # TODO: 2 matrixes as result? log.debug("CF_FINISH") if CfFinisher.KEY_FOR_DATA in kwargs: self._insert_data(kwargs[CfFinisher.KEY_FOR_DATA], kwargs[CfFinisher.KEY_NAME]) return CfFinisher.STATUS_CODE_SUCCESSFUL else: raise CfFinisherException("No data for insert retrieved from a previous step. Failing...") def _insert_data(self, data_to_insert, measurement): result = [] for name, data in data_to_insert.iteritems(): json_record = self._create_record_json(measurement + "_" + name, data) result.append(json_record) self._dbConnector.write_data(result) def _create_record_json(self, measurement, data): record_json = {"measurement": measurement + "_" + self.node, "fields": {}} for i, val in enumerate(data): record_json["fields"]["f" + str(i + 1)] = val return record_json
class MetricsFetcher(Filter): def __init__(self, conf): super(MetricsFetcher, self).__init__(conf) # TODO: type of the database engine # should be loaded from the workflow configuration file self._dbConnector = InfluxDbConnector(conf) self._dbConnector.connect() self._result = None self.source = conf['default']['node'] def run(self, **kwargs): self._result = kwargs # data filtered from Serenity input... data_to_test = self._filter_data_from_serenity(kwargs) # ... will be updated by database data metrics_to_query = self._get_which_metrics_query() log.debug(metrics_to_query) for metric, fields in metrics_to_query.iteritems(): # TODO: How big set of data we should analyze? # In the meaning of time (where statement) where_clause = "time > now() - " + TIME_FROM_NOW +\ " and source = \'%s\'" % self.source query_to_execute = compile( Q().tables('"' + metric + '"'). fields(fields["field"]).where(where_clause) ) log.debug(self._format_query_to_string(query_to_execute)) database_output = self._get_data_from_database( self._format_query_to_string(query_to_execute)) data_to_test[metric] = (database_output, fields) self._result[DATA_FIELD] = data_to_test return self._result def _filter_data_from_serenity(self, serenity_data): metric_value = dict() conf_file = open(SERENITY_METRICS_CONF_FILE, 'r') for record in conf_file: if record.startswith('#'): # it's just a comment, move along continue if record in serenity_data: metric_value[record] = serenity_data[record] return metric_value def _get_which_metrics_query(self): table_field_dict = dict() conf_file = open(DATABASE_METRICS_CONF_FILE, 'r') for record in conf_file: if record.startswith('#'): # it's just a comment, move along continue key_value = record.split() table_field_dict[key_value[0]] = dict() table_field_dict[key_value[0]]["field"] = key_value[1] table_field_dict[key_value[0]]["cumulated"] = False table_field_dict[key_value[0]]["save"] = False table_field_dict[key_value[0]]["mul"] = 1 table_field_dict[key_value[0]]["post_effect_needed"] = False for i in xrange(1, (len(key_value)-1)): if key_value[i] == "cumulated": table_field_dict[key_value[0]]["cumulated"] = True table_field_dict[key_value[0]]["post_effect_needed"] = True elif key_value[i] == "save": table_field_dict[key_value[0]]["save"] = True table_field_dict[key_value[0]]["post_effect_needed"] = True elif "mul" in key_value[i]: mul = key_value[i].split('=') table_field_dict[key_value[0]]["mul"] = float(mul[1]) table_field_dict[key_value[0]]["post_effect_needed"] = True return table_field_dict def _format_query_to_string(self, query): return query[0] % tuple(query[1]) def _get_data_from_database(self, query_to_execute): return self._dbConnector.query_data(query_to_execute)
class PcaFinisher(Filter): KEY_FOR_DATA = 'data_to_insert' KEY_NAME = 'measurement' DEFAULT_CORR_PREFIX = 'corr_' DEFAULT_SEPARATOR = '_' STATUS_CODE_SUCCESSFUL = 0 def __init__(self, conf): super(PcaFinisher, self).__init__(conf) # TODO: type of database engine # should be loaded from the workflow configuration file self._dbConnector = InfluxDbConnector(conf) self._dbConnector.connect() self._result = None self.node = conf['default']['node'] def run(self, **kwargs): if PcaFinisher.KEY_FOR_DATA in kwargs: self._insert_data(kwargs[PcaFinisher.KEY_FOR_DATA], kwargs[PcaFinisher.KEY_NAME]) return PcaFinisher.STATUS_CODE_SUCCESSFUL else: raise PcaFinisherException( 'No data for insert retrieved from a previous step. Failing...') def _insert_data(self, data_to_insert, measurement): result = [] for name, data in data_to_insert.iteritems(): for key, val in data.iteritems(): if math.isnan(val): val = 0.00001 elif val == 0.0: val = 0.00001 json_record = self._create_record_json( measurement, name, key, val) result.append(json_record) self._dbConnector.write_data(result) def _create_record_json(self, measurement, name, tag, value): name = name.replace('/', '_') tag = tag.replace('/', '_') record_json = { "measurement": measurement + '_' + self.node, "tags": { "corr_name": name, "corr_with": tag }, "fields": { "value": value } } return record_json