class MetricsCollector(): def __init__(self): self.tsdb_register = TsdbRegister() def run(self): while True: start = time.time() self.collect_metrics() self.tsdb_register.register_new_keys_to_tsdb() self.batch_output_to_tsdb() end = time.time() to_sleep_time = collect_period - (end - start) if to_sleep_time > 0: time.sleep(to_sleep_time) def collect_metrics(self): try: out_file = open(local_data_path, 'w') json_string = urllib.urlopen(metrics_url).read() metrics = json.loads(json_string) timestamp = metrics['timestamp'] for endpoint, group_metrics in metrics['data'].iteritems(): for group, key_metrics in group_metrics.iteritems(): for key, metric in key_metrics.iteritems(): value = metric['value'] self.append_to_file(out_file, timestamp, key, value, endpoint, group) if key not in self.tsdb_register.register_keys: self.tsdb_register.new_keys.append(key) self.tsdb_register.register_keys.add(key) out_file.close() except Exception, e: logger_metrics.error("collect_metrics exception: %s", e)
class MetricsCollector(): def __init__(self): self.tsdb_register = TsdbRegister() def run(self): while True: start = time.time() self.collect_metrics() self.tsdb_register.register_new_keys_to_tsdb() self.batch_output_to_tsdb() end = time.time() to_sleep_time = collect_period - (end - start) if to_sleep_time > 0: time.sleep(to_sleep_time) def collect_metrics(self): try: out_file = open(local_data_path, 'w') json_string = urllib.urlopen(metrics_url).read() metrics = json.loads(json_string) timestamp = metrics['timestamp'] for endpoint, group_metrics in metrics['data'].iteritems(): for group, key_metrics in group_metrics.iteritems(): for key, metric in key_metrics.iteritems(): if key.find('#') != -1: key = key.replace("#", "_") value = metric['value'] self.append_to_file(out_file, timestamp, key, value, endpoint, group) if key not in self.tsdb_register.register_keys: self.tsdb_register.new_keys.append(key) self.tsdb_register.register_keys.add(key) out_file.close() except Exception, e: logger_metrics.error("collect_metrics exception: %s", e)
class QuotaInjector(): ''' Push quota information into opentsdb ''' def __init__(self): self.tsdb_register = TsdbRegister() def check_quota_new_keys(self, quota_list): if len(quota_list) > 0: for quota_key in quota_list[0].keys(): if quota_key not in self.tsdb_register.register_keys and quota_key != 'name': self.tsdb_register.new_keys.append(quota_key) self.tsdb_register.register_keys.add(quota_key) self.tsdb_register.register_new_keys_to_tsdb() def push_quota_to_tsdb(self, quota_list, cluster_name): self.check_quota_new_keys(quota_list) timestamp = int(time.time()) # reset the quota_total_dict quota_total_dict = dict.fromkeys(QUOTA_TOTAL_DICT, 0) # push every user's quota to tsdb for cluster_name for quota_dict in quota_list: for quota_key, quota_value in quota_dict.iteritems(): if quota_key != 'name': if not quota_value.isdigit(): quota_value = '0' quota_record = "%s %d %d user_id=%s cluster=%s" % ( quota_key, timestamp, int(quota_value), quota_dict['name'], cluster_name) put_operation = 'echo put %s | nc -w 10 %s %s' % ( quota_record, tsdb_host, tsdb_port) logger_quota.info(put_operation) os.system(put_operation) if quota_key in quota_total_dict.keys(): quota_total_dict[quota_key] += int(quota_value) # push the total values to tsdb for quota_key, quota_value in quota_total_dict.iteritems(): quota_record = "%s %d %d user_id=%s cluster=%s" % ( quota_key, timestamp, quota_value, quota_key + '_total', cluster_name) put_operation = 'echo put %s | nc -w 10 %s %s' % ( quota_record, tsdb_host, tsdb_port) logger_quota.info(put_operation) os.system(put_operation)
class QuotaInjector(): ''' Push quota information into opentsdb ''' def __init__(self): self.tsdb_register = TsdbRegister() def check_quota_new_keys(self, quota_list): if len(quota_list) > 0: for quota_key in quota_list[0].keys(): if quota_key not in self.tsdb_register.register_keys and quota_key != 'name': self.tsdb_register.new_keys.append(quota_key) self.tsdb_register.register_keys.add(quota_key) self.tsdb_register.register_new_keys_to_tsdb() def push_quota_to_tsdb(self, quota_list, cluster_name): self.check_quota_new_keys(quota_list) timestamp = int(time.time()) # reset the quota_total_dict quota_total_dict = dict.fromkeys(QUOTA_TOTAL_DICT, 0) # push every user's quota to tsdb for cluster_name for quota_dict in quota_list: for quota_key, quota_value in quota_dict.iteritems(): if quota_key != 'name': if not quota_value.isdigit(): quota_value = '0' quota_record = "%s %d %d user_id=%s cluster=%s" % ( quota_key, timestamp, int(quota_value), quota_dict['name'], cluster_name) put_operation = 'echo put %s | nc -w 10 %s %s' % (quota_record, tsdb_host, tsdb_port) logger_quota.info(put_operation) os.system(put_operation) if quota_key in quota_total_dict.keys(): quota_total_dict[quota_key] += int(quota_value) # push the total values to tsdb for quota_key, quota_value in quota_total_dict.iteritems(): quota_record = "%s %d %d user_id=%s cluster=%s" % ( quota_key, timestamp, quota_value, quota_key+'_total', cluster_name) put_operation = 'echo put %s | nc -w 10 %s %s' % (quota_record, tsdb_host, tsdb_port) logger_quota.info(put_operation) os.system(put_operation)
def __init__(self): self.tsdb_register = TsdbRegister()