class MetricsCollector():
def __init__(self):
self.tsdb_register = TsdbRegister()
def run(self):
while True:
start = time.time()
self.collect_metrics()
self.tsdb_register.register_new_keys_to_tsdb()
self.batch_output_to_tsdb()
end = time.time()
to_sleep_time = collect_period - (end - start)
if to_sleep_time > 0:
time.sleep(to_sleep_time)
def collect_metrics(self):
try:
out_file = open(local_data_path, 'w')
json_string = urllib.urlopen(metrics_url).read()
metrics = json.loads(json_string)
timestamp = metrics['timestamp']
for endpoint, group_metrics in metrics['data'].iteritems():
for group, key_metrics in group_metrics.iteritems():
for key, metric in key_metrics.iteritems():
value = metric['value']
self.append_to_file(out_file, timestamp, key, value, endpoint, group)
if key not in self.tsdb_register.register_keys:
self.tsdb_register.new_keys.append(key)
self.tsdb_register.register_keys.add(key)
out_file.close()
except Exception, e:
logger_metrics.error("collect_metrics exception: %s", e)
class MetricsCollector():
def __init__(self):
self.tsdb_register = TsdbRegister()
def run(self):
while True:
start = time.time()
self.collect_metrics()
self.tsdb_register.register_new_keys_to_tsdb()
self.batch_output_to_tsdb()
end = time.time()
to_sleep_time = collect_period - (end - start)
if to_sleep_time > 0:
time.sleep(to_sleep_time)
def collect_metrics(self):
try:
out_file = open(local_data_path, 'w')
json_string = urllib.urlopen(metrics_url).read()
metrics = json.loads(json_string)
timestamp = metrics['timestamp']
for endpoint, group_metrics in metrics['data'].iteritems():
for group, key_metrics in group_metrics.iteritems():
for key, metric in key_metrics.iteritems():
if key.find('#') != -1:
key = key.replace("#", "_")
value = metric['value']
self.append_to_file(out_file, timestamp, key, value,
endpoint, group)
if key not in self.tsdb_register.register_keys:
self.tsdb_register.new_keys.append(key)
self.tsdb_register.register_keys.add(key)
out_file.close()
except Exception, e:
logger_metrics.error("collect_metrics exception: %s", e)
class QuotaInjector():
'''
Push quota information into opentsdb
'''
def __init__(self):
self.tsdb_register = TsdbRegister()
def check_quota_new_keys(self, quota_list):
if len(quota_list) > 0:
for quota_key in quota_list[0].keys():
if quota_key not in self.tsdb_register.register_keys and quota_key != 'name':
self.tsdb_register.new_keys.append(quota_key)
self.tsdb_register.register_keys.add(quota_key)
self.tsdb_register.register_new_keys_to_tsdb()
def push_quota_to_tsdb(self, quota_list, cluster_name):
self.check_quota_new_keys(quota_list)
timestamp = int(time.time())
# reset the quota_total_dict
quota_total_dict = dict.fromkeys(QUOTA_TOTAL_DICT, 0)
# push every user's quota to tsdb for cluster_name
for quota_dict in quota_list:
for quota_key, quota_value in quota_dict.iteritems():
if quota_key != 'name':
if not quota_value.isdigit():
quota_value = '0'
quota_record = "%s %d %d user_id=%s cluster=%s" % (
quota_key, timestamp, int(quota_value),
quota_dict['name'], cluster_name)
put_operation = 'echo put %s | nc -w 10 %s %s' % (
quota_record, tsdb_host, tsdb_port)
logger_quota.info(put_operation)
os.system(put_operation)
if quota_key in quota_total_dict.keys():
quota_total_dict[quota_key] += int(quota_value)
# push the total values to tsdb
for quota_key, quota_value in quota_total_dict.iteritems():
quota_record = "%s %d %d user_id=%s cluster=%s" % (
quota_key, timestamp, quota_value, quota_key + '_total',
cluster_name)
put_operation = 'echo put %s | nc -w 10 %s %s' % (
quota_record, tsdb_host, tsdb_port)
logger_quota.info(put_operation)
os.system(put_operation)
class QuotaInjector():
'''
Push quota information into opentsdb
'''
def __init__(self):
self.tsdb_register = TsdbRegister()
def check_quota_new_keys(self, quota_list):
if len(quota_list) > 0:
for quota_key in quota_list[0].keys():
if quota_key not in self.tsdb_register.register_keys and quota_key != 'name':
self.tsdb_register.new_keys.append(quota_key)
self.tsdb_register.register_keys.add(quota_key)
self.tsdb_register.register_new_keys_to_tsdb()
def push_quota_to_tsdb(self, quota_list, cluster_name):
self.check_quota_new_keys(quota_list)
timestamp = int(time.time())
# reset the quota_total_dict
quota_total_dict = dict.fromkeys(QUOTA_TOTAL_DICT, 0)
# push every user's quota to tsdb for cluster_name
for quota_dict in quota_list:
for quota_key, quota_value in quota_dict.iteritems():
if quota_key != 'name':
if not quota_value.isdigit():
quota_value = '0'
quota_record = "%s %d %d user_id=%s cluster=%s" % (
quota_key, timestamp, int(quota_value), quota_dict['name'], cluster_name)
put_operation = 'echo put %s | nc -w 10 %s %s' % (quota_record, tsdb_host, tsdb_port)
logger_quota.info(put_operation)
os.system(put_operation)
if quota_key in quota_total_dict.keys():
quota_total_dict[quota_key] += int(quota_value)
# push the total values to tsdb
for quota_key, quota_value in quota_total_dict.iteritems():
quota_record = "%s %d %d user_id=%s cluster=%s" % (
quota_key, timestamp, quota_value, quota_key+'_total', cluster_name)
put_operation = 'echo put %s | nc -w 10 %s %s' % (quota_record, tsdb_host, tsdb_port)
logger_quota.info(put_operation)
os.system(put_operation)
def __init__(self): self.tsdb_register = TsdbRegister()
def __init__(self):
self.tsdb_register = TsdbRegister()