class MeasuredClientMixIn(object):
def initialize(self, client, address, dimensions, *args, **kwargs):
super(MeasuredClientMixIn, self).initialize(*args, **kwargs)
self.measure = Measure(client, address)
self.dimensions = dimensions
def fetch_impl(self, request, callback):
now = time.time()
def new_callback(response):
response_time = time.time() - now
self.write_metric(request, response, response_time)
return callback(response)
return super(MeasuredClientMixIn, self).fetch_impl(request, new_callback)
def write_metric(self, request, response, response_time):
dimensions = {
'url': request.url,
'host': urlparse(request.url).netloc,
'status_code': response.code,
'response_time': response_time
}
if self.dimensions:
dimensions.update(self.dimensions)
self.measure.count('http_response', dimensions=dimensions)
def __init__(self):
es_url = app.config['ELASTICSEARCH_URL']
es_port = app.config['ELASTICSEARCH_PORT']
logstash_host = app.config['LOGSTASH_HOST']
logstash_port = int(app.config['LOGSTASH_PORT'])
self.measure = Measure(app.config['ELASTICSEARCH_CLIENT'],
(logstash_host, logstash_port))
self.es = connections.create_connection(hosts=[es_url + ':' + es_port])
class LogstashBackend(object):
def __init__(self):
envs = os.environ
self.app_name = envs.get("TSURU_APPNAME")
self.host_name = socket.gethostname()
# initialize measure
host = envs.get("LOGSTASH_HOST") or 'localhost'
port = envs.get("LOGSTASH_PORT") or '1984'
from measures import Measure
self.client = Measure(envs.get("LOGSTASH_CLIENT", "tsuru"), (host, int(port)))
def stop(self):
pass
def gauge(self, key, value):
dimensions = {"app": self.app_name, "host": self.host_name, "value": float(value)}
self.client.count(key, dimensions=dimensions)
def disk_usage(self, value):
self.gauge("disk_usage", value)
def net_sent(self, value):
self.gauge("net_sent", value)
def net_recv(self, value):
self.gauge("net_recv", value)
def net_connections(self, value):
self.gauge("net_connections", value)
def connections(self, connection_list):
for conn in connection_list:
dimensions = {"app": self.app_name, "host": self.host_name, "connection": conn}
self.client.count("connection", dimensions=dimensions)
def cpu_max(self, name, value):
self.gauge("cpu_max", value)
def cpu_sum(self, name, value):
self.gauge("cpu_sum", value)
def mem_max(self, name, value):
self.gauge("mem_max", value)
def mem_sum(self, name, value):
self.gauge("mem_sum", value)
def mem_pct_max(self, name, value):
self.gauge("mem_pct_max", value)
def mem_pct_sum(self, name, value):
self.gauge("mem_pct_sum", value)
def __init__(self):
envs = os.environ
self.app_name = envs.get("TSURU_APPNAME")
self.host_name = socket.gethostname()
# initialize measure
host = envs.get("LOGSTASH_HOST") or 'localhost'
port = envs.get("LOGSTASH_PORT") or '1984'
from measures import Measure
self.client = Measure(envs.get("LOGSTASH_CLIENT", "tsuru"),
(host, int(port)))
class ELKClient:
def __init__(self):
es_url = app.config['ELASTICSEARCH_URL']
es_port = app.config['ELASTICSEARCH_PORT']
logstash_host = app.config['LOGSTASH_HOST']
logstash_port = int(app.config['LOGSTASH_PORT'])
self.measure = Measure(app.config['ELASTICSEARCH_CLIENT'],
(logstash_host, logstash_port))
self.es = connections.create_connection(hosts=[es_url + ':' + es_port])
def create_usage_record(self, data):
self.measure.count(app.config['ELASTICSEARCH_TYPE'], dimensions=data)
def find_usage_records(self, region, account, start, end):
s = Search(using=self.es,
index=app.config['ELASTICSEARCH_INDEX'],
doc_type=app.config['ELASTICSEARCH_TYPE'])
s = s.filter('term', region=region)
if account is not None:
s = s.filter('term', account=account)
s = s.filter('range',
date={
'gte': parse(start).date().isoformat(),
'lte': parse(end).date().isoformat()
})[0:0]
s.aggs.bucket('by_project', 'terms', field='projectid.raw', size=0) \
.bucket('by_type', 'terms', field='usagetype.raw') \
.bucket('by_offering', 'terms', field='offering_struct.raw') \
.metric('rawusage_sum', 'sum', field='rawusage')
return s.execute().aggregations.to_dict()
def delete_usage_records(self, region, date):
index = app.config['ELASTICSEARCH_INDEX']
doc_type = app.config['ELASTICSEARCH_TYPE']
s = Search(using=self.es, index=app.config['ELASTICSEARCH_INDEX'], doc_type=doc_type) \
.filter('term', region=region) \
.filter('term', date=date)
self.es.delete_by_query(index=index,
doc_type=doc_type,
body=s.to_dict())
def health(self):
return self.es.cluster.health()
def __init__(self):
es_url = app.config['ELASTICSEARCH_URL']
es_port = app.config['ELASTICSEARCH_PORT']
logstash_host = app.config['LOGSTASH_HOST']
logstash_port = int(app.config['LOGSTASH_PORT'])
self.measure = Measure(app.config['ELASTICSEARCH_CLIENT'], (logstash_host, logstash_port))
self.es = connections.create_connection(hosts=[es_url + ':' + es_port])
def test_must_create_a_measure_object_with_correct_attributes(self):
measure = Measure('myclient', ('localhost', 1984))
self.assertEqual(measure.client, 'myclient')
self.assertEqual(measure.address, ('localhost', 1984))
self.assertEqual(
measure.socket.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE),
socket.SOCK_DGRAM)
# socket is non-blocking
self.assertEqual(measure.socket.gettimeout(), 0.0)
class MeasureClient:
def __init__(self):
es_url = app.config['ELASTICSEARCH_URL']
es_port = app.config['ELASTICSEARCH_PORT']
logstash_host = app.config['LOGSTASH_HOST']
logstash_port = int(app.config['LOGSTASH_PORT'])
self.measure = Measure(app.config['ELASTICSEARCH_CLIENT'], (logstash_host, logstash_port))
self.es = connections.create_connection(hosts=[es_url + ':' + es_port])
def create(self, data):
self.measure.count(app.config['ELASTICSEARCH_TYPE'], dimensions=data)
def find(self, region, account, start, end):
s = Search(using=self.es, index=app.config['ELASTICSEARCH_INDEX'], doc_type=app.config['ELASTICSEARCH_TYPE'])
s = s.filter('term', region=region)
if account is not None:
s = s.filter('term', account=account)
s = s.filter('range', date={
'gte': parse(start).date().isoformat(),
'lte': parse(end).date().isoformat()
})[0:0]
s.aggs.bucket('by_project', 'terms', field='projectid.raw', size=0) \
.bucket('by_type', 'terms', field='usagetype.raw') \
.bucket('by_offering', 'terms', field='offeringid.raw') \
.metric('rawusage_sum', 'sum', field='rawusage')
return s.execute().aggregations.to_dict()
def delete(self, region, date):
index = app.config['ELASTICSEARCH_INDEX']
doc_type = app.config['ELASTICSEARCH_TYPE']
s = Search(using=self.es, index=app.config['ELASTICSEARCH_INDEX'], doc_type=doc_type) \
.filter('term', region=region) \
.filter('term', date=date)
self.es.delete_by_query(index=index, doc_type=doc_type, body=s.to_dict())
def health(self):
return self.es.cluster.health()
def __init__(self):
envs = os.environ
self.app_name = envs.get("TSURU_APPNAME")
self.host_name = socket.gethostname()
# initialize measure
host = envs.get("LOGSTASH_HOST") or 'localhost'
port = envs.get("LOGSTASH_PORT") or '1984'
from measures import Measure
self.client = Measure(envs.get("LOGSTASH_CLIENT", "tsuru"), (host, int(port)))
def setUp(self):
self.measure = Measure('myclient', ('localhost', 1984))
def setUp(self):
super(MeasureTestCase, self).setUp()
self.measure = Measure('myclient', ('127.0.0.1', 1984))
class MeasureTestCase(UDPServerTestCase):
def setUp(self):
super(MeasureTestCase, self).setUp()
self.measure = Measure('myclient', ('127.0.0.1', 1984))
def test_recieve_count_message_correctly_over_network(self):
self.measure.count('mymetric')
self.wait_for(lambda: len(self.messages))
self.assertEqual(len(self.messages), 1)
message = json.loads(self.messages[0].decode('utf-8'))
expected_message = {
'client': 'myclient',
'metric': 'mymetric',
'count': 1,
}
self.assertDictEqual(message, expected_message)
def test_recieve_time_message_correctly_over_network(self):
with self.measure.time('mymetric'):
pass
while True:
self.wait_for(lambda: len(self.messages))
message = json.loads(self.messages.pop().decode('utf-8'))
if 'time' in message:
break
self.assertIsInstance(message['time'], float)
self.assertIn('client', message)
self.assertEqual(message['client'], 'myclient')
self.assertIn('metric', message)
self.assertEqual(message['metric'], 'mymetric')
self.assertGreater(message['time'], 0)
self.assertIn('error_type', message)
self.assertEqual(message['error_type'], '')
self.assertIn('error_value', message)
self.assertEqual(message['error_value'], '')
def test_count_must_not_hang_if_server_is_up(self):
p = Process(target=lambda: self.measure.count('mymetric'))
p.start()
p.join(0.3)
self.assertFalse(p.is_alive())
p.terminate()
def test_time_must_not_hang_if_server_is_up(self):
p = Process(target=lambda: self.measure.time('mymetric'))
p.start()
p.join(0.3)
self.assertFalse(p.is_alive())
p.terminate()
def test_count_must_not_hang_if_server_is_down(self):
self.__class__.stop_server()
time.sleep(0.5)
p = Process(target=lambda: self.measure.count('mymetric'))
p.start()
p.join(0.3)
self.assertFalse(p.is_alive())
p.terminate()
def test_time_must_not_hang_if_server_is_down(self):
self.__class__.stop_server()
time.sleep(0.5)
p = Process(target=lambda: self.measure.time('mymetric'))
p.start()
p.join(0.3)
self.assertFalse(p.is_alive())
p.terminate()
def initialize(self, client, address, dimensions, *args, **kwargs):
super(MeasuredClientMixIn, self).initialize(*args, **kwargs)
self.measure = Measure(client, address)
self.dimensions = dimensions
class LogstashBackend(object):
def __init__(self):
envs = os.environ
self.app_name = envs.get("TSURU_APPNAME")
self.host_name = socket.gethostname()
# initialize measure
host = envs.get("LOGSTASH_HOST") or 'localhost'
port = envs.get("LOGSTASH_PORT") or '1984'
from measures import Measure
self.client = Measure(envs.get("LOGSTASH_CLIENT", "tsuru"),
(host, int(port)))
def stop(self):
pass
def gauge(self, key, value):
dimensions = {
"app": self.app_name,
"host": self.host_name,
"value": float(value)
}
self.client.count(key, dimensions=dimensions)
def disk_usage(self, value):
self.gauge("disk_usage", value)
def net_sent(self, value):
self.gauge("net_sent", value)
def net_recv(self, value):
self.gauge("net_recv", value)
def net_connections(self, value):
self.gauge("net_connections", value)
def connections(self, connection_list):
for conn in connection_list:
dimensions = {
"app": self.app_name,
"host": self.host_name,
"connection": conn
}
self.client.count("connection", dimensions=dimensions)
def cpu_max(self, name, value):
self.gauge("cpu_max", value)
def cpu_sum(self, name, value):
self.gauge("cpu_sum", value)
def mem_max(self, name, value):
self.gauge("mem_max", value)
def mem_sum(self, name, value):
self.gauge("mem_sum", value)
def mem_pct_max(self, name, value):
self.gauge("mem_pct_max", value)
def mem_pct_sum(self, name, value):
self.gauge("mem_pct_sum", value)
def setUp(self):
super(MeasureTestCase, self).setUp()
self.measure = Measure('myclient', ('127.0.0.1', 1984))
class MeasureTestCase(UDPServerTestCase):
def setUp(self):
super(MeasureTestCase, self).setUp()
self.measure = Measure('myclient', ('127.0.0.1', 1984))
def test_recieve_count_message_correctly_over_network(self):
self.measure.count('mymetric')
self.wait_for(lambda: len(self.messages))
self.assertEqual(len(self.messages), 1)
message = json.loads(self.messages[0].decode('utf-8'))
expected_message = {
'client': 'myclient',
'metric': 'mymetric',
'count': 1,
}
self.assertDictEqual(message, expected_message)
def test_recieve_time_message_correctly_over_network(self):
with self.measure.time('mymetric'):
pass
while True:
self.wait_for(lambda: len(self.messages))
message = json.loads(self.messages.pop().decode('utf-8'))
if 'time' in message:
break
self.assertIsInstance(message['time'], float)
self.assertIn('client', message)
self.assertEqual(message['client'], 'myclient')
self.assertIn('metric', message)
self.assertEqual(message['metric'], 'mymetric')
self.assertGreater(message['time'], 0)
self.assertIn('error_type', message)
self.assertEqual(message['error_type'], '')
self.assertIn('error_value', message)
self.assertEqual(message['error_value'], '')
def test_count_must_not_hang_if_server_is_up(self):
p = Process(target=lambda: self.measure.count('mymetric'))
p.start()
p.join(0.3)
self.assertFalse(p.is_alive())
p.terminate()
def test_time_must_not_hang_if_server_is_up(self):
p = Process(target=lambda: self.measure.time('mymetric'))
p.start()
p.join(0.3)
self.assertFalse(p.is_alive())
p.terminate()
def test_count_must_not_hang_if_server_is_down(self):
self.__class__.stop_server()
time.sleep(0.5)
p = Process(target=lambda: self.measure.count('mymetric'))
p.start()
p.join(0.3)
self.assertFalse(p.is_alive())
p.terminate()
def test_time_must_not_hang_if_server_is_down(self):
self.__class__.stop_server()
time.sleep(0.5)
p = Process(target=lambda: self.measure.time('mymetric'))
p.start()
p.join(0.3)
self.assertFalse(p.is_alive())
p.terminate()
def run_model(inputs, file_name=None):
####################### Reading Parameters ########################
param = Parameters(inputs, file_name)
AppLogging.startMessage(param)
AppLogging.setConfig(param.output_directory)
####################### Reading Input Data ########################
input_data = PreProcessing.read_input_data(param.input_file)
input_data, alerts = PreProcessing.validate_input_data(input_data)
AppLogging.inputValidationMessage(alerts)
data_repartition = PreProcessing.split_input_data(
input_data, param.vol_filter_lower_bound, param.vol_filter_upper_bound,
param.vol_filter_min_cluster_size)
AppLogging.volumeFilteringMessage(data_repartition, param)
for (key, value) in data_repartition.items():
AppLogging.dataMessage(key, value)
split_data = input_data.iloc[value].reset_index(drop=True)
split_data = input_data
try:
####################### Pre Processing Data ########################
filter_data = PreProcessing.filter_input_data(split_data)
geo_data = PreProcessing.get_geolocate_data(filter_data)
model_data = PreProcessing.get_scaled_data(geo_data)
#Plots.data_iteractive_view(geo_data, param.svc_name)
############### Choosing the Appropriate Number of Clusters #############
min_num_cluster = Measure.calculate_min_num_cluster(
model_data, param)
num_cluster, num_tests = Measure.calculate_best_num_cluster(
model_data, param, min_num_cluster)
AppLogging.metricsMessage(min_num_cluster, num_tests, num_cluster)
################### 1) Call Clustering Algorithm ####################
model = Clusters(model_data, geo_data, num_cluster, param)
labels, sizes, algo_name = model.clustering_algorithm()
AppLogging.algorithmNameMessage(algo_name)
################# 4) Call Cluster Aglumerative Algorithm #################
violation = model.check_aglomerative_violation(sizes)
if violation:
labels, sizes, clusters_viol = model.aglomerative_clusters(
labels, sizes)
AppLogging.aglomerativeMessage(clusters_viol)
AppLogging.numClustersMessage(model.num_cluster)
################# 2) Call Cluster Balance Algorithm #################
violation = model.check_balance_violation(sizes)
if violation:
labels, sizes, clusters_viol = model.balance_clusters(
labels, sizes)
if param.type == 'dense':
labels, sizes = model.simple_center_adjustment(
labels, sizes)
AppLogging.balanceMessage(clusters_viol)
################# 3) Call Fine Adjustment ########################
labels, sizes = model.fine_adjustement(labels, sizes)
################# Post Processing Data ########################
labels, sizes, violated_points = PostProcessing.validate_output_data(
filter_data, labels, sizes)
AppLogging.latlongAdjustmentMessage(violated_points)
output_data = PostProcessing.generate_output_data(
filter_data, labels)
output_folder, output_name = PostProcessing.generate_output_folder(
param.output_directory, param.svc_name, len(output_data), key)
PostProcessing.write_output_data(output_data, output_folder)
################# Print Maps ########################
if param.plot:
centers = list(
map(
lambda x: model.calculate_geo_cluster_center(
labels, x), range(model.num_cluster)))
cluster_map = Plots.cluster_iteractive_view(
model.geo_data, labels, sizes, centers)
PostProcessing.write_cluster_map(cluster_map,
model.num_cluster,
output_name, output_folder)
AppLogging.endMessage(sizes, output_folder)
except Exception:
AppLogging.errorMessage()
