class MeasuredClientMixIn(object): def initialize(self, client, address, dimensions, *args, **kwargs): super(MeasuredClientMixIn, self).initialize(*args, **kwargs) self.measure = Measure(client, address) self.dimensions = dimensions def fetch_impl(self, request, callback): now = time.time() def new_callback(response): response_time = time.time() - now self.write_metric(request, response, response_time) return callback(response) return super(MeasuredClientMixIn, self).fetch_impl(request, new_callback) def write_metric(self, request, response, response_time): dimensions = { 'url': request.url, 'host': urlparse(request.url).netloc, 'status_code': response.code, 'response_time': response_time } if self.dimensions: dimensions.update(self.dimensions) self.measure.count('http_response', dimensions=dimensions)
def __init__(self): es_url = app.config['ELASTICSEARCH_URL'] es_port = app.config['ELASTICSEARCH_PORT'] logstash_host = app.config['LOGSTASH_HOST'] logstash_port = int(app.config['LOGSTASH_PORT']) self.measure = Measure(app.config['ELASTICSEARCH_CLIENT'], (logstash_host, logstash_port)) self.es = connections.create_connection(hosts=[es_url + ':' + es_port])
class LogstashBackend(object): def __init__(self): envs = os.environ self.app_name = envs.get("TSURU_APPNAME") self.host_name = socket.gethostname() # initialize measure host = envs.get("LOGSTASH_HOST") or 'localhost' port = envs.get("LOGSTASH_PORT") or '1984' from measures import Measure self.client = Measure(envs.get("LOGSTASH_CLIENT", "tsuru"), (host, int(port))) def stop(self): pass def gauge(self, key, value): dimensions = {"app": self.app_name, "host": self.host_name, "value": float(value)} self.client.count(key, dimensions=dimensions) def disk_usage(self, value): self.gauge("disk_usage", value) def net_sent(self, value): self.gauge("net_sent", value) def net_recv(self, value): self.gauge("net_recv", value) def net_connections(self, value): self.gauge("net_connections", value) def connections(self, connection_list): for conn in connection_list: dimensions = {"app": self.app_name, "host": self.host_name, "connection": conn} self.client.count("connection", dimensions=dimensions) def cpu_max(self, name, value): self.gauge("cpu_max", value) def cpu_sum(self, name, value): self.gauge("cpu_sum", value) def mem_max(self, name, value): self.gauge("mem_max", value) def mem_sum(self, name, value): self.gauge("mem_sum", value) def mem_pct_max(self, name, value): self.gauge("mem_pct_max", value) def mem_pct_sum(self, name, value): self.gauge("mem_pct_sum", value)
def __init__(self): envs = os.environ self.app_name = envs.get("TSURU_APPNAME") self.host_name = socket.gethostname() # initialize measure host = envs.get("LOGSTASH_HOST") or 'localhost' port = envs.get("LOGSTASH_PORT") or '1984' from measures import Measure self.client = Measure(envs.get("LOGSTASH_CLIENT", "tsuru"), (host, int(port)))
class ELKClient: def __init__(self): es_url = app.config['ELASTICSEARCH_URL'] es_port = app.config['ELASTICSEARCH_PORT'] logstash_host = app.config['LOGSTASH_HOST'] logstash_port = int(app.config['LOGSTASH_PORT']) self.measure = Measure(app.config['ELASTICSEARCH_CLIENT'], (logstash_host, logstash_port)) self.es = connections.create_connection(hosts=[es_url + ':' + es_port]) def create_usage_record(self, data): self.measure.count(app.config['ELASTICSEARCH_TYPE'], dimensions=data) def find_usage_records(self, region, account, start, end): s = Search(using=self.es, index=app.config['ELASTICSEARCH_INDEX'], doc_type=app.config['ELASTICSEARCH_TYPE']) s = s.filter('term', region=region) if account is not None: s = s.filter('term', account=account) s = s.filter('range', date={ 'gte': parse(start).date().isoformat(), 'lte': parse(end).date().isoformat() })[0:0] s.aggs.bucket('by_project', 'terms', field='projectid.raw', size=0) \ .bucket('by_type', 'terms', field='usagetype.raw') \ .bucket('by_offering', 'terms', field='offering_struct.raw') \ .metric('rawusage_sum', 'sum', field='rawusage') return s.execute().aggregations.to_dict() def delete_usage_records(self, region, date): index = app.config['ELASTICSEARCH_INDEX'] doc_type = app.config['ELASTICSEARCH_TYPE'] s = Search(using=self.es, index=app.config['ELASTICSEARCH_INDEX'], doc_type=doc_type) \ .filter('term', region=region) \ .filter('term', date=date) self.es.delete_by_query(index=index, doc_type=doc_type, body=s.to_dict()) def health(self): return self.es.cluster.health()
def test_must_create_a_measure_object_with_correct_attributes(self): measure = Measure('myclient', ('localhost', 1984)) self.assertEqual(measure.client, 'myclient') self.assertEqual(measure.address, ('localhost', 1984)) self.assertEqual( measure.socket.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE), socket.SOCK_DGRAM) # socket is non-blocking self.assertEqual(measure.socket.gettimeout(), 0.0)
class MeasureClient: def __init__(self): es_url = app.config['ELASTICSEARCH_URL'] es_port = app.config['ELASTICSEARCH_PORT'] logstash_host = app.config['LOGSTASH_HOST'] logstash_port = int(app.config['LOGSTASH_PORT']) self.measure = Measure(app.config['ELASTICSEARCH_CLIENT'], (logstash_host, logstash_port)) self.es = connections.create_connection(hosts=[es_url + ':' + es_port]) def create(self, data): self.measure.count(app.config['ELASTICSEARCH_TYPE'], dimensions=data) def find(self, region, account, start, end): s = Search(using=self.es, index=app.config['ELASTICSEARCH_INDEX'], doc_type=app.config['ELASTICSEARCH_TYPE']) s = s.filter('term', region=region) if account is not None: s = s.filter('term', account=account) s = s.filter('range', date={ 'gte': parse(start).date().isoformat(), 'lte': parse(end).date().isoformat() })[0:0] s.aggs.bucket('by_project', 'terms', field='projectid.raw', size=0) \ .bucket('by_type', 'terms', field='usagetype.raw') \ .bucket('by_offering', 'terms', field='offeringid.raw') \ .metric('rawusage_sum', 'sum', field='rawusage') return s.execute().aggregations.to_dict() def delete(self, region, date): index = app.config['ELASTICSEARCH_INDEX'] doc_type = app.config['ELASTICSEARCH_TYPE'] s = Search(using=self.es, index=app.config['ELASTICSEARCH_INDEX'], doc_type=doc_type) \ .filter('term', region=region) \ .filter('term', date=date) self.es.delete_by_query(index=index, doc_type=doc_type, body=s.to_dict()) def health(self): return self.es.cluster.health()
def setUp(self): self.measure = Measure('myclient', ('localhost', 1984))
def setUp(self): super(MeasureTestCase, self).setUp() self.measure = Measure('myclient', ('127.0.0.1', 1984))
class MeasureTestCase(UDPServerTestCase): def setUp(self): super(MeasureTestCase, self).setUp() self.measure = Measure('myclient', ('127.0.0.1', 1984)) def test_recieve_count_message_correctly_over_network(self): self.measure.count('mymetric') self.wait_for(lambda: len(self.messages)) self.assertEqual(len(self.messages), 1) message = json.loads(self.messages[0].decode('utf-8')) expected_message = { 'client': 'myclient', 'metric': 'mymetric', 'count': 1, } self.assertDictEqual(message, expected_message) def test_recieve_time_message_correctly_over_network(self): with self.measure.time('mymetric'): pass while True: self.wait_for(lambda: len(self.messages)) message = json.loads(self.messages.pop().decode('utf-8')) if 'time' in message: break self.assertIsInstance(message['time'], float) self.assertIn('client', message) self.assertEqual(message['client'], 'myclient') self.assertIn('metric', message) self.assertEqual(message['metric'], 'mymetric') self.assertGreater(message['time'], 0) self.assertIn('error_type', message) self.assertEqual(message['error_type'], '') self.assertIn('error_value', message) self.assertEqual(message['error_value'], '') def test_count_must_not_hang_if_server_is_up(self): p = Process(target=lambda: self.measure.count('mymetric')) p.start() p.join(0.3) self.assertFalse(p.is_alive()) p.terminate() def test_time_must_not_hang_if_server_is_up(self): p = Process(target=lambda: self.measure.time('mymetric')) p.start() p.join(0.3) self.assertFalse(p.is_alive()) p.terminate() def test_count_must_not_hang_if_server_is_down(self): self.__class__.stop_server() time.sleep(0.5) p = Process(target=lambda: self.measure.count('mymetric')) p.start() p.join(0.3) self.assertFalse(p.is_alive()) p.terminate() def test_time_must_not_hang_if_server_is_down(self): self.__class__.stop_server() time.sleep(0.5) p = Process(target=lambda: self.measure.time('mymetric')) p.start() p.join(0.3) self.assertFalse(p.is_alive()) p.terminate()
def initialize(self, client, address, dimensions, *args, **kwargs): super(MeasuredClientMixIn, self).initialize(*args, **kwargs) self.measure = Measure(client, address) self.dimensions = dimensions
class LogstashBackend(object): def __init__(self): envs = os.environ self.app_name = envs.get("TSURU_APPNAME") self.host_name = socket.gethostname() # initialize measure host = envs.get("LOGSTASH_HOST") or 'localhost' port = envs.get("LOGSTASH_PORT") or '1984' from measures import Measure self.client = Measure(envs.get("LOGSTASH_CLIENT", "tsuru"), (host, int(port))) def stop(self): pass def gauge(self, key, value): dimensions = { "app": self.app_name, "host": self.host_name, "value": float(value) } self.client.count(key, dimensions=dimensions) def disk_usage(self, value): self.gauge("disk_usage", value) def net_sent(self, value): self.gauge("net_sent", value) def net_recv(self, value): self.gauge("net_recv", value) def net_connections(self, value): self.gauge("net_connections", value) def connections(self, connection_list): for conn in connection_list: dimensions = { "app": self.app_name, "host": self.host_name, "connection": conn } self.client.count("connection", dimensions=dimensions) def cpu_max(self, name, value): self.gauge("cpu_max", value) def cpu_sum(self, name, value): self.gauge("cpu_sum", value) def mem_max(self, name, value): self.gauge("mem_max", value) def mem_sum(self, name, value): self.gauge("mem_sum", value) def mem_pct_max(self, name, value): self.gauge("mem_pct_max", value) def mem_pct_sum(self, name, value): self.gauge("mem_pct_sum", value)
def run_model(inputs, file_name=None): ####################### Reading Parameters ######################## param = Parameters(inputs, file_name) AppLogging.startMessage(param) AppLogging.setConfig(param.output_directory) ####################### Reading Input Data ######################## input_data = PreProcessing.read_input_data(param.input_file) input_data, alerts = PreProcessing.validate_input_data(input_data) AppLogging.inputValidationMessage(alerts) data_repartition = PreProcessing.split_input_data( input_data, param.vol_filter_lower_bound, param.vol_filter_upper_bound, param.vol_filter_min_cluster_size) AppLogging.volumeFilteringMessage(data_repartition, param) for (key, value) in data_repartition.items(): AppLogging.dataMessage(key, value) split_data = input_data.iloc[value].reset_index(drop=True) split_data = input_data try: ####################### Pre Processing Data ######################## filter_data = PreProcessing.filter_input_data(split_data) geo_data = PreProcessing.get_geolocate_data(filter_data) model_data = PreProcessing.get_scaled_data(geo_data) #Plots.data_iteractive_view(geo_data, param.svc_name) ############### Choosing the Appropriate Number of Clusters ############# min_num_cluster = Measure.calculate_min_num_cluster( model_data, param) num_cluster, num_tests = Measure.calculate_best_num_cluster( model_data, param, min_num_cluster) AppLogging.metricsMessage(min_num_cluster, num_tests, num_cluster) ################### 1) Call Clustering Algorithm #################### model = Clusters(model_data, geo_data, num_cluster, param) labels, sizes, algo_name = model.clustering_algorithm() AppLogging.algorithmNameMessage(algo_name) ################# 4) Call Cluster Aglumerative Algorithm ################# violation = model.check_aglomerative_violation(sizes) if violation: labels, sizes, clusters_viol = model.aglomerative_clusters( labels, sizes) AppLogging.aglomerativeMessage(clusters_viol) AppLogging.numClustersMessage(model.num_cluster) ################# 2) Call Cluster Balance Algorithm ################# violation = model.check_balance_violation(sizes) if violation: labels, sizes, clusters_viol = model.balance_clusters( labels, sizes) if param.type == 'dense': labels, sizes = model.simple_center_adjustment( labels, sizes) AppLogging.balanceMessage(clusters_viol) ################# 3) Call Fine Adjustment ######################## labels, sizes = model.fine_adjustement(labels, sizes) ################# Post Processing Data ######################## labels, sizes, violated_points = PostProcessing.validate_output_data( filter_data, labels, sizes) AppLogging.latlongAdjustmentMessage(violated_points) output_data = PostProcessing.generate_output_data( filter_data, labels) output_folder, output_name = PostProcessing.generate_output_folder( param.output_directory, param.svc_name, len(output_data), key) PostProcessing.write_output_data(output_data, output_folder) ################# Print Maps ######################## if param.plot: centers = list( map( lambda x: model.calculate_geo_cluster_center( labels, x), range(model.num_cluster))) cluster_map = Plots.cluster_iteractive_view( model.geo_data, labels, sizes, centers) PostProcessing.write_cluster_map(cluster_map, model.num_cluster, output_name, output_folder) AppLogging.endMessage(sizes, output_folder) except Exception: AppLogging.errorMessage()