def get_filtered_search_results(self, search_type, search_query, search_filter, **kwargs): ''' Filter search. Provides results for searches that are faceted. search_type = one of ['locations', 'servers', 'clients'] search_query = input query from api search_filter = {type: ['locations', 'servers', 'clients'], value:[id1, id2]} ''' if not search_filter['type'] or search_filter['type'] == search_type: return [] table_name = self.get_table_name(search_type, search_filter) table_config = get_table_config(self.table_configs, None, table_name) all_results = [] for filter_value in sorted(search_filter['value'], reverse=False): # we always want this filter value to be the first key key_prefix = du.get_key_field(filter_value, 0, table_config) key_prefix += du.BIGTABLE_KEY_DELIM # filter only the `meta` column family - for speed. tablefilter = FamilyNameRegexFilter('meta') all_results += bt.scan_table(table_config, self.get_pool(), prefix=key_prefix, filter=tablefilter, **kwargs) filtered_results = self.filter_results(search_type, search_query, all_results) return self.prepare_filtered_search_results(filtered_results)
def get_filtered_search_results(self, search_type, search_query, search_filter, **kwargs): ''' Filter search. Provides results for searches that are faceted. search_type = one of ['locations', 'servers', 'clients', 'asn_numbers'] search_query = input query from api search_filter = { type: ['locations', 'servers', 'clients', 'asn_numbers'], value:[id1, id2] } ''' if not search_filter['type'] or search_filter['type'] == search_type: return [] table_name = self.get_table_name(search_type, search_filter) table_config = get_table_config(self.table_configs, None, table_name) all_results = [] for filter_value in sorted(search_filter['value'], reverse=False): # we always want this filter value to be the first key key_prefix = du.get_key_field(filter_value, 0, table_config) key_prefix += du.BIGTABLE_KEY_DELIM # filter only the `meta` column family - for speed. tablefilter = FamilyNameRegexFilter('meta') all_results += bt.scan_table( table_config, self.get_pool(), prefix=key_prefix, filter=tablefilter, **kwargs) filtered_results = self.filter_results(search_type, search_query, all_results) return self.prepare_filtered_search_results(filtered_results)
def get_raw_test_results(self): ''' Extract sample raw data. ''' table_name = 'raw_sample' table_config = get_table_config(self.table_configs, None, table_name) results = bt.scan_table(table_config, self.get_pool(), limit=1000) return {"results": results}
def get_raw_test_results(self): ''' Extract sample raw data. ''' table_name = 'raw_sample' table_config = get_table_config(self.table_configs, None, table_name) results = bt.scan_table(table_config, self.get_pool(), limit=1000) return {"results": results}
def get_basic_search_results(self, search_type, search_query): ''' Provide basic search with no filtering logic. Basic search is based purely on bigtable prefix scans. ''' table_name = du.search_table(search_type) table_config = get_table_config(self.table_configs, None, table_name) results = bt.scan_table(table_config, self.get_pool(), prefix=search_query) return results
def get_basic_search_results(self, search_type, search_query): ''' Provide basic search with no filtering logic. Basic search is based purely on bigtable prefix scans. ''' table_name = du.search_table(search_type) table_config = get_table_config(self.table_configs, None, table_name) results = bt.scan_table(table_config, self.get_pool(), prefix=search_query) return results
def get_server_info(self, server_id): ''' Get info for a client server_id = id of server. ''' # we are using a hack from list tables # so grab the first match from a list table faceted by server ids' table_name = du.list_table("clients", "servers") table_config = get_table_config(self.table_configs, None, table_name) key_fields = du.get_key_fields([server_id], table_config) return get_bt_results(key_fields, table_config, self.get_pool())
def get_server_info(self, server_id): ''' Get info for a client server_id = id of server. ''' # we are using a hack from list tables # so grab the first match from a list table faceted by server ids' table_name = du.list_table("clients", "servers") table_config = get_table_config(self.table_configs, None, table_name) key_fields = du.get_key_fields([server_id], table_config) return get_bt_results(key_fields, table_config, self.get_pool())
def get_client_metrics(self, client_id, timebin, starttime, endtime): ''' Get data for client location at a specific timebin between start and stop times. client_id = id string of client. timebin = time aggregation key. starttime = start time for metric query. endtime = end time for metric query. ''' table_config = get_table_config(self.table_configs, timebin, TABLE_KEYS["clients"]) key_fields = du.get_key_fields([client_id], table_config) formatted = bt.get_time_metric_results(key_fields, self.get_pool(), timebin, starttime, endtime, table_config, "clients") return formatted
def get_location_client_isp_info(self, location_id, client_id): ''' Get static information about location_id = id string of location. client_id = id string of client. ''' config_id = du.list_table('clients', 'locations') table_config = get_table_config(self.table_configs, None, config_id) key_fields = du.get_key_fields([location_id, client_id], table_config) row_key = du.BIGTABLE_KEY_DELIM.join(key_fields) results = [] results = bt.get_row(table_config, self.get_pool(), row_key) return results
def get_location_info(self, location_id): ''' Get info about specific location location_id = id string of location. ''' table_config = get_table_config(self.table_configs, None, du.list_table('locations')) # add empty field to get child location in there location_key_fields = du.get_key_fields(["info", location_id], table_config) row_key = du.BIGTABLE_KEY_DELIM.join(location_key_fields) row = "" row = bt.get_row(table_config, self.get_pool(), row_key) return row
def get_client_metrics(self, client_id, timebin, starttime, endtime): ''' Get data for client location at a specific timebin between start and stop times. client_id = id string of client. timebin = time aggregation key. starttime = start time for metric query. endtime = end time for metric query. ''' table_config = get_table_config(self.table_configs, timebin, TABLE_KEYS["clients"]) key_fields = du.get_key_fields([client_id], table_config) formatted = bt.get_time_metric_results( key_fields, self.get_pool(), timebin, starttime, endtime, table_config, "clients") return formatted
def get_server_metrics(self, server_id, timebin, starttime, endtime): ''' Get data for specific location at a specific frequency between start and stop times. server_id = id of server. timebin = time aggregation key. starttime = start time for metric query. endtime = end time for metric query. ''' table_config = get_table_config(self.table_configs, timebin, TABLE_KEYS["servers"]) location_key_fields = du.get_key_fields([server_id], table_config) formatted = bt.get_time_metric_results( location_key_fields, self.get_pool(), timebin, starttime, endtime, table_config, "servers") return formatted
def get_client_server_metrics(self, client_id, server_id, timebin, starttime, endtime): ''' Get data for a specific client + server at a timebin between start and stop times. client_id = id string of client. server_id = id of server. timebin = time aggregation key. starttime = start time for metric query. endtime = end time for metric query. ''' agg_name = TABLE_KEYS["servers"] + '_' + TABLE_KEYS["clients"] table_config = get_table_config(self.table_configs, timebin, agg_name) key_fields = du.get_key_fields([server_id, client_id], table_config) formatted = bt.get_time_metric_results(key_fields, self.get_pool(), timebin, starttime, endtime, table_config, "clients") return formatted
def get_list_data(self, entity_id, entity_type, query_type, include_data): ''' Helper method to get out data from a list table. entity_id = id of entity to look for entity_type = [locations, clients, servers] query_type = [locations, clients, servers] - what we are faceting on include_data = boolean to indicate if data should be queried and returned. ''' config_id = du.list_table(query_type, entity_type) metric_name = "_".join([entity_type, query_type]) table_config = get_table_config(self.table_configs, None, config_id) key_fields = du.get_key_fields([entity_id], table_config) results = bt.get_list_table_results(key_fields, self.get_pool(), include_data, table_config, metric_name) return {"results": results}
def get_server_metrics(self, server_id, timebin, starttime, endtime): ''' Get data for specific location at a specific frequency between start and stop times. server_id = id of server. timebin = time aggregation key. starttime = start time for metric query. endtime = end time for metric query. ''' table_config = get_table_config(self.table_configs, timebin, TABLE_KEYS["servers"]) location_key_fields = du.get_key_fields([server_id], table_config) formatted = bt.get_time_metric_results(location_key_fields, self.get_pool(), timebin, starttime, endtime, table_config, "servers") return formatted
def get_list_data(self, entity_id, entity_type, query_type, include_data): ''' Helper method to get out data from a list table. entity_id = id of entity to look for entity_type = [locations, clients, servers] query_type = [locations, clients, servers] - what we are faceting on include_data = boolean indicates if data should be queried and returned. ''' config_id = du.list_table(query_type, entity_type) metric_name = "_".join([entity_type, query_type]) table_config = get_table_config(self.table_configs, None, config_id) key_fields = du.get_key_fields([entity_id], table_config) results = bt.get_list_table_results( key_fields, self.get_pool(), include_data, table_config, metric_name) return {"results": results}
def get_location_children(self, location_id, type_filter=None): ''' Return information about children regions of a location location_id = id string of location. type_filter = optionally restrict results to a location type. ''' table_config = get_table_config(self.table_configs, None, du.list_table('locations')) location_key_fields = du.get_location_key_fields(location_id, table_config) location_key_field = du.BIGTABLE_KEY_DELIM.join(location_key_fields) results = [] results = bt.scan_table(table_config, self.get_pool(), prefix=location_key_field) if type_filter: results = [r for r in results if r['meta']['type'] == type_filter] return {"results": results}
def get_client_server_metrics(self, client_id, server_id, timebin, starttime, endtime): ''' Get data for a specific client + server at a timebin between start and stop times. client_id = id string of client. server_id = id of server. timebin = time aggregation key. starttime = start time for metric query. endtime = end time for metric query. ''' agg_name = TABLE_KEYS["servers"] + '_' + TABLE_KEYS["clients"] table_config = get_table_config(self.table_configs, timebin, agg_name) key_fields = du.get_key_fields([server_id, client_id], table_config) formatted = bt.get_time_metric_results( key_fields, self.get_pool(), timebin, starttime, endtime, table_config, "clients") return formatted
def get_client_info(self, client_id): ''' Get info for a client. client_id = id string of client. ''' # we are using a hack from list tables # so grab the first match from a list table faceted by client ids' table_name = du.list_table("servers", "clients") table_config = get_table_config(self.table_configs, None, table_name) key_fields = du.get_key_fields([client_id], table_config) prefix_key = du.BIGTABLE_KEY_DELIM.join(key_fields) results = bt.scan_table(table_config, self.get_pool(), prefix=prefix_key, limit=1, filter=FamilyNameRegexFilter('meta')) result = {} if len(results) > 0: result = results[0] return result
# -*- coding: utf-8 -*- ''' Test whether BigTable connection works ''' import os from mlab_api.data.bigtable_utils import init_pool, scan_table from mlab_api.data.table_config import get_table_config, read_table_configs from google.oauth2 import service_account CREDENTIALS = service_account.Credentials.from_service_account_file( os.path.abspath(os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"))) # credentials = GoogleCredentials.get_application_default() print 'Using credentials' + CREDENTIALS.service_account_email print 'Environment variables' print os.environ print 'Connect to db' POOL = init_pool() TABLE_CONFIGS = read_table_configs("bigtable_configs") TABLE_NAME = 'client_loc_search' TABLE_CONFIG = get_table_config(TABLE_CONFIGS, None, TABLE_NAME) print 'Query db' RESULTS = scan_table(TABLE_CONFIG, POOL, limit=3) print 'Results' print RESULTS
# -*- coding: utf-8 -*- ''' Test whether BigTable connection works ''' import os from mlab_api.data.bigtable_utils import init_pool, scan_table from mlab_api.data.table_config import get_table_config, read_table_configs from google.oauth2 import service_account CREDENTIALS = service_account.Credentials.from_service_account_file( os.path.abspath(os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")) ) # credentials = GoogleCredentials.get_application_default() print 'Using credentials' + CREDENTIALS.service_account_email print 'Environment variables' print os.environ print 'Connect to db' POOL = init_pool() TABLE_CONFIGS = read_table_configs("bigtable_configs") TABLE_NAME = 'client_loc_search' TABLE_CONFIG = get_table_config(TABLE_CONFIGS, None, TABLE_NAME) print 'Query db' RESULTS = scan_table(TABLE_CONFIG, POOL, limit=3) print 'Results' print RESULTS