def get_filtered_search_results(self, search_type, search_query,
search_filter, **kwargs):
'''
Filter search. Provides results for searches that are faceted.
search_type = one of ['locations', 'servers', 'clients']
search_query = input query from api
search_filter = {type: ['locations', 'servers', 'clients'], value:[id1, id2]}
'''
if not search_filter['type'] or search_filter['type'] == search_type:
return []
table_name = self.get_table_name(search_type, search_filter)
table_config = get_table_config(self.table_configs, None, table_name)
all_results = []
for filter_value in sorted(search_filter['value'], reverse=False):
# we always want this filter value to be the first key
key_prefix = du.get_key_field(filter_value, 0, table_config)
key_prefix += du.BIGTABLE_KEY_DELIM
# filter only the `meta` column family - for speed.
tablefilter = FamilyNameRegexFilter('meta')
all_results += bt.scan_table(table_config,
self.get_pool(),
prefix=key_prefix,
filter=tablefilter,
**kwargs)
filtered_results = self.filter_results(search_type, search_query,
all_results)
return self.prepare_filtered_search_results(filtered_results)
def get_filtered_search_results(self, search_type, search_query,
search_filter, **kwargs):
'''
Filter search. Provides results for searches that are faceted.
search_type = one of ['locations', 'servers', 'clients', 'asn_numbers']
search_query = input query from api
search_filter = {
type: ['locations', 'servers', 'clients', 'asn_numbers'],
value:[id1, id2]
}
'''
if not search_filter['type'] or search_filter['type'] == search_type:
return []
table_name = self.get_table_name(search_type, search_filter)
table_config = get_table_config(self.table_configs, None, table_name)
all_results = []
for filter_value in sorted(search_filter['value'], reverse=False):
# we always want this filter value to be the first key
key_prefix = du.get_key_field(filter_value, 0, table_config)
key_prefix += du.BIGTABLE_KEY_DELIM
# filter only the `meta` column family - for speed.
tablefilter = FamilyNameRegexFilter('meta')
all_results += bt.scan_table(
table_config, self.get_pool(), prefix=key_prefix,
filter=tablefilter, **kwargs)
filtered_results = self.filter_results(search_type, search_query,
all_results)
return self.prepare_filtered_search_results(filtered_results)
def get_raw_test_results(self):
'''
Extract sample raw data.
'''
table_name = 'raw_sample'
table_config = get_table_config(self.table_configs, None, table_name)
results = bt.scan_table(table_config, self.get_pool(), limit=1000)
return {"results": results}
def get_raw_test_results(self):
'''
Extract sample raw data.
'''
table_name = 'raw_sample'
table_config = get_table_config(self.table_configs, None, table_name)
results = bt.scan_table(table_config, self.get_pool(), limit=1000)
return {"results": results}
def get_basic_search_results(self, search_type, search_query):
'''
Provide basic search with no filtering logic.
Basic search is based purely on bigtable prefix scans.
'''
table_name = du.search_table(search_type)
table_config = get_table_config(self.table_configs, None, table_name)
results = bt.scan_table(table_config, self.get_pool(),
prefix=search_query)
return results
def get_basic_search_results(self, search_type, search_query):
'''
Provide basic search with no filtering logic.
Basic search is based purely on bigtable prefix scans.
'''
table_name = du.search_table(search_type)
table_config = get_table_config(self.table_configs, None, table_name)
results = bt.scan_table(table_config,
self.get_pool(),
prefix=search_query)
return results
def get_server_info(self, server_id):
'''
Get info for a client
server_id = id of server.
'''
# we are using a hack from list tables
# so grab the first match from a list table faceted by server ids'
table_name = du.list_table("clients", "servers")
table_config = get_table_config(self.table_configs, None, table_name)
key_fields = du.get_key_fields([server_id], table_config)
return get_bt_results(key_fields, table_config, self.get_pool())
def get_server_info(self, server_id):
'''
Get info for a client
server_id = id of server.
'''
# we are using a hack from list tables
# so grab the first match from a list table faceted by server ids'
table_name = du.list_table("clients", "servers")
table_config = get_table_config(self.table_configs, None, table_name)
key_fields = du.get_key_fields([server_id], table_config)
return get_bt_results(key_fields, table_config, self.get_pool())
def get_client_metrics(self, client_id, timebin, starttime, endtime):
'''
Get data for client location at a specific
timebin between start and stop times.
client_id = id string of client.
timebin = time aggregation key.
starttime = start time for metric query.
endtime = end time for metric query.
'''
table_config = get_table_config(self.table_configs, timebin, TABLE_KEYS["clients"])
key_fields = du.get_key_fields([client_id], table_config)
formatted = bt.get_time_metric_results(key_fields, self.get_pool(), timebin, starttime, endtime, table_config, "clients")
return formatted
def get_location_client_isp_info(self, location_id, client_id):
'''
Get static information about
location_id = id string of location.
client_id = id string of client.
'''
config_id = du.list_table('clients', 'locations')
table_config = get_table_config(self.table_configs, None, config_id)
key_fields = du.get_key_fields([location_id, client_id], table_config)
row_key = du.BIGTABLE_KEY_DELIM.join(key_fields)
results = []
results = bt.get_row(table_config, self.get_pool(), row_key)
return results
def get_location_info(self, location_id):
'''
Get info about specific location
location_id = id string of location.
'''
table_config = get_table_config(self.table_configs, None,
du.list_table('locations'))
# add empty field to get child location in there
location_key_fields = du.get_key_fields(["info", location_id],
table_config)
row_key = du.BIGTABLE_KEY_DELIM.join(location_key_fields)
row = ""
row = bt.get_row(table_config, self.get_pool(), row_key)
return row
def get_client_metrics(self, client_id, timebin, starttime, endtime):
'''
Get data for client location at a specific
timebin between start and stop times.
client_id = id string of client.
timebin = time aggregation key.
starttime = start time for metric query.
endtime = end time for metric query.
'''
table_config = get_table_config(self.table_configs, timebin,
TABLE_KEYS["clients"])
key_fields = du.get_key_fields([client_id], table_config)
formatted = bt.get_time_metric_results(
key_fields, self.get_pool(), timebin, starttime, endtime,
table_config, "clients")
return formatted
def get_server_metrics(self, server_id, timebin, starttime, endtime):
'''
Get data for specific location at a specific
frequency between start and stop times.
server_id = id of server.
timebin = time aggregation key.
starttime = start time for metric query.
endtime = end time for metric query.
'''
table_config = get_table_config(self.table_configs, timebin,
TABLE_KEYS["servers"])
location_key_fields = du.get_key_fields([server_id], table_config)
formatted = bt.get_time_metric_results(
location_key_fields, self.get_pool(), timebin, starttime, endtime,
table_config, "servers")
return formatted
def get_client_server_metrics(self, client_id, server_id, timebin, starttime, endtime):
'''
Get data for a specific client + server at a
timebin between start and stop times.
client_id = id string of client.
server_id = id of server.
timebin = time aggregation key.
starttime = start time for metric query.
endtime = end time for metric query.
'''
agg_name = TABLE_KEYS["servers"] + '_' + TABLE_KEYS["clients"]
table_config = get_table_config(self.table_configs, timebin, agg_name)
key_fields = du.get_key_fields([server_id, client_id], table_config)
formatted = bt.get_time_metric_results(key_fields, self.get_pool(), timebin, starttime, endtime, table_config, "clients")
return formatted
def get_list_data(self, entity_id, entity_type, query_type, include_data):
'''
Helper method to get out data from a list table.
entity_id = id of entity to look for
entity_type = [locations, clients, servers]
query_type = [locations, clients, servers] - what we are faceting on
include_data = boolean to indicate if data should be queried and returned.
'''
config_id = du.list_table(query_type, entity_type)
metric_name = "_".join([entity_type, query_type])
table_config = get_table_config(self.table_configs, None, config_id)
key_fields = du.get_key_fields([entity_id], table_config)
results = bt.get_list_table_results(key_fields, self.get_pool(), include_data, table_config, metric_name)
return {"results": results}
def get_server_metrics(self, server_id, timebin, starttime, endtime):
'''
Get data for specific location at a specific
frequency between start and stop times.
server_id = id of server.
timebin = time aggregation key.
starttime = start time for metric query.
endtime = end time for metric query.
'''
table_config = get_table_config(self.table_configs, timebin,
TABLE_KEYS["servers"])
location_key_fields = du.get_key_fields([server_id], table_config)
formatted = bt.get_time_metric_results(location_key_fields,
self.get_pool(), timebin,
starttime, endtime,
table_config, "servers")
return formatted
def get_list_data(self, entity_id, entity_type, query_type, include_data):
'''
Helper method to get out data from a list table.
entity_id = id of entity to look for
entity_type = [locations, clients, servers]
query_type = [locations, clients, servers] - what we are faceting on
include_data = boolean indicates if data should be queried and returned.
'''
config_id = du.list_table(query_type, entity_type)
metric_name = "_".join([entity_type, query_type])
table_config = get_table_config(self.table_configs, None, config_id)
key_fields = du.get_key_fields([entity_id], table_config)
results = bt.get_list_table_results(
key_fields, self.get_pool(), include_data, table_config,
metric_name)
return {"results": results}
def get_location_children(self, location_id, type_filter=None):
'''
Return information about children regions of a location
location_id = id string of location.
type_filter = optionally restrict results to a location type.
'''
table_config = get_table_config(self.table_configs,
None,
du.list_table('locations'))
location_key_fields = du.get_location_key_fields(location_id,
table_config)
location_key_field = du.BIGTABLE_KEY_DELIM.join(location_key_fields)
results = []
results = bt.scan_table(table_config, self.get_pool(),
prefix=location_key_field)
if type_filter:
results = [r for r in results if r['meta']['type'] == type_filter]
return {"results": results}
def get_client_server_metrics(self, client_id, server_id, timebin,
starttime, endtime):
'''
Get data for a specific client + server at a
timebin between start and stop times.
client_id = id string of client.
server_id = id of server.
timebin = time aggregation key.
starttime = start time for metric query.
endtime = end time for metric query.
'''
agg_name = TABLE_KEYS["servers"] + '_' + TABLE_KEYS["clients"]
table_config = get_table_config(self.table_configs, timebin, agg_name)
key_fields = du.get_key_fields([server_id, client_id], table_config)
formatted = bt.get_time_metric_results(
key_fields, self.get_pool(), timebin, starttime, endtime,
table_config, "clients")
return formatted
def get_client_info(self, client_id):
'''
Get info for a client.
client_id = id string of client.
'''
# we are using a hack from list tables
# so grab the first match from a list table faceted by client ids'
table_name = du.list_table("servers", "clients")
table_config = get_table_config(self.table_configs, None, table_name)
key_fields = du.get_key_fields([client_id], table_config)
prefix_key = du.BIGTABLE_KEY_DELIM.join(key_fields)
results = bt.scan_table(table_config, self.get_pool(), prefix=prefix_key, limit=1, filter=FamilyNameRegexFilter('meta'))
result = {}
if len(results) > 0:
result = results[0]
return result
# -*- coding: utf-8 -*-
'''
Test whether BigTable connection works
'''
import os
from mlab_api.data.bigtable_utils import init_pool, scan_table
from mlab_api.data.table_config import get_table_config, read_table_configs
from google.oauth2 import service_account
CREDENTIALS = service_account.Credentials.from_service_account_file(
os.path.abspath(os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")))
# credentials = GoogleCredentials.get_application_default()
print 'Using credentials' + CREDENTIALS.service_account_email
print 'Environment variables'
print os.environ
print 'Connect to db'
POOL = init_pool()
TABLE_CONFIGS = read_table_configs("bigtable_configs")
TABLE_NAME = 'client_loc_search'
TABLE_CONFIG = get_table_config(TABLE_CONFIGS, None, TABLE_NAME)
print 'Query db'
RESULTS = scan_table(TABLE_CONFIG, POOL, limit=3)
print 'Results'
print RESULTS
# -*- coding: utf-8 -*-
'''
Test whether BigTable connection works
'''
import os
from mlab_api.data.bigtable_utils import init_pool, scan_table
from mlab_api.data.table_config import get_table_config, read_table_configs
from google.oauth2 import service_account
CREDENTIALS = service_account.Credentials.from_service_account_file(
os.path.abspath(os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"))
)
# credentials = GoogleCredentials.get_application_default()
print 'Using credentials' + CREDENTIALS.service_account_email
print 'Environment variables'
print os.environ
print 'Connect to db'
POOL = init_pool()
TABLE_CONFIGS = read_table_configs("bigtable_configs")
TABLE_NAME = 'client_loc_search'
TABLE_CONFIG = get_table_config(TABLE_CONFIGS, None, TABLE_NAME)
print 'Query db'
RESULTS = scan_table(TABLE_CONFIG, POOL, limit=3)
print 'Results'
print RESULTS
