def get_filtered_search_results(self, search_type, search_query,
search_filter, **kwargs):
'''
Filter search. Provides results for searches that are faceted.
search_type = one of ['locations', 'servers', 'clients']
search_query = input query from api
search_filter = {
type: ['locations', 'servers', 'clients'],
value:[id1, id2]
}
'''
if not search_filter['type'] or search_filter['type'] == search_type:
return []
table_name = self.get_table_name(search_type, search_filter)
table_config = get_table_config(self.table_configs, None, table_name)
all_results = []
for filter_value in sorted(search_filter['value'], reverse=False):
# we always want this filter value to be the first key
key_prefix = du.get_key_field(filter_value, 0, table_config)
key_prefix += du.BIGTABLE_KEY_DELIM
# filter only the `meta` column family - for speed.
tablefilter = FamilyNameRegexFilter('meta')
all_results += bt.scan_table(table_config,
self.get_pool(),
prefix=key_prefix,
filter=tablefilter,
**kwargs)
filtered_results = self.filter_results(search_type, search_query,
all_results)
return self.prepare_filtered_search_results(filtered_results)
def get_server_info(self, server_id):
'''
Get info for a client
server_id = id of server.
'''
# we are using a hack from list tables
# so grab the first match from a list table faceted by server ids'
table_name = du.list_table("clients", "servers")
table_config = get_table_config(self.table_configs, None, table_name)
key_fields = du.get_key_fields([server_id], table_config)
prefix_key = du.BIGTABLE_KEY_DELIM.join(key_fields)
results = bt.scan_table(table_config,
self.get_pool(),
prefix=prefix_key,
limit=1,
filter=FamilyNameRegexFilter('meta'))
result = {}
if len(results) > 0:
result = results[0]
return result
def get_filtered_search_results(self, search_type, search_query,
search_filter, **kwargs):
'''
Filter search. Provides results for searches that are faceted.
search_type = one of ['locations', 'servers', 'clients', 'asn_numbers']
search_query = input query from api
search_filter = {
type: ['locations', 'servers', 'clients', 'asn_numbers'],
value:[id1, id2]
}
'''
if not search_filter['type'] or search_filter['type'] == search_type:
return []
table_name = self.get_table_name(search_type, search_filter)
table_config = get_table_config(self.table_configs, None, table_name)
all_results = []
for filter_value in sorted(search_filter['value'], reverse=False):
# we always want this filter value to be the first key
key_prefix = du.get_key_field(filter_value, 0, table_config)
key_prefix += du.BIGTABLE_KEY_DELIM
# filter only the `meta` column family - for speed.
tablefilter = FamilyNameRegexFilter('meta')
all_results += bt.scan_table(
table_config, self.get_pool(), prefix=key_prefix,
filter=tablefilter, **kwargs)
filtered_results = self.filter_results(search_type, search_query,
all_results)
return self.prepare_filtered_search_results(filtered_results)
def get_raw_test_results(self):
'''
Extract sample raw data.
'''
table_name = 'raw_sample'
table_config = get_table_config(self.table_configs, None, table_name)
results = bt.scan_table(table_config, self.get_pool(), limit=1000)
return {"results": results}
def get_raw_test_results(self):
'''
Extract sample raw data.
'''
table_name = 'raw_sample'
table_config = get_table_config(self.table_configs, None, table_name)
results = bt.scan_table(table_config, self.get_pool(), limit=1000)
return {"results": results}
def get_basic_search_results(self, search_type, search_query):
'''
Provide basic search with no filtering logic.
Basic search is based purely on bigtable prefix scans.
'''
table_name = du.search_table(search_type)
table_config = get_table_config(self.table_configs, None, table_name)
results = bt.scan_table(table_config, self.get_pool(),
prefix=search_query)
return results
def get_basic_search_results(self, search_type, search_query):
'''
Provide basic search with no filtering logic.
Basic search is based purely on bigtable prefix scans.
'''
table_name = du.search_table(search_type)
table_config = get_table_config(self.table_configs, None, table_name)
results = bt.scan_table(table_config,
self.get_pool(),
prefix=search_query)
return results
def get_location_children(self, location_id, type_filter=None):
'''
Return information about children regions of a location
location_id = id string of location.
type_filter = optionally restrict results to a location type.
'''
table_config = get_table_config(self.table_configs,
None,
du.list_table('locations'))
location_key_fields = du.get_location_key_fields(location_id,
table_config)
location_key_field = du.BIGTABLE_KEY_DELIM.join(location_key_fields)
results = []
results = bt.scan_table(table_config, self.get_pool(),
prefix=location_key_field)
if type_filter:
results = [r for r in results if r['meta']['type'] == type_filter]
return {"results": results}
def get_bt_results(key_fields, table_config, pool):
'''
Scans for and returns big table results based on key fields
and a table config
key_fields = key fields for the search
table_config = the table to scan
pool = a big table connection pool
'''
prefix_key = du.BIGTABLE_KEY_DELIM.join(key_fields)
results = bt.scan_table(table_config,
pool,
prefix=prefix_key,
limit=1,
filter=FamilyNameRegexFilter('meta'))
result = {}
if results:
result = results[0]
return result
# -*- coding: utf-8 -*-
'''
Test whether BigTable connection works
'''
import os
from mlab_api.data.bigtable_utils import init_pool, scan_table
from mlab_api.data.table_config import get_table_config, read_table_configs
from google.oauth2 import service_account
CREDENTIALS = service_account.Credentials.from_service_account_file(
os.path.abspath(os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")))
# credentials = GoogleCredentials.get_application_default()
print 'Using credentials' + CREDENTIALS.service_account_email
print 'Environment variables'
print os.environ
print 'Connect to db'
POOL = init_pool()
TABLE_CONFIGS = read_table_configs("bigtable_configs")
TABLE_NAME = 'client_loc_search'
TABLE_CONFIG = get_table_config(TABLE_CONFIGS, None, TABLE_NAME)
print 'Query db'
RESULTS = scan_table(TABLE_CONFIG, POOL, limit=3)
print 'Results'
print RESULTS
# -*- coding: utf-8 -*-
'''
Test whether BigTable connection works
'''
import os
from mlab_api.data.bigtable_utils import init_pool, scan_table
from mlab_api.data.table_config import get_table_config, read_table_configs
from google.oauth2 import service_account
CREDENTIALS = service_account.Credentials.from_service_account_file(
os.path.abspath(os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"))
)
# credentials = GoogleCredentials.get_application_default()
print 'Using credentials' + CREDENTIALS.service_account_email
print 'Environment variables'
print os.environ
print 'Connect to db'
POOL = init_pool()
TABLE_CONFIGS = read_table_configs("bigtable_configs")
TABLE_NAME = 'client_loc_search'
TABLE_CONFIG = get_table_config(TABLE_CONFIGS, None, TABLE_NAME)
print 'Query db'
RESULTS = scan_table(TABLE_CONFIG, POOL, limit=3)
print 'Results'
print RESULTS
