def __init__(self, memcached_client, interaction_pool):
self._log = logging.getLogger("InteractionPoolAuthenticator")
self._interaction_pool = interaction_pool
self._collection_lookup = CollectionLookup(memcached_client,
interaction_pool)
self._customer_lookup = CustomerIdLookup(memcached_client,
interaction_pool)
self._customer_key_lookup = CustomerKeyLookup(memcached_client,
interaction_pool)
def init(self):
#import logging
#import traceback
log = logging.getLogger("init")
log.info("init start")
self.central_conn_pool = DBInteractionPool(
get_central_database_dsn(),
pool_size=CENTRAL_DB_POOL_SIZE,
do_log=True)
self.redis = StrictRedis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB)
self.memcached_client = memcache.Client(MEMCACHED_NODES)
self.collection_lookup = CollectionLookup(self.memcached_client,
self.central_conn_pool)
log.info("init complete")
self.init_complete.set(True)
class InteractionPoolAuthenticator(object):
def __init__(self, memcached_client, interaction_pool):
self._log = logging.getLogger("InteractionPoolAuthenticator")
self._interaction_pool = interaction_pool
self._collection_lookup = CollectionLookup(memcached_client, interaction_pool)
self._customer_lookup = CustomerIdLookup(memcached_client, interaction_pool)
self._customer_key_lookup = CustomerKeyLookup(memcached_client, interaction_pool)
def authenticate(self, collection_name, access_type, req):
"""
establish that this is a valid user and a valid collection
return collection_entry if valid
raise AccessUnauthorized(error_message) if invalid
"""
collection_row = self._collection_lookup.get(collection_name.lower())
if collection_row is None:
error_message = "unknown collection {0}".format(collection_name)
self._log.error(error_message)
raise AccessUnauthorized(error_message)
if access_type is not None:
access_result = check_access_control(access_type, req, collection_row["access_control"])
if access_result == access_allowed:
return collection_row
if access_result == access_forbidden:
raise AccessForbidden()
assert access_result == access_requires_password_authentication
customer_row = self._customer_lookup.get(collection_row["customer_id"])
if customer_row is None:
error_message = "unknown customer {0}".format(collection_name)
self._log.error(error_message)
raise AccessUnauthorized(error_message)
try:
auth_type, auth_string = req.authorization
except Exception, instance:
error_message = "invalid req.authorization {0} {1}".format(instance, req.authorization)
self._log.error(error_message)
raise AccessUnauthorized(error_message)
if auth_type != "NIMBUS.IO":
error_message = "unknown auth_type %r" % (auth_type,)
self._log.error(error_message)
raise AccessUnauthorized(error_message)
try:
key_id, signature = auth_string.split(":", 1)
except Exception, instance:
error_message = "invalid auth_string {0} {1}".format(instance, auth_string)
self._log.error(error_message)
raise AccessUnauthorized(error_message)
def init(self):
#import logging
#import traceback
log = logging.getLogger("init")
log.info("init start")
self.central_conn_pool = DBInteractionPool(
get_central_database_dsn(),
pool_size = CENTRAL_DB_POOL_SIZE,
do_log = True )
self.redis = StrictRedis(host = REDIS_HOST,
port = REDIS_PORT,
db = REDIS_DB)
self.memcached_client = memcache.Client(MEMCACHED_NODES)
self.collection_lookup = CollectionLookup(self.memcached_client,
self.central_conn_pool)
log.info("init complete")
self.init_complete.set(True)
class Router(object):
"""
Router object for assisting the proxy function (below.)
Holds database connection, state for caching, etc.
"""
def __init__(self):
self.init_complete = AsyncResult()
self.central_conn_pool = None
self.redis = None
self.service_domain = NIMBUS_IO_SERVICE_DOMAIN
self.read_dest_port = NIMBUSIO_WEB_PUBLIC_READER_PORT
self.write_dest_port = NIMBUSIO_WEB_WRITER_PORT
self.known_clusters = dict()
self.management_api_request_dest_hosts = \
deque(NIMBUSIO_MANAGEMENT_API_REQUEST_DEST.strip().split())
self.memcached_client = None
self.collection_lookup = None
self.request_counter = 0
self.path_hash_base = hmac.new(
key = NIMBUSIO_URL_DEST_HASH_KEY,
digestmod=sha256)
# start the round robin dispatcher at a random number, so all the
# workers don't start on the same point.
self.round_robin_dispatch_counter = random.choice(range(10))
def init(self):
#import logging
#import traceback
log = logging.getLogger("init")
log.info("init start")
self.central_conn_pool = DBInteractionPool(
get_central_database_dsn(),
pool_size = CENTRAL_DB_POOL_SIZE,
do_log = True )
self.redis = StrictRedis(host = REDIS_HOST,
port = REDIS_PORT,
db = REDIS_DB)
self.memcached_client = memcache.Client(MEMCACHED_NODES)
self.collection_lookup = CollectionLookup(self.memcached_client,
self.central_conn_pool)
log.info("init complete")
self.init_complete.set(True)
def _parse_collection(self, hostname):
"return the Nimbus.io collection name from host name"
offset = -1 * ( len(self.service_domain) + 1 )
return hostname[:offset].lower()
def _hosts_for_collection(self, collection):
"return a list of hosts for this collection"
cluster_id = self._cluster_for_collection(collection)
if cluster_id is None:
return None
cluster_info = self._cluster_info(cluster_id)
return cluster_info['hosts']
def _cluster_for_collection(self, collection, _retries=0):
"return cluster ID for collection"
collection_row = self.collection_lookup.get(collection)
if not collection_row:
return None
return collection_row['cluster_id']
def _db_cluster_info(self, cluster_id):
async_result = self.central_conn_pool.run("""
select name, hostname, node_number_in_cluster
from nimbusio_central.node
where cluster_id=%s
order by node_number_in_cluster""",
[cluster_id, ])
rows = async_result.get()
info = dict(rows = rows,
hosts = [r['hostname'] for r in rows])
return info
def _cluster_info(self, cluster_id):
"return info about a cluster and its hosts"
if cluster_id in self.known_clusters:
return self.known_clusters[cluster_id]
info = self._db_cluster_info(cluster_id)
self.known_clusters[cluster_id] = info
return info
def check_availability(self, hosts, dest_port, _resolve_cache=dict()):
"return set of hosts we think are available"
log = logging.getLogger("check_availability")
available = set()
if not hosts:
return available
addresses = []
for host in hosts:
if not host in _resolve_cache:
_resolve_cache[host] = socket.gethostbyname(host)
addresses.append(_resolve_cache[host])
redis_keys = [ REDIS_WEB_MONITOR_HASHKEY_FORMAT % (a, dest_port, )
for a in addresses ]
try:
redis_values = self.redis.hmget(REDIS_WEB_MONITOR_HASH_NAME,
redis_keys)
except RedisError as err:
log.warn("redis error querying availability for %s: %s, %r"
% ( REDIS_WEB_MONITOR_HASH_NAME, err, redis_keys, ))
# just consider everything available. it's the best we can do.
available.update(hosts)
return available
unknown = []
for idx, val in enumerate(redis_values):
if val is None:
unknown.append((hosts[idx], redis_keys[idx], ))
continue
try:
status = json.loads(val)
except Exception, err:
log.warn("cannot decode %s %s %s %r" % (
REDIS_WEB_MONITOR_HASH_NAME, hosts[idx],
redis_keys[idx], val, ))
else:
if status["reachable"]:
available.add(hosts[idx])
if unknown:
log.warn("no availability info in redis for hkeys: %s %r" %
( REDIS_WEB_MONITOR_HASH_NAME, unknown, ))
# if every host is unknown, just consider them all available
if len(unknown) == len(hosts):
available.update(hosts)
return available
class Router(object):
"""
Router object for assisting the proxy function (below.)
Holds database connection, state for caching, etc.
"""
def __init__(self):
self.init_complete = AsyncResult()
self.central_conn_pool = None
self.redis = None
self.service_domain = NIMBUS_IO_SERVICE_DOMAIN
self.read_dest_port = NIMBUSIO_WEB_PUBLIC_READER_PORT
self.write_dest_port = NIMBUSIO_WEB_WRITER_PORT
self.known_clusters = dict()
self.management_api_request_dest_hosts = \
deque(NIMBUSIO_MANAGEMENT_API_REQUEST_DEST.strip().split())
self.memcached_client = None
self.collection_lookup = None
self.request_counter = 0
self.path_hash_base = hmac.new(key=NIMBUSIO_URL_DEST_HASH_KEY,
digestmod=sha256)
# start the round robin dispatcher at a random number, so all the
# workers don't start on the same point.
self.round_robin_dispatch_counter = random.choice(range(10))
def init(self):
#import logging
#import traceback
log = logging.getLogger("init")
log.info("init start")
self.central_conn_pool = DBInteractionPool(
get_central_database_dsn(),
pool_size=CENTRAL_DB_POOL_SIZE,
do_log=True)
self.redis = StrictRedis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB)
self.memcached_client = memcache.Client(MEMCACHED_NODES)
self.collection_lookup = CollectionLookup(self.memcached_client,
self.central_conn_pool)
log.info("init complete")
self.init_complete.set(True)
def _parse_collection(self, hostname):
"return the Nimbus.io collection name from host name"
offset = -1 * (len(self.service_domain) + 1)
return hostname[:offset].lower()
def _hosts_for_collection(self, collection):
"return a list of hosts for this collection"
cluster_id = self._cluster_for_collection(collection)
if cluster_id is None:
return None
cluster_info = self._cluster_info(cluster_id)
return cluster_info['hosts']
def _cluster_for_collection(self, collection, _retries=0):
"return cluster ID for collection"
collection_row = self.collection_lookup.get(collection)
if not collection_row:
return None
return collection_row['cluster_id']
def _db_cluster_info(self, cluster_id):
async_result = self.central_conn_pool.run(
"""
select name, hostname, node_number_in_cluster
from nimbusio_central.node
where cluster_id=%s
order by node_number_in_cluster""", [
cluster_id,
])
rows = async_result.get()
info = dict(rows=rows, hosts=[r['hostname'] for r in rows])
return info
def _cluster_info(self, cluster_id):
"return info about a cluster and its hosts"
if cluster_id in self.known_clusters:
return self.known_clusters[cluster_id]
info = self._db_cluster_info(cluster_id)
self.known_clusters[cluster_id] = info
return info
def check_availability(self, hosts, dest_port, _resolve_cache=dict()):
"return set of hosts we think are available"
log = logging.getLogger("check_availability")
available = set()
if not hosts:
return available
addresses = []
for host in hosts:
if not host in _resolve_cache:
_resolve_cache[host] = socket.gethostbyname(host)
addresses.append(_resolve_cache[host])
redis_keys = [
REDIS_WEB_MONITOR_HASHKEY_FORMAT % (
a,
dest_port,
) for a in addresses
]
try:
redis_values = self.redis.hmget(REDIS_WEB_MONITOR_HASH_NAME,
redis_keys)
except RedisError as err:
log.warn("redis error querying availability for %s: %s, %r" % (
REDIS_WEB_MONITOR_HASH_NAME,
err,
redis_keys,
))
# just consider everything available. it's the best we can do.
available.update(hosts)
return available
unknown = []
for idx, val in enumerate(redis_values):
if val is None:
unknown.append((
hosts[idx],
redis_keys[idx],
))
continue
try:
status = json.loads(val)
except Exception, err:
log.warn("cannot decode %s %s %s %r" % (
REDIS_WEB_MONITOR_HASH_NAME,
hosts[idx],
redis_keys[idx],
val,
))
else:
if status["reachable"]:
available.add(hosts[idx])
if unknown:
log.warn("no availability info in redis for hkeys: %s %r" % (
REDIS_WEB_MONITOR_HASH_NAME,
unknown,
))
# if every host is unknown, just consider them all available
if len(unknown) == len(hosts):
available.update(hosts)
return available
def __init__(self, memcached_client, interaction_pool):
self._log = logging.getLogger("InteractionPoolAuthenticator")
self._interaction_pool = interaction_pool
self._collection_lookup = CollectionLookup(memcached_client, interaction_pool)
self._customer_lookup = CustomerIdLookup(memcached_client, interaction_pool)
self._customer_key_lookup = CustomerKeyLookup(memcached_client, interaction_pool)
class InteractionPoolAuthenticator(object):
def __init__(self, memcached_client, interaction_pool):
self._log = logging.getLogger("InteractionPoolAuthenticator")
self._interaction_pool = interaction_pool
self._collection_lookup = CollectionLookup(memcached_client,
interaction_pool)
self._customer_lookup = CustomerIdLookup(memcached_client,
interaction_pool)
self._customer_key_lookup = CustomerKeyLookup(memcached_client,
interaction_pool)
def authenticate(self, collection_name, access_type, req):
"""
establish that this is a valid user and a valid collection
return collection_entry if valid
raise AccessUnauthorized(error_message) if invalid
"""
collection_row = self._collection_lookup.get(collection_name.lower())
if collection_row is None:
error_message = "unknown collection {0}".format(collection_name)
self._log.error(error_message)
raise AccessUnauthorized(error_message)
if access_type is not None:
access_result = \
check_access_control(access_type,
req,
collection_row["access_control"])
if access_result == access_allowed:
return collection_row
if access_result == access_forbidden:
raise AccessForbidden()
assert access_result == access_requires_password_authentication
customer_row = self._customer_lookup.get(collection_row["customer_id"])
if customer_row is None:
error_message = "unknown customer {0}".format(collection_name)
self._log.error(error_message)
raise AccessUnauthorized(error_message)
try:
auth_type, auth_string = req.authorization
except Exception, instance:
error_message = "invalid req.authorization {0} {1}".format(
instance,
req.authorization)
self._log.error(error_message)
raise AccessUnauthorized(error_message)
if auth_type != 'NIMBUS.IO':
error_message = "unknown auth_type %r" % (auth_type, )
self._log.error(error_message)
raise AccessUnauthorized(error_message)
try:
key_id, signature = auth_string.split(':', 1)
except Exception, instance:
error_message = "invalid auth_string {0} {1}".format(
instance, auth_string)
self._log.error(error_message)
raise AccessUnauthorized(error_message)
