class InteractionPoolAuthenticator(object): def __init__(self, memcached_client, interaction_pool): self._log = logging.getLogger("InteractionPoolAuthenticator") self._interaction_pool = interaction_pool self._collection_lookup = CollectionLookup(memcached_client, interaction_pool) self._customer_lookup = CustomerIdLookup(memcached_client, interaction_pool) self._customer_key_lookup = CustomerKeyLookup(memcached_client, interaction_pool) def authenticate(self, collection_name, access_type, req): """ establish that this is a valid user and a valid collection return collection_entry if valid raise AccessUnauthorized(error_message) if invalid """ collection_row = self._collection_lookup.get(collection_name.lower()) if collection_row is None: error_message = "unknown collection {0}".format(collection_name) self._log.error(error_message) raise AccessUnauthorized(error_message) if access_type is not None: access_result = check_access_control(access_type, req, collection_row["access_control"]) if access_result == access_allowed: return collection_row if access_result == access_forbidden: raise AccessForbidden() assert access_result == access_requires_password_authentication customer_row = self._customer_lookup.get(collection_row["customer_id"]) if customer_row is None: error_message = "unknown customer {0}".format(collection_name) self._log.error(error_message) raise AccessUnauthorized(error_message) try: auth_type, auth_string = req.authorization except Exception, instance: error_message = "invalid req.authorization {0} {1}".format(instance, req.authorization) self._log.error(error_message) raise AccessUnauthorized(error_message) if auth_type != "NIMBUS.IO": error_message = "unknown auth_type %r" % (auth_type,) self._log.error(error_message) raise AccessUnauthorized(error_message) try: key_id, signature = auth_string.split(":", 1) except Exception, instance: error_message = "invalid auth_string {0} {1}".format(instance, auth_string) self._log.error(error_message) raise AccessUnauthorized(error_message)
class Router(object): """ Router object for assisting the proxy function (below.) Holds database connection, state for caching, etc. """ def __init__(self): self.init_complete = AsyncResult() self.central_conn_pool = None self.redis = None self.service_domain = NIMBUS_IO_SERVICE_DOMAIN self.read_dest_port = NIMBUSIO_WEB_PUBLIC_READER_PORT self.write_dest_port = NIMBUSIO_WEB_WRITER_PORT self.known_clusters = dict() self.management_api_request_dest_hosts = \ deque(NIMBUSIO_MANAGEMENT_API_REQUEST_DEST.strip().split()) self.memcached_client = None self.collection_lookup = None self.request_counter = 0 self.path_hash_base = hmac.new( key = NIMBUSIO_URL_DEST_HASH_KEY, digestmod=sha256) # start the round robin dispatcher at a random number, so all the # workers don't start on the same point. self.round_robin_dispatch_counter = random.choice(range(10)) def init(self): #import logging #import traceback log = logging.getLogger("init") log.info("init start") self.central_conn_pool = DBInteractionPool( get_central_database_dsn(), pool_size = CENTRAL_DB_POOL_SIZE, do_log = True ) self.redis = StrictRedis(host = REDIS_HOST, port = REDIS_PORT, db = REDIS_DB) self.memcached_client = memcache.Client(MEMCACHED_NODES) self.collection_lookup = CollectionLookup(self.memcached_client, self.central_conn_pool) log.info("init complete") self.init_complete.set(True) def _parse_collection(self, hostname): "return the Nimbus.io collection name from host name" offset = -1 * ( len(self.service_domain) + 1 ) return hostname[:offset].lower() def _hosts_for_collection(self, collection): "return a list of hosts for this collection" cluster_id = self._cluster_for_collection(collection) if cluster_id is None: return None cluster_info = self._cluster_info(cluster_id) return cluster_info['hosts'] def _cluster_for_collection(self, collection, _retries=0): "return cluster ID for collection" collection_row = self.collection_lookup.get(collection) if not collection_row: return None return collection_row['cluster_id'] def _db_cluster_info(self, cluster_id): async_result = self.central_conn_pool.run(""" select name, hostname, node_number_in_cluster from nimbusio_central.node where cluster_id=%s order by node_number_in_cluster""", [cluster_id, ]) rows = async_result.get() info = dict(rows = rows, hosts = [r['hostname'] for r in rows]) return info def _cluster_info(self, cluster_id): "return info about a cluster and its hosts" if cluster_id in self.known_clusters: return self.known_clusters[cluster_id] info = self._db_cluster_info(cluster_id) self.known_clusters[cluster_id] = info return info def check_availability(self, hosts, dest_port, _resolve_cache=dict()): "return set of hosts we think are available" log = logging.getLogger("check_availability") available = set() if not hosts: return available addresses = [] for host in hosts: if not host in _resolve_cache: _resolve_cache[host] = socket.gethostbyname(host) addresses.append(_resolve_cache[host]) redis_keys = [ REDIS_WEB_MONITOR_HASHKEY_FORMAT % (a, dest_port, ) for a in addresses ] try: redis_values = self.redis.hmget(REDIS_WEB_MONITOR_HASH_NAME, redis_keys) except RedisError as err: log.warn("redis error querying availability for %s: %s, %r" % ( REDIS_WEB_MONITOR_HASH_NAME, err, redis_keys, )) # just consider everything available. it's the best we can do. available.update(hosts) return available unknown = [] for idx, val in enumerate(redis_values): if val is None: unknown.append((hosts[idx], redis_keys[idx], )) continue try: status = json.loads(val) except Exception, err: log.warn("cannot decode %s %s %s %r" % ( REDIS_WEB_MONITOR_HASH_NAME, hosts[idx], redis_keys[idx], val, )) else: if status["reachable"]: available.add(hosts[idx]) if unknown: log.warn("no availability info in redis for hkeys: %s %r" % ( REDIS_WEB_MONITOR_HASH_NAME, unknown, )) # if every host is unknown, just consider them all available if len(unknown) == len(hosts): available.update(hosts) return available
class Router(object): """ Router object for assisting the proxy function (below.) Holds database connection, state for caching, etc. """ def __init__(self): self.init_complete = AsyncResult() self.central_conn_pool = None self.redis = None self.service_domain = NIMBUS_IO_SERVICE_DOMAIN self.read_dest_port = NIMBUSIO_WEB_PUBLIC_READER_PORT self.write_dest_port = NIMBUSIO_WEB_WRITER_PORT self.known_clusters = dict() self.management_api_request_dest_hosts = \ deque(NIMBUSIO_MANAGEMENT_API_REQUEST_DEST.strip().split()) self.memcached_client = None self.collection_lookup = None self.request_counter = 0 self.path_hash_base = hmac.new(key=NIMBUSIO_URL_DEST_HASH_KEY, digestmod=sha256) # start the round robin dispatcher at a random number, so all the # workers don't start on the same point. self.round_robin_dispatch_counter = random.choice(range(10)) def init(self): #import logging #import traceback log = logging.getLogger("init") log.info("init start") self.central_conn_pool = DBInteractionPool( get_central_database_dsn(), pool_size=CENTRAL_DB_POOL_SIZE, do_log=True) self.redis = StrictRedis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB) self.memcached_client = memcache.Client(MEMCACHED_NODES) self.collection_lookup = CollectionLookup(self.memcached_client, self.central_conn_pool) log.info("init complete") self.init_complete.set(True) def _parse_collection(self, hostname): "return the Nimbus.io collection name from host name" offset = -1 * (len(self.service_domain) + 1) return hostname[:offset].lower() def _hosts_for_collection(self, collection): "return a list of hosts for this collection" cluster_id = self._cluster_for_collection(collection) if cluster_id is None: return None cluster_info = self._cluster_info(cluster_id) return cluster_info['hosts'] def _cluster_for_collection(self, collection, _retries=0): "return cluster ID for collection" collection_row = self.collection_lookup.get(collection) if not collection_row: return None return collection_row['cluster_id'] def _db_cluster_info(self, cluster_id): async_result = self.central_conn_pool.run( """ select name, hostname, node_number_in_cluster from nimbusio_central.node where cluster_id=%s order by node_number_in_cluster""", [ cluster_id, ]) rows = async_result.get() info = dict(rows=rows, hosts=[r['hostname'] for r in rows]) return info def _cluster_info(self, cluster_id): "return info about a cluster and its hosts" if cluster_id in self.known_clusters: return self.known_clusters[cluster_id] info = self._db_cluster_info(cluster_id) self.known_clusters[cluster_id] = info return info def check_availability(self, hosts, dest_port, _resolve_cache=dict()): "return set of hosts we think are available" log = logging.getLogger("check_availability") available = set() if not hosts: return available addresses = [] for host in hosts: if not host in _resolve_cache: _resolve_cache[host] = socket.gethostbyname(host) addresses.append(_resolve_cache[host]) redis_keys = [ REDIS_WEB_MONITOR_HASHKEY_FORMAT % ( a, dest_port, ) for a in addresses ] try: redis_values = self.redis.hmget(REDIS_WEB_MONITOR_HASH_NAME, redis_keys) except RedisError as err: log.warn("redis error querying availability for %s: %s, %r" % ( REDIS_WEB_MONITOR_HASH_NAME, err, redis_keys, )) # just consider everything available. it's the best we can do. available.update(hosts) return available unknown = [] for idx, val in enumerate(redis_values): if val is None: unknown.append(( hosts[idx], redis_keys[idx], )) continue try: status = json.loads(val) except Exception, err: log.warn("cannot decode %s %s %s %r" % ( REDIS_WEB_MONITOR_HASH_NAME, hosts[idx], redis_keys[idx], val, )) else: if status["reachable"]: available.add(hosts[idx]) if unknown: log.warn("no availability info in redis for hkeys: %s %r" % ( REDIS_WEB_MONITOR_HASH_NAME, unknown, )) # if every host is unknown, just consider them all available if len(unknown) == len(hosts): available.update(hosts) return available
class InteractionPoolAuthenticator(object): def __init__(self, memcached_client, interaction_pool): self._log = logging.getLogger("InteractionPoolAuthenticator") self._interaction_pool = interaction_pool self._collection_lookup = CollectionLookup(memcached_client, interaction_pool) self._customer_lookup = CustomerIdLookup(memcached_client, interaction_pool) self._customer_key_lookup = CustomerKeyLookup(memcached_client, interaction_pool) def authenticate(self, collection_name, access_type, req): """ establish that this is a valid user and a valid collection return collection_entry if valid raise AccessUnauthorized(error_message) if invalid """ collection_row = self._collection_lookup.get(collection_name.lower()) if collection_row is None: error_message = "unknown collection {0}".format(collection_name) self._log.error(error_message) raise AccessUnauthorized(error_message) if access_type is not None: access_result = \ check_access_control(access_type, req, collection_row["access_control"]) if access_result == access_allowed: return collection_row if access_result == access_forbidden: raise AccessForbidden() assert access_result == access_requires_password_authentication customer_row = self._customer_lookup.get(collection_row["customer_id"]) if customer_row is None: error_message = "unknown customer {0}".format(collection_name) self._log.error(error_message) raise AccessUnauthorized(error_message) try: auth_type, auth_string = req.authorization except Exception, instance: error_message = "invalid req.authorization {0} {1}".format( instance, req.authorization) self._log.error(error_message) raise AccessUnauthorized(error_message) if auth_type != 'NIMBUS.IO': error_message = "unknown auth_type %r" % (auth_type, ) self._log.error(error_message) raise AccessUnauthorized(error_message) try: key_id, signature = auth_string.split(':', 1) except Exception, instance: error_message = "invalid auth_string {0} {1}".format( instance, auth_string) self._log.error(error_message) raise AccessUnauthorized(error_message)