def __lookup_function(lookup_field_value): log = logging.getLogger("CollectionLookup") async_result = \ interaction_pool.run(interaction=_query, interaction_args=[lookup_field_value, ], pool=_central_pool_name) try: result_list = async_result.get(block=True, timeout=_query_timeout) except Exception: log.exception(lookup_field_value) raise if len(result_list) == 0: return None assert len(result_list) == 1 result = result_list[0] return_result = dict() for key, value in result.items(): if key in _timestamp_columns: return_result[key] = http_timestamp_str(value) else: return_result[key] = value return return_result
def _get_collection_info(cursor, username, customer_id, collection_name): """ get basic information about the collection See Ticket #51 Implement GET JSON for a collection """ log = logging.getLogger("_get_collection_info") log.debug("_list_collection(cursor, {0}, {1}".format(customer_id, collection_name)) row = _list_collection(cursor, customer_id, collection_name) if row is None: collection_dict = {"success" : False, "error_message" : "No such collection"} return httplib.NOT_FOUND, collection_dict default_collection_name = compute_default_collection_name(username) name, versioning, raw_access_control, raw_creation_time = row if raw_access_control is None: access_control = None else: access_control = json.loads(raw_access_control) collection_dict = {"success" : True, "name" : name, "default_collection" : name == default_collection_name, "versioning" : versioning, "access_control" : access_control, "creation-time" : http_timestamp_str(raw_creation_time)} return httplib.OK, collection_dict
def _get_collection_info(cursor, username, customer_id, collection_name): """ get basic information about the collection See Ticket #51 Implement GET JSON for a collection """ log = logging.getLogger("_get_collection_info") log.debug("_list_collection(cursor, {0}, {1}".format( customer_id, collection_name)) row = _list_collection(cursor, customer_id, collection_name) if row is None: collection_dict = { "success": False, "error_message": "No such collection" } return httplib.NOT_FOUND, collection_dict default_collection_name = compute_default_collection_name(username) name, versioning, raw_access_control, raw_creation_time = row if raw_access_control is None: access_control = None else: access_control = json.loads(raw_access_control) collection_dict = { "success": True, "name": name, "default_collection": name == default_collection_name, "versioning": versioning, "access_control": access_control, "creation-time": http_timestamp_str(raw_creation_time) } return httplib.OK, collection_dict
def list_versions(interaction_pool, collection_id, versioned, prefix=None, max_keys=1000, delimiter="", key_marker=None, version_id_marker=None): """ retrieve infromation about versions which are visible: not deleted, etc """ # ask for one more than max_keys, so we can tell if we are truncated max_keys = int(max_keys) request_count = max_keys + 1 sql_text = sql_factory.list_versions(collection_id, versioned=versioned, prefix=prefix, key_marker=key_marker, version_marker=version_id_marker, limit=request_count) args = {"collection_id" : collection_id, "prefix" : (prefix if prefix is not None else ""), "key_marker" : (key_marker if key_marker is not None else ""), "version_marker": (version_id_marker if version_id_marker is not None else 0), } async_result = interaction_pool.run(interaction=sql_text.encode("utf-8"), interaction_args=args, pool=_local_node_name) result = async_result.get() truncated = len(result) == request_count key_list = list() for row in result[:max_keys]: key_list.append( {"key" : row["key"], "version_identifier" : row["unified_id"], "timestamp" : http_timestamp_str(row["timestamp"])}) if delimiter == "": return {"key_data" : key_list, "truncated" : truncated} # XXX: there may be some SQL way to do this efficiently prefix_set = set() offset = (len(prefix) if prefix is not None else 0) for key_entry in key_list: delimiter_pos = key_entry["key"].find(delimiter, offset) if delimiter_pos > 0: prefix_set.add(key_entry["key"][:delimiter_pos+1]) return {"prefixes" : list(prefix_set), "truncated" : truncated}
def _process_result_list(result_list): if len(result_list) == 0: return None assert len(result_list) == 1 result = result_list[0] return_result = dict() for key, value in result.items(): if key in _timestamp_columns: return_result[key] = http_timestamp_str(value) else: return_result[key] = value return return_result
def dispatch_request(self, username): log = logging.getLogger("ListCollectionsView") try: user_request_id = \ flask.request.headers['x-nimbus-io-user-request-id'] except KeyError: user_request_id = str(uuid.uuid4()) log.warn("user_request_id = {0}, " \ "no x-nimbus-io-user-request-id " \ "header".format(user_request_id)) log.info("user_request_id = {0}, " \ "user_name = {1}".format(user_request_id, username)) with GetConnection(self.connection_pool) as connection: customer_key_lookup = \ CustomerKeyConnectionLookup(self.memcached_client, connection) customer_id = authenticate(customer_key_lookup, username, flask.request) if customer_id is None: log.info("user_request_id = {0}, unauthorized".format( user_request_id)) flask.abort(httplib.UNAUTHORIZED) try: raw_collection_list = _list_collections( connection, customer_id) except Exception: log.exception("user_request_id = {0}".format(user_request_id)) raise # ticket #50 When listing collections for a user, show whether a # collection is a default collection. default_collection_name = compute_default_collection_name(username) collection_list = list() for raw_entry in raw_collection_list: name, versioning, raw_access_control, raw_creation_time = raw_entry if raw_access_control is None: access_control = None else: access_control = json.loads(raw_access_control) entry = { "name": name, "default_collection": name == default_collection_name, "versioning": versioning, "access_control": access_control, "creation-time": http_timestamp_str(raw_creation_time) } collection_list.append(entry) log.info("user_request_id = {0}, found {1} collections".format( user_request_id, len(collection_list))) # 2012-08-16 dougfort Ticket #29 - format json for debuging data = json.dumps(collection_list, sort_keys=True, indent=4) # 2012-08-16 dougfort Ticket #28 - set content_type response = flask.Response(data, status=httplib.OK, content_type="application/json") response.headers["content-length"] = str(len(data)) return response
def _get_collection_space_usage(memcached_client, cursor, customer_id, collection_name, args): """ get usage information for the collection See Ticket #66 Include operational stats in API queries for space usage """ log = logging.getLogger("_get_collection_space_usage") if "days_of_history" in args: # if N is specified, it is always rounded up to the nearest multiple # of 30 (for caching) days_of_history = ((int(args["days_of_history"]) / 30) + 1) * 30 else: days_of_history = _default_days_of_history log.debug("seeking {0} days of history".format(days_of_history)) memcached_key = \ _memcached_space_accounting_template.format(customer_id, collection_name, days_of_history) cached_dict = memcached_client.get(memcached_key) if cached_dict is not None: log.debug("cache hit {0} days {1}".format( len(cached_dict["operational_stats"]), memcached_key)) return httplib.OK, cached_dict collection_id = _get_collection_id(cursor, customer_id, collection_name) if collection_id is None: collection_dict = {"success" : False, "error_message" : "No such collection"} return httplib.NOT_FOUND, collection_dict # 2012-12-10 dougfort -- for reasons I don't understand, success_bytes_in # and success_bytes_out emerge as type Dec. So I force them to int to # keep JSON happy. if days_of_history > _default_days_of_history: cursor.execute(_long_day_query, [collection_id, days_of_history, ]) else: cursor.execute(_short_day_query, [collection_id, days_of_history, ]) collection_dict = {"success" : True, "operational_stats" : list()} for row in map(_operational_stats_row._make, cursor.fetchall()): stats_dict = { "day" : http_timestamp_str(row.day), "retrieve_success" : row.retrieve_success, "archive_success" : row.archive_success, "listmatch_success": row.listmatch_success, "delete_success" : row.delete_success, "success_bytes_in" : int(row.success_bytes_in), "success_bytes_out": int(row.success_bytes_out), } collection_dict["operational_stats"].append(stats_dict) log.debug("database hit {0} days {1}".format( len(collection_dict["operational_stats"]), memcached_key)) success = memcached_client.set(memcached_key, collection_dict, time=_expiration_time_in_seconds) if not success: log.error("memcached_client.set({0}...) returned {1}".format( memcached_key, success)) return httplib.OK, collection_dict
def _fix_timestamp(timestamp): return (None if timestamp is None else http_timestamp_str(timestamp))
def dispatch_request(self, username): log = logging.getLogger("CreateCollectionView") try: user_request_id = \ flask.request.headers['x-nimbus-io-user-request-id'] except KeyError: user_request_id = str(uuid.uuid4()) log.warn("user_request_id = {0}, " \ "no x-nimbus-io-user-request-id " \ "header".format(user_request_id)) log.info("user_request_id = {0}, user_name = {1}, " \ "collection_name = {2}".format(user_request_id, username, flask.request.args["name"])) assert flask.request.args["action"] == "create", flask.request.args collection_name = flask.request.args["name"] if not valid_collection_name(collection_name): # Ticket #48 Creating collection incorrectly handles # creating colliding collections log.error("user_request_id = {0}, " \ "invalid collection name '{1}'".format(user_request_id, collection_name)) collection_dict = { "name" : collection_name, "error-messages" : ["Invalid Name"]} data = json.dumps(collection_dict, sort_keys=True, indent=4) response = flask.Response(data, status=httplib.CONFLICT, content_type="application/json") response.headers["content-length"] = str(len(data)) return response versioning = False # Ticket # 43 Implement access_control properties for collections if "Content-Type" in flask.request.headers and \ flask.request.headers['Content-Type'] == 'application/json': access_control, error_list = \ cleanse_access_control(flask.request.data) if error_list is not None: log.error("user_request_id = {0}, " \ "invalid access control '{1}'".format(user_request_id, error_list)) result_dict = {"success" : False, "error_list" : error_list, } data = json.dumps(result_dict, sort_keys=True, indent=4) response = flask.Response(data, status=httplib.BAD_REQUEST, content_type="application/json") response.headers["content-length"] = str(len(data)) return response else: access_control = None with GetConnection(self.connection_pool) as connection: customer_key_lookup = \ CustomerKeyConnectionLookup(self.memcached_client, connection) customer_id = authenticate(customer_key_lookup, username, flask.request) if customer_id is None: log.info("user_request_id = {0}, unauthorized") flask.abort(httplib.UNAUTHORIZED) cursor = connection.cursor() cursor.execute("begin") try: creation_time = _create_collection(cursor, customer_id, collection_name, versioning, access_control) except DuplicateCollection: cursor.close() connection.rollback() # Ticket #48 Creating collection incorrectly handles # creating colliding collections log.error("user_request_id = {0}, " \ "duplicate collection name '{1}'".format( user_request_id, collection_name)) collection_dict = { "name" : collection_name, "error-messages" : ["Invalid Name"]} data = json.dumps(collection_dict, sort_keys=True, indent=4) response = flask.Response(data, status=httplib.CONFLICT, content_type="application/json") response.headers["content-length"] = str(len(data)) return response except Exception: log.exception("user_request_id = {0}".format(user_request_id)) cursor.close() connection.rollback() raise else: cursor.close() connection.commit() log.info("user_request_id = {0}, created {1}".format(user_request_id, collection_name)) # this is the same format returned by list_collection collection_dict = { "name" : collection_name, "versioning" : versioning, "creation-time" : http_timestamp_str(creation_time)} # 2012-08-16 dougfort Ticket #29 - format json for debuging data = json.dumps(collection_dict, sort_keys=True, indent=4) # 2012-04-15 dougfort Ticket #12 - return 201 'created' # 2012-08-16 dougfort Ticket #28 - set content_type response = flask.Response(data, status=httplib.CREATED, content_type="application/json") response.headers["content-length"] = str(len(data)) return response
def _fix_timestamp(timestamp): return None if timestamp is None else http_timestamp_str(timestamp)
def dispatch_request(self, username): log = logging.getLogger("ListCollectionsView") try: user_request_id = \ flask.request.headers['x-nimbus-io-user-request-id'] except KeyError: user_request_id = str(uuid.uuid4()) log.warn("user_request_id = {0}, " \ "no x-nimbus-io-user-request-id " \ "header".format(user_request_id)) log.info("user_request_id = {0}, " \ "user_name = {1}".format(user_request_id, username)) with GetConnection(self.connection_pool) as connection: customer_key_lookup = \ CustomerKeyConnectionLookup(self.memcached_client, connection) customer_id = authenticate(customer_key_lookup, username, flask.request) if customer_id is None: log.info("user_request_id = {0}, unauthorized".format( user_request_id)) flask.abort(httplib.UNAUTHORIZED) try: raw_collection_list = _list_collections(connection, customer_id) except Exception: log.exception("user_request_id = {0}".format(user_request_id)) raise # ticket #50 When listing collections for a user, show whether a # collection is a default collection. default_collection_name = compute_default_collection_name(username) collection_list = list() for raw_entry in raw_collection_list: name, versioning, raw_access_control, raw_creation_time = raw_entry if raw_access_control is None: access_control = None else: access_control = json.loads(raw_access_control) entry = {"name" : name, "default_collection" : name == default_collection_name, "versioning" : versioning, "access_control" : access_control, "creation-time" : http_timestamp_str(raw_creation_time)} collection_list.append(entry) log.info("user_request_id = {0}, found {1} collections".format( user_request_id, len(collection_list))) # 2012-08-16 dougfort Ticket #29 - format json for debuging data = json.dumps(collection_list, sort_keys=True, indent=4) # 2012-08-16 dougfort Ticket #28 - set content_type response = flask.Response(data, status=httplib.OK, content_type="application/json") response.headers["content-length"] = str(len(data)) return response
def _get_collection_space_usage(memcached_client, cursor, customer_id, collection_name, args): """ get usage information for the collection See Ticket #66 Include operational stats in API queries for space usage """ log = logging.getLogger("_get_collection_space_usage") if "days_of_history" in args: # if N is specified, it is always rounded up to the nearest multiple # of 30 (for caching) days_of_history = ((int(args["days_of_history"]) / 30) + 1) * 30 else: days_of_history = _default_days_of_history log.debug("seeking {0} days of history".format(days_of_history)) memcached_key = \ _memcached_space_accounting_template.format(customer_id, collection_name, days_of_history) cached_dict = memcached_client.get(memcached_key) if cached_dict is not None: log.debug("cache hit {0} days {1}".format( len(cached_dict["operational_stats"]), memcached_key)) return httplib.OK, cached_dict collection_id = _get_collection_id(cursor, customer_id, collection_name) if collection_id is None: collection_dict = { "success": False, "error_message": "No such collection" } return httplib.NOT_FOUND, collection_dict # 2012-12-10 dougfort -- for reasons I don't understand, success_bytes_in # and success_bytes_out emerge as type Dec. So I force them to int to # keep JSON happy. if days_of_history > _default_days_of_history: cursor.execute(_long_day_query, [ collection_id, days_of_history, ]) else: cursor.execute(_short_day_query, [ collection_id, days_of_history, ]) collection_dict = {"success": True, "operational_stats": list()} for row in map(_operational_stats_row._make, cursor.fetchall()): stats_dict = { "day": http_timestamp_str(row.day), "retrieve_success": row.retrieve_success, "archive_success": row.archive_success, "listmatch_success": row.listmatch_success, "delete_success": row.delete_success, "success_bytes_in": int(row.success_bytes_in), "success_bytes_out": int(row.success_bytes_out), } collection_dict["operational_stats"].append(stats_dict) log.debug("database hit {0} days {1}".format( len(collection_dict["operational_stats"]), memcached_key)) success = memcached_client.set(memcached_key, collection_dict, time=_expiration_time_in_seconds) if not success: log.error("memcached_client.set({0}...) returned {1}".format( memcached_key, success)) return httplib.OK, collection_dict