def __lookup_function(lookup_field_value):
log = logging.getLogger("CollectionLookup")
async_result = \
interaction_pool.run(interaction=_query,
interaction_args=[lookup_field_value, ],
pool=_central_pool_name)
try:
result_list = async_result.get(block=True,
timeout=_query_timeout)
except Exception:
log.exception(lookup_field_value)
raise
if len(result_list) == 0:
return None
assert len(result_list) == 1
result = result_list[0]
return_result = dict()
for key, value in result.items():
if key in _timestamp_columns:
return_result[key] = http_timestamp_str(value)
else:
return_result[key] = value
return return_result
def _get_collection_info(cursor, username, customer_id, collection_name):
"""
get basic information about the collection
See Ticket #51 Implement GET JSON for a collection
"""
log = logging.getLogger("_get_collection_info")
log.debug("_list_collection(cursor, {0}, {1}".format(customer_id,
collection_name))
row = _list_collection(cursor, customer_id, collection_name)
if row is None:
collection_dict = {"success" : False,
"error_message" : "No such collection"}
return httplib.NOT_FOUND, collection_dict
default_collection_name = compute_default_collection_name(username)
name, versioning, raw_access_control, raw_creation_time = row
if raw_access_control is None:
access_control = None
else:
access_control = json.loads(raw_access_control)
collection_dict = {"success" : True,
"name" : name,
"default_collection" : name == default_collection_name,
"versioning" : versioning,
"access_control" : access_control,
"creation-time" : http_timestamp_str(raw_creation_time)}
return httplib.OK, collection_dict
def _get_collection_info(cursor, username, customer_id, collection_name):
"""
get basic information about the collection
See Ticket #51 Implement GET JSON for a collection
"""
log = logging.getLogger("_get_collection_info")
log.debug("_list_collection(cursor, {0}, {1}".format(
customer_id, collection_name))
row = _list_collection(cursor, customer_id, collection_name)
if row is None:
collection_dict = {
"success": False,
"error_message": "No such collection"
}
return httplib.NOT_FOUND, collection_dict
default_collection_name = compute_default_collection_name(username)
name, versioning, raw_access_control, raw_creation_time = row
if raw_access_control is None:
access_control = None
else:
access_control = json.loads(raw_access_control)
collection_dict = {
"success": True,
"name": name,
"default_collection": name == default_collection_name,
"versioning": versioning,
"access_control": access_control,
"creation-time": http_timestamp_str(raw_creation_time)
}
return httplib.OK, collection_dict
def __lookup_function(lookup_field_value):
log = logging.getLogger("CollectionLookup")
async_result = \
interaction_pool.run(interaction=_query,
interaction_args=[lookup_field_value, ],
pool=_central_pool_name)
try:
result_list = async_result.get(block=True, timeout=_query_timeout)
except Exception:
log.exception(lookup_field_value)
raise
if len(result_list) == 0:
return None
assert len(result_list) == 1
result = result_list[0]
return_result = dict()
for key, value in result.items():
if key in _timestamp_columns:
return_result[key] = http_timestamp_str(value)
else:
return_result[key] = value
return return_result
def list_versions(interaction_pool,
collection_id,
versioned,
prefix=None,
max_keys=1000,
delimiter="",
key_marker=None,
version_id_marker=None):
"""
retrieve infromation about versions which are visible: not deleted, etc
"""
# ask for one more than max_keys, so we can tell if we are truncated
max_keys = int(max_keys)
request_count = max_keys + 1
sql_text = sql_factory.list_versions(collection_id,
versioned=versioned,
prefix=prefix,
key_marker=key_marker,
version_marker=version_id_marker,
limit=request_count)
args = {"collection_id" : collection_id,
"prefix" : (prefix if prefix is not None else ""),
"key_marker" : (key_marker if key_marker is not None else ""),
"version_marker":
(version_id_marker if version_id_marker is not None else 0), }
async_result = interaction_pool.run(interaction=sql_text.encode("utf-8"),
interaction_args=args,
pool=_local_node_name)
result = async_result.get()
truncated = len(result) == request_count
key_list = list()
for row in result[:max_keys]:
key_list.append(
{"key" : row["key"],
"version_identifier" : row["unified_id"],
"timestamp" : http_timestamp_str(row["timestamp"])})
if delimiter == "":
return {"key_data" : key_list, "truncated" : truncated}
# XXX: there may be some SQL way to do this efficiently
prefix_set = set()
offset = (len(prefix) if prefix is not None else 0)
for key_entry in key_list:
delimiter_pos = key_entry["key"].find(delimiter, offset)
if delimiter_pos > 0:
prefix_set.add(key_entry["key"][:delimiter_pos+1])
return {"prefixes" : list(prefix_set), "truncated" : truncated}
def _process_result_list(result_list):
if len(result_list) == 0:
return None
assert len(result_list) == 1
result = result_list[0]
return_result = dict()
for key, value in result.items():
if key in _timestamp_columns:
return_result[key] = http_timestamp_str(value)
else:
return_result[key] = value
return return_result
def _process_result_list(result_list):
if len(result_list) == 0:
return None
assert len(result_list) == 1
result = result_list[0]
return_result = dict()
for key, value in result.items():
if key in _timestamp_columns:
return_result[key] = http_timestamp_str(value)
else:
return_result[key] = value
return return_result
def dispatch_request(self, username):
log = logging.getLogger("ListCollectionsView")
try:
user_request_id = \
flask.request.headers['x-nimbus-io-user-request-id']
except KeyError:
user_request_id = str(uuid.uuid4())
log.warn("user_request_id = {0}, " \
"no x-nimbus-io-user-request-id " \
"header".format(user_request_id))
log.info("user_request_id = {0}, " \
"user_name = {1}".format(user_request_id,
username))
with GetConnection(self.connection_pool) as connection:
customer_key_lookup = \
CustomerKeyConnectionLookup(self.memcached_client,
connection)
customer_id = authenticate(customer_key_lookup, username,
flask.request)
if customer_id is None:
log.info("user_request_id = {0}, unauthorized".format(
user_request_id))
flask.abort(httplib.UNAUTHORIZED)
try:
raw_collection_list = _list_collections(
connection, customer_id)
except Exception:
log.exception("user_request_id = {0}".format(user_request_id))
raise
# ticket #50 When listing collections for a user, show whether a
# collection is a default collection.
default_collection_name = compute_default_collection_name(username)
collection_list = list()
for raw_entry in raw_collection_list:
name, versioning, raw_access_control, raw_creation_time = raw_entry
if raw_access_control is None:
access_control = None
else:
access_control = json.loads(raw_access_control)
entry = {
"name": name,
"default_collection": name == default_collection_name,
"versioning": versioning,
"access_control": access_control,
"creation-time": http_timestamp_str(raw_creation_time)
}
collection_list.append(entry)
log.info("user_request_id = {0}, found {1} collections".format(
user_request_id, len(collection_list)))
# 2012-08-16 dougfort Ticket #29 - format json for debuging
data = json.dumps(collection_list, sort_keys=True, indent=4)
# 2012-08-16 dougfort Ticket #28 - set content_type
response = flask.Response(data,
status=httplib.OK,
content_type="application/json")
response.headers["content-length"] = str(len(data))
return response
def _get_collection_space_usage(memcached_client,
cursor,
customer_id,
collection_name,
args):
"""
get usage information for the collection
See Ticket #66 Include operational stats in API queries for space usage
"""
log = logging.getLogger("_get_collection_space_usage")
if "days_of_history" in args:
# if N is specified, it is always rounded up to the nearest multiple
# of 30 (for caching)
days_of_history = ((int(args["days_of_history"]) / 30) + 1) * 30
else:
days_of_history = _default_days_of_history
log.debug("seeking {0} days of history".format(days_of_history))
memcached_key = \
_memcached_space_accounting_template.format(customer_id,
collection_name,
days_of_history)
cached_dict = memcached_client.get(memcached_key)
if cached_dict is not None:
log.debug("cache hit {0} days {1}".format(
len(cached_dict["operational_stats"]), memcached_key))
return httplib.OK, cached_dict
collection_id = _get_collection_id(cursor, customer_id, collection_name)
if collection_id is None:
collection_dict = {"success" : False,
"error_message" : "No such collection"}
return httplib.NOT_FOUND, collection_dict
# 2012-12-10 dougfort -- for reasons I don't understand, success_bytes_in
# and success_bytes_out emerge as type Dec. So I force them to int to
# keep JSON happy.
if days_of_history > _default_days_of_history:
cursor.execute(_long_day_query, [collection_id, days_of_history, ])
else:
cursor.execute(_short_day_query, [collection_id, days_of_history, ])
collection_dict = {"success" : True, "operational_stats" : list()}
for row in map(_operational_stats_row._make, cursor.fetchall()):
stats_dict = { "day" : http_timestamp_str(row.day),
"retrieve_success" : row.retrieve_success,
"archive_success" : row.archive_success,
"listmatch_success": row.listmatch_success,
"delete_success" : row.delete_success,
"success_bytes_in" : int(row.success_bytes_in),
"success_bytes_out": int(row.success_bytes_out), }
collection_dict["operational_stats"].append(stats_dict)
log.debug("database hit {0} days {1}".format(
len(collection_dict["operational_stats"]), memcached_key))
success = memcached_client.set(memcached_key,
collection_dict,
time=_expiration_time_in_seconds)
if not success:
log.error("memcached_client.set({0}...) returned {1}".format(
memcached_key, success))
return httplib.OK, collection_dict
def _fix_timestamp(timestamp):
return (None if timestamp is None else http_timestamp_str(timestamp))
def dispatch_request(self, username):
log = logging.getLogger("CreateCollectionView")
try:
user_request_id = \
flask.request.headers['x-nimbus-io-user-request-id']
except KeyError:
user_request_id = str(uuid.uuid4())
log.warn("user_request_id = {0}, " \
"no x-nimbus-io-user-request-id " \
"header".format(user_request_id))
log.info("user_request_id = {0}, user_name = {1}, " \
"collection_name = {2}".format(user_request_id,
username,
flask.request.args["name"]))
assert flask.request.args["action"] == "create", flask.request.args
collection_name = flask.request.args["name"]
if not valid_collection_name(collection_name):
# Ticket #48 Creating collection incorrectly handles
# creating colliding collections
log.error("user_request_id = {0}, " \
"invalid collection name '{1}'".format(user_request_id,
collection_name))
collection_dict = {
"name" : collection_name,
"error-messages" : ["Invalid Name"]}
data = json.dumps(collection_dict, sort_keys=True, indent=4)
response = flask.Response(data,
status=httplib.CONFLICT,
content_type="application/json")
response.headers["content-length"] = str(len(data))
return response
versioning = False
# Ticket # 43 Implement access_control properties for collections
if "Content-Type" in flask.request.headers and \
flask.request.headers['Content-Type'] == 'application/json':
access_control, error_list = \
cleanse_access_control(flask.request.data)
if error_list is not None:
log.error("user_request_id = {0}, " \
"invalid access control '{1}'".format(user_request_id,
error_list))
result_dict = {"success" : False,
"error_list" : error_list, }
data = json.dumps(result_dict, sort_keys=True, indent=4)
response = flask.Response(data,
status=httplib.BAD_REQUEST,
content_type="application/json")
response.headers["content-length"] = str(len(data))
return response
else:
access_control = None
with GetConnection(self.connection_pool) as connection:
customer_key_lookup = \
CustomerKeyConnectionLookup(self.memcached_client,
connection)
customer_id = authenticate(customer_key_lookup,
username,
flask.request)
if customer_id is None:
log.info("user_request_id = {0}, unauthorized")
flask.abort(httplib.UNAUTHORIZED)
cursor = connection.cursor()
cursor.execute("begin")
try:
creation_time = _create_collection(cursor,
customer_id,
collection_name,
versioning,
access_control)
except DuplicateCollection:
cursor.close()
connection.rollback()
# Ticket #48 Creating collection incorrectly handles
# creating colliding collections
log.error("user_request_id = {0}, " \
"duplicate collection name '{1}'".format(
user_request_id, collection_name))
collection_dict = {
"name" : collection_name,
"error-messages" : ["Invalid Name"]}
data = json.dumps(collection_dict, sort_keys=True, indent=4)
response = flask.Response(data,
status=httplib.CONFLICT,
content_type="application/json")
response.headers["content-length"] = str(len(data))
return response
except Exception:
log.exception("user_request_id = {0}".format(user_request_id))
cursor.close()
connection.rollback()
raise
else:
cursor.close()
connection.commit()
log.info("user_request_id = {0}, created {1}".format(user_request_id,
collection_name))
# this is the same format returned by list_collection
collection_dict = {
"name" : collection_name,
"versioning" : versioning,
"creation-time" : http_timestamp_str(creation_time)}
# 2012-08-16 dougfort Ticket #29 - format json for debuging
data = json.dumps(collection_dict, sort_keys=True, indent=4)
# 2012-04-15 dougfort Ticket #12 - return 201 'created'
# 2012-08-16 dougfort Ticket #28 - set content_type
response = flask.Response(data,
status=httplib.CREATED,
content_type="application/json")
response.headers["content-length"] = str(len(data))
return response
def _fix_timestamp(timestamp):
return None if timestamp is None else http_timestamp_str(timestamp)
def dispatch_request(self, username):
log = logging.getLogger("ListCollectionsView")
try:
user_request_id = \
flask.request.headers['x-nimbus-io-user-request-id']
except KeyError:
user_request_id = str(uuid.uuid4())
log.warn("user_request_id = {0}, " \
"no x-nimbus-io-user-request-id " \
"header".format(user_request_id))
log.info("user_request_id = {0}, " \
"user_name = {1}".format(user_request_id,
username))
with GetConnection(self.connection_pool) as connection:
customer_key_lookup = \
CustomerKeyConnectionLookup(self.memcached_client,
connection)
customer_id = authenticate(customer_key_lookup,
username,
flask.request)
if customer_id is None:
log.info("user_request_id = {0}, unauthorized".format(
user_request_id))
flask.abort(httplib.UNAUTHORIZED)
try:
raw_collection_list = _list_collections(connection, customer_id)
except Exception:
log.exception("user_request_id = {0}".format(user_request_id))
raise
# ticket #50 When listing collections for a user, show whether a
# collection is a default collection.
default_collection_name = compute_default_collection_name(username)
collection_list = list()
for raw_entry in raw_collection_list:
name, versioning, raw_access_control, raw_creation_time = raw_entry
if raw_access_control is None:
access_control = None
else:
access_control = json.loads(raw_access_control)
entry = {"name" : name,
"default_collection" : name == default_collection_name,
"versioning" : versioning,
"access_control" : access_control,
"creation-time" : http_timestamp_str(raw_creation_time)}
collection_list.append(entry)
log.info("user_request_id = {0}, found {1} collections".format(
user_request_id, len(collection_list)))
# 2012-08-16 dougfort Ticket #29 - format json for debuging
data = json.dumps(collection_list, sort_keys=True, indent=4)
# 2012-08-16 dougfort Ticket #28 - set content_type
response = flask.Response(data,
status=httplib.OK,
content_type="application/json")
response.headers["content-length"] = str(len(data))
return response
def _get_collection_space_usage(memcached_client, cursor, customer_id,
collection_name, args):
"""
get usage information for the collection
See Ticket #66 Include operational stats in API queries for space usage
"""
log = logging.getLogger("_get_collection_space_usage")
if "days_of_history" in args:
# if N is specified, it is always rounded up to the nearest multiple
# of 30 (for caching)
days_of_history = ((int(args["days_of_history"]) / 30) + 1) * 30
else:
days_of_history = _default_days_of_history
log.debug("seeking {0} days of history".format(days_of_history))
memcached_key = \
_memcached_space_accounting_template.format(customer_id,
collection_name,
days_of_history)
cached_dict = memcached_client.get(memcached_key)
if cached_dict is not None:
log.debug("cache hit {0} days {1}".format(
len(cached_dict["operational_stats"]), memcached_key))
return httplib.OK, cached_dict
collection_id = _get_collection_id(cursor, customer_id, collection_name)
if collection_id is None:
collection_dict = {
"success": False,
"error_message": "No such collection"
}
return httplib.NOT_FOUND, collection_dict
# 2012-12-10 dougfort -- for reasons I don't understand, success_bytes_in
# and success_bytes_out emerge as type Dec. So I force them to int to
# keep JSON happy.
if days_of_history > _default_days_of_history:
cursor.execute(_long_day_query, [
collection_id,
days_of_history,
])
else:
cursor.execute(_short_day_query, [
collection_id,
days_of_history,
])
collection_dict = {"success": True, "operational_stats": list()}
for row in map(_operational_stats_row._make, cursor.fetchall()):
stats_dict = {
"day": http_timestamp_str(row.day),
"retrieve_success": row.retrieve_success,
"archive_success": row.archive_success,
"listmatch_success": row.listmatch_success,
"delete_success": row.delete_success,
"success_bytes_in": int(row.success_bytes_in),
"success_bytes_out": int(row.success_bytes_out),
}
collection_dict["operational_stats"].append(stats_dict)
log.debug("database hit {0} days {1}".format(
len(collection_dict["operational_stats"]), memcached_key))
success = memcached_client.set(memcached_key,
collection_dict,
time=_expiration_time_in_seconds)
if not success:
log.error("memcached_client.set({0}...) returned {1}".format(
memcached_key, success))
return httplib.OK, collection_dict
