def delete_main(path):
valid_keys = ['containers', 'keyvals']
try:
container = ccget.get_container(path)
except ce.noResult:
return(error(
404,
"Container you want to delete from is not found",
requested_container_tree=" -> ".join(path.split("/"))))
# Check the validity of request
if len(flask.request.args) > 0:
return(error(
400,
"DELETE requests should not have querystring params",
querystring_params=flask.request.args))
# Load the JSON data, and validate it
try:
request_data = json.loads(flask.request.get_data())
except json.JSONDecodeError as e:
return(error(
400,
"JSON Data is invalid",
escaped_data_submitted=flask.request.get_data(),
parsing_error=e.message))
# Check for invalid keys
for key in sorted(request_data.keys()):
if key not in valid_keys:
return(error(
400,
"JSON contained bad key",
errored_on_key=key))
return_data = list()
for key in valid_keys:
try:
local_data = request_data[key]
except (KeyError) as e:
continue
return_data = globals()["delete_process_" + key](
local_data,
container_id=container.id,
processed_so_far=return_data,
path=path)
# Return data will become a TUPLE if there was a processing error, in
# which case we need to return this back up the stack
if type(return_data) is tuple:
return return_data
# Time to clean up the caches...
# post_invalidate_cache(path, request_data)
purge(path, request_data)
return json.dumps({"results": return_data})
def put_main(path):
# A function that matches post_process_{valid_key} must exist
valid_keys = ['containers', 'keyvals']
try:
container = ccget.get_container(path)
except ce.noResult:
return(error(
404,
"Container you want to PUT into is not found",
requested_container_tree=" -> ".join(path.split("/"))))
# Check the validity of request
if len(flask.request.args) > 0:
return(error(
400,
"PUT requests should not have querystring params",
querystring_params=flask.request.args))
# Load the JSON data, and validate it
try:
request_data = json.loads(flask.request.get_data())
except json.JSONDecodeError as e:
return(error(
400,
"JSON Data is invalid",
escaped_data_submitted=flask.request.get_data(),
parsing_error=e.message))
# Check for invalid keys
for key in sorted(request_data.keys()):
if key not in valid_keys:
return(error(
400,
"JSON contained bad key",
errored_on_key=key))
return_data = list()
for key in valid_keys:
try:
local_data = request_data[key]
except (KeyError) as e:
continue
return_data = globals()["put_process_" + key](
local_data,
path=path,
container_id=container.id,
processed_so_far=return_data)
# Return data will become a TUPLE if there was a processing error, in
# which case we need to return this back up the stack
if type(return_data) is tuple:
return return_data
# Time to clean up the caches...
purge(path, request_data)
return json.dumps({"results": return_data})
def before_request():
try:
auth = ("APIKEY", flask.request.headers['X-API-Auth'], flask.request)
except KeyError as e:
try:
auth = ("LDAP", flask.request.headers['X-SSO-Auth'], flask.request)
except KeyError as e:
return (error(403,
"authentication headers not provided",
validheaders=['X-API-Auth', 'X-SSO-Auth']))
if util.checkAuth(auth) is not True:
return (error(403, "invalid authentication informationed provided"))
def before_request():
try:
auth = ("APIKEY", flask.request.headers['X-API-Auth'], flask.request)
except KeyError as e:
try:
auth = ("LDAP", flask.request.headers['X-SSO-Auth'], flask.request)
except KeyError as e:
return(error(
403,
"authentication headers not provided",
validheaders=['X-API-Auth', 'X-SSO-Auth']))
if util.checkAuth(auth) is not True:
return(error(403, "invalid authentication informationed provided"))
def post_process_containers(data, **kwargs):
cid = kwargs['container_id']
return_data = kwargs['processed_so_far']
try:
for container in data:
try:
name = container['name']
except TypeError as e:
return(error(
400,
"containers JSON should be a list of JSON"
" dictionaries, and yours is not",
processed_so_far=return_data))
except KeyError as e:
return(error(
400,
"unable to add container, forgot to specify name!",
processed_so_far=return_data))
try:
desc = container['description']
except KeyError:
desc = None
try:
return_data.append("Created:" +
str(ccpost.add_container(
cid,
name,
desc)))
except ce.notUnique:
return_data.append("Container Not Added, Already Exists: %s" %
(name))
except TypeError as e:
return(error(
400,
"containers JSON should be a list of JSON dictionaries",
processed_so_far=return_data),
True)
return return_data
def post_process_keyvals(data, **kwargs):
cid = kwargs['container_id']
return_data = kwargs['processed_so_far']
try:
for keyval in data:
try:
key = keyval['key'].lower()
except TypeError as e:
return(error(
400,
"containers JSON should be a list of JSON dictionaries",
processed_so_far=return_data))
except KeyError as e:
return(error(
400,
"must specify key name for all keyvals",
processed_so_far=return_data))
try:
value = keyval['value']
except KeyError as e:
return(error(
400,
"must specify value for all keyvals",
processed_so_far=return_data))
try:
tag = keyval['tag']
except KeyError as e:
tag = None
return_data.append("Created:" + str(
ccpost.add_keyval(cid, key, value, tag)))
except TypeError as e:
raise
return(error(
400,
"keyvals JSON should be a list of JSON dictionaries",
processed_so_far=return_data))
return return_data
def put_process_keyvals(data, **kwargs):
return_data = kwargs['processed_so_far']
for item in data:
try:
k = item['key'].lower()
v = item['value']
except KeyError as e:
return(error(
400,
"Key Val items must contain both a key and a value",
processed_so_far=kwargs['processed_so_far']))
try:
t = item['tag']
except KeyError as e:
t = app.config['DEFAULT_TAG']
try:
ccput.update_keyval(kwargs['path'], k, v, tag=t)
return_data.append("set /%s :%s=%s?TAG=%s" %
(kwargs['path'], k, v, t))
except ce.noResult as e:
return(error(400,
"key '%s' not found at /%s" % (k, kwargs['path']),
processed_so_far=kwargs['processed_so_far']))
return return_data
def get_content(path):
"""
This will return a dict of the content in containers matching key or key and value provided
in arguments.
"""
cache_key = "/operations/container/content/" + path + "?" + "&".join(
["%s=%s" % (k.upper(), v.upper()) for k, v in sorted(
flask.request.args.iteritems())])
app.logger.debug("Using Cache Key: %s" % (cache_key))
loadkey = "loading-" + cache_key
cache_result = cache.get(cache_key)
loops = 0
while cache_result is None:
loops += 1
if loops > 70:
return error(500,
message="item never became available in cache",
debug="%d: %s" % (loops, cache_key))
app.logger.debug("%s is not cached, checked %d times" %
(cache_key, loops))
sleeptime = random() / 10
sleep(sleeptime)
if ((cache.get("loading-" + cache_key) is None and
cache.get(cache_key) is None)):
cache.set("loading-" + cache_key, "True",
timeout=app.config['CACHE_LOCKING_SECONDS'])
max_recursion = int(app.config['TREE_MAX_RECURSION'])
request_key = None
request_val = None
for (k, v) in flask.request.args.iteritems():
if k.upper() == "KEY":
request_key = v.lower()
elif k.upper() == "VALUE":
request_val = v.lower()
else:
return error(400,
"client error",
"invalid query string",
key=k,
value=v,
path=cache_key
)
app.logger.debug("loading %s after %f seconds wait" % (cache_key, sleeptime))
# Check that key is provided
if request_key == None:
return error(400,
"client error",
"No key provided"
)
try:
container = ccget.get_container(path)
except ce.noResult as e:
return error(404,
"container not found",
container=path)
if request_key is not None and request_val is None:
#app.logger.debug("Content with no key")
return_hash = dict()
a = container.dumptree_with_key(max_recursion=max_recursion, key=request_key, container_id=True)
for container in a:
c = ccget.get_container(container)
all_values = dict()
for key in c.keys:
all_values[key.name] = list()
for value in key.values:
all_values[key.name].append(value.value)
return_hash[c.name]= all_values
cache.set(cache_key,
json.dumps(return_hash),
timeout=app.config['CACHE_DEFAULT_AGE_SECONDS'])
elif request_key is not None and request_val is not None:
#app.logger.debug("Content with given key")
return_hash = dict()
a = container.dumptree_with_key_val(max_recursion=max_recursion, key=request_key, r_val=request_val, container_id=True)
for container in a:
c = ccget.get_container(container)
all_values = dict()
for key in c.keys:
all_values[key.name] = list()
for value in key.values:
all_values[key.name].append(value.value)
return_hash[c.name]=all_values
cache.set(cache_key,
json.dumps(return_hash),
timeout=app.config['CACHE_DEFAULT_AGE_SECONDS'])
## Shouldn't get here TODO fix or error
else:
app.logger.debug("Should not get here!")
cache.set(cache_key,
json.dumps([x for x in container.children.keys()]),
timeout=app.config['CACHE_DEFAULT_AGE_SECONDS'])
cache.delete("loading-" + cache_key)
cache_result = cache.get(cache_key)
return cache_result
def get_main(path):
# This will handle all gets that are not otherwise defined.
cache_key = "/" + path + "?" + "&".join(
["%s=%s" % (k.upper(), v.upper()) for k, v in sorted(
flask.request.args.iteritems())])
app.logger.debug("Using Cache Key: %s" % (cache_key))
loadkey = "loading-" + cache_key
cache_result = cache.get(cache_key)
loops = 0
# All get's are served up from the cache.
while cache_result is None:
# Keep track of loops, abort if it get's to be excessive, this may
# need tuned later depending on how long DB operations take, right now
# the number of wait loops is excessive
loops += 1
if loops > 50:
return(error(
500,
message="item never became available in cache",
debug="%d: %s" % (loops, cache_key)))
app.logger.debug("%s is not cached, checked %d times" %
(cache_key, loops))
sleeptime = random() / 100
sleep(sleeptime)
# For other threads that might be requesting the same thing, announce
# to the cache that we're processing this page, help to eliminate
# overloading during bursts for the same key.
if ((cache.get("loading-" + cache_key) is None and
cache.get(cache_key) is None)):
# Announce we're loading to the cache
cache.set("loading-" + cache_key, "True",
timeout=app.config['CACHE_LOCKING_SECONDS'])
# Set values for the request
request_tag = app.config['DEFAULT_TAG']
request_return = "VALUES"
request_key = None
for (k, v) in flask.request.args.iteritems():
if k.upper() == "TAG":
request_tag = v.upper()
elif k.upper() == "KEY":
request_key = v.lower()
elif k.upper() == "RETURN":
request_return = v.upper()
else:
# Clean up known query string params, avoid cachebusting
# and make clearing the cache possible for posts/puts
return(error(
400,
"client error",
"invalid query string parameters",
key=k,
value=v,
path=cache_key))
app.logger.debug(
"I'm doing the loading after a sleep of %f seconds" %
(sleeptime))
# Can't request a key in conjuction with some request_return types
if request_key is not None and request_return is not "VALUES":
return(error(
400,
"client error",
"paramater conflict",
message="%s can't be return value when requesting a single key" %(request_return),
))
if request_key is not None:
# A single Key is requested
try:
key = ccget.get_key(path, request_key, tag=request_tag)
except ce.noResult:
return error(404, "key not found", requested_key=request_key, requested_container=path, applied_tag=request_tag)
return_key = {key.name: {"values": list()}}
return_key[key.name]['tag'] = key.tag
# @TODO: Fix date added/date updated
# return_key['added'] = key.added
# return_key['updated'] = key.updated
for v in key.values:
return_key[key.name]['values'].append(
{'id': v.id, 'value': v.value } )
cache.set(cache_key,
json.dumps(return_key),
timeout=app.config['CACHE_DEFAULT_AGE_SECONDS'])
else:
try:
container = ccget.get_container(path)
except ce.noResult:
return error(404, "container not found", requested_container=path)
# Process a container list
if request_return == "ALL" or request_return == "CONTAINERS":
child_containers = list()
all_child_containers = dict()
app.logger.debug(request_return)
for child_container in container.children:
child_containers.append(path + "/" + child_container)
app.logger.debug(all_child_containers)
for child_container in child_containers:
c = ccget.get_container(child_container)
all_child_containers[c.name] = { "description": c.description,
"id": c.id }
# Process a value list
if request_return == "ALL" or request_return =="VALUES":
all_containers = [container.id]
all_values = dict()
while container.parent is not None:
container = container.parent
if container is not None:
all_containers.append(container.id)
for container in reversed(all_containers):
c = ccget.get_container(container)
for key in c.keys:
if ((key.tag != request_tag and
key.tag != app.config['DEFAULT_TAG'])):
continue
try:
if ((all_values[key.name]['tag'] !=
app.config['DEFAULT_TAG'])):
continue
except KeyError:
pass
all_values[key.name] = {'values': list(),
'tag': key.tag,
'id': key.id}
for value in key.values:
all_values[key.name]['values'].append(
{'id': value.id, 'value': value.value })
if request_return == "ALL":
return_hash = { "containers": all_child_containers,
"key_values": all_values }
elif request_return == "CONTAINERS":
return_hash = all_child_containers
elif request_return == "VALUES":
return_hash = all_values
else:
return(error(400,
"client_error",
"invalid paramater value",
key=k,
value=v,
path=cache_key))
cache.set(cache_key,
json.dumps(return_hash),
timeout=app.config['CACHE_DEFAULT_AGE_SECONDS'])
cache.delete("loading-" + cache_key)
cache_result = cache.get(cache_key)
return cache_result
def get_list(path):
"""
This will return a list of all containers inside of another
container
Setting ?recursive=true will get all the child containers,
until there are no more children and provide a full container
tree
"""
cache_key = "/operations/container/list/" + path + "?" + "&".join(
["%s=%s" % (k.upper(), v.upper()) for k, v in sorted(
flask.request.args.iteritems())])
app.logger.debug("Using Cache Key: %s" % (cache_key))
loadkey = "loading-" + cache_key
cache_result = cache.get(cache_key)
loops = 0
while cache_result is None:
loops += 1
if loops > 50:
return error(500,
message="item never became available in cache",
debug="%d: %s" % (loops, cache_key))
app.logger.debug("%s is not cached, checked %d times" %
(cache_key, loops))
sleeptime = random() / 10
sleep(sleeptime)
if ((cache.get("loading-" + cache_key) is None and
cache.get(cache_key) is None)):
cache.set("loading-" + cache_key, "True",
timeout=app.config['CACHE_LOCKING_SECONDS'])
request_recursive = False
max_recursion = int(app.config['TREE_MAX_RECURSION'])
request_key = None
request_val = None
for (k, v) in flask.request.args.iteritems():
if k.upper() == "DEPTH":
try:
max_recursion = int(v)
except ValueError as e:
return error(400,
"client_error",
"invalid querystring",
recursion_depth="%s does not appear to be" +
" an integer" % (v))
elif k.upper() == "RECURSIVE":
try:
request_recursive = crud_config.util.toBool(v)
except ValueError as e:
return error(400,
"client error",
"invalid querystring",
recursion_request_value=v)
elif k.upper() == "KEY":
request_key = v.lower()
elif k.upper() == "VALUE":
request_val = v.lower()
else:
return error(400,
"client error",
"invalid query string",
key=k,
value=v,
path=cache_key
)
app.logger.debug("loading %s after %f seconds wait" % (cache_key, sleeptime))
try:
container = ccget.get_container(path)
except ce.noResult as e:
return error(404,
"container not found",
container=path)
if request_recursive is True and request_key is None:
#app.logger.debug("recursive with NO key")
cache.set(cache_key,
json.dumps(container.dumptree(max_recursion=max_recursion)),
timeout=app.config['CACHE_DEFAULT_AGE_SECONDS'])
elif request_recursive is True and request_key is not None and request_val is None:
#app.logger.debug("recursive with key")
cache.set(cache_key,
json.dumps(container.dumptree_with_key(max_recursion=max_recursion, key=request_key)),
timeout=app.config['CACHE_DEFAULT_AGE_SECONDS'])
elif request_recursive is True and request_key is not None and request_val is not None:
#app.logger.debug("recursive with key and val")
cache.set(cache_key,
json.dumps(container.dumptree_with_key_val(max_recursion=max_recursion, key=request_key, r_val=request_val)),
timeout=app.config['CACHE_DEFAULT_AGE_SECONDS'])
else:
cache.set(cache_key,
json.dumps([x for x in container.children.keys()]),
timeout=app.config['CACHE_DEFAULT_AGE_SECONDS'])
cache.delete("loading-" + cache_key)
cache_result = cache.get(cache_key)
return cache_result