def filter_undeploy(request, filter_id, project_id, container=None, swift_object=None):
"""
Undeploy a filter from a specific swift project.
"""
if request.method == 'PUT':
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Problems to connect with the DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
filter_data = r.hgetall("filter:" + str(filter_id))
if not filter_data:
return JSONResponse('Filter does not exist', status=status.HTTP_404_NOT_FOUND)
if container and swift_object:
target = project_id + "/" + container + "/" + swift_object
elif container:
target = project_id + "/" + container
else:
target = project_id
token = get_token_connection(request)
return unset_filter(r, target, filter_data, token)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def dependency_undeploy(request, dependency_id, project_id):
if request.method == 'PUT':
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Problems to connect with the DB', status=500)
dependency = r.hgetall("dependency:" + str(dependency_id))
if not dependency:
return JSONResponse('Dependency does not exist', status=404)
if not r.exists(str(project_id) + ":dependency:" + str(dependency["name"])):
return JSONResponse('Dependency ' + str(dependency["name"]) + ' has not been deployed already', status=404)
try:
token = get_token_connection(request)
url = settings.SWIFT_URL + "/AUTH_" + project_id
swift_response = dict()
swift_client.delete_object(url, token, 'dependency', dependency["name"], None, None, None, None, swift_response)
except ClientException:
return JSONResponse(swift_response.get("reason"), status=swift_response.get('status'))
swift_status = swift_response.get('status')
if 200 <= swift_status < 300:
r.delete(str(project_id) + ":dependency:" + str(dependency["name"]))
r.lrem(str(project_id) + ":dependencies", str(dependency["name"]), 1)
return JSONResponse('The dependency has been deleted', status=swift_status)
return JSONResponse(swift_response.get("reason"), status=swift_status)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=405)
def containers_list(request, project_id):
if request.method == 'GET':
token = get_token_connection(request)
url = settings.SWIFT_URL + "/AUTH_" + project_id
_, containers = swift_client.get_account(url, token)
for c_id in reversed(range(len(containers))):
if containers[c_id]['name'] in ('.dependency', '.storlet'):
del containers[c_id]
return JSONResponse(containers, status=status.HTTP_200_OK)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def containers_list(request, project_id):
if request.method == 'GET':
token = get_token_connection(request)
url = settings.SWIFT_URL + "/AUTH_" + project_id
_, containers = swift_client.get_account(url, token)
for c_id in reversed(range(len(containers))):
if containers[c_id]['name'] in ('.dependency', '.storlet'):
del containers[c_id]
return JSONResponse(containers, status=status.HTTP_200_OK)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def create_container(request, project_id, container_name):
if request.method == 'POST':
try:
headers = JSONParser().parse(request)
token = get_token_connection(request)
url = settings.SWIFT_URL + "/AUTH_" + project_id
swift_client.put_container(url, token, container_name, headers)
except Exception as ex:
return JSONResponse(ex.message, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return JSONResponse("Container Policy updated correctly", status=status.HTTP_201_CREATED)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def create_container(request, project_id, container_name):
if request.method == 'POST':
try:
headers = JSONParser().parse(request)
token = get_token_connection(request)
url = settings.SWIFT_URL + "/AUTH_" + project_id
swift_client.put_container(url, token, container_name, headers)
except Exception as e:
return JSONResponse(e.message, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return JSONResponse("Container Policy updated correctly", status=status.HTTP_201_CREATED)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def update_container(request, project_id, container_name):
if request.method == 'PUT':
sp = JSONParser().parse(request)
token = get_token_connection(request)
url = settings.SWIFT_URL + "/AUTH_" + project_id
headers, obj_list = swift_client.get_container(url, token,
container_name)
headers['X-Storage-Policy'] = sp
path_container = settings.SWIFT_CFG_TMP_DIR + "/" + container_name
os.mkdir(path_container)
for obj in obj_list:
file = open(path_container + "/" + obj["name"], "w")
obj_headers, obj_body = swift_client.get_object(
url, token, container_name, obj["name"])
file.write(obj_body)
file.close()
obj["headers"] = obj_headers
swift_client.delete_object(url, token, container_name, obj["name"])
swift_client.delete_container(url, token, container_name)
swift_client.put_container(url, token, container_name, headers)
for obj in obj_list:
obj_path = os.path.join(path_container, obj["name"])
obj_body = open(obj_path, "r")
content_length = os.stat(obj_path).st_size
swift_response = {}
swift_client.put_object(url, token, container_name, obj["name"],
obj_body, content_length, None, None,
obj['content_type'], obj["headers"], None,
None, None, swift_response)
obj_body.close()
os.remove(obj_path)
os.rmdir(path_container)
return JSONResponse("Container Policy updated correctly",
status=status.HTTP_201_CREATED)
return JSONResponse('Method ' + str(request.method) + ' not allowed.',
status=status.HTTP_405_METHOD_NOT_ALLOWED)
def update_container(request, project_id, container_name):
if request.method == 'PUT':
sp = JSONParser().parse(request)
token = get_token_connection(request)
url = settings.SWIFT_URL + "/AUTH_" + project_id
headers, obj_list = swift_client.get_container(url, token, container_name)
headers['X-Storage-Policy'] = sp
path_container = settings.SWIFT_CFG_TMP_DIR + "/" + container_name
os.mkdir(path_container)
for obj in obj_list:
fle = open(path_container + "/" + obj["name"], "w")
obj_headers, obj_body = swift_client.get_object(url, token, container_name, obj["name"])
fle.write(obj_body)
fle.close()
obj["headers"] = obj_headers
swift_client.delete_object(url, token, container_name, obj["name"])
swift_client.delete_container(url, token, container_name)
swift_client.put_container(url, token, container_name, headers)
for obj in obj_list:
obj_path = os.path.join(path_container, obj["name"])
obj_body = open(obj_path, "r")
content_length = os.stat(obj_path).st_size
swift_response = {}
swift_client.put_object(url, token, container_name, obj["name"],
obj_body, content_length,
None, None, obj['content_type'],
obj["headers"], None, None, None, swift_response)
obj_body.close()
os.remove(obj_path)
os.rmdir(path_container)
return JSONResponse("Container Policy updated correctly", status=status.HTTP_201_CREATED)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def dependency_deploy(request, dependency_id, project_id):
if request.method == 'PUT':
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Problems to connect with the DB', status=500)
dependency = r.hgetall("dependency:" + str(dependency_id))
if not dependency:
return JSONResponse('Dependency does not exist', status=404)
metadata = {'X-Object-Meta-Storlet-Dependency-Version': str(dependency["version"])}
if "path" not in dependency.keys():
return JSONResponse('Dependency path does not exist', status=404)
try:
dependency_file = open(dependency["path"], 'r')
content_length = None
response = dict()
token = get_token_connection(request)
url = settings.SWIFT_URL + "/AUTH_" + project_id
swift_client.put_object(url, token, 'dependency', dependency["name"], dependency_file, content_length,
None, None, "application/octet-stream", metadata, None, None, None, response)
except ClientException:
return JSONResponse(response.get("reason"), status=response.get('status'))
finally:
dependency_file.close()
status = response.get('status')
if status == 201:
if r.exists(str(project_id) + ":dependency:" + str(dependency['name'])):
return JSONResponse("Already deployed", status=200)
if r.lpush(str(project_id) + ":dependencies", str(dependency['name'])):
return JSONResponse("Deployed", status=201)
return JSONResponse("error", status=400)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=405)
def filter_deploy(request, filter_id, project_id, container=None, swift_object=None):
"""
Deploy a filter to a specific swift account.
"""
if request.method == 'PUT':
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Problems to connect with the DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
filter_data = r.hgetall("filter:" + str(filter_id))
if not filter_data:
return JSONResponse('Filter does not exist', status=status.HTTP_404_NOT_FOUND)
try:
params = JSONParser().parse(request)
except ParseError:
return JSONResponse("Invalid format or empty request params", status=status.HTTP_400_BAD_REQUEST)
# Get an identifier of this new policy
policy_id = r.incr("policies:id")
# Set the policy data
policy_data = {
"policy_id": policy_id,
"object_type": params['object_type'],
"object_size": params['object_size'],
"object_tag": params['object_tag'],
"object_name": ', '.join(r.lrange('object_type:' + params['object_type'], 0, -1)),
"execution_order": policy_id,
"params": params['params'],
"callable": False
}
if 'execution_server' in params:
if params['execution_server'] != 'default':
policy_data['execution_server'] = params['execution_server']
if 'execution_server_reverse' in params:
if params['execution_server_reverse'] != 'default':
policy_data['execution_server_reverse'] = params['execution_server_reverse']
if project_id.startswith('group:'):
projects_id = json.loads(r.hgetall('project_group:' + project_id.split(':')[1])['attached_projects'])
else:
projects_id = [project_id]
try:
for project in projects_id:
if container and swift_object:
target = ':'.join([project, container, swift_object])
elif container:
target = ':'.join([project, container])
else:
target = project
token = get_token_connection(request)
set_filter(r, target, filter_data.copy(), policy_data.copy(), token)
return JSONResponse(policy_id, status=status.HTTP_201_CREATED)
except SwiftClientError:
return JSONResponse('Error accessing Swift.', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
except StorletNotFoundException:
return JSONResponse('Storlet not found.', status=status.HTTP_404_NOT_FOUND)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def put(self, request, filter_id, format=None):
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
filter_name = "filter:" + str(filter_id)
if r.exists(filter_name):
file_obj = request.FILES['file']
filter_type = r.hget(filter_name, 'filter_type')
if (filter_type == 'storlet' and not (file_obj.name.endswith('.jar') or file_obj.name.endswith('.py'))) or \
(filter_type == 'native' and not file_obj.name.endswith('.py')):
return JSONResponse('Uploaded file is incompatible with filter type', status=status.HTTP_400_BAD_REQUEST)
if filter_type == 'storlet':
filter_dir = settings.STORLET_FILTERS_DIR
elif filter_type == 'native':
filter_dir = settings.NATIVE_FILTERS_DIR
make_sure_path_exists(filter_dir)
path = save_file(file_obj, filter_dir)
md5_etag = md5(path)
try:
filter_basename = os.path.basename(path)
content_length = os.stat(path).st_size
etag = str(md5_etag)
path = str(path)
r.hset(filter_name, "filter_name", filter_basename)
r.hset(filter_name, "path", path)
r.hset(filter_name, "content_length", content_length)
r.hset(filter_name, "etag", etag)
except RedisError:
return JSONResponse('Problems connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# Update info in already deployed filters
filter_data = r.hgetall(filter_name)
main = filter_data['main']
token = get_token_connection(request)
pipelines = r.keys('pipeline:*')
for pipeline in pipelines:
target = pipeline.replace('pipeline:', '')
filters_data = r.hgetall(pipeline)
for policy_id in filters_data:
parameters = {}
parameters["policy_id"] = policy_id
cfilter = eval(filters_data[policy_id].replace('true', '"True"').replace('false', '"False"'))
if cfilter['dsl_name'] == filter_id:
cfilter['filter_name'] = filter_basename
cfilter['content_length'] = content_length
cfilter['etag'] = etag
cfilter['path'] = path
cfilter['main'] = main
set_filter(r, target, cfilter, parameters, token)
if filter_type == 'native':
# synchronize metrics directory with all nodes
try:
rsync_dir_with_nodes(filter_dir, filter_dir)
except FileSynchronizationException as e:
return JSONResponse(e.message, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return JSONResponse('Filter has been updated', status=status.HTTP_201_CREATED)
return JSONResponse('Filter does not exist', status=status.HTTP_404_NOT_FOUND)
def deploy_static_policy(request, r, parsed_rule):
token = get_token_connection(request)
container = None
rules_to_parse = dict()
projects_crystal_enabled = r.lrange('projects_crystal_enabled', 0, -1)
project_list = get_project_list()
for target in parsed_rule.target:
if target[0] == 'TENANT':
project = target[1]
elif target[0] == 'CONTAINER':
project, container = target[1].split('/')
if project in project_list:
# Project ID
project_id = project
elif project in project_list.values():
# Project name
project_id = project_list.keys()[project_list.values().index(project)]
else:
raise ProjectNotFound()
if project_id not in projects_crystal_enabled:
raise ProjectNotCrystalEnabled()
if container:
target = os.path.join(project_id, container)
else:
target = project_id
rules_to_parse[target] = parsed_rule
for target in rules_to_parse.keys():
for action_info in rules_to_parse[target].action_list:
logger.info("Static policy, target rule: " + str(action_info))
cfilter = r.hgetall("filter:"+str(action_info.filter))
if not cfilter:
return JSONResponse("Filter does not exist", status=status.HTTP_404_NOT_FOUND)
if action_info.action == "SET":
# Get an identifier of this new policy
policy_id = r.incr("policies:id")
# Set the policy data
policy_data = {
"policy_id": policy_id,
"object_type": "",
"object_size": "",
"object_tag": "",
"object_name": "",
"execution_order": policy_id,
"params": "",
"callable": False
}
# Rewrite default values
if parsed_rule.object_list:
if parsed_rule.object_list.object_type:
policy_data["object_type"] = parsed_rule.object_list.object_type.object_value
policy_data["object_name"] = ', '.join(r.lrange('object_type:' + policy_data['object_type'], 0, -1))
if parsed_rule.object_list.object_size:
policy_data["object_size"] = [parsed_rule.object_list.object_size.operand,
parsed_rule.object_list.object_size.object_value]
if action_info.server_execution:
policy_data["execution_server"] = action_info.server_execution
if action_info.params:
policy_data["params"] = action_info.params
if action_info.callable:
policy_data["callable"] = True
# Deploy (an exception is raised if something goes wrong)
set_filter(r, target, cfilter, policy_data, token)
elif action_info.action == "DELETE":
unset_filter(r, target, cfilter, token)