def static_policy_detail(request, policy_id):
"""
Retrieve, update or delete a static policy.
"""
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
target = str(policy_id).split(':')[:-1]
target = ':'.join(target)
policy = str(policy_id).split(':')[-1]
if request.method == 'GET':
project_list = get_project_list()
project_list['global'] = 'Global'
policy_redis = r.hget("pipeline:" + str(target), policy)
data = json.loads(policy_redis)
filter_data = r.hgetall('filter:' + str(data['dsl_name']))
to_json_bools(filter_data, 'get', 'put', 'post', 'head', 'delete')
data['get'] = filter_data['get']
data['put'] = filter_data['put']
if 'post' in filter_data:
data['post'] = filter_data['post']
if 'head' in filter_data:
data['head'] = filter_data['head']
if 'delete' in filter_data:
data['delete'] = filter_data['delete']
data["id"] = policy
data["target_id"] = target
data["target_name"] = project_list[target.split(':')[0]]
return JSONResponse(data, status=200)
elif request.method == 'PUT':
data = JSONParser().parse(request)
try:
policy_redis = r.hget("pipeline:" + str(target), policy)
json_data = json.loads(policy_redis)
json_data.update(data)
json_data['object_name'] = ', '.join(r.lrange('object_type:' + json_data['object_type'], 0, -1))
json_data['execution_order'] = int(json_data['execution_order'])
r.hset("pipeline:" + str(target), policy, json.dumps(json_data))
return JSONResponse("Data updated", status=201)
except DataError:
return JSONResponse("Error updating data", status=400)
elif request.method == 'DELETE':
r.hdel('pipeline:' + target, policy)
policies_ids = r.keys('policy:*')
pipelines_ids = r.keys('pipeline:*')
if len(policies_ids) == 0 and len(pipelines_ids) == 0:
r.set('policies:id', 0)
# token = get_token_connection(request)
# unset_filter(r, target, filter_data, token)
return JSONResponse('Policy has been deleted', status=status.HTTP_204_NO_CONTENT)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def projects(request, project_id=None):
"""
GET: List all projects ordered by name
PUT: Save a project (enable)
DELETE: Delete a project (disable)
POST: Check if a project exist or is enabled
"""
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
if request.method == 'GET':
enabled_projects = r.lrange('projects_crystal_enabled', 0, -1)
return JSONResponse(enabled_projects, status=status.HTTP_200_OK)
if request.method == 'PUT':
project_list = get_project_list()
project_name = project_list[project_id]
if project_name == settings.MANAGEMENT_ACCOUNT:
return JSONResponse("Management project could not be set as Crystal project",
status=status.HTTP_400_BAD_REQUEST)
try:
# Set Manager as admin of the Crystal Project
keystone_client = get_keystone_admin_auth()
admin_role_id, reseller_admin_role_id, admin_user_id = get_admin_role_user_ids(keystone_client)
keystone_client.roles.grant(role=admin_role_id, user=admin_user_id, project=project_id)
keystone_client.roles.grant(role=reseller_admin_role_id, user=admin_user_id, project=project_id)
# Post Storlet and Dependency containers
url, token = get_swift_url_and_token(project_name)
swift_client.put_container(url, token, ".storlet")
swift_client.put_container(url, token, ".dependency")
headers = {'X-Account-Meta-Crystal-Enabled': True, 'X-Account-Meta-Storlet-Enabled': True}
swift_client.post_account(url, token, headers)
# Create project docker image
create_docker_image(r, project_id)
r.lpush('projects_crystal_enabled', project_id)
return JSONResponse("Crystal Project correctly enabled", status=status.HTTP_201_CREATED)
except:
return JSONResponse("Error Enabling Crystal Project", status=status.HTTP_400_BAD_REQUEST)
if request.method == 'DELETE':
try:
project_list = get_project_list()
project_name = project_list[project_id]
# Delete Storlet and Dependency containers
try:
url, token = get_swift_url_and_token(project_name)
headers = {'X-Account-Meta-Crystal-Enabled': '', 'X-Account-Meta-Storlet-Enabled': ''}
swift_client.post_account(url, token, headers)
swift_client.delete_container(url, token, ".storlet")
swift_client.delete_container(url, token, ".dependency")
except:
pass
# Delete Manager as admin of the Crystal Project
keystone_client = get_keystone_admin_auth()
admin_role_id, reseller_admin_role_id, admin_user_id = get_admin_role_user_ids(keystone_client)
try:
keystone_client.roles.revoke(role=admin_role_id, user=admin_user_id, project=project_id)
keystone_client.roles.revoke(role=reseller_admin_role_id, user=admin_user_id, project=project_id)
except:
pass
# Delete project docker image
delete_docker_image(r, project_id)
r.lrem('projects_crystal_enabled', project_id)
return JSONResponse("Crystal project correctly disabled.", status=status.HTTP_201_CREATED)
except RedisError:
return JSONResponse("Error inserting data", status=status.HTTP_400_BAD_REQUEST)
if request.method == 'POST':
enabled_projects = r.lrange('projects_crystal_enabled', 0, -1)
if project_id in enabled_projects:
return JSONResponse(project_id, status=status.HTTP_200_OK)
return JSONResponse('The project with id: ' + str(project_id) + ' does not exist.',
status=status.HTTP_404_NOT_FOUND)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def projects(request, project_id=None):
"""
GET: List all projects ordered by name
PUT: Save a project (enable)
DELETE: Delete a project (disable)
POST: Check if a project exist or is enabled
"""
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
if request.method == 'GET':
enabled_projects = r.lrange('projects_crystal_enabled', 0, -1)
return JSONResponse(enabled_projects, status=status.HTTP_200_OK)
if request.method == 'PUT':
project_list = get_project_list()
project_name = project_list[project_id]
if project_name == settings.MANAGEMENT_ACCOUNT:
return JSONResponse(
"Management project could not be set as Crystal project",
status=status.HTTP_400_BAD_REQUEST)
try:
# Set Manager as admin of the Crystal Project
keystone_client = get_keystone_admin_auth()
admin_role_id, reseller_admin_role_id, admin_user_id = get_admin_role_user_ids(
keystone_client)
keystone_client.roles.grant(role=admin_role_id,
user=admin_user_id,
project=project_id)
keystone_client.roles.grant(role=reseller_admin_role_id,
user=admin_user_id,
project=project_id)
# Post Storlet and Dependency containers
url, token = get_swift_url_and_token(project_name)
swift_client.put_container(url, token, ".storlet")
swift_client.put_container(url, token, ".dependency")
headers = {
'X-Account-Meta-Crystal-Enabled': True,
'X-Account-Meta-Storlet-Enabled': True
}
swift_client.post_account(url, token, headers)
# Create project docker image
create_docker_image(r, project_id)
r.lpush('projects_crystal_enabled', project_id)
return JSONResponse("Crystal Project correctly enabled",
status=status.HTTP_201_CREATED)
except:
return JSONResponse("Error Enabling Crystal Project",
status=status.HTTP_400_BAD_REQUEST)
if request.method == 'DELETE':
try:
project_list = get_project_list()
project_name = project_list[project_id]
# Delete Storlet and Dependency containers
try:
url, token = get_swift_url_and_token(project_name)
headers = {
'X-Account-Meta-Crystal-Enabled': '',
'X-Account-Meta-Storlet-Enabled': ''
}
swift_client.post_account(url, token, headers)
swift_client.delete_container(url, token, ".storlet")
swift_client.delete_container(url, token, ".dependency")
except:
pass
# Delete Manager as admin of the Crystal Project
keystone_client = get_keystone_admin_auth()
admin_role_id, reseller_admin_role_id, admin_user_id = get_admin_role_user_ids(
keystone_client)
try:
keystone_client.roles.revoke(role=admin_role_id,
user=admin_user_id,
project=project_id)
keystone_client.roles.revoke(role=reseller_admin_role_id,
user=admin_user_id,
project=project_id)
except:
pass
# Delete project docker image
delete_docker_image(r, project_id)
r.lrem('projects_crystal_enabled', project_id)
return JSONResponse("Crystal project correctly disabled.",
status=status.HTTP_201_CREATED)
except RedisError:
return JSONResponse("Error inserting data",
status=status.HTTP_400_BAD_REQUEST)
if request.method == 'POST':
enabled_projects = r.lrange('projects_crystal_enabled', 0, -1)
if project_id in enabled_projects:
return JSONResponse(project_id, status=status.HTTP_200_OK)
return JSONResponse('The project with id: ' + str(project_id) +
' does not exist.',
status=status.HTTP_404_NOT_FOUND)
return JSONResponse('Method ' + str(request.method) + ' not allowed.',
status=status.HTTP_405_METHOD_NOT_ALLOWED)
def access_control_detail(request, policy_id):
"""
Get or delete an access control.
"""
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB', status=500)
target_id = str(policy_id).split(':')[:-1]
target_id = ':'.join(target_id)
acl_id = str(policy_id).split(':')[-1]
if request.method == 'GET':
try:
project_list = get_project_list()
policy_redis = r.hget("acl:" + str(target_id), acl_id)
policy = json.loads(policy_redis)
to_json_bools(policy, 'list', 'write', 'read')
target_split = target_id.split(':')
if len(target_split) > 1:
target_name = project_list[target_id.split(':')[0]]+'/'+target_id.split(':')[1]
else:
target_name = project_list[target_id.split(':')[0]]
p = {'id': acl_id,
'target_id': target_id,
'target_name': target_name}
p.update(policy)
return JSONResponse(p, status=status.HTTP_200_OK)
except DataError:
return JSONResponse("Error retrieving policy", status=400)
if request.method == 'DELETE':
try:
r.hdel("acl:" + str(target_id), acl_id)
except DataError:
return JSONResponse("Error retrieving policy", status=400)
return JSONResponse("Access policy correctly removed", status=status.HTTP_200_OK)
if request.method == 'PUT':
data = JSONParser().parse(request)
try:
policy_redis = r.hget("acl:" + str(target_id), acl_id)
policy = json.loads(policy_redis)
access = data.pop('access')
if access == 'list':
data['list'] = True
data['read'] = False
data['write'] = False
elif access == 'read':
data['list'] = False
data['read'] = True
data['write'] = False
elif access == 'read-write':
data['list'] = False
data['read'] = True
data['write'] = True
policy.update(data)
policy['object_name'] = ', '.join(r.lrange('object_type:' + policy['object_type'], 0, -1))
r.hset("acl:" + str(target_id), acl_id, json.dumps(policy))
return JSONResponse('Data updated', status=status.HTTP_201_CREATED)
except DataError:
return JSONResponse("Error creating policy", status=400)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=405)
def access_control(request):
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB', status=500)
if request.method == 'GET':
acl = []
project_list = get_project_list()
try:
keys = r.keys('acl:*')
for it in keys:
for key, value in r.hgetall(it).items():
policy = json.loads(value)
to_json_bools(policy, 'list', 'write', 'read')
target_id = it.replace('acl:', '')
target_split = target_id.split(':')
if len(target_split) > 1:
target_name = project_list[target_id.split(':')[0]]+'/'+target_id.split(':')[1]
else:
target_name = project_list[target_id.split(':')[0]]
p = {'id': key,
'target_id': target_id,
'target_name': target_name}
p.update(policy)
acl.append(p)
except DataError:
return JSONResponse("Error retrieving policy", status=400)
return JSONResponse(acl, status=status.HTTP_200_OK)
if request.method == 'POST':
data = JSONParser().parse(request)
try:
if data['container_id']:
key = 'acl:' + data['project_id'] + ':' + data['container_id']
else:
key = 'acl:' + data['project_id']
acl_id = str(r.incr('acls:id'))
data.pop('container_id')
data.pop('project_id')
data['object_name'] = ', '.join(r.lrange('object_type:' + data['object_type'], 0, -1))
identity = data.pop('identity')
access = data.pop('access')
if access == 'list':
data['list'] = True
data['read'] = False
data['write'] = False
elif access == 'read':
data['list'] = False
data['read'] = True
data['write'] = False
elif access == 'read-write':
data['list'] = False
data['read'] = True
data['write'] = True
if 'user_id' in identity:
data['user_id'] = identity.replace('user_id:', '')
data['group_id'] = ''
elif 'group_id' in identity:
data['group_id'] = identity.replace('group_id:', '')
data['user_id'] = ''
r.hset(key, acl_id, json.dumps(data))
return JSONResponse("Access control policy created", status=201)
except DataError:
return JSONResponse("Error creating policy", status=400)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=405)
def deploy_dynamic_policy(r, rule_string, parsed_rule, http_host):
rules_to_parse = dict()
project = None
container = None
# TODO: get only the Crystal enabled projects
projects_crystal_enabled = r.lrange('projects_crystal_enabled', 0, -1)
project_list = get_project_list()
for target in parsed_rule.target:
if target[0] == 'TENANT':
project = target[1]
elif target[0] == 'CONTAINER':
project, container = target[1].split('/')
if project in project_list:
# Project ID
project_id = project
project_name = project_list[project_id]
elif project in project_list.values():
# Project name
project_name = project
project_id = project_list.keys()[project_list.values().index(project)]
else:
raise ProjectNotFound()
if project_id not in projects_crystal_enabled:
raise ProjectNotCrystalEnabled()
if container:
target = project_name+":"+os.path.join(project_id, container)
# target = crystal:f1bf1d778939445dbd20734cbd98de16/data
else:
target = project_name+":"+project_id
# target = crystal:f1bf1d778939445dbd20734cbd98de16
rules_to_parse[target] = parsed_rule
for target in rules_to_parse.keys():
for action_info in rules_to_parse[target].action_list:
container = None
if '/' in target:
# target includes a container
project, container = target.split('/')
project_name, project_id = project.split(':')
target_id = os.path.join(project_id, container)
target_name = os.path.join(project_name, container)
else:
target_name, target_id = target.split(':')
policy_id = r.incr("policies:id")
rule_id = 'policy:' + str(policy_id)
if action_info.transient:
transient = True
location = settings.RULE_TRANSIENT_MODULE
else:
transient = False
location = settings.RULE_MODULE
policy_location = os.path.join(settings.PYACTOR_URL, location, str(rule_id))
# FIXME Should we recreate a static rule for each target and action??
condition_re = re.compile(r'.* (WHEN .*) DO .*', re.M | re.I)
condition_str = condition_re.match(rule_string).group(1)
object_type = ""
object_size = ""
object_tag = ""
object_name = ""
if parsed_rule.object_list:
if parsed_rule.object_list.object_type:
object_type = parsed_rule.object_list.object_type.object_value
object_name = ', '.join(r.lrange('object_type:' + object_type, 0, -1))
if parsed_rule.object_list.object_tag:
object_tag = parsed_rule.object_list.object_tag.object_value
if parsed_rule.object_list.object_size:
object_size = [parsed_rule.object_list.object_size.operand,
parsed_rule.object_list.object_size.object_value]
policy_data = {"id": policy_id,
"target_id": target_id,
"target_name": target_name,
"filter": action_info.filter,
"parameters": action_info.params,
"action": action_info.action,
"condition": condition_str.replace('WHEN ', ''),
"object_name": object_name,
"object_type": object_type,
"object_size": object_size,
"object_tag": object_tag,
"transient": transient,
"policy_location": policy_location,
"status": 'Alive'}
start_dynamic_policy_actor(policy_data, http_host)
# Add policy into Redis
r.hmset('policy:' + str(policy_id), policy_data)
def deploy_static_policy(request, r, parsed_rule):
token = get_token_connection(request)
container = None
rules_to_parse = dict()
projects_crystal_enabled = r.lrange('projects_crystal_enabled', 0, -1)
project_list = get_project_list()
for target in parsed_rule.target:
if target[0] == 'TENANT':
project = target[1]
elif target[0] == 'CONTAINER':
project, container = target[1].split('/')
if project in project_list:
# Project ID
project_id = project
elif project in project_list.values():
# Project name
project_id = project_list.keys()[project_list.values().index(project)]
else:
raise ProjectNotFound()
if project_id not in projects_crystal_enabled:
raise ProjectNotCrystalEnabled()
if container:
target = os.path.join(project_id, container)
else:
target = project_id
rules_to_parse[target] = parsed_rule
for target in rules_to_parse.keys():
for action_info in rules_to_parse[target].action_list:
logger.info("Static policy, target rule: " + str(action_info))
cfilter = r.hgetall("filter:"+str(action_info.filter))
if not cfilter:
return JSONResponse("Filter does not exist", status=status.HTTP_404_NOT_FOUND)
if action_info.action == "SET":
# Get an identifier of this new policy
policy_id = r.incr("policies:id")
# Set the policy data
policy_data = {
"policy_id": policy_id,
"object_type": "",
"object_size": "",
"object_tag": "",
"object_name": "",
"execution_order": policy_id,
"params": "",
"callable": False
}
# Rewrite default values
if parsed_rule.object_list:
if parsed_rule.object_list.object_type:
policy_data["object_type"] = parsed_rule.object_list.object_type.object_value
policy_data["object_name"] = ', '.join(r.lrange('object_type:' + policy_data['object_type'], 0, -1))
if parsed_rule.object_list.object_size:
policy_data["object_size"] = [parsed_rule.object_list.object_size.operand,
parsed_rule.object_list.object_size.object_value]
if action_info.server_execution:
policy_data["execution_server"] = action_info.server_execution
if action_info.params:
policy_data["params"] = action_info.params
if action_info.callable:
policy_data["callable"] = True
# Deploy (an exception is raised if something goes wrong)
set_filter(r, target, cfilter, policy_data, token)
elif action_info.action == "DELETE":
unset_filter(r, target, cfilter, token)
def policy_list(request):
"""
List all policies (sorted by execution_order). Deploy new policies via DSL.
"""
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
if request.method == 'GET':
if 'static' in str(request.path):
project_list = get_project_list()
project_list['global'] = 'Global'
keys = r.keys("pipeline:*")
policies = []
for it in keys:
for key, value in r.hgetall(it).items():
policy = json.loads(value)
filter_data = r.hgetall('filter:' + str(policy['dsl_name']))
to_json_bools(filter_data, 'get', 'put', 'post', 'head', 'delete')
target_id = it.replace('pipeline:', '')
policy = {'id': key, 'target_id': target_id,
'target_name': project_list[target_id.split(':')[0]],
'filter_name': policy['filter_name'],
'object_type': policy['object_type'],
'object_size': policy['object_size'],
'object_tag': policy['object_tag'],
'object_name': ', '.join(r.lrange('object_type:' + policy['object_type'], 0, -1)),
'execution_server': policy['execution_server'],
'reverse': policy['reverse'],
'execution_order': policy['execution_order'],
'params': policy['params'],
'put': filter_data['put'],
'get': filter_data['get']}
if 'post' in filter_data:
policy['post'] = filter_data['post']
if 'head' in filter_data:
policy['head'] = filter_data['head']
if 'delete' in filter_data:
policy['delete'] = filter_data['delete']
policies.append(policy)
sorted_policies = sorted(policies, key=lambda x: int(itemgetter('execution_order')(x)))
return JSONResponse(sorted_policies, status=status.HTTP_200_OK)
elif 'dynamic' in str(request.path):
keys = r.keys("policy:*")
policies = []
for key in keys:
policy = r.hgetall(key)
policies.append(policy)
return JSONResponse(policies, status=status.HTTP_200_OK)
else:
return JSONResponse("Invalid request", status=status.HTTP_400_BAD_REQUEST)
if request.method == 'POST':
# New Policy
rules_string = request.body.splitlines()
for rule_string in rules_string:
#
# Rules improved:
# TODO: Handle the new parameters of the rule
# Add containers and object in rules
# Add execution server in rules
# Add object type in rules
#
try:
condition_list, rule_parsed = dsl_parser.parse(rule_string)
if condition_list:
# Dynamic Rule
http_host = request.META['HTTP_HOST']
deploy_dynamic_policy(r, rule_string, rule_parsed, http_host)
else:
# Static Rule
deploy_static_policy(request, r, rule_parsed)
except SwiftClientError:
return JSONResponse('Error accessing Swift.', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
except StorletNotFoundException:
return JSONResponse('Storlet not found.', status=status.HTTP_404_NOT_FOUND)
except ProjectNotFound:
return JSONResponse('Invalid Project Name/ID. The Project does not exist.', status=status.HTTP_404_NOT_FOUND)
except ProjectNotCrystalEnabled:
return JSONResponse('The project is not Crystal Enabled. Verify it in the Projects panel.',
status=status.HTTP_404_NOT_FOUND)
except Exception as e:
logger.info('Unexpected exception: ' + e.message)
return JSONResponse('Please, review the rule, and start the related workload '
'metric before creating a new policy', status=status.HTTP_401_UNAUTHORIZED)
return JSONResponse('Policies added successfully!', status=status.HTTP_201_CREATED)
if request.method == 'PUT':
# Dynamic Policy From form
http_host = request.META['HTTP_HOST']
data = JSONParser().parse(request)
policy_id = r.incr("policies:id")
rule_id = 'policy:' + str(policy_id)
action = data['action']
project_id = data['project_id']
container = data['container_id']
if project_id == 'global':
project_name = 'Global'
else:
project_list = get_project_list()
project_name = project_list[project_id]
if container:
target_id = os.path.join(project_id, container)
target_name = os.path.join(project_name, container)
else:
target_id = project_id
target_name = project_name
if data['transient']:
location = settings.RULE_TRANSIENT_MODULE
else:
location = settings.RULE_MODULE
policy_location = os.path.join(settings.PYACTOR_URL, location, str(rule_id))
policy_data = {"id": policy_id,
"target_id": target_id,
"target_name": target_name,
"filter": data['filter_id'],
"parameters": data['params'],
"action": action,
"condition": data['workload_metric']+' '+data['condition'],
"object_type": data['object_type'],
"object_size": data['object_size'],
"object_tag": data['object_tag'],
"object_name": ', '.join(r.lrange('object_type:' + data['object_type'], 0, -1)),
"transient": data['transient'],
"policy_location": policy_location,
"status": 'Alive'}
start_dynamic_policy_actor(policy_data, http_host)
try:
r.hmset(rule_id, policy_data)
return JSONResponse("Policy inserted correctly", status=status.HTTP_201_CREATED)
except RedisError:
return JSONResponse("Error inserting policy", status=status.HTTP_400_BAD_REQUEST)
return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)