def put_data_object(self, path):
# Check if a collection with the name exists
collection = Collection.find(path)
if collection:
# Try to put a data_object when a collection of the same name
# already exists
self.logger.info(u"Impossible to create a new resource, the collection '{}' already exists, try to update it".format(path))
return self.put_container(path)
parent, name = split(path)
# Check if the resource already exists
resource = Resource.find(path)
# Check permissions
if resource:
# Update Resource
if not resource.user_can(self.user, "edit"):
self.logger.warning(u"User {} tried to modify resource at '{}'".format(self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
else:
# Create Resource
parent_collection = Collection.find(parent)
if not parent_collection:
self.logger.info(u"Fail to create a resource at '{}', collection doesn't exist".format(path))
return Response(status=HTTP_404_NOT_FOUND)
# Check if user can create a new resource in the collection
if not parent_collection.user_can(self.user, "write"):
self.logger.warning(u"User {} tried to create new resource at '{}'".format(self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
# All permissions are checked, we can proceed to create/update
if self.http_mode:
return self.put_data_object_http(parent, name, resource)
else:
return self.put_data_object_cdmi(parent, name, resource)
def view_collection(request, path):
if not path:
path = '/'
collection = Collection.find(path)
if not collection:
raise Http404()
if not collection.user_can(request.user, "read") and not collection.is_root:
# If the user can't read, then return 404 rather than 403 so that
# we don't leak information.
raise Http404()
paths = []
full = ""
for p in collection.path.split('/'):
if not p:
continue
full = u"{}/{}".format(full, p)
paths.append((p, full))
children_c, children_r = collection.get_child()
children_c.sort(key=lambda x: x.lower())
children_r.sort(key=lambda x: x.lower())
ctx = {
'collection': collection.to_dict(request.user),
'children_c': [Collection.find(merge(path,c)).to_dict(request.user) for c in children_c],
'children_r': [Resource.find(merge(path,c)).simple_dict(request.user) for c in children_r],
'collection_paths': paths,
'empty': len(children_c) + len(children_r) == 0,
}
return render(request, 'archive/index.html', ctx)
def view_resource(request, path):
resource = Resource.find(path)
if not resource:
raise Http404()
if not resource.user_can(request.user, "read"):
raise PermissionDenied
container = Collection.find(resource.container)
if not container:
# TODO: the container has to be there. If not it may be a network
# issue with Cassandra so we try again before raising an error to the
# user
container = Collection.find(resource.container)
if not container:
return HttpResponse(status=408,
content="Unable to find parent container '{}'".format(resource.container))
paths = []
full = ""
for p in container.path.split('/'):
if not p:
continue
full = u"{}/{}".format(full, p)
paths.append((p, full))
ctx = {
"resource": resource.full_dict(request.user),
"container": container,
"container_path": container.path,
"collection_paths": paths
}
return render(request, 'archive/resource/view.html', ctx)
def new_collection(request, parent):
parent_collection = Collection.find(parent)
if not parent_collection.user_can(request.user, "write"):
raise PermissionDenied
keys = get_collection_keys()
mdata = collections.OrderedDict()
for k in keys:
mdata[k] = ""
if not mdata:
mdata[""] = ""
read_access, write_access = parent_collection.get_acl_list()
initial = {
'metadata': json.dumps(mdata),
"read_access": read_access,
"write_access": write_access
}
form = CollectionNewForm(request.POST or None, initial=initial)
if request.method == 'POST':
if form.is_valid():
data = form.cleaned_data
try:
name = data['name']
parent = parent_collection.path
metadata = {}
for k, v in json.loads(data['metadata']):
if k in metadata:
if isinstance(metadata[k], list):
metadata[k].append(v)
else:
metadata[k] = [metadata[k], v]
else:
metadata[k] = v
collection = Collection.create(name=name,
container=parent,
metadata=metadata,
username=request.user.name)
collection.create_acl_list(data['read_access'], data['write_access'])
messages.add_message(request, messages.INFO,
u"New collection '{}' created" .format(collection.name))
return redirect('archive:view', path=collection.path)
except CollectionConflictError:
messages.add_message(request, messages.ERROR,
"That name is in use in the current collection")
except ResourceConflictError:
messages.add_message(request, messages.ERROR,
"That name is in use in the current collection")
groups = Group.objects.all()
return render(request, 'archive/new.html', {'form': form, "parent": parent_collection, "groups": groups})
def delete_container(self, path):
collection = Collection.find(path)
if not collection:
self.logger.info(u"Fail to delete collection at '{}'".format(path))
return Response(status=HTTP_404_NOT_FOUND)
if not collection.user_can(self.user, "delete"):
self.logger.warning(u"User {} tried to delete container '{}'".format(self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
Collection.delete_all(collection.path)
self.logger.info(u"The container '{}' was successfully deleted".format(path))
return Response(status=HTTP_204_NO_CONTENT)
def create_collection(self, parent_path, name, path):
try:
c = Collection.create(name,
container=parent_path,
metadata=None,
username=self.user.name)
c.create_acl_list(self.groups, self.groups)
except CollectionConflictError:
# Collection already exists
c = Collection.find(path)
self.collection_cache[path] = c
return c
def get_authorized_actions(self, user):
""""Get available actions for user according to a group"""
# Check permission on the parent container if there's no action
# defined at this level
acl = self.get_acl()
if not acl:
from indigo.models import Collection
parent_container = Collection.find(self.container)
return parent_container.get_authorized_actions(user)
actions = set([])
for gid in user.groups:
if gid in acl:
ace = acl[gid]
level = acemask_to_str(ace.acemask, True)
if level == "read":
actions.add("read")
elif level == "write":
actions.add("write")
actions.add("delete")
actions.add("edit")
elif level == "read/write":
actions.add("read")
actions.add("write")
actions.add("delete")
actions.add("edit")
return actions
def edit_resource(request, path):
# Requires edit on resource
resource = Resource.find(path)
if not resource:
raise Http404()
container = Collection.find(resource.container)
if not container:
raise Http404()
if not resource.user_can(request.user, "edit"):
raise PermissionDenied
if request.method == "POST":
form = ResourceForm(request.POST)
if form.is_valid():
metadata = {}
for k, v in json.loads(form.cleaned_data['metadata']):
if k in metadata:
if isinstance(metadata[k], list):
metadata[k].append(v)
else:
metadata[k] = [metadata[k], v]
else:
metadata[k] = v
try:
data = form.cleaned_data
resource.update(metadata=metadata, username=request.user.name)
resource.create_acl_list(data['read_access'], data['write_access'])
return redirect('archive:resource_view', path=resource.path)
except ResourceConflictError:
messages.add_message(request, messages.ERROR,
"That name is in use withinin the current collection")
else:
md = resource.get_cdmi_metadata()
metadata = json.dumps(md)
if not md:
metadata = '{"":""}'
read_access, write_access = resource.get_acl_list()
initial_data = {'name': resource.name, 'metadata': metadata,
'read_access': read_access,
'write_access': write_access
}
form = ResourceForm(initial=initial_data)
ctx = {
"form": form,
"resource": resource,
"container": container,
"groups": Group.objects.all()
}
return render(request, 'archive/resource/edit.html', ctx)
def get_collection(self, path):
c = self.collection_cache.get(path, None)
if c:
return c
c = Collection.find(path)
if c:
self.collection_cache[path] = c
return c
return None
def crud_id(request, id):
# The URL should end with a '/'
id = id.replace('/', '')
collection = Collection.find_by_uuid(id)
if collection:
return redirect('cdmi:api_cdmi', path=collection.path())
else:
resource = Resource.find_by_uuid(id)
if resource:
return redirect('cdmi:api_cdmi', path=resource.path())
else:
return Response(status=HTTP_404_NOT_FOUND)
def delete_collection(request, path):
"delete_coll"
coll = Collection.find(path)
if not coll:
raise Http404
if not coll.user_can(request.user, "delete"):
raise PermissionDenied
if request.method == "POST":
parent_coll = Collection.find(coll.path)
if parent_coll:
parent_path = parent_coll.container
else:
# Just in case
parent_path = ''
Collection.delete_all(coll.path, username=request.user.name)
messages.add_message(request, messages.INFO,
u"The collection '{}' has been deleted".format(coll.name))
return redirect('archive:view', path=parent_path)
return render(request, 'archive/delete.html', {'collection': coll})
def read_container(self, path):
collection = Collection.find(path)
if not collection:
self.logger.info(u"Fail to read a collection at '{}'".format(path))
return Response(status=HTTP_404_NOT_FOUND)
if not collection.user_can(self.user, "read"):
self.logger.warning(u"User {} tried to read container at '{}'".format(self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
cdmi_container = CDMIContainer(collection, self.api_root)
if self.http_mode:
# HTTP Request, unsupported
self.logger.warning(u"Read container '{}' using HTTP is undefined".format(path))
return Response(status=HTTP_406_NOT_ACCEPTABLE)
else:
# Read using CDMI
return self.read_container_cdmi(cdmi_container)
def delete_data_object(self, path):
resource = Resource.find(path)
if not resource:
collection = Collection.find(path)
if collection:
self.logger.info(u"Fail to delete resource at '{}', test if it's a collection".format(path))
return self.delete_container(path)
else:
self.logger.info(u"Fail to delete resource at '{}'".format(path))
return Response(status=HTTP_404_NOT_FOUND)
if not resource.user_can(self.user, "delete"):
self.logger.warning(u"User {} tried to delete resource '{}'".format(self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
resource.delete()
self.logger.info(u"The resource '{}' was successfully deleted".format(path))
return Response(status=HTTP_204_NO_CONTENT)
def ready(self):
from indigo.models import initialise, Collection, User, Group
logging.basicConfig(level=logging.WARNING)
logging.getLogger("models").setLevel(logging.WARNING)
logging.getLogger("dse.policies").setLevel(logging.WARNING)
logging.getLogger("dse.cluster").setLevel(logging.WARNING)
logging.getLogger("dse.cqlengine.management").setLevel(logging.WARNING)
cfg = get_config(None)
initialise(keyspace=cfg.get('KEYSPACE', 'indigo'),
hosts=cfg.get('CASSANDRA_HOSTS', ('127.0.0.1', )))
# Try to get the root. It will create it if it doesn't exist
root = Collection.get_root()
# TODO: Review that at some point
# Check that the graph vertices for users are still there
User.check_graph_users()
def edit_collection(request, path):
coll = Collection.find(path)
if not coll:
raise Http404
if not coll.user_can(request.user, "edit"):
raise PermissionDenied
if request.method == "POST":
form = CollectionForm(request.POST)
if form.is_valid():
metadata = {}
for k, v in json.loads(form.cleaned_data['metadata']):
if k in metadata:
if isinstance(metadata[k], list):
metadata[k].append(v)
else:
metadata[k] = [metadata[k], v]
else:
metadata[k] = v
try:
data = form.cleaned_data
coll.update(metadata=metadata, username=request.user.name)
coll.create_acl_list(data['read_access'], data['write_access'])
return redirect('archive:view', path=coll.path)
except CollectionConflictError:
messages.add_message(request, messages.ERROR,
"That name is in use in the current collection")
else:
md = coll.get_cdmi_metadata()
metadata = json.dumps(md)
if not md:
metadata = '{"":""}'
read_access, write_access = coll.get_acl_list()
initial_data = {'name': coll.name,
'metadata': metadata,
'read_access': read_access,
'write_access': write_access}
form = CollectionForm(initial=initial_data)
groups = Group.objects.all()
return render(request, 'archive/edit.html', {'form': form, 'collection': coll, 'groups': groups})
def delete_resource(request, path):
resource = Resource.find(path)
if not resource:
raise Http404
if not resource.user_can(request.user, "delete"):
raise PermissionDenied
container = Collection.find(resource.container)
if request.method == "POST":
resource.delete(username=request.user.name)
messages.add_message(request, messages.INFO,
"The resource '{}' has been deleted".format(resource.name))
return redirect('archive:view', path=container.path)
# Requires delete on resource
ctx = {
"resource": resource,
"container": container,
}
return render(request, 'archive/resource/delete.html', ctx)
def read_data_object(self, path):
"""Read a resource"""
resource = Resource.find(path)
if not resource:
collection = Collection.find(path)
if collection:
self.logger.info(u"Fail to read a resource at '{}', test if it's a collection".format(path))
return self.read_container(path)
else:
self.logger.info(u"Fail to read a resource at '{}'".format(path))
return Response(status=HTTP_404_NOT_FOUND)
if not resource.user_can(self.user, "read"):
self.logger.warning(u"User {} tried to read resource at '{}'".format(self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
cdmi_resource = CDMIResource(resource, self.api_root)
if self.http_mode:
if cdmi_resource.is_reference():
return self.read_data_object_reference(cdmi_resource)
else:
return self.read_data_object_http(cdmi_resource)
else:
return self.read_data_object_cdmi(cdmi_resource)
def create_resource(self, path):
container, name = split(path)
Collection.create(name, container)
def put_container(self, path):
# Check if the container already exists
collection = Collection.find(path)
if collection:
# Update
if not collection.user_can(self.user, "edit"):
self.logger.warning(u"User {} tried to modify collection at '{}'".format(self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
if self.http_mode:
# HTTP Request, unsupported
self.logger.warning(u"Update collection '{}' using HTTP is undefined".format(path))
return Response(status=HTTP_406_NOT_ACCEPTABLE)
res = self.put_container_metadata(collection)
return Response(status=res)
# Create Collection
parent, name = split(path)
if name.startswith("cdmi_"):
return Response("cdmi_ prefix is not a valid name for a container",
status=HTTP_400_BAD_REQUEST)
parent_collection = Collection.find(parent)
if not parent_collection:
self.logger.info(u"Fail to create a collection at '{}', parent collection doesn't exist".format(path))
return Response(status=HTTP_404_NOT_FOUND)
# Check if user can create a new collection in the collection
if not parent_collection.user_can(self.user, "write"):
self.logger.warning(u"User {} tried to create new collection at '{}'".format(self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
body = OrderedDict()
try:
collection = Collection.create(name=name,
container=parent)
except ResourceConflictError:
return Response(status=HTTP_409_CONFLICT)
cdmi_container = CDMIContainer(collection, self.api_root)
delayed = False
res = self.put_container_metadata(collection)
if res != HTTP_204_NO_CONTENT:
return Response(status=res)
if self.http_mode:
# Specification states that:
#
# A response message body may be provided as per RFC 2616.
#
# Send the CDMI response but with Content-Type = application/json
content_type = 'application/json'
# Mandatory completionStatus
# Does not accept CDMI - cannot return "202 Accepted"
# Try to wait until complete
while not path_exists(path):
# Wait for 5 seconds
time.sleep(5)
response_status = "201 Created"
body['completionStatus'] = "Complete"
else:
# CDMI mode
for field, value in FIELDS_CONTAINER.items():
get_field = getattr(cdmi_container, 'get_{}'.format(field))
body[field] = get_field()
if delayed:
response_status = HTTP_202_ACCEPTED
body['completionStatus'] = "Processing"
else:
response_status = HTTP_201_CREATED
body['completionStatus'] = "Complete"
return JsonResponse(body,
content_type=body['objectType'],
status=response_status)
def home(request):
notifications = Notification.recent(10)
activities = []
for notif in notifications:
t = template.Template(notif['tmpl'])
obj_uuid = notif['object_uuid']
object = None
if notif['object_type'] == OBJ_RESOURCE:
object = Resource.find(obj_uuid)
if object:
object_dict = object.to_dict()
else:
object_dict = {'name': obj_uuid}
elif notif['object_type'] == OBJ_COLLECTION:
object = Collection.find(obj_uuid)
if object:
object_dict = object.to_dict()
else:
object_dict = {'name': obj_uuid}
elif notif['object_type'] == OBJ_USER:
object = User.find(obj_uuid)
if object:
object_dict = object.to_dict()
else:
# User has been deleted it can't be find by uuid
# look in payload of the message to get the name
if notif['operation'] in [OP_CREATE, OP_UPDATE]:
name = notif['payload']['post']['name']
else: # OP_DELETE
name = notif['payload']['pre']['name']
object_dict = {'name': name}
elif notif['object_type'] == OBJ_GROUP:
object = Group.find(obj_uuid)
if object:
object_dict = object.to_dict()
else:
# User has been deleted it can't be find by uuid
# look in payload of the message to get the name
if notif['operation'] in [OP_CREATE, OP_UPDATE]:
name = notif['payload']['post']['name']
else: # OP_DELETE
name = notif['payload']['pre']['name']
object_dict = {'uuid': obj_uuid,
'name': name}
user_dict = {}
if notif['username']:
user = User.find(notif['username'])
if user:
user_dict = user.to_dict()
variables = {
'user': user_dict,
'when': notif['when'],
'object': object_dict
}
ctx = template.Context(variables)
activities.append({'html': t.render(ctx)})
return render(request, 'activity/index.html', {'activities': activities})
def do_work(self):
# if self.include_pattern:
# include_pattern = self.include_pattern.lower()
# matcher = lambda x: include_pattern in x.lower()
# else:
# matcher = lambda x: True
timer = TimerCounter()
root_collection = Collection.get_root()
self.collection_cache["/"] = root_collection
t0 = time.time()
if self.include_pattern:
start_dir = None
for (path, dirs, files) in os.walk(self.folder,
topdown=True,
followlinks=True):
for k in dirs:
if self.include_pattern.lower() in k.lower():
start_dir = os.path.join(path, k)
break
if start_dir:
break
else:
start_dir = self.folder
if start_dir:
print "STARTING AT ", start_dir
else:
print "No start dir matching {0} found ... giving up ".format(
self.include_pattern)
return None
for (path, dirs, files) in os.walk(start_dir,
topdown=True,
followlinks=True):
if '/.' in path:
continue # Ignore .paths
path = path.replace(self.folder, '')
parent_path, name = split(path)
t1 = time.time()
msg = "Processing {0} - '{1}' Previous={2:02f}s , this={3} files".format(
path, name, t1 - t0, len(files))
t0 = t1
logger.info("Processing {} - '{}'".format(path, name))
print(msg)
if name:
timer.enter('get-collection')
# parent = self.get_collection(parent_path)
current_collection = self.get_collection(path)
if not current_collection:
current_collection = self.create_collection(
parent_path, # parent.path,
name,
path)
timer.exit('get-collection')
else:
current_collection = root_collection
# Now we can add the resources from self.folder + path
for filename in files:
fullpath = self.folder + path + '/' + filename
if filename.startswith("."):
continue
if filename.endswith(SKIP):
continue
if not os.path.isfile(fullpath):
continue
# Extract information needed to create the new resources
resc_dict = local_file_dict(fullpath)
resc_dict["read_access"] = self.groups
resc_dict["write_access"] = self.groups
resc_dict["container"] = current_collection.path
resc_dict['compress'] = self.compress
context = {
"fullpath": fullpath,
"local_ip": self.local_ip,
"path": path,
"filename": filename,
"user": self.user.name
}
timer.enter('push')
self.create_entry(resc_dict, context, self.is_reference)
timer.exit('push')
timer.summary()
def new_resource(request, parent):
parent_collection = Collection.find(parent)
# Inherits perms from container by default.
if not parent_collection:
raise Http404()
# User must be able to write to this collection
if not parent_collection.user_can(request.user, "write"):
raise PermissionDenied
keys = get_resource_keys()
mdata = collections.OrderedDict()
for k in keys:
mdata[k] = ""
if not mdata:
mdata[""] = ""
read_access, write_access = parent_collection.get_acl_list()
initial = {
'metadata': json.dumps(mdata),
'read_access': read_access,
'write_access': write_access
}
if request.method == 'POST':
form = ResourceNewForm(request.POST, files=request.FILES, initial=initial)
if form.is_valid():
data = form.cleaned_data
try:
blob_id = data['file'].read()
url = "cassandra://{}".format(blob_id)
name = data['name']
metadata = {}
for k, v in json.loads(data['metadata']):
if k in metadata:
if isinstance(metadata[k], list):
metadata[k].append(v)
else:
metadata[k] = [metadata[k], v]
else:
metadata[k] = v
resource = Resource.create(container=parent_collection.path,
name=name,
metadata=metadata,
url=url,
mimetype=data['file'].content_type,
username=request.user.name,
size=data['file'].size)
resource.create_acl_list(data['read_access'], data['write_access'])
messages.add_message(request, messages.INFO,
u"New resource '{}' created" .format(resource.get_name()))
except ResourceConflictError:
messages.add_message(request, messages.ERROR,
"That name is in use within the current collection")
return redirect('archive:view', path=parent_collection.path)
else:
form = ResourceNewForm(initial=initial)
ctx = {
"form": form,
"container": parent_collection,
"groups": Group.objects.all()
}
return render(request, 'archive/resource/new.html', ctx)
def create(cls,
container,
name,
uuid=None,
metadata=None,
url=None,
mimetype=None,
username=None,
size=None):
"""Create a new resource in the tree_entry table"""
from indigo.models import Collection
from indigo.models import Notification
# Check if parent collection exists
parent = Collection.find(container)
if parent is None:
raise NoSuchCollectionError(container)
if uuid is None:
uuid = default_cdmi_id()
create_ts = datetime.now()
modified_ts = create_ts
path = merge(container, name)
if metadata:
metadata_cass = meta_cdmi_to_cassandra(metadata)
# Check the container exists
collection = Collection.find(container)
if not collection:
raise NoSuchCollectionError(container)
# Make sure parent/name are not in use.
existing = cls.find(path)
if existing:
raise ResourceConflictError(path)
kwargs = {
"container": container,
"name": name,
"url": url,
"uuid": uuid,
}
if is_reference(url):
kwargs["create_ts"] = create_ts
kwargs["modified_ts"] = modified_ts
kwargs["mimetype"] = mimetype
if metadata:
kwargs["metadata"] = metadata_cass
else:
obj_id = url.replace("cassandra://", "")
data_obj = DataObject.find(obj_id)
if metadata:
data_obj.update(mimetype=mimetype, metadata=metadata_cass)
else:
if mimetype:
data_obj.update(mimetype=mimetype)
if size:
data_obj.update(size=size)
data_entry = TreeEntry.create(**kwargs)
new = Resource(data_entry)
session = get_graph_session()
add_user_edge = ""
if username:
user = User.find(username)
if user:
add_user_edge = """v_user = {}.next();
v_user.addEdge('owns', v_new);
""".format(gq_get_vertex_user(user))
else:
add_user_edge = ""
session.execute_graph("""v_parent = {}.next();
v_new = {};
v_parent.addEdge('son', v_new);
{}
""".format(gq_get_vertex_collection(parent),
gq_add_vertex_resource(new),
add_user_edge))
if metadata:
new.update_graph(metadata)
state = new.mqtt_get_state()
payload = new.mqtt_payload({}, state)
Notification.create_resource(username, path, payload)
# Index the resource
new.index()
return new
def delete_collection(self, path):
Collection.delete_all(path)