def put_data_object(self, path):
# Check if a collection with the name exists
collection = Collection.find(path)
if collection:
# Try to put a data_object when a collection of the same name
# already exists
self.logger.info(u"Impossible to create a new resource, the collection '{}' already exists, try to update it".format(path))
return self.put_container(path)
parent, name = split(path)
# Check if the resource already exists
resource = Resource.find(path)
# Check permissions
if resource:
# Update Resource
if not resource.user_can(self.user, "edit"):
self.logger.warning(u"User {} tried to modify resource at '{}'".format(self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
else:
# Create Resource
parent_collection = Collection.find(parent)
if not parent_collection:
self.logger.info(u"Fail to create a resource at '{}', collection doesn't exist".format(path))
return Response(status=HTTP_404_NOT_FOUND)
# Check if user can create a new resource in the collection
if not parent_collection.user_can(self.user, "write"):
self.logger.warning(u"User {} tried to create new resource at '{}'".format(self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
# All permissions are checked, we can proceed to create/update
if self.http_mode:
return self.put_data_object_http(parent, name, resource)
else:
return self.put_data_object_cdmi(parent, name, resource)
def view_collection(request, path):
if not path:
path = '/'
collection = Collection.find(path)
if not collection:
raise Http404()
if not collection.user_can(request.user, "read") and not collection.is_root:
# If the user can't read, then return 404 rather than 403 so that
# we don't leak information.
raise Http404()
paths = []
full = ""
for p in collection.path.split('/'):
if not p:
continue
full = u"{}/{}".format(full, p)
paths.append((p, full))
children_c, children_r = collection.get_child()
children_c.sort(key=lambda x: x.lower())
children_r.sort(key=lambda x: x.lower())
ctx = {
'collection': collection.to_dict(request.user),
'children_c': [Collection.find(merge(path,c)).to_dict(request.user) for c in children_c],
'children_r': [Resource.find(merge(path,c)).simple_dict(request.user) for c in children_r],
'collection_paths': paths,
'empty': len(children_c) + len(children_r) == 0,
}
return render(request, 'archive/index.html', ctx)
def view_resource(request, path):
resource = Resource.find(path)
if not resource:
raise Http404()
if not resource.user_can(request.user, "read"):
raise PermissionDenied
container = Collection.find(resource.container)
if not container:
# TODO: the container has to be there. If not it may be a network
# issue with Cassandra so we try again before raising an error to the
# user
container = Collection.find(resource.container)
if not container:
return HttpResponse(status=408,
content="Unable to find parent container '{}'".format(resource.container))
paths = []
full = ""
for p in container.path.split('/'):
if not p:
continue
full = u"{}/{}".format(full, p)
paths.append((p, full))
ctx = {
"resource": resource.full_dict(request.user),
"container": container,
"container_path": container.path,
"collection_paths": paths
}
return render(request, 'archive/resource/view.html', ctx)
def edit_resource(request, path):
# Requires edit on resource
resource = Resource.find(path)
if not resource:
raise Http404()
container = Collection.find(resource.container)
if not container:
raise Http404()
if not resource.user_can(request.user, "edit"):
raise PermissionDenied
if request.method == "POST":
form = ResourceForm(request.POST)
if form.is_valid():
metadata = {}
for k, v in json.loads(form.cleaned_data['metadata']):
if k in metadata:
if isinstance(metadata[k], list):
metadata[k].append(v)
else:
metadata[k] = [metadata[k], v]
else:
metadata[k] = v
try:
data = form.cleaned_data
resource.update(metadata=metadata, username=request.user.name)
resource.create_acl_list(data['read_access'], data['write_access'])
return redirect('archive:resource_view', path=resource.path)
except ResourceConflictError:
messages.add_message(request, messages.ERROR,
"That name is in use withinin the current collection")
else:
md = resource.get_cdmi_metadata()
metadata = json.dumps(md)
if not md:
metadata = '{"":""}'
read_access, write_access = resource.get_acl_list()
initial_data = {'name': resource.name, 'metadata': metadata,
'read_access': read_access,
'write_access': write_access
}
form = ResourceForm(initial=initial_data)
ctx = {
"form": form,
"resource": resource,
"container": container,
"groups": Group.objects.all()
}
return render(request, 'archive/resource/edit.html', ctx)
def crud_id(request, id):
# The URL should end with a '/'
id = id.replace('/', '')
collection = Collection.find_by_uuid(id)
if collection:
return redirect('cdmi:api_cdmi', path=collection.path())
else:
resource = Resource.find_by_uuid(id)
if resource:
return redirect('cdmi:api_cdmi', path=resource.path())
else:
return Response(status=HTTP_404_NOT_FOUND)
def delete_all(cls, path, username=None):
"""Delete recursively all sub-collections and all resources contained
in a collection at 'path'"""
from indigo.models import Resource
parent = Collection.find(path)
if not parent:
return
collections, resources = parent.get_child()
collections = [Collection.find(merge(path, c)) for c in collections]
resources = [Resource.find(merge(path, c)) for c in resources]
for resource in resources:
resource.delete(username)
for collection in collections:
Collection.delete_all(collection.path, username)
parent.delete(username)
def preview(request, path):
"""
Find the preview of the resource with the given ID and deliver it. This will
be rendered in the iframe of the resource view page.
"""
resource = Resource.find(path)
if not resource:
raise Http404
preview_info = {
# "type": "image",
# "url": "http://....."
}
return render(request, 'archive/preview.html', {'preview': preview_info})
def delete_data_object(self, path):
resource = Resource.find(path)
if not resource:
collection = Collection.find(path)
if collection:
self.logger.info(u"Fail to delete resource at '{}', test if it's a collection".format(path))
return self.delete_container(path)
else:
self.logger.info(u"Fail to delete resource at '{}'".format(path))
return Response(status=HTTP_404_NOT_FOUND)
if not resource.user_can(self.user, "delete"):
self.logger.warning(u"User {} tried to delete resource '{}'".format(self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
resource.delete()
self.logger.info(u"The resource '{}' was successfully deleted".format(path))
return Response(status=HTTP_204_NO_CONTENT)
def create_resource(self, parent, name, content, mimetype):
uuid = None
seq_num = 0
for chk in chunkstring(content, CHUNK_SIZE):
if uuid is None:
uuid = self.create_data_object(chk)
else:
self.append_data_object(uuid, seq_num, chk)
seq_num += 1
if uuid is None: # Content is null
uuid = self.create_empty_data_object()
url = "cassandra://{}".format(uuid)
resource = Resource.create(name=name,
container=parent,
url=url,
mimetype=mimetype,
size=len(content))
return resource
def delete_resource(request, path):
resource = Resource.find(path)
if not resource:
raise Http404
if not resource.user_can(request.user, "delete"):
raise PermissionDenied
container = Collection.find(resource.container)
if request.method == "POST":
resource.delete(username=request.user.name)
messages.add_message(request, messages.INFO,
"The resource '{}' has been deleted".format(resource.name))
return redirect('archive:view', path=container.path)
# Requires delete on resource
ctx = {
"resource": resource,
"container": container,
}
return render(request, 'archive/resource/delete.html', ctx)
def read_data_object(self, path):
"""Read a resource"""
resource = Resource.find(path)
if not resource:
collection = Collection.find(path)
if collection:
self.logger.info(u"Fail to read a resource at '{}', test if it's a collection".format(path))
return self.read_container(path)
else:
self.logger.info(u"Fail to read a resource at '{}'".format(path))
return Response(status=HTTP_404_NOT_FOUND)
if not resource.user_can(self.user, "read"):
self.logger.warning(u"User {} tried to read resource at '{}'".format(self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
cdmi_resource = CDMIResource(resource, self.api_root)
if self.http_mode:
if cdmi_resource.is_reference():
return self.read_data_object_reference(cdmi_resource)
else:
return self.read_data_object_http(cdmi_resource)
else:
return self.read_data_object_cdmi(cdmi_resource)
def download(request, path):
"""
Requests for download are redirected to the agent via the agent,
but for debugging the requests are served directly.
We will send appropriate user auth to the agent.
"""
resource = Resource.find(path)
if not resource:
raise Http404
if not resource.user_can(request.user, "read"):
raise PermissionDenied
if resource.is_reference:
r = requests.get(resource.url, stream=True)
resp = StreamingHttpResponse(streaming_content=r,
content_type=resource.get_mimetype())
else:
resp = StreamingHttpResponse(streaming_content=resource.chunk_content(),
content_type=resource.get_mimetype())
resp['Content-Disposition'] = u'attachment; filename="{}"'.format(resource.name)
return resp
def new_resource(request, parent):
parent_collection = Collection.find(parent)
# Inherits perms from container by default.
if not parent_collection:
raise Http404()
# User must be able to write to this collection
if not parent_collection.user_can(request.user, "write"):
raise PermissionDenied
keys = get_resource_keys()
mdata = collections.OrderedDict()
for k in keys:
mdata[k] = ""
if not mdata:
mdata[""] = ""
read_access, write_access = parent_collection.get_acl_list()
initial = {
'metadata': json.dumps(mdata),
'read_access': read_access,
'write_access': write_access
}
if request.method == 'POST':
form = ResourceNewForm(request.POST, files=request.FILES, initial=initial)
if form.is_valid():
data = form.cleaned_data
try:
blob_id = data['file'].read()
url = "cassandra://{}".format(blob_id)
name = data['name']
metadata = {}
for k, v in json.loads(data['metadata']):
if k in metadata:
if isinstance(metadata[k], list):
metadata[k].append(v)
else:
metadata[k] = [metadata[k], v]
else:
metadata[k] = v
resource = Resource.create(container=parent_collection.path,
name=name,
metadata=metadata,
url=url,
mimetype=data['file'].content_type,
username=request.user.name,
size=data['file'].size)
resource.create_acl_list(data['read_access'], data['write_access'])
messages.add_message(request, messages.INFO,
u"New resource '{}' created" .format(resource.get_name()))
except ResourceConflictError:
messages.add_message(request, messages.ERROR,
"That name is in use within the current collection")
return redirect('archive:view', path=parent_collection.path)
else:
form = ResourceNewForm(initial=initial)
ctx = {
"form": form,
"container": parent_collection,
"groups": Group.objects.all()
}
return render(request, 'archive/resource/new.html', ctx)
def create(cls, name, container='/', metadata=None, username=None):
"""Create a new collection"""
from indigo.models import Notification
from indigo.models import Resource
path = merge(container, name)
# Check if parent collection exists
parent = Collection.find(container)
if parent is None:
raise NoSuchCollectionError(container)
resource = Resource.find(merge(container, name))
if resource is not None:
raise ResourceConflictError(container)
collection = Collection.find(path)
if collection is not None:
raise CollectionConflictError(container)
now = datetime.now()
# If we try to create a tree entry with no metadata, cassandra-driver
# will fail as it tries to delete a static column
if metadata:
metadata_cass = meta_cdmi_to_cassandra(metadata)
coll_entry = TreeEntry.create(container=path,
name='.',
container_create_ts=now,
container_modified_ts=now,
container_metadata=metadata_cass)
else:
coll_entry = TreeEntry.create(container=path,
name='.',
container_create_ts=now,
container_modified_ts=now)
coll_entry.update(uuid=coll_entry.container_uuid)
child_entry = TreeEntry.create(container=container,
name=name + '/',
uuid=coll_entry.container_uuid)
new = Collection.find(path)
add_user_edge = ""
if username:
user = User.find(username)
if user:
add_user_edge = """v_user = {}.next();
v_user.addEdge('owns', v_new);
""".format(gq_get_vertex_user(user))
else:
add_user_edge = ""
session = get_graph_session()
session.execute_graph("""v_parent = {}.next();
v_new = {};
v_parent.addEdge('son', v_new);
{}
""".format(gq_get_vertex_collection(parent),
gq_add_vertex_collection(new),
add_user_edge))
if metadata:
new.update_graph(metadata)
state = new.mqtt_get_state()
payload = new.mqtt_payload({}, state)
Notification.create_collection(username, path, payload)
# Index the collection
new.index()
return new
def process_create_entry_work(self, resc_dict, context, is_reference):
# MOSTLY the resource will not exist... so start by calculating the URL and trying to insert the entire record..
if is_reference:
url = "file://{}{}/{}".format(context['local_ip'], context['path'],
context['entry'])
else:
with open(context['fullpath'], 'r') as f:
seq_number = 0
data_uuid = None
for chk in read_in_chunks(f):
if seq_number == 0:
data_object = DataObject.create(
chk, resc_dict['compress'])
data_uuid = data_object.uuid
else:
DataObject.append_chunk(data_uuid, chk, seq_number,
resc_dict['compress'])
seq_number += 1
if data_uuid:
url = "cassandra://{}".format(data_uuid)
else:
return None
try:
# OK -- try to insert ( create ) the record...
t1 = time.time()
resource = Resource.create(container=resc_dict['container'],
name=resc_dict['name'],
url=url,
mimetype=resc_dict['mimetype'],
username=context['user'],
size=resc_dict['size'])
resource.create_acl_list(resc_dict['read_access'],
resc_dict['write_access'])
msg = 'Resource {} created --> {}'.format(resource.get_name(),
time.time() - t1)
logger.info(msg)
except ResourceConflictError:
# If the create fails, the record already exists... so retrieve it...
t1 = time.time()
resource = Resource.find(
merge(resc_dict['container'], resc_dict['name']))
msg = "{} ::: Fetch Object -> {}".format(resource.get_name(),
time.time() - t1)
logger.info(msg)
# if the url is not correct then update
# TODO: if the url is a block set that is stored internally then reduce its count so that it can be GC'd.
# t3 = None
if resource.url != url:
t2 = time.time()
# if url.startswith('cassandra://') : tidy up the stored block count...
resource.update(url=url)
t3 = time.time()
msg = "{} ::: update -> {}".format(resource.get_name(), t3 - t2)
logger.info(msg)
# t1 = time.time()
SearchIndex.reset(resource.uuid)
SearchIndex.index(resource, ['name', 'metadata'])
def home(request):
notifications = Notification.recent(10)
activities = []
for notif in notifications:
t = template.Template(notif['tmpl'])
obj_uuid = notif['object_uuid']
object = None
if notif['object_type'] == OBJ_RESOURCE:
object = Resource.find(obj_uuid)
if object:
object_dict = object.to_dict()
else:
object_dict = {'name': obj_uuid}
elif notif['object_type'] == OBJ_COLLECTION:
object = Collection.find(obj_uuid)
if object:
object_dict = object.to_dict()
else:
object_dict = {'name': obj_uuid}
elif notif['object_type'] == OBJ_USER:
object = User.find(obj_uuid)
if object:
object_dict = object.to_dict()
else:
# User has been deleted it can't be find by uuid
# look in payload of the message to get the name
if notif['operation'] in [OP_CREATE, OP_UPDATE]:
name = notif['payload']['post']['name']
else: # OP_DELETE
name = notif['payload']['pre']['name']
object_dict = {'name': name}
elif notif['object_type'] == OBJ_GROUP:
object = Group.find(obj_uuid)
if object:
object_dict = object.to_dict()
else:
# User has been deleted it can't be find by uuid
# look in payload of the message to get the name
if notif['operation'] in [OP_CREATE, OP_UPDATE]:
name = notif['payload']['post']['name']
else: # OP_DELETE
name = notif['payload']['pre']['name']
object_dict = {'uuid': obj_uuid,
'name': name}
user_dict = {}
if notif['username']:
user = User.find(notif['username'])
if user:
user_dict = user.to_dict()
variables = {
'user': user_dict,
'when': notif['when'],
'object': object_dict
}
ctx = template.Context(variables)
activities.append({'html': t.render(ctx)})
return render(request, 'activity/index.html', {'activities': activities})
def create_reference(self, parent, name, url, mimetype="application/cdmi-object"):
resource = Resource.create(name=name,
container=parent,
url=url,
mimetype=mimetype)
return resource