def test_put():
myFactory = Faker()
content = myFactory.text()
# Create a new resource with a random name
resc_name = uuid.uuid4().hex
resc = Resource.create('/', resc_name)
resc.put(content.encode())
assert resc.get_size() == len(content)
# Force small chunk size for testing
cfg.chunk_size = 4
fh = io.BytesIO(content.encode())
resc.put(fh)
assert resc.get_size() == len(content)
resc.delete()
resc = Resource.create('/',
resc_name,
url="http://www.google.fr",
mimetype="text/plain")
with pytest.raises(NotImplementedError):
resc.put("test")
resc.delete()
def test_dict():
coll_name = uuid.uuid4().hex
coll = Collection.create("/", coll_name)
myFactory = Faker()
content = myFactory.text()
# Read/Write resource stored in Cassandra
do = DataObject.create(content.encode())
resc_name = uuid.uuid4().hex
resc = Resource.create(coll.path,
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid))
resc = Resource.find(resc.path)
resc_dict = resc.full_dict(User.find(USR1_NAME))
assert resc_dict['size'] == len(content)
assert resc_dict['can_read']
assert resc_dict['can_write']
assert resc_dict['uuid'] == resc.uuid
resc_dict = resc.simple_dict(User.find(USR1_NAME))
assert resc_dict['name'] == resc_name
assert resc_dict['is_reference'] == False
assert resc_dict['can_read']
assert resc_dict['can_write']
assert resc_dict['id'] == resc.uuid
assert resc.simple_dict() == resc.to_dict()
resc.delete()
coll.delete()
def test_get_child():
# Create a new collection with a random name
coll_name = uuid.uuid4().hex
coll1 = Collection.create('/', coll_name)
coll2 = Collection.create(coll1.path, uuid.uuid4().hex)
coll3 = Collection.create(coll1.path, uuid.uuid4().hex)
coll4 = Collection.create(coll1.path, uuid.uuid4().hex)
resc1 = Resource.create(coll1.path,
uuid.uuid4().hex,
url="http://www.google.fr")
resc2 = Resource.create(coll1.path,
uuid.uuid4().hex,
url="http://www.google.fr")
coll_childs, resc_childs = coll1.get_child()
assert set(coll_childs) == set([coll2.name, coll3.name, coll4.name])
assert set(resc_childs) == set([resc1.get_name(), resc2.get_name()])
assert coll1.get_child_resource_count() == 2
coll_root = Collection.find("/")
# Test for a resource where the url has been lost somehow
resc3 = Resource.create(coll_root.path, uuid.uuid4().hex)
resc3.update(object_url=None)
resc3 = Resource.find(resc3.path)
coll_childs, resc_childs = coll_root.get_child()
assert set(coll_childs) == set(["1/", "2/", coll1.name])
assert set(resc_childs) == set([resc3.get_name(), 'g'])
coll1.delete()
def test_chunk_content():
coll_name = uuid.uuid4().hex
coll = Collection.create("/", coll_name)
myFactory = Faker()
content = myFactory.text()
do = DataObject.create(content.encode())
resc_name = uuid.uuid4().hex
resc = Resource.create(coll.path,
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid))
data = []
for chk in resc.chunk_content():
data.append(chk)
res = b"".join([s for s in data])
assert res == content.encode()
resc.obj = None
assert resc.chunk_content() == None
TEST_URL = "http://www.google.fr"
resc_name = uuid.uuid4().hex
resc = Resource.create(coll.path, resc_name, url=TEST_URL)
data = []
for chk in resc.chunk_content():
data.append(chk)
res = b"".join(data)
assert res
coll.delete()
def test_size():
coll_name = uuid.uuid4().hex
coll = Collection.create('/', coll_name)
myFactory = Faker()
content = myFactory.text()
chk = hashlib.sha224(content.encode()).hexdigest()
do = DataObject.create(content.encode())
resc_name = uuid.uuid4().hex
resc = Resource.create(coll.path,
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid))
resc = Resource.find(resc.path)
assert resc.get_size() == len(content)
resc.obj = None
assert resc.get_size() == 0
resc.delete()
# Size for a reference
resc = Resource.create(coll.path,
resc_name,
url="http://www.google.fr",
mimetype="text/plain")
resc = Resource.find(resc.path)
# Size stored in the tree entry
assert resc.get_size() == 0
resc.delete()
coll.delete()
def setup_module():
cfg.dse_keyspace = TEST_KEYSPACE
initialise()
connect()
create_tables()
create_root()
try:
coll = Collection.create("/", "coll1")
ref1 = Resource.create("/", "test.url", url=TEST_URL)
resc = Resource.create("/coll1", "test.txt")
ref2 = Resource.create("/coll1", "test.url", url=TEST_URL)
except: # If collections or resources already exist
pass
def put(self, args):
"Put a file to a path."
is_reference = args["--ref"]
if is_reference:
url = args["<url>"]
dest_path = args["<dest>"]
# Get the full destination path of the new resource
dest_path = self.get_full_path(dest_path)
else:
src = args["<src>"]
# Absolutize local path
local_path = os.path.abspath(src)
# Check that local file exists
if not os.path.exists(local_path):
self.print_error("File '{}' doesn't exist".format(local_path))
return errno.ENOENT
if args["<dest>"]:
dest_path = args["<dest>"]
# We try to put the new file in a subcollection
if dest_path.endswith('/'):
dest_path = "{}{}".format(dest_path,
os.path.basename(local_path))
else:
# PUT to same name in pwd on server
dest_path = os.path.basename(local_path)
# Get the full destination path of the new resource
dest_path = self.get_full_path(dest_path)
# Check resource objects on the database
resc = Resource.find(dest_path)
if resc:
self.print_error(MSG_RESC_EXIST.format(dest_path))
return
parent, name = split(dest_path)
try:
if is_reference:
resc = Resource.create(parent, name, url=url)
else:
resc = Resource.create(parent, name)
with open(local_path, "rb") as fh:
resc.put(fh)
print(resc)
except NoSuchCollectionError:
self.print_error(MSG_COLL_NOT_EXIST.format(os.path.dirname(dest_path)))
def delete_resource(request, path):
"""Display the page to delete a resource"""
resource = Resource.find(path)
if not resource:
raise Http404
if not resource.user_can(request.user, "delete"):
raise PermissionDenied
container = Collection.find(resource.container)
if request.method == "POST":
resource.delete(username=request.user.name)
messages.add_message(
request,
messages.INFO,
"The resource '{}' has been deleted".format(resource.name),
)
return redirect("archive:view", path=container.path)
# Requires delete on resource
ctx = {
"resource": resource,
"container": container,
}
return render(request, "archive/resource/delete.html", ctx)
def read_data_object(self, path):
"""Read a resource"""
resource = Resource.find(path)
if not resource:
collection = Collection.find(path)
if collection:
self.logger.info(
u"Fail to read a resource at '{}', test if it's a collection"
.format(path))
return self.read_container(path)
else:
self.logger.info(
u"Fail to read a resource at '{}'".format(path))
return Response(status=HTTP_404_NOT_FOUND)
if not resource.user_can(self.user, "read"):
self.logger.warning(
u"User {} tried to read resource at '{}'".format(
self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
cdmi_resource = CDMIResource(resource, self.api_root)
if self.http_mode:
if cdmi_resource.is_reference():
return self.read_data_object_reference(cdmi_resource)
else:
return self.read_data_object_http(cdmi_resource)
else:
return self.read_data_object_cdmi(cdmi_resource)
def test_collection():
coll1 = Collection.create("/", uuid.uuid4().hex)
coll2 = Collection.create(coll1.path, uuid.uuid4().hex)
meta_dict = {"meta": "val"}
coll3 = Collection.create("/", uuid.uuid4().hex, metadata=meta_dict)
assert coll3.get_cdmi_user_meta() == meta_dict
coll4 = Collection.create("/", uuid.uuid4().hex, creator="test")
# test if name ends with '/'
coll5 = Collection.create("/", uuid.uuid4().hex + "/")
with pytest.raises(NoSuchCollectionError):
Collection.create("unknown", uuid.uuid4().hex)
r = Resource.find("/g")
with pytest.raises(ResourceConflictError):
coll = Collection.create("/", "g")
with pytest.raises(CollectionConflictError):
Collection.create("/", coll1.name)
coll1.delete()
coll2.delete()
coll3.delete()
coll4.delete()
coll5.delete()
def home(request):
"""Default view for Activities"""
notifications = Notification.recent(10)
activities = []
for notif in notifications:
tmpl = template.Template(notif["tmpl"])
obj_uuid = notif["object_uuid"]
obj = None
if notif["object_type"] == OBJ_RESOURCE:
obj = Resource.find(obj_uuid)
if obj:
object_dict = obj.to_dict()
else:
object_dict = {"name": obj_uuid}
elif notif["object_type"] == OBJ_COLLECTION:
obj = Collection.find(obj_uuid)
if obj:
object_dict = obj.to_dict()
else:
object_dict = {"name": obj_uuid}
elif notif["object_type"] == OBJ_USER:
obj = User.find(obj_uuid)
if obj:
object_dict = obj.to_dict()
else:
# User has been deleted it can't be find by uuid
# look in payload of the message to get the name
if notif["operation"] in [OP_CREATE, OP_UPDATE]:
name = notif["payload"]["post"]["name"]
else: # OP_DELETE
name = notif["payload"]["pre"]["name"]
object_dict = {"name": name}
elif notif["object_type"] == OBJ_GROUP:
obj = Group.find(obj_uuid)
if obj:
object_dict = obj.to_dict()
else:
# User has been deleted it can't be find by uuid
# look in payload of the message to get the name
if notif["operation"] in [OP_CREATE, OP_UPDATE]:
name = notif["payload"]["post"]["name"]
else: # OP_DELETE
name = notif["payload"]["pre"]["name"]
object_dict = {"uuid": obj_uuid, "name": name}
user_dict = {}
if notif["username"]:
user = User.find(notif["username"])
if user:
user_dict = user.to_dict()
else:
user_dict = { 'name': notif["username"],
'email': notif["username"]+ '@radon.org' }
variables = {"user": user_dict, "when": notif["when"], "object": object_dict}
ctx = template.Context(variables)
activities.append({"html": tmpl.render(ctx)})
return render(request, "activity/index.html", {"activities": activities})
def test_update():
coll_name = uuid.uuid4().hex
coll = Collection.create('/', coll_name)
myFactory = Faker()
content = myFactory.text()
chk = hashlib.sha224(content.encode()).hexdigest()
metadata = {"test": "val", "test_json": '["t", "e", "s", "t"]'}
# Simple update
do = DataObject.create(content.encode())
resc_name = uuid.uuid4().hex
resc = Resource.create(coll.path,
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid))
resc.update(mimetype="text/plain")
resc = Resource.find(resc.path)
assert resc.get_mimetype() == "text/plain"
resc.delete()
# update with metadata and a username for notification
do = DataObject.create(content.encode())
resc_name = uuid.uuid4().hex
resc = Resource.create(coll.path,
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid))
resc.update(username="******", metadata=metadata)
resc = Resource.find(resc.path)
assert resc.get_cdmi_user_meta()['test'] == metadata['test']
resc.delete()
# Update with a change of url (new dataObject)
do = DataObject.create(content.encode())
resc_name = uuid.uuid4().hex
resc = Resource.create(coll.path,
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid))
DataObject.delete_id(do.uuid)
do = DataObject.create(content.encode())
resc.update(object_url="{}{}".format(cfg.protocol_cassandra, do.uuid))
resc = Resource.find(resc.path)
assert resc.get_size() == len(content)
resc.delete()
# Update for a reference
resc = Resource.create(coll.path, resc_name, url="http://www.google.fr")
resc.update(mimetype="text/plain")
resc = Resource.find(resc.path)
# Mimetype stored in the tree entry
assert resc.get_mimetype() == "text/plain"
resc.delete()
coll.delete()
def test_path():
coll_name = uuid.uuid4().hex
coll = Collection.create('/', coll_name)
myFactory = Faker()
content = myFactory.text()
chk = hashlib.sha224(content.encode()).hexdigest()
do = DataObject.create(content.encode())
resc_name = uuid.uuid4().hex
resc = Resource.create(coll.path,
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid))
resc = Resource.find(resc.path)
assert resc.path == "{}{}".format(coll.path, resc_name)
assert resc.get_path() == "{}{}".format(coll.path, resc_name)
resc.delete()
coll.delete()
def create_reference(self,
parent,
name,
url,
mimetype="application/cdmi-object"):
"""Create a new reference"""
resource = Resource.create(name=name,
container=parent,
url=url,
mimetype=mimetype)
return resource
def is_resource(path):
"""Check if the resource exists
:param path: The path of the resource in Radon
:type path: str
:return: a boolean
:rtype: bool
"""
from radon.model import Resource
return Resource.find(path) is not None
def test_create_acl_fail(mocker):
list_read = [GRP1_NAME]
list_write = [GRP1_NAME]
myFactory = Faker()
content = myFactory.text()
# Create a new resource with a random name
do = DataObject.create(content.encode())
resc_name = uuid.uuid4().hex
resc = Resource.create('/',
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid))
mocker.patch('radon.model.resource.acemask_to_str',
return_value="wrong_oper")
resc.create_acl_list(list_read, list_write)
resc = Resource.find('/{}'.format(resc_name))
# Test get_acl_list wrong operation name
acl_list = resc.get_acl_list()
assert acl_list == ([], [])
resc.delete()
def crud_id(request, obj_id):
"""Call the correct method from a call with the uuid of a resource/collection"""
# The URL should end with a '/'
obj_id = obj_id.replace("/", "")
collection = Collection.find_by_uuid(obj_id)
if collection:
return redirect("rest_cdmi:api_cdmi", path=collection.path())
else:
resource = Resource.find_by_uuid(obj_id)
if resource:
return redirect("rest_cdmi:api_cdmi", path=resource.path())
else:
return Response(status=HTTP_404_NOT_FOUND)
def test_delete():
coll1_name = uuid.uuid4().hex
coll1 = Collection.create("/", coll1_name)
coll2 = Collection.create(coll1.path, uuid.uuid4().hex)
coll3 = Collection.create(coll1.path, uuid.uuid4().hex)
coll4 = Collection.create(coll2.path, uuid.uuid4().hex)
coll5 = Collection.create(coll4.path, uuid.uuid4().hex)
resc1 = Resource.create(coll1.path,
uuid.uuid4().hex,
url="http://www.google.fr")
coll1.delete()
assert Collection.find(coll1_name) == None
def test_delete_all():
coll1_name = uuid.uuid4().hex
coll1 = Collection.create("/", coll1_name)
coll2 = Collection.create(coll1.path, uuid.uuid4().hex)
coll5 = Collection.create(coll2.path, uuid.uuid4().hex)
resc1 = Resource.create(coll1.path,
uuid.uuid4().hex,
url="http://www.google.fr")
Collection.delete_all("/{}/".format(coll1_name))
assert Collection.find(coll1_name) == None
assert Collection.delete_all("/unknown/") == None
def preview(request, path):
"""
Find the preview of the resource with the given ID and deliver it. This will
be rendered in the iframe of the resource view page.
Deprecated for the moment as the iframe isn't working
"""
resource = Resource.find(path)
if not resource:
raise Http404
preview_info = {
# "type": "image",
# "url": "http://....."
}
return render(request, "archive/preview.html", {"preview": preview_info})
def test_metadata():
coll_name = uuid.uuid4().hex
coll = Collection.create('/', coll_name)
myFactory = Faker()
content = myFactory.text()
chk = hashlib.sha224(content.encode()).hexdigest()
metadata = {"test": "val", "test_list": ['t', 'e', 's', 't']}
# Checksum passed at creation
do = DataObject.create(content.encode())
resc_name = uuid.uuid4().hex
resc = Resource.create(coll.path,
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid),
metadata=metadata)
resc = Resource.find(resc.path)
meta_dict = resc.get_cdmi_user_meta()
assert meta_dict['test'] == metadata['test']
assert meta_dict['test_list'] == metadata['test_list']
sys_meta = resc.get_cdmi_sys_meta()
assert "radon_create_ts" in sys_meta
assert "radon_modify_ts" in sys_meta
resc.delete()
# Checksum passed at creation
do = DataObject.create(content.encode())
resc_name = uuid.uuid4().hex
resc = Resource.create(coll.path,
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid),
mimetype="text/plain")
resc = Resource.find(resc.path)
assert resc.get_mimetype() == "text/plain"
resc.delete()
# Mimetype for a reference
resc = Resource.create(coll.path,
resc_name,
url="http://www.google.fr",
mimetype="text/plain")
resc = Resource.find(resc.path)
# Mimetype stored in the tree entry
assert resc.get_mimetype() == "text/plain"
resc.delete()
coll.delete()
def rm(self, args):
"""Remove a data object or a collection.
"""
path = args["<path>"]
# Get the full path of the object to delete
path = self.get_full_path(path)
resc = Resource.find(path)
if resc:
resc.delete()
return
coll = Collection.find(path)
if coll:
coll.delete()
return
self.print_error(MSG_NO_OBJECT.format(path))
def test_resource():
coll_name = uuid.uuid4().hex
with pytest.raises(NoSuchCollectionError):
resc = Resource.create("/{}".format(coll_name), uuid.uuid4().hex)
coll = Collection.create("/", coll_name)
do = create_data_object()
resc_name = uuid.uuid4().hex
resc = Resource.create(coll.path,
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid))
assert str(resc) == resc.path
assert resc.get_name() == resc_name
with pytest.raises(ResourceConflictError):
resc = Resource.create(coll.path, resc_name)
resc.delete()
# Check resource is gone
resc = Resource.find(resc.path)
assert resc == None
do = create_data_object()
resc = Resource.create(coll.path,
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid))
resc.delete()
resc = Resource.create(coll.path, resc_name, url="http://www.google.fr")
assert resc.get_name() == resc_name + '?'
resc.delete()
# Check deleting resource also deleted data object
do = DataObject.find(do.uuid)
assert do == None
do = create_data_object()
resc = Resource.create(coll.path,
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid),
size=do.size)
assert resc.get_size() == do.size
resc.delete()
coll.delete()
def create_resource(self, parent, name, content, mimetype):
"""Create a new resource"""
uuid = None
seq_num = 0
for chk in chunkstring(content, CHUNK_SIZE):
if uuid is None:
uuid = self.create_data_object(chk)
else:
self.append_data_object(uuid, seq_num, chk)
seq_num += 1
if uuid is None: # Content is null
uuid = self.create_empty_data_object()
url = "cassandra://{}".format(uuid)
resource = Resource.create(name=name,
container=parent,
url=url,
mimetype=mimetype,
size=len(content))
return resource
def put_data_object(self, path):
"""Put a data object to a specific collection"""
# Check if a collection with the name exists
collection = Collection.find(path)
if collection:
# Try to put a data_object when a collection of the same name
# already exists
self.logger.info(
"Impossible to create a new resource, the collection '{}' already exists, try to update it"
.format(path))
return self.put_container(path)
parent, name = split(path)
# Check if the resource already exists
resource = Resource.find(path)
# Check permissions
if resource:
# Update Resource
if not resource.user_can(self.user, "edit"):
self.logger.warning(
"User {} tried to modify resource at '{}'".format(
self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
else:
# Create Resource
parent_collection = Collection.find(parent)
if not parent_collection:
self.logger.info(
"Fail to create a resource at '{}', collection doesn't exist"
.format(path))
return Response(status=HTTP_404_NOT_FOUND)
# Check if user can create a new resource in the collection
if not parent_collection.user_can(self.user, "write"):
self.logger.warning(
"User {} tried to create new resource at '{}'".format(
self.user, path))
return Response(status=HTTP_403_FORBIDDEN)
# All permissions are checked, we can proceed to create/update
if self.http_mode:
return self.put_data_object_http(parent, name, resource)
else:
return self.put_data_object_cdmi(parent, name, resource)
def test_user_can():
myFactory = Faker()
content = myFactory.text()
# Create a new resource with a random name
do = DataObject.create(content.encode())
resc_name = uuid.uuid4().hex
resc = Resource.create('/',
resc_name,
url="{}{}".format(cfg.protocol_cassandra, do.uuid))
usr1 = User.find(USR1_NAME)
usr2 = User.find(USR2_NAME)
# usr1 should be admin
assert resc.user_can(usr1, "read")
# usr2 should not be admin, on root collection, only read
assert resc.user_can(usr2, "read")
assert not resc.user_can(usr2, "write")
resc.delete()
def delete(self, username=None):
"""
Delete a collection and the associated row in the tree_node table
:param username: The name of the user who deleted the collection
:type username: str, optional
"""
from radon.model import Resource
if not username:
username = radon.cfg.sys_lib_user
if self.is_root:
return
# We don't need the suffixes for the resources, otherwise we won't
# find them
child_container, child_dataobject = self.get_child(False)
for child_str in child_container:
child = Collection.find(self.path + child_str)
if child:
child.delete()
for child_str in child_dataobject:
child = Resource.find(self.path + child_str)
if child:
child.delete()
session = connection.get_session()
keyspace = radon.cfg.dse_keyspace
session.set_keyspace(keyspace)
query = SimpleStatement(
"""DELETE FROM tree_node WHERE container=%s and name=%s""")
session.execute(query, (
self.container,
self.name,
))
from radon.model import Notification
state = self.mqtt_get_state()
payload = self.mqtt_payload(state, {})
Notification.delete_collection(username, self.path, payload)
def download(request, path):
""" Download the content of a resource"""
resource = Resource.find(path)
if not resource:
raise Http404
if not resource.user_can(request.user, "read"):
raise PermissionDenied
if resource.is_reference():
r = requests.get(resource.url, stream=True)
resp = StreamingHttpResponse(
streaming_content=r, content_type=resource.get_mimetype()
)
else:
resp = StreamingHttpResponse(
streaming_content=resource.chunk_content(),
content_type=resource.get_mimetype(),
)
resp["Content-Disposition"] = u'attachment; filename="{}"'.format(resource.name)
return resp
def view_resource(request, path):
"""Display the page for a resource in the archive"""
resource = Resource.find(path)
if not resource:
raise Http404()
if not resource.user_can(request.user, "read"):
raise PermissionDenied
container = Collection.find(resource.container)
if not container:
# TODO: the container has to be there. If not it may be a network
# issue with Cassandra so we try again before raising an error to the
# user
container = Collection.find(resource.container)
if not container:
return HttpResponse(
status=408,
content="Unable to find parent container '{}'".format(
resource.container
),
)
paths = []
full = "/"
for pth in container.path.split("/"):
if not pth:
continue
full = u"{}{}/".format(full, pth)
paths.append((pth, full))
ctx = {
"resource": resource.full_dict(request.user),
"container": container,
"container_path": container.path,
"collection_paths": paths,
}
return render(request, "archive/resource/view.html", ctx)
def view_collection(request, path='/'):
"""Display the page which shows the subcollections/resources of a collection"""
if not path:
path = "/"
collection = Collection.find(path)
if not collection:
raise Http404()
if not collection.user_can(request.user, "read") and not collection.is_root:
# If the user can't read, then return 404 rather than 403 so that
# we don't leak information.
raise Http404()
paths = []
full = "/"
for p in collection.path.split("/"):
if not p:
continue
full = u"{}{}/".format(full, p)
paths.append((p, full))
children_c, children_r = collection.get_child(False)
children_c.sort(key=lambda x: x.lower())
children_r.sort(key=lambda x: x.lower())
ctx = {
"collection": collection.to_dict(request.user),
"children_c": [
Collection.find(merge(path, c)).to_dict(request.user) for c in children_c
],
"children_r": [
Resource.find(merge(path, c)).simple_dict(request.user) for c in children_r
],
"collection_paths": paths,
"empty": len(children_c) + len(children_r) == 0,
}
return render(request, "archive/index.html", ctx)
