class DummyUDT(Model):
__keyspace__ = 'ks'
primary_key = columns.Integer(primary_key=True)
value = columns.UserDefinedType(Value)
class TreeEntry(Model):
"""TreeEntry model"""
# Partitioned by container, clustered by name, so all files for a directory
# are in the same bucket and share the single instance of the static
# container data
container = columns.Text(partition_key=True)
name = columns.Text(primary_key=True, partition_key=False)
# The following set of columns are shared between all entries with the same
# container name. i.e. it removes the need for a separate container table,
# removes the need for extra lookups and avoids the container / objects
# getting out of sync
#
# It also facilitates _some_ directory operations, e.g. removal.
#
# Renaming is still slow because the container and the name are primary
# keys, so you have to create a new record and delete the old one...
# It is suggested to use the batch system to make such an operation (more
# or less) atomic.
#
container_metadata = columns.Map(columns.Text, columns.Text, static=True)
container_uuid = columns.Text(default=default_cdmi_id, static=True)
container_create_ts = columns.DateTime(static=True)
container_modified_ts = columns.DateTime(static=True)
container_acl = columns.Map(columns.Text,
columns.UserDefinedType(Ace),
static=True)
# This is the actual directory entry per-se, i.e. unique per name....
# As with a conventional filesystem this is simply a reference to the 'real'
# data where ACLs, system metadata &c are held.
# per-record, but only for externals (see DataObject)
metadata = columns.Map(columns.Text, columns.Text)
create_ts = columns.DateTime(default=datetime.now)
modified_ts = columns.DateTime()
acl = columns.Map(columns.Text, columns.UserDefinedType(Ace))
mimetype = columns.Text()
# Use the url schema (file:// , cdmi:// &c ) to route the request...
# Only cdmi:// does anything everything else results in a redirect
url = columns.Text()
uuid = columns.Text()
def add_default_acl(self):
"""Add read access to all authenticated users"""
self.create_container_acl_list(["AUTHENTICATED@"], [])
@classmethod
def create(cls, **kwargs):
"""Create"""
# if "mimetype" in kwargs:
# metadata = kwargs.get('metadata', {})
# metadata["cdmi_mimetype"] = kwargs["mimetype"]
# kwargs['metadata'] = meta_cdmi_to_cassandra(metadata)
# del kwargs['mimetype']
new = super(TreeEntry, cls).create(**kwargs)
return new
def create_container_acl(self, acl_cql):
"""Replace the static acl with the given cql string
"""
cfg = get_config(None)
session = connection.get_session()
keyspace = cfg.get('KEYSPACE', 'indigo')
session.set_keyspace(keyspace)
query = SimpleStatement(u"""UPDATE tree_entry SET container_acl={}
WHERE container=%s""".format(acl_cql))
session.execute(query, (self.container, ))
def create_container_acl_cdmi(self, cdmi_acl):
"""""Create static ACL from a cdmi object (list of dict)"""
cql_string = acl_cdmi_to_cql(cdmi_acl)
self.create_container_acl(cql_string)
def create_container_acl_list(self, read_access, write_access):
"""""Create static ACL from lists of group uuids"""
cql_string = acl_list_to_cql(read_access, write_access)
self.create_container_acl(cql_string)
def create_entry_acl(self, acl_cql):
"""Replace the acl with the given cql string
"""
cfg = get_config(None)
session = connection.get_session()
keyspace = cfg.get('KEYSPACE', 'indigo')
session.set_keyspace(keyspace)
query = SimpleStatement(u"""UPDATE tree_entry SET acl={}
WHERE container=%s and name=%s""".format(acl_cql))
session.execute(query, (
self.container,
self.name,
))
def create_entry_acl_list(self, read_access, write_access):
"""""Create entry ACL from lists of group uuids"""
cql_string = acl_list_to_cql(read_access, write_access)
self.create_entry_acl(cql_string)
def create_entry_acl_cdmi(self, cdmi_acl):
"""""Create entry ACL from a cdmi object (list of dict)"""
cql_string = acl_cdmi_to_cql(cdmi_acl)
self.create_entry_acl(cql_string)
def path(self):
"""Get the full path of the specific entry"""
return merge(self.container, self.name)
def update(self, **kwargs):
"""Update a collection"""
cfg = get_config(None)
session = connection.get_session()
keyspace = cfg.get('KEYSPACE', 'indigo')
session.set_keyspace(keyspace)
for arg in kwargs:
# For static fields we can't use the name in the where condition
if arg in static_fields:
query = SimpleStatement(u"""UPDATE tree_entry SET {}=%s
WHERE container=%s""".format(arg))
session.execute(query, (kwargs[arg], self.container))
else:
query = SimpleStatement(u"""UPDATE tree_entry SET {}=%s
WHERE container=%s and name=%s""".format(arg))
session.execute(query,
(kwargs[arg], self.container, self.name))
return self
def update_container_acl(self, acl_cql):
"""Update the static acl with the given cql string"""
cfg = get_config(None)
session = connection.get_session()
keyspace = cfg.get('KEYSPACE', 'indigo')
session.set_keyspace(keyspace)
query = SimpleStatement(
u"""UPDATE tree_entry SET container_acl=container_acl+{}
WHERE container=%s""".format(acl_cql))
session.execute(query, (self.container, ))
def update_container_acl_cdmi(self, cdmi_acl):
""""Update static ACL from a cdmi object (list of dict)"""
cql_string = acl_cdmi_to_cql(cdmi_acl)
self.update_container_acl(cql_string)
def update_container_acl_list(self, read_access, write_access):
""""Update static ACL from lists of group uuids"""
cql_string = acl_list_to_cql(read_access, write_access)
self.update_container_acl(cql_string)
def update_entry_acl(self, acl_cql):
"""Update the acl with the given cql string"""
cfg = get_config(None)
session = connection.get_session()
keyspace = cfg.get('KEYSPACE', 'indigo')
session.set_keyspace(keyspace)
query = SimpleStatement(u"""UPDATE tree_entry SET acl=acl+{}
WHERE container=%s and name=%s""".format(acl_cql))
session.execute(query, (
self.container,
self.name,
))
def update_entry_acl_list(self, read_access, write_access):
""""Update entry ACL from lists of group uuids"""
cql_string = acl_list_to_cql(read_access, write_access)
self.update_entry_acl(cql_string)
def update_entry_acl_cdmi(self, cdmi_acl):
""""Update entry ACL from a cdmi object (list of dict)"""
cql_string = acl_cdmi_to_cql(cdmi_acl)
self.update_entry_acl(cql_string)
class AllDatatypesModel(Model):
id = columns.Integer(primary_key=True)
data = columns.UserDefinedType(AllDatatypes)
class UserModelValidateDefault(Model):
id = columns.Integer(primary_key=True)
info = columns.UserDefinedType(UserValidateDefault)
class TheModel(Model):
id = columns.Integer(primary_key=True)
info = columns.UserDefinedType(db_field_different)
class OuterModel(Model):
name = columns.Text(primary_key=True)
first_name = columns.Text()
nested = columns.List(columns.UserDefinedType(NestedUdt))
simple = columns.UserDefinedType(NestedUdt)
class UserModel(Model):
id = columns.Integer(primary_key=True)
info = columns.UserDefinedType(User)
class UserModelText(Model):
id = columns.Text(primary_key=True)
info = columns.UserDefinedType(User)
class Container(Model):
id = columns.UUID(primary_key=True, default=uuid4)
names = columns.List(columns.UserDefinedType(Name))
class DepthModel(Model):
id = columns.Integer(primary_key=True)
v_0 = columns.UserDefinedType(Depth_0)
v_1 = columns.UserDefinedType(Depth_1)
v_2 = columns.UserDefinedType(Depth_2)
v_3 = columns.UserDefinedType(Depth_3)
class Depth_3(UserType):
value = columns.UserDefinedType(Depth_2)
class TreeNode(Model):
"""TreeNode model
This is used to store the hierarchy in Cassandra, Collections or Data
Objects.
(container, name) is the partition key, it's the path of the element in the
hierarchy. Collections ends with a '/' like in the CDMI standard. That way
subcollections are stored closely in Cassandra nodes.
version is the last part of the primary key so we can keep several versions
of the hierarchy.
:param container: The parent path of the object/collection
:type container: :class:`columns.Text`
:param name: The name of the object/collection. Collections ends with '/'
:type name: :class:`columns.Text`
:param version: The version of the object/collection
:type version: :class:`columns.Integer`
:param uuid: A CDMI uuid
:type uuid: :class:`columns.Text`
:param is_object: A boolean to simplify the test
:type is_object: :class:`columns.Boolean`
:param object_url: For data object the url to the content of the object. It
can starts with 'cassandra:// if data is stored in Radon (See
:class:`radon.model.DataObject`
:type object_url: :class:`columns.Text()`
:param sys_meta: A Key/Value pair dictionary for system metadata
:type sys_meta: :class:`columns.Map(columns.Text, columns.Text)`
:param user_meta: A Key/Value pair dictionary for user metadata. Values are
stored in JSON
:type user_meta: :class:`columns.Map(columns.Text, columns.Text)`
:param acl: A Key/Value pair dictionary for ACL, a group name and the
associated ACE
:type acl: :class:`columns.Map(columns.Text, columns.UserDefinedType(Ace))`
"""
# Partitioned by container, clustered by name, so all files for a directory
# are in the same partition
container = columns.Text(partition_key=True)
name = columns.Text(primary_key=True, partition_key=False)
version = columns.Integer(primary_key=True,
partition_key=False,
default=0,
clustering_order="DESC")
# UUID are not indexed
uuid = columns.Text(default=default_cdmi_id)
is_object = columns.Boolean(default=False)
# URL to a data object if the Tree node is not a container
# (radon:// for internal objects or anything else for a reference, we do not
# restrict the syntax of the URL yet, it's up to the client to manage the
# different URL stored in Cassandra)
object_url = columns.Text()
sys_meta = columns.Map(columns.Text, columns.Text)
user_meta = columns.Map(columns.Text, columns.Text)
acl = columns.Map(columns.Text, columns.UserDefinedType(Ace))
def add_default_acl(self):
"""Add read access to all authenticated users"""
self.create_acl_list(["AUTHENTICATED@"], [])
def create_acl(self, acl_cql):
"""
Replace the acl with the given cql string
:param acl_cql: The acl string to put in Cassandra, can be easily
generated in :meth:`radon.model.acl.acl_list_to_cql`
:type acl_cql: str
"""
session = connection.get_session()
keyspace = radon.cfg.dse_keyspace
session.set_keyspace(keyspace)
query = SimpleStatement(u"""UPDATE tree_node SET acl={}
WHERE container=%s and name=%s and version=%s""".format(acl_cql))
session.execute(query, (self.container, self.name, self.version))
def create_acl_list(self, read_access, write_access):
"""
Create ACL from lists of group uuids
:param read_access: A list of group names which have read access
:type read_access: List[str]
:param write_access: A list of group names which have write access
:type write_access: List[str]
"""
cql_string = acl_list_to_cql(read_access, write_access)
self.create_acl(cql_string)
def path(self):
"""
Get the full path of the element. See :meth:`radon.util.merge`
:return: The merged path
:rtype: str
"""
return merge(self.container, self.name)
def update_acl(self, acl_cql):
"""
Update the acl with the given cql string that will be added
:param acl_cql: The acl string to put in Cassandra, can be easily
generated in :meth:`radon.model.acl.acl_list_to_cql`
:type acl_cql: str
"""
session = connection.get_session()
keyspace = radon.cfg.dse_keyspace
session.set_keyspace(keyspace)
query = SimpleStatement(u"""UPDATE tree_node SET acl=acl+{}
WHERE container=%s and name=%s and version=%s""".format(acl_cql))
session.execute(query, (self.container, self.name, self.version))
class DataObject(Model):
""" The DataObject represents actual data objects, the tree structure
merely references it.
Each partition key gathers together all the data under one partition (the
CDMI ID ) and the object properties are represented using static columns
(one instance per partition)
It has a similar effect to a join to a properties table, except the
properties are stored with the rest of the partition
This is an 'efficient' model optimised for Cassandra's quirks.
N.B. by default Cassandra compresses its data ( using LZW ), so we get that
for free."""
# The 'name' of the object
uuid = columns.Text(default=default_cdmi_id, required=True,
partition_key=True)
#####################
# These columns are the same (shared) between all entries with same id
# (they use the static attribute , [ like an inode or a header ])
#####################
checksum = columns.Text(static=True)
size = columns.BigInt(default=0, static=True)
metadata = columns.Map(columns.Text, columns.Text, static=True)
mimetype = columns.Text(static=True)
alt_url = columns.Set(columns.Text, static=True)
create_ts = columns.DateTime(default=datetime.now, static=True)
modified_ts = columns.DateTime(default=datetime.now, static=True)
type = columns.Text(required=False, static=True, default='UNKNOWN')
acl = columns.Map(columns.Text, columns.UserDefinedType(Ace), static=True)
# A general aid to integrity ...
treepath = columns.Text(static=True, required=False)
#####################
# And 'clever' bit -- 'here' data, These will be the only per-record-fields
# in the partition (i.e. object)
# So the datastructure looks like a header , with an ordered list of blobs
#####################
# This is the 'clustering' key...
sequence_number = columns.Integer(primary_key=True, partition_key=False)
blob = columns.Blob(required=False)
compressed = columns.Boolean(default=False)
#####################
@classmethod
def append_chunk(cls, uuid, raw_data, sequence_number, compressed=False):
"""Create a new blob for an existing data_object"""
if compressed:
f = StringIO()
z = zipfile.ZipFile(f, "w", zipfile.ZIP_DEFLATED)
z.writestr("data", raw_data)
z.close()
data = f.getvalue()
f.close()
else:
data = raw_data
data_object = cls(uuid=uuid,
sequence_number=sequence_number,
blob=data,
compressed=compressed)
data_object.save()
return data_object
def chunk_content(self):
"""
Yields the content for the driver's URL, if any
a chunk at a time. The value yielded is the size of
the chunk and the content chunk itself.
"""
entries = DataObject.objects.filter(uuid=self.uuid)
for entry in entries:
if entry.compressed:
data = StringIO(entry.blob)
z = zipfile.ZipFile(data, 'r')
content = z.read("data")
data.close()
z.close()
yield content
else:
yield entry.blob
@classmethod
def create(cls, raw_data, compressed=False, metadata=None, create_ts=None, acl=None):
"""data: initial data"""
new_id = default_cdmi_id()
now = datetime.now()
if compressed:
f = StringIO()
z = zipfile.ZipFile(f, "w", zipfile.ZIP_DEFLATED)
z.writestr("data", raw_data)
z.close()
data = f.getvalue()
f.close()
else:
data = raw_data
kwargs = {
"uuid": new_id,
"sequence_number": 0,
"blob": data,
"compressed": compressed,
"modified_ts": now
}
if metadata:
kwargs['metadata'] = metadata
if create_ts:
kwargs['create_ts'] = create_ts
else:
kwargs['create_ts'] = now
if acl:
kwargs['acl'] = acl
new = super(DataObject, cls).create(**kwargs)
return new
def create_acl(self, acl_cql):
"""Replace the static acl with the given cql string"""
cfg = get_config(None)
session = connection.get_session()
keyspace = cfg.get('KEYSPACE', 'indigo')
session.set_keyspace(keyspace)
query = SimpleStatement(u"""UPDATE data_object SET acl = {}
WHERE uuid=%s""".format(acl_cql))
session.execute(query, (self.uuid,))
def create_acl_cdmi(self, cdmi_acl):
"""""Create entry ACL from a cdmi object (list of dict)"""
cql_string = acl_cdmi_to_cql(cdmi_acl)
self.create_acl(cql_string)
def create_acl_list(self, read_access, write_access):
"""Create ACL from two lists of groups id, existing ACL are replaced"""
cql_string = acl_list_to_cql(read_access, write_access)
self.create_acl(cql_string)
@classmethod
def delete_id(cls, uuid):
"""Delete all blobs for the specified uuid"""
cfg = get_config(None)
session = connection.get_session()
keyspace = cfg.get('KEYSPACE', 'indigo')
session.set_keyspace(keyspace)
query = SimpleStatement("""DELETE FROM data_object WHERE uuid=%s""")
session.execute(query, (uuid,))
@classmethod
def find(cls, uuid):
"""Find an object by uuid"""
entries = cls.objects.filter(uuid=uuid)
if not entries:
return None
else:
return entries.first()
def update(self, **kwargs):
"""Update a data object"""
cfg = get_config(None)
session = connection.get_session()
keyspace = cfg.get('KEYSPACE', 'indigo')
session.set_keyspace(keyspace)
for arg in kwargs:
# For static fields we can't use the name in the where condition
if arg in static_fields:
query = SimpleStatement("""UPDATE data_object SET {}=%s
WHERE uuid=%s""".format(arg))
session.execute(query, (kwargs[arg], self.uuid))
else:
print """UPDATE data_object SET {}=%s
WHERE uuid=%s and sequence_number=%s""".format(arg)
query = SimpleStatement("""UPDATE data_object SET {}=%s
WHERE uuid=%s and sequence_number=%s""".format(arg))
session.execute(query, (kwargs[arg], self.uuid, self.sequence_number))
return self
def update_acl(self, acl_cql):
"""Update the static acl with the given cql string
"""
cfg = get_config(None)
session = connection.get_session()
keyspace = cfg.get('KEYSPACE', 'indigo')
session.set_keyspace(keyspace)
query = SimpleStatement(u"""UPDATE data_object SET acl = acl + {}
WHERE uuid=%s""".format(acl_cql))
session.execute(query, (self.uuid,))
def update_acl_cdmi(self, cdmi_acl):
""""Update entry ACL from a cdmi object (list of dict)"""
cql_string = acl_cdmi_to_cql(cdmi_acl)
self.update_acl(cql_string)
def update_acl_list(self, read_access, write_access):
"""Update ACL from two lists of groups id, existing ACL are replaced"""
cql_string = acl_list_to_cql(read_access, write_access)
self.update_acl(cql_string)
