class Ace(UserType):
"""A user type to describe an Access Control Entry for a specific user
or group. This can be used in an Access Control List in a resource or a
collection
The two types of ACEs in CDMI are ALLOW and DENY. An ALLOW ACE grants some
form of access to a principal. Principals are either users or groups and
are represented by identifiers. A DENY ACE denies access of some kind to a
principal.
In addition to principals some special identifiers can be used:
- "EVERYONE@": The world
- "ANONYMOUS@": Access without authentication
- "AUTHENTICATED@": Any authenticated user (opposite of ANONYMOUS)
CDMI allows for nested containers and mandates that objects and
subcontainers be able to inherit access permissions from their parent
containers. However, it is not enough to simply inherit all permissions
from the parent; it might be desirable, for example, to have different
default permissions on child objects and subcontainers of a given
container.
The mask field of an ACE contains a set of permissions allowed or denied.
"""
acetype = columns.Text()
identifier = columns.Text()
# aceflags isn't used yet, future versions may use it
aceflags = columns.Integer()
acemask = columns.Integer()
class Checkpoint(CustomDjangoCassandraModel):
__table_name__ = "davinci_checkpoint"
source = columns.Text(partition_key=True)
key = columns.Text(primary_key=True)
# When was created the entity and the last modification date
created_at = columns.DateTime(default=datetime.utcnow)
updated_at = columns.DateTime(default=datetime.utcnow)
# Controls if the entity is active or has been deleted
is_deleted = columns.Boolean(default=False)
deleted_reason = columns.Text()
data = columns.Text(required=False)
class Meta:
get_pk_field = "source"
def get_data(self):
return json.loads(self.data)
def set_data(self, json_data):
self.data = json.dumps(json_data, sort_keys=True, indent=4, default=default)
class SecondModel(Model):
__table_name__ = 'first_model'
first_key = columns.UUID(primary_key=True)
second_key = columns.UUID()
third_key = columns.Text()
fourth_key = columns.Text()
class UserModel(Model):
"""Model class that maps to the user table"""
__table_name__ = 'users'
user_id = columns.UUID(db_field='userid', primary_key=True)
first_name = columns.Text(db_field='firstname')
last_name = columns.Text(db_field='lastname')
email = columns.Text()
created_date = columns.Date()
class ComplexModelRouting(Model):
__table_name__ = 'complex_model_routing'
partition = columns.UUID(partition_key=True, default=uuid4)
cluster = columns.Integer(partition_key=True)
count = columns.Integer()
text = columns.Text(partition_key=True)
float = columns.Float(partition_key=True)
text_2 = columns.Text()
class TaskMoreInfo(UserType):
__type_name__ = "task_more_info"
# from where the more info was created
source = columns.Text()
created_at = columns.DateTime()
# details about the error
details = columns.Text()
class UserVideosModel(Model):
"""Model class that maps to the user_videos table"""
__table_name__ = 'user_videos'
user_id = columns.UUID(primary_key=True, db_field='userid')
added_date = columns.DateTime(primary_key=True, clustering_order='DESC')
video_id = columns.UUID(primary_key=True,
clustering_order='ASC',
db_field='videoid')
name = columns.Text()
preview_image_location = columns.Text()
class IDSearch(Model):
"""Reverse Search Model (lookup table for search from object_path)
"""
object_path = columns.Text(required=True, primary_key=True)
term = columns.Text(required=True, primary_key=True)
term_type = columns.Text(required=True, primary_key=True)
@classmethod
def find(cls, object_path):
"""Find all terms associated to an object id"""
return cls.objects.filter(object_path=object_path).all()
class VideosModel(Model):
"""Model class that maps to the videos table"""
__table_name__ = 'videos'
video_id = columns.UUID(primary_key=True, db_field='videoid')
user_id = columns.UUID(db_field='userid')
name = columns.Text()
description = columns.Text()
location = columns.Text()
location_type = columns.Integer()
preview_image_location = columns.Text()
tags = columns.Set(columns.Text)
added_date = columns.DateTime()
class BovespaAccount(CustomDjangoCassandraModel):
__table_name__ = "bovespa_account"
# ID of the company in B3
ccvm = columns.Text(partition_key=True)
# Date of the account value
period = columns.Date(primary_key=True, clustering_order="DESC")
# The version of the account. The company could present different
# versions of the files
version = columns.Text(primary_key=True, clustering_order="DESC")
# The account number. Ex. "1.01.01"
number = columns.Text(primary_key=True, max_length=15)
# Financial type account (instant/individual or consolidated)
financial_info_type = columns.Text(primary_key=True, max_length=15)
# Type of financial statement
balance_type = columns.Text(max_length=15, required=True)
# The account name. Ex. "Receita de Venda de Bens e/ou Serviços"
name = columns.Text(max_length=200, required=True)
# Company sector
sector = columns.Integer(default=0, required=True)
# The amount of the account
amount = Decimal(required=True, max_digits=20, decimal_places=2)
# The comments. Used for "DFP_BALANCE_DMPL" accounts, explaining the
# meaning of the account: Shareholder's Equity, Accrued Profit/Loss, etc.
comments = columns.Text()
# When was created the entity and the last modification date
created_at = columns.DateTime(default=datetime.utcnow)
updated_at = columns.DateTime(default=datetime.utcnow)
# Controls if the entity is active or has been deleted
is_deleted = columns.Boolean(default=False)
deleted_reason = columns.Text()
class Meta:
get_pk_field = "ccvm"
def validate(self):
super().validate()
if self.financial_info_type not in FINANCIAL_INFO_TYPES:
raise ValidationError("Invalid financial type [{0}] for account "
"[{1} {2}]. Valid types are: {3}.".format(
self.financial_info_type, self.number,
self.name, FINANCIAL_INFO_TYPES))
if self.balance_type not in BALANCE_TYPES:
raise ValidationError(
"Invalid balance type [{0}]. Valid types are: {1}.".format(
self.balance_type, BALANCE_TYPES))
class Ace(UserType):
"""A user type to describe an Access Control Entry for a specific user
or group. This can be used in an Access Control List in a resource or a
collection
The two types of ACEs in CDMI are ALLOW and DENY. An ALLOW ACE grants some
form of access to a principal. Principals are either users or groups and
are represented by identifiers. A DENY ACE denies access of some kind to a
principal.
In addition to principals some special identifiers can be used:
- "EVERYONE@": The world
- "ANONYMOUS@": Access without authentication
- "AUTHENTICATED@": Any authenticated user (opposite of ANONYMOUS)
CDMI allows for nested containers and mandates that objects and
subcontainers be able to inherit access permissions from their parent
containers. However, it is not enough to simply inherit all permissions
from the parent; it might be desirable, for example, to have different
default permissions on child objects and subcontainers of a given
container.
The mask field of an ACE contains a set of permissions allowed or denied.
:param acetype: ALLOW or DENY
:type acetype: :class:`columns.Text`
:param identifier: groups the ACL refers to
:type identifier: :class:`columns.Text`
:param aceflags: not used yet
:type aceflags: :class:`columns.Integer`
:param acemask: the mask that defines the access (read, write, r/w or none)
:type acemask: :class:`columns.Integer`
"""
acetype = columns.Text()
identifier = columns.Text()
# aceflags isn't used yet, future versions may use it
aceflags = columns.Integer()
acemask = columns.Integer()
def __str__(self):
"""
ACE object represented as a string
:return: A string representation
:rtype: str
"""
return "(acetype: {}, identifier: {}, aceflags: {}, acemask: {})".format(
self.acetype, self.identifier, self.aceflags, self.acemask)
def test_instantiation_with_column_instance(self):
"""
Tests that columns instantiated with a column instance work properly
"""
column = columns.Map(columns.Text(min_length=100), columns.Integer())
self.assertIsInstance(column.key_col, columns.Text)
self.assertIsInstance(column.value_col, columns.Integer)
class FamilyMembers(Model):
__keyspace__ = KEYSPACE
id = columns.UUID(primary_key=True, default=uuid4)
surname = columns.Text(primary_key=True)
name = columns.Text(primary_key=True)
birth_year = columns.Integer()
sex = columns.Text(min_length=1, max_length=1)
def validate(self):
super(FamilyMembers, self).validate()
if self.sex and self.sex not in 'mf':
raise ValidationError("FamilyMember.sex must be one of ['m', 'f']")
if self.birth_year and self.sex == 'f':
raise ValidationError(
"FamilyMember.birth_year is set, and 'a lady never tells'")
class AnotherTestModel(Model):
__keyspace__ = 'ks1'
partition = columns.Integer(primary_key=True)
cluster = columns.Integer(primary_key=True)
count = columns.Integer()
text = columns.Text()
class FourthModel(Model):
__table_name__ = 'first_model'
first_key = columns.UUID(primary_key=True)
second_key = columns.UUID()
third_key = columns.Text()
# renamed model field, but map to existing column
renamed = columns.Map(columns.Text, columns.Text, db_field='blah')
class TestModelSave(Model):
partition = columns.UUID(primary_key=True, default=uuid4)
cluster = columns.Integer(primary_key=True)
count = columns.Integer(required=False)
text = columns.Text(required=False, index=True)
text_set = columns.Set(columns.Text, required=False)
text_list = columns.List(columns.Text, required=False)
text_map = columns.Map(columns.Text, columns.Text, required=False)
class ThirdModel(Model):
__table_name__ = 'first_model'
first_key = columns.UUID(primary_key=True)
second_key = columns.UUID()
third_key = columns.Text()
# removed fourth key, but it should stay in the DB
blah = columns.Map(columns.Text, columns.Text)
class CommentsByVideoModel(Model):
"""Model class that maps to the comments_by_video table"""
__table_name__ = 'comments_by_video'
video_id = columns.UUID(db_field='videoid', primary_key=True)
comment_id = columns.UUID(db_field='commentid',
primary_key=True,
clustering_order='DESC')
user_id = columns.UUID(db_field='userid')
comment = columns.Text()
class AllLeveledOptionsModel(Model):
__options__ = {
'compaction': {
'class':
'org.apache.cassandra.db.compaction.LeveledCompactionStrategy',
'sstable_size_in_mb': '64'
}
}
cid = columns.UUID(primary_key=True)
name = columns.Text()
class ListenerLog(Model):
"""Listener Log Model"""
script_name = columns.Text(partition_key=True)
when = columns.TimeUUID(primary_key=True,
default=default_time,
clustering_order="DESC")
stdout = columns.Text()
stderr = columns.Text()
@classmethod
def recent(cls, script_name, count=20):
"""Return the last logs"""
cfg = get_config(None)
session = connection.get_session()
keyspace = cfg.get('KEYSPACE', 'indigo')
session.set_keyspace(keyspace)
# I couldn't find how to disable paging in cqlengine in the "model" view
# so I create the cal query directly
query = SimpleStatement(u"""SELECT * from listener_log WHERE
script_name = '{}'
ORDER BY when DESC
limit {}""".format(script_name, count))
# Disable paging for this query (we use IN and ORDER BY in the same
# query
query.fetch_size = None
res = []
for row in session.execute(query):
res.append(ListenerLog(**row).to_dict())
return res
def to_dict(self):
"""Return a dictionary which describes a log for the web ui"""
data = {
'when': datetime_from_uuid1(self.when),
'script_name': self.script_name,
'stdout': self.stdout,
'stderr': self.stderr,
}
return data
class AllDatatypes(UserType):
a = columns.Ascii()
b = columns.BigInt()
c = columns.Blob()
d = columns.Boolean()
e = columns.DateTime()
f = columns.Decimal()
g = columns.Double()
h = columns.Float()
i = columns.Inet()
j = columns.Integer()
k = columns.Text()
l = columns.TimeUUID()
m = columns.UUID()
n = columns.VarInt()
class AllSizeTieredOptionsModel(Model):
__options__ = {
'compaction': {
'class':
'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy',
'bucket_low': '.3',
'bucket_high': '2',
'min_threshold': '2',
'max_threshold': '64',
'tombstone_compaction_interval': '86400'
}
}
cid = columns.UUID(primary_key=True)
name = columns.Text()
class AllDatatypesModel(Model):
id = columns.Integer(primary_key=True)
a = columns.Ascii()
b = columns.BigInt()
c = columns.Blob()
d = columns.Boolean()
e = columns.DateTime()
f = columns.Decimal()
g = columns.Double()
h = columns.Float()
i = columns.Inet()
j = columns.Integer()
k = columns.Text()
l = columns.TimeUUID()
m = columns.UUID()
n = columns.VarInt()
class Address(UserType):
"""
A User Defined type for model an Address, a unit value to be consolidated
"""
__type_name__ = "address"
street_type = columns.Text()
street_name = columns.Text()
street_number = columns.Integer()
city = columns.Text()
region = columns.Text()
state = columns.Text()
country_code = columns.Text(min_length=3, max_length=3)
zipcode = columns.Text()
class BaseEntity(CustomDjangoCassandraModel):
"""
The common field that will be shared between all the managed entities
"""
__abstract__ = True
# A unique identifier of the entity
_id = columns.UUID(primary_key=True, default=uuid.uuid4)
# When was created the entity and the last modification date
created_at = columns.DateTime(default=datetime.utcnow)
updated_at = columns.DateTime(default=datetime.utcnow)
# Controls if the entity is active or has been deleted
is_deleted = columns.Boolean(default=False)
deleted_reason = columns.Text()
class Meta:
get_pk_field = '_id'
class BovespaCompany(CustomDjangoCassandraModel):
__table_name__ = "bovespa_company"
# Force that all the values will reside in the seam node of the cluster
entity_type = columns.Text(partition_key=True, default="company")
# ID of the company in B3
ccvm = columns.Text(primary_key=True)
# When was created the entity and the last modification date
created_at = columns.DateTime(default=datetime.utcnow)
updated_at = columns.DateTime(default=datetime.utcnow)
# Controls if the entity is active or has been deleted
is_deleted = columns.Boolean(default=False)
deleted_reason = columns.Text()
company_name = columns.Text(required=True)
cnpj = columns.Text()
company_type = columns.Text()
situation = columns.Text(required=True)
granted_date = columns.Date()
canceled_date = columns.Date()
class Meta:
get_pk_field = "entity_type"
def validate(self):
super().validate()
if self.situation not in SITUATIONS:
raise ValidationError(
"Invalid situation [{0}]. Valid situations are: {1}.".format(
self.situation, SITUATIONS))
class TestIfNotExistsModel(Model):
id = columns.UUID(primary_key=True, default=lambda: uuid4())
count = columns.Integer()
text = columns.Text(required=False)
class SearchIndex(Model):
"""SearchIndex Model"""
term = columns.Text(required=True, primary_key=True)
term_type = columns.Text(required=True, primary_key=True)
object_path = columns.Text(required=True, primary_key=True)
object_type = columns.Text(required=True)
uuid = columns.Text(default=default_uuid)
@classmethod
def create(cls, **kwargs):
"""Create a new indexed term"""
from indigo.models import IDSearch
idx = super(SearchIndex, cls).create(**kwargs)
# Create a row in the ID search table
idx = IDSearch.create(object_path=idx.object_path,
term=idx.term,
term_type=idx.term_type)
return idx
@classmethod
def find(cls, termstrings, user):
from indigo.models.collection import Collection
from indigo.models.resource import Resource
def get_object(obj, user):
"""Return the object corresponding to the SearchIndex object"""
if obj.object_type == 'Collection':
result_obj = Collection.find(obj.object_path)
if not result_obj or not result_obj.user_can(user, "read"):
return None
result_obj = result_obj.to_dict(user)
result_obj['result_type'] = 'Collection'
return result_obj
elif obj.object_type == 'Resource':
result_obj = Resource.find(obj.object_path)
# Check the resource's collection for read permission
if not result_obj or not result_obj.user_can(user, "read"):
return None
result_obj = result_obj.to_dict(user)
result_obj['result_type'] = 'Resource'
return result_obj
return None
result_objects = []
for t in termstrings:
if cls.is_stop_word(t):
continue
result_objects.extend(cls.objects.filter(term=t).all())
results = []
for result in result_objects:
try:
results.append(get_object(result, user))
except AttributeError:
logging.warning(
u"Problem with SearchIndex('{}','{}','{}','{}')".format(
result.uuid, result.term, result.object_type,
result.uuid))
results = [x for x in results if x]
# Do some sane ordering here to group together by ID and
# order by frequency. Add the hit_count to the object dictionary
# and then we can order on that
keys = set(r['id'] for r in results)
result_list = []
for k in keys:
# get each element with this key, count them, store the hit
# count and only add one to results
matches = [x for x in results if x['id'] == k]
match = matches[0]
match['hit_count'] = len(matches)
result_list.append(match)
return sorted(result_list,
key=lambda res: res.get('hit_count', 0),
reverse=True)
@classmethod
def is_stop_word(cls, term):
"""Check if a term is a stop word"""
return term in [
"a", "an", "and", "the", "of", "is", "in", "it", "or", "to"
]
@classmethod
def reset(cls, object_path):
"""Delete objects from the SearchIndex"""
from indigo.models import IDSearch
rows = IDSearch.find(object_path)
for id_obj in rows:
obj = cls.objects.filter(term=id_obj.term,
term_type=id_obj.term_type,
object_path=id_obj.object_path).first()
if obj:
obj.delete()
id_obj.delete()
@classmethod
def index(cls, object, fields=['name']):
"""Index"""
result_count = 0
def clean(t):
"""Clean a term"""
if t:
return t.lower().replace('.', ' ').replace('_', ' ').split(' ')
else:
return []
def clean_full(t):
"""Clean a term but keep all chars"""
if t:
return t.lower()
else:
return ""
terms = []
if 'metadata' in fields:
metadata = object.get_cdmi_metadata()
# Metadata are stored as json string, get_metadata() returns it as
# a Python dictionary
for k, v in metadata.iteritems():
# A value can be a string or a list of string
if isinstance(v, list):
for vv in v:
terms.extend([('metadata', el)
for el in clean(vv.strip())])
else:
terms.extend([('metadata', el) for el in clean(v.strip())])
fields.remove('metadata')
for f in fields:
attr = getattr(object, f)
if isinstance(attr, dict):
for k, v in attr.iteritems():
terms.extend([(f, el) for el in clean(v.strip())])
terms.append((f, clean_full(v.strip())))
else:
terms.extend([(f, el) for el in clean(attr)])
terms.append((f, clean_full(attr)))
object_type = object.__class__.__name__
for term_type, term in terms:
if cls.is_stop_word(term):
continue
if len(term) < 2:
continue
SearchIndex.create(term=term,
term_type=term_type,
object_type=object_type,
object_path=object.path)
result_count += 1
return result_count
def __unicode__(self):
return unicode("".format(self.term, self.object_type))
class CapitalizedKeyModel(Model):
firstKey = columns.Integer(primary_key=True)
secondKey = columns.Integer(primary_key=True)
someData = columns.Text()
class LowercaseKeyModel(Model):
first_key = columns.Integer(primary_key=True)
second_key = columns.Integer(primary_key=True)
some_data = columns.Text()
