class Ace(UserType): """A user type to describe an Access Control Entry for a specific user or group. This can be used in an Access Control List in a resource or a collection The two types of ACEs in CDMI are ALLOW and DENY. An ALLOW ACE grants some form of access to a principal. Principals are either users or groups and are represented by identifiers. A DENY ACE denies access of some kind to a principal. In addition to principals some special identifiers can be used: - "EVERYONE@": The world - "ANONYMOUS@": Access without authentication - "AUTHENTICATED@": Any authenticated user (opposite of ANONYMOUS) CDMI allows for nested containers and mandates that objects and subcontainers be able to inherit access permissions from their parent containers. However, it is not enough to simply inherit all permissions from the parent; it might be desirable, for example, to have different default permissions on child objects and subcontainers of a given container. The mask field of an ACE contains a set of permissions allowed or denied. """ acetype = columns.Text() identifier = columns.Text() # aceflags isn't used yet, future versions may use it aceflags = columns.Integer() acemask = columns.Integer()
class Checkpoint(CustomDjangoCassandraModel): __table_name__ = "davinci_checkpoint" source = columns.Text(partition_key=True) key = columns.Text(primary_key=True) # When was created the entity and the last modification date created_at = columns.DateTime(default=datetime.utcnow) updated_at = columns.DateTime(default=datetime.utcnow) # Controls if the entity is active or has been deleted is_deleted = columns.Boolean(default=False) deleted_reason = columns.Text() data = columns.Text(required=False) class Meta: get_pk_field = "source" def get_data(self): return json.loads(self.data) def set_data(self, json_data): self.data = json.dumps(json_data, sort_keys=True, indent=4, default=default)
class SecondModel(Model): __table_name__ = 'first_model' first_key = columns.UUID(primary_key=True) second_key = columns.UUID() third_key = columns.Text() fourth_key = columns.Text()
class UserModel(Model): """Model class that maps to the user table""" __table_name__ = 'users' user_id = columns.UUID(db_field='userid', primary_key=True) first_name = columns.Text(db_field='firstname') last_name = columns.Text(db_field='lastname') email = columns.Text() created_date = columns.Date()
class ComplexModelRouting(Model): __table_name__ = 'complex_model_routing' partition = columns.UUID(partition_key=True, default=uuid4) cluster = columns.Integer(partition_key=True) count = columns.Integer() text = columns.Text(partition_key=True) float = columns.Float(partition_key=True) text_2 = columns.Text()
class TaskMoreInfo(UserType): __type_name__ = "task_more_info" # from where the more info was created source = columns.Text() created_at = columns.DateTime() # details about the error details = columns.Text()
class UserVideosModel(Model): """Model class that maps to the user_videos table""" __table_name__ = 'user_videos' user_id = columns.UUID(primary_key=True, db_field='userid') added_date = columns.DateTime(primary_key=True, clustering_order='DESC') video_id = columns.UUID(primary_key=True, clustering_order='ASC', db_field='videoid') name = columns.Text() preview_image_location = columns.Text()
class IDSearch(Model): """Reverse Search Model (lookup table for search from object_path) """ object_path = columns.Text(required=True, primary_key=True) term = columns.Text(required=True, primary_key=True) term_type = columns.Text(required=True, primary_key=True) @classmethod def find(cls, object_path): """Find all terms associated to an object id""" return cls.objects.filter(object_path=object_path).all()
class VideosModel(Model): """Model class that maps to the videos table""" __table_name__ = 'videos' video_id = columns.UUID(primary_key=True, db_field='videoid') user_id = columns.UUID(db_field='userid') name = columns.Text() description = columns.Text() location = columns.Text() location_type = columns.Integer() preview_image_location = columns.Text() tags = columns.Set(columns.Text) added_date = columns.DateTime()
class BovespaAccount(CustomDjangoCassandraModel): __table_name__ = "bovespa_account" # ID of the company in B3 ccvm = columns.Text(partition_key=True) # Date of the account value period = columns.Date(primary_key=True, clustering_order="DESC") # The version of the account. The company could present different # versions of the files version = columns.Text(primary_key=True, clustering_order="DESC") # The account number. Ex. "1.01.01" number = columns.Text(primary_key=True, max_length=15) # Financial type account (instant/individual or consolidated) financial_info_type = columns.Text(primary_key=True, max_length=15) # Type of financial statement balance_type = columns.Text(max_length=15, required=True) # The account name. Ex. "Receita de Venda de Bens e/ou Serviços" name = columns.Text(max_length=200, required=True) # Company sector sector = columns.Integer(default=0, required=True) # The amount of the account amount = Decimal(required=True, max_digits=20, decimal_places=2) # The comments. Used for "DFP_BALANCE_DMPL" accounts, explaining the # meaning of the account: Shareholder's Equity, Accrued Profit/Loss, etc. comments = columns.Text() # When was created the entity and the last modification date created_at = columns.DateTime(default=datetime.utcnow) updated_at = columns.DateTime(default=datetime.utcnow) # Controls if the entity is active or has been deleted is_deleted = columns.Boolean(default=False) deleted_reason = columns.Text() class Meta: get_pk_field = "ccvm" def validate(self): super().validate() if self.financial_info_type not in FINANCIAL_INFO_TYPES: raise ValidationError("Invalid financial type [{0}] for account " "[{1} {2}]. Valid types are: {3}.".format( self.financial_info_type, self.number, self.name, FINANCIAL_INFO_TYPES)) if self.balance_type not in BALANCE_TYPES: raise ValidationError( "Invalid balance type [{0}]. Valid types are: {1}.".format( self.balance_type, BALANCE_TYPES))
class Ace(UserType): """A user type to describe an Access Control Entry for a specific user or group. This can be used in an Access Control List in a resource or a collection The two types of ACEs in CDMI are ALLOW and DENY. An ALLOW ACE grants some form of access to a principal. Principals are either users or groups and are represented by identifiers. A DENY ACE denies access of some kind to a principal. In addition to principals some special identifiers can be used: - "EVERYONE@": The world - "ANONYMOUS@": Access without authentication - "AUTHENTICATED@": Any authenticated user (opposite of ANONYMOUS) CDMI allows for nested containers and mandates that objects and subcontainers be able to inherit access permissions from their parent containers. However, it is not enough to simply inherit all permissions from the parent; it might be desirable, for example, to have different default permissions on child objects and subcontainers of a given container. The mask field of an ACE contains a set of permissions allowed or denied. :param acetype: ALLOW or DENY :type acetype: :class:`columns.Text` :param identifier: groups the ACL refers to :type identifier: :class:`columns.Text` :param aceflags: not used yet :type aceflags: :class:`columns.Integer` :param acemask: the mask that defines the access (read, write, r/w or none) :type acemask: :class:`columns.Integer` """ acetype = columns.Text() identifier = columns.Text() # aceflags isn't used yet, future versions may use it aceflags = columns.Integer() acemask = columns.Integer() def __str__(self): """ ACE object represented as a string :return: A string representation :rtype: str """ return "(acetype: {}, identifier: {}, aceflags: {}, acemask: {})".format( self.acetype, self.identifier, self.aceflags, self.acemask)
def test_instantiation_with_column_instance(self): """ Tests that columns instantiated with a column instance work properly """ column = columns.Map(columns.Text(min_length=100), columns.Integer()) self.assertIsInstance(column.key_col, columns.Text) self.assertIsInstance(column.value_col, columns.Integer)
class FamilyMembers(Model): __keyspace__ = KEYSPACE id = columns.UUID(primary_key=True, default=uuid4) surname = columns.Text(primary_key=True) name = columns.Text(primary_key=True) birth_year = columns.Integer() sex = columns.Text(min_length=1, max_length=1) def validate(self): super(FamilyMembers, self).validate() if self.sex and self.sex not in 'mf': raise ValidationError("FamilyMember.sex must be one of ['m', 'f']") if self.birth_year and self.sex == 'f': raise ValidationError( "FamilyMember.birth_year is set, and 'a lady never tells'")
class AnotherTestModel(Model): __keyspace__ = 'ks1' partition = columns.Integer(primary_key=True) cluster = columns.Integer(primary_key=True) count = columns.Integer() text = columns.Text()
class FourthModel(Model): __table_name__ = 'first_model' first_key = columns.UUID(primary_key=True) second_key = columns.UUID() third_key = columns.Text() # renamed model field, but map to existing column renamed = columns.Map(columns.Text, columns.Text, db_field='blah')
class TestModelSave(Model): partition = columns.UUID(primary_key=True, default=uuid4) cluster = columns.Integer(primary_key=True) count = columns.Integer(required=False) text = columns.Text(required=False, index=True) text_set = columns.Set(columns.Text, required=False) text_list = columns.List(columns.Text, required=False) text_map = columns.Map(columns.Text, columns.Text, required=False)
class ThirdModel(Model): __table_name__ = 'first_model' first_key = columns.UUID(primary_key=True) second_key = columns.UUID() third_key = columns.Text() # removed fourth key, but it should stay in the DB blah = columns.Map(columns.Text, columns.Text)
class CommentsByVideoModel(Model): """Model class that maps to the comments_by_video table""" __table_name__ = 'comments_by_video' video_id = columns.UUID(db_field='videoid', primary_key=True) comment_id = columns.UUID(db_field='commentid', primary_key=True, clustering_order='DESC') user_id = columns.UUID(db_field='userid') comment = columns.Text()
class AllLeveledOptionsModel(Model): __options__ = { 'compaction': { 'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy', 'sstable_size_in_mb': '64' } } cid = columns.UUID(primary_key=True) name = columns.Text()
class ListenerLog(Model): """Listener Log Model""" script_name = columns.Text(partition_key=True) when = columns.TimeUUID(primary_key=True, default=default_time, clustering_order="DESC") stdout = columns.Text() stderr = columns.Text() @classmethod def recent(cls, script_name, count=20): """Return the last logs""" cfg = get_config(None) session = connection.get_session() keyspace = cfg.get('KEYSPACE', 'indigo') session.set_keyspace(keyspace) # I couldn't find how to disable paging in cqlengine in the "model" view # so I create the cal query directly query = SimpleStatement(u"""SELECT * from listener_log WHERE script_name = '{}' ORDER BY when DESC limit {}""".format(script_name, count)) # Disable paging for this query (we use IN and ORDER BY in the same # query query.fetch_size = None res = [] for row in session.execute(query): res.append(ListenerLog(**row).to_dict()) return res def to_dict(self): """Return a dictionary which describes a log for the web ui""" data = { 'when': datetime_from_uuid1(self.when), 'script_name': self.script_name, 'stdout': self.stdout, 'stderr': self.stderr, } return data
class AllDatatypes(UserType): a = columns.Ascii() b = columns.BigInt() c = columns.Blob() d = columns.Boolean() e = columns.DateTime() f = columns.Decimal() g = columns.Double() h = columns.Float() i = columns.Inet() j = columns.Integer() k = columns.Text() l = columns.TimeUUID() m = columns.UUID() n = columns.VarInt()
class AllSizeTieredOptionsModel(Model): __options__ = { 'compaction': { 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'bucket_low': '.3', 'bucket_high': '2', 'min_threshold': '2', 'max_threshold': '64', 'tombstone_compaction_interval': '86400' } } cid = columns.UUID(primary_key=True) name = columns.Text()
class AllDatatypesModel(Model): id = columns.Integer(primary_key=True) a = columns.Ascii() b = columns.BigInt() c = columns.Blob() d = columns.Boolean() e = columns.DateTime() f = columns.Decimal() g = columns.Double() h = columns.Float() i = columns.Inet() j = columns.Integer() k = columns.Text() l = columns.TimeUUID() m = columns.UUID() n = columns.VarInt()
class Address(UserType): """ A User Defined type for model an Address, a unit value to be consolidated """ __type_name__ = "address" street_type = columns.Text() street_name = columns.Text() street_number = columns.Integer() city = columns.Text() region = columns.Text() state = columns.Text() country_code = columns.Text(min_length=3, max_length=3) zipcode = columns.Text()
class BaseEntity(CustomDjangoCassandraModel): """ The common field that will be shared between all the managed entities """ __abstract__ = True # A unique identifier of the entity _id = columns.UUID(primary_key=True, default=uuid.uuid4) # When was created the entity and the last modification date created_at = columns.DateTime(default=datetime.utcnow) updated_at = columns.DateTime(default=datetime.utcnow) # Controls if the entity is active or has been deleted is_deleted = columns.Boolean(default=False) deleted_reason = columns.Text() class Meta: get_pk_field = '_id'
class BovespaCompany(CustomDjangoCassandraModel): __table_name__ = "bovespa_company" # Force that all the values will reside in the seam node of the cluster entity_type = columns.Text(partition_key=True, default="company") # ID of the company in B3 ccvm = columns.Text(primary_key=True) # When was created the entity and the last modification date created_at = columns.DateTime(default=datetime.utcnow) updated_at = columns.DateTime(default=datetime.utcnow) # Controls if the entity is active or has been deleted is_deleted = columns.Boolean(default=False) deleted_reason = columns.Text() company_name = columns.Text(required=True) cnpj = columns.Text() company_type = columns.Text() situation = columns.Text(required=True) granted_date = columns.Date() canceled_date = columns.Date() class Meta: get_pk_field = "entity_type" def validate(self): super().validate() if self.situation not in SITUATIONS: raise ValidationError( "Invalid situation [{0}]. Valid situations are: {1}.".format( self.situation, SITUATIONS))
class TestIfNotExistsModel(Model): id = columns.UUID(primary_key=True, default=lambda: uuid4()) count = columns.Integer() text = columns.Text(required=False)
class SearchIndex(Model): """SearchIndex Model""" term = columns.Text(required=True, primary_key=True) term_type = columns.Text(required=True, primary_key=True) object_path = columns.Text(required=True, primary_key=True) object_type = columns.Text(required=True) uuid = columns.Text(default=default_uuid) @classmethod def create(cls, **kwargs): """Create a new indexed term""" from indigo.models import IDSearch idx = super(SearchIndex, cls).create(**kwargs) # Create a row in the ID search table idx = IDSearch.create(object_path=idx.object_path, term=idx.term, term_type=idx.term_type) return idx @classmethod def find(cls, termstrings, user): from indigo.models.collection import Collection from indigo.models.resource import Resource def get_object(obj, user): """Return the object corresponding to the SearchIndex object""" if obj.object_type == 'Collection': result_obj = Collection.find(obj.object_path) if not result_obj or not result_obj.user_can(user, "read"): return None result_obj = result_obj.to_dict(user) result_obj['result_type'] = 'Collection' return result_obj elif obj.object_type == 'Resource': result_obj = Resource.find(obj.object_path) # Check the resource's collection for read permission if not result_obj or not result_obj.user_can(user, "read"): return None result_obj = result_obj.to_dict(user) result_obj['result_type'] = 'Resource' return result_obj return None result_objects = [] for t in termstrings: if cls.is_stop_word(t): continue result_objects.extend(cls.objects.filter(term=t).all()) results = [] for result in result_objects: try: results.append(get_object(result, user)) except AttributeError: logging.warning( u"Problem with SearchIndex('{}','{}','{}','{}')".format( result.uuid, result.term, result.object_type, result.uuid)) results = [x for x in results if x] # Do some sane ordering here to group together by ID and # order by frequency. Add the hit_count to the object dictionary # and then we can order on that keys = set(r['id'] for r in results) result_list = [] for k in keys: # get each element with this key, count them, store the hit # count and only add one to results matches = [x for x in results if x['id'] == k] match = matches[0] match['hit_count'] = len(matches) result_list.append(match) return sorted(result_list, key=lambda res: res.get('hit_count', 0), reverse=True) @classmethod def is_stop_word(cls, term): """Check if a term is a stop word""" return term in [ "a", "an", "and", "the", "of", "is", "in", "it", "or", "to" ] @classmethod def reset(cls, object_path): """Delete objects from the SearchIndex""" from indigo.models import IDSearch rows = IDSearch.find(object_path) for id_obj in rows: obj = cls.objects.filter(term=id_obj.term, term_type=id_obj.term_type, object_path=id_obj.object_path).first() if obj: obj.delete() id_obj.delete() @classmethod def index(cls, object, fields=['name']): """Index""" result_count = 0 def clean(t): """Clean a term""" if t: return t.lower().replace('.', ' ').replace('_', ' ').split(' ') else: return [] def clean_full(t): """Clean a term but keep all chars""" if t: return t.lower() else: return "" terms = [] if 'metadata' in fields: metadata = object.get_cdmi_metadata() # Metadata are stored as json string, get_metadata() returns it as # a Python dictionary for k, v in metadata.iteritems(): # A value can be a string or a list of string if isinstance(v, list): for vv in v: terms.extend([('metadata', el) for el in clean(vv.strip())]) else: terms.extend([('metadata', el) for el in clean(v.strip())]) fields.remove('metadata') for f in fields: attr = getattr(object, f) if isinstance(attr, dict): for k, v in attr.iteritems(): terms.extend([(f, el) for el in clean(v.strip())]) terms.append((f, clean_full(v.strip()))) else: terms.extend([(f, el) for el in clean(attr)]) terms.append((f, clean_full(attr))) object_type = object.__class__.__name__ for term_type, term in terms: if cls.is_stop_word(term): continue if len(term) < 2: continue SearchIndex.create(term=term, term_type=term_type, object_type=object_type, object_path=object.path) result_count += 1 return result_count def __unicode__(self): return unicode("".format(self.term, self.object_type))
class CapitalizedKeyModel(Model): firstKey = columns.Integer(primary_key=True) secondKey = columns.Integer(primary_key=True) someData = columns.Text()
class LowercaseKeyModel(Model): first_key = columns.Integer(primary_key=True) second_key = columns.Integer(primary_key=True) some_data = columns.Text()