def read(self, id, mode=FetchMode.Property): """ Reads a Model from Cassandra @param id: The `id` of the property in its natural python state before conversion. @param mode: specifies whether you want to read all the columns or just the static properties of the Model """ from homer.core.models import CqlProperty q = "SELECT {mode} FROM {keyspace}.{kind} WHERE id={id};" if mode == FetchMode.All: mode = "*" else: columns = [] properties = fields(self.model, CqlProperty) for name, prop in properties.items(): if prop.saveable(): columns.append(name) mode = ",".join(columns) key = properties.get("id") if "id" in properties else self.model.default[0] id = key.convert(self.model, id) # Convert the id to something CQL can use. query = q.format(mode=mode, kind=self.model.kind(), id=id, keyspace=self.model.keyspace) try: query = CqlQuery(clasz=self.model, query=query) query.execute() return query.one() except Exception as e: logging.exception(e) raise e
def __delitem__(self, key): '''Allows dictionary style item deletions to work properly''' props = fields(self, Property) if key in props: delattr(self, key) else: del self.__store__[key]
def convert(self, instance, value, insert=False, update=False): '''References are stored as serialized Key objects in the datastore''' assert isinstance(value, Model), "You can only use the Reference descriptor with Models" value = self.validate(value) properties = fields(value, CqlProperty) key = properties.get("id") if "id" in properties else value.default[0] id = key.convert(value, value.id) pointer = { "keyspace" : value.keyspace, "kind" : value.kind(), "id" : id } value = yaml.dump(pointer) #Serialize it with YAML and then quote it. value = quote(value) if update and not insert: arguments = { "name":self.name, "value":value, "id":id, "keyspace":instance.keyspace, "kind":instance.kind(), } q = "UPDATE {keyspace}.{kind} {ttl} SET {name}={value} WHERE id={id}" if self.expire: query = q.format(ttl="USING TTL %s" % self.expire, **arguments) else: query = q.format(ttl="", **arguments) return query else: return value
def __iter__(self): """CqlQuery objects yield Model objects or longs during counts.""" from homer.core.models import CqlProperty from homer.core.commons import Integer try: if not self.results: print("Executing Query: {}".format(self.query)) self.execute() except Exception as e: logging.exception(e) traceback.print_exc(e) # PROCESS THE RESULTS OF THE QUERY. if self.results.type == CqlResultType.ROWS: print("CqlResultType.ROWS: Decoding Rows.") if not self.clasz: raise CqlQueryException("No Model attached to this CqlQuery") else: rows = self.results.rows if re.search("COUNT\(\*\)", self.query): #Pragma no cover, C9 doesn't give us neat way to do this. row = rows[0] column = row.columns[0] assert column.name == "count", "Please contact dev support, this is unexpected behaviour" yield Integer().deconvert(column.value) properties = fields(self.clasz, CqlProperty) for row in rows: arguments = {} for column in row.columns: print("PROCESSING RAW COLUMN: " + str(column)) name = column.name arguments[name] = properties[name].deconvert(column.value) yield self.clasz(**arguments) else: print("CqlResultType.VOID: Nothing to do..")
def __parse_where_keywords(self, keywords): '''An internal helper method for query and count''' from homer.core.models import CqlProperty, Operator, EQ properties = fields(self.model, CqlProperty) extension = "" for name, value in keywords.items(): part = "" if isinstance(value, Operator): assert value.right is not None, "Your Operator must have its RHS set to be valid" operator = value operator.model = self.model operator.left = name part = str(operator) else: property = properties.get(name, None) #If an operator is not set, assume the EQ operator is being used. if not property: raise ValueError("The {name} property doesn't exist on {model}".format(name=name, model=self.model)) operator = EQ(right=value) operator.left = name operator.model = self.model part = str(operator) if not self.__where_used: extension += "WHERE {part}".format(part=part) self.__where_used = True else: extension += " AND {part}".format(part=part) return extension
def load(self, key, coscs): '''Creates a Model from an iterable of ColumnOrSuperColumns''' if not coscs: return None cls = Schema.ClassForModel(key.namespace, key.kind) info = Schema.Get(cls) model = cls() descriptors = fields(cls, Property) for cosc in coscs: name = cosc.column.name if name in descriptors: # Deconvert static properties first. prop = descriptors[name] deconverted = prop.deconvert(cosc.column.value) model[name] = deconverted else: # Deconvert dynamic properties, this deconverts column names, and column values k, v = model.default k = k() if isinstance(k, type) else k v = v() if isinstance(v, type) else v name = k.deconvert(cosc.column.name) value = v.deconvert(cosc.column.value) model[name] = value keyname = info[2] setattr(model, keyname, key.id) #Make sure the newly returned model has the same key key = model.key() key.saved = True return model
def read(clasz, key, fetchmode=FetchMode.Property): '''Read a Model from Cassandra''' assert key and fetchmode, "specify key and fetchmode" assert key.complete(), "your key has to be complete" parent = ColumnParent(column_family = key.kind) predicate = None if fetchmode == FetchMode.Property: if key.columns: predicate = SlicePredicate(column_names = key.columns) else: type = Schema.ClassForModel(key.namespace, key.kind) names = fields(type, Property).keys() columns = list(names) predicate = SlicePredicate(column_names = columns) elif fetchmode == FetchMode.All: range = SliceRange(start='', finish='', count = FETCHSIZE ) predicate = SlicePredicate(slice_range=range) found = None pool = poolFor(key.namespace) with using(pool) as conn: keyspace = keyspaceFor(key.namespace) conn.client.set_keyspace(keyspace) coscs = conn.client.get_slice(key.id, parent, predicate, clasz.consistency) found = MetaModel.load(key, coscs) return found
def convert(self, instance=None, value=None, insert=False, update=False): '''Converts the basic type with the str operation, which we can do an eval() on.''' from homer.core.types import blob value = self.validate(value) if update and not insert: assert isinstance(instance, Model), "We only accept Model instances for inserts or updates" value = quote(value) properties = fields(instance, CqlProperty) key = properties.get("id") if "id" in properties else instance.default[0] id = key.convert(instance, instance.id) arguments = {"name":self.name, "value":value, "keyspace":instance.keyspace, "kind":instance.kind(), "id":id} q = "UPDATE {keyspace}.{kind} {ttl} SET {name}={value} WHERE id={id}" if self.expire: query = q.format(ttl="USING TTL %s" % self.expire, **arguments) else: query = q.format(ttl="", **arguments) return query else: if isinstance(value, basestring): return quote(value) if isinstance(value, bool): value = quote("true") if value else quote("false") return value if isinstance(value, blob): return repr(value) val = self.type(value) return str(val)
def delete(self, *ids): """ Delete the Model(s) whose keys are in *ids @param *ids: A list of the ids of the models you want to delete in their natural python form. """ from homer.core.models import CqlProperty properties = fields(self.model, CqlProperty) key = properties.get("id") if "id" in properties else self.model.default[0] q = "DELETE FROM {keyspace}.{kind} WHERE id={id};" deletes = [] for id in ids: new = key.convert(value=id) query = q.format(kind=self.model.kind(), id=new, keyspace=self.model.keyspace) deletes.append(query) joined = ";".join(deletes) timestamp = self.now() batch = """ BEGIN BATCH USING TIMESTAMP {timestamp} {deletes} APPLY BATCH; """ batch = batch.format(deletes=joined, timestamp=timestamp) try: CqlQuery(query=batch).execute() except Exception as e: logging.exception(e) raise e
def __init__(self, model): '''Creates a Transform for this BaseModel''' info = Schema.Get(model) self.model = model self.namespace = info[0] self.keyspace = keyspaceFor(self.namespace) self.kind = info[1] self.key = info[2] self.comment = self.kind.__doc__ self.fields = fields(model, Property)
def convert(self): '''Generic implementation for the conversion routine.''' assert hasattr(self, "left") and hasattr(self, "model") and hasattr(self, "right"), "This Operator isn't complete." assert isinstance(self.left, basestring), "The LHS of a EQ query has to be a valid property name" property = fields(self.model, CqlProperty).get(self.left, None) if not property or not property.indexed(): raise ValueError("{self.left} is not an indexed property".format(self=self)) left = self.left right = property.convert(self.model, self.right) #Normal conversion. return left, right
def __setitem__(self, key, value): '''Allows dictionary style item sets to behave properly''' props = fields(self, Property) if key in props: setattr(self, key, value) else: # The converter for default properties are instantiated everytime, authors of descriptors should note. k, v = self.default k = k() if isinstance(k, type) else k #Construct the converter object if necessary v = v() if isinstance(v, type) else v self.__store__[k(key)] = v(value)
def __new__(cls, *arguments, **keywords): '''Customizes all Model instances to include special attributes''' instance = BaseModel.__new__(cls, *arguments, **keywords) if not hasattr(cls, "default"): cls.default = Default() if not hasattr(cls, "keyspace"): cls.keyspace = KeyspaceProperty() if not hasattr(cls, "expire"): cls.expire = ExpiryProperty() [prop.configure(name, cls) for name, prop in fields(cls, Property).items()] cls.objects = ObjectsProperty() return instance
def validate(self): '''Checks if a Model can be persisted to Cassandra''' # MAKES SURE THAT ALL REQUIRED PROPERTIES ARE AVAILABLE BEFORE PERSISTENCE. properties = fields(self, CqlProperty) assert "id" in properties, "Every Model must have an id property" for name, prop in properties.items(): if hasattr(prop, 'required') and prop.required: value = getattr(self, name) if prop.empty(value): raise BadValueError("Property: %s is required" % name) elif hasattr(prop, 'key') and prop.key: value = getattr(self, name) if prop.empty(value): raise BadValueError("Property: %s is a key so its required" % name)
def save(self, unique=False): """Stores this Model in Cassandra in one batch update.""" from homer.core.types import TypedCollection model = MetaModel(self) if model.created: model.update() else: model.new(unique) properties = fields(self, CqlProperty) for name, property in properties.items(): #Commit the changes in all collections within this model. if isinstance(property, CqlCollection): value = self[name] if isinstance(value, TypedCollection): value.commit() self.differ.commit() #Commit the differ for the model finally. self.__saved = True #Mark this model as saved.
def __iter__(self): '''Execute your queries and converts data to python data models''' # EXECUTE THE QUERY IF IT HASN'T BEEN EXECUTED try: if self.cursor is None: self.execute() except Exception as e: logging.exception("Something wen't wrong when executing the query: %s, error: %s" % self, str(e)) if Settings.debug(): print_exc() # FOR SOME ODD REASON CASSANDRA 1.0.0 ALWAYS RETURNS CqlResultType.ROWS, # SO TO FIGURE OUT COUNTS I MANUALLY SEARCH THE QUERY WITH A REGEX if re.search(self.pattern, self.query): logging.info("Count expression found;") yield self.cursor.fetchone()[0] else: logging.info("Deciphering rows as usual") cursor = self.cursor description = self.cursor.description if not description: raise StopIteration names = [tuple[0] for tuple in description] row = cursor.fetchone() while row: model = self.kind() descs = fields(model, Property) values = {} for name, value in zip(names, row): if not value: continue if name == "KEY": continue #Ignore the KEY attribute prop = descs.get(name, None) if prop: found = prop.deconvert(value) model[name] = found else: k, v = model.default k = k() if isinstance(k, type) else k v = v() if isinstance(v, type) else v name = k.deconvert(value) value = v.deconvert(value) model[name] = value yield model row = cursor.fetchone()
def update(self): '''Updates, deletes, and adds values to the underlying model in one batch.''' from homer.core.models import CqlProperty self.model.validate() batch = """ BEGIN BATCH USING TIMESTAMP {timestamp} {modifications} {deletions} APPLY BATCH; """ properties = fields(self.model, CqlProperty) key = properties.get("id") if "id" in properties else self.model.default[0] id = key.convert(self.model, self.model.id) added = list(self.model.differ.added()) modified = list(self.model.differ.modified()) # MARSHALL ADDITIONS & MODIFICATIONS modifications = [] for name in itertools.chain(added, modified): property = properties[name] if not property.saveable(): continue query = property.convert(self.model, self.model[name], update=True, insert=False) modifications.append(query) # MARSHALL DELETIONS deleted = list(self.model.differ.deleted()) if deleted: q = """DELETE ({names}) FROM {keyspace}.{kind} WHERE id={id} USING TIMESTAMP={timestamp};""" names = ",".join(deleted) + ";" deletion = q.format(names=names, id=id, timestamp=self.now(), keyspace=self.model.keyspace, kind=self.model.kind()) else: deletion = "" # BUILD AND EXECUTE THE BATCH QUERY modifications = ";".join(modifications) batch = batch.format(timestamp=self.now(), modifications=modifications, deletions=deletion) try: return CqlQuery(query=batch).execute() except Exception as e: logging.exception(e) raise e
def convert(self, instance=None, value=None, insert=False, update=False): '''Converts the basic type with the str operation, which we can do an eval() on.''' value = self.validate(value) if update and not insert: # updates would occur in a batch statement so we return a CQL assert isinstance(instance, Model), "A model instance must be provided for updates to work." properties = fields(instance, CqlProperty) key = properties.get("id") if "id" in properties else instance.default[0] id = key.convert(instance, instance.id) arguments = {"name": self.name, "value": value, "keyspace":instance.keyspace, "kind":instance.kind(), "id": id} q = "UPDATE {keyspace}.{kind} {ttl} SET {name}={value} WHERE id={id}".format(**arguments) if self.expire: query = q.format(ttl="USING TTL %s" % self.expire) else: query = q.format(ttl="") return query else: return str(value)
def convert(self, instance=None, value=None, insert=False, update=False): '''Converts data to queries''' assert isinstance(instance, Model), "We can only run conversion with Model objects" value = self.validate(value) property = self.converter all = fields(instance, CqlProperty) key = all.get("id") if "id" in all else instance.default[0] def escape(iterable): '''Useful for changing a list to it appropriate CQL3 representation''' return '[' + ', '.join(converted) + ']' if update and not insert: assert isinstance(value, TypedList), "We can only support typed lists at this point." ttl = "USING TTL %s" % self.expire if self.expire else "" if value.rewrite(): converted = [property.convert(value=val) for val in value] format = { "name" : self.name, "value" : escape(converted), "ttl" : ttl, "keyspace" : instance.keyspace, "kind" : instance.kind(), "id" : key.convert(value=instance.id) } q = "UPDATE {keyspace}.{kind} {ttl} SET {name} = {value} WHERE id={id}".format(**format) return q else: changes = value.modifications() # Generate CQL for appends and prepends format = { "keyspace" : instance.keyspace, "kind" : instance.kind(), "ttl" : ttl, "id" : key.convert(value=instance.id), "name" : self.name, } prepend = [property.convert(value=value) for val in changes["prepend"]] format["value"] = escape(prepend) prepend = "UPDATE {keyspace}.{kind} {ttl} SET {name} = {value} + {name} WHERE id={id}".format(**format) append = [property.convert(value=value) for val in changes["append"]] format["value"] = escape(append) # The format dictionary is the same except for the value item. append = "UPDATE {keyspace}.{kind} {ttl} SET {name} = {name} + {value} WHERE id={id}".format(**format) return ";".join([prepend, append]) else: converted = [property.convert(value=v) for v in value] return escape(converted)
def convert(self, instance, value, insert=False, update=False): '''Yields the datastore representation of its value''' value = self.validate(value) value = repr(value) value = quote(value) if update and not insert: assert isinstance(instance, Model), "We only accept Model instances for inserts or updates" properties = fields(instance, CqlProperty) key = properties.get("id") if "id" in properties else instance.default[0] id = key.convert(instance, instance.id) arguments = {"name":self.name, "value":value, "keyspace":instance.keyspace, "kind":instance.kind(), "id":id} q = "UPDATE {keyspace}.{kind} {ttl} SET {name}={value} WHERE id={id}" if self.expire: query = q.format(ttl="USING TTL %s" % self.expire, **arguments) else: query = q.format(ttl="", **arguments) return query else: return value
def distinct(self, *names): '''Adds the DISTINCT clause to the query''' from homer.core.models import CqlProperty if self.__count_used: raise CqlQueryException("You cannot use the DISTINCT and COUNT in the same query") properties = fields(self.model, CqlProperty) for name in names: property = properties.get(name, None) if not property: raise CqlQueryException("Property: {name} does not exist in {model}".format(name=name, model=self.model)) if not property.key and name != "id": raise CqlQueryException("You can only use the DISTINCT clause on clustering/compound keys") if not self.__distinct_used: part = "DISTINCT {names}".format(names=",".join(names)) self.__distinct = part self.__distinct_used = True else: part = ",".join(names) self.__distinct = part return self
def parse(self, keywords): '''Uses the descriptors in the Model to convert keywords''' if not self.convert: return keywords converted = {} props = fields(self.kind, Property) for name, value in keywords.items(): converter = props.get(name, None) if converter: value = converter.convert(value) if not isinstance(value, basestring): value = str(value) converted[name] = encode(value) else: T, V = self.kind.default value = V.convert(value) if not isinstance(value, basestring): value = str(value) converted[name] = encode(value) return converted
def new(self, unique=False): '''Saves a new instance of a self.model to the datastore''' from homer.core.models import CqlProperty if not Schema.contains(self.model): # Check if this Model has been created before. print("{model} does not seem to exist yet, trying to create model...".format(model=self.model)) self.create() print("Preparing to persist a new Model :%s" % self.model) self.model.validate() q = "INSERT INTO {keyspace}.{kind} ({columns}) VALUES ({values}){unique}{ttl}{timestamp};" unique = " IF NOT EXISTS" if unique else "" expire = self.model.expire ttl = " USING TTL {expire}".format(expire=expire) if expire else "" timestamp = " AND TIMESTAMP %s" % self.now() if ttl else " USING TIMESTAMP %s" % self.now() properties = fields(self.model, CqlProperty) # Fill up the cql3 template query columns = sorted(properties.keys()) values = [] for name in columns: property = properties[name] if not property.saveable(): continue # Insert only saveable properties. value = property.convert(self.model, self.model[name], insert=True) values.append(value) # BUILD THE QUERY. if unique: query = q.format(keyspace=self.model.keyspace, kind=self.model.kind(), unique=unique, ttl=ttl, columns=", ".join(columns), values=", ".join(values), timestamp="") else: query = q.format(keyspace=self.model.keyspace, kind=self.model.kind(), unique=unique, ttl=ttl, columns=", ".join(columns), values=", ".join(values), timestamp=timestamp) try: CqlQuery(query=query).execute() # Execute the Insert Query except Exception as e: logging.exception(e) raise e
def convert(self, instance, value, insert=False, update=False): '''Generates the CQL query for a particular set object''' assert isinstance(instance, Model), "We can only run conversion with Model objects" value = self.validate(value) property = self.converter all = fields(instance, CqlProperty) key = all.get("id") if "id" in all else instance.default[0] def escape(iterable): '''Useful for changing a set to it appropriate CQL3 representation''' return '{' + ', '.join(iterable) + '}' if update and not insert: if isinstance(value, TypedSet): added = [self.converter.convert(value=v) for v in value.added()] removed = [self.converter.convert(value=v) for v in value.deleted()] format = { "name" : self.name, "value" : escape(converted), "ttl" : ttl, "keyspace" : instance.keyspace, "kind" : instance.kind(), "id" : key.convert(value=instance.id) } ttl = "USING TTL %s" % self.expire if self.expire else "" add = "UPDATE {keyspace}.{kind} {ttl} SET {name} = {name} + {value} WHERE id={id}" format["value"] = escape(added) add = add.format(**format) remove = "UPDATE {keyspace}.{kind} {ttl} SET {name} = {name} - {value} WHERE id={id}" format["value"] = escape(removed) remove.format(**format) modifications = [add, remove] return ";".join(modifications) # Return both queries. else: raise AssertionError("We expect a TypedSet instance at this point; please contact the support channels") else: converted = [property.convert(value=val) for val in value] return escape(converted) # Just return the value directly.
def create(self): '''Creates the Keyspace and Model for self.keyspace and Model''' from homer.core.models import CqlProperty if Schema.contains(self.model): print("{model} has already been created before".format(model=self.model)) return # MAKE THE KEYSPACE q = """CREATE KEYSPACE IF NOT EXISTS {keyspace} WITH REPLICATION = {replication} AND DURABLE_WRITES = {durable};""" config = settings() replication = config.get("strategy", None) durable = config.get("durable", True) if not replication: raise ConfigurationError("No replication strategy specified.") replication = '{' + ', '.join([quote(k) + ':' + quote(v) for k,v in replication.items()]) + '}' print("GOT REPLICATION CONFIGURATION: " + replication) keyspace = self.model.keyspace durable = str(durable).lower() q = q.format(keyspace=keyspace, replication=replication, durable=durable) try: print("Creating keyspace: {0}".format(q)) CqlQuery(query=q).execute() except Exception as e: logging.exception(e) # raise e # FIND ALL THE CQLPROPERTY DESCRIPTORS FROM THIS MODEL. properties = fields(self.model, CqlProperty) # BUILD THE CQL3 QUERY FOR MAKING THE TABLE. id, type = False, None columns = [] keys = [] for name, property in properties.items(): if name == "id": id = True type = property.ctype else: fragment = "{0} {1}".format(name, property.ctype) columns.append(fragment) if property.key: keys.append(name) # CONSTRUCT THE QUERY STRING FOR CREATING TABLES. q = """ CREATE TABLE IF NOT EXISTS {keyspace}.{kind} ( {key} {columns} {keys} ); """ assert id, "Every Model must have an id property" if keys: keys = sorted(keys) keys.insert(0, "id") columns.append("{0} {1}".format("id", type)) keys = ", PRIMARY KEY(" + ",".join(keys) + ")" key = "" else: keys = "" key = "id {type} PRIMARY KEY,".format(type=type) values = {"columns" : ",".join(sorted(columns))} values["key"] = key values["keys"] = keys values["kind"] = self.model.kind() values["keyspace"] = self.model.keyspace values["type"] = type query = q.format(**values) try: print("Creating Table for Model with Query: " + query) CqlQuery(query=query).execute() except Exception as e: logging.exception(e) raise e # BUILD INDEXES, FOR EACH INDEXABLE PROPERTY. q = "CREATE INDEX IF NOT EXISTS {name} ON {keyspace}.{kind}({name});" for name, property in properties.items(): if name == "id": continue if property.indexed() and not property.key: query = q.format(name=name, keyspace=self.model.keyspace, kind=self.model.kind()) try: CqlQuery(query=query).execute() except Exception as e: logging.exception(e) raise e # If everything goes well, mark this model as created by adding it to the global schema cache. Schema.add(self.model)