def _store_meta(storage_args): """ Saves the information of the object in the istorage table. Args:. storage_args (object): contains all data needed to restore the object from the workers """ log.debug("StorageObj: storing media %s", storage_args) try: config.session.execute(StorageNumpy._prepared_store_meta, [ storage_args.storage_id, storage_args.class_name, storage_args.name ]) except Exception as ex: log.warn("Error creating the StorageNumpy metadata with args: %s" % str(storage_args)) raise ex
def __contains__(self, key): """ Method that checks if a given key exists in a StorageDict. Args: key: the position that we want to check if exists. Returns: boolean (true - exists, false - doesn't exist). """ if not self._is_persistent: return dict.__contains__(self, key) else: try: # TODO we should save this value in a cache self._hcache.get_row(self._make_key(key)) return True except Exception as ex: log.warn("persistentDict.__contains__ ex %s", ex) raise ex
def __getattr__(self, attribute): """ Given an attribute, this function returns the value, obtaining it from either: a) memory b) the Database Args: attribute: name of the value that we want to obtain Returns: value: obtained value """ if attribute[ 0] != '_' and self._is_persistent and attribute in self._persistent_attrs: try: query = "SELECT %s FROM %s.%s WHERE storage_id = %s;" \ % (attribute, self._ksp, self._table, self._storage_id) log.debug("GETATTR: %s", query) result = config.session.execute(query) for row in result: for row_key, row_var in vars(row).iteritems(): if row_var is not None: if isinstance(row_var, list) and isinstance( row_var[0], unicode): new_toreturn = [] for entry in row_var: new_toreturn.append(str(entry)) return new_toreturn else: return row_var else: raise AttributeError except Exception as ex: log.warn("GETATTR ex %s", ex) raise AttributeError('value not found') else: return object.__getattribute__(self, attribute)
def make_persistent(self, name): """ Method to transform a StorageDict into a persistent object. This will make it use a persistent DB as the main location of its data. Args: name: """ if self._is_persistent: raise AlreadyPersistentError( "This StorageDict is already persistent [Before:{}.{}][After:{}]", self._ksp, self._table, name) self._is_persistent = True (self._ksp, self._table) = self._extract_ks_tab(name) if self._storage_id is None: self._storage_id = uuid.uuid3(uuid.NAMESPACE_DNS, self._ksp + '.' + self._table) self._build_args = self._build_args._replace( storage_id=self._storage_id, name=self._ksp + "." + self._table) self._store_meta(self._build_args) if config.id_create_schema == -1: query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % ( self._ksp, config.replication) try: log.debug('MAKE PERSISTENCE: %s', query_keyspace) config.session.execute(query_keyspace) except Exception as ex: log.warn("Error creating the StorageDict keyspace %s, %s", (query_keyspace), ex) raise ex for key, value in dict.iteritems(self): if issubclass(value.__class__, IStorage): # new name as ksp+table+obj_class_name val_name = self._ksp + '.' + self._table + type( value).__name__.lower() value.make_persistent(val_name) columns = self._primary_keys + self._columns for ind, entry in enumerate(columns): n = StorageDict._other_case.match(entry[1]) if n is not None: iter_type, intra_type = n.groups() else: iter_type = entry[1] if iter_type not in IStorage._basic_types: columns[ind] = entry[0], 'uuid' pks = map(lambda a: a[0], self._primary_keys) query_table = "CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));" \ % (self._ksp, self._table, ",".join("%s %s" % tup for tup in columns), str.join(',', pks)) try: log.debug('MAKE PERSISTENCE: %s', query_table) config.session.execute(query_table) except Exception as ex: log.warn("Error creating the StorageDict table: %s %s", query_table, ex) raise ex key_names = map(lambda a: a[0].encode('UTF8'), self._primary_keys) column_names = self._columns self._hcache_params = (self._ksp, self._table, self._storage_id, self._tokens, key_names, map(lambda x: { "name": x[0], "type": x[1] }, column_names), { 'cache_size': config.max_cache_size, 'writer_par': config.write_callbacks_number, 'write_buffer': config.write_buffer_size }) log.debug("HCACHE params %s", self._hcache_params) self._hcache = Hcache(*self._hcache_params) # Storing all in-memory values to cassandra for key, value in dict.iteritems(self): self._hcache.put_row(self._make_key(key), self._make_value(value)) if hasattr(self, '_indexed_args') and self._indexed_args is not None: index_query = 'CREATE CUSTOM INDEX IF NOT EXISTS ' + self._table + '_idx ON ' index_query += self._ksp + '.' + self._table + ' (' + str.join( ',', self._indexed_args) + ') ' index_query += "using 'es.bsc.qbeast.index.QbeastIndex';" try: config.session.execute(index_query) except Exception as ex: log.error("Error creating the Qbeast custom index: %s %s", index_query, ex) raise ex