def _store_meta(storage_args):
"""
Saves the information of the object in the istorage table.
Args:.
storage_args (object): contains all data needed to restore the object from the workers
"""
log.debug("StorageObj: storing media %s", storage_args)
try:
config.session.execute(StorageNumpy._prepared_store_meta, [
storage_args.storage_id, storage_args.class_name,
storage_args.name
])
except Exception as ex:
log.warn("Error creating the StorageNumpy metadata with args: %s" %
str(storage_args))
raise ex
def __contains__(self, key):
"""
Method that checks if a given key exists in a StorageDict.
Args:
key: the position that we want to check if exists.
Returns:
boolean (true - exists, false - doesn't exist).
"""
if not self._is_persistent:
return dict.__contains__(self, key)
else:
try:
# TODO we should save this value in a cache
self._hcache.get_row(self._make_key(key))
return True
except Exception as ex:
log.warn("persistentDict.__contains__ ex %s", ex)
raise ex
def __getattr__(self, attribute):
"""
Given an attribute, this function returns the value, obtaining it from either:
a) memory
b) the Database
Args:
attribute: name of the value that we want to obtain
Returns:
value: obtained value
"""
if attribute[
0] != '_' and self._is_persistent and attribute in self._persistent_attrs:
try:
query = "SELECT %s FROM %s.%s WHERE storage_id = %s;" \
% (attribute, self._ksp,
self._table,
self._storage_id)
log.debug("GETATTR: %s", query)
result = config.session.execute(query)
for row in result:
for row_key, row_var in vars(row).iteritems():
if row_var is not None:
if isinstance(row_var, list) and isinstance(
row_var[0], unicode):
new_toreturn = []
for entry in row_var:
new_toreturn.append(str(entry))
return new_toreturn
else:
return row_var
else:
raise AttributeError
except Exception as ex:
log.warn("GETATTR ex %s", ex)
raise AttributeError('value not found')
else:
return object.__getattribute__(self, attribute)
def make_persistent(self, name):
"""
Method to transform a StorageDict into a persistent object.
This will make it use a persistent DB as the main location
of its data.
Args:
name:
"""
if self._is_persistent:
raise AlreadyPersistentError(
"This StorageDict is already persistent [Before:{}.{}][After:{}]",
self._ksp, self._table, name)
self._is_persistent = True
(self._ksp, self._table) = self._extract_ks_tab(name)
if self._storage_id is None:
self._storage_id = uuid.uuid3(uuid.NAMESPACE_DNS,
self._ksp + '.' + self._table)
self._build_args = self._build_args._replace(
storage_id=self._storage_id, name=self._ksp + "." + self._table)
self._store_meta(self._build_args)
if config.id_create_schema == -1:
query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (
self._ksp, config.replication)
try:
log.debug('MAKE PERSISTENCE: %s', query_keyspace)
config.session.execute(query_keyspace)
except Exception as ex:
log.warn("Error creating the StorageDict keyspace %s, %s",
(query_keyspace), ex)
raise ex
for key, value in dict.iteritems(self):
if issubclass(value.__class__, IStorage):
# new name as ksp+table+obj_class_name
val_name = self._ksp + '.' + self._table + type(
value).__name__.lower()
value.make_persistent(val_name)
columns = self._primary_keys + self._columns
for ind, entry in enumerate(columns):
n = StorageDict._other_case.match(entry[1])
if n is not None:
iter_type, intra_type = n.groups()
else:
iter_type = entry[1]
if iter_type not in IStorage._basic_types:
columns[ind] = entry[0], 'uuid'
pks = map(lambda a: a[0], self._primary_keys)
query_table = "CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));" \
% (self._ksp,
self._table,
",".join("%s %s" % tup for tup in columns),
str.join(',', pks))
try:
log.debug('MAKE PERSISTENCE: %s', query_table)
config.session.execute(query_table)
except Exception as ex:
log.warn("Error creating the StorageDict table: %s %s",
query_table, ex)
raise ex
key_names = map(lambda a: a[0].encode('UTF8'), self._primary_keys)
column_names = self._columns
self._hcache_params = (self._ksp, self._table, self._storage_id,
self._tokens, key_names,
map(lambda x: {
"name": x[0],
"type": x[1]
}, column_names), {
'cache_size': config.max_cache_size,
'writer_par': config.write_callbacks_number,
'write_buffer': config.write_buffer_size
})
log.debug("HCACHE params %s", self._hcache_params)
self._hcache = Hcache(*self._hcache_params)
# Storing all in-memory values to cassandra
for key, value in dict.iteritems(self):
self._hcache.put_row(self._make_key(key), self._make_value(value))
if hasattr(self, '_indexed_args') and self._indexed_args is not None:
index_query = 'CREATE CUSTOM INDEX IF NOT EXISTS ' + self._table + '_idx ON '
index_query += self._ksp + '.' + self._table + ' (' + str.join(
',', self._indexed_args) + ') '
index_query += "using 'es.bsc.qbeast.index.QbeastIndex';"
try:
config.session.execute(index_query)
except Exception as ex:
log.error("Error creating the Qbeast custom index: %s %s",
index_query, ex)
raise ex