def getByID(objid):
"""
We rebuild the object from its id. The id can either be:
block: UUID (eg. f291f008-a520-11e6-b42e-5b582e04fd70)
storageobj: UUID_(version) (eg. f291f008-a520-11e6-b42e-5b582e04fd70_1)
Args:
objid (str): object identifier
Returns:
(Block| Storageobj)
"""
"""
TODO
Args:
objid (str): object identifier
Returns:
(Block| Storageobj)
"""
from hecuba import log
from hecuba.IStorage import IStorage
try:
from hecuba import config
query = "SELECT * FROM hecuba.istorage WHERE storage_id = %s"
results = config.session.execute(query, [uuid.UUID(objid)])[0]
except Exception as e:
log.error("Query %s failed", query)
raise e
log.debug("IStorage API:getByID(%s) of class %s", objid,
results.class_name)
return IStorage.build_remotely(results._asdict())
def make_persistent(self, name):
"""
Once a StorageObj has been created, it can be made persistent. This function retrieves the information about
the Object class schema, and creates a Cassandra table with those parameters, where information will be
saved from now on, until execution finishes or StorageObj is no longer persistent.
It also inserts into the new table all information that was in memory assigned to the StorageObj prior to
this call.
Args:
name (string): name with which the table in the DB will be created
"""
if self._is_persistent:
raise AlreadyPersistentError(
"This StorageObj is already persistent [Before:{}.{}][After:{}]",
self._ksp, self._table, name)
self._is_persistent = True
(self._ksp, self._table) = self._extract_ks_tab(name)
if self._storage_id is None:
self._storage_id = uuid.uuid3(uuid.NAMESPACE_DNS,
self._ksp + '.' + self._table)
self._build_args = self.args(self._ksp + '.' + self._table,
self._tokens, self._storage_id,
self._istorage_props, self._class_name)
log.info("PERSISTING DATA INTO %s %s", self._ksp, self._table)
query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (
self._ksp, config.replication)
config.session.execute(query_keyspace)
query_simple = 'CREATE TABLE IF NOT EXISTS ' + self._ksp + '.' + self._table + \
'( storage_id uuid PRIMARY KEY, '
for key, entry in self._persistent_props.items():
query_simple += str(key) + ' '
if entry['type'] != 'dict' and entry[
'type'] in IStorage._valid_types:
if entry['type'] == 'list' or entry['type'] == 'tuple':
query_simple += entry['type'] + '<' + entry[
'columns'] + '>, '
else:
query_simple += entry['type'] + ', '
else:
query_simple += 'uuid, '
try:
config.session.execute(query_simple[:-2] + ' )')
except Exception as ir:
log.error("Unable to execute %s", query_simple)
raise ir
for obj_name, obj_info in self._persistent_props.items():
if hasattr(self, obj_name):
pd = getattr(self, obj_name)
if obj_info['type'] not in IStorage._basic_types:
sd_name = self._ksp + "." + self._table + "_" + obj_name
pd.make_persistent(sd_name)
setattr(
self, obj_name, pd
) # super(StorageObj, self).__setattr__(obj_name, pd) why?
self._store_meta(self._build_args)
def _store_meta(storage_args):
log.debug("QbeastIterator: storing metas %s", '')
try:
config.session.execute(QbeastIterator._prepared_store_meta, [
storage_args.primary_keys, storage_args.columns,
storage_args.name, storage_args.qbeast_meta,
storage_args.qbeast_id, storage_args.entry_point,
storage_args.storage_id, storage_args.tokens,
storage_args.class_name
])
except Exception as ex:
log.error("Error creating the StorageDictIx metadata: %s %s",
storage_args, ex)
raise ex
def _store_meta(storage_args):
"""
Method to update the info about the StorageDict in the DB metadata table
Args:
storage_args: structure with all data needed to update the metadata
"""
log.debug("StorageDict: storing metas %s", storage_args)
try:
config.session.execute(StorageDict._prepared_store_meta, [
storage_args.storage_id, storage_args.class_name,
storage_args.name, storage_args.tokens,
storage_args.primary_keys, storage_args.columns,
storage_args.indexed_on
])
except Exception as ex:
log.error("Error creating the StorageDict metadata: %s %s",
storage_args, ex)
raise ex
def getByID(objid):
"""
We rebuild the object from its id. The id can either be:
block: UUID (eg. f291f008-a520-11e6-b42e-5b582e04fd70)
storageobj: UUID_(version) (eg. f291f008-a520-11e6-b42e-5b582e04fd70_1)
Args:
objid (str): object identifier
Returns:
(Block| Storageobj)
"""
"""
TODO
Args:
objid (str): object identifier
Returns:
(Block| Storageobj)
"""
from hecuba import log
try:
from hecuba import config
query = "SELECT * FROM hecuba.istorage WHERE storage_id = %s"
results = config.session.execute(query, [uuid.UUID(objid)])[0]
except Exception as e:
log.error("Query %s failed", query)
raise e
class_name = results.class_name
log.debug("IStorage API:getByID(%s) of class %s", objid, class_name)
last = 0
for key, i in enumerate(class_name):
if i == '.' and key > last:
last = key
module = class_name[:last]
cname = class_name[last + 1:]
mod = __import__(module, globals(), locals(), [cname], 0)
b = getattr(mod, cname).build_remotely(results)
return b
def make_persistent(self, name):
"""
Method to transform a StorageDict into a persistent object.
This will make it use a persistent DB as the main location
of its data.
Args:
name:
"""
if self._is_persistent:
raise AlreadyPersistentError(
"This StorageDict is already persistent [Before:{}.{}][After:{}]",
self._ksp, self._table, name)
self._is_persistent = True
(self._ksp, self._table) = self._extract_ks_tab(name)
if self._storage_id is None:
self._storage_id = uuid.uuid3(uuid.NAMESPACE_DNS,
self._ksp + '.' + self._table)
self._build_args = self._build_args._replace(
storage_id=self._storage_id, name=self._ksp + "." + self._table)
self._store_meta(self._build_args)
if config.id_create_schema == -1:
query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (
self._ksp, config.replication)
try:
log.debug('MAKE PERSISTENCE: %s', query_keyspace)
config.session.execute(query_keyspace)
except Exception as ex:
log.warn("Error creating the StorageDict keyspace %s, %s",
(query_keyspace), ex)
raise ex
for key, value in dict.iteritems(self):
if issubclass(value.__class__, IStorage):
# new name as ksp+table+obj_class_name
val_name = self._ksp + '.' + self._table + type(
value).__name__.lower()
value.make_persistent(val_name)
columns = self._primary_keys + self._columns
for ind, entry in enumerate(columns):
n = StorageDict._other_case.match(entry[1])
if n is not None:
iter_type, intra_type = n.groups()
else:
iter_type = entry[1]
if iter_type not in IStorage._basic_types:
columns[ind] = entry[0], 'uuid'
pks = map(lambda a: a[0], self._primary_keys)
query_table = "CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));" \
% (self._ksp,
self._table,
",".join("%s %s" % tup for tup in columns),
str.join(',', pks))
try:
log.debug('MAKE PERSISTENCE: %s', query_table)
config.session.execute(query_table)
except Exception as ex:
log.warn("Error creating the StorageDict table: %s %s",
query_table, ex)
raise ex
key_names = map(lambda a: a[0].encode('UTF8'), self._primary_keys)
column_names = self._columns
self._hcache_params = (self._ksp, self._table, self._storage_id,
self._tokens, key_names,
map(lambda x: {
"name": x[0],
"type": x[1]
}, column_names), {
'cache_size': config.max_cache_size,
'writer_par': config.write_callbacks_number,
'write_buffer': config.write_buffer_size
})
log.debug("HCACHE params %s", self._hcache_params)
self._hcache = Hcache(*self._hcache_params)
# Storing all in-memory values to cassandra
for key, value in dict.iteritems(self):
self._hcache.put_row(self._make_key(key), self._make_value(value))
if hasattr(self, '_indexed_args') and self._indexed_args is not None:
index_query = 'CREATE CUSTOM INDEX IF NOT EXISTS ' + self._table + '_idx ON '
index_query += self._ksp + '.' + self._table + ' (' + str.join(
',', self._indexed_args) + ') '
index_query += "using 'es.bsc.qbeast.index.QbeastIndex';"
try:
config.session.execute(index_query)
except Exception as ex:
log.error("Error creating the Qbeast custom index: %s %s",
index_query, ex)
raise ex