class StorageDict(IStorage, dict):
# """
# Object used to access data from workers.
# """
args_names = [
"name", "primary_keys", "columns", "tokens", "storage_id",
"indexed_on", "class_name", "built_remotely"
]
args = namedtuple('StorageDictArgs', args_names)
_prepared_store_meta = config.session.prepare(
'INSERT INTO hecuba.istorage'
'(storage_id, class_name, name, tokens, '
'primary_keys, columns, indexed_on)'
'VALUES (?,?,?,?,?,?,?)')
@staticmethod
def _store_meta(storage_args):
"""
Method to update the info about the StorageDict in the DB metadata table
Args:
storage_args: structure with all data needed to update the metadata
"""
log.debug("StorageDict: storing metas %s", storage_args)
try:
config.session.execute(StorageDict._prepared_store_meta, [
storage_args.storage_id, storage_args.class_name,
storage_args.name, storage_args.tokens,
storage_args.primary_keys, storage_args.columns,
storage_args.indexed_on
])
except Exception as ex:
log.error("Error creating the StorageDict metadata: %s %s",
storage_args, ex)
raise ex
def __init__(self,
name=None,
primary_keys=None,
columns=None,
indexed_on=None,
storage_id=None,
**kwargs):
"""
Creates a new StorageDict.
Args:
name (string): the name of the collection/table (keyspace is optional)
primary_keys (list(tuple)): a list of (key,type) primary keys (primary + clustering).
columns (list(tuple)): a list of (key,type) columns
tokens (list): list of tokens
storage_id (string): the storage id identifier
indexed_on (list): values that will be used as index
kwargs: other parameters
"""
super().__init__((), name=name, storage_id=storage_id, **kwargs)
log.debug("CREATE StorageDict(%s,%s)", primary_keys, columns)
'''
yolandab
kwargs of the init should contain metas: all the row in the istorage if exists
after super().__init__
if kwargs is empty --> this is a new object
generate build args parsing the _doc_ string or using the parameters
we need to generate the column info of sets with the format to persist it (name--> _set_)
if name or storage id --> call to store_metas
else --> this is an already existing objects
metas and tokens should form the attributes of self
we need to convert the column info of sets to the format in memory ( _set_name --> name)
TODO: implement a cleaner version of embedded sets
'''
build_column = None
build_keys = None
if self.__doc__ is not None:
self._persistent_props = self._parse_comments(self.__doc__)
self._primary_keys = self._persistent_props['primary_keys']
self._columns = self._persistent_props['columns']
self._indexed_on = self._persistent_props.get(
'indexed_on', indexed_on)
# Field '_istorage_metas' will be set if it exists in HECUBA.istorage
initialized = (getattr(self, '_istorage_metas', None) is not None)
if not initialized and self.__doc__ is None:
#info is not in the doc string, should be passed in the parameters
if primary_keys == None or columns == None:
raise RuntimeError(
"StorageDict: missed specification. Type of Primary Key or Column undefined"
)
self._primary_keys = primary_keys
self._columns = columns
self._indexed_on = indexed_on
if initialized: #object already in istorage
# if (primary_keys is not None or columns is not None):
# raise RuntimeError("StorageDict: Trying to define a new schema, but it is already persistent")
# --> this check would be necessary if passing columns/key spec
# as parameter was part of the user interface. As it is intended
# just for internal use we skip this check. If the spec does not
# match the actual schema access to the object will fail.
if getattr(self, "_persistent_props",
None) is not None: # __doc__ and disk: do they match?
self._check_schema_and_raise("__init__")
else: # _persistent_props == None (only in disk)
# Parse _istorage_metas to fulfill the _primary_keys, _columns
self._primary_keys = self._istorage_metas.primary_keys
self._columns = self._istorage_metas.columns
build_column = self._columns # Keep a copy from the disk to avoid recalculate it later
build_keys = self._primary_keys # Keep a copy from the disk to avoid recalculate it later
self._indexed_on = self._istorage_metas.indexed_on
#we manipulate the info about sets retrieved from istorage
# (_set_s1_0,int), (_set_s1_1,int) --> {name: s1, type: set , column:((s1_0, int), (s1_1, int))}
has_embedded_set = False
set_pks = []
normal_columns = []
for column_name, column_type in self._columns:
if column_name.find("_set_") == 0:
attr_name = column_name[
5:] # Remove '_set_' The attribute name also contains the "column_name" needed later...
set_pks.append((attr_name, column_type))
has_embedded_set = True
else:
normal_columns.append((column_name, column_type))
if has_embedded_set: # Embedded set has a different layout {name,type:set, columns:[(name,type),(name,type)]}
column_name = attr_name.split(
"_", 1
)[0] # Get the 1st name (attr_1, attr_2... -> attr or attr -> attr)
self._columns = [{
"name": column_name,
"type": "set",
"columns": set_pks
}]
else:
self._columns = [{
"type": col[1],
"name": col[0]
} for col in normal_columns]
# COMMON CODE: new and instantiation
# Special case:Do we have an embedded set?
self._has_embedded_set = False
if isinstance(self._columns[0], dict):
if self._columns[0]['type'] == 'set':
self._has_embedded_set = True
self._primary_keys = [{
"type": key[1],
"name": key[0]
} if isinstance(key, tuple) else key for key in self._primary_keys]
self._columns = [{
"type": col[1],
"name": col[0]
} if isinstance(col, tuple) else col for col in self._columns]
# POST: _primary_keys and _columns are list of DICTS> [ {name:..., type:...}, {name:..., type:set, columns:[(name,type),...]},...]
log.debug("CREATED StorageDict(%s,%s)", self._primary_keys,
self._columns)
key_names = [key["name"] for key in self._primary_keys]
column_names = [col["name"] for col in self._columns]
if len(key_names) > 1:
self._key_builder = namedtuple('row', key_names)
else: # 1
self._key_builder = None
if self._has_embedded_set:
set_names = [colname for (colname, dt) in self._get_set_types()]
self._column_builder = namedtuple('row', set_names)
elif len(column_names) > 1:
self._column_builder = namedtuple('row', column_names)
else:
self._column_builder = None
self._k_size = len(key_names)
class_name = '%s.%s' % (self.__class__.__module__,
self.__class__.__name__)
if build_keys == None:
build_keys = [(key["name"], key["type"])
for key in self._primary_keys]
# Define 'build_column': it will contain the column info stored in istorage. For the sets we manipulate the parsed data
if build_column == None:
build_column = []
for col in self._columns:
if col["type"] == "set":
types = col["columns"]
for t in types:
build_column.append(("_set_" + t[0], t[1]))
else:
build_column.append((col["name"], col["type"]))
self._build_args = self.args(self._get_name(), build_keys,
build_column, self._tokens,
self.storage_id, self._indexed_on,
class_name, self._built_remotely)
if name and storage_id and (storage_id != storage_id_from_name(name)
): # instantiating an splitted object
self._persist_metadata()
elif name or storage_id: # instantiating a persistent object
if initialized: # already existint
self._setup_hcache()
else: # new object
self._persist_metadata()
@classmethod
def _parse_comments(self, comments):
parser = Parser("TypeSpec")
return parser._parse_comments(comments)
def __contains__(self, key):
"""
Method that checks if a given key exists in a StorageDict.
Args:
key: the position that we want to check if exists.
Returns:
boolean (true - exists, false - doesn't exist).
"""
if not self.storage_id:
return dict.__contains__(self, key)
else:
try:
# TODO we should save this value in a cache
self._hcache.get_row(self._make_key(key))
return True
except Exception as ex:
log.warn("persistentDict.__contains__ ex %s", ex)
return False
def _create_tables(self):
# Prepare data
persistent_keys = [
(key["name"], "tuple<" + ",".join(key["columns"]) +
">") if key["type"] == "tuple" else (key["name"], key["type"])
for key in self._primary_keys
] + self._get_set_types()
persistent_values = []
if not self._has_embedded_set:
for col in self._columns:
if col["type"] == "tuple":
persistent_values.append({
"name":
col["name"],
"type":
"tuple<" + ",".join(col["columns"]) + ">"
})
elif col["type"] not in basic_types:
persistent_values.append({
"name": col["name"],
"type": "uuid"
})
else:
persistent_values.append({
"name": col["name"],
"type": col["type"]
})
key_names = [
col[0] if isinstance(col, tuple) else col["name"]
for col in persistent_keys
]
query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (
self._ksp, config.replication)
try:
log.debug('MAKE PERSISTENCE: %s', query_keyspace)
config.executelocked(query_keyspace)
except Exception as ex:
log.warn("Error creating the StorageDict keyspace %s, %s",
(query_keyspace), ex)
raise ex
persistent_columns = [(col["name"], col["type"])
for col in persistent_values]
query_table = "CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));" \
% (self._ksp,
self._table,
",".join("%s %s" % tup for tup in persistent_keys + persistent_columns),
str.join(',', key_names))
try:
log.debug('MAKE PERSISTENCE: %s', query_table)
config.executelocked(query_table)
except Exception as ex:
log.warn("Error creating the StorageDict table: %s %s",
query_table, ex)
raise ex
if hasattr(self, '_indexed_on') and self._indexed_on is not None:
index_query = 'CREATE CUSTOM INDEX IF NOT EXISTS ' + self._table + '_idx ON '
index_query += self._ksp + '.' + self._table + ' (' + str.join(
',', self._indexed_on) + ') '
index_query += "using 'es.bsc.qbeast.index.QbeastIndex';"
try:
config.executelocked(index_query)
except Exception as ex:
log.error("Error creating the Qbeast custom index: %s %s",
index_query, ex)
raise ex
trigger_query = "CREATE TRIGGER IF NOT EXISTS %s%s_qtr ON %s.%s USING 'es.bsc.qbeast.index.QbeastTrigger';" % \
(self._ksp, self._table, self._ksp, self._table)
try:
config.executelocked(trigger_query)
except Exception as ex:
log.error("Error creating the Qbeast trigger: %s %s",
trigger_query, ex)
raise ex
def _persist_data_from_memory(self):
for k, v in super().items():
self[k] = v
if config.max_cache_size != 0: #if C++ cache is enabled, clear Python memory, otherwise keep it
super().clear()
def sync(self):
super().sync()
self._hcache.flush()
def _setup_hcache(self):
key_names = [key["name"] for key in self._primary_keys]
key_names = key_names + [name for name, dt in self._get_set_types()]
persistent_values = []
if not self._has_embedded_set:
persistent_values = [{
"name": col["name"]
} for col in self._columns]
if self._tokens is None:
raise RuntimeError("Tokens for object {} are null".format(
self._get_name()))
self._hcache_params = (self._ksp, self._table, self.storage_id,
self._tokens, key_names, persistent_values, {
'cache_size': config.max_cache_size,
'writer_par': config.write_callbacks_number,
'writer_buffer': config.write_buffer_size,
'timestamped_writes':
config.timestamped_writes
})
log.debug("HCACHE params %s", self._hcache_params)
self._hcache = Hcache(*self._hcache_params)
def _make_key(self, key):
"""
Method used to pass the key data to the StorageDict cache in a proper way.
Args:
key: the data that needs to get the correct format
"""
if isinstance(key, str) or not isinstance(key, Iterable):
if len(self._primary_keys) == 1:
return [key]
else:
raise Exception('missing a primary key')
if isinstance(key, Iterable) and len(key) == len(self._primary_keys):
return list(key)
elif self._has_embedded_set and isinstance(
key, Iterable) and len(key) == (len(self._primary_keys) +
len(self._get_set_types())):
return list(key)
else:
raise Exception('wrong primary key')
@staticmethod
def _make_value(value):
"""
Method used to pass the value data to the StorageDict cache in a proper way.
Args:
value: the data that needs to get the correct format
"""
if issubclass(value.__class__, IStorage):
return [value.storage_id]
elif isinstance(value,
str) or not isinstance(value, Iterable) or isinstance(
value, np.ndarray):
return [value]
elif isinstance(value, tuple):
return [value]
elif isinstance(value, Iterable):
val = []
for v in value:
if isinstance(v, IStorage):
val.append(v.storage_id)
else:
val.append(v)
return val
else:
return list(value)
def _count_elements(self, query):
try:
result = config.session.execute(query)
return result[0][0]
except OperationTimedOut as ex:
import warnings
warnings.warn(
"len() operation on {} from class {} failed by timeout."
"Use len() on split() results if you must".format(
self._get_name(), self.__class__.__name__))
raise ex
except Exception as ir:
log.error("Unable to execute %s", query)
raise ir
def __iter__(self):
"""
Method that overloads the python dict basic iteration, which returns
an iterator over the dictionary keys.
"""
return self.keys()
def _persist_metadata(self):
"""
Private Method to create tables, setup the cache and store the metadata
of a StorageDict.
Used for NEW storage dicts, that do no need to persist any data.
"""
if not self._built_remotely:
self._create_tables()
self._setup_hcache()
StorageDict._store_meta(self._build_args)
def _persist_data(self, name):
"""
Private Method to store a StorageDict into cassandra
This will make it use a persistent DB as the main location
of its data.
Args:
name:
"""
# Update local StorageDict metadata
self._build_args = self._build_args._replace(
storage_id=self.storage_id,
name=self._ksp + "." + self._table,
tokens=self._tokens)
self._persist_metadata()
self._persist_data_from_memory()
def make_persistent(self, name):
"""
Method to transform a StorageDict into a persistent object.
This will make it use a persistent DB as the main location
of its data.
Args:
name:
"""
super().make_persistent(name)
if getattr(self, "_istorage_metas", None) is not None:
self._check_schema_and_raise("make_persistent")
self._persist_data(name)
def stop_persistent(self):
"""
Method to turn a StorageDict into non-persistent.
"""
super().stop_persistent()
log.debug('STOP PERSISTENCE: %s', self._table)
self._hcache = None
self.storage_id = None
def delete_persistent(self):
"""
Method to empty all data assigned to a StorageDict.
"""
self.sync()
super().delete_persistent()
log.debug('DELETE PERSISTENT: %s', self._table)
query = "DROP TABLE %s.%s;" % (self._ksp, self._table)
config.session.execute(query)
query = "DELETE FROM hecuba.istorage where storage_id={}".format(
self.storage_id)
config.session.execute(query)
self.storage_id = None
def __delitem__(self, key):
"""
Method to delete a specific entry in the dict in the key position.
Args:
key: position of the entry that we want to delete
"""
if not self.storage_id:
dict.__delitem__(self, key)
elif self._has_embedded_set:
self._hcache.delete_row(key)
elif isinstance(key, Iterable) and not isinstance(key, str):
self._hcache.delete_row(list(key))
else:
self._hcache.delete_row([key])
def __create_embeddedset(self, key, val=None):
if not isinstance(key, Iterable) or isinstance(key, str):
return EmbeddedSet(self, [key], val)
else:
return EmbeddedSet(self, list(key), val)
def _check_schema_and_raise(self, txt):
"""
Raises an exception if the schema stored in the database does not match
with the description of the object in memory. This may happen if the
user specifies an already used name for its data.
PRE:
self._istorage_metas contains a list of tuples (name, type)
self._primary_keys contains a list of tuples (name, type) or list of dicts {'name':value, 'type':value}
self._columns may contain:
a list of tuples (name, type) or
a list of dicts {'name':value, 'type':value} or
a list of dicts with a set {'name':value, 'type':'set','columns':[(name1,type1),....]}
"""
# TODO: Change parser to have a consistent behaviour
# try to send a useful message if it is a problem with a mismatched schema
if getattr(self, "_istorage_metas", None) is None:
self._istorage_metas = get_istorage_attrs(self.storage_id)
if len(self._primary_keys) != len(self._istorage_metas.primary_keys):
raise RuntimeError(
"StorageDict: {}: key Metadata does not match specification. Trying {} but stored specification {}"
.format(txt, self._primary_keys,
self._istorage_metas.primary_keys))
pk = [{
"type": key[1],
"name": key[0]
} if isinstance(key, tuple) else key for key in self._primary_keys]
for pos, key in enumerate(pk):
if self._istorage_metas.primary_keys[pos][0] != key[
'name'] or self._istorage_metas.primary_keys[pos][
1] != key['type']:
raise RuntimeError(
"StorageDict: {}: key Metadata does not match specification. Trying {} but stored specification {}"
.format(txt, self._primary_keys,
self._istorage_metas.primary_keys))
columns = self._columns
# Treat the embedded set case...
if type(self._columns[0]) == dict:
if self._columns[0]['type'] == 'set':
columns = self._columns[0]['columns']
if len(columns) != len(self._istorage_metas.columns):
raise RuntimeError(
"StorageDict: {}: column Metadata does not match specification. Trying {} but stored specification {}"
.format(txt, self._columns, self._istorage_metas.columns))
columns = [{
"type": col[1],
"name": col[0]
} if isinstance(col, tuple) else col for col in columns]
for pos, val in enumerate(columns):
#istorage_metas.columns[pos] -->[(_set_s1_0,int),(_set_s1_1,int)]
mykey = self._istorage_metas.columns[pos][0]
mytype = self._istorage_metas.columns[pos][1]
if mykey.find("_set_") == 0:
mykey = mykey[
5:] # Skip the '_set_' '_set_s1_0' ==> 's1_0' TODO Change the set identification method
if (mykey != val['name']) or (mytype != val['type']):
raise RuntimeError(
"StorageDict: {}: column Metadata does not match specification. Trying {} but stored specification {}"
.format(txt, self._columns, self._istorage_metas.columns))
def __getitem__(self, key):
"""
If the object is persistent, each request goes to the hfetch.
Args:
key: the dictionary key
Returns
item: value found in position key
"""
log.debug('GET ITEM %s', key)
if not self.storage_id:
return dict.__getitem__(self, key)
elif self._has_embedded_set:
return self.__create_embeddedset(key=key)
else:
# Returns always a list with a single entry for the key
if config.max_cache_size == 0: # if C++ cache is disabled, use Python memory
try:
result = dict.__getitem__(self, key)
return result
except:
pass
persistent_result = self._hcache.get_row(self._make_key(key))
log.debug("GET ITEM %s[%s]", persistent_result,
persistent_result.__class__)
# we need to transform UUIDs belonging to IStorage objects and rebuild them
# TODO hcache should return objects of the class uuid, not str
final_results = []
for index, col in enumerate(self._columns):
col_type = col["type"]
element = persistent_result[index]
if col_type not in basic_types:
# element is not a built-in type
info = {
"storage_id": element,
"tokens": self._build_args.tokens,
"class_name": col_type
}
element = build_remotely(info)
final_results.append(element)
if self._column_builder is not None:
return self._column_builder(*final_results)
else:
return final_results[0]
def __make_val_persistent(self, val, col=0):
if isinstance(val, list):
for index, element in enumerate(val):
val[index] = self.__make_val_persistent(element, index)
elif isinstance(val, IStorage) and not val._is_persistent:
valstorage_id = uuid.uuid4()
attribute = self._columns[col]["name"]
name = self._ksp + "." + (
"D" + str(valstorage_id).replace('-', '_') + self._table +
attribute
)[:
40] # 48 is the max length of table names, this may have collisions but this would only affect to object instantiation that are not really expected (user should give the name of the object instead of relying on the system to generate it)
# new name as ksp.Dra_n_dom_table_attrname[:40]
val.make_persistent(name)
return val
def __setitem__(self, key, val):
"""
Method to insert values in the StorageDict
Args:
key: the position of the value that we want to save
val: the value that we want to save in that position
"""
if isinstance(val, list):
vals_istorage = []
for element in val:
if isinstance(element, np.ndarray) and not isinstance(
element, StorageNumpy):
val_istorage = StorageNumpy(element)
else:
val_istorage = element
vals_istorage.append(val_istorage)
val = vals_istorage
elif isinstance(val, np.ndarray) and not isinstance(val, StorageNumpy):
val = StorageNumpy(val)
elif isinstance(val, set):
val = self.__create_embeddedset(key=key, val=val)
log.debug('SET ITEM %s->%s', key, val)
if self.storage_id is None:
dict.__setitem__(self, key, val)
elif not isinstance(val, EmbeddedSet):
# Not needed because it is made persistent and inserted to hcache when calling to self.__create_embeddedset
val = self.__make_val_persistent(val)
self._hcache.put_row(self._make_key(key), self._make_value(val))
if config.max_cache_size == 0: # If C++ cache is disabled, use python memory
dict.__setitem__(self, key, val)
def __len__(self):
if not self.storage_id:
return super().__len__()
self.sync()
if self._tokens[0][0] == _min_token and self._tokens[-1][
1] == _max_token:
query = f"SELECT COUNT(*) FROM {self._ksp}.{self._table}"
return self._count_elements(query)
else:
keys = []
for pkey in self._primary_keys:
template = "'{}'" if pkey["type"] == "text" else "{}"
keys.append(template.format(pkey["name"]))
all_keys = ",".join(keys)
total = 0
for (token_start, token_end) in self._tokens:
query = f"SELECT COUNT(*) FROM {self._ksp}.{self._table} " \
f"WHERE token({all_keys})>={token_start} AND token({all_keys})<{token_end}"
total = total + self._count_elements(query)
return total
def __repr__(self):
"""
Overloads the method used by print to show a StorageDict
Returns: The representation of the data stored in the StorageDict
"""
to_return = {}
for item in self.items():
to_return[item[0]] = item[1]
if len(to_return) == config.hecuba_print_limit:
return str(to_return)
if len(to_return) > 0:
return str(to_return)
return ""
def update(self, other=None, **kwargs):
"""
Updates the current dict with a new dictionary or set of attr,value pairs
(those must follow the current dict data model).
Args:
other: python dictionary or StorageDict. All key,val values in it will
be inserted in the current dict.
**kwargs: set of attr:val pairs, to be treated as key,val and inserted
in the current dict.
"""
if other is not None:
if isinstance(other, StorageDict):
for k, v in other.items():
self[k] = v
else:
for k, v in other.items() if isinstance(other,
Mapping) else other:
self[k] = v
for k, v in kwargs.items():
self[k] = v
def keys(self):
"""
Obtains the iterator for the keys of the StorageDict
Returns:
if persistent:
iterkeys(self): list of keys
if not persistent:
dict.keys(self)
"""
if self.storage_id:
self.sync()
ik = self._hcache.iterkeys(config.prefetch_size)
iterator = NamedIterator(ik, self._key_builder, self)
if self._has_embedded_set:
iterator = iter(set(iterator))
return iterator
else:
return dict.keys(self)
def items(self):
"""
Obtains the iterator for the key,val pairs of the StorageDict
Returns:
if persistent:
NamedItemsIterator(self): list of key,val pairs
if not persistent:
dict.items(self)
"""
if self.storage_id:
self.sync()
ik = self._hcache.iteritems(config.prefetch_size)
iterator = NamedItemsIterator(self._key_builder,
self._column_builder, self._k_size,
ik, self)
if self._has_embedded_set:
d = defaultdict(set)
# iteritems has the set values in different rows, this puts all the set values in the same row
if len(self._get_set_types()) == 1:
for row in iterator:
d[row.key].add(row.value[0])
else:
for row in iterator:
d[row.key].add(tuple(row.value))
iterator = d.items()
return iterator
else:
return dict.items(self)
def values(self):
"""
Obtains the iterator for the values of the StorageDict
Returns:
if persistent:
NamedIterator(self): list of valuesStorageDict
if not persistent:
dict.values(self)
"""
if self.storage_id:
self.sync()
if self._has_embedded_set:
items = self.items()
return dict(items).values()
else:
ik = self._hcache.itervalues(config.prefetch_size)
return NamedIterator(ik, self._column_builder, self)
else:
return dict.values(self)
def get(self, key, default=None):
try:
value = self.__getitem__(key)
except KeyError:
value = default
return value
def _get_set_types(self):
"""
Returns a list of tuples (name,type) for the types of the set
"""
if self._has_embedded_set:
set_types = [
col.get("columns", []) for col in self._columns
if isinstance(col, dict)
]
return sum(set_types, [])
else:
return []
class StorageDict(dict, IStorage):
# """
# Object used to access data from workers.
# """
args_names = [
"name", "primary_keys", "columns", "tokens", "storage_id",
"indexed_on", "class_name"
]
args = namedtuple('StorageDictArgs', args_names)
_prepared_store_meta = config.session.prepare(
'INSERT INTO hecuba.istorage'
'(storage_id, class_name, name, tokens, '
'primary_keys, columns, indexed_on)'
'VALUES (?,?,?,?,?,?,?)')
@staticmethod
def build_remotely(result):
"""
Launches the StorageDict.__init__ from the api.getByID
Args:
result: a namedtuple with all the information needed to create again the StorageDict
"""
log.debug("Building Storage dict with %s", result)
return StorageDict(result.name, result.primary_keys, result.columns,
result.tokens, result.storage_id, result.indexed_on)
@staticmethod
def _store_meta(storage_args):
"""
Method to update the info about the StorageDict in the DB metadata table
Args:
storage_args: structure with all data needed to update the metadata
"""
log.debug("StorageDict: storing metas %s", storage_args)
try:
config.session.execute(StorageDict._prepared_store_meta, [
storage_args.storage_id, storage_args.class_name,
storage_args.name, storage_args.tokens,
storage_args.primary_keys, storage_args.columns,
storage_args.indexed_on
])
except Exception as ex:
log.error("Error creating the StorageDict metadata: %s %s",
storage_args, ex)
raise ex
def __init__(self,
name=None,
primary_keys=None,
columns=None,
tokens=None,
storage_id=None,
indexed_args=None,
**kwargs):
"""
Creates a new StorageDict.
Args:
name (string): the name of the collection/table (keyspace is optional)
primary_keys (list(tuple)): a list of (key,type) primary keys (primary + clustering).
columns (list(tuple)): a list of (key,type) columns
tokens (list): list of tokens
storage_id (string): the storage id identifier
indexed_args (list): values that will be used as index
kwargs: other parameters
"""
super(StorageDict, self).__init__(**kwargs)
self._is_persistent = False
log.debug("CREATED StorageDict(%s,%s,%s,%s,%s,%s)", primary_keys,
columns, name, tokens, storage_id, kwargs)
if tokens is None:
log.info('using all tokens')
tokens = map(lambda a: a.value,
config.cluster.metadata.token_map.ring)
self._tokens = IStorage._discrete_token_ranges(tokens)
else:
self._tokens = tokens
self._storage_id = storage_id
if self.__doc__ is not None:
self._persistent_props = self._parse_comments(self.__doc__)
self._primary_keys = self._persistent_props[
self.__class__.__name__]['primary_keys']
self._columns = self._persistent_props[
self.__class__.__name__]['columns']
try:
self._indexed_args = self._persistent_props[
self.__class__.__name__]['indexed_values']
except KeyError:
self._indexed_args = indexed_args
else:
self._primary_keys = primary_keys
self._columns = columns
self._indexed_args = indexed_args
key_names = [pkname for (pkname, dt) in self._primary_keys]
column_names = [colname for (colname, dt) in self._columns]
self._item_builder = namedtuple('row', key_names + column_names)
if len(key_names) > 1:
self._key_builder = namedtuple('row', key_names)
else:
self._key_builder = None
if len(column_names) > 1:
self._column_builder = namedtuple('row', column_names)
else:
self._column_builder = None
self._k_size = len(key_names)
class_name = '%s.%s' % (self.__class__.__module__,
self.__class__.__name__)
self._build_args = self.args(name, self._primary_keys, self._columns,
self._tokens, self._storage_id,
self._indexed_args, class_name)
if name is not None:
self.make_persistent(name)
else:
self._is_persistent = False
def __eq__(self, other):
"""
Method to compare a StorageDict with another one.
Args:
other: StorageDict to be compared with.
Returns:
boolean (true - equals, false - not equals).
"""
return self._storage_id == other._storage_id and self._tokens == other.token_ranges and \
self._table == other.table_name and self._ksp == other.keyspace
_dict_case = re.compile(
'.*@TypeSpec + *< *< *([\w:, ]+)+ *> *, *([\w+:., <>]+) *>')
_tuple_case = re.compile('.*@TypeSpec +(\w+) +tuple+ *< *([\w, +]+) *>')
_index_vars = re.compile('.*@Index_on *([A-z0-9, ]+)')
_other_case = re.compile(' *(\w+) *< *([\w, +]+) *>')
@classmethod
def _parse_comments(self, comments):
"""
Parses de comments in a class file to save them in the class information
Args:
comments: the comment in the class file
Returns:
this: a structure with all the information of the comment
"""
this = {}
for line in comments.split('\n'):
m = StorageDict._dict_case.match(line)
if m is not None:
# Matching @TypeSpec of a dict
dict_keys, dict_values = m.groups()
primary_keys = []
for ind, key in enumerate(dict_keys.split(",")):
key = key.replace(' ', '')
match = IStorage._data_type.match(key)
if match is not None:
# an IStorage with a name
name, value = match.groups()
elif ':' in key:
raise SyntaxError
else:
name = "key" + str(ind)
value = key
name = name.replace(' ', '')
value = value.replace(' ', '')
primary_keys.append(
(name, StorageDict._conversions[value]))
dict_values = dict_values.replace(' ', '')
if dict_values.startswith('dict'):
n = IStorage._sub_dict_case.match(dict_values[4:])
# Matching @TypeSpec of a sub dict
dict_keys2, dict_values2 = n.groups()
primary_keys2 = []
for ind, key in enumerate(dict_keys2.split(",")):
try:
name, value = IStorage._data_type.match(
key).groups()
except ValueError:
if ':' in key:
raise SyntaxError
else:
name = "key" + str(ind)
value = key
name = name.replace(' ', '')
primary_keys2.append(
(name, StorageDict._conversions[value]))
columns2 = []
dict_values2 = dict_values2.replace(' ', '')
if dict_values2.startswith('tuple'):
dict_values2 = dict_values2[6:]
for ind, val in enumerate(dict_values2.split(",")):
try:
name, value = IStorage._data_type.match(
val).groups()
except ValueError:
if ':' in key:
raise SyntaxError
else:
name = "val" + str(ind)
value = val
columns2.append(
(name, StorageDict._conversions[value]))
columns = {
'type': 'dict',
'primary_keys': primary_keys2,
'columns': columns2
}
elif dict_values.startswith('tuple'):
n = IStorage._sub_tuple_case.match(dict_values[5:])
tuple_values = list(n.groups())[0]
columns = []
for ind, val in enumerate(tuple_values.split(",")):
try:
name, value = val.split(':')
except ValueError:
if ':' in key:
raise SyntaxError
else:
name = "val" + str(ind)
value = val
name = name.replace(' ', '')
columns.append((name, StorageDict._conversions[value]))
else:
columns = []
for ind, val in enumerate(dict_values.split(",")):
match = IStorage._data_type.match(val)
if match is not None:
# an IStorage with a name
name, value = match.groups()
elif ':' in val:
name, value = IStorage._so_data_type.match(
val).groups()
else:
name = "val" + str(ind)
value = val
name = name.replace(' ', '')
try:
columns.append(
(name, StorageDict._conversions[value]))
except KeyError:
columns.append((name, value))
name = str(self).replace('\'>', '').split('.')[-1]
if self.__class__.__name__ in this:
this[name].update({
'type': 'dict',
'primary_keys': primary_keys,
'columns': columns
})
else:
this[name] = {
'type': 'dict',
'primary_keys': primary_keys,
'columns': columns
}
m = StorageDict._index_vars.match(line)
if m is not None:
name = str(self).replace('\'>', '').split('.')[-1]
indexed_values = m.groups()
indexed_values = indexed_values.replace(' ', '').split(',')
if name in this:
this[name].update({'indexed_values': indexed_values})
else:
this[name] = {'indexed_values': indexed_values}
return this
def __contains__(self, key):
"""
Method that checks if a given key exists in a StorageDict.
Args:
key: the position that we want to check if exists.
Returns:
boolean (true - exists, false - doesn't exist).
"""
if not self._is_persistent:
return dict.__contains__(self, key)
else:
try:
# TODO we should save this value in a cache
self._hcache.get_row(self._make_key(key))
return True
except Exception as ex:
log.warn("persistentDict.__contains__ ex %s", ex)
raise ex
def _make_key(self, key):
"""
Method used to pass the key data to the StorageDict cache in a proper way.
Args:
key: the data that needs to get the correct format
"""
if isinstance(key, str) or isinstance(
key, unicode) or not isinstance(key, Iterable):
if len(self._primary_keys) == 1:
if isinstance(key, unicode):
return [key.encode('ascii', 'ignore')]
return [key]
else:
raise Exception('missing a primary key')
if isinstance(key, Iterable) and len(key) == len(self._primary_keys):
return list(key)
else:
raise Exception('wrong primary key')
@staticmethod
def _make_value(value):
"""
Method used to pass the value data to the StorageDict cache in a proper way.
Args:
value: the data that needs to get the correct format
"""
if issubclass(value.__class__, IStorage):
return [uuid.UUID(value.getID())]
elif isinstance(value,
str) or not isinstance(value, Iterable) or isinstance(
value, np.ndarray):
return [value]
elif isinstance(value, unicode):
return [value.encode('ascii', 'ignore')]
else:
return list(value)
def keys(self):
"""
This method return a list of all the keys of the StorageDict.
Returns:
list: a list of keys
"""
return [i for i in self.iterkeys()]
def values(self):
"""
This method return a list of all the values of the StorageDict.
Returns:
list: a list of values
"""
return [i for i in self.itervalues()]
def __iter__(self):
"""
Method that overloads the python dict basic iteration, which returns
an iterator over the dictionary keys.
"""
return self.iterkeys()
def make_persistent(self, name):
"""
Method to transform a StorageDict into a persistent object.
This will make it use a persistent DB as the main location
of its data.
Args:
name:
"""
if self._is_persistent:
raise AlreadyPersistentError(
"This StorageDict is already persistent [Before:{}.{}][After:{}]",
self._ksp, self._table, name)
self._is_persistent = True
(self._ksp, self._table) = self._extract_ks_tab(name)
if self._storage_id is None:
self._storage_id = uuid.uuid3(uuid.NAMESPACE_DNS,
self._ksp + '.' + self._table)
self._build_args = self._build_args._replace(
storage_id=self._storage_id, name=self._ksp + "." + self._table)
self._store_meta(self._build_args)
if config.id_create_schema == -1:
query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (
self._ksp, config.replication)
try:
log.debug('MAKE PERSISTENCE: %s', query_keyspace)
config.session.execute(query_keyspace)
except Exception as ex:
log.warn("Error creating the StorageDict keyspace %s, %s",
(query_keyspace), ex)
raise ex
for key, value in dict.iteritems(self):
if issubclass(value.__class__, IStorage):
# new name as ksp+table+obj_class_name
val_name = self._ksp + '.' + self._table + type(
value).__name__.lower()
value.make_persistent(val_name)
columns = self._primary_keys + self._columns
for ind, entry in enumerate(columns):
n = StorageDict._other_case.match(entry[1])
if n is not None:
iter_type, intra_type = n.groups()
else:
iter_type = entry[1]
if iter_type not in IStorage._basic_types:
columns[ind] = entry[0], 'uuid'
pks = map(lambda a: a[0], self._primary_keys)
query_table = "CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));" \
% (self._ksp,
self._table,
",".join("%s %s" % tup for tup in columns),
str.join(',', pks))
try:
log.debug('MAKE PERSISTENCE: %s', query_table)
config.session.execute(query_table)
except Exception as ex:
log.warn("Error creating the StorageDict table: %s %s",
query_table, ex)
raise ex
key_names = map(lambda a: a[0].encode('UTF8'), self._primary_keys)
column_names = self._columns
self._hcache_params = (self._ksp, self._table, self._storage_id,
self._tokens, key_names,
map(lambda x: {
"name": x[0],
"type": x[1]
}, column_names), {
'cache_size': config.max_cache_size,
'writer_par': config.write_callbacks_number,
'write_buffer': config.write_buffer_size
})
log.debug("HCACHE params %s", self._hcache_params)
self._hcache = Hcache(*self._hcache_params)
# Storing all in-memory values to cassandra
for key, value in dict.iteritems(self):
self._hcache.put_row(self._make_key(key), self._make_value(value))
if hasattr(self, '_indexed_args') and self._indexed_args is not None:
index_query = 'CREATE CUSTOM INDEX IF NOT EXISTS ' + self._table + '_idx ON '
index_query += self._ksp + '.' + self._table + ' (' + str.join(
',', self._indexed_args) + ') '
index_query += "using 'es.bsc.qbeast.index.QbeastIndex';"
try:
config.session.execute(index_query)
except Exception as ex:
log.error("Error creating the Qbeast custom index: %s %s",
index_query, ex)
raise ex
def stop_persistent(self):
"""
Method to turn a StorageDict into non-persistent.
"""
log.debug('STOP PERSISTENCE: %s', self._table)
self._is_persistent = False
self._hcache = None
def delete_persistent(self):
"""
Method to empty all data assigned to a StorageDict.
"""
query = "TRUNCATE TABLE %s.%s;" % (self._ksp, self._table)
log.debug('DELETE PERSISTENT: %s', query)
config.session.execute(query)
def _build_istorage_obj(self, obj_type, so_name, storage_id):
cname, module = IStorage.process_path(obj_type)
mod = __import__(module, globals(), locals(), [cname], 0)
# new name as ksp+table+obj_class_name
so = getattr(mod, cname)(name=so_name + cname.lower(),
storage_id=storage_id)
# sso._storage_id = storage_id
return so
def __delitem__(self, key):
"""
Method to delete a specific entry in the dict in the key position.
Args:
key: position of the entry that we want to delete
"""
if not self._is_persistent:
dict.__delitem__(self, key)
else:
self._hcache.delete_row([key])
def __getitem__(self, key):
"""
If the object is persistent, each request goes to the hfetch.
Args:
key: the dictionary key
Returns
item: value found in position key
"""
log.debug('GET ITEM %s', key)
if not self._is_persistent:
to_return = dict.__getitem__(self, key)
return to_return
else:
cres = self._hcache.get_row(self._make_key(key))
log.debug("GET ITEM %s[%s]", cres, cres.__class__)
final_results = []
for index, (name, col_type) in enumerate(self._columns):
if col_type not in IStorage._basic_types:
table_name = self._ksp + '.' + self._table
element = (self._build_istorage_obj(
col_type, table_name, uuid.UUID(cres[index])))
else:
element = cres[index]
final_results.append(element)
cres = final_results
if issubclass(cres.__class__, NoneType):
return None
elif self._column_builder is not None:
if len(cres) > 0 and isinstance(cres[0], list):
return [self._column_builder(*row) for row in cres]
else:
return self._column_builder(*cres)
else:
return cres[0]
def __setitem__(self, key, val):
"""
Method to insert values in the StorageDict
Args:
key: the position of the value that we want to save
val: the value that we want to save in that position
"""
if isinstance(val, np.ndarray):
val = StorageNumpy(val)
log.debug('SET ITEM %s->%s', key, val)
if not config.hecuba_type_checking:
if not self._is_persistent:
dict.__setitem__(self, key, val)
else:
if isinstance(val, IStorage) and not val._is_persistent:
attribute = val.__class__.__name__.lower()
count = self._count_name_collision(attribute)
# new name as ksp+table+obj_class_name
val.make_persistent(self._ksp + '.' + self._table + "_" +
attribute + "_" + str(count))
self._hcache.put_row(self._make_key(key),
self._make_value(val))
else:
if isinstance(val, Iterable) and not isinstance(val, str):
col_types = map(
lambda x: IStorage._conversions[x.__class__.__name__], val)
spec_col_types = map(lambda x: x[1], self._columns)
for idx, value in enumerate(spec_col_types):
if value == 'double':
spec_col_types[idx] = 'float'
else:
col_types = IStorage._conversions[val.__class__.__name__]
spec_col_types = map(lambda x: x[1], self._columns)[0]
if spec_col_types == 'double':
spec_col_types = 'float'
if isinstance(key, Iterable) and not isinstance(key, str):
key_types = map(
lambda x: IStorage._conversions[x.__class__.__name__], key)
spec_key_types = map(lambda x: x[1], self._primary_keys)
for idx, value in enumerate(spec_key_types):
if value == 'double':
spec_key_types[idx] = 'float'
else:
key_types = IStorage._conversions[key.__class__.__name__]
spec_key_types = map(lambda x: x[1], self._primary_keys)[0]
if spec_key_types == 'double':
spec_key_types = 'float'
if (col_types == spec_col_types):
if (key_types == spec_key_types):
if not self._is_persistent:
dict.__setitem__(self, key, val)
else:
self._hcache.put_row(self._make_key(key),
self._make_value(val))
else:
raise KeyError
else:
raise ValueError
def __repr__(self):
"""
Overloads the method used by print to show a StorageDict
Returns: The representation of the data stored in the StorageDict
"""
to_return = {}
for item in self.iteritems():
to_return[item[0]] = item[1]
if len(to_return) == config.hecuba_print_limit:
return str(to_return)
if len(to_return) > 0:
return str(to_return)
return ""
def update(self, other=None, **kwargs):
"""
Updates the current dict with a new dictionary or set of attr,value pairs
(those must follow the current dict data model).
Args:
other: python dictionary or StorageDict. All key,val values in it will
be inserted in the current dict.
**kwargs: set of attr:val pairs, to be treated as key,val and inserted
in the current dict.
"""
if other is not None:
if isinstance(other, StorageDict):
for k, v in other.iteritems():
self[k] = v
else:
for k, v in other.items() if isinstance(other,
Mapping) else other:
self[k] = v
for k, v in kwargs.items():
self[k] = v
def iterkeys(self):
"""
Obtains the iterator for the keys of the StorageDict
Returns:
if persistent:
iterkeys(self): list of keys
if not persistent:
dict.iterkeys(self)
"""
if self._is_persistent:
ik = self._hcache.iterkeys(config.prefetch_size)
return NamedIterator(ik, self._key_builder, self)
else:
return dict.iterkeys(self)
def iteritems(self):
"""
Obtains the iterator for the key,val pairs of the StorageDict
Returns:
if persistent:
NamedItemsIterator(self): list of key,val pairs
if not persistent:
dict.iteritems(self)
"""
if self._is_persistent:
ik = self._hcache.iteritems(config.prefetch_size)
return NamedItemsIterator(self._key_builder, self._column_builder,
self._k_size, ik, self)
else:
return dict.iteritems(self)
def itervalues(self):
"""
Obtains the iterator for the values of the StorageDict
Returns:
if persistent:
NamedIterator(self): list of valuesStorageDict
if not persistent:
dict.itervalues(self)
"""
if self._is_persistent:
ik = self._hcache.itervalues(config.prefetch_size)
return NamedIterator(ik, self._column_builder, self)
else:
return dict.itervalues(self)
def keys(self):
return [i for i in self.iterkeys()]
def values(self):
return [i for i in self.itervalues()]
def items(self):
return [i for i in self.iteritems()]
def get(self, key, default):
try:
value = self.__getitem__(key)
except KeyError:
value = default
return value