def getByID(objid):
"""
We rebuild the object from its id. The id can either be:
block: UUID (eg. f291f008-a520-11e6-b42e-5b582e04fd70)
storageobj: UUID_(version) (eg. f291f008-a520-11e6-b42e-5b582e04fd70_1)
Args:
objid (str): object identifier
Returns:
(Block| Storageobj)
"""
"""
TODO
Args:
objid (str): object identifier
Returns:
(Block| Storageobj)
"""
from hecuba import log
from hecuba.IStorage import IStorage
try:
from hecuba import config
query = "SELECT * FROM hecuba.istorage WHERE storage_id = %s"
results = config.session.execute(query, [uuid.UUID(objid)])[0]
except Exception as e:
log.error("Query %s failed", query)
raise e
log.debug("IStorage API:getByID(%s) of class %s", objid,
results.class_name)
return IStorage.build_remotely(results._asdict())
def __init__(self,
primary_keys,
columns,
name,
qbeast_meta,
qbeast_id=None,
entry_point=None,
storage_id=None,
tokens=None):
"""
Creates a new block.
Args:
table_name (string): the name of the collection/table
keyspace_name (string): name of the Cassandra keyspace.
primary_keys (list(tuple)): a list of (key,type) primary keys (primary + clustering).
columns (list(tuple)): a list of (key,type) columns
tokens (list): list of tokens
storage_id (uuid): the storage id identifier
"""
log.debug(
"CREATED QbeastIterator(%s,%s,%s,%s)",
storage_id,
tokens,
)
self._selects = map(lambda a: a[0], primary_keys + columns)
key_namedtuple = namedtuple("key", map(lambda a: a[0], primary_keys))
value_namedtuple = namedtuple("value", map(lambda a: a[0], columns))
div = len(primary_keys)
self._row_builder = lambda a: self._row_namedtuple(
key_namedtuple(*a[:div]), value_namedtuple(*a[div:]))
(self._ksp, self._table) = self._extract_ks_tab(name)
self._qbeast_meta = qbeast_meta
self._qbeast_id = qbeast_id
self._entry_point = entry_point
if tokens is None:
log.info('using all tokens')
tokens = map(lambda a: a.value,
config.cluster.metadata.token_map.ring)
self._tokens = IStorage._discrete_token_ranges(tokens)
else:
self._tokens = tokens
class_name = '%s.%s' % (self.__class__.__module__,
self.__class__.__name__)
# primary_keys columns name tokens
# indexed_args nonindexed_args value_list
# mem_filter port storage_id class_name
if storage_id is None:
self._storage_id = uuid.uuid4()
save = True
else:
self._storage_id = storage_id
save = False
self._build_args = self._building_args(primary_keys, columns, name,
qbeast_meta, qbeast_id,
entry_point, self._storage_id,
self._tokens, class_name)
if save:
self._store_meta(self._build_args)
def tokens512_partition_16_nodes_test(self):
partitions = [
i for i in IStorage._tokens_partitions(self.t16, 512, 16)
]
flat = reduce(list.__add__, partitions)
self.check_full_range(flat)
self.assertGreaterEqual(len(partitions), 16)
self.assertGreater(len(set(flat)), 512)
def tokens16_partition_16_nodes_test(self):
partitions = [i for i in IStorage._tokens_partitions(self.t16, 16, 16)]
flat = reduce(list.__add__, partitions)
self.check_full_range(flat)
self.assertGreaterEqual(16, len(partitions))
self.assertEqual((2**63) - 1, reduce(max, map(lambda a: a[1], flat)))
self.assertEqual(-9223372036854775808,
reduce(min, map(lambda a: a[0], flat)))
self.assertEqual(16, len(set(flat)))
def test_init_create_pdict(self):
config.session.execute("DROP TABLE IF EXISTS my_app.tt1")
config.session.execute("DROP TABLE IF EXISTS my_app.tt1_instances")
class res:
pass
config.session.execute("DROP TABLE IF EXISTS " + config.execution_name + '.tt1')
r = res()
r.ksp = config.execution_name
r.name = 'tt1'
r.class_name = r.class_name = str(TestStorageObj.__module__) + "." + TestStorageObj.__name__
r.storage_id = uuid.uuid3(uuid.NAMESPACE_DNS, config.execution_name + '.' + r.name)
r.tokens = IStorage._discrete_token_ranges(
[8508619251581300691, 8514581128764531689, 8577968535836399533, 8596162846302799189,
8603491526474728284, 8628291680139169981, 8687301163739303017, 9111581078517061776])
r.istorage_props = {}
nopars = StorageObj.build_remotely(r)
self.assertEqual('tt1', nopars._table)
self.assertEqual(config.execution_name, nopars._ksp)
self.assertEqual(uuid.uuid3(uuid.NAMESPACE_DNS, config.execution_name + '.tt1'), nopars._storage_id)
name, tkns = \
config.session.execute("SELECT name,tokens FROM hecuba.istorage WHERE storage_id = %s",
[nopars._storage_id])[0]
self.assertEqual(name, config.execution_name + '.' + r.name)
self.assertEqual(tkns, r.tokens)
tkns = IStorage._discrete_token_ranges(
[8508619251581300691, 8514581128764531689, 8577968535836399533, 8596162846302799189,
8603491526474728284, 8628291680139169981, 8687301163739303017, 9111581078517061776])
config.session.execute("DROP TABLE IF EXISTS " + config.execution_name + '.tt2')
nopars = Result(name='tt2',
tokens=tkns)
self.assertEqual('tt2', nopars._table)
self.assertEqual(config.execution_name, nopars._ksp)
self.assertEqual(uuid.uuid3(uuid.NAMESPACE_DNS, config.execution_name + '.tt2'), nopars._storage_id)
self.assertEqual(True, nopars._is_persistent)
self.assertTrue(hasattr(nopars, 'instances'))
name, read_tkns = config.session.execute("SELECT name,tokens FROM hecuba.istorage WHERE storage_id = %s",
[nopars._storage_id])[0]
self.assertEqual(name, config.execution_name + '.tt2')
self.assertEqual(tkns, read_tkns)
def tokens1024_16_blocks_with_16_vtokens_test(self):
partitions = [
i for i in IStorage._tokens_partitions(self.t1024_tuples, 16, 16)
]
flat = reduce(list.__add__, partitions)
self.check_full_range(flat)
self.assertGreaterEqual(len(partitions), 16)
self.assertEqual((2**63) - 1, reduce(max, map(lambda a: a[1], flat)))
self.assertEqual(-(2**63), reduce(min, map(lambda a: a[0], flat)))
#self.assertEqual({64}, reduce(set.union, map(lambda a: {len(a)}, partitions)))
self.assertGreaterEqual(len(set(flat)), 1024)
def split(self):
"""
Initializes the iterator, and saves the information about the token ranges of each block
Args:
my_dict (PersistentDict): Hecuba PersistentDict
"""
splits = [s for s in IStorage.split(self)]
if type(config.qbeast_entry_node) == list:
qbeast_node = config.qbeast_entry_node[0]
else:
qbeast_node = config.qbeast_entry_node
transport = TSocket.TSocket(qbeast_node, config.qbeast_master_port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TFramedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = QbeastMaster.Client(protocol)
# Connect!
transport.open()
area = FilteringArea(fromPoint=self._qbeast_meta.from_point,
toPoint=self._qbeast_meta.to_point)
uuids = map(lambda x: str(x._storage_id), splits)
log.info(
"calling initQuery (%s, %s, %s, precision=%f ,area=%s, uuids=%s, max_results=%f",
self._selects, self._ksp, self._table, self._qbeast_meta.precision,
area, uuids, config.qbeast_max_results)
self._qbeast_id = uuid.UUID(
client.initQuery(self._selects, self._ksp + '_qbeast',
self._table + '_' + self._table + '_idx_d8tree',
area, self._qbeast_meta.precision,
config.qbeast_max_results, uuids))
transport.close()
for i in splits:
i._set_qbeast_id(self._qbeast_id)
return iter(splits)
def test_build_remotely(self):
class res:
pass
r = res()
r.ksp = config.execution_name
r.name = 'tt1'
r.class_name = str(TestStorageObj.__module__) + "." + TestStorageObj.__name__
r.tokens = IStorage._discrete_token_ranges(
[8508619251581300691, 8514581128764531689, 8577968535836399533, 8596162846302799189,
8603491526474728284, 8628291680139169981, 8687301163739303017, 9111581078517061776])
r.storage_id = uuid.uuid3(uuid.NAMESPACE_DNS, config.execution_name + '.tt1')
r.istorage_props = {}
nopars = StorageObj.build_remotely(r)
self.assertEqual('tt1', nopars._table)
self.assertEqual(config.execution_name, nopars._ksp)
self.assertEqual(uuid.uuid3(uuid.NAMESPACE_DNS, config.execution_name + '.tt1'), nopars._storage_id)
name, tkns = \
config.session.execute("SELECT name, tokens FROM hecuba.istorage WHERE storage_id = %s", [nopars._storage_id])[
0]
self.assertEqual(name, config.execution_name + '.tt1')
self.assertEqual(tkns, r.tokens)