def post(self):
"""
Creates a new node based on data provided into a POST request.
The data can be provided in the body as JSON or HTTP form-data.
HTTP Data:
name (str): The name of the node that will be created
cluster_name (str): The cluster that the node will belong
"""
args = self.reqparse.parse_args()
# check if cluster_name actually exists
log.debug("Checking if cluster %s exists." % args['cluster_name'])
c = models.Cluster.query.get(args['cluster_name'])
if c is None:
log.debug("Cluster %s doesn't exist." % args['cluster_name'])
abort(404, message="Cluster {} doesn't exist.".format(args['cluster_name']))
# check if http auth user has permission to modify
auth.check_permission(auth.http_auth.username(), args['cluster_name'])
# added if doesn't exist
n = models.Node(name=args['name'], cluster_name=args['cluster_name'])
db.session.add(n)
try:
db.session.commit()
log.info("Node %s added to cluster %s." % (args['name'], args['cluster_name']))
except IntegrityError:
log.debug("Tried to add node %s but it already exists." % args['name'])
return { 'message': "Node already exists." }, 409
return { 'nodes': marshal(n, models.node_fields) }, 201, { 'Location': "/node/%s" % args['name'] }
def make_persistent(self, name):
"""
Once a StorageObj has been created, it can be made persistent. This function retrieves the information about
the Object class schema, and creates a Cassandra table with those parameters, where information will be
saved from now on, until execution finishes or StorageObj is no longer persistent.
It also inserts into the new table all information that was in memory assigned to the StorageObj prior to
this call.
Args:
name (string): name with which the table in the DB will be created
"""
if self._is_persistent:
raise AlreadyPersistentError(
"This StorageObj is already persistent [Before:{}.{}][After:{}]",
self._ksp, self._table, name)
self._is_persistent = True
(self._ksp, self._table) = self._extract_ks_tab(name)
if self._storage_id is None:
self._storage_id = uuid.uuid3(uuid.NAMESPACE_DNS,
self._ksp + '.' + self._table)
self._build_args = self.args(self._ksp + '.' + self._table,
self._tokens, self._storage_id,
self._istorage_props, self._class_name)
log.info("PERSISTING DATA INTO %s %s", self._ksp, self._table)
query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (
self._ksp, config.replication)
config.session.execute(query_keyspace)
query_simple = 'CREATE TABLE IF NOT EXISTS ' + self._ksp + '.' + self._table + \
'( storage_id uuid PRIMARY KEY, '
for key, entry in self._persistent_props.items():
query_simple += str(key) + ' '
if entry['type'] != 'dict' and entry[
'type'] in IStorage._valid_types:
if entry['type'] == 'list' or entry['type'] == 'tuple':
query_simple += entry['type'] + '<' + entry[
'columns'] + '>, '
else:
query_simple += entry['type'] + ', '
else:
query_simple += 'uuid, '
try:
config.session.execute(query_simple[:-2] + ' )')
except Exception as ir:
log.error("Unable to execute %s", query_simple)
raise ir
for obj_name, obj_info in self._persistent_props.items():
if hasattr(self, obj_name):
pd = getattr(self, obj_name)
if obj_info['type'] not in IStorage._basic_types:
sd_name = self._ksp + "." + self._table + "_" + obj_name
pd.make_persistent(sd_name)
setattr(
self, obj_name, pd
) # super(StorageObj, self).__setattr__(obj_name, pd) why?
self._store_meta(self._build_args)
def delete(self, name):
"""
Deletes an existing node
Args:
name (str): The name of the node that will be deleted
"""
log.debug("Checking if node %s exists." % name)
n = models.Node.query.get(name)
if (n is None):
log.debug("Node %s doesn't exist." % name)
abort(404, message="Node {} doesn't exist.".format(name))
# check if http auth user has permission to modify
auth.check_permission(auth.http_auth.username(), n.cluster_name)
db.session.delete(n)
try:
db.session.commit()
log.info("Node %s was deleted." % name)
except Exception as ex:
log.debug(ex)
return { 'message': "Error removing node." }, 500
return { 'message': "Node removed." }
def __init__(self,
primary_keys,
columns,
name,
qbeast_meta,
qbeast_id=None,
entry_point=None,
storage_id=None,
tokens=None):
"""
Creates a new block.
Args:
table_name (string): the name of the collection/table
keyspace_name (string): name of the Cassandra keyspace.
primary_keys (list(tuple)): a list of (key,type) primary keys (primary + clustering).
columns (list(tuple)): a list of (key,type) columns
tokens (list): list of tokens
storage_id (uuid): the storage id identifier
"""
log.debug(
"CREATED QbeastIterator(%s,%s,%s,%s)",
storage_id,
tokens,
)
self._selects = map(lambda a: a[0], primary_keys + columns)
key_namedtuple = namedtuple("key", map(lambda a: a[0], primary_keys))
value_namedtuple = namedtuple("value", map(lambda a: a[0], columns))
div = len(primary_keys)
self._row_builder = lambda a: self._row_namedtuple(
key_namedtuple(*a[:div]), value_namedtuple(*a[div:]))
(self._ksp, self._table) = self._extract_ks_tab(name)
self._qbeast_meta = qbeast_meta
self._qbeast_id = qbeast_id
self._entry_point = entry_point
if tokens is None:
log.info('using all tokens')
tokens = map(lambda a: a.value,
config.cluster.metadata.token_map.ring)
self._tokens = IStorage._discrete_token_ranges(tokens)
else:
self._tokens = tokens
class_name = '%s.%s' % (self.__class__.__module__,
self.__class__.__name__)
# primary_keys columns name tokens
# indexed_args nonindexed_args value_list
# mem_filter port storage_id class_name
if storage_id is None:
self._storage_id = uuid.uuid4()
save = True
else:
self._storage_id = storage_id
save = False
self._build_args = self._building_args(primary_keys, columns, name,
qbeast_meta, qbeast_id,
entry_point, self._storage_id,
self._tokens, class_name)
if save:
self._store_meta(self._build_args)
def __iter__(self):
if self._storage_id is None or self._qbeast_id is None:
'''
In this case, we are doing a query without splitting the object,
and thus we have to initialize the query for only this iterator
'''
if type(config.qbeast_entry_node) == list:
qbeast_node = config.qbeast_entry_node[0]
else:
qbeast_node = config.qbeast_entry_node
transport = TSocket.TSocket(qbeast_node, config.qbeast_master_port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TFramedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = QbeastMaster.Client(protocol)
# Connect!
transport.open()
area = FilteringArea(fromPoint=self._qbeast_meta.from_point,
toPoint=self._qbeast_meta.to_point)
if self._storage_id is None:
self._storage_id = uuid.uuid4()
self._build_args = self._build_args._replace(
storage_id=self._storage_id)
self._store_meta(self._build_args)
uuids = [str(self._storage_id)]
log.info(
"calling initQuery (%s, %s, %s, precision=%f ,area=%s, uuids=%s, max_results=%f",
self._selects, self._ksp, self._table,
self._qbeast_meta.precision, area, uuids,
config.qbeast_max_results)
self._qbeast_id = client.initQuery(
self._selects, self._ksp + '_qbeast',
self._table + '_' + self._table + '_idx_d8tree', area,
self._qbeast_meta.precision, config.qbeast_max_results, uuids)
self._store_meta(self._build_args)
transport.close()
return IndexedIterValue(self._storage_id, self._entry_point,
self._row_builder)
def split(self):
"""
Initializes the iterator, and saves the information about the token ranges of each block
Args:
my_dict (PersistentDict): Hecuba PersistentDict
"""
splits = [s for s in IStorage.split(self)]
if type(config.qbeast_entry_node) == list:
qbeast_node = config.qbeast_entry_node[0]
else:
qbeast_node = config.qbeast_entry_node
transport = TSocket.TSocket(qbeast_node, config.qbeast_master_port)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TFramedTransport(transport)
# Wrap in a protocol
protocol = TBinaryProtocol.TBinaryProtocol(transport)
# Create a client to use the protocol encoder
client = QbeastMaster.Client(protocol)
# Connect!
transport.open()
area = FilteringArea(fromPoint=self._qbeast_meta.from_point,
toPoint=self._qbeast_meta.to_point)
uuids = map(lambda x: str(x._storage_id), splits)
log.info(
"calling initQuery (%s, %s, %s, precision=%f ,area=%s, uuids=%s, max_results=%f",
self._selects, self._ksp, self._table, self._qbeast_meta.precision,
area, uuids, config.qbeast_max_results)
self._qbeast_id = uuid.UUID(
client.initQuery(self._selects, self._ksp + '_qbeast',
self._table + '_' + self._table + '_idx_d8tree',
area, self._qbeast_meta.precision,
config.qbeast_max_results, uuids))
transport.close()
for i in splits:
i._set_qbeast_id(self._qbeast_id)
return iter(splits)
def make_persistent(self, name):
if self._is_persistent:
raise AlreadyPersistentError(
"This StorageNumpy is already persistent [Before:{}.{}][After:{}]",
self._ksp, self._table, name)
self._is_persistent = True
(self._ksp, self._table) = self._extract_ks_tab(name)
if self._storage_id is None:
self._storage_id = uuid.uuid3(
uuid.NAMESPACE_DNS, self._ksp + '.' + self._table + '_numpies')
self._build_args = self.args(self._storage_id, self._class_name, name)
log.info("PERSISTING DATA INTO %s %s", self._ksp, self._table)
query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (
self._ksp, config.replication)
config.session.execute(query_keyspace)
config.session.execute(
'CREATE TABLE IF NOT EXISTS ' + self._ksp + '.' + self._table +
'_numpies'
'(storage_id uuid , '
'cluster_id int, '
'block_id int, '
'payload blob, '
'PRIMARY KEY((storage_id,cluster_id),block_id))')
self._hcache_params = (self._ksp, self._table + '_numpies',
self._storage_id, [],
['storage_id', 'cluster_id', 'block_id'], [{
'name':
"payload",
'type':
'numpy'
}], {
'cache_size': config.max_cache_size,
'writer_par': config.write_callbacks_number,
'write_buffer': config.write_buffer_size
})
self._hcache = Hcache(*self._hcache_params)
if len(self.shape) != 0:
self._hcache.put_row([self._storage_id, -1, -1], [self])
self._store_meta(self._build_args)
def post(self):
"""
Creates a new cluster based on data provided into a POST request.
The data can be provided in the body as JSON or HTTP form-data.
Note:
The "hecuba_admin" cluster name is reserved and cannot be used.
HTTP Data:
name (str): The name of the cluster that will be created
secret (Optional|str): Password for managing/updating the cluster
"""
args = self.reqparse.parse_args()
# check if http auth user has permission to modify
auth.check_permission(auth.http_auth.username(), "hecuba_admin")
# hash secret if needed
if (args['secret']):
hashed = auth.generate_password(args['secret'])
else:
hashed = None
# hecuba_admin cluster name is reserved for global authentication
if (args['name'] == "hecuba_admin"):
return { 'message': "Cannot create cluster: name \"hecuba_admin\" reserved." }, 403
# added if doesn't exist
c = models.Cluster(name=args['name'], secret=hashed)
db.session.add(c)
try:
db.session.commit()
log.info("Created cluster named %s." % args['name'])
except IntegrityError:
db.session.rollback()
log.debug("Tried to create a cluster named %s, but it already exists." % args['name'])
return { 'message': "Cluster already exists." }, 409
else:
return { 'clusters': marshal(c, models.cluster_fields) }, 201, { 'Location': "/cluster/%s" % args['name'] }
def delete(self, name):
"""
Deletes an existing cluster
Args:
name (str): The name of the cluster that will be deleted
"""
# check if http auth user has permission to modify
auth.check_permission(auth.http_auth.username(), "hecuba_admin")
log.debug("Checking if cluster %s exists." % name)
c = models.Cluster.query.get(name)
if (c is None):
log.debug("Cluster %s doesn't exist." % name)
abort(404, message="Cluster {} doesn't exist.".format(name))
# grab all nodes from current cluster to delete
log.info("Deleting all nodes from cluster %s." % name)
nodes = c.nodes.all()
for n in nodes:
db.session.delete(n)
try:
db.session.commit()
log.info("Node %s deleted." % n.name)
except Exception as ex:
db.session.rollback()
log.debug(ex)
return { 'message': "Error removing nodes from cluster." }, 500
log.debug("Deleting cluster %s." % name)
db.session.delete(c)
try:
db.session.commit()
log.info("Cluster %s was deleted." % name)
except Exception as ex:
db.session.rollback()
log.debug(ex)
return { 'message': "Error removing cluster." }, 500
return { 'message': "Cluster removed." }
def put(self, name):
"""
Updates an existing node based on data provided into a PUT request.
Args:
name (str): The name of the node that will be updated
"""
# get node and corresponding cluster
log.debug("Checking if node %s exists." % name)
n = models.Node.query.get(name)
if (n is None):
log.debug("Node %s doesn't exist." % name)
abort(404, message="Node {} doesn't exist.".format(name))
# check if http auth user has permission to modify
auth.check_permission(auth.http_auth.username(), n.cluster_name)
c = models.Cluster.query.get(n.cluster_name)
# only update node if cluster task is not running
if (n.running == False and c.running == True):
log.debug("Cluster %s has a task running, cannot update node %s." % (c.name, name))
return { 'message': "Cluster task already running. Cannot update." }, 403
# parse arguments from request
args = self.reqparse.parse_args()
for key, value in args.iteritems():
if (value != None and value != getattr(n, key)):
log.debug("Setting node '%s' attribute '%s' to: %s." % (name, key, value))
setattr(n, key, value)
n.last_updated = datetime.datetime.utcnow()
# lock the cluster if needed
if (n.running == True and c.running == False):
log.debug("Trying to lock cluster %s." % c.name)
rs = db.session.query(models.Cluster).filter_by(
name=n.cluster_name,
running=0
).update({
"running": True,
"last_updated": datetime.datetime.utcnow()
})
if (rs):
log.info("Locked cluster %s because node '%s' will run the task." % (c.name, name))
else:
log.debug("Cluster %s has a task running, cannot update node %s." % (c.name, name))
return { 'message': "Cluster task already running. Cannot update." }, 403
# or free the cluster and update node
elif (n.running == False and c.running == True):
rs = db.session.query(models.Cluster).filter_by(
name=n.cluster_name,
running=1
).update({
"running": False,
"last_updated": datetime.datetime.utcnow()
})
if (rs):
log.info("Unlocked cluster %s because node %s completed the task." % (c.name, name))
else:
log.debug("Cluster %s hasn't a task running, cannot update node %s." % (c.name, name))
return { 'message': "Cluster task not running. Cannot update." }, 403
try:
db.session.commit()
log.info("Node %s was updated." % (name))
except IntegrityError:
db.session.rollback()
except Exception as ex:
log.debug(ex)
return { 'message': "Error updating node." }, 500
return { 'message': "Node updated." }, 204
def __init__(self,
name=None,
primary_keys=None,
columns=None,
tokens=None,
storage_id=None,
indexed_args=None,
**kwargs):
"""
Creates a new StorageDict.
Args:
name (string): the name of the collection/table (keyspace is optional)
primary_keys (list(tuple)): a list of (key,type) primary keys (primary + clustering).
columns (list(tuple)): a list of (key,type) columns
tokens (list): list of tokens
storage_id (string): the storage id identifier
indexed_args (list): values that will be used as index
kwargs: other parameters
"""
super(StorageDict, self).__init__(**kwargs)
self._is_persistent = False
log.debug("CREATED StorageDict(%s,%s,%s,%s,%s,%s)", primary_keys,
columns, name, tokens, storage_id, kwargs)
if tokens is None:
log.info('using all tokens')
tokens = map(lambda a: a.value,
config.cluster.metadata.token_map.ring)
self._tokens = IStorage._discrete_token_ranges(tokens)
else:
self._tokens = tokens
self._storage_id = storage_id
if self.__doc__ is not None:
self._persistent_props = self._parse_comments(self.__doc__)
self._primary_keys = self._persistent_props[
self.__class__.__name__]['primary_keys']
self._columns = self._persistent_props[
self.__class__.__name__]['columns']
try:
self._indexed_args = self._persistent_props[
self.__class__.__name__]['indexed_values']
except KeyError:
self._indexed_args = indexed_args
else:
self._primary_keys = primary_keys
self._columns = columns
self._indexed_args = indexed_args
key_names = [pkname for (pkname, dt) in self._primary_keys]
column_names = [colname for (colname, dt) in self._columns]
self._item_builder = namedtuple('row', key_names + column_names)
if len(key_names) > 1:
self._key_builder = namedtuple('row', key_names)
else:
self._key_builder = None
if len(column_names) > 1:
self._column_builder = namedtuple('row', column_names)
else:
self._column_builder = None
self._k_size = len(key_names)
class_name = '%s.%s' % (self.__class__.__module__,
self.__class__.__name__)
self._build_args = self.args(name, self._primary_keys, self._columns,
self._tokens, self._storage_id,
self._indexed_args, class_name)
if name is not None:
self.make_persistent(name)
else:
self._is_persistent = False