def post(self): """ Creates a new node based on data provided into a POST request. The data can be provided in the body as JSON or HTTP form-data. HTTP Data: name (str): The name of the node that will be created cluster_name (str): The cluster that the node will belong """ args = self.reqparse.parse_args() # check if cluster_name actually exists log.debug("Checking if cluster %s exists." % args['cluster_name']) c = models.Cluster.query.get(args['cluster_name']) if c is None: log.debug("Cluster %s doesn't exist." % args['cluster_name']) abort(404, message="Cluster {} doesn't exist.".format(args['cluster_name'])) # check if http auth user has permission to modify auth.check_permission(auth.http_auth.username(), args['cluster_name']) # added if doesn't exist n = models.Node(name=args['name'], cluster_name=args['cluster_name']) db.session.add(n) try: db.session.commit() log.info("Node %s added to cluster %s." % (args['name'], args['cluster_name'])) except IntegrityError: log.debug("Tried to add node %s but it already exists." % args['name']) return { 'message': "Node already exists." }, 409 return { 'nodes': marshal(n, models.node_fields) }, 201, { 'Location': "/node/%s" % args['name'] }
def make_persistent(self, name): """ Once a StorageObj has been created, it can be made persistent. This function retrieves the information about the Object class schema, and creates a Cassandra table with those parameters, where information will be saved from now on, until execution finishes or StorageObj is no longer persistent. It also inserts into the new table all information that was in memory assigned to the StorageObj prior to this call. Args: name (string): name with which the table in the DB will be created """ if self._is_persistent: raise AlreadyPersistentError( "This StorageObj is already persistent [Before:{}.{}][After:{}]", self._ksp, self._table, name) self._is_persistent = True (self._ksp, self._table) = self._extract_ks_tab(name) if self._storage_id is None: self._storage_id = uuid.uuid3(uuid.NAMESPACE_DNS, self._ksp + '.' + self._table) self._build_args = self.args(self._ksp + '.' + self._table, self._tokens, self._storage_id, self._istorage_props, self._class_name) log.info("PERSISTING DATA INTO %s %s", self._ksp, self._table) query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % ( self._ksp, config.replication) config.session.execute(query_keyspace) query_simple = 'CREATE TABLE IF NOT EXISTS ' + self._ksp + '.' + self._table + \ '( storage_id uuid PRIMARY KEY, ' for key, entry in self._persistent_props.items(): query_simple += str(key) + ' ' if entry['type'] != 'dict' and entry[ 'type'] in IStorage._valid_types: if entry['type'] == 'list' or entry['type'] == 'tuple': query_simple += entry['type'] + '<' + entry[ 'columns'] + '>, ' else: query_simple += entry['type'] + ', ' else: query_simple += 'uuid, ' try: config.session.execute(query_simple[:-2] + ' )') except Exception as ir: log.error("Unable to execute %s", query_simple) raise ir for obj_name, obj_info in self._persistent_props.items(): if hasattr(self, obj_name): pd = getattr(self, obj_name) if obj_info['type'] not in IStorage._basic_types: sd_name = self._ksp + "." + self._table + "_" + obj_name pd.make_persistent(sd_name) setattr( self, obj_name, pd ) # super(StorageObj, self).__setattr__(obj_name, pd) why? self._store_meta(self._build_args)
def delete(self, name): """ Deletes an existing node Args: name (str): The name of the node that will be deleted """ log.debug("Checking if node %s exists." % name) n = models.Node.query.get(name) if (n is None): log.debug("Node %s doesn't exist." % name) abort(404, message="Node {} doesn't exist.".format(name)) # check if http auth user has permission to modify auth.check_permission(auth.http_auth.username(), n.cluster_name) db.session.delete(n) try: db.session.commit() log.info("Node %s was deleted." % name) except Exception as ex: log.debug(ex) return { 'message': "Error removing node." }, 500 return { 'message': "Node removed." }
def __init__(self, primary_keys, columns, name, qbeast_meta, qbeast_id=None, entry_point=None, storage_id=None, tokens=None): """ Creates a new block. Args: table_name (string): the name of the collection/table keyspace_name (string): name of the Cassandra keyspace. primary_keys (list(tuple)): a list of (key,type) primary keys (primary + clustering). columns (list(tuple)): a list of (key,type) columns tokens (list): list of tokens storage_id (uuid): the storage id identifier """ log.debug( "CREATED QbeastIterator(%s,%s,%s,%s)", storage_id, tokens, ) self._selects = map(lambda a: a[0], primary_keys + columns) key_namedtuple = namedtuple("key", map(lambda a: a[0], primary_keys)) value_namedtuple = namedtuple("value", map(lambda a: a[0], columns)) div = len(primary_keys) self._row_builder = lambda a: self._row_namedtuple( key_namedtuple(*a[:div]), value_namedtuple(*a[div:])) (self._ksp, self._table) = self._extract_ks_tab(name) self._qbeast_meta = qbeast_meta self._qbeast_id = qbeast_id self._entry_point = entry_point if tokens is None: log.info('using all tokens') tokens = map(lambda a: a.value, config.cluster.metadata.token_map.ring) self._tokens = IStorage._discrete_token_ranges(tokens) else: self._tokens = tokens class_name = '%s.%s' % (self.__class__.__module__, self.__class__.__name__) # primary_keys columns name tokens # indexed_args nonindexed_args value_list # mem_filter port storage_id class_name if storage_id is None: self._storage_id = uuid.uuid4() save = True else: self._storage_id = storage_id save = False self._build_args = self._building_args(primary_keys, columns, name, qbeast_meta, qbeast_id, entry_point, self._storage_id, self._tokens, class_name) if save: self._store_meta(self._build_args)
def __iter__(self): if self._storage_id is None or self._qbeast_id is None: ''' In this case, we are doing a query without splitting the object, and thus we have to initialize the query for only this iterator ''' if type(config.qbeast_entry_node) == list: qbeast_node = config.qbeast_entry_node[0] else: qbeast_node = config.qbeast_entry_node transport = TSocket.TSocket(qbeast_node, config.qbeast_master_port) # Buffering is critical. Raw sockets are very slow transport = TTransport.TFramedTransport(transport) # Wrap in a protocol protocol = TBinaryProtocol.TBinaryProtocol(transport) # Create a client to use the protocol encoder client = QbeastMaster.Client(protocol) # Connect! transport.open() area = FilteringArea(fromPoint=self._qbeast_meta.from_point, toPoint=self._qbeast_meta.to_point) if self._storage_id is None: self._storage_id = uuid.uuid4() self._build_args = self._build_args._replace( storage_id=self._storage_id) self._store_meta(self._build_args) uuids = [str(self._storage_id)] log.info( "calling initQuery (%s, %s, %s, precision=%f ,area=%s, uuids=%s, max_results=%f", self._selects, self._ksp, self._table, self._qbeast_meta.precision, area, uuids, config.qbeast_max_results) self._qbeast_id = client.initQuery( self._selects, self._ksp + '_qbeast', self._table + '_' + self._table + '_idx_d8tree', area, self._qbeast_meta.precision, config.qbeast_max_results, uuids) self._store_meta(self._build_args) transport.close() return IndexedIterValue(self._storage_id, self._entry_point, self._row_builder)
def split(self): """ Initializes the iterator, and saves the information about the token ranges of each block Args: my_dict (PersistentDict): Hecuba PersistentDict """ splits = [s for s in IStorage.split(self)] if type(config.qbeast_entry_node) == list: qbeast_node = config.qbeast_entry_node[0] else: qbeast_node = config.qbeast_entry_node transport = TSocket.TSocket(qbeast_node, config.qbeast_master_port) # Buffering is critical. Raw sockets are very slow transport = TTransport.TFramedTransport(transport) # Wrap in a protocol protocol = TBinaryProtocol.TBinaryProtocol(transport) # Create a client to use the protocol encoder client = QbeastMaster.Client(protocol) # Connect! transport.open() area = FilteringArea(fromPoint=self._qbeast_meta.from_point, toPoint=self._qbeast_meta.to_point) uuids = map(lambda x: str(x._storage_id), splits) log.info( "calling initQuery (%s, %s, %s, precision=%f ,area=%s, uuids=%s, max_results=%f", self._selects, self._ksp, self._table, self._qbeast_meta.precision, area, uuids, config.qbeast_max_results) self._qbeast_id = uuid.UUID( client.initQuery(self._selects, self._ksp + '_qbeast', self._table + '_' + self._table + '_idx_d8tree', area, self._qbeast_meta.precision, config.qbeast_max_results, uuids)) transport.close() for i in splits: i._set_qbeast_id(self._qbeast_id) return iter(splits)
def make_persistent(self, name): if self._is_persistent: raise AlreadyPersistentError( "This StorageNumpy is already persistent [Before:{}.{}][After:{}]", self._ksp, self._table, name) self._is_persistent = True (self._ksp, self._table) = self._extract_ks_tab(name) if self._storage_id is None: self._storage_id = uuid.uuid3( uuid.NAMESPACE_DNS, self._ksp + '.' + self._table + '_numpies') self._build_args = self.args(self._storage_id, self._class_name, name) log.info("PERSISTING DATA INTO %s %s", self._ksp, self._table) query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % ( self._ksp, config.replication) config.session.execute(query_keyspace) config.session.execute( 'CREATE TABLE IF NOT EXISTS ' + self._ksp + '.' + self._table + '_numpies' '(storage_id uuid , ' 'cluster_id int, ' 'block_id int, ' 'payload blob, ' 'PRIMARY KEY((storage_id,cluster_id),block_id))') self._hcache_params = (self._ksp, self._table + '_numpies', self._storage_id, [], ['storage_id', 'cluster_id', 'block_id'], [{ 'name': "payload", 'type': 'numpy' }], { 'cache_size': config.max_cache_size, 'writer_par': config.write_callbacks_number, 'write_buffer': config.write_buffer_size }) self._hcache = Hcache(*self._hcache_params) if len(self.shape) != 0: self._hcache.put_row([self._storage_id, -1, -1], [self]) self._store_meta(self._build_args)
def post(self): """ Creates a new cluster based on data provided into a POST request. The data can be provided in the body as JSON or HTTP form-data. Note: The "hecuba_admin" cluster name is reserved and cannot be used. HTTP Data: name (str): The name of the cluster that will be created secret (Optional|str): Password for managing/updating the cluster """ args = self.reqparse.parse_args() # check if http auth user has permission to modify auth.check_permission(auth.http_auth.username(), "hecuba_admin") # hash secret if needed if (args['secret']): hashed = auth.generate_password(args['secret']) else: hashed = None # hecuba_admin cluster name is reserved for global authentication if (args['name'] == "hecuba_admin"): return { 'message': "Cannot create cluster: name \"hecuba_admin\" reserved." }, 403 # added if doesn't exist c = models.Cluster(name=args['name'], secret=hashed) db.session.add(c) try: db.session.commit() log.info("Created cluster named %s." % args['name']) except IntegrityError: db.session.rollback() log.debug("Tried to create a cluster named %s, but it already exists." % args['name']) return { 'message': "Cluster already exists." }, 409 else: return { 'clusters': marshal(c, models.cluster_fields) }, 201, { 'Location': "/cluster/%s" % args['name'] }
def delete(self, name): """ Deletes an existing cluster Args: name (str): The name of the cluster that will be deleted """ # check if http auth user has permission to modify auth.check_permission(auth.http_auth.username(), "hecuba_admin") log.debug("Checking if cluster %s exists." % name) c = models.Cluster.query.get(name) if (c is None): log.debug("Cluster %s doesn't exist." % name) abort(404, message="Cluster {} doesn't exist.".format(name)) # grab all nodes from current cluster to delete log.info("Deleting all nodes from cluster %s." % name) nodes = c.nodes.all() for n in nodes: db.session.delete(n) try: db.session.commit() log.info("Node %s deleted." % n.name) except Exception as ex: db.session.rollback() log.debug(ex) return { 'message': "Error removing nodes from cluster." }, 500 log.debug("Deleting cluster %s." % name) db.session.delete(c) try: db.session.commit() log.info("Cluster %s was deleted." % name) except Exception as ex: db.session.rollback() log.debug(ex) return { 'message': "Error removing cluster." }, 500 return { 'message': "Cluster removed." }
def put(self, name): """ Updates an existing node based on data provided into a PUT request. Args: name (str): The name of the node that will be updated """ # get node and corresponding cluster log.debug("Checking if node %s exists." % name) n = models.Node.query.get(name) if (n is None): log.debug("Node %s doesn't exist." % name) abort(404, message="Node {} doesn't exist.".format(name)) # check if http auth user has permission to modify auth.check_permission(auth.http_auth.username(), n.cluster_name) c = models.Cluster.query.get(n.cluster_name) # only update node if cluster task is not running if (n.running == False and c.running == True): log.debug("Cluster %s has a task running, cannot update node %s." % (c.name, name)) return { 'message': "Cluster task already running. Cannot update." }, 403 # parse arguments from request args = self.reqparse.parse_args() for key, value in args.iteritems(): if (value != None and value != getattr(n, key)): log.debug("Setting node '%s' attribute '%s' to: %s." % (name, key, value)) setattr(n, key, value) n.last_updated = datetime.datetime.utcnow() # lock the cluster if needed if (n.running == True and c.running == False): log.debug("Trying to lock cluster %s." % c.name) rs = db.session.query(models.Cluster).filter_by( name=n.cluster_name, running=0 ).update({ "running": True, "last_updated": datetime.datetime.utcnow() }) if (rs): log.info("Locked cluster %s because node '%s' will run the task." % (c.name, name)) else: log.debug("Cluster %s has a task running, cannot update node %s." % (c.name, name)) return { 'message': "Cluster task already running. Cannot update." }, 403 # or free the cluster and update node elif (n.running == False and c.running == True): rs = db.session.query(models.Cluster).filter_by( name=n.cluster_name, running=1 ).update({ "running": False, "last_updated": datetime.datetime.utcnow() }) if (rs): log.info("Unlocked cluster %s because node %s completed the task." % (c.name, name)) else: log.debug("Cluster %s hasn't a task running, cannot update node %s." % (c.name, name)) return { 'message': "Cluster task not running. Cannot update." }, 403 try: db.session.commit() log.info("Node %s was updated." % (name)) except IntegrityError: db.session.rollback() except Exception as ex: log.debug(ex) return { 'message': "Error updating node." }, 500 return { 'message': "Node updated." }, 204
def __init__(self, name=None, primary_keys=None, columns=None, tokens=None, storage_id=None, indexed_args=None, **kwargs): """ Creates a new StorageDict. Args: name (string): the name of the collection/table (keyspace is optional) primary_keys (list(tuple)): a list of (key,type) primary keys (primary + clustering). columns (list(tuple)): a list of (key,type) columns tokens (list): list of tokens storage_id (string): the storage id identifier indexed_args (list): values that will be used as index kwargs: other parameters """ super(StorageDict, self).__init__(**kwargs) self._is_persistent = False log.debug("CREATED StorageDict(%s,%s,%s,%s,%s,%s)", primary_keys, columns, name, tokens, storage_id, kwargs) if tokens is None: log.info('using all tokens') tokens = map(lambda a: a.value, config.cluster.metadata.token_map.ring) self._tokens = IStorage._discrete_token_ranges(tokens) else: self._tokens = tokens self._storage_id = storage_id if self.__doc__ is not None: self._persistent_props = self._parse_comments(self.__doc__) self._primary_keys = self._persistent_props[ self.__class__.__name__]['primary_keys'] self._columns = self._persistent_props[ self.__class__.__name__]['columns'] try: self._indexed_args = self._persistent_props[ self.__class__.__name__]['indexed_values'] except KeyError: self._indexed_args = indexed_args else: self._primary_keys = primary_keys self._columns = columns self._indexed_args = indexed_args key_names = [pkname for (pkname, dt) in self._primary_keys] column_names = [colname for (colname, dt) in self._columns] self._item_builder = namedtuple('row', key_names + column_names) if len(key_names) > 1: self._key_builder = namedtuple('row', key_names) else: self._key_builder = None if len(column_names) > 1: self._column_builder = namedtuple('row', column_names) else: self._column_builder = None self._k_size = len(key_names) class_name = '%s.%s' % (self.__class__.__module__, self.__class__.__name__) self._build_args = self.args(name, self._primary_keys, self._columns, self._tokens, self._storage_id, self._indexed_args, class_name) if name is not None: self.make_persistent(name) else: self._is_persistent = False