def getByID(objid): """ We rebuild the object from its id. The id can either be: block: UUID (eg. f291f008-a520-11e6-b42e-5b582e04fd70) storageobj: UUID_(version) (eg. f291f008-a520-11e6-b42e-5b582e04fd70_1) Args: objid (str): object identifier Returns: (Block| Storageobj) """ """ TODO Args: objid (str): object identifier Returns: (Block| Storageobj) """ from hecuba import log from hecuba.IStorage import IStorage try: from hecuba import config query = "SELECT * FROM hecuba.istorage WHERE storage_id = %s" results = config.session.execute(query, [uuid.UUID(objid)])[0] except Exception as e: log.error("Query %s failed", query) raise e log.debug("IStorage API:getByID(%s) of class %s", objid, results.class_name) return IStorage.build_remotely(results._asdict())
def delete_persistent(self): """ Method to empty all data assigned to a StorageDict. """ query = "TRUNCATE TABLE %s.%s;" % (self._ksp, self._table) log.debug('DELETE PERSISTENT: %s', query) config.session.execute(query)
def stop_persistent(self): """ The StorageObj stops being persistent, but keeps the information already stored in Cassandra """ log.debug("STOP PERSISTENT") for persistent_dict in self._persistent_dicts: persistent_dict.stop_persistent()
def __init__(self, primary_keys, columns, name, qbeast_meta, qbeast_id=None, entry_point=None, storage_id=None, tokens=None): """ Creates a new block. Args: table_name (string): the name of the collection/table keyspace_name (string): name of the Cassandra keyspace. primary_keys (list(tuple)): a list of (key,type) primary keys (primary + clustering). columns (list(tuple)): a list of (key,type) columns tokens (list): list of tokens storage_id (uuid): the storage id identifier """ log.debug( "CREATED QbeastIterator(%s,%s,%s,%s)", storage_id, tokens, ) self._selects = map(lambda a: a[0], primary_keys + columns) key_namedtuple = namedtuple("key", map(lambda a: a[0], primary_keys)) value_namedtuple = namedtuple("value", map(lambda a: a[0], columns)) div = len(primary_keys) self._row_builder = lambda a: self._row_namedtuple( key_namedtuple(*a[:div]), value_namedtuple(*a[div:])) (self._ksp, self._table) = self._extract_ks_tab(name) self._qbeast_meta = qbeast_meta self._qbeast_id = qbeast_id self._entry_point = entry_point if tokens is None: log.info('using all tokens') tokens = map(lambda a: a.value, config.cluster.metadata.token_map.ring) self._tokens = IStorage._discrete_token_ranges(tokens) else: self._tokens = tokens class_name = '%s.%s' % (self.__class__.__module__, self.__class__.__name__) # primary_keys columns name tokens # indexed_args nonindexed_args value_list # mem_filter port storage_id class_name if storage_id is None: self._storage_id = uuid.uuid4() save = True else: self._storage_id = storage_id save = False self._build_args = self._building_args(primary_keys, columns, name, qbeast_meta, qbeast_id, entry_point, self._storage_id, self._tokens, class_name) if save: self._store_meta(self._build_args)
def get(self, name, attribute=None): """ Returns a filtered json object with a specific cluster and its attributes when a HTTP GET request is made. Args: name (str): The unique name of the cluster attribute (Optional|str): The name of a unique cluster attribute to show. Defaults to None. """ log.debug("Checking if cluster %s exists." % name) rs = models.Cluster.query.get(name) if (rs is None): log.debug("Cluster %s doesn't exist." % name) abort(404, message="Cluster {} doesn't exist.".format(name)) c = rs.serialize if attribute is None: # grab all nodes from current cluster c['nodes'] = [] log.debug("Getting all nodes for cluster %s." % name) nodes = rs.nodes.all() for n in nodes: c['nodes'].append(n.serialize) return c else: try: log.debug("Returning cluster %s attribute %s with value %s." % (name, attribute, c[attribute])) return { attribute: c[attribute] } except KeyError: log.debug("Client requested attribute %s from cluster %s which is invalid." % (attribute, name)) return { 'message': "Requested attribute is invalid." }, 400
def get(self, name, attribute=None): """ Returns a filtered json object with a specific node and its attributes when a HTTP GET request is made. Args: name (str): The unique name of the node attribute (Optional|str): The name of a unique node attribute to show. Defaults to None. """ log.debug("Checking if node %s exists." % name) n = models.Node.query.get(name) if (n is None): log.debug("Node %s doesn't exist." % name) abort(404, message="Node {} doesn't exist.".format(name)) n = n.serialize if attribute is None: log.debug("Showing node %s." % name) return n else: try: log.debug("Returning node %s attribute %s with value %s." % (name, attribute, n[attribute])) return { attribute: n[attribute] } except KeyError: log.debug("Client requested attribute %s from node %s which is invalid." % (attribute, name)) return { 'message': "Requested attribute is invalid." }, 400
def stop_persistent(self): """ Method to turn a StorageDict into non-persistent. """ log.debug('STOP PERSISTENCE: %s', self._table) self._is_persistent = False self._hcache = None
def split(self): ''' config.partition_strategy == "DYNAMIC" Data will be partitioned in config.number_of_partitions different chunks :return: an iterator over partitions ''' st = time.time() tokens = self._father._build_args.tokens for token_split in self._tokens_partitions( tokens, config.min_number_of_tokens): storage_id = uuid.uuid4() log.debug('assigning to %s %d tokens', str(storage_id), len(token_split)) new_args = self._father._build_args._replace(tokens=token_split, storage_id=storage_id) partitioned_object = self._father.__class__.build_remotely( new_args._asdict()) config.session.execute(self._prepared_store_id, [ self._partitioning_uuid, partitioned_object._storage_id, config.number_of_partitions ]) self._idle_nodes -= 1 yield partitioned_object log.debug('completed split of %s in %f', self._father.__class__.__name__, time.time() - st)
def error_handler(): """ Returns HTTP Forbidden when a authentication error happens. Also used as a callback to Flask-HTTPAuth not authenticating the user. """ log.debug("Access was denied while requesting a view.") abort(403, message="Access denied.")
def __setitem__(self, key, val): """ Method to insert values in the StorageDict Args: key: the position of the value that we want to save val: the value that we want to save in that position """ if isinstance(val, np.ndarray): val = StorageNumpy(val) log.debug('SET ITEM %s->%s', key, val) if not config.hecuba_type_checking: if not self._is_persistent: dict.__setitem__(self, key, val) else: if isinstance(val, IStorage) and not val._is_persistent: attribute = val.__class__.__name__.lower() count = self._count_name_collision(attribute) # new name as ksp+table+obj_class_name val.make_persistent(self._ksp + '.' + self._table + "_" + attribute + "_" + str(count)) self._hcache.put_row(self._make_key(key), self._make_value(val)) else: if isinstance(val, Iterable) and not isinstance(val, str): col_types = map( lambda x: IStorage._conversions[x.__class__.__name__], val) spec_col_types = map(lambda x: x[1], self._columns) for idx, value in enumerate(spec_col_types): if value == 'double': spec_col_types[idx] = 'float' else: col_types = IStorage._conversions[val.__class__.__name__] spec_col_types = map(lambda x: x[1], self._columns)[0] if spec_col_types == 'double': spec_col_types = 'float' if isinstance(key, Iterable) and not isinstance(key, str): key_types = map( lambda x: IStorage._conversions[x.__class__.__name__], key) spec_key_types = map(lambda x: x[1], self._primary_keys) for idx, value in enumerate(spec_key_types): if value == 'double': spec_key_types[idx] = 'float' else: key_types = IStorage._conversions[key.__class__.__name__] spec_key_types = map(lambda x: x[1], self._primary_keys)[0] if spec_key_types == 'double': spec_key_types = 'float' if (col_types == spec_col_types): if (key_types == spec_key_types): if not self._is_persistent: dict.__setitem__(self, key, val) else: self._hcache.put_row(self._make_key(key), self._make_value(val)) else: raise KeyError else: raise ValueError
def build_remotely(result): """ Launches the StorageDict.__init__ from the api.getByID Args: result: a namedtuple with all the information needed to create again the StorageDict """ log.debug("Building Storage dict with %s", result) return StorageDict(result.name, result.primary_keys, result.columns, result.tokens, result.storage_id, result.indexed_on)
def build_remotely(new_args): """ Launches the StorageNumpy.__init__ from the uuid api.getByID Args: new_args: a list of all information needed to create again the StorageNumpy Returns: so: the created StorageNumpy """ log.debug("Building StorageNumpy object with %s", new_args) return StorageNumpy(new_args.storage_id)
def delete_persistent(self): """ Deletes the Cassandra table where the persistent StorageObj stores data """ self._is_persistent = False query = "DELETE FROM %s.%s WHERE storage_id = %s;" % ( self._ksp, self._table + '_numpies', self._storage_id) log.debug("DELETE PERSISTENT: %s", query) config.session.execute(query)
def get(self): """ Returns a filtered json object with the all nodes and their attributes when a HTTP GET request is made. """ log.debug("Getting nodes collection information.") nodes = [] for n in models.Node.query.all(): nodes.append(n.serialize) return { 'nodes': marshal(nodes, models.node_fields) }
def get(self): """ Returns a filtered json object with the all clusters and their attributes when a HTTP GET request is made. """ log.debug("Getting clusters collection information.") clusters = [] for c in models.Cluster.query.all(): clusters.append(c.serialize) return { 'clusters': marshal(clusters, models.cluster_fields) }
def build_remotely(result): """ Launches the Block.__init__ from the api.getByID Args: result: a namedtuple with all the information needed to create again the block """ log.debug("Building Storage dict with %s", result) return QbeastIterator(result.primary_keys, result.columns, result.name, result.qbeast_meta, result.qbeast_id, result.entry_point, result.storage_id, result.tokens)
def generate_password(password): """ Generates a bcrypt hash using "BCRYPT_ROUNDS" rounds from configuration Args: password (str): The password that will be hashed Returns: str: The hashed password """ log.debug("Generating password with %s bcrypt rounds" % hecuba.config['BCRYPT_ROUNDS']) return bcrypt.hashpw(password, bcrypt.gensalt(hecuba.config['BCRYPT_ROUNDS']))
def __setattr__(self, attribute, value): """ Given a key and its value, this function saves it (depending on if it's persistent or not): a) In memory b) In the DB Args: attribute: name of the value that we want to set value: value that we want to save """ if attribute[0] is '_': object.__setattr__(self, attribute, value) return if attribute in self._persistent_attrs: if config.hecuba_type_checking and value is not None and not isinstance(value, dict) and \ IStorage._conversions[value.__class__.__name__] != self._persistent_props[attribute][ 'type']: raise TypeError if not isinstance(value, IStorage): if isinstance(value, np.ndarray): value = StorageNumpy(value) elif isinstance(value, dict): per_dict = self._persistent_props[attribute] indexed_args = per_dict.get('indexed_values', None) new_value = StorageDict(None, per_dict['primary_keys'], per_dict['columns'], tokens=self._tokens, indexed_args=indexed_args) new_value.update(value) value = new_value if self._is_persistent: if issubclass(value.__class__, IStorage): if not value._is_persistent: count = self._count_name_collision(attribute) value.make_persistent(self._ksp + '.' + self._table + '_' + attribute + '_' + str(count)) values = [self._storage_id, value._storage_id] else: values = [self._storage_id, value] query = "INSERT INTO %s.%s (storage_id,%s)" % ( self._ksp, self._table, attribute) query += " VALUES (%s,%s)" log.debug("SETATTR: ", query) config.session.execute(query, values) object.__setattr__(self, attribute, value)
def _store_meta(storage_args): log.debug("QbeastIterator: storing metas %s", '') try: config.session.execute(QbeastIterator._prepared_store_meta, [ storage_args.primary_keys, storage_args.columns, storage_args.name, storage_args.qbeast_meta, storage_args.qbeast_id, storage_args.entry_point, storage_args.storage_id, storage_args.tokens, storage_args.class_name ]) except Exception as ex: log.error("Error creating the StorageDictIx metadata: %s %s", storage_args, ex) raise ex
def _store_meta(storage_args): """ Saves the information of the object in the istorage table. Args:. storage_args (object): contains all data needed to restore the object from the workers """ log.debug("StorageObj: storing media %s", storage_args) try: config.session.execute(StorageNumpy._prepared_store_meta, [ storage_args.storage_id, storage_args.class_name, storage_args.name ]) except Exception as ex: log.warn("Error creating the StorageNumpy metadata with args: %s" % str(storage_args)) raise ex
def delete_persistent(self): """ Deletes the Cassandra table where the persistent StorageObj stores data """ self._is_persistent = False if hasattr(self, '_persistent_dicts'): for pers_dict in self._persistent_dicts: pers_dict.delete_persistent() if hasattr(self, '_storage_objs'): for so in self._storage_objs: so.delete_persistent() query = "TRUNCATE TABLE %s.%s;" % (self._ksp, self._table) log.debug("DELETE PERSISTENT: %s", query) config.session.execute(query)
def _store_meta(storage_args): """ Method to update the info about the StorageDict in the DB metadata table Args: storage_args: structure with all data needed to update the metadata """ log.debug("StorageDict: storing metas %s", storage_args) try: config.session.execute(StorageDict._prepared_store_meta, [ storage_args.storage_id, storage_args.class_name, storage_args.name, storage_args.tokens, storage_args.primary_keys, storage_args.columns, storage_args.indexed_on ]) except Exception as ex: log.error("Error creating the StorageDict metadata: %s %s", storage_args, ex) raise ex
def split(self): ''' config.partition_strategy == "SIMPLE" Data will be partitioned in config.number_of_partitions different chunks :return: an iterator over partitions ''' st = time.time() tokens = self._father._build_args.tokens for token_split in self._tokens_partitions( tokens, config.min_number_of_tokens, config.number_of_partitions): storage_id = uuid.uuid4() log.debug('assigning to %s %d tokens', str(storage_id), len(token_split)) new_args = self._father._build_args._replace(tokens=token_split, storage_id=storage_id) yield self._father.__class__.build_remotely(new_args._asdict()) log.debug('completed split of %s in %f', self._father.__class__.__name__, time.time() - st)
def delete(self, name): """ Deletes an existing cluster Args: name (str): The name of the cluster that will be deleted """ # check if http auth user has permission to modify auth.check_permission(auth.http_auth.username(), "hecuba_admin") log.debug("Checking if cluster %s exists." % name) c = models.Cluster.query.get(name) if (c is None): log.debug("Cluster %s doesn't exist." % name) abort(404, message="Cluster {} doesn't exist.".format(name)) # grab all nodes from current cluster to delete log.info("Deleting all nodes from cluster %s." % name) nodes = c.nodes.all() for n in nodes: db.session.delete(n) try: db.session.commit() log.info("Node %s deleted." % n.name) except Exception as ex: db.session.rollback() log.debug(ex) return { 'message': "Error removing nodes from cluster." }, 500 log.debug("Deleting cluster %s." % name) db.session.delete(c) try: db.session.commit() log.info("Cluster %s was deleted." % name) except Exception as ex: db.session.rollback() log.debug(ex) return { 'message': "Error removing cluster." }, 500 return { 'message': "Cluster removed." }
def post(self): """ Creates a new cluster based on data provided into a POST request. The data can be provided in the body as JSON or HTTP form-data. Note: The "hecuba_admin" cluster name is reserved and cannot be used. HTTP Data: name (str): The name of the cluster that will be created secret (Optional|str): Password for managing/updating the cluster """ args = self.reqparse.parse_args() # check if http auth user has permission to modify auth.check_permission(auth.http_auth.username(), "hecuba_admin") # hash secret if needed if (args['secret']): hashed = auth.generate_password(args['secret']) else: hashed = None # hecuba_admin cluster name is reserved for global authentication if (args['name'] == "hecuba_admin"): return { 'message': "Cannot create cluster: name \"hecuba_admin\" reserved." }, 403 # added if doesn't exist c = models.Cluster(name=args['name'], secret=hashed) db.session.add(c) try: db.session.commit() log.info("Created cluster named %s." % args['name']) except IntegrityError: db.session.rollback() log.debug("Tried to create a cluster named %s, but it already exists." % args['name']) return { 'message': "Cluster already exists." }, 409 else: return { 'clusters': marshal(c, models.cluster_fields) }, 201, { 'Location': "/cluster/%s" % args['name'] }
def delete(self, name): """ Deletes an existing node Args: name (str): The name of the node that will be deleted """ log.debug("Checking if node %s exists." % name) n = models.Node.query.get(name) if (n is None): log.debug("Node %s doesn't exist." % name) abort(404, message="Node {} doesn't exist.".format(name)) # check if http auth user has permission to modify auth.check_permission(auth.http_auth.username(), n.cluster_name) db.session.delete(n) try: db.session.commit() log.info("Node %s was deleted." % name) except Exception as ex: log.debug(ex) return { 'message': "Error removing node." }, 500 return { 'message': "Node removed." }
def getByID(objid): """ We rebuild the object from its id. The id can either be: block: UUID (eg. f291f008-a520-11e6-b42e-5b582e04fd70) storageobj: UUID_(version) (eg. f291f008-a520-11e6-b42e-5b582e04fd70_1) Args: objid (str): object identifier Returns: (Block| Storageobj) """ """ TODO Args: objid (str): object identifier Returns: (Block| Storageobj) """ from hecuba import log try: from hecuba import config query = "SELECT * FROM hecuba.istorage WHERE storage_id = %s" results = config.session.execute(query, [uuid.UUID(objid)])[0] except Exception as e: log.error("Query %s failed", query) raise e class_name = results.class_name log.debug("IStorage API:getByID(%s) of class %s", objid, class_name) last = 0 for key, i in enumerate(class_name): if i == '.' and key > last: last = key module = class_name[:last] cname = class_name[last + 1:] mod = __import__(module, globals(), locals(), [cname], 0) b = getattr(mod, cname).build_remotely(results) return b
def post(self): """ Creates a new node based on data provided into a POST request. The data can be provided in the body as JSON or HTTP form-data. HTTP Data: name (str): The name of the node that will be created cluster_name (str): The cluster that the node will belong """ args = self.reqparse.parse_args() # check if cluster_name actually exists log.debug("Checking if cluster %s exists." % args['cluster_name']) c = models.Cluster.query.get(args['cluster_name']) if c is None: log.debug("Cluster %s doesn't exist." % args['cluster_name']) abort(404, message="Cluster {} doesn't exist.".format(args['cluster_name'])) # check if http auth user has permission to modify auth.check_permission(auth.http_auth.username(), args['cluster_name']) # added if doesn't exist n = models.Node(name=args['name'], cluster_name=args['cluster_name']) db.session.add(n) try: db.session.commit() log.info("Node %s added to cluster %s." % (args['name'], args['cluster_name'])) except IntegrityError: log.debug("Tried to add node %s but it already exists." % args['name']) return { 'message': "Node already exists." }, 409 return { 'nodes': marshal(n, models.node_fields) }, 201, { 'Location': "/node/%s" % args['name'] }
def split(self): """ Method used to divide an object into sub-objects. Returns: a subobject everytime is called """ st = time() tokens = self._build_args.tokens for token_split in IStorage._tokens_partitions( tokens, config.min_number_of_tokens, config.number_of_partitions): storage_id = uuid.uuid4() log.debug('assigning to %s %d tokens', str(storage_id), len(token_split)) new_args = self._build_args._replace(tokens=token_split, storage_id=storage_id) self.__class__._store_meta(new_args) yield self.__class__.build_remotely(new_args) log.debug('completed split of %s in %f', self.__class__.__name__, time() - st)
def getByID(objid): """ We rebuild the object from its id. Args: objid (str): object identifier Returns: Hecuba Object """ """ TODO Args: objid (str): object identifier Returns: (Block| Storageobj) """ from hecuba import log from hecuba.IStorage import build_remotely from hecuba import config from hecuba import StorageNumpy, StorageDict from hecuba import StorageObj as StorageObject import uuid query = "SELECT * FROM hecuba.istorage WHERE storage_id = %s" if isinstance(objid, str): objid = uuid.UUID(objid) results = config.session.execute(query, [objid]) if not results: raise RuntimeError("Object {} not found on hecuba.istorage".format(objid)) results = results[0] log.debug("IStorage API:getByID(%s) of class %s", objid, results.class_name) return build_remotely(results._asdict())
def build_remotely(new_args): """ Launches the StorageObj.__init__ from the uuid api.getByID Args: new_args: a list of all information needed to create again the storageobj Returns: so: the created storageobj """ log.debug("Building Storage object with %s", new_args) class_name = new_args.class_name if class_name is 'StorageObj': so = StorageObj(new_args.name.encode('utf8'), new_args.tokens, new_args.storage_id, new_args.istorage_props) else: class_name, mod_name = IStorage.process_path(class_name) mod = __import__(mod_name, globals(), locals(), [class_name], 0) so = getattr(mod, class_name)(new_args.name.encode('utf8'), new_args.tokens, new_args.storage_id, new_args.istorage_props) return so
def __getattr__(self, attribute): """ Given an attribute, this function returns the value, obtaining it from either: a) memory b) the Database Args: attribute: name of the value that we want to obtain Returns: value: obtained value """ if attribute[ 0] != '_' and self._is_persistent and attribute in self._persistent_attrs: try: query = "SELECT %s FROM %s.%s WHERE storage_id = %s;" \ % (attribute, self._ksp, self._table, self._storage_id) log.debug("GETATTR: %s", query) result = config.session.execute(query) for row in result: for row_key, row_var in vars(row).iteritems(): if row_var is not None: if isinstance(row_var, list) and isinstance( row_var[0], unicode): new_toreturn = [] for entry in row_var: new_toreturn.append(str(entry)) return new_toreturn else: return row_var else: raise AttributeError except Exception as ex: log.warn("GETATTR ex %s", ex) raise AttributeError('value not found') else: return object.__getattribute__(self, attribute)
def __getitem__(self, key): """ If the object is persistent, each request goes to the hfetch. Args: key: the dictionary key Returns item: value found in position key """ log.debug('GET ITEM %s', key) if not self._is_persistent: to_return = dict.__getitem__(self, key) return to_return else: cres = self._hcache.get_row(self._make_key(key)) log.debug("GET ITEM %s[%s]", cres, cres.__class__) final_results = [] for index, (name, col_type) in enumerate(self._columns): if col_type not in IStorage._basic_types: table_name = self._ksp + '.' + self._table element = (self._build_istorage_obj( col_type, table_name, uuid.UUID(cres[index]))) else: element = cres[index] final_results.append(element) cres = final_results if issubclass(cres.__class__, NoneType): return None elif self._column_builder is not None: if len(cres) > 0 and isinstance(cres[0], list): return [self._column_builder(*row) for row in cres] else: return self._column_builder(*cres) else: return cres[0]
def check_permission(auth_user, cluster): """ Checks if an authenticated user has access to a cluster Args: auth_user (str): The HTTP-Auth username provided cluster (str): The name of the cluster Returns: mixed: True if user has permission. Call error_handler if user doesn't have permission. """ # operations that don't need cluster secret if (cluster == "hecuba_admin"): if (hecuba.config['ADMIN_SECRET'] and auth_user == "hecuba_admin"): log.debug("Action access granted using hecuba_admin") return True elif (not hecuba.config['ADMIN_SECRET']): return True else: log.debug("Action access denied using hecuba_admin") error_handler() # operations that uses cluster secret c = models.Cluster.query.get(cluster) if (c is not None): if (c.secret is not None): if (hecuba.config['ADMIN_SECRET'] and auth_user == "hecuba_admin"): log.debug("Access on cluster %s granted using hecuba_admin" % cluster) return True elif (auth_user != cluster): log.debug("Access denied to username \"%s\" on cluster %s" % (auth_user, cluster)) error_handler() return True else: error_handler()
def verify_password(cluster, password, ws=True): """ Checks if password is valid for a cluster or whole program. Also used as a callback to Flask-HTTPAuth verify the password. Args: cluster (str): The cluster name for checking its password. If "hecuba_admin" is provided, assume the password is for the global scope (whole program, configured with the ADMIN_SECRET variable on the configuration file). password (str): The password to be checked. It will be hashed and compared to the stored one. ws (bool): If true, this means the method is called on a web server and it will return an error_handler(). If false, it will simple return false (for using it at the manager cli). Defaults to true. """ # specific cluster or free authentication if (cluster != "hecuba_admin" and cluster != ""): log.debug("Verifying credentials for cluster %s" % cluster) c = models.Cluster.query.get(cluster) if (c is None): # cluster doesn't exist if (ws == True): error_handler() else: return False if (c.secret): return bcrypt.hashpw(password, c.secret) == c.secret else: return password == "None" else: if hecuba.config['ADMIN_SECRET'] and cluster == "hecuba_admin": log.debug("Verifying credentials for hecuba_admin") hashed = hecuba.config['ADMIN_SECRET'] return bcrypt.hashpw(password, hashed) == hashed else: log.debug("Admin secret disabled. Request can be served.") return True
def __init__(self, name=None, tokens=None, storage_id=None, istorage_props=None, **kwargs): """ Creates a new storageobj. Args: name (string): the name of the Cassandra Keyspace + table where information can be found tokens (list of tuples): token ranges assigned to the new StorageObj storage_id (string): an unique storageobj identifier istorage_props dict(string,string): a map with the storage id of each contained istorage object. kwargs: more optional parameters """ log.debug("CREATED StorageObj(%s)", name) self._is_persistent = False self._persistent_dicts = [] self._storage_objs = [] if name is None: self._ksp = config.execution_name else: (self._ksp, self._table) = self._extract_ks_tab(name) self._persistent_props = StorageObj._parse_comments(self.__doc__) self._persistent_attrs = self._persistent_props.keys() if tokens is None: # log.info('using all tokens') tokens = map(lambda a: a.value, config.cluster.metadata.token_map.ring) self._tokens = IStorage._discrete_token_ranges(tokens) else: self._tokens = tokens self._storage_id = storage_id self._istorage_props = istorage_props self._class_name = '%s.%s' % (self.__class__.__module__, self.__class__.__name__) if name is not None: self._build_args = self.args(self._ksp + '.' + self._table, self._tokens, self._storage_id, self._istorage_props, self._class_name) dictionaries = filter(lambda (k, t): t['type'] == 'StorageDict', self._persistent_props.iteritems()) for table_name, per_dict in dictionaries: dict_name = "%s.%s" % (self._ksp, table_name) if istorage_props is not None and dict_name in istorage_props: args = config.session.execute(IStorage._select_istorage_meta, (istorage_props[dict_name], ))[0] # The internal objects must have the same father's tokens args = args._replace(tokens=self._tokens) log.debug("CREATING INTERNAL StorageDict with %s", args) pd = StorageDict.build_remotely(args) else: if 'indexed_values' in per_dict: indexed_args = per_dict['indexed_values'] else: indexed_args = None pd = StorageDict(None, per_dict['primary_keys'], per_dict['columns'], tokens=self._tokens, indexed_args=indexed_args) setattr(self, table_name, pd) self._persistent_dicts.append(pd) storageobjs = filter( lambda (k, t): t['type'] not in IStorage._basic_types and t['type'] != 'StorageDict', self._persistent_props.iteritems()) for table_name, per_dict in storageobjs: so_name = "%s.%s" % (self._ksp, table_name) cname, module = IStorage.process_path(per_dict['type']) mod = __import__(module, globals(), locals(), [cname], 0) so = getattr(mod, cname)() setattr(self, table_name, so) if name is not None: self.make_persistent(name)
def __init__(self, name=None, primary_keys=None, columns=None, tokens=None, storage_id=None, indexed_args=None, **kwargs): """ Creates a new StorageDict. Args: name (string): the name of the collection/table (keyspace is optional) primary_keys (list(tuple)): a list of (key,type) primary keys (primary + clustering). columns (list(tuple)): a list of (key,type) columns tokens (list): list of tokens storage_id (string): the storage id identifier indexed_args (list): values that will be used as index kwargs: other parameters """ super(StorageDict, self).__init__(**kwargs) self._is_persistent = False log.debug("CREATED StorageDict(%s,%s,%s,%s,%s,%s)", primary_keys, columns, name, tokens, storage_id, kwargs) if tokens is None: log.info('using all tokens') tokens = map(lambda a: a.value, config.cluster.metadata.token_map.ring) self._tokens = IStorage._discrete_token_ranges(tokens) else: self._tokens = tokens self._storage_id = storage_id if self.__doc__ is not None: self._persistent_props = self._parse_comments(self.__doc__) self._primary_keys = self._persistent_props[ self.__class__.__name__]['primary_keys'] self._columns = self._persistent_props[ self.__class__.__name__]['columns'] try: self._indexed_args = self._persistent_props[ self.__class__.__name__]['indexed_values'] except KeyError: self._indexed_args = indexed_args else: self._primary_keys = primary_keys self._columns = columns self._indexed_args = indexed_args key_names = [pkname for (pkname, dt) in self._primary_keys] column_names = [colname for (colname, dt) in self._columns] self._item_builder = namedtuple('row', key_names + column_names) if len(key_names) > 1: self._key_builder = namedtuple('row', key_names) else: self._key_builder = None if len(column_names) > 1: self._column_builder = namedtuple('row', column_names) else: self._column_builder = None self._k_size = len(key_names) class_name = '%s.%s' % (self.__class__.__module__, self.__class__.__name__) self._build_args = self.args(name, self._primary_keys, self._columns, self._tokens, self._storage_id, self._indexed_args, class_name) if name is not None: self.make_persistent(name) else: self._is_persistent = False
def make_persistent(self, name): """ Method to transform a StorageDict into a persistent object. This will make it use a persistent DB as the main location of its data. Args: name: """ if self._is_persistent: raise AlreadyPersistentError( "This StorageDict is already persistent [Before:{}.{}][After:{}]", self._ksp, self._table, name) self._is_persistent = True (self._ksp, self._table) = self._extract_ks_tab(name) if self._storage_id is None: self._storage_id = uuid.uuid3(uuid.NAMESPACE_DNS, self._ksp + '.' + self._table) self._build_args = self._build_args._replace( storage_id=self._storage_id, name=self._ksp + "." + self._table) self._store_meta(self._build_args) if config.id_create_schema == -1: query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % ( self._ksp, config.replication) try: log.debug('MAKE PERSISTENCE: %s', query_keyspace) config.session.execute(query_keyspace) except Exception as ex: log.warn("Error creating the StorageDict keyspace %s, %s", (query_keyspace), ex) raise ex for key, value in dict.iteritems(self): if issubclass(value.__class__, IStorage): # new name as ksp+table+obj_class_name val_name = self._ksp + '.' + self._table + type( value).__name__.lower() value.make_persistent(val_name) columns = self._primary_keys + self._columns for ind, entry in enumerate(columns): n = StorageDict._other_case.match(entry[1]) if n is not None: iter_type, intra_type = n.groups() else: iter_type = entry[1] if iter_type not in IStorage._basic_types: columns[ind] = entry[0], 'uuid' pks = map(lambda a: a[0], self._primary_keys) query_table = "CREATE TABLE IF NOT EXISTS %s.%s (%s, PRIMARY KEY (%s));" \ % (self._ksp, self._table, ",".join("%s %s" % tup for tup in columns), str.join(',', pks)) try: log.debug('MAKE PERSISTENCE: %s', query_table) config.session.execute(query_table) except Exception as ex: log.warn("Error creating the StorageDict table: %s %s", query_table, ex) raise ex key_names = map(lambda a: a[0].encode('UTF8'), self._primary_keys) column_names = self._columns self._hcache_params = (self._ksp, self._table, self._storage_id, self._tokens, key_names, map(lambda x: { "name": x[0], "type": x[1] }, column_names), { 'cache_size': config.max_cache_size, 'writer_par': config.write_callbacks_number, 'write_buffer': config.write_buffer_size }) log.debug("HCACHE params %s", self._hcache_params) self._hcache = Hcache(*self._hcache_params) # Storing all in-memory values to cassandra for key, value in dict.iteritems(self): self._hcache.put_row(self._make_key(key), self._make_value(value)) if hasattr(self, '_indexed_args') and self._indexed_args is not None: index_query = 'CREATE CUSTOM INDEX IF NOT EXISTS ' + self._table + '_idx ON ' index_query += self._ksp + '.' + self._table + ' (' + str.join( ',', self._indexed_args) + ') ' index_query += "using 'es.bsc.qbeast.index.QbeastIndex';" try: config.session.execute(index_query) except Exception as ex: log.error("Error creating the Qbeast custom index: %s %s", index_query, ex) raise ex
def put(self, name): """ Updates an existing node based on data provided into a PUT request. Args: name (str): The name of the node that will be updated """ # get node and corresponding cluster log.debug("Checking if node %s exists." % name) n = models.Node.query.get(name) if (n is None): log.debug("Node %s doesn't exist." % name) abort(404, message="Node {} doesn't exist.".format(name)) # check if http auth user has permission to modify auth.check_permission(auth.http_auth.username(), n.cluster_name) c = models.Cluster.query.get(n.cluster_name) # only update node if cluster task is not running if (n.running == False and c.running == True): log.debug("Cluster %s has a task running, cannot update node %s." % (c.name, name)) return { 'message': "Cluster task already running. Cannot update." }, 403 # parse arguments from request args = self.reqparse.parse_args() for key, value in args.iteritems(): if (value != None and value != getattr(n, key)): log.debug("Setting node '%s' attribute '%s' to: %s." % (name, key, value)) setattr(n, key, value) n.last_updated = datetime.datetime.utcnow() # lock the cluster if needed if (n.running == True and c.running == False): log.debug("Trying to lock cluster %s." % c.name) rs = db.session.query(models.Cluster).filter_by( name=n.cluster_name, running=0 ).update({ "running": True, "last_updated": datetime.datetime.utcnow() }) if (rs): log.info("Locked cluster %s because node '%s' will run the task." % (c.name, name)) else: log.debug("Cluster %s has a task running, cannot update node %s." % (c.name, name)) return { 'message': "Cluster task already running. Cannot update." }, 403 # or free the cluster and update node elif (n.running == False and c.running == True): rs = db.session.query(models.Cluster).filter_by( name=n.cluster_name, running=1 ).update({ "running": False, "last_updated": datetime.datetime.utcnow() }) if (rs): log.info("Unlocked cluster %s because node %s completed the task." % (c.name, name)) else: log.debug("Cluster %s hasn't a task running, cannot update node %s." % (c.name, name)) return { 'message': "Cluster task not running. Cannot update." }, 403 try: db.session.commit() log.info("Node %s was updated." % (name)) except IntegrityError: db.session.rollback() except Exception as ex: log.debug(ex) return { 'message': "Error updating node." }, 500 return { 'message': "Node updated." }, 204
def get(self): """ Checks internal commands and database and report if OK or not when a HTTP GET request is made. """ log.debug("Performing a health-check.") log.debug("Health-check: Testing version view") try: v = Version_Handler() v = v.get() except: log.debug("Health-check: Error testing version view") return { 'message': 'Error testing version view.' }, 500 log.debug("Health-check: Testing models and database") try: c = models.Cluster.query.first() n = models.Node.query.first() except: log.debug("Health-check: Error testing models and database") return { 'message': 'Error testing models and database' }, 500 log.debug("Health-check passed.") return { 'message': 'All checks passed.' }
""" hecuba.auth ~~~~~~~~~~~~ Authentication classes, functions and decorators :copyright: (c) 2016 by Hugo Cisneiros. :license: GPLv2, see LICENSE for more details. """ from hecuba import hecuba, db, models, log from flask.ext.httpauth import HTTPBasicAuth from flask_restful import abort import bcrypt # setup http basic auth log.debug("Setting up HTTP Basic Auth capability") http_auth = HTTPBasicAuth() @http_auth.error_handler def error_handler(): """ Returns HTTP Forbidden when a authentication error happens. Also used as a callback to Flask-HTTPAuth not authenticating the user. """ log.debug("Access was denied while requesting a view.") abort(403, message="Access denied.") @http_auth.verify_password def verify_password(cluster, password, ws=True): """