def update(self, key, data): """Update an entity. Update an existing entity with the given key and the given data. key (str): the key of the entity that has to be updated data (str): the new properties of the entity (a dict encoded in JSON) Raise InvalidKey if key isn't a str or if no entity with that key is present in the store. Raise InvalidData if data cannot be parsed, if it's missing some properties or if properties are of the wrong type. """ self._verify_key(key, must_be_present=True) # update entity try: item = self._entity() item.set(json.loads(data)) if not item.consistent(): raise InvalidData('Inconsistent data') item.key = key self._store[key] = item except ValueError: raise InvalidData('Invalid JSON') # notify callbacks for callback in self._update_callbacks: callback(key) # reflect changes on the persistent storage try: with open(os.path.join(self._path, key + '.json'), 'w') as rec: rec.write(json.dumps(self._store[key].dump())) except IOError: logger.error("IOError occured", exc_info=True)
def delete(self, key): """Delete an entity. Delete an existing entity from the store. key (unicode): the key of the entity that has to be deleted raise (InvalidKey): if key isn't a unicode or if no entity with that key is present in the store. """ self._verify_key(key, must_be_present=True) with LOCK: # delete entity old_value = self._store[key] del self._store[key] # enforce consistency for depend in self._depends: for o_key, o_value in list(depend._store.iteritems()): if not o_value.consistent(): depend.delete(o_key) # notify callbacks for callback in self._delete_callbacks: callback(key, old_value) # reflect changes on the persistent storage try: os.remove(os.path.join(self._path, key + '.json')) except OSError: logger.error("Unable to delete entity", exc_info=True)
def create(self, key, data): """Create a new entity. Create a new entity with the given key and the given data. key (str): the key with which the entity will be later accessed data (dict): the properties of the entity Raise InvalidKey if key isn't a str or if an entity with the same key is already present in the store. Raise InvalidData if data cannot be parsed, if it's missing some properties or if properties are of the wrong type. """ self._verify_key(key) # create entity try: item = self._entity() item.set(json.loads(data)) item.key = key self._store[key] = item except ValueError: raise InvalidData("Invalid JSON") # notify callbacks for callback in self._create_callbacks: callback(key) # reflect changes on the persistent storage try: with open(os.path.join(self._path, key + ".json"), "w") as rec: rec.write(json.dumps(self._store[key].dump())) except IOError: logger.error("IOError occured", exc_info=True)
def delete(self, key, confirm=None): """Delete an entity. Delete an existing entity from the store. key (str): the key of the entity that has to be deleted confirm (callable): action to be performed as soon as we're sure that the action won't fail (in particular, before notifying the callbacks). Raise InvalidKey if key isn't a str or if no entity with that key is present in the store. """ self._verify_key(key, must_be_present=True) # confirm the operation if confirm is not None: confirm() # delete entity old_value = self._store[key] del self._store[key] # enforce consistency for depend in self._depends: for o_key, o_value in list(depend.store._store.iteritems()): if not o_value.consistent(): depend.store.delete(o_key) # notify callbacks for callback in self._delete_callbacks: callback(key, old_value) # reflect changes on the persistent storage try: os.remove(os.path.join(self._path, key + '.json')) except OSError: logger.error("Unable to delete entity", exc_info=True)
def merge_list(self, data_dict): """Merge a list of entities. Take a dictionary of entites and, for each of them: - if it's not present in the store, create it - if it's present, update it data_dict (dict): the dictionary of entities raise (InvalidData) if data cannot be parsed, if an entity is missing some properties or if properties are of the wrong type. """ with LOCK: if not isinstance(data_dict, dict): raise InvalidData("Not a dictionary") item_dict = dict() for key, value in data_dict.iteritems(): try: # FIXME We should allow keys to be arbitrary unicode # strings, so this just needs to be a non-empty check. if not re.match('[A-Za-z0-9_]+', key): raise InvalidData("Invalid key") item = self._entity() item.set(value) if not item.consistent(): raise InvalidData("Inconsistent data") item.key = key item_dict[key] = item except InvalidData as exc: exc.message = "[entity %s] %s" % (key, exc) exc.args = exc.message, raise exc for key, value in item_dict.iteritems(): is_new = key not in self._store old_value = self._store.get(key) # insert entity self._store[key] = value # notify callbacks if is_new: for callback in self._create_callbacks: callback(key, value) else: for callback in self._update_callbacks: callback(key, old_value, value) # reflect changes on the persistent storage try: path = os.path.join(self._path, key + '.json') with io.open(path, 'wb') as rec: json.dump(value.get(), rec, encoding='utf-8') except IOError: logger.error( "I/O error occured while merging entity lists", exc_info=True)
def merge_list(self, data_dict): """Merge a list of entities. Take a dictionary of entites and, for each of them: - if it's not present in the store, create it - if it's present, update it data_dict (dict): the dictionary of entities raise (InvalidData) if data cannot be parsed, if an entity is missing some properties or if properties are of the wrong type. """ with LOCK: if not isinstance(data_dict, dict): raise InvalidData("Not a dictionary") item_dict = dict() for key, value in data_dict.iteritems(): try: # FIXME We should allow keys to be arbitrary unicode # strings, so this just needs to be a non-empty check. if not re.match('[A-Za-z0-9_]+', key): raise InvalidData("Invalid key") item = self._entity() item.set(value) if not item.consistent(): raise InvalidData("Inconsistent data") item.key = key item_dict[key] = item except InvalidData as exc: exc.message = "[entity %s] %s" % (key, exc) exc.args = exc.message, raise exc for key, value in item_dict.iteritems(): is_new = key not in self._store old_value = self._store.get(key) # insert entity self._store[key] = value # notify callbacks if is_new: for callback in self._create_callbacks: callback(key, value) else: for callback in self._update_callbacks: callback(key, old_value, value) # reflect changes on the persistent storage try: path = os.path.join(self._path, key + '.json') with io.open(path, 'wb') as rec: json.dump(value.get(), rec, encoding='utf-8') except IOError: logger.error("I/O error occured while merging entity lists", exc_info=True)
def delete(self, entity_id): if not entity_id: # delete list entity_store.delete_list() elif entity_id in entity_store: # delete try: entity_store.delete(entity_id) except InvalidKey: logger.error("Entity %s doesn't exist" % entity_id, extra={"location": self.request.full_url()}) raise tornado.web.HTTPError(404)
def delete(self, entity_id): if not entity_id: # delete list entity_store.delete_list(self.finish) elif entity_id in entity_store: # delete try: entity_store.delete(entity_id, self.finish) except InvalidKey: logger.error("Entity %s doesn't exist" % entity_id, extra={'location': self.request.full_url()}) raise tornado.web.HTTPError(404)
def merge_list(self, data): """Merge a list of entities. Take a dictionary of entites and, for each of them: - if it's not present in the store, create it - if it's present, update it data (str): the dictionary of entities (a dict encoded in JSON) Raise InvalidData if data cannot be parsed, if an entity is missing some properties or if properties are of the wrong type. """ try: data_dict = json.loads(data) assert type(data_dict) is dict, "Not a dictionary" item_dict = dict() for key, value in data_dict.iteritems(): try: item = self._entity() item.set(value) if not item.consistent(): raise InvalidData('Inconsistent data') item.key = key item_dict[key] = item except InvalidData as exc: exc.message = '[entity %s] %s' % (key, exc) exc.args = exc.message, raise exc except ValueError: raise InvalidData('Invalid JSON') except AssertionError as message: raise InvalidData(str(message)) for key, value in item_dict.iteritems(): # insert entity self._store[key] = value # notify callbacks for callback in self._update_callbacks: callback(key) # reflect changes on the persistent storage try: with open(os.path.join(self._path, key + '.json'), 'w') as rec: rec.write(json.dumps(value.dump())) except IOError: logger.error("IOError occured", exc_info=True)
def update(self, key, data, confirm=None): """Update an entity. Update an existing entity with the given key and the given data. key (str): the key of the entity that has to be updated data (str): the new properties of the entity (a dict encoded in JSON) confirm (callable): action to be performed as soon as we're sure that the action won't fail (in particular, before notifying the callbacks). Raise InvalidKey if key isn't a str or if no entity with that key is present in the store. Raise InvalidData if data cannot be parsed, if it's missing some properties or if properties are of the wrong type. """ self._verify_key(key, must_be_present=True) # update entity try: item = self._entity() item.set(json.loads(data)) if not item.consistent(): raise InvalidData('Inconsistent data') item.key = key old_item = self._store[key] self._store[key] = item except ValueError: raise InvalidData('Invalid JSON') # confirm the operation if confirm is not None: confirm() # notify callbacks for callback in self._update_callbacks: callback(key, old_item, item) # reflect changes on the persistent storage try: with open(os.path.join(self._path, key + '.json'), 'w') as rec: rec.write(json.dumps(self._store[key].dump())) except IOError: logger.error("I/O error occured while updating entity", exc_info=True)
def update(self, key, data, confirm=None): """Update an entity. Update an existing entity with the given key and the given data. key (str): the key of the entity that has to be updated data (str): the new properties of the entity (a dict encoded in JSON) confirm (callable): action to be performed as soon as we're sure that the action won't fail (in particular, before notifying the callbacks). Raise InvalidKey if key isn't a str or if no entity with that key is present in the store. Raise InvalidData if data cannot be parsed, if it's missing some properties or if properties are of the wrong type. """ self._verify_key(key, must_be_present=True) # update entity try: item = self._entity() item.set(json.loads(data)) if not item.consistent(): raise InvalidData('Inconsistent data') item.key = key self._store[key] = item except ValueError: raise InvalidData('Invalid JSON') # confirm the operation if confirm is not None: confirm() # notify callbacks for callback in self._update_callbacks: callback(key) # reflect changes on the persistent storage try: with open(os.path.join(self._path, key + '.json'), 'w') as rec: rec.write(json.dumps(self._store[key].dump())) except IOError: logger.error("I/O error occured while updating entity", exc_info=True)
def put(self, entity_id): if not entity_id: # merge list try: entity_store.merge_list(self.request.body, self.finish) except InvalidData as exc: logger.error(str(exc), exc_info=False, extra={'location': self.request.full_url(), 'details': self.request.body}) raise tornado.web.HTTPError(400) elif entity_id not in entity_store: # create try: entity_store.create(entity_id, self.request.body, self.finish) except InvalidData as exc: logger.error(str(exc), exc_info=False, extra={'location': self.request.full_url(), 'details': self.request.body}) raise tornado.web.HTTPError(400) else: # update try: entity_store.update(entity_id, self.request.body, self.finish) except InvalidData as exc: logger.error(str(exc), exc_info=False, extra={'location': self.request.full_url(), 'details': self.request.body}) raise tornado.web.HTTPError(400)
def __init__(self, entity, dir_name, depends=None): """Initialize an empty EntityStore. The entity definition given as argument will define what kind of entities will be stored. It cannot be changed. entity (class): the class definition of the entities that will be stored """ if not issubclass(entity, Entity): raise ValueError("The 'entity' parameter " "isn't a subclass of Entity") self._entity = entity self._path = os.path.join(config.lib_dir, dir_name) self._depends = depends if depends is not None else [] self._store = dict() self._create_callbacks = list() self._update_callbacks = list() self._delete_callbacks = list() try: os.mkdir(self._path) except OSError: # it's ok: it means the directory already exists pass try: for name in os.listdir(self._path): # TODO check that the key is '[A-Za-z0-9_]+' if name[-5:] == '.json' and name[:-5] != '': with open(os.path.join(self._path, name), 'r') as rec: item = self._entity() item.load(json.load(rec)) item.key = name[:-5] self._store[name[:-5]] = item except OSError: # the path isn't a directory or is inaccessible logger.error("Path is not a directory or is not accessible", exc_info=True) except IOError: logger.error("I/O error occured", exc_info=True) except ValueError: logger.error("Invalid JSON", exc_info=False, extra={'location': os.path.join(self._path, name)}) except InvalidData, exc: logger.error(str(exc), exc_info=False, extra={'location': os.path.join(self._path, name)})
def __init__(self, entity, dir_name, depends=[]): """Initialize an empty EntityStore. The entity definition given as argument will define what kind of entities will be stored. It cannot be changed. entity (class): the class definition of the entities that will be stored """ if not issubclass(entity, Entity): raise ValueError("The 'entity' parameter " "isn't a subclass of Entity") self._entity = entity self._path = os.path.join(config.lib_dir, dir_name) self._depends = depends self._store = dict() self._create_callbacks = list() self._update_callbacks = list() self._delete_callbacks = list() try: os.mkdir(self._path) except OSError: # it's ok: it means the directory already exists pass try: for name in os.listdir(self._path): # TODO check that the key is '[A-Za-z0-9_]+' if name[-5:] == '.json' and name[:-5] != '': with open(os.path.join(self._path, name), 'r') as rec: item = self._entity() item.load(json.load(rec)) item.key = name[:-5] self._store[name[:-5]] = item except OSError: # the path isn't a directory or is inaccessible logger.error("OSError occured", exc_info=True) except IOError: logger.error("IOError occured", exc_info=True) except ValueError: logger.error("Invalid JSON", exc_info=False, extra={'location': os.path.join(self._path, name), 'details': data}) except InvalidData, exc: logger.error(str(exc), exc_info=False, extra={'location': os.path.join(self._path, name), 'details': data})
def delete(self, key): """Delete an entity. Delete an existing entity from the store. key (str): the key of the entity that has to be deleted Raise InvalidKey if key isn't a str or if no entity with that key is present in the store. """ self._verify_key(key, must_be_present=True) # delete entity del self._store[key] # notify callbacks for callback in self._delete_callbacks: callback(key) # reflect changes on the persistent storage try: os.remove(os.path.join(self._path, key + ".json")) except OSError: logger.error("OSError occured", exc_info=True)
def update(self, key, data): """Update an entity. Update an existing entity with the given key and the given data. key (unicode): the key of the entity that has to be updated data (dict): the new properties of the entity raise (InvalidKey): if key isn't a unicode or if no entity with that key is present in the store. raise (InvalidData): if data cannot be parsed, if it's missing some properties or if properties are of the wrong type. """ self._verify_key(key, must_be_present=True) # update entity with LOCK: item = self._entity() item.set(data) if not item.consistent(): raise InvalidData("Inconsistent data") item.key = key old_item = self._store[key] self._store[key] = item # notify callbacks for callback in self._update_callbacks: callback(key, old_item, item) # reflect changes on the persistent storage try: path = os.path.join(self._path, key + '.json') with io.open(path, 'wb') as rec: json.dump(self._store[key].get(), rec, encoding='utf-8') except IOError: logger.error("I/O error occured while updating entity", exc_info=True)
def merge_list(self, data, confirm=None): """Merge a list of entities. Take a dictionary of entites and, for each of them: - if it's not present in the store, create it - if it's present, update it data (str): the dictionary of entities (a dict encoded in JSON) confirm (callable): action to be performed as soon as we're sure that the action won't fail (in particular, before notifying the callbacks). Raise InvalidData if data cannot be parsed, if an entity is missing some properties or if properties are of the wrong type. """ try: data_dict = json.loads(data) assert type(data_dict) is dict, "Not a dictionary" item_dict = dict() for key, value in data_dict.iteritems(): try: # FIXME We should allow keys to be arbitrary unicode # strings, so this just needs to be a non-empty check. if not re.match("[A-Za-z0-9_]+", key): raise InvalidData('Invalid key') item = self._entity() item.set(value) if not item.consistent(): raise InvalidData('Inconsistent data') item.key = key item_dict[key] = item except InvalidData as exc: exc.message = '[entity %s] %s' % (key, exc) exc.args = exc.message, raise exc except ValueError: raise InvalidData('Invalid JSON') except AssertionError as message: raise InvalidData(str(message)) # confirm the operation if confirm is not None: confirm() for key, value in item_dict.iteritems(): is_new = key not in self._store # insert entity self._store[key] = value # notify callbacks if is_new: for callback in self._create_callbacks: callback(key) else: for callback in self._update_callbacks: callback(key) # reflect changes on the persistent storage try: with open(os.path.join(self._path, key + '.json'), 'w') as rec: rec.write(json.dumps(value.dump())) except IOError: logger.error("IOError occured", exc_info=True)
def merge_list(self, data, confirm=None): """Merge a list of entities. Take a dictionary of entites and, for each of them: - if it's not present in the store, create it - if it's present, update it data (str): the dictionary of entities (a dict encoded in JSON) confirm (callable): action to be performed as soon as we're sure that the action won't fail (in particular, before notifying the callbacks). Raise InvalidData if data cannot be parsed, if an entity is missing some properties or if properties are of the wrong type. """ try: data_dict = json.loads(data) assert type(data_dict) is dict, "Not a dictionary" item_dict = dict() for key, value in data_dict.iteritems(): try: # FIXME We should allow keys to be arbitrary unicode # strings, so this just needs to be a non-empty check. if not re.match("[A-Za-z0-9_]+", key): raise InvalidData('Invalid key') item = self._entity() item.set(value) if not item.consistent(): raise InvalidData('Inconsistent data') item.key = key item_dict[key] = item except InvalidData as exc: exc.message = '[entity %s] %s' % (key, exc) exc.args = exc.message, raise exc except ValueError: raise InvalidData('Invalid JSON') except AssertionError as message: raise InvalidData(str(message)) # confirm the operation if confirm is not None: confirm() for key, value in item_dict.iteritems(): is_new = key not in self._store old_value = self._store.get(key) # insert entity self._store[key] = value # notify callbacks if is_new: for callback in self._create_callbacks: callback(key, value) else: for callback in self._update_callbacks: callback(key, old_value, value) # reflect changes on the persistent storage try: with open(os.path.join(self._path, key + '.json'), 'w') as rec: rec.write(json.dumps(value.dump())) except IOError: logger.error("I/O error occured while merging entity lists", exc_info=True)