def update(self, key, data):
"""Update an entity.
Update an existing entity with the given key and the given data.
key (str): the key of the entity that has to be updated
data (str): the new properties of the entity (a dict encoded in JSON)
Raise InvalidKey if key isn't a str or if no entity with that key
is present in the store.
Raise InvalidData if data cannot be parsed, if it's missing some
properties or if properties are of the wrong type.
"""
self._verify_key(key, must_be_present=True)
# update entity
try:
item = self._entity()
item.set(json.loads(data))
if not item.consistent():
raise InvalidData('Inconsistent data')
item.key = key
self._store[key] = item
except ValueError:
raise InvalidData('Invalid JSON')
# notify callbacks
for callback in self._update_callbacks:
callback(key)
# reflect changes on the persistent storage
try:
with open(os.path.join(self._path, key + '.json'), 'w') as rec:
rec.write(json.dumps(self._store[key].dump()))
except IOError:
logger.error("IOError occured", exc_info=True)
def delete(self, key):
"""Delete an entity.
Delete an existing entity from the store.
key (unicode): the key of the entity that has to be deleted
raise (InvalidKey): if key isn't a unicode or if no entity
with that key is present in the store.
"""
self._verify_key(key, must_be_present=True)
with LOCK:
# delete entity
old_value = self._store[key]
del self._store[key]
# enforce consistency
for depend in self._depends:
for o_key, o_value in list(depend._store.iteritems()):
if not o_value.consistent():
depend.delete(o_key)
# notify callbacks
for callback in self._delete_callbacks:
callback(key, old_value)
# reflect changes on the persistent storage
try:
os.remove(os.path.join(self._path, key + '.json'))
except OSError:
logger.error("Unable to delete entity", exc_info=True)
def create(self, key, data):
"""Create a new entity.
Create a new entity with the given key and the given data.
key (str): the key with which the entity will be later accessed
data (dict): the properties of the entity
Raise InvalidKey if key isn't a str or if an entity with the same
key is already present in the store.
Raise InvalidData if data cannot be parsed, if it's missing some
properties or if properties are of the wrong type.
"""
self._verify_key(key)
# create entity
try:
item = self._entity()
item.set(json.loads(data))
item.key = key
self._store[key] = item
except ValueError:
raise InvalidData("Invalid JSON")
# notify callbacks
for callback in self._create_callbacks:
callback(key)
# reflect changes on the persistent storage
try:
with open(os.path.join(self._path, key + ".json"), "w") as rec:
rec.write(json.dumps(self._store[key].dump()))
except IOError:
logger.error("IOError occured", exc_info=True)
def delete(self, key, confirm=None):
"""Delete an entity.
Delete an existing entity from the store.
key (str): the key of the entity that has to be deleted
confirm (callable): action to be performed as soon as we're sure
that the action won't fail (in particular,
before notifying the callbacks).
Raise InvalidKey if key isn't a str or if no entity with that key
is present in the store.
"""
self._verify_key(key, must_be_present=True)
# confirm the operation
if confirm is not None:
confirm()
# delete entity
old_value = self._store[key]
del self._store[key]
# enforce consistency
for depend in self._depends:
for o_key, o_value in list(depend.store._store.iteritems()):
if not o_value.consistent():
depend.store.delete(o_key)
# notify callbacks
for callback in self._delete_callbacks:
callback(key, old_value)
# reflect changes on the persistent storage
try:
os.remove(os.path.join(self._path, key + '.json'))
except OSError:
logger.error("Unable to delete entity", exc_info=True)
def delete(self, key):
"""Delete an entity.
Delete an existing entity from the store.
key (unicode): the key of the entity that has to be deleted
raise (InvalidKey): if key isn't a unicode or if no entity
with that key is present in the store.
"""
self._verify_key(key, must_be_present=True)
with LOCK:
# delete entity
old_value = self._store[key]
del self._store[key]
# enforce consistency
for depend in self._depends:
for o_key, o_value in list(depend._store.iteritems()):
if not o_value.consistent():
depend.delete(o_key)
# notify callbacks
for callback in self._delete_callbacks:
callback(key, old_value)
# reflect changes on the persistent storage
try:
os.remove(os.path.join(self._path, key + '.json'))
except OSError:
logger.error("Unable to delete entity", exc_info=True)
def merge_list(self, data_dict):
"""Merge a list of entities.
Take a dictionary of entites and, for each of them:
- if it's not present in the store, create it
- if it's present, update it
data_dict (dict): the dictionary of entities
raise (InvalidData) if data cannot be parsed, if an entity is
missing some properties or if properties are of the wrong
type.
"""
with LOCK:
if not isinstance(data_dict, dict):
raise InvalidData("Not a dictionary")
item_dict = dict()
for key, value in data_dict.iteritems():
try:
# FIXME We should allow keys to be arbitrary unicode
# strings, so this just needs to be a non-empty check.
if not re.match('[A-Za-z0-9_]+', key):
raise InvalidData("Invalid key")
item = self._entity()
item.set(value)
if not item.consistent():
raise InvalidData("Inconsistent data")
item.key = key
item_dict[key] = item
except InvalidData as exc:
exc.message = "[entity %s] %s" % (key, exc)
exc.args = exc.message,
raise exc
for key, value in item_dict.iteritems():
is_new = key not in self._store
old_value = self._store.get(key)
# insert entity
self._store[key] = value
# notify callbacks
if is_new:
for callback in self._create_callbacks:
callback(key, value)
else:
for callback in self._update_callbacks:
callback(key, old_value, value)
# reflect changes on the persistent storage
try:
path = os.path.join(self._path, key + '.json')
with io.open(path, 'wb') as rec:
json.dump(value.get(), rec, encoding='utf-8')
except IOError:
logger.error(
"I/O error occured while merging entity lists",
exc_info=True)
def merge_list(self, data_dict):
"""Merge a list of entities.
Take a dictionary of entites and, for each of them:
- if it's not present in the store, create it
- if it's present, update it
data_dict (dict): the dictionary of entities
raise (InvalidData) if data cannot be parsed, if an entity is
missing some properties or if properties are of the wrong
type.
"""
with LOCK:
if not isinstance(data_dict, dict):
raise InvalidData("Not a dictionary")
item_dict = dict()
for key, value in data_dict.iteritems():
try:
# FIXME We should allow keys to be arbitrary unicode
# strings, so this just needs to be a non-empty check.
if not re.match('[A-Za-z0-9_]+', key):
raise InvalidData("Invalid key")
item = self._entity()
item.set(value)
if not item.consistent():
raise InvalidData("Inconsistent data")
item.key = key
item_dict[key] = item
except InvalidData as exc:
exc.message = "[entity %s] %s" % (key, exc)
exc.args = exc.message,
raise exc
for key, value in item_dict.iteritems():
is_new = key not in self._store
old_value = self._store.get(key)
# insert entity
self._store[key] = value
# notify callbacks
if is_new:
for callback in self._create_callbacks:
callback(key, value)
else:
for callback in self._update_callbacks:
callback(key, old_value, value)
# reflect changes on the persistent storage
try:
path = os.path.join(self._path, key + '.json')
with io.open(path, 'wb') as rec:
json.dump(value.get(), rec, encoding='utf-8')
except IOError:
logger.error("I/O error occured while merging entity lists",
exc_info=True)
def delete(self, entity_id):
if not entity_id:
# delete list
entity_store.delete_list()
elif entity_id in entity_store:
# delete
try:
entity_store.delete(entity_id)
except InvalidKey:
logger.error("Entity %s doesn't exist" % entity_id, extra={"location": self.request.full_url()})
raise tornado.web.HTTPError(404)
def delete(self, entity_id):
if not entity_id:
# delete list
entity_store.delete_list(self.finish)
elif entity_id in entity_store:
# delete
try:
entity_store.delete(entity_id, self.finish)
except InvalidKey:
logger.error("Entity %s doesn't exist" % entity_id,
extra={'location': self.request.full_url()})
raise tornado.web.HTTPError(404)
def merge_list(self, data):
"""Merge a list of entities.
Take a dictionary of entites and, for each of them:
- if it's not present in the store, create it
- if it's present, update it
data (str): the dictionary of entities (a dict encoded in JSON)
Raise InvalidData if data cannot be parsed, if an entity is missing
some properties or if properties are of the wrong type.
"""
try:
data_dict = json.loads(data)
assert type(data_dict) is dict, "Not a dictionary"
item_dict = dict()
for key, value in data_dict.iteritems():
try:
item = self._entity()
item.set(value)
if not item.consistent():
raise InvalidData('Inconsistent data')
item.key = key
item_dict[key] = item
except InvalidData as exc:
exc.message = '[entity %s] %s' % (key, exc)
exc.args = exc.message,
raise exc
except ValueError:
raise InvalidData('Invalid JSON')
except AssertionError as message:
raise InvalidData(str(message))
for key, value in item_dict.iteritems():
# insert entity
self._store[key] = value
# notify callbacks
for callback in self._update_callbacks:
callback(key)
# reflect changes on the persistent storage
try:
with open(os.path.join(self._path, key + '.json'), 'w') as rec:
rec.write(json.dumps(value.dump()))
except IOError:
logger.error("IOError occured", exc_info=True)
def update(self, key, data, confirm=None):
"""Update an entity.
Update an existing entity with the given key and the given data.
key (str): the key of the entity that has to be updated
data (str): the new properties of the entity (a dict encoded in JSON)
confirm (callable): action to be performed as soon as we're sure
that the action won't fail (in particular,
before notifying the callbacks).
Raise InvalidKey if key isn't a str or if no entity with that key
is present in the store.
Raise InvalidData if data cannot be parsed, if it's missing some
properties or if properties are of the wrong type.
"""
self._verify_key(key, must_be_present=True)
# update entity
try:
item = self._entity()
item.set(json.loads(data))
if not item.consistent():
raise InvalidData('Inconsistent data')
item.key = key
old_item = self._store[key]
self._store[key] = item
except ValueError:
raise InvalidData('Invalid JSON')
# confirm the operation
if confirm is not None:
confirm()
# notify callbacks
for callback in self._update_callbacks:
callback(key, old_item, item)
# reflect changes on the persistent storage
try:
with open(os.path.join(self._path, key + '.json'), 'w') as rec:
rec.write(json.dumps(self._store[key].dump()))
except IOError:
logger.error("I/O error occured while updating entity",
exc_info=True)
def update(self, key, data, confirm=None):
"""Update an entity.
Update an existing entity with the given key and the given data.
key (str): the key of the entity that has to be updated
data (str): the new properties of the entity (a dict encoded in JSON)
confirm (callable): action to be performed as soon as we're sure
that the action won't fail (in particular,
before notifying the callbacks).
Raise InvalidKey if key isn't a str or if no entity with that key
is present in the store.
Raise InvalidData if data cannot be parsed, if it's missing some
properties or if properties are of the wrong type.
"""
self._verify_key(key, must_be_present=True)
# update entity
try:
item = self._entity()
item.set(json.loads(data))
if not item.consistent():
raise InvalidData('Inconsistent data')
item.key = key
self._store[key] = item
except ValueError:
raise InvalidData('Invalid JSON')
# confirm the operation
if confirm is not None:
confirm()
# notify callbacks
for callback in self._update_callbacks:
callback(key)
# reflect changes on the persistent storage
try:
with open(os.path.join(self._path, key + '.json'), 'w') as rec:
rec.write(json.dumps(self._store[key].dump()))
except IOError:
logger.error("I/O error occured while updating entity",
exc_info=True)
def put(self, entity_id):
if not entity_id:
# merge list
try:
entity_store.merge_list(self.request.body,
self.finish)
except InvalidData as exc:
logger.error(str(exc), exc_info=False,
extra={'location': self.request.full_url(),
'details': self.request.body})
raise tornado.web.HTTPError(400)
elif entity_id not in entity_store:
# create
try:
entity_store.create(entity_id, self.request.body,
self.finish)
except InvalidData as exc:
logger.error(str(exc), exc_info=False,
extra={'location': self.request.full_url(),
'details': self.request.body})
raise tornado.web.HTTPError(400)
else:
# update
try:
entity_store.update(entity_id, self.request.body,
self.finish)
except InvalidData as exc:
logger.error(str(exc), exc_info=False,
extra={'location': self.request.full_url(),
'details': self.request.body})
raise tornado.web.HTTPError(400)
def put(self, entity_id):
if not entity_id:
# merge list
try:
entity_store.merge_list(self.request.body,
self.finish)
except InvalidData as exc:
logger.error(str(exc), exc_info=False,
extra={'location': self.request.full_url(),
'details': self.request.body})
raise tornado.web.HTTPError(400)
elif entity_id not in entity_store:
# create
try:
entity_store.create(entity_id, self.request.body,
self.finish)
except InvalidData as exc:
logger.error(str(exc), exc_info=False,
extra={'location': self.request.full_url(),
'details': self.request.body})
raise tornado.web.HTTPError(400)
else:
# update
try:
entity_store.update(entity_id, self.request.body,
self.finish)
except InvalidData as exc:
logger.error(str(exc), exc_info=False,
extra={'location': self.request.full_url(),
'details': self.request.body})
raise tornado.web.HTTPError(400)
def __init__(self, entity, dir_name, depends=None):
"""Initialize an empty EntityStore.
The entity definition given as argument will define what kind of
entities will be stored. It cannot be changed.
entity (class): the class definition of the entities that will
be stored
"""
if not issubclass(entity, Entity):
raise ValueError("The 'entity' parameter "
"isn't a subclass of Entity")
self._entity = entity
self._path = os.path.join(config.lib_dir, dir_name)
self._depends = depends if depends is not None else []
self._store = dict()
self._create_callbacks = list()
self._update_callbacks = list()
self._delete_callbacks = list()
try:
os.mkdir(self._path)
except OSError:
# it's ok: it means the directory already exists
pass
try:
for name in os.listdir(self._path):
# TODO check that the key is '[A-Za-z0-9_]+'
if name[-5:] == '.json' and name[:-5] != '':
with open(os.path.join(self._path, name), 'r') as rec:
item = self._entity()
item.load(json.load(rec))
item.key = name[:-5]
self._store[name[:-5]] = item
except OSError:
# the path isn't a directory or is inaccessible
logger.error("Path is not a directory or is not accessible",
exc_info=True)
except IOError:
logger.error("I/O error occured", exc_info=True)
except ValueError:
logger.error("Invalid JSON",
exc_info=False,
extra={'location': os.path.join(self._path, name)})
except InvalidData, exc:
logger.error(str(exc),
exc_info=False,
extra={'location': os.path.join(self._path, name)})
def __init__(self, entity, dir_name, depends=[]):
"""Initialize an empty EntityStore.
The entity definition given as argument will define what kind of
entities will be stored. It cannot be changed.
entity (class): the class definition of the entities that will
be stored
"""
if not issubclass(entity, Entity):
raise ValueError("The 'entity' parameter "
"isn't a subclass of Entity")
self._entity = entity
self._path = os.path.join(config.lib_dir, dir_name)
self._depends = depends
self._store = dict()
self._create_callbacks = list()
self._update_callbacks = list()
self._delete_callbacks = list()
try:
os.mkdir(self._path)
except OSError:
# it's ok: it means the directory already exists
pass
try:
for name in os.listdir(self._path):
# TODO check that the key is '[A-Za-z0-9_]+'
if name[-5:] == '.json' and name[:-5] != '':
with open(os.path.join(self._path, name), 'r') as rec:
item = self._entity()
item.load(json.load(rec))
item.key = name[:-5]
self._store[name[:-5]] = item
except OSError:
# the path isn't a directory or is inaccessible
logger.error("OSError occured", exc_info=True)
except IOError:
logger.error("IOError occured", exc_info=True)
except ValueError:
logger.error("Invalid JSON", exc_info=False,
extra={'location': os.path.join(self._path, name),
'details': data})
except InvalidData, exc:
logger.error(str(exc), exc_info=False,
extra={'location': os.path.join(self._path, name),
'details': data})
def delete(self, key):
"""Delete an entity.
Delete an existing entity from the store.
key (str): the key of the entity that has to be deleted
Raise InvalidKey if key isn't a str or if no entity with that key
is present in the store.
"""
self._verify_key(key, must_be_present=True)
# delete entity
del self._store[key]
# notify callbacks
for callback in self._delete_callbacks:
callback(key)
# reflect changes on the persistent storage
try:
os.remove(os.path.join(self._path, key + ".json"))
except OSError:
logger.error("OSError occured", exc_info=True)
def update(self, key, data):
"""Update an entity.
Update an existing entity with the given key and the given
data.
key (unicode): the key of the entity that has to be updated
data (dict): the new properties of the entity
raise (InvalidKey): if key isn't a unicode or if no entity
with that key is present in the store.
raise (InvalidData): if data cannot be parsed, if it's missing
some properties or if properties are of the wrong type.
"""
self._verify_key(key, must_be_present=True)
# update entity
with LOCK:
item = self._entity()
item.set(data)
if not item.consistent():
raise InvalidData("Inconsistent data")
item.key = key
old_item = self._store[key]
self._store[key] = item
# notify callbacks
for callback in self._update_callbacks:
callback(key, old_item, item)
# reflect changes on the persistent storage
try:
path = os.path.join(self._path, key + '.json')
with io.open(path, 'wb') as rec:
json.dump(self._store[key].get(), rec, encoding='utf-8')
except IOError:
logger.error("I/O error occured while updating entity",
exc_info=True)
def update(self, key, data):
"""Update an entity.
Update an existing entity with the given key and the given
data.
key (unicode): the key of the entity that has to be updated
data (dict): the new properties of the entity
raise (InvalidKey): if key isn't a unicode or if no entity
with that key is present in the store.
raise (InvalidData): if data cannot be parsed, if it's missing
some properties or if properties are of the wrong type.
"""
self._verify_key(key, must_be_present=True)
# update entity
with LOCK:
item = self._entity()
item.set(data)
if not item.consistent():
raise InvalidData("Inconsistent data")
item.key = key
old_item = self._store[key]
self._store[key] = item
# notify callbacks
for callback in self._update_callbacks:
callback(key, old_item, item)
# reflect changes on the persistent storage
try:
path = os.path.join(self._path, key + '.json')
with io.open(path, 'wb') as rec:
json.dump(self._store[key].get(), rec, encoding='utf-8')
except IOError:
logger.error("I/O error occured while updating entity",
exc_info=True)
def merge_list(self, data, confirm=None):
"""Merge a list of entities.
Take a dictionary of entites and, for each of them:
- if it's not present in the store, create it
- if it's present, update it
data (str): the dictionary of entities (a dict encoded in JSON)
confirm (callable): action to be performed as soon as we're sure
that the action won't fail (in particular,
before notifying the callbacks).
Raise InvalidData if data cannot be parsed, if an entity is missing
some properties or if properties are of the wrong type.
"""
try:
data_dict = json.loads(data)
assert type(data_dict) is dict, "Not a dictionary"
item_dict = dict()
for key, value in data_dict.iteritems():
try:
# FIXME We should allow keys to be arbitrary unicode
# strings, so this just needs to be a non-empty check.
if not re.match("[A-Za-z0-9_]+", key):
raise InvalidData('Invalid key')
item = self._entity()
item.set(value)
if not item.consistent():
raise InvalidData('Inconsistent data')
item.key = key
item_dict[key] = item
except InvalidData as exc:
exc.message = '[entity %s] %s' % (key, exc)
exc.args = exc.message,
raise exc
except ValueError:
raise InvalidData('Invalid JSON')
except AssertionError as message:
raise InvalidData(str(message))
# confirm the operation
if confirm is not None:
confirm()
for key, value in item_dict.iteritems():
is_new = key not in self._store
# insert entity
self._store[key] = value
# notify callbacks
if is_new:
for callback in self._create_callbacks:
callback(key)
else:
for callback in self._update_callbacks:
callback(key)
# reflect changes on the persistent storage
try:
with open(os.path.join(self._path, key + '.json'), 'w') as rec:
rec.write(json.dumps(value.dump()))
except IOError:
logger.error("IOError occured", exc_info=True)
def merge_list(self, data, confirm=None):
"""Merge a list of entities.
Take a dictionary of entites and, for each of them:
- if it's not present in the store, create it
- if it's present, update it
data (str): the dictionary of entities (a dict encoded in JSON)
confirm (callable): action to be performed as soon as we're sure
that the action won't fail (in particular,
before notifying the callbacks).
Raise InvalidData if data cannot be parsed, if an entity is missing
some properties or if properties are of the wrong type.
"""
try:
data_dict = json.loads(data)
assert type(data_dict) is dict, "Not a dictionary"
item_dict = dict()
for key, value in data_dict.iteritems():
try:
# FIXME We should allow keys to be arbitrary unicode
# strings, so this just needs to be a non-empty check.
if not re.match("[A-Za-z0-9_]+", key):
raise InvalidData('Invalid key')
item = self._entity()
item.set(value)
if not item.consistent():
raise InvalidData('Inconsistent data')
item.key = key
item_dict[key] = item
except InvalidData as exc:
exc.message = '[entity %s] %s' % (key, exc)
exc.args = exc.message,
raise exc
except ValueError:
raise InvalidData('Invalid JSON')
except AssertionError as message:
raise InvalidData(str(message))
# confirm the operation
if confirm is not None:
confirm()
for key, value in item_dict.iteritems():
is_new = key not in self._store
old_value = self._store.get(key)
# insert entity
self._store[key] = value
# notify callbacks
if is_new:
for callback in self._create_callbacks:
callback(key, value)
else:
for callback in self._update_callbacks:
callback(key, old_value, value)
# reflect changes on the persistent storage
try:
with open(os.path.join(self._path, key + '.json'), 'w') as rec:
rec.write(json.dumps(value.dump()))
except IOError:
logger.error("I/O error occured while merging entity lists",
exc_info=True)