def register(self, parent, key, create_only=False, **kwargs):
"""
Add/replace an entry within directory, below a parent node or "/".
Note: Replaces (not merges) the attribute values of the entry if existing
@param create_only If True, does not change an existing entry
@retval DirEntry if previously existing
"""
if not (parent and key):
raise BadRequest("Illegal arguments")
if not type(parent) is str or not parent.startswith("/"):
raise BadRequest("Illegal arguments: parent")
dn = self._get_path(parent, key)
log.debug("Directory.register(%s): %s", dn, kwargs)
entry_old = None
cur_time = get_ion_ts()
# Must read existing entry by path to make sure to not create path twice
direntry = self._read_by_path(dn)
if direntry and create_only:
# We only wanted to make sure entry exists. Do not change
return direntry
elif direntry:
entry_old = direntry.attributes
direntry.attributes = kwargs
direntry.ts_updated = cur_time
# TODO: This may fail because of concurrent update
self.dir_store.update(direntry)
else:
direntry = self._create_dir_entry(parent, key, attributes=kwargs, ts=cur_time)
self.dir_store.create(direntry, create_unique_directory_id())
return entry_old
def register(self, parent, key, create_only=False, **kwargs):
"""
Add/replace an entry within directory, below a parent node or "/".
Note: Replaces (not merges) the attribute values of the entry if existing
@retval DirEntry if previously existing
"""
if not (parent and key):
raise BadRequest("Illegal arguments")
if not type(parent) is str or not parent.startswith("/"):
raise BadRequest("Illegal arguments: parent")
dn = self._get_path(parent, key)
entry_old = None
direntry = self._read_by_path(dn)
cur_time = get_ion_ts()
if direntry and create_only:
# We only wanted to make sure entry exists. Do not change
return direntry
elif direntry:
# Change existing entry's attributes
entry_old = direntry.get('attributes')
direntry['attributes'] = kwargs
direntry['ts_updated'] = cur_time
self.datastore.update_doc(direntry)
else:
doc = self._create_dir_entry(object_id=create_unique_directory_id(), parent=parent, key=key,
attributes=kwargs)
self.datastore.create_doc(doc)
return entry_old
def register(self, parent, key, create_only=False, **kwargs):
"""
Add/replace an entry within directory, below a parent node or "/".
Note: Replaces (not merges) the attribute values of the entry if existing
@param create_only If True, does not change an existing entry
@retval DirEntry if previously existing
"""
if not (parent and key):
raise BadRequest("Illegal arguments")
if not type(parent) is str or not parent.startswith("/"):
raise BadRequest("Illegal arguments: parent")
dn = self._get_path(parent, key)
log.debug("Directory.register(%s): %s", dn, kwargs)
entry_old = None
cur_time = get_ion_ts()
# Must read existing entry by path to make sure to not create path twice
direntry = self._read_by_path(dn)
if direntry and create_only:
# We only wanted to make sure entry exists. Do not change
return direntry
elif direntry:
entry_old = direntry.attributes
direntry.attributes = kwargs
direntry.ts_updated = cur_time
# TODO: This may fail because of concurrent update
self.dir_store.update(direntry)
else:
direntry = self._create_dir_entry(parent, key, attributes=kwargs, ts=cur_time)
self._ensure_parents_exist([direntry])
self.dir_store.create(direntry, create_unique_directory_id())
return entry_old
def register(self, parent, key, create_only=False, **kwargs):
"""
Add/replace an entry within directory, below a parent node or "/".
Note: Replaces (not merges) the attribute values of the entry if existing
@retval DirEntry if previously existing
"""
if not (parent and key):
raise BadRequest("Illegal arguments")
if not type(parent) is str or not parent.startswith("/"):
raise BadRequest("Illegal arguments: parent")
dn = self._get_path(parent, key)
entry_old = None
direntry = self._read_by_path(dn)
cur_time = get_ion_ts()
if direntry and create_only:
# We only wanted to make sure entry exists. Do not change
return direntry
elif direntry:
# Change existing entry's attributes
entry_old = direntry.get('attributes')
direntry['attributes'] = kwargs
direntry['ts_updated'] = cur_time
self.datastore.update_doc(direntry)
else:
doc = self._create_dir_entry(object_id=create_unique_directory_id(), parent=parent, key=key,
attributes=kwargs)
self.datastore.create_doc(doc)
return entry_old
def register_mult(self, entries):
"""
Registers multiple directory entries efficiently in one datastore access.
Note: this fails of entries are currently existing, so works for create only.
"""
if type(entries) not in (list, tuple):
raise BadRequest("Bad entries type")
de_list = []
cur_time = get_ion_ts()
for parent, key, attrs in entries:
de = self._create_dir_entry(object_id=create_unique_directory_id(), parent=parent, key=key,
attributes=attrs, ts_created=cur_time, ts_updated=cur_time)
de_list.append(de)
self.datastore.create_doc_mult(de_list)
def register_mult(self, entries):
"""
Registers multiple directory entries efficiently in one datastore access.
Note: this fails of entries are currently existing, so works for create only.
"""
if type(entries) not in (list, tuple):
raise BadRequest("Bad entries type")
de_list = []
cur_time = get_ion_ts()
for parent, key, attrs in entries:
direntry = self._create_dir_entry(parent, key, attributes=attrs, ts=cur_time)
de_list.append(direntry)
deid_list = [create_unique_directory_id() for i in xrange(len(de_list))]
self.dir_store.create_mult(de_list, deid_list)
def register_mult(self, entries):
"""
Registers multiple directory entries efficiently in one datastore access.
Note: this fails of entries are currently existing, so works for create only.
"""
if type(entries) not in (list, tuple):
raise BadRequest("Bad entries type")
de_list = []
cur_time = get_ion_ts()
for parent, key, attrs in entries:
direntry = self._create_dir_entry(parent, key, attributes=attrs, ts=cur_time)
de_list.append(direntry)
deid_list = [create_unique_directory_id() for i in xrange(len(de_list))]
self.dir_store.create_mult(de_list, deid_list)
def register_mult(self, entries):
"""
Registers multiple directory entries efficiently in one datastore access.
Note: this fails of entries are currently existing, so works for create only.
"""
if type(entries) not in (list, tuple):
raise BadRequest("Bad entries type")
de_list = []
cur_time = get_ion_ts()
for parent, key, attrs in entries:
de = self._create_dir_entry(object_id=create_unique_directory_id(), parent=parent, key=key,
attributes=attrs, ts_created=cur_time, ts_updated=cur_time)
de_list.append(de)
self.datastore.create_doc_mult(de_list)
def _ensure_parents_exist(self, entry_list, create=True):
parents_list = self._get_unique_parents(entry_list)
pe_list = []
try:
for parent in parents_list:
pe = self.lookup(parent)
if pe is None:
pp, pk = parent.rsplit("/", 1)
direntry = self._create_dir_entry(parent=pp, key=pk)
pe_list.append(direntry)
if create:
self.dir_store.create(direntry, create_unique_directory_id())
except Exception as ex:
log.warn("_ensure_parents_exist(): Error creating directory parents", exc_info=True)
return pe_list
def _ensure_parents_exist(self, entry_list, create=True):
parents_list = self._get_unique_parents(entry_list)
pe_list = []
try:
for parent in parents_list:
pe = self.lookup(parent)
if pe is None:
pp, pk = parent.rsplit("/", 1)
doc = self._create_dir_entry(object_id=create_unique_directory_id(), parent=pp, key=pk)
pe_list.append(doc)
if create:
self.datastore.create_doc(doc)
except Exception as ex:
print "_ensure_parents_exist(): Error creating directory parents", ex
return pe_list
def register(self, parent, key, create_only=False, ensure_parents=True, **kwargs):
"""
Add/replace an entry within directory, below a parent node or "/".
Note: Replaces (not merges) the attribute values of the entry if existing
@retval DirEntry if previously existing
"""
if not (parent and key):
raise BadRequest("Illegal arguments")
if not type(parent) is str or not parent.startswith("/"):
raise BadRequest("Illegal arguments: parent")
dn = self._get_path(parent, key)
entry_old = None
direntry = self._read_by_path(dn)
cur_time = get_ion_ts()
if direntry and create_only:
# We only wanted to make sure entry exists. Do not change
return direntry
elif direntry:
# Change existing entry's attributes
entry_old = direntry.get("attributes")
direntry["attributes"] = kwargs
direntry["ts_updated"] = cur_time
try:
self.datastore.update_doc(direntry)
except Conflict:
# Concurrent update - we accept that we finished the race second and give up
pass
else:
doc = self._create_dir_entry(
object_id=create_unique_directory_id(), parent=parent, key=key, attributes=kwargs
)
if ensure_parents:
self._ensure_parents_exist([doc])
try:
self.datastore.create_doc(doc)
except BadRequest as ex:
if not ex.message.startswith("DirEntry already exists"):
raise
# Concurrent create - we accept that we finished the race second and give up
return entry_old
def register(self, parent, key, create_only=False, ensure_parents=True, **kwargs):
"""
Add/replace an entry within directory, below a parent node or "/".
Note: Replaces (not merges) the attribute values of the entry if existing
@retval DirEntry if previously existing
"""
if not (parent and key):
raise BadRequest("Illegal arguments")
if not type(parent) is str or not parent.startswith("/"):
raise BadRequest("Illegal arguments: parent")
dn = self._get_path(parent, key)
entry_old = None
direntry = self._read_by_path(dn)
cur_time = get_ion_ts()
if direntry and create_only:
# We only wanted to make sure entry exists. Do not change
return direntry
elif direntry:
# Change existing entry's attributes
entry_old = direntry.get('attributes')
direntry['attributes'] = kwargs
direntry['ts_updated'] = cur_time
try:
self.datastore.update_doc(direntry)
except Conflict:
# Concurrent update - we accept that we finished the race second and give up
pass
else:
doc = self._create_dir_entry(object_id=create_unique_directory_id(), parent=parent, key=key,
attributes=kwargs)
if ensure_parents:
self._ensure_parents_exist([doc])
try:
self.datastore.create_doc(doc)
except BadRequest as ex:
if not ex.message.startswith("DirEntry already exists"):
raise
# Concurrent create - we accept that we finished the race second and give up
return entry_old
def _ensure_parents_exist(self, entry_list, create=True):
parents_list = self._get_unique_parents(entry_list)
pe_list = []
try:
for parent in parents_list:
pe = self.lookup(parent)
if pe is None:
pp, pk = parent.rsplit("/", 1)
doc = self._create_dir_entry(object_id=create_unique_directory_id(), parent=pp, key=pk)
pe_list.append(doc)
if create:
try:
self.datastore.create_doc(doc)
except BadRequest as ex:
if not ex.message.startswith("DirEntry already exists"):
raise
# Else: Concurrent create
except Exception as ex:
print "_ensure_parents_exist(): Error creating directory parents", ex
return pe_list
def _ensure_parents_exist(self, entry_list, create=True):
parents_list = self._get_unique_parents(entry_list)
pe_list = []
try:
for parent in parents_list:
pe = self.lookup(parent)
if pe is None:
pp, pk = parent.rsplit("/", 1)
doc = self._create_dir_entry(object_id=create_unique_directory_id(), parent=pp, key=pk)
pe_list.append(doc)
if create:
try:
self.datastore.create_doc(doc)
except BadRequest as ex:
if not ex.message.startswith("DirEntry already exists"):
raise
# Else: Concurrent create
except Exception as ex:
print "_ensure_parents_exist(): Error creating directory parents", ex
return pe_list
def register_mult(self, entries):
"""
Registers multiple directory entries efficiently in one datastore access.
Note: this fails if entries are already existing, so works for create only.
"""
if type(entries) not in (list, tuple):
raise BadRequest("Bad entries type")
de_list = []
cur_time = get_ion_ts()
for parent, key, attrs in entries:
direntry = self._create_dir_entry(parent, key, attributes=attrs, ts=cur_time)
de_list.append(direntry)
pe_list = self._ensure_parents_exist(de_list, create=False)
de_list.extend(pe_list)
deid_list = [create_unique_directory_id() for i in xrange(len(de_list))]
self.dir_store.create_mult(de_list, deid_list)
if self.events_enabled and self.container.has_capability(CCAP.EXCHANGE_MANAGER):
for de in de_list:
self.event_pub.publish_event(event_type="DirectoryModifiedEvent",
origin=self.orgname + ".DIR", origin_type="DIR",
key=de.key, parent=de.parent, org=self.orgname,
sub_type="REGISTER." + de.parent[1:].replace("/", "."),
mod_type=DirectoryModificationType.CREATE)
def register_mult(self, entries):
"""
Registers multiple directory entries efficiently in one datastore access.
Note: this fails if entries are already existing, so works for create only.
"""
if type(entries) not in (list, tuple):
raise BadRequest("Bad entries type")
de_list = []
cur_time = get_ion_ts()
for parent, key, attrs in entries:
direntry = self._create_dir_entry(parent, key, attributes=attrs, ts=cur_time)
de_list.append(direntry)
pe_list = self._ensure_parents_exist(de_list, create=False)
de_list.extend(pe_list)
deid_list = [create_unique_directory_id() for i in xrange(len(de_list))]
self.dir_store.create_mult(de_list, deid_list)
if self.events_enabled and self.container.has_capability(CCAP.EXCHANGE_MANAGER):
for de in de_list:
self.event_pub.publish_event(event_type="DirectoryModifiedEvent",
origin=self.orgname + ".DIR", origin_type="DIR",
key=de.key, parent=de.parent, org=self.orgname,
sub_type="REGISTER." + de.parent[1:].replace("/", "."),
mod_type=DirectoryModificationType.CREATE)
def register(self, parent, key, create_only=False, return_entry=False, ensure_parents=True, **kwargs):
"""
Add/replace an entry within directory, below a parent node or "/" root.
Note: Replaces (not merges) the attribute values of the entry if existing.
register will fail when a concurrent write was detected, meaning that the other writer wins.
@param create_only If True, does not change an already existing entry
@param return_entry If True, returns DirEntry object of prior entry, otherwise DirEntry attributes dict
@param ensure_parents If True, make sure that parent nodes exist
@retval DirEntry if previously existing
"""
if not (parent and key):
raise BadRequest("Illegal arguments")
if not type(parent) is str or not parent.startswith("/"):
raise BadRequest("Illegal arguments: parent")
dn = self._get_path(parent, key)
log.debug("Directory.register(%s): %s", dn, kwargs)
entry_old = None
cur_time = get_ion_ts()
# Must read existing entry by path to make sure to not create path twice
direntry = self._read_by_path(dn)
if direntry and create_only:
# We only wanted to make sure entry exists. Do not change
# NOTE: It is ambiguous to the caller whether we ran into this situation. Seems OK.
return direntry if return_entry else direntry.attributes
elif direntry:
old_rev, old_ts, old_attr = direntry._rev, direntry.ts_updated, direntry.attributes
direntry.attributes = kwargs
direntry.ts_updated = cur_time
try:
self.dir_store.update(direntry)
if self.events_enabled and self.container.has_capability(CCAP.EXCHANGE_MANAGER):
self.event_pub.publish_event(event_type="DirectoryModifiedEvent",
origin=self.orgname + ".DIR", origin_type="DIR",
key=key, parent=parent, org=self.orgname,
sub_type="REGISTER." + parent[1:].replace("/", "."),
mod_type=DirectoryModificationType.UPDATE)
except Conflict:
# Concurrent update - we accept that we finished the race second and give up
log.warn("Concurrent update to %s detected. We lost: %s", dn, kwargs)
if return_entry:
# Reset object back to prior state
direntry.attributes = old_attr
direntry.ts_updated = old_ts
direntry._rev = old_rev
entry_old = direntry
else:
entry_old = old_attr
else:
direntry = self._create_dir_entry(parent, key, attributes=kwargs, ts=cur_time)
if ensure_parents:
self._ensure_parents_exist([direntry])
try:
self.dir_store.create(direntry, create_unique_directory_id())
if self.events_enabled and self.container.has_capability(CCAP.EXCHANGE_MANAGER):
self.event_pub.publish_event(event_type="DirectoryModifiedEvent",
origin=self.orgname + ".DIR", origin_type="DIR",
key=key, parent=parent, org=self.orgname,
sub_type="REGISTER." + parent[1:].replace("/", "."),
mod_type=DirectoryModificationType.CREATE)
except BadRequest as ex:
if not ex.message.startswith("DirEntry already exists"):
raise
# Concurrent create - we accept that we finished the race second and give up
log.warn("Concurrent create of %s detected. We lost: %s", dn, kwargs)
return entry_old
def acquire_lock(self, key, timeout=LOCK_EXPIRES_DEFAULT, lock_holder=None, lock_info=None):
"""
Attempts to atomically acquire a lock with the given key and namespace.
If holder is given and holder already has the lock, renew.
Checks for expired locks.
@param timeout Int value of millis until lock expiration or 0 for no expiration
@param lock_holder Str value identifying lock holder for subsequent exclusive access
@param lock_info Dict value for additional attributes describing lock
@retval bool - could lock be acquired?
"""
if not key:
raise BadRequest("Missing argument: key")
if "/" in key:
raise BadRequest("Invalid argument value: key")
lock_attrs = {LOCK_EXPIRES_ATTR: get_ion_ts_millis() + timeout if timeout else 0,
LOCK_HOLDER_ATTR: lock_holder or ""}
if lock_info:
lock_attrs.update(lock_info)
expires = int(lock_attrs[LOCK_EXPIRES_ATTR]) # Check type just to be sure
if expires and get_ion_ts_millis() > expires:
raise BadRequest("Invalid lock expiration value: %s", expires)
direntry = self._create_dir_entry(LOCK_DIR_PATH, key, attributes=lock_attrs)
lock_result = False
try:
# This is an atomic operation. It relies on the unique key constraint of the directory service
self.dir_store.create(direntry, create_unique_directory_id())
lock_result = True
except BadRequest as ex:
if ex.message.startswith("DirEntry already exists"):
de_old = self.lookup(LOCK_DIR_PATH, key, return_entry=True)
if de_old:
if self._is_lock_expired(de_old):
# Lock is expired: remove, try to relock
# Note: even as holder, it's safer to reacquire in this case than renew
log.warn("Removing expired lock: %s/%s", de_old.parent, de_old.key)
try:
# This is safe, because of lock was deleted + recreated in the meantime, it has different id
self._delete_lock(de_old)
# Try recreate - may fail again due to concurrency
self.dir_store.create(direntry, create_unique_directory_id())
lock_result = True
except Exception:
log.exception("Error releasing/reacquiring expired lock %s", de_old.key)
elif lock_holder and de_old.attributes[LOCK_HOLDER_ATTR] == lock_holder:
# Holder currently holds the lock: renew
log.info("Renewing lock %s/%s for holder %s", de_old.parent, de_old.key, lock_holder)
de_old.attributes = lock_attrs
try:
self.dir_store.update(de_old)
lock_result = True
except Exception:
log.exception("Error renewing expired lock %s", de_old.key)
# We do nothing if we could not find the lock now...
else:
raise
log.debug("Directory.acquire_lock(%s): %s -> %s", key, lock_attrs, lock_result)
return lock_result
def register(self, parent, key, create_only=False, return_entry=False, ensure_parents=True, **kwargs):
"""
Add/replace an entry within directory, below a parent node or "/" root.
Note: Replaces (not merges) the attribute values of the entry if existing.
register will fail when a concurrent write was detected, meaning that the other writer wins.
@param create_only If True, does not change an already existing entry
@param return_entry If True, returns DirEntry object of prior entry, otherwise DirEntry attributes dict
@param ensure_parents If True, make sure that parent nodes exist
@retval DirEntry if previously existing
"""
if not (parent and key):
raise BadRequest("Illegal arguments")
if not type(parent) is str or not parent.startswith("/"):
raise BadRequest("Illegal arguments: parent")
dn = self._get_path(parent, key)
log.debug("Directory.register(%s): %s", dn, kwargs)
entry_old = None
cur_time = get_ion_ts()
# Must read existing entry by path to make sure to not create path twice
direntry = self._read_by_path(dn)
if direntry and create_only:
# We only wanted to make sure entry exists. Do not change
# NOTE: It is ambiguous to the caller whether we ran into this situation. Seems OK.
return direntry if return_entry else direntry.attributes
elif direntry:
old_rev, old_ts, old_attr = direntry._rev, direntry.ts_updated, direntry.attributes
direntry.attributes = kwargs
direntry.ts_updated = cur_time
try:
self.dir_store.update(direntry)
if self.events_enabled and self.container.has_capability(CCAP.EXCHANGE_MANAGER):
self.event_pub.publish_event(event_type="DirectoryModifiedEvent",
origin=self.orgname + ".DIR", origin_type="DIR",
key=key, parent=parent, org=self.orgname,
sub_type="REGISTER." + parent[1:].replace("/", "."),
mod_type=DirectoryModificationType.UPDATE)
except Conflict:
# Concurrent update - we accept that we finished the race second and give up
log.warn("Concurrent update to %s detected. We lost: %s", dn, kwargs)
if return_entry:
# Reset object back to prior state
direntry.attributes = old_attr
direntry.ts_updated = old_ts
direntry._rev = old_rev
entry_old = direntry
else:
entry_old = old_attr
else:
direntry = self._create_dir_entry(parent, key, attributes=kwargs, ts=cur_time)
if ensure_parents:
self._ensure_parents_exist([direntry])
try:
self.dir_store.create(direntry, create_unique_directory_id())
if self.events_enabled and self.container.has_capability(CCAP.EXCHANGE_MANAGER):
self.event_pub.publish_event(event_type="DirectoryModifiedEvent",
origin=self.orgname + ".DIR", origin_type="DIR",
key=key, parent=parent, org=self.orgname,
sub_type="REGISTER." + parent[1:].replace("/", "."),
mod_type=DirectoryModificationType.CREATE)
except BadRequest as ex:
if not ex.message.startswith("DirEntry already exists"):
raise
# Concurrent create - we accept that we finished the race second and give up
log.warn("Concurrent create of %s detected. We lost: %s", dn, kwargs)
return entry_old
def acquire_lock(self, key, timeout=LOCK_EXPIRES_DEFAULT, lock_holder=None, lock_info=None):
"""
Attempts to atomically acquire a lock with the given key and namespace.
If holder is given and holder already has the lock, renew.
Checks for expired locks.
@param timeout Secs until lock expiration or 0 for no expiration
@param lock_holder Str value identifying lock holder for subsequent exclusive access
@param lock_info Dict value for additional attributes describing lock
@retval bool - could lock be acquired?
"""
if not key:
raise BadRequest("Missing argument: key")
if "/" in key:
raise BadRequest("Invalid argument value: key")
lock_attrs = {LOCK_EXPIRES_ATTR: get_ion_ts_millis() + int(1000*timeout) if timeout else 0,
LOCK_HOLDER_ATTR: lock_holder or ""}
if lock_info:
lock_attrs.update(lock_info)
expires = int(lock_attrs[LOCK_EXPIRES_ATTR]) # Check type just to be sure
if expires and get_ion_ts_millis() > expires:
raise BadRequest("Invalid lock expiration value: %s", expires)
direntry = self._create_dir_entry(LOCK_DIR_PATH, key, attributes=lock_attrs)
lock_result = False
try:
# This is an atomic operation. It relies on the unique key constraint of the directory service
self.dir_store.create(direntry, create_unique_directory_id())
lock_result = True
except BadRequest as ex:
if ex.message.startswith("DirEntry already exists"):
de_old = self.lookup(LOCK_DIR_PATH, key, return_entry=True)
if de_old:
if self._is_lock_expired(de_old):
# Lock is expired: remove, try to relock
# Note: even as holder, it's safer to reacquire in this case than renew
log.warn("Removing expired lock: %s/%s", de_old.parent, de_old.key)
try:
# This is safe, because of lock was deleted + recreated in the meantime, it has different id
self._delete_lock(de_old)
# Try recreate - may fail again due to concurrency
self.dir_store.create(direntry, create_unique_directory_id())
lock_result = True
except BadRequest as ex:
if not ex.message.startswith("DirEntry already exists"):
log.exception("Error releasing/reacquiring expired lock %s", de_old.key)
except Exception:
log.exception("Error releasing/reacquiring expired lock %s", de_old.key)
elif lock_holder and de_old.attributes[LOCK_HOLDER_ATTR] == lock_holder:
# Holder currently holds the lock: renew
log.debug("Renewing lock %s/%s for holder %s", de_old.parent, de_old.key, lock_holder)
de_old.attributes = lock_attrs
try:
self.dir_store.update(de_old)
lock_result = True
except Exception:
log.exception("Error renewing expired lock %s", de_old.key)
# We do nothing if we could not find the lock now...
else:
raise
log.debug("Directory.acquire_lock(%s): %s -> %s", key, lock_attrs, lock_result)
return lock_result
