def register(self, parent, key, create_only=False, **kwargs): """ Add/replace an entry within directory, below a parent node or "/". Note: Replaces (not merges) the attribute values of the entry if existing @param create_only If True, does not change an existing entry @retval DirEntry if previously existing """ if not (parent and key): raise BadRequest("Illegal arguments") if not type(parent) is str or not parent.startswith("/"): raise BadRequest("Illegal arguments: parent") dn = self._get_path(parent, key) log.debug("Directory.register(%s): %s", dn, kwargs) entry_old = None cur_time = get_ion_ts() # Must read existing entry by path to make sure to not create path twice direntry = self._read_by_path(dn) if direntry and create_only: # We only wanted to make sure entry exists. Do not change return direntry elif direntry: entry_old = direntry.attributes direntry.attributes = kwargs direntry.ts_updated = cur_time # TODO: This may fail because of concurrent update self.dir_store.update(direntry) else: direntry = self._create_dir_entry(parent, key, attributes=kwargs, ts=cur_time) self.dir_store.create(direntry, create_unique_directory_id()) return entry_old
def register(self, parent, key, create_only=False, **kwargs): """ Add/replace an entry within directory, below a parent node or "/". Note: Replaces (not merges) the attribute values of the entry if existing @retval DirEntry if previously existing """ if not (parent and key): raise BadRequest("Illegal arguments") if not type(parent) is str or not parent.startswith("/"): raise BadRequest("Illegal arguments: parent") dn = self._get_path(parent, key) entry_old = None direntry = self._read_by_path(dn) cur_time = get_ion_ts() if direntry and create_only: # We only wanted to make sure entry exists. Do not change return direntry elif direntry: # Change existing entry's attributes entry_old = direntry.get('attributes') direntry['attributes'] = kwargs direntry['ts_updated'] = cur_time self.datastore.update_doc(direntry) else: doc = self._create_dir_entry(object_id=create_unique_directory_id(), parent=parent, key=key, attributes=kwargs) self.datastore.create_doc(doc) return entry_old
def register(self, parent, key, create_only=False, **kwargs): """ Add/replace an entry within directory, below a parent node or "/". Note: Replaces (not merges) the attribute values of the entry if existing @param create_only If True, does not change an existing entry @retval DirEntry if previously existing """ if not (parent and key): raise BadRequest("Illegal arguments") if not type(parent) is str or not parent.startswith("/"): raise BadRequest("Illegal arguments: parent") dn = self._get_path(parent, key) log.debug("Directory.register(%s): %s", dn, kwargs) entry_old = None cur_time = get_ion_ts() # Must read existing entry by path to make sure to not create path twice direntry = self._read_by_path(dn) if direntry and create_only: # We only wanted to make sure entry exists. Do not change return direntry elif direntry: entry_old = direntry.attributes direntry.attributes = kwargs direntry.ts_updated = cur_time # TODO: This may fail because of concurrent update self.dir_store.update(direntry) else: direntry = self._create_dir_entry(parent, key, attributes=kwargs, ts=cur_time) self._ensure_parents_exist([direntry]) self.dir_store.create(direntry, create_unique_directory_id()) return entry_old
def register_mult(self, entries): """ Registers multiple directory entries efficiently in one datastore access. Note: this fails of entries are currently existing, so works for create only. """ if type(entries) not in (list, tuple): raise BadRequest("Bad entries type") de_list = [] cur_time = get_ion_ts() for parent, key, attrs in entries: de = self._create_dir_entry(object_id=create_unique_directory_id(), parent=parent, key=key, attributes=attrs, ts_created=cur_time, ts_updated=cur_time) de_list.append(de) self.datastore.create_doc_mult(de_list)
def register_mult(self, entries): """ Registers multiple directory entries efficiently in one datastore access. Note: this fails of entries are currently existing, so works for create only. """ if type(entries) not in (list, tuple): raise BadRequest("Bad entries type") de_list = [] cur_time = get_ion_ts() for parent, key, attrs in entries: direntry = self._create_dir_entry(parent, key, attributes=attrs, ts=cur_time) de_list.append(direntry) deid_list = [create_unique_directory_id() for i in xrange(len(de_list))] self.dir_store.create_mult(de_list, deid_list)
def _ensure_parents_exist(self, entry_list, create=True): parents_list = self._get_unique_parents(entry_list) pe_list = [] try: for parent in parents_list: pe = self.lookup(parent) if pe is None: pp, pk = parent.rsplit("/", 1) direntry = self._create_dir_entry(parent=pp, key=pk) pe_list.append(direntry) if create: self.dir_store.create(direntry, create_unique_directory_id()) except Exception as ex: log.warn("_ensure_parents_exist(): Error creating directory parents", exc_info=True) return pe_list
def _ensure_parents_exist(self, entry_list, create=True): parents_list = self._get_unique_parents(entry_list) pe_list = [] try: for parent in parents_list: pe = self.lookup(parent) if pe is None: pp, pk = parent.rsplit("/", 1) doc = self._create_dir_entry(object_id=create_unique_directory_id(), parent=pp, key=pk) pe_list.append(doc) if create: self.datastore.create_doc(doc) except Exception as ex: print "_ensure_parents_exist(): Error creating directory parents", ex return pe_list
def register(self, parent, key, create_only=False, ensure_parents=True, **kwargs): """ Add/replace an entry within directory, below a parent node or "/". Note: Replaces (not merges) the attribute values of the entry if existing @retval DirEntry if previously existing """ if not (parent and key): raise BadRequest("Illegal arguments") if not type(parent) is str or not parent.startswith("/"): raise BadRequest("Illegal arguments: parent") dn = self._get_path(parent, key) entry_old = None direntry = self._read_by_path(dn) cur_time = get_ion_ts() if direntry and create_only: # We only wanted to make sure entry exists. Do not change return direntry elif direntry: # Change existing entry's attributes entry_old = direntry.get("attributes") direntry["attributes"] = kwargs direntry["ts_updated"] = cur_time try: self.datastore.update_doc(direntry) except Conflict: # Concurrent update - we accept that we finished the race second and give up pass else: doc = self._create_dir_entry( object_id=create_unique_directory_id(), parent=parent, key=key, attributes=kwargs ) if ensure_parents: self._ensure_parents_exist([doc]) try: self.datastore.create_doc(doc) except BadRequest as ex: if not ex.message.startswith("DirEntry already exists"): raise # Concurrent create - we accept that we finished the race second and give up return entry_old
def register(self, parent, key, create_only=False, ensure_parents=True, **kwargs): """ Add/replace an entry within directory, below a parent node or "/". Note: Replaces (not merges) the attribute values of the entry if existing @retval DirEntry if previously existing """ if not (parent and key): raise BadRequest("Illegal arguments") if not type(parent) is str or not parent.startswith("/"): raise BadRequest("Illegal arguments: parent") dn = self._get_path(parent, key) entry_old = None direntry = self._read_by_path(dn) cur_time = get_ion_ts() if direntry and create_only: # We only wanted to make sure entry exists. Do not change return direntry elif direntry: # Change existing entry's attributes entry_old = direntry.get('attributes') direntry['attributes'] = kwargs direntry['ts_updated'] = cur_time try: self.datastore.update_doc(direntry) except Conflict: # Concurrent update - we accept that we finished the race second and give up pass else: doc = self._create_dir_entry(object_id=create_unique_directory_id(), parent=parent, key=key, attributes=kwargs) if ensure_parents: self._ensure_parents_exist([doc]) try: self.datastore.create_doc(doc) except BadRequest as ex: if not ex.message.startswith("DirEntry already exists"): raise # Concurrent create - we accept that we finished the race second and give up return entry_old
def _ensure_parents_exist(self, entry_list, create=True): parents_list = self._get_unique_parents(entry_list) pe_list = [] try: for parent in parents_list: pe = self.lookup(parent) if pe is None: pp, pk = parent.rsplit("/", 1) doc = self._create_dir_entry(object_id=create_unique_directory_id(), parent=pp, key=pk) pe_list.append(doc) if create: try: self.datastore.create_doc(doc) except BadRequest as ex: if not ex.message.startswith("DirEntry already exists"): raise # Else: Concurrent create except Exception as ex: print "_ensure_parents_exist(): Error creating directory parents", ex return pe_list
def register_mult(self, entries): """ Registers multiple directory entries efficiently in one datastore access. Note: this fails if entries are already existing, so works for create only. """ if type(entries) not in (list, tuple): raise BadRequest("Bad entries type") de_list = [] cur_time = get_ion_ts() for parent, key, attrs in entries: direntry = self._create_dir_entry(parent, key, attributes=attrs, ts=cur_time) de_list.append(direntry) pe_list = self._ensure_parents_exist(de_list, create=False) de_list.extend(pe_list) deid_list = [create_unique_directory_id() for i in xrange(len(de_list))] self.dir_store.create_mult(de_list, deid_list) if self.events_enabled and self.container.has_capability(CCAP.EXCHANGE_MANAGER): for de in de_list: self.event_pub.publish_event(event_type="DirectoryModifiedEvent", origin=self.orgname + ".DIR", origin_type="DIR", key=de.key, parent=de.parent, org=self.orgname, sub_type="REGISTER." + de.parent[1:].replace("/", "."), mod_type=DirectoryModificationType.CREATE)
def register(self, parent, key, create_only=False, return_entry=False, ensure_parents=True, **kwargs): """ Add/replace an entry within directory, below a parent node or "/" root. Note: Replaces (not merges) the attribute values of the entry if existing. register will fail when a concurrent write was detected, meaning that the other writer wins. @param create_only If True, does not change an already existing entry @param return_entry If True, returns DirEntry object of prior entry, otherwise DirEntry attributes dict @param ensure_parents If True, make sure that parent nodes exist @retval DirEntry if previously existing """ if not (parent and key): raise BadRequest("Illegal arguments") if not type(parent) is str or not parent.startswith("/"): raise BadRequest("Illegal arguments: parent") dn = self._get_path(parent, key) log.debug("Directory.register(%s): %s", dn, kwargs) entry_old = None cur_time = get_ion_ts() # Must read existing entry by path to make sure to not create path twice direntry = self._read_by_path(dn) if direntry and create_only: # We only wanted to make sure entry exists. Do not change # NOTE: It is ambiguous to the caller whether we ran into this situation. Seems OK. return direntry if return_entry else direntry.attributes elif direntry: old_rev, old_ts, old_attr = direntry._rev, direntry.ts_updated, direntry.attributes direntry.attributes = kwargs direntry.ts_updated = cur_time try: self.dir_store.update(direntry) if self.events_enabled and self.container.has_capability(CCAP.EXCHANGE_MANAGER): self.event_pub.publish_event(event_type="DirectoryModifiedEvent", origin=self.orgname + ".DIR", origin_type="DIR", key=key, parent=parent, org=self.orgname, sub_type="REGISTER." + parent[1:].replace("/", "."), mod_type=DirectoryModificationType.UPDATE) except Conflict: # Concurrent update - we accept that we finished the race second and give up log.warn("Concurrent update to %s detected. We lost: %s", dn, kwargs) if return_entry: # Reset object back to prior state direntry.attributes = old_attr direntry.ts_updated = old_ts direntry._rev = old_rev entry_old = direntry else: entry_old = old_attr else: direntry = self._create_dir_entry(parent, key, attributes=kwargs, ts=cur_time) if ensure_parents: self._ensure_parents_exist([direntry]) try: self.dir_store.create(direntry, create_unique_directory_id()) if self.events_enabled and self.container.has_capability(CCAP.EXCHANGE_MANAGER): self.event_pub.publish_event(event_type="DirectoryModifiedEvent", origin=self.orgname + ".DIR", origin_type="DIR", key=key, parent=parent, org=self.orgname, sub_type="REGISTER." + parent[1:].replace("/", "."), mod_type=DirectoryModificationType.CREATE) except BadRequest as ex: if not ex.message.startswith("DirEntry already exists"): raise # Concurrent create - we accept that we finished the race second and give up log.warn("Concurrent create of %s detected. We lost: %s", dn, kwargs) return entry_old
def acquire_lock(self, key, timeout=LOCK_EXPIRES_DEFAULT, lock_holder=None, lock_info=None): """ Attempts to atomically acquire a lock with the given key and namespace. If holder is given and holder already has the lock, renew. Checks for expired locks. @param timeout Int value of millis until lock expiration or 0 for no expiration @param lock_holder Str value identifying lock holder for subsequent exclusive access @param lock_info Dict value for additional attributes describing lock @retval bool - could lock be acquired? """ if not key: raise BadRequest("Missing argument: key") if "/" in key: raise BadRequest("Invalid argument value: key") lock_attrs = {LOCK_EXPIRES_ATTR: get_ion_ts_millis() + timeout if timeout else 0, LOCK_HOLDER_ATTR: lock_holder or ""} if lock_info: lock_attrs.update(lock_info) expires = int(lock_attrs[LOCK_EXPIRES_ATTR]) # Check type just to be sure if expires and get_ion_ts_millis() > expires: raise BadRequest("Invalid lock expiration value: %s", expires) direntry = self._create_dir_entry(LOCK_DIR_PATH, key, attributes=lock_attrs) lock_result = False try: # This is an atomic operation. It relies on the unique key constraint of the directory service self.dir_store.create(direntry, create_unique_directory_id()) lock_result = True except BadRequest as ex: if ex.message.startswith("DirEntry already exists"): de_old = self.lookup(LOCK_DIR_PATH, key, return_entry=True) if de_old: if self._is_lock_expired(de_old): # Lock is expired: remove, try to relock # Note: even as holder, it's safer to reacquire in this case than renew log.warn("Removing expired lock: %s/%s", de_old.parent, de_old.key) try: # This is safe, because of lock was deleted + recreated in the meantime, it has different id self._delete_lock(de_old) # Try recreate - may fail again due to concurrency self.dir_store.create(direntry, create_unique_directory_id()) lock_result = True except Exception: log.exception("Error releasing/reacquiring expired lock %s", de_old.key) elif lock_holder and de_old.attributes[LOCK_HOLDER_ATTR] == lock_holder: # Holder currently holds the lock: renew log.info("Renewing lock %s/%s for holder %s", de_old.parent, de_old.key, lock_holder) de_old.attributes = lock_attrs try: self.dir_store.update(de_old) lock_result = True except Exception: log.exception("Error renewing expired lock %s", de_old.key) # We do nothing if we could not find the lock now... else: raise log.debug("Directory.acquire_lock(%s): %s -> %s", key, lock_attrs, lock_result) return lock_result
def acquire_lock(self, key, timeout=LOCK_EXPIRES_DEFAULT, lock_holder=None, lock_info=None): """ Attempts to atomically acquire a lock with the given key and namespace. If holder is given and holder already has the lock, renew. Checks for expired locks. @param timeout Secs until lock expiration or 0 for no expiration @param lock_holder Str value identifying lock holder for subsequent exclusive access @param lock_info Dict value for additional attributes describing lock @retval bool - could lock be acquired? """ if not key: raise BadRequest("Missing argument: key") if "/" in key: raise BadRequest("Invalid argument value: key") lock_attrs = {LOCK_EXPIRES_ATTR: get_ion_ts_millis() + int(1000*timeout) if timeout else 0, LOCK_HOLDER_ATTR: lock_holder or ""} if lock_info: lock_attrs.update(lock_info) expires = int(lock_attrs[LOCK_EXPIRES_ATTR]) # Check type just to be sure if expires and get_ion_ts_millis() > expires: raise BadRequest("Invalid lock expiration value: %s", expires) direntry = self._create_dir_entry(LOCK_DIR_PATH, key, attributes=lock_attrs) lock_result = False try: # This is an atomic operation. It relies on the unique key constraint of the directory service self.dir_store.create(direntry, create_unique_directory_id()) lock_result = True except BadRequest as ex: if ex.message.startswith("DirEntry already exists"): de_old = self.lookup(LOCK_DIR_PATH, key, return_entry=True) if de_old: if self._is_lock_expired(de_old): # Lock is expired: remove, try to relock # Note: even as holder, it's safer to reacquire in this case than renew log.warn("Removing expired lock: %s/%s", de_old.parent, de_old.key) try: # This is safe, because of lock was deleted + recreated in the meantime, it has different id self._delete_lock(de_old) # Try recreate - may fail again due to concurrency self.dir_store.create(direntry, create_unique_directory_id()) lock_result = True except BadRequest as ex: if not ex.message.startswith("DirEntry already exists"): log.exception("Error releasing/reacquiring expired lock %s", de_old.key) except Exception: log.exception("Error releasing/reacquiring expired lock %s", de_old.key) elif lock_holder and de_old.attributes[LOCK_HOLDER_ATTR] == lock_holder: # Holder currently holds the lock: renew log.debug("Renewing lock %s/%s for holder %s", de_old.parent, de_old.key, lock_holder) de_old.attributes = lock_attrs try: self.dir_store.update(de_old) lock_result = True except Exception: log.exception("Error renewing expired lock %s", de_old.key) # We do nothing if we could not find the lock now... else: raise log.debug("Directory.acquire_lock(%s): %s -> %s", key, lock_attrs, lock_result) return lock_result