def _load_datastore(self, path=None, ds_name=None, ignore_errors=True):
ds = CouchDataStore(ds_name, config=self.config, scope=self.sysname)
try:
objects = []
for fn in os.listdir(path):
fp = os.path.join(path, fn)
try:
with open(fp, 'r') as f:
yaml_text = f.read()
obj = yaml.load(yaml_text)
if obj and type(obj) is list and obj[0] == "COMPACTDUMP":
objects.extend(obj[1:])
else:
objects.append(obj)
except Exception as ex:
if ignore_errors:
log.warn("load error id=%s err=%s" % (fn, str(ex)))
else:
raise ex
if objects:
for obj in objects:
if "_rev" in obj:
del obj["_rev"]
try:
res = ds.create_doc_mult(objects)
log.info("DatastoreLoader: Loaded %s objects into %s" % (len(res), ds_name))
except Exception as ex:
if ignore_errors:
log.warn("load error id=%s err=%s" % (fn, str(ex)))
else:
raise ex
finally:
ds.close()
def _load_datastore(self, path=None, ds_name=None, ignore_errors=True):
ds = CouchDataStore(ds_name, config=self.config, scope=self.sysname)
try:
objects = []
for fn in os.listdir(path):
fp = os.path.join(path, fn)
try:
with open(fp, 'r') as f:
yaml_text = f.read()
obj = yaml.load(yaml_text)
if obj and type(obj) is list and obj[0] == "COMPACTDUMP":
objects.extend(obj[1:])
else:
objects.append(obj)
except Exception as ex:
if ignore_errors:
log.warn("load error id=%s err=%s" % (fn, str(ex)))
else:
raise ex
if objects:
for obj in objects:
if "_rev" in obj:
del obj["_rev"]
try:
res = ds.create_doc_mult(objects)
log.info("DatastoreLoader: Loaded %s objects into %s" %
(len(res), ds_name))
except Exception as ex:
if ignore_errors:
log.warn("load error id=%s err=%s" % (fn, str(ex)))
else:
raise ex
finally:
ds.close()
class ResourceRegistryStandalone(object):
"""
Resource Registry service standalone class
"""
def __init__(self, sysname=None, orgname=None, config=None):
self.orgname = orgname or get_safe(config, 'system.root_org', 'ION')
sysname = sysname or get_default_sysname()
self.datastore_name = "resources"
self.datastore = CouchDataStore(self.datastore_name, config=config, scope=sysname)
try:
self.datastore.read_doc("_design/directory")
except NotFound:
self.datastore.define_profile_views("RESOURCES")
def close(self):
self.datastore.close()
self.datastore = None
def create(self, object=None, actor_id=None, lcstate=None):
if object is None:
raise BadRequest("Object not present")
if not "type_" in object:
raise BadRequest("Object is not an IonObject")
cur_time = get_ion_ts()
object['lcstate'] = lcstate or "DEPLOYED_AVAILABLE"
object['ts_created'] = cur_time
object['ts_updated'] = cur_time
new_res_id = create_unique_resource_id()
res_id, rev = self.datastore.create_doc(object, new_res_id)
if actor_id and actor_id != 'anonymous':
self.create_association(res_id, "hasOwner", actor_id)
return res
def create_mult(self, res_list, lcstate=None):
cur_time = get_ion_ts()
for resobj in res_list:
resobj['lcstate'] = lcstate or "DEPLOYED_AVAILABLE"
resobj['ts_created'] = cur_time
resobj['ts_updated'] = cur_time
id_list = [create_unique_resource_id() for i in xrange(len(res_list))]
res = self.datastore.create_doc_mult(res_list, id_list)
res_list = [(rid,rrv) for success,rid,rrv in res]
return res_list
def read(self, object_id='', rev_id=''):
if not object_id:
raise BadRequest("The object_id parameter is an empty string")
return self.datastore.read_doc(object_id, rev_id)
def read_mult(self, object_ids=None):
if not object_ids:
raise BadRequest("The object_ids parameter is empty")
return self.datastore.read_doc_mult(object_ids)
def create_association(self, subject=None, predicate=None, obj=None, assoc_type='H2H'):
"""
Create an association between two IonObjects with a given predicate
"""
if not subject or not predicate or not obj:
raise BadRequest("Association must have all elements set")
if type(subject) is str:
subject_id = subject
subject = self.read(subject_id)
else:
if "_id" not in subject or "_rev" not in subject:
raise BadRequest("Subject id or rev not available")
subject_id = subject._id
st = type(subject).__name__
if type(obj) is str:
object_id = obj
obj = self.read(object_id)
else:
if "_id" not in obj or "_rev" not in obj:
raise BadRequest("Object id or rev not available")
object_id = obj._id
ot = type(obj).__name__
assoc_type = assoc_type or 'H2H'
if not assoc_type in AT:
raise BadRequest("Unsupported assoc_type: %s" % assoc_type)
# Check that subject and object type are permitted by association definition
# Note: Need import here, so that import orders are not screwed up
from pyon.core.registry import getextends
from pyon.ion.resource import Predicates
from pyon.core.bootstrap import IonObject
assoc = dict(type_="Association",
at=assoc_type,
s=subject_id, st=st, srv=subject._rev,
p=predicate,
o=object_id, ot=ot, orv=obj._rev,
ts=get_ion_ts())
return self.datastore.create_doc(assoc, create_unique_association_id())
def find_by_type(self, restype, id_only=False, **kwargs):
start_key = [restype]
end_key = [restype]
res = self.datastore.find_docs_by_view('resource', 'by_type',
start_key=start_key, end_key=end_key, id_only=id_only, **kwargs)
if id_only:
match = [docid for docid, indexkey, doc in res]
else:
match = [doc for docid, indexkey, doc in res]
return match
class ResourceRegistryStandalone(object):
"""
Resource Registry service standalone class
"""
def __init__(self, sysname=None, orgname=None, config=None):
self.orgname = orgname or get_safe(config, 'system.root_org', 'ION')
sysname = sysname or get_default_sysname()
self.datastore_name = "resources"
self.datastore = CouchDataStore(self.datastore_name,
config=config,
scope=sysname)
try:
self.datastore.read_doc("_design/directory")
except NotFound:
self.datastore.define_profile_views("RESOURCES")
def close(self):
self.datastore.close()
self.datastore = None
def create(self, object=None, actor_id=None, lcstate=None):
if object is None:
raise BadRequest("Object not present")
if not "type_" in object:
raise BadRequest("Object is not an IonObject")
cur_time = get_ion_ts()
object['lcstate'] = lcstate or "DEPLOYED_AVAILABLE"
object['ts_created'] = cur_time
object['ts_updated'] = cur_time
new_res_id = create_unique_resource_id()
res_id, rev = self.datastore.create_doc(object, new_res_id)
if actor_id and actor_id != 'anonymous':
self.create_association(res_id, "hasOwner", actor_id)
return res
def create_mult(self, res_list, lcstate=None):
cur_time = get_ion_ts()
for resobj in res_list:
resobj['lcstate'] = lcstate or "DEPLOYED_AVAILABLE"
resobj['ts_created'] = cur_time
resobj['ts_updated'] = cur_time
id_list = [create_unique_resource_id() for i in xrange(len(res_list))]
res = self.datastore.create_doc_mult(res_list, id_list)
res_list = [(rid, rrv) for success, rid, rrv in res]
return res_list
def read(self, object_id='', rev_id=''):
if not object_id:
raise BadRequest("The object_id parameter is an empty string")
return self.datastore.read_doc(object_id, rev_id)
def read_mult(self, object_ids=None):
if not object_ids:
raise BadRequest("The object_ids parameter is empty")
return self.datastore.read_doc_mult(object_ids)
def create_association(self,
subject=None,
predicate=None,
obj=None,
assoc_type='H2H'):
"""
Create an association between two IonObjects with a given predicate
"""
if not subject or not predicate or not obj:
raise BadRequest("Association must have all elements set")
if type(subject) is str:
subject_id = subject
subject = self.read(subject_id)
else:
if "_id" not in subject or "_rev" not in subject:
raise BadRequest("Subject id or rev not available")
subject_id = subject._id
st = type(subject).__name__
if type(obj) is str:
object_id = obj
obj = self.read(object_id)
else:
if "_id" not in obj or "_rev" not in obj:
raise BadRequest("Object id or rev not available")
object_id = obj._id
ot = type(obj).__name__
assoc_type = assoc_type or 'H2H'
if not assoc_type in AT:
raise BadRequest("Unsupported assoc_type: %s" % assoc_type)
# Check that subject and object type are permitted by association definition
# Note: Need import here, so that import orders are not screwed up
from pyon.core.registry import getextends
from pyon.ion.resource import Predicates
from pyon.core.bootstrap import IonObject
assoc = dict(type_="Association",
at=assoc_type,
s=subject_id,
st=st,
srv=subject._rev,
p=predicate,
o=object_id,
ot=ot,
orv=obj._rev,
ts=get_ion_ts())
return self.datastore.create_doc(assoc, create_unique_association_id())
def find_by_type(self, restype, id_only=False, **kwargs):
start_key = [restype]
end_key = [restype]
res = self.datastore.find_docs_by_view('resource',
'by_type',
start_key=start_key,
end_key=end_key,
id_only=id_only,
**kwargs)
if id_only:
match = [docid for docid, indexkey, doc in res]
else:
match = [doc for docid, indexkey, doc in res]
return match
class DirectoryStandalone(object):
"""
Directory service standalone class
"""
def __init__(self, sysname=None, orgname=None, config=None):
self.orgname = orgname or get_safe(config, 'system.root_org', 'ION')
sysname = sysname or get_default_sysname()
self.datastore_name = "resources"
self.datastore = CouchDataStore(self.datastore_name, config=config, scope=sysname)
try:
self.datastore.read_doc("_design/directory")
except NotFound:
self.datastore.define_profile_views("RESOURCES")
def close(self):
self.datastore.close()
self.datastore = None
def _get_path(self, parent, key):
"""
Returns the qualified directory path for a directory entry.
"""
if parent == "/":
return parent + key
elif parent.startswith("/"):
return parent + "/" + key
else:
raise BadRequest("Illegal parent: %s" % parent)
def _get_key(self, path):
"""
Returns the key from a qualified directory path
"""
parent, key = path.rsplit("/", 1)
return key
def _read_by_path(self, path, orgname=None):
"""
Given a qualified path, find entry in directory and return DirEntry
document or None if not found
"""
if path is None:
raise BadRequest("Illegal arguments")
orgname = orgname or self.orgname
parent, key = path.rsplit("/", 1)
parent = parent or "/"
find_key = [orgname, key, parent]
view_res = self.datastore.find_docs_by_view('directory', 'by_key', key=find_key, id_only=True)
match = [doc for docid, index, doc in view_res]
if len(match) > 1:
raise Inconsistent("More than one directory entry found for key %s" % path)
elif match:
return match[0]
return None
def lookup(self, parent, key=None, return_entry=False):
"""
Read entry residing in directory at parent node level.
"""
path = self._get_path(parent, key) if key else parent
direntry = self._read_by_path(path)
if return_entry:
return direntry
else:
return direntry['attributes'] if direntry else None
def register(self, parent, key, create_only=False, **kwargs):
"""
Add/replace an entry within directory, below a parent node or "/".
Note: Replaces (not merges) the attribute values of the entry if existing
@retval DirEntry if previously existing
"""
if not (parent and key):
raise BadRequest("Illegal arguments")
if not type(parent) is str or not parent.startswith("/"):
raise BadRequest("Illegal arguments: parent")
dn = self._get_path(parent, key)
entry_old = None
direntry = self._read_by_path(dn)
cur_time = get_ion_ts()
if direntry and create_only:
# We only wanted to make sure entry exists. Do not change
return direntry
elif direntry:
# Change existing entry's attributes
entry_old = direntry.get('attributes')
direntry['attributes'] = kwargs
direntry['ts_updated'] = cur_time
self.datastore.update_doc(direntry)
else:
doc = self._create_dir_entry(object_id=create_unique_directory_id(), parent=parent, key=key,
attributes=kwargs)
self.datastore.create_doc(doc)
return entry_old
def register_mult(self, entries):
"""
Registers multiple directory entries efficiently in one datastore access.
Note: this fails of entries are currently existing, so works for create only.
"""
if type(entries) not in (list, tuple):
raise BadRequest("Bad entries type")
de_list = []
cur_time = get_ion_ts()
for parent, key, attrs in entries:
de = self._create_dir_entry(object_id=create_unique_directory_id(), parent=parent, key=key,
attributes=attrs, ts_created=cur_time, ts_updated=cur_time)
de_list.append(de)
self.datastore.create_doc_mult(de_list)
def _update_dir_entry(self, doc, parent, key, attributes=None, ts_updated=''):
doc['attributes'] = attributes or {}
doc['key'] = key
doc['parent'] = parent
doc['ts_updated'] = ts_updated or get_ion_ts()
return doc
def _create_dir_entry(self, object_id, parent, key, attributes=None, ts_created='', ts_updated=''):
doc = {}
doc['_id'] = object_id
doc['type_'] = 'DirEntry'
doc['attributes'] = attributes or {}
doc['key'] = key
doc['parent'] = parent
doc['org'] = self.orgname
doc['ts_created'] = ts_created or get_ion_ts()
doc['ts_updated'] = ts_updated or get_ion_ts()
return doc
def find_child_entries(self, parent='/', direct_only=True, **kwargs):
"""
Return all child entries (ordered by path) for the given parent path.
Does not return the parent itself. Optionally returns child of child entries.
Additional kwargs are applied to constrain the search results (limit, descending, skip).
@param parent Path to parent (must start with "/")
@param direct_only If False, includes child of child entries
@retval A list of DirEntry objects for the matches
"""
if not type(parent) is str or not parent.startswith("/"):
raise BadRequest("Illegal argument parent: %s" % parent)
if direct_only:
start_key = [self.orgname, parent, 0]
end_key = [self.orgname, parent]
res = self.datastore.find_docs_by_view('directory', 'by_parent',
start_key=start_key, end_key=end_key, id_only=True, **kwargs)
else:
path = parent[1:].split("/")
start_key = [self.orgname, path, 0]
end_key = [self.orgname, list(path) + ["ZZZZZZ"]]
res = self.datastore.find_docs_by_view('directory', 'by_path',
start_key=start_key, end_key=end_key, id_only=True, **kwargs)
match = [doc for docid, indexkey, doc in res]
return match
class DirectoryStandalone(object):
"""
Directory service standalone class
"""
def __init__(self, sysname=None, orgname=None, config=None):
self.orgname = orgname or get_safe(config, 'system.root_org', 'ION')
sysname = sysname or get_default_sysname()
self.datastore_name = "resources"
self.datastore = CouchDataStore(self.datastore_name, config=config, scope=sysname)
try:
self.datastore.read_doc("_design/directory")
except NotFound:
self.datastore.define_profile_views("RESOURCES")
def close(self):
self.datastore.close()
self.datastore = None
def _get_path(self, parent, key):
"""
Returns the qualified directory path for a directory entry.
"""
if parent == "/":
return parent + key
elif parent.startswith("/"):
return parent + "/" + key
else:
raise BadRequest("Illegal parent: %s" % parent)
def _get_key(self, path):
"""
Returns the key from a qualified directory path
"""
parent, key = path.rsplit("/", 1)
return key
def _read_by_path(self, path, orgname=None):
"""
Given a qualified path, find entry in directory and return DirEntry
document or None if not found
"""
if path is None:
raise BadRequest("Illegal arguments")
orgname = orgname or self.orgname
parent, key = path.rsplit("/", 1)
parent = parent or "/"
find_key = [orgname, key, parent]
view_res = self.datastore.find_docs_by_view('directory', 'by_key', key=find_key, id_only=True)
match = [doc for docid, index, doc in view_res]
if len(match) > 1:
raise Inconsistent("More than one directory entry found for key %s" % path)
elif match:
return match[0]
return None
def lookup(self, parent, key=None, return_entry=False):
"""
Read entry residing in directory at parent node level.
"""
path = self._get_path(parent, key) if key else parent
direntry = self._read_by_path(path)
if return_entry:
return direntry
else:
return direntry['attributes'] if direntry else None
def register(self, parent, key, create_only=False, **kwargs):
"""
Add/replace an entry within directory, below a parent node or "/".
Note: Replaces (not merges) the attribute values of the entry if existing
@retval DirEntry if previously existing
"""
if not (parent and key):
raise BadRequest("Illegal arguments")
if not type(parent) is str or not parent.startswith("/"):
raise BadRequest("Illegal arguments: parent")
dn = self._get_path(parent, key)
entry_old = None
direntry = self._read_by_path(dn)
cur_time = get_ion_ts()
if direntry and create_only:
# We only wanted to make sure entry exists. Do not change
return direntry
elif direntry:
# Change existing entry's attributes
entry_old = direntry.get('attributes')
direntry['attributes'] = kwargs
direntry['ts_updated'] = cur_time
self.datastore.update_doc(direntry)
else:
doc = self._create_dir_entry(object_id=create_unique_directory_id(), parent=parent, key=key,
attributes=kwargs)
self.datastore.create_doc(doc)
return entry_old
def register_mult(self, entries):
"""
Registers multiple directory entries efficiently in one datastore access.
Note: this fails of entries are currently existing, so works for create only.
"""
if type(entries) not in (list, tuple):
raise BadRequest("Bad entries type")
de_list = []
cur_time = get_ion_ts()
for parent, key, attrs in entries:
de = self._create_dir_entry(object_id=create_unique_directory_id(), parent=parent, key=key,
attributes=attrs, ts_created=cur_time, ts_updated=cur_time)
de_list.append(de)
self.datastore.create_doc_mult(de_list)
def _update_dir_entry(self, doc, parent, key, attributes=None, ts_updated=''):
doc['attributes'] = attributes or {}
doc['key'] = key
doc['parent'] = parent
doc['ts_updated'] = ts_updated or get_ion_ts()
return doc
def _create_dir_entry(self, object_id, parent, key, attributes=None, ts_created='', ts_updated=''):
doc = {}
doc['_id'] = object_id
doc['type_'] = 'DirEntry'
doc['attributes'] = attributes or {}
doc['key'] = key
doc['parent'] = parent
doc['org'] = self.orgname
doc['ts_created'] = ts_created or get_ion_ts()
doc['ts_updated'] = ts_updated or get_ion_ts()
return doc
def find_child_entries(self, parent='/', direct_only=True, **kwargs):
"""
Return all child entries (ordered by path) for the given parent path.
Does not return the parent itself. Optionally returns child of child entries.
Additional kwargs are applied to constrain the search results (limit, descending, skip).
@param parent Path to parent (must start with "/")
@param direct_only If False, includes child of child entries
@retval A list of DirEntry objects for the matches
"""
if not type(parent) is str or not parent.startswith("/"):
raise BadRequest("Illegal argument parent: %s" % parent)
if direct_only:
start_key = [self.orgname, parent, 0]
end_key = [self.orgname, parent]
res = self.datastore.find_docs_by_view('directory', 'by_parent',
start_key=start_key, end_key=end_key, id_only=True, **kwargs)
else:
path = parent[1:].split("/")
start_key = [self.orgname, path, 0]
end_key = [self.orgname, list(path) + ["ZZZZZZ"]]
res = self.datastore.find_docs_by_view('directory', 'by_path',
start_key=start_key, end_key=end_key, id_only=True, **kwargs)
match = [doc for docid, indexkey, doc in res]
return match
