def on_init(self):
     resource_registry_name = sys_name + "_resources"
     resource_registry_name = resource_registry_name.lower()
     persistent = False
     force_clean = False
     if 'resource_registry' in self.CFG:
         resource_registry_cfg = self.CFG['resource_registry']
         if 'persistent' in resource_registry_cfg:
             if resource_registry_cfg['persistent'] == True:
                 persistent = True
         if 'force_clean' in resource_registry_cfg:
             if resource_registry_cfg['force_clean'] == True:
                 force_clean = True
     if persistent:
         self.resource_registry = CouchDB_DataStore(datastore_name=resource_registry_name)
     else:
         self.resource_registry = MockDB_DataStore(datastore_name=resource_registry_name)
     if force_clean:
         try:
             self.resource_registry.delete_datastore(resource_registry_name)
         except NotFound:
             pass
     if not self.resource_registry.datastore_exists(resource_registry_name):
         self.resource_registry.create_datastore(resource_registry_name)
     # For easier interactive debugging
     self.dss = self.resource_registry.server[resource_registry_name] if persistent else None
     self.ds = self.resource_registry
 def on_init(self):
     datastore_name = sys_name + "_scratch"
     datastore_name = datastore_name.lower()
     persistent = False
     force_clean = False
     if 'datastore' in self.CFG:
         datastore_cfg = self.CFG['datastore']
         if 'persistent' in datastore_cfg:
             if datastore_cfg['persistent'] == True:
                 persistent = True
         if 'force_clean' in datastore_cfg:
             if datastore_cfg['force_clean'] == True:
                 force_clean = True
     if persistent:
         self.datastore = CouchDB_DataStore(datastore_name=datastore_name)
     else:
         self.datastore = MockDB_DataStore(datastore_name=datastore_name)
     if force_clean:
         try:
             self.datastore.delete_datastore(datastore_name)
         except NotFound:
             pass
     if not self.datastore_exists(datastore_name):
         self.datastore.create_datastore(datastore_name)
     # For easier interactive debugging
     self.dss = self.datastore.server[datastore_name] if persistent else None
     self.ds = self.datastore
示例#3
0
    def _force_clean(self, recreate=False):
        from pyon.core.bootstrap import get_sys_name
        from pyon.datastore.datastore import DatastoreManager
        if DatastoreManager.persistent is None:
            DatastoreManager.persistent = not CFG.system.mockdb
        if not DatastoreManager.persistent:
            datastore = MockDB_DataStore()
        else:
            datastore = CouchDB_DataStore()
        dbs = datastore.list_datastores()
        things_to_clean = filter(
            lambda x: x.startswith('%s_' % get_sys_name()), dbs)
        try:
            for thing in things_to_clean:
                datastore.delete_datastore(datastore_name=thing)
                if recreate:
                    datastore.create_datastore(datastore_name=thing)

        finally:
            datastore.close()
示例#4
0
文件: int_test.py 项目: tgiguere/pyon
 def _force_clean(self):
     from pyon.core.bootstrap import get_sys_name
     from pyon.datastore.datastore import DatastoreManager
     if DatastoreManager.persistent is None:
         DatastoreManager.persistent = not CFG.system.mockdb
     if not DatastoreManager.persistent:
         datastore = MockDB_DataStore()
     else:
         datastore = CouchDB_DataStore()
     dbs = datastore.list_datastores()
     things_to_clean = filter(lambda x: x.startswith('%s_' % get_sys_name()), dbs)
     try:
         for thing in things_to_clean:
             datastore.delete_datastore(datastore_name=thing)
             datastore.create_datastore(datastore_name=thing)
     finally:
         datastore.close()
示例#5
0
文件: int_test.py 项目: wfrench/pyon
 def _force_clean(self):
 # Force clean Couch in between tests, which is taken care of normally during process_start.
     from pyon.core.bootstrap import get_sys_name
     things_to_clean = ["%s_%s" % (str(get_sys_name()).lower(), thing_name) for thing_name in ('resources', 'objects', 'directory', 'events', 'state')]
     if CFG.system.mockdb:
         datastore = MockDB_DataStore()
     else:
         datastore = CouchDB_DataStore()
     try:
         for thing in things_to_clean:
             datastore.delete_datastore(datastore_name=thing)
             datastore.create_datastore(datastore_name=thing)
     finally:
         datastore.close()
示例#6
0
        def __init__(self):
            persistent = False
            force_clean = False

            # couchdb cannot handle capital letters, apparently
            datastore_name = bootstrap.sys_name.lower() + "_directory"

            if 'directory' in CFG:
                directory_cfg = CFG['directory']
                if 'persistent' in directory_cfg:
                    persistent = directory_cfg['persistent']
                if 'force_clean' in directory_cfg:
                    force_clean = directory_cfg['force_clean']
            if persistent:
                self.datastore = CouchDB_DataStore(datastore_name=datastore_name)
            else:
                self.datastore = MockDB_DataStore(datastore_name=datastore_name)

            if force_clean:
                self._delete()

            if not self.datastore.datastore_exists(datastore_name):
                self._create()
            self._init()
示例#7
0
    def get_datastore_instance(cls,
                               ds_name,
                               persistent=None,
                               profile=DataStore.DS_PROFILE.BASIC):
        scoped_name = DatastoreManager.get_scoped_name(ds_name)

        # Imports here to prevent cyclic module dependency
        from pyon.core.bootstrap import CFG
        if persistent is None:
            persistent = not bool(get_safe(CFG, "system.mockdb"))

        # Persistent (CouchDB) or MockDB?
        if persistent:
            # Use inline import to prevent circular import dependency
            from pyon.datastore.couchdb.couchdb_datastore import CouchDB_DataStore
            new_ds = CouchDB_DataStore(datastore_name=scoped_name,
                                       profile=profile)
        else:
            # Use inline import to prevent circular import dependency
            from pyon.datastore.mockdb.mockdb_datastore import MockDB_DataStore
            new_ds = MockDB_DataStore(datastore_name=scoped_name)

        return new_ds
示例#8
0
    class __impl:

        def __init__(self):
            persistent = False
            force_clean = False

            # couchdb cannot handle capital letters, apparently
            datastore_name = bootstrap.sys_name.lower() + "_directory"

            if 'directory' in CFG:
                directory_cfg = CFG['directory']
                if 'persistent' in directory_cfg:
                    persistent = directory_cfg['persistent']
                if 'force_clean' in directory_cfg:
                    force_clean = directory_cfg['force_clean']
            if persistent:
                self.datastore = CouchDB_DataStore(datastore_name=datastore_name)
            else:
                self.datastore = MockDB_DataStore(datastore_name=datastore_name)

            if force_clean:
                self._delete()

            if not self.datastore.datastore_exists(datastore_name):
                self._create()
            self._init()

        def _delete(self):
            """
            Method to delete directory.  Delete occurs as side effect
            of deleting the underlying data store.
            TODO: Change this functionality in the future?
            """
            log.debug("Deleting data store and Directory")
            try:
                self.datastore.delete_datastore()
            except NotFound:
                pass

        def _get_dn(self, parent, key):
            if parent == '/':
                return "/%s" % (key)
            else:
                return "%s/%s" % (parent,key)

        def _init(self):
            self._assert_existence("/", "Agents")
            self._assert_existence("/", "Config")
            self._assert_existence("/", "Containers")
            self._assert_existence("/", "ObjectTypes")
            self._assert_existence("/", "ResourceTypes")
            self._assert_existence("/", "ServiceInterfaces")
            self._assert_existence("/", "Services")

        def _assert_existence(self, parent, key):
            dn = self._get_dn(parent, key)
            direntry = self._safe_read(dn)
            if not direntry:
                direntry = IonObject("DirEntry", parent=parent, key=key, attributes={})
                # TODO: This may fail because of concurrent create
                self.datastore.create(direntry, dn)

        def _safe_read(self, oid):
            try:
                res = self.datastore.read(oid)
                return res
            except NotFound:
                return None

        def _create(self):
            """
            Method which will create the underlying data store and
            persist an empty Directory object.
            """
            log.debug("Creating data store and Directory")
            self.datastore.create_datastore()

            # Persist empty Directory object under known name
            self.dir_name = bootstrap.sys_name
            directory_obj = IonObject('Directory', name=self.dir_name)
            dir_id,rev = self.datastore.create(directory_obj, 'DIR')

            # Persist ROOT Directory object
            root_obj = IonObject('DirEntry', parent='/', key="ROOT", attributes=dict(sys_name=bootstrap.sys_name))
            root_id,rev = self.datastore.create(root_obj, self._get_dn(root_obj.parent, root_obj.key))

        def register(self, parent, key, **kwargs):
            """
            Add/replace an entry to directory below a parent node.
            Note: Does not merge the attribute values of the entry if existing
            """
            assert parent and key, "Malformed Directory register"
            dn = self._get_dn(parent, key)
            log.debug("Directory.add(%s): %s" % (dn, kwargs))

            entry_old = None
            direntry = self._safe_read(dn)
            if direntry:
                entry_old = direntry.attributes
                direntry.attributes = kwargs
                # TODO: This may fail because of concurrent update
                self.datastore.update(direntry)
            else:
                direntry = IonObject("DirEntry", parent=parent, key=key, attributes=kwargs)
                # TODO: This may fail because of concurrent create
                self.datastore.create(direntry, dn)

            return entry_old

        def lookup(self, qualified_key='/'):
            """
            Read entry residing in directory at parent node level.
            """
            log.debug("Reading content at path %s" % qualified_key)
            direntry = self._safe_read(qualified_key)
            return direntry.attributes if direntry else None

        def unregister(self, parent, key):
            """
            Remove entry residing in directory at parent node level.
            """
            dn = self._get_dn(parent, key)
            log.debug("Removing content at path %s" % dn)

            entry_old = None
            direntry = self._safe_read(dn)
            if direntry:
                entry_old = direntry.attributes
                self.datastore.delete(direntry)

            return entry_old

        def find_entries(self, qname='/'):
            if not str(qname).startswith('/'):
                raise BadRequest("Illegal directory node: qname=%s" % qname)
            delist = self.datastore.find_dir_entries(qname)
            return delist
class DataStoreService(BaseDatastoreService):

    """
    	The Data Store Service manages the persistence of Structured Objects (i.e. the "Business Objects") in the
    	ION system, and the definition of Structured Object types. It makes use of DM Preservation services and
    	components to persist the actual information on disk or in the network.
    """

    def on_init(self):
        datastore_name = sys_name + "_scratch"
        datastore_name = datastore_name.lower()
        persistent = False
        force_clean = False
        if 'datastore' in self.CFG:
            datastore_cfg = self.CFG['datastore']
            if 'persistent' in datastore_cfg:
                if datastore_cfg['persistent'] == True:
                    persistent = True
            if 'force_clean' in datastore_cfg:
                if datastore_cfg['force_clean'] == True:
                    force_clean = True
        if persistent:
            self.datastore = CouchDB_DataStore(datastore_name=datastore_name)
        else:
            self.datastore = MockDB_DataStore(datastore_name=datastore_name)
        if force_clean:
            try:
                self.datastore.delete_datastore(datastore_name)
            except NotFound:
                pass
        if not self.datastore_exists(datastore_name):
            self.datastore.create_datastore(datastore_name)
        # For easier interactive debugging
        self.dss = self.datastore.server[datastore_name] if persistent else None
        self.ds = self.datastore

    def create_datastore(self, datastore_name=''):
        return self.datastore.create_datastore(datastore_name)

    def delete_datastore(self, datastore_name=''):
        return self.datastore.delete_datastore(datastore_name)

    def list_datastores(self):
        return self.datastore.list_datastores()

    def info_datastore(self, datastore_name=''):
        return self.datastore.info_datastore(datastore_name)

    def datastore_exists(self, datastore_name=''):
        return self.datastore.datastore_exists(datastore_name)

    def list_objects(self, datastore_name=''):
        return self.datastore.list_objects(datastore_name)

    def list_object_revisions(self, object_id='', datastore_name=''):
        return self.datastore.list_object_revisions(object_id, datastore_name)

    def create(self, object={}, object_id='', datastore_name=''):
        return self.datastore.create(object, object_id=object_id, datastore_name=datastore_name)

    def create_doc(self, object={}, object_id='', datastore_name=''):
        return self.datastore.create_doc(object, object_id=object_id, datastore_name=datastore_name)

    def read(self, object_id='', rev_id='', datastore_name=''):
        return self.datastore.read(object_id, rev_id, datastore_name)

    def read_doc(self, object_id='', rev_id='', datastore_name=''):
        return self.datastore.read_doc(object_id, rev_id, datastore_name)

    def update(self, object={}, datastore_name=''):
        return self.datastore.update(object, datastore_name)

    def update_doc(self, object={}, datastore_name=''):
        return self.datastore.update_doc(object, datastore_name)

    def delete(self, object={}, datastore_name=''):
        return self.datastore.delete_doc(object, datastore_name)

    def delete_doc(self, object={}, datastore_name=''):
        return self.datastore.delete_doc(object, datastore_name)

    def find(self, criteria=[], datastore_name=''):
        return self.datastore.find(criteria, datastore_name)

    def find_doc(self, criteria=[], datastore_name=''):
        return self.datastore.find_doc(criteria, datastore_name)

    def find_by_idref(self, criteria=[], association="", datastore_name=""):
        return self.datastore.find_by_idref(criteria, association, datastore_name)

    def find_by_idref_doc(self, criteria=[], association="", datastore_name=""):
        return self.datastore.find_by_idref_doc(criteria, association, datastore_name)

    def resolve_idref(self, subject="", predicate="", object="", datastore_name=""):
        return self.datastore.resolve_idref(subject, predicate, object, datastore_name)

    def resolve_idref_doc(self, subject="", predicate="", object="", datastore_name=""):
        return self.datastore.resolve_idref_doc(subject, predicate, object, datastore_name)

    def create_association(self, subject=None, predicate=None, object=None, assoc_type=None):
        return self.datastore.create_association(subject, predicate, object, assoc_type)

    def delete_association(self, association=''):
        return self.datastore.delete_association(association)

    def find_objects(self, subject="", predicate="", object_type="", id_only=False):
        return self.datastore.find_objects(subject, predicate, object_type, id_only=id_only)

    def find_subjects(self, subject_type="", predicate="", object="", id_only=False):
        return self.datastore.find_subjects(subject_type, predicate, object, id_only=id_only)

    def find_associations(self, subject="", predicate="", object="", id_only=False):
        return self.datastore.find_associations(subject, predicate, object, id_only=id_only)
示例#10
0
    def test_non_persistent(self):
        self._do_test(MockDB_DataStore(datastore_name='ion_test_ds'))

        self._do_test_views(MockDB_DataStore(datastore_name='ion_test_ds'))
class ResourceRegistryService(BaseResourceRegistryService):

    """
    Service that manages the definition of types of resources and all cross-cutting concerns of all system resources.
    The resource registry uses the underlying push and pull ops of the datastore to fetch, retrieve and create
    resource objects.
    """
    def on_init(self):
        resource_registry_name = sys_name + "_resources"
        resource_registry_name = resource_registry_name.lower()
        persistent = False
        force_clean = False
        if 'resource_registry' in self.CFG:
            resource_registry_cfg = self.CFG['resource_registry']
            if 'persistent' in resource_registry_cfg:
                if resource_registry_cfg['persistent'] == True:
                    persistent = True
            if 'force_clean' in resource_registry_cfg:
                if resource_registry_cfg['force_clean'] == True:
                    force_clean = True
        if persistent:
            self.resource_registry = CouchDB_DataStore(datastore_name=resource_registry_name)
        else:
            self.resource_registry = MockDB_DataStore(datastore_name=resource_registry_name)
        if force_clean:
            try:
                self.resource_registry.delete_datastore(resource_registry_name)
            except NotFound:
                pass
        if not self.resource_registry.datastore_exists(resource_registry_name):
            self.resource_registry.create_datastore(resource_registry_name)
        # For easier interactive debugging
        self.dss = self.resource_registry.server[resource_registry_name] if persistent else None
        self.ds = self.resource_registry

    def create(self, object={}):
        cur_time = get_ion_ts()
        object.ts_created = cur_time
        object.ts_updated = cur_time
        return self.resource_registry.create(object)

    def read(self, object_id='', rev_id=''):
        return self.resource_registry.read(object_id, rev_id)

    def update(self, object={}):
        # Do an check whether LCS has been modified
        res_obj = self.read(object._id, object._rev)
        self.assert_condition(res_obj.lcstate == object.lcstate, "Cannot modify life cycle state in update!")
        object.ts_updated = get_ion_ts()
        return self.resource_registry.update(object)

    def delete(self, object_id=''):
        res_obj = self.read(object_id)
        if not res_obj:
            raise NotFound("Resource %s does not exist" % object_id)
        return self.resource_registry.delete(res_obj)

    def execute_lifecycle_transition(self, resource_id='', transition_event='', current_lcstate=''):
        self.assert_condition(not current_lcstate or current_lcstate in LCS, "Unknown life-cycle state %s" % current_lcstate)
        res_obj = self.read(resource_id)

        if current_lcstate and res_obj.lcstate != current_lcstate:
            raise Inconsistent("Resource id=%s lcstate is %s, expected was %s" % (
                                resource_id, res_obj.lcstate, current_lcstate))

        restype = type(res_obj).__name__
        restype_workflow = lcs_workflows.get(restype, None)
        if not restype_workflow:
            restype_workflow = lcs_workflows['Resource']

        new_state = restype_workflow.get_successor(res_obj.lcstate, transition_event)
        if not new_state:
            raise Inconsistent("Resource id=%s, type=%s, lcstate=%s has no transition for event %s" % (
                                resource_id, restype, res_obj.lcstate, transition_event))

        res_obj.lcstate = new_state
        res_obj.ts_updated = get_ion_ts()
        updres = self.resource_registry.update(res_obj)
        return new_state

    def create_association(self, subject=None, predicate=None, object=None, assoc_type=None):
        return self.resource_registry.create_association(subject, predicate, object, assoc_type)

    def delete_association(self, association=''):
        return self.resource_registry.delete_association(association)

    def find(self, **kwargs):
        raise NotImplementedError("Do not use find. Use a specific find operation instead.")

    def find_objects(self, subject="", predicate="", object_type="", id_only=False):
        return self.resource_registry.find_objects(subject, predicate, object_type, id_only=id_only)

    def find_subjects(self, subject_type="", predicate="", object="", id_only=False):
        return self.resource_registry.find_subjects(subject_type, predicate, object, id_only=id_only)

    def find_associations(self, subject="", predicate="", object="", id_only=False):
        return self.resource_registry.find_associations(subject, predicate, object, id_only=id_only)

    def get_association(self, subject="", predicate="", object=""):
        assoc = self.resource_registry.find_associations(subject, predicate, object, id_only=True)
        if not assoc:
            raise NotFound("Association for subject/predicate/object %s/%s/%s not found" % (str(subject),str(predicate),str(object)))
        elif len(assoc) > 1:
            raise Inconsistent("Duplicate associations found for subject/predicate/object %s/%s/%s" % (str(subject),str(predicate),str(object)))
        return assoc[0]

    def find_resources(self, restype="", lcstate="", name="", id_only=False):
        
        return self.resource_registry.find_resources(restype, lcstate, name, id_only=id_only)