def create_cache(self): """Create instance of the project cache.""" return CommonProjectCache( datastores=FileSystemDatastoreFactory(DATASTORES_DIR), filestores=FileSystemFilestoreFactory(FILESTORES_DIR), viztrails=OSViztrailRepository(base_path=VIZTRAILS_DIR) )
def setUp(self): """Create an instance of the default vizier processor for an empty server directory. """ # Drop directory if it exists if os.path.isdir(SERVER_DIR): shutil.rmtree(SERVER_DIR) os.makedirs(SERVER_DIR) vizual = VizualTaskProcessor(api=DefaultVizualApi()) pycell = PyCellTaskProcessor() projects = CommonProjectCache( datastores=FileSystemDatastoreFactory(DATASTORES_DIR), filestores=FileSystemFilestoreFactory(FILESTORES_DIR), viztrails=OSViztrailRepository(base_path=VIZTRAILS_DIR)) self.PROJECT_ID = projects.create_project().identifier self.backend = SynchronousTaskEngine(commands={ PACKAGE_PYTHON: { PYTHON_CODE: pycell }, PACKAGE_VIZUAL: { VIZUAL_LOAD: vizual, VIZUAL_UPD_CELL: vizual } }, projects=projects)
def test_create_and_delete_datastore(self): """Test the basic functionality of the file store factory.""" fact = FileSystemDatastoreFactory( properties={PARA_DIRECTORY: SERVER_DIR}) db = fact.get_datastore('0123') db = fact.get_datastore('4567') fact.delete_datastore('0123') self.assertFalse(os.path.isdir(os.path.join(SERVER_DIR, '0123'))) self.assertTrue(os.path.isdir(os.path.join(SERVER_DIR, '4567'))) db = fact.get_datastore('0123') self.assertTrue(os.path.isdir(os.path.join(SERVER_DIR, '0123'))) self.assertTrue(os.path.isdir(os.path.join(SERVER_DIR, '4567'))) # ValueError if no base path is given with self.assertRaises(ValueError): FileSystemDatastoreFactory()
def get_env(config): """Get teh celery worker environment based on the value of the environment identifier. If the value of the environment variable VIZIERWORKER_ENV is 'DEV' the environment variable VIZIERENGINE_DATA_DIR is used to instantiate the datastore and filestore factory. If the value of VIZIERWORKER_ENV is 'REMOTE' the variable VIZIERWORKER_CONTROLLER_URL is used to instantiate the datastore factory. In a remote environment a dummy filestore factory is used. Parameters ---------- config: vizier.config.worker.WorkerConfig Worker configuration object Returns ------- vizier.engine.backend.remote.celery.env.WorkerEnvironment """ if config.env.identifier == base.DEV_ENGINE: base_dir = base.get_config_value( env_variable=app.VIZIERENGINE_DATA_DIR, default_values={app.VIZIERENGINE_DATA_DIR: base.ENV_DIRECTORY}) datastores_dir = os.path.join(base_dir, app.DEFAULT_DATASTORES_DIR) filestores_dir = os.path.join(base_dir, app.DEFAULT_FILESTORES_DIR) datastore_factory = FileSystemDatastoreFactory(datastores_dir) filestore_factory = FileSystemFilestoreFactory(filestores_dir) elif config.env.identifier == 'REMOTE': datastore_factory = DatastoreClientFactory( base_url=config.controller.url) filestore_factory = DevNullFilestoreFactory() else: raise ValueError('unknown worker environment identifier \'' + config.env.identifier + "\'") return WorkerEnvironment(controller_url=config.controller.url, processors=load_processors( config.env.processor_path), datastores=datastore_factory, filestores=filestore_factory)
def setUp(self): """Create an instance of the default vizier processor for an empty server directory. """ # Drop directory if it exists if os.path.isdir(SERVER_DIR): shutil.rmtree(SERVER_DIR) os.makedirs(SERVER_DIR) projects = CommonProjectCache( datastores=FileSystemDatastoreFactory(DATASTORES_DIR), filestores=FileSystemFilestoreFactory(FILESTORES_DIR), viztrails=OSViztrailRepository(base_path=VIZTRAILS_DIR)) self.PROJECT_ID = projects.create_project().identifier self.backend = MultiProcessBackend(processors={ PACKAGE_PYTHON: PyCellTaskProcessor(), PACKAGE_VIZUAL: VizualTaskProcessor(api=MimirVizualApi()), 'error': FakeTaskProcessor() }, projects=projects)
def get_engine(config): """Create instance of vizier engine using the default datastore, filestore and viztrails factories. The default engine may use a multi-process backend or a celery backend. Parameters ---------- config: vizier.config.app.AppConfig Application configuration object Returns ------- vizier.engine.base.VizierEngine """ # Get backend identifier. Raise ValueError if value does not identify # a valid backend. backend_id = config.engine.backend.identifier if not backend_id in base.BACKENDS: raise ValueError('unknown backend \'' + str(backend_id) + '\'') # Get the identifier factory for the viztrails repository and create # the object store. At this point we use the default object store only. # We could add another environment variable to use different object # stores (once implemented). if config.engine.use_short_ids: id_factory = get_short_identifier else: id_factory = get_unique_identifier object_store = DefaultObjectStore( identifier_factory=id_factory ) # By default the vizier engine uses the objectstore implementation for # the viztrails repository. The datastore and filestore factories depend # on the values of engine identifier (DEV or MIMIR). base_dir = config.engine.data_dir viztrails_dir = os.path.join(base_dir, app.DEFAULT_VIZTRAILS_DIR) if config.engine.identifier in [base.DEV_ENGINE, base.MIMIR_ENGINE]: filestores_dir = os.path.join(base_dir, app.DEFAULT_FILESTORES_DIR) filestore_factory=FileSystemFilestoreFactory(filestores_dir) datastores_dir = os.path.join(base_dir, app.DEFAULT_DATASTORES_DIR) if config.engine.identifier == base.DEV_ENGINE: datastore_factory = FileSystemDatastoreFactory(datastores_dir) else: datastore_factory = MimirDatastoreFactory(datastores_dir) else: raise ValueError('unknown vizier engine \'' + str(config.engine.identifier) + '\'') # The default engine uses a common project cache. projects = SingleProjectCache( ProjectHandle( viztrail=ViztrailHandle(identifier=config.project_id), datastore=datastore_factory.get_datastore(config.project_id), filestore=filestore_factory.get_filestore(config.project_id) ) ) # Create workflow execution backend and processor for synchronous task packages = load_packages(config.engine.package_path) processors = load_processors(config.engine.processor_path) # Create the backend if backend_id == base.BACKEND_MULTIPROCESS: backend = MultiProcessBackend( processors=processors, projects=projects, synchronous=None ) elif backend_id == base.BACKEND_CELERY: # Create and configure routing information (if given) backend = CeleryBackend( routes=config_routes(config), synchronous=None ) else: # For completeness. Validity of the backend id is be checked before. raise ValueError('unknown backend \'' + str(backend_id) + '\'') return VizierEngine( name=config.engine.identifier + ' (' + backend_id + ')', projects=projects, backend=backend, packages=packages )
def get_engine(config: AppConfig) -> VizierEngine: """Create instance of the default vizual engine using the default datastore, filestore and viztrails factories. The default engine may use a multi-process backend or a celery backend. Parameters ---------- config: vizier.config.app.AppConfig Application configuration object Returns ------- vizier.engine.base.VizierEngine """ # Get backend identifier. Raise ValueError if value does not identify # a valid backend. backend_id = config.engine.backend.identifier if not backend_id in base.BACKENDS: raise ValueError('unknown backend \'' + str(backend_id) + '\'') # Get the identifier factory for the viztrails repository and create # the object store. At this point we use the default object store only. # We could add another environment variable to use different object # stores (once implemented). if config.engine.use_short_ids: id_factory = get_short_identifier else: id_factory = get_unique_identifier object_store = DefaultObjectStore( identifier_factory=id_factory ) # Create index of supported packages packages = load_packages(config.engine.package_path) # By default the vizier engine uses the objectstore implementation for # the viztrails repository. The datastore and filestore factories depend # on the values of engine identifier (DEV or MIMIR). base_dir = config.engine.data_dir # Create the local viztrails repository viztrails = OSViztrailRepository( base_path=os.path.join(base_dir, app.DEFAULT_VIZTRAILS_DIR), object_store=object_store ) filestores_dir = os.path.join(base_dir, app.DEFAULT_FILESTORES_DIR) datastores_dir = os.path.join(base_dir, app.DEFAULT_DATASTORES_DIR) if config.engine.identifier in [base.DEV_ENGINE, base.MIMIR_ENGINE]: filestore_factory=FileSystemFilestoreFactory(filestores_dir) datastore_factory: DatastoreFactory if config.engine.identifier == base.DEV_ENGINE: datastore_factory = FileSystemDatastoreFactory(datastores_dir) else: datastore_factory = MimirDatastoreFactory(datastores_dir) # The default engine uses a common project cache. projects: ProjectCache = CommonProjectCache( datastores=datastore_factory, filestores=filestore_factory, viztrails=viztrails ) # Get set of task processors for supported packages processors = load_processors(config.engine.processor_path) # Create an optional task processor for synchronous tasks if given sync_commands_list = config.engine.sync_commands if not sync_commands_list is None: commands:Dict[str,Dict[str,TaskProcessor]] = dict() for el in sync_commands_list.split(':'): package_id, command_id = el.split('.') if not package_id in commands: commands[package_id] = dict() commands[package_id][command_id] = processors[package_id] synchronous: TaskExecEngine = SynchronousTaskEngine( commands=commands, projects=projects ) else: synchronous = NonSynchronousEngine() # Create the backend backend: VizierBackend if backend_id == base.BACKEND_MULTIPROCESS: backend = MultiProcessBackend( processors=processors, projects=projects, synchronous=synchronous ) elif backend_id == base.BACKEND_CELERY: # Create and configure routing information (if given) backend = CeleryBackend( routes=config_routes(config), synchronous=synchronous ) else: # Not all combinations of engine identifier and backend identifier # are valid. raise ValueError('invalid backend \'' + str(backend_id) + '\'') elif config.engine.identifier == base.CONTAINER_ENGINE: if backend_id == base.BACKEND_CONTAINER: projects = ContainerProjectCache( viztrails=viztrails, container_file=os.path.join(base_dir, app.DEFAULT_CONTAINER_FILE), config=config, datastores=MimirDatastoreFactory(datastores_dir), filestores=FileSystemFilestoreFactory(filestores_dir) ) backend = ContainerBackend(projects=projects) else: # The container engine only supports a single backend type. raise ValueError('invalid backend \'' + str(backend_id) + '\'') else: raise ValueError('unknown vizier engine \'' + str(config.engine.identifier) + '\'') return VizierEngine( name=config.engine.identifier + ' (' + backend_id + ')', projects=projects, backend=backend, packages=packages )