def create_cache(self):
"""Create instance of the project cache."""
return CommonProjectCache(
datastores=FileSystemDatastoreFactory(DATASTORES_DIR),
filestores=FileSystemFilestoreFactory(FILESTORES_DIR),
viztrails=OSViztrailRepository(base_path=VIZTRAILS_DIR)
)
def setUp(self):
"""Create an instance of the default vizier processor for an empty server
directory.
"""
# Drop directory if it exists
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.makedirs(SERVER_DIR)
vizual = VizualTaskProcessor(api=DefaultVizualApi())
pycell = PyCellTaskProcessor()
projects = CommonProjectCache(
datastores=FileSystemDatastoreFactory(DATASTORES_DIR),
filestores=FileSystemFilestoreFactory(FILESTORES_DIR),
viztrails=OSViztrailRepository(base_path=VIZTRAILS_DIR))
self.PROJECT_ID = projects.create_project().identifier
self.backend = SynchronousTaskEngine(commands={
PACKAGE_PYTHON: {
PYTHON_CODE: pycell
},
PACKAGE_VIZUAL: {
VIZUAL_LOAD: vizual,
VIZUAL_UPD_CELL: vizual
}
},
projects=projects)
def test_create_and_delete_filestore(self):
"""Test the basic functionality of the file store factory."""
fact = FileSystemFilestoreFactory(
properties={PARA_DIRECTORY: SERVER_DIR})
db = fact.get_filestore('0123')
fh = db.upload_file(CSV_FILE)
self.assertTrue(os.path.isdir(os.path.join(SERVER_DIR, '0123')))
self.assertTrue(os.path.isfile(fh.filepath))
db = fact.get_filestore('4567')
fact.delete_filestore('0123')
self.assertFalse(os.path.isdir(os.path.join(SERVER_DIR, '0123')))
self.assertFalse(os.path.isfile(fh.filepath))
self.assertTrue(os.path.isdir(os.path.join(SERVER_DIR, '4567')))
db = fact.get_filestore('0123')
self.assertTrue(os.path.isdir(os.path.join(SERVER_DIR, '0123')))
self.assertTrue(os.path.isdir(os.path.join(SERVER_DIR, '4567')))
# ValueError if no base path is given
with self.assertRaises(ValueError):
FileSystemFilestoreFactory()
def test_create_cache(self):
"""Test accessing and deleting projects for an empty repository."""
viztrails = OSViztrailRepository(base_path=VIZTRAILS_DIR)
vt1 = viztrails.create_viztrail(
properties={PROPERTY_NAME: 'My Project'})
vt2 = viztrails.create_viztrail(
properties={PROPERTY_NAME: 'A Project'})
filename = os.path.join(SERVER_DIR, 'container.json')
DefaultObjectStore().write_object(object_path=filename,
content=[{
'projectId': vt1.identifier,
'url': 'API1',
'port': 80,
'containerId': 'ID1'
}, {
'projectId': vt2.identifier,
'url': 'API2',
'port': 81,
'containerId': 'ID2'
}])
# Initialize the project cache
viztrails = OSViztrailRepository(base_path=VIZTRAILS_DIR)
filestores_dir = os.path.join(SERVER_DIR, DEFAULT_FILESTORES_DIR)
datastores_dir = os.path.join(SERVER_DIR, DEFAULT_DATASTORES_DIR)
projects = ContainerProjectCache(
viztrails=viztrails,
container_file=filename,
config=AppConfig(),
datastores=MimirDatastoreFactory(datastores_dir),
filestores=FileSystemFilestoreFactory(filestores_dir))
self.assertEqual(len(projects.list_projects()), 2)
self.assertEqual(
projects.get_project(vt1.identifier).container_api, 'API1')
self.assertEqual(
projects.get_project(vt2.identifier).container_api, 'API2')
self.assertEqual(
projects.get_project(vt1.identifier).container_id, 'ID1')
self.assertEqual(
projects.get_project(vt2.identifier).container_id, 'ID2')
self.assertEqual(projects.get_project(vt1.identifier).port, 80)
self.assertEqual(projects.get_project(vt2.identifier).port, 81)
def get_env(config):
"""Get teh celery worker environment based on the value of the
environment identifier.
If the value of the environment variable VIZIERWORKER_ENV is 'DEV' the
environment variable VIZIERENGINE_DATA_DIR is used to instantiate the
datastore and filestore factory. If the value of VIZIERWORKER_ENV is
'REMOTE' the variable VIZIERWORKER_CONTROLLER_URL is used to instantiate
the datastore factory. In a remote environment a dummy filestore factory
is used.
Parameters
----------
config: vizier.config.worker.WorkerConfig
Worker configuration object
Returns
-------
vizier.engine.backend.remote.celery.env.WorkerEnvironment
"""
if config.env.identifier == base.DEV_ENGINE:
base_dir = base.get_config_value(
env_variable=app.VIZIERENGINE_DATA_DIR,
default_values={app.VIZIERENGINE_DATA_DIR: base.ENV_DIRECTORY})
datastores_dir = os.path.join(base_dir, app.DEFAULT_DATASTORES_DIR)
filestores_dir = os.path.join(base_dir, app.DEFAULT_FILESTORES_DIR)
datastore_factory = FileSystemDatastoreFactory(datastores_dir)
filestore_factory = FileSystemFilestoreFactory(filestores_dir)
elif config.env.identifier == 'REMOTE':
datastore_factory = DatastoreClientFactory(
base_url=config.controller.url)
filestore_factory = DevNullFilestoreFactory()
else:
raise ValueError('unknown worker environment identifier \'' +
config.env.identifier + "\'")
return WorkerEnvironment(controller_url=config.controller.url,
processors=load_processors(
config.env.processor_path),
datastores=datastore_factory,
filestores=filestore_factory)
def setUp(self):
"""Create an instance of the default vizier processor for an empty server
directory.
"""
# Drop directory if it exists
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.makedirs(SERVER_DIR)
projects = CommonProjectCache(
datastores=FileSystemDatastoreFactory(DATASTORES_DIR),
filestores=FileSystemFilestoreFactory(FILESTORES_DIR),
viztrails=OSViztrailRepository(base_path=VIZTRAILS_DIR))
self.PROJECT_ID = projects.create_project().identifier
self.backend = MultiProcessBackend(processors={
PACKAGE_PYTHON:
PyCellTaskProcessor(),
PACKAGE_VIZUAL:
VizualTaskProcessor(api=MimirVizualApi()),
'error':
FakeTaskProcessor()
},
projects=projects)
def get_engine(config):
"""Create instance of vizier engine using the default datastore, filestore
and viztrails factories. The default engine may use a multi-process backend
or a celery backend.
Parameters
----------
config: vizier.config.app.AppConfig
Application configuration object
Returns
-------
vizier.engine.base.VizierEngine
"""
# Get backend identifier. Raise ValueError if value does not identify
# a valid backend.
backend_id = config.engine.backend.identifier
if not backend_id in base.BACKENDS:
raise ValueError('unknown backend \'' + str(backend_id) + '\'')
# Get the identifier factory for the viztrails repository and create
# the object store. At this point we use the default object store only.
# We could add another environment variable to use different object
# stores (once implemented).
if config.engine.use_short_ids:
id_factory = get_short_identifier
else:
id_factory = get_unique_identifier
object_store = DefaultObjectStore(
identifier_factory=id_factory
)
# By default the vizier engine uses the objectstore implementation for
# the viztrails repository. The datastore and filestore factories depend
# on the values of engine identifier (DEV or MIMIR).
base_dir = config.engine.data_dir
viztrails_dir = os.path.join(base_dir, app.DEFAULT_VIZTRAILS_DIR)
if config.engine.identifier in [base.DEV_ENGINE, base.MIMIR_ENGINE]:
filestores_dir = os.path.join(base_dir, app.DEFAULT_FILESTORES_DIR)
filestore_factory=FileSystemFilestoreFactory(filestores_dir)
datastores_dir = os.path.join(base_dir, app.DEFAULT_DATASTORES_DIR)
if config.engine.identifier == base.DEV_ENGINE:
datastore_factory = FileSystemDatastoreFactory(datastores_dir)
else:
datastore_factory = MimirDatastoreFactory(datastores_dir)
else:
raise ValueError('unknown vizier engine \'' + str(config.engine.identifier) + '\'')
# The default engine uses a common project cache.
projects = SingleProjectCache(
ProjectHandle(
viztrail=ViztrailHandle(identifier=config.project_id),
datastore=datastore_factory.get_datastore(config.project_id),
filestore=filestore_factory.get_filestore(config.project_id)
)
)
# Create workflow execution backend and processor for synchronous task
packages = load_packages(config.engine.package_path)
processors = load_processors(config.engine.processor_path)
# Create the backend
if backend_id == base.BACKEND_MULTIPROCESS:
backend = MultiProcessBackend(
processors=processors,
projects=projects,
synchronous=None
)
elif backend_id == base.BACKEND_CELERY:
# Create and configure routing information (if given)
backend = CeleryBackend(
routes=config_routes(config),
synchronous=None
)
else:
# For completeness. Validity of the backend id is be checked before.
raise ValueError('unknown backend \'' + str(backend_id) + '\'')
return VizierEngine(
name=config.engine.identifier + ' (' + backend_id + ')',
projects=projects,
backend=backend,
packages=packages
)
def get_engine(config: AppConfig) -> VizierEngine:
"""Create instance of the default vizual engine using the default datastore,
filestore and viztrails factories. The default engine may use a
multi-process backend or a celery backend.
Parameters
----------
config: vizier.config.app.AppConfig
Application configuration object
Returns
-------
vizier.engine.base.VizierEngine
"""
# Get backend identifier. Raise ValueError if value does not identify
# a valid backend.
backend_id = config.engine.backend.identifier
if not backend_id in base.BACKENDS:
raise ValueError('unknown backend \'' + str(backend_id) + '\'')
# Get the identifier factory for the viztrails repository and create
# the object store. At this point we use the default object store only.
# We could add another environment variable to use different object
# stores (once implemented).
if config.engine.use_short_ids:
id_factory = get_short_identifier
else:
id_factory = get_unique_identifier
object_store = DefaultObjectStore(
identifier_factory=id_factory
)
# Create index of supported packages
packages = load_packages(config.engine.package_path)
# By default the vizier engine uses the objectstore implementation for
# the viztrails repository. The datastore and filestore factories depend
# on the values of engine identifier (DEV or MIMIR).
base_dir = config.engine.data_dir
# Create the local viztrails repository
viztrails = OSViztrailRepository(
base_path=os.path.join(base_dir, app.DEFAULT_VIZTRAILS_DIR),
object_store=object_store
)
filestores_dir = os.path.join(base_dir, app.DEFAULT_FILESTORES_DIR)
datastores_dir = os.path.join(base_dir, app.DEFAULT_DATASTORES_DIR)
if config.engine.identifier in [base.DEV_ENGINE, base.MIMIR_ENGINE]:
filestore_factory=FileSystemFilestoreFactory(filestores_dir)
datastore_factory: DatastoreFactory
if config.engine.identifier == base.DEV_ENGINE:
datastore_factory = FileSystemDatastoreFactory(datastores_dir)
else:
datastore_factory = MimirDatastoreFactory(datastores_dir)
# The default engine uses a common project cache.
projects: ProjectCache = CommonProjectCache(
datastores=datastore_factory,
filestores=filestore_factory,
viztrails=viztrails
)
# Get set of task processors for supported packages
processors = load_processors(config.engine.processor_path)
# Create an optional task processor for synchronous tasks if given
sync_commands_list = config.engine.sync_commands
if not sync_commands_list is None:
commands:Dict[str,Dict[str,TaskProcessor]] = dict()
for el in sync_commands_list.split(':'):
package_id, command_id = el.split('.')
if not package_id in commands:
commands[package_id] = dict()
commands[package_id][command_id] = processors[package_id]
synchronous: TaskExecEngine = SynchronousTaskEngine(
commands=commands,
projects=projects
)
else:
synchronous = NonSynchronousEngine()
# Create the backend
backend: VizierBackend
if backend_id == base.BACKEND_MULTIPROCESS:
backend = MultiProcessBackend(
processors=processors,
projects=projects,
synchronous=synchronous
)
elif backend_id == base.BACKEND_CELERY:
# Create and configure routing information (if given)
backend = CeleryBackend(
routes=config_routes(config),
synchronous=synchronous
)
else:
# Not all combinations of engine identifier and backend identifier
# are valid.
raise ValueError('invalid backend \'' + str(backend_id) + '\'')
elif config.engine.identifier == base.CONTAINER_ENGINE:
if backend_id == base.BACKEND_CONTAINER:
projects = ContainerProjectCache(
viztrails=viztrails,
container_file=os.path.join(base_dir, app.DEFAULT_CONTAINER_FILE),
config=config,
datastores=MimirDatastoreFactory(datastores_dir),
filestores=FileSystemFilestoreFactory(filestores_dir)
)
backend = ContainerBackend(projects=projects)
else:
# The container engine only supports a single backend type.
raise ValueError('invalid backend \'' + str(backend_id) + '\'')
else:
raise ValueError('unknown vizier engine \'' + str(config.engine.identifier) + '\'')
return VizierEngine(
name=config.engine.identifier + ' (' + backend_id + ')',
projects=projects,
backend=backend,
packages=packages
)