def test_error_on_missing(self):
"""Test that reading a missing object will raise a ValueError."""
store = DefaultObjectStore()
filename = store.join(BASE_DIRECTORY, 'A.file')
store.create_object(BASE_DIRECTORY,
identifier='A.file',
content={'A': 1})
self.assertTrue(store.exists(filename))
# Re-create the store to ensure that this has no effect
store = DefaultObjectStore()
store.read_object(filename)
os.remove(filename)
with self.assertRaises(ValueError):
store.read_object(filename)
def test_exists(self):
"""Test exists method."""
store = DefaultObjectStore()
filename = store.join(BASE_DIRECTORY, 'A.file')
dirname = store.join(BASE_DIRECTORY, 'A.dir')
self.assertFalse(store.exists(filename))
store.create_object(BASE_DIRECTORY, identifier='A.file')
self.assertTrue(store.exists(filename))
self.assertFalse(store.exists(dirname))
os.makedirs(dirname)
self.assertTrue(store.exists(dirname))
# Re-create the store to ensure that this has no effect
store = DefaultObjectStore()
self.assertTrue(store.exists(filename))
self.assertTrue(store.exists(dirname))
def test_create_file_repeat(self):
"""Test create file with identifier factory that not always returns
a unique identifier.
"""
store = DefaultObjectStore(identifier_factory=IdFactory(
max_attempts=MAX_ATTEMPS - 1))
id1 = store.create_object(BASE_DIRECTORY)
id2 = store.create_object(BASE_DIRECTORY)
self.assertNotEqual(id1, id2)
self.assertTrue(store.exists(store.join(BASE_DIRECTORY, id1)))
self.assertTrue(store.exists(store.join(BASE_DIRECTORY, id2)))
store.delete_object(store.join(BASE_DIRECTORY, id1))
store.delete_object(store.join(BASE_DIRECTORY, id2))
store = DefaultObjectStore(identifier_factory=IdFactory(
max_attempts=MAX_ATTEMPS + 1))
id1 = store.create_object(BASE_DIRECTORY)
with self.assertRaises(RuntimeError):
store.create_object(BASE_DIRECTORY)
def test_list_folders(self):
"""Test list_folders method."""
store = DefaultObjectStore()
# The result is an empty list even if the folder does not exist and
# is not created using the create flag
dirname = store.join(BASE_DIRECTORY, 'A')
dirs = store.list_folders(parent_folder=dirname, create=False)
self.assertEqual(len(dirs), 0)
self.assertFalse(store.exists(dirname))
# The result is an empty list after the folder is created using the
# create flag
dirs = store.list_folders(parent_folder=dirname, create=True)
self.assertEqual(len(dirs), 0)
self.assertTrue(store.exists(dirname))
# Create directories and files
os.makedirs(store.join(dirname, 'A'))
dirs = store.list_folders(parent_folder=dirname)
self.assertEqual(len(dirs), 1)
self.assertTrue('A' in dirs)
os.makedirs(store.join(dirname, 'B'))
dirs = store.list_folders(parent_folder=dirname, create=True)
self.assertEqual(len(dirs), 2)
self.assertTrue('A' in dirs)
self.assertTrue('B' in dirs)
filename = store.join(BASE_DIRECTORY, 'A.file')
store.create_object(BASE_DIRECTORY, identifier='A.file')
dirs = store.list_folders(parent_folder=dirname, create=True)
self.assertEqual(len(dirs), 2)
self.assertTrue('A' in dirs)
self.assertTrue('B' in dirs)
# Re-create the store to ensure that this has no effect
store = DefaultObjectStore()
dirs = store.list_folders(parent_folder=dirname, create=True)
self.assertEqual(len(dirs), 2)
self.assertTrue('A' in dirs)
self.assertTrue('B' in dirs)
def test_create_object_with_identifier(self):
"""Test creating a new object with a given identifier."""
store = DefaultObjectStore()
store.create_object(BASE_DIRECTORY, identifier='A')
self.assertTrue(os.path.isfile(os.path.join(BASE_DIRECTORY, 'A')))
with self.assertRaises(ValueError):
store.read_object(store.join(BASE_DIRECTORY, 'A'))
store.create_object(BASE_DIRECTORY,
identifier='B',
content={'id': 100})
self.assertTrue(os.path.isfile(os.path.join(BASE_DIRECTORY, 'B')))
content = store.read_object(store.join(BASE_DIRECTORY, 'B'))
self.assertEqual(content['id'], 100)
store.create_object(BASE_DIRECTORY,
identifier='A',
content={'id': 100})
self.assertTrue(os.path.isfile(os.path.join(BASE_DIRECTORY, 'A')))
content = store.read_object(store.join(BASE_DIRECTORY, 'A'))
self.assertEqual(content['id'], 100)
store.create_object(BASE_DIRECTORY, identifier='B')
self.assertTrue(os.path.isfile(os.path.join(BASE_DIRECTORY, 'B')))
with self.assertRaises(ValueError):
store.read_object(store.join(BASE_DIRECTORY, 'B'))
def create_module(command,
external_form,
state,
timestamp,
outputs,
provenance,
module_folder,
datasets=None,
object_store=None):
"""Create a new materialized module instance for the given values.
Parameters
----------
command : vizier.viztrail.command.ModuleCommand
Specification of the module (i.e., package, name, and arguments)
external_form: string
Printable representation of module command
state: int
Module state (one of PENDING, RUNNING, CANCELED, ERROR, SUCCESS)
timestamp: vizier.viztrail.module.timestamp.ModuleTimestamp
Module timestamp
datasets : dict(vizier.datastore.dataset.DatasetDescriptor)
Dictionary of resulting datasets. Dataset descriptors are keyed by
the user-specified dataset name.
outputs: vizier.viztrail.module.output.ModuleOutputs
Module output streams STDOUT and STDERR
provenance: vizier.viztrail.module.provenance.ModuleProvenance
Provenance information about datasets that were read and writen by
previous execution of the module.
module_folder: string
Object store folder containing module resources
object_store: vizier.core.io.base.ObjectStore, optional
Object store implementation to access and maintain resources
Returns
-------
vizier.viztrail.objectstore.module.OSModuleHandle
"""
# Make sure the object store is not None
if object_store is None:
object_store = DefaultObjectStore()
# Serialize module components and materialize
obj = serialize_module(command=command,
external_form=external_form,
state=state,
timestamp=timestamp,
outputs=outputs,
provenance=provenance)
identifier = object_store.create_object(parent_folder=module_folder,
content=obj)
# Return handle for created module
return OSModuleHandle(
identifier=identifier,
command=command,
external_form=external_form,
module_path=object_store.join(module_folder, identifier),
state=state,
timestamp=timestamp,
datasets=datasets if not datasets is None else dict(),
outputs=outputs,
provenance=provenance,
object_store=object_store)
class OSViztrailRepository(ViztrailRepository):
"""Repository for viztrails. This implementation maintains all resources
that are managed by the repository as objects in an object store. The base
path is the identifier prefix for all managed resources.
By default all resources are mantained as directories and files on the local
file system. The viztrails index is a list object that contains the
identifier of active viztrails.
Folders and Resources
---------------------
viztrails : List of active viztrails
<vt-identifier>/ : Folder with resources for individual viztrail
"""
def __init__(self, base_path, object_store=None):
"""Initialize the repository from a configuration dictionary. Expects
a dictionary that contains at least the base path for the repository.
The definition of the object store is optional. If none is given the
default object store will be used.
Parameters
---------
base_path: string
Path to the base directory for viztrail resources
object_store: vizier.core.io.base.ObjectStore, optional
Store for objects that represent viztrail resources
not
"""
# Raise an exception if the base directory argument is not given
if base_path is None:
raise ValueError('missing path for base directory')
# Create the base directory if it does not exist
self.base_path = base_path
if not os.path.isdir(self.base_path):
os.makedirs(self.base_path)
# The object store element is optional. If not given the default object
# store is used.
if not object_store is None:
self.object_store = object_store
else:
self.object_store = DefaultObjectStore()
# Initialize the viztrails index. Create the index file if it does not
# exist.
self.viztrails_index = self.object_store.join(self.base_path,
OBJ_VIZTRAILINDEX)
if not self.object_store.exists(self.viztrails_index):
self.object_store.create_object(parent_folder=self.base_path,
identifier=OBJ_VIZTRAILINDEX,
content=list())
# Load viztrails and intialize the remaining instance variables by
# calling the constructor of the super class
self.viztrails = dict()
for identifier in self.object_store.read_object(self.viztrails_index):
vt = OSViztrailHandle.load_viztrail(
base_path=self.object_store.join(self.base_path, identifier),
object_store=self.object_store)
self.viztrails[vt.identifier] = vt
def create_viztrail(self, properties=None):
"""Create a new viztrail. The initial set of properties is an optional
dictionary of (key,value)-pairs where all values are expected to either
be scalar values or a list of scalar values.
Parameters
----------
properties: dict, optional
Set of properties for the new viztrail
Returns
-------
vizier.viztrail.objectstore.viztrail.OSViztrailHandle
"""
# Get unique identifier for new viztrail and viztrail directory. Raise
# runtime error if the returned identifier is not unique.
identifier = self.object_store.create_folder(
parent_folder=self.base_path)
viztrail_path = self.object_store.join(self.base_path, identifier)
# Create materialized viztrail resource
vt = OSViztrailHandle.create_viztrail(identifier=identifier,
properties=properties,
base_path=viztrail_path,
object_store=self.object_store)
# Add the new resource to the viztrails index. Write updated index to
# object store before returning the new viztrail handle
self.viztrails[vt.identifier] = vt
self.object_store.write_object(
object_path=self.viztrails_index,
content=[vt_id for vt_id in self.viztrails])
return vt
def delete_viztrail(self, viztrail_id):
"""Delete the viztrail with given identifier. The result is True if a
viztrail with the given identifier existed, False otherwise.
Parameters
----------
viztrail_id : string
Unique viztrail identifier
Returns
-------
bool
"""
# Get the viztrail handle if it exists
if viztrail_id in self.viztrails:
# Call the delete method of the OSViztrailHandle to delete the
# files that are associated with the viztrail
self.viztrails[viztrail_id].delete_viztrail()
# Remove viztrail from the internal cache and write the updated
# viztrails index
del self.viztrails[viztrail_id]
self.object_store.write_object(
object_path=self.viztrails_index,
content=[vt for vt in self.viztrails])
return True
else:
return False
def get_viztrail(self, viztrail_id):
"""Retrieve the viztrail with the given identifier. The result is None
if no viztrail with given identifier exists.
Parameters
----------
viztrail_id : string
Unique viztrail identifier
Returns
-------
vizier.viztrail.base.ViztrailHandle
"""
if viztrail_id in self.viztrails:
return self.viztrails[viztrail_id]
else:
return None
def list_viztrails(self):
"""List handles for all viztrails in the repository.
Returns
-------
list(vizier.viztrail.base.ViztrailHandle)
"""
return list(self.viztrails.values())
