def test_error_on_missing(self): """Test that reading a missing object will raise a ValueError.""" store = DefaultObjectStore() filename = store.join(BASE_DIRECTORY, 'A.file') store.create_object(BASE_DIRECTORY, identifier='A.file', content={'A': 1}) self.assertTrue(store.exists(filename)) # Re-create the store to ensure that this has no effect store = DefaultObjectStore() store.read_object(filename) os.remove(filename) with self.assertRaises(ValueError): store.read_object(filename)
def test_exists(self): """Test exists method.""" store = DefaultObjectStore() filename = store.join(BASE_DIRECTORY, 'A.file') dirname = store.join(BASE_DIRECTORY, 'A.dir') self.assertFalse(store.exists(filename)) store.create_object(BASE_DIRECTORY, identifier='A.file') self.assertTrue(store.exists(filename)) self.assertFalse(store.exists(dirname)) os.makedirs(dirname) self.assertTrue(store.exists(dirname)) # Re-create the store to ensure that this has no effect store = DefaultObjectStore() self.assertTrue(store.exists(filename)) self.assertTrue(store.exists(dirname))
def test_create_file_repeat(self): """Test create file with identifier factory that not always returns a unique identifier. """ store = DefaultObjectStore(identifier_factory=IdFactory( max_attempts=MAX_ATTEMPS - 1)) id1 = store.create_object(BASE_DIRECTORY) id2 = store.create_object(BASE_DIRECTORY) self.assertNotEqual(id1, id2) self.assertTrue(store.exists(store.join(BASE_DIRECTORY, id1))) self.assertTrue(store.exists(store.join(BASE_DIRECTORY, id2))) store.delete_object(store.join(BASE_DIRECTORY, id1)) store.delete_object(store.join(BASE_DIRECTORY, id2)) store = DefaultObjectStore(identifier_factory=IdFactory( max_attempts=MAX_ATTEMPS + 1)) id1 = store.create_object(BASE_DIRECTORY) with self.assertRaises(RuntimeError): store.create_object(BASE_DIRECTORY)
def test_list_folders(self): """Test list_folders method.""" store = DefaultObjectStore() # The result is an empty list even if the folder does not exist and # is not created using the create flag dirname = store.join(BASE_DIRECTORY, 'A') dirs = store.list_folders(parent_folder=dirname, create=False) self.assertEqual(len(dirs), 0) self.assertFalse(store.exists(dirname)) # The result is an empty list after the folder is created using the # create flag dirs = store.list_folders(parent_folder=dirname, create=True) self.assertEqual(len(dirs), 0) self.assertTrue(store.exists(dirname)) # Create directories and files os.makedirs(store.join(dirname, 'A')) dirs = store.list_folders(parent_folder=dirname) self.assertEqual(len(dirs), 1) self.assertTrue('A' in dirs) os.makedirs(store.join(dirname, 'B')) dirs = store.list_folders(parent_folder=dirname, create=True) self.assertEqual(len(dirs), 2) self.assertTrue('A' in dirs) self.assertTrue('B' in dirs) filename = store.join(BASE_DIRECTORY, 'A.file') store.create_object(BASE_DIRECTORY, identifier='A.file') dirs = store.list_folders(parent_folder=dirname, create=True) self.assertEqual(len(dirs), 2) self.assertTrue('A' in dirs) self.assertTrue('B' in dirs) # Re-create the store to ensure that this has no effect store = DefaultObjectStore() dirs = store.list_folders(parent_folder=dirname, create=True) self.assertEqual(len(dirs), 2) self.assertTrue('A' in dirs) self.assertTrue('B' in dirs)
def test_create_object_with_identifier(self): """Test creating a new object with a given identifier.""" store = DefaultObjectStore() store.create_object(BASE_DIRECTORY, identifier='A') self.assertTrue(os.path.isfile(os.path.join(BASE_DIRECTORY, 'A'))) with self.assertRaises(ValueError): store.read_object(store.join(BASE_DIRECTORY, 'A')) store.create_object(BASE_DIRECTORY, identifier='B', content={'id': 100}) self.assertTrue(os.path.isfile(os.path.join(BASE_DIRECTORY, 'B'))) content = store.read_object(store.join(BASE_DIRECTORY, 'B')) self.assertEqual(content['id'], 100) store.create_object(BASE_DIRECTORY, identifier='A', content={'id': 100}) self.assertTrue(os.path.isfile(os.path.join(BASE_DIRECTORY, 'A'))) content = store.read_object(store.join(BASE_DIRECTORY, 'A')) self.assertEqual(content['id'], 100) store.create_object(BASE_DIRECTORY, identifier='B') self.assertTrue(os.path.isfile(os.path.join(BASE_DIRECTORY, 'B'))) with self.assertRaises(ValueError): store.read_object(store.join(BASE_DIRECTORY, 'B'))
def create_module(command, external_form, state, timestamp, outputs, provenance, module_folder, datasets=None, object_store=None): """Create a new materialized module instance for the given values. Parameters ---------- command : vizier.viztrail.command.ModuleCommand Specification of the module (i.e., package, name, and arguments) external_form: string Printable representation of module command state: int Module state (one of PENDING, RUNNING, CANCELED, ERROR, SUCCESS) timestamp: vizier.viztrail.module.timestamp.ModuleTimestamp Module timestamp datasets : dict(vizier.datastore.dataset.DatasetDescriptor) Dictionary of resulting datasets. Dataset descriptors are keyed by the user-specified dataset name. outputs: vizier.viztrail.module.output.ModuleOutputs Module output streams STDOUT and STDERR provenance: vizier.viztrail.module.provenance.ModuleProvenance Provenance information about datasets that were read and writen by previous execution of the module. module_folder: string Object store folder containing module resources object_store: vizier.core.io.base.ObjectStore, optional Object store implementation to access and maintain resources Returns ------- vizier.viztrail.objectstore.module.OSModuleHandle """ # Make sure the object store is not None if object_store is None: object_store = DefaultObjectStore() # Serialize module components and materialize obj = serialize_module(command=command, external_form=external_form, state=state, timestamp=timestamp, outputs=outputs, provenance=provenance) identifier = object_store.create_object(parent_folder=module_folder, content=obj) # Return handle for created module return OSModuleHandle( identifier=identifier, command=command, external_form=external_form, module_path=object_store.join(module_folder, identifier), state=state, timestamp=timestamp, datasets=datasets if not datasets is None else dict(), outputs=outputs, provenance=provenance, object_store=object_store)
class OSViztrailRepository(ViztrailRepository): """Repository for viztrails. This implementation maintains all resources that are managed by the repository as objects in an object store. The base path is the identifier prefix for all managed resources. By default all resources are mantained as directories and files on the local file system. The viztrails index is a list object that contains the identifier of active viztrails. Folders and Resources --------------------- viztrails : List of active viztrails <vt-identifier>/ : Folder with resources for individual viztrail """ def __init__(self, base_path, object_store=None): """Initialize the repository from a configuration dictionary. Expects a dictionary that contains at least the base path for the repository. The definition of the object store is optional. If none is given the default object store will be used. Parameters --------- base_path: string Path to the base directory for viztrail resources object_store: vizier.core.io.base.ObjectStore, optional Store for objects that represent viztrail resources not """ # Raise an exception if the base directory argument is not given if base_path is None: raise ValueError('missing path for base directory') # Create the base directory if it does not exist self.base_path = base_path if not os.path.isdir(self.base_path): os.makedirs(self.base_path) # The object store element is optional. If not given the default object # store is used. if not object_store is None: self.object_store = object_store else: self.object_store = DefaultObjectStore() # Initialize the viztrails index. Create the index file if it does not # exist. self.viztrails_index = self.object_store.join(self.base_path, OBJ_VIZTRAILINDEX) if not self.object_store.exists(self.viztrails_index): self.object_store.create_object(parent_folder=self.base_path, identifier=OBJ_VIZTRAILINDEX, content=list()) # Load viztrails and intialize the remaining instance variables by # calling the constructor of the super class self.viztrails = dict() for identifier in self.object_store.read_object(self.viztrails_index): vt = OSViztrailHandle.load_viztrail( base_path=self.object_store.join(self.base_path, identifier), object_store=self.object_store) self.viztrails[vt.identifier] = vt def create_viztrail(self, properties=None): """Create a new viztrail. The initial set of properties is an optional dictionary of (key,value)-pairs where all values are expected to either be scalar values or a list of scalar values. Parameters ---------- properties: dict, optional Set of properties for the new viztrail Returns ------- vizier.viztrail.objectstore.viztrail.OSViztrailHandle """ # Get unique identifier for new viztrail and viztrail directory. Raise # runtime error if the returned identifier is not unique. identifier = self.object_store.create_folder( parent_folder=self.base_path) viztrail_path = self.object_store.join(self.base_path, identifier) # Create materialized viztrail resource vt = OSViztrailHandle.create_viztrail(identifier=identifier, properties=properties, base_path=viztrail_path, object_store=self.object_store) # Add the new resource to the viztrails index. Write updated index to # object store before returning the new viztrail handle self.viztrails[vt.identifier] = vt self.object_store.write_object( object_path=self.viztrails_index, content=[vt_id for vt_id in self.viztrails]) return vt def delete_viztrail(self, viztrail_id): """Delete the viztrail with given identifier. The result is True if a viztrail with the given identifier existed, False otherwise. Parameters ---------- viztrail_id : string Unique viztrail identifier Returns ------- bool """ # Get the viztrail handle if it exists if viztrail_id in self.viztrails: # Call the delete method of the OSViztrailHandle to delete the # files that are associated with the viztrail self.viztrails[viztrail_id].delete_viztrail() # Remove viztrail from the internal cache and write the updated # viztrails index del self.viztrails[viztrail_id] self.object_store.write_object( object_path=self.viztrails_index, content=[vt for vt in self.viztrails]) return True else: return False def get_viztrail(self, viztrail_id): """Retrieve the viztrail with the given identifier. The result is None if no viztrail with given identifier exists. Parameters ---------- viztrail_id : string Unique viztrail identifier Returns ------- vizier.viztrail.base.ViztrailHandle """ if viztrail_id in self.viztrails: return self.viztrails[viztrail_id] else: return None def list_viztrails(self): """List handles for all viztrails in the repository. Returns ------- list(vizier.viztrail.base.ViztrailHandle) """ return list(self.viztrails.values())