def test_reconfigureOrGetConfiguration_with_None_doesnt_affect_config(self): from niprov.dependencies import Dependencies dependencies = Dependencies() self.assertEqual(dependencies.getListener().verbosity, "info") dependencies.reconfigureOrGetConfiguration(None) self.assertEqual(dependencies.getListener().verbosity, "info")
def test_reconfigureOrGetConfiguration_with_new_config_returns_it(self): from niprov.dependencies import Dependencies, Configuration dependencies = Dependencies() newSettings = Configuration() newSettings.verbose = True outConfig = dependencies.reconfigureOrGetConfiguration(newSettings) self.assertEqual(newSettings, outConfig)
def test_Changing_setting_directly_affects_component_setting(self): from niprov.dependencies import Dependencies dependencies = Dependencies() self.assertEqual(dependencies.getListener().verbosity, "info") dependencies.config.verbosity = "warning" self.assertEqual(dependencies.getListener().verbosity, "warning")
def test_reconfigureOrGetConfiguration_creates_new_dependencies(self): from niprov.dependencies import Dependencies, Configuration dependencies = Dependencies() self.assertEqual(dependencies.getListener().verbosity, "info") newSettings = Configuration() newSettings.verbosity = "warning" dependencies.reconfigureOrGetConfiguration(newSettings) self.assertEqual(dependencies.getListener().verbosity, "warning")
def test_Changing_storage_setting_changes_repository_provided(self): from niprov.dependencies import Dependencies from niprov.jsonfile import JsonFile from niprov.mongo import MongoRepository dependencies = Dependencies() with patch('niprov.mongo.pymongo'): dependencies.config.database_type = 'file' self.assertIsInstance(dependencies.getRepository(), JsonFile) dependencies.config.database_type = 'MongoDB' self.assertIsInstance(dependencies.getRepository(), MongoRepository)
def test_Changing_storage_setting_changes_repository_provided(self): from niprov.dependencies import Dependencies from niprov.jsonfile import JsonFile from niprov.mongo import MongoRepository dependencies = Dependencies() with patch("niprov.mongo.pymongo"): dependencies.config.database_type = "file" self.assertIsInstance(dependencies.getRepository(), JsonFile) dependencies.config.database_type = "MongoDB" self.assertIsInstance(dependencies.getRepository(), MongoRepository)
def importp(filepath, dependencies=Dependencies()): """Add provenance in bulk from a file, such as saved by export(). Named importp as opposed to import because the latter is a reserved word in Python. This can serve as a backup, migration tool, or for exchange. """ repository = dependencies.getRepository() importDeps = Dependencies() importDeps.getConfiguration().database_url = filepath importRepo = JsonFile(importDeps) allFiles = importRepo.all() for pfile in allFiles: repository.add(pfile)
def main(global_config, **settings): """ This function returns a Pyramid WSGI application. """ # settings = {} # settings['reload_all'] = True # settings['debug_all'] = True # settings['mako.directories'] = os.path.join(here, 'templates') config = Configurator(settings=settings) config.include('pyramid_mako') config.add_static_view('static', 'static', cache_max_age=10) config.add_static_view('snapshots', os.path.expanduser('~/.niprov-snapshots'), cache_max_age=10) config.add_route('home', '/') config.add_route('latest', '/latest') config.add_route('stats', '/stats') config.add_route('short', '/id/{id}') config.add_route('pipeline', '/id/{id}/pipeline') config.add_route('location', '/locations/{host}*path') config.add_route('modality', '/modalities/{modality}') config.add_route('subject', '/subjects/{subject}') config.add_route('project', '/projects/{project}') config.add_route('user', '/users/{user}') config.add_route('search', '/search') config.add_route('modalities', '/modalities') config.add_route('projects', '/projects') config.add_route('users', '/users') config.add_request_method(lambda r: Dependencies(), 'dependencies', reify=True) config.scan() return config.make_wsgi_app()
def discover(root, dependencies=Dependencies()): """ Search a directory for image files, and add them to your provenance collection. Files are only included if they match the filters in the 'discover_file_extensions' settings. Refer to niprov.add for details on what happens to individual files. Args: root (str): The top directory in which to look for new files. """ filesys = dependencies.getFilesystem() filefilter = dependencies.getFileFilter() listener = dependencies.getListener() dirs = filesys.walk(root) stats = { 'total': 0, 'new': 0, 'series-new-file': 0, 'failed': 0, 'new-version': 0 } for (root, sdirs, files) in dirs: for filename in files: filepath = os.path.join(root, filename) if filefilter.include(filename): stats['total'] = stats['total'] + 1 p = add(filepath, transient=False, dependencies=dependencies) stats[p.status] = stats[p.status] + 1 listener.discoveryFinished(nnew=stats['new'], nadded=stats['series-new-file'], nfailed=stats['failed'], ntotal=stats['total'])
def export(provenance, medium, form, pipeline=False, dependencies=Dependencies()): """Publish or simply return provenance for selected files. To get provenance on one specific file, pass its path as the 'forFile' argument. Alternatively, to get all files associated with a certain subject, use the 'forSubject' argument. If none of these is used, provenance for the most recently registered files is reported. Args: provenance: Niprov BaseFile object or list of such. medium (str): The medium in which to publish the provenance. One of: 'stdout' (print the provenance to the terminal), 'direct' (return object to caller), 'file' (write to a text file), 'viewer' (open in the system image viewer). form (str): The format in which to serialize the provenance. One of 'json','xml','narrated','simple','dict','picture'. Returns: Depends on medium selected. """ formatFactory = dependencies.getFormatFactory() mediumFactory = dependencies.getMediumFactory() makePipeline = dependencies.getPipelineFactory() form = formatFactory.create(form) medium = mediumFactory.create(medium) if pipeline: provenance = makePipeline.forFile(provenance) formattedProvenance = form.serialize(provenance) return medium.export(formattedProvenance, form)
def __init__(self, dependencies=Dependencies()): self.filesys = dependencies.getFilesystem() self.json = dependencies.getSerializer() self.factory = dependencies.getFileFactory() self.pictureCache = dependencies.getPictureCache() url = dependencies.getConfiguration().database_url self.datafile = os.path.expanduser(url)
def markedForApproval(dependencies=Dependencies()): """List files marked for approval by a human. """ query = dependencies.getQuery() listener = dependencies.getListener() markedFiles = query.byApproval('pending') listener.filesMarkedForApproval(markedFiles) return markedFiles
def approve(filepath, dependencies=Dependencies()): """Mark this file as approved. Args: filepath (str): Path to the tracked file that has been found valid. """ loc = dependencies.getLocationFactory().completeString(filepath) repository = dependencies.getRepository() repository.updateApproval(loc, 'granted')
def search(text, dependencies=Dependencies()): """ Search for files with the given text in their provenance fields. Args: text (str): Words to look for. Returns: list: List of BaseFile objects """ return dependencies.getRepository().search(text)
def renameDicoms(dicomdir, dependencies=Dependencies()): """ Add the .dcm extension to any non-hidden files without extension. Args: dicomdir (str): Directory in which to rename files. """ listener = dependencies.getListener() for f in glob.glob(os.path.join(dicomdir, '*')): if '.' not in os.path.basename(f): listener.renamedDicom(f) shutil.move(f, f + '.dcm')
def test_reconfigureOrGetConfiguration_with_None_doesnt_affect_config( self): from niprov.dependencies import Dependencies dependencies = Dependencies() self.assertEqual(dependencies.getListener().verbosity, 'info') dependencies.reconfigureOrGetConfiguration(None) self.assertEqual(dependencies.getListener().verbosity, 'info')
def test_reconfigureOrGetConfiguration_creates_new_dependencies(self): from niprov.dependencies import Dependencies, Configuration dependencies = Dependencies() self.assertEqual(dependencies.getListener().verbosity, 'info') newSettings = Configuration() newSettings.verbosity = 'warning' dependencies.reconfigureOrGetConfiguration(newSettings) self.assertEqual(dependencies.getListener().verbosity, 'warning')
def selectApproved(files, dependencies=Dependencies()): """Return only files that have approval status 'granted'. Args: files (list): List of paths of files to check for approval status. """ repository = dependencies.getRepository() location = dependencies.getLocationFactory() selection = [] for filepath in files: locationString = location.completeString(filepath) img = repository.byLocation(locationString) if img.provenance.get('approval') == 'granted': selection.append(img.path) return selection
def markForApproval(files, reset=False, dependencies=Dependencies()): """Mark a list of files for approval by a human. Args: files (list): List of paths of files tracked by niprov to mark for approval. reset (bool): Also mark files that have already been approved. False by default. """ repository = dependencies.getRepository() location = dependencies.getLocationFactory() for filepath in files: loc = location.completeString(filepath) if not reset: image = repository.byLocation(loc) if image is None: raise ValueError('Unknown file: ' + filepath) if image.provenance.get('approval') == 'granted': continue repository.updateApproval(loc, 'pending')
def test_provides_PipelineFactory(self): from niprov.dependencies import Dependencies from niprov.pipelinefactory import PipelineFactory dependencies = Dependencies() self.assertIsInstance(dependencies.getPipelineFactory(), PipelineFactory)
def test_provides_Query(self): from niprov.dependencies import Dependencies from niprov.querying import Query dependencies = Dependencies() self.assertIsInstance(dependencies.getQuery(), Query)
def test_provides_PictureCache(self): from niprov.dependencies import Dependencies from niprov.pictures import PictureCache dependencies = Dependencies() self.assertIsInstance(dependencies.getPictureCache(), PictureCache)
def test_Changing_setting_directly_affects_component_setting(self): from niprov.dependencies import Dependencies dependencies = Dependencies() self.assertEqual(dependencies.getListener().verbosity, 'info') dependencies.config.verbosity = 'warning' self.assertEqual(dependencies.getListener().verbosity, 'warning')
def test_provides_MediumFactory(self): from niprov.dependencies import Dependencies from niprov.mediumfactory import MediumFactory dependencies = Dependencies() self.assertIsInstance(dependencies.getMediumFactory(), MediumFactory)
def print_(provenance, pipeline=False, dependencies=Dependencies()): """Shortcut for export(medium='stdout', form='simple'). """ return export(provenance, medium='stdout', form='simple', pipeline=pipeline, dependencies=dependencies)
def __init__(self, dependencies=Dependencies()): self.files = dependencies.getRepository()
def test_provides_LocationFactory(self): from niprov.dependencies import Dependencies from niprov.locationfactory import LocationFactory dependencies = Dependencies() self.assertIsInstance(dependencies.getLocationFactory(), LocationFactory)
def test_GetConfiguration_returns_config(self): from niprov.dependencies import Dependencies dependencies = Dependencies() outConfig = dependencies.getConfiguration() self.assertEqual(dependencies.config, outConfig)
def __init__(self, dependencies=Dependencies()): self.dependencies = dependencies
def test_provides_Externals(self): from niprov.dependencies import Dependencies from niprov.externals import Externals dependencies = Dependencies() self.assertIsInstance(dependencies.getExternals(), Externals)
def __init__(self): self.config = Configuration() self.deps = Dependencies(self.config)
def test_reconfigureOrGetConfiguration_with_None_returns_config(self): from niprov.dependencies import Dependencies dependencies = Dependencies() outConfig = dependencies.reconfigureOrGetConfiguration(None) self.assertEqual(dependencies.config, outConfig)
def __init__(self, dependencies=Dependencies()): self.fileExtension = 'txt'
class ProvenanceContext(object): def __init__(self): self.config = Configuration() self.deps = Dependencies(self.config) def add(self, filepath, transient=False, provenance=None): """See :py:mod:`niprov.adding` """ return niprov.adding.add(filepath, transient, provenance, self.deps) def approve(self, filepath): """See :py:mod:`niprov.approval` """ return niprov.approval.approve(filepath, dependencies=self.deps) def backup(self): """See :py:mod:`niprov.exporting` """ return niprov.exporting.backup(self.deps) def compare(self, file1, file2): """See :py:mod:`niprov.comparing` """ return niprov.comparing.compare(file1, file2, dependencies=self.deps) def discover(self, root): """See :py:mod:`niprov.discovery` """ return niprov.discovery.discover(root, dependencies=self.deps) def export(self, images, medium, form, pipeline=False): """See :py:mod:`niprov.exporting` """ return niprov.exporting.export(images, medium, form, pipeline, self.deps) def get(self): """See :py:mod:`niprov.querying` """ return self.deps.getQuery() def importp(self, fpath): """See :py:mod:`niprov.importing` """ return niprov.importing.importp(fpath, self.deps) def inspect(self, location): """See :py:mod:`niprov.inspection` """ return niprov.inspection.inspect(location, dependencies=self.deps) def log(self, new, transformation, parents, code=None, logtext=None, transient=False, script=None, user=None, provenance=None, opts=None): """See :py:mod:`niprov.plogging` """ return niprov.plogging.log(new, transformation, parents, code, logtext, transient, script, user, provenance, opts, self.deps) def markForApproval(self, files): """See :py:mod:`niprov.approval` """ return niprov.approval.markForApproval(files, dependencies=self.deps) def markedForApproval(self): """See :py:mod:`niprov.approval` """ return niprov.approval.markedForApproval(dependencies=self.deps) def print_(self, images, pipeline=False): """See :py:mod:`niprov.exporting` """ return niprov.exporting.print_(images, pipeline, self.deps) def renameDicoms(self, dicomdir): """See :py:mod:`niprov.renaming` """ return niprov.renaming.renameDicoms(dicomdir, dependencies=self.deps) def record(self, command, new=None, parents=None, transient=False, args=None, kwargs=None, user=None, opts=None): """See :py:mod:`niprov.recording` """ return niprov.recording.record(command, new, parents, transient, args, kwargs, user, opts, self.deps) def search(self, text): """See :py:mod:`niprov.searching` """ return niprov.searching.search(text, dependencies=self.deps) def selectApproved(self, files): """See :py:mod:`niprov.approval` """ return niprov.approval.selectApproved(files, dependencies=self.deps) def view(self, images, pipeline=False): """See :py:mod:`niprov.exporting` """ return niprov.exporting.view(images, pipeline, self.deps)
def test_Filefactory_provided(self): from niprov.dependencies import Dependencies from niprov.files import FileFactory dependencies = Dependencies() self.assertIsInstance(dependencies.getFileFactory(), FileFactory)
def test_provides_FileFilter(self): from niprov.dependencies import Dependencies from niprov.filefilter import FileFilter dependencies = Dependencies() self.assertIsInstance(dependencies.getFileFilter(), FileFilter)
def test_provides_FormatFactory(self): from niprov.dependencies import Dependencies from niprov.formatfactory import FormatFactory dependencies = Dependencies() self.assertIsInstance(dependencies.getFormatFactory(), FormatFactory)
def test_provides_Hasher(self): from niprov.dependencies import Dependencies from niprov.hashing import Hasher dependencies = Dependencies() self.assertIsInstance(dependencies.getHasher(), Hasher)
def __init__(self, dependencies=Dependencies()): self.libs = dependencies.getLibraries() self.listener = dependencies.getListener() self.dependencies = dependencies
def test_provides_Users(self): from niprov.dependencies import Dependencies from niprov.users import Users dependencies = Dependencies() self.assertIsInstance(dependencies.getUsers(), Users)
def backup(dependencies=Dependencies()): """Shortcut for export(medium='file', form='json') for all provenance. """ provenance = dependencies.getQuery().all() return export(provenance, medium='file', form='json', dependencies=dependencies)
def test_provides_Camera(self): from niprov.dependencies import Dependencies from niprov.camera import Camera dependencies = Dependencies() self.assertIsInstance(dependencies.getCamera(), Camera)
def view(provenance, pipeline=False, dependencies=Dependencies()): """Shortcut for export(medium='viewer', form='picture'). """ return export(provenance, medium='viewer', form='picture', pipeline=pipeline, dependencies=dependencies)