コード例 #1
0
ファイル: discovery.py プロジェクト: ktavabi/niprov
def discover(root, dependencies=Dependencies()):
    """
    Search a directory for image files, and add them to your provenance collection.

    Files are only included if they match the filters in the 
    'discover_file_extensions' settings.
    Refer to niprov.add for details on what happens to individual files.

    Args:
        root (str): The top directory in which to look for new files.
    """
    filesys = dependencies.getFilesystem()
    filefilter = dependencies.getFileFilter()
    listener = dependencies.getListener()

    dirs = filesys.walk(root)
    stats = {
        'total': 0,
        'new': 0,
        'series-new-file': 0,
        'failed': 0,
        'new-version': 0
    }
    for (root, sdirs, files) in dirs:
        for filename in files:
            filepath = os.path.join(root, filename)
            if filefilter.include(filename):
                stats['total'] = stats['total'] + 1
                p = add(filepath, transient=False, dependencies=dependencies)
                stats[p.status] = stats[p.status] + 1
    listener.discoveryFinished(nnew=stats['new'],
                               nadded=stats['series-new-file'],
                               nfailed=stats['failed'],
                               ntotal=stats['total'])
コード例 #2
0
ファイル: discovery.py プロジェクト: ilogue/niprov
def discover(root, dependencies=Dependencies()):
    """
    Search a directory for image files, and add them to your provenance collection.

    Files are only included if they match the filters in the 
    'discover_file_extensions' settings.
    Refer to niprov.add for details on what happens to individual files.

    Args:
        root (str): The top directory in which to look for new files.
    """
    filesys = dependencies.getFilesystem()
    filefilter = dependencies.getFileFilter()
    listener = dependencies.getListener()

    dirs = filesys.walk(root)
    stats = {'total':0, 'new':0, 'series-new-file':0, 'failed':0, 
        'new-version':0}
    for (root, sdirs, files) in dirs:
        for filename in files:
            filepath = os.path.join(root, filename)
            if filefilter.include(filename):
                stats['total'] = stats['total'] + 1
                p = add(filepath, transient=False, dependencies=dependencies)
                stats[p.status] = stats[p.status] + 1
    listener.discoveryFinished(nnew=stats['new'], 
        nadded=stats['series-new-file'], nfailed=stats['failed'], 
        ntotal=stats['total'])
コード例 #3
0
ファイル: test_add.py プロジェクト: ktavabi/niprov
 def add(self, path, **kwargs):
     from niprov.adding import add
     with patch('niprov.adding.inheritFrom') as self.inheritFrom:
         return add(path, dependencies=self.dependencies, **kwargs)
コード例 #4
0
ファイル: plogging.py プロジェクト: ilogue/niprov
def log(new, transformation, parents, code=None, logtext=None, transient=False,
        script=None, user=None, provenance=None, opts=None,
        dependencies=Dependencies()):
    """
    Register a transformation that creates a new image (or several).

    This will retrieve the primary parent's provenance. if no provenance is 
    availale for the primary parent, calls listener.unknownFile. Otherwise,
    some fields are copied from the primary parent, subject to availability. 
    For instance, if the parent has no 'subject' field, the new file's 
    provenance won't either.

    Args:
        new (str or list): Path(s) to the newly created file(s).
        transformation (str): Name of the operation that has been used.
        parents (str or list): Path(s) to the file(s) that were used as the 
            basis of the transformation. Assumes that the first file in the 
            list is the primary parent for which basic provenance is known.
        code (str, optional): Code that was used to generate the new file
        logtext (str, optional): Any information about the transformation that 
            was logged.
        script (str, optional): Path to the code file that contains the 
            transformation code.
        transient (bool, optional): Set this to True to indicate that the file 
            is only temporary and future checks should not expect it to be 
            physically present. Defaults to False, assuming that the file 
            remains.
        user (string, optional): Name of the user logging provenance.
        provenance (dict, optional): Add the key-value pairs in this dictionary 
            to the provenance record for the new files.
        opts (Configuration): General settings for niprov. 
            See :py:mod:`niprov.config`

    Raises:
      IOError: '[Errno 2] File not found' is raised if the new file does not
        exist on the filesystem and is not marked as transient.

    Returns:
        BaseFile: New provenance, if multiple files were created, this is 
            a list of images, otherwise, it is a single object.
    """
    repository = dependencies.getRepository()
    listener = dependencies.getListener()
    factory = dependencies.getFileFactory()
    filesys = dependencies.getFilesystem()
    opts = dependencies.reconfigureOrGetConfiguration(opts)
    location = dependencies.getLocationFactory()
    users = dependencies.getUsers()

    if isinstance(new, basestring):
        new = [new]
    if isinstance(parents, basestring):
        parents = [parents]
    if provenance is None:
        provenance = {}

    #gather provenance common to all new files
    parents = [location.completeString(p) for p in parents]
    commonProvenance = provenance
    commonProvenance['parents'] = parents
    commonProvenance['transformation'] = transformation
    commonProvenance['script'] = script
    commonProvenance['user'] = users.determineUser(user)
    if code:
        commonProvenance['code'] = code
    if logtext:
        commonProvenance['logtext'] = logtext
    parent = repository.byLocation(parents[0])
    if parent is None:
        parent = add(parents[0], dependencies=dependencies)
        listener.addUnknownParent(parents[0])

    inheritFrom(commonProvenance, parent.provenance)

    # do things specific to each new file
    newImages = []
    for newfile in new:
        singleProvenance = copy.deepcopy(commonProvenance)
        image = add(newfile, transient=transient, provenance=singleProvenance, 
            dependencies=dependencies)
        newImages.append(image)

    # only return one image if only one file was created
    if len(new) == 1:
        return image

    return newImages
コード例 #5
0
def log(new,
        transformation,
        parents,
        code=None,
        logtext=None,
        transient=False,
        script=None,
        user=None,
        provenance=None,
        opts=None,
        dependencies=Dependencies()):
    """
    Register a transformation that creates a new image (or several).

    This will retrieve the primary parent's provenance. if no provenance is 
    availale for the primary parent, calls listener.unknownFile. Otherwise,
    some fields are copied from the primary parent, subject to availability. 
    For instance, if the parent has no 'subject' field, the new file's 
    provenance won't either.

    Args:
        new (str or list): Path(s) to the newly created file(s).
        transformation (str): Name of the operation that has been used.
        parents (str or list): Path(s) to the file(s) that were used as the 
            basis of the transformation. Assumes that the first file in the 
            list is the primary parent for which basic provenance is known.
        code (str, optional): Code that was used to generate the new file
        logtext (str, optional): Any information about the transformation that 
            was logged.
        script (str, optional): Path to the code file that contains the 
            transformation code.
        transient (bool, optional): Set this to True to indicate that the file 
            is only temporary and future checks should not expect it to be 
            physically present. Defaults to False, assuming that the file 
            remains.
        user (string, optional): Name of the user logging provenance.
        provenance (dict, optional): Add the key-value pairs in this dictionary 
            to the provenance record for the new files.
        opts (Configuration): General settings for niprov. 
            See :py:mod:`niprov.config`

    Raises:
      IOError: '[Errno 2] File not found' is raised if the new file does not
        exist on the filesystem and is not marked as transient.

    Returns:
        BaseFile: New provenance, if multiple files were created, this is 
            a list of images, otherwise, it is a single object.
    """
    repository = dependencies.getRepository()
    listener = dependencies.getListener()
    factory = dependencies.getFileFactory()
    filesys = dependencies.getFilesystem()
    opts = dependencies.reconfigureOrGetConfiguration(opts)
    location = dependencies.getLocationFactory()
    users = dependencies.getUsers()

    if isinstance(new, basestring):
        new = [new]
    if isinstance(parents, basestring):
        parents = [parents]
    if provenance is None:
        provenance = {}

    #gather provenance common to all new files
    parents = [location.completeString(p) for p in parents]
    commonProvenance = provenance
    commonProvenance['parents'] = parents
    commonProvenance['transformation'] = transformation
    commonProvenance['script'] = script
    commonProvenance['user'] = users.determineUser(user)
    if code:
        commonProvenance['code'] = code
    if logtext:
        commonProvenance['logtext'] = logtext
    parent = repository.byLocation(parents[0])
    if parent is None:
        parent = add(parents[0], dependencies=dependencies)
        listener.addUnknownParent(parents[0])

    inheritFrom(commonProvenance, parent.provenance)

    # do things specific to each new file
    newImages = []
    for newfile in new:
        singleProvenance = copy.deepcopy(commonProvenance)
        image = add(newfile,
                    transient=transient,
                    provenance=singleProvenance,
                    dependencies=dependencies)
        newImages.append(image)

    # only return one image if only one file was created
    if len(new) == 1:
        return image

    return newImages