Beispiel #1
0
def test_has_files(tmp_crumb):
    assert not os.path.exists(tmp_crumb.path)

    assert not tmp_crumb.has_files()

    values_dict = {
        'session_id': ['session_{:02}'.format(i) for i in range(2)],
        'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
        'modality': ['anat'],
        'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
    }

    paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    assert os.path.exists(tmp_crumb.split()[0])

    assert not tmp_crumb.has_files()

    pa = Path(str(paths[0]))
    pa.rmdir()
    pa.touch()

    assert pa.exists()
    assert pa.is_file()
    assert tmp_crumb.has_files()
Beispiel #2
0
    def _extract_to_cache(cls, cached_file, name):
        """
        Extract cached file zip file to cache folder
        :param str cached_file: local copy of archive file
        :param str name: cache context
        :return: cached folder containing the extracted archive content
        """
        # only zip files
        if not cached_file or not str(cached_file).lower().endswith('.zip'):
            return cached_file

        archive_suffix = cached_file.rpartition(".")[0]
        target_folder = Path("{0}_artifact_archive_{1}".format(
            archive_suffix, name))
        base_logger = LoggerRoot.get_base_logger()
        try:
            temp_target_folder = "{0}_{1}".format(target_folder.name,
                                                  time() * 1000)
            os.mkdir(path=temp_target_folder)
            ZipFile(cached_file).extractall(path=temp_target_folder)
            # we assume we will have such folder if we already extract the zip file
            # noinspection PyBroadException
            try:
                # if rename fails, it means that someone else already manged to extract the zip, delete the current
                # folder and return the already existing cached zip folder
                shutil.move(temp_target_folder, str(target_folder))
            except Exception:
                if target_folder.exists():
                    target_folder.touch(exist_ok=True)
                else:
                    base_logger.warning("Failed renaming {0} to {1}".format(
                        temp_target_folder, target_folder))
                try:
                    shutil.rmtree(temp_target_folder)
                except Exception as ex:
                    base_logger.warning(
                        "Exception {}\nFailed deleting folder {}".format(
                            ex, temp_target_folder))
        except Exception as ex:
            # failed extracting zip file:
            base_logger.warning(
                "Exception {}\nFailed extracting zip file {}".format(
                    ex, cached_file))
            # noinspection PyBroadException
            try:
                target_folder.rmdir()
            except Exception:
                pass
            return cached_file
        return target_folder
Beispiel #3
0
def test_has_files(tmp_crumb):
    assert not op.exists(tmp_crumb.path)

    assert not tmp_crumb.has_files()

    values_dict = {'session_id': ['session_{:02}'.format(i) for i in range( 2)],
                   'subject_id': ['subj_{:03}'.format(i)    for i in range( 3)],
                   'modality':   ['anat'],
                   'image':      ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
                   }

    paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))

    assert op.exists(tmp_crumb.split()[0])

    assert not tmp_crumb.has_files()

    pa = Path(str(paths[0]))
    pa.rmdir()
    pa.touch()

    assert pa.exists()
    assert pa.is_file()
    assert tmp_crumb.has_files()
Beispiel #4
0
    def _extract_to_cache(
            cls,
            cached_file,  # type: str
            name,  # type: str
            cache_context=None,  # type: Optional[str]
            target_folder=None,  # type: Optional[str]
            cache_path_encoding=None,  # type: Optional[str]
            force=False,  # type: bool
    ):
        # type: (...) -> str
        """
        Extract cached file to cache folder
        :param str cached_file: local copy of archive file
        :param str name: name of the target file
        :param str cache_context: cache context id
        :param str target_folder: specify target path to use for archive extraction
        :param str cache_path_encoding: specify representation of the local path of the cached files,
            this will always point to local cache folder, even if we have direct access file.
            Used for extracting the cached archived based on cache_path_encoding
        :param bool force: Force archive extraction even if target folder exists
        :return: cached folder containing the extracted archive content
        """
        if not cached_file:
            return cached_file

        cached_file = Path(cached_file)
        cache_path_encoding = Path(
            cache_path_encoding) if cache_path_encoding else None

        # we support zip and tar.gz files auto-extraction
        suffix = cached_file.suffix.lower()
        if suffix == '.gz':
            suffix = ''.join(a.lower() for a in cached_file.suffixes[-2:])

        if suffix not in (".zip", ".tgz", ".tar.gz"):
            return str(cached_file)

        cache_folder = Path(cache_path_encoding or cached_file).parent
        archive_suffix = (cache_path_encoding
                          or cached_file).name[:-len(suffix)]
        name = encode_string_to_filename(name) if name else name
        if target_folder:
            target_folder = Path(target_folder)
        else:
            target_folder = cache_folder / CacheManager.get_context_folder_lookup(
                cache_context).format(archive_suffix, name)

        if target_folder.is_dir() and not force:
            # noinspection PyBroadException
            try:
                target_folder.touch(exist_ok=True)
                return target_folder.as_posix()
            except Exception:
                pass

        base_logger = LoggerRoot.get_base_logger()
        try:
            # if target folder exists, meaning this is forced ao we extract directly into target folder
            if target_folder.is_dir():
                temp_target_folder = target_folder
            else:
                temp_target_folder = cache_folder / "{0}_{1}_{2}".format(
                    target_folder.name,
                    time() * 1000,
                    str(random()).replace('.', ''))
                temp_target_folder.mkdir(parents=True, exist_ok=True)

            if suffix == ".zip":
                ZipFile(cached_file.as_posix()).extractall(
                    path=temp_target_folder.as_posix())
            elif suffix == ".tar.gz":
                with tarfile.open(cached_file.as_posix()) as file:
                    file.extractall(temp_target_folder.as_posix())
            elif suffix == ".tgz":
                with tarfile.open(cached_file.as_posix(), mode='r:gz') as file:
                    file.extractall(temp_target_folder.as_posix())

            if temp_target_folder != target_folder:
                # we assume we will have such folder if we already extract the file
                # noinspection PyBroadException
                try:
                    # if rename fails, it means that someone else already manged to extract the file, delete the current
                    # folder and return the already existing cached zip folder
                    shutil.move(temp_target_folder.as_posix(),
                                target_folder.as_posix())
                except Exception:
                    if target_folder.exists():
                        target_folder.touch(exist_ok=True)
                    else:
                        base_logger.warning(
                            "Failed renaming {0} to {1}".format(
                                temp_target_folder.as_posix(),
                                target_folder.as_posix()))
                    try:
                        shutil.rmtree(temp_target_folder.as_posix())
                    except Exception as ex:
                        base_logger.warning(
                            "Exception {}\nFailed deleting folder {}".format(
                                ex, temp_target_folder.as_posix()))
        except Exception as ex:
            # failed extracting the file:
            base_logger.warning(
                "Exception {}\nFailed extracting zip file {}".format(
                    ex, cached_file.as_posix()))
            # noinspection PyBroadException
            try:
                target_folder.rmdir()
            except Exception:
                pass
            return cached_file.as_posix()
        return target_folder.as_posix()
Beispiel #5
0
    def _extract_to_cache(cls,
                          cached_file,
                          name,
                          cache_context=None,
                          target_folder=None):
        # type: (str, str, Optional[str], Optional[str]) -> str
        """
        Extract cached file to cache folder
        :param str cached_file: local copy of archive file
        :param str name: name of the target file
        :param str cache_context: cache context id
        :param str target_folder: specify target path to use for archive extraction
        :return: cached folder containing the extracted archive content
        """
        if not cached_file:
            return cached_file

        cached_file = Path(cached_file)

        # we support zip and tar.gz files auto-extraction
        suffix = cached_file.suffix.lower()
        if suffix == '.gz':
            suffix = ''.join(a.lower() for a in cached_file.suffixes[-2:])

        if suffix not in (".zip", ".tgz", ".tar.gz"):
            return str(cached_file)

        cached_folder = Path(cached_file).parent
        archive_suffix = cached_file.name[:-len(suffix)]
        name = encode_string_to_filename(name)
        target_folder = Path(
            target_folder
            or CacheManager.get_context_folder_lookup(cache_context).format(
                archive_suffix, name))

        if target_folder.exists():
            # noinspection PyBroadException
            try:
                target_folder.touch(exist_ok=True)
                return target_folder.as_posix()
            except Exception:
                pass

        base_logger = LoggerRoot.get_base_logger()
        try:
            temp_target_folder = cached_folder / "{0}_{1}_{2}".format(
                target_folder.name,
                time() * 1000,
                str(random()).replace('.', ''))
            temp_target_folder.mkdir(parents=True, exist_ok=True)
            if suffix == ".zip":
                ZipFile(cached_file.as_posix()).extractall(
                    path=temp_target_folder.as_posix())
            elif suffix == ".tar.gz":
                with tarfile.open(cached_file.as_posix()) as file:
                    file.extractall(temp_target_folder.as_posix())
            elif suffix == ".tgz":
                with tarfile.open(cached_file.as_posix(), mode='r:gz') as file:
                    file.extractall(temp_target_folder.as_posix())

            # we assume we will have such folder if we already extract the file
            # noinspection PyBroadException
            try:
                # if rename fails, it means that someone else already manged to extract the file, delete the current
                # folder and return the already existing cached zip folder
                shutil.move(temp_target_folder.as_posix(),
                            target_folder.as_posix())
            except Exception:
                if target_folder.exists():
                    target_folder.touch(exist_ok=True)
                else:
                    base_logger.warning("Failed renaming {0} to {1}".format(
                        temp_target_folder.as_posix(),
                        target_folder.as_posix()))
                try:
                    shutil.rmtree(temp_target_folder.as_posix())
                except Exception as ex:
                    base_logger.warning(
                        "Exception {}\nFailed deleting folder {}".format(
                            ex, temp_target_folder.as_posix()))
        except Exception as ex:
            # failed extracting the file:
            base_logger.warning(
                "Exception {}\nFailed extracting zip file {}".format(
                    ex, cached_file.as_posix()))
            # noinspection PyBroadException
            try:
                target_folder.rmdir()
            except Exception:
                pass
            return cached_file.as_posix()
        return target_folder.as_posix()
Beispiel #6
0
    def _extract_to_cache(cls, cached_file, name):
        """
        Extract cached file to cache folder
        :param str cached_file: local copy of archive file
        :param str name: cache context
        :return: cached folder containing the extracted archive content
        """
        if not cached_file:
            return cached_file

        cached_file = Path(cached_file)

        # we support zip and tar.gz files auto-extraction
        if (not cached_file.suffix == ".zip"
                and not cached_file.suffixes[-2:] == [".tar", ".gz"]):
            return str(cached_file)

        cached_folder = cached_file.parent

        name = encode_string_to_filename(name) if name else name
        target_folder = Path("{0}/{1}_artifacts_archive_{2}".format(
            cached_folder, cached_file.stem, name))
        if target_folder.exists():
            # noinspection PyBroadException
            try:
                target_folder.touch(exist_ok=True)
                return target_folder
            except Exception:
                pass

        base_logger = LoggerRoot.get_base_logger()
        try:
            temp_target_folder = cached_folder / "{0}_{1}_{2}".format(
                target_folder.name,
                time() * 1000,
                str(random()).replace('.', ''))
            temp_target_folder.mkdir(parents=True, exist_ok=True)
            if cached_file.suffix == ".zip":
                ZipFile(cached_file).extractall(
                    path=temp_target_folder.as_posix())
            elif cached_file.suffixes[-2:] == [".tar", ".gz"]:
                with tarfile.open(cached_file) as file:
                    file.extractall(temp_target_folder)

            # we assume we will have such folder if we already extract the file
            # noinspection PyBroadException
            try:
                # if rename fails, it means that someone else already manged to extract the file, delete the current
                # folder and return the already existing cached zip folder
                shutil.move(temp_target_folder.as_posix(),
                            target_folder.as_posix())
            except Exception:
                if target_folder.exists():
                    target_folder.touch(exist_ok=True)
                else:
                    base_logger.warning("Failed renaming {0} to {1}".format(
                        temp_target_folder, target_folder))
                try:
                    shutil.rmtree(temp_target_folder)
                except Exception as ex:
                    base_logger.warning(
                        "Exception {}\nFailed deleting folder {}".format(
                            ex, temp_target_folder))
        except Exception as ex:
            # failed extracting the file:
            base_logger.warning(
                "Exception {}\nFailed extracting zip file {}".format(
                    ex, str(cached_file)))
            # noinspection PyBroadException
            try:
                target_folder.rmdir()
            except Exception:
                pass
            return cached_file
        return target_folder