def initialize_logging(self):
logging_config = self._config.get("logging", None)
if not logging_config:
return False
# handle incomplete file handlers
deleted = []
handlers = logging_config.get("handlers", {})
for name, handler in list(handlers.items()):
cls = handler.get("class", None)
is_file = cls and "FileHandler" in cls
if cls is None or (is_file and "filename" not in handler):
deleted.append(name)
del handlers[name]
elif is_file:
file = Path(handler.get("filename"))
if not file.is_file():
file.parent.mkdir(parents=True, exist_ok=True)
file.touch()
# remove dependency in deleted handlers
root_logger = logging_config.get("root", None)
loggers = list(logging_config.get(
"loggers", {}).values()) + ([root_logger] if root_logger else [])
for logger in loggers:
handlers = logger.get("handlers", None)
if not handlers:
continue
logger["handlers"] = [h for h in handlers if h not in deleted]
extra = None
if self._app:
extra = {"app": self._app}
initialize_log(logging_config, extra=extra)
return True
def test_inputs_archive_with_subfolder_is_supplied_no_ri___archive_is_extracted_into_inputs(
self):
with TemporaryDirectory() as run_dir, TemporaryDirectory(
) as oasis_src_fp, TemporaryDirectory() as model_data_fp:
analysis_settings_fp = os.path.join(oasis_src_fp, "settings.json")
Path(analysis_settings_fp).touch()
tar_path = os.path.join(oasis_src_fp, 'archive.tar')
with tarfile.open(tar_path, 'w') as tar:
archived_file_path = Path(oasis_src_fp, 'archived_file')
archived_file_path.touch()
tar.add(str(archived_file_path), arcname='archived_file')
os.remove(str(archived_file_path))
os.mkdir(os.path.join(oasis_src_fp, "sub1"))
archived_file_path = Path(oasis_src_fp, "sub1",
'archived_file')
archived_file_path.touch()
tar.add(str(archived_file_path),
arcname='sub1{}archived_file'.format(os.sep))
shutil.rmtree(os.path.join(oasis_src_fp, "sub1"))
prepare_run_directory(run_dir,
oasis_src_fp,
model_data_fp,
analysis_settings_fp,
inputs_archive=tar_path)
self.assertTrue(Path(run_dir, 'input', 'archived_file').exists())
self.assertTrue(
Path(run_dir, 'input', 'sub1', 'archived_file').exists())
def test_has_files(tmp_crumb):
assert not os.path.exists(tmp_crumb.path)
assert not tmp_crumb.has_files()
values_dict = {
'session_id': ['session_{:02}'.format(i) for i in range(2)],
'subject_id': ['subj_{:03}'.format(i) for i in range(3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
assert os.path.exists(tmp_crumb.split()[0])
assert not tmp_crumb.has_files()
pa = Path(str(paths[0]))
pa.rmdir()
pa.touch()
assert pa.exists()
assert pa.is_file()
assert tmp_crumb.has_files()
def run():
import sys
from spil.libs.util.log import setLevel, DEBUG, INFO
print('')
print('Tests start')
print('')
setLevel(INFO)
# setLevel(DEBUG) # In case of problems, use DEBUG mode
print('*' * 60)
test_paths = []
# tests = [r'Z:\Projects\Vic_Movie\Prod']
# if not input('Create if not existing ?'):
# sys.exit()
from spil.conf.sid_conf import test_sids as tests
test_sids = []
for test in tests:
sid = Sid(test)
print('Sid : {}'.format(sid))
# We fill defaults, and strip
sid.set_defaults()
print('Sid with defaults: {}'.format(sid))
sid = sid.get_stripped()
print('Sid stripped: {}'.format(sid))
path = sid.path
if path:
test_paths.append(path)
print('Appended : {}'.format(path))
print('')
for path in test_paths:
path = Path(path)
print(path)
if not path.exists():
if is_filename(path):
print(path, 'is a file')
if not path.parent.exists():
os.makedirs(str(path.parent))
path.touch()
else:
print(path, 'is a dir')
os.makedirs(str(path))
print('Created test paths : ')
pprint(test_paths)
def test_inputs_archive_is_supplied___archive_is_extracted_into_inputs(self):
with TemporaryDirectory() as output_path, TemporaryDirectory() as input_path:
tar_path = os.path.join(input_path, 'archive.tar')
with tarfile.open(tar_path, 'w', encoding='utf-8') as tar:
archived_file_path = Path(input_path, 'archived_file')
archived_file_path.touch()
tar.add(str(archived_file_path), arcname='archived_file')
prepare_model_run_directory(output_path, inputs_archive=tar_path)
self.assertTrue(Path(output_path, 'input', 'archived_file').exists())
def test_inputs_archive_is_supplied_with_ri___archive_is_extracted_into_run_dir(self):
with TemporaryDirectory() as run_dir, TemporaryDirectory() as oasis_src_fp, TemporaryDirectory() as model_data_fp, NamedTemporaryFile() as analysis_settings_fp:
tar_path = os.path.join(oasis_src_fp, 'archive.tar')
with tarfile.open(tar_path, 'w', encoding='utf-8') as tar:
archived_file_path = Path(oasis_src_fp, 'archived_file')
archived_file_path.touch()
tar.add(str(archived_file_path), arcname='archived_file')
prepare_run_directory(run_dir, oasis_src_fp, model_data_fp, analysis_settings_fp.name, inputs_archive=tar_path, ri=True)
self.assertTrue(Path(run_dir, 'archived_file').exists())
def _extract_to_cache(cls, cached_file, name):
"""
Extract cached file zip file to cache folder
:param str cached_file: local copy of archive file
:param str name: cache context
:return: cached folder containing the extracted archive content
"""
# only zip files
if not cached_file or not str(cached_file).lower().endswith('.zip'):
return cached_file
archive_suffix = cached_file.rpartition(".")[0]
target_folder = Path("{0}_artifact_archive_{1}".format(
archive_suffix, name))
base_logger = LoggerRoot.get_base_logger()
try:
temp_target_folder = "{0}_{1}".format(target_folder.name,
time() * 1000)
os.mkdir(path=temp_target_folder)
ZipFile(cached_file).extractall(path=temp_target_folder)
# we assume we will have such folder if we already extract the zip file
# noinspection PyBroadException
try:
# if rename fails, it means that someone else already manged to extract the zip, delete the current
# folder and return the already existing cached zip folder
shutil.move(temp_target_folder, str(target_folder))
except Exception:
if target_folder.exists():
target_folder.touch(exist_ok=True)
else:
base_logger.warning("Failed renaming {0} to {1}".format(
temp_target_folder, target_folder))
try:
shutil.rmtree(temp_target_folder)
except Exception as ex:
base_logger.warning(
"Exception {}\nFailed deleting folder {}".format(
ex, temp_target_folder))
except Exception as ex:
# failed extracting zip file:
base_logger.warning(
"Exception {}\nFailed extracting zip file {}".format(
ex, cached_file))
# noinspection PyBroadException
try:
target_folder.rmdir()
except Exception:
pass
return cached_file
return target_folder
def convert_to_csv(self,
dat_path: str,
csv_path: Optional[str] = '') -> None:
"""Convert a .dat file to csv
Check whether the .dat file exists
Then read it
Finally store its .csv equivalent
:see: https://tools.ietf.org/html/rfc4180
:param dat_path: path to the .dat file
:param csv_path: name and location of the generated .csv file
:return: None
"""
# checking source file integrity
source = Path(dat_path)
if not source.exists():
raise FileNotFoundError
if source.suffix != Dat.ext:
raise BadFileFormatException(
f'source file should contains the extension: '
f'{Dat.ext}')
# checking output file integrity
if not csv_path:
csv_path = f'{self.DEFAULT_OUTPUT_FOLDER}' \
f'{source.name.replace(Dat.ext, Csv.ext)}'
else:
if not csv_path.endswith(Csv.ext):
raise BadFileFormatException(
f'output should contains the extension: '
f'{Csv.ext}')
output = Path(csv_path)
if not output.exists():
output.touch()
# formatting content
with source.open(mode='r', encoding=Dat.encoding) as src:
content = src.readlines()
# writing the formatted content
with output.open(mode='w', encoding=Csv.encoding) as dest:
for row in self._format_dirty_content(content):
dest.write(row)
def download_stream(self, url, file, chunk_size=1024):
"""Download url to file and get download progress through generator
Args:
file: Can either be a pathlike obj, str or filelike obj
Yield:
Tuple like (chunks downloaded, total chunks)
"""
file_obj = None
if isinstance(file, Path) or (file is not None
and isinstance(file, py3Path)):
pass
elif isinstance(file, six.string_types):
file = Path(file)
elif hasattr(file, 'write'):
file_obj = file
else:
raise TypeError('File is neither pathlike, filelike or a sring')
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36',
'Referer':
self.base_url
}
response = requests.get(url, headers=headers, stream=True)
file_size = int(response.headers['Content-Length'])
toatal_chunks = int(file_size / chunk_size)
yield 0, toatal_chunks
if file_obj:
for chunks_downloaded, chunk in enumerate(
response.iter_content(chunk_size=chunk_size), 1):
file_obj.write(chunk)
yield chunks_downloaded, toatal_chunks
else:
file.touch()
with file.open('wb') as file_obj:
for chunks_downloaded, chunk in enumerate(
response.iter_content(chunk_size=chunk_size), 1):
file_obj.write(chunk)
yield chunks_downloaded, toatal_chunks
def test_has_files(tmp_crumb):
assert not op.exists(tmp_crumb.path)
assert not tmp_crumb.has_files()
values_dict = {'session_id': ['session_{:02}'.format(i) for i in range( 2)],
'subject_id': ['subj_{:03}'.format(i) for i in range( 3)],
'modality': ['anat'],
'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'],
}
paths = mktree(tmp_crumb, list(ParameterGrid(values_dict)))
assert op.exists(tmp_crumb.split()[0])
assert not tmp_crumb.has_files()
pa = Path(str(paths[0]))
pa.rmdir()
pa.touch()
assert pa.exists()
assert pa.is_file()
assert tmp_crumb.has_files()
def _extract_to_cache(
cls,
cached_file, # type: str
name, # type: str
cache_context=None, # type: Optional[str]
target_folder=None, # type: Optional[str]
cache_path_encoding=None, # type: Optional[str]
force=False, # type: bool
):
# type: (...) -> str
"""
Extract cached file to cache folder
:param str cached_file: local copy of archive file
:param str name: name of the target file
:param str cache_context: cache context id
:param str target_folder: specify target path to use for archive extraction
:param str cache_path_encoding: specify representation of the local path of the cached files,
this will always point to local cache folder, even if we have direct access file.
Used for extracting the cached archived based on cache_path_encoding
:param bool force: Force archive extraction even if target folder exists
:return: cached folder containing the extracted archive content
"""
if not cached_file:
return cached_file
cached_file = Path(cached_file)
cache_path_encoding = Path(
cache_path_encoding) if cache_path_encoding else None
# we support zip and tar.gz files auto-extraction
suffix = cached_file.suffix.lower()
if suffix == '.gz':
suffix = ''.join(a.lower() for a in cached_file.suffixes[-2:])
if suffix not in (".zip", ".tgz", ".tar.gz"):
return str(cached_file)
cache_folder = Path(cache_path_encoding or cached_file).parent
archive_suffix = (cache_path_encoding
or cached_file).name[:-len(suffix)]
name = encode_string_to_filename(name) if name else name
if target_folder:
target_folder = Path(target_folder)
else:
target_folder = cache_folder / CacheManager.get_context_folder_lookup(
cache_context).format(archive_suffix, name)
if target_folder.is_dir() and not force:
# noinspection PyBroadException
try:
target_folder.touch(exist_ok=True)
return target_folder.as_posix()
except Exception:
pass
base_logger = LoggerRoot.get_base_logger()
try:
# if target folder exists, meaning this is forced ao we extract directly into target folder
if target_folder.is_dir():
temp_target_folder = target_folder
else:
temp_target_folder = cache_folder / "{0}_{1}_{2}".format(
target_folder.name,
time() * 1000,
str(random()).replace('.', ''))
temp_target_folder.mkdir(parents=True, exist_ok=True)
if suffix == ".zip":
ZipFile(cached_file.as_posix()).extractall(
path=temp_target_folder.as_posix())
elif suffix == ".tar.gz":
with tarfile.open(cached_file.as_posix()) as file:
file.extractall(temp_target_folder.as_posix())
elif suffix == ".tgz":
with tarfile.open(cached_file.as_posix(), mode='r:gz') as file:
file.extractall(temp_target_folder.as_posix())
if temp_target_folder != target_folder:
# we assume we will have such folder if we already extract the file
# noinspection PyBroadException
try:
# if rename fails, it means that someone else already manged to extract the file, delete the current
# folder and return the already existing cached zip folder
shutil.move(temp_target_folder.as_posix(),
target_folder.as_posix())
except Exception:
if target_folder.exists():
target_folder.touch(exist_ok=True)
else:
base_logger.warning(
"Failed renaming {0} to {1}".format(
temp_target_folder.as_posix(),
target_folder.as_posix()))
try:
shutil.rmtree(temp_target_folder.as_posix())
except Exception as ex:
base_logger.warning(
"Exception {}\nFailed deleting folder {}".format(
ex, temp_target_folder.as_posix()))
except Exception as ex:
# failed extracting the file:
base_logger.warning(
"Exception {}\nFailed extracting zip file {}".format(
ex, cached_file.as_posix()))
# noinspection PyBroadException
try:
target_folder.rmdir()
except Exception:
pass
return cached_file.as_posix()
return target_folder.as_posix()
def _extract_to_cache(cls,
cached_file,
name,
cache_context=None,
target_folder=None):
# type: (str, str, Optional[str], Optional[str]) -> str
"""
Extract cached file to cache folder
:param str cached_file: local copy of archive file
:param str name: name of the target file
:param str cache_context: cache context id
:param str target_folder: specify target path to use for archive extraction
:return: cached folder containing the extracted archive content
"""
if not cached_file:
return cached_file
cached_file = Path(cached_file)
# we support zip and tar.gz files auto-extraction
suffix = cached_file.suffix.lower()
if suffix == '.gz':
suffix = ''.join(a.lower() for a in cached_file.suffixes[-2:])
if suffix not in (".zip", ".tgz", ".tar.gz"):
return str(cached_file)
cached_folder = Path(cached_file).parent
archive_suffix = cached_file.name[:-len(suffix)]
name = encode_string_to_filename(name)
target_folder = Path(
target_folder
or CacheManager.get_context_folder_lookup(cache_context).format(
archive_suffix, name))
if target_folder.exists():
# noinspection PyBroadException
try:
target_folder.touch(exist_ok=True)
return target_folder.as_posix()
except Exception:
pass
base_logger = LoggerRoot.get_base_logger()
try:
temp_target_folder = cached_folder / "{0}_{1}_{2}".format(
target_folder.name,
time() * 1000,
str(random()).replace('.', ''))
temp_target_folder.mkdir(parents=True, exist_ok=True)
if suffix == ".zip":
ZipFile(cached_file.as_posix()).extractall(
path=temp_target_folder.as_posix())
elif suffix == ".tar.gz":
with tarfile.open(cached_file.as_posix()) as file:
file.extractall(temp_target_folder.as_posix())
elif suffix == ".tgz":
with tarfile.open(cached_file.as_posix(), mode='r:gz') as file:
file.extractall(temp_target_folder.as_posix())
# we assume we will have such folder if we already extract the file
# noinspection PyBroadException
try:
# if rename fails, it means that someone else already manged to extract the file, delete the current
# folder and return the already existing cached zip folder
shutil.move(temp_target_folder.as_posix(),
target_folder.as_posix())
except Exception:
if target_folder.exists():
target_folder.touch(exist_ok=True)
else:
base_logger.warning("Failed renaming {0} to {1}".format(
temp_target_folder.as_posix(),
target_folder.as_posix()))
try:
shutil.rmtree(temp_target_folder.as_posix())
except Exception as ex:
base_logger.warning(
"Exception {}\nFailed deleting folder {}".format(
ex, temp_target_folder.as_posix()))
except Exception as ex:
# failed extracting the file:
base_logger.warning(
"Exception {}\nFailed extracting zip file {}".format(
ex, cached_file.as_posix()))
# noinspection PyBroadException
try:
target_folder.rmdir()
except Exception:
pass
return cached_file.as_posix()
return target_folder.as_posix()
class TestReader(TestCase):
"""References TestReader
Test suite for the Reader class.
`Dat` and `Csv` may be used in unit tests because they does not
contains any logic.
"""
def setUp(self) -> None:
"""Initializing the object to test
"""
# dummy simple csv for tests purpose
self.dummy_csv = Path(FileUtils.Csv.CSV_NAME)
self.dummy_csv.touch()
self.dummy_csv.write_text(FileUtils.Csv.CSV_CONTENT)
def tearDown(self) -> None:
"""Reinitialize state after unit tests execution
"""
self.dummy_csv.unlink()
def test_invalid_initialization_unknown_file(self):
"""A non-existing file should throw an exception
"""
with self.assertRaises(FileNotFoundError):
Reader(FileUtils.Csv.NON_EXISTING_NAME)
def test_valid_properties_columns(self):
"""The reader should correctly get the columns number
"""
# arrange
expected = len(
FileUtils.Csv.CSV_CONTENT.split(Csv.line_end)[0].split(
Csv.separator))
# act
reader = Reader(str(self.dummy_csv))
# assert
self.assertEqual(expected, reader.columns)
def test_valid_properties_rows(self):
"""The reader should correctly get the columns number
"""
# arrange
expected = len(FileUtils.Csv.CSV_CONTENT.split(Csv.line_end))
# act
reader = Reader(str(self.dummy_csv))
# assert
self.assertEqual(expected, reader.rows)
def test_valid_read_content(self):
"""The reader should correctly gather the fields
"""
# arrange
first_line = FileUtils.Csv.CSV_CONTENT \
.split(Csv.line_end)[0].split(Csv.separator)
# extract data from formatted string
first_line = [
field[1:-1] if field.startswith(Csv.delimiter)
and field.endswith(Csv.delimiter) else field
for field in first_line
]
expected_len = len(first_line)
reader = Reader(str(self.dummy_csv))
# act
# read the content of the file
content = list(reader.read_content())
# only stores the first line
content = content[0]
# assert
# the length of the content read should be the same as specified
self.assertEqual(expected_len, len(content))
# the files writen and loaded should be the same
self.assertListEqual(first_line, content)
def _extract_to_cache(cls, cached_file, name):
"""
Extract cached file to cache folder
:param str cached_file: local copy of archive file
:param str name: cache context
:return: cached folder containing the extracted archive content
"""
if not cached_file:
return cached_file
cached_file = Path(cached_file)
# we support zip and tar.gz files auto-extraction
if (not cached_file.suffix == ".zip"
and not cached_file.suffixes[-2:] == [".tar", ".gz"]):
return str(cached_file)
cached_folder = cached_file.parent
name = encode_string_to_filename(name) if name else name
target_folder = Path("{0}/{1}_artifacts_archive_{2}".format(
cached_folder, cached_file.stem, name))
if target_folder.exists():
# noinspection PyBroadException
try:
target_folder.touch(exist_ok=True)
return target_folder
except Exception:
pass
base_logger = LoggerRoot.get_base_logger()
try:
temp_target_folder = cached_folder / "{0}_{1}_{2}".format(
target_folder.name,
time() * 1000,
str(random()).replace('.', ''))
temp_target_folder.mkdir(parents=True, exist_ok=True)
if cached_file.suffix == ".zip":
ZipFile(cached_file).extractall(
path=temp_target_folder.as_posix())
elif cached_file.suffixes[-2:] == [".tar", ".gz"]:
with tarfile.open(cached_file) as file:
file.extractall(temp_target_folder)
# we assume we will have such folder if we already extract the file
# noinspection PyBroadException
try:
# if rename fails, it means that someone else already manged to extract the file, delete the current
# folder and return the already existing cached zip folder
shutil.move(temp_target_folder.as_posix(),
target_folder.as_posix())
except Exception:
if target_folder.exists():
target_folder.touch(exist_ok=True)
else:
base_logger.warning("Failed renaming {0} to {1}".format(
temp_target_folder, target_folder))
try:
shutil.rmtree(temp_target_folder)
except Exception as ex:
base_logger.warning(
"Exception {}\nFailed deleting folder {}".format(
ex, temp_target_folder))
except Exception as ex:
# failed extracting the file:
base_logger.warning(
"Exception {}\nFailed extracting zip file {}".format(
ex, str(cached_file)))
# noinspection PyBroadException
try:
target_folder.rmdir()
except Exception:
pass
return cached_file
return target_folder
class TestNormalizer(TestCase):
"""References TestNormalizer
Test suite for the Normalizer class.
`Dat` and `Csv` may be used in unit tests because they does not
contains any logic.
"""
def setUp(self) -> None:
"""Initializing the object to test
"""
self.normalizer = Normalizer(to_normalize_ext=Dat.ext,
separator=Dat.separator)
self.dummy_csv = Path(FileUtils.Csv.CSV_NAME)
self.dummy_csv.touch()
self.dummy_dat = Path(FileUtils.Csv.DAT_NAME)
self.dummy_dat.touch()
def tearDown(self) -> None:
"""Reinitialize state after unit tests execution
"""
self.dummy_csv.unlink()
self.dummy_dat.unlink()
def test_invalid_is_valid_csv_field_number(self):
"""A bad formatted number should be invalid
"""
# trailing quotes
self.assertFalse(Normalizer.is_valid_csv_field('1337"'))
# beginning quotes
self.assertFalse(Normalizer.is_valid_csv_field('"1337'))
# no quotes
self.assertFalse(Normalizer.is_valid_csv_field('1337'))
def test_valid_is_valid_csv_field_number(self):
"""A well formatted number should be valid
"""
# int
self.assertTrue(Normalizer.is_valid_csv_field('"42"'))
# float
self.assertTrue(Normalizer.is_valid_csv_field('"13.37"'))
# negative
self.assertTrue(Normalizer.is_valid_csv_field('"-3.14"'))
def test_valid_is_valid_csv_field_string(self):
"""A well formatted string should be valid
"""
# single string
self.assertTrue(Normalizer.is_valid_csv_field('"field"'))
# with spaces
self.assertTrue(Normalizer.is_valid_csv_field('"some field"'))
def test_invalid_convert_to_csv_no_file(self):
"""A non-existing file should throw an exception
"""
# with an incorrect extension too
with self.assertRaises(FileNotFoundError):
self.normalizer.convert_to_csv(
dat_path=FileUtils.Csv.NON_EXISTING_NAME)
# with the appropriate extension
with self.assertRaises(FileNotFoundError):
self.normalizer.convert_to_csv(
dat_path=FileUtils.Csv.NON_EXISTING_NAME + Dat.ext)
def test_invalid_convert_to_csv_bad_file_dat_ext(self):
"""A bad DAT file extension should throw an exception
"""
with self.assertRaises(BadFileFormatException):
self.normalizer.convert_to_csv(dat_path=str(self.dummy_csv))
def test_invalid_convert_to_csv_bad_file_dat_csv(self):
"""A bad CSV file extension should throw an exception
"""
with self.assertRaises(BadFileFormatException):
self.normalizer.convert_to_csv(dat_path=str(self.dummy_dat),
csv_path=str(self.dummy_dat))
def test_invalid_convert_to_csv_from_folder_non_existing_folder(self):
"""A non-existing folder should throw an exception
"""
with self.assertRaises(BadFileFormatException):
self.normalizer.convert_to_csv_from_folder(
dat_folder=FileUtils.Csv.NON_EXISTING_NAME)
def test_invalid_convert_to_csv_from_folder_not_folder(self):
"""A non-existing folder should throw an exception
"""
with self.assertRaises(BadFileFormatException):
self.normalizer.convert_to_csv_from_folder(
dat_folder=self.dummy_dat)
