def get_hierarchy_dict(dct, path):
from dxpy.filesystem import Path
if isinstance(path, str):
path = Path(path)
result = dct
for k in path.rel_parts():
result = result.get(k)
if result is None:
return dict()
return result
def create_node(self, name=None, path_parent=None, data=None):
path_parent, path_current = self._resolve_path(name, path_parent)
if path_parent is not None and self.get_node(path_parent) is None:
self.create_node(
Path(path_parent).name,
Path(path_parent).father, None)
return self.tree.create_node(name,
path_current,
parent=path_parent,
data=data)
def _resolve_path(cls, name, path_parent):
if name is None:
if path_parent is None:
return None, Path('/').abs
else:
raise TypeError("Name is None while path_parent is not None")
else:
if path_parent is None:
return Path('/').abs, (Path('/') / name).abs
else:
return Path(path_parent).abs, (Path(path_parent) / name).abs
def _path_config_file(self, path):
from dxpy.filesystem import Directory, File
if path is None:
return None
filename = Path(path)
if self.module_name is not None and Directory(filename).exists:
possible_suffix = ['', '.yml', '.yaml', '.json']
for sfx in possible_suffix:
name = '{0}{1}'.format(self.module_name, sfx)
if File(filename / name).exists:
filename = filename / name
break
if not File(filename).exists:
return None
return Path(filename).abs
def task_slurm(file, *args, workdir, **kwargs):
return task.Task(*args,
**kwargs,
workdir=workdir,
worker=task.Worker.Slurm,
ttype=task.Type.Script,
data={'file': (Path(workdir) / file).abs})
def __resolve_path_load(self, feeds):
from fs.osfs import OSFS
import re
from dxpy.filesystem import Path
path_check_point = (Path(self.param('model_dir', feeds)) /
'checkpoint').abs
pp = re.compile('^.*: "(.*)".*$')
ps = re.compile('.*' + self.param('ckpt_name', feeds) + '-([0-9]+)-*')
paths = []
with OSFS('/') as fs:
if not fs.exists(path_check_point):
return fs.getsyspath(path_check_point), False
with fs.open(path_check_point) as fin:
for l in fin.readlines():
mat_path = pp.match(l)
if mat_path is not None:
path_load = mat_path[1]
mat_step = ps.match(path_load)
if mat_step is not None:
paths.append([path_load, int(mat_step[1])])
step = self.param('step', feeds, default=-1)
if step == -1:
step = max(list(zip(*paths))[1])
for p, s in paths:
if s == step:
return p, True
return step, False
def test_detail(self):
p = Path('/tmp/test')
self.assertEqual(
p.detail, {
'name': 'test',
'path': '/tmp/test',
'parent': '/tmp',
'url': '%2Ftmp%2Ftest',
'suffix': '',
'parts': ('/', 'tmp', 'test')
})
def _unified_keys(cls, key_or_keys_or_path):
if isinstance(key_or_keys_or_path, (list, tuple)):
return tuple(key_or_keys_or_path)
if isinstance(key_or_keys_or_path, str):
result = tuple(Path(key_or_keys_or_path).parts)
if len(result) > 0 and result[0] == '/':
result = result[1:]
return result
if isinstance(key_or_keys_or_path, int):
return tuple(str(key_or_keys_or_path))
else:
raise TypeError(
"Key not supported {}.".format(key_or_keys_or_path))
def negtive():
fs = MagicMock()
fs_exists = MagicMock(return_value=False)
fs.exists = fs_exists
p = Path('/tmp/temp.txt')
assert p.check_exists(fs) == False
def task_script(file, *args, **kwargs):
return task.Task(*args,
**kwargs,
ttype=task.Type.Script,
data={'file': (Path(kwargs['workdir']) / file).abs})
def test_str_basic(self):
p = Path('/tmp')
self.assertEqual(p.abs, '/tmp')
def test_url(self):
p = Path('%2Ftmp')
self.assertEqual(p.abs, '/tmp')
def test_parts(self):
p = Path('/tmp/base')
self.assertEqual(p.parts, ('/', 'tmp', 'base'))
from dxpy.filesystem import Path
import os
datasets_configs = {
'dataset_root_path': os.environ.get('PATHS_DATASET', str(Path(os.environ.get('HOME')) / 'Datas')),
'path': '/home/hongxwing/Datas/',
'analytical_phantom_sinogram': {
'path': '/home/hongxwing/Datas/Phantom',
},
'apssr': {
'image_type': 'sinogram',
'target_shape': [320, 320],
'super_resolution': {
'nb_down_sample': 3
},
},
'mice_sinograms': {
'path': str(Path(os.environ.get('HOME'))/'Datas'/'Mice')
}
}
config = {
'train': {
'summary_freq': 60,
'ckpt_path': './save'
},
'datasets': datasets_configs
}
def get_config():
from dxpy.configs import ConfigsView
def path_of_script_folder_of_module(module, relative_path='../scripts'):
from dxpy.filesystem import Path
return Path(Path(module.__file__) / relative_path)
def test_brief(self):
p = Path('/tmp/test')
self.assertEqual(p.brief, {'name': 'test', 'path': '/tmp/test'})
def test_name_dir(self):
p = Path('/tmp/base/')
self.assertEqual(p.name, 'base')
def test_parent(self):
p = Path('/tmp/base')
self.assertEqual(p.father, '/tmp')
def path_of_global_defaults(root_module='dxpy'):
from dxpy.filesystem import Path
return Path(Path('~') / '.{m}'.format(m=root_module))
def test_copy_init(self):
p = Path('/tmp/file')
p2 = Path(p)
assert p.abs == p2.abs
def _default_dataset_path():
return str(Path(os.environ.get('HOME')) / 'Datas')
def test_div(self):
p = Path('/tmp')
p = p / 'sub'
assert p.abs == '/tmp/sub'
def _model_path(self):
return (Path(self.param('model_dir')) / self.param('ckpt_name')).abs
def test_str_root(self):
p = Path('/')
self.assertEqual(p.abs, '/')
def _refine_path_user(self, path_user):
import os
if isinstance(path_user, str) and not path_user.startswith('/'):
return Path(os.environ['HOME']) / '.dxl' / path_user
else:
return Path(path_user)
def test_yaml_dump(self):
p = Path('/tmp/test')
assert yaml.dump(p) == "!path '/tmp/test'\n"
def template_dir():
return Path(Path(__file__).father)
def template_path():
from dxpy.filesystem import Path
import dxpy
p = Path(dxpy.__file__).father
def get_scripts_path():
import pygate
from dxpy.filesystem import Path
p = Path(pygate.__file__)
return Path(p.father) / 'scripts'
import os
from collections import UserDict
from dxpy.filesystem import Path
_default = {
'path': str(Path(os.environ.get('PATH_DATABASE')) / 'taskdb.db'),
'name': 'task',
'names': 'tasks',
'use_web_api': True,
'echo': False,
'default_state': 'BeforeSubmit',
'host': '0.0.0.0',
'ip': '127.0.0.1',
'port': 23300,
'debug': False,
'version': 0.2,
'base': '/'
}
class Config(UserDict):
def __init__(self):
super(__class__, self).__init__()
self.data.update(_default)
@property
def path_sqllite(self):
return 'sqlite:///' + self.data['path']
def back_to_default(self):
self.data.update(_default)