def load_data(cls, *, data_path: PurePath, user=None, decrypt=True):
"""Load data from pickle."""
base = {}
if data_path.exists():
base_bytes = data_path.read_bytes()
if decrypt:
base_bytes = cls.decrypt_data(user.temp_datakey, base_bytes)
base = dill.loads(base_bytes)
return base
def load_configs(configs_path: PurePath):
configs_path = Path(configs_path)
if not configs_path.exists():
return None
with open(str(configs_path), 'r') as file:
configs = util.yaml_load(file.read())
return configs
def __init__(self,
path: PurePath,
tokenizer: Callable,
*,
url: Optional[str] = None,
filter_subset: Optional[int] = None):
path = Path(path)
if not path.exists():
if not url:
raise FileNotFoundError(str(path))
else:
download_file(url, path)
with monit.section("Load data"):
text = self.load(path)
if filter_subset:
text = text[:filter_subset]
split = int(len(text) * .9)
train = text[:split]
valid = text[split:]
super().__init__(path, tokenizer, train, valid, '')
def _check_dir(cls, dir: PurePath):
"""Create empty directory if not exists."""
if not dir.exists():
dir.mkdir(parents=True)
pprint(p.anchor)
pprint(list(p.parents))
pprint(p.parent)
pprint(p.name)
pprint(p.suffix)
pprint(p.stem)
pprint(p.is_absolute())
pprint(p.joinpath('foo', 'bar', 'baz'))
pprint(p.match('piyo.*'))
# 具象パスを扱う(Path)--------------------------------------------------------
from pathlib import Path
p = Path.cwd() / 'newfile.txt'
pprint(p)
pprint(p.exists())
f = p.open('w+')
pprint(p.exists())
pprint(p.resolve())
# ディレクトリ探索--------------------------------------------------------
from pathlib import Path
p = Path('.')
pprint(p.iterdir())
pprint(sorted(p.iterdir()))
pprint(p.glob('**/*.txt'))
pprint(sorted(p.glob('**/*.txt')))
