def load_data(cls, *, data_path: PurePath, user=None, decrypt=True): """Load data from pickle.""" base = {} if data_path.exists(): base_bytes = data_path.read_bytes() if decrypt: base_bytes = cls.decrypt_data(user.temp_datakey, base_bytes) base = dill.loads(base_bytes) return base
def load_configs(configs_path: PurePath): configs_path = Path(configs_path) if not configs_path.exists(): return None with open(str(configs_path), 'r') as file: configs = util.yaml_load(file.read()) return configs
def __init__(self, path: PurePath, tokenizer: Callable, *, url: Optional[str] = None, filter_subset: Optional[int] = None): path = Path(path) if not path.exists(): if not url: raise FileNotFoundError(str(path)) else: download_file(url, path) with monit.section("Load data"): text = self.load(path) if filter_subset: text = text[:filter_subset] split = int(len(text) * .9) train = text[:split] valid = text[split:] super().__init__(path, tokenizer, train, valid, '')
def _check_dir(cls, dir: PurePath): """Create empty directory if not exists.""" if not dir.exists(): dir.mkdir(parents=True)
pprint(p.anchor) pprint(list(p.parents)) pprint(p.parent) pprint(p.name) pprint(p.suffix) pprint(p.stem) pprint(p.is_absolute()) pprint(p.joinpath('foo', 'bar', 'baz')) pprint(p.match('piyo.*')) # 具象パスを扱う(Path)-------------------------------------------------------- from pathlib import Path p = Path.cwd() / 'newfile.txt' pprint(p) pprint(p.exists()) f = p.open('w+') pprint(p.exists()) pprint(p.resolve()) # ディレクトリ探索-------------------------------------------------------- from pathlib import Path p = Path('.') pprint(p.iterdir()) pprint(sorted(p.iterdir())) pprint(p.glob('**/*.txt')) pprint(sorted(p.glob('**/*.txt')))