def main(): parser = argparse.ArgumentParser() parser.add_argument("yaml_file", help="input file", type=str) parser.add_argument("db_file", nargs="?", default="runs.db", help="output file", type=str) parser.add_argument("--column-width", default=100, help="output file", type=int) args = parser.parse_args() if args.yaml_file.endswith("yml") or args.yaml_file.endswith("yaml"): with Path(args.yaml_file).open() as f: data = yaml.load(f) elif args.yaml_file.endswith("pkl"): with Path(args.yaml_file).open("rb") as f: data = pickle.load(f) else: raise RuntimeError("This script works on yaml or pickle files only") with DataBase(args.db_file, Logger(quiet=False)) as db: for run in yaml_to_run_entry(data): db.append(run) print(ls.string(db=db))
def main(): parser = argparse.ArgumentParser() parser.add_argument('yaml_file', help='input file', type=str) parser.add_argument('db_file', nargs='?', default='runs.db', help='output file', type=str) parser.add_argument('--column-width', default=100, help='output file', type=int) args = parser.parse_args() if args.yaml_file.endswith('yml') or args.yaml_file.endswith('yaml'): with Path(args.yaml_file).open() as f: data = yaml.load(f) elif args.yaml_file.endswith('pkl'): with Path(args.yaml_file).open('rb') as f: data = pickle.load(f) else: raise RuntimeError('This script works on yaml or pickle files only') with DataBase(args.db_file, Logger(quiet=False)) as db: for run in yaml_to_run_entry(data): db.append(run) print(table.string(db=db))
def get_table(tag, db_path, patterns, smoothing, tensorboard_dir, use_cache): logger = Logger(quiet=False) with DataBase(db_path, logger) as db: entries = {entry.path: entry for entry in db.get(patterns)} dirs = [Path(tensorboard_dir, path) for path in entries] data_points = crawl(dirs=dirs, tag=tag, smoothing=smoothing, use_cache=use_cache, quiet=True) rewards = { event_file.relative_to(tensorboard_dir).parent: data for data, event_file in data_points } def format_flag_name(name): return name.lstrip('-').replace('-', '_') commands = [e.command for e in entries.values()] flag_names = parse_flags(commands, delimiter='=').keys() flag_names = [format_flag_name(n) for n in flag_names] Row = namedtuple('Row', ['reward'] + list(RunEntry.fields()) + list(flag_names)) def get_row(path): entry = entries[path] # type: RunEntry flags = parse_flags([entry.command], delimiter='=') flags = {format_flag_name(k): v.pop() for k, v in flags.items()} entry_dict = {str(k): str(v) for k, v in entry.asdict().items()} return Row(reward=rewards[path], **entry_dict, **flags) return Row._fields, map(get_row, rewards)
def _wrapper(db_path, quiet, assume_yes, root, dir_names, *args, **kwargs): ui = UI(assume_yes=assume_yes, quiet=quiet) with DataBase(path=db_path, logger=ui) as db: transaction = Transaction(ui=ui, db=db, root=root, dir_names=dir_names) with transaction as open_transaction: return func( db=db, logger=ui, bash=open_transaction.bash, transaction=open_transaction, *args, **kwargs )
def move( query_args: QueryArgs, dest_path: str, kill_tmux: bool, transaction: Transaction, db: DataBase, ): dest_path_is_dir = any( [dest_path == ".", f"{dest_path}/%" in db, dest_path.endswith("/")]) for src_pattern in query_args.patterns: dest_to_src = defaultdict(list) src_entries = db.get(**query_args._replace( patterns=[src_pattern])._asdict()) for entry in src_entries: # parent, grandparent, great-grandparent, etc. parents = [str(entry.path) ] + [str(p) + "/" for p in entry.path.parents] matches = [ p for p in reversed(parents) if like(str(p), str(src_pattern) + "%") ] head = next(iter(matches)) # a/b/% -> a/b tail = PurePath(*[PurePath(m).name for m in matches]) # a/b/% -> c/d if dest_path_is_dir: dest = PurePath(dest_path, tail) else: dest = str(entry.path).replace(head.rstrip("/"), dest_path, 1) dest_to_src[dest] += [entry.path] for dest, srcs in dest_to_src.items(): for i, src in enumerate(srcs): if len(srcs) > 1: dest = PurePath(dest, str(i)) else: dest = PurePath(dest) transaction.move(src=src, dest=dest, kill_tmux=kill_tmux) if dest in db: transaction.remove(dest)
def move(query_args: QueryArgs, dest_path: str, kill_tmux: bool, transaction: Transaction, db: DataBase): dest_path_is_dir = any([ dest_path == PurePath('.'), f'{dest_path}/%' in db, str(dest_path).endswith('/') ]) if dest_path_is_dir: dest_path = add_slash(dest_path) for src_pattern in query_args.patterns: dest_to_src = defaultdict(list) src_entries = db.get(**query_args._replace( patterns=[src_pattern])._asdict()) for entry in src_entries: parents = [entry.path] + [str(p) + '/' for p in entry.path.parents] matching = None for p in parents: if like(str(p), str(src_pattern) + '%'): matching = PurePath(p) if matching is None: raise RuntimeError( f'Somehow, {entry.path} does not match with {src_pattern}.' ) part_to_replace = add_root(matching) if dest_path_is_dir: part_to_replace = add_root(add_slash(src_pattern.parent)) path = add_root(entry.path) dest = path.replace(str(part_to_replace), str(dest_path)) dest_to_src[dest] += [entry.path] for dest, srcs in dest_to_src.items(): for i, src in enumerate(srcs): if len(srcs) > 1: dest = PurePath(dest, str(i)) else: dest = PurePath(dest) transaction.move(src=src, dest=dest, kill_tmux=kill_tmux) if dest in db: transaction.remove(dest)
parser = argparse.ArgumentParser() parser.add_argument('--option', default=0) print(vars(parser.parse_args()))\ """ COMMAND = "python3 test.py" WORK_DIR = "/tmp/test-run-manager" DB_PATH = Path(WORK_DIR, "runs.db") ROOT = WORK_DIR + "/.runs" DESCRIPTION = "test 'new' command" SEP = "/" SUBDIR = "subdir" TEST_RUN = "test_run" LOGGER = UI(quiet=True, assume_yes=True, raise_on_exit=True) BASH = Bash(logger=LOGGER) DB = DataBase(DB_PATH, LOGGER) def sessions(): try: output = BASH.cmd( 'tmux list-session -F "#{session_name}"'.split(), fail_ok=True ) assert isinstance(output, str) return output.split("\n") except subprocess.CalledProcessError: return [] def quote(string): return '"' + string + '"'