def main():
parser = argparse.ArgumentParser()
parser.add_argument("yaml_file", help="input file", type=str)
parser.add_argument("db_file",
nargs="?",
default="runs.db",
help="output file",
type=str)
parser.add_argument("--column-width",
default=100,
help="output file",
type=int)
args = parser.parse_args()
if args.yaml_file.endswith("yml") or args.yaml_file.endswith("yaml"):
with Path(args.yaml_file).open() as f:
data = yaml.load(f)
elif args.yaml_file.endswith("pkl"):
with Path(args.yaml_file).open("rb") as f:
data = pickle.load(f)
else:
raise RuntimeError("This script works on yaml or pickle files only")
with DataBase(args.db_file, Logger(quiet=False)) as db:
for run in yaml_to_run_entry(data):
db.append(run)
print(ls.string(db=db))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('yaml_file', help='input file', type=str)
parser.add_argument('db_file',
nargs='?',
default='runs.db',
help='output file',
type=str)
parser.add_argument('--column-width',
default=100,
help='output file',
type=int)
args = parser.parse_args()
if args.yaml_file.endswith('yml') or args.yaml_file.endswith('yaml'):
with Path(args.yaml_file).open() as f:
data = yaml.load(f)
elif args.yaml_file.endswith('pkl'):
with Path(args.yaml_file).open('rb') as f:
data = pickle.load(f)
else:
raise RuntimeError('This script works on yaml or pickle files only')
with DataBase(args.db_file, Logger(quiet=False)) as db:
for run in yaml_to_run_entry(data):
db.append(run)
print(table.string(db=db))
def get_table(tag, db_path, patterns, smoothing, tensorboard_dir, use_cache):
logger = Logger(quiet=False)
with DataBase(db_path, logger) as db:
entries = {entry.path: entry for entry in db.get(patterns)}
dirs = [Path(tensorboard_dir, path) for path in entries]
data_points = crawl(dirs=dirs,
tag=tag,
smoothing=smoothing,
use_cache=use_cache,
quiet=True)
rewards = {
event_file.relative_to(tensorboard_dir).parent: data
for data, event_file in data_points
}
def format_flag_name(name):
return name.lstrip('-').replace('-', '_')
commands = [e.command for e in entries.values()]
flag_names = parse_flags(commands, delimiter='=').keys()
flag_names = [format_flag_name(n) for n in flag_names]
Row = namedtuple('Row',
['reward'] + list(RunEntry.fields()) + list(flag_names))
def get_row(path):
entry = entries[path] # type: RunEntry
flags = parse_flags([entry.command], delimiter='=')
flags = {format_flag_name(k): v.pop() for k, v in flags.items()}
entry_dict = {str(k): str(v) for k, v in entry.asdict().items()}
return Row(reward=rewards[path], **entry_dict, **flags)
return Row._fields, map(get_row, rewards)
def _wrapper(db_path, quiet, assume_yes, root, dir_names, *args, **kwargs):
ui = UI(assume_yes=assume_yes, quiet=quiet)
with DataBase(path=db_path, logger=ui) as db:
transaction = Transaction(ui=ui, db=db, root=root, dir_names=dir_names)
with transaction as open_transaction:
return func(
db=db,
logger=ui,
bash=open_transaction.bash,
transaction=open_transaction,
*args,
**kwargs
)
def move(
query_args: QueryArgs,
dest_path: str,
kill_tmux: bool,
transaction: Transaction,
db: DataBase,
):
dest_path_is_dir = any(
[dest_path == ".", f"{dest_path}/%" in db,
dest_path.endswith("/")])
for src_pattern in query_args.patterns:
dest_to_src = defaultdict(list)
src_entries = db.get(**query_args._replace(
patterns=[src_pattern])._asdict())
for entry in src_entries:
# parent, grandparent, great-grandparent, etc.
parents = [str(entry.path)
] + [str(p) + "/" for p in entry.path.parents]
matches = [
p for p in reversed(parents) if like(str(p),
str(src_pattern) + "%")
]
head = next(iter(matches)) # a/b/% -> a/b
tail = PurePath(*[PurePath(m).name
for m in matches]) # a/b/% -> c/d
if dest_path_is_dir:
dest = PurePath(dest_path, tail)
else:
dest = str(entry.path).replace(head.rstrip("/"), dest_path, 1)
dest_to_src[dest] += [entry.path]
for dest, srcs in dest_to_src.items():
for i, src in enumerate(srcs):
if len(srcs) > 1:
dest = PurePath(dest, str(i))
else:
dest = PurePath(dest)
transaction.move(src=src, dest=dest, kill_tmux=kill_tmux)
if dest in db:
transaction.remove(dest)
def move(query_args: QueryArgs, dest_path: str, kill_tmux: bool,
transaction: Transaction, db: DataBase):
dest_path_is_dir = any([
dest_path == PurePath('.'), f'{dest_path}/%' in db,
str(dest_path).endswith('/')
])
if dest_path_is_dir:
dest_path = add_slash(dest_path)
for src_pattern in query_args.patterns:
dest_to_src = defaultdict(list)
src_entries = db.get(**query_args._replace(
patterns=[src_pattern])._asdict())
for entry in src_entries:
parents = [entry.path] + [str(p) + '/' for p in entry.path.parents]
matching = None
for p in parents:
if like(str(p), str(src_pattern) + '%'):
matching = PurePath(p)
if matching is None:
raise RuntimeError(
f'Somehow, {entry.path} does not match with {src_pattern}.'
)
part_to_replace = add_root(matching)
if dest_path_is_dir:
part_to_replace = add_root(add_slash(src_pattern.parent))
path = add_root(entry.path)
dest = path.replace(str(part_to_replace), str(dest_path))
dest_to_src[dest] += [entry.path]
for dest, srcs in dest_to_src.items():
for i, src in enumerate(srcs):
if len(srcs) > 1:
dest = PurePath(dest, str(i))
else:
dest = PurePath(dest)
transaction.move(src=src, dest=dest, kill_tmux=kill_tmux)
if dest in db:
transaction.remove(dest)
parser = argparse.ArgumentParser()
parser.add_argument('--option', default=0)
print(vars(parser.parse_args()))\
"""
COMMAND = "python3 test.py"
WORK_DIR = "/tmp/test-run-manager"
DB_PATH = Path(WORK_DIR, "runs.db")
ROOT = WORK_DIR + "/.runs"
DESCRIPTION = "test 'new' command"
SEP = "/"
SUBDIR = "subdir"
TEST_RUN = "test_run"
LOGGER = UI(quiet=True, assume_yes=True, raise_on_exit=True)
BASH = Bash(logger=LOGGER)
DB = DataBase(DB_PATH, LOGGER)
def sessions():
try:
output = BASH.cmd(
'tmux list-session -F "#{session_name}"'.split(), fail_ok=True
)
assert isinstance(output, str)
return output.split("\n")
except subprocess.CalledProcessError:
return []
def quote(string):
return '"' + string + '"'