def kaggle_command():
cmd = Command("kaggle", "kaggleコマンドを実行する")
cmd.option("args", nargs="*")
@cmd
def kaggle_main(options, *_, **__):
sub = subprocess.Popen(["kaggle", *options.args])
sub.wait()
return cmd
def arxiv2vec_command():
cmd = Command("arxiv2vec", "論文ベクトル化")
cmd >> dataset_command()
cmd >> infer_command()
cmd >> train_command()
return cmd
def train_command():
"""
trainコマンド
"""
cmd = Command("train", "訓練する")
cmd.option("--model", required=True)
cmd.option("--preprocess", required=True)
cmd.option("--train_data", required=True)
cmd.option("--save_model", required=True)
cmd(train.run)
return cmd
def test_something(self):
root = Command("root", "A_")
cmd = root >> Command("cmd1", "cmd1") >> Command("cmd2", "cmd2")
@cmd
def run_test(args):
self.assertEqual(get_meta(args).parents, ["root", "cmd1"])
argv = ["cmd1", "cmd2"]
root.start(argv)
def build(config: Optional[MLBaseConfig] = None) -> Command:
if config is None:
config = load_config()
cmd = __entry_command()
cmd >> note_command()
cmd >> kaggle_command()
cmd >> estimate_command()
cmd_dataset = Command("dataset", "データセットに関するコマンドです。") << cmd
cmd_dataset >> rough_estimate_command()
cmd >> Command("research", "研究段階のもののコマンドです。") >> train_cifar_command()
cmd >> arxiv2vec_command()
cmd >> Command("tree", "コマンドの一覧表示します。")(show_tree(cmd))
# config
config_file_map = {
"mlbase": config.mlbase_config,
"plugins": config.plugin_config,
}
config_edit = cmd >> Command("config", "設定を行います。") >> Command("edit", "エディタで編集を行います。")
config_edit.option("target", choices=config_file_map.keys())
@config_edit
def config_edit_main(args, *_, **__):
path = config_file_map[args.target]
if not path.exists():
path.parent.mkdir(exist_ok=True, parents=True)
subprocess.Popen([*config.editor, path]).wait()
# plugin
plugin_manager = PluginManger(cmd)
plugin_manager.add_plugin_type("local", LocalPluginType)
plugin_manager.load_yml(config.plugin_config)
update = cmd >> Command("plugins", "プラグインに関するもの") >> Command("update", "プラグインの更新をします。")
@update
def run_update(*_, **__):
plugin_manager.update()
return cmd
def rough_estimate_command() -> Command:
cmd = Command("rough_estimate", "概算する")
data_size_cmd = Command("datasize", "データサイズを概算") << cmd
data_size_cmd.option("--width", type=int, required=True)
data_size_cmd.option("--height", type=int, required=True)
data_size_cmd.option("--channel", type=int, required=True)
data_size_cmd.option("--count", type=int, required=True)
@data_size_cmd
def _(args, *_, **__):
print(args.width * args.height * args.channel * args.count // 1000000000, "[GB]")
return cmd
def __entry_command():
cmd = Command("mlbase", "機械学習を支援するコマンドです。")
cmd.option("--config", dest=META_CONFIG, help="設定ファイルのパス")
cmd.option("--no_config", action="store_true", dest=META_NO_CONFIG, help="設定ファイルを用いない。--configよりも優先される。")
# cmd.option("--interactive", action="store_true", dest=META_INTERACTIVE, help="対話モード")
return cmd
def train_cifar_command() -> Command:
cmd = Command("train_cifar", "cifar100の訓練")
cmd(run)
cmd.option("--checkpoint")
cmd.option("--data_path",
required=True,
type=Path,
help="~/data/cifar-100-binary")
cmd.option("--cache_path", default="_cifar100_cache.pkl")
cmd.option("--output", required=True)
cmd.option("--param", required=True)
return cmd
def infer_command():
"""
inferコマンド
"""
cmd = Command("infer", "実行する")
cmd_show_vector = Command("show_vector", "ベクトル表示") << cmd
cmd_show_vector.option('--load_model')
cmd_show_vector.option('input_texts', nargs="+")
@cmd_show_vector
def run_show_vector(args, *_, **__):
model = infer.load_model(args.load_model)
infer.run_action("show_vector", model, args)
cmd_compare = Command("compare", "比較") << cmd
@cmd_compare
def run_compare(args, *_, **__):
model = infer.load_model(args.load_model)
infer.run_action("compare", model, args)
cmd_find_neighbors = Command("find_neighbors", "近傍探索") << cmd
cmd_find_neighbors.option("--train_data")
cmd_find_neighbors.option("--load_model")
cmd_find_neighbors.option("--input_texts", nargs="+")
@cmd_find_neighbors
def run_find_neighbors(args, *_, **__):
model = infer.load_model(args.load_model)
infer.run_action("find_neighbors", model, args)
cmd_output_vectors = Command("output_vectors", "ベクトル保存") << cmd
cmd_output_vectors.option("--load_model")
cmd_output_vectors.option("--input_texts", nargs="+")
@cmd_output_vectors
def run_output_vectors(args, *_, **__):
model = infer.load_model(args.load_model)
infer.run_action("output_vectors", model, args)
return cmd
def dataset_command():
"""
datasetコマンド
"""
cmd = Command("dataset", "データセットを整理するコマンド")
cmd_json = Command("json_arxiv", "arXivのデータを今回用に加工") << cmd
cmd_json.option('--input', required=True, nargs='+')
cmd_json.option('--output', required=True)
@cmd_json
def run_json_arxiv(args, *_, **__):
json_arxiv.run(args.input, args.output)
cmd_merge_json = Command("merge_json", "jsonファイルの統合") << cmd
cmd_merge_json.option('--input', required=True, nargs='+')
cmd_merge_json.option('--output', required=True)
@cmd_merge_json
def run_merge_json(args, *_, **__):
merge_json(args.input, args.output)
return cmd
def arxiv_command():
cmd_arxiv = Command("arxiv", "arXivの論文を管理")
cmd_arxiv_add = cmd_arxiv >> Command("add", "論文を追加する")
cmd_arxiv_remove = cmd_arxiv >> Command("remove", "論文を削除する")
def note_command():
cmd = Command("note", "ノートを書く")
# add
add_cmd = Command("add", "追加する") << cmd
add_cmd.option("--name", required=True, help="名前")
add_cmd.option("--message", required=True, help="説明")
add_cmd.option("--storage", help="保存先")
add_cmd.option("--tags", nargs="*", help="タグ")
add_cmd.option("--entory_point", required=True, help="編集時に開く対象")
add_cmd.option("--edit_command",
default="{editor} {entory_point}",
help="編集コマンド")
@add_cmd
def _(args, config: MLBaseConfig, *_, **__):
note = NoteObject(
name=args.name,
message=args.message,
tags=args.tags,
storage=args.storage or config.storage,
entory_point=args.entory_point,
edit_command=args.edit_command.format(editor=config.editor))
__add_note(note)
# edit
edit_cmd = Command("edit", "編集する") << cmd
edit_cmd.option("name")
@edit_cmd
def run_edit(args, config: MLBaseConfig, *_, **__):
pass
# remove
remove_cmd = Command("remove", "消す（隠す）") << cmd
remove_cmd.option("name")
@remove_cmd
def run_remove(args, *_, **__):
pass
# show
show_cmd = Command("show", "一覧する") << cmd
show_cmd.option("--tag")
@show_cmd
def run_show(args, *_, **__):
pass
return cmd
def estimate_command():
cmd = Command("estimate", "見積もり計算をする")
cmd_tpu = Command("tpu", "TPU見積もりを行う") << cmd
cmd_tpu.option("--num_core", help="全コア数", required=True, type=float)
cmd_tpu.option("--core_per_tpu", help="TPUあたりのコア数", default=8, type=float)
cmd_tpu.option("--hours", help="何時間実行するか", type=float)
cmd_tpu.option("--dollar_per_hour", help="コア数", default=5.22, type=float)
cmd_tpu.option("--yen_per_dollar", help="1ドル何円か", default=110, type=float)
@cmd_tpu
def tpu_case(args, *others, **kwargs):
num_tpu = args.num_core / args.core_per_tpu
dollar_per_hour = args.dollar_per_hour * num_tpu
yen_per_hour = dollar_per_hour * args.yen_per_dollar
print(f"コア数: {args.num_core}")
print(f"1TPUあたりコア数: {args.core_per_tpu}")
print(f"TPU数: {num_tpu}")
print(f"1TPU値段(ドル/h): {args.dollar_per_hour}")
print(f"1ドル(円): {args.yen_per_dollar}")
print(f"単位時間値段(ドル/h): {dollar_per_hour}")
print(f"単位時間値段(円/h): {yen_per_hour}")
if args.hours is not None:
total_dollar = dollar_per_hour * args.hours
total_yen = total_dollar * args.yen_per_dollar
print(f"時間(h): {args.hours}")
print(f"値段(ドル): {total_dollar}")
print(f"値段(円): {total_yen}")
return cmd