def wf_single_fit(fworker, fit_name, pipe_config, name, df, target, tags=None): """ Submit a dataset to be fit for a single pipeline (i.e., to train on a dataset for real predictions). """ # todo this is not working probably warnings.warn("Single fitted MatPipe not being stored in automatminer db " "collections. Please consult fw_spec to find the benchmark " "on {}".format(fworker)) if fworker not in VALID_FWORKERS: raise ValueError("fworker must be in {}".format(VALID_FWORKERS)) data_file = None now = get_time_str() base_save_dir = now + "_single_fit" spec = { "pipe_config": pipe_config, "base_save_dir": base_save_dir, "data_file": data_file, "target": target, "automatminer_commit": get_last_commit(), "tags": tags if tags else [], "_fworker": fworker, } fw_name = "{} single fit".format(name) wf_name = "single fit: {} ({}) [{}]".format(name, fit_name, fworker) fw = Firework(RunSingleFit(), spec=spec, name=fw_name) wf = Workflow([fw], metadata={"tags": tags}, name=wf_name) return wf
def wf_run_test(fworker, test_name): commit = get_last_commit() wf_name = "run tests: {} [{}]".format(test_name, commit) add_to_spec = { "commit": commit, } test_fw = get_test_fw(fworker, add_to_spec=add_to_spec) wf = Workflow([test_fw], metadata={"tags": "test"}, name=wf_name) return wf
def wf_evaluate_build( fworker, build_name, dataset_set, pipe_config, include_tests=False, cache=True, kfold_config=KFOLD_DEFAULT, tags=None, ): """ Current fworkers: - "local": Alex's local computer - "cori": Cori - "lrc": Lawrencium """ if fworker not in VALID_FWORKERS: raise ValueError("fworker must be in {}".format(VALID_FWORKERS)) # Get a fun unique id for this build word_file = "/usr/share/dict/words" words = open(word_file).read().splitlines() words_short = [w for w in words if 4 <= len(w) <= 6] build_id = None while (LP.db.automatminer_builds.find({ "build_id": build_id }).count() != 0 or not build_id): build_id = " ".join([w.lower() for w in random.sample(words_short, 2)]) print("build id: {}".format(build_id)) all_links = {} fws_fold0 = [] fws_consolidate = [] benchmark_hashes = [] for benchmark in dataset_set: links, fw_fold0, fw_consolidate = wf_benchmark( fworker, pipe_config, **benchmark, tags=tags, kfold_config=kfold_config, cache=cache, return_fireworks=True, build_id=build_id, add_dataset_to_names=True) all_links.update(links) fws_fold0.extend(fw_fold0) fws_consolidate.append(fw_consolidate) # benchmark has is the same between all fws in one benchmark benchmark_hashes.append( fw_fold0[0].to_dict()["spec"]["benchmark_hash"]) fw_build_merge = Firework( ConsolidateBenchmarksToBuild(), spec={ "benchmark_hashes": benchmark_hashes, "build_id": build_id, "pipe_config": pipe_config, "build_name": build_name, "commit": get_last_commit(), "_fworker": fworker, "tags": tags, }, name="build merge ({})".format(build_id), ) for fw in fws_consolidate: all_links[fw] = [fw_build_merge] if include_tests: fw_test = get_test_fw(fworker, build_id) all_links[fw_test] = fws_fold0 all_links[fw_build_merge] = [] wf_name = "build: {} ({}) [{}]".format(build_id, build_name, fworker) wf = Workflow( list(all_links.keys()), all_links, name=wf_name, metadata={ "build_id": build_id, "tags": tags, "benchmark_hashes": benchmark_hashes, }, ) return wf
def wf_benchmark( fworker, pipe_config, name, data_file, target, problem_type, clf_pos_label, cache=True, kfold_config=KFOLD_DEFAULT, tags=None, return_fireworks=False, add_dataset_to_names=True, build_id=None, prepend_name="", ): if fworker not in VALID_FWORKERS: raise ValueError("fworker must be in {}".format(VALID_FWORKERS)) # if fworker == "cori": # n_cori_jobs = 32 # warnings.warn( # "Worker is cori. Overriding n_jobs to {}".format(n_cori_jobs)) # pipe_config["learner_kwargs"]["n_jobs"] = n_cori_jobs # pipe_config["autofeaturizer_kwargs"]["n_jobs"] = n_cori_jobs # Single (run) hash is the combination of pipe configuration + last commit # + data_file last_commit = get_last_commit() benchmark_config_for_hash = copy.deepcopy(pipe_config) benchmark_config_for_hash["last_commit"] = last_commit benchmark_config_for_hash["data_file"] = data_file benchmark_config_for_hash["worker"] = fworker benchmark_config_for_hash = str(benchmark_config_for_hash).encode("UTF-8") benchmark_hash = hashlib.sha1(benchmark_config_for_hash).hexdigest()[:10] base_save_dir = get_time_str() + "_" + benchmark_hash common_spec = { "pipe_config": pipe_config, "base_save_dir": base_save_dir, "kfold_config": kfold_config, "data_file": data_file, "target": target, "clf_pos_label": clf_pos_label, "problem_type": problem_type, "automatminer_commit": last_commit, "name": name, "benchmark_hash": benchmark_hash, "tags": tags if tags else [], "cache": cache, "build_id": build_id, "_fworker": fworker, } dataset_name = "" if not add_dataset_to_names else name + " " fws_all_folds = [] kfold = KFold(**kfold_config) for fold in range(kfold.n_splits): save_dir = os.path.join("fold_{}".format(fold)) foldspec = copy.deepcopy(common_spec) foldspec["fold"] = fold foldspec["save_dir"] = save_dir if fold == 0 and cache: pipename = "{}fold {} + featurization ({})".format( dataset_name, fold, benchmark_hash) else: pipename = "{}fold {} ({})".format(dataset_name, fold, benchmark_hash) fws_all_folds.append( Firework([RunPipe(), StorePipeResults()], spec=foldspec, name=pipename)) fw_consolidate = Firework( ConsolidatePipesToBenchmark(), spec=common_spec, name="bench merge ({})".format(benchmark_hash), ) if cache: fw_fold0 = fws_all_folds[0] fws_folds = fws_all_folds[1:] links = {fw: [fw_consolidate] for fw in fws_folds} links[fw_fold0] = fws_folds links[fw_consolidate] = [] else: links = {fw: [fw_consolidate] for fw in fws_all_folds} links[fw_consolidate] = [] fw_fold0 = fws_all_folds if return_fireworks: connected_to_top_wf = [fw_fold0] if cache else fw_fold0 return links, connected_to_top_wf, fw_consolidate else: wf_name = "benchmark {}: ({}) [{}]".format(benchmark_hash, name, fworker) if prepend_name: wf_name = "<<{}>> {}".format(prepend_name, wf_name) wf = Workflow( list(links.keys()), links_dict=links, name=wf_name, metadata={ "benchmark_hash": benchmark_hash, "tags": tags }, ) return wf