def import_experiments(input_dir, use_src_user_id, import_mlflow_tags,
import_metadata_tags):
start_time = time.time()
manifest_path = os.path.join(input_dir, "experiments", "manifest.json")
manifest = utils.read_json_file(manifest_path)
exps = manifest["experiments"]
importer = ExperimentImporter(None, use_src_user_id, import_mlflow_tags,
import_metadata_tags)
print("Experiments:")
for exp in exps:
print(" ", exp)
run_info_map = {}
exceptions = []
for exp in exps:
exp_input_dir = os.path.join(input_dir, "experiments", exp["id"])
try:
_run_info_map = importer.import_experiment(exp["name"],
exp_input_dir)
run_info_map[exp["id"]] = _run_info_map
except Exception as e:
exceptions.append(e)
import traceback
traceback.print_exc()
duration = round(time.time() - start_time, 1)
if len(exceptions) > 0:
print(f"Errors: {len(exceptions)}")
print(f"Duration: {duration} seconds")
return run_info_map, {
"experiments": len(exps),
"exceptions": exceptions,
"duration": duration
}
def main(input_dir, experiment_name_prefix, use_src_user_id, import_mlflow_tags, import_metadata_tags): # pragma: no cover
print("Options:")
for k,v in locals().items():
print(f" {k}: {v}")
path = os.path.join(input_dir,"manifest.json")
with open(path, "r") as f:
dct = json.loads(f.read())
for exp in dct["experiments"]:
print(" ",exp)
importer = ExperimentImporter(None, use_src_user_id, import_mlflow_tags, import_metadata_tags)
for exp in dct["experiments"]:
exp_input = os.path.join(input_dir,exp["id"])
exp_name = experiment_name_prefix + exp["name"] if experiment_name_prefix else exp["name"]
importer.import_experiment(exp_name, exp_input)
def main(input_dir, experiment_name_prefix, use_src_user_id,
import_mlflow_tags, import_metadata_tags, use_threads):
print("Options:")
for k, v in locals().items():
print(f" {k}: {v}")
path = os.path.join(input_dir, "manifest.json")
with open(path, "r") as f:
dct = json.loads(f.read())
for exp in dct["experiments"]:
print(" ", exp)
importer = ExperimentImporter(None, use_src_user_id, import_mlflow_tags,
import_metadata_tags)
max_workers = os.cpu_count() or 4 if use_threads else 1
with ThreadPoolExecutor(max_workers=max_workers) as executor:
for exp in dct["experiments"]:
exp_input_dir = os.path.join(input_dir, exp["id"])
exp_name = experiment_name_prefix + exp[
"name"] if experiment_name_prefix else exp["name"]
executor.submit(_import_experiment, importer, exp_name,
exp_input_dir)
dbutils.widgets.dropdown("Import metadata tags","no",["yes","no"])
import_metadata_tags = dbutils.widgets.get("Import metadata tags") == "yes"
dbutils.widgets.dropdown("Just peek","no",["yes","no"])
just_peek = dbutils.widgets.get("Just peek") == "yes"
if len(input_dir)==0: raise Exception("ERROR: Input is required")
print("input_dir:",input_dir)
print("experiment_name:",experiment_name)
print("import_metadata_tags:",import_metadata_tags)
print("just_peek:",just_peek)
# COMMAND ----------
from mlflow_export_import.experiment.import_experiment import ExperimentImporter
from mlflow_export_import import peek_at_experiment
importer = ExperimentImporter(import_metadata_tags=import_metadata_tags)
peek_at_experiment(input_dir)
# COMMAND ----------
if not just_peek:
if len(experiment_name)==0: raise Exception("ERROR: Destination Experiment Name is required")
importer.import_experiment(experiment_name, input_dir)
# COMMAND ----------
display_experiment_uri(experiment_name)
def test_exp_import_metadata_tags():
run1, run2 = init_exp_test(ExperimentExporter(export_metadata_tags=True),
ExperimentImporter(import_metadata_tags=True),
verbose=False)
compare_run_import_metadata_tags(run1, run2)
def test_exp_no_import_mlflow_tags():
run1, run2 = init_exp_test(ExperimentExporter(),
ExperimentImporter(import_mlflow_tags=False))
compare_run_no_import_mlflow_tags(run1, run2)
def test_exp_basic():
run1, run2 = init_exp_test(ExperimentExporter(), ExperimentImporter(),
True)
compare_runs(run1, run2)