def _path_to_id(path, run_id=None):
from pypads.app.pypads import get_current_pads
if run_id is None:
run = get_current_pads().api.active_run()
run_id = run.info.run_id
experiment_name = mlflow.get_experiment(
mlflow.active_run().info.experiment_id).name
else:
experiment_name = mlflow.get_experiment(
mlflow.active_run().info.experiment_id).name
return os.path.sep.join([experiment_name, run_id, path])
def test_get_experiment_by_id():
with TempDir(chdr=True):
name = "Random experiment %d" % random.randint(1, 1e6)
exp_id = mlflow.create_experiment(name)
experiment = mlflow.get_experiment(exp_id)
assert experiment.experiment_id == exp_id
def mlflow_get_experiment_run_info() -> Optional[Tuple[mlflow.entities.Experiment, mlflow.entities.RunInfo]]:
""" Returns active MLFLow active experiment object and run infos if any """
run = mlflow.active_run()
if run is not None:
experiment = mlflow.get_experiment(run.info.experiment_id)
return experiment, run.info
return (None, None)
def test_get_or_create_experiment():
expr_name = "test"
expr_id = exp.get_or_create_experiment(expr_name)
expr = mlflow.get_experiment(expr_id)
assert expr.experiment_id == expr_id
assert expr.name == expr_name
def active_experiment(self):
run = mlflow.active_run()
r_id = run.info.run_id
if not self.pypads.cache.run_exists(f"experiment_for_run_{r_id}"):
self.pypads.cache.run_add(
f"experiment_for_run_{r_id}",
mlflow.get_experiment(run.info.experiment_id))
return self.pypads.cache.run_get(f"experiment_for_run_{r_id}")
def print_experiment_info(self):
experiment = mlflow.get_experiment(self.experiment_id)
logger.info("Name: {}".format(experiment.name))
logger.info("Experiment_id: {}".format(experiment.experiment_id))
logger.info("Artifact Location: {}".format(
experiment.artifact_location))
logger.info("Tags: {}".format(experiment.tags))
logger.info("Lifecycle_stage: {}".format(experiment.lifecycle_stage))
def _init(self):
mlflow = _import_mlflow()
logger_config = self.config.pop("logger_config", {})
tracking_uri = logger_config.get("mlflow_tracking_uri")
registry_uri = logger_config.get("mlflow_registry_uri")
experiment_id = logger_config.get("mlflow_experiment_id")
if experiment_id is None or not mlflow.get_experiment(experiment_id):
raise ValueError(
"You must provide a valid `mlflow_experiment_id` "
"in your `logger_config` dict in the `config` "
"dict passed to `tune.run`. "
"Are you sure you passed in a `experiment_id` and "
"the experiment exists?")
else:
experiment_name = mlflow.get_experiment(experiment_id).name
self._trial_experiment_logger = self._experiment_logger_cls(
tracking_uri, registry_uri, experiment_name)
self._trial_experiment_logger.log_trial_start(self.trial)
def extract_experiment_name(yaml_path: str) -> Optional[str]:
"""
Reads the supplied backstage yaml configuration file and returns the MLFlow Experiment that it is configured
to use.
"""
mlflow.set_tracking_uri(TRACKING_URI)
with open(yaml_path, "r") as fin:
parsed = yaml.safe_load(fin.read())
experiment_id = parsed["metadata"]["annotations"]["mlflow.org/experiment"]
print(experiment_id)
experiment: Experiment = mlflow.get_experiment(experiment_id)
return experiment.name
def _export_model(self, model_name, output_dir):
fs = _filesystem.get_filesystem(output_dir)
model = self.http_client.get(f"registered-models/get",
{"name": model_name})
fs.mkdirs(output_dir)
model["registered_model"]["latest_versions"] = []
versions = self.mlflow_client.search_model_versions(
f"name='{model_name}'")
print(f"Found {len(versions)} versions for model {model_name}")
manifest = []
exported_versions = 0
for vr in versions:
if len(self.stages) > 0 and not vr.current_stage.lower(
) in self.stages:
continue
run_id = vr.run_id
opath = os.path.join(output_dir, run_id)
opath = opath.replace("dbfs:", "/dbfs")
dct = {
"version": vr.version,
"stage": vr.current_stage,
"run_id": run_id
}
print(f"Exporting: {dct}")
manifest.append(dct)
try:
if self.export_run:
self.run_exporter.export_run(run_id, opath)
run = self.mlflow_client.get_run(run_id)
dct = dict(vr)
dct["_run_artifact_uri"] = run.info.artifact_uri
experiment = mlflow.get_experiment(run.info.experiment_id)
dct["_experiment_name"] = experiment.name
model["registered_model"]["latest_versions"].append(dct)
exported_versions += 1
except mlflow.exceptions.RestException as e:
if "RESOURCE_DOES_NOT_EXIST: Run" in str(e):
print(
f"WARNING: Run for version {vr.version} does not exist. {e}"
)
else:
import traceback
traceback.print_exc()
print(
f"Exported {exported_versions}/{len(versions)} versions for model {model_name}"
)
path = os.path.join(output_dir, "model.json")
utils.write_json_file(fs, path, model)
return manifest
def __init__(self,
identifier,
shape,
latent_dim,
p_fliplabel=0.,
output_path=None,
experiment_path=None,
cuda=False,
ngpu=1):
self.cuda = cuda
self.ngpu = 0 if not self.cuda else ngpu
self.shape = shape
self.latent_dim = latent_dim
self.identifier = identifier
self.p_fliplabel = p_fliplabel
self.output_path = output_path or os.path.join(
config.default_output_dir)
self.tracking_path = os.path.join(self.output_path, "mlruns")
self.experiment_path = experiment_path or os.path.join(
self.output_path, identifier)
mlflow.set_tracking_uri(self.tracking_path)
self.experiment = mlflow.get_experiment_by_name(
identifier) or mlflow.get_experiment(
mlflow.create_experiment(identifier))
if torch.cuda.is_available() and not self.cuda:
print(
"[WARNING] You have a CUDA device. You probably want to run with CUDA enabled"
)
self.device = torch.device("cuda" if self.cuda else "cpu")
self.generator = None
self.discriminator = None
self.latent_vector_sampler = self._get_default_latent_vector_sampler()
self.model_params = {
"shape": shape,
"latent_dim": latent_dim,
"p_fliplabel": p_fliplabel,
"sampler": "normal"
}
self.Tensor = torch.cuda.FloatTensor if self.cuda else torch.FloatTensor
import constants as c
logging.basicConfig(filename=os.path.join(c.LOGS_DIR, f'{datetime.now()}.txt'),
filemode='w+',
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.DEBUG)
logging.getLogger().setLevel(logging.INFO)
parser = argparse.ArgumentParser(description='Train & Save Model')
parser.add_argument('--epochs', type=int, dest="epochs", help="Num of epochs to run", default=3)
parser.add_argument('--batch_size', type=int, dest="batch_size", help="Batch Size", default=32)
parser.add_argument('--beta_1', type=float, dest="beta_1", help="beta_1 hyper-parameter", default=0.9)
parser.add_argument('--beta_2', type=float, dest="beta_2", help="beta_2 hyper-parameter", default=0.999)
parser.add_argument('--lr', type=float, dest="lr", help="Initial Learning rate(Default: 2e-5)", default=2e-5)
parser.add_argument('--epsilon', type=float, dest="epsilon", help="Epsilon(Default: 1e-7)", default=1e-7)
args = parser.parse_args()
experiment = mlflow.get_experiment(c.ML_FLOW_EXPERIMENT_ID)
logging.info("Name: {}".format(experiment.name))
logging.info("Experiment_id: {}".format(experiment.experiment_id))
logging.info("Artifact Location: {}".format(experiment.artifact_location))
logging.info("Tags: {}".format(experiment.tags))
logging.info("Lifecycle_stage: {}".format(experiment.lifecycle_stage))
mlflow.tensorflow.autolog(every_n_iter=1, log_models=True, disable=False, exclusive=False)
with mlflow.start_run(experiment_id=c.ML_FLOW_EXPERIMENT_ID):
save_dir_path, tags, signature = train_test(epochs=args.epochs, eval_batch_size=args.batch_size, beta_1=args.beta_1,
beta_2=args.beta_2, init_lr=args.lr, epsilon=args.epsilon)
import mlflow
try:
experiment_id = mlflow.create_experiment("mung")
experiment = mlflow.get_experiment(experiment_id)
print("Name: {}".format(experiment.name))
print("experiment_id: {}".format(experiment.experiment_id))
print("artifact_location: {}".format(experiment.artifact_location))
print("tags: {}".format(experiment.tags))
print("lifecycle_stage: {}".format(experiment.lifecycle_stage))
except Exception as err:
print('=========== Aready Experiment Message ===========')
print(err)
print('=========== Aready Experiment Message ===========')
print()
print()
from mlflow.tracking.client import MlflowClient
import mlflow
import numpy as np
start_experiment = 3
end_experiment = 9
metric = 'val_accuracy'
num_epochs = 50
experiment_ids = [1, 2, 3, 4, 5, 6]
means = []
stds = []
for experiment_id in experiment_ids:
experiment_id = str(experiment_id)
experiment_name = mlflow.get_experiment(experiment_id).name
print(experiment_name)
results = mlflow.search_runs(experiment_ids=[experiment_id])
max_accuracies = []
for run_id in (results['run_id']):
val_accuracies = MlflowClient().get_metric_history(run_id, metric)
max_accuracy = 0.
for epoch in range(num_epochs):
accuracy = val_accuracies[epoch].value
if val_accuracies[epoch].value > max_accuracy:
max_accuracy = val_accuracies[epoch].value
max_accuracies.append(max_accuracy)
mean = np.mean(max_accuracies)
std = np.std(max_accuracies)
print('Mean:{} Std:{}'.format(mean, std))
def cli_main():
pl.seed_everything(1234)
# ------------
# args
# ------------
parser = ArgumentParser()
parser.add_argument("--batch_size", default=32, type=int)
parser.add_argument("--hidden_dim", type=int, default=128)
parser = pl.Trainer.add_argparse_args(parser)
args = parser.parse_args()
# ------------
# data
# ------------
dataset = MNIST("",
train=True,
download=True,
transform=transforms.ToTensor())
mnist_test = MNIST("",
train=False,
download=True,
transform=transforms.ToTensor())
mnist_train, mnist_val = random_split(dataset, [55000, 5000])
train_loader = DataLoader(mnist_train, batch_size=args.batch_size)
val_loader = DataLoader(mnist_val, batch_size=args.batch_size)
test_loader = DataLoader(mnist_test, batch_size=args.batch_size)
# ------------
# model
# ------------
model = LitAutoEncoder()
# ------------
# logging
# ------------
# get run object using mlflow
with mlflow.start_run() as run:
experiment_id = run.info.experiment_id
# get the experiment name
exp_name = mlflow.get_experiment(experiment_id).name
# get the mlflow tracking uri
mlflow_uri = mlflow.get_tracking_uri()
mlf_logger = MLFlowLogger(experiment_name=exp_name,
tracking_uri=mlflow_uri)
# link the mlflowlogger run ID to the azureml run ID
mlf_logger._run_id = run.info.run_id
# ------------
# training
# ------------
trainer = pl.Trainer.from_argparse_args(args, logger=mlf_logger)
trainer.fit(model, train_loader, val_loader)
# ------------
# testing
# ------------
result = trainer.test(test_dataloaders=test_loader)
print(result)
def create_experiment(base_name: str) -> mlflow.entities.experiment.Experiment:
uid = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
experiment_id = mlflow.create_experiment(name=f"{base_name}_{uid}")
return mlflow.get_experiment(experiment_id=experiment_id)
#
# Code snippet for https://mlflow.org/docs/latest/python_api/mlflow.html#get_experement
#
import warnings
import mlflow
if __name__ == "__main__":
warnings.filterwarnings("ignore")
print(mlflow.__version__)
# Convert experiment ID as a string argument
experiment = mlflow.get_experiment("0")
# Print the contents of Experiment data
print("Name: {}".format(experiment.name))
print("Artifact Location: {}".format(experiment.artifact_location))
print("Tags: {}".format(experiment.tags))
print("Lifecycle_stage: {}".format(experiment.lifecycle_stage))
def get_experiment(self, experiment_id):
return mlflow.get_experiment(experiment_id)
def _experiment_name(self):
return mlflow.get_experiment(self._experiment_id).name
