def start_run(
mlflow_config: MlflowConfig,
exp: str,
name: str,
verbose: bool = True,
):
if mlflow.active_run() is not None:
mlflow.end_run()
mlflow.set_tracking_uri(mlflow_config.uri)
exp_name = mlflow_config.exp_name
if mlflow.get_experiment_by_name(exp_name) is None:
mlflow.create_experiment(exp_name)
experiment_id = mlflow.get_experiment_by_name(exp_name).experiment_id
search_df: Any = mlflow.search_runs(
[experiment_id], filter_string=f"tags.exp = '{exp}'")
resume = len(search_df) > 0
if resume:
run_id = search_df["run_id"].iloc[0]
mlflow.start_run(run_id=run_id, experiment_id=experiment_id)
else:
mlflow.start_run(run_name=f"{exp}: {name}",
experiment_id=experiment_id)
mlflow.set_tag("exp", exp)
if verbose:
print(f"mlflow started. experiment name: {exp_name}")
print(f"{mlflow_config.uri}/#/experiments/{experiment_id}")
def __get_mlflow_experiment(name):
"""
Get an experiment from the MLFlow server
:param name - The name of the experiment
"""
if not mlflow.get_experiment_by_name(name):
mlflow.create_experiment(name=name)
return mlflow.get_experiment_by_name(name)
def start_mlflow(exp_name: str = "cv_new") -> int:
try:
mlflow.end_run()
except Exception:
pass
if mlflow.get_experiment_by_name(exp_name) is None:
mlflow.create_experiment(exp_name)
experiment_id = mlflow.get_experiment_by_name(exp_name).experiment_id
return experiment_id
def mlflow_del_default_experiment():
print(f"\nDelete 'Default' experiment from MLflow loggs...")
try:
default_id = mlflow.get_experiment_by_name("Default").experiment_id
default_loc = mlflow.get_experiment_by_name(
"Default").artifact_location
mlflow.delete_experiment(default_id)
print(f"'Default' experiment located:'{default_loc}' was deleted.\n")
except Exception as e:
print(f"'Default' experiment doesnt exist: {e}\n")
def setupMlflowConf(conf):
mlflow.set_experiment(conf['experiment-path'])
try:
experimentID = mlflow.get_experiment_by_name(
conf['experiment-path']).experiment_id
return experimentID
except FileNotFoundError as e:
time.sleep(10)
experimentID = mlflow.get_experiment_by_name(
conf['experiment-path']).experiment_id
return experimentID
def start_run(models: str):
name = input("name: ")
if mlflow.active_run() is not None:
mlflow.end_run()
if mlflow.get_experiment_by_name(Config.exp_name) is None:
mlflow.create_experiment(Config.exp_name)
experiment_id = mlflow.get_experiment_by_name(
Config.exp_name).experiment_id
exp = str(Path(__file__)).split("/")[-1].replace(".py", "")
mlflow.start_run(run_name=f"{exp}: ({models}) {name}",
experiment_id=experiment_id)
mlflow.log_param("exp", exp)
def start_mlflow(config: Config) -> Tuple[str, str]:
try:
mlflow.end_run()
except Exception:
pass
if mlflow.get_experiment_by_name(config.exp_name) is None:
mlflow.create_experiment(config.exp_name)
experiment_id: str = mlflow.get_experiment_by_name(
config.exp_name).experiment_id
print("put the run name")
run_name: str = input()
mlflow.start_run(experiment_id=experiment_id, run_name=run_name)
config.log_mlflow_params()
return experiment_id, run_name
def process_log_func(exp_name):
import mlflow
import random
mlflow.set_experiment(exp_name)
uri = mlflow.get_tracking_uri()
mlflow.set_tracking_uri(uri)
my_pid = os.getpid()
parent_pid = os.getppid()
artifact_path = "/tmp/features-" + str(my_pid) + ".txt"
# create an artifact
features = "rooms, zipcode, median_price, school_rating, transport"
with open(artifact_path, 'w') as f:
f.write(features)
print("Running in PID: {}".format(my_pid))
with mlflow.start_run():
pid = "PID-" + str(my_pid)
ppid = "PPID-" + str(parent_pid)
mlflow.log_param(pid, my_pid)
mlflow.log_param(ppid, parent_pid)
mlflow.log_metric("metric", random.randint(1, 10))
mlflow.log_artifact(artifact_path)
experiment = mlflow.get_experiment_by_name(exp_name)
if experiment:
print("In pid: {}, experiment_id: {}".format(my_pid,
experiment.experiment_id))
def get_latest_mlrun(params):
"""Get latest mlflow run
:param params: gdl parameters dictionary
:return: mlflow run object
"""
tracking_uri = params['global']['mlflow_uri']
mlflow.set_tracking_uri(tracking_uri)
mlexp = mlflow.get_experiment_by_name(
params['global']['mlflow_experiment_name'])
exp_id = mlexp.experiment_id
try:
run_ids = ([
x.run_id for x in mlflow.list_run_infos(
exp_id, max_results=1, order_by=["tag.release DESC"])
])
except AttributeError:
mlflow_client = mlflow.tracking.MlflowClient(tracking_uri=tracking_uri)
run_ids = [
x.run_id
for x in mlflow_client.list_run_infos(exp_id, run_view_type=3)[0:1]
]
mlrun = mlflow.get_run(run_ids[0])
return mlrun
def mlflow_start_run(
uri, experiment_name, artifact_location, run_name=None, enable_mlflow=True
):
if enable_mlflow:
from mlflow import (
create_experiment,
get_experiment_by_name,
start_run,
set_tracking_uri,
)
from mlflow.exceptions import MlflowException
if uri:
set_tracking_uri(uri)
if experiment_name:
try:
experiment_id = create_experiment(
experiment_name,
artifact_location=artifact_location,
)
except MlflowException:
experiment_id = get_experiment_by_name(experiment_name).experiment_id
start_run(experiment_id=experiment_id, run_name=run_name)
def prepare_environment(environment: str) -> ApiClient:
environment_data = get_environment_data(environment)
config_type, config = pick_config(environment_data)
api_client = _get_api_client(config, command_name="cicdtemplates-")
_prepare_workspace_dir(api_client, environment_data["workspace_dir"])
if config_type == "ENV":
mlflow.set_tracking_uri(DATABRICKS_MLFLOW_URI)
elif config_type == "PROFILE":
mlflow.set_tracking_uri(f'{DATABRICKS_MLFLOW_URI}://{environment_data["profile"]}')
else:
raise NotImplementedError(f"Config type: {config_type} is not implemented")
experiment: Optional[mlflow.entities.Experiment] = mlflow.get_experiment_by_name(environment_data["workspace_dir"])
# if there is no experiment
if not experiment:
mlflow.create_experiment(environment_data["workspace_dir"], environment_data["artifact_location"])
else:
# verify experiment location
if experiment.artifact_location != environment_data["artifact_location"]:
raise Exception(
f"Required location of experiment {environment_data['workspace_dir']} "
f"doesn't match the project defined one: \n"
f"\t experiment artifact location: {experiment.artifact_location} \n"
f"\t project artifact location : {environment_data['artifact_location']} \n"
f"Change of experiment location is currently not supported in MLflow. "
f"Please change the experiment name to create a new experiment."
)
mlflow.set_experiment(environment_data["workspace_dir"])
return api_client
def get_model_path_local(run_name):
"""
.
"""
experiment_id = mlflow.get_experiment_by_name(
name=EXPERIMENT_NAME).experiment_id
return os.path.join(DATA_PATH, 'edge_models', experiment_id, run_name)
def __call__(self, config: Any, command: str = ""):
if mlflow.active_run() is not None:
LOGGER.info(f"Active run found: {get_url(mlflow.active_run())}")
self.log_and_launch(config=config, command=command)
else:
# Create run from params in config
params = config.get("mlflow") or {}
run_id = params.get("run_id")
run_name = params.get("run_name")
tracking_uri = params.get("tracking_uri")
experiment_name = params.get("experiment_name")
artifact_location = params.get("artifact_location")
# Setup experiment and general MlFlow parameters
if tracking_uri is not None:
mlflow.set_tracking_uri(tracking_uri)
if experiment_name is not None:
if mlflow.get_experiment_by_name(experiment_name) is None:
mlflow.create_experiment(experiment_name, artifact_location=artifact_location)
mlflow.set_experiment(experiment_name)
# Start MlFlow run, log information and launch
with mlflow.start_run(run_id=run_id, run_name=run_name) as run:
LOGGER.info(f"Started run: {get_url(run)}")
config = fromconfig.utils.merge_dict(config, {"mlflow": {"run_id": run.info.run_id}})
self.log_and_launch(config=config, command=command)
def _save(self, data: Any) -> None:
self._init_dataset()
self._dataset.save(data)
if self._cache:
self._cache.save(data)
if find_spec("mlflow"):
import mlflow
if self.saving_tracking_uri:
mlflow.set_tracking_uri(self.saving_tracking_uri)
if self.saving_run_id:
mlflow.start_run(run_id=self.saving_run_id)
elif self.saving_experiment_name:
experiment_id = mlflow.get_experiment_by_name(
self.saving_experiment_name).experiment_id
mlflow.start_run(run_id=self.saving_run_id,
experiment_id=experiment_id)
if self.dataset in {"p"}:
mlflow_log_params({self.dataset_name: data})
elif self.dataset in {"m"}:
mlflow_log_metrics({self.dataset_name: data})
else:
mlflow_log_artifacts([self.filepath])
if self.saving_run_id or self.saving_experiment_name:
mlflow.end_run()
def main(cfg: DictConfig) -> None:
# set up mlflow experiment id
mlflow.set_tracking_uri(f"file://{to_absolute_path(cfg.path_to_mlflow)}")
experiment = mlflow.get_experiment_by_name(cfg.experiment_name)
if experiment is not None: # fetch existing experiment id
run_kwargs = {'experiment_id': experiment.experiment_id}
else: # create new experiment
experiment_id = mlflow.create_experiment(cfg.experiment_name)
run_kwargs = {'experiment_id': experiment_id}
# run the training with mlflow tracking
with mlflow.start_run(**run_kwargs) as active_run:
setup_gpu(cfg.gpu_cfg)
training_cfg = OmegaConf.to_object(cfg.training_cfg) # convert to python dictionary
scaling_cfg = to_absolute_path(cfg.scaling_cfg)
dataloader = DataLoaderReco.DataLoader(training_cfg, scaling_cfg)
dl_config = dataloader.config
model = model = MyGNN(dl_config)
input_shape, _ = dataloader.get_shape()
# print(input_shape[0])
# compile_build = tf.ones(input_shape[0], dtype=tf.float32, name=None)
model.build(list(input_shape[0]))
compile_model(model, dl_config["SetupNN"]["mode"], dl_config["SetupNN"]["learning_rate"])
fit_hist = run_training(model, dataloader, False, cfg.log_suffix)
mlflow.log_dict(training_cfg, 'input_cfg/training_cfg.yaml')
mlflow.log_artifact(scaling_cfg, 'input_cfg')
mlflow.log_artifact(to_absolute_path("Training_SNNv0.py"), 'input_cfg')
mlflow.log_artifact(to_absolute_path("../commonReco.py"), 'input_cfg')
mlflow.log_artifacts('.hydra', 'input_cfg/hydra')
mlflow.log_artifact('Training_SNNv0.log', 'input_cfg/hydra')
mlflow.log_param('run_id', active_run.info.run_id)
print(f'\nTraining has finished! Corresponding MLflow experiment name (ID): {cfg.experiment_name}({run_kwargs["experiment_id"]}), and run ID: {active_run.info.run_id}\n')
def __init__(self, context):
# Context associated attributes
self.log = context.log
self.run_name = context.pipeline_run.pipeline_name
self.dagster_run_id = context.run_id
# resource config attributes
resource_config = context.resource_config
self.tracking_uri = resource_config.get("mlflow_tracking_uri")
if self.tracking_uri:
mlflow.set_tracking_uri(self.tracking_uri)
self.parent_run_id = resource_config.get("parent_run_id")
self.experiment_name = resource_config["experiment_name"]
self.env_tags_to_log = resource_config.get("env_to_tag") or []
self.extra_tags = resource_config.get("extra_tags")
# Update env variables if any are given
self.env_vars = resource_config.get("env", {})
if self.env_vars:
environ.update(self.env_vars)
# If the experiment exists then the set won't do anything
mlflow.set_experiment(self.experiment_name)
self.experiment = mlflow.get_experiment_by_name(self.experiment_name)
# Get the client object
self.tracking_client = mlflow.tracking.MlflowClient()
# Set up the active run and tags
self._setup()
def set_experiment_id(self, name):
"""
Get experiment_id using self.exp_folder
"""
if name is None:
experiment_id = None
else:
# Get experiment_id from already created experiment name -> 'NLP'
try:
exp_id = mlflow.get_experiment_by_name(name).experiment_id
# Inputs:
# name = the experiment's name
# Returns: Experiment object
# .experiment_id = Integer - value wanted from object
# Create new experiment and get experiment_id
except AttributeError: # Error: if experiment doesn't exist ...
exp_id = mlflow.create_experiment(name)
# Inputs:
# name = the experiment's name (unique)
# artifact_location (optional) = location to store run artifacts
# Returns: integer Id of the experiment
return exp_id
def test_get_experiment_by_name():
with TempDir(chdr=True):
name = "Random experiment %d" % random.randint(1, 1e6)
exp_id = mlflow.create_experiment(name)
experiment = mlflow.get_experiment_by_name(name)
assert experiment.experiment_id == exp_id
def Experiment_Id(self):
print("Attaining 'experiment_id' ...")
experiment_name = self.experiment_name
if experiment_name == None:
experiment_id = None
else:
# Get experiment_id from already created experiment
try:
experiment_id = mlflow.get_experiment_by_name(experiment_name).experiment_id
# Inputs:
# name = the experiment's name
# Returns: Experiment object
# .experiment_id = Integer - value wanted from object
# Create new experiment and get experiment_id
except AttributeError: # Error: if experiment doesn't exist ...
experiment_id = mlflow.create_experiment(experiment_name)
# Inputs:
# name = the experiment's name (unique)
# artifact_location (optional) = location to store run artifacts
# Returns: integer Id of the experiment
print("New experiment created in tracking_uri filepath:\n" + str(self.tracking_uri))
return experiment_id
def __init__(self,
root: str,
tracking_uri: str,
experiment_name: str = 'nlkda'):
"""
Constructor.
Connects to a running MLFlow instance in which the runs of the experiments are stored.
Also creates an output root directory. The directory will be created if it does not exist.
:param root: str
The path of the output root.
:param tracking_uri: str
The uri where the MLFlow instance is running.
:param experiment_name: str
The name of the experiment on the MLFlow instance server.
"""
mlflow.set_tracking_uri(uri=tracking_uri)
experiment = mlflow.get_experiment_by_name(name=experiment_name)
if experiment is None:
experiment_id = mlflow.create_experiment(name=experiment_name)
else:
experiment_id = experiment.experiment_id
self.experiment_id = experiment_id
self.root = pathlib.Path(root)
self.current_run = None
def create_experiment(experiment_name):
experiment = mlflow.get_experiment_by_name(experiment_name)
experiment_artifact = experiment.artifact_location if experiment else ""
experiment_id = experiment.experiment_id if experiment else None
blob_storage_name = environ.get("AZURE_STORAGE_ACCOUNT_NAME")
blob_container_name = environ.get("AZURE_STORAGE_CONTAINER_NAME")
blob_uri = (
f"wasbs://{blob_container_name}@{blob_storage_name}.blob.core.windows.net"
if blob_container_name and blob_storage_name else None)
if experiment is None:
print(
f"Experiment {experiment_name} doesn't exist, creating new with artifact_location {blob_uri}"
)
experiment_id = mlflow.create_experiment(experiment_name,
artifact_location=blob_uri)
# artifact_location are set at the experiment level upon creation
# if this changes, the experiment needs to be recreated
elif experiment_artifact != blob_uri and not (
experiment_artifact.startswith("dbfs") and blob_uri is None):
print(
f"Experiment {experiment_name} already exist with artifact_location {experiment_artifact}"
)
mlflow.delete_experiment(experiment_id)
experiment_id = mlflow.create_experiment(experiment_name,
artifact_location=blob_uri)
print(
f"Experiment {experiment_name} recreated with artifact_location {blob_uri}"
)
return experiment_id
def predict_tags(
text: str = "Transfer learning with BERT for self-supervised learning",
run_id: str = "",
) -> Dict:
"""Predict tags for a give input text using a trained model.
Warning:
Make sure that you have a trained model first!
Args:
text (str, optional): Input text to predict tags for.
Defaults to "Transfer learning with BERT for self-supervised learning".
run_id (str, optional): ID of the run to load model artifacts from.
Defaults to model with lowest `best_val_loss` from the `best` experiment.
Returns:
Predicted tags for input text.
"""
# Get best run
if not run_id:
experiment_id = mlflow.get_experiment_by_name("best").experiment_id
all_runs = mlflow.search_runs(
experiment_ids=experiment_id,
order_by=["metrics.best_val_loss ASC"],
)
run_id = all_runs.iloc[0].run_id
# Predict
prediction = predict.predict(texts=[text], run_id=run_id)
logger.info(json.dumps(prediction, indent=2))
return prediction
def test_tune_exp_default_trainable(
dataset, pipeline_config, trainer_config, monkeypatch
):
# avoid logging to wandb
monkeypatch.setenv("WANDB_MODE", "dryrun")
pipeline_config["features"]["word"]["embedding_dim"] = tune.choice([2, 4])
trainer_config["optimizer"]["lr"] = tune.loguniform(0.001, 0.01)
my_exp = TuneExperiment(
pipeline_config=pipeline_config,
trainer_config=trainer_config,
train_dataset=dataset,
valid_dataset=dataset,
num_samples=1,
)
assert my_exp._name.startswith("HPO on")
assert my_exp.name == my_exp._name
assert my_exp._run_identifier == "_default_trainable"
analysis = tune.run(my_exp)
assert len(analysis.trials) == 1
mlflow.set_tracking_uri(mlflow.get_tracking_uri())
assert mlflow.get_experiment_by_name(my_exp._name)
def get_or_create_experiment(name: str,
artifact_location: Optional[str] = None) -> str:
"""
Get or create an experiment.
Parameters
----------
name : str
Experiment name
artifact_location : str, default None
Location to store run artifacts. If unspecified, the server
picks an appropriate default.
Returns
-------
str
ID of the experiment.
Examples
--------
>>> with mlflow.start_run():
... expr_id = mlflow.get_or_create_experiment('test')
"""
expr = mlflow.get_experiment_by_name(name)
if expr is None:
return mlflow.create_experiment(name, artifact_location)
return expr.experiment_id
def start_track(self, experiment_name=None, disable_run_init=False):
"""
Start a new run to track.
:param experiment_name: The name of the mlflow experiment
:param disable_run_init: Flag to indicate if the run_init functions are to be run on an already existing run.
:return:
"""
if not tracking_active:
self.activate_tracking()
# check if there is already an active run
run = mlflow.active_run()
experiment = None
if run is None:
experiment_name = experiment_name or DEFAULT_EXPERIMENT_NAME
# Create run if run doesn't already exist
experiment = mlflow.get_experiment_by_name(experiment_name)
experiment_id = experiment.experiment_id if experiment else mlflow.create_experiment(
experiment_name)
run = self.api.start_run(experiment_id=experiment_id)
else:
# if not disable_run_init:
# self.api.run_setups(_pypads_env=LoggerEnv(parameter=dict(), experiment_id=experiment_id, run_id=run_id,
# data={"category": "SetupFn"}))
if experiment_name:
# Check if we're still in the same experiment
experiment = mlflow.get_experiment_by_name(experiment_name)
experiment_id = experiment.experiment_id if experiment else mlflow.create_experiment(
experiment_name)
if run.info.experiment_id != experiment_id:
experiment = mlflow.get_experiment_by_name(experiment_name)
if experiment is None:
experiment = self.backend.get_experiment(run.info.experiment_id)
# override active run if used
if experiment_name and run.info.experiment_id != experiment.experiment_id:
logger.warning(
"Active experiment_id of run doesn't match given input name " +
experiment_name + ". Recreating new run.")
try:
self.api.start_run(experiment_id=experiment.experiment_id,
nested=True)
except Exception:
mlflow.end_run()
self.api.start_run(experiment_id=experiment.experiment_id)
return self
def setup_mlflow(cfg, features_cfg, data_cfg):
import mlflow
experiment_id = OmegaConf.select(cfg, "experiment_id", default=None)
if experiment_id is None and cfg.experiment_name is not None:
mlflow.set_experiment(cfg.experiment_name)
experiment = mlflow.get_experiment_by_name(cfg.experiment_name)
if cfg.experiment_name is not None:
logger.debug(f"MLFlow Experiment: {cfg.experiment_name}")
experiment_id = experiment.experiment_id
orig_cwd = get_original_cwd()
tracking_uri = f"file://{orig_cwd}/mlruns"
mlflow.set_tracking_uri(tracking_uri)
logger.info(f"MLFlow Tracking URI: {tracking_uri}")
# if cfg.model == "xgboost":
# import mlflow.xgboost
# logger.debug("Turning on MLFlow autologging for XGBoost...")
# mlflow.xgboost.autolog()
run = mlflow.start_run(experiment_id=experiment_id)
# tracking_uri = mlflow.get_tracking_uri()
# logger.info(f"MLFlow Tracking URI: {tracking_uri}")
processed_data_dir = Path(to_absolute_path(data_cfg.hydra.run.dir))
if processed_data_dir is not None:
data_hydra_dir = processed_data_dir / ".hydra"
mlflow.log_artifacts(data_hydra_dir,
artifact_path="processed_data_configs")
data_cfg = OmegaConf.load(data_hydra_dir / "config.yaml")
for name, param_name in DATA_CONFIGS_TO_LOG.items():
param = OmegaConf.select(data_cfg, param_name)
if param is not None:
if isinstance(param, list):
param = ", ".join([str(p) for p in param])
mlflow.log_param(name, param)
model_hydra_dir = Path(".hydra")
mlflow.log_artifacts(model_hydra_dir, artifact_path="model_configs")
mlflow.log_params({
"model": cfg.model,
"lag": features_cfg.lag,
"exog_lag": features_cfg.exog_lag,
"lead": features_cfg.lead,
"cv_method": cfg.cv.method,
})
mlflow.log_params(cfg.cv.params)
tags = OmegaConf.select(cfg, "tags", default={})
if bool(tags):
mlflow.set_tags(tags)
return run
def setExperiment():
experiment_path = "/Shared/MLflow_Experiments/Modeler_2_Experiments/"
experiment_name = "Financial Fraud Experiment"
mlflow.set_experiment(experiment_path + experiment_name)
experiment = mlflow.get_experiment_by_name(experiment_path +
experiment_name)
return experiment
def getLatestRunId(pathToExperiment):
import mlflow
from pyspark.sql.functions import col
expId = mlflow.get_experiment_by_name(pathToExperiment).experiment_id
lastest_run_id = spark.read.format("mlflow-experiment").load(
expId).orderBy(
col("end_time").desc()).select("run_id").limit(1).collect()[0][0]
return lastest_run_id
def setOrCeateMLFlowExperiment(experimentPath,mlflow_model_save_dir):
from mlflow.exceptions import MlflowException
try:
experiment_id = mlflow.create_experiment(experimentPath, "dbfs:" + mlflow_model_save_dir)
except MlflowException: # if experiment is already created
experiment_id = mlflow.get_experiment_by_name(experimentPath).experiment_id
mlflow.set_experiment(experimentPath)
return experiment_id
def setExperiment():
experiment_path = "/Shared/MLflow_Experiments/Modeler_1_Experiments/"
experiment_name = "Wine Quality Experiment"
mlflow.set_experiment(experiment_path + experiment_name)
experiment = mlflow.get_experiment_by_name(experiment_path +
experiment_name)
return experiment