def start_run( mlflow_config: MlflowConfig, exp: str, name: str, verbose: bool = True, ): if mlflow.active_run() is not None: mlflow.end_run() mlflow.set_tracking_uri(mlflow_config.uri) exp_name = mlflow_config.exp_name if mlflow.get_experiment_by_name(exp_name) is None: mlflow.create_experiment(exp_name) experiment_id = mlflow.get_experiment_by_name(exp_name).experiment_id search_df: Any = mlflow.search_runs( [experiment_id], filter_string=f"tags.exp = '{exp}'") resume = len(search_df) > 0 if resume: run_id = search_df["run_id"].iloc[0] mlflow.start_run(run_id=run_id, experiment_id=experiment_id) else: mlflow.start_run(run_name=f"{exp}: {name}", experiment_id=experiment_id) mlflow.set_tag("exp", exp) if verbose: print(f"mlflow started. experiment name: {exp_name}") print(f"{mlflow_config.uri}/#/experiments/{experiment_id}")
def __get_mlflow_experiment(name): """ Get an experiment from the MLFlow server :param name - The name of the experiment """ if not mlflow.get_experiment_by_name(name): mlflow.create_experiment(name=name) return mlflow.get_experiment_by_name(name)
def start_mlflow(exp_name: str = "cv_new") -> int: try: mlflow.end_run() except Exception: pass if mlflow.get_experiment_by_name(exp_name) is None: mlflow.create_experiment(exp_name) experiment_id = mlflow.get_experiment_by_name(exp_name).experiment_id return experiment_id
def mlflow_del_default_experiment(): print(f"\nDelete 'Default' experiment from MLflow loggs...") try: default_id = mlflow.get_experiment_by_name("Default").experiment_id default_loc = mlflow.get_experiment_by_name( "Default").artifact_location mlflow.delete_experiment(default_id) print(f"'Default' experiment located:'{default_loc}' was deleted.\n") except Exception as e: print(f"'Default' experiment doesnt exist: {e}\n")
def setupMlflowConf(conf): mlflow.set_experiment(conf['experiment-path']) try: experimentID = mlflow.get_experiment_by_name( conf['experiment-path']).experiment_id return experimentID except FileNotFoundError as e: time.sleep(10) experimentID = mlflow.get_experiment_by_name( conf['experiment-path']).experiment_id return experimentID
def start_run(models: str): name = input("name: ") if mlflow.active_run() is not None: mlflow.end_run() if mlflow.get_experiment_by_name(Config.exp_name) is None: mlflow.create_experiment(Config.exp_name) experiment_id = mlflow.get_experiment_by_name( Config.exp_name).experiment_id exp = str(Path(__file__)).split("/")[-1].replace(".py", "") mlflow.start_run(run_name=f"{exp}: ({models}) {name}", experiment_id=experiment_id) mlflow.log_param("exp", exp)
def start_mlflow(config: Config) -> Tuple[str, str]: try: mlflow.end_run() except Exception: pass if mlflow.get_experiment_by_name(config.exp_name) is None: mlflow.create_experiment(config.exp_name) experiment_id: str = mlflow.get_experiment_by_name( config.exp_name).experiment_id print("put the run name") run_name: str = input() mlflow.start_run(experiment_id=experiment_id, run_name=run_name) config.log_mlflow_params() return experiment_id, run_name
def process_log_func(exp_name): import mlflow import random mlflow.set_experiment(exp_name) uri = mlflow.get_tracking_uri() mlflow.set_tracking_uri(uri) my_pid = os.getpid() parent_pid = os.getppid() artifact_path = "/tmp/features-" + str(my_pid) + ".txt" # create an artifact features = "rooms, zipcode, median_price, school_rating, transport" with open(artifact_path, 'w') as f: f.write(features) print("Running in PID: {}".format(my_pid)) with mlflow.start_run(): pid = "PID-" + str(my_pid) ppid = "PPID-" + str(parent_pid) mlflow.log_param(pid, my_pid) mlflow.log_param(ppid, parent_pid) mlflow.log_metric("metric", random.randint(1, 10)) mlflow.log_artifact(artifact_path) experiment = mlflow.get_experiment_by_name(exp_name) if experiment: print("In pid: {}, experiment_id: {}".format(my_pid, experiment.experiment_id))
def get_latest_mlrun(params): """Get latest mlflow run :param params: gdl parameters dictionary :return: mlflow run object """ tracking_uri = params['global']['mlflow_uri'] mlflow.set_tracking_uri(tracking_uri) mlexp = mlflow.get_experiment_by_name( params['global']['mlflow_experiment_name']) exp_id = mlexp.experiment_id try: run_ids = ([ x.run_id for x in mlflow.list_run_infos( exp_id, max_results=1, order_by=["tag.release DESC"]) ]) except AttributeError: mlflow_client = mlflow.tracking.MlflowClient(tracking_uri=tracking_uri) run_ids = [ x.run_id for x in mlflow_client.list_run_infos(exp_id, run_view_type=3)[0:1] ] mlrun = mlflow.get_run(run_ids[0]) return mlrun
def mlflow_start_run( uri, experiment_name, artifact_location, run_name=None, enable_mlflow=True ): if enable_mlflow: from mlflow import ( create_experiment, get_experiment_by_name, start_run, set_tracking_uri, ) from mlflow.exceptions import MlflowException if uri: set_tracking_uri(uri) if experiment_name: try: experiment_id = create_experiment( experiment_name, artifact_location=artifact_location, ) except MlflowException: experiment_id = get_experiment_by_name(experiment_name).experiment_id start_run(experiment_id=experiment_id, run_name=run_name)
def prepare_environment(environment: str) -> ApiClient: environment_data = get_environment_data(environment) config_type, config = pick_config(environment_data) api_client = _get_api_client(config, command_name="cicdtemplates-") _prepare_workspace_dir(api_client, environment_data["workspace_dir"]) if config_type == "ENV": mlflow.set_tracking_uri(DATABRICKS_MLFLOW_URI) elif config_type == "PROFILE": mlflow.set_tracking_uri(f'{DATABRICKS_MLFLOW_URI}://{environment_data["profile"]}') else: raise NotImplementedError(f"Config type: {config_type} is not implemented") experiment: Optional[mlflow.entities.Experiment] = mlflow.get_experiment_by_name(environment_data["workspace_dir"]) # if there is no experiment if not experiment: mlflow.create_experiment(environment_data["workspace_dir"], environment_data["artifact_location"]) else: # verify experiment location if experiment.artifact_location != environment_data["artifact_location"]: raise Exception( f"Required location of experiment {environment_data['workspace_dir']} " f"doesn't match the project defined one: \n" f"\t experiment artifact location: {experiment.artifact_location} \n" f"\t project artifact location : {environment_data['artifact_location']} \n" f"Change of experiment location is currently not supported in MLflow. " f"Please change the experiment name to create a new experiment." ) mlflow.set_experiment(environment_data["workspace_dir"]) return api_client
def get_model_path_local(run_name): """ . """ experiment_id = mlflow.get_experiment_by_name( name=EXPERIMENT_NAME).experiment_id return os.path.join(DATA_PATH, 'edge_models', experiment_id, run_name)
def __call__(self, config: Any, command: str = ""): if mlflow.active_run() is not None: LOGGER.info(f"Active run found: {get_url(mlflow.active_run())}") self.log_and_launch(config=config, command=command) else: # Create run from params in config params = config.get("mlflow") or {} run_id = params.get("run_id") run_name = params.get("run_name") tracking_uri = params.get("tracking_uri") experiment_name = params.get("experiment_name") artifact_location = params.get("artifact_location") # Setup experiment and general MlFlow parameters if tracking_uri is not None: mlflow.set_tracking_uri(tracking_uri) if experiment_name is not None: if mlflow.get_experiment_by_name(experiment_name) is None: mlflow.create_experiment(experiment_name, artifact_location=artifact_location) mlflow.set_experiment(experiment_name) # Start MlFlow run, log information and launch with mlflow.start_run(run_id=run_id, run_name=run_name) as run: LOGGER.info(f"Started run: {get_url(run)}") config = fromconfig.utils.merge_dict(config, {"mlflow": {"run_id": run.info.run_id}}) self.log_and_launch(config=config, command=command)
def _save(self, data: Any) -> None: self._init_dataset() self._dataset.save(data) if self._cache: self._cache.save(data) if find_spec("mlflow"): import mlflow if self.saving_tracking_uri: mlflow.set_tracking_uri(self.saving_tracking_uri) if self.saving_run_id: mlflow.start_run(run_id=self.saving_run_id) elif self.saving_experiment_name: experiment_id = mlflow.get_experiment_by_name( self.saving_experiment_name).experiment_id mlflow.start_run(run_id=self.saving_run_id, experiment_id=experiment_id) if self.dataset in {"p"}: mlflow_log_params({self.dataset_name: data}) elif self.dataset in {"m"}: mlflow_log_metrics({self.dataset_name: data}) else: mlflow_log_artifacts([self.filepath]) if self.saving_run_id or self.saving_experiment_name: mlflow.end_run()
def main(cfg: DictConfig) -> None: # set up mlflow experiment id mlflow.set_tracking_uri(f"file://{to_absolute_path(cfg.path_to_mlflow)}") experiment = mlflow.get_experiment_by_name(cfg.experiment_name) if experiment is not None: # fetch existing experiment id run_kwargs = {'experiment_id': experiment.experiment_id} else: # create new experiment experiment_id = mlflow.create_experiment(cfg.experiment_name) run_kwargs = {'experiment_id': experiment_id} # run the training with mlflow tracking with mlflow.start_run(**run_kwargs) as active_run: setup_gpu(cfg.gpu_cfg) training_cfg = OmegaConf.to_object(cfg.training_cfg) # convert to python dictionary scaling_cfg = to_absolute_path(cfg.scaling_cfg) dataloader = DataLoaderReco.DataLoader(training_cfg, scaling_cfg) dl_config = dataloader.config model = model = MyGNN(dl_config) input_shape, _ = dataloader.get_shape() # print(input_shape[0]) # compile_build = tf.ones(input_shape[0], dtype=tf.float32, name=None) model.build(list(input_shape[0])) compile_model(model, dl_config["SetupNN"]["mode"], dl_config["SetupNN"]["learning_rate"]) fit_hist = run_training(model, dataloader, False, cfg.log_suffix) mlflow.log_dict(training_cfg, 'input_cfg/training_cfg.yaml') mlflow.log_artifact(scaling_cfg, 'input_cfg') mlflow.log_artifact(to_absolute_path("Training_SNNv0.py"), 'input_cfg') mlflow.log_artifact(to_absolute_path("../commonReco.py"), 'input_cfg') mlflow.log_artifacts('.hydra', 'input_cfg/hydra') mlflow.log_artifact('Training_SNNv0.log', 'input_cfg/hydra') mlflow.log_param('run_id', active_run.info.run_id) print(f'\nTraining has finished! Corresponding MLflow experiment name (ID): {cfg.experiment_name}({run_kwargs["experiment_id"]}), and run ID: {active_run.info.run_id}\n')
def __init__(self, context): # Context associated attributes self.log = context.log self.run_name = context.pipeline_run.pipeline_name self.dagster_run_id = context.run_id # resource config attributes resource_config = context.resource_config self.tracking_uri = resource_config.get("mlflow_tracking_uri") if self.tracking_uri: mlflow.set_tracking_uri(self.tracking_uri) self.parent_run_id = resource_config.get("parent_run_id") self.experiment_name = resource_config["experiment_name"] self.env_tags_to_log = resource_config.get("env_to_tag") or [] self.extra_tags = resource_config.get("extra_tags") # Update env variables if any are given self.env_vars = resource_config.get("env", {}) if self.env_vars: environ.update(self.env_vars) # If the experiment exists then the set won't do anything mlflow.set_experiment(self.experiment_name) self.experiment = mlflow.get_experiment_by_name(self.experiment_name) # Get the client object self.tracking_client = mlflow.tracking.MlflowClient() # Set up the active run and tags self._setup()
def set_experiment_id(self, name): """ Get experiment_id using self.exp_folder """ if name is None: experiment_id = None else: # Get experiment_id from already created experiment name -> 'NLP' try: exp_id = mlflow.get_experiment_by_name(name).experiment_id # Inputs: # name = the experiment's name # Returns: Experiment object # .experiment_id = Integer - value wanted from object # Create new experiment and get experiment_id except AttributeError: # Error: if experiment doesn't exist ... exp_id = mlflow.create_experiment(name) # Inputs: # name = the experiment's name (unique) # artifact_location (optional) = location to store run artifacts # Returns: integer Id of the experiment return exp_id
def test_get_experiment_by_name(): with TempDir(chdr=True): name = "Random experiment %d" % random.randint(1, 1e6) exp_id = mlflow.create_experiment(name) experiment = mlflow.get_experiment_by_name(name) assert experiment.experiment_id == exp_id
def Experiment_Id(self): print("Attaining 'experiment_id' ...") experiment_name = self.experiment_name if experiment_name == None: experiment_id = None else: # Get experiment_id from already created experiment try: experiment_id = mlflow.get_experiment_by_name(experiment_name).experiment_id # Inputs: # name = the experiment's name # Returns: Experiment object # .experiment_id = Integer - value wanted from object # Create new experiment and get experiment_id except AttributeError: # Error: if experiment doesn't exist ... experiment_id = mlflow.create_experiment(experiment_name) # Inputs: # name = the experiment's name (unique) # artifact_location (optional) = location to store run artifacts # Returns: integer Id of the experiment print("New experiment created in tracking_uri filepath:\n" + str(self.tracking_uri)) return experiment_id
def __init__(self, root: str, tracking_uri: str, experiment_name: str = 'nlkda'): """ Constructor. Connects to a running MLFlow instance in which the runs of the experiments are stored. Also creates an output root directory. The directory will be created if it does not exist. :param root: str The path of the output root. :param tracking_uri: str The uri where the MLFlow instance is running. :param experiment_name: str The name of the experiment on the MLFlow instance server. """ mlflow.set_tracking_uri(uri=tracking_uri) experiment = mlflow.get_experiment_by_name(name=experiment_name) if experiment is None: experiment_id = mlflow.create_experiment(name=experiment_name) else: experiment_id = experiment.experiment_id self.experiment_id = experiment_id self.root = pathlib.Path(root) self.current_run = None
def create_experiment(experiment_name): experiment = mlflow.get_experiment_by_name(experiment_name) experiment_artifact = experiment.artifact_location if experiment else "" experiment_id = experiment.experiment_id if experiment else None blob_storage_name = environ.get("AZURE_STORAGE_ACCOUNT_NAME") blob_container_name = environ.get("AZURE_STORAGE_CONTAINER_NAME") blob_uri = ( f"wasbs://{blob_container_name}@{blob_storage_name}.blob.core.windows.net" if blob_container_name and blob_storage_name else None) if experiment is None: print( f"Experiment {experiment_name} doesn't exist, creating new with artifact_location {blob_uri}" ) experiment_id = mlflow.create_experiment(experiment_name, artifact_location=blob_uri) # artifact_location are set at the experiment level upon creation # if this changes, the experiment needs to be recreated elif experiment_artifact != blob_uri and not ( experiment_artifact.startswith("dbfs") and blob_uri is None): print( f"Experiment {experiment_name} already exist with artifact_location {experiment_artifact}" ) mlflow.delete_experiment(experiment_id) experiment_id = mlflow.create_experiment(experiment_name, artifact_location=blob_uri) print( f"Experiment {experiment_name} recreated with artifact_location {blob_uri}" ) return experiment_id
def predict_tags( text: str = "Transfer learning with BERT for self-supervised learning", run_id: str = "", ) -> Dict: """Predict tags for a give input text using a trained model. Warning: Make sure that you have a trained model first! Args: text (str, optional): Input text to predict tags for. Defaults to "Transfer learning with BERT for self-supervised learning". run_id (str, optional): ID of the run to load model artifacts from. Defaults to model with lowest `best_val_loss` from the `best` experiment. Returns: Predicted tags for input text. """ # Get best run if not run_id: experiment_id = mlflow.get_experiment_by_name("best").experiment_id all_runs = mlflow.search_runs( experiment_ids=experiment_id, order_by=["metrics.best_val_loss ASC"], ) run_id = all_runs.iloc[0].run_id # Predict prediction = predict.predict(texts=[text], run_id=run_id) logger.info(json.dumps(prediction, indent=2)) return prediction
def test_tune_exp_default_trainable( dataset, pipeline_config, trainer_config, monkeypatch ): # avoid logging to wandb monkeypatch.setenv("WANDB_MODE", "dryrun") pipeline_config["features"]["word"]["embedding_dim"] = tune.choice([2, 4]) trainer_config["optimizer"]["lr"] = tune.loguniform(0.001, 0.01) my_exp = TuneExperiment( pipeline_config=pipeline_config, trainer_config=trainer_config, train_dataset=dataset, valid_dataset=dataset, num_samples=1, ) assert my_exp._name.startswith("HPO on") assert my_exp.name == my_exp._name assert my_exp._run_identifier == "_default_trainable" analysis = tune.run(my_exp) assert len(analysis.trials) == 1 mlflow.set_tracking_uri(mlflow.get_tracking_uri()) assert mlflow.get_experiment_by_name(my_exp._name)
def get_or_create_experiment(name: str, artifact_location: Optional[str] = None) -> str: """ Get or create an experiment. Parameters ---------- name : str Experiment name artifact_location : str, default None Location to store run artifacts. If unspecified, the server picks an appropriate default. Returns ------- str ID of the experiment. Examples -------- >>> with mlflow.start_run(): ... expr_id = mlflow.get_or_create_experiment('test') """ expr = mlflow.get_experiment_by_name(name) if expr is None: return mlflow.create_experiment(name, artifact_location) return expr.experiment_id
def start_track(self, experiment_name=None, disable_run_init=False): """ Start a new run to track. :param experiment_name: The name of the mlflow experiment :param disable_run_init: Flag to indicate if the run_init functions are to be run on an already existing run. :return: """ if not tracking_active: self.activate_tracking() # check if there is already an active run run = mlflow.active_run() experiment = None if run is None: experiment_name = experiment_name or DEFAULT_EXPERIMENT_NAME # Create run if run doesn't already exist experiment = mlflow.get_experiment_by_name(experiment_name) experiment_id = experiment.experiment_id if experiment else mlflow.create_experiment( experiment_name) run = self.api.start_run(experiment_id=experiment_id) else: # if not disable_run_init: # self.api.run_setups(_pypads_env=LoggerEnv(parameter=dict(), experiment_id=experiment_id, run_id=run_id, # data={"category": "SetupFn"})) if experiment_name: # Check if we're still in the same experiment experiment = mlflow.get_experiment_by_name(experiment_name) experiment_id = experiment.experiment_id if experiment else mlflow.create_experiment( experiment_name) if run.info.experiment_id != experiment_id: experiment = mlflow.get_experiment_by_name(experiment_name) if experiment is None: experiment = self.backend.get_experiment(run.info.experiment_id) # override active run if used if experiment_name and run.info.experiment_id != experiment.experiment_id: logger.warning( "Active experiment_id of run doesn't match given input name " + experiment_name + ". Recreating new run.") try: self.api.start_run(experiment_id=experiment.experiment_id, nested=True) except Exception: mlflow.end_run() self.api.start_run(experiment_id=experiment.experiment_id) return self
def setup_mlflow(cfg, features_cfg, data_cfg): import mlflow experiment_id = OmegaConf.select(cfg, "experiment_id", default=None) if experiment_id is None and cfg.experiment_name is not None: mlflow.set_experiment(cfg.experiment_name) experiment = mlflow.get_experiment_by_name(cfg.experiment_name) if cfg.experiment_name is not None: logger.debug(f"MLFlow Experiment: {cfg.experiment_name}") experiment_id = experiment.experiment_id orig_cwd = get_original_cwd() tracking_uri = f"file://{orig_cwd}/mlruns" mlflow.set_tracking_uri(tracking_uri) logger.info(f"MLFlow Tracking URI: {tracking_uri}") # if cfg.model == "xgboost": # import mlflow.xgboost # logger.debug("Turning on MLFlow autologging for XGBoost...") # mlflow.xgboost.autolog() run = mlflow.start_run(experiment_id=experiment_id) # tracking_uri = mlflow.get_tracking_uri() # logger.info(f"MLFlow Tracking URI: {tracking_uri}") processed_data_dir = Path(to_absolute_path(data_cfg.hydra.run.dir)) if processed_data_dir is not None: data_hydra_dir = processed_data_dir / ".hydra" mlflow.log_artifacts(data_hydra_dir, artifact_path="processed_data_configs") data_cfg = OmegaConf.load(data_hydra_dir / "config.yaml") for name, param_name in DATA_CONFIGS_TO_LOG.items(): param = OmegaConf.select(data_cfg, param_name) if param is not None: if isinstance(param, list): param = ", ".join([str(p) for p in param]) mlflow.log_param(name, param) model_hydra_dir = Path(".hydra") mlflow.log_artifacts(model_hydra_dir, artifact_path="model_configs") mlflow.log_params({ "model": cfg.model, "lag": features_cfg.lag, "exog_lag": features_cfg.exog_lag, "lead": features_cfg.lead, "cv_method": cfg.cv.method, }) mlflow.log_params(cfg.cv.params) tags = OmegaConf.select(cfg, "tags", default={}) if bool(tags): mlflow.set_tags(tags) return run
def setExperiment(): experiment_path = "/Shared/MLflow_Experiments/Modeler_2_Experiments/" experiment_name = "Financial Fraud Experiment" mlflow.set_experiment(experiment_path + experiment_name) experiment = mlflow.get_experiment_by_name(experiment_path + experiment_name) return experiment
def getLatestRunId(pathToExperiment): import mlflow from pyspark.sql.functions import col expId = mlflow.get_experiment_by_name(pathToExperiment).experiment_id lastest_run_id = spark.read.format("mlflow-experiment").load( expId).orderBy( col("end_time").desc()).select("run_id").limit(1).collect()[0][0] return lastest_run_id
def setOrCeateMLFlowExperiment(experimentPath,mlflow_model_save_dir): from mlflow.exceptions import MlflowException try: experiment_id = mlflow.create_experiment(experimentPath, "dbfs:" + mlflow_model_save_dir) except MlflowException: # if experiment is already created experiment_id = mlflow.get_experiment_by_name(experimentPath).experiment_id mlflow.set_experiment(experimentPath) return experiment_id
def setExperiment(): experiment_path = "/Shared/MLflow_Experiments/Modeler_1_Experiments/" experiment_name = "Wine Quality Experiment" mlflow.set_experiment(experiment_path + experiment_name) experiment = mlflow.get_experiment_by_name(experiment_path + experiment_name) return experiment