def __init__(self, url, experiment): self.url = url self.experiment = experiment mlflow.set_tracking_uri(self.url) mlflow.set_experiment(self.experiment)
import pandas as pd import mlflow import mlflow.sklearn from sklearn.model_selection import train_test_split from sklearn.ensemble import RandomForestRegressor mlflow.set_tracking_uri("http://localhost:5000") mlflow.set_experiment("kyle-test") df = pd.read_csv("kc_house_data.csv") # choose features features = ["bedrooms","bathrooms","sqft_living","sqft_above","grade", "floors","view",'sqft_lot','floors','waterfront','zipcode'] # getting those features from the dataframe x = df[features] y = df["price"] # splits data into 80% train 20% test x_train, x_test, y_train, y_test = train_test_split(x, y, train_size=0.7, random_state=3) # choose model with settings model = RandomForestRegressor(n_estimators=100) model.fit(x_train, y_train) # define and print metrics = {"train_score": model.score(x_train, y_train), "test_score": model.score(x_test, y_test)} print(metrics)
import mlflow from cls.rfr_model import RFRModel from cls.utils import Utils if __name__ == "__main__": # Use sqlite:///mlruns.db as the local store for tracking and registery mlflow.set_tracking_uri("sqlite:///mlruns.db") # load and print dataset csv_path = "data/windfarm_data.csv" wind_farm_data = Utils.load_data(csv_path, index_col=0) Utils.print_pandas_dataset(wind_farm_data) # Get Validation data X_train, y_train = Utils.get_training_data(wind_farm_data) val_x, val_y = Utils.get_validation_data(wind_farm_data) # train, fit and register our model params_list = [ {"n_estimators": 100}, {"n_estimators": 200}, {"n_estimators": 300}] # Iterate over few different tuning parameters model_name = "SKLearnWeatherForestModel" for params in params_list: rfr = RFRModel.new_instance(params) print("Using paramerts={}".format(params)) runID = rfr.mlflow_run(X_train, y_train, val_x, val_y, model_name) print("MLflow run_id={} completed with MSE={} and RMSE={}".format(runID, rfr.mse, rfr.rsme))
def run_experiments(*, data_file_path: str = None, ground_truth_path: str = None, train_size: int, val_size: float = 0.1, sub_test_size: int, channels_idx: int = 0, neighborhood_size: int = None, save_data: bool = False, n_runs: int = 1, dest_path: str, models_path: str, model_name: str, n_classes: int, use_ensemble: bool = False, ensemble_copies: int = None, voting: str = 'mean', voting_model: str = None, voting_model_params: str = None, batch_size: int = 256, noise_params: str = None, endmembers_path: str = None, use_mlflow: bool = False, experiment_name: str = None, model_exp_name: str = None, run_name: str = None): """ Function for running the inference for the unmixing problem given a set of hyperparameters. :param data_file_path: Path to the data file. It should be a numpy array. :param ground_truth_path: Path to the ground-truth data file. It should be a numpy array. :param train_size: If float, should be between 0.0 and 1.0, if int, it represents number of samples to draw from data. :param val_size: Should be between 0.0 and 1.0. Represents the percentage of samples to extract from the training set. :param sub_test_size: Number of pixels to subsample the test set instead of performing the inference on the entire subset. :param channels_idx: Index specifying the channels position in the provided data. :param neighborhood_size: Size of the spatial patch. :param save_data: Boolean indicating whether to save the prepared dataset. :param n_runs: Number of total experiment runs. :param dest_path: Path to the directory where all experiment runs will be saved as subdirectories. :param models_path: Path to the directory where the previously trained models are stored. :param model_name: Name of the model, it serves as a key in the dictionary holding all functions returning models. :param n_classes: Number of classes. :param use_ensemble: Boolean indicating whether to use the ensemble functionality for prediction. :param ensemble_copies: Number of model copies for the ensemble. :param voting: Method of ensemble voting. If 'booster', employs a new model, which is trained on the ensemble predictions on the training set. Else if 'mean', averages the predictions of all models, without any weights. :param voting_model: Type of the model to use when the voting argument is set to 'booster'. This indicates, that a new model is trained on the ensemble's predictions on the learning set, to leverage the quality of the regression. Supported models are: SVR (support vector machine for regression), RFR (random forest for regression) and DTR (decision tree for regression). :param voting_model_params: Parameters of the voting model. Used only when the type of voting is set to 'booster'. Should be specified analogously to the noise injection parameters in the 'noise' module. :param batch_size: Size of the batch used in training phase, it is the number of samples to utilize per single gradient step. :param noise_params: Parameters for the noise when creating copies of the base model. Those can be for instance the mean, or standard deviation of the noise. For the details see the 'noise' module. Exemplary value for this parameter is "{"mean": 0, "std": 1}". :param endmembers_path: Path to the endmembers file containing the average reflectances for each class. Used only when 'use_unmixing' is set to True. :param use_mlflow: Boolean indicating whether to log metrics and artifacts to mlflow. :param experiment_name: Name of the experiment. Used only if 'use_mlflow' is set to True. :param model_exp_name: Name of the experiment. Used only if 'use_mlflow' is set to True. :param run_name: Name of the run. Used only if 'use_mlflow' is set to True. """ if use_mlflow: args = locals() mlflow.set_tracking_uri("http://beetle.mlflow.kplabs.pl") mlflow.set_experiment(experiment_name) mlflow.start_run(run_name=run_name) log_params_to_mlflow(args) log_tags_to_mlflow(args['run_name']) models_path = get_mlflow_artifacts_path(models_path, model_exp_name) for experiment_id in range(n_runs): experiment_dest_path = os.path.join( dest_path, 'experiment_' + str(experiment_id)) model_name_regex = re.compile('unmixing_.*') model_dir = os.path.join(models_path, f'experiment_{experiment_id}') model_name = list(filter(model_name_regex.match, os.listdir(model_dir)))[0] model_path = os.path.join(model_dir, model_name) os.makedirs(experiment_dest_path, exist_ok=True) data_source = prepare_data.main(data_file_path=data_file_path, ground_truth_path=ground_truth_path, train_size=train_size, val_size=val_size, stratified=False, background_label=-1, channels_idx=channels_idx, neighborhood_size=neighborhood_size, save_data=save_data, seed=experiment_id, use_unmixing=True) if sub_test_size is not None: subsample_test_set(data_source[enums.Dataset.TEST], sub_test_size) evaluate_unmixing.evaluate( model_path=model_path, data=data_source, dest_path=experiment_dest_path, use_ensemble=use_ensemble, ensemble_copies=ensemble_copies, endmembers_path=endmembers_path, voting=voting, voting_model=voting_model, noise_params=noise_params, batch_size=batch_size, seed=experiment_id, neighborhood_size=neighborhood_size, voting_model_params=voting_model_params) tf.keras.backend.clear_session() artifacts_reporter.collect_artifacts_report(experiments_path=dest_path, dest_path=dest_path, use_mlflow=use_mlflow) if use_mlflow: mlflow.set_experiment(experiment_name) mlflow.log_artifacts(dest_path, artifact_path=dest_path) shutil.rmtree(dest_path)
import mlflow import shutil import inspect import collections from conf import Config from pathlib import Path from src.data_connectors import PandasFileConnector mlflow.set_tracking_uri( Config.MLFLOW["TRACKING_URI"]) # Setting location to save models mlflow.set_experiment(Config.MLFLOW["EXPERIMENT_NAME"]) class MLFlowLogger: @classmethod def log(cls, post_process_output): with mlflow.start_run(): cls.__log_config() cls.__log_opt_model(post_process_output) @classmethod def __log_opt_model(cls, post_process_output): artifact_folder = Config.MLFLOW['TEMP_ARTIFACT_DIR'] Path(artifact_folder).mkdir(parents=True, exist_ok=True) # Solver Results post_process_output.solver_results.results.write(filename=str( Path(artifact_folder, 'solver_results.json')), format='json')
# In[26]: yPredict = gnb.predict(dfToPredict) print('La classe predite est : ', yPredict) # # Integration de MLFlow # In[27]: import mlflow import mlflow.sklearn # In[28]: #mlflow.set_experiment(experiment_name='Examen_A57') mlflow.set_tracking_uri("http://benmassaoud.com:5000") # In[29]: with mlflow.start_run(): mlflow.log_metric("recall_score_test", recall_score_test) mlflow.log_metric("f1_score_test", f1_score_test) mlflow.log_metric("accuracy_test", accuracy_test) mlflow.sklearn.log_model(gnb, "model") # # Export des metriques # In[32]: with open("metrics.txt", 'w') as outfile:
def prepare_mlflow(params): mlflow.set_tracking_uri(params['tracking_uri']) mlflow.set_experiment(params['experiment'])
def __enter__(self): if USE_MLFLOW: mlflow.set_tracking_uri(uri=MLFLOW_TRACKING_URL) mlflow.set_experiment(TENANT) mlflow.start_run(run_name=RUN_LABEL) return self
from datetime import date import mlflow import pandas as pd from sklearn.metrics import mean_squared_error from sklearn.model_selection import train_test_split from xgboost import XGBRegressor mlflow_settings = dict( username="******", password="******", host="127.0.0.1", port=5000, ) mlflow.set_tracking_uri( "http://{username}:{password}@{host}:{port}".format(**mlflow_settings)) current_date = date.today() experiment_id = mlflow.set_experiment("Web Traffic Forecast") def prepare_data(df): df["ds"] = pd.to_datetime(df["ds"]) df['weekday'] = df['ds'].apply(lambda x: x.weekday()) df['year'] = df.ds.dt.year df['month'] = df.ds.dt.month df['day'] = df.ds.dt.day X = df.set_index("ds").drop(columns=["y"], errors="ignore") return X
def http_tracking_uri_mock(): mlflow.set_tracking_uri("http://some-cool-uri") yield mlflow.set_tracking_uri(None)
return [s] def predict(self, context, model_input): model_input[['name']] = model_input.apply(self.summarize_article) return model_input # Input and Output formats input = json.dumps([{'name': 'text', 'type': 'string'}]) output = json.dumps([{'name': 'text', 'type': 'string'}]) # Load model from spec signature = ModelSignature.from_dict({'inputs': input, 'outputs': output}) #MLFlow Operations mlflow.set_tracking_uri("") tracking_uri = mlflow.get_tracking_uri() print("Current tracking uri: {}".format(tracking_uri)) # Start tracking with mlflow.start_run(run_name="hf_summarizer") as run: print(run.info.run_id) runner = run.info.run_id print("mlflow models serve -m runs:/" + run.info.run_id + "/model --no-conda") mlflow.pyfunc.log_model('model', loader_module=None, data_path=None, code_path=None, conda_env=None, python_model=Summarizer(),
def mlflow_client(self): mlflow.set_tracking_uri(MLFLOW_URI) return MlflowClient()
if __name__ == '__main__': parser = ArgumentParser(description="Training of Sentence VAE") parser.add_argument("--config", type=str, required=True, metavar='PATH', help="Path to a configuration file.") parser.add_argument("--hyper-parameters", type=str, metavar='PATH', help="Path to a hyper parameters file.") parser.add_argument("--run-dir", type=str, required=True, metavar='PATH', help="Path to a directory where model checkpoints will be stored.") parser.add_argument("--force", action='store_true', help="Whether to rewrite data if run directory already exists.") parser.add_argument("--experiment-name", type=str, metavar="ID", help="Name of experiment if training process is run under mlflow") parser.add_argument("--verbose", action='store_true', help="Verbosity of the training script.") args = parser.parse_args() if args.experiment_name is not None: if args.hyper_parameters is None: raise ValueError("You should provide hyper-parameters file to log into mlflow.") with open(args.hyper_parameters) as fp: h_params = json.load(fp) mlflow.set_tracking_uri(args.run_dir) mlflow_client = MlflowClient(args.run_dir) experiment_id = get_experiment_id(mlflow_client, args.experiment_name) tags = get_git_tags(Path.cwd()) run_experiment(h_params, args.config, mlflow_client, experiment_id, tags=tags, verbose=args.verbose) else: params = json.loads(evaluate_file(args.config)) train(args.run_dir, params, args.force, verbose=args.verbose)
def test_start_run(monkeypatch): _reset_experiment() with tempfile.TemporaryDirectory() as tmpdir: mlf.set_tracking_uri(f'file:{tmpdir}/foo') # no run should be active initially assert mlf.active_run() is None # test default args with uv.start_run() as r: active_run = mlf.active_run() assert active_run is not None assert active_run == r # test explicit experiment name, run name, artifact location cfg = { 'experiment_name': 'experiment_0', 'run_name': 'bar', 'artifact_location': '/foo/bar', } with uv.start_run(**cfg) as r: active_run = mlf.active_run() assert active_run is not None assert active_run == r assert r.data.tags['mlflow.runName'] == cfg['run_name'] assert mlf.get_experiment_by_name(cfg['experiment_name']) is not None assert mlf.get_artifact_uri().startswith(cfg['artifact_location']) # test env var experiment name, run name, path-based artifact location cfg = { 'MLFLOW_EXPERIMENT_NAME': 'env_foo', 'MLFLOW_RUN_NAME': 'env_bar', 'MLFLOW_ARTIFACT_ROOT': '/tmp/foo/bar' } for k, v in cfg.items(): monkeypatch.setenv(k, v) with uv.start_run() as r: active_run = mlf.active_run() assert active_run is not None assert active_run == r assert r.data.tags['mlflow.runName'] == cfg['MLFLOW_RUN_NAME'] assert mlf.get_experiment_by_name( cfg['MLFLOW_EXPERIMENT_NAME']) is not None assert mlf.get_artifact_uri().startswith(cfg['MLFLOW_ARTIFACT_ROOT']) for k, v in cfg.items(): monkeypatch.delenv(k) # test env var tags cfg = { 'tag0': 'foo', 'tag1': 'bar', } for k, v in cfg.items(): monkeypatch.setenv(f'ENVVAR_{k}', v) with uv.start_run() as r: client = mlf.tracking.MlflowClient() tags = client.get_run(r.info.run_id).data.tags for k, v in cfg.items(): assert k in tags, pp.pformat(tags) assert tags[k] == v, pp.pformat(tags) for k in cfg: monkeypatch.delenv(f'ENVVAR_{k}') # test CAIP tags monkeypatch.setenv('CLOUD_ML_JOB_ID', 'foo_cloud_job') with uv.start_run() as r: client = mlf.tracking.MlflowClient() tags = client.get_run(r.info.run_id).data.tags assert 'cloud_ml_job_details' in tags, pp.pformat(tags) assert tags['cloud_ml_job_details'] == ( 'https://console.cloud.google.com/ai-platform/jobs/foo_cloud_job') assert 'cloud_ml_job_id' in tags, pp.pformat(tags) assert tags['cloud_ml_job_id'] == 'foo_cloud_job' monkeypatch.delenv('CLOUD_ML_JOB_ID') # test case where no gcp project is set with gcs artifact store def mock_default(scopes=None, request=None, quota_project_id=None): return (google.auth.credentials.AnonymousCredentials(), None) monkeypatch.setattr('google.auth.default', mock_default) cfg = { 'experiment_name': 'experiment_1', 'run_name': 'bar', 'artifact_location': 'gs://foo/bar', } with uv.start_run(**cfg) as r: active_run = mlf.active_run() assert active_run is not None assert active_run == r assert r.data.tags['mlflow.runName'] == cfg['run_name'] assert mlf.get_experiment_by_name(cfg['experiment_name']) is not None assert mlf.get_artifact_uri().startswith( cfg['artifact_location']), mlf.get_artifact_uri() assert os.environ.get('GOOGLE_CLOUD_PROJECT') is not None # test case where gcp project is set with gcs artifact storage def mock_default(scopes=None, request=None, quota_project_id=None): return (google.auth.credentials.AnonymousCredentials(), 'test_project') monkeypatch.setattr('google.auth.default', mock_default) cfg = { 'experiment_name': 'experiment_2', 'run_name': 'bar', 'artifact_location': 'gs://foo/bar', } with uv.start_run(**cfg) as r: active_run = mlf.active_run() assert active_run is not None assert active_run == r assert r.data.tags['mlflow.runName'] == cfg['run_name'] assert mlf.get_experiment_by_name(cfg['experiment_name']) is not None assert mlf.get_artifact_uri().startswith( cfg['artifact_location']), mlf.get_artifact_uri() # test using existing experiment with different artifact location # - this should use original artifact location cfg = { 'experiment_name': 'experiment_2', 'run_name': 'bar2', 'artifact_location': '/a/b/c', } with uv.start_run(**cfg) as r: active_run = mlf.active_run() assert active_run is not None assert active_run == r assert r.data.tags['mlflow.runName'] == cfg['run_name'] assert mlf.get_experiment_by_name(cfg['experiment_name']) is not None assert not mlf.get_artifact_uri().startswith( cfg['artifact_location']), mlf.get_artifact_uri()
def main(args): def do_eda(args): show_ner_datainfo(ner_labels, train_data_generator, args.train_file, test_data_generator, args.test_file) def do_submit(args): generate_submission(args) if args.do_eda: do_eda(args) elif args.do_submit: do_submit(args) elif args.to_train_poplar: from theta.modeling import to_train_poplar to_train_poplar(args, train_data_generator, ner_labels=ner_labels, ner_connections=[], start_page=args.start_page, max_pages=args.max_pages) elif args.to_reviews_poplar: from theta.modeling import to_reviews_poplar to_reviews_poplar(args, ner_labels=ner_labels, ner_connections=[], start_page=args.start_page, max_pages=args.max_pages) else: # -------------------- Model -------------------- if args.ner_type == 'span': from theta.modeling.ner_span import NerTrainer else: from theta.modeling.ner import NerTrainer class AppTrainer(NerTrainer): def __init__(self, args, ner_labels): super(AppTrainer, self).__init__(args, ner_labels, build_model=None) # def on_predict_end(self, args, test_dataset): # super(Trainer, self).on_predict_end(args, test_dataset) trainer = AppTrainer(args, ner_labels) def do_train(args): train_examples, val_examples = load_train_val_examples(args) trainer.train(args, train_examples, val_examples) def do_eval(args): args.model_path = args.best_model_path _, eval_examples = load_train_val_examples(args) model = load_model(args) trainer.evaluate(args, model, eval_examples) def do_predict(args): args.model_path = args.best_model_path test_examples = load_test_examples(args) model = load_model(args) trainer.predict(args, model, test_examples) reviews_file, category_mentions_file = save_ner_preds( args, trainer.pred_results, test_examples) return reviews_file, category_mentions_file if args.do_train: do_train(args) elif args.do_eval: do_eval(args) elif args.do_predict: do_predict(args) elif args.do_experiment: if args.tracking_uri: mlflow.set_tracking_uri(args.tracking_uri) mlflow.set_experiment(args.experiment_name) with mlflow.start_run(run_name=f"{args.local_id}") as mlrun: log_global_params(args, experiment_params) # ----- Train ----- do_train(args) # ----- Predict ----- do_predict(args) # ----- Submit ----- do_submit(args)
from sklearn.model_selection import train_test_split from sklearn.linear_model import ElasticNet import mlflow import mlflow.sklearn def eval_metrics(actual, pred): rmse = np.sqrt(mean_squared_error(actual, pred)) mae = mean_absolute_error(actual, pred) r2 = r2_score(actual, pred) return rmse, mae, r2 if __name__ == "__main__": mlflow.set_tracking_uri("http://mlflow.bayescluster.com") warnings.filterwarnings("ignore") np.random.seed(40) # Read the wine-quality csv file (make sure you're running this from the root of MLflow!) wine_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "wine-quality.csv") data = pd.read_csv(wine_path) # Split the data into training and test sets. (0.75, 0.25) split. train, test = train_test_split(data) # The predicted column is "quality" which is a scalar from [3, 9] train_x = train.drop(["quality"], axis=1) test_x = test.drop(["quality"], axis=1) train_y = train[["quality"]]
from pathlib import Path from azureml.core import Workspace # get workspace ws = Workspace.from_config() # get root of git repo prefix = Path(__file__).parent.parent.parent.absolute() # project settings project_uri = prefix.joinpath("mlprojects", "sklearn-diabetes") # azure ml settings experiment_name = "sklearn-diabetes-mlproject-example" compute_name = "cpu-cluster" # setup mlflow tracking mlflow.set_tracking_uri(ws.get_mlflow_tracking_uri()) mlflow.set_experiment(experiment_name) # setup backend config backend_config = {"COMPUTE": compute_name} # run mlflow project run = mlflow.projects.run( uri=str(project_uri), parameters={"alpha": 0.3}, backend="azureml", backend_config=backend_config, )
def format_layers(config): formatted_input_layer = [format_layer("input", config['input'])] formatted_hidden_layers = reduce(format_hidden_layer, config['stacks'], { 'index': 1, 'stacks': [] }) formatted_output_layer = [format_layer("output", config['output'])] return formatted_input_layer + formatted_hidden_layers[ 'stacks'] + formatted_output_layer if __name__ == '__main__': LOG.info('Start connecting to mlFlow instance') mlflow.set_tracking_uri(os.environ['MLFLOW_TRACKING_URI']) mlflow.set_experiment(EXPERIMENT_NAME) mlflow.start_run(run_name=JOB) LOG.info('Done connecting to mlFlow instance') try: LOG.info('Start loading datasets') LOG.info('Start downloading datasets') datalake = os.environ['DATALAKE'].replace('s3://', '') mlflow.log_param('input', os.environ['DATALAKE'] + '/pinkman/') datasets.download_s3_folder(datalake, 'pinkman/dictionary.csv', 'dictionary') datasets.download_s3_folder(datalake, 'pinkman/test.csv', './test') datasets.download_s3_folder(datalake, 'pinkman/train.csv', './train') LOG.info('Done downloading datasets')
import os import json import amphora_client import mlflow import time from datetime import datetime from src.mapping import water_save, water_load from src.sites import site_info from src.signals import signals from src.upload_signals import create_or_update_amphorae, upload_signals_to_amphora ## Set up log metrics start = time.time() sep = '_' mlflow.set_tracking_uri( "http://aci-mlflow-dns.australiaeast.azurecontainer.io:5000/") runName = sep.join(['Job_at', str(datetime.utcnow())]) mlflow.start_run(experiment_id=1, run_name=runName) mlflow.log_metric("time_to_complete", 0) mlflow.log_metric("sites_analysed", 0) mlflow.log_metric("run_complete", 0) sites = site_info() water_locations = dict() location_infos = dict() # check we have all the amphora we need for key, value in sites.items(): store = dict() location_info = dict() code = key
def search_plasticc(sim_sn_path, training_cosmos_path, test_cosmos_path, model_dir, batch_size, optimizer, adabound_gamma, adabound_final_lr, lr, seed, epochs, patience, n_trials, norm, flux_err, input1, input2, mixup, threads, eval_frequency, binary, mixup_alpha, mixup_beta): storage = 'sqlite:///{}/example.db'.format(model_dir) if not os.path.exists(model_dir): os.makedirs(model_dir) if platform.system() == 'Windows': tmp = (Path(__file__).parents[1] / 'mlruns' / 'search-plasticc-classification' / 'mlruns') uri = str(tmp.absolute().as_uri()) # uri = 'file://' + str(tmp.absolute()) else: tmp = (Path(__file__).parents[1] / 'mlruns' / 'search-plasticc-classification' / 'mlruns') uri = str(tmp.absolute().as_uri()) mlflow.set_tracking_uri(uri) n_classes = 2 if binary else 3 name = '{n_classes}-{input1}-{input2}'.format( n_classes=n_classes, input1=input1, input2=input2 ) mlflow.set_experiment(name) db_path = os.path.join(model_dir, 'example.db') sampler = MyTPESampler() if os.path.exists(db_path): study = optuna.Study(study_name='study190513', storage=storage, sampler=sampler) else: study = optuna.create_study(study_name='study190513', storage=storage, sampler=sampler) input_setting = InputSetting( batch_size=batch_size, mixup=mixup, mixup_alpha=mixup_alpha, mixup_beta=mixup_beta, balance=False ) input_data = InputData( training_data=None, validation_data=None, test_data=None, mean=None, std=None, input1=input1, input2=input2, remove_y=False, is_hsc=False, n_classes=n_classes, input_setting=input_setting ) optimizer_setting = OptimizerSetting( name=optimizer, lr=lr, gamma=adabound_gamma, final_lr=adabound_final_lr ) loop_setting = LoopSetting(epochs=epochs, patience=patience, eval_frequency=eval_frequency, end_by_epochs=False) print('loading data') # 今までのflux_errなら1, 新しいflux_errなら2 sim_sn, training_cosmos, _ = load_plasticc_data( sim_sn_path=sim_sn_path, training_cosmos_path=training_cosmos_path, test_cosmos_path=test_cosmos_path, use_flux_err2=flux_err == 2 ) sim_sn = sklearn.utils.shuffle(sim_sn, random_state=seed) training_cosmos = sklearn.utils.shuffle(training_cosmos, random_state=seed + 1) for data in (sim_sn, training_cosmos): for key in ('flux', 'flux_err'): tmp = data[key] data[key][np.isnan(tmp)] = 0 # クラスラベルを数字にする label_map = get_label_map(binary=binary) sim_sn_y = np.array([label_map[c] for c in sim_sn['sn_type']]) training_cosmos_y = np.array([label_map[c] for c in training_cosmos['sn_type']]) sim_x1, sim_x2, sim_y1, sim_y2 = train_test_split( sim_sn, sim_sn_y, test_size=0.3, random_state=42, stratify=sim_sn_y ) cosmos_x1, cosmos_x2, cosmos_y1, cosmos_y2 = train_test_split( training_cosmos, training_cosmos_y, test_size=0.3, random_state=43, stratify=training_cosmos_y ) sim_dev_x, sim_val_x, sim_dev_y, sim_val_y = train_test_split( sim_x1, sim_y1, test_size=0.3, random_state=44, stratify=sim_y1 ) cosmos_dev_x, cosmos_val_x, cosmos_dev_y, cosmos_val_y = train_test_split( cosmos_x1, cosmos_y1, test_size=0.3, random_state=45, stratify=cosmos_y1 ) weight = np.asarray([0.9 / len(sim_dev_y)] * len(sim_dev_y) + [0.1 / len(cosmos_dev_y)] * len(cosmos_dev_y)) training_data = Data(x=np.hstack([sim_dev_x, cosmos_dev_x]), y=np.hstack([sim_dev_y, cosmos_dev_y]), weight=weight) validation_data = Data(x=np.hstack([sim_val_x, cosmos_val_x]), y=np.hstack([sim_val_y, cosmos_val_y])) test_data = Data(x=np.hstack([sim_x2, cosmos_x2]), y=np.hstack([sim_y2, cosmos_y2])) input_data.training_data = training_data input_data.validation_data = validation_data input_data.test_data = test_data mean, std = compute_moments( train_data=training_data.x, input1=input1, input2=input2, norm=norm, use_redshift=False, is_hsc=False, threads=threads ) input_data.mean, input_data.std = mean, std for i in range(n_trials): study.optimize( lambda trial: objective_plasticc( trial=trial, input_data=input_data, optimizer_setting=optimizer_setting, seed=seed, loop_setting=loop_setting, normalization=norm, threads=threads, binary=binary, sim_sn_path=sim_sn_path, training_cosmos_path=training_cosmos_path, flux_err=flux_err ), n_trials=1 ) df = study.trials_dataframe() df.to_csv(os.path.join(model_dir, 'result.csv'))
import mlflow from mlflow.tracking._tracking_service import utils import os if __name__ == "__main__": mlflow.set_tracking_uri('databricks') # Note: get_host_creds will be undefined if not logging to a remote tracking server, e.g. if logging to the local filesystem host_creds = utils._get_store().get_host_creds() token = host_creds.token host = host_creds.host print(host, token, os.getgid())
def search_hsc(sim_sn_path, hsc_path, model_dir, batch_size, optimizer, adabound_gamma, adabound_final_lr, lr, seed, epochs, patience, n_trials, norm, input1, input2, mixup, threads, eval_frequency, binary, task_name, remove_y, mixup_alpha, mixup_beta): storage = 'sqlite:///{}/example.db'.format(model_dir) if not os.path.exists(model_dir): os.makedirs(model_dir) if platform.system() == 'Windows': tmp = (Path(__file__).parents[1] / 'mlruns' / 'search-hsc-classification' / 'mlruns') uri = str(tmp.absolute().as_uri()) # uri = 'file://' + str(tmp.absolute()) else: tmp = (Path(__file__).absolute().parents[1] / 'mlruns' / 'search-hsc-classification' / 'mlruns') uri = str(tmp.absolute().as_uri()) mlflow.set_tracking_uri(uri) mlflow.set_tracking_uri(uri) n_classes = 2 if binary else 3 name = '{n_classes}-{task_name}-{input1}-{input2}'.format( n_classes=n_classes, task_name=task_name, input1=input1, input2=input2 ) if remove_y: name += '-remove-y' mlflow.set_experiment(name) print(model_dir) db_path = os.path.join(model_dir, 'example.db') sampler = MyTPESampler() if os.path.exists(db_path): study = optuna.Study(study_name='study190513', storage=storage, sampler=sampler) else: study = optuna.create_study(study_name='study190513', storage=storage, sampler=sampler) input_setting = InputSetting( batch_size=batch_size, mixup=mixup, mixup_alpha=mixup_alpha, mixup_beta=mixup_beta ) input_data = InputData( training_data=None, validation_data=None, test_data=None, mean=None, std=None, input1=input1, input2=input2, remove_y=remove_y, is_hsc=True, n_classes=n_classes, input_setting=input_setting ) optimizer_setting = OptimizerSetting( name=optimizer, lr=lr, gamma=adabound_gamma, final_lr=adabound_final_lr ) loop_setting = LoopSetting(epochs=epochs, patience=patience, eval_frequency=eval_frequency, end_by_epochs=False) print('loading data') sim_sn, _ = load_hsc_data( sim_sn_path=sim_sn_path, hsc_path=hsc_path, remove_y=input_data.remove_y ) sim_sn = sklearn.utils.shuffle(sim_sn, random_state=seed) # クラスラベルを数字にする label_map = get_label_map(binary=binary) sim_sn_y = np.array([label_map[c] for c in sim_sn['sn_type']]) sim_x1, sim_x2, sim_y1, sim_y2 = train_test_split( sim_sn, sim_sn_y, test_size=0.3, random_state=42, stratify=sim_sn_y ) sim_dev_x, sim_val_x, sim_dev_y, sim_val_y = train_test_split( sim_x1, sim_y1, test_size=0.3, random_state=44, stratify=sim_y1 ) training_data = Data(x=sim_dev_x, y=sim_dev_y) validation_data = Data(x=sim_val_x, y=sim_val_y) test_data = Data(x=sim_x2, y=sim_y2) input_data.training_data = training_data input_data.validation_data = validation_data input_data.test_data = test_data mean, std = compute_moments( train_data=training_data.x, input1=input1, input2=input2, norm=norm, use_redshift=False, is_hsc=True, threads=threads ) input_data.mean, input_data.std = mean, std for i in range(n_trials): study.optimize( lambda trial: objective_hsc( trial=trial, sim_sn_path=sim_sn_path, hsc_path=hsc_path, optimizer_setting=optimizer_setting, seed=seed, loop_setting=loop_setting, normalization=norm, threads=threads, binary=binary, input_data=input_data ), n_trials=1 ) df = study.trials_dataframe() df.to_csv(os.path.join(model_dir, 'result.csv'))
def run_experiments(*, data_file_path: str, ground_truth_path: str = None, train_size: ('train_size', multi(min=0)), val_size: float = 0.1, stratified: bool = True, background_label: int = 0, channels_idx: int = 0, n_runs: int, model_name: str, kernel_size: int = 3, n_kernels: int = 16, save_data: bool = 0, n_layers: int = 1, dest_path: str = None, sample_size: int, n_classes: int, lr: float = 0.005, batch_size: int = 150, epochs: int = 10, verbose: int = 2, shuffle: bool = True, patience: int = 3, pre_noise: ('pre', multi(min=0)), pre_noise_sets: ('spre', multi(min=0)), post_noise: ('post', multi(min=0)), post_noise_sets: ('spost', multi(min=0)), noise_params: str = None, use_mlflow: bool = False, experiment_name: str = None, run_name: str = None): """ Function for running experiments given a set of hyper parameters. :param data_file_path: Path to the data file. Supported types are: .npy :param ground_truth_path: Path to the ground-truth data file. :param train_size: If float, should be between 0.0 and 1.0, if stratified = True, it represents percentage of each class to be extracted, If float and stratified = False, it represents percentage of the whole dataset to be extracted with samples drawn randomly, regardless of their class. If int and stratified = True, it represents number of samples to be drawn from each class. If int and stratified = False, it represents overall number of samples to be drawn regardless of their class, randomly. Defaults to 0.8 :param val_size: Should be between 0.0 and 1.0. Represents the percentage of each class from the training set to be extracted as a validation set, defaults to 0.1 :param stratified: Indicated whether the extracted training set should be stratified, defaults to True :param background_label: Label indicating the background in GT file :param channels_idx: Index specifying the channels position in the provided data :param save_data: Whether to save the prepared dataset :param n_runs: Number of total experiment runs. :param model_name: Name of the model, it serves as a key in the dictionary holding all functions returning models. :param kernel_size: Size of ech kernel in each layer. :param n_kernels: Number of kernels in each layer. :param n_layers: Number of layers in the model. :param dest_path: Path to where all experiment runs will be saved as subfolders in this directory. :param sample_size: Size of the input sample. :param n_classes: Number of classes. :param lr: Learning rate for the model, i.e., regulates the size of the step in the gradient descent process. :param batch_size: Size of the batch used in training phase, it is the size of samples per gradient step. :param epochs: Number of epochs for model to train. :param verbose: Verbosity mode used in training, (0, 1 or 2). :param shuffle: Boolean indicating whether to shuffle dataset dataset_key each epoch. :param patience: Number of epochs without improvement in order to stop the training phase. :param pre_noise: The list of names of noise injection methods before the normalization transformations. Exemplary names are "gaussian" or "impulsive". :param pre_noise_sets: The list of sets to which the noise will be injected. One element can either be "train", "val" or "test". :param post_noise: The list of names of noise injection methods after the normalization transformations. :param post_noise_sets: The list of sets to which the noise will be injected. :param noise_params: JSON containing the parameter setting of injection methods. Exemplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}". This JSON should include all parameters for noise injection functions that are specified in pre_noise and post_noise arguments. For the accurate description of each parameter, please refer to the ml_intuition/data/noise.py module. :param use_mlflow: Whether to log metrics and artifacts to mlflow. :param experiment_name: Name of the experiment. Used only if use_mlflow = True :param run_name: Name of the run. Used only if use_mlflow = True. """ train_size = parse_train_size(train_size) if use_mlflow: args = locals() mlflow.set_tracking_uri("http://beetle.mlflow.kplabs.pl") mlflow.set_experiment(experiment_name) mlflow.start_run(run_name=run_name) log_params_to_mlflow(args) log_tags_to_mlflow(args['run_name']) if dest_path is None: dest_path = os.path.join(os.path.curdir, "temp_artifacts") for experiment_id in range(n_runs): experiment_dest_path = os.path.join( dest_path, '{}_{}'.format(enums.Experiment.EXPERIMENT, str(experiment_id))) if save_data: data_source = os.path.join(experiment_dest_path, 'data.h5') else: data_source = None os.makedirs(experiment_dest_path, exist_ok=True) if data_file_path.endswith('.h5') and ground_truth_path is None: data = load_processed_h5(data_file_path=data_file_path) else: data = prepare_data.main(data_file_path=data_file_path, ground_truth_path=ground_truth_path, output_path=data_source, train_size=train_size, val_size=val_size, stratified=stratified, background_label=background_label, channels_idx=channels_idx, save_data=save_data, seed=experiment_id) if not save_data: data_source = data if len(pre_noise) > 0: noise.inject_noise(data_source=data_source, affected_subsets=pre_noise_sets, noise_injectors=pre_noise, noise_params=noise_params) train_model.train(model_name=model_name, kernel_size=kernel_size, n_kernels=n_kernels, n_layers=n_layers, dest_path=experiment_dest_path, data=data_source, sample_size=sample_size, n_classes=n_classes, lr=lr, batch_size=batch_size, epochs=epochs, verbose=verbose, shuffle=shuffle, patience=patience, noise=post_noise, noise_sets=pre_noise_sets, noise_params=noise_params) evaluate_model.evaluate(model_path=os.path.join( experiment_dest_path, model_name), data=data_source, dest_path=experiment_dest_path, n_classes=n_classes, batch_size=batch_size, noise=post_noise, noise_sets=pre_noise_sets, noise_params=noise_params) tf.keras.backend.clear_session() artifacts_reporter.collect_artifacts_report(experiments_path=dest_path, dest_path=dest_path, use_mlflow=use_mlflow) if enums.Splits.GRIDS in data_file_path: fair_report_path = os.path.join(dest_path, enums.Experiment.REPORT_FAIR) artifacts_reporter.collect_artifacts_report( experiments_path=dest_path, dest_path=fair_report_path, filename=enums.Experiment.INFERENCE_FAIR_METRICS, use_mlflow=use_mlflow) if use_mlflow: mlflow.log_artifacts(dest_path, artifact_path=dest_path) shutil.rmtree(dest_path)
def log_cv(self, experiment, name, tracking_uri=None): """Logging of cross validation results to mlflow tracking server Args: experiment (str): experiment ID name (str): Name of the experiment artifact (prefix) tracking_uri (str, optional): URI of the tracking server. Defaults to None, which will use remote tracking in remote case """ cv_results = self.results.cv_results_ best = self.results.best_index_ timestamp = datetime.datetime.now().isoformat().split(".")[0].replace( ":", ".") num_runs = len(cv_results["rank_test_score"]) run_name = "run %d (best run of %d):" % (self.results.best_index_, num_runs) if tracking_uri: mlflow.set_tracking_uri(tracking_uri) mlflow.set_experiment(experiment) with mlflow.start_run(run_name=run_name): # as run: mlflow.log_param("folds", self.results.cv) print("Logging parameters") params = list(self.results.param_grid.keys()) for param in params: mlflow.log_param(param, cv_results["param_%s" % param][best]) print("Logging metrics") mlflow.log_metric("mean_test_score", cv_results["mean_test_score"][best]) mlflow.log_metric("std_test_score", cv_results["std_test_score"][best]) print("Logging model") mlflow.sklearn.log_model(self.results.best_estimator_, "model") print("Logging CV results matrix") tempdir = tempfile.TemporaryDirectory().name os.mkdir(tempdir) filename = "%s-%s-cv_results.csv" % (name, timestamp) csv = os.path.join(tempdir, filename) with warnings.catch_warnings(): warnings.simplefilter("ignore") pd.DataFrame(cv_results).sort_values( by="rank_test_score").to_csv(csv, index=False) mlflow.log_artifact(csv, "cv_results") client = MlflowClient() experiment_id = client.get_experiment_by_name(experiment).experiment_id if is_remote(): if os.environ.get("DBJL_ORG", None) is None: display( HTML( "<a href=%s/#mlflow/experiments/%s>Goto experiment</a>" % (os.environ["DBJL_HOST"], experiment_id))) else: display( HTML( "<a href=%s?o=%s#mlflow/experiments/%s>Goto experiment</a>" % (os.environ["DBJL_HOST"], os.environ["DBJL_ORG"], experiment_id))) else: display( HTML("<a href=%s/#/experiments/%s>Goto experiment</a>" % (tracking_uri, experiment_id)))
import os from random import random, randint import mlflow from mlflow import log_metric, log_param, log_artifacts tracking_uri = 'file:///root/mlflow' mlflow.set_tracking_uri(tracking_uri) experiment_name = 'hello_world' mlflow.set_experiment(experiment_name) if __name__ == "__main__": print("Running mlflow_tracking.py") log_param("hyperparam1", randint(0, 100)) log_metric("accuracy", random()) log_metric("accuracy", random() + 1) log_metric("accuracy", random() + 2) if not os.path.exists("outputs"): os.makedirs("outputs") with open("outputs/model.txt", "w") as f: f.write("hello world!") log_artifacts("outputs")
def setUp(self): TestCaseWithReset.setUp(self) TestCaseWithTempDir.setUp(self) if "MLFLOW_TRACKING_URI" in os.environ: del os.environ["MLFLOW_TRACKING_URI"] mlflow.set_tracking_uri(None)
parser.add_argument("dir", help="Directory") parser.add_argument("pipeline", help="Pipeline Name") parser.add_argument("--cluster-id", help="Cluster ID") parser.add_argument("--new-cluster", help="Create new cluster", action="store_true") args = parser.parse_args() print(args) if not isdir(args.dir): print('Please specify existing directory with pipelines! ', args.dir, ' directory does not exist.') sys.exit(-100) if not isdir(args.dir+'/'+args.pipeline): print('Please specify existing pipeline name as --pipeline-name ',args.dir+'/'+args.pipeline,' drectory does not exist.') sys.exit(-100) if args.cluster_id and args.new_cluster: print('create_cluster parameter is set to True and cluster_id is specified. Exiting...') sys.exit(-100) if not (args.cluster_id) and not (args.new_cluster): print('create_cluster parameter is set to False and cluster_id is not specified. Exiting...') sys.exit(-100) import mlflow mlflow.set_tracking_uri("databricks") from setuptools import sandbox sandbox.run_setup('setup.py', ['clean', 'bdist_wheel']) from databrickslabs_cicdtemplates import cluster_and_libraries cluster_and_libraries.main(args.dir, args.pipeline, args.cluster_id)
import mlflow import json import os from elasticsearch import Elasticsearch from pipeline.util import get_or_create_experiment_id from pipeline.util import MlflowReporter from pyformance.registry import MetricsRegistry from pyformance.reporters.influx import InfluxReporter mlflow.set_tracking_uri("http://127.0.0.1:5000") def fetch_docs(base_path, es_host, es_index, es_query=None, limit=-1): exp_name = "fetch_docs" exp_path = f"{base_path}/{exp_name}" os.makedirs(exp_path, exist_ok=True) run = mlflow.start_run(experiment_id=get_or_create_experiment_id(exp_name)) docs_path = f"{exp_path}/{run.run_info.run_uuid}" registry = MetricsRegistry() mlflow_reporter = MlflowReporter(registry=registry, active_run=run, reporting_interval=10) mlflow_reporter.start() influx_reporter = InfluxReporter(registry=registry, reporting_interval=10, autocreate_database=True) influx_reporter.start()
def test_mlflow_hook_save_pipeline_ml_with_copy_mode( kedro_project_with_mlflow_conf, dummy_pipeline_ml, dummy_catalog, dummy_run_params, copy_mode, expected, ): # config_with_base_mlflow_conf is a conftest fixture bootstrap_project(kedro_project_with_mlflow_conf) with KedroSession.create( project_path=kedro_project_with_mlflow_conf) as session: context = session.load_context() mlflow_hook = MlflowHook() runner = SequentialRunner() mlflow_hook.after_context_created(context) mlflow_hook.after_catalog_created( catalog=dummy_catalog, # `after_catalog_created` is not using any of arguments bellow, # so we are setting them to empty values. conf_catalog={}, conf_creds={}, feed_dict={}, save_version="", load_versions="", ) pipeline_to_run = pipeline_ml_factory( training=dummy_pipeline_ml.training, inference=dummy_pipeline_ml.inference, input_name=dummy_pipeline_ml.input_name, log_model_kwargs={ "artifact_path": dummy_pipeline_ml.log_model_kwargs["artifact_path"], "conda_env": { "python": "3.7.0", "dependencies": ["kedro==0.16.5"] }, }, kpm_kwargs={ "copy_mode": copy_mode, }, ) mlflow_hook.before_pipeline_run(run_params=dummy_run_params, pipeline=pipeline_to_run, catalog=dummy_catalog) runner.run(pipeline_to_run, dummy_catalog, session._hook_manager) run_id = mlflow.active_run().info.run_id mlflow_hook.after_pipeline_run(run_params=dummy_run_params, pipeline=pipeline_to_run, catalog=dummy_catalog) mlflow_tracking_uri = (kedro_project_with_mlflow_conf / "mlruns").as_uri() mlflow.set_tracking_uri(mlflow_tracking_uri) loaded_model = mlflow.pyfunc.load_model( model_uri=f"runs:/{run_id}/model") actual_copy_mode = { name: ds._copy_mode for name, ds in loaded_model._model_impl.python_model. loaded_catalog._data_sets.items() } assert actual_copy_mode == expected
import mlflow import logging from cortex.main import run from src.models.fix_match.controller import FixMatchController from src.data.dataset_plugins import SSLDatasetPlugin from src import MLFLOW_SSL_URI logger = logging.getLogger('ssl_evaluation') if __name__ == '__main__': # if exp.ARGS mlflow.set_tracking_uri(MLFLOW_SSL_URI) controller = FixMatchController() run(model=controller)