def register_model(model_id, host_name, did_pass_acceptance_test): logger = logging.getLogger(__name__) mlflow.set_tracking_uri(uri=host_name) logger.info(f"Reporting data for model {model_id}") file_names = get_model_files(model_id) specification = get_json(file_names['model_specification']) mlflow.set_experiment(specification['problem_name']) with mlflow.start_run(run_name=model_id): log_param("ProblemName", specification['problem_name']) log_param("MLPipelineParamsName", specification['ml_pipeline_params_name']) log_param("FeatureSetName", specification['feature_set_name']) log_param("AlgorithmName", specification['algorithm_name']) log_param("AlgorithmParamsName", specification['algorithm_params_name']) set_tag("DidPassAcceptanceTest", did_pass_acceptance_test) set_tag("BuildNumber", os.getenv('BUILD_NUMBER')) log_model_metrics_file(file_names["model_metrics"]) log_ml_pipeline_params_file(file_names["ml_pipeline_params"]) log_artifacts(file_names['results_folder'])
def get_last_model_subdir(): # use empty string to get the root model_results_dir file_names = get_model_files('') results_dir = file_names['results_folder'] subdirs = get_subdirs(results_dir) if len(subdirs) == 0: return None latest_subdir = max(subdirs, key=os.path.getmtime) return latest_subdir
def get_model_id_location(model_id_arg): logger = logging.getLogger(__name__) if model_id_arg is None: latest_subdir = get_last_model_subdir() if latest_subdir is None: results_dir = get_model_files('')['results_folder'] raise IOError(f'No models found in {results_dir}') model_id = latest_subdir.split('/')[-1] logger.info(f"Using model_id {model_id}") else: model_id = model_id_arg return model_id
def test_acceptance_with_no_arguments(tmp_path): os.environ["CD4ML_DATA_DIR"] = str(tmp_path) files = get_model_files('', base_data_dir=tmp_path) base_results_directory = files['results_folder'] earlier_folder = Path(base_results_directory, "earlier") earlier_folder.mkdir(parents=True) os.utime(earlier_folder, (earlier_time, earlier_time)) latest_folder = Path(base_results_directory, "later") latest_folder.mkdir(parents=True) os.utime(earlier_folder, (later_time, later_time)) model_id = acceptance_script.parse_arguments([]) del os.environ["CD4ML_DATA_DIR"] assert model_id == 'later'
def test_tracking_dictionaries_only(self, tmp_path): specification = {'problem_name': "foo_problem"} os.environ["CD4ML_DATA_DIR"] = str(tmp_path) tracking = Track("unit-test-id", specification) tracking.log_param("my_param", 1) tracking.log_metrics({"my_metric": 2}) tracking.save_results() filenames = get_model_files('unit-test-id', base_data_dir=tmp_path) model_dir = filenames['results_folder'] files = os.listdir(model_dir) assert set(files) == {"model_metrics.json", "ml_pipeline_params.json", "model_specification.json"} metrics_json = get_json(filenames['model_metrics']) assert metrics_json["my_metric"] == 2 params_json = get_json(filenames['ml_pipeline_params']) assert params_json["my_param"] == 1
def is_model_accepted(model_id): model_files = get_model_files(model_id) params = get_json(model_files['ml_pipeline_params']) metric_name = params['acceptance_metric'].replace("'", "") threshold_min = float(params['acceptance_threshold_min']) threshold_max = float(params['acceptance_threshold_max']) metrics = get_json(model_files['model_metrics']) metric_value = metrics[metric_name] accepted = threshold_min <= metric_value <= threshold_max message = get_message(model_id, metric_name, metric_value, threshold_min, threshold_max, accepted) return accepted, message
def save_results(self): filenames = get_model_files(self.model_id) self.logger.info("Recording run information for model %s" % self.model_id) if self.model is not None: self.model.save(filenames['full_model']) if self.plot is not None: import bokeh.plotting as bokeh_saver bokeh_saver.save(obj=self.plot, filename=filenames['validation_plot'], title='Validation Plot') self._write_dictionary_to_file(self.params, filenames['ml_pipeline_params']) self._write_dictionary_to_file(self.metrics, filenames['model_metrics']) self._write_dictionary_to_file(self.specification, filenames['model_specification'])
def get_trained_encoder(stream, ml_fields, problem_name, write=True, read_from_file=False, base_features_omitted=None): file_names = get_model_files(problem_name) encoder_file = file_names.get('encoder') if encoder_file and os.path.exists(encoder_file) and read_from_file: logger.info('Reading encoder from : {}'.format(encoder_file)) logger.info(base_features_omitted) encoder_from_file = OneHotEncoder([], [], omit_cols=base_features_omitted) encoder_from_file.load_from_file(encoder_file) return encoder_from_file logger.info('Building encoder') encoder = get_encoder_from_stream(stream, ml_fields, omit_cols=base_features_omitted) if write: if encoder_file: logger.info('Writing encoder to: %s' % encoder_file) encoder.save(encoder_file) return encoder
def test_writing_bokeh_plot(self, tmp_path): from bokeh.plotting import figure, output_file from bokeh.sampledata.iris import flowers specification = {'problem_name': "foo_problem"} colormap = {'setosa': 'red', 'versicolor': 'green', 'virginica': 'blue'} colors = [colormap[x] for x in flowers['species']] output_file(NamedTemporaryFile().name) p = figure(title="Iris Morphology") p.xaxis.axis_label = 'Petal Length' p.yaxis.axis_label = 'Petal Width' p.circle(flowers["petal_length"], flowers["petal_width"], color=colors, fill_alpha=0.2, size=10) os.environ["CD4ML_DATA_DIR"] = str(tmp_path) tracking = Track("unit-test-id", specification) tracking.log_validation_plot(p) tracking.save_results() filenames = get_model_files('unit-test-id', base_data_dir=tmp_path) model_dir = filenames['results_folder'] files = os.listdir(model_dir) assert set(files) == {"validation_plot.html", "model_specification.json"}