def main(*, listen:('l', parameters.multi(min=1, max=3))): """Listens on the given addresses :param listen: An address to listen on. """ for address in listen: print('Listening on {0}'.format(address))
def run_evaluation( *, model_hash: str, thr: float = 0.5, dpath: str = "datasets/clouds/Landsat-Cloud-Cover-Assessment-" + "Validation-Data-Partial", rpath: str = "artifacts/", vids: ("v", multi(min=1)), batch_size: int = 32, bands: ("b", multi(min=1)), bands_names: ("bn", multi(min=1)), img_ids: ("iid", multi(min=0)), resize: bool = False, normalize: bool = False, standardize: bool = False, mlflow: bool = False, run_name: str = None, ): """ Load model given model hash and get evaluation metrics on L8CCA testset. :param model_hash: MLFlow hash of the model to load. :param thr: threshold to be used during evaluation. :param dpath: path to dataset. :param rpath: path to directory where results and artifacts should be logged. :param vids: tuple of ids of images which should be used to create visualisations. If contains '*' visualisations will be created for all images in the dataset. :type vids: tuple[str] :param batch_size: size of generated batches, only one batch is loaded to memory at a time. :param bands: band numbers to load :type bands: list[int] :param bands_names: names of the bands to load. Should have the same number of elements as bands. :type bands_names: list[str] :param img_ids: if given, process only these images. :type img_ids: list[int] :param resize: whether to resize loaded img to gt. :param normalize: whether to normalize the image. :param standardize: whether to standardize the image. :param mlflow: whether to use MLFlow. :param run_name: name of the run. """ snow_imgs = ["LC82271192014287LGN00", "LC81321192014054LGN00"] if img_ids == []: img_ids = None else: snow_imgs = list(set(snow_imgs) & set(img_ids)) dpath, rpath = make_paths(dpath, rpath) rpath = rpath / uuid.uuid4().hex print(f"Working dir: {os.getcwd()}, " + f"artifacts dir: {rpath}", flush=True) mpath = Path(f"/media/ML/mlflow/beetle/artifacts/34/{model_hash}/" # Change init_model to model for old models + "artifacts/init_model/data/model.h5") # WARNING: If ever upgraded to tf2.x, this way of using metrics will not work # because loaded metrics will become MeanMetricWrapper objects in tf2.x and # this script isn't prepared for such objects (because MeanMetricWrapper has state, # as opposed to present stateless metric functions). model = keras.models.load_model( mpath, custom_objects={ "jaccard_index_loss": losses.JaccardIndexLoss(), "jaccard_index_metric": losses.JaccardIndexMetric(), "dice_coeff_metric": losses.DiceCoefMetric(), "recall": losses.recall, "precision": losses.precision, "specificity": losses.specificity, # F1 score is needed for old models # "f1_score": losses.f1_score, "tf": tf, }, ) model.load_weights(f"/media/ML/mlflow/beetle/artifacts/34/{model_hash}/" + "artifacts/best_weights/best_weights") metrics, _ = evaluate_model( model=model, thr=thr, dpath=dpath, rpath=rpath, vids=vids, batch_size=batch_size, bands=bands, bands_names=bands_names, img_ids=img_ids, resize=resize, normalize=normalize, standardize=standardize, mlflow=mlflow, run_name=run_name, ) mean_metrics = {} mean_metrics_snow = {} for key, value in metrics.items(): mean_metrics[key] = np.mean(list(value.values())) mean_metrics_snow[f"snow_{key}"] = np.mean( [value[x] for x in snow_imgs]) print(mean_metrics, mean_metrics_snow) if mlflow: log_metrics(mean_metrics) log_metrics(mean_metrics_snow) log_artifacts(rpath)
def main(*, data_file_path: str, ground_truth_path: str, output_path: str = None, train_size: ('train_size', multi(min=0)), val_size: float = 0.1, stratified: bool = True, background_label: int = 0, channels_idx: int = 0, save_data: bool = False, seed: int = 0): """ :param data_file_path: Path to the data file. Supported types are: .npy :param ground_truth_path: Path to the data file. :param output_path: Path under in which the output .h5 file will be stored. Used only if the parameter save_data is set to True :param train_size: If float, should be between 0.0 and 1.0, if stratified = True, it represents percentage of each class to be extracted, If float and stratified = False, it represents percentage of the whole dataset to be extracted with samples drawn randomly, regardless of their class. If int and stratified = True, it represents number of samples to be drawn from each class. If int and stratified = False, it represents overall number of samples to be drawn regardless of their class, randomly. Defaults to 0.8 :type train_size: float or int :param val_size: Should be between 0.0 and 1.0. Represents the percentage of each class from the training set to be extracted as a validation set, defaults to 0.1 :param stratified: Indicated whether the extracted training set should be stratified, defaults to True :param background_label: Label indicating the background in GT file :param channels_idx: Index specifying the channels position in the provided data :param save_data: Whether to save data as .md5 or to return it as a dict :param seed: Seed used for data shuffling :raises TypeError: When provided data or labels file is not supported """ train_size = utils.parse_train_size(train_size) if data_file_path.endswith('.npy') and ground_truth_path.endswith('.npy'): data, labels = io.load_npy(data_file_path, ground_truth_path) data, labels = preprocessing.reshape_cube_to_2d_samples( data, labels, channels_idx) elif data_file_path.endswith('.h5') and ground_truth_path.endswith( '.tiff'): data, gt_transform_mat = io.load_satellite_h5(data_file_path) labels = io.load_tiff(ground_truth_path) data_2d_shape = data.shape[1:] labels = preprocessing.align_ground_truth(data_2d_shape, labels, gt_transform_mat) data, labels = preprocessing.reshape_cube_to_2d_samples( data, labels, channels_idx) data, labels = preprocessing.remove_nan_samples(data, labels) else: raise ValueError( "The following data file type is not supported: {}".format( os.path.splitext(data_file_path)[EXTENSION])) data = data[labels != background_label] labels = labels[labels != background_label] labels = preprocessing.normalize_labels(labels) train_x, train_y, val_x, val_y, test_x, test_y = \ preprocessing.train_val_test_split(data, labels, train_size, val_size, stratified, seed=seed) if save_data: io.save_md5(output_path, train_x, train_y, val_x, val_y, test_x, test_y) return None else: return utils.build_data_dict(train_x, train_y, val_x, val_y, test_x, test_y)
def main_(*, config: 'c' = 'config.yml', debug: 'd' = False, extra: ('e', str, parameters.multi()) = None, optimistic = False, quiet: 'q' = False, vim_is_fucking_retarded: Parameter.UNDOCUMENTED = False): """ Run cms7. config: Path to project configuration debug: Print obnoxious debugging output extra: Path to additional configuration (e.g. site local overrides). Can be specified multiple times. Later configurations override. optimistic: Try to continue processing after rendering errors quiet: Only ever print warnings """ rl = logging.getLogger() h = logging.StreamHandler() try: if not sys.stdin.isatty(): raise Exception import colorlog h.setFormatter(colorlog.ColoredFormatter( "%(log_color)s%(levelname)-8s%(reset)s %(message_log_color)s%(name)s:%(message)s", secondary_log_colors={ 'message': { 'WARNING': 'yellow', 'ERROR': 'red', 'CRITICAL': 'red', } })) except: h.setFormatter(logging.Formatter( "%(levelname)-8s %(name)s:%(message)s")) rl.addHandler(h) if vim_is_fucking_retarded: report_error.quiet = False if debug: rl.setLevel(logging.DEBUG) elif quiet: if vim_is_fucking_retarded: rl.setLevel(logging.CRITICAL + 1) else: rl.setLevel(logging.WARNING) else: rl.setLevel(logging.INFO) try: cfg = _config.load(config, *extra) if optimistic: cfg.optimistic = True gen = Generator(cfg) for m in cfg.modules(): m.prepare() for m in cfg.modules(): m.run(gen) gen.run() for r in cfg.resources: r.run() except CMS7Error as e: logger.critical('%s', e.message, exc_info=debug) if not debug: logger.warning('exiting for exception. use --debug to get a traceback') sys.exit(1) except Exception: logger.critical('unexpected exception', exc_info=True) sys.exit(1)
def train(*, data, model_name: str, dest_path: str, sample_size: int, n_classes: int, kernel_size: int = 3, n_kernels: int = 16, n_layers: int = 1, lr: float = 0.005, batch_size: int = 150, epochs: int = 10, verbose: int = 2, shuffle: bool = True, patience: int = 3, seed: int = 0, noise: ('post', multi(min=0)), noise_sets: ('spost', multi(min=0)), noise_params: str = None): """ Function for training tensorflow models given a dataset. :param model_name: Name of the model, it serves as a key in the dictionary holding all functions returning models. :param kernel_size: Size of ech kernel in each layer. :param n_kernels: Number of kernels in each layer. :param n_layers: Number of layers in the model. :param dest_path: Path to where to save the model under the name "model_name". :param sample_size: Size of the input sample. :param n_classes: Number of classes. :param lr: Learning rate for the model, i.e., regulates the size of the step in the gradient descent process. :param data: Either path to the input data or the data dict itself. First dimension of the dataset should be the number of samples. :param batch_size: Size of the batch used in training phase, it is the size of samples per gradient step. :param epochs: Number of epochs for model to train. :param verbose: Verbosity mode used in training, (0, 1 or 2). :param shuffle: Boolean indicating whether to shuffle dataset dataset_key each epoch. :param patience: Number of epochs without improvement in order to stop the training phase. :param seed: Seed for training reproducibility. :param noise: List containing names of used noise injection methods that are performed after the normalization transformations. :type noise: list[str] :param noise_sets: List of sets that are affected by the noise injection methods. For this module single element can be either "train" or "val". :type noise_sets: list[str] :param noise_params: JSON containing the parameters setting of injection methods. Exemplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}". This JSON should include all parameters for noise injection functions that are specified in the noise argument. For the accurate description of each parameter, please refer to the ml_intuition/data/noise.py module. """ # Reproducibility tf.reset_default_graph() tf.set_random_seed(seed=seed) np.random.seed(seed=seed) if type(data) is str: train_dict = io.extract_set(data, enums.Dataset.TRAIN) val_dict = io.extract_set(data, enums.Dataset.VAL) min_, max_ = train_dict[enums.DataStats.MIN], \ train_dict[enums.DataStats.MAX] else: train_dict = data[enums.Dataset.TRAIN] val_dict = data[enums.Dataset.VAL] min_, max_ = data[enums.DataStats.MIN], \ data[enums.DataStats.MAX] transformations = [transforms.SpectralTransform(), transforms.OneHotEncode(n_classes=n_classes), transforms.MinMaxNormalize(min_=min_, max_=max_)] tr_transformations = transformations + get_noise_functions(noise, noise_params) \ if enums.Dataset.TRAIN in noise_sets else transformations val_transformations = transformations + get_noise_functions(noise, noise_params) \ if enums.Dataset.VAL in noise_sets else transformations train_dict = transforms.apply_transformations(train_dict, tr_transformations) val_dict = transforms.apply_transformations(val_dict, val_transformations) model = models.get_model(model_key=model_name, kernel_size=kernel_size, n_kernels=n_kernels, n_layers=n_layers, input_size=sample_size, n_classes=n_classes) model.summary() model.compile(tf.keras.optimizers.Adam(lr=lr), 'categorical_crossentropy', metrics=['accuracy']) time_history = time_metrics.TimeHistory() mcp_save = tf.keras.callbacks.ModelCheckpoint( os.path.join(dest_path, model_name), save_best_only=True, monitor='val_acc', mode='max') early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=patience) callbacks = [time_history, mcp_save, early_stopping] history = model.fit(x=train_dict[enums.Dataset.DATA], y=train_dict[enums.Dataset.LABELS], epochs=epochs, verbose=verbose, shuffle=shuffle, validation_data=(val_dict[enums.Dataset.DATA], val_dict[enums.Dataset.LABELS]), callbacks=callbacks, batch_size=batch_size) history.history[time_metrics.TimeHistory.__name__] = time_history.average io.save_metrics(dest_path=dest_path, file_name='training_metrics.csv', metrics=history.history) np.savetxt(os.path.join(dest_path, 'min-max.csv'), np.array([min_, max_]), delimiter=',', fmt='%f')
from sigtools import modifiers from clize import run, parameters @modifiers.kwoargs('listen') @modifiers.annotate(listen=('l', parameters.multi(min=1, max=3))) def main(listen): """Listens on the given addresses listen: An address to listen on. """ for address in listen: print('Listening on {0}'.format(address)) run(main)
def _main(*steps: use_mixin(StepList), each_line: 'l' = False, # noqa: F821 raw: 'r' = False, # noqa: F821 start: (int, 's') = 0, end: (int, 'e') = None, prelude: (multi(), 'p') = 'pass', pipe_name: Parameter.UNDOCUMENTED = PIPE_NAME, no_default_fragments: Parameter.UNDOCUMENTED = False, no_exception_handling: Parameter.UNDOCUMENTED = False, show_fragments: Parameter.UNDOCUMENTED = False, break_: Parameter.UNDOCUMENTED = False): """Run Python code. :param steps: At least one Python expression (or suite) to execute :param each_line: If specified, process lines as strings rather than all of stdin as a file :param start: Don't print before this result (zero-based) :param end: Stop after getting this result (zero-based) :param prelude: Execute this statement before running any steps. Can be specified more than once. :param raw: Don't add helper functionality to stdin """ pipe_name = sys.intern(pipe_name) spy.context = context = make_context() for stmt in prelude: exec(stmt, context, context.view()) step_src = steps steps = [] for i, code in enumerate(step_src): fragment_name = 'Fragment {}'.format(i + 1) source = code if isinstance(code, _Decorated): source = '{} {!r}'.format(code.name, code.value) code, funcseq = code.value, code.funcseq else: funcseq = () try: co, is_expr = compile_(code, filename=fragment_name) except SyntaxError as e: pretty_syntax_error(code, e) if break_: # pragma: no cover debugger() sys.exit(1) debuginfo = (fragment_name, source) ca = make_callable(co, is_expr, context, pipe_name, debuginfo) for fn in funcseq: try: ca = fn(ca, debuginfo=debuginfo) except TypeError: ca = fn(ca) steps.append(spy.fragment(ca)) index_offset = 0 if not no_default_fragments: steps.append(fragments.make_limit(start=start, end=end)) steps.append(fragments.print) if each_line: steps.insert(0, fragments.foreach) index_offset -= 1 if raw: steps.insert(0, fragments.raw_stdin) else: steps.insert(0, fragments.stdin) index_offset -= 1 chain = spy.chain(steps, index_offset=index_offset) data = [None] if show_fragments: print(chain.format()) return with ExitStack() as stack: if not no_exception_handling: stack.enter_context(catcher.handler(delete_all=True)) if break_: # pragma: no cover stack.enter_context(DebuggerContext()) chain.run_to_exhaustion(data)
def run_experiments(*, data_file_path: str, ground_truth_path: str = None, train_size: ('train_size', multi(min=0)), val_size: float = 0.1, stratified: bool = True, background_label: int = 0, channels_idx: int = 0, neighborhood_size: int = None, n_runs: int, model_name: str, kernel_size: int = 5, n_kernels: int = 200, save_data: bool = 0, n_layers: int = 1, dest_path: str = None, sample_size: int, n_classes: int, lr: float = 0.001, batch_size: int = 128, epochs: int = 200, verbose: int = 2, shuffle: bool = True, patience: int = 15, pre_noise: ('pre', multi(min=0)), pre_noise_sets: ('spre', multi(min=0)), post_noise: ('post', multi(min=0)), post_noise_sets: ('spost', multi(min=0)), noise_params: str = None, use_mlflow: bool = False, experiment_name: str = None, run_name: str = None): """ Function for running experiments given a set of hyper parameters. :param data_file_path: Path to the data file. Supported types are: .npy. :param ground_truth_path: Path to the ground-truth data file. :param train_size: If float, should be between 0.0 and 1.0. If stratified = True, it represents percentage of each class to be extracted. If float and stratified = False, it represents percentage of the whole dataset to be extracted with samples drawn randomly, regardless of their class. If int and stratified = True, it represents number of samples to be drawn from each class. If int and stratified = False, it represents overall number of samples to be drawn regardless of their class, randomly. Defaults to 0.8. :type train_size: Union[int, float] :param val_size: Should be between 0.0 and 1.0. Represents the percentage of each class from the training set to be extracted as a validation set. Defaults to 0.1. :param stratified: Indicated whether the extracted training set should be stratified. Defaults to True. :param background_label: Label indicating the background in GT file. :param channels_idx: Index specifying the channels position in the provided data. :param neighborhood_size: Size of the spatial patch. :param save_data: Whether to save the prepared dataset. :param n_runs: Number of total experiment runs. :param model_name: Name of the model, it serves as a key in the dictionary holding all functions returning models. :param kernel_size: Size of ech kernel in each layer. :param n_kernels: Number of kernels in each layer. :param n_layers: Number of layers in the model. :param dest_path: Path to where all experiment runs will be saved as subfolders in this directory. :param sample_size: Size of the input sample. :param n_classes: Number of classes. :param lr: Learning rate for the model, i.e., regulates the size of the step in the gradient descent process. :param batch_size: Size of the batch used in training phase, it is the size of samples per gradient step. :param epochs: Number of epochs for model to train. :param verbose: Verbosity mode used in training, (0, 1 or 2). :param shuffle: Boolean indicating whether to shuffle dataset dataset_key each epoch. :param patience: Number of epochs without improvement in order to stop the training phase. :param pre_noise: The list of names of noise injection methods before the normalization transformations. Exemplary names are "gaussian" or "impulsive". :type pre_noise: list[str] :param pre_noise_sets: The list of sets to which the noise will be injected. One element can either be "train", "val" or "test". :type pre_noise_sets: list[str] :param post_noise: The list of names of noise injection methods after the normalization transformations. :type post_noise: list[str] :param post_noise_sets: The list of sets to which the noise will be injected. :type post_noise_sets: list[str] :param noise_params: JSON containing the parameter setting of injection methods. Exemplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}". This JSON should include all parameters for noise injection functions that are specified in pre_noise and post_noise arguments. For the accurate description of each parameter, please refer to the ml_intuition/data/noise.py module. :param use_mlflow: Whether to log metrics and artifacts to mlflow. :param experiment_name: Name of the experiment. Used only if use_mlflow = True :param run_name: Name of the run. Used only if use_mlflow = True. """ train_size = parse_train_size(train_size) if use_mlflow: args = locals() mlflow.set_tracking_uri("http://beetle.mlflow.kplabs.pl") mlflow.set_experiment(experiment_name) mlflow.start_run(run_name=run_name) log_params_to_mlflow(args) log_tags_to_mlflow(args['run_name']) if dest_path is None: dest_path = os.path.join(os.path.curdir, "temp_artifacts") for experiment_id in range(n_runs): experiment_dest_path = os.path.join( dest_path, '{}_{}'.format(enums.Experiment.EXPERIMENT, str(experiment_id))) if save_data: data_source = os.path.join(experiment_dest_path, 'data.h5') else: data_source = None os.makedirs(experiment_dest_path, exist_ok=True) if data_file_path.endswith('.h5') and ground_truth_path is None: data = load_processed_h5(data_file_path=data_file_path) else: data = prepare_data.main(data_file_path=data_file_path, ground_truth_path=ground_truth_path, output_path=data_source, train_size=train_size, val_size=val_size, stratified=stratified, background_label=background_label, channels_idx=channels_idx, save_data=save_data, seed=experiment_id) if not save_data: data_source = data if len(pre_noise) > 0: noise.inject_noise(data_source=data_source, affected_subsets=pre_noise_sets, noise_injectors=pre_noise, noise_params=noise_params) train_model.train(model_name=model_name, kernel_size=kernel_size, n_kernels=n_kernels, n_layers=n_layers, dest_path=experiment_dest_path, data=data_source, sample_size=sample_size, n_classes=n_classes, lr=lr, batch_size=batch_size, epochs=epochs, verbose=verbose, shuffle=shuffle, patience=patience, noise=post_noise, noise_sets=pre_noise_sets, noise_params=noise_params) evaluate_model.evaluate(model_path=os.path.join( experiment_dest_path, model_name), data=data_source, dest_path=experiment_dest_path, n_classes=n_classes, batch_size=batch_size, noise=post_noise, noise_sets=pre_noise_sets, noise_params=noise_params) tf.keras.backend.clear_session() artifacts_reporter.collect_artifacts_report(experiments_path=dest_path, dest_path=dest_path, use_mlflow=use_mlflow) if enums.Splits.GRIDS in data_file_path: fair_report_path = os.path.join(dest_path, enums.Experiment.REPORT_FAIR) artifacts_reporter.collect_artifacts_report( experiments_path=dest_path, dest_path=fair_report_path, filename=enums.Experiment.INFERENCE_FAIR_METRICS, use_mlflow=use_mlflow) if use_mlflow: mlflow.log_artifacts(dest_path, artifact_path=dest_path) shutil.rmtree(dest_path)
def evaluate(*, data, model_path: str, dest_path: str, n_classes: int, batch_size: int = 1024, noise: ('post', multi(min=0)), noise_sets: ('spost', multi(min=0)), noise_params: str = None): """ Function for evaluating the trained model. :param model_path: Path to the model. :param data: Either path to the input data or the data dict. :param dest_path: Directory in which to store the calculated metrics :param n_classes: Number of classes. :param batch_size: Size of the batch for inference :param noise: List containing names of used noise injection methods that are performed after the normalization transformations. :param noise_sets: List of sets that are affected by the noise injection. For this module single element can be "test". :param noise_params: JSON containing the parameters setting of noise injection methods. Exemplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}". This JSON should include all parameters for noise injection functions that are specified in the noise argument. For the accurate description of each parameter, please refer to the ml_intuition/data/noise.py module. """ if type(data) is str: test_dict = io.extract_set(data, enums.Dataset.TEST) else: test_dict = data[enums.Dataset.TEST] min_max_path = os.path.join(os.path.dirname(model_path), "min-max.csv") if os.path.exists(min_max_path): min_value, max_value = io.read_min_max(min_max_path) else: min_value, max_value = data[enums.DataStats.MIN], \ data[enums.DataStats.MAX] transformations = [transforms.SpectralTransform(), transforms.OneHotEncode(n_classes=n_classes), transforms.MinMaxNormalize(min_=min_value, max_=max_value)] transformations = transformations + get_noise_functions(noise, noise_params) \ if enums.Dataset.TEST in noise_sets else transformations test_dict = transforms.apply_transformations(test_dict, transformations) model = tf.keras.models.load_model(model_path, compile=True) predict = timeit(model.predict) y_pred, inference_time = predict(test_dict[enums.Dataset.DATA], batch_size=batch_size) y_pred = np.argmax(y_pred, axis=-1) y_true = np.argmax(test_dict[enums.Dataset.LABELS], axis=-1) model_metrics = get_model_metrics(y_true, y_pred) model_metrics['inference_time'] = [inference_time] conf_matrix = confusion_matrix(y_true, y_pred) io.save_metrics(dest_path=dest_path, file_name=enums.Experiment.INFERENCE_METRICS, metrics=model_metrics) io.save_confusion_matrix(conf_matrix, dest_path) if enums.Splits.GRIDS in model_path: if type(data) is str: train_dict = io.extract_set(data, enums.Dataset.TRAIN) labels_in_train = np.unique(train_dict[enums.Dataset.LABELS]) else: train_labels = data[enums.Dataset.TRAIN][enums.Dataset.LABELS] if train_labels.ndim > 1: train_labels = np.argmax(train_labels, axis=-1) labels_in_train = np.unique(train_labels) fair_metrics = get_fair_model_metrics(conf_matrix, labels_in_train) io.save_metrics(dest_path=dest_path, file_name=enums.Experiment.INFERENCE_FAIR_METRICS, metrics=fair_metrics)
class RepTests(object): mapped_basic = ('par:a', parameters.mapped([ ('greeting', ['hello'], 'h1'), ('parting', ['goodbye'], 'h2'), ]), 'par') mapped_default = ('par:a="greeting"', parameters.mapped([ ('greeting', ['hello'], 'h1'), ('parting', ['goodbye'], 'h2'), ]), '[par]') mapped_alternate_list = ('par:a', parameters.mapped([ ('greeting', ['hello'], 'h1'), ('parting', ['goodbye'], 'h2'), ], list_name="options"), 'par') mapped_no_list = ('par:a', parameters.mapped([ ('greeting', ['hello'], 'h1'), ('parting', ['goodbye'], 'h2'), ], list_name=None), 'par') mapped_kw = ('*, par:a', parameters.mapped([ ('greeting', ['hello'], 'h1'), ('parting', ['goodbye'], 'h2'), ], list_name=None), '--par=STR') mapped_force_icase = ('par:a', parameters.mapped([(1, ['thing'], 'h')], case_sensitive=False), 'par') mapped_force_scase = ('par:a', parameters.mapped([ (1, ['Thing'], 'h'), ], case_sensitive=True), 'par') mapped_imply_scase = ('par:a', parameters.mapped([ (1, ['thing'], 'h'), (2, ['Thing'], 'h'), ]), 'par') mapped_bad_icase = ('par:a', parameters.mapped([ (1, ['thing'], 'h'), (2, ['Thing'], 'h'), ], case_sensitive=False), 'par') oneof_basic = 'par:a', parameters.one_of('hello', 'goodbye', 'bye'), 'par' oneof_help = ('par:a', parameters.one_of(('hello', 'h1'), ('bye', 'h2')), 'par') multi_basic = '*, par:a', parameters.multi(), '[--par=STR...]' multi_req = '*, par:a', parameters.multi(1), '--par=STR...' multi_min = '*, par:a', parameters.multi(2), '--par=STR...' multi_max = '*, par:a', parameters.multi(max=2), '[--par=STR...]' multi_bound = '*, par:a', parameters.multi(min=2, max=3), '--par=STR...' multi_conv = '*, par:a', (parameters.multi(), int), '[--par=INT...]' multi_last_opt = ('*args, par:a', (parameters.multi(), Parameter.L), '[--par=STR...] [args...]') margs_basic = '*args:a', parameters.multi(), '[args...]' margs_req = '*args:a', parameters.multi(1), 'args...' margs_min = '*args:a', parameters.multi(2), 'args...' margs_max = '*args:a', parameters.multi(max=2), '[args...]' margs_bound = '*args:a', parameters.multi(min=2, max=3), 'args...' margs_conv = '*args:a', (parameters.multi(), int), '[args...]' margs_last_opt = ('*args:a, par=""', (parameters.multi(), Parameter.L), '[--par=STR] [args...]') @parameters.argument_decorator def _blank(arg): return arg + 'x' deco_blank_pos = 'par:a', _blank, 'par' deco_blank_posd = 'par:a="d"', _blank, '[par]' deco_blank_kw = '*, par:a', _blank, '--par=STR' deco_blank_kwd = '*, par:a="d"', _blank, '[--par=STR]' deco_blank_args = '*par:a', _blank, '[par...]' @parameters.argument_decorator @modifiers.kwoargs(start='kw') def _kw(arg, kw): return arg + kw deco_kw_pos = 'par:a', _kw, '--kw=STR par' deco_kw_posd = 'par:a="d"', _kw, '[--kw=STR par]' deco_kw_kw = '*, par:a', _kw, '--kw=STR --par=STR' deco_kw_kwd = '*, par:a="d"', _kw, '[--kw=STR --par=STR]' deco_kw_args = '*par:a', _kw, '[--kw=STR par...]' @parameters.argument_decorator @modifiers.autokwoargs def _kwdef(arg, kw='D'): return arg + kw deco_def_pos = 'par:a', _kwdef, '[--kw=STR] par' deco_def_posd = 'par:a="d"', _kwdef, '[[--kw=STR] par]' deco_def_kw = '*, par:a', _kwdef, '[--kw=STR] --par=STR' deco_def_kwd = '*, par:a="d"', _kwdef, '[[--kw=STR] --par=STR]' deco_def_args = '*par:a', _kwdef, '[[--kw=STR] par...]' @parameters.argument_decorator @modifiers.autokwoargs def _flag(arg, f=False): return arg + ('y' if f else 'n') deco_flag_pos = 'par:a', _flag, '[-f] par' deco_flag_posd = 'par:a="d"', _flag, '[[-f] par]' deco_flag_kw = '*, par:a', _flag, '[-f] --par=STR' deco_flag_kwd = '*, par:a="d"', _flag, '[[-f] --par=STR]' deco_flag_args = '*par:a', _flag, '[[-f] par...]' deco_flag_other = 'par:a, *, x=False', _flag, '[-x] [-f] par' @parameters.argument_decorator @modifiers.autokwoargs def _int(arg, i=0): return arg + str(i) deco_int_pos = 'par:a', _int, '[-i INT] par' deco_int_posd = 'par:a="d"', _int, '[[-i INT] par]' deco_int_kw = '*, par:a', _int, '[-i INT] --par=STR' deco_int_kwd = '*, par:a="d"', _int, '[[-i INT] --par=STR]' deco_int_args = '*par:a', _int, '[[-i INT] par...]' deco_int_other = 'par:a, *, x=False', _int, '[-x] [-i INT] par' @parameters.argument_decorator @modifiers.kwoargs(start='kw') def _all(arg, kw, flag=False, kwd='D', i=0): return arg, kw, flag, kwd, i _all_rep = '--kw=STR [--flag] [--kwd=STR] [-i INT] ' deco_all_pos = 'par:a', _all, _all_rep + 'par' deco_all_posd = 'par:a="d"', _all, '[' + _all_rep + 'par]' deco_all_kw = '*, par:a', _all, _all_rep + '--par=STR' deco_all_kwd = '*, par:a="d"', _all, '[' + _all_rep + '--par=STR]' deco_all_args = '*par:a', _all, '[' + _all_rep + 'par...]' @parameters.argument_decorator @modifiers.kwoargs(start='why') def _inner(arg, why, zed='zed'): return '(' + arg + why + zed + ')' @parameters.argument_decorator @modifiers.kwoargs(start='ab') @modifiers.annotate(cd=_inner) def _nest(arg, ab, cd='cd'): return '(' + arg + ab + cd + ')' _nest_rep = '--ab=STR [--why=STR [--zed=STR] --cd=STR] ' deco_nest_pos = 'par:a', _nest, _nest_rep + 'par' deco_nest_posd = 'par:a="d"', _nest, '[' + _nest_rep + 'par]' deco_nest_kw = '*, par:a', _nest, _nest_rep + '--par=STR' deco_nest_kwd = '*, par:a="d"', _nest, '[' + _nest_rep + '--par=STR]' deco_nest_args = '*par:a', _nest, '[' + _nest_rep + 'par...]' pn_pos = 'par:a', parameters.pass_name, '' pn_pos_first = 'par:a, other', parameters.pass_name, 'other' pn_pos_nextpicky = 'par:a, other:int', parameters.pass_name, 'other' pn_kw = '*, par:a', parameters.pass_name, ''
def run_experiments(*, input_dir: str, n_runs: int, dest_path: str, data_file_path: str = None, ground_truth_path: str = None, dataset_path: str = None, background_label: int = 0, channels_idx: int = 2, channels_count: int = 103, train_size: ('train_size', multi(min=0)), batch_size: int = 64, stratified: bool = True, gpu: bool = 0): """ Freeze model, quantize it and evaluate N times. :param input_dir: Directory with saved data and models, each in separate `experiment_n` folder. :param n_runs: Number of total experiment runs. :param dest_path: Path to where all experiment runs will be saved as sub folders in this directory. :param data_file_path: Path to the data file. Supported types are: .npy and .md5. This is optional, if the data is not already saved in the input_dir. :param ground_truth_path: Path to the data file. :param dataset_path: Path to the already extracted .h5 dataset :param background_label: Label indicating the background in GT file :param channels_idx: Index specifying the channels position in the provided data :param channels_count: Number of channels (bands) in the image. :param train_size: If float, should be between 0.0 and 1.0. If stratified = True, it represents percentage of each class to be extracted, If float and stratified = False, it represents percentage of the whole dataset to be extracted with samples drawn randomly, regardless of their class. If int and stratified = True, it represents number of samples to be drawn from each class. If int and stratified = False, it represents overall number of samples to be drawn regardless of their class, randomly. Defaults to 0.8 :type train_size: Union[int, float] :param stratified: Indicated whether the extracted training set should be stratified, defaults to True :param batch_size: Batch size :param gpu: Whether to run quantization on gpu. """ for experiment_id in range(n_runs): experiment_dest_path = os.path.join( dest_path, 'experiment_' + str(experiment_id)) model_path = os.path.join(input_dir, 'experiment_' + str(experiment_id), 'model_2d') created_dataset = False if dataset_path is None: data_path = os.path.join(input_dir, 'experiment_' + str(experiment_id), 'data.h5') created_dataset = True else: data_path = dataset_path os.makedirs(experiment_dest_path, exist_ok=True) if not os.path.exists(data_path): data_path = os.path.join(experiment_dest_path, 'data.md5') prepare_data.main(data_file_path=data_file_path, ground_truth_path=ground_truth_path, output_path=data_path, background_label=background_label, channels_idx=channels_idx, save_data=True, seed=experiment_id, train_size=train_size, stratified=stratified) freeze_model.main(model_path=model_path, output_dir=experiment_dest_path) node_names_file = os.path.join(experiment_dest_path, 'freeze_input_output_node_name.json') frozen_graph_path = os.path.join(experiment_dest_path, 'frozen_graph.pb') cmd = 'scripts/quantize.sh ' + node_names_file + ' ' \ + frozen_graph_path + ' ' + data_path + ' ' + \ '?,{},1,1'.format(channels_count) + ' ' + \ 'ml_intuition.data.input_fn.calibrate_2d_input' + ' ' + \ '128' + ' ' + experiment_dest_path + \ ' ' + str(gpu) subprocess.call(cmd, shell=True, env=os.environ.copy()) graph_path = os.path.join(experiment_dest_path, 'quantize_eval_model.pb') evaluate_graph.main(graph_path=graph_path, node_names_path=node_names_file, dataset_path=data_path, batch_size=batch_size) if created_dataset: os.remove(data_path) artifacts_reporter.collect_artifacts_report(experiments_path=dest_path, dest_path=dest_path, filename='inference_graph_metrics.csv') tf.keras.backend.clear_session()
def run_experiments(*, data_file_paths: ('d', multi(min=1)), ground_truth_path: str, train_size: float = 0.8, val_size: float = 0.1, stratified: bool = True, background_label: int = 0, channels_idx: int = 0, save_data: bool = False, n_runs: int, model_name: str, kernel_size: int = 3, n_kernels: int = 16, n_layers: int = 1, dest_path: str, sample_size: int, n_classes: int, lr: float = 0.005, batch_size: int = 150, epochs: int = 10, verbose: int = 2, shuffle: bool = True, patience: int = 3, pre_noise: ('pre', multi(min=0)), pre_noise_sets: ('spre', multi(min=0)), post_noise: ('post', multi(min=0)), post_noise_sets: ('spost', multi(min=0)), noise_params: str = None): """ Function for running experiments given a set of hyperparameters. :param data_file_paths: Paths to the data files. Supported types are: .npy and .h5 :param ground_truth_path: Path to the ground-truth data file. :param train_size: If float, should be between 0.0 and 1.0, if stratified = True, it represents percentage of each class to be extracted, If float and stratified = False, it represents percentage of the whole dataset to be extracted with samples drawn randomly, regardless of their class. If int and stratified = True, it represents number of samples to be drawn from each class. If int and stratified = False, it represents overall number of samples to be drawn regardless of their class, randomly. Defaults to 0.8 :param val_size: Should be between 0.0 and 1.0. Represents the percentage of each class from the training set to be extracted as a validation set, defaults to 0.1 :param stratified: Indicated whether the extracted training set should be stratified, defaults to True :param background_label: Label indicating the background in GT file :param channels_idx: Index specifying the channels position in the provided data :param save_data: Whether to save the prepared dataset :param n_runs: Number of total experiment runs. :param model_name: Name of the model, it serves as a key in the dictionary holding all functions returning models. :param kernel_size: Size of ech kernel in each layer. :param n_kernels: Number of kernels in each layer. :param n_layers: Number of layers in the model. :param dest_path: Path to where all experiment runs will be saved as subfolders in this directory. :param sample_size: Size of the input sample. :param n_classes: Number of classes. :param lr: Learning rate for the model, i.e., regulates the size of the step in the gradient descent process. :param batch_size: Size of the batch used in training phase, it is the size of samples per gradient step. :param epochs: Number of epochs for model to train. :param verbose: Verbosity mode used in training, (0, 1 or 2). :param shuffle: Boolean indicating whether to shuffle dataset dataset_key each epoch. :param patience: Number of epochs without improvement in order to stop the training phase. :param pre_noise: The list of names of noise injection methods before the normalization transformations. Examplary names are "gaussian" or "impulsive". :param pre_noise_sets: The list of sets to which the noise will be injected. One element can either be "train", "val" or "test". :param post_noise: The list of names of noise injection metods after the normalization transformations. :param post_noise_sets: The list of sets to which the noise will be injected. :param noise_params: JSON containing the parameter setting of injection methods. Examplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}". This JSON should include all parameters for noise injection functions that are specified in pre_noise and post_noise arguments. For the accurate description of each parameter, please refer to the ml_intuition/data/noise.py module. """ for experiment_id in range(n_runs): experiment_dest_path = os.path.join(dest_path, 'experiment_' + str(experiment_id)) if save_data: data_source = os.path.join(experiment_dest_path, 'data.h5') else: data_source = None os.makedirs(experiment_dest_path, exist_ok=True) data_to_merge = [] for data_file_path in data_file_paths: data = prepare_data.main(data_file_path=data_file_path, ground_truth_path=ground_truth_path, output_path=data_source, train_size=train_size, val_size=val_size, stratified=stratified, background_label=background_label, channels_idx=channels_idx, save_data=save_data, seed=experiment_id) del data[enums.Dataset.TEST] data_to_merge.append(data) data = utils.merge_datasets(data_to_merge) del data_to_merge if not save_data: data_source = data if len(pre_noise) > 0: noise.inject_noise(data_source=data_source, affected_subsets=pre_noise_sets, noise_injectors=pre_noise, noise_params=noise_params) train_model.train(model_name=model_name, kernel_size=kernel_size, n_kernels=n_kernels, n_layers=n_layers, dest_path=experiment_dest_path, data=data_source, sample_size=sample_size, n_classes=n_classes, lr=lr, batch_size=batch_size, epochs=epochs, verbose=verbose, shuffle=shuffle, patience=patience, noise=post_noise, noise_sets=pre_noise_sets, noise_params=noise_params) tf.keras.backend.clear_session()
def run_experiments(*, data_file_path: str, ground_truth_path: str = None, train_size: ('train_size', multi(min=0)), val_size: float = 0.1, sub_test_size: int = None, channels_idx: int = -1, neighborhood_size: int = None, n_runs: int = 1, model_name: str, save_data: bool = 0, dest_path: str = None, sample_size: int, n_classes: int, lr: float = None, batch_size: int = 256, epochs: int = 100, verbose: int = 2, shuffle: bool = True, patience: int = 15, use_mlflow: bool = False, endmembers_path: str = None, experiment_name: str = None, run_name: str = None): """ Function for running experiments on unmixing given a set of hyperparameters. :param data_file_path: Path to the data file. Supported types are: .npy. :param ground_truth_path: Path to the ground-truth data file. :param train_size: If float, should be between 0.0 and 1.0, if int, it represents number of samples to draw from data. :type train_size: Union[int, float] :param val_size: Should be between 0.0 and 1.0. Represents the percentage of samples to extract from the training set. :param sub_test_size: Number of pixels to subsample the test set instead of performing the inference on the entire subset. :param channels_idx: Index specifying the channels position in the provided data. :param neighborhood_size: Size of the spatial patch. :param save_data: Boolean indicating whether to save the prepared dataset. :param n_runs: Number of total experiment runs. :param model_name: Name of the model, it serves as a key in the dictionary holding all functions returning models. :param dest_path: Path to the directory where all experiment runs will be saved as subdirectories. :param sample_size: Spectral size of the input sample. :param n_classes: Number of classes. :param lr: Learning rate for the model i.e., it regulates the size of the step in the gradient descent process. :param batch_size: Size of the batch used in training phase, it is the number of samples to utilize per single gradient step. :param epochs: Total number of epochs for model to train. :param verbose: Verbosity mode used in training, (0, 1 or 2). :param shuffle: Boolean indicating whether to shuffle dataset. :param patience: Number of epochs without improvement in order to stop the training phase. :param use_mlflow: Boolean indicating whether to log metrics and artifacts to mlflow. :param endmembers_path: Path to the endmembers file containing the average reflectances for each class. Used only when 'use_unmixing' is set to True. :param experiment_name: Name of the experiment. Used only if 'use_mlflow' is set to True. :param run_name: Name of the run. Used only if 'use_mlflow' is set to True. """ if use_mlflow: args = locals() mlflow.set_tracking_uri("http://beetle.mlflow.kplabs.pl") mlflow.set_experiment(experiment_name) mlflow.start_run(run_name=run_name) log_params_to_mlflow(args) log_tags_to_mlflow(args['run_name']) if dest_path is None: dest_path = os.path.join(os.path.curdir, "temp_artifacts") for experiment_id in range(n_runs): experiment_dest_path = os.path.join( dest_path, '{}_{}'.format(enums.Experiment.EXPERIMENT, str(experiment_id))) os.makedirs(experiment_dest_path, exist_ok=True) # Apply default literature hyperparameters: if neighborhood_size is None and model_name in NEIGHBORHOOD_SIZES: neighborhood_size = NEIGHBORHOOD_SIZES[model_name] if lr is None and model_name in LEARNING_RATES: lr = LEARNING_RATES[model_name] data = prepare_data.main(data_file_path=data_file_path, ground_truth_path=ground_truth_path, train_size=parse_train_size(train_size), val_size=val_size, stratified=False, background_label=-1, channels_idx=channels_idx, neighborhood_size=neighborhood_size, save_data=save_data, seed=experiment_id, use_unmixing=True) if sub_test_size is not None: subsample_test_set(data[enums.Dataset.TEST], sub_test_size) train_unmixing.train(model_name=model_name, dest_path=experiment_dest_path, data=data, sample_size=sample_size, neighborhood_size=neighborhood_size, n_classes=n_classes, lr=lr, batch_size=batch_size, epochs=epochs, verbose=verbose, shuffle=shuffle, patience=patience, endmembers_path=endmembers_path, seed=experiment_id) evaluate_unmixing.evaluate( model_path=os.path.join(experiment_dest_path, model_name), data=data, dest_path=experiment_dest_path, neighborhood_size=neighborhood_size, batch_size=batch_size, endmembers_path=endmembers_path) tf.keras.backend.clear_session() artifacts_reporter.collect_artifacts_report( experiments_path=dest_path, dest_path=dest_path, use_mlflow=use_mlflow) if use_mlflow: mlflow.log_artifacts(dest_path, artifact_path=dest_path) shutil.rmtree(dest_path)
def _main( *steps: (use_mixin(StepList), Parameter.REQUIRED), each_line: 'l' = False, # noqa: F821 raw: 'r' = False, # noqa: F821 start: (int, 's') = 0, # noqa: F821 end: (int, 'e') = None, # noqa: F821 prelude: (multi(), 'p') = 'pass', # noqa: F821 pipe_name: Parameter.UNDOCUMENTED = PIPE_NAME, no_default_fragments: Parameter.UNDOCUMENTED = False, no_exception_handling: Parameter.UNDOCUMENTED = False, show_fragments: Parameter.UNDOCUMENTED = False, break_: Parameter.UNDOCUMENTED = False): """Feed data through a sequence of Python expressions. :param steps: At least one Python expression (or suite) to execute :param each_line: Process lines as strings rather than all of stdin as a file :param start: Don't print before this result (zero-based) :param end: Stop after getting this result (zero-based) :param prelude: Execute a statement before running any steps. Can be specified more than once. :param raw: Don't add helper functionality to stdin """ pipe_name = sys.intern(pipe_name) spy.context = context = make_context() for stmt in prelude: exec(stmt, context, context.view()) step_src = steps steps = [] for i, code in enumerate(step_src): fragment_name = 'Fragment {}'.format(i + 1) source = code literal = False if isinstance(code, _Decorated): source = '{} {!r}'.format(code.name, code.value) literal = code.LITERAL code, funcseq = code.value, code.funcseq else: funcseq = () debuginfo = (fragment_name, source) if literal: ca = make_literal(code, context, pipe_name, debuginfo) else: try: co, is_expr = compile_(code, filename=fragment_name) except SyntaxError as e: pretty_syntax_error(code, e) if break_: # pragma: no cover debugger() sys.exit(1) ca = make_callable(co, is_expr, context, pipe_name, debuginfo) for fn in funcseq: ca = fn(ca) steps.append(spy.fragment(ca)) index_offset = 0 if not no_default_fragments: steps.append(fragments.make_limit(start=start, end=end)) steps.append(fragments.print) if each_line: steps.insert(0, fragments.foreach) index_offset -= 1 chain = spy.chain(steps, index_offset=index_offset) if raw: data = [sys.stdin] else: data = [SpyFile(sys.stdin)] if show_fragments: print(chain.format()) return with ExitStack() as stack: if not no_exception_handling: stack.enter_context(catcher.handler(delete_all=True)) if break_: # pragma: no cover stack.enter_context(DebuggerContext()) chain.run_to_exhaustion(data)
def main_(*, config: 'c' = 'config.yml', debug: 'd' = False, quiet: 'q' = False, extra: ('e', str, parameters.multi()) = None): """ Run cms7. config: Path to project configuration extra: Path to additional configuration (e.g. site local overrides). Can be specified multiple times. Later configurations override. debug: Print obnoxious debugging output quiet: Only ever print warnings """ rl = logging.getLogger() h = logging.StreamHandler() try: import colorlog h.setFormatter(colorlog.ColoredFormatter( "%(log_color)s%(levelname)-8s%(reset)s %(message_log_color)s%(name)s:%(message)s", secondary_log_colors={ 'message': { 'WARNING': 'yellow', 'ERROR': 'red', 'CRITICAL': 'red', } })) except ImportError: h.setFormatter(logging.Formatter( "%(levelname)-8s %(name)s:%(message)s")) rl.addHandler(h) if debug: rl.setLevel(logging.DEBUG) elif quiet: rl.setLevel(logging.WARNING) else: rl.setLevel(logging.INFO) try: cfg = _config.load(config, *extra) gen = Generator(cfg) for m in cfg.modules(): m.prepare() for m in cfg.modules(): m.run(gen) gen.run() for r in cfg.resources: r.run() except CMS7Error as e: logger.critical('%s', e.message, exc_info=debug) if not debug: logger.warning('exiting for exception. use --debug to get a traceback') sys.exit(1) except Exception: logger.critical('unexpected exception', exc_info=True) sys.exit(1)
def run_evaluation( *, model_hash: str, thr: float = 0.5, dpath: str = "datasets/clouds/38-Cloud/38-Cloud_test", gtpath: str = "datasets/clouds/38-Cloud/38-Cloud_test/Entire_scene_gts", vpath: str = "datasets/clouds/38-Cloud/38-Cloud_test/Natural_False_Color", rpath: str = "artifacts/", vids: ("v", multi(min=1)), batch_size: int = 32, img_ids: ("iid", multi(min=0)), mlflow: bool = False, run_name: str = None, ): """ Load model given model hash and get evaluation metrics on 38-Cloud testset. :param model_hash: MLFlow hash of the model to load. :param thr: threshold to be used during evaluation. :param dpath: path to dataset. :param gtpath: path to dataset ground truths. :param vpath: path to dataset (false color) visualisation images. :param rpath: path to directory where results and artifacts should be logged. :param vids: tuple of ids of images which should be used to create visualisations. If contains '*' visualisations will be created for all images in the dataset. :type vids: tuple[str] :param batch_size: size of generated batches, only one batch is loaded to memory at a time. :param img_ids: if given, process only these images. :type img_ids: list[int] :param mlflow: whether to use MLFlow. :param run_name: name of the run. """ snow_imgs = [ "LC08_L1TP_064015_20160420_20170223_01_T1", "LC08_L1TP_035035_20160120_20170224_01_T1", "LC08_L1TP_050024_20160520_20170324_01_T1", ] if img_ids == []: img_ids = None else: snow_imgs = list(set(snow_imgs) & set(img_ids)) dpath, gtpath, vpath, rpath = make_paths(dpath, gtpath, vpath, rpath) rpath = rpath / uuid.uuid4().hex print(f"Working dir: {os.getcwd()}, " + f"artifacts dir: {rpath}", flush=True) mpath = Path( f"/media/ML/mlflow/beetle/artifacts/34/{model_hash}/" # Change init_model to model for old models + "artifacts/init_model/data/model.h5" ) # WARNING: If ever upgraded to tf2.x, this way of using metrics will not work # because loaded metrics will become MeanMetricWrapper objects in tf2.x and # this script isn't prepared for such objects (because MeanMetricWrapper has state, # as opposed to present stateless metric functions). model = keras.models.load_model( mpath, custom_objects={ "jaccard_index_loss": losses.JaccardIndexLoss(), "jaccard_index_metric": losses.JaccardIndexMetric(), "dice_coeff_metric": losses.DiceCoefMetric(), "recall": losses.recall, "precision": losses.precision, "specificity": losses.specificity, # F1 score is needed for old models # "f1_score": losses.f1_score, "tf": tf, }, ) model.load_weights( f"/media/ML/mlflow/beetle/artifacts/34/{model_hash}/" + "artifacts/best_weights/best_weights" ) metrics, _ = evaluate_model( model=model, thr=thr, dpath=dpath, gtpath=gtpath, vpath=vpath, rpath=rpath, vids=vids, batch_size=batch_size, img_ids=img_ids, mlflow=mlflow, run_name=run_name, ) mean_metrics = {} mean_metrics_snow = {} for key, value in metrics.items(): mean_metrics[key] = np.mean(list(value.values())) mean_metrics_snow[f"snow_{key}"] = np.mean( [value[x] for x in snow_imgs]) print(mean_metrics, mean_metrics_snow) if mlflow: log_metrics(mean_metrics) log_metrics(mean_metrics_snow) log_artifacts(rpath)
def evaluate(*, data, model_path: str, dest_path: str, n_classes: int, batch_size: int = 1024, use_ensemble: bool = False, ensemble_copies: int = 1, voting: str = 'hard', noise: ('post', multi(min=0)), noise_sets: ('spost', multi(min=0)), noise_params: str = None, seed: int = 0): """ Function for evaluating the trained model. :param model_path: Path to the model. :param data: Either path to the input data or the data dict. :param dest_path: Directory in which to store the calculated metrics. :param n_classes: Number of classes. :param batch_size: Size of the batch for inference. :param use_ensemble: Use ensemble for prediction. :param ensemble_copies: Number of model copies for the ensemble. :param voting: Method of ensemble voting. If ‘hard’, uses predicted class labels for majority rule voting. Else if ‘soft’, predicts the class label based on the argmax of the sums of the predicted probabilities. :param noise: List containing names of used noise injection methods that are performed after the normalization transformations. :type noise: list[str] :param noise_sets: List of sets that are affected by the noise injection. For this module single element can be "test". :type noise_sets: list[str] :param noise_params: JSON containing the parameters setting of noise injection methods. Exemplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}". This JSON should include all parameters for noise injection functions that are specified in the noise argument. For the accurate description of each parameter, please refer to the ml_intuition/data/noise.py module. :param seed: Seed for RNG. """ os.makedirs(dest_path, exist_ok=True) if type(data) is str: test_dict = io.extract_set(data, enums.Dataset.TEST) else: test_dict = copy(data[enums.Dataset.TEST]) min_max_path = os.path.join(os.path.dirname(model_path), "min-max.csv") if os.path.exists(min_max_path): min_value, max_value = io.read_min_max(min_max_path) else: min_value, max_value = data[enums.DataStats.MIN], \ data[enums.DataStats.MAX] transformations = [transforms.SpectralTransform(), transforms.OneHotEncode(n_classes=n_classes), transforms.MinMaxNormalize(min_=min_value, max_=max_value)] transformations = transformations + \ get_noise_functions(noise, noise_params) \ if enums.Dataset.TEST in noise_sets else transformations test_dict = transforms.apply_transformations(test_dict, transformations) model = tf.keras.models.load_model(model_path, compile=True) if use_ensemble: model = Ensemble(model, voting=voting) if ensemble_copies is not None: noise_params = yaml.load(noise_params) model.generate_models_with_noise(copies=ensemble_copies, mean=noise_params['mean'], seed=seed) if voting == 'classifier': if type(data) is str: train_dict = io.extract_set(data, enums.Dataset.TRAIN) else: train_dict = data[enums.Dataset.TRAIN] train_dict = transforms.apply_transformations(train_dict, transformations) train_probabilities = model.predict_probabilities(train_dict[enums.Dataset.DATA]) model.train_ensemble_predictor(train_probabilities, train_dict[enums.Dataset.LABELS]) predict = timeit(model.predict) y_pred, inference_time = predict(test_dict[enums.Dataset.DATA], batch_size=batch_size) if voting == 'classifier': y_pred = np.argmax(y_pred, axis=-1) y_true = np.argmax(test_dict[enums.Dataset.LABELS], axis=-1) model_metrics = get_model_metrics(y_true, y_pred) model_metrics['inference_time'] = [inference_time] conf_matrix = confusion_matrix(y_true, y_pred) io.save_metrics(dest_path=dest_path, file_name=enums.Experiment.INFERENCE_METRICS, metrics=model_metrics) io.save_confusion_matrix(conf_matrix, dest_path) if enums.Splits.GRIDS in model_path: if type(data) is str: train_dict = io.extract_set(data, enums.Dataset.TRAIN) labels_in_train = np.unique(train_dict[enums.Dataset.LABELS]) else: train_labels = data[enums.Dataset.TRAIN][enums.Dataset.LABELS] if train_labels.ndim > 1: train_labels = np.argmax(train_labels, axis=-1) labels_in_train = np.unique(train_labels) fair_metrics = get_fair_model_metrics(conf_matrix, labels_in_train) io.save_metrics(dest_path=dest_path, file_name=enums.Experiment.INFERENCE_FAIR_METRICS, metrics=fair_metrics)
def run_experiments(*, data_file_path: str = None, ground_truth_path: str = None, dataset_path: str = None, train_size: ('train_size', multi(min=0)), val_size: float = 0.1, stratified: bool = True, background_label: int = 0, channels_idx: int = 0, neighborhood_size: int = None, save_data: bool = False, n_runs: int, dest_path: str, models_path: str, model_name: str = 'model_2d', n_classes: int, use_ensemble: bool = False, ensemble_copies: int = None, voting: str = 'hard', batch_size: int = 1024, post_noise_sets: ('spost', multi(min=0)), post_noise: ('post', multi(min=0)), noise_params: str = None, use_mlflow: bool = False, experiment_name: str = None, model_exp_name: str = None, run_name: str = None): """ Run inference on the provided model given set of hyperparameters. :param data_file_path: Path to the data file. Supported types are: .npy :param ground_truth_path: Path to the ground-truth data file. :param dataset_path: Path to the already extracted .h5 dataset :param train_size: If float, should be between 0.0 and 1.0. If stratified = True, it represents percentage of each class to be extracted, If float and stratified = False, it represents percentage of the whole dataset to be extracted with samples drawn randomly, regardless of their class. If int and stratified = True, it represents number of samples to be drawn from each class. If int and stratified = False, it represents overall number of samples to be drawn regardless of their class, randomly. Defaults to 0.8 :type train_size: Union[int, float] :param val_size: Should be between 0.0 and 1.0. Represents the percentage of each class from the training set to be extracted as a validation set. Defaults to 0.1. :param stratified: Indicated whether the extracted training set should be stratified. Defaults to True. :param background_label: Label indicating the background in GT file. :param channels_idx: Index specifying the channels position in the provided data. :param neighborhood_size: Size of the neighbourhood for the model. :param save_data: Whether to save the prepared dataset :param n_runs: Number of total experiment runs. :param dest_path: Path to where all experiment runs will be saved as subfolders in this directory. :param models_path: Name of the model, it serves as a key in the dictionary holding all functions returning models. :param model_name: The name of model for the inference. :param n_classes: Number of classes. :param use_ensemble: Use ensemble for prediction. :param ensemble_copies: Number of model copies for the ensemble. :param voting: Method of ensemble voting. If ‘hard’, uses predicted class labels for majority rule voting. Else if ‘soft’, predicts the class label based on the argmax of the sums of the predicted probabilities. :param batch_size: Size of the batch for the inference :param post_noise_sets: The list of sets to which the noise will be injected. One element can either be "train", "val" or "test". :type post_noise_sets: list[str] :param post_noise: The list of names of noise injection methods after the normalization transformations. :type post_noise: list[str] :param noise_params: JSON containing the parameter setting of injection methods. Exemplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}". This JSON should include all parameters for noise injection functions that are specified in pre_noise and post_noise arguments. For the accurate description of each parameter, please refer to the ml_intuition/data/noise.py module. :param use_mlflow: Whether to log metrics and artifacts to mlflow. :param experiment_name: Name of the experiment. Used only if use_mlflow = True. :param run_name: Name of the run. Used only if use_mlflow = True. """ train_size = parse_train_size(train_size) if use_mlflow: args = locals() mlflow.set_tracking_uri("http://beetle.mlflow.kplabs.pl") mlflow.set_experiment(experiment_name) mlflow.start_run(run_name=run_name) log_params_to_mlflow(args) log_tags_to_mlflow(args['run_name']) models_path = get_mlflow_artifacts_path(models_path, model_exp_name) for experiment_id in range(n_runs): experiment_dest_path = os.path.join( dest_path, 'experiment_' + str(experiment_id)) model_name_regex = re.compile('model_.*') model_dir = os.path.join(models_path, f'experiment_{experiment_id}') model_name = list(filter(model_name_regex.match, os.listdir(model_dir)))[0] model_path = os.path.join(model_dir, model_name) if dataset_path is None: data_source = os.path.join(models_path, 'experiment_' + str(experiment_id), 'data.h5') else: data_source = dataset_path os.makedirs(experiment_dest_path, exist_ok=True) if data_file_path.endswith('.h5') and ground_truth_path is None and 'patches' not in data_file_path: data_source = load_processed_h5(data_file_path=data_file_path) elif not os.path.exists(data_source): data_source = prepare_data.main(data_file_path=data_file_path, ground_truth_path=ground_truth_path, output_path=data_source, train_size=train_size, val_size=val_size, stratified=stratified, background_label=background_label, channels_idx=channels_idx, neighborhood_size=neighborhood_size, save_data=save_data, seed=experiment_id) evaluate_model.evaluate( model_path=model_path, data=data_source, dest_path=experiment_dest_path, n_classes=n_classes, use_ensemble=use_ensemble, ensemble_copies=ensemble_copies, voting=voting, noise=post_noise, noise_sets=post_noise_sets, noise_params=noise_params, batch_size=batch_size, seed=experiment_id) tf.keras.backend.clear_session() artifacts_reporter.collect_artifacts_report(experiments_path=dest_path, dest_path=dest_path, use_mlflow=use_mlflow) if Splits.GRIDS in data_file_path: fair_report_path = os.path.join(dest_path, Experiment.REPORT_FAIR) artifacts_reporter.collect_artifacts_report(experiments_path=dest_path, dest_path=fair_report_path, filename=Experiment.INFERENCE_FAIR_METRICS, use_mlflow=use_mlflow) if use_mlflow: mlflow.set_experiment(experiment_name) mlflow.log_artifacts(dest_path, artifact_path=dest_path) shutil.rmtree(dest_path)
def run_experiments(*, data_file_paths: ('d', multi(min=1)), train_size: ('train_size', multi(min=0)), val_size: float = 0.1, stratified: bool = True, background_label: int = 0, channels_idx: int = 0, neighborhood_sizes: ('n', multi(min=1)), save_data: bool = False, n_runs: int, dest_path: str, model_paths: ('m', multi(min=1)), model_experiment_names: ('e', multi(min=1)), n_classes: int, voting: str = 'hard', voting_model: str = None, voting_model_params: str = None, batch_size: int = 1024, post_noise_sets: ('spost', multi(min=0)), post_noise: ('post', multi(min=0)), noise_params: str = None, use_mlflow: bool = False, experiment_name: str = None, run_name: str = None, use_unmixing: bool = False, gt_file_paths: ('g', multi(min=0)), sub_test_size: int = None): """ Function for running experiments given a set of hyper parameters. :param data_file_paths: Path to the data file. Supported types are: .npy :type data_file_paths: list[str] :param train_size: If float, should be between 0.0 and 1.0. If stratified = True, it represents percentage of each class to be extracted, If float and stratified = False, it represents percentage of the whole dataset to be extracted with samples drawn randomly, regardless of their class. If int and stratified = True, it represents number of samples to be drawn from each class. If int and stratified = False, it represents overall number of samples to be drawn regardless of their class, randomly. Defaults to 0.8 :type train_size: Union[int, float] :param val_size: Should be between 0.0 and 1.0. Represents the percentage of each class from the training set to be extracted as a validation set, defaults to 0.1 :param stratified: Indicated whether the extracted training set should be stratified, defaults to True :param background_label: Label indicating the background in GT file :param channels_idx: Index specifying the channels position in the provided data. :param neighborhood_sizes: List of sizes of neighborhoods of provided models. :type neighborhood_sizes: list[str] :param save_data: Whether to save the prepared dataset :param n_runs: Number of total experiment runs. :param dest_path: Path to where all experiment runs will be saved as subfolders in this directory. :param model_paths: Paths to all models to be used in ensemble :type model_paths: list[str] :param model_experiment_names: Names of MLFlow experiments :type model_experiment_names: list[str] :param n_classes: Number of classes. :param voting: Method of ensemble voting. If ‘hard’, uses predicted class labels for majority rule voting. Else if ‘soft’, predicts the class label based on the argmax of the sums of the predicted probabilities. Else if 'booster', employs a new model, which is trained on the ensemble predictions on the training set. :param voting_model: Type of model to use when the voting argument is set to "booster". Supported models are: SVR, SVC, RFR, RFC, DTR, DTC. :param voting_model_params: Parameters of the voting model, same as the parameters for the noise injection. :param batch_size: Size of the batch for the inference :param post_noise_sets: The list of sets to which the noise will be injected. One element can either be "train", "val" or "test". :type post_noise_sets: list[str] :param post_noise: The list of names of noise injection methods after the normalization transformations. :type post_noise: list[str] :param noise_params: JSON containing the parameter setting of injection methods. Exemplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}". This JSON should include all parameters for noise injection functions that are specified in pre_noise and post_noise arguments. For the accurate description of each parameter, please refer to the ml_intuition/data/noise.py module. :param use_mlflow: Whether to log metrics and artifacts to mlflow. :param experiment_name: Name of the experiment. Used only if use_mlflow = True :param run_name: Name of the run. Used only if use_mlflow = True. :param use_unmixing: Boolean indicating whether to utilize the unmixing functionality. :param gt_file_paths: Path to the ground-truth data files. Supported types are: .npy :type gt_file_paths: list[str] :param sub_test_size: Number of pixels to subsample from the test set instead of performing the inference on all untrained samples. """ train_size = parse_train_size(train_size) if use_mlflow: args = locals() mlflow.set_tracking_uri("http://beetle.mlflow.kplabs.pl") mlflow.set_experiment(experiment_name) mlflow.start_run(run_name=run_name) log_params_to_mlflow(args) log_tags_to_mlflow(args['run_name']) for experiment_id in range(n_runs): experiment_dest_path = os.path.join(dest_path, 'experiment_' + str(experiment_id)) os.makedirs(experiment_dest_path, exist_ok=True) models_test_predictions = [] models_train_predictions = [] for data_file_path, model_path, model_experiment_name, \ neighborhood_size, gt_file_path in \ zip_longest(data_file_paths, model_paths, model_experiment_names, neighborhood_sizes, gt_file_paths, fillvalue=None): if use_mlflow: model_path = get_mlflow_artifacts_path(model_path, model_experiment_name) model_name_regex = re.compile('unmixing_.*') \ if use_unmixing else re.compile('model_.*') model_dir = os.path.join(model_path, f'experiment_{experiment_id}') model_name = \ list(filter(model_name_regex.match, os.listdir(model_dir)))[0] model_path = os.path.join(model_dir, model_name) if neighborhood_size is not None: neighborhood_size = int(neighborhood_size) if data_file_path.endswith( '.h5') and 'patches' not in data_file_path: data_source = load_processed_h5(data_file_path=data_file_path) else: data_source = prepare_data.main( data_file_path=data_file_path, ground_truth_path='' if gt_file_path is None else gt_file_path, train_size=train_size, val_size=val_size, stratified=stratified, background_label=background_label, channels_idx=channels_idx, neighborhood_size=neighborhood_size, save_data=save_data, seed=experiment_id, use_unmixing=use_unmixing) if sub_test_size is not None: subsample_test_set(data_source[enums.Dataset.TEST], sub_test_size) test_predictions = predict_with_model.predict( model_path=model_path, data=data_source, batch_size=batch_size, dataset_to_predict=enums.Dataset.TEST, use_unmixing=use_unmixing, neighborhood_size=neighborhood_size) models_test_predictions.append(test_predictions) if voting == 'booster': train_predictions = predict_with_model.predict( model_path=model_path, data=data_source, batch_size=batch_size, dataset_to_predict=enums.Dataset.TRAIN, use_unmixing=use_unmixing) models_train_predictions.append(train_predictions) tf.keras.backend.clear_session() models_test_predictions = np.asarray(models_test_predictions) if use_unmixing: evaluate_unmixing_with_ensemble.evaluate( y_pred=models_test_predictions, data=data_source, dest_path=experiment_dest_path, voting=voting, train_set_predictions=models_train_predictions, voting_model=voting_model, voting_model_params=voting_model_params) else: evaluate_with_ensemble.evaluate( y_pred=models_test_predictions, model_path=model_path, data=data_source, dest_path=experiment_dest_path, voting=voting, train_set_predictions=models_train_predictions, n_classes=n_classes, voting_model=voting_model, voting_model_params=voting_model_params) artifacts_reporter.collect_artifacts_report(experiments_path=dest_path, dest_path=dest_path, use_mlflow=use_mlflow) if not use_unmixing: fair_report_path = os.path.join(dest_path, Experiment.REPORT_FAIR) artifacts_reporter.collect_artifacts_report( experiments_path=dest_path, dest_path=fair_report_path, filename=Experiment.INFERENCE_FAIR_METRICS, use_mlflow=use_mlflow) if use_mlflow: mlflow.set_experiment(experiment_name) mlflow.log_artifacts(dest_path, artifact_path=dest_path) shutil.rmtree(dest_path)