def main(*, listen:('l', parameters.multi(min=1, max=3))):
"""Listens on the given addresses
:param listen: An address to listen on.
"""
for address in listen:
print('Listening on {0}'.format(address))
def run_evaluation(
*,
model_hash: str,
thr: float = 0.5,
dpath: str = "datasets/clouds/Landsat-Cloud-Cover-Assessment-" +
"Validation-Data-Partial",
rpath: str = "artifacts/",
vids: ("v", multi(min=1)),
batch_size: int = 32,
bands: ("b", multi(min=1)),
bands_names: ("bn", multi(min=1)),
img_ids: ("iid", multi(min=0)),
resize: bool = False,
normalize: bool = False,
standardize: bool = False,
mlflow: bool = False,
run_name: str = None,
):
"""
Load model given model hash and get evaluation metrics on L8CCA testset.
:param model_hash: MLFlow hash of the model to load.
:param thr: threshold to be used during evaluation.
:param dpath: path to dataset.
:param rpath: path to directory where results
and artifacts should be logged.
:param vids: tuple of ids of images which should be used to create
visualisations. If contains '*' visualisations will be
created for all images in the dataset.
:type vids: tuple[str]
:param batch_size: size of generated batches, only one batch is loaded
to memory at a time.
:param bands: band numbers to load
:type bands: list[int]
:param bands_names: names of the bands to load. Should have the same number
of elements as bands.
:type bands_names: list[str]
:param img_ids: if given, process only these images.
:type img_ids: list[int]
:param resize: whether to resize loaded img to gt.
:param normalize: whether to normalize the image.
:param standardize: whether to standardize the image.
:param mlflow: whether to use MLFlow.
:param run_name: name of the run.
"""
snow_imgs = ["LC82271192014287LGN00", "LC81321192014054LGN00"]
if img_ids == []:
img_ids = None
else:
snow_imgs = list(set(snow_imgs) & set(img_ids))
dpath, rpath = make_paths(dpath, rpath)
rpath = rpath / uuid.uuid4().hex
print(f"Working dir: {os.getcwd()}, " + f"artifacts dir: {rpath}",
flush=True)
mpath = Path(f"/media/ML/mlflow/beetle/artifacts/34/{model_hash}/"
# Change init_model to model for old models
+ "artifacts/init_model/data/model.h5")
# WARNING: If ever upgraded to tf2.x, this way of using metrics will not work
# because loaded metrics will become MeanMetricWrapper objects in tf2.x and
# this script isn't prepared for such objects (because MeanMetricWrapper has state,
# as opposed to present stateless metric functions).
model = keras.models.load_model(
mpath,
custom_objects={
"jaccard_index_loss": losses.JaccardIndexLoss(),
"jaccard_index_metric": losses.JaccardIndexMetric(),
"dice_coeff_metric": losses.DiceCoefMetric(),
"recall": losses.recall,
"precision": losses.precision,
"specificity": losses.specificity,
# F1 score is needed for old models
# "f1_score": losses.f1_score,
"tf": tf,
},
)
model.load_weights(f"/media/ML/mlflow/beetle/artifacts/34/{model_hash}/" +
"artifacts/best_weights/best_weights")
metrics, _ = evaluate_model(
model=model,
thr=thr,
dpath=dpath,
rpath=rpath,
vids=vids,
batch_size=batch_size,
bands=bands,
bands_names=bands_names,
img_ids=img_ids,
resize=resize,
normalize=normalize,
standardize=standardize,
mlflow=mlflow,
run_name=run_name,
)
mean_metrics = {}
mean_metrics_snow = {}
for key, value in metrics.items():
mean_metrics[key] = np.mean(list(value.values()))
mean_metrics_snow[f"snow_{key}"] = np.mean(
[value[x] for x in snow_imgs])
print(mean_metrics, mean_metrics_snow)
if mlflow:
log_metrics(mean_metrics)
log_metrics(mean_metrics_snow)
log_artifacts(rpath)
def main(*,
data_file_path: str,
ground_truth_path: str,
output_path: str = None,
train_size: ('train_size', multi(min=0)),
val_size: float = 0.1,
stratified: bool = True,
background_label: int = 0,
channels_idx: int = 0,
save_data: bool = False,
seed: int = 0):
"""
:param data_file_path: Path to the data file. Supported types are: .npy
:param ground_truth_path: Path to the data file.
:param output_path: Path under in which the output .h5 file will be stored.
Used only if the parameter save_data is set to True
:param train_size: If float, should be between 0.0 and 1.0,
if stratified = True, it represents percentage of each
class to be extracted,
If float and stratified = False, it represents percentage of the
whole dataset to be extracted with samples drawn randomly,
regardless of their class.
If int and stratified = True, it represents number of samples
to be drawn from each class.
If int and stratified = False, it represents overall number of
samples to be drawn regardless of their class, randomly.
Defaults to 0.8
:type train_size: float or int
:param val_size: Should be between 0.0 and 1.0. Represents the percentage of
each class from the training set to be extracted as a
validation set, defaults to 0.1
:param stratified: Indicated whether the extracted training set should be
stratified, defaults to True
:param background_label: Label indicating the background in GT file
:param channels_idx: Index specifying the channels position in the provided
data
:param save_data: Whether to save data as .md5 or to return it as a dict
:param seed: Seed used for data shuffling
:raises TypeError: When provided data or labels file is not supported
"""
train_size = utils.parse_train_size(train_size)
if data_file_path.endswith('.npy') and ground_truth_path.endswith('.npy'):
data, labels = io.load_npy(data_file_path, ground_truth_path)
data, labels = preprocessing.reshape_cube_to_2d_samples(
data, labels, channels_idx)
elif data_file_path.endswith('.h5') and ground_truth_path.endswith(
'.tiff'):
data, gt_transform_mat = io.load_satellite_h5(data_file_path)
labels = io.load_tiff(ground_truth_path)
data_2d_shape = data.shape[1:]
labels = preprocessing.align_ground_truth(data_2d_shape, labels,
gt_transform_mat)
data, labels = preprocessing.reshape_cube_to_2d_samples(
data, labels, channels_idx)
data, labels = preprocessing.remove_nan_samples(data, labels)
else:
raise ValueError(
"The following data file type is not supported: {}".format(
os.path.splitext(data_file_path)[EXTENSION]))
data = data[labels != background_label]
labels = labels[labels != background_label]
labels = preprocessing.normalize_labels(labels)
train_x, train_y, val_x, val_y, test_x, test_y = \
preprocessing.train_val_test_split(data, labels, train_size, val_size,
stratified, seed=seed)
if save_data:
io.save_md5(output_path, train_x, train_y, val_x, val_y, test_x,
test_y)
return None
else:
return utils.build_data_dict(train_x, train_y, val_x, val_y, test_x,
test_y)
def main_(*,
config: 'c' = 'config.yml',
debug: 'd' = False,
extra: ('e', str, parameters.multi()) = None,
optimistic = False,
quiet: 'q' = False,
vim_is_fucking_retarded: Parameter.UNDOCUMENTED = False):
"""
Run cms7.
config: Path to project configuration
debug: Print obnoxious debugging output
extra: Path to additional configuration (e.g. site local overrides). Can
be specified multiple times. Later configurations override.
optimistic: Try to continue processing after rendering errors
quiet: Only ever print warnings
"""
rl = logging.getLogger()
h = logging.StreamHandler()
try:
if not sys.stdin.isatty():
raise Exception
import colorlog
h.setFormatter(colorlog.ColoredFormatter(
"%(log_color)s%(levelname)-8s%(reset)s %(message_log_color)s%(name)s:%(message)s",
secondary_log_colors={
'message': {
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'red',
}
}))
except:
h.setFormatter(logging.Formatter(
"%(levelname)-8s %(name)s:%(message)s"))
rl.addHandler(h)
if vim_is_fucking_retarded:
report_error.quiet = False
if debug:
rl.setLevel(logging.DEBUG)
elif quiet:
if vim_is_fucking_retarded:
rl.setLevel(logging.CRITICAL + 1)
else:
rl.setLevel(logging.WARNING)
else:
rl.setLevel(logging.INFO)
try:
cfg = _config.load(config, *extra)
if optimistic:
cfg.optimistic = True
gen = Generator(cfg)
for m in cfg.modules():
m.prepare()
for m in cfg.modules():
m.run(gen)
gen.run()
for r in cfg.resources:
r.run()
except CMS7Error as e:
logger.critical('%s', e.message, exc_info=debug)
if not debug:
logger.warning('exiting for exception. use --debug to get a traceback')
sys.exit(1)
except Exception:
logger.critical('unexpected exception', exc_info=True)
sys.exit(1)
def train(*,
data,
model_name: str,
dest_path: str,
sample_size: int,
n_classes: int,
kernel_size: int = 3,
n_kernels: int = 16,
n_layers: int = 1,
lr: float = 0.005,
batch_size: int = 150,
epochs: int = 10,
verbose: int = 2,
shuffle: bool = True,
patience: int = 3,
seed: int = 0,
noise: ('post', multi(min=0)),
noise_sets: ('spost', multi(min=0)),
noise_params: str = None):
"""
Function for training tensorflow models given a dataset.
:param model_name: Name of the model, it serves as a key in the
dictionary holding all functions returning models.
:param kernel_size: Size of ech kernel in each layer.
:param n_kernels: Number of kernels in each layer.
:param n_layers: Number of layers in the model.
:param dest_path: Path to where to save the model
under the name "model_name".
:param sample_size: Size of the input sample.
:param n_classes: Number of classes.
:param lr: Learning rate for the model, i.e., regulates the size of the step
in the gradient descent process.
:param data: Either path to the input data or the data dict itself.
First dimension of the dataset should be the number of samples.
:param batch_size: Size of the batch used in training phase,
it is the size of samples per gradient step.
:param epochs: Number of epochs for model to train.
:param verbose: Verbosity mode used in training, (0, 1 or 2).
:param shuffle: Boolean indicating whether to shuffle dataset
dataset_key each epoch.
:param patience: Number of epochs without improvement in order to
stop the training phase.
:param seed: Seed for training reproducibility.
:param noise: List containing names of used noise injection methods
that are performed after the normalization transformations.
:type noise: list[str]
:param noise_sets: List of sets that are affected by
the noise injection methods.
For this module single element can be either "train" or "val".
:type noise_sets: list[str]
:param noise_params: JSON containing the parameters
setting of injection methods.
Exemplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}".
This JSON should include all parameters for noise injection
functions that are specified in the noise argument.
For the accurate description of each parameter, please
refer to the ml_intuition/data/noise.py module.
"""
# Reproducibility
tf.reset_default_graph()
tf.set_random_seed(seed=seed)
np.random.seed(seed=seed)
if type(data) is str:
train_dict = io.extract_set(data, enums.Dataset.TRAIN)
val_dict = io.extract_set(data, enums.Dataset.VAL)
min_, max_ = train_dict[enums.DataStats.MIN], \
train_dict[enums.DataStats.MAX]
else:
train_dict = data[enums.Dataset.TRAIN]
val_dict = data[enums.Dataset.VAL]
min_, max_ = data[enums.DataStats.MIN], \
data[enums.DataStats.MAX]
transformations = [transforms.SpectralTransform(),
transforms.OneHotEncode(n_classes=n_classes),
transforms.MinMaxNormalize(min_=min_, max_=max_)]
tr_transformations = transformations + get_noise_functions(noise, noise_params) \
if enums.Dataset.TRAIN in noise_sets else transformations
val_transformations = transformations + get_noise_functions(noise, noise_params) \
if enums.Dataset.VAL in noise_sets else transformations
train_dict = transforms.apply_transformations(train_dict, tr_transformations)
val_dict = transforms.apply_transformations(val_dict, val_transformations)
model = models.get_model(model_key=model_name, kernel_size=kernel_size,
n_kernels=n_kernels, n_layers=n_layers,
input_size=sample_size, n_classes=n_classes)
model.summary()
model.compile(tf.keras.optimizers.Adam(lr=lr),
'categorical_crossentropy',
metrics=['accuracy'])
time_history = time_metrics.TimeHistory()
mcp_save = tf.keras.callbacks.ModelCheckpoint(
os.path.join(dest_path, model_name), save_best_only=True,
monitor='val_acc', mode='max')
early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss',
patience=patience)
callbacks = [time_history, mcp_save, early_stopping]
history = model.fit(x=train_dict[enums.Dataset.DATA],
y=train_dict[enums.Dataset.LABELS],
epochs=epochs,
verbose=verbose,
shuffle=shuffle,
validation_data=(val_dict[enums.Dataset.DATA],
val_dict[enums.Dataset.LABELS]),
callbacks=callbacks,
batch_size=batch_size)
history.history[time_metrics.TimeHistory.__name__] = time_history.average
io.save_metrics(dest_path=dest_path,
file_name='training_metrics.csv',
metrics=history.history)
np.savetxt(os.path.join(dest_path, 'min-max.csv'),
np.array([min_, max_]), delimiter=',', fmt='%f')
from sigtools import modifiers
from clize import run, parameters
@modifiers.kwoargs('listen')
@modifiers.annotate(listen=('l', parameters.multi(min=1, max=3)))
def main(listen):
"""Listens on the given addresses
listen: An address to listen on.
"""
for address in listen:
print('Listening on {0}'.format(address))
run(main)
def _main(*steps: use_mixin(StepList),
each_line: 'l' = False, # noqa: F821
raw: 'r' = False, # noqa: F821
start: (int, 's') = 0,
end: (int, 'e') = None,
prelude: (multi(), 'p') = 'pass',
pipe_name: Parameter.UNDOCUMENTED = PIPE_NAME,
no_default_fragments: Parameter.UNDOCUMENTED = False,
no_exception_handling: Parameter.UNDOCUMENTED = False,
show_fragments: Parameter.UNDOCUMENTED = False,
break_: Parameter.UNDOCUMENTED = False):
"""Run Python code.
:param steps: At least one Python expression (or suite) to execute
:param each_line: If specified, process lines as strings rather than all of stdin as a file
:param start: Don't print before this result (zero-based)
:param end: Stop after getting this result (zero-based)
:param prelude: Execute this statement before running any steps. Can be specified more than once.
:param raw: Don't add helper functionality to stdin
"""
pipe_name = sys.intern(pipe_name)
spy.context = context = make_context()
for stmt in prelude:
exec(stmt, context, context.view())
step_src = steps
steps = []
for i, code in enumerate(step_src):
fragment_name = 'Fragment {}'.format(i + 1)
source = code
if isinstance(code, _Decorated):
source = '{} {!r}'.format(code.name, code.value)
code, funcseq = code.value, code.funcseq
else:
funcseq = ()
try:
co, is_expr = compile_(code, filename=fragment_name)
except SyntaxError as e:
pretty_syntax_error(code, e)
if break_: # pragma: no cover
debugger()
sys.exit(1)
debuginfo = (fragment_name, source)
ca = make_callable(co, is_expr, context, pipe_name, debuginfo)
for fn in funcseq:
try:
ca = fn(ca, debuginfo=debuginfo)
except TypeError:
ca = fn(ca)
steps.append(spy.fragment(ca))
index_offset = 0
if not no_default_fragments:
steps.append(fragments.make_limit(start=start, end=end))
steps.append(fragments.print)
if each_line:
steps.insert(0, fragments.foreach)
index_offset -= 1
if raw:
steps.insert(0, fragments.raw_stdin)
else:
steps.insert(0, fragments.stdin)
index_offset -= 1
chain = spy.chain(steps, index_offset=index_offset)
data = [None]
if show_fragments:
print(chain.format())
return
with ExitStack() as stack:
if not no_exception_handling:
stack.enter_context(catcher.handler(delete_all=True))
if break_: # pragma: no cover
stack.enter_context(DebuggerContext())
chain.run_to_exhaustion(data)
def run_experiments(*,
data_file_path: str,
ground_truth_path: str = None,
train_size: ('train_size', multi(min=0)),
val_size: float = 0.1,
stratified: bool = True,
background_label: int = 0,
channels_idx: int = 0,
neighborhood_size: int = None,
n_runs: int,
model_name: str,
kernel_size: int = 5,
n_kernels: int = 200,
save_data: bool = 0,
n_layers: int = 1,
dest_path: str = None,
sample_size: int,
n_classes: int,
lr: float = 0.001,
batch_size: int = 128,
epochs: int = 200,
verbose: int = 2,
shuffle: bool = True,
patience: int = 15,
pre_noise: ('pre', multi(min=0)),
pre_noise_sets: ('spre', multi(min=0)),
post_noise: ('post', multi(min=0)),
post_noise_sets: ('spost', multi(min=0)),
noise_params: str = None,
use_mlflow: bool = False,
experiment_name: str = None,
run_name: str = None):
"""
Function for running experiments given a set of hyper parameters.
:param data_file_path: Path to the data file. Supported types are: .npy.
:param ground_truth_path: Path to the ground-truth data file.
:param train_size: If float, should be between 0.0 and 1.0.
If stratified = True, it represents percentage of each class to be extracted.
If float and stratified = False, it represents percentage of the whole
dataset to be extracted with samples drawn randomly, regardless of their class.
If int and stratified = True, it represents number of samples to be
drawn from each class.
If int and stratified = False, it represents overall number of samples
to be drawn regardless of their class, randomly.
Defaults to 0.8.
:type train_size: Union[int, float]
:param val_size: Should be between 0.0 and 1.0. Represents the
percentage of each class from the training set to be
extracted as a validation set.
Defaults to 0.1.
:param stratified: Indicated whether the extracted training set should be
stratified.
Defaults to True.
:param background_label: Label indicating the background in GT file.
:param channels_idx: Index specifying the channels position in the provided
data.
:param neighborhood_size: Size of the spatial patch.
:param save_data: Whether to save the prepared dataset.
:param n_runs: Number of total experiment runs.
:param model_name: Name of the model, it serves as a key in the
dictionary holding all functions returning models.
:param kernel_size: Size of ech kernel in each layer.
:param n_kernels: Number of kernels in each layer.
:param n_layers: Number of layers in the model.
:param dest_path: Path to where all experiment runs will be saved as
subfolders in this directory.
:param sample_size: Size of the input sample.
:param n_classes: Number of classes.
:param lr: Learning rate for the model, i.e., regulates
the size of the step in the gradient descent process.
:param batch_size: Size of the batch used in training phase,
it is the size of samples per gradient step.
:param epochs: Number of epochs for model to train.
:param verbose: Verbosity mode used in training, (0, 1 or 2).
:param shuffle: Boolean indicating whether to shuffle dataset
dataset_key each epoch.
:param patience: Number of epochs without improvement in order to
stop the training phase.
:param pre_noise: The list of names of noise injection methods before
the normalization transformations. Exemplary names are "gaussian"
or "impulsive".
:type pre_noise: list[str]
:param pre_noise_sets: The list of sets to which the noise will be
injected. One element can either be "train", "val" or "test".
:type pre_noise_sets: list[str]
:param post_noise: The list of names of noise injection methods after
the normalization transformations.
:type post_noise: list[str]
:param post_noise_sets: The list of sets to which
the noise will be injected.
:type post_noise_sets: list[str]
:param noise_params: JSON containing the parameter
setting of injection methods.
Exemplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}".
This JSON should include all parameters for noise injection
functions that are specified in pre_noise and post_noise arguments.
For the accurate description of each parameter, please
refer to the ml_intuition/data/noise.py module.
:param use_mlflow: Whether to log metrics and artifacts to mlflow.
:param experiment_name: Name of the experiment. Used only if
use_mlflow = True
:param run_name: Name of the run. Used only if use_mlflow = True.
"""
train_size = parse_train_size(train_size)
if use_mlflow:
args = locals()
mlflow.set_tracking_uri("http://beetle.mlflow.kplabs.pl")
mlflow.set_experiment(experiment_name)
mlflow.start_run(run_name=run_name)
log_params_to_mlflow(args)
log_tags_to_mlflow(args['run_name'])
if dest_path is None:
dest_path = os.path.join(os.path.curdir, "temp_artifacts")
for experiment_id in range(n_runs):
experiment_dest_path = os.path.join(
dest_path, '{}_{}'.format(enums.Experiment.EXPERIMENT,
str(experiment_id)))
if save_data:
data_source = os.path.join(experiment_dest_path, 'data.h5')
else:
data_source = None
os.makedirs(experiment_dest_path, exist_ok=True)
if data_file_path.endswith('.h5') and ground_truth_path is None:
data = load_processed_h5(data_file_path=data_file_path)
else:
data = prepare_data.main(data_file_path=data_file_path,
ground_truth_path=ground_truth_path,
output_path=data_source,
train_size=train_size,
val_size=val_size,
stratified=stratified,
background_label=background_label,
channels_idx=channels_idx,
save_data=save_data,
seed=experiment_id)
if not save_data:
data_source = data
if len(pre_noise) > 0:
noise.inject_noise(data_source=data_source,
affected_subsets=pre_noise_sets,
noise_injectors=pre_noise,
noise_params=noise_params)
train_model.train(model_name=model_name,
kernel_size=kernel_size,
n_kernels=n_kernels,
n_layers=n_layers,
dest_path=experiment_dest_path,
data=data_source,
sample_size=sample_size,
n_classes=n_classes,
lr=lr,
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
shuffle=shuffle,
patience=patience,
noise=post_noise,
noise_sets=pre_noise_sets,
noise_params=noise_params)
evaluate_model.evaluate(model_path=os.path.join(
experiment_dest_path, model_name),
data=data_source,
dest_path=experiment_dest_path,
n_classes=n_classes,
batch_size=batch_size,
noise=post_noise,
noise_sets=pre_noise_sets,
noise_params=noise_params)
tf.keras.backend.clear_session()
artifacts_reporter.collect_artifacts_report(experiments_path=dest_path,
dest_path=dest_path,
use_mlflow=use_mlflow)
if enums.Splits.GRIDS in data_file_path:
fair_report_path = os.path.join(dest_path,
enums.Experiment.REPORT_FAIR)
artifacts_reporter.collect_artifacts_report(
experiments_path=dest_path,
dest_path=fair_report_path,
filename=enums.Experiment.INFERENCE_FAIR_METRICS,
use_mlflow=use_mlflow)
if use_mlflow:
mlflow.log_artifacts(dest_path, artifact_path=dest_path)
shutil.rmtree(dest_path)
def evaluate(*,
data,
model_path: str,
dest_path: str,
n_classes: int,
batch_size: int = 1024,
noise: ('post', multi(min=0)),
noise_sets: ('spost', multi(min=0)),
noise_params: str = None):
"""
Function for evaluating the trained model.
:param model_path: Path to the model.
:param data: Either path to the input data or the data dict.
:param dest_path: Directory in which to store the calculated metrics
:param n_classes: Number of classes.
:param batch_size: Size of the batch for inference
:param noise: List containing names of used noise injection methods
that are performed after the normalization transformations.
:param noise_sets: List of sets that are affected by the noise injection.
For this module single element can be "test".
:param noise_params: JSON containing the parameters
setting of noise injection methods.
Exemplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}".
This JSON should include all parameters for noise injection
functions that are specified in the noise argument.
For the accurate description of each parameter, please
refer to the ml_intuition/data/noise.py module.
"""
if type(data) is str:
test_dict = io.extract_set(data, enums.Dataset.TEST)
else:
test_dict = data[enums.Dataset.TEST]
min_max_path = os.path.join(os.path.dirname(model_path), "min-max.csv")
if os.path.exists(min_max_path):
min_value, max_value = io.read_min_max(min_max_path)
else:
min_value, max_value = data[enums.DataStats.MIN], \
data[enums.DataStats.MAX]
transformations = [transforms.SpectralTransform(),
transforms.OneHotEncode(n_classes=n_classes),
transforms.MinMaxNormalize(min_=min_value, max_=max_value)]
transformations = transformations + get_noise_functions(noise, noise_params) \
if enums.Dataset.TEST in noise_sets else transformations
test_dict = transforms.apply_transformations(test_dict, transformations)
model = tf.keras.models.load_model(model_path, compile=True)
predict = timeit(model.predict)
y_pred, inference_time = predict(test_dict[enums.Dataset.DATA],
batch_size=batch_size)
y_pred = np.argmax(y_pred, axis=-1)
y_true = np.argmax(test_dict[enums.Dataset.LABELS], axis=-1)
model_metrics = get_model_metrics(y_true, y_pred)
model_metrics['inference_time'] = [inference_time]
conf_matrix = confusion_matrix(y_true, y_pred)
io.save_metrics(dest_path=dest_path,
file_name=enums.Experiment.INFERENCE_METRICS,
metrics=model_metrics)
io.save_confusion_matrix(conf_matrix, dest_path)
if enums.Splits.GRIDS in model_path:
if type(data) is str:
train_dict = io.extract_set(data, enums.Dataset.TRAIN)
labels_in_train = np.unique(train_dict[enums.Dataset.LABELS])
else:
train_labels = data[enums.Dataset.TRAIN][enums.Dataset.LABELS]
if train_labels.ndim > 1:
train_labels = np.argmax(train_labels, axis=-1)
labels_in_train = np.unique(train_labels)
fair_metrics = get_fair_model_metrics(conf_matrix, labels_in_train)
io.save_metrics(dest_path=dest_path,
file_name=enums.Experiment.INFERENCE_FAIR_METRICS,
metrics=fair_metrics)
class RepTests(object):
mapped_basic = ('par:a',
parameters.mapped([
('greeting', ['hello'], 'h1'),
('parting', ['goodbye'], 'h2'),
]), 'par')
mapped_default = ('par:a="greeting"',
parameters.mapped([
('greeting', ['hello'], 'h1'),
('parting', ['goodbye'], 'h2'),
]), '[par]')
mapped_alternate_list = ('par:a',
parameters.mapped([
('greeting', ['hello'], 'h1'),
('parting', ['goodbye'], 'h2'),
],
list_name="options"), 'par')
mapped_no_list = ('par:a',
parameters.mapped([
('greeting', ['hello'], 'h1'),
('parting', ['goodbye'], 'h2'),
],
list_name=None), 'par')
mapped_kw = ('*, par:a',
parameters.mapped([
('greeting', ['hello'], 'h1'),
('parting', ['goodbye'], 'h2'),
],
list_name=None), '--par=STR')
mapped_force_icase = ('par:a',
parameters.mapped([(1, ['thing'], 'h')],
case_sensitive=False), 'par')
mapped_force_scase = ('par:a',
parameters.mapped([
(1, ['Thing'], 'h'),
],
case_sensitive=True), 'par')
mapped_imply_scase = ('par:a',
parameters.mapped([
(1, ['thing'], 'h'),
(2, ['Thing'], 'h'),
]), 'par')
mapped_bad_icase = ('par:a',
parameters.mapped([
(1, ['thing'], 'h'),
(2, ['Thing'], 'h'),
],
case_sensitive=False), 'par')
oneof_basic = 'par:a', parameters.one_of('hello', 'goodbye', 'bye'), 'par'
oneof_help = ('par:a', parameters.one_of(('hello', 'h1'),
('bye', 'h2')), 'par')
multi_basic = '*, par:a', parameters.multi(), '[--par=STR...]'
multi_req = '*, par:a', parameters.multi(1), '--par=STR...'
multi_min = '*, par:a', parameters.multi(2), '--par=STR...'
multi_max = '*, par:a', parameters.multi(max=2), '[--par=STR...]'
multi_bound = '*, par:a', parameters.multi(min=2, max=3), '--par=STR...'
multi_conv = '*, par:a', (parameters.multi(), int), '[--par=INT...]'
multi_last_opt = ('*args, par:a', (parameters.multi(), Parameter.L),
'[--par=STR...] [args...]')
margs_basic = '*args:a', parameters.multi(), '[args...]'
margs_req = '*args:a', parameters.multi(1), 'args...'
margs_min = '*args:a', parameters.multi(2), 'args...'
margs_max = '*args:a', parameters.multi(max=2), '[args...]'
margs_bound = '*args:a', parameters.multi(min=2, max=3), 'args...'
margs_conv = '*args:a', (parameters.multi(), int), '[args...]'
margs_last_opt = ('*args:a, par=""', (parameters.multi(), Parameter.L),
'[--par=STR] [args...]')
@parameters.argument_decorator
def _blank(arg):
return arg + 'x'
deco_blank_pos = 'par:a', _blank, 'par'
deco_blank_posd = 'par:a="d"', _blank, '[par]'
deco_blank_kw = '*, par:a', _blank, '--par=STR'
deco_blank_kwd = '*, par:a="d"', _blank, '[--par=STR]'
deco_blank_args = '*par:a', _blank, '[par...]'
@parameters.argument_decorator
@modifiers.kwoargs(start='kw')
def _kw(arg, kw):
return arg + kw
deco_kw_pos = 'par:a', _kw, '--kw=STR par'
deco_kw_posd = 'par:a="d"', _kw, '[--kw=STR par]'
deco_kw_kw = '*, par:a', _kw, '--kw=STR --par=STR'
deco_kw_kwd = '*, par:a="d"', _kw, '[--kw=STR --par=STR]'
deco_kw_args = '*par:a', _kw, '[--kw=STR par...]'
@parameters.argument_decorator
@modifiers.autokwoargs
def _kwdef(arg, kw='D'):
return arg + kw
deco_def_pos = 'par:a', _kwdef, '[--kw=STR] par'
deco_def_posd = 'par:a="d"', _kwdef, '[[--kw=STR] par]'
deco_def_kw = '*, par:a', _kwdef, '[--kw=STR] --par=STR'
deco_def_kwd = '*, par:a="d"', _kwdef, '[[--kw=STR] --par=STR]'
deco_def_args = '*par:a', _kwdef, '[[--kw=STR] par...]'
@parameters.argument_decorator
@modifiers.autokwoargs
def _flag(arg, f=False):
return arg + ('y' if f else 'n')
deco_flag_pos = 'par:a', _flag, '[-f] par'
deco_flag_posd = 'par:a="d"', _flag, '[[-f] par]'
deco_flag_kw = '*, par:a', _flag, '[-f] --par=STR'
deco_flag_kwd = '*, par:a="d"', _flag, '[[-f] --par=STR]'
deco_flag_args = '*par:a', _flag, '[[-f] par...]'
deco_flag_other = 'par:a, *, x=False', _flag, '[-x] [-f] par'
@parameters.argument_decorator
@modifiers.autokwoargs
def _int(arg, i=0):
return arg + str(i)
deco_int_pos = 'par:a', _int, '[-i INT] par'
deco_int_posd = 'par:a="d"', _int, '[[-i INT] par]'
deco_int_kw = '*, par:a', _int, '[-i INT] --par=STR'
deco_int_kwd = '*, par:a="d"', _int, '[[-i INT] --par=STR]'
deco_int_args = '*par:a', _int, '[[-i INT] par...]'
deco_int_other = 'par:a, *, x=False', _int, '[-x] [-i INT] par'
@parameters.argument_decorator
@modifiers.kwoargs(start='kw')
def _all(arg, kw, flag=False, kwd='D', i=0):
return arg, kw, flag, kwd, i
_all_rep = '--kw=STR [--flag] [--kwd=STR] [-i INT] '
deco_all_pos = 'par:a', _all, _all_rep + 'par'
deco_all_posd = 'par:a="d"', _all, '[' + _all_rep + 'par]'
deco_all_kw = '*, par:a', _all, _all_rep + '--par=STR'
deco_all_kwd = '*, par:a="d"', _all, '[' + _all_rep + '--par=STR]'
deco_all_args = '*par:a', _all, '[' + _all_rep + 'par...]'
@parameters.argument_decorator
@modifiers.kwoargs(start='why')
def _inner(arg, why, zed='zed'):
return '(' + arg + why + zed + ')'
@parameters.argument_decorator
@modifiers.kwoargs(start='ab')
@modifiers.annotate(cd=_inner)
def _nest(arg, ab, cd='cd'):
return '(' + arg + ab + cd + ')'
_nest_rep = '--ab=STR [--why=STR [--zed=STR] --cd=STR] '
deco_nest_pos = 'par:a', _nest, _nest_rep + 'par'
deco_nest_posd = 'par:a="d"', _nest, '[' + _nest_rep + 'par]'
deco_nest_kw = '*, par:a', _nest, _nest_rep + '--par=STR'
deco_nest_kwd = '*, par:a="d"', _nest, '[' + _nest_rep + '--par=STR]'
deco_nest_args = '*par:a', _nest, '[' + _nest_rep + 'par...]'
pn_pos = 'par:a', parameters.pass_name, ''
pn_pos_first = 'par:a, other', parameters.pass_name, 'other'
pn_pos_nextpicky = 'par:a, other:int', parameters.pass_name, 'other'
pn_kw = '*, par:a', parameters.pass_name, ''
def run_experiments(*,
input_dir: str,
n_runs: int,
dest_path: str,
data_file_path: str = None,
ground_truth_path: str = None,
dataset_path: str = None,
background_label: int = 0,
channels_idx: int = 2,
channels_count: int = 103,
train_size: ('train_size', multi(min=0)),
batch_size: int = 64,
stratified: bool = True,
gpu: bool = 0):
"""
Freeze model, quantize it and evaluate N times.
:param input_dir: Directory with saved data and models, each in separate
`experiment_n` folder.
:param n_runs: Number of total experiment runs.
:param dest_path: Path to where all experiment runs will be saved as
sub folders in this directory.
:param data_file_path: Path to the data file. Supported types are: .npy and
.md5. This is optional, if the data is not already saved in the
input_dir.
:param ground_truth_path: Path to the data file.
:param dataset_path: Path to the already extracted .h5 dataset
:param background_label: Label indicating the background in GT file
:param channels_idx: Index specifying the channels position in the provided
data
:param channels_count: Number of channels (bands) in the image.
:param train_size: If float, should be between 0.0 and 1.0.
If stratified = True, it represents percentage of each class to be extracted,
If float and stratified = False, it represents percentage of the whole
dataset to be extracted with samples drawn randomly, regardless of their class.
If int and stratified = True, it represents number of samples to be
drawn from each class.
If int and stratified = False, it represents overall number of samples
to be drawn regardless of their class, randomly.
Defaults to 0.8
:type train_size: Union[int, float]
:param stratified: Indicated whether the extracted training set should be
stratified, defaults to True
:param batch_size: Batch size
:param gpu: Whether to run quantization on gpu.
"""
for experiment_id in range(n_runs):
experiment_dest_path = os.path.join(
dest_path, 'experiment_' + str(experiment_id))
model_path = os.path.join(input_dir,
'experiment_' + str(experiment_id),
'model_2d')
created_dataset = False
if dataset_path is None:
data_path = os.path.join(input_dir, 'experiment_' + str(experiment_id),
'data.h5')
created_dataset = True
else:
data_path = dataset_path
os.makedirs(experiment_dest_path, exist_ok=True)
if not os.path.exists(data_path):
data_path = os.path.join(experiment_dest_path, 'data.md5')
prepare_data.main(data_file_path=data_file_path,
ground_truth_path=ground_truth_path,
output_path=data_path,
background_label=background_label,
channels_idx=channels_idx,
save_data=True,
seed=experiment_id,
train_size=train_size,
stratified=stratified)
freeze_model.main(model_path=model_path,
output_dir=experiment_dest_path)
node_names_file = os.path.join(experiment_dest_path,
'freeze_input_output_node_name.json')
frozen_graph_path = os.path.join(experiment_dest_path,
'frozen_graph.pb')
cmd = 'scripts/quantize.sh ' + node_names_file + ' ' \
+ frozen_graph_path + ' ' + data_path + ' ' + \
'?,{},1,1'.format(channels_count) + ' ' + \
'ml_intuition.data.input_fn.calibrate_2d_input' + ' ' + \
'128' + ' ' + experiment_dest_path + \
' ' + str(gpu)
subprocess.call(cmd, shell=True, env=os.environ.copy())
graph_path = os.path.join(experiment_dest_path,
'quantize_eval_model.pb')
evaluate_graph.main(graph_path=graph_path,
node_names_path=node_names_file,
dataset_path=data_path,
batch_size=batch_size)
if created_dataset:
os.remove(data_path)
artifacts_reporter.collect_artifacts_report(experiments_path=dest_path,
dest_path=dest_path,
filename='inference_graph_metrics.csv')
tf.keras.backend.clear_session()
def run_experiments(*,
data_file_paths: ('d', multi(min=1)),
ground_truth_path: str,
train_size: float = 0.8,
val_size: float = 0.1,
stratified: bool = True,
background_label: int = 0,
channels_idx: int = 0,
save_data: bool = False,
n_runs: int,
model_name: str,
kernel_size: int = 3,
n_kernels: int = 16,
n_layers: int = 1,
dest_path: str,
sample_size: int,
n_classes: int,
lr: float = 0.005,
batch_size: int = 150,
epochs: int = 10,
verbose: int = 2,
shuffle: bool = True,
patience: int = 3,
pre_noise: ('pre', multi(min=0)),
pre_noise_sets: ('spre', multi(min=0)),
post_noise: ('post', multi(min=0)),
post_noise_sets: ('spost', multi(min=0)),
noise_params: str = None):
"""
Function for running experiments given a set of hyperparameters.
:param data_file_paths: Paths to the data files. Supported types are:
.npy and .h5
:param ground_truth_path: Path to the ground-truth data file.
:param train_size: If float, should be between 0.0 and 1.0,
if stratified = True, it represents percentage of each
class to be extracted,
If float and stratified = False, it represents percentage of the
whole dataset to be extracted with samples drawn randomly,
regardless of their class.
If int and stratified = True, it represents number of samples
to be drawn from each class.
If int and stratified = False, it represents overall number of
samples to be drawn regardless of their class, randomly.
Defaults to 0.8
:param val_size: Should be between 0.0 and 1.0. Represents the percentage of
each class from the training set to be extracted as a
validation set, defaults to 0.1
:param stratified: Indicated whether the extracted training set should be
stratified, defaults to True
:param background_label: Label indicating the background in GT file
:param channels_idx: Index specifying the channels position in the provided
data
:param save_data: Whether to save the prepared dataset
:param n_runs: Number of total experiment runs.
:param model_name: Name of the model, it serves as a key in the
dictionary holding all functions returning models.
:param kernel_size: Size of ech kernel in each layer.
:param n_kernels: Number of kernels in each layer.
:param n_layers: Number of layers in the model.
:param dest_path: Path to where all experiment runs will be saved as subfolders
in this directory.
:param sample_size: Size of the input sample.
:param n_classes: Number of classes.
:param lr: Learning rate for the model, i.e., regulates the size of the step
in the gradient descent process.
:param batch_size: Size of the batch used in training phase,
it is the size of samples per gradient step.
:param epochs: Number of epochs for model to train.
:param verbose: Verbosity mode used in training, (0, 1 or 2).
:param shuffle: Boolean indicating whether to shuffle dataset
dataset_key each epoch.
:param patience: Number of epochs without improvement in order to
stop the training phase.
:param pre_noise: The list of names of noise injection methods before
the normalization transformations. Examplary names are "gaussian"
or "impulsive".
:param pre_noise_sets: The list of sets to which the noise will be
injected. One element can either be "train", "val" or "test".
:param post_noise: The list of names of noise injection metods after
the normalization transformations.
:param post_noise_sets: The list of sets to which the noise will be injected.
:param noise_params: JSON containing the parameter setting of injection methods.
Examplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}".
This JSON should include all parameters for noise injection
functions that are specified in pre_noise and post_noise arguments.
For the accurate description of each parameter, please
refer to the ml_intuition/data/noise.py module.
"""
for experiment_id in range(n_runs):
experiment_dest_path = os.path.join(dest_path,
'experiment_' + str(experiment_id))
if save_data:
data_source = os.path.join(experiment_dest_path, 'data.h5')
else:
data_source = None
os.makedirs(experiment_dest_path, exist_ok=True)
data_to_merge = []
for data_file_path in data_file_paths:
data = prepare_data.main(data_file_path=data_file_path,
ground_truth_path=ground_truth_path,
output_path=data_source,
train_size=train_size,
val_size=val_size,
stratified=stratified,
background_label=background_label,
channels_idx=channels_idx,
save_data=save_data,
seed=experiment_id)
del data[enums.Dataset.TEST]
data_to_merge.append(data)
data = utils.merge_datasets(data_to_merge)
del data_to_merge
if not save_data:
data_source = data
if len(pre_noise) > 0:
noise.inject_noise(data_source=data_source,
affected_subsets=pre_noise_sets,
noise_injectors=pre_noise,
noise_params=noise_params)
train_model.train(model_name=model_name,
kernel_size=kernel_size,
n_kernels=n_kernels,
n_layers=n_layers,
dest_path=experiment_dest_path,
data=data_source,
sample_size=sample_size,
n_classes=n_classes,
lr=lr,
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
shuffle=shuffle,
patience=patience,
noise=post_noise,
noise_sets=pre_noise_sets,
noise_params=noise_params)
tf.keras.backend.clear_session()
def run_experiments(*,
data_file_path: str,
ground_truth_path: str = None,
train_size: ('train_size', multi(min=0)),
val_size: float = 0.1,
sub_test_size: int = None,
channels_idx: int = -1,
neighborhood_size: int = None,
n_runs: int = 1,
model_name: str,
save_data: bool = 0,
dest_path: str = None,
sample_size: int,
n_classes: int,
lr: float = None,
batch_size: int = 256,
epochs: int = 100,
verbose: int = 2,
shuffle: bool = True,
patience: int = 15,
use_mlflow: bool = False,
endmembers_path: str = None,
experiment_name: str = None,
run_name: str = None):
"""
Function for running experiments on unmixing given a set of hyperparameters.
:param data_file_path: Path to the data file. Supported types are: .npy.
:param ground_truth_path: Path to the ground-truth data file.
:param train_size: If float, should be between 0.0 and 1.0,
if int, it represents number of samples to draw from data.
:type train_size: Union[int, float]
:param val_size: Should be between 0.0 and 1.0. Represents the
percentage of samples to extract from the training set.
:param sub_test_size: Number of pixels to subsample the test set
instead of performing the inference on the entire subset.
:param channels_idx: Index specifying the channels
position in the provided data.
:param neighborhood_size: Size of the spatial patch.
:param save_data: Boolean indicating whether to save the prepared dataset.
:param n_runs: Number of total experiment runs.
:param model_name: Name of the model, it serves as a key in the
dictionary holding all functions returning models.
:param dest_path: Path to the directory where all experiment runs
will be saved as subdirectories.
:param sample_size: Spectral size of the input sample.
:param n_classes: Number of classes.
:param lr: Learning rate for the model i.e., it regulates
the size of the step in the gradient descent process.
:param batch_size: Size of the batch used in training phase,
it is the number of samples to utilize per single gradient step.
:param epochs: Total number of epochs for model to train.
:param verbose: Verbosity mode used in training, (0, 1 or 2).
:param shuffle: Boolean indicating whether to shuffle dataset.
:param patience: Number of epochs without improvement in order to
stop the training phase.
:param use_mlflow: Boolean indicating whether to log metrics
and artifacts to mlflow.
:param endmembers_path: Path to the endmembers file containing
the average reflectances for each class. Used only when
'use_unmixing' is set to True.
:param experiment_name: Name of the experiment. Used only if
'use_mlflow' is set to True.
:param run_name: Name of the run. Used only if 'use_mlflow' is set to True.
"""
if use_mlflow:
args = locals()
mlflow.set_tracking_uri("http://beetle.mlflow.kplabs.pl")
mlflow.set_experiment(experiment_name)
mlflow.start_run(run_name=run_name)
log_params_to_mlflow(args)
log_tags_to_mlflow(args['run_name'])
if dest_path is None:
dest_path = os.path.join(os.path.curdir, "temp_artifacts")
for experiment_id in range(n_runs):
experiment_dest_path = os.path.join(
dest_path,
'{}_{}'.format(enums.Experiment.EXPERIMENT, str(experiment_id)))
os.makedirs(experiment_dest_path, exist_ok=True)
# Apply default literature hyperparameters:
if neighborhood_size is None and model_name in NEIGHBORHOOD_SIZES:
neighborhood_size = NEIGHBORHOOD_SIZES[model_name]
if lr is None and model_name in LEARNING_RATES:
lr = LEARNING_RATES[model_name]
data = prepare_data.main(data_file_path=data_file_path,
ground_truth_path=ground_truth_path,
train_size=parse_train_size(train_size),
val_size=val_size,
stratified=False,
background_label=-1,
channels_idx=channels_idx,
neighborhood_size=neighborhood_size,
save_data=save_data,
seed=experiment_id,
use_unmixing=True)
if sub_test_size is not None:
subsample_test_set(data[enums.Dataset.TEST], sub_test_size)
train_unmixing.train(model_name=model_name,
dest_path=experiment_dest_path,
data=data,
sample_size=sample_size,
neighborhood_size=neighborhood_size,
n_classes=n_classes,
lr=lr,
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
shuffle=shuffle,
patience=patience,
endmembers_path=endmembers_path,
seed=experiment_id)
evaluate_unmixing.evaluate(
model_path=os.path.join(experiment_dest_path, model_name),
data=data,
dest_path=experiment_dest_path,
neighborhood_size=neighborhood_size,
batch_size=batch_size,
endmembers_path=endmembers_path)
tf.keras.backend.clear_session()
artifacts_reporter.collect_artifacts_report(
experiments_path=dest_path,
dest_path=dest_path,
use_mlflow=use_mlflow)
if use_mlflow:
mlflow.log_artifacts(dest_path, artifact_path=dest_path)
shutil.rmtree(dest_path)
def _main(
*steps: (use_mixin(StepList), Parameter.REQUIRED),
each_line: 'l' = False, # noqa: F821
raw: 'r' = False, # noqa: F821
start: (int, 's') = 0, # noqa: F821
end: (int, 'e') = None, # noqa: F821
prelude: (multi(), 'p') = 'pass', # noqa: F821
pipe_name: Parameter.UNDOCUMENTED = PIPE_NAME,
no_default_fragments: Parameter.UNDOCUMENTED = False,
no_exception_handling: Parameter.UNDOCUMENTED = False,
show_fragments: Parameter.UNDOCUMENTED = False,
break_: Parameter.UNDOCUMENTED = False):
"""Feed data through a sequence of Python expressions.
:param steps: At least one Python expression (or suite) to execute
:param each_line: Process lines as strings rather than all of stdin as a file
:param start: Don't print before this result (zero-based)
:param end: Stop after getting this result (zero-based)
:param prelude: Execute a statement before running any steps. Can be specified more than once.
:param raw: Don't add helper functionality to stdin
"""
pipe_name = sys.intern(pipe_name)
spy.context = context = make_context()
for stmt in prelude:
exec(stmt, context, context.view())
step_src = steps
steps = []
for i, code in enumerate(step_src):
fragment_name = 'Fragment {}'.format(i + 1)
source = code
literal = False
if isinstance(code, _Decorated):
source = '{} {!r}'.format(code.name, code.value)
literal = code.LITERAL
code, funcseq = code.value, code.funcseq
else:
funcseq = ()
debuginfo = (fragment_name, source)
if literal:
ca = make_literal(code, context, pipe_name, debuginfo)
else:
try:
co, is_expr = compile_(code, filename=fragment_name)
except SyntaxError as e:
pretty_syntax_error(code, e)
if break_: # pragma: no cover
debugger()
sys.exit(1)
ca = make_callable(co, is_expr, context, pipe_name, debuginfo)
for fn in funcseq:
ca = fn(ca)
steps.append(spy.fragment(ca))
index_offset = 0
if not no_default_fragments:
steps.append(fragments.make_limit(start=start, end=end))
steps.append(fragments.print)
if each_line:
steps.insert(0, fragments.foreach)
index_offset -= 1
chain = spy.chain(steps, index_offset=index_offset)
if raw:
data = [sys.stdin]
else:
data = [SpyFile(sys.stdin)]
if show_fragments:
print(chain.format())
return
with ExitStack() as stack:
if not no_exception_handling:
stack.enter_context(catcher.handler(delete_all=True))
if break_: # pragma: no cover
stack.enter_context(DebuggerContext())
chain.run_to_exhaustion(data)
def main_(*,
config: 'c' = 'config.yml',
debug: 'd' = False,
quiet: 'q' = False,
extra: ('e', str, parameters.multi()) = None):
"""
Run cms7.
config: Path to project configuration
extra: Path to additional configuration (e.g. site local overrides). Can
be specified multiple times. Later configurations override.
debug: Print obnoxious debugging output
quiet: Only ever print warnings
"""
rl = logging.getLogger()
h = logging.StreamHandler()
try:
import colorlog
h.setFormatter(colorlog.ColoredFormatter(
"%(log_color)s%(levelname)-8s%(reset)s %(message_log_color)s%(name)s:%(message)s",
secondary_log_colors={
'message': {
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'red',
}
}))
except ImportError:
h.setFormatter(logging.Formatter(
"%(levelname)-8s %(name)s:%(message)s"))
rl.addHandler(h)
if debug:
rl.setLevel(logging.DEBUG)
elif quiet:
rl.setLevel(logging.WARNING)
else:
rl.setLevel(logging.INFO)
try:
cfg = _config.load(config, *extra)
gen = Generator(cfg)
for m in cfg.modules():
m.prepare()
for m in cfg.modules():
m.run(gen)
gen.run()
for r in cfg.resources:
r.run()
except CMS7Error as e:
logger.critical('%s', e.message, exc_info=debug)
if not debug:
logger.warning('exiting for exception. use --debug to get a traceback')
sys.exit(1)
except Exception:
logger.critical('unexpected exception', exc_info=True)
sys.exit(1)
def run_evaluation(
*,
model_hash: str,
thr: float = 0.5,
dpath: str = "datasets/clouds/38-Cloud/38-Cloud_test",
gtpath: str = "datasets/clouds/38-Cloud/38-Cloud_test/Entire_scene_gts",
vpath: str = "datasets/clouds/38-Cloud/38-Cloud_test/Natural_False_Color",
rpath: str = "artifacts/",
vids: ("v", multi(min=1)),
batch_size: int = 32,
img_ids: ("iid", multi(min=0)),
mlflow: bool = False,
run_name: str = None,
):
"""
Load model given model hash and get evaluation metrics on 38-Cloud testset.
:param model_hash: MLFlow hash of the model to load.
:param thr: threshold to be used during evaluation.
:param dpath: path to dataset.
:param gtpath: path to dataset ground truths.
:param vpath: path to dataset (false color) visualisation images.
:param rpath: path to directory where results
and artifacts should be logged.
:param vids: tuple of ids of images which should be used to create
visualisations. If contains '*' visualisations will be
created for all images in the dataset.
:type vids: tuple[str]
:param batch_size: size of generated batches, only one batch is loaded
to memory at a time.
:param img_ids: if given, process only these images.
:type img_ids: list[int]
:param mlflow: whether to use MLFlow.
:param run_name: name of the run.
"""
snow_imgs = [
"LC08_L1TP_064015_20160420_20170223_01_T1",
"LC08_L1TP_035035_20160120_20170224_01_T1",
"LC08_L1TP_050024_20160520_20170324_01_T1",
]
if img_ids == []:
img_ids = None
else:
snow_imgs = list(set(snow_imgs) & set(img_ids))
dpath, gtpath, vpath, rpath = make_paths(dpath, gtpath, vpath, rpath)
rpath = rpath / uuid.uuid4().hex
print(f"Working dir: {os.getcwd()}, "
+ f"artifacts dir: {rpath}",
flush=True)
mpath = Path(
f"/media/ML/mlflow/beetle/artifacts/34/{model_hash}/"
# Change init_model to model for old models
+ "artifacts/init_model/data/model.h5"
)
# WARNING: If ever upgraded to tf2.x, this way of using metrics will not work
# because loaded metrics will become MeanMetricWrapper objects in tf2.x and
# this script isn't prepared for such objects (because MeanMetricWrapper has state,
# as opposed to present stateless metric functions).
model = keras.models.load_model(
mpath,
custom_objects={
"jaccard_index_loss": losses.JaccardIndexLoss(),
"jaccard_index_metric": losses.JaccardIndexMetric(),
"dice_coeff_metric": losses.DiceCoefMetric(),
"recall": losses.recall,
"precision": losses.precision,
"specificity": losses.specificity,
# F1 score is needed for old models
# "f1_score": losses.f1_score,
"tf": tf,
},
)
model.load_weights(
f"/media/ML/mlflow/beetle/artifacts/34/{model_hash}/"
+ "artifacts/best_weights/best_weights"
)
metrics, _ = evaluate_model(
model=model,
thr=thr,
dpath=dpath,
gtpath=gtpath,
vpath=vpath,
rpath=rpath,
vids=vids,
batch_size=batch_size,
img_ids=img_ids,
mlflow=mlflow,
run_name=run_name,
)
mean_metrics = {}
mean_metrics_snow = {}
for key, value in metrics.items():
mean_metrics[key] = np.mean(list(value.values()))
mean_metrics_snow[f"snow_{key}"] = np.mean(
[value[x] for x in snow_imgs])
print(mean_metrics, mean_metrics_snow)
if mlflow:
log_metrics(mean_metrics)
log_metrics(mean_metrics_snow)
log_artifacts(rpath)
def evaluate(*,
data,
model_path: str,
dest_path: str,
n_classes: int,
batch_size: int = 1024,
use_ensemble: bool = False,
ensemble_copies: int = 1,
voting: str = 'hard',
noise: ('post', multi(min=0)),
noise_sets: ('spost', multi(min=0)),
noise_params: str = None,
seed: int = 0):
"""
Function for evaluating the trained model.
:param model_path: Path to the model.
:param data: Either path to the input data or the data dict.
:param dest_path: Directory in which to store the calculated metrics.
:param n_classes: Number of classes.
:param batch_size: Size of the batch for inference.
:param use_ensemble: Use ensemble for prediction.
:param ensemble_copies: Number of model copies for the ensemble.
:param voting: Method of ensemble voting. If ‘hard’, uses predicted class
labels for majority rule voting. Else if ‘soft’, predicts the class
label based on the argmax of the sums of the predicted probabilities.
:param noise: List containing names of used noise injection methods
that are performed after the normalization transformations.
:type noise: list[str]
:param noise_sets: List of sets that are affected by the noise injection.
For this module single element can be "test".
:type noise_sets: list[str]
:param noise_params: JSON containing the parameters
setting of noise injection methods.
Exemplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}".
This JSON should include all parameters for noise injection
functions that are specified in the noise argument.
For the accurate description of each parameter, please
refer to the ml_intuition/data/noise.py module.
:param seed: Seed for RNG.
"""
os.makedirs(dest_path, exist_ok=True)
if type(data) is str:
test_dict = io.extract_set(data, enums.Dataset.TEST)
else:
test_dict = copy(data[enums.Dataset.TEST])
min_max_path = os.path.join(os.path.dirname(model_path), "min-max.csv")
if os.path.exists(min_max_path):
min_value, max_value = io.read_min_max(min_max_path)
else:
min_value, max_value = data[enums.DataStats.MIN], \
data[enums.DataStats.MAX]
transformations = [transforms.SpectralTransform(),
transforms.OneHotEncode(n_classes=n_classes),
transforms.MinMaxNormalize(min_=min_value,
max_=max_value)]
transformations = transformations + \
get_noise_functions(noise, noise_params) \
if enums.Dataset.TEST in noise_sets else transformations
test_dict = transforms.apply_transformations(test_dict, transformations)
model = tf.keras.models.load_model(model_path, compile=True)
if use_ensemble:
model = Ensemble(model, voting=voting)
if ensemble_copies is not None:
noise_params = yaml.load(noise_params)
model.generate_models_with_noise(copies=ensemble_copies,
mean=noise_params['mean'],
seed=seed)
if voting == 'classifier':
if type(data) is str:
train_dict = io.extract_set(data, enums.Dataset.TRAIN)
else:
train_dict = data[enums.Dataset.TRAIN]
train_dict = transforms.apply_transformations(train_dict, transformations)
train_probabilities = model.predict_probabilities(train_dict[enums.Dataset.DATA])
model.train_ensemble_predictor(train_probabilities, train_dict[enums.Dataset.LABELS])
predict = timeit(model.predict)
y_pred, inference_time = predict(test_dict[enums.Dataset.DATA],
batch_size=batch_size)
if voting == 'classifier':
y_pred = np.argmax(y_pred, axis=-1)
y_true = np.argmax(test_dict[enums.Dataset.LABELS], axis=-1)
model_metrics = get_model_metrics(y_true, y_pred)
model_metrics['inference_time'] = [inference_time]
conf_matrix = confusion_matrix(y_true, y_pred)
io.save_metrics(dest_path=dest_path,
file_name=enums.Experiment.INFERENCE_METRICS,
metrics=model_metrics)
io.save_confusion_matrix(conf_matrix, dest_path)
if enums.Splits.GRIDS in model_path:
if type(data) is str:
train_dict = io.extract_set(data, enums.Dataset.TRAIN)
labels_in_train = np.unique(train_dict[enums.Dataset.LABELS])
else:
train_labels = data[enums.Dataset.TRAIN][enums.Dataset.LABELS]
if train_labels.ndim > 1:
train_labels = np.argmax(train_labels, axis=-1)
labels_in_train = np.unique(train_labels)
fair_metrics = get_fair_model_metrics(conf_matrix, labels_in_train)
io.save_metrics(dest_path=dest_path,
file_name=enums.Experiment.INFERENCE_FAIR_METRICS,
metrics=fair_metrics)
def run_experiments(*,
data_file_path: str = None,
ground_truth_path: str = None,
dataset_path: str = None,
train_size: ('train_size', multi(min=0)),
val_size: float = 0.1,
stratified: bool = True,
background_label: int = 0,
channels_idx: int = 0,
neighborhood_size: int = None,
save_data: bool = False,
n_runs: int,
dest_path: str,
models_path: str,
model_name: str = 'model_2d',
n_classes: int,
use_ensemble: bool = False,
ensemble_copies: int = None,
voting: str = 'hard',
batch_size: int = 1024,
post_noise_sets: ('spost', multi(min=0)),
post_noise: ('post', multi(min=0)),
noise_params: str = None,
use_mlflow: bool = False,
experiment_name: str = None,
model_exp_name: str = None,
run_name: str = None):
"""
Run inference on the provided model given set of hyperparameters.
:param data_file_path: Path to the data file. Supported types are: .npy
:param ground_truth_path: Path to the ground-truth data file.
:param dataset_path: Path to the already extracted .h5 dataset
:param train_size: If float, should be between 0.0 and 1.0.
If stratified = True, it represents percentage of each class to be extracted,
If float and stratified = False, it represents percentage of the whole
dataset to be extracted with samples drawn randomly, regardless of their class.
If int and stratified = True, it represents number of samples to be
drawn from each class.
If int and stratified = False, it represents overall number of samples
to be drawn regardless of their class, randomly.
Defaults to 0.8
:type train_size: Union[int, float]
:param val_size: Should be between 0.0 and 1.0. Represents the percentage of
each class from the training set to be extracted as a
validation set.
Defaults to 0.1.
:param stratified: Indicated whether the extracted training set should be
stratified.
Defaults to True.
:param background_label: Label indicating the background in GT file.
:param channels_idx: Index specifying the channels position in the provided
data.
:param neighborhood_size: Size of the neighbourhood for the model.
:param save_data: Whether to save the prepared dataset
:param n_runs: Number of total experiment runs.
:param dest_path: Path to where all experiment runs will be saved as
subfolders in this directory.
:param models_path: Name of the model, it serves as a key in the
dictionary holding all functions returning models.
:param model_name: The name of model for the inference.
:param n_classes: Number of classes.
:param use_ensemble: Use ensemble for prediction.
:param ensemble_copies: Number of model copies for the ensemble.
:param voting: Method of ensemble voting. If ‘hard’, uses predicted class
labels for majority rule voting. Else if ‘soft’, predicts the class
label based on the argmax of the sums of the predicted probabilities.
:param batch_size: Size of the batch for the inference
:param post_noise_sets: The list of sets to which the noise will be
injected. One element can either be "train", "val" or "test".
:type post_noise_sets: list[str]
:param post_noise: The list of names of noise injection methods after
the normalization transformations.
:type post_noise: list[str]
:param noise_params: JSON containing the parameter setting of injection methods.
Exemplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}".
This JSON should include all parameters for noise injection
functions that are specified in pre_noise and post_noise arguments.
For the accurate description of each parameter, please
refer to the ml_intuition/data/noise.py module.
:param use_mlflow: Whether to log metrics and artifacts to mlflow.
:param experiment_name: Name of the experiment. Used only if
use_mlflow = True.
:param run_name: Name of the run. Used only if use_mlflow = True.
"""
train_size = parse_train_size(train_size)
if use_mlflow:
args = locals()
mlflow.set_tracking_uri("http://beetle.mlflow.kplabs.pl")
mlflow.set_experiment(experiment_name)
mlflow.start_run(run_name=run_name)
log_params_to_mlflow(args)
log_tags_to_mlflow(args['run_name'])
models_path = get_mlflow_artifacts_path(models_path, model_exp_name)
for experiment_id in range(n_runs):
experiment_dest_path = os.path.join(
dest_path, 'experiment_' + str(experiment_id))
model_name_regex = re.compile('model_.*')
model_dir = os.path.join(models_path, f'experiment_{experiment_id}')
model_name = list(filter(model_name_regex.match, os.listdir(model_dir)))[0]
model_path = os.path.join(model_dir, model_name)
if dataset_path is None:
data_source = os.path.join(models_path,
'experiment_' + str(experiment_id),
'data.h5')
else:
data_source = dataset_path
os.makedirs(experiment_dest_path, exist_ok=True)
if data_file_path.endswith('.h5') and ground_truth_path is None and 'patches' not in data_file_path:
data_source = load_processed_h5(data_file_path=data_file_path)
elif not os.path.exists(data_source):
data_source = prepare_data.main(data_file_path=data_file_path,
ground_truth_path=ground_truth_path,
output_path=data_source,
train_size=train_size,
val_size=val_size,
stratified=stratified,
background_label=background_label,
channels_idx=channels_idx,
neighborhood_size=neighborhood_size,
save_data=save_data,
seed=experiment_id)
evaluate_model.evaluate(
model_path=model_path,
data=data_source,
dest_path=experiment_dest_path,
n_classes=n_classes,
use_ensemble=use_ensemble,
ensemble_copies=ensemble_copies,
voting=voting,
noise=post_noise,
noise_sets=post_noise_sets,
noise_params=noise_params,
batch_size=batch_size,
seed=experiment_id)
tf.keras.backend.clear_session()
artifacts_reporter.collect_artifacts_report(experiments_path=dest_path,
dest_path=dest_path,
use_mlflow=use_mlflow)
if Splits.GRIDS in data_file_path:
fair_report_path = os.path.join(dest_path, Experiment.REPORT_FAIR)
artifacts_reporter.collect_artifacts_report(experiments_path=dest_path,
dest_path=fair_report_path,
filename=Experiment.INFERENCE_FAIR_METRICS,
use_mlflow=use_mlflow)
if use_mlflow:
mlflow.set_experiment(experiment_name)
mlflow.log_artifacts(dest_path, artifact_path=dest_path)
shutil.rmtree(dest_path)
def run_experiments(*,
data_file_paths: ('d', multi(min=1)),
train_size: ('train_size', multi(min=0)),
val_size: float = 0.1,
stratified: bool = True,
background_label: int = 0,
channels_idx: int = 0,
neighborhood_sizes: ('n', multi(min=1)),
save_data: bool = False,
n_runs: int,
dest_path: str,
model_paths: ('m', multi(min=1)),
model_experiment_names: ('e', multi(min=1)),
n_classes: int,
voting: str = 'hard',
voting_model: str = None,
voting_model_params: str = None,
batch_size: int = 1024,
post_noise_sets: ('spost', multi(min=0)),
post_noise: ('post', multi(min=0)),
noise_params: str = None,
use_mlflow: bool = False,
experiment_name: str = None,
run_name: str = None,
use_unmixing: bool = False,
gt_file_paths: ('g', multi(min=0)),
sub_test_size: int = None):
"""
Function for running experiments given a set of hyper parameters.
:param data_file_paths: Path to the data file. Supported types are: .npy
:type data_file_paths: list[str]
:param train_size: If float, should be between 0.0 and 1.0.
If stratified = True, it represents percentage of each class to be extracted,
If float and stratified = False, it represents percentage of the whole
dataset to be extracted with samples drawn randomly, regardless of their class.
If int and stratified = True, it represents number of samples to be
drawn from each class.
If int and stratified = False, it represents overall number of samples
to be drawn regardless of their class, randomly.
Defaults to 0.8
:type train_size: Union[int, float]
:param val_size: Should be between 0.0 and 1.0. Represents the percentage of
each class from the training set to be extracted as a
validation set, defaults to 0.1
:param stratified: Indicated whether the extracted training set should be
stratified, defaults to True
:param background_label: Label indicating the background in GT file
:param channels_idx: Index specifying the channels position in the provided data.
:param neighborhood_sizes: List of sizes of neighborhoods of provided models.
:type neighborhood_sizes: list[str]
:param save_data: Whether to save the prepared dataset
:param n_runs: Number of total experiment runs.
:param dest_path: Path to where all experiment runs will be saved as
subfolders in this directory.
:param model_paths: Paths to all models to be used in ensemble
:type model_paths: list[str]
:param model_experiment_names: Names of MLFlow experiments
:type model_experiment_names: list[str]
:param n_classes: Number of classes.
:param voting: Method of ensemble voting. If ‘hard’, uses predicted class
labels for majority rule voting. Else if ‘soft’, predicts the class
label based on the argmax of the sums of the predicted probabilities.
Else if 'booster', employs a new model, which is trained on the
ensemble predictions on the training set.
:param voting_model: Type of model to use when the voting argument is set
to "booster". Supported models are: SVR, SVC, RFR, RFC, DTR, DTC.
:param voting_model_params: Parameters of the voting model, same
as the parameters for the noise injection.
:param batch_size: Size of the batch for the inference
:param post_noise_sets: The list of sets to which the noise will be
injected. One element can either be "train", "val" or "test".
:type post_noise_sets: list[str]
:param post_noise: The list of names of noise injection methods after
the normalization transformations.
:type post_noise: list[str]
:param noise_params: JSON containing the parameter setting of injection methods.
Exemplary value for this parameter: "{"mean": 0, "std": 1, "pa": 0.1}".
This JSON should include all parameters for noise injection
functions that are specified in pre_noise and post_noise arguments.
For the accurate description of each parameter, please
refer to the ml_intuition/data/noise.py module.
:param use_mlflow: Whether to log metrics and artifacts to mlflow.
:param experiment_name: Name of the experiment. Used only if
use_mlflow = True
:param run_name: Name of the run. Used only if use_mlflow = True.
:param use_unmixing: Boolean indicating whether
to utilize the unmixing functionality.
:param gt_file_paths: Path to the ground-truth data files.
Supported types are: .npy
:type gt_file_paths: list[str]
:param sub_test_size: Number of pixels to subsample from the test set
instead of performing the inference on all untrained samples.
"""
train_size = parse_train_size(train_size)
if use_mlflow:
args = locals()
mlflow.set_tracking_uri("http://beetle.mlflow.kplabs.pl")
mlflow.set_experiment(experiment_name)
mlflow.start_run(run_name=run_name)
log_params_to_mlflow(args)
log_tags_to_mlflow(args['run_name'])
for experiment_id in range(n_runs):
experiment_dest_path = os.path.join(dest_path,
'experiment_' + str(experiment_id))
os.makedirs(experiment_dest_path, exist_ok=True)
models_test_predictions = []
models_train_predictions = []
for data_file_path, model_path, model_experiment_name, \
neighborhood_size, gt_file_path in \
zip_longest(data_file_paths,
model_paths,
model_experiment_names,
neighborhood_sizes,
gt_file_paths,
fillvalue=None):
if use_mlflow:
model_path = get_mlflow_artifacts_path(model_path,
model_experiment_name)
model_name_regex = re.compile('unmixing_.*') \
if use_unmixing else re.compile('model_.*')
model_dir = os.path.join(model_path, f'experiment_{experiment_id}')
model_name = \
list(filter(model_name_regex.match, os.listdir(model_dir)))[0]
model_path = os.path.join(model_dir, model_name)
if neighborhood_size is not None:
neighborhood_size = int(neighborhood_size)
if data_file_path.endswith(
'.h5') and 'patches' not in data_file_path:
data_source = load_processed_h5(data_file_path=data_file_path)
else:
data_source = prepare_data.main(
data_file_path=data_file_path,
ground_truth_path=''
if gt_file_path is None else gt_file_path,
train_size=train_size,
val_size=val_size,
stratified=stratified,
background_label=background_label,
channels_idx=channels_idx,
neighborhood_size=neighborhood_size,
save_data=save_data,
seed=experiment_id,
use_unmixing=use_unmixing)
if sub_test_size is not None:
subsample_test_set(data_source[enums.Dataset.TEST],
sub_test_size)
test_predictions = predict_with_model.predict(
model_path=model_path,
data=data_source,
batch_size=batch_size,
dataset_to_predict=enums.Dataset.TEST,
use_unmixing=use_unmixing,
neighborhood_size=neighborhood_size)
models_test_predictions.append(test_predictions)
if voting == 'booster':
train_predictions = predict_with_model.predict(
model_path=model_path,
data=data_source,
batch_size=batch_size,
dataset_to_predict=enums.Dataset.TRAIN,
use_unmixing=use_unmixing)
models_train_predictions.append(train_predictions)
tf.keras.backend.clear_session()
models_test_predictions = np.asarray(models_test_predictions)
if use_unmixing:
evaluate_unmixing_with_ensemble.evaluate(
y_pred=models_test_predictions,
data=data_source,
dest_path=experiment_dest_path,
voting=voting,
train_set_predictions=models_train_predictions,
voting_model=voting_model,
voting_model_params=voting_model_params)
else:
evaluate_with_ensemble.evaluate(
y_pred=models_test_predictions,
model_path=model_path,
data=data_source,
dest_path=experiment_dest_path,
voting=voting,
train_set_predictions=models_train_predictions,
n_classes=n_classes,
voting_model=voting_model,
voting_model_params=voting_model_params)
artifacts_reporter.collect_artifacts_report(experiments_path=dest_path,
dest_path=dest_path,
use_mlflow=use_mlflow)
if not use_unmixing:
fair_report_path = os.path.join(dest_path, Experiment.REPORT_FAIR)
artifacts_reporter.collect_artifacts_report(
experiments_path=dest_path,
dest_path=fair_report_path,
filename=Experiment.INFERENCE_FAIR_METRICS,
use_mlflow=use_mlflow)
if use_mlflow:
mlflow.set_experiment(experiment_name)
mlflow.log_artifacts(dest_path, artifact_path=dest_path)
shutil.rmtree(dest_path)
