def get_model():
logger.info(f"Loading model from autosaved file: {tomo2seg_model.autosaved_model_path.name}")
model = tf.keras.models.load_model(
tomo2seg_model.autosaved_model_path_str,
compile=False
)
logger.debug("Changing the model's input type to accept any size of crop.")
in_ = model.layers[0]
in_shape = in_.input_shape[0]
input_n_channels = in_shape[-1:]
logger.debug(f"{input_n_channels=}")
# make it capable of getting any dimension in the input
anysize_input = layers.Input(
shape=[None, None, None] + list(input_n_channels),
name="input_any_image_size"
)
logger.debug(f"{anysize_input=}")
model.layers[0] = anysize_input
# todo keep this somewhere instead of copying and pasting
optimizer = optimizers.Adam()
loss_func = keras_custom_loss.jaccard2_loss
model.compile(loss=loss_func, optimizer=optimizer)
return model
def validate_partitions_to_compute(partitions_to_compute, volume):
if partitions_to_compute is None:
logger.info("Using all available parittions.")
return tuple(volume.metadata.set_partitions.keys())
assert len(partitions_to_compute) > 0
for part_alias in partitions_to_compute:
try:
volume[part_alias]
except KeyError as ex:
logger.exception(ex)
raise ValueError(
f"Invalid volume partition. {volume.fullname=} {partitions_to_compute=}"
)
return tuple(partitions_to_compute)
def get_2d_blob_props(
label_volume: ndarray,
data_volume: ndarray,
axes: Tuple[int] = (0, 1, 2),
parallel_nprocs: Optional[int] = None,
) -> DataFrame:
assert min(axes) >= 0, f"{min(axes)=}"
assert max(axes) <= 2, f"{max(axes)=}"
all_blob_props = []
for axis in axes:
logger.info(f"computing 2d_blob_props on plane normal to {axis=}")
all_blob_props.append(
get_slice_props_parallel(
label_volume,
data_volume,
normal_axis=axis,
nprocs=parallel_nprocs,
))
logger.debug("Converting 2d blob props dicts to data frames.")
for axis in axes:
blob_props = all_blob_props[axis]
ref_shape = len(blob_props["area"])
for k in blob_props.keys():
assert (shap := len(
blob_props[k])) == ref_shape, f"{k=} {shap=} {ref_shape=}"
all_blob_props[axis] = pd.DataFrame(blob_props)
logger.debug(f"{all_blob_props[axis].shape=}")
return pd.concat(all_blob_props, axis=0)
def __post_init__(self):
if self.runid is None:
self.runid = int(time.time())
logger.info(f"Using auto runid={self.runid}")
if self.random_state_seed is None:
self.random_state_seed = int(time.time()) % 1000
logger.info(
f"Using auto random_state_seed={self.random_state_seed}")
if self.host is None:
self.host = get_host()
logger.info(f"Using auto host={self.host.hostname=}")
'batch_norm': False,
'dropout': 0.10
},
'unet_down_kwargs': {'batchnorm': False},
'unet_up_kwargs': {'batchnorm': False}
}
try:
try:
t2s_model
except NameError:
logger.info("Creating a T2SModel.")
t2s_model = T2SModel(
model_master_name,
model_version,
runid=args.runid,
factory_function=model_factory_function,
factory_kwargs=model_factory_kwargs,
)
else:
logger.warning("The model is already defined. To create a new one: `del t2s_model`")
finally:
logger.info(f"t2s_model\n{dict2str(asdict(t2s_model))}")
logger.info(f"{t2s_model.name=}")
def notify_finished():
logger.info("Sending notification of finished training.")
notify("Training finished!")
def main():
# ===================================================== models =====================================================
# todo update these with the correct values
pa66gf30_proportions = [
.809861, # matrix
.189801, # fiber
.000338, # porosity
]
pa66gf30_classwise_histograms = np.load(
"../data/PA66GF30.v1/ground-truth-analysis/histogram-per-label.npy")
pa66gf30_models = [
UniformProbabilitiesClassifier(name="pa66gf30",
proportions=pa66gf30_proportions),
Order0Classifier(
name="pa66gf30",
n_classes=len(pa66gf30_proportions),
p0=pa66gf30_proportions[0],
),
BinwiseOrder0Classifier(
name="pa66gf30",
classwise_histograms=pa66gf30_classwise_histograms,
),
]
models = []
models.extend(pa66gf30_models)
# ===================================================== losses =====================================================
global_losses = [
attr for attr, _ in inspect.getmembers(TheoreticalModel)
if isinstance(attr, str) and attr.endswith("loss")
]
classwise_losses = [
attr for attr, _ in inspect.getmembers(TheoreticalModel)
if isinstance(attr, str) and attr.endswith("classwise_losses")
]
losses = global_losses + classwise_losses
logger.debug(f"{global_losses=} {classwise_losses=}")
logger.info(f"{losses=}")
# ===================================================== coeffs =====================================================
global_coeffs = [
attr for attr, _ in inspect.getmembers(TheoreticalModel)
if isinstance(attr, str) and attr.endswith("coeff")
]
classwise_coeffs = [
attr for attr, _ in inspect.getmembers(TheoreticalModel)
if isinstance(attr, str) and attr.endswith("classwise_coeffs")
]
coeffs = global_coeffs + classwise_coeffs
logger.debug(f"{global_coeffs=} {classwise_coeffs=}")
logger.info(f"{coeffs=}")
# ===================================================== table ======================================================
def get_value(model: TheoreticalModel, attr: str,
is_classwise: bool) -> str:
try:
val = getattr(model, attr)
except AttributeError:
return "not def"
if val is None:
return "not def"
if not is_classwise:
return f'{float(f"{val:.4g}"):.2%}'
else:
return ", ".join(f'{float(f"{v:.4g}"):.2%}' for v in val)
table_losses = pd.DataFrame(
data={
**{
attr: [getattr(model, attr) for model in models]
for attr in ["fullname"]
},
**{
attr.split("_loss")[0]: [
get_value(model, attr, False) for model in models
]
for attr in global_losses
},
**{
attr.split("_losses")[0]: [
get_value(model, attr, True) for model in models
]
for attr in classwise_losses
},
}).set_index("fullname")
table_coeffs = pd.DataFrame(
data={
**{
attr: [getattr(model, attr) for model in models]
for attr in ["fullname"]
},
**{
attr.split("_coeff")[0]: [
get_value(model, attr, False) for model in models
]
for attr in global_coeffs
},
**{
attr.split("_coeffs")[0]: [
get_value(model, attr, True) for model in models
]
for attr in classwise_coeffs
},
}).set_index("fullname")
print("losses:")
print(tabulate(table_losses, headers="keys", tablefmt="psql"))
print("coefficients:")
print(tabulate(table_coeffs, headers="keys", tablefmt="psql"))
def u_net(
input_shape,
nb_filters_0,
output_channels,
depth,
sigma_noise,
convlayer,
updown_conv_sampling,
unet_block_kwargs,
unet_down_kwargs,
unet_up_kwargs,
normlayer,
normtype=NormType.channel,
name=None,
norm_kwargs=dict(),
):
"""Modular U-Net.
Note that the dimensions of the input images should be
multiples of 16.
todo make this multichannel enabled
"""
unet_block_kwargs = {
**dict(
convlayer=convlayer,
normlayer=normlayer,
normtype=normtype,
norm_kwargs=norm_kwargs,
),
**unet_block_kwargs,
}
logger.info(f"unet_block_kwargs\n{dict2str(unet_block_kwargs)}")
unet_block = functools.partial(generic_unet_block, **unet_block_kwargs)
unet_down_kwargs = {
**dict(
conv_sampling=updown_conv_sampling,
convlayer=convlayer,
normlayer=normlayer,
normtype=normtype,
norm_kwargs=norm_kwargs,
),
**unet_down_kwargs,
}
logger.info(f"unet_down_kwargs\n{dict2str(unet_down_kwargs)}")
unet_down = functools.partial(generic_unet_down, **unet_down_kwargs)
unet_up_kwargs = {
**dict(
conv_sampling=updown_conv_sampling,
convlayer=convlayer,
normlayer=normlayer,
normtype=normtype,
norm_kwargs=norm_kwargs,
),
**unet_up_kwargs,
}
logger.info(f"unet_up_kwargs\n{dict2str(unet_up_kwargs)}")
unet_up = functools.partial(
generic_unet_up,
**unet_up_kwargs,
)
x = x0 = layers.Input(input_shape, name="input")
skips = {}
for i in range(depth):
nb_filters_begin = nb_filters_0 * 2**i
nb_filters_end = nb_filters_0 * 2**(i + 1)
x = unet_block(f"enc-block-{i}",
nb_filters_1=nb_filters_begin,
nb_filters_2=nb_filters_end)(x)
skips[i] = x
x = unet_down(nb_filters=nb_filters_end, name=f"enc-block-{i}")(x)
nb_filters_begin = nb_filters_0 * 2**depth
nb_filters_end = nb_filters_0 * 2**(depth + 1)
x = unet_block(f"enc-block-{depth}",
nb_filters_1=nb_filters_begin,
nb_filters_2=nb_filters_end)(x)
for i in reversed(range(depth)):
nb_filters_up = nb_filters_0 * 2**(i + 2)
nb_filters_conv = nb_filters_0 * 2**(i + 1)
x = unet_up(nb_filters=nb_filters_up, name=f"dec-block-{i}")(x)
x_skip = skips[i]
x = layers.concatenate([x_skip, x], axis=-1)
x = unet_block(f"dec-block-{i}",
nb_filters_1=nb_filters_conv,
nb_filters_2=nb_filters_conv)(x)
if sigma_noise > 0:
x = layers.GaussianNoise(sigma_noise, name="gaussian-noise")(x)
if convlayer in (ConvLayer.conv2d, ConvLayer.conv2d_separable):
x = layers.Conv2D(output_channels, 1, activation="softmax",
name="out")(x)
elif convlayer in (ConvLayer.conv3d, ConvLayer.conv3d_separable):
x = layers.Conv3D(output_channels, 1, activation="softmax",
name="out")(x)
else:
raise ValueError(f"{convlayer=}")
return Model(x0, x, name=name)
def __post_init__(self):
super().__post_init__()
# todo: move me to the parent class
logger.info(f"{self.__class__.__name__}\n{dict2str(asdict(self))}")
# # Setup
# In[4]:
logger.setLevel(logging.DEBUG)
# In[5]:
random_state = 42
random_state = np.random.RandomState(random_state)
runid = int(time.time())
logger.info(f"{runid=}")
# In[6]:
logger.debug(f"{tf.__version__=}")
logger.info(f"Num GPUs Available: {len(tf.config.list_physical_devices('GPU'))}\nThis should be 2 on R790-TOMO.")
logger.debug(f"Both here should return 2 devices...\n{tf.config.list_physical_devices('GPU')=}\n{tf.config.list_logical_devices('GPU')=}")
# xla auto-clustering optimization (see: https://www.tensorflow.org/xla#auto-clustering)
# this seems to break the training
tf.config.optimizer.set_jit(False)
# get a distribution strategy to use both gpus (see https://www.tensorflow.org/guide/distributed_training)
strategy = tf.distribute.MirroredStrategy()
model=tomo2seg_model,
set_partition=partition,
runid=runid,
)
# this is informal metadata for human use
estimation_volume["aggregation_strategy"] = args.aggregation_strategy.name
estimation_volume["cropping_strategy"] = args.cropping_strategy.name
estimation_volume["probabilities_dtype"] = args.probabilities_dtype.__name__
if args.opts.save_logs:
fh = logging.FileHandler(estimation_volume.exec_log_path_str)
fh.setFormatter(logger_get_formatter())
logger.addHandler(fh)
logger.info(f"Added a new file handler to the logger. {estimation_volume.exec_log_path_str=}")
logger.setLevel(logging.DEBUG)
# show inputs
# In[ ]:
logger.info(f"args\n{pprint_module.PrettyPrinter(indent=4, compact=False).pformat(dataclasses.asdict(args))}")
logger.info(f"{estimation_volume=}")
logger.debug(f"{volume=}")
logger.debug(f"{partition=}")
logger.debug(f"{tomo2seg_model=}")
if args.model_type == process.ModelType.input2halfd:
def get_model():
try:
best_autosaved_model_path = tomo2seg_model.autosaved2_best_model_path # it's a property
assert best_autosaved_model_path is not None, "no-autosaved2"
except ValueError as ex:
if ex.args[0] != "min() arg is an empty sequence":
raise ex
logger.warning(
f"{tomo2seg_model.name=} did not use autosaved2 apparently, falling back to autosaved."
)
best_autosaved_model_path = tomo2seg_model.autosaved_model_path
except AssertionError as ex:
if ex.args[0] != "no-autosaved2":
raise ex
logger.warning(
f"{tomo2seg_model.name=} did not use autosaved2 apparently, falling back to autosaved."
)
best_autosaved_model_path = tomo2seg_model.autosaved_model_path
print(best_autosaved_model_path)
logger.info(
f"Loading model from autosaved file: {best_autosaved_model_path.name}"
)
model = tf.keras.models.load_model(str(best_autosaved_model_path),
compile=False)
logger.debug(
"Changing the model's input type to accept any size of crop.")
in_ = model.layers[0]
in_shape = in_.input_shape[0]
input_n_channels = in_shape[-1]
logger.debug(f"{input_n_channels=}")
if input_n_channels > 1:
if args.model_type == Args.ModelType.input2halfd:
if len(in_shape) != 4:
raise f"len({in_shape=}) > 4, so this model must be multi-channel. Not supported yet..."
else:
raise NotImplementedError(f"{input_n_channels=} > 1")
# make it capable of getting any dimension in the input
# "-2" = 1 for the batch size, 1 for the nb.channels
anysize_target_shape = (len(in_shape) - 2) * [None] + [
input_n_channels
]
logger.debug(f"{anysize_target_shape=}")
anysize_input = layers.Input(shape=anysize_target_shape,
name="input_any_image_size")
logger.debug(f"{anysize_input=}")
model.layers[0] = anysize_input
# this doesn't really matter bc this script will not fit the model
optimizer = optimizers.Adam()
loss_func = keras_custom_loss.jaccard2_loss
logger.debug("Starting model compilation")
model.compile(loss=loss_func, optimizer=optimizer)
logger.debug("Done!")
return model
# [manual-input]
volume_name_version = t2s_datasets.VOLUME_COMPOSITE_V1
args = Args.setup00_process_test(
# 3d
model_type=Args.ModelType.input2d,
model_name=model_name,
volume_name=volume_name_version[0],
volume_version=volume_name_version[1],
script_name="process-volume-07.ipynb",
host=None, # get from socket.hostname
runid=None, # default is time.time()
random_state_seed=42, # None = auto value
)
logger.info(f"args={dict2str(asdict(args))}")
# build `tomo2seg` objects
tomo2seg_model = Tomo2SegModel.build_from_model_name(args.model_name)
volume = Volume.with_check(name=args.volume_name,
version=args.volume_version)
partition = volume[
args.partition_alias] if args.partition_alias is not None else None
estimation_volume = EstimationVolume.from_objects(
volume=volume,
model=tomo2seg_model,
set_partition=partition,
