def get_model(): logger.info(f"Loading model from autosaved file: {tomo2seg_model.autosaved_model_path.name}") model = tf.keras.models.load_model( tomo2seg_model.autosaved_model_path_str, compile=False ) logger.debug("Changing the model's input type to accept any size of crop.") in_ = model.layers[0] in_shape = in_.input_shape[0] input_n_channels = in_shape[-1:] logger.debug(f"{input_n_channels=}") # make it capable of getting any dimension in the input anysize_input = layers.Input( shape=[None, None, None] + list(input_n_channels), name="input_any_image_size" ) logger.debug(f"{anysize_input=}") model.layers[0] = anysize_input # todo keep this somewhere instead of copying and pasting optimizer = optimizers.Adam() loss_func = keras_custom_loss.jaccard2_loss model.compile(loss=loss_func, optimizer=optimizer) return model
def validate_partitions_to_compute(partitions_to_compute, volume): if partitions_to_compute is None: logger.info("Using all available parittions.") return tuple(volume.metadata.set_partitions.keys()) assert len(partitions_to_compute) > 0 for part_alias in partitions_to_compute: try: volume[part_alias] except KeyError as ex: logger.exception(ex) raise ValueError( f"Invalid volume partition. {volume.fullname=} {partitions_to_compute=}" ) return tuple(partitions_to_compute)
def get_2d_blob_props( label_volume: ndarray, data_volume: ndarray, axes: Tuple[int] = (0, 1, 2), parallel_nprocs: Optional[int] = None, ) -> DataFrame: assert min(axes) >= 0, f"{min(axes)=}" assert max(axes) <= 2, f"{max(axes)=}" all_blob_props = [] for axis in axes: logger.info(f"computing 2d_blob_props on plane normal to {axis=}") all_blob_props.append( get_slice_props_parallel( label_volume, data_volume, normal_axis=axis, nprocs=parallel_nprocs, )) logger.debug("Converting 2d blob props dicts to data frames.") for axis in axes: blob_props = all_blob_props[axis] ref_shape = len(blob_props["area"]) for k in blob_props.keys(): assert (shap := len( blob_props[k])) == ref_shape, f"{k=} {shap=} {ref_shape=}" all_blob_props[axis] = pd.DataFrame(blob_props) logger.debug(f"{all_blob_props[axis].shape=}") return pd.concat(all_blob_props, axis=0)
def __post_init__(self): if self.runid is None: self.runid = int(time.time()) logger.info(f"Using auto runid={self.runid}") if self.random_state_seed is None: self.random_state_seed = int(time.time()) % 1000 logger.info( f"Using auto random_state_seed={self.random_state_seed}") if self.host is None: self.host = get_host() logger.info(f"Using auto host={self.host.hostname=}")
'batch_norm': False, 'dropout': 0.10 }, 'unet_down_kwargs': {'batchnorm': False}, 'unet_up_kwargs': {'batchnorm': False} } try: try: t2s_model except NameError: logger.info("Creating a T2SModel.") t2s_model = T2SModel( model_master_name, model_version, runid=args.runid, factory_function=model_factory_function, factory_kwargs=model_factory_kwargs, ) else: logger.warning("The model is already defined. To create a new one: `del t2s_model`") finally: logger.info(f"t2s_model\n{dict2str(asdict(t2s_model))}") logger.info(f"{t2s_model.name=}")
def notify_finished(): logger.info("Sending notification of finished training.") notify("Training finished!")
def main(): # ===================================================== models ===================================================== # todo update these with the correct values pa66gf30_proportions = [ .809861, # matrix .189801, # fiber .000338, # porosity ] pa66gf30_classwise_histograms = np.load( "../data/PA66GF30.v1/ground-truth-analysis/histogram-per-label.npy") pa66gf30_models = [ UniformProbabilitiesClassifier(name="pa66gf30", proportions=pa66gf30_proportions), Order0Classifier( name="pa66gf30", n_classes=len(pa66gf30_proportions), p0=pa66gf30_proportions[0], ), BinwiseOrder0Classifier( name="pa66gf30", classwise_histograms=pa66gf30_classwise_histograms, ), ] models = [] models.extend(pa66gf30_models) # ===================================================== losses ===================================================== global_losses = [ attr for attr, _ in inspect.getmembers(TheoreticalModel) if isinstance(attr, str) and attr.endswith("loss") ] classwise_losses = [ attr for attr, _ in inspect.getmembers(TheoreticalModel) if isinstance(attr, str) and attr.endswith("classwise_losses") ] losses = global_losses + classwise_losses logger.debug(f"{global_losses=} {classwise_losses=}") logger.info(f"{losses=}") # ===================================================== coeffs ===================================================== global_coeffs = [ attr for attr, _ in inspect.getmembers(TheoreticalModel) if isinstance(attr, str) and attr.endswith("coeff") ] classwise_coeffs = [ attr for attr, _ in inspect.getmembers(TheoreticalModel) if isinstance(attr, str) and attr.endswith("classwise_coeffs") ] coeffs = global_coeffs + classwise_coeffs logger.debug(f"{global_coeffs=} {classwise_coeffs=}") logger.info(f"{coeffs=}") # ===================================================== table ====================================================== def get_value(model: TheoreticalModel, attr: str, is_classwise: bool) -> str: try: val = getattr(model, attr) except AttributeError: return "not def" if val is None: return "not def" if not is_classwise: return f'{float(f"{val:.4g}"):.2%}' else: return ", ".join(f'{float(f"{v:.4g}"):.2%}' for v in val) table_losses = pd.DataFrame( data={ **{ attr: [getattr(model, attr) for model in models] for attr in ["fullname"] }, **{ attr.split("_loss")[0]: [ get_value(model, attr, False) for model in models ] for attr in global_losses }, **{ attr.split("_losses")[0]: [ get_value(model, attr, True) for model in models ] for attr in classwise_losses }, }).set_index("fullname") table_coeffs = pd.DataFrame( data={ **{ attr: [getattr(model, attr) for model in models] for attr in ["fullname"] }, **{ attr.split("_coeff")[0]: [ get_value(model, attr, False) for model in models ] for attr in global_coeffs }, **{ attr.split("_coeffs")[0]: [ get_value(model, attr, True) for model in models ] for attr in classwise_coeffs }, }).set_index("fullname") print("losses:") print(tabulate(table_losses, headers="keys", tablefmt="psql")) print("coefficients:") print(tabulate(table_coeffs, headers="keys", tablefmt="psql"))
def u_net( input_shape, nb_filters_0, output_channels, depth, sigma_noise, convlayer, updown_conv_sampling, unet_block_kwargs, unet_down_kwargs, unet_up_kwargs, normlayer, normtype=NormType.channel, name=None, norm_kwargs=dict(), ): """Modular U-Net. Note that the dimensions of the input images should be multiples of 16. todo make this multichannel enabled """ unet_block_kwargs = { **dict( convlayer=convlayer, normlayer=normlayer, normtype=normtype, norm_kwargs=norm_kwargs, ), **unet_block_kwargs, } logger.info(f"unet_block_kwargs\n{dict2str(unet_block_kwargs)}") unet_block = functools.partial(generic_unet_block, **unet_block_kwargs) unet_down_kwargs = { **dict( conv_sampling=updown_conv_sampling, convlayer=convlayer, normlayer=normlayer, normtype=normtype, norm_kwargs=norm_kwargs, ), **unet_down_kwargs, } logger.info(f"unet_down_kwargs\n{dict2str(unet_down_kwargs)}") unet_down = functools.partial(generic_unet_down, **unet_down_kwargs) unet_up_kwargs = { **dict( conv_sampling=updown_conv_sampling, convlayer=convlayer, normlayer=normlayer, normtype=normtype, norm_kwargs=norm_kwargs, ), **unet_up_kwargs, } logger.info(f"unet_up_kwargs\n{dict2str(unet_up_kwargs)}") unet_up = functools.partial( generic_unet_up, **unet_up_kwargs, ) x = x0 = layers.Input(input_shape, name="input") skips = {} for i in range(depth): nb_filters_begin = nb_filters_0 * 2**i nb_filters_end = nb_filters_0 * 2**(i + 1) x = unet_block(f"enc-block-{i}", nb_filters_1=nb_filters_begin, nb_filters_2=nb_filters_end)(x) skips[i] = x x = unet_down(nb_filters=nb_filters_end, name=f"enc-block-{i}")(x) nb_filters_begin = nb_filters_0 * 2**depth nb_filters_end = nb_filters_0 * 2**(depth + 1) x = unet_block(f"enc-block-{depth}", nb_filters_1=nb_filters_begin, nb_filters_2=nb_filters_end)(x) for i in reversed(range(depth)): nb_filters_up = nb_filters_0 * 2**(i + 2) nb_filters_conv = nb_filters_0 * 2**(i + 1) x = unet_up(nb_filters=nb_filters_up, name=f"dec-block-{i}")(x) x_skip = skips[i] x = layers.concatenate([x_skip, x], axis=-1) x = unet_block(f"dec-block-{i}", nb_filters_1=nb_filters_conv, nb_filters_2=nb_filters_conv)(x) if sigma_noise > 0: x = layers.GaussianNoise(sigma_noise, name="gaussian-noise")(x) if convlayer in (ConvLayer.conv2d, ConvLayer.conv2d_separable): x = layers.Conv2D(output_channels, 1, activation="softmax", name="out")(x) elif convlayer in (ConvLayer.conv3d, ConvLayer.conv3d_separable): x = layers.Conv3D(output_channels, 1, activation="softmax", name="out")(x) else: raise ValueError(f"{convlayer=}") return Model(x0, x, name=name)
def __post_init__(self): super().__post_init__() # todo: move me to the parent class logger.info(f"{self.__class__.__name__}\n{dict2str(asdict(self))}")
# # Setup # In[4]: logger.setLevel(logging.DEBUG) # In[5]: random_state = 42 random_state = np.random.RandomState(random_state) runid = int(time.time()) logger.info(f"{runid=}") # In[6]: logger.debug(f"{tf.__version__=}") logger.info(f"Num GPUs Available: {len(tf.config.list_physical_devices('GPU'))}\nThis should be 2 on R790-TOMO.") logger.debug(f"Both here should return 2 devices...\n{tf.config.list_physical_devices('GPU')=}\n{tf.config.list_logical_devices('GPU')=}") # xla auto-clustering optimization (see: https://www.tensorflow.org/xla#auto-clustering) # this seems to break the training tf.config.optimizer.set_jit(False) # get a distribution strategy to use both gpus (see https://www.tensorflow.org/guide/distributed_training) strategy = tf.distribute.MirroredStrategy()
model=tomo2seg_model, set_partition=partition, runid=runid, ) # this is informal metadata for human use estimation_volume["aggregation_strategy"] = args.aggregation_strategy.name estimation_volume["cropping_strategy"] = args.cropping_strategy.name estimation_volume["probabilities_dtype"] = args.probabilities_dtype.__name__ if args.opts.save_logs: fh = logging.FileHandler(estimation_volume.exec_log_path_str) fh.setFormatter(logger_get_formatter()) logger.addHandler(fh) logger.info(f"Added a new file handler to the logger. {estimation_volume.exec_log_path_str=}") logger.setLevel(logging.DEBUG) # show inputs # In[ ]: logger.info(f"args\n{pprint_module.PrettyPrinter(indent=4, compact=False).pformat(dataclasses.asdict(args))}") logger.info(f"{estimation_volume=}") logger.debug(f"{volume=}") logger.debug(f"{partition=}") logger.debug(f"{tomo2seg_model=}") if args.model_type == process.ModelType.input2halfd:
def get_model(): try: best_autosaved_model_path = tomo2seg_model.autosaved2_best_model_path # it's a property assert best_autosaved_model_path is not None, "no-autosaved2" except ValueError as ex: if ex.args[0] != "min() arg is an empty sequence": raise ex logger.warning( f"{tomo2seg_model.name=} did not use autosaved2 apparently, falling back to autosaved." ) best_autosaved_model_path = tomo2seg_model.autosaved_model_path except AssertionError as ex: if ex.args[0] != "no-autosaved2": raise ex logger.warning( f"{tomo2seg_model.name=} did not use autosaved2 apparently, falling back to autosaved." ) best_autosaved_model_path = tomo2seg_model.autosaved_model_path print(best_autosaved_model_path) logger.info( f"Loading model from autosaved file: {best_autosaved_model_path.name}" ) model = tf.keras.models.load_model(str(best_autosaved_model_path), compile=False) logger.debug( "Changing the model's input type to accept any size of crop.") in_ = model.layers[0] in_shape = in_.input_shape[0] input_n_channels = in_shape[-1] logger.debug(f"{input_n_channels=}") if input_n_channels > 1: if args.model_type == Args.ModelType.input2halfd: if len(in_shape) != 4: raise f"len({in_shape=}) > 4, so this model must be multi-channel. Not supported yet..." else: raise NotImplementedError(f"{input_n_channels=} > 1") # make it capable of getting any dimension in the input # "-2" = 1 for the batch size, 1 for the nb.channels anysize_target_shape = (len(in_shape) - 2) * [None] + [ input_n_channels ] logger.debug(f"{anysize_target_shape=}") anysize_input = layers.Input(shape=anysize_target_shape, name="input_any_image_size") logger.debug(f"{anysize_input=}") model.layers[0] = anysize_input # this doesn't really matter bc this script will not fit the model optimizer = optimizers.Adam() loss_func = keras_custom_loss.jaccard2_loss logger.debug("Starting model compilation") model.compile(loss=loss_func, optimizer=optimizer) logger.debug("Done!") return model
# [manual-input] volume_name_version = t2s_datasets.VOLUME_COMPOSITE_V1 args = Args.setup00_process_test( # 3d model_type=Args.ModelType.input2d, model_name=model_name, volume_name=volume_name_version[0], volume_version=volume_name_version[1], script_name="process-volume-07.ipynb", host=None, # get from socket.hostname runid=None, # default is time.time() random_state_seed=42, # None = auto value ) logger.info(f"args={dict2str(asdict(args))}") # build `tomo2seg` objects tomo2seg_model = Tomo2SegModel.build_from_model_name(args.model_name) volume = Volume.with_check(name=args.volume_name, version=args.volume_version) partition = volume[ args.partition_alias] if args.partition_alias is not None else None estimation_volume = EstimationVolume.from_objects( volume=volume, model=tomo2seg_model, set_partition=partition,