def build_optimizer(cfg_solver: dict) -> optimizer_v2.OptimizerV2:
"""Build the optimizer.
Args:
cfg_solver: dict, solver subsection of config.
Returns:
optimizer_v2.OptimizerV2, tf.keras v2 optimizer.
Raises:
TypeError, optimizer not an OptimizerV2.
"""
path = cfg_solver["optimizer"]["import"]
params = cfg_solver["optimizer"].get("params", {})
learning_rate = cfg_solver["optimizer"]["learning_rate"]
if isinstance(learning_rate, dict):
lr_cls = import_utils.import_obj_with_search_modules(
learning_rate["import"], ["tensorflow.keras.optimizers.schedules"])
lr = lr_cls(**learning_rate["params"])
if not isinstance(lr, learning_rate_schedule.LearningRateSchedule):
raise TypeError(
f"import learning rate: {lr} is not a LearningRateSchedule")
else:
lr = learning_rate
opt_cls = import_utils.import_obj_with_search_modules(
path, ["tensorflow.keras.optimizers"], True)
opt = opt_cls(learning_rate=lr, **params)
if not isinstance(opt, optimizer_v2.OptimizerV2):
raise TypeError(f"import optimizer: {opt} is not an OptimizerV2")
return opt
def test_import_obj_with_search_modules(python_path, search_modules,
both_cases, expected):
if expected:
obj = import_utils.import_obj_with_search_modules(
python_path, search_modules, both_cases)
assert obj() == "test"
else:
with pytest.raises(ImportError):
import_utils.import_obj_with_search_modules(
python_path, search_modules, both_cases)
def sequential_from_config(layers: List[dict], **kwargs) -> tf.keras.Model:
"""Build a sequential model from a list of layer specifications.
Supports references to network_params computed inside Transformers by specifying
{{variable name}}.
Args:
layers: list[dict], layer imports.
Returns:
tf.keras.Model, network.
"""
layers = config._render_params(layers, kwargs)
network = tf.keras.models.Sequential()
for layer in layers:
if "import" not in layer:
raise KeyError(f"layer {layer} missing 'import' key")
if not layer.keys() <= {"import", "params"}:
unexpected_keys = set(layer.keys()).difference(
{"import", "params"})
raise KeyError(
f"layer {layer} unexpected key(s): {unexpected_keys}")
layer_cls = import_utils.import_obj_with_search_modules(
layer["import"], search_modules=["tensorflow.keras.layers"])
layer_params = layer.get("params", {})
network.add(layer_cls(**layer_params))
return network
def build_network(cfg_model: dict, transform_params: dict) -> tf.keras.Model:
"""Build the network.
Args:
cfg_model: dict, model subsection of config.
transform_params: dict, params from transformer.
Returns:
tf.keras.Model, network.
Raises:
TypeError, network not a tf.keras.Model.
"""
path = cfg_model["network"]["import"]
network_params = cfg_model["network"].get("params", {})
net_func = import_utils.import_obj_with_search_modules(path)
net = net_func(**network_params, **transform_params)
if not isinstance(net, tf.keras.Model):
raise TypeError(f"import network: {net} is not a tf.keras.Model")
return net
def __init__(
self,
artifact_dir: str,
cfg_dataset: dict,
records: Union[pd.DataFrame, core.Records],
mode: core.RecordMode,
batch_size: int,
):
if not isinstance(mode, core.RecordMode):
raise TypeError("mode must be type RecordMode")
if isinstance(records, pd.DataFrame):
records.reset_index(drop=True, inplace=True)
self.records = records.to_dict(orient="records")
elif all(isinstance(record, dict) for record in records):
self.records = records
else:
raise TypeError(
"record must be a list of dicts or pandas DataFrame")
self.num_records = len(records)
logger.info(f"Building {mode} dataset with {self.num_records} records")
self.mode = mode
self.batch_size = batch_size
self.seed = cfg_dataset.get("seed")
np.random.seed(self.seed)
sample_count = cfg_dataset.get("sample_count")
if self.mode == core.RecordMode.TRAIN and sample_count is not None:
self._sample_inds = convert_sample_count_to_inds(
[record[sample_count] for record in self.records])
else:
self._sample_inds = list(range(self.num_records))
self.shuffle()
logger.info(f"Creating record loader")
loader_cls = import_utils.import_obj_with_search_modules(
cfg_dataset["loader"]["import"], search_modules=SEARCH_MODULES)
self.loader = loader_cls(mode=mode,
params=cfg_dataset["loader"].get(
"params", {}))
if not isinstance(self.loader, RecordLoader):
raise TypeError(
f"loader {self.loader} is not of type RecordLoader")
logger.info(f"Creating record transformer")
transformer_cls = import_utils.import_obj_with_search_modules(
cfg_dataset["transformer"]["import"],
search_modules=SEARCH_MODULES)
self.transformer = transformer_cls(
mode=self.mode,
loader=self.loader,
params=cfg_dataset["transformer"].get("params", {}),
)
if not isinstance(self.transformer, RecordTransformer):
raise TypeError(
f"transformer {self.transformer} is not of type RecordTransformer"
)
dataset_dir = os.path.join(artifact_dir, "dataset")
if self.mode == core.RecordMode.TRAIN:
logger.info("Creating record augmentor")
self.augmentor = RecordAugmentor(cfg_dataset["augmentor"])
logger.info(
f"Fitting transform: {self.transformer.__class__.__name__}")
self.transformer.fit(copy.deepcopy(self.records))
logger.info(
f"Transformer network params: {self.transformer.network_params}"
)
logger.info("Saving transformer")
self.transformer.save(dataset_dir)
else:
logger.info(
f"Loading transform: {self.transformer.__class__.__name__}")
self.transformer.load(dataset_dir)