def _initialize_stat_values_store_if_needed(
self, shape: Tuple[int, ...]) -> None:
"""
Initialize storage for the benchmark statistics if it wasn't created yet.
:param shape: Shape of the stats map.
"""
if self.__tiledb_stats_array is not None and tiledb.array_exists(
self.__tiledb_stats_array):
return
# Create array with one dense dimension to store read statistics from the latest benchmark run.
dom = tiledb.Domain(
tiledb.Dim(name='n',
domain=(0, shape[0] - 1),
tile=shape[0] - 1,
dtype=np.int64),
tiledb.Dim(name='f',
domain=(0, shape[1] - 1),
tile=(shape[1] - 1),
dtype=np.int64))
# Schema contains one attribute for READ count
schema = tiledb.ArraySchema(
domain=dom,
sparse=False,
attrs=[tiledb.Attr(name='read', dtype=np.int32)])
# Create the (empty) array on disk.
tiledb.DenseArray.create(self.__tiledb_stats_array, schema)
# Fill with zeroes
with tiledb.DenseArray(self.__tiledb_stats_array, mode='w') as rr:
zero_data = np.zeros(shape, dtype=np.int32)
rr[:] = zero_data
def fragments_copy(uri_src, uri_dst, time_start, time_end, verbose, dry_run, force):
"""
(POSIX only). Copy a range of fragments from time-start to time-end (inclusive)
in an array located at uri-src to an array at uri-dst. If the array does not
exist, it will be created. The range may be formatted in UNIX seconds or ISO 8601.
"""
if not force:
prompt_poweruser()
if time_start:
time_start = to_unix_time(time_start)
if time_end:
time_end = to_unix_time(time_end)
copy_fragments_to_array = (
tiledb.copy_fragments_to_existing_array
if tiledb.array_exists(uri_dst)
else tiledb.create_array_from_fragments
)
copy_fragments_to_array(
uri_src,
uri_dst,
timestamp_range=(time_start, time_end),
verbose=verbose,
dry_run=dry_run,
)
def test_save_model_to_tiledb_array_without_compile_functional(self):
functional_model = testing_utils.get_small_functional_mlp(
num_hidden=1, num_classes=2, input_dim=3)
tiledb_uri = os.path.join(self.get_temp_dir(), "model_array")
tiledb_model_obj = TensorflowTileDB(uri=tiledb_uri)
tiledb_model_obj.save(model=functional_model, include_optimizer=False)
self.assertTrue(tiledb.array_exists(tiledb_uri))
def _initialize_stat_values_store_if_needed_read_obs_space(
self, registry_read: PRegistryReadClient) -> None:
"""
Initialize the TileDB stats array with no pre-supplied observations shape. It will retrieve, observations, and
create a new array based on that. Only use in cases where it's uncertain if the array was initialized or not but
there is no observation space yet.
:param registry_read: Read-only client for the registry.
"""
if self.__tiledb_stats_array is not None and tiledb.array_exists(
self.__tiledb_stats_array):
return
agent_nodes, fragments, obs = self.get_full_allocation_observation(
registry_read=registry_read)
# Expand observation space per agent to include read frequencies from the latest benchmark run or with
# default values if no benchmark values yet available
self._initialize_stat_values_store_if_needed(obs.shape)
def test_save_model_to_tiledb_array(self, tmpdir, api, loss, optimizer,
metrics):
model = (api(num_hidden=1, num_classes=2, input_dim=3)
if api != ConfigSubclassModel else api(
hidden_units=[16, 16, 10]))
tiledb_uri = os.path.join(tmpdir, "model_array")
# Compiles the model if optimizer is present
if optimizer:
model.compile(loss=loss, optimizer=optimizer, metrics=[metrics])
if not model.built:
model.build(tuple(np.random.randint(20, size=2)))
tiledb_model_obj = TensorflowKerasTileDBModel(uri=tiledb_uri,
model=model)
tiledb_model_obj.save(include_optimizer=True if optimizer else False)
assert tiledb.array_exists(tiledb_uri)