def _initialize_stat_values_store_if_needed( self, shape: Tuple[int, ...]) -> None: """ Initialize storage for the benchmark statistics if it wasn't created yet. :param shape: Shape of the stats map. """ if self.__tiledb_stats_array is not None and tiledb.array_exists( self.__tiledb_stats_array): return # Create array with one dense dimension to store read statistics from the latest benchmark run. dom = tiledb.Domain( tiledb.Dim(name='n', domain=(0, shape[0] - 1), tile=shape[0] - 1, dtype=np.int64), tiledb.Dim(name='f', domain=(0, shape[1] - 1), tile=(shape[1] - 1), dtype=np.int64)) # Schema contains one attribute for READ count schema = tiledb.ArraySchema( domain=dom, sparse=False, attrs=[tiledb.Attr(name='read', dtype=np.int32)]) # Create the (empty) array on disk. tiledb.DenseArray.create(self.__tiledb_stats_array, schema) # Fill with zeroes with tiledb.DenseArray(self.__tiledb_stats_array, mode='w') as rr: zero_data = np.zeros(shape, dtype=np.int32) rr[:] = zero_data
def fragments_copy(uri_src, uri_dst, time_start, time_end, verbose, dry_run, force): """ (POSIX only). Copy a range of fragments from time-start to time-end (inclusive) in an array located at uri-src to an array at uri-dst. If the array does not exist, it will be created. The range may be formatted in UNIX seconds or ISO 8601. """ if not force: prompt_poweruser() if time_start: time_start = to_unix_time(time_start) if time_end: time_end = to_unix_time(time_end) copy_fragments_to_array = ( tiledb.copy_fragments_to_existing_array if tiledb.array_exists(uri_dst) else tiledb.create_array_from_fragments ) copy_fragments_to_array( uri_src, uri_dst, timestamp_range=(time_start, time_end), verbose=verbose, dry_run=dry_run, )
def test_save_model_to_tiledb_array_without_compile_functional(self): functional_model = testing_utils.get_small_functional_mlp( num_hidden=1, num_classes=2, input_dim=3) tiledb_uri = os.path.join(self.get_temp_dir(), "model_array") tiledb_model_obj = TensorflowTileDB(uri=tiledb_uri) tiledb_model_obj.save(model=functional_model, include_optimizer=False) self.assertTrue(tiledb.array_exists(tiledb_uri))
def _initialize_stat_values_store_if_needed_read_obs_space( self, registry_read: PRegistryReadClient) -> None: """ Initialize the TileDB stats array with no pre-supplied observations shape. It will retrieve, observations, and create a new array based on that. Only use in cases where it's uncertain if the array was initialized or not but there is no observation space yet. :param registry_read: Read-only client for the registry. """ if self.__tiledb_stats_array is not None and tiledb.array_exists( self.__tiledb_stats_array): return agent_nodes, fragments, obs = self.get_full_allocation_observation( registry_read=registry_read) # Expand observation space per agent to include read frequencies from the latest benchmark run or with # default values if no benchmark values yet available self._initialize_stat_values_store_if_needed(obs.shape)
def test_save_model_to_tiledb_array(self, tmpdir, api, loss, optimizer, metrics): model = (api(num_hidden=1, num_classes=2, input_dim=3) if api != ConfigSubclassModel else api( hidden_units=[16, 16, 10])) tiledb_uri = os.path.join(tmpdir, "model_array") # Compiles the model if optimizer is present if optimizer: model.compile(loss=loss, optimizer=optimizer, metrics=[metrics]) if not model.built: model.build(tuple(np.random.randint(20, size=2))) tiledb_model_obj = TensorflowKerasTileDBModel(uri=tiledb_uri, model=model) tiledb_model_obj.save(include_optimizer=True if optimizer else False) assert tiledb.array_exists(tiledb_uri)