def from_files(files, read_fct): """ :param files: collections of nz absolute file path names that point to 2D images with same nx*ny size. One 2D image at evenly spaced z position. So a 3D array (nx, ny, nz) is obtained :param read_fct: fonction that gives for a path filename a python array (with array.shape, array.dtype) :return: a RasGMArray """ array = read_fct(files[0]) gm_array = RasGMArray() shape = (len(files)-1,) + array.shape gm_array.spatial_domain = MInterval.from_shape(shape) gm_array.type_name = RasGMArrayBuilder.get_type_name( array.dtype, gm_array.spatial_domain.shape ) gm_array.type_length = array.dtype.itemsize gm_array.data = b'' storage_layout = FileStorageLayout(files, read_fct) storage_layout.spatial_domain = MInterval.from_shape((1,) + array.shape) gm_array.storage_layout = storage_layout print(gm_array) return gm_array
def from_np_array(array: np.array): """ :param array: the numpy array used to create this new RasGMArray :return: a RasGMArray """ gm_array = RasGMArray() shape = array.shape intervals = [] for i_max in shape: intervals.append(SInterval(0, i_max - 1)) gm_array.spatial_domain = MInterval(intervals) gm_array.type_name = RasGMArrayBuilder.get_type_name( array.dtype, array.shape) gm_array.type_length = array.dtype.itemsize gm_array.data = bytes(array) storage_layout = BandStorageLayout() storage_layout.compute_spatial_domain(gm_array) gm_array.storage_layout = storage_layout return gm_array
def _create_ras_array(self, barray, offset, step_size, y_interval, z_interval): ras_array = RasGMArray() ras_array.spatial_domain = MInterval( [z_interval, y_interval, self.storage_intervals[2]]) ras_array.storage_layout = RasStorageLayOut(self.gm_array.type_length, ras_array.spatial_domain) ras_array.type_name = self.gm_array.type_name ras_array.type_length = self.gm_array.type_length ras_array.data = barray[offset:offset + step_size - 1] return ras_array
def _create_ras_array(self, size): i_interval = SInterval(self.lo, self.hi) print( f"get buffer from {self.offset} to {self.offset + size - 1} with {i_interval}" ) ras_array = RasGMArray() ras_array.spatial_domain = MInterval( [i_interval, self.storage_intervals[1], self.storage_intervals[2]]) ras_array.storage_layout = self.gm_array.storage_layout ras_array.type_name = self.gm_array.type_name ras_array.type_length = self.gm_array.type_length ras_array.data = bytes(self.barray[self.offset:self.offset + size - 1]) return ras_array
def from_file(file_path, mdd_domain=None, mdd_type=RasGMArray.DEFAULT_MDD_TYPE, mdd_type_length=RasGMArray.DEFAULT_TYPE_LENGTH, tile_domain=None, tile_size=RasStorageLayOut.DEFAULT_TILE_SIZE): if not os.path.isfile(file_path): raise Exception("File {} to be inserted to rasdaman collection does not exist.".format(file_path)) # The data sent to rasserver is binary file content, not file_path with open(file_path, mode='rb') as file: data = file.read() # NOTE: with insert into values decode($1), it doesn't need to specify --mdddomain and --mddtype # but with values $1 it needs to specify these parameters if mdd_domain is None: mdd_domain = "[0:" + str(len(data) - 1) + "]" if mdd_type is None: mdd_type = RasGMArray.DEFAULT_MDD_TYPE # Parse str values of mdd_domain and tile_domain to MInterval objects mdd_domain = MInterval.from_str(mdd_domain) # tile* are not parameters from "insert into value from file" in rasql client, so calculate it internally if tile_domain is None: dim = len(mdd_domain.intervals) tile_domain = get_tiling_domain(dim, mdd_type_length, tile_size) else: tile_domain = MInterval.from_str(tile_domain) # Create helper objects to build POST query string to rasserver storage_layout = RasStorageLayOut(spatial_domain=tile_domain, tile_size=tile_size) return RasGMArray(mdd_domain, mdd_type, mdd_type_length, data, storage_layout)
def _get_mdd_core(self, mdd_resp): if mdd_resp.current_format != rDataFormat.r_Array.value: # So far, for other format than binary array, only one tile is used. tileresp = rassrvr_get_next_tile( self.transaction.database.stub, self.transaction.database.connection.session.clientId) tile_status = tileresp.status if tile_status == 3: raise Exception( f"multipart format {mdd_resp.current_format} is not handled" ) if tile_status == 4: raise Exception( "rpcGetNextTile - no tile to transfer or empty collection") return tile_status, tileresp.data # At that point, we try to read a binary array. mdd_result = None tile_idx = 0 tile_assigner = None read_size = 0 tile_status = 2 # call get_next_tile at least once while tile_status == 2 or tile_status == 3: tileresp = rassrvr_get_next_tile( self.transaction.database.stub, self.transaction.database.connection.session.clientId) tile_status = tileresp.status if tile_status == 4: raise Exception( "rpcGetNextTile - no tile to transfer or empty collection") if mdd_result is None: mdd_result = RasGMArray(spatial_domain=MInterval.from_str( mdd_resp.domain), type_length=tileresp.cell_type_length) #print(f"full domain {mdd_result.spatial_domain} with format {mdd_result.format}") #print(f"need to read {mdd_result.byte_size} bytes") tile_idx += 1 #print(f"tile {tile_idx}") # Creates current tile array mdd_tile = RasGMArray(spatial_domain=MInterval.from_str( tileresp.domain), type_length=mdd_result.type_length) if tile_status == 3: tile_status = self.read_tile_by_parts(mdd_tile, tileresp.data) else: mdd_tile.data = tileresp.data # At that point, data array of the mdd_tile should be complete for its spatial_domain read_size += len(mdd_tile.data) #print(f"read {read_size} bytes so far") #print(f"tile domain {mdd_tile.spatial_domain}") if tile_status < 2 and tile_idx == 1 and str( mdd_tile.spatial_domain) == str(mdd_result.spatial_domain): # MDD consists of just one tile that is the same size of the mdd mdd_result.data = mdd_tile.data else: # MDD consists of more than one tile or the tile does not cover the whole domain if tile_idx == 1: mdd_result.data = bytearray(mdd_result.byte_size) tile_assigner = TileAssigner(mdd_result) tile_assigner.start() # copy tile data into global MDD data space # optimized, relying on the internal representation of an MDD tile_assigner.add_tile(mdd_tile) if tile_assigner is not None: tile_assigner.go_on = False tile_assigner.join() return tile_status, mdd_result.data