def from_files(files, read_fct):
"""
:param files: collections of nz absolute file path names that point to 2D images with same nx*ny size.
One 2D image at evenly spaced z position. So a 3D array (nx, ny, nz) is obtained
:param read_fct: fonction that gives for a path filename a python array (with array.shape, array.dtype)
:return: a RasGMArray
"""
array = read_fct(files[0])
gm_array = RasGMArray()
shape = (len(files)-1,) + array.shape
gm_array.spatial_domain = MInterval.from_shape(shape)
gm_array.type_name = RasGMArrayBuilder.get_type_name(
array.dtype,
gm_array.spatial_domain.shape
)
gm_array.type_length = array.dtype.itemsize
gm_array.data = b''
storage_layout = FileStorageLayout(files, read_fct)
storage_layout.spatial_domain = MInterval.from_shape((1,) + array.shape)
gm_array.storage_layout = storage_layout
print(gm_array)
return gm_array
def from_np_array(array: np.array):
"""
:param array: the numpy array used to create this new RasGMArray
:return: a RasGMArray
"""
gm_array = RasGMArray()
shape = array.shape
intervals = []
for i_max in shape:
intervals.append(SInterval(0, i_max - 1))
gm_array.spatial_domain = MInterval(intervals)
gm_array.type_name = RasGMArrayBuilder.get_type_name(
array.dtype, array.shape)
gm_array.type_length = array.dtype.itemsize
gm_array.data = bytes(array)
storage_layout = BandStorageLayout()
storage_layout.compute_spatial_domain(gm_array)
gm_array.storage_layout = storage_layout
return gm_array
def _create_ras_array(self, barray, offset, step_size, y_interval,
z_interval):
ras_array = RasGMArray()
ras_array.spatial_domain = MInterval(
[z_interval, y_interval, self.storage_intervals[2]])
ras_array.storage_layout = RasStorageLayOut(self.gm_array.type_length,
ras_array.spatial_domain)
ras_array.type_name = self.gm_array.type_name
ras_array.type_length = self.gm_array.type_length
ras_array.data = barray[offset:offset + step_size - 1]
return ras_array
def _create_ras_array(self, size):
i_interval = SInterval(self.lo, self.hi)
print(
f"get buffer from {self.offset} to {self.offset + size - 1} with {i_interval}"
)
ras_array = RasGMArray()
ras_array.spatial_domain = MInterval(
[i_interval, self.storage_intervals[1], self.storage_intervals[2]])
ras_array.storage_layout = self.gm_array.storage_layout
ras_array.type_name = self.gm_array.type_name
ras_array.type_length = self.gm_array.type_length
ras_array.data = bytes(self.barray[self.offset:self.offset + size - 1])
return ras_array
def from_file(file_path,
mdd_domain=None,
mdd_type=RasGMArray.DEFAULT_MDD_TYPE,
mdd_type_length=RasGMArray.DEFAULT_TYPE_LENGTH,
tile_domain=None,
tile_size=RasStorageLayOut.DEFAULT_TILE_SIZE):
if not os.path.isfile(file_path):
raise Exception("File {} to be inserted to rasdaman collection does not exist.".format(file_path))
# The data sent to rasserver is binary file content, not file_path
with open(file_path, mode='rb') as file:
data = file.read()
# NOTE: with insert into values decode($1), it doesn't need to specify --mdddomain and --mddtype
# but with values $1 it needs to specify these parameters
if mdd_domain is None:
mdd_domain = "[0:" + str(len(data) - 1) + "]"
if mdd_type is None:
mdd_type = RasGMArray.DEFAULT_MDD_TYPE
# Parse str values of mdd_domain and tile_domain to MInterval objects
mdd_domain = MInterval.from_str(mdd_domain)
# tile* are not parameters from "insert into value from file" in rasql client, so calculate it internally
if tile_domain is None:
dim = len(mdd_domain.intervals)
tile_domain = get_tiling_domain(dim, mdd_type_length, tile_size)
else:
tile_domain = MInterval.from_str(tile_domain)
# Create helper objects to build POST query string to rasserver
storage_layout = RasStorageLayOut(spatial_domain=tile_domain, tile_size=tile_size)
return RasGMArray(mdd_domain,
mdd_type,
mdd_type_length,
data,
storage_layout)
def _get_mdd_core(self, mdd_resp):
if mdd_resp.current_format != rDataFormat.r_Array.value:
# So far, for other format than binary array, only one tile is used.
tileresp = rassrvr_get_next_tile(
self.transaction.database.stub,
self.transaction.database.connection.session.clientId)
tile_status = tileresp.status
if tile_status == 3:
raise Exception(
f"multipart format {mdd_resp.current_format} is not handled"
)
if tile_status == 4:
raise Exception(
"rpcGetNextTile - no tile to transfer or empty collection")
return tile_status, tileresp.data
# At that point, we try to read a binary array.
mdd_result = None
tile_idx = 0
tile_assigner = None
read_size = 0
tile_status = 2 # call get_next_tile at least once
while tile_status == 2 or tile_status == 3:
tileresp = rassrvr_get_next_tile(
self.transaction.database.stub,
self.transaction.database.connection.session.clientId)
tile_status = tileresp.status
if tile_status == 4:
raise Exception(
"rpcGetNextTile - no tile to transfer or empty collection")
if mdd_result is None:
mdd_result = RasGMArray(spatial_domain=MInterval.from_str(
mdd_resp.domain),
type_length=tileresp.cell_type_length)
#print(f"full domain {mdd_result.spatial_domain} with format {mdd_result.format}")
#print(f"need to read {mdd_result.byte_size} bytes")
tile_idx += 1
#print(f"tile {tile_idx}")
# Creates current tile array
mdd_tile = RasGMArray(spatial_domain=MInterval.from_str(
tileresp.domain),
type_length=mdd_result.type_length)
if tile_status == 3:
tile_status = self.read_tile_by_parts(mdd_tile, tileresp.data)
else:
mdd_tile.data = tileresp.data
# At that point, data array of the mdd_tile should be complete for its spatial_domain
read_size += len(mdd_tile.data)
#print(f"read {read_size} bytes so far")
#print(f"tile domain {mdd_tile.spatial_domain}")
if tile_status < 2 and tile_idx == 1 and str(
mdd_tile.spatial_domain) == str(mdd_result.spatial_domain):
# MDD consists of just one tile that is the same size of the mdd
mdd_result.data = mdd_tile.data
else:
# MDD consists of more than one tile or the tile does not cover the whole domain
if tile_idx == 1:
mdd_result.data = bytearray(mdd_result.byte_size)
tile_assigner = TileAssigner(mdd_result)
tile_assigner.start()
# copy tile data into global MDD data space
# optimized, relying on the internal representation of an MDD
tile_assigner.add_tile(mdd_tile)
if tile_assigner is not None:
tile_assigner.go_on = False
tile_assigner.join()
return tile_status, mdd_result.data