def compute_spatial_domain(self, gm_array):
# Size is small enough, no need to split domain, use spatial domain.
if gm_array.data_length <= RasStorageLayOut.DEFAULT_TILE_SIZE:
return str(gm_array.spatial_domain)
mintervals = gm_array.spatial_domain.intervals
bytes_incr = self.get_hyper_plane_bytesize(gm_array)
if bytes_incr > RasStorageLayOut.DEFAULT_TILE_SIZE:
raise Exception(
f"tile size {bytes_incr} is bigger than maximum {RasStorageLayOut.DEFAULT_TILE_SIZE} size"
)
i_dim = int(RasStorageLayOut.DEFAULT_TILE_SIZE / bytes_incr)
self.tile_size = i_dim * bytes_incr
intervals = [SInterval(0, i_dim - 1)]
for i in range(1, len(mintervals)):
hi_value = mintervals[i].hi
intervals.append(SInterval(0, hi_value))
self.spatial_domain = MInterval(intervals)
print(self.spatial_domain)
def _fill_rasarray_list(self, ras_list, barray, itemsize, z):
full_size = len(barray)
print("fullsize", full_size)
n_iter = full_size // RasStorageLayOut.DEFAULT_TILE_SIZE + 1
y_width = self.storage_intervals[1].extent // n_iter
step_size = y_width * self.storage_intervals[2].extent * itemsize
offset = 0
lo = self.storage_intervals[1].lo
hi = y_width - 1
while offset + step_size <= full_size:
y_interval = SInterval(lo, hi)
z_interval = SInterval(z, z)
ras_array = self._create_ras_array(barray, offset, step_size,
y_interval, z_interval)
ras_list.append(ras_array)
lo += y_width
hi += y_width
offset += step_size
print(y_interval, offset)
remaining_size = full_size - offset
print("offset", offset, "remaining", remaining_size)
if remaining_size > 0:
y_interval = SInterval(lo, self.storage_intervals[1].hi)
z_interval = SInterval(z, z)
ras_array = self._create_ras_array(barray, offset, remaining_size,
y_interval, z_interval)
ras_list.append(ras_array)
def test_cartesian_product(self):
i1 = SInterval(200, 210)
i2 = SInterval(300, 305)
i3 = SInterval(400, 400)
m = MInterval([i1, i2, i3])
l = m.cartesian_product()
print(l)
def from_np_array(array: np.array):
"""
:param array: the numpy array used to create this new RasGMArray
:return: a RasGMArray
"""
gm_array = RasGMArray()
shape = array.shape
intervals = []
for i_max in shape:
intervals.append(SInterval(0, i_max - 1))
gm_array.spatial_domain = MInterval(intervals)
gm_array.type_name = RasGMArrayBuilder.get_type_name(
array.dtype, array.shape)
gm_array.type_length = array.dtype.itemsize
gm_array.data = bytes(array)
storage_layout = BandStorageLayout()
storage_layout.compute_spatial_domain(gm_array)
gm_array.storage_layout = storage_layout
return gm_array
def _create_ras_array(self, size):
i_interval = SInterval(self.lo, self.hi)
print(
f"get buffer from {self.offset} to {self.offset + size - 1} with {i_interval}"
)
ras_array = RasGMArray()
ras_array.spatial_domain = MInterval(
[i_interval, self.storage_intervals[1], self.storage_intervals[2]])
ras_array.storage_layout = self.gm_array.storage_layout
ras_array.type_name = self.gm_array.type_name
ras_array.type_length = self.gm_array.type_length
ras_array.data = bytes(self.barray[self.offset:self.offset + size - 1])
return ras_array
def from_str(str_mdd):
"""
Parse the string representing MDD domain (e.g: "[0:18,0:18]") to a MInterval object with list of SIntervals
:param str str_mdd: represents MDD domain
:return: MInterval result: a MInterval object
"""
tmp = str_mdd.strip("[]")
sinterval_str_arr = tmp.split(",")
result_arr = []
for tmp in sinterval_str_arr:
tmp_arr = tmp.split(":")
sinterval = SInterval(tmp_arr[0], tmp_arr[1])
result_arr.append(sinterval)
minterval = MInterval(result_arr)
return minterval
def get_spatial_domain_from_type_structure(input_str):
"""
Parse the sdom from input_str
:param input_str: the string containing sdom (e.g: u'set <marray <long, [0:250,0:210]>>')
:return: MInterval object containing the domain of array (e.g: [0:250,0:210])
"""
primary_regex = ".*\[(.*)\].*"
primary_match = re.match(primary_regex, input_str)
sinterlval_array = []
if primary_match is not None:
matches = primary_match.groups()[0].split(",")
for match in matches:
tmp_array = match.split(":")
sinterval = SInterval(tmp_array[0], tmp_array[1])
sinterlval_array.append(sinterval)
minterval = MInterval(sinterlval_array)
return minterval
def __iter__(self):
for z, file in enumerate(self.layout.files):
array = self.layout.reader(file)
byte_size = array.size * self.gm_array.type_length
barray = bytes(array)
ras_list = []
if byte_size > RasStorageLayOut.DEFAULT_TILE_SIZE:
self._fill_rasarray_list(ras_list, barray, array.itemsize, z)
else:
z_interval = SInterval(z, z)
y_interval = self.storage_intervals[1]
ras_array = self._create_ras_array(barray, 0, byte_size,
y_interval, z_interval)
ras_list.append(ras_array)
for ras_array in ras_list:
print(ras_array.spatial_domain)
yield ras_array
def convert_binary_data_stream(dtype, data):
"""
Convert a set of binary data into meaningful data for next_element
:param dtype: data type object to determine the type of parsing data
:param data: string binary needed to be unpack by the length of data type (e.g: short - 2 bytes, int - 4 bytes,...)
:return: object according to query
"""
# base is either marray or scalar
base_type = dtype["base_type"]
# type is the data type of elements in base
type = dtype["type"]
if base_type != "marray" and base_type != "scalar":
raise Exception("Unknown base_type {} and type {} ".format(
dtype["base_type"], dtype["type"]))
if base_type == "scalar" and type == "struct":
# e.g: select {1, 2, 3}
temp_array = []
cell_counter = 0
last_byte = 0
for idx, dt in enumerate(dtype["sub_type"]["types"]):
dtsize = get_size_from_data_type(dt)
from_byte = last_byte
to_byte = last_byte + dtsize
last_byte = to_byte
scalar_value = convert_data_from_bin(dt, data[from_byte:to_byte])
scalar_value = get_scalar_result(scalar_value)
temp_array.append(scalar_value)
cell_counter += 1
composite_type = CompositeType(temp_array)
return composite_type
elif base_type == "scalar" and type == "minterval":
# e.g: select sdom(c) from test_mr as c ([0:250,0:210])
temp = ""
# strip the [] of the string to parse
length = len(data) - 2
for i in range(1, length):
s = str(convert_data_from_bin(type, data[i:i + 1]))
temp = temp + s
temp_array = temp.split(",")
intervals = []
for temp in temp_array:
lo = temp.split(":")[0]
hi = temp.split(":")[1]
sinterval = SInterval(lo, hi)
# e.g: 0:250
intervals.append(sinterval)
# e.g: [0:250,0:211]
minterval = MInterval(intervals)
return minterval
elif base_type == "scalar" and type == "sinterval":
# e.g: select sdom(c)[0] from test_mr as c
temp = ""
length = len(data) - 1
for i in range(0, length):
s = str(convert_data_from_bin(type, data[i:i + 1]))
temp = temp + s
# e.g: 0:250
tmp_array = temp.split(":")
sinterval = SInterval(tmp_array[0], tmp_array[1])
return sinterval
elif base_type == "scalar" and type == "complexd":
# e.g: select complex(0.5, 2.5) from test_mr
# complexd is 8 bytes: 8 bytes
dtsize = get_size_from_data_type(type)
real_number = convert_data_from_bin(type, data[0:dtsize])
imagine_number = convert_data_from_bin(type, data[dtsize:dtsize * 2])
real_number = get_scalar_result(real_number)
imagine_number = get_scalar_result(imagine_number)
complex_number = Complex(real_number, imagine_number)
return complex_number
else:
# e.g: query return 1 double value and data will have length = 8 bytes
scalar_value = convert_data_from_bin(type, data)
scalar_value = get_scalar_result(scalar_value)
return scalar_value
def from_shape(shape):
intervals = []
for i_max in shape:
intervals.append(SInterval(0, i_max - 1))
return MInterval(intervals)