def test_3():
# perform assignment stack level testing
ne = NDexpr()
ne.test_1_level()
ne.test_2_level()
def main():
print('Instantiating API and NDexpr')
g = API()
nd = NDexpr()
print('Retrieving data from API')
# construct data request parameters for band_30 and band_40
data_request_descriptor = {
'platform': 'LANDSAT_5',
'product': 'nbar',
'variables': ('red', 'nir'),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1990, 1, 1), datetime(1990, 12, 31))
}
}
}
# get data
d1 = g.get_data(data_request_descriptor)
# construct data request parameters for PQ
pq_request_descriptor = {
'platform': 'LANDSAT_5',
'product': 'pqa',
'variables': ('pixelquality'),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1990, 1, 1), datetime(1990, 12, 31))
}
}
}
# get data
d2 = g.get_data(pq_request_descriptor)
# The following 3 lines shouldn't be done like this
# Currently done like this for the sake of the example.
b30 = d1['arrays']['red']
b40 = d1['arrays']['nir']
pq = d2['arrays']['pixelquality']
print('NDexpr demo begins here')
# perform ndvi as expressed in this language.
ndvi = nd.evaluate('((b40 - b30) / (b40 + b30))')
# perform mask on ndvi as expressed in this language.
masked_ndvi = nd.evaluate('ndvi{(pq == 32767) | (pq == 16383) | (pq == 2457)}')
print(masked_ndvi.values)
def __init__(self, api=None, index=None):
LOG.info('Initialise Execution Module.')
self.cache = {}
self.nd = NDexpr()
self.nd.set_ae(True)
self.api = api or API(index=index)
self.udfuncs = {}
def __init__(self, api=None, index=None):
LOG.info('Initialise Execution Module.')
self.cache = {}
self.nd = NDexpr()
self.nd.set_ae(True)
self.api = api or API(index=index)
self.udfuncs = {}
def test_3():
# perform assignment stack level testing
ne = NDexpr()
ne.test_1_level()
ne.test_2_level()
def main():
print('Instantiating API and NDexpr')
g = API()
nd = NDexpr()
print('Retrieving data from API')
# construct data request parameters for band_30 and band_40
data_request_descriptor = {
'platform': 'LANDSAT_5',
'product': 'nbar',
'variables': ('red', 'nir'),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1990, 1, 1), datetime(1990, 12, 31))
}
}
}
# get data
d1 = g.get_data(data_request_descriptor)
# construct data request parameters for PQ
pq_request_descriptor = {
'platform': 'LANDSAT_5',
'product': 'pqa',
'variables': ('pixelquality'),
'dimensions': {
'longitude': {
'range': (149.07, 149.18)
},
'latitude': {
'range': (-35.32, -35.28)
},
'time': {
'range': (datetime(1990, 1, 1), datetime(1990, 12, 31))
}
}
}
# get data
d2 = g.get_data(pq_request_descriptor)
# The following 3 lines shouldn't be done like this
# Currently done like this for the sake of the example.
b30 = d1['arrays']['red']
b40 = d1['arrays']['nir']
pq = d2['arrays']['pixelquality']
print('NDexpr demo begins here')
# perform ndvi as expressed in this language.
ndvi = nd.evaluate('((b40 - b30) / (b40 + b30))')
# perform mask on ndvi as expressed in this language.
masked_ndvi = nd.evaluate(
'ndvi{(pq == 32767) | (pq == 16383) | (pq == 2457)}')
# currently dimensions are integer indices, later will be labels when
# Data Access API Interface has been finalised.
reduction_on_dim0 = nd.evaluate('median(masked_ndvi, 0)')
reduction_on_dim01 = nd.evaluate('median(masked_ndvi, 0, 1)')
reduction_on_dim012 = nd.evaluate('median(masked_ndvi, 0, 1, 2)')
print(reduction_on_dim0)
print(reduction_on_dim01)
print(reduction_on_dim012)
class ExecutionEngine(object):
REDUCTION_FNS = {"all": xr.DataArray.all,
"any": xr.DataArray.any,
"argmax": xr.DataArray.argmax,
"argmin": xr.DataArray.argmin,
"max": xr.DataArray.max,
"mean": xr.DataArray.mean,
"median": xr.DataArray.median,
"min": xr.DataArray.min,
"prod": xr.DataArray.prod,
"sum": xr.DataArray.sum,
"std": xr.DataArray.std,
"var": xr.DataArray.var}
def __init__(self, api=None, index=None):
LOG.info('Initialise Execution Module.')
self.cache = {}
self.nd = NDexpr()
self.nd.set_ae(True)
self.api = api or API(index=index)
self.udfuncs = {}
def add_function(self, name, func):
self.udfuncs[name] = func
def execute_plan(self, plan):
for task in plan:
function = next(iter(task.values()))['orig_function']
op_type = next(iter(task.values()))['operation_type']
if op_type == OperationType.Get_Data:
self.execute_get_data(task)
elif op_type == OperationType.Expression:
self.execute_expression(task)
elif op_type == OperationType.Cloud_Mask:
self.execute_cloud_mask(task)
elif op_type == OperationType.Reduction and \
len([s for s in self.REDUCTION_FNS.keys() if s in function]) > 0:
self.execute_reduction(task)
elif op_type == OperationType.Bandmath:
self.execute_bandmath(task)
def execute_get_data(self, task):
value = next(iter(task.values()))
array_input_value = next(iter(value['array_input'][0].values()))
data_request_param = {}
data_request_param['dimensions'] = array_input_value['dimensions']
if 'storage_type' in array_input_value:
data_request_param['storage_type'] = array_input_value['storage_type']
else:
data_request_param['product'] = array_input_value['product']
data_request_param['platform'] = array_input_value['platform']
data_request_param['variables'] = ()
for array in value['array_input']:
data_request_param['variables'] += (next(iter(array.values()))['variable'],)
data_response = self.api.get_data(data_request_param)
no_data_value = value['array_output']['no_data_value']
if no_data_value is not None:
for k, v in data_response['arrays'].items():
data_response['arrays'][k] = data_response['arrays'][k].where(v != no_data_value)
key = next(iter(task.keys()))
self.cache[key] = {}
self.cache[key]['array_result'] = copy.deepcopy(data_response['arrays'])
self.cache[key]['array_indices'] = copy.deepcopy(data_response['indices'])
self.cache[key]['array_dimensions'] = copy.deepcopy(data_response['dimensions'])
self.cache[key]['array_output'] = copy.deepcopy(value['array_output'])
self.cache[key]['crs'] = copy.deepcopy(data_response['coordinate_reference_systems'])
del data_request_param
del data_response
return self.cache[key]
def execute_cloud_mask(self, task):
key = next(iter(task.keys()))
value = next(iter(task.values()))
data_key = value['array_input'][0]
mask_key = value['array_mask']
no_data_value = value['array_output']['no_data_value']
array_desc = self.cache[value['array_input'][0]]
data_array = next(iter(self.cache[data_key]['array_result'].values()))
mask_array = next(iter(self.cache[mask_key]['array_result'].values()))
pqa_mask = get_pqa_mask(mask_array.values)
masked_array = xr.DataArray.where(data_array, pqa_mask)
#masked_array = masked_array.fillna(no_data_value)
self.cache[key] = {}
self.cache[key]['array_result'] = {}
self.cache[key]['array_result'][key] = masked_array
self.cache[key]['array_indices'] = copy.deepcopy(array_desc['array_indices'])
self.cache[key]['array_dimensions'] = copy.deepcopy(array_desc['array_dimensions'])
self.cache[key]['array_output'] = copy.deepcopy(value['array_output'])
self.cache[key]['crs'] = copy.deepcopy(array_desc['crs'])
def execute_expression(self, task):
key = next(iter(task.keys()))
value = next(iter(task.values()))
no_data_value = value['array_output']['no_data_value']
arrays = {}
for task_name in value['array_input']:
arrays[task_name] = next(iter(self.cache[task_name]['array_result'].values()))
for i in arrays:
if arrays[i].dtype == type(no_data_value):
arrays[i] = arrays[i].where(arrays[i] != no_data_value)
array_result = {}
array_result['array_result'] = {}
array_result['array_result'][key] = self.nd.evaluate(value['function'],
local_dict=arrays,
user_functions=self.udfuncs)
#array_result['array_result'][key] = array_result['array_result'][key].fillna(no_data_value)
array_desc = self.cache[value['array_input'][0]]
array_result['array_indices'] = copy.deepcopy(array_desc['array_indices'])
array_result['array_dimensions'] = copy.deepcopy(array_desc['array_dimensions'])
array_result['array_output'] = copy.deepcopy(value['array_output'])
array_result['crs'] = copy.deepcopy(array_desc['crs'])
self.cache[key] = array_result
return self.cache[key]
def execute_bandmath(self, task):
key = next(iter(task.keys()))
value = next(iter(task.values()))
arrays = {}
for task_name in value['array_input']:
#for k, v in self.cache[task_name]['array_result'].items():
# arrays[k] = v.astype(float).values
arrays.update(self.cache[task_name]['array_result'])
array_result = {}
array_result['array_result'] = {}
array_result['array_result'][key] = xr.DataArray(ne.evaluate(value['function'], arrays))
#array_result['array_result'][key] = self.nd.evaluate(value['function'], arrays)
array_desc = self.cache[value['array_input'][0]]
array_result['array_indices'] = copy.deepcopy(array_desc['array_indices'])
array_result['array_dimensions'] = copy.deepcopy(array_desc['array_dimensions'])
array_result['array_output'] = copy.deepcopy(value['array_output'])
array_result['crs'] = copy.deepcopy(array_desc['crs'])
self.cache[key] = array_result
return self.cache[key]
def execute_reduction(self, task):
key = next(iter(task.keys()))
value = next(iter(task.values()))
function_name = value['orig_function'].replace(")", " ").replace("(", " ").split()[0]
func = self.REDUCTION_FNS[function_name]
data_key = value['array_input'][0]
data = self.cache[data_key]['array_dimensions']
no_data_value = value['array_output']['no_data_value']
array_data = next(iter(self.cache[data_key]['array_result'].values()))
array_desc = self.cache[value['array_input'][0]]
array_result = {}
array_result['array_result'] = {}
array_result['array_output'] = copy.deepcopy(value['array_output'])
dims = tuple((self.cache[data_key]['array_dimensions'].index(p) for p in value['dimension']))
args = {}
if function_name == 'argmax' or function_name == 'argmin':
if len(dims) != 1:
args['axis'] = dims[0]
else:
args['axis'] = dims
if sys.version_info >= (3, 0):
if 'skipna' in list(inspect.signature(self.REDUCTION_FNS[function_name]).parameters.keys()) and \
function_name != 'prod':
args['skipna'] = True
else:
if 'skipna' in inspect.getargspec(self.REDUCTION_FNS[function_name])[0] and \
function_name != 'prod':
args['skipna'] = True
array_result['array_result'][key] = func(xr.DataArray(array_data), **args)
array_result['array_indices'] = copy.deepcopy(array_desc['array_indices'])
array_result['array_dimensions'] = copy.deepcopy(array_result['array_output']['dimensions_order'])
array_result['crs'] = copy.deepcopy(array_desc['crs'])
self.cache[key] = array_result
return self.cache[key]
def test_2():
# perform language test
ne = NDexpr()
ne.set_ae(False)
x1 = xr.DataArray(np.random.randn(2, 3))
y1 = xr.DataArray(np.random.randn(2, 3))
z1 = xr.DataArray(
np.array([[[0, 1, 2], [3, 4, 5], [6, 7, 8]],
[[9, 10, 11], [12, 13, 14], [15, 16, 17]],
[[18, 19, 20], [21, 22, 23], [24, 25, 26]]]))
z2 = z1 * 2
z3 = np.arange(27)
mask1 = z1 > 4
assert ne.test("arccos(z1)", xr.ufuncs.arccos(z1))
assert ne.test("angle(z1)", xr.ufuncs.angle(z1))
assert ne.test("arccos(z1)", xr.ufuncs.arccos(z1))
assert ne.test("arccosh(z1)", xr.ufuncs.arccosh(z1))
assert ne.test("arcsin(z1)", xr.ufuncs.arcsin(z1))
assert ne.test("arcsinh(z1)", xr.ufuncs.arcsinh(z1))
assert ne.test("arctan(z1)", xr.ufuncs.arctan(z1))
assert ne.test("arctanh(z1)", xr.ufuncs.arctanh(z1))
assert ne.test("ceil(z1)", xr.ufuncs.ceil(z1))
assert ne.test("conj(z1)", xr.ufuncs.conj(z1))
assert ne.test("cos(z1)", xr.ufuncs.cos(z1))
assert ne.test("cosh(z1)", xr.ufuncs.cosh(z1))
assert ne.test("deg2rad(z1)", xr.ufuncs.deg2rad(z1))
assert ne.test("degrees(z1)", xr.ufuncs.degrees(z1))
assert ne.test("exp(z1)", xr.ufuncs.exp(z1))
assert ne.test("expm1(z1)", xr.ufuncs.expm1(z1))
assert ne.test("fabs(z1)", xr.ufuncs.fabs(z1))
assert ne.test("fix(z1)", xr.ufuncs.fix(z1))
assert ne.test("floor(z1)", xr.ufuncs.floor(z1))
assert ne.test("frexp(z3)", xr.ufuncs.frexp(z3))
assert ne.test("imag(z1)", xr.ufuncs.imag(z1))
assert ne.test("iscomplex(z1)", xr.ufuncs.iscomplex(z1))
assert ne.test("isfinite(z1)", xr.ufuncs.isfinite(z1))
assert ne.test("isinf(z1)", xr.ufuncs.isinf(z1))
assert ne.test("isnan(z1)", xr.ufuncs.isnan(z1))
assert ne.test("isreal(z1)", xr.ufuncs.isreal(z1))
assert ne.test("log(z1)", xr.ufuncs.log(z1))
assert ne.test("log10(z1)", xr.ufuncs.log10(z1))
assert ne.test("log1p(z1)", xr.ufuncs.log1p(z1))
assert ne.test("log2(z1)", xr.ufuncs.log2(z1))
assert ne.test("pow(z1, 2.0)", np.power(z1, 2.0))
assert ne.test("pow(z1, 2)", np.power(z1, 2))
assert ne.test("z1^2", np.power(z1, 2))
assert ne.test("z1**2", np.power(z1, 2))
assert ne.test("rad2deg(z1)", xr.ufuncs.rad2deg(z1))
assert ne.test("radians(z1)", xr.ufuncs.radians(z1))
assert ne.test("real(z1)", xr.ufuncs.real(z1))
assert ne.test("rint(z1)", xr.ufuncs.rint(z1))
assert ne.test("sign(z1)", xr.ufuncs.sign(z1))
assert ne.test("signbit(z1)", xr.ufuncs.signbit(z1))
assert ne.test("sin(z1)", xr.ufuncs.sin(z1))
assert ne.test("sinh(z1)", xr.ufuncs.sinh(z1))
assert ne.test("sqrt(z1)", xr.ufuncs.sqrt(z1))
assert ne.test("square(z1)", xr.ufuncs.square(z1))
assert ne.test("tan(z1)", xr.ufuncs.tan(z1))
assert ne.test("tanh(z1)", xr.ufuncs.tanh(z1))
assert ne.test("trunc(z1)", xr.ufuncs.trunc(z1))
assert ne.test("arctan2(z1, z2)", xr.ufuncs.arctan2(z1, z2))
assert ne.test("copysign(z1, z2)", xr.ufuncs.copysign(z1, z2))
assert ne.test("fmax(z1, z2)", xr.ufuncs.fmax(z1, z2))
assert ne.test("fmin(z1, z2)", xr.ufuncs.fmin(z1, z2))
assert ne.test("fmod(z1, z2)", xr.ufuncs.fmod(z1, z2))
assert ne.test("hypot(z1, z2)", xr.ufuncs.hypot(z1, z2))
assert ne.test("ldexp(z1, z2)", xr.DataArray(xr.ufuncs.ldexp(z1, z2)))
assert ne.test("logaddexp(z1, z2)", xr.ufuncs.logaddexp(z1, z2))
assert ne.test("logaddexp2(z1, z2)", xr.ufuncs.logaddexp2(z1, z2))
assert ne.test("logicaland(z1, z2)", xr.ufuncs.logical_and(z1, z2))
assert ne.test("logicalnot(z1, z2)", xr.ufuncs.logical_not(z1, z2))
assert ne.test("logicalor(z1, z2)", xr.ufuncs.logical_or(z1, z2))
assert ne.test("logicalxor(z1, z2)", xr.ufuncs.logical_xor(z1, z2))
assert ne.test("maximum(z1, z2)", xr.ufuncs.maximum(z1, z2))
assert ne.test("minimum(z1, z2)", xr.ufuncs.minimum(z1, z2))
assert ne.test("nextafter(z1, z2)", xr.ufuncs.nextafter(z1, z2))
assert ne.test("all(z1)", xr.DataArray.all(z1))
assert ne.test("all(z1, 0)", xr.DataArray.all(z1, axis=0))
assert ne.test("all(z1, 0, 1)", xr.DataArray.all(z1, axis=(0, 1)))
assert ne.test("all(z1, 0, 1, 2)", xr.DataArray.all(z1, axis=(0, 1, 2)))
assert ne.test("any(z1)", xr.DataArray.any(z1))
assert ne.test("any(z1, 0)", xr.DataArray.any(z1, axis=0))
assert ne.test("any(z1, 0, 1)", xr.DataArray.any(z1, axis=(0, 1)))
assert ne.test("any(z1, 0, 1, 2)", xr.DataArray.any(z1, axis=(0, 1, 2)))
assert ne.test("argmax(z1)", xr.DataArray.argmax(z1))
assert ne.test("argmax(z1, 0)", xr.DataArray.argmax(z1, axis=0))
assert ne.test("argmax(z1, 1)", xr.DataArray.argmax(z1, axis=1))
assert ne.test("argmax(z1, 2)", xr.DataArray.argmax(z1, axis=2))
assert ne.test("argmin(z1)", xr.DataArray.argmin(z1))
assert ne.test("argmin(z1, 0)", xr.DataArray.argmin(z1, axis=0))
assert ne.test("argmin(z1, 1)", xr.DataArray.argmin(z1, axis=1))
assert ne.test("argmin(z1, 2)", xr.DataArray.argmin(z1, axis=2))
assert ne.test("max(z1)", xr.DataArray.max(z1))
assert ne.test("max(z1, 0)", xr.DataArray.max(z1, axis=0))
assert ne.test("max(z1, 0, 1)", xr.DataArray.max(z1, axis=(0, 1)))
assert ne.test("max(z1, 0, 1, 2)", xr.DataArray.max(z1, axis=(0, 1, 2)))
assert ne.test("mean(z1)", xr.DataArray.mean(z1))
assert ne.test("mean(z1, 0)", xr.DataArray.mean(z1, axis=0))
assert ne.test("mean(z1, 0, 1)", xr.DataArray.mean(z1, axis=(0, 1)))
assert ne.test("mean(z1, 0, 1, 2)", xr.DataArray.mean(z1, axis=(0, 1, 2)))
assert ne.test("median(z1)", xr.DataArray.median(z1))
assert ne.test("median(z1, 0)", xr.DataArray.median(z1, axis=0))
assert ne.test("median(z1, 0, 1)", xr.DataArray.median(z1, axis=(0, 1)))
assert ne.test("median(z1, 0, 1, 2)",
xr.DataArray.median(z1, axis=(0, 1, 2)))
assert ne.test("min(z1)", xr.DataArray.min(z1))
assert ne.test("min(z1, 0)", xr.DataArray.min(z1, axis=0))
assert ne.test("min(z1, 0, 1)", xr.DataArray.min(z1, axis=(0, 1)))
assert ne.test("min(z1, 0, 1, 2)", xr.DataArray.min(z1, axis=(0, 1, 2)))
assert ne.test("prod(z1)", xr.DataArray.prod(z1))
assert ne.test("prod(z1, 0)", xr.DataArray.prod(z1, axis=0))
assert ne.test("prod(z1, 0, 1)", xr.DataArray.prod(z1, axis=(0, 1)))
assert ne.test("prod(z1, 0, 1, 2)", xr.DataArray.prod(z1, axis=(0, 1, 2)))
assert ne.test("sum(z1)", xr.DataArray.sum(z1))
assert ne.test("sum(z1, 0)", xr.DataArray.sum(z1, axis=0))
assert ne.test("sum(z1, 0, 1)", xr.DataArray.sum(z1, axis=(0, 1)))
assert ne.test("sum(z1, 0, 1, 2)", xr.DataArray.sum(z1, axis=(0, 1, 2)))
assert ne.test("std(z1)", xr.DataArray.std(z1))
assert ne.test("std(z1, 0)", xr.DataArray.std(z1, axis=0))
assert ne.test("std(z1, 0, 1)", xr.DataArray.std(z1, axis=(0, 1)))
assert ne.test("std(z1, 0, 1, 2)", xr.DataArray.std(z1, axis=(0, 1, 2)))
assert ne.test("var(z1)", xr.DataArray.var(z1))
assert ne.test("var(z1, 0)", xr.DataArray.var(z1, axis=0))
assert ne.test("var(z1, 0, 1)", xr.DataArray.var(z1, axis=(0, 1)))
assert ne.test("var(z1, 0, 1, 2)", xr.DataArray.var(z1, axis=(0, 1, 2)))
assert ne.test("percentile(z1, 50)", np.percentile(z1, 50))
assert ne.test("percentile(z1, 50)+percentile(z1, 50)",
np.percentile(z1, 50) + np.percentile(z1, 50))
assert ne.test("percentile(z1, (50))", np.percentile(z1, (50)))
assert ne.test("percentile(z1, (50, 60))", np.percentile(z1, (50, 60)))
assert ne.test("percentile(z1, (50, 60, 70))",
np.percentile(z1, (50, 60, 70)))
assert ne.test(
"percentile(z1, (50, 60, 70)) + percentile(z1, (50, 60, 70))",
np.percentile(z1, (50, 60, 70)) + np.percentile(z1, (50, 60, 70)))
assert ne.test("1 + var(z1, 0, 0+1, 2) + 1",
1 + xr.DataArray.var(z1, axis=(0, 0 + 1, 2)) + 1)
assert ne.test("1 + ((z1+z1)*0.0005)**2 + 1", 1 + np.power(
(z1 + z1) * 0.0005, 2) + 1)
assert ne.test("z1{mask1}", xr.DataArray.where(z1, mask1))
assert ne.test("z1{z1>2}", xr.DataArray.where(z1, z1 > 2))
assert ne.test("z1{z1>=2}", xr.DataArray.where(z1, z1 >= 2))
assert ne.test("z1{z1<2}", xr.DataArray.where(z1, z1 < 2))
assert ne.test("z1{z1<=2}", xr.DataArray.where(z1, z1 <= 2))
assert ne.test("z1{z1==2}", xr.DataArray.where(z1, z1 == 2))
assert ne.test("z1{z1!=2}", xr.DataArray.where(z1, z1 != 2))
assert ne.test("z1{z1<2 | z1>5}",
xr.DataArray.where(z1, (z1 < 2) | (z1 > 5)))
assert ne.test("z1{z1>2 & z1<5}",
xr.DataArray.where(z1, (z1 > 2) & (z1 < 5)))
ne.evaluate("m = z1+1")
assert ne.test("m", z1 + 1)
assert ne.test("z1{~mask1}", xr.DataArray.where(z1, ~mask1))
assert ne.test("(1<0?1+1;2+2)", 4)
assert ne.test("(0<1?1+1;2+2)", 2)
assert ne.test("z1+mask1", xr.DataArray.where(z1, mask1))
assert ne.is_number(1) is True
assert ne.test("-z1", -z1)
assert ne.test("z1{!(z1<2)}",
xr.DataArray.where(z1, xr.ufuncs.logical_not(z1 < 2)))
assert ne.test("z1>>1", np.right_shift(z1, 1))
assert ne.test("z1<<1", np.left_shift(z1, 1))
def test_1():
test_ndexpr = NDexpr()
class ExecutionEngine(object):
REDUCTION_FNS = {"all": xr.DataArray.all,
"any": xr.DataArray.any,
"argmax": xr.DataArray.argmax,
"argmin": xr.DataArray.argmin,
"max": xr.DataArray.max,
"mean": xr.DataArray.mean,
"median": xr.DataArray.median,
"min": xr.DataArray.min,
"prod": xr.DataArray.prod,
"sum": xr.DataArray.sum,
"std": xr.DataArray.std,
"var": xr.DataArray.var}
def __init__(self, api=None, index=None):
LOG.info('Initialise Execution Module.')
self.cache = {}
self.nd = NDexpr()
self.nd.set_ae(True)
self.api = api or API(index=index)
self.udfuncs = {}
def add_function(self, name, func):
self.udfuncs[name] = func
def execute_plan(self, plan):
for task in plan:
function = next(iter(task.values()))['orig_function']
op_type = next(iter(task.values()))['operation_type']
if op_type == OperationType.Get_Data:
self.execute_get_data(task)
elif op_type == OperationType.Expression:
self.execute_expression(task)
elif op_type == OperationType.Cloud_Mask:
self.execute_cloud_mask(task)
elif op_type == OperationType.Reduction and \
len([s for s in self.REDUCTION_FNS.keys() if s in function]) > 0:
self.execute_reduction(task)
elif op_type == OperationType.Bandmath:
self.execute_bandmath(task)
def execute_get_data(self, task):
value = next(iter(task.values()))
array_input_value = next(iter(value['array_input'][0].values()))
data_request_param = {}
data_request_param['dimensions'] = array_input_value['dimensions']
if 'storage_type' in array_input_value:
data_request_param['storage_type'] = array_input_value['storage_type']
else:
data_request_param['product'] = array_input_value['product']
data_request_param['platform'] = array_input_value['platform']
data_request_param['variables'] = ()
for array in value['array_input']:
data_request_param['variables'] += (next(iter(array.values()))['variable'],)
data_response = self.api.get_data(data_request_param)
no_data_value = value['array_output']['no_data_value']
if no_data_value is not None:
for k, v in data_response['arrays'].items():
data_response['arrays'][k] = data_response['arrays'][k].where(v != no_data_value)
key = next(iter(task.keys()))
self.cache[key] = {}
self.cache[key]['array_result'] = copy.deepcopy(data_response['arrays'])
self.cache[key]['array_indices'] = copy.deepcopy(data_response['indices'])
self.cache[key]['array_dimensions'] = copy.deepcopy(data_response['dimensions'])
self.cache[key]['array_output'] = copy.deepcopy(value['array_output'])
self.cache[key]['crs'] = copy.deepcopy(data_response['coordinate_reference_systems'])
del data_request_param
del data_response
return self.cache[key]
def execute_cloud_mask(self, task):
key = next(iter(task.keys()))
value = next(iter(task.values()))
data_key = value['array_input'][0]
mask_key = value['array_mask']
no_data_value = value['array_output']['no_data_value']
array_desc = self.cache[value['array_input'][0]]
data_array = next(iter(self.cache[data_key]['array_result'].values()))
mask_array = next(iter(self.cache[mask_key]['array_result'].values()))
pqa_mask = get_pqa_mask(mask_array.values)
masked_array = xr.DataArray.where(data_array, pqa_mask)
#masked_array = masked_array.fillna(no_data_value)
self.cache[key] = {}
self.cache[key]['array_result'] = {}
self.cache[key]['array_result'][key] = masked_array
self.cache[key]['array_indices'] = copy.deepcopy(array_desc['array_indices'])
self.cache[key]['array_dimensions'] = copy.deepcopy(array_desc['array_dimensions'])
self.cache[key]['array_output'] = copy.deepcopy(value['array_output'])
self.cache[key]['crs'] = copy.deepcopy(array_desc['crs'])
def execute_expression(self, task):
key = next(iter(task.keys()))
value = next(iter(task.values()))
no_data_value = value['array_output']['no_data_value']
arrays = {}
for task_name in value['array_input']:
arrays[task_name] = next(iter(self.cache[task_name]['array_result'].values()))
for i in arrays.keys():
if arrays[i].dtype == type(no_data_value):
arrays[i] = arrays[i].where(arrays[i] != no_data_value)
array_result = {}
array_result['array_result'] = {}
array_result['array_result'][key] = self.nd.evaluate(value['function'],
local_dict=arrays,
user_functions=self.udfuncs)
#array_result['array_result'][key] = array_result['array_result'][key].fillna(no_data_value)
array_desc = self.cache[value['array_input'][0]]
array_result['array_indices'] = copy.deepcopy(array_desc['array_indices'])
array_result['array_dimensions'] = copy.deepcopy(array_desc['array_dimensions'])
array_result['array_output'] = copy.deepcopy(value['array_output'])
array_result['crs'] = copy.deepcopy(array_desc['crs'])
self.cache[key] = array_result
return self.cache[key]
def execute_bandmath(self, task):
key = next(iter(task.keys()))
value = next(iter(task.values()))
arrays = {}
for task_name in value['array_input']:
#for k, v in self.cache[task_name]['array_result'].items():
# arrays[k] = v.astype(float).values
arrays.update(self.cache[task_name]['array_result'])
array_result = {}
array_result['array_result'] = {}
array_result['array_result'][key] = xr.DataArray(ne.evaluate(value['function'], arrays))
#array_result['array_result'][key] = self.nd.evaluate(value['function'], arrays)
array_desc = self.cache[value['array_input'][0]]
array_result['array_indices'] = copy.deepcopy(array_desc['array_indices'])
array_result['array_dimensions'] = copy.deepcopy(array_desc['array_dimensions'])
array_result['array_output'] = copy.deepcopy(value['array_output'])
array_result['crs'] = copy.deepcopy(array_desc['crs'])
self.cache[key] = array_result
return self.cache[key]
def execute_reduction(self, task):
key = next(iter(task.keys()))
value = next(iter(task.values()))
function_name = value['orig_function'].replace(")", " ").replace("(", " ").split()[0]
func = self.REDUCTION_FNS[function_name]
data_key = value['array_input'][0]
data = self.cache[data_key]['array_dimensions']
no_data_value = value['array_output']['no_data_value']
array_data = next(iter(self.cache[data_key]['array_result'].values()))
array_desc = self.cache[value['array_input'][0]]
array_result = {}
array_result['array_result'] = {}
array_result['array_output'] = copy.deepcopy(value['array_output'])
dims = tuple((self.cache[data_key]['array_dimensions'].index(p) for p in value['dimension']))
args = {}
if function_name == 'argmax' or function_name == 'argmin':
if len(dims) != 1:
args['axis'] = dims[0]
else:
args['axis'] = dims
if sys.version_info >= (3, 0):
if 'skipna' in list(inspect.signature(self.REDUCTION_FNS[function_name]).parameters.keys()) and \
function_name != 'prod':
args['skipna'] = True
else:
if 'skipna' in inspect.getargspec(self.REDUCTION_FNS[function_name])[0] and \
function_name != 'prod':
args['skipna'] = True
array_result['array_result'][key] = func(xr.DataArray(array_data), **args)
array_result['array_indices'] = copy.deepcopy(array_desc['array_indices'])
array_result['array_dimensions'] = copy.deepcopy(array_result['array_output']['dimensions_order'])
array_result['crs'] = copy.deepcopy(array_desc['crs'])
self.cache[key] = array_result
return self.cache[key]
def test_2():
# perform language test
ne = NDexpr()
ne.set_ae(False)
x1 = xr.DataArray(np.random.randn(2, 3))
y1 = xr.DataArray(np.random.randn(2, 3))
z1 = xr.DataArray(np.array([[[0, 1, 2], [3, 4, 5], [6, 7, 8]],
[[9, 10, 11], [12, 13, 14], [15, 16, 17]],
[[18, 19, 20], [21, 22, 23], [24, 25, 26]]
]))
z2 = z1*2
z3 = np.arange(27)
mask1 = z1 > 4
assert ne.test("arccos(z1)", xr.ufuncs.arccos(z1))
assert ne.test("angle(z1)", xr.ufuncs.angle(z1))
assert ne.test("arccos(z1)", xr.ufuncs.arccos(z1))
assert ne.test("arccosh(z1)", xr.ufuncs.arccosh(z1))
assert ne.test("arcsin(z1)", xr.ufuncs.arcsin(z1))
assert ne.test("arcsinh(z1)", xr.ufuncs.arcsinh(z1))
assert ne.test("arctan(z1)", xr.ufuncs.arctan(z1))
assert ne.test("arctanh(z1)", xr.ufuncs.arctanh(z1))
assert ne.test("ceil(z1)", xr.ufuncs.ceil(z1))
assert ne.test("conj(z1)", xr.ufuncs.conj(z1))
assert ne.test("cos(z1)", xr.ufuncs.cos(z1))
assert ne.test("cosh(z1)", xr.ufuncs.cosh(z1))
assert ne.test("deg2rad(z1)", xr.ufuncs.deg2rad(z1))
assert ne.test("degrees(z1)", xr.ufuncs.degrees(z1))
assert ne.test("exp(z1)", xr.ufuncs.exp(z1))
assert ne.test("expm1(z1)", xr.ufuncs.expm1(z1))
assert ne.test("fabs(z1)", xr.ufuncs.fabs(z1))
assert ne.test("fix(z1)", xr.ufuncs.fix(z1))
assert ne.test("floor(z1)", xr.ufuncs.floor(z1))
assert ne.test("frexp(z3)", xr.DataArray(xr.ufuncs.frexp(z3)))
assert ne.test("imag(z1)", xr.ufuncs.imag(z1))
assert ne.test("iscomplex(z1)", xr.ufuncs.iscomplex(z1))
assert ne.test("isfinite(z1)", xr.ufuncs.isfinite(z1))
assert ne.test("isinf(z1)", xr.ufuncs.isinf(z1))
assert ne.test("isnan(z1)", xr.ufuncs.isnan(z1))
assert ne.test("isreal(z1)", xr.ufuncs.isreal(z1))
assert ne.test("log(z1)", xr.ufuncs.log(z1))
assert ne.test("log10(z1)", xr.ufuncs.log10(z1))
assert ne.test("log1p(z1)", xr.ufuncs.log1p(z1))
assert ne.test("log2(z1)", xr.ufuncs.log2(z1))
assert ne.test("rad2deg(z1)", xr.ufuncs.rad2deg(z1))
assert ne.test("radians(z1)", xr.ufuncs.radians(z1))
assert ne.test("real(z1)", xr.ufuncs.real(z1))
assert ne.test("rint(z1)", xr.ufuncs.rint(z1))
assert ne.test("sign(z1)", xr.ufuncs.sign(z1))
assert ne.test("signbit(z1)", xr.ufuncs.signbit(z1))
assert ne.test("sin(z1)", xr.ufuncs.sin(z1))
assert ne.test("sinh(z1)", xr.ufuncs.sinh(z1))
assert ne.test("sqrt(z1)", xr.ufuncs.sqrt(z1))
assert ne.test("square(z1)", xr.ufuncs.square(z1))
assert ne.test("tan(z1)", xr.ufuncs.tan(z1))
assert ne.test("tanh(z1)", xr.ufuncs.tanh(z1))
assert ne.test("trunc(z1)", xr.ufuncs.trunc(z1))
assert ne.test("arctan2(z1, z2)", xr.ufuncs.arctan2(z1, z2))
assert ne.test("copysign(z1, z2)", xr.ufuncs.copysign(z1, z2))
assert ne.test("fmax(z1, z2)", xr.ufuncs.fmax(z1, z2))
assert ne.test("fmin(z1, z2)", xr.ufuncs.fmin(z1, z2))
assert ne.test("fmod(z1, z2)", xr.ufuncs.fmod(z1, z2))
assert ne.test("hypot(z1, z2)", xr.ufuncs.hypot(z1, z2))
assert ne.test("ldexp(z1, z2)", xr.DataArray(xr.ufuncs.ldexp(z1, z2)))
assert ne.test("logaddexp(z1, z2)", xr.ufuncs.logaddexp(z1, z2))
assert ne.test("logaddexp2(z1, z2)", xr.ufuncs.logaddexp2(z1, z2))
assert ne.test("logicaland(z1, z2)", xr.ufuncs.logical_and(z1, z2))
assert ne.test("logicalnot(z1, z2)", xr.ufuncs.logical_not(z1, z2))
assert ne.test("logicalor(z1, z2)", xr.ufuncs.logical_or(z1, z2))
assert ne.test("logicalxor(z1, z2)", xr.ufuncs.logical_xor(z1, z2))
assert ne.test("maximum(z1, z2)", xr.ufuncs.maximum(z1, z2))
assert ne.test("minimum(z1, z2)", xr.ufuncs.minimum(z1, z2))
assert ne.test("nextafter(z1, z2)", xr.ufuncs.nextafter(z1, z2))
assert ne.test("all(z1)", xr.DataArray.all(z1))
assert ne.test("all(z1, 0)", xr.DataArray.all(z1, axis=0))
assert ne.test("all(z1, 0, 1)", xr.DataArray.all(z1, axis=(0, 1)))
assert ne.test("all(z1, 0, 1, 2)", xr.DataArray.all(z1, axis=(0, 1, 2)))
assert ne.test("any(z1)", xr.DataArray.any(z1))
assert ne.test("any(z1, 0)", xr.DataArray.any(z1, axis=0))
assert ne.test("any(z1, 0, 1)", xr.DataArray.any(z1, axis=(0, 1)))
assert ne.test("any(z1, 0, 1, 2)", xr.DataArray.any(z1, axis=(0, 1, 2)))
assert ne.test("argmax(z1)", xr.DataArray.argmax(z1))
assert ne.test("argmax(z1, 0)", xr.DataArray.argmax(z1, axis=0))
assert ne.test("argmax(z1, 1)", xr.DataArray.argmax(z1, axis=1))
assert ne.test("argmax(z1, 2)", xr.DataArray.argmax(z1, axis=2))
assert ne.test("argmin(z1)", xr.DataArray.argmin(z1))
assert ne.test("argmin(z1, 0)", xr.DataArray.argmin(z1, axis=0))
assert ne.test("argmin(z1, 1)", xr.DataArray.argmin(z1, axis=1))
assert ne.test("argmin(z1, 2)", xr.DataArray.argmin(z1, axis=2))
assert ne.test("max(z1)", xr.DataArray.max(z1))
assert ne.test("max(z1, 0)", xr.DataArray.max(z1, axis=0))
assert ne.test("max(z1, 0, 1)", xr.DataArray.max(z1, axis=(0, 1)))
assert ne.test("max(z1, 0, 1, 2)", xr.DataArray.max(z1, axis=(0, 1, 2)))
assert ne.test("mean(z1)", xr.DataArray.mean(z1))
assert ne.test("mean(z1, 0)", xr.DataArray.mean(z1, axis=0))
assert ne.test("mean(z1, 0, 1)", xr.DataArray.mean(z1, axis=(0, 1)))
assert ne.test("mean(z1, 0, 1, 2)", xr.DataArray.mean(z1, axis=(0, 1, 2)))
assert ne.test("median(z1)", xr.DataArray.median(z1))
assert ne.test("median(z1, 0)", xr.DataArray.median(z1, axis=0))
assert ne.test("median(z1, 0, 1)", xr.DataArray.median(z1, axis=(0, 1)))
assert ne.test("median(z1, 0, 1, 2)", xr.DataArray.median(z1, axis=(0, 1, 2)))
assert ne.test("min(z1)", xr.DataArray.min(z1))
assert ne.test("min(z1, 0)", xr.DataArray.min(z1, axis=0))
assert ne.test("min(z1, 0, 1)", xr.DataArray.min(z1, axis=(0, 1)))
assert ne.test("min(z1, 0, 1, 2)", xr.DataArray.min(z1, axis=(0, 1, 2)))
assert ne.test("prod(z1)", xr.DataArray.prod(z1))
assert ne.test("prod(z1, 0)", xr.DataArray.prod(z1, axis=0))
assert ne.test("prod(z1, 0, 1)", xr.DataArray.prod(z1, axis=(0, 1)))
assert ne.test("prod(z1, 0, 1, 2)", xr.DataArray.prod(z1, axis=(0, 1, 2)))
assert ne.test("sum(z1)", xr.DataArray.sum(z1))
assert ne.test("sum(z1, 0)", xr.DataArray.sum(z1, axis=0))
assert ne.test("sum(z1, 0, 1)", xr.DataArray.sum(z1, axis=(0, 1)))
assert ne.test("sum(z1, 0, 1, 2)", xr.DataArray.sum(z1, axis=(0, 1, 2)))
assert ne.test("std(z1)", xr.DataArray.std(z1))
assert ne.test("std(z1, 0)", xr.DataArray.std(z1, axis=0))
assert ne.test("std(z1, 0, 1)", xr.DataArray.std(z1, axis=(0, 1)))
assert ne.test("std(z1, 0, 1, 2)", xr.DataArray.std(z1, axis=(0, 1, 2)))
assert ne.test("var(z1)", xr.DataArray.var(z1))
assert ne.test("var(z1, 0)", xr.DataArray.var(z1, axis=0))
assert ne.test("var(z1, 0, 1)", xr.DataArray.var(z1, axis=(0, 1)))
assert ne.test("var(z1, 0, 1, 2)", xr.DataArray.var(z1, axis=(0, 1, 2)))
assert ne.test("percentile(z1, 50)", np.percentile(z1, 50))
assert ne.test("percentile(z1, 50)+percentile(z1, 50)", np.percentile(z1, 50) + np.percentile(z1, 50))
assert ne.test("percentile(z1, (50))", np.percentile(z1, (50)))
assert ne.test("percentile(z1, (50, 60))", np.percentile(z1, (50, 60)))
assert ne.test("percentile(z1, (50, 60, 70))", np.percentile(z1, (50, 60, 70)))
assert ne.test("percentile(z1, (50, 60, 70)) + percentile(z1, (50, 60, 70))",
np.percentile(z1, (50, 60, 70)) + np.percentile(z1, (50, 60, 70)))
assert ne.test("1 + var(z1, 0, 0+1, 2) + 1", 1+xr.DataArray.var(z1, axis=(0, 0+1, 2))+1)
assert ne.test("z1{mask1}", xr.DataArray.where(z1, mask1))
assert ne.test("z1{z1>2}", xr.DataArray.where(z1, z1 > 2))
assert ne.test("z1{z1>=2}", xr.DataArray.where(z1, z1 >= 2))
assert ne.test("z1{z1<2}", xr.DataArray.where(z1, z1 < 2))
assert ne.test("z1{z1<=2}", xr.DataArray.where(z1, z1 <= 2))
assert ne.test("z1{z1==2}", xr.DataArray.where(z1, z1 == 2))
assert ne.test("z1{z1!=2}", xr.DataArray.where(z1, z1 != 2))
assert ne.test("z1{z1<2 | z1>5}", xr.DataArray.where(z1, (z1 < 2) | (z1 > 5)))
assert ne.test("z1{z1>2 & z1<5}", xr.DataArray.where(z1, (z1 > 2) & (z1 < 5)))
ne.evaluate("m = z1+1")
assert ne.test("m", z1+1)
assert ne.test("z1{~mask1}", xr.DataArray.where(z1, ~mask1))
assert ne.test("(1<0?1+1;2+2)", 4)
assert ne.test("(0<1?1+1;2+2)", 2)
assert ne.test("z1+mask1", xr.DataArray.where(z1, mask1))
assert ne.is_number(1) is True
assert ne.test("-z1", -z1)
assert ne.test("z1{!(z1<2)}", xr.DataArray.where(z1, xr.ufuncs.logical_not(z1 < 2)))
