def main():
# Testing stuff
g = GDF()
# g.debug = True
# pprint(g.storage_config['LS5TM'])
# pprint(dict(g.storage_config['LS5TM']['dimensions']))
# pprint(dict(g.storage_config['LS5TM']['measurement_types']))
# pprint(g.storage_config['LS8OLI'])
# pprint(dict(g.storage_config['LS8OLI']['dimensions']))
# pprint(dict(g.storage_config['LS8OLI']['measurement_types']))
data_request_descriptor = {'storage_type': 'LS5TM',
'variables': ('B30',),
'dimensions': {'X': {'range': (147.968, 148.032)},
'Y': {'range': (-36.032, -35.968)},
'T': {'range': (1262304000.0, 1325375999.999999)}, # 2010-01-01 00:00:00.0 - 2011-12-31 23:59:59.999999
}
}
t0 = datetime.now()
print 'Starting 256 x 256 single-band cross-boundary descriptor at ', t0
d = g.get_descriptor(data_request_descriptor)
t1 = datetime.now()
print 'Finishing 256 x 256 cross-boundary descriptor at %s (Elapsed time %s)' % (t1, t1 - t0)
pprint(d)
t0 = datetime.now()
print 'Starting 256 x 256 single-band cross-boundary selection at ', t0
a = g.get_data(data_request_descriptor)
t1 = datetime.now()
print 'Finishing 256 x 256 cross-boundary selection at %s (Elapsed time %s)' % (t1, t1 - t0)
pprint(a)
def main():
# Testing stuff
g = GDF(opendap=True)
# g.debug = True
# pprint(g.storage_config['LS5TM'])
# pprint(dict(g.storage_config['LS5TM']['dimensions']))
# pprint(dict(g.storage_config['LS5TM']['measurement_types']))
# pprint(g.storage_config['LS8OLI'])
# pprint(dict(g.storage_config['LS8OLI']['dimensions']))
# pprint(dict(g.storage_config['LS8OLI']['measurement_types']))
data_request_descriptor = {
"storage_type": "LS5TM",
"variables": ("B30",),
"dimensions": {
"X": {"range": (147.968, 148.032)},
"Y": {"range": (-36.032, -35.968)},
"T": {"range": (1262304000.0, 1325375999.999999)}, # 2010-01-01 00:00:00.0 - 2011-12-31 23:59:59.999999
},
}
t0 = datetime.now()
print "Starting 256 x 256 single-band cross-boundary descriptor at ", t0
d = g.get_descriptor(data_request_descriptor)
t1 = datetime.now()
print "Finishing 256 x 256 cross-boundary descriptor at %s (Elapsed time %s)" % (t1, t1 - t0)
pprint(d)
t0 = datetime.now()
print "Starting 256 x 256 single-band cross-boundary selection at ", t0
a = g.get_data(data_request_descriptor)
t1 = datetime.now()
print "Finishing 256 x 256 cross-boundary selection at %s (Elapsed time %s)" % (t1, t1 - t0)
pprint(a)
def test_GDF_get_descriptor(self):
"Test GDF get_descriptor function"
#TODO: Define tests which check DB contents
test_gdf = GDF() # Test default configuration
descriptor = test_gdf.get_descriptor(self.TEST_3D_PARAMETER)
descriptor = test_gdf.get_descriptor(self.TEST_2D_PARAMETER)
descriptor = test_gdf.get_descriptor()
def test_GDF_get_descriptor(self):
"Test GDF get_descriptor function"
#TODO: Define tests which check DB contents
test_gdf = GDF() # Test default configuration
descriptor = test_gdf.get_descriptor(self.TEST_3D_PARAMETER)
descriptor = test_gdf.get_descriptor(self.TEST_2D_PARAMETER)
descriptor = test_gdf.get_descriptor()
def open_raw_record_file(self):
self.ws_client.raw_data.clear()
self.ws_client.raw_data_ticks.clear()
self.ws_client.raw_data_events.clear()
self.mkdir_p(os.path.dirname(self.filepath))
self.target_file = GDF(filename=self.filepath,
ch_num=self.ch_num,
ch_label=self.ch_label,
sps=self.raw_sample_rate)
def test_GDF(self):
"Test GDF constructor"
test_gdf = GDF() # Test default configuration
assert test_gdf.code_root == os.path.abspath(
os.path.join(os.path.dirname(os.path.dirname(__file__)),
'gdf')), 'Code root is incorrect'
assert len(test_gdf.config_files
) == 1, 'Default config path list should have one entry'
assert test_gdf.config_files[0] == os.path.abspath(
os.path.join(
test_gdf.code_root,
GDF.DEFAULT_CONFIG_FILE)), 'Default config path is incorrect'
assert test_gdf.command_line_params is not None, 'Command line parameter dict not set'
assert test_gdf.configuration is not None, 'Configurations dict not set'
assert len(test_gdf.configuration
) > 0, 'Config files must define at least one setup'
assert len(
test_gdf.databases) > 0, 'At least one database must be set up'
assert test_gdf.storage_config is not None, 'storage configuration dict not set'
assert len(
test_gdf.storage_config
) > 0, 'storage configuration dict must contain at least one storage_type definition'
def __init__(self):
logger.debug('Initialise Execution Module.')
self.gdf = GDF()
self.gdf.debug = False
self.cache = {}
class ExecutionEngine(object):
SUPPORTED_REDUCTION_OPERATORS = [ 'min', 'max', 'amin', 'amax', 'nanmin', 'nanmax', 'ptp',
'median', 'average', 'mean', 'std', 'var', 'nanmean', 'nanstd', 'nanvar',
'argmax', 'argmin', 'sum', 'prod', 'all', 'any'
]
def __init__(self):
logger.debug('Initialise Execution Module.')
self.gdf = GDF()
self.gdf.debug = False
self.cache = {}
def executePlan(self, plan):
for task in plan:
function = task.values()[0]['orig_function']
print 'function =', function
if function == 'get_data': # get data
self.executeGetData(task)
elif function == 'apply_cloud_mask': # apply cloud mask
self.executeCloudMask(task)
elif len([s for s in self.SUPPORTED_REDUCTION_OPERATORS if s in function]) > 0: # reduction operator
self.executeReduction(task)
else: # bandmath
self.executeBandmath(task)
def executeGetData(self, task):
data_request_param = {}
data_request_param['dimensions'] = task.values()[0]['array_input'][0].values()[0]['dimensions']
data_request_param['storage_type'] = task.values()[0]['array_input'][0].values()[0]['storage_type']
data_request_param['variables'] = ()
for array in task.values()[0]['array_input']:
data_request_param['variables'] += (array.values()[0]['variable'],)
data_response = self.gdf.get_data(data_request_param)
key = task.keys()[0]
self.cache[key] = {}
self.cache[key]['array_result'] = copy.deepcopy(data_response['arrays'])
self.cache[key]['array_indices'] = copy.deepcopy(data_response['indices'])
self.cache[key]['array_dimensions'] = copy.deepcopy(data_response['dimensions'])
self.cache[key]['array_output'] = copy.deepcopy(task.values()[0]['array_output'])
del data_request_param
del data_response
return self.cache[key]
def executeCloudMask(self, task):
key = task.keys()[0]
data_key = task.values()[0]['array_input'][0]
mask_key = task.values()[0]['array_mask']
no_data_value = task.values()[0]['array_output']['no_data_value']
print 'key =', key
print 'data key =', data_key
print 'data mask_key =', mask_key
print 'no_data_value =', no_data_value
array_desc = self.cache[task.values()[0]['array_input'][0]]
data_array = self.cache[data_key]['array_result'].values()[0]
mask_array = self.cache[mask_key]['array_result'].values()[0]
pqa_mask = get_pqa_mask(mask_array)
masked_array = copy.deepcopy(data_array)
masked_array[~pqa_mask] = no_data_value
self.cache[key] = {}
self.cache[key]['array_result'] = {}
self.cache[key]['array_result'][key] = masked_array
self.cache[key]['array_indices'] = copy.deepcopy(array_desc['array_indices'])
self.cache[key]['array_dimensions'] = copy.deepcopy(array_desc['array_dimensions'])
self.cache[key]['array_output'] = copy.deepcopy(task.values()[0]['array_output'])
def executeBandmath(self, task):
key = task.keys()[0]
# TODO: check all input arrays are the same shape and parameters
arrays = {}
for task_name in task.values()[0]['array_input']:
arrays.update(self.cache[task_name]['array_result'])
arrayResult = {}
arrayResult['array_result'] = {}
arrayResult['array_result'][key] = ne.evaluate(task.values()[0]['function'], arrays)
no_data_value = task.values()[0]['array_output']['no_data_value']
array_desc = self.cache[task.values()[0]['array_input'][0]]
for array in array_desc['array_result'].values():
arrayResult['array_result'][key][array == no_data_value] = no_data_value
arrayResult['array_indices'] = copy.deepcopy(array_desc['array_indices'])
arrayResult['array_dimensions'] = copy.deepcopy(array_desc['array_dimensions'])
arrayResult['array_output'] = copy.deepcopy(task.values()[0]['array_output'])
self.cache[key] = arrayResult
return self.cache[key]
def executeReduction(self, task):
function_name = task.values()[0]['orig_function'].replace(")"," ").replace("("," ").split()[0]
func = getattr(np, function_name)
key = key = task.keys()[0]
data_key = task.values()[0]['array_input'][0]
print 'key =', key
print 'data key =', data_key
data = self.cache[data_key]['array_dimensions']
no_data_value = task.values()[0]['array_output']['no_data_value']
array_data = self.cache[data_key]['array_result'].values()[0]
array_desc = self.cache[task.values()[0]['array_input'][0]]
arrayResult = {}
arrayResult['array_result'] = {}
arrayResult['array_output'] = copy.deepcopy(task.values()[0]['array_output'])
if len(task.values()[0]['dimension']) == 1: # 3D -> 2D reduction
pprint(self.cache[data_key]['array_dimensions'])
dim = self.cache[data_key]['array_dimensions'].index(task.values()[0]['dimension'][0])
arrayResult['array_result'][key]=np.apply_along_axis(lambda x: func(x[x!=no_data_value]), dim, array_data)
arrayResult['array_indices'] = copy.deepcopy(array_desc['array_indices'])
arrayResult['array_dimensions'] = copy.deepcopy(arrayResult['array_output']['dimensions_order'])
for index in array_desc['array_indices']:
if index not in arrayResult['array_dimensions'] and index in arrayResult['array_indices']:
del arrayResult['array_indices'][index]
elif len(task.values()[0]['dimension']) == 2: # 3D -> 1D reduction
size = task.values()[0]['array_output']['shape'][0]
print 'size =', size
out = np.empty([size])
dim = self.cache[data_key]['array_dimensions'].index(task.values()[0]['array_output']['dimensions_order'][0])
print 'dim =', dim
#to fix bug in gdf
#size = self.cache[data_key]['array_result'].values()[0].shape[dim]
#print 'size =', size
#out = np.empty([size])
for i in range(size):
if dim == 0:
out[i] = func(array_data[i,:,:][array_data[i,:,:] != no_data_value])
elif dim == 1:
out[i] = func(array_data[:,i,:][array_data[:,i,:] != no_data_value])
elif dim == 2:
out[i] = func(array_data[:,:,i][array_data[:,:,i] != no_data_value])
if np.isnan(out[i]):
out[i] = no_data_value
arrayResult['array_result'][key] = out
arrayResult['array_indices'] = copy.deepcopy(array_desc['array_indices'])
arrayResult['array_dimensions'] = copy.deepcopy(arrayResult['array_output']['dimensions_order'])
for index in array_desc['array_indices']:
if index not in arrayResult['array_dimensions'] and index in arrayResult['array_indices']:
del arrayResult['array_indices'][index]
self.cache[key] = arrayResult
return self.cache[key]
def __init__(self):
logger.debug("Initialise Execution Module.")
self.gdf = GDF()
self.gdf.debug = False
self.cache = {}
num_t = data_t.shape[0]
num_rowcol = math.ceil(math.sqrt(num_t))
fig = plt.figure()
fig.clf()
plot_count = 1
for i in range(data_t.shape[0]):
data = data_t[i]
ax = fig.add_subplot(num_rowcol,num_rowcol,plot_count)
plt.contourf(lat, lon, data, 64)
plot_count += 1
fig.tight_layout()
plt.show()
# In[4]:
g = GDF()
g.debug = False
# In[5]:
start_date = dt2secs(date(year=2010,month=1,day=1))
end_date = dt2secs(date(year=2010, month=1, day=18))
data_request_descriptors = []
data_request_descriptors.append( {'storage_type': 'LS5TM',
'variables': ('B30',),
'dimensions': {'X': {'range': (140.0, 141.0)},
'Y': {'range': (-36.0, -35.0)},
'T': {'range': (start_date, end_date),
# 'array_range': (0, 4)
#'crs': 'SSE', # Seconds since epoch
#get_ipython().magic(u'pylab')
# %pylab inline
# import mpld3; mpld3.enable_notebook()
# In[2]:
import calendar
from datetime import datetime, date, timedelta
from gdf import GDF
from pprint import pprint
from gdf import dt2secs, secs2dt
import matplotlib.pyplot as plt
# In[4]:
g = GDF()
g.debug = False
# In[5]:
start_date = dt2secs(date(year=2010, month=1, day=1))
end_date = dt2secs(date(year=2010, month=1, day=18))
data_request_descriptor = {'storage_type': 'LS5TM',
'variables': ('B40', 'B30',),
'dimensions': {'X': {'range': (147.875, 148.125)},
'Y': {'range': (-37.0 + 0.875, -36.0 + 0.125)},
# 'T': {'range': (start_date, end_date),
# 'array_range': (0, 4)
# 'crs': 'SSE', # Seconds since epoch
# 'grouping_function': g.null_grouping
class GDF_FILE(object):
def __init__(self, filepath, ws_client):
self.filepath = filepath
self.ws_client = ws_client
self.filepath = filepath
self.ws_client = ws_client
self.raw_sample_rate = self.ws_client.raw_sample_rate
self.ch_num = self.ws_client.ch_num
self.ch_label = self.ws_client.ch_label
self.raw_cache_size = 1000
self.record_start_tick = None
self.open_raw_record_file()
def mkdir_p(self, path):
try:
os.makedirs(path)
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
def open_raw_record_file(self):
self.ws_client.raw_data.clear()
self.ws_client.raw_data_ticks.clear()
self.ws_client.raw_data_events.clear()
self.mkdir_p(os.path.dirname(self.filepath))
self.target_file = GDF(filename=self.filepath,
ch_num=self.ch_num,
ch_label=self.ch_label,
sps=self.raw_sample_rate)
def close_raw_record_file(self):
while self.ws_client.raw_data:
self.write_raw_data_to_file()
self.ws_client.raw_data.clear()
self.ws_client.raw_data_ticks.clear()
self.ws_client.raw_data_events.clear()
self.target_file.close_and_save()
self.target_file = None
def write_raw_data_to_file(self):
if self.target_file is not None:
if self.record_start_tick is None:
self.record_start_tick = self.ws_client.raw_data_ticks[0]
copied_data = self.ws_client.raw_data[:self.raw_cache_size]
self.ws_client.raw_data = self.ws_client.raw_data[self.
raw_cache_size:]
copied_ticks = self.ws_client.raw_data_ticks[:self.raw_cache_size]
self.ws_client.raw_data_ticks = self.ws_client.raw_data_ticks[
self.raw_cache_size:]
copied_events = self.ws_client.raw_data_events[:self.
raw_cache_size]
self.ws_client.raw_data_events = self.ws_client.raw_data_events[
self.raw_cache_size:]
assert len(copied_data) == len(copied_ticks)
assert len(copied_data) == len(copied_events)
self.target_file.writeSamples(copied_data)
for tick, events in zip(copied_ticks, copied_events):
for event_id, event_duration in zip(events["event_id"],
events["event_duration"]):
ed = None
if event_duration is None:
ed = 0
else:
ed = round(event_duration * self.raw_sample_rate)
self.target_file.writeEvent(
first_tick=self.record_start_tick,
tick=tick,
duratoin=ed,
event_id=event_id)
@author: woo409
"""
# In[2]:
from glue.core import Data, DataCollection
from glue.qt.glue_application import GlueApplication
from datetime import datetime, date, timedelta
from gdf import GDF
from gdf import dt2secs, secs2dt
from analytics import Analytics
from execution_engine import ExecutionEngine
# In[4]:
a = Analytics()
e = ExecutionEngine()
g = GDF()
g.debug = False
# In[5]:
start_date = dt2secs(date(year=2010,month=1,day=1))
end_date = dt2secs(date(year=2010, month=1, day=18))
data_request_descriptor = {'storage_type': 'LS5TM',
'variables': ('B40',),
'dimensions': {'X': {'range': (149.0699, 149.152)},
'Y': {'range': (-35.3117, -35.2842)},
#'T': {'range': (start_date, end_date),
# 'array_range': (0, 4)
#'crs': 'SSE', # Seconds since epoch
#'grouping_function': g.null_grouping
class Analytics(object):
SUPPORTED_OUTPUT_TYPES = ['netcdf-cf', 'geotiff']
OPERATORS_SENSOR_SPECIFIC_BANDMATH = \
{
'ndvi': \
{
'sensors': \
{
'LS5TM' : { 'input': ['B40', 'B30'], 'function': 'ndvi'},
'LS7ETM' : { 'input': ['B40', 'B30'], 'function': 'ndvi'},
'LS8OLI' : { 'input': ['B5', 'B4'], 'function': 'ndvi'},
'LS8OLITIRS' : { 'input': ['B5', 'B4'], 'function': 'ndvi'},
},
'functions': \
{
'ndvi' : '((array1 - array2) / (array1 + array2))'
}
}
}
OPERATORS_REDUCTION = \
{
'median': 'median(array1)'
}
def __init__(self):
logger.debug('Initialise Analytics Module.')
self.gdf = GDF()
self.gdf.debug = False
self.plan = []
self.planDict = {}
def task(self, name):
return self.plan[self.planDict[name]]
def add_to_plan(self, name, task):
self.plan.append( { name : task })
size = len(self.plan)
self.planDict[name] = size-1
return self.plan[size-1]
def createArray(self, storage_type, variables, dimensions, name):
query_parameters = {}
query_parameters['storage_type'] = storage_type
query_parameters['dimensions'] = dimensions
query_parameters['variables'] = ()
for array in variables:
query_parameters['variables'] += (array,)
arrayDescriptors = self.gdf.get_descriptor(query_parameters)
if storage_type not in arrayDescriptors.keys():
raise AssertionError(storage_type, "not present in descriptor")
logger.debug('storage_type = %s', storage_type)
arrayResults = []
for variable in variables:
if variable not in arrayDescriptors[storage_type]['variables']:
raise AssertionError(variable, "not present in", storage_type, "descriptor")
logger.debug('variable = %s', variable)
arrayResult = {}
arrayResult['storage_type'] = storage_type
arrayResult['variable'] = variable
arrayResult['dimensions_order'] = arrayDescriptors[storage_type]['dimensions']
arrayResult['dimensions'] = dimensions
arrayResult['shape'] = arrayDescriptors[storage_type]['result_shape']
arrayResult['data_type'] = arrayDescriptors[storage_type]['variables'][variable]['numpy_datatype_name']
arrayResult['no_data_value'] = arrayDescriptors[storage_type]['variables'][variable]['nodata_value']
arrayResults.append({variable : arrayResult})
task = {}
task['array_input'] = arrayResults
task['array_output'] = copy.deepcopy(arrayResult)
task['array_output']['variable'] = 'data'
task['function'] = 'get_data'
task['orig_function'] = 'get_data'
return self.add_to_plan(name, task)
def applyCloudMask(self, arrays, mask, name):
size = len(arrays)
if size == 0:
raise AssertionError("Input array is empty")
task = {}
task['array_input'] = []
for array in arrays.keys():
task['array_input'].append(array)
task['array_mask'] = mask.keys()[0]
task['orig_function'] = 'apply_cloud_mask'
task['function'] = 'apply_cloud_mask'
task['array_output'] = copy.deepcopy(arrays.values()[0]['array_output'])
task['array_output']['variable'] = name
return self.add_to_plan(name, task)
# generic version
def applyBandMath(self, arrays, function, name):
# Arrays is a list
# output Array is same shape as input array
size = len(arrays)
if size == 0:
raise AssertionError("Input array is empty")
variables = []
if size == 1: # 1 input
if arrays.values()[0]['function'] == 'get_data':
# check shape is same for all input arrays
shape = arrays.values()[0]['array_input'][0].values()[0]['shape']
for variable in arrays.values()[0]['array_input']:
if shape != variable.values()[0]['shape']:
raise AssertionError("Shape is different")
variables.append(variable.keys()[0])
else:
variables.append(arrays.keys()[0])
orig_function = function
logger.debug('function before = %s', function)
count = 1
for variable in variables:
function = function.replace('array'+str(count), variable)
count += 1
logger.debug('function after = %s', function)
task = {}
task['array_input'] = []
task['array_input'].append(arrays.keys()[0])
task['orig_function'] = orig_function
task['function'] = function
task['array_output'] = copy.deepcopy(arrays.values()[0]['array_output'])
task['array_output']['variable'] = name
return self.add_to_plan(name, task)
else: # multi-dependencies
pprint(arrays)
for array in arrays:
variables.append(array.keys()[0])
orig_function = function
logger.debug('function before = %s', function)
count = 1
for variable in variables:
function = function.replace('array'+str(count), variable)
count += 1
logger.debug('function after = %s', function)
task = {}
task['array_input'] = []
for array in arrays:
task['array_input'].append(array.keys()[0])
task['orig_function'] = orig_function
task['function'] = function
task['array_output'] = copy.deepcopy(arrays[0].values()[0]['array_output'])
task['array_output']['variable'] = name
return self.add_to_plan(name, task)
# sensor specific version
def applySensorSpecificBandMath(self, storage_types, function, dimensions, name_data, name_result):
variables = self.getPredefinedInputs(storage_types, function)
arrays = self.createArray(storage_types, variables, dimensions, name_data)
function_text = self.getPredefinedFunction(storage_types, function)
return self.applyBandMath(arrays, function_text, name_result)
def applyReduction(self, array1, dimensions, function, name):
function = self.OPERATORS_REDUCTION.get(function)
return self.applyGenericReductionFunction(array1, dimensions, function, name)
def applyGenericReduction(self, arrays, dimensions, function, name):
size = len(arrays)
if size != 1:
raise AssertionError("Input array should be 1")
if arrays.values()[0]['function'] == 'get_data':
variable = arrays.values()[0]['array_input'][0].values()[0]['variable']
orig_function = function
logger.debug('function before = %s', function)
function = function.replace('array1', variable)
logger.debug('function after = %s', function)
task = {}
task['array_input'] = []
task['array_input'].append(arrays.keys()[0])
task['orig_function'] = orig_function
task['function'] = function
task['dimension'] = copy.deepcopy(dimensions)
task['array_output'] = copy.deepcopy(arrays.values()[0]['array_output'])
task['array_output']['variable'] = name
task['array_output']['dimensions_order'] = self.diffList(arrays.values()[0]['array_input'][0].values()[0]['dimensions_order'], dimensions)
result = ()
for value in task['array_output']['dimensions_order']:
input_task = self.task(task['array_input'][0])
index = input_task.values()[0]['array_output']['dimensions_order'].index(value)
value = input_task.values()[0]['array_output']['shape'][index]
result += (value,)
task['array_output']['shape'] = result
return self.add_to_plan(name, task)
else:
variable = arrays.keys()[0]
orig_function = function
logger.debug('function before = %s', function)
function = function.replace('array1', variable)
logger.debug('function after = %s', function)
task = {}
task['array_input'] = []
task['array_input'].append(variable)
task['orig_function'] = orig_function
task['function'] = function
task['dimension'] = copy.deepcopy(dimensions)
task['array_output'] = copy.deepcopy(arrays.values()[0]['array_output'])
task['array_output']['variable'] = name
task['array_output']['dimensions_order'] = self.diffList(arrays.values()[0]['array_output']['dimensions_order'], dimensions)
result = ()
for value in task['array_output']['dimensions_order']:
input_task = self.task(task['array_input'][0])
pprint(input_task)
index = input_task.values()[0]['array_output']['dimensions_order'].index(value)
value = input_task.values()[0]['array_output']['shape'][index]
result += (value,)
task['array_output']['shape'] = result
return self.add_to_plan(name, task)
def diffList(self, list1, list2):
list2 = set(list2)
return [result for result in list1 if result not in list2]
def getPredefinedInputs(self, storage_type, function):
return self.OPERATORS_SENSOR_SPECIFIC_BANDMATH.get(function).get('sensors').get(storage_type).get('input')
def getPredefinedFunction(self, storage_type, function):
function_id = self.OPERATORS_SENSOR_SPECIFIC_BANDMATH.get(function).get('sensors').get(storage_type).get('function')
return self.OPERATORS_SENSOR_SPECIFIC_BANDMATH.get(function).get('functions').get(function_id)
class ExecutionEngine(object):
SUPPORTED_REDUCTION_OPERATORS = [
"min",
"max",
"amin",
"amax",
"nanmin",
"nanmax",
"ptp",
"median",
"average",
"mean",
"std",
"var",
"nanmean",
"nanstd",
"nanvar",
"argmax",
"argmin",
"sum",
"prod",
"all",
"any",
]
def __init__(self):
logger.debug("Initialise Execution Module.")
self.gdf = GDF()
self.gdf.debug = False
self.cache = {}
def executePlan(self, plan):
for task in plan:
function = task.values()[0]["orig_function"]
print "function =", function
if function == "get_data": # get data
self.executeGetData(task)
elif function == "apply_cloud_mask": # apply cloud mask
self.executeCloudMask(task)
elif len([s for s in self.SUPPORTED_REDUCTION_OPERATORS if s in function]) > 0: # reduction operator
self.executeReduction(task)
else: # bandmath
self.executeBandmath(task)
def executeGetData(self, task):
data_request_param = {}
data_request_param["dimensions"] = task.values()[0]["array_input"][0].values()[0]["dimensions"]
data_request_param["storage_type"] = task.values()[0]["array_input"][0].values()[0]["storage_type"]
data_request_param["variables"] = ()
for array in task.values()[0]["array_input"]:
data_request_param["variables"] += (array.values()[0]["variable"],)
data_response = self.gdf.get_data(data_request_param)
key = task.keys()[0]
self.cache[key] = {}
self.cache[key]["array_result"] = copy.deepcopy(data_response["arrays"])
self.cache[key]["array_indices"] = copy.deepcopy(data_response["indices"])
self.cache[key]["array_dimensions"] = copy.deepcopy(data_response["dimensions"])
self.cache[key]["array_output"] = copy.deepcopy(task.values()[0]["array_output"])
del data_request_param
del data_response
return self.cache[key]
def executeCloudMask(self, task):
key = task.keys()[0]
data_key = task.values()[0]["array_input"][0]
mask_key = task.values()[0]["array_mask"]
no_data_value = task.values()[0]["array_output"]["no_data_value"]
print "key =", key
print "data key =", data_key
print "data mask_key =", mask_key
print "no_data_value =", no_data_value
array_desc = self.cache[task.values()[0]["array_input"][0]]
data_array = self.cache[data_key]["array_result"].values()[0]
mask_array = self.cache[mask_key]["array_result"].values()[0]
pqa_mask = get_pqa_mask(mask_array)
masked_array = copy.deepcopy(data_array)
masked_array[~pqa_mask] = no_data_value
self.cache[key] = {}
self.cache[key]["array_result"] = {}
self.cache[key]["array_result"][key] = masked_array
self.cache[key]["array_indices"] = copy.deepcopy(array_desc["array_indices"])
self.cache[key]["array_dimensions"] = copy.deepcopy(array_desc["array_dimensions"])
self.cache[key]["array_output"] = copy.deepcopy(task.values()[0]["array_output"])
def executeBandmath(self, task):
key = task.keys()[0]
# TODO: check all input arrays are the same shape and parameters
arrays = {}
for task_name in task.values()[0]["array_input"]:
arrays.update(self.cache[task_name]["array_result"])
arrayResult = {}
arrayResult["array_result"] = {}
arrayResult["array_result"][key] = ne.evaluate(task.values()[0]["function"], arrays)
no_data_value = task.values()[0]["array_output"]["no_data_value"]
array_desc = self.cache[task.values()[0]["array_input"][0]]
for array in array_desc["array_result"].values():
arrayResult["array_result"][key][array == no_data_value] = no_data_value
arrayResult["array_indices"] = copy.deepcopy(array_desc["array_indices"])
arrayResult["array_dimensions"] = copy.deepcopy(array_desc["array_dimensions"])
arrayResult["array_output"] = copy.deepcopy(task.values()[0]["array_output"])
self.cache[key] = arrayResult
return self.cache[key]
def executeReduction(self, task):
function_name = task.values()[0]["orig_function"].replace(")", " ").replace("(", " ").split()[0]
func = getattr(np, function_name)
key = key = task.keys()[0]
data_key = task.values()[0]["array_input"][0]
print "key =", key
print "data key =", data_key
data = self.cache[data_key]["array_dimensions"]
no_data_value = task.values()[0]["array_output"]["no_data_value"]
array_data = self.cache[data_key]["array_result"].values()[0]
array_desc = self.cache[task.values()[0]["array_input"][0]]
arrayResult = {}
arrayResult["array_result"] = {}
arrayResult["array_output"] = copy.deepcopy(task.values()[0]["array_output"])
if len(task.values()[0]["dimension"]) == 1: # 3D -> 2D reduction
pprint(self.cache[data_key]["array_dimensions"])
dim = self.cache[data_key]["array_dimensions"].index(task.values()[0]["dimension"][0])
arrayResult["array_result"][key] = np.apply_along_axis(
lambda x: func(x[x != no_data_value]), dim, array_data
)
arrayResult["array_indices"] = copy.deepcopy(array_desc["array_indices"])
arrayResult["array_dimensions"] = copy.deepcopy(arrayResult["array_output"]["dimensions_order"])
for index in array_desc["array_indices"]:
if index not in arrayResult["array_dimensions"] and index in arrayResult["array_indices"]:
del arrayResult["array_indices"][index]
elif len(task.values()[0]["dimension"]) == 2: # 3D -> 1D reduction
size = task.values()[0]["array_output"]["shape"][0]
print "size =", size
out = np.empty([size])
dim = self.cache[data_key]["array_dimensions"].index(
task.values()[0]["array_output"]["dimensions_order"][0]
)
print "dim =", dim
# to fix bug in gdf
# size = self.cache[data_key]['array_result'].values()[0].shape[dim]
# print 'size =', size
# out = np.empty([size])
for i in range(size):
if dim == 0:
out[i] = func(array_data[i, :, :][array_data[i, :, :] != no_data_value])
elif dim == 1:
out[i] = func(array_data[:, i, :][array_data[:, i, :] != no_data_value])
elif dim == 2:
out[i] = func(array_data[:, :, i][array_data[:, :, i] != no_data_value])
if np.isnan(out[i]):
out[i] = no_data_value
arrayResult["array_result"][key] = out
arrayResult["array_indices"] = copy.deepcopy(array_desc["array_indices"])
arrayResult["array_dimensions"] = copy.deepcopy(arrayResult["array_output"]["dimensions_order"])
for index in array_desc["array_indices"]:
if index not in arrayResult["array_dimensions"] and index in arrayResult["array_indices"]:
del arrayResult["array_indices"][index]
self.cache[key] = arrayResult
return self.cache[key]
def __init__(self):
logger.debug('Initialise Analytics Module.')
self.gdf = GDF()
self.gdf.debug = False
self.plan = []
self.planDict = {}
plot_count = 1
for i in range(img.shape[0]):
data = img[i]
ax = fig.add_subplot(num_rowcol,num_rowcol,plot_count)
plt.setp(ax, xticks=[], yticks=[])
#cax = ax.imshow(data, interpolation='nearest', aspect = 'equal')
cax = ax.pcolormesh(data) #, interpolation='nearest', aspect = 'equal')
#fig.colorbar(cax)
plot_count += 1
fig.tight_layout()
plt.show()
# In[4]:
g = GDF()
#g.debug = True
# In[5]:
data_request_descriptor = {'storage_type': 'LS7ETM',
'variables': ('B30', 'B40'),
'dimensions': {'X': {'range': (140.0, 140.125)},
'Y': {'range': (-35.0, -35.0+0.125)},
#'T': {'range': (0, 6325376000),
# 'array_range': (0, 4)
#'crs': 'SSE', # Seconds since epoch
#'grouping_function': g.null_grouping
# }
}