예제 #1
0
def main():
    # Testing stuff
    g = GDF()
    # g.debug = True
    # pprint(g.storage_config['LS5TM'])
    # pprint(dict(g.storage_config['LS5TM']['dimensions']))
    # pprint(dict(g.storage_config['LS5TM']['measurement_types']))
    # pprint(g.storage_config['LS8OLI'])
    # pprint(dict(g.storage_config['LS8OLI']['dimensions']))
    # pprint(dict(g.storage_config['LS8OLI']['measurement_types']))
    
    data_request_descriptor = {'storage_type': 'LS5TM', 
                               'variables': ('B30',), 
                               'dimensions': {'X': {'range': (147.968, 148.032)}, 
                                              'Y': {'range': (-36.032, -35.968)},
                                              'T': {'range': (1262304000.0, 1325375999.999999)}, # 2010-01-01 00:00:00.0 - 2011-12-31 23:59:59.999999
                                              }
                               }
    t0 = datetime.now()
    print 'Starting 256 x 256 single-band cross-boundary descriptor at ', t0
    d = g.get_descriptor(data_request_descriptor)
    t1 = datetime.now()
    print 'Finishing 256 x 256 cross-boundary descriptor at %s (Elapsed time %s)' % (t1, t1 - t0)
    pprint(d)

    t0 = datetime.now()
    print 'Starting 256 x 256 single-band cross-boundary selection at ', t0
    a = g.get_data(data_request_descriptor)
    t1 = datetime.now()
    print 'Finishing 256 x 256 cross-boundary selection at %s (Elapsed time %s)' % (t1, t1 - t0)
    pprint(a)
예제 #2
0
def main():
    # Testing stuff
    g = GDF(opendap=True)
    # g.debug = True
    # pprint(g.storage_config['LS5TM'])
    # pprint(dict(g.storage_config['LS5TM']['dimensions']))
    # pprint(dict(g.storage_config['LS5TM']['measurement_types']))
    # pprint(g.storage_config['LS8OLI'])
    # pprint(dict(g.storage_config['LS8OLI']['dimensions']))
    # pprint(dict(g.storage_config['LS8OLI']['measurement_types']))

    data_request_descriptor = {
        "storage_type": "LS5TM",
        "variables": ("B30",),
        "dimensions": {
            "X": {"range": (147.968, 148.032)},
            "Y": {"range": (-36.032, -35.968)},
            "T": {"range": (1262304000.0, 1325375999.999999)},  # 2010-01-01 00:00:00.0 - 2011-12-31 23:59:59.999999
        },
    }
    t0 = datetime.now()
    print "Starting 256 x 256 single-band cross-boundary descriptor at ", t0
    d = g.get_descriptor(data_request_descriptor)
    t1 = datetime.now()
    print "Finishing 256 x 256 cross-boundary descriptor at %s (Elapsed time %s)" % (t1, t1 - t0)
    pprint(d)

    t0 = datetime.now()
    print "Starting 256 x 256 single-band cross-boundary selection at ", t0
    a = g.get_data(data_request_descriptor)
    t1 = datetime.now()
    print "Finishing 256 x 256 cross-boundary selection at %s (Elapsed time %s)" % (t1, t1 - t0)
    pprint(a)
예제 #3
0
    def test_GDF_get_descriptor(self):
        "Test GDF get_descriptor function"
        #TODO: Define tests which check DB contents
        test_gdf = GDF()  # Test default configuration

        descriptor = test_gdf.get_descriptor(self.TEST_3D_PARAMETER)
        descriptor = test_gdf.get_descriptor(self.TEST_2D_PARAMETER)
        descriptor = test_gdf.get_descriptor()
예제 #4
0
 def test_GDF_get_descriptor(self):
     "Test GDF get_descriptor function"
     #TODO: Define tests which check DB contents
     test_gdf = GDF() # Test default configuration
     
     descriptor = test_gdf.get_descriptor(self.TEST_3D_PARAMETER)
     descriptor = test_gdf.get_descriptor(self.TEST_2D_PARAMETER)
     descriptor = test_gdf.get_descriptor()
예제 #5
0
    def open_raw_record_file(self):
        self.ws_client.raw_data.clear()
        self.ws_client.raw_data_ticks.clear()
        self.ws_client.raw_data_events.clear()

        self.mkdir_p(os.path.dirname(self.filepath))
        self.target_file = GDF(filename=self.filepath,
                               ch_num=self.ch_num,
                               ch_label=self.ch_label,
                               sps=self.raw_sample_rate)
예제 #6
0
    def test_GDF(self):
        "Test GDF constructor"
        test_gdf = GDF()  # Test default configuration

        assert test_gdf.code_root == os.path.abspath(
            os.path.join(os.path.dirname(os.path.dirname(__file__)),
                         'gdf')), 'Code root is incorrect'
        assert len(test_gdf.config_files
                   ) == 1, 'Default config path list should have one entry'
        assert test_gdf.config_files[0] == os.path.abspath(
            os.path.join(
                test_gdf.code_root,
                GDF.DEFAULT_CONFIG_FILE)), 'Default config path is incorrect'
        assert test_gdf.command_line_params is not None, 'Command line parameter dict not set'
        assert test_gdf.configuration is not None, 'Configurations dict not set'
        assert len(test_gdf.configuration
                   ) > 0, 'Config files must define at least one setup'
        assert len(
            test_gdf.databases) > 0, 'At least one database must be set up'
        assert test_gdf.storage_config is not None, 'storage configuration dict not set'
        assert len(
            test_gdf.storage_config
        ) > 0, 'storage configuration dict must contain at least one storage_type definition'
예제 #7
0
파일: __init__.py 프로젝트: bingo456/gdf
	def __init__(self):
		logger.debug('Initialise Execution Module.')
		self.gdf = GDF()
		self.gdf.debug = False
		self.cache = {}
예제 #8
0
파일: __init__.py 프로젝트: bingo456/gdf
class ExecutionEngine(object):

	SUPPORTED_REDUCTION_OPERATORS = [ 'min', 'max', 'amin', 'amax', 'nanmin', 'nanmax', 'ptp',
									  'median', 'average', 'mean', 'std', 'var', 'nanmean', 'nanstd', 'nanvar',
									  'argmax', 'argmin', 'sum', 'prod', 'all', 'any' 
									]

	def __init__(self):
		logger.debug('Initialise Execution Module.')
		self.gdf = GDF()
		self.gdf.debug = False
		self.cache = {}

	def executePlan(self, plan):

		for task in plan:
			function = task.values()[0]['orig_function']
			print 'function =', function
			if function == 'get_data': # get data
				self.executeGetData(task)
			elif function == 'apply_cloud_mask': # apply cloud mask
				self.executeCloudMask(task)
			elif len([s for s in self.SUPPORTED_REDUCTION_OPERATORS if s in function]) > 0: # reduction operator
				self.executeReduction(task)
			else: # bandmath
				self.executeBandmath(task)

	def executeGetData(self, task):
		
		data_request_param = {}
		data_request_param['dimensions'] = task.values()[0]['array_input'][0].values()[0]['dimensions']
		data_request_param['storage_type'] = task.values()[0]['array_input'][0].values()[0]['storage_type']
		data_request_param['variables'] = ()

		for array in task.values()[0]['array_input']:
			data_request_param['variables'] += (array.values()[0]['variable'],)
		
		data_response = self.gdf.get_data(data_request_param)

		key = task.keys()[0]
		self.cache[key] = {}
		self.cache[key]['array_result'] = copy.deepcopy(data_response['arrays'])
		self.cache[key]['array_indices'] = copy.deepcopy(data_response['indices'])
		self.cache[key]['array_dimensions'] = copy.deepcopy(data_response['dimensions'])
		self.cache[key]['array_output'] = copy.deepcopy(task.values()[0]['array_output'])

		del data_request_param
		del data_response

		return self.cache[key]		

	def executeCloudMask(self, task):

		key = task.keys()[0]
		data_key = task.values()[0]['array_input'][0]
		mask_key = task.values()[0]['array_mask']
		no_data_value = task.values()[0]['array_output']['no_data_value']
		print 'key =', key
		print 'data key =', data_key
		print 'data mask_key =', mask_key
		print 'no_data_value =', no_data_value
		
		array_desc = self.cache[task.values()[0]['array_input'][0]]

		data_array = self.cache[data_key]['array_result'].values()[0]
		mask_array = self.cache[mask_key]['array_result'].values()[0]

		pqa_mask = get_pqa_mask(mask_array)

		masked_array = copy.deepcopy(data_array)
		masked_array[~pqa_mask] = no_data_value
		self.cache[key] = {}
		
		self.cache[key]['array_result'] = {}
		self.cache[key]['array_result'][key] = masked_array
		self.cache[key]['array_indices'] = copy.deepcopy(array_desc['array_indices'])
		self.cache[key]['array_dimensions'] = copy.deepcopy(array_desc['array_dimensions'])
		self.cache[key]['array_output'] = copy.deepcopy(task.values()[0]['array_output'])

	def executeBandmath(self, task):

		key = task.keys()[0]
		
		# TODO: check all input arrays are the same shape and parameters

		arrays = {}
		for task_name in task.values()[0]['array_input']:
			arrays.update(self.cache[task_name]['array_result'])

		arrayResult = {}
		arrayResult['array_result'] = {}
		arrayResult['array_result'][key] = ne.evaluate(task.values()[0]['function'],  arrays)

		no_data_value = task.values()[0]['array_output']['no_data_value']

		array_desc = self.cache[task.values()[0]['array_input'][0]]

		for array in array_desc['array_result'].values():
			arrayResult['array_result'][key][array == no_data_value] = no_data_value

		arrayResult['array_indices'] = copy.deepcopy(array_desc['array_indices'])
		arrayResult['array_dimensions'] = copy.deepcopy(array_desc['array_dimensions'])
		arrayResult['array_output'] = copy.deepcopy(task.values()[0]['array_output'])

		self.cache[key] = arrayResult
		return self.cache[key]

	def executeReduction(self, task):

		function_name = task.values()[0]['orig_function'].replace(")"," ").replace("("," ").split()[0]
		func = getattr(np, function_name)

		key = key = task.keys()[0]
		data_key = task.values()[0]['array_input'][0]
		print 'key =', key
		print 'data key =', data_key

		data = self.cache[data_key]['array_dimensions']

		no_data_value = task.values()[0]['array_output']['no_data_value']

		array_data = self.cache[data_key]['array_result'].values()[0]

		array_desc = self.cache[task.values()[0]['array_input'][0]]
	
		arrayResult = {}
		arrayResult['array_result'] = {}
		arrayResult['array_output'] = copy.deepcopy(task.values()[0]['array_output'])

		if len(task.values()[0]['dimension']) == 1: # 3D -> 2D reduction
			pprint(self.cache[data_key]['array_dimensions'])
			dim = self.cache[data_key]['array_dimensions'].index(task.values()[0]['dimension'][0])

			arrayResult['array_result'][key]=np.apply_along_axis(lambda x: func(x[x!=no_data_value]), dim, array_data)
		
			arrayResult['array_indices'] = copy.deepcopy(array_desc['array_indices'])
			arrayResult['array_dimensions'] = copy.deepcopy(arrayResult['array_output']['dimensions_order'])

			for index in array_desc['array_indices']:
				if index not in arrayResult['array_dimensions'] and index in arrayResult['array_indices']:
					del arrayResult['array_indices'][index]
		elif len(task.values()[0]['dimension']) == 2: # 3D -> 1D reduction
			size = task.values()[0]['array_output']['shape'][0]
			print 'size =', size
			out = np.empty([size])
			dim = self.cache[data_key]['array_dimensions'].index(task.values()[0]['array_output']['dimensions_order'][0])
			print 'dim =', dim
			
			#to fix bug in gdf
			#size = self.cache[data_key]['array_result'].values()[0].shape[dim]
			#print 'size =', size
			#out = np.empty([size])

			for i in range(size):
				if dim == 0:
					out[i] = func(array_data[i,:,:][array_data[i,:,:] != no_data_value])
				elif dim == 1:
					out[i] = func(array_data[:,i,:][array_data[:,i,:] != no_data_value])
				elif dim == 2:
					out[i] = func(array_data[:,:,i][array_data[:,:,i] != no_data_value])

				if np.isnan(out[i]):
					out[i] = no_data_value

			arrayResult['array_result'][key] = out
			arrayResult['array_indices'] = copy.deepcopy(array_desc['array_indices'])
			arrayResult['array_dimensions'] = copy.deepcopy(arrayResult['array_output']['dimensions_order'])
		
			for index in array_desc['array_indices']:
				if index not in arrayResult['array_dimensions'] and index in arrayResult['array_indices']:
					del arrayResult['array_indices'][index]

		self.cache[key] = arrayResult
		return self.cache[key]
예제 #9
0
파일: __init__.py 프로젝트: omad/agdc-v2
 def __init__(self):
     logger.debug("Initialise Execution Module.")
     self.gdf = GDF()
     self.gdf.debug = False
     self.cache = {}
    num_t = data_t.shape[0]
    num_rowcol = math.ceil(math.sqrt(num_t))
    fig = plt.figure()
    fig.clf()
    plot_count = 1
    for i in range(data_t.shape[0]):
       data = data_t[i]
       ax = fig.add_subplot(num_rowcol,num_rowcol,plot_count)
       plt.contourf(lat, lon, data, 64)
       plot_count += 1
    fig.tight_layout()
    plt.show()

# In[4]:

g = GDF()
g.debug = False


# In[5]:

start_date = dt2secs(date(year=2010,month=1,day=1))
end_date = dt2secs(date(year=2010, month=1, day=18))
data_request_descriptors = []
data_request_descriptors.append( {'storage_type': 'LS5TM',
                               'variables': ('B30',),
                               'dimensions': {'X': {'range': (140.0, 141.0)},
                                              'Y': {'range': (-36.0, -35.0)},
                                              'T': {'range': (start_date, end_date),
                                              #      'array_range': (0, 4)
                                                    #'crs': 'SSE', # Seconds since epoch
예제 #11
0
#get_ipython().magic(u'pylab')
# %pylab inline
# import mpld3; mpld3.enable_notebook()

# In[2]:
import calendar
from datetime import datetime, date, timedelta
from gdf import GDF
from pprint import pprint
from gdf import dt2secs, secs2dt
import matplotlib.pyplot as plt

# In[4]:

g = GDF()
g.debug = False


# In[5]:

start_date = dt2secs(date(year=2010, month=1, day=1))
end_date = dt2secs(date(year=2010, month=1, day=18))
data_request_descriptor = {'storage_type': 'LS5TM',
                           'variables': ('B40', 'B30',),
                           'dimensions': {'X': {'range': (147.875, 148.125)},
                                          'Y': {'range': (-37.0 + 0.875, -36.0 + 0.125)},
                                          # 'T': {'range': (start_date, end_date),
                                          #      'array_range': (0, 4)
                                          # 'crs': 'SSE', # Seconds since epoch
                                          # 'grouping_function': g.null_grouping
예제 #12
0
class GDF_FILE(object):
    def __init__(self, filepath, ws_client):
        self.filepath = filepath
        self.ws_client = ws_client
        self.filepath = filepath
        self.ws_client = ws_client
        self.raw_sample_rate = self.ws_client.raw_sample_rate
        self.ch_num = self.ws_client.ch_num
        self.ch_label = self.ws_client.ch_label
        self.raw_cache_size = 1000
        self.record_start_tick = None

        self.open_raw_record_file()

    def mkdir_p(self, path):
        try:
            os.makedirs(path)
        except OSError as exc:  # Python >2.5
            if exc.errno == errno.EEXIST and os.path.isdir(path):
                pass
            else:
                raise

    def open_raw_record_file(self):
        self.ws_client.raw_data.clear()
        self.ws_client.raw_data_ticks.clear()
        self.ws_client.raw_data_events.clear()

        self.mkdir_p(os.path.dirname(self.filepath))
        self.target_file = GDF(filename=self.filepath,
                               ch_num=self.ch_num,
                               ch_label=self.ch_label,
                               sps=self.raw_sample_rate)

    def close_raw_record_file(self):
        while self.ws_client.raw_data:
            self.write_raw_data_to_file()

        self.ws_client.raw_data.clear()
        self.ws_client.raw_data_ticks.clear()
        self.ws_client.raw_data_events.clear()

        self.target_file.close_and_save()
        self.target_file = None

    def write_raw_data_to_file(self):
        if self.target_file is not None:
            if self.record_start_tick is None:
                self.record_start_tick = self.ws_client.raw_data_ticks[0]

            copied_data = self.ws_client.raw_data[:self.raw_cache_size]
            self.ws_client.raw_data = self.ws_client.raw_data[self.
                                                              raw_cache_size:]

            copied_ticks = self.ws_client.raw_data_ticks[:self.raw_cache_size]
            self.ws_client.raw_data_ticks = self.ws_client.raw_data_ticks[
                self.raw_cache_size:]

            copied_events = self.ws_client.raw_data_events[:self.
                                                           raw_cache_size]
            self.ws_client.raw_data_events = self.ws_client.raw_data_events[
                self.raw_cache_size:]

            assert len(copied_data) == len(copied_ticks)
            assert len(copied_data) == len(copied_events)

            self.target_file.writeSamples(copied_data)

            for tick, events in zip(copied_ticks, copied_events):
                for event_id, event_duration in zip(events["event_id"],
                                                    events["event_duration"]):
                    ed = None
                    if event_duration is None:
                        ed = 0
                    else:
                        ed = round(event_duration * self.raw_sample_rate)
                    self.target_file.writeEvent(
                        first_tick=self.record_start_tick,
                        tick=tick,
                        duratoin=ed,
                        event_id=event_id)
예제 #13
0
@author: woo409
"""
# In[2]:
from glue.core import Data, DataCollection
from glue.qt.glue_application import GlueApplication
from datetime import datetime, date, timedelta
from gdf import GDF
from gdf import dt2secs, secs2dt
from analytics import Analytics
from execution_engine import ExecutionEngine

# In[4]:

a = Analytics()
e = ExecutionEngine()
g = GDF()
g.debug = False


# In[5]:

start_date = dt2secs(date(year=2010,month=1,day=1))
end_date = dt2secs(date(year=2010, month=1, day=18))
data_request_descriptor = {'storage_type': 'LS5TM',
                               'variables': ('B40',),
                               'dimensions': {'X': {'range': (149.0699, 149.152)},
                                              'Y': {'range': (-35.3117, -35.2842)},
                                              #'T': {'range': (start_date, end_date),
                                              #      'array_range': (0, 4)
                                                    #'crs': 'SSE', # Seconds since epoch
                                                    #'grouping_function': g.null_grouping
예제 #14
0
파일: __init__.py 프로젝트: bingo456/gdf
class Analytics(object):

	SUPPORTED_OUTPUT_TYPES = ['netcdf-cf', 'geotiff']

	OPERATORS_SENSOR_SPECIFIC_BANDMATH = \
	{
		'ndvi': \
		{
			'sensors': \
			{
				'LS5TM' : { 'input': ['B40', 'B30'], 'function': 'ndvi'},
				'LS7ETM' : { 'input': ['B40', 'B30'], 'function': 'ndvi'},
				'LS8OLI' : { 'input': ['B5', 'B4'], 'function': 'ndvi'},
				'LS8OLITIRS' : { 'input': ['B5', 'B4'], 'function': 'ndvi'},
			},
			'functions': \
			{
				'ndvi' : '((array1 - array2) / (array1 + array2))'
			}
		}
	}

	OPERATORS_REDUCTION = \
	{
		'median': 'median(array1)'
	}

	def __init__(self):
		logger.debug('Initialise Analytics Module.')
		self.gdf = GDF()
		self.gdf.debug = False
		self.plan = []
		self.planDict = {}

	def task(self, name):

		return self.plan[self.planDict[name]]

	def add_to_plan(self, name, task):
		
		self.plan.append( { name : task })
		size = len(self.plan)
		self.planDict[name] = size-1

		return self.plan[size-1]

	def createArray(self, storage_type, variables, dimensions, name):
		query_parameters = {}
		query_parameters['storage_type'] = storage_type
		query_parameters['dimensions'] = dimensions
		query_parameters['variables'] = ()

		for array in variables:
			query_parameters['variables'] += (array,)

		arrayDescriptors = self.gdf.get_descriptor(query_parameters)

		if storage_type not in arrayDescriptors.keys():
			raise AssertionError(storage_type, "not present in descriptor")

		logger.debug('storage_type = %s', storage_type)

		arrayResults = []

		for variable in variables:
			if variable not in arrayDescriptors[storage_type]['variables']:
				raise AssertionError(variable, "not present in", storage_type, "descriptor")

			logger.debug('variable = %s', variable)

			arrayResult = {}
			arrayResult['storage_type'] = storage_type
			arrayResult['variable'] = variable
			arrayResult['dimensions_order'] = arrayDescriptors[storage_type]['dimensions']
			arrayResult['dimensions'] = dimensions
			arrayResult['shape'] = arrayDescriptors[storage_type]['result_shape']
			arrayResult['data_type'] = arrayDescriptors[storage_type]['variables'][variable]['numpy_datatype_name']
			arrayResult['no_data_value'] = arrayDescriptors[storage_type]['variables'][variable]['nodata_value']

			arrayResults.append({variable : arrayResult})

		task = {}
		task['array_input'] = arrayResults
		task['array_output'] = copy.deepcopy(arrayResult)
		task['array_output']['variable'] = 'data'
		task['function'] = 'get_data'
		task['orig_function'] = 'get_data'

		return self.add_to_plan(name, task)

	def applyCloudMask(self, arrays, mask, name):
		size = len(arrays)
		if size == 0:
			raise AssertionError("Input array is empty")

		task = {}
		task['array_input'] = []
		for array in arrays.keys():
			task['array_input'].append(array)

		task['array_mask'] = mask.keys()[0]
		task['orig_function'] = 'apply_cloud_mask'
		task['function'] = 'apply_cloud_mask'
		task['array_output'] = copy.deepcopy(arrays.values()[0]['array_output'])
		task['array_output']['variable'] = name

		return self.add_to_plan(name, task)

	# generic version
	def applyBandMath(self, arrays, function, name):
		# Arrays is a list
		# output Array is same shape as input array
		size = len(arrays)
		if size == 0:
			raise AssertionError("Input array is empty")

		variables = []
		if size == 1: # 1 input
			if arrays.values()[0]['function'] == 'get_data':

				# check shape is same for all input arrays
				shape = arrays.values()[0]['array_input'][0].values()[0]['shape']
				for variable in arrays.values()[0]['array_input']:
					if shape != variable.values()[0]['shape']:
						raise AssertionError("Shape is different")
					variables.append(variable.keys()[0])
			else:
				variables.append(arrays.keys()[0])

			orig_function = function
			logger.debug('function before = %s', function)
			count = 1
			for variable in variables:
				function = function.replace('array'+str(count), variable)
				count += 1
			logger.debug('function after = %s', function)

			task = {}

			task['array_input'] = []
			task['array_input'].append(arrays.keys()[0])
			task['orig_function'] = orig_function
			task['function'] = function
			task['array_output'] = copy.deepcopy(arrays.values()[0]['array_output'])
			task['array_output']['variable'] = name

			return self.add_to_plan(name, task)

		else: # multi-dependencies
			pprint(arrays)
			for array in arrays:
				variables.append(array.keys()[0])

			orig_function = function
			logger.debug('function before = %s', function)
			count = 1
			for variable in variables:
				function = function.replace('array'+str(count), variable)
				count += 1
			logger.debug('function after = %s', function)

			task = {}

			task['array_input'] = []

			for array in arrays:
				task['array_input'].append(array.keys()[0])

			task['orig_function'] = orig_function
			task['function'] = function
			task['array_output'] = copy.deepcopy(arrays[0].values()[0]['array_output'])
			task['array_output']['variable'] = name

			return self.add_to_plan(name, task)

	# sensor specific version
	def applySensorSpecificBandMath(self, storage_types, function, dimensions, name_data, name_result):
		variables = self.getPredefinedInputs(storage_types, function)
		arrays = self.createArray(storage_types, variables, dimensions, name_data)

		function_text = self.getPredefinedFunction(storage_types, function)
		return self.applyBandMath(arrays, function_text, name_result)

	def applyReduction(self, array1, dimensions, function, name):

		function = self.OPERATORS_REDUCTION.get(function)
		return self.applyGenericReductionFunction(array1, dimensions, function, name)

	def applyGenericReduction(self, arrays, dimensions, function, name):
		
		size = len(arrays)
		if size != 1:
			raise AssertionError("Input array should be 1")

		if arrays.values()[0]['function'] == 'get_data':

			variable = arrays.values()[0]['array_input'][0].values()[0]['variable']
			orig_function = function
			logger.debug('function before = %s', function)
			function = function.replace('array1', variable)
			logger.debug('function after = %s', function)

			task = {}

			task['array_input'] = []
			task['array_input'].append(arrays.keys()[0])
			task['orig_function'] = orig_function
			task['function'] = function
			task['dimension'] = copy.deepcopy(dimensions)

			task['array_output'] = copy.deepcopy(arrays.values()[0]['array_output'])
			task['array_output']['variable'] = name
			task['array_output']['dimensions_order'] = self.diffList(arrays.values()[0]['array_input'][0].values()[0]['dimensions_order'], dimensions)

			result = ()
			for value in task['array_output']['dimensions_order']:
				input_task = self.task(task['array_input'][0])
				index = input_task.values()[0]['array_output']['dimensions_order'].index(value)
				value = input_task.values()[0]['array_output']['shape'][index]
				result += (value,)
			task['array_output']['shape'] = result

			return self.add_to_plan(name, task)
		else:
			variable = arrays.keys()[0]
			orig_function = function
			logger.debug('function before = %s', function)
			function = function.replace('array1', variable)
			logger.debug('function after = %s', function)

			task = {}
			
			task['array_input'] = []
			task['array_input'].append(variable)
			task['orig_function'] = orig_function
			task['function'] = function
			task['dimension'] = copy.deepcopy(dimensions)

			task['array_output'] = copy.deepcopy(arrays.values()[0]['array_output'])
			task['array_output']['variable'] = name
			task['array_output']['dimensions_order'] = self.diffList(arrays.values()[0]['array_output']['dimensions_order'], dimensions)

			result = ()
			for value in task['array_output']['dimensions_order']:
				input_task = self.task(task['array_input'][0])
				pprint(input_task)
				index = input_task.values()[0]['array_output']['dimensions_order'].index(value)
				value = input_task.values()[0]['array_output']['shape'][index]
				result += (value,)
			task['array_output']['shape'] = result

			return self.add_to_plan(name, task)

	def diffList(self, list1, list2):
		list2 = set(list2)
		return [result for result in list1 if result not in list2]

	def getPredefinedInputs(self, storage_type, function):
		return self.OPERATORS_SENSOR_SPECIFIC_BANDMATH.get(function).get('sensors').get(storage_type).get('input')

	def getPredefinedFunction(self, storage_type, function):
		function_id = self.OPERATORS_SENSOR_SPECIFIC_BANDMATH.get(function).get('sensors').get(storage_type).get('function')
		return self.OPERATORS_SENSOR_SPECIFIC_BANDMATH.get(function).get('functions').get(function_id)
예제 #15
0
파일: __init__.py 프로젝트: omad/agdc-v2
class ExecutionEngine(object):

    SUPPORTED_REDUCTION_OPERATORS = [
        "min",
        "max",
        "amin",
        "amax",
        "nanmin",
        "nanmax",
        "ptp",
        "median",
        "average",
        "mean",
        "std",
        "var",
        "nanmean",
        "nanstd",
        "nanvar",
        "argmax",
        "argmin",
        "sum",
        "prod",
        "all",
        "any",
    ]

    def __init__(self):
        logger.debug("Initialise Execution Module.")
        self.gdf = GDF()
        self.gdf.debug = False
        self.cache = {}

    def executePlan(self, plan):

        for task in plan:
            function = task.values()[0]["orig_function"]
            print "function =", function
            if function == "get_data":  # get data
                self.executeGetData(task)
            elif function == "apply_cloud_mask":  # apply cloud mask
                self.executeCloudMask(task)
            elif len([s for s in self.SUPPORTED_REDUCTION_OPERATORS if s in function]) > 0:  # reduction operator
                self.executeReduction(task)
            else:  # bandmath
                self.executeBandmath(task)

    def executeGetData(self, task):

        data_request_param = {}
        data_request_param["dimensions"] = task.values()[0]["array_input"][0].values()[0]["dimensions"]
        data_request_param["storage_type"] = task.values()[0]["array_input"][0].values()[0]["storage_type"]
        data_request_param["variables"] = ()

        for array in task.values()[0]["array_input"]:
            data_request_param["variables"] += (array.values()[0]["variable"],)

        data_response = self.gdf.get_data(data_request_param)

        key = task.keys()[0]
        self.cache[key] = {}
        self.cache[key]["array_result"] = copy.deepcopy(data_response["arrays"])
        self.cache[key]["array_indices"] = copy.deepcopy(data_response["indices"])
        self.cache[key]["array_dimensions"] = copy.deepcopy(data_response["dimensions"])
        self.cache[key]["array_output"] = copy.deepcopy(task.values()[0]["array_output"])

        del data_request_param
        del data_response

        return self.cache[key]

    def executeCloudMask(self, task):

        key = task.keys()[0]
        data_key = task.values()[0]["array_input"][0]
        mask_key = task.values()[0]["array_mask"]
        no_data_value = task.values()[0]["array_output"]["no_data_value"]
        print "key =", key
        print "data key =", data_key
        print "data mask_key =", mask_key
        print "no_data_value =", no_data_value

        array_desc = self.cache[task.values()[0]["array_input"][0]]

        data_array = self.cache[data_key]["array_result"].values()[0]
        mask_array = self.cache[mask_key]["array_result"].values()[0]

        pqa_mask = get_pqa_mask(mask_array)

        masked_array = copy.deepcopy(data_array)
        masked_array[~pqa_mask] = no_data_value
        self.cache[key] = {}

        self.cache[key]["array_result"] = {}
        self.cache[key]["array_result"][key] = masked_array
        self.cache[key]["array_indices"] = copy.deepcopy(array_desc["array_indices"])
        self.cache[key]["array_dimensions"] = copy.deepcopy(array_desc["array_dimensions"])
        self.cache[key]["array_output"] = copy.deepcopy(task.values()[0]["array_output"])

    def executeBandmath(self, task):

        key = task.keys()[0]

        # TODO: check all input arrays are the same shape and parameters

        arrays = {}
        for task_name in task.values()[0]["array_input"]:
            arrays.update(self.cache[task_name]["array_result"])

        arrayResult = {}
        arrayResult["array_result"] = {}
        arrayResult["array_result"][key] = ne.evaluate(task.values()[0]["function"], arrays)

        no_data_value = task.values()[0]["array_output"]["no_data_value"]

        array_desc = self.cache[task.values()[0]["array_input"][0]]

        for array in array_desc["array_result"].values():
            arrayResult["array_result"][key][array == no_data_value] = no_data_value

        arrayResult["array_indices"] = copy.deepcopy(array_desc["array_indices"])
        arrayResult["array_dimensions"] = copy.deepcopy(array_desc["array_dimensions"])
        arrayResult["array_output"] = copy.deepcopy(task.values()[0]["array_output"])

        self.cache[key] = arrayResult
        return self.cache[key]

    def executeReduction(self, task):

        function_name = task.values()[0]["orig_function"].replace(")", " ").replace("(", " ").split()[0]
        func = getattr(np, function_name)

        key = key = task.keys()[0]
        data_key = task.values()[0]["array_input"][0]
        print "key =", key
        print "data key =", data_key

        data = self.cache[data_key]["array_dimensions"]

        no_data_value = task.values()[0]["array_output"]["no_data_value"]

        array_data = self.cache[data_key]["array_result"].values()[0]

        array_desc = self.cache[task.values()[0]["array_input"][0]]

        arrayResult = {}
        arrayResult["array_result"] = {}
        arrayResult["array_output"] = copy.deepcopy(task.values()[0]["array_output"])

        if len(task.values()[0]["dimension"]) == 1:  # 3D -> 2D reduction
            pprint(self.cache[data_key]["array_dimensions"])
            dim = self.cache[data_key]["array_dimensions"].index(task.values()[0]["dimension"][0])

            arrayResult["array_result"][key] = np.apply_along_axis(
                lambda x: func(x[x != no_data_value]), dim, array_data
            )

            arrayResult["array_indices"] = copy.deepcopy(array_desc["array_indices"])
            arrayResult["array_dimensions"] = copy.deepcopy(arrayResult["array_output"]["dimensions_order"])

            for index in array_desc["array_indices"]:
                if index not in arrayResult["array_dimensions"] and index in arrayResult["array_indices"]:
                    del arrayResult["array_indices"][index]
        elif len(task.values()[0]["dimension"]) == 2:  # 3D -> 1D reduction
            size = task.values()[0]["array_output"]["shape"][0]
            print "size =", size
            out = np.empty([size])
            dim = self.cache[data_key]["array_dimensions"].index(
                task.values()[0]["array_output"]["dimensions_order"][0]
            )
            print "dim =", dim

            # to fix bug in gdf
            # size = self.cache[data_key]['array_result'].values()[0].shape[dim]
            # print 'size =', size
            # out = np.empty([size])

            for i in range(size):
                if dim == 0:
                    out[i] = func(array_data[i, :, :][array_data[i, :, :] != no_data_value])
                elif dim == 1:
                    out[i] = func(array_data[:, i, :][array_data[:, i, :] != no_data_value])
                elif dim == 2:
                    out[i] = func(array_data[:, :, i][array_data[:, :, i] != no_data_value])

                if np.isnan(out[i]):
                    out[i] = no_data_value

            arrayResult["array_result"][key] = out
            arrayResult["array_indices"] = copy.deepcopy(array_desc["array_indices"])
            arrayResult["array_dimensions"] = copy.deepcopy(arrayResult["array_output"]["dimensions_order"])

            for index in array_desc["array_indices"]:
                if index not in arrayResult["array_dimensions"] and index in arrayResult["array_indices"]:
                    del arrayResult["array_indices"][index]

        self.cache[key] = arrayResult
        return self.cache[key]
예제 #16
0
파일: __init__.py 프로젝트: bingo456/gdf
	def __init__(self):
		logger.debug('Initialise Analytics Module.')
		self.gdf = GDF()
		self.gdf.debug = False
		self.plan = []
		self.planDict = {}
예제 #17
0
    plot_count = 1
    for i in range(img.shape[0]):
       data = img[i]
       ax = fig.add_subplot(num_rowcol,num_rowcol,plot_count)
       plt.setp(ax, xticks=[], yticks=[])
       #cax = ax.imshow(data, interpolation='nearest', aspect = 'equal')
       cax = ax.pcolormesh(data) #, interpolation='nearest', aspect = 'equal')
       #fig.colorbar(cax)
       plot_count += 1
    fig.tight_layout()
    plt.show()


# In[4]:

g = GDF()
#g.debug = True


# In[5]:

data_request_descriptor = {'storage_type': 'LS7ETM',
                               'variables': ('B30', 'B40'),
                               'dimensions': {'X': {'range': (140.0, 140.125)},
                                              'Y': {'range': (-35.0, -35.0+0.125)},
                                              #'T': {'range': (0, 6325376000),
                                              #      'array_range': (0, 4)
                                                    #'crs': 'SSE', # Seconds since epoch
                                                    #'grouping_function': g.null_grouping
                                              #      }
                                            }