def get_time_idx(cls, coverage, timeval): temporal_variable = coverage.temporal_parameter_name uom = coverage.get_parameter_context(temporal_variable).uom units = TimeUtils.ts_to_units(uom, timeval) idx = TimeUtils.get_relative_time(coverage, units) return idx
def dataset_temporal_bounds(self, dataset_id): coverage = self._get_coverage(dataset_id) temporal_param = coverage.temporal_parameter_name try: bounds = coverage.get_data_bounds(temporal_param) except ValueError: return (coverage.get_parameter_context(temporal_param).fill_value,) * 2 uom = coverage.get_parameter_context(temporal_param).uom new_bounds = (TimeUtils.units_to_ts(uom,bounds[0]), TimeUtils.units_to_ts(uom,bounds[1])) return new_bounds
def sync_rdt_with_coverage(self, coverage=None, tdoa=None, start_time=None, end_time=None, stride_time=None, parameters=None): ''' Builds a granule based on the coverage ''' if coverage is None: coverage = self.coverage slice_ = slice(None) # Defaults to all values if tdoa is not None and isinstance(tdoa,slice): slice_ = tdoa elif stride_time is not None: validate_is_instance(start_time, Number, 'start_time must be a number for striding.') validate_is_instance(end_time, Number, 'end_time must be a number for striding.') validate_is_instance(stride_time, Number, 'stride_time must be a number for striding.') ugly_range = np.arange(start_time, end_time, stride_time) idx_values = [TimeUtils.get_relative_time(coverage,i) for i in ugly_range] slice_ = [idx_values] elif not (start_time is None and end_time is None): time_var = coverage._temporal_param_name uom = coverage.get_parameter_context(time_var).uom if start_time is not None: start_units = TimeUtils.ts_to_units(uom,start_time) log.info('Units: %s', start_units) start_idx = TimeUtils.get_relative_time(coverage,start_units) log.info('Start Index: %s', start_idx) start_time = start_idx if end_time is not None: end_units = TimeUtils.ts_to_units(uom,end_time) log.info('End units: %s', end_units) end_idx = TimeUtils.get_relative_time(coverage,end_units) log.info('End index: %s', end_idx) end_time = end_idx slice_ = slice(start_time,end_time,stride_time) log.info('Slice: %s', slice_) if parameters is not None: pdict = ParameterDictionary() params = set(coverage.list_parameters()).intersection(parameters) for param in params: pdict.add_context(coverage.get_parameter_context(param)) rdt = RecordDictionaryTool(param_dictionary=pdict) self.pdict = pdict else: rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary) fields = coverage.list_parameters() if parameters is not None: fields = set(fields).intersection(parameters) for d in fields: rdt[d] = coverage.get_parameter_values(d,tdoa=slice_) self.rdt = rdt # Sync
def dataset_temporal_bounds(self, dataset_id): coverage = self._get_coverage(dataset_id) temporal_param = coverage.temporal_parameter_name try: bounds = coverage.get_data_bounds(temporal_param) except ValueError: return (coverage.get_parameter_context(temporal_param).fill_value, ) * 2 uom = coverage.get_parameter_context(temporal_param).uom new_bounds = (TimeUtils.units_to_ts(uom, bounds[0]), TimeUtils.units_to_ts(uom, bounds[1])) return new_bounds
def insert_values(self, coverage, rdt, stream_id): elements = len(rdt) start_index = coverage.num_timesteps - elements for k, v in rdt.iteritems(): if isinstance(v, SparseConstantValue): continue slice_ = slice(start_index, None) try: coverage.set_parameter_values(param_name=k, tdoa=slice_, value=v) except IOError as e: log.error("Couldn't insert values for coverage: %s", coverage.persistence_dir, exc_info=True) try: coverage.close() finally: self._bad_coverages[stream_id] = 1 raise CorruptionError(e.message) if 'ingestion_timestamp' in coverage.list_parameters(): t_now = time.time() ntp_time = TimeUtils.ts_to_units( coverage.get_parameter_context('ingestion_timestamp').uom, t_now) coverage.set_parameter_values(param_name='ingestion_timestamp', tdoa=slice_, value=ntp_time)
def insert_values(self, coverage, rdt, stream_id): elements = len(rdt) start_index = coverage.num_timesteps - elements for k,v in rdt.iteritems(): if isinstance(v, SparseConstantValue): continue slice_ = slice(start_index, None) try: coverage.set_parameter_values(param_name=k, tdoa=slice_, value=v) except IOError as e: log.error("Couldn't insert values for coverage: %s", coverage.persistence_dir, exc_info=True) try: coverage.close() finally: self._bad_coverages[stream_id] = 1 raise CorruptionError(e.message) except IndexError as e: log.error("Value set: %s", v[:]) data_products, _ = self.container.resource_registry.find_subjects(object=stream_id, predicate=PRED.hasStream, subject_type=RT.DataProduct) for data_product in data_products: log.exception("Index exception with %s, trying to insert %s into coverage with shape %s", data_product.name, k, v.shape) if 'ingestion_timestamp' in coverage.list_parameters(): t_now = time.time() ntp_time = TimeUtils.ts_to_units(coverage.get_parameter_context('ingestion_timestamp').uom, t_now) coverage.set_parameter_values(param_name='ingestion_timestamp', tdoa=slice_, value=ntp_time)
def dataset_temporal_bounds(self, dataset_id): dataset = self.read_dataset(dataset_id) if not dataset: return {} pdict = ParameterDictionary.load(dataset.parameter_dictionary) temporal_parameter = pdict.temporal_parameter_name units = pdict.get_temporal_context().uom bounds = self.dataset_bounds(dataset_id) if not bounds: return {} bounds = bounds[temporal_parameter or 'time'] bounds = [TimeUtils.units_to_ts(units, i) for i in bounds] return bounds
def dataset_temporal_bounds(self, dataset_id): dataset = self.read_dataset(dataset_id) parameter_dictionary_id = self.clients.resource_registry.find_objects(dataset_id, PRED.hasParameterDictionary, id_only=True)[0][0] pdict = self._coverage_parameter_dictionary(parameter_dictionary_id) if not dataset: return {} temporal_parameter = pdict.temporal_parameter_name units = pdict.get_temporal_context().uom bounds = self.dataset_bounds(dataset_id) if not bounds: return {} bounds = bounds[temporal_parameter or 'time'] bounds = [TimeUtils.units_to_ts(units, i) for i in bounds] return bounds
def recv_packet(self, msg, stream_route, stream_id): validate_is_instance(msg, Granule, "Incoming packet must be of type granule") cov = self.get_coverage(stream_id) if cov: cov.insert_timesteps(1) if "raw" in cov.list_parameters(): gran = IonObjectSerializer().serialize(msg) cov.set_parameter_values(param_name="raw", value=[gran]) if "ingestion_timestamp" in cov.list_parameters(): t_now = time.time() ntp_time = TimeUtils.ts_to_units(cov.get_parameter_context("ingestion_timestamp").uom, t_now) cov.set_parameter_values(param_name="ingestion_timestamp", value=ntp_time) self.dataset_changed(self.get_dataset(stream_id), cov.num_timesteps)
def dataset_temporal_bounds(self, dataset_id): dataset = self.read_dataset(dataset_id) parameter_dictionary_id = self.clients.resource_registry.find_objects( dataset_id, PRED.hasParameterDictionary, id_only=True)[0][0] pdict = self._coverage_parameter_dictionary(parameter_dictionary_id) if not dataset: return {} temporal_parameter = pdict.temporal_parameter_name units = pdict.get_temporal_context().uom bounds = self.dataset_bounds(dataset_id) if not bounds: return {} bounds = bounds[temporal_parameter or 'time'] bounds = [TimeUtils.units_to_ts(units, i) for i in bounds] return bounds
def recv_packet(self, msg, stream_route, stream_id): validate_is_instance(msg, Granule, 'Incoming packet must be of type granule') cov = self.get_coverage(stream_id) if cov: cov.insert_timesteps(1) if 'raw' in cov.list_parameters(): gran = IonObjectSerializer().serialize(msg) cov.set_parameter_values(param_name='raw', value=[gran]) if 'ingestion_timestamp' in cov.list_parameters(): t_now = time.time() ntp_time = TimeUtils.ts_to_units( cov.get_parameter_context('ingestion_timestamp').uom, t_now) cov.set_parameter_values(param_name='ingestion_timestamp', value=ntp_time) self.dataset_changed(self.get_dataset(stream_id), cov.num_timesteps)
def insert_values(self, coverage, rdt, stream_id): elements = len(rdt) start_index = coverage.num_timesteps - elements for k,v in rdt.iteritems(): if isinstance(v, SparseConstantValue): continue slice_ = slice(start_index, None) try: coverage.set_parameter_values(param_name=k, tdoa=slice_, value=v) except IOError as e: log.error("Couldn't insert values for coverage: %s", coverage.persistence_dir, exc_info=True) try: coverage.close() finally: self._bad_coverages[stream_id] = 1 raise CorruptionError(e.message) if 'ingestion_timestamp' in coverage.list_parameters(): t_now = time.time() ntp_time = TimeUtils.ts_to_units(coverage.get_parameter_context('ingestion_timestamp').uom, t_now) coverage.set_parameter_values(param_name='ingestion_timestamp', tdoa=slice_, value=ntp_time)
def execute(input=None, context=None, config=None, params=None, state=None): stream_definition_id = params #init stuff rdt_for_nones = {} data_table_content = [] gdt_allowed_numerical_types = [ 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64', 'str' ] # TODO : Move this in to container parameter default_precision = 5 rdt = RecordDictionaryTool.load_from_granule(input) data_description = [] # Buid a local precisions and fill value dictionary to use for parsing data correctly precisions = {} fill_values = {} for field in rdt.fields: precision_str = rdt.context(field).precision if not precision_str: precisions[field] = default_precision else: try: precisions[field] = int(precision_str) except ValueError: precisions[field] = default_precision fill_values[field] = rdt.fill_value(field) if stream_definition_id == None: log.error( "GoogleDT transform: Need a output stream definition to process graphs" ) return None fields = [] fields = rdt.fields # Ascertain temporal field. Use 'time' as backup time_field = rdt.temporal_parameter or 'time' #if 'time' not in rdt: return None if rdt[time_field] is None: return None #time_fill_value = 0.0 # should be derived from the granule's param dict. time_fill_value = fill_values[ time_field] # should be derived from the granule's param dict. total_num_of_records = len(rdt[time_field]) data_description.append(('time', 'number', 'time')) ###### DEBUG ########## #for field in fields: # if hasattr(rdt.context(field),'visible'): # print " >>>>>>>> '", field, "' [visible = ", rdt.context(field).visible,"] : ", rdt[field] # else: # print " >>>>>>>> '", field, "' [visible = NOT SPECIFIED] : ", rdt[field] import re for field in fields: if field == time_field: continue # If a config block was passed, consider only the params listed in it if config and 'parameters' in config and len( config['parameters']) > 0: if not field in config['parameters']: log.info( "Skipping ", field, " since it was not present in the list of allowed parameters" ) continue # If the value is none, assign it a small one fill_value array for now to generate description, # Actual array of fill_values will be assigned later rdt_field = rdt[field] if rdt_field == None: rdt_for_nones[field] = np.array([fill_values[field]] * total_num_of_records) rdt_field = rdt_for_nones[field] # Check if visibility is false (system generated params) or not specified explicitly if hasattr(rdt.context(field), 'visible') and not rdt.context(field).visible: continue # If it's a QC parameter ignore it if field.endswith('_qc'): continue # Handle string type or if its an unknown type, convert to string context = rdt.context(field) if (rdt_field.dtype == 'string' or rdt_field.dtype not in gdt_allowed_numerical_types): data_description.append((field, 'string', field)) elif (isinstance(context.param_type, ArrayType) or isinstance( context.param_type, ParameterFunctionType)) and len(rdt_field.shape) > 1: for i in xrange(rdt_field.shape[1]): data_description.append(('%s[%s]' % (field, i), 'number', '%s[%s]' % (field, i), { 'precision': str(precisions[field]) })) else: data_description.append((field, 'number', field, { 'precision': str(precisions[field]) })) for i in xrange(len(rdt)): varTuple = [] # Put time first if its not zero. Retrieval returns 0 secs for malformed entries if rdt[time_field][i] == time_fill_value: continue # convert timestamp from instrument to UNIX time stamp since thats what Google DT expects varTuple.append(ntplib.ntp_to_system_time(rdt[time_field][i])) for dd in data_description: field = dd[0] field_type = dd[1] # ignore time since its been already added if field == None or field == time_field: continue """ rdt_field = rdt[field] if rdt_field == None: rdt_field = np.array([fill_values[field]] * total_num_of_records) """ if re.match(r'.*\[\d+\]', field): field, j = re.match(r'(.*)\[(\d+)\]', field).groups() j = int(j) varTuple.append(float(rdt[field][i][j])) else: if (field_type == 'number'): if rdt[field] == None or rdt[field][i] == fill_values[ field]: varTuple.append(None) else: # Adjust float for precision if (precisions[field] == None): varTuple.append(float(rdt[field][i])) else: varTuple.append( round(float(rdt[field][i]), precisions[field])) # if field type is string, there are two possibilities. Either it really is a string or # its an object that needs to be converted to string. if (field_type == 'string'): if rdt[field] == None or rdt[field][i] == fill_values[ field]: varTuple.append(None) else: varTuple.append(str(rdt[field][i])) # Append the tuples to the data table if len(varTuple) > 0: data_table_content.append(varTuple) out_rdt = RecordDictionaryTool( stream_definition_id=stream_definition_id) # Prepare granule content out_dict = { "viz_product_type": "google_dt", "data_description": data_description, "data_content": data_table_content } out_rdt["google_dt_components"] = np.array([out_dict]) out_rdt["viz_timestamp"] = TimeUtils.ts_to_units( rdt.context(time_field).uom, time.time()) log.debug('Google DT transform: Sending a granule') out_granule = out_rdt.to_granule() return out_granule
def ntp_from_iso(self, iso): return TimeUtils.ntp_from_iso(iso)
def execute(input=None, context=None, config=None, params=None, state=None): stream_definition_id = params #init stuff data_description = [] data_table_content = [] gdt_allowed_numerical_types = ['int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64','str'] # TODO : Move this in to container parameter default_precision = 5 rdt = RecordDictionaryTool.load_from_granule(input) data_description = [] # Buid a local precisions and fill value dictionary to use for parsing data correctly precisions = {} fill_values = {} for field in rdt.fields: precision_str = rdt.context(field).precision if not precision_str: precisions[field] = default_precision else: try: precisions[field] = int(precision_str) except ValueError: precisions[field] = default_precision fill_values[field] = rdt.fill_value(field) if stream_definition_id == None: log.error("GoogleDT transform: Need a output stream definition to process graphs") return None fields = [] fields = rdt.fields # if time was null or missing, do not process if 'time' not in rdt: return None if rdt['time'] is None: return None time_fill_value = 0.0 # should be derived from the granule's param dict. data_description.append(('time','number','time')) import re for field in fields: if field == rdt.temporal_parameter: continue # If a config block was passed, consider only the params listed in it if config and 'parameters' in config and len(config['parameters']) > 0: if not field in config['parameters']: log.info("Skipping ", field, " since it was not present in the list of allowed parameters") continue # only consider fields which are allowed. if rdt[field] == None: continue # Check if visibility is false (system generated params) if hasattr(rdt.context(field),'visible') and not rdt.context(field).visible: continue # If it's a QC parameter ignore it if field.endswith('_qc'): continue # Handle string type or if its an unknown type, convert to string context = rdt.context(field) if (rdt[field].dtype == 'string' or rdt[field].dtype not in gdt_allowed_numerical_types): data_description.append((field, 'string', field )) elif (isinstance(context.param_type, ArrayType) or isinstance(context.param_type,ParameterFunctionType)) and len(rdt[field].shape)>1: for i in xrange(rdt[field].shape[1]): data_description.append(('%s[%s]' % (field,i), 'number', '%s[%s]' % (field,i), {'precision':str(precisions[field])})) else: data_description.append((field, 'number', field, {'precision':str(precisions[field])} )) for i in xrange(len(rdt)): varTuple = [] # Put time first if its not zero. Retrieval returns 0 secs for malformed entries if rdt['time'][i] == time_fill_value: continue # convert timestamp from instrument to UNIX time stamp since thats what Google DT expects varTuple.append(ntplib.ntp_to_system_time(rdt['time'][i])) for dd in data_description: field = dd[0] field_type = dd[1] # ignore time since its been already added if field == None or field == 'time': continue if re.match(r'.*\[\d+\]', field): field, j = re.match(r'(.*)\[(\d+)\]', field).groups() j = int(j) varTuple.append(float(rdt[field][i][j])) elif rdt[field] == None or rdt[field][i] == None: varTuple.append(None) else: if(field_type == 'number'): if rdt[field][i] == None or rdt[field][i] == fill_values[field]: varTuple.append(None) else: # Adjust float for precision if (precisions[field] == None): varTuple.append(float(rdt[field][i])) else: varTuple.append(round(float(rdt[field][i]), precisions[field])) # if field type is string, there are two possibilities. Either it really is a string or # its an object that needs to be converted to string. if(field_type == 'string'): varTuple.append(str(rdt[field][i])) # Append the tuples to the data table if len(varTuple) > 0: data_table_content.append(varTuple) out_rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id) # Prepare granule content out_dict = {"viz_product_type" : "google_dt", "data_description" : data_description, "data_content" : data_table_content} out_rdt["google_dt_components"] = np.array([out_dict]) out_rdt["viz_timestamp"] = TimeUtils.ts_to_units(rdt.context(rdt.temporal_parameter).uom, time.time()) log.debug('Google DT transform: Sending a granule') out_granule = out_rdt.to_granule() return out_granule
def render_graphs(cls, rdt, graph_data, stream_definition_id, fileName=None, resolution=None, time_field='time'): # init Matplotlib with passsed parameters x_res = y_res = 100 # some default in case nothing is provided if resolution: x_res, y_res = resolution.split('x') x_res = int(x_res) y_res = int(y_res) fig = Figure(figsize=(x_res / 100, y_res / 100), dpi=100, frameon=True) ax = fig.add_subplot(111) canvas = FigureCanvas(fig) imgInMem = StringIO.StringIO() # Guess xticks for number of labels along x-axis. Place a label every 100 pixels ax.locator_params(x_res / 100) # If there's no data, wait # For the simple case of testing, lets plot all time variant variables one at a time xAxisVar = time_field # Prepare the set of y axis variables that will be plotted. This needs to be smarter and passed as # config variable to the transform yAxisVars = [] for varName, varData in graph_data.iteritems(): if varName == time_field or varName == 'height' or varName == 'longitude' or varName == 'latitude': continue yAxisVars.append(varName) # Do a error check for incorrect time values. Ignore all time == fill_values time_fill_value = 0.0 clean_data_flag = False while not clean_data_flag: clean_data_flag = True # Go through the data and see if we can find empty time entries. Delete it along # with all corresponding variables for idx in xrange(len(graph_data[xAxisVar])): if graph_data[xAxisVar][idx] == time_fill_value: graph_data[xAxisVar].pop(idx) for varName in yAxisVars: try: graph_data[varName].pop(idx) except IndexError: # Do nothing really. Must be a malformed granule pass clean_data_flag = False break #xAxisFloatData = graph_data[xAxisVar] xAxisFloatData = [ datetime.fromtimestamp(ntplib.ntp_to_system_time(t)) for t in graph_data[xAxisVar] ] #print " >>>>>>>>>>>>>>>> xAxisFloatData : ", xAxisFloatData idx = 0 for varName in yAxisVars: yAxisFloatData = graph_data[varName] # Generate the plot ax.plot(xAxisFloatData, yAxisFloatData, cls.line_style(idx), label=varName) idx += 1 yAxisLabel = "" # generate a filename for the output image for varName in yAxisVars: if yAxisLabel: yAxisLabel = yAxisLabel + "-" + varName else: yAxisLabel = varName if not fileName: fileName = yAxisLabel + '_vs_' + xAxisVar fileName = fileName + "." + mpl_output_image_format # Choose a small font for the legend legend_font_prop = FontProperties() legend_font_prop.set_size('small') ax.set_xlabel(xAxisVar) ax.set_ylabel(yAxisLabel) ax.set_title(yAxisLabel + ' vs ' + xAxisVar) ax.set_autoscale_on(False) ax.legend(loc='upper left', ncol=3, fancybox=True, shadow=True, prop=legend_font_prop) # Date formatting # matplotlib date format object #hfmt = dates.DateFormatter('%m/%d %H:%M') #ax.xaxis.set_major_locator(dates.MinuteLocator()) #ax.xaxis.set_major_formatter(hfmt) # Save the figure to the in memory file canvas.print_figure(imgInMem, format=mpl_output_image_format) imgInMem.seek(0) # Create output dictionary from the param dict out_rdt = RecordDictionaryTool( stream_definition_id=stream_definition_id) # Prepare granule content #out_dict = {} out_rdt["viz_product_type"] = ["matplotlib_graphs"] out_rdt["image_obj"] = [imgInMem.getvalue()] out_rdt["image_name"] = [fileName] out_rdt["content_type"] = ["image/png"] #print " >>>>>>>>>> OUT_IMAGE_NAME : ", out_rdt['image_name'] #out_rdt["graph_image_param_dict"] = np.array([out_dict]) out_rdt["viz_timestamp"] = TimeUtils.ts_to_units( rdt.context(rdt.temporal_parameter).uom, time.time()) return out_rdt.to_granule()
def render_graphs(cls, rdt, graph_data, stream_definition_id, fileName = None, resolution = None, time_field = 'time'): # init Matplotlib with passsed parameters x_res = y_res = 100 # some default in case nothing is provided if resolution: x_res, y_res = resolution.split('x') x_res = int(x_res) y_res = int(y_res) fig = Figure(figsize=(x_res / 100, y_res / 100), dpi=100, frameon=True) ax = fig.add_subplot(111) canvas = FigureCanvas(fig) imgInMem = StringIO.StringIO() # Guess xticks for number of labels along x-axis. Place a label every 100 pixels ax.locator_params(x_res/100) # If there's no data, wait # For the simple case of testing, lets plot all time variant variables one at a time xAxisVar = time_field # Prepare the set of y axis variables that will be plotted. This needs to be smarter and passed as # config variable to the transform yAxisVars = [] for varName, varData in graph_data.iteritems(): if varName == time_field or varName == 'height' or varName == 'longitude' or varName == 'latitude': continue yAxisVars.append(varName) # Do a error check for incorrect time values. Ignore all time == fill_values time_fill_value = 0.0 clean_data_flag = False while not clean_data_flag: clean_data_flag = True # Go through the data and see if we can find empty time entries. Delete it along # with all corresponding variables for idx in xrange(len(graph_data[xAxisVar])): if graph_data[xAxisVar][idx] == time_fill_value: graph_data[xAxisVar].pop(idx) for varName in yAxisVars: try: graph_data[varName].pop(idx) except IndexError: # Do nothing really. Must be a malformed granule pass clean_data_flag = False break #xAxisFloatData = graph_data[xAxisVar] xAxisFloatData = [datetime.fromtimestamp(ntplib.ntp_to_system_time(t)) for t in graph_data[xAxisVar]] #print " >>>>>>>>>>>>>>>> xAxisFloatData : ", xAxisFloatData idx = 0 for varName in yAxisVars: yAxisFloatData = graph_data[varName] # Generate the plot ax.plot(xAxisFloatData, yAxisFloatData, cls.line_style(idx), label=varName) idx += 1 yAxisLabel = "" # generate a filename for the output image for varName in yAxisVars: if yAxisLabel: yAxisLabel = yAxisLabel + "-" + varName else: yAxisLabel = varName if not fileName: fileName = yAxisLabel + '_vs_' + xAxisVar fileName = fileName + "." + mpl_output_image_format # Choose a small font for the legend legend_font_prop = FontProperties() legend_font_prop.set_size('small') ax.set_xlabel(xAxisVar) ax.set_ylabel(yAxisLabel) ax.set_title(yAxisLabel + ' vs ' + xAxisVar) ax.set_autoscale_on(False) ax.legend(loc='upper left', ncol=3, fancybox=True, shadow=True, prop=legend_font_prop) # Date formatting # matplotlib date format object #hfmt = dates.DateFormatter('%m/%d %H:%M') #ax.xaxis.set_major_locator(dates.MinuteLocator()) #ax.xaxis.set_major_formatter(hfmt) # Save the figure to the in memory file canvas.print_figure(imgInMem, format=mpl_output_image_format) imgInMem.seek(0) # Create output dictionary from the param dict out_rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id) # Prepare granule content #out_dict = {} out_rdt["viz_product_type"] = ["matplotlib_graphs"] out_rdt["image_obj"] = [imgInMem.getvalue()] out_rdt["image_name"] = [fileName] out_rdt["content_type"] = ["image/png"] #print " >>>>>>>>>> OUT_IMAGE_NAME : ", out_rdt['image_name'] #out_rdt["graph_image_param_dict"] = np.array([out_dict]) out_rdt["viz_timestamp"] = TimeUtils.ts_to_units(rdt.context(rdt.temporal_parameter).uom, time.time()) return out_rdt.to_granule()
def get_time_idx(cls, coverage, timeval): corrected_time = cls.convert_time(coverage, timeval) idx = TimeUtils.get_relative_time(coverage, corrected_time) return idx
def convert_time(cls, coverage, timeval): tname = coverage.temporal_parameter_name uom = coverage.get_parameter_context(tname).uom corrected_time = TimeUtils.ts_to_units(uom, timeval) return corrected_time
def execute(input=None, context=None, config=None, params=None, state=None): stream_definition_id = params #init stuff rdt_for_nones = {} hc_data = [] normalized_ts = [] hc_allowed_numerical_types = ['int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64','str'] # TODO : Move this in to container parameter default_precision = 5 rdt = RecordDictionaryTool.load_from_granule(input) # Buid a local precisions and fill value dictionary to use for parsing data correctly precisions = {} fill_values = {} for field in rdt.fields: precision_str = rdt.context(field).precision if not precision_str: precisions[field] = default_precision else: try: precisions[field] = int(precision_str) except ValueError: precisions[field] = default_precision fill_values[field] = rdt.fill_value(field) if stream_definition_id == None: log.error("HighCharts transform: Need a output stream definition to process graphs") return None fields = [] fields = rdt.fields # Ascertain temporal field. Use 'time' as backup time_field = rdt.temporal_parameter or 'time' #if 'time' not in rdt: return None if rdt[time_field] is None: return None #time_fill_value = 0.0 # should be derived from the granule's param dict. time_fill_value = fill_values[time_field] # should be derived from the granule's param dict. total_num_of_records = len(rdt[time_field]) # convert timestamps from ntp to system count = 0 normalized_ts = [None] * total_num_of_records for ts in rdt[time_field]: if ts == time_fill_value: normalized_ts[count] = time_fill_value else: normalized_ts[count] = float(ntplib.ntp_to_system_time(ts)) count += 1 ###### DEBUG ########## #for field in fields: # if hasattr(rdt.context(field),'visible'): # print " >>>>>>>> '", field, "' [visible = ", rdt.context(field).visible,"] : ", rdt[field] # else: # print " >>>>>>>> '", field, "' [visible = NOT SPECIFIED] : ", rdt[field] # Convert the fields in to HC series format import re for field in fields: field_precision = precisions[field] if field == time_field: continue # If a config block was passed, consider only the params listed in it if config and 'parameters' in config and len(config['parameters']) > 0: if not field in config['parameters']: log.info("Skipping ", field, " since it was not present in the list of allowed parameters") continue # If the value is none, assign it a small one fill_value array for now to generate description, # Actual array of fill_values will be assigned later rdt_field =rdt[field] if rdt_field == None: rdt_for_nones[field] = np.array([fill_values[field]] * total_num_of_records) #rdt_for_nones[field] = [fill_values[field]] * total_num_of_records rdt_field = rdt_for_nones[field] # Check if visibility is false (system generated params) or not specified explicitly if hasattr(rdt.context(field),'visible') and not rdt.context(field).visible: continue # If it's a QC parameter ignore it if field.endswith('_qc'): continue # NOTE: The reason why the following libes of code seem to make a lot of branches are to pull the major decisions # outside the primary loops responsible for arranging data in the data structures to be sent to the charts. # This is better than having to make the decisions on a per record entry. # Handle arrays by spliting them them in to individual parameters context = rdt.context(field) if (isinstance(context.param_type, ArrayType) or isinstance(context.param_type,ParameterFunctionType)) and len(rdt_field.shape)>1: if (rdt_field.dtype == 'string' or rdt_field.dtype not in hc_allowed_numerical_types): for i in xrange(rdt_field.shape[1]): series = {} series["name"] = field + "[" + str(i) + "]" series["visible"] = False series["data"] = VizTransformHighChartsAlgorithm.form_series_data_str(normalized_ts, rdt_field, i, time_fill_value, fill_values[field]) hc_data.append(series) else: # Assume its a float or number for i in xrange(rdt_field.shape[1]): series = {} series["name"] = field + "[" + str(i) + "]" series["visible"] = True series["tooltip"] = {"valueDecimals":field_precision} series["data"] = VizTransformHighChartsAlgorithm.form_series_data_num(normalized_ts, rdt_field, i, time_fill_value, fill_values[field], field_precision) hc_data.append(series) else: if (rdt_field.dtype == 'string' or rdt_field.dtype not in hc_allowed_numerical_types): series = {} series["name"] = field series["visible"] = False series["data"] = VizTransformHighChartsAlgorithm.form_series_data_str(normalized_ts, rdt_field, None, time_fill_value, fill_values[field]) else: series = {} series["name"] = field series["tooltip"] = {"valueDecimals":field_precision} series["visible"] = True series["data"] = VizTransformHighChartsAlgorithm.form_series_data_num(normalized_ts, rdt_field, None, time_fill_value, fill_values[field], field_precision) # Append series to the hc data hc_data.append(series) # Prep the outgoing granule out_rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id) #out_dict = {"hc_data": hc_data} #out_rdt["hc_data"] = np.array([out_dict]) out_rdt["hc_data"] = [hc_data] out_rdt["viz_timestamp"] = TimeUtils.ts_to_units(rdt.context(time_field).uom, time.time()) log.debug('HighCharts Transform: Sending a granule') out_granule = out_rdt.to_granule() return out_granule
def add_granule(self,stream_id, rdt): ''' Appends the granule's data to the coverage and persists it. ''' debugging = log.isEnabledFor(DEBUG) timer = Timer() if debugging else None if stream_id in self._bad_coverages: log.info('Message attempting to be inserted into bad coverage: %s', DatasetManagementService._get_coverage_path(self.get_dataset(stream_id))) #-------------------------------------------------------------------------------- # Coverage determiniation and appending #-------------------------------------------------------------------------------- dataset_id = self.get_dataset(stream_id) if not dataset_id: log.error('No dataset could be determined on this stream: %s', stream_id) return try: coverage = self.get_coverage(stream_id) except IOError as e: log.error("Couldn't open coverage: %s", DatasetManagementService._get_coverage_path(self.get_dataset(stream_id))) raise CorruptionError(e.message) if debugging: path = DatasetManagementService._get_coverage_path(dataset_id) log.debug('%s: add_granule stream %s dataset %s coverage %r file %s', self._id, stream_id, dataset_id, coverage, path) if not coverage: log.error('Could not persist coverage from granule, coverage is None') return #-------------------------------------------------------------------------------- # Actual persistence #-------------------------------------------------------------------------------- elements = len(rdt) if debugging: timer.complete_step('checks') # lightweight ops, should be zero try: coverage.insert_timesteps(elements, oob=False) except IOError as e: log.error("Couldn't insert time steps for coverage: %s", DatasetManagementService._get_coverage_path(self.get_dataset(stream_id)), exc_info=True) try: coverage.close() finally: self._bad_coverages[stream_id] = 1 raise CorruptionError(e.message) if debugging: timer.complete_step('insert') start_index = coverage.num_timesteps - elements for k,v in rdt.iteritems(): slice_ = slice(start_index, None) try: coverage.set_parameter_values(param_name=k, tdoa=slice_, value=v) except IOError as e: log.error("Couldn't insert values for coverage: %s", DatasetManagementService._get_coverage_path(self.get_dataset(stream_id)), exc_info=True) try: coverage.close() finally: self._bad_coverages[stream_id] = 1 raise CorruptionError(e.message) if 'ingestion_timestamp' in coverage.list_parameters(): t_now = time.time() ntp_time = TimeUtils.ts_to_units(coverage.get_parameter_context('ingestion_timestamp').uom, t_now) coverage.set_parameter_values(param_name='ingestion_timestamp', tdoa=slice_, value=ntp_time) if debugging: timer.complete_step('keys') DatasetManagementService._save_coverage(coverage) if debugging: timer.complete_step('save') self.dataset_changed(dataset_id,coverage.num_timesteps,(start_index,start_index+elements)) if debugging: timer.complete_step('notify') self._add_timing_stats(timer)
def sync_rdt_with_coverage(self, coverage=None, tdoa=None, start_time=None, end_time=None, stride_time=None, parameters=None): ''' Builds a granule based on the coverage ''' if coverage is None: coverage = self.coverage slice_ = slice(None) # Defaults to all values if tdoa is not None and isinstance(tdoa, slice): slice_ = tdoa elif stride_time is not None: validate_is_instance(start_time, Number, 'start_time must be a number for striding.') validate_is_instance(end_time, Number, 'end_time must be a number for striding.') validate_is_instance(stride_time, Number, 'stride_time must be a number for striding.') ugly_range = np.arange(start_time, end_time, stride_time) idx_values = [ TimeUtils.get_relative_time(coverage, i) for i in ugly_range ] slice_ = [idx_values] elif not (start_time is None and end_time is None): time_var = coverage._temporal_param_name uom = coverage.get_parameter_context(time_var).uom if start_time is not None: start_units = TimeUtils.ts_to_units(uom, start_time) log.info('Units: %s', start_units) start_idx = TimeUtils.get_relative_time(coverage, start_units) log.info('Start Index: %s', start_idx) start_time = start_idx if end_time is not None: end_units = TimeUtils.ts_to_units(uom, end_time) log.info('End units: %s', end_units) end_idx = TimeUtils.get_relative_time(coverage, end_units) log.info('End index: %s', end_idx) end_time = end_idx slice_ = slice(start_time, end_time, stride_time) log.info('Slice: %s', slice_) if parameters is not None: pdict = ParameterDictionary() params = set(coverage.list_parameters()).intersection(parameters) for param in params: pdict.add_context(coverage.get_parameter_context(param)) rdt = RecordDictionaryTool(param_dictionary=pdict) self.pdict = pdict else: rdt = RecordDictionaryTool( param_dictionary=coverage.parameter_dictionary) fields = coverage.list_parameters() if parameters is not None: fields = set(fields).intersection(parameters) for d in fields: rdt[d] = coverage.get_parameter_values(d, tdoa=slice_) self.rdt = rdt # Sync
def add_granule(self, stream_id, rdt): ''' Appends the granule's data to the coverage and persists it. ''' debugging = log.isEnabledFor(DEBUG) timer = Timer() if debugging else None if stream_id in self._bad_coverages: log.info( 'Message attempting to be inserted into bad coverage: %s', DatasetManagementService._get_coverage_path( self.get_dataset(stream_id))) #-------------------------------------------------------------------------------- # Coverage determiniation and appending #-------------------------------------------------------------------------------- dataset_id = self.get_dataset(stream_id) if not dataset_id: log.error('No dataset could be determined on this stream: %s', stream_id) return try: coverage = self.get_coverage(stream_id) except IOError as e: log.error( "Couldn't open coverage: %s", DatasetManagementService._get_coverage_path( self.get_dataset(stream_id))) raise CorruptionError(e.message) if debugging: path = DatasetManagementService._get_coverage_path(dataset_id) log.debug( '%s: add_granule stream %s dataset %s coverage %r file %s', self._id, stream_id, dataset_id, coverage, path) if not coverage: log.error( 'Could not persist coverage from granule, coverage is None') return #-------------------------------------------------------------------------------- # Actual persistence #-------------------------------------------------------------------------------- elements = len(rdt) if debugging: timer.complete_step('checks') # lightweight ops, should be zero try: coverage.insert_timesteps(elements, oob=False) except IOError as e: log.error("Couldn't insert time steps for coverage: %s", DatasetManagementService._get_coverage_path( self.get_dataset(stream_id)), exc_info=True) try: coverage.close() finally: self._bad_coverages[stream_id] = 1 raise CorruptionError(e.message) if debugging: timer.complete_step('insert') start_index = coverage.num_timesteps - elements for k, v in rdt.iteritems(): slice_ = slice(start_index, None) try: coverage.set_parameter_values(param_name=k, tdoa=slice_, value=v) except IOError as e: log.error("Couldn't insert values for coverage: %s", DatasetManagementService._get_coverage_path( self.get_dataset(stream_id)), exc_info=True) try: coverage.close() finally: self._bad_coverages[stream_id] = 1 raise CorruptionError(e.message) if 'ingestion_timestamp' in coverage.list_parameters(): t_now = time.time() ntp_time = TimeUtils.ts_to_units( coverage.get_parameter_context('ingestion_timestamp').uom, t_now) coverage.set_parameter_values(param_name='ingestion_timestamp', tdoa=slice_, value=ntp_time) if debugging: timer.complete_step('keys') DatasetManagementService._save_coverage(coverage) if debugging: timer.complete_step('save') self.dataset_changed(dataset_id, coverage.num_timesteps, (start_index, start_index + elements)) if debugging: timer.complete_step('notify') self._add_timing_stats(timer)