def test_exclude_data(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] times = ctd_ds.time.values ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.exclude_flagged_data() np.testing.assert_array_equal(times, ctd_stream_dataset.datasets[2].time.values) # exclude a bit start = ntplib.ntp_to_system_time(times[0]) * 1000 stop = ntplib.ntp_to_system_time(times[100]) * 1000 anno = self._create_exclusion_anno(start, stop) ctd_stream_dataset.annotation_store.add_annotations([anno]) ctd_stream_dataset.exclude_flagged_data() np.testing.assert_array_equal(times[101:], ctd_stream_dataset.datasets[2].time.values) # exclude everything start = ntplib.ntp_to_system_time(times[0]) * 1000 stop = ntplib.ntp_to_system_time(times[-1]) * 1000 anno = self._create_exclusion_anno(start, stop) ctd_stream_dataset.annotation_store.add_annotations([anno]) ctd_stream_dataset.exclude_flagged_data() self.assertNotIn(2, ctd_stream_dataset.datasets)
def test_exclude_data(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] times = ctd_ds.time.values store = AnnotationStore() ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.exclude_flagged_data(store) np.testing.assert_array_equal(times, ctd_stream_dataset.datasets[2].time.values) # exclude a bit start = ntplib.ntp_to_system_time(times[0]) * 1000 stop = ntplib.ntp_to_system_time(times[100]) * 1000 anno = self._create_exclusion_anno(self.ctdpf_sk, start, stop) store.add_annotations([anno]) ctd_stream_dataset.exclude_flagged_data(store) np.testing.assert_array_equal(times[101:], ctd_stream_dataset.datasets[2].time.values) # exclude everything start = ntplib.ntp_to_system_time(times[0]) * 1000 stop = ntplib.ntp_to_system_time(times[-1]) * 1000 anno = self._create_exclusion_anno(self.ctdpf_sk, start, stop) store.add_annotations([anno]) ctd_stream_dataset.exclude_flagged_data(store) self.assertNotIn(2, ctd_stream_dataset.datasets)
def as_millis(self): """ Return the start/stop times in milliseconds since 1-1-1970 :return: (start, stop) """ return int(ntplib.ntp_to_system_time(self.start) * 1000), int( ntplib.ntp_to_system_time(self.stop) * 1000)
def __restore_ntp_pck(self, ntp: NTP) -> NTP: self.log.debug('Send timestamp for reconstruction: ' + str(ntp.sent)) sent_time_stamp = datetime.fromtimestamp( ntplib.ntp_to_system_time(ntp.sent)) sent_time_stamp = sent_time_stamp.replace(year=datetime.now().year) sent_time_stamp_as_ntp = ntplib.system_to_ntp_time( sent_time_stamp.timestamp()) ntp.sent = sent_time_stamp_as_ntp self.log.debug('Send timestamp after reconstruction: ' + str(ntp.sent)) pck = CP3Package(ntp) if NTPMode.from_bit_string(pck.mode()) is NTPMode.CLIENT: self.log.debug("Restored in Client mode") ntp.ref = 0 ntp.orig = 0 ntp.recv = 0 if NTPMode.from_bit_string(pck.mode()) is NTPMode.SERVER \ or NTPMode.from_bit_string(pck.mode()) is NTPMode.BROADCAST_SERVER: self.log.debug("Restored in Server mode") origin_last_32 = pck.origin_timestamp()[32:64] received_last_32 = pck.receive_timestamp()[32:64] transmit_first_32 = pck.origin_timestamp()[0:32] pck.set_origin_timestamp(transmit_first_32 + origin_last_32) pck.set_receive_timestamp(transmit_first_32 + received_last_32) ntp = pck.ntp() self.log.debug("Reconstruction complete.") #ntp.show() return ntp
def test_instrument_autosample_samples(self): """ @brief Test for putting instrument in 'auto-sample' state and receiving samples """ self.assert_initialize_driver() # command the instrument to auto-sample mode. self.driver_client.cmd_dvr('execute_resource', ProtocolEvent.START_AUTOSAMPLE) self.assert_current_state(ProtocolStates.AUTOSAMPLE) # wait for some samples to be generated log.debug('test_instrument_start_stop_autosample: waiting 5 seconds for samples') gevent.sleep(5) # Verify we received at least 2 samples. sample_events = [evt for evt in self.events if evt['type']==DriverAsyncEvent.SAMPLE] log.debug('test_instrument_start_stop_autosample: # 0f samples = %d' %len(sample_events)) for sample in sample_events: if sample['value'].find(DataParticleType.SAMPLE) != -1: log.debug('parsed sample=%s\n' %sample) sample_dict = eval(sample['value']) # turn string into dictionary values = sample_dict['values'] # get particle dictionary # pull timestamp out of particle ntp_timestamp = [item for item in values if item["value_id"] == "timestamp"][0]['value'] float_timestamp = ntplib.ntp_to_system_time(ntp_timestamp) log.debug('dt=%s' %time.ctime(float_timestamp)) self.assertTrue(len(sample_events) >= 2) # stop autosample and return to command mode self.driver_client.cmd_dvr('execute_resource', ProtocolEvent.STOP_AUTOSAMPLE) self.assert_current_state(ProtocolStates.COMMAND)
def ntp_to_ISO_date(ntp_time): try: ntp_time = float(ntp_time) dt = datetime.datetime.utcfromtimestamp(ntplib.ntp_to_system_time(ntp_time)) return dt.isoformat() except Exception as e: return str(ntp_time)
def test_restore_pck_server_pck_other_timestamps_filled_with_send(self): # Arrange client = CP3Handler('') pck = CP3Package() first_32 = '11111111111111111111000000011111' origin_last_32 = '11111111111110011110111111111111' received_last_32 = '11111111110111011110111111111111' transmit = '1111111111111111111111111111111111111111111111111111111111111111' pck.set_origin_timestamp(first_32 + origin_last_32) pck.set_receive_timestamp(first_32 + received_last_32) pck.set_transmit_timestamp(transmit) pck.set_mode(NTPMode.to_bit_string(NTPMode.SERVER)) # Act result_pck = RawNTP(client.restore_pck(pck)) # Assert self.assertNotEqual(result_pck.transmit_timestamp(), transmit) self.assertEqual(result_pck.origin_timestamp()[32:64], origin_last_32) self.assertEqual(result_pck.receive_timestamp()[32:64], received_last_32) self.assertEqual(result_pck.origin_timestamp()[0:32], result_pck.origin_timestamp()[0:32]) self.assertEqual(result_pck.receive_timestamp()[0:32], result_pck.origin_timestamp()[0:32]) self.assertEqual( datetime.fromtimestamp( ntplib.ntp_to_system_time(result_pck.ntp().sent)).year, datetime.now().year)
def _set_cp3_mode(self,year:int): ntp = self.ntp() time = ntplib.system_to_ntp_time(datetime.fromtimestamp(ntplib.ntp_to_system_time(ntp.sent)) .replace(year=year).timestamp()) ntp.sent = time raw = RawNTP(ntp) self.set_transmit_timestamp(raw.transmit_timestamp())
def ntp_2_ion_ts(ntp_time): """ Converts an NTP time into ION system timestamp. The converse operation is ion_ts_2_ntp(ion_ts). Note: this should probably be a utility provided by pyon. @see https://jira.oceanobservatories.org/tasks/browse/OOIION-631 @param ntp_time float representing an NTP time. @retval str representing an integer number, the millis in UNIX epoch, corresponding to the given NTP time. This is calculated as str( int(round(ntplib.ntp_to_system_time(ntp_time) * 1000)) ) """ # convert to system time: sys_time = ntplib.ntp_to_system_time(ntp_time) # convert to milliseconds rounding to the nearest integer: sys_time = int(round(sys_time * 1000)) # return as str return str(sys_time)
def ntp_to_datetime(ntp_time): try: ntp_time = float(ntp_time) unix_time = ntplib.ntp_to_system_time(ntp_time) dt = datetime.datetime.utcfromtimestamp(unix_time) return dt except (ValueError, TypeError): return None
def _last_sync(self) -> float: """Get last sync time in NTP epoch Returns ------- The amplifer time of last sync in system time """ return ntp_to_system_time(self._mstime)
def test_restore_pck_client_pck_with_changed_year(self): # Arrange client = CP3Handler('') pck = init_ntp_pck() pck.sent = ntplib.system_to_ntp_time( datetime.fromtimestamp(ntplib.ntp_to_system_time( pck.sent)).replace(year=2006).timestamp()) # Act result_pck = client.restore_pck(CP3Package(ntp_pck=pck)) # Assert self.assertEqual(result_pck.ref, 0) self.assertEqual(result_pck.recv, 0) self.assertEqual(result_pck.orig, 0) self.assertEqual( datetime.fromtimestamp(ntplib.ntp_to_system_time( result_pck.sent)).year, datetime.now().year)
def ntp_to_string(timestamp, time_format=DATE_FORMAT): """ takes an NTP timestamp (seconds since 1900/1/1) and outputs in provided format :param timestamp: ntp timestamp :param time_format: datetime compatible time string format :return: """ unix_time = ntplib.ntp_to_system_time(timestamp) dt = datetime.datetime.utcfromtimestamp(unix_time) return dt.strftime(time_format)
def test_set_cp3_mode_2_correct_datetime_set(self): # Arrange pck = CP3Package(ntp_pck=init_ntp_pck()) # Act pck.set_cp3_mode_2() # Assert time = datetime.fromtimestamp(ntplib.ntp_to_system_time(pck.ntp().sent)).year self.assertEqual(time, 2000)
def resync(self) -> float: """Re-synchronize with the amplifier Returns ------- The current NTP epoch time """ t = (datetime.now(timezone.utc) - unix_epoch).total_seconds() data = system_to_ntp_time(t) self._mstime = self._command('NTPReturnClock', data) # TODO: remove this debug info print('Sent local time: ' + format_time(t)) amptime = ntp_to_system_time(self._mstime) print('Received amp time: ' + format_time(amptime))
def test_helpers(self): """Helper methods tests.""" client = ntplib.NTPClient() time.sleep(self.POLL_DELAY) info = client.request(self.NTP_SERVER) self.assertEqual(int(info.tx_time), ntplib.ntp_to_system_time( ntplib.system_to_ntp_time(int(info.tx_time)))) self.assertTrue(isinstance(ntplib.leap_to_text(info.leap), str)) self.assertTrue(isinstance(ntplib.mode_to_text(info.mode), str)) self.assertTrue(isinstance(ntplib.stratum_to_text(info.stratum), str)) self.assertTrue(isinstance(ntplib.ref_id_to_text(info.ref_id, info.stratum), str))
def flatten(particle): for each in particle.get('values', []): particle[each['value_id']] = each['value'] del (particle['values']) del (particle['quality_flag']) del (particle['stream_name']) del (particle['pkt_format_id']) del (particle['pkt_version']) particle['time'] = particle.get(particle.get('preferred_timestamp', {})) particle['ingestion_timestamp'] = ntplib.system_to_ntp_time(time.time()) ts = ntplib.ntp_to_system_time(particle['time']) ts = time.gmtime(ts) particle['refdesig'] = 'TEST' particle['year'] = ts.tm_year particle['jday'] = ts.tm_yday
def flatten(particle): for each in particle.get('values', []): particle[each['value_id']] = each['value'] del(particle['values']) del(particle['quality_flag']) del(particle['stream_name']) del(particle['pkt_format_id']) del(particle['pkt_version']) particle['time'] = particle.get(particle.get('preferred_timestamp', {})) particle['ingestion_timestamp'] = ntplib.system_to_ntp_time(time.time()) ts = ntplib.ntp_to_system_time(particle['time']) ts = time.gmtime(ts) particle['refdesig'] = 'TEST' particle['year'] = ts.tm_year particle['jday'] = ts.tm_yday
def test_helpers(self): """Helper methods tests.""" client = ntplib.NTPClient() time.sleep(self.POLL_DELAY) info = client.request(self.NTP_SERVER) self.assertEqual( int(info.tx_time), ntplib.ntp_to_system_time( ntplib.system_to_ntp_time(int(info.tx_time)))) self.assertTrue(isinstance(ntplib.leap_to_text(info.leap), str)) self.assertTrue(isinstance(ntplib.mode_to_text(info.mode), str)) self.assertTrue(isinstance(ntplib.stratum_to_text(info.stratum), str)) self.assertTrue( isinstance(ntplib.ref_id_to_text(info.ref_id, info.stratum), str))
def execute(input=None, context=None, config=None, params=None, state=None): stream_definition_id = params #init stuff var_tuple = [] data_description = [] data_table_content = [] gdt_allowed_numerical_types = [ 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64', 'str' ] rdt = RecordDictionaryTool.load_from_granule(input) data_description = [] if stream_definition_id == None: log.error( "GoogleDT transform: Need a output stream definition to process graphs" ) return None fields = [] if config and config['parameters']: fields = config['parameters'] else: fields = rdt.fields # if time was null or missing, do not process if 'time' not in rdt: return None if rdt['time'] is None: return None time_fill_value = 0.0 # should be derived from the granule's param dict. data_description.append(('time', 'number', 'time')) for field in fields: if field == 'time': continue # If a config block was passed, consider only the params listed in it if config and config['parameters']: if not field in config['parameters']: continue # only consider fields which are supposed to be numbers. if (rdt[field] != None) and (rdt[field].dtype not in gdt_allowed_numerical_types): continue data_description.append((field, 'number', field)) for i in xrange(len(rdt)): varTuple = [] # Put time first if its not zero. Retrieval returns 0 secs for malformed entries if rdt['time'][i] == time_fill_value: continue # convert timestamp from instrument to UNIX time stamp since thats what Google DT expects varTuple.append(ntplib.ntp_to_system_time(rdt['time'][i])) for dd in data_description: field = dd[0] # ignore time since its been already added if field == None or field == 'time': continue if rdt[field] == None or rdt[field][i] == None: varTuple.append(0.0) else: varTuple.append(rdt[field][i]) # Append the tuples to the data table if len(varTuple) > 0: data_table_content.append(varTuple) out_rdt = RecordDictionaryTool( stream_definition_id=stream_definition_id) # Prepare granule content out_dict = { "viz_product_type": "google_dt", "data_description": data_description, "data_content": data_table_content } out_rdt["google_dt_components"] = np.array([out_dict]) log.debug('Google DT transform: Sending a granule') out_granule = out_rdt.to_granule() return out_granule
def _process_visualization_message(self, messages): gdt_description = None gdt_content = [] viz_product_type = '' for message in messages: if message == None: continue message_data = message.body if isinstance(message_data,Granule): rdt = RecordDictionaryTool.load_from_granule(message_data) gdt_components = get_safe(rdt, 'google_dt_components') # IF this granule does not contain google dt, skip if gdt_components is None: continue gdt_component = gdt_components[0] viz_product_type = gdt_component['viz_product_type'] # Process Google DataTable messages if viz_product_type == 'google_dt': # If the data description is being put together for the first time, # switch the time format from float to datetime if (gdt_description == None): temp_gdt_description = gdt_component['data_description'] gdt_description = [('time', 'datetime', 'time')] for idx in range(1,len(temp_gdt_description)): # for some weird reason need to force convert to tuples temp_arr = temp_gdt_description[idx] if temp_arr != None and temp_arr[0] != 'time': gdt_description.append((temp_arr[0], temp_arr[1], temp_arr[2])) # append all content to one big array temp_gdt_content = gdt_component['data_content'] for tempTuple in temp_gdt_content: # sometimes there are inexplicable empty tuples in the content. Drop them if tempTuple == [] or len(tempTuple) == 0: continue varTuple = [] # Adjust the ntp time stamp from instruments to standard date tine #varTuple.append(datetime.fromtimestamp(tempTuple[0])) varTuple.append(datetime.fromtimestamp(ntplib.ntp_to_system_time(tempTuple[0]))) for idx in range(1,len(tempTuple)): # some silly numpy format won't go away so need to cast numbers to floats if(gdt_description[idx][1] == 'number'): varTuple.append((float)(tempTuple[idx])) else: varTuple.append(tempTuple[idx]) gdt_content.append(varTuple) #TODO - what to do if this is not a valid visualization message? # Now that all the messages have been parsed, any last processing should be done here if viz_product_type == "google_dt": # Using the description and content, build the google data table gdt = gviz_api.DataTable(gdt_description) gdt.LoadData(gdt_content) return gdt.ToJSonResponse() return None
def render_graphs(cls, graph_data, stream_definition_id, fileName=None, resolution=None): # init Matplotlib with passsed parameters x_res = y_res = 100 # some default in case nothing is provided if resolution: x_res, y_res = resolution.split('x') x_res = int(x_res) y_res = int(y_res) fig = Figure(figsize=(x_res / 100, y_res / 100), dpi=100, frameon=True) ax = fig.add_subplot(111) canvas = FigureCanvas(fig) imgInMem = StringIO.StringIO() # Guess xticks for number of labels along x-axis. Place a label every 100 pixels ax.locator_params(x_res / 100) # If there's no data, wait # For the simple case of testing, lets plot all time variant variables one at a time xAxisVar = 'time' # Prepare the set of y axis variables that will be plotted. This needs to be smarter and passed as # config variable to the transform yAxisVars = [] for varName, varData in graph_data.iteritems(): if varName == 'time' or varName == 'height' or varName == 'longitude' or varName == 'latitude': continue yAxisVars.append(varName) # Do a error check for incorrect time values. Ignore all time == fill_values time_fill_value = 0.0 clean_data_flag = False while not clean_data_flag: clean_data_flag = True # Go through the data and see if we can find empty time entries. Delete it along # with all corresponding variables for idx in xrange(len(graph_data[xAxisVar])): if graph_data[xAxisVar][idx] == time_fill_value: graph_data[xAxisVar].pop(idx) for varName in yAxisVars: try: graph_data[varName].pop(idx) except IndexError: # Do nothing really. Must be a malformed granule pass clean_data_flag = False break #xAxisFloatData = graph_data[xAxisVar] xAxisFloatData = [ datetime.fromtimestamp(ntplib.ntp_to_system_time(t)) for t in graph_data[xAxisVar] ] #print " >>>>>>>>>>>>>>>> xAxisFloatData : ", xAxisFloatData idx = 0 for varName in yAxisVars: yAxisFloatData = graph_data[varName] # Generate the plot ax.plot(xAxisFloatData, yAxisFloatData, cls.line_style(idx), label=varName) idx += 1 yAxisLabel = "" # generate a filename for the output image for varName in yAxisVars: if yAxisLabel: yAxisLabel = yAxisLabel + "-" + varName else: yAxisLabel = varName if not fileName: fileName = yAxisLabel + '_vs_' + xAxisVar fileName = fileName + "." + mpl_output_image_format # Choose a small font for the legend legend_font_prop = FontProperties() legend_font_prop.set_size('small') ax.set_xlabel(xAxisVar) ax.set_ylabel(yAxisLabel) ax.set_title(yAxisLabel + ' vs ' + xAxisVar) ax.set_autoscale_on(False) ax.legend(loc='upper left', ncol=3, fancybox=True, shadow=True, prop=legend_font_prop) # Date formatting # matplotlib date format object #hfmt = dates.DateFormatter('%m/%d %H:%M') #ax.xaxis.set_major_locator(dates.MinuteLocator()) #ax.xaxis.set_major_formatter(hfmt) # Save the figure to the in memory file canvas.print_figure(imgInMem, format=mpl_output_image_format) imgInMem.seek(0) # Create output dictionary from the param dict out_rdt = RecordDictionaryTool( stream_definition_id=stream_definition_id) # Prepare granule content #out_dict = {} out_rdt["viz_product_type"] = ["matplotlib_graphs"] out_rdt["image_obj"] = [imgInMem.getvalue()] out_rdt["image_name"] = [fileName] out_rdt["content_type"] = ["image/png"] #print " >>>>>>>>>> OUT_IMAGE_NAME : ", out_rdt['image_name'] #out_rdt["graph_image_param_dict"] = np.array([out_dict]) return out_rdt.to_granule()
def execute(input=None, context=None, config=None, params=None, state=None): stream_definition_id = params #init stuff data_description = [] data_table_content = [] gdt_allowed_numerical_types = ['int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64','str'] # TODO : Move this in to container parameter default_precision = 5 rdt = RecordDictionaryTool.load_from_granule(input) data_description = [] # Buid a local precisions and fill value dictionary to use for parsing data correctly precisions = {} fill_values = {} for field in rdt.fields: precision_str = rdt.context(field).precision if not precision_str: precisions[field] = default_precision else: try: precisions[field] = int(precision_str) except ValueError: precisions[field] = default_precision fill_values[field] = rdt.fill_value(field) if stream_definition_id == None: log.error("GoogleDT transform: Need a output stream definition to process graphs") return None fields = [] fields = rdt.fields # if time was null or missing, do not process if 'time' not in rdt: return None if rdt['time'] is None: return None time_fill_value = 0.0 # should be derived from the granule's param dict. data_description.append(('time','number','time')) import re for field in fields: if field == rdt.temporal_parameter: continue # If a config block was passed, consider only the params listed in it if config and 'parameters' in config and len(config['parameters']) > 0: if not field in config['parameters']: log.info("Skipping ", field, " since it was not present in the list of allowed parameters") continue # only consider fields which are allowed. if rdt[field] == None: continue # Check if visibility is false (system generated params) if hasattr(rdt.context(field),'visible') and not rdt.context(field).visible: continue # If it's a QC parameter ignore it if field.endswith('_qc'): continue # Handle string type or if its an unknown type, convert to string context = rdt.context(field) if (rdt[field].dtype == 'string' or rdt[field].dtype not in gdt_allowed_numerical_types): data_description.append((field, 'string', field )) elif (isinstance(context.param_type, ArrayType) or isinstance(context.param_type,ParameterFunctionType)) and len(rdt[field].shape)>1: for i in xrange(rdt[field].shape[1]): data_description.append(('%s[%s]' % (field,i), 'number', '%s[%s]' % (field,i), {'precision':str(precisions[field])})) else: data_description.append((field, 'number', field, {'precision':str(precisions[field])} )) for i in xrange(len(rdt)): varTuple = [] # Put time first if its not zero. Retrieval returns 0 secs for malformed entries if rdt['time'][i] == time_fill_value: continue # convert timestamp from instrument to UNIX time stamp since thats what Google DT expects varTuple.append(ntplib.ntp_to_system_time(rdt['time'][i])) for dd in data_description: field = dd[0] field_type = dd[1] # ignore time since its been already added if field == None or field == 'time': continue if re.match(r'.*\[\d+\]', field): field, j = re.match(r'(.*)\[(\d+)\]', field).groups() j = int(j) varTuple.append(float(rdt[field][i][j])) elif rdt[field] == None or rdt[field][i] == None: varTuple.append(None) else: if(field_type == 'number'): if rdt[field][i] == None or rdt[field][i] == fill_values[field]: varTuple.append(None) else: # Adjust float for precision if (precisions[field] == None): varTuple.append(float(rdt[field][i])) else: varTuple.append(round(float(rdt[field][i]), precisions[field])) # if field type is string, there are two possibilities. Either it really is a string or # its an object that needs to be converted to string. if(field_type == 'string'): varTuple.append(str(rdt[field][i])) # Append the tuples to the data table if len(varTuple) > 0: data_table_content.append(varTuple) out_rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id) # Prepare granule content out_dict = {"viz_product_type" : "google_dt", "data_description" : data_description, "data_content" : data_table_content} out_rdt["google_dt_components"] = np.array([out_dict]) out_rdt["viz_timestamp"] = TimeUtils.ts_to_units(rdt.context(rdt.temporal_parameter).uom, time.time()) log.debug('Google DT transform: Sending a granule') out_granule = out_rdt.to_granule() return out_granule
def render_graphs(cls, rdt, graph_data, stream_definition_id, fileName = None, resolution = None, time_field = 'time'): # init Matplotlib with passsed parameters x_res = y_res = 100 # some default in case nothing is provided if resolution: x_res, y_res = resolution.split('x') x_res = int(x_res) y_res = int(y_res) fig = Figure(figsize=(x_res / 100, y_res / 100), dpi=100, frameon=True) ax = fig.add_subplot(111) canvas = FigureCanvas(fig) imgInMem = StringIO.StringIO() # Guess xticks for number of labels along x-axis. Place a label every 100 pixels ax.locator_params(x_res/100) # If there's no data, wait # For the simple case of testing, lets plot all time variant variables one at a time xAxisVar = time_field # Prepare the set of y axis variables that will be plotted. This needs to be smarter and passed as # config variable to the transform yAxisVars = [] for varName, varData in graph_data.iteritems(): if varName == time_field or varName == 'height' or varName == 'longitude' or varName == 'latitude': continue yAxisVars.append(varName) # Do a error check for incorrect time values. Ignore all time == fill_values time_fill_value = 0.0 clean_data_flag = False while not clean_data_flag: clean_data_flag = True # Go through the data and see if we can find empty time entries. Delete it along # with all corresponding variables for idx in xrange(len(graph_data[xAxisVar])): if graph_data[xAxisVar][idx] == time_fill_value: graph_data[xAxisVar].pop(idx) for varName in yAxisVars: try: graph_data[varName].pop(idx) except IndexError: # Do nothing really. Must be a malformed granule pass clean_data_flag = False break #xAxisFloatData = graph_data[xAxisVar] xAxisFloatData = [datetime.fromtimestamp(ntplib.ntp_to_system_time(t)) for t in graph_data[xAxisVar]] #print " >>>>>>>>>>>>>>>> xAxisFloatData : ", xAxisFloatData idx = 0 for varName in yAxisVars: yAxisFloatData = graph_data[varName] # Generate the plot ax.plot(xAxisFloatData, yAxisFloatData, cls.line_style(idx), label=varName) idx += 1 yAxisLabel = "" # generate a filename for the output image for varName in yAxisVars: if yAxisLabel: yAxisLabel = yAxisLabel + "-" + varName else: yAxisLabel = varName if not fileName: fileName = yAxisLabel + '_vs_' + xAxisVar fileName = fileName + "." + mpl_output_image_format # Choose a small font for the legend legend_font_prop = FontProperties() legend_font_prop.set_size('small') ax.set_xlabel(xAxisVar) ax.set_ylabel(yAxisLabel) ax.set_title(yAxisLabel + ' vs ' + xAxisVar) ax.set_autoscale_on(False) ax.legend(loc='upper left', ncol=3, fancybox=True, shadow=True, prop=legend_font_prop) # Date formatting # matplotlib date format object #hfmt = dates.DateFormatter('%m/%d %H:%M') #ax.xaxis.set_major_locator(dates.MinuteLocator()) #ax.xaxis.set_major_formatter(hfmt) # Save the figure to the in memory file canvas.print_figure(imgInMem, format=mpl_output_image_format) imgInMem.seek(0) # Create output dictionary from the param dict out_rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id) # Prepare granule content #out_dict = {} out_rdt["viz_product_type"] = ["matplotlib_graphs"] out_rdt["image_obj"] = [imgInMem.getvalue()] out_rdt["image_name"] = [fileName] out_rdt["content_type"] = ["image/png"] #print " >>>>>>>>>> OUT_IMAGE_NAME : ", out_rdt['image_name'] #out_rdt["graph_image_param_dict"] = np.array([out_dict]) out_rdt["viz_timestamp"] = TimeUtils.ts_to_units(rdt.context(rdt.temporal_parameter).uom, time.time()) return out_rdt.to_granule()
def execute(input=None, context=None, config=None, params=None, state=None): stream_definition_id = params #init stuff rdt_for_nones = {} data_table_content = [] gdt_allowed_numerical_types = [ 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64', 'str' ] # TODO : Move this in to container parameter default_precision = 5 rdt = RecordDictionaryTool.load_from_granule(input) data_description = [] # Buid a local precisions and fill value dictionary to use for parsing data correctly precisions = {} fill_values = {} for field in rdt.fields: precision_str = rdt.context(field).precision if not precision_str: precisions[field] = default_precision else: try: precisions[field] = int(precision_str) except ValueError: precisions[field] = default_precision fill_values[field] = rdt.fill_value(field) if stream_definition_id == None: log.error( "GoogleDT transform: Need a output stream definition to process graphs" ) return None fields = [] fields = rdt.fields # Ascertain temporal field. Use 'time' as backup time_field = rdt.temporal_parameter or 'time' #if 'time' not in rdt: return None if rdt[time_field] is None: return None #time_fill_value = 0.0 # should be derived from the granule's param dict. time_fill_value = fill_values[ time_field] # should be derived from the granule's param dict. total_num_of_records = len(rdt[time_field]) data_description.append(('time', 'number', 'time')) ###### DEBUG ########## #for field in fields: # if hasattr(rdt.context(field),'visible'): # print " >>>>>>>> '", field, "' [visible = ", rdt.context(field).visible,"] : ", rdt[field] # else: # print " >>>>>>>> '", field, "' [visible = NOT SPECIFIED] : ", rdt[field] import re for field in fields: if field == time_field: continue # If a config block was passed, consider only the params listed in it if config and 'parameters' in config and len( config['parameters']) > 0: if not field in config['parameters']: log.info( "Skipping ", field, " since it was not present in the list of allowed parameters" ) continue # If the value is none, assign it a small one fill_value array for now to generate description, # Actual array of fill_values will be assigned later rdt_field = rdt[field] if rdt_field == None: rdt_for_nones[field] = np.array([fill_values[field]] * total_num_of_records) rdt_field = rdt_for_nones[field] # Check if visibility is false (system generated params) or not specified explicitly if hasattr(rdt.context(field), 'visible') and not rdt.context(field).visible: continue # If it's a QC parameter ignore it if field.endswith('_qc'): continue # Handle string type or if its an unknown type, convert to string context = rdt.context(field) if (rdt_field.dtype == 'string' or rdt_field.dtype not in gdt_allowed_numerical_types): data_description.append((field, 'string', field)) elif (isinstance(context.param_type, ArrayType) or isinstance( context.param_type, ParameterFunctionType)) and len(rdt_field.shape) > 1: for i in xrange(rdt_field.shape[1]): data_description.append(('%s[%s]' % (field, i), 'number', '%s[%s]' % (field, i), { 'precision': str(precisions[field]) })) else: data_description.append((field, 'number', field, { 'precision': str(precisions[field]) })) for i in xrange(len(rdt)): varTuple = [] # Put time first if its not zero. Retrieval returns 0 secs for malformed entries if rdt[time_field][i] == time_fill_value: continue # convert timestamp from instrument to UNIX time stamp since thats what Google DT expects varTuple.append(ntplib.ntp_to_system_time(rdt[time_field][i])) for dd in data_description: field = dd[0] field_type = dd[1] # ignore time since its been already added if field == None or field == time_field: continue """ rdt_field = rdt[field] if rdt_field == None: rdt_field = np.array([fill_values[field]] * total_num_of_records) """ if re.match(r'.*\[\d+\]', field): field, j = re.match(r'(.*)\[(\d+)\]', field).groups() j = int(j) varTuple.append(float(rdt[field][i][j])) else: if (field_type == 'number'): if rdt[field] == None or rdt[field][i] == fill_values[ field]: varTuple.append(None) else: # Adjust float for precision if (precisions[field] == None): varTuple.append(float(rdt[field][i])) else: varTuple.append( round(float(rdt[field][i]), precisions[field])) # if field type is string, there are two possibilities. Either it really is a string or # its an object that needs to be converted to string. if (field_type == 'string'): if rdt[field] == None or rdt[field][i] == fill_values[ field]: varTuple.append(None) else: varTuple.append(str(rdt[field][i])) # Append the tuples to the data table if len(varTuple) > 0: data_table_content.append(varTuple) out_rdt = RecordDictionaryTool( stream_definition_id=stream_definition_id) # Prepare granule content out_dict = { "viz_product_type": "google_dt", "data_description": data_description, "data_content": data_table_content } out_rdt["google_dt_components"] = np.array([out_dict]) out_rdt["viz_timestamp"] = TimeUtils.ts_to_units( rdt.context(time_field).uom, time.time()) log.debug('Google DT transform: Sending a granule') out_granule = out_rdt.to_granule() return out_granule
def execute(input=None, context=None, config=None, params=None, state=None): stream_definition_id = params #init stuff var_tuple = [] data_description = [] data_table_content = [] gdt_allowed_numerical_types = ['int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64','str'] rdt = RecordDictionaryTool.load_from_granule(input) data_description = [] if stream_definition_id == None: log.error("GoogleDT transform: Need a output stream definition to process graphs") return None fields = [] if config and config['parameters']: fields = config['parameters'] else: fields = rdt.fields # if time was null or missing, do not process if 'time' not in rdt: return None if rdt['time'] is None: return None time_fill_value = 0.0 # should be derived from the granule's param dict. data_description.append(('time','number','time')) for field in fields: if field == 'time': continue # If a config block was passed, consider only the params listed in it if config and config['parameters']: if not field in config['parameters']: continue # only consider fields which are supposed to be numbers. if (rdt[field] != None) and (rdt[field].dtype not in gdt_allowed_numerical_types): continue data_description.append((field, 'number', field)) for i in xrange(len(rdt)): varTuple = [] # Put time first if its not zero. Retrieval returns 0 secs for malformed entries if rdt['time'][i] == time_fill_value: continue # convert timestamp from instrument to UNIX time stamp since thats what Google DT expects varTuple.append(ntplib.ntp_to_system_time(rdt['time'][i])) for dd in data_description: field = dd[0] # ignore time since its been already added if field == None or field == 'time': continue if rdt[field] == None or rdt[field][i] == None: varTuple.append(0.0) else: varTuple.append(rdt[field][i]) # Append the tuples to the data table if len(varTuple) > 0: data_table_content.append(varTuple) out_rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id) # Prepare granule content out_dict = {"viz_product_type" : "google_dt", "data_description" : data_description, "data_content" : data_table_content} out_rdt["google_dt_components"] = np.array([out_dict]) log.debug('Google DT transform: Sending a granule') out_granule = out_rdt.to_granule() return out_granule
def execute(input=None, context=None, config=None, params=None, state=None): stream_definition_id = params #init stuff var_tuple = [] data_description = [] data_table_content = [] gdt_allowed_numerical_types = ['int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64','str'] # TODO : Move this in to container parameter default_precision = 5 rdt = RecordDictionaryTool.load_from_granule(input) data_description = [] # Buid a local precisions and fill value dictionary to use for parsing data correctly precisions = {} fill_values = {} for field in rdt._pdict: _precision_str = rdt._pdict.get_context(field).precision if _precision_str == None or _precision_str == '': precisions[field] = default_precision else: precisions[field] = int(_precision_str) _fv_str = rdt._pdict.get_context(field).fill_value if _fv_str == None or _fv_str == '': fill_values[field] = None else: fill_values[field] = int(_fv_str) if stream_definition_id == None: log.error("GoogleDT transform: Need a output stream definition to process graphs") return None fields = [] if config and config['parameters']: fields = config['parameters'] else: fields = rdt.fields # if time was null or missing, do not process if 'time' not in rdt: return None if rdt['time'] is None: return None time_fill_value = 0.0 # should be derived from the granule's param dict. data_description.append(('time','number','time')) for field in fields: if field == 'time': continue # If a config block was passed, consider only the params listed in it if config and config['parameters']: if not field in config['parameters']: continue # only consider fields which are allowed. if rdt[field] == None: continue if (rdt[field] != None) and (rdt[field].dtype not in gdt_allowed_numerical_types): print ">>>>>>>>>>>>>> DONT KNOW HOW TO HANDLE : ", rdt[field].dtype continue if (rdt[field].dtype == 'string'): data_description.append((field, 'string', field )) else: data_description.append((field, 'number', field, {'precision':str(precisions[field])} )) for i in xrange(len(rdt)): varTuple = [] # Put time first if its not zero. Retrieval returns 0 secs for malformed entries if rdt['time'][i] == time_fill_value: continue # convert timestamp from instrument to UNIX time stamp since thats what Google DT expects varTuple.append(ntplib.ntp_to_system_time(rdt['time'][i])) for dd in data_description: field = dd[0] field_type = dd[1] # ignore time since its been already added if field == None or field == 'time': continue if rdt[field] == None or rdt[field][i] == None: varTuple.append(None) else: if(field_type == 'number'): if rdt[field][i] == None or rdt[field][i] == fill_values[field]: varTuple.append(None) else: # Adjust float for precision if (precisions[field] == None): varTuple.append(float(rdt[field][i])) else: varTuple.append(round(float(rdt[field][i]), precisions[field])) else: varTuple.append(rdt[field][i]) # Append the tuples to the data table if len(varTuple) > 0: data_table_content.append(varTuple) out_rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id) # Prepare granule content out_dict = {"viz_product_type" : "google_dt", "data_description" : data_description, "data_content" : data_table_content} out_rdt["google_dt_components"] = np.array([out_dict]) log.debug('Google DT transform: Sending a granule') out_granule = out_rdt.to_granule() return out_granule
def execute(input=None, context=None, config=None, params=None, state=None): stream_definition_id = params #init stuff rdt_for_nones = {} hc_data = [] normalized_ts = [] hc_allowed_numerical_types = ['int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64','str'] # TODO : Move this in to container parameter default_precision = 5 rdt = RecordDictionaryTool.load_from_granule(input) # Buid a local precisions and fill value dictionary to use for parsing data correctly precisions = {} fill_values = {} for field in rdt.fields: precision_str = rdt.context(field).precision if not precision_str: precisions[field] = default_precision else: try: precisions[field] = int(precision_str) except ValueError: precisions[field] = default_precision fill_values[field] = rdt.fill_value(field) if stream_definition_id == None: log.error("HighCharts transform: Need a output stream definition to process graphs") return None fields = [] fields = rdt.fields # Ascertain temporal field. Use 'time' as backup time_field = rdt.temporal_parameter or 'time' #if 'time' not in rdt: return None if rdt[time_field] is None: return None #time_fill_value = 0.0 # should be derived from the granule's param dict. time_fill_value = fill_values[time_field] # should be derived from the granule's param dict. total_num_of_records = len(rdt[time_field]) # convert timestamps from ntp to system count = 0 normalized_ts = [None] * total_num_of_records for ts in rdt[time_field]: if ts == time_fill_value: normalized_ts[count] = time_fill_value else: normalized_ts[count] = float(ntplib.ntp_to_system_time(ts)) count += 1 ###### DEBUG ########## #for field in fields: # if hasattr(rdt.context(field),'visible'): # print " >>>>>>>> '", field, "' [visible = ", rdt.context(field).visible,"] : ", rdt[field] # else: # print " >>>>>>>> '", field, "' [visible = NOT SPECIFIED] : ", rdt[field] # Convert the fields in to HC series format import re for field in fields: field_precision = precisions[field] if field == time_field: continue # If a config block was passed, consider only the params listed in it if config and 'parameters' in config and len(config['parameters']) > 0: if not field in config['parameters']: log.info("Skipping ", field, " since it was not present in the list of allowed parameters") continue # If the value is none, assign it a small one fill_value array for now to generate description, # Actual array of fill_values will be assigned later rdt_field =rdt[field] if rdt_field == None: rdt_for_nones[field] = np.array([fill_values[field]] * total_num_of_records) #rdt_for_nones[field] = [fill_values[field]] * total_num_of_records rdt_field = rdt_for_nones[field] # Check if visibility is false (system generated params) or not specified explicitly if hasattr(rdt.context(field),'visible') and not rdt.context(field).visible: continue # If it's a QC parameter ignore it if field.endswith('_qc'): continue # NOTE: The reason why the following libes of code seem to make a lot of branches are to pull the major decisions # outside the primary loops responsible for arranging data in the data structures to be sent to the charts. # This is better than having to make the decisions on a per record entry. # Handle arrays by spliting them them in to individual parameters context = rdt.context(field) if (isinstance(context.param_type, ArrayType) or isinstance(context.param_type,ParameterFunctionType)) and len(rdt_field.shape)>1: if (rdt_field.dtype == 'string' or rdt_field.dtype not in hc_allowed_numerical_types): for i in xrange(rdt_field.shape[1]): series = {} series["name"] = field + "[" + str(i) + "]" series["visible"] = False series["data"] = VizTransformHighChartsAlgorithm.form_series_data_str(normalized_ts, rdt_field, i, time_fill_value, fill_values[field]) hc_data.append(series) else: # Assume its a float or number for i in xrange(rdt_field.shape[1]): series = {} series["name"] = field + "[" + str(i) + "]" series["visible"] = True series["tooltip"] = {"valueDecimals":field_precision} series["data"] = VizTransformHighChartsAlgorithm.form_series_data_num(normalized_ts, rdt_field, i, time_fill_value, fill_values[field], field_precision) hc_data.append(series) else: if (rdt_field.dtype == 'string' or rdt_field.dtype not in hc_allowed_numerical_types): series = {} series["name"] = field series["visible"] = False series["data"] = VizTransformHighChartsAlgorithm.form_series_data_str(normalized_ts, rdt_field, None, time_fill_value, fill_values[field]) else: series = {} series["name"] = field series["tooltip"] = {"valueDecimals":field_precision} series["visible"] = True series["data"] = VizTransformHighChartsAlgorithm.form_series_data_num(normalized_ts, rdt_field, None, time_fill_value, fill_values[field], field_precision) # Append series to the hc data hc_data.append(series) # Prep the outgoing granule out_rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id) #out_dict = {"hc_data": hc_data} #out_rdt["hc_data"] = np.array([out_dict]) out_rdt["hc_data"] = [hc_data] out_rdt["viz_timestamp"] = TimeUtils.ts_to_units(rdt.context(time_field).uom, time.time()) log.debug('HighCharts Transform: Sending a granule') out_granule = out_rdt.to_granule() return out_granule
def _extract_transmit_year(self) -> int: year = datetime.fromtimestamp(ntplib.ntp_to_system_time(self.ntp().sent)).year return year
def ntp_to_datestring(ntp_time): try: ntp_time = float(ntp_time) return datetime.datetime.utcfromtimestamp(ntplib.ntp_to_system_time(ntp_time)).strftime("%Y-%m-%d %H:%M:%S") except: return str(ntp_time)
def ntptime_to_string(t): t = ntplib.ntp_to_system_time(t) millis = '%f' % (t-int(t)) millis = millis[1:5] return time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime(t)) + millis + 'Z'
def ntptime_to_string(t): t = ntplib.ntp_to_system_time(t) millis = '%f' % (t - int(t)) millis = millis[1:5] return time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime(t)) + millis + 'Z'