def test_remove_noncontiguous(self): tb1 = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) tb2 = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) tb3 = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) # nonzero signal mean tsd1 = TimeseriesData(timebase=tb1, signal=Signal(np.arange(len(tb1))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0))))) tsd2 = TimeseriesData(timebase=tb2, signal=Signal(np.arange(len(tb2))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0))))) tsd3 = TimeseriesData(timebase=tb3, signal=Signal(np.arange(len(tb3))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0))))) self.assertTrue(tb1.is_contiguous()) self.assertTrue(tb2.is_contiguous()) self.assertTrue(tb3.is_contiguous()) tsd2.timebase[-50:] += 1.0 self.assertFalse(tb2.is_contiguous()) ds = DataSet('ds') for tsd in [tsd1, tsd2, tsd3]: ds.add(tsd) for tsd in [tsd1, tsd2, tsd3]: self.assertTrue(tsd in ds) filtered_ds = ds.remove_noncontiguous() for tsd in [tsd1, tsd3]: self.assertTrue(tsd in filtered_ds) self.assertFalse(tsd2 in filtered_ds)
def test_n_channels(self): test_sig_1a = Signal(np.random.rand(10)) self.assertEqual(test_sig_1a.n_channels(), 1) test_sig_1b = Signal(np.random.rand(1,10)) self.assertEqual(test_sig_1b.n_channels(), 1) test_sig_2 = Signal(np.random.rand(2,10)) self.assertEqual(test_sig_2.n_channels(), 2)
def test_n_samples(self): test_sig_1a = Signal(np.random.rand(10)) self.assertEqual(test_sig_1a.n_samples(), 10) test_sig_1b = Signal(np.random.rand(1,10)) self.assertEqual(test_sig_1b.n_samples(), 10) test_sig_2 = Signal(np.random.rand(2,10)) self.assertEqual(test_sig_2.n_samples(), 10)
def test_reduce_time_dataset(self): new_times = [-0.25, 0.25] tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) tsd_1 = TimeseriesData(timebase=tb, signal=Signal(np.resize(np.arange(5*len(tb)),(5,len(tb)))), channels=get_n_channels(5)) tsd_2 = TimeseriesData(timebase=tb, signal=Signal(np.resize(np.arange(5*len(tb))+1,(5,len(tb)))), channels=get_n_channels(5)) test_dataset = DataSet('test_dataset') test_dataset.add(tsd_1) test_dataset.add(tsd_2) test_dataset.reduce_time(new_times)
def test_dataset(self): tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) tsd_1 = TimeseriesData(timebase=tb, signal=Signal(np.resize(np.arange(3*len(tb)), (3,len(tb)))), channels=get_n_channels(3)) tsd_2 = TimeseriesData(timebase=tb, signal=Signal(np.resize(np.arange(3*len(tb)+1),(3,len(tb)))), channels=get_n_channels(3)) input_dataset = DataSet('test_dataset') input_dataset.add(tsd_1) input_dataset.add(tsd_2) seg_dataset = input_dataset.segment(n_samples=10) self.assertTrue(len(seg_dataset)==20)
def test_dataset(self): ch=get_n_channels(5) new_times = [-0.25, 0.25] tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) tsd_1 = TimeseriesData(timebase=tb, signal=Signal(np.resize(np.arange(5*len(tb)),(5,len(tb)))), channels=ch) tsd_2 = TimeseriesData(timebase=tb, signal=Signal(np.resize(np.arange(5*len(tb))+1, (5,len(tb)))), channels=ch) test_dataset = DataSet('test_ds_1') test_dataset.add(tsd_1) test_dataset.add(tsd_2) self.assertTrue(tsd_1 in test_dataset) """
def test_multi_channel_timeseries(self): tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) tsd = TimeseriesData(timebase=tb, signal=Signal(np.resize(np.arange(3*len(tb)), (3,len(tb)))), channels=get_n_channels(3)) seg_dataset = tsd.segment(n_samples=10) self.assertTrue(len(seg_dataset)==10)
def do_fetch(self): chan_name = (self.diag_name.split('-'))[-1] # remove - filename_dict = {'diag_name': chan_name, 'shot': self.shot} self.basename = path.join(pf.config.get('global', 'localdatapath'), data_filename % filename_dict) files_exist = path.exists(self.basename) if not files_exist: raise Exception, "file " + self.basename + " not found." else: signal_dict = newload(self.basename) if ((chan_name == array(['MP5', 'HMP13', 'HMP05'])).any()): flip = -1. else: flip = 1. if self.diag_name[0] == '-': flip = -flip # coords = get_coords_for_channel(**self.__dict__) ch = Channel(self.diag_name, Coords('dummy', (0, 0, 0))) output_data = TimeseriesData( timebase=Timebase(signal_dict['timebase']), signal=Signal(flip * signal_dict['signal']), channels=ch) output_data.meta.update({'shot': self.shot}) return output_data
def test_remove_mean_single_channel(self): tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) # nonzero signal mean tsd = TimeseriesData(timebase=tb, signal=Signal(np.arange(len(tb))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0))))) filtered_tsd = tsd.subtract_mean() assert_almost_equal(np.mean(filtered_tsd.signal), 0)
def do_fetch(self): print self.shot, self.senal data_dim = tjiidata.dimens(self.shot, self.senal) if data_dim[0] < MAX_SIGNAL_LENGTH: data_dict = tjiidata.lectur(self.shot, self.senal, data_dim[0], data_dim[0], data_dim[1]) else: raise ValueError, 'Not loading data to avoid segmentation fault in tjiidata.lectur' ch = Channel(self.senal, Coords('dummy', (0, 0, 0))) if self.invert == 'true': #yuk - TODO: use boolean type from config s = Signal(-np.array(data_dict['y'])) else: s = Signal(np.array(data_dict['y'])) output_data = TimeseriesData(timebase=Timebase(data_dict['x']), signal=s, channels=ch) output_data.meta.update({'shot': self.shot}) return output_data
def test_reduce_time_filter_multi_channel_attached_method(self): new_times = [-0.25, 0.25] tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) tsd = TimeseriesData(timebase=tb, signal=Signal(np.resize(np.arange(5*len(tb)),(5,len(tb)))), channels=get_n_channels(5)) new_time_args = np.searchsorted(tb, new_times) timebase_test = tsd.timebase[new_time_args[0]:new_time_args[1]].copy() signal_test = tsd.signal[:,new_time_args[0]:new_time_args[1]].copy() reduced_tsd = tsd.reduce_time(new_times) self.assertTrue(isinstance(reduced_tsd, TimeseriesData)) assert_array_almost_equal(reduced_tsd.timebase, timebase_test) assert_array_almost_equal(reduced_tsd.signal, signal_test)
def get_probe_angles(input_data, closed=False): """ return a list of thetas for a given signal (timeseries) or a string that specifies it. get_probe_angles('W7X:W7X_MIRNOV_41_BEST_LOOP:(20180912,43)') This is a kludgey way to read coordinates. Should be through acquisition.base or acquisition.'device' rather than looking up config directly """ import pyfusion if isinstance(input_data, str): pieces = input_data.split(':') if len(pieces) == 3: dev_name, diag_name, shotstr = pieces shot_number = eval(shotstr) dev = pyfusion.getDevice(dev_name) data = dev.acq.getdata(shot_number, diag_name, time_range=[0, 0.1]) else: from pyfusion.data.timeseries import TimeseriesData, Timebase, Signal from pyfusion.data.base import Channel, ChannelList, Coords input_data = TimeseriesData(Timebase([0, 1]), Signal([0, 1])) dev_name, diag_name = pieces # channels are amongst options opts = pyfusion.config.pf_options('Diagnostic', diag_name) chans = [ pyfusion.config.pf_get('Diagnostic', diag_name, opt) for opt in opts if 'channel_' in opt ] # for now, assume config_name is some as name input_data.channels = ChannelList( *[Channel(ch, Coords('?', [0, 0, 0])) for ch in chans]) Phi = np.array([ 2 * np.pi / 360 * float( pyfusion.config.get( 'Diagnostic:{cn}'.format( cn=c.config_name if c.config_name != '' else c.name), 'Coords_reduced').split(',')[0]) for c in input_data.channels ]) Theta = np.array([ 2 * np.pi / 360 * float( pyfusion.config.get( 'Diagnostic:{cn}'.format( cn=c.config_name if c.config_name != '' else c.name), 'Coords_reduced').split(',')[1]) for c in input_data.channels ]) if closed: Phi = np.append(Phi, Phi[0]) Theta = np.append(Theta, Theta[0]) return (dict(Theta=Theta, Phi=Phi))
def do_fetch(self): channel_length = int(self.length) outdata = np.zeros(1024 * 2 * 256 + 1) ## !! really should put a wrapper around gethjdata to do common stuff # outfile is only needed if the direct passing of binary won't work # with tempfile.NamedTemporaryFile(prefix="pyfusion_") as outfile: # get in two steps to make debugging easier allrets = gethjdata.gethjdata(self.shot, channel_length, self.path, verbose=VERBOSE, opt=1, ierror=2, isample=-1, outdata=outdata, outname='') ierror, isample, getrets = allrets if ierror != 0: raise LookupError( 'hj Okada style data not found for {s}:{c}'.format( s=self.shot, c=self.path)) ch = Channel(self.path, Coords('dummy', (0, 0, 0))) # the intent statement causes the out var to be returned in the result lsit # looks like the time,data is interleaved in a 1x256 array # it is fed in as real*64, but returns as real*32! (as per fortran decl) debug_(pyfusion.DEBUG, 4, key='Heliotron_fetch', msg='after call to getdata') # timebase in secs (ms in raw data) - could add a preferred unit? # this is partly allowed for in savez_compressed, newload, and # for plotting, in the config file. # important that the 1e-3 be inside the Timebase() output_data = TimeseriesData(timebase=Timebase( 1e-3 * getrets[1::2][0:isample]), signal=Signal(getrets[2::2][0:isample]), channels=ch) output_data.meta.update({'shot': self.shot}) if pyfusion.VERBOSE > 0: print('HJ config name', self.config_name) output_data.config_name = self.config_name stprms = get_static_params(shot=self.shot, signal=self.path) if len(list(stprms) ) == 0: # maybe this should be ignored - how do we use it? raise LookupError( ' failure to get params for {shot}:{path}'.format( shot=self.shot, path=self.path)) output_data.params = stprms return output_data
def do_fetch(self): channel_length = int(self.length) outdata=np.zeros(1024*2*256+1) with tempfile.NamedTemporaryFile(prefix="pyfusion_") as outfile: getrets=gethjdata.gethjdata(self.shot,channel_length,self.path, VERBOSE, OPT, outfile.name, outdata) ch = Channel(self.path, Coords('dummy', (0,0,0))) output_data = TimeseriesData(timebase=Timebase(getrets[1::2]), signal=Signal(getrets[2::2]), channels=ch) output_data.meta.update({'shot':self.shot}) return output_data
def do_fetch(self): delimiter = self.__dict__.get("delimiter", None) data = genfromtxt(self.filename.replace("(shot)", str(self.shot)), unpack=True, delimiter=delimiter) # len(data) is number of channels + 1 (timebase) n_channels = len(data) - 1 ch_generator = (generic_ch(i) for i in range(n_channels)) ch = ChannelList(*ch_generator) return TimeseriesData(timebase=Timebase(data[0]), signal=Signal(data[1:]), channels=ch)
def do_fetch(self): print(self.pointname) print(self.shot) if self.NC!=None: print(self.NC) t_name = '{}_time'.format(self.pointname) NC_vars = self.NC.variables.keys() if self.pointname in NC_vars: print('Reading cache!!!!') t_axis = self.NC.variables[t_name].data[:].copy() data = self.NC.variables[self.pointname].data[:].copy() else: tmp = self.acq.connection.get('ptdata2("{}",{})'.format(self.pointname, self.shot)) data = tmp.data() tmp = self.acq.connection.get('dim_of(ptdata2("{}",{}))'.format(self.pointname, self.shot)) t_axis = tmp.data() self.write_cache = True print(t_axis) print(data) coords = get_coords_for_channel(**self.__dict__) ch = Channel(self.pointname, coords) # con=MDS.Connection('atlas.gat.com::') # pointname = 'MPI66M067D' # shot = 164950 # tmp = con.get('ptdata2("{}",{})'.format(pointname, shot)) # dat = tmp.data() # tmp = con.get('dim_of(ptdata2("{}",{}))'.format(pointname, shot)) # t = tmp.data() if self.NC!=None and self.write_cache: print self.pointname self.NC.createDimension(t_name, len(t_axis)) f_time = self.NC.createVariable(t_name,'d',(t_name,)) f_time[:] = +t_axis print('Wrote time') sig = self.NC.createVariable(self.pointname,'f',(t_name,)) sig[:] = +data print('Wrote signal') output_data = TimeseriesData(timebase=Timebase(t_axis), signal=Signal(data), channels=ch) # output_data = super(DIIIDDataFetcherPTdata, self).do_fetch() # coords = get_coords_for_channel(**self.__dict__) # ch = Channel(self.mds_path, coords) # output_data.channels = ch # output_data.meta.update({'shot':self.shot, 'kh':self.get_kh()}) # print(ch) output_data.config_name = ch self.fetch_mode = 'ptdata' return output_data
def get_multimode_test_data(channels = get_n_channels(DEFAULT_N_CHANNELS), timebase = DEFAULT_TIMEBASE, modes = [mode_1, mode_2], noise = DEFAULT_NOISE): """Generate synthetic multi-channel data for testing.""" n_channels = len(channels) data_size = (n_channels, timebase.size) data_array = noise*2*(np.random.random(data_size)-0.5) timebase_matrix = np.resize(timebase, data_size) angle_matrix = np.resize(np.array([i.coords.cylindrical[1] for i in channels]), data_size[::-1]).T for m in modes: data_array += m['amp']*np.cos(2*np.pi*m['freq']*timebase_matrix + m['mode_number']*angle_matrix + m['phase']) output = TimeseriesData(timebase=timebase, signal=Signal(data_array), channels=channels) return output
def testFilteredDataHistory_nocopy(self): tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) # nonzero signal mean ch = get_n_channels(1) tsd = TimeseriesData(timebase=tb, signal=Signal(np.arange(len(tb))), channels=ch) filtered_tsd = tsd.subtract_mean() #self.assertEqual(len(filtered_tsd.history.split('\n')), 3) self.assertEqual(len(filtered_tsd.history.split('\n')), 5) # bdb thinks extra 2 is OK output_data = filtered_tsd.normalise(method='rms', copy=False) #self.assertEqual(filtered_tsd.history.split('> ')[-1], "normalise(method='rms')") self.assertEqual(filtered_tsd.history.split('> ')[-1].split('\n')[0], "normalise(method='rms')") #self.assertEqual(output_data.history.split('> ')[-1], "normalise(method='rms')") self.assertEqual(output_data.history.split('> ')[-1].split('\n')[0], "normalise(method='rms')")
def fetch(self, interp_if_diff = True): """Fetch each channel and combine into a multichannel instance of :py:class:`~pyfusion.data.timeseries.TimeseriesData`. :rtype: :py:class:`~pyfusion.data.timeseries.TimeseriesData` """ print('******** hello world ***********') ## initially, assume only single channel signals ordered_channel_names = self.ordered_channel_names() data_list = [] channels = ChannelList() timebase = None meta_dict={} from scipy.io import netcdf fname = '/u/haskeysr/tmp/{}.nc'.format(self.shot) write_cache=False; read_cache=False if os.path.exists(fname): NC = netcdf.netcdf_file(fname,'a',version=2) else: NC = netcdf.netcdf_file(fname,'w',version=2) for chan in ordered_channel_names: fetcher_class = import_setting('Diagnostic', chan, 'data_fetcher') tmp_data = fetcher_class(self.acq, self.shot, config_name=chan, NC=NC).fetch() channels.append(tmp_data.channels) meta_dict.update(tmp_data.meta) if timebase == None: timebase = tmp_data.timebase data_list.append(tmp_data.signal) else: try: assert_array_almost_equal(timebase, tmp_data.timebase) data_list.append(tmp_data.signal) except: if interp_if_diff: data_list.append(np.interp(timebase, tmp_data.timebase, tmp_data.signal)) else: raise NC.close() signal=Signal(data_list) output_data = TimeseriesData(signal=signal, timebase=timebase, channels=channels) #output_data.meta.update({'shot':self.shot}) output_data.meta.update(meta_dict) return output_data
def testFilteredDataHistory_copy(self): """ make sure that _copy version does NOT alter original """ tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) # nonzero signal mean ch = get_n_channels(1) tsd = TimeseriesData(timebase=tb, signal=Signal(np.arange(len(tb))), channels=ch) filtered_tsd = tsd.subtract_mean() # bdb in 4 places, assume that the xtra info (norm_value) is supposed to be there. #self.assertEqual(len(filtered_tsd.history.split('\n')), 3) self.assertEqual(len(filtered_tsd.history.split('\n')), 5) # bdb thinks extra 2 is OK output_data = filtered_tsd.normalise(method='rms', copy=True) #self.assertEqual(output_data.history.split('> ')[-1], "normalise(method='rms')") self.assertEqual(output_data.history.split('> ')[-1].split('\n')[0], "normalise(method='rms')") #self.assertEqual(filtered_tsd.history.split('> ')[-1], "subtract_mean()") self.assertEqual(filtered_tsd.history.split('> ')[-1].split('\n')[0], "subtract_mean()")
def do_fetch(self): print("Shot: {}\nPoint Name: {}".format(self.shot, self.pointname)) if not hasattr(self, 'NC'): self.NC = None if self.NC is not None: #print(self.NC) t_name = '{}_time'.format(self.pointname) NC_vars = self.NC.variables.keys() else: NC_vars = [] if self.pointname in NC_vars: print(' Pointname in NC cache, Reading...\n') t_axis = self.NC.variables[t_name].data[:].copy() data = self.NC.variables[self.pointname].data[:].copy() self.write_cache = False else: print(' Fetching from ptdata') tmp = self.acq.connection.get('ptdata2("{}",{})'.format( self.pointname, self.shot)) data = tmp.data() tmp = self.acq.connection.get('dim_of(ptdata2("{}",{}))'.format( self.pointname, self.shot)) t_axis = tmp.data() self.write_cache = True coords = get_coords_for_channel(**self.__dict__) ch = Channel(self.pointname, coords) if self.NC is not None and self.write_cache: print("\t Writing to NC file disabled temporarily.") #print(' Writing pointname to NC file\n') #self.NC.createDimension(t_name, len(t_axis)) #f_time = self.NC.createVariable(t_name,'d',(t_name,)) #f_time[:] = +t_axis # sig = self.NC.createVariable(self.pointname,'f',(t_name,)) #sig[:] = +data output_data = TimeseriesData(timebase=Timebase(t_axis), signal=Signal(data), channels=ch) output_data.config_name = ch self.fetch_mode = 'ptdata' return output_data
def do_fetch(self): sig = self.conn.get(self.mds_path) dim = self.conn.get('DIM_OF(' + self.mds_path + ')') scl = 1.0 coords = get_coords_for_channel(**self.__dict__) ch = Channel(self.config_name, coords) timedata = dim.data() output_data = TimeseriesData(timebase=Timebase(1e-9 * timedata), signal=scl * Signal(sig), channels=ch) output_data.meta.update({'shot': self.shot}) if hasattr(self, 'mdsshot'): # intended for checks - not yet used. output_data.mdsshot = self.mdsshot output_data.config_name = self.config_name output_data.utc = [timedata[0], timedata[-1]] #output_data.units = dat['units'] if 'units' in dat else '' debug_(pyfusion.DEBUG, level=1, key='W7M_do_fetch', msg='entering W7X MDS do_fetch') return (output_data)