def test_remove_noncontiguous(self): tb1 = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) tb2 = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) tb3 = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) # nonzero signal mean tsd1 = TimeseriesData(timebase=tb1, signal=Signal(np.arange(len(tb1))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0))))) tsd2 = TimeseriesData(timebase=tb2, signal=Signal(np.arange(len(tb2))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0))))) tsd3 = TimeseriesData(timebase=tb3, signal=Signal(np.arange(len(tb3))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0))))) self.assertTrue(tb1.is_contiguous()) self.assertTrue(tb2.is_contiguous()) self.assertTrue(tb3.is_contiguous()) tsd2.timebase[-50:] += 1.0 self.assertFalse(tb2.is_contiguous()) ds = DataSet('ds') for tsd in [tsd1, tsd2, tsd3]: ds.add(tsd) for tsd in [tsd1, tsd2, tsd3]: self.assertTrue(tsd in ds) filtered_ds = ds.remove_noncontiguous() for tsd in [tsd1, tsd3]: self.assertTrue(tsd in filtered_ds) self.assertFalse(tsd2 in filtered_ds)
def fetch(self): """Fetch each channel and combine into a multichannel instance of :py:class:`~pyfusion.data.timeseries.TimeseriesData`. :rtype: :py:class:`~pyfusion.data.timeseries.TimeseriesData` """ ## initially, assume only single channel signals # this base debug breakpoint will apply to all flavours of acquisition debug_(pyfusion.DEBUG, level=2, key='base_multi_fetch') ordered_channel_names = self.ordered_channel_names() data_list = [] channels = ChannelList() timebase = None meta_dict={} if hasattr(self, 't_min') and hasattr(self, 't_max'): t_range = [float(self.t_min), float(self.t_max)] else: t_range = [] for chan in ordered_channel_names: sgn = 1 if chan[0]=='-': sgn = -sgn bare_chan = (chan.split('-'))[-1] fetcher_class = import_setting('Diagnostic', bare_chan, 'data_fetcher') tmp_data = fetcher_class(self.acq, self.shot, config_name=bare_chan).fetch() if len(t_range) == 2: tmp_data = tmp_data.reduce_time(t_range) channels.append(tmp_data.channels) # two tricky things here - tmp.data.channels only gets one channel hhere #config_name for a channel is attached to the multi part - #we need to move it to the particular channel # was channels[-1].config_name = chan channels[-1].config_name = tmp_data.config_name meta_dict.update(tmp_data.meta) #print(tmp_data.signal[-1], sgn) tmp_data.signal = sgn * tmp_data.signal #print(tmp_data.signal[-1], sgn) if timebase == None: timebase = tmp_data.timebase data_list.append(tmp_data.signal) else: if hasattr(self, 'skip_timebase_check') and self.skip_timebase_check == 'true': data_list.append(tmp_data.signal) else: try: assert_array_almost_equal(timebase, tmp_data.timebase) data_list.append(tmp_data.signal) except: raise signal=Signal(data_list) output_data = TimeseriesData(signal=signal, timebase=timebase, channels=channels) #output_data.meta.update({'shot':self.shot}) output_data.meta.update(meta_dict) return output_data
def fetch(self, interp_if_diff=True): """Fetch each channel and combine into a multichannel instance of :py:class:`~pyfusion.data.timeseries.TimeseriesData`. :rtype: :py:class:`~pyfusion.data.timeseries.TimeseriesData` """ ## initially, assume only single channel signals ordered_channel_names = self.ordered_channel_names() data_list = [] channels = ChannelList() timebase = None meta_dict = {} #from scipy.io import netcdf #home = os.environ['HOME'] #os.system('mkdir -p {}/tmp_pyfusion/'.format(home)) #fname = '{}/tmp_pyfusion/{}.nc'.format(home,self.shot) #if os.path.exists(fname): # NC = netcdf.netcdf_file(fname,'r',version=2) #else: # NC = netcdf.netcdf_file(fname,'w',version=2) for chan in ordered_channel_names: fetcher_class = import_setting('Diagnostic', chan, 'data_fetcher') tmp_data = fetcher_class(self.acq, self.shot, config_name=chan).fetch() channels.append(tmp_data.channels) meta_dict.update(tmp_data.meta) if timebase is None: timebase = tmp_data.timebase data_list.append(tmp_data.signal) else: try: assert_array_almost_equal(timebase, tmp_data.timebase) data_list.append(tmp_data.signal) except: if interp_if_diff: data_list.append( np.interp(timebase, tmp_data.timebase, tmp_data.signal)) else: raise #NC.close() signal = Signal(data_list) output_data = TimeseriesData(signal=signal, timebase=timebase, channels=channels) #output_data.meta.update({'shot':self.shot}) output_data.meta.update(meta_dict) return output_data
def test_channel_list(self): ch01 = Channel('test_1', Coords('dummy', (0,0,0))) ch02 = Channel('test_2', Coords('dummy', (0,0,0))) ch03 = Channel('test_3', Coords('dummy', (0,0,0))) new_cl = ChannelList([ch01, ch02, ch03])
def test_timebase_and_coords(self): n_ch = 10 n_samples = 1024 timebase = Timebase(np.arange(n_samples)*1.e-6) channels = ChannelList(*(Channel('ch_%d' %i, Coords('cylindrical',(1.0,i,0.0))) for i in 2*np.pi*np.arange(n_ch)/n_ch)) multichannel_data = get_multimode_test_data(channels = channels, timebase = timebase, noise = 0.5)
def fetch(self, interp_if_diff = True): """Fetch each channel and combine into a multichannel instance of :py:class:`~pyfusion.data.timeseries.TimeseriesData`. :rtype: :py:class:`~pyfusion.data.timeseries.TimeseriesData` """ print('******** hello world ***********') ## initially, assume only single channel signals ordered_channel_names = self.ordered_channel_names() data_list = [] channels = ChannelList() timebase = None meta_dict={} from scipy.io import netcdf fname = '/u/haskeysr/tmp/{}.nc'.format(self.shot) write_cache=False; read_cache=False if os.path.exists(fname): NC = netcdf.netcdf_file(fname,'a',version=2) else: NC = netcdf.netcdf_file(fname,'w',version=2) for chan in ordered_channel_names: fetcher_class = import_setting('Diagnostic', chan, 'data_fetcher') tmp_data = fetcher_class(self.acq, self.shot, config_name=chan, NC=NC).fetch() channels.append(tmp_data.channels) meta_dict.update(tmp_data.meta) if timebase == None: timebase = tmp_data.timebase data_list.append(tmp_data.signal) else: try: assert_array_almost_equal(timebase, tmp_data.timebase) data_list.append(tmp_data.signal) except: if interp_if_diff: data_list.append(np.interp(timebase, tmp_data.timebase, tmp_data.signal)) else: raise NC.close() signal=Signal(data_list) output_data = TimeseriesData(signal=signal, timebase=timebase, channels=channels) #output_data.meta.update({'shot':self.shot}) output_data.meta.update(meta_dict) return output_data
def test_channellist_ORM(self): ch01 = Channel('test_1', Coords('dummy', (0,0,0))) ch02 = Channel('test_2', Coords('dummy', (0,0,0))) ch03 = Channel('test_3', Coords('dummy', (0,0,0))) new_cl = ChannelList(ch03, ch01, ch02) new_cl.save() # get our channellist if pyfusion.orm_manager.IS_ACTIVE: session = pyfusion.orm_manager.Session() our_channellist = session.query(ChannelList).order_by("id").first() self.assertEqual(our_channellist[0].name, 'test_3') self.assertEqual(our_channellist[1].name, 'test_1') self.assertEqual(our_channellist[2].name, 'test_2')
def test_remove_mean_single_channel(self): tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2) # nonzero signal mean tsd = TimeseriesData(timebase=tb, signal=Signal(np.arange(len(tb))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0))))) filtered_tsd = tsd.subtract_mean() assert_almost_equal(np.mean(filtered_tsd.signal), 0)
def test_dataset_filter_nocopy(self): n_ch = 10 n_samples = 640 timebase = Timebase(np.arange(n_samples)*1.e-6) channels = ChannelList(*(Channel('ch_%d' %i, Coords('cylindrical',(1.0,i,0.0))) for i in 2*np.pi*np.arange(n_ch)/n_ch)) multichannel_data = get_multimode_test_data(channels = channels, timebase = timebase, noise = 0.5) dataset = multichannel_data.segment(64, copy=False) new_dataset = dataset.segment(16, copy=False)
def test_timeseries_filter_nocopy(self): # Use reduce_time filter for testing... n_ch = 10 n_samples = 5000 timebase = Timebase(np.arange(n_samples)*1.e-6) channels = ChannelList(*(Channel('ch_%d' %i, Coords('cylindrical',(1.0,i,0.0))) for i in 2*np.pi*np.arange(n_ch)/n_ch)) multichannel_data = get_multimode_test_data(channels = channels, timebase = timebase, noise = 0.5) new_data = multichannel_data.reduce_time([0,1.e-3], copy=False) self.assertTrue(new_data is multichannel_data)
def fetch(self): tb = generate_timebase(t0=float(self.t0), n_samples=int(self.n_samples), sample_freq=float(self.sample_freq)) sig = Signal( float(self.amplitude) * sin(2 * pi * float(self.frequency) * tb)) dummy_channel = Channel('ch_01', Coords('dummy', (0, 0, 0))) output_data = TimeseriesData(timebase=tb, signal=sig, channels=ChannelList(dummy_channel)) output_data.meta.update({'shot': self.shot}) return output_data
def fetch(self, interp_if_diff=True): """Fetch each channel and combine into a multichannel instance of :py:class:`~pyfusion.data.timeseries.TimeseriesData`. :rtype: :py:class:`~pyfusion.data.timeseries.TimeseriesData` """ ## initially, assume only single channel signals ordered_channel_names = self.ordered_channel_names() data_list = [] channels = ChannelList() timebase = None meta_dict = {} for chan in ordered_channel_names: fetcher_class = import_setting('Diagnostic', chan, 'data_fetcher') tmp_data = fetcher_class(self.acq, self.shot, config_name=chan).fetch() channels.append(tmp_data.channels) meta_dict.update(tmp_data.meta) if timebase == None: timebase = tmp_data.timebase data_list.append(tmp_data.signal) else: try: assert_array_almost_equal(timebase, tmp_data.timebase) data_list.append(tmp_data.signal) except: if interp_if_diff: data_list.append( np.interp(timebase, tmp_data.timebase, tmp_data.signal)) else: raise signal = Signal(data_list) output_data = TimeseriesData(signal=signal, timebase=timebase, channels=channels) #output_data.meta.update({'shot':self.shot}) output_data.meta.update(meta_dict) return output_data
def get_probe_angles(input_data, closed=False): """ return a list of thetas for a given signal (timeseries) or a string that specifies it. get_probe_angles('W7X:W7X_MIRNOV_41_BEST_LOOP:(20180912,43)') This is a kludgey way to read coordinates. Should be through acquisition.base or acquisition.'device' rather than looking up config directly """ import pyfusion if isinstance(input_data, str): pieces = input_data.split(':') if len(pieces) == 3: dev_name, diag_name, shotstr = pieces shot_number = eval(shotstr) dev = pyfusion.getDevice(dev_name) data = dev.acq.getdata(shot_number, diag_name, time_range=[0, 0.1]) else: from pyfusion.data.timeseries import TimeseriesData, Timebase, Signal from pyfusion.data.base import Channel, ChannelList, Coords input_data = TimeseriesData(Timebase([0, 1]), Signal([0, 1])) dev_name, diag_name = pieces # channels are amongst options opts = pyfusion.config.pf_options('Diagnostic', diag_name) chans = [ pyfusion.config.pf_get('Diagnostic', diag_name, opt) for opt in opts if 'channel_' in opt ] # for now, assume config_name is some as name input_data.channels = ChannelList( *[Channel(ch, Coords('?', [0, 0, 0])) for ch in chans]) Phi = np.array([ 2 * np.pi / 360 * float( pyfusion.config.get( 'Diagnostic:{cn}'.format( cn=c.config_name if c.config_name != '' else c.name), 'Coords_reduced').split(',')[0]) for c in input_data.channels ]) Theta = np.array([ 2 * np.pi / 360 * float( pyfusion.config.get( 'Diagnostic:{cn}'.format( cn=c.config_name if c.config_name != '' else c.name), 'Coords_reduced').split(',')[1]) for c in input_data.channels ]) if closed: Phi = np.append(Phi, Phi[0]) Theta = np.append(Theta, Theta[0]) return (dict(Theta=Theta, Phi=Phi))
def do_fetch(self): delimiter = self.__dict__.get("delimiter", None) data = genfromtxt(self.filename.replace("(shot)", str(self.shot)), unpack=True, delimiter=delimiter) # len(data) is number of channels + 1 (timebase) n_channels = len(data) - 1 ch_generator = (generic_ch(i) for i in range(n_channels)) ch = ChannelList(*ch_generator) return TimeseriesData(timebase=Timebase(data[0]), signal=Signal(data[1:]), channels=ch)
def test_svd_data(self): n_ch = 10 n_samples = 1024 timebase = Timebase(np.arange(n_samples)*1.e-6) channels = ChannelList(*(Channel('ch_%02d' %i, Coords('cylindrical',(1.0,i,0.0))) for i in 2*np.pi*np.arange(n_ch)/n_ch)) multichannel_data = get_multimode_test_data(channels = channels, timebase = timebase, noise = 0.5) test_svd = multichannel_data.svd() self.assertTrue(isinstance(test_svd, SVDData)) self.assertEqual(len(test_svd.topos[0]), n_ch) self.assertEqual(len(test_svd.chronos[0]), n_samples) assert_array_almost_equal(test_svd.chrono_labels, timebase) for c_i, ch in enumerate(channels): self.assertEqual(ch, test_svd.channels[c_i])
def do_fetch(self): # evaluate filename list try: filenames = eval(self.__dict__.get("filenames", "[]")) except TypeError: # assume we have been given a list of filenames as a keyword argument, rather than # reading the config file. filenames = self.__dict__.get("filenames") data_array = [] channel_names = [] dtypes = [] for fn_i, fn in enumerate(filenames): dt = eval(self.__dict__.get("dtype_%d" % (fn_i + 1), None)) dtypes.append(dt) if fn.endswith('.bz2'): f = bz2.BZ2File(fn.replace("(shot)", str(self.shot))) data_array.append(np.fromstring(f.read(), dtype=dt)) f.close() else: data_array.append( np.fromfile(fn.replace("(shot)", str(self.shot)), dtype=dt)) channel_names.extend( [i for i in dt.names if i.startswith('channel_')]) ch_generator = (named_ch(i) for i in channel_names) ch = ChannelList(*ch_generator) signal_data = np.zeros((len(channel_names), data_array[0].shape[0]), dtype=dtypes[0][channel_names[0]]) sig_counter = 0 for d_i, d in enumerate(data_array): for ch_name in dtypes[d_i].names: if ch_name.startswith('channel_'): signal_data[sig_counter, :] = d[ch_name] sig_counter += 1 tsd = TimeseriesData(timebase=Timebase(data_array[0]['timebase']), signal=Signal(signal_data), channels=ch) tsd.phase_pairs = self.__dict__.get("phase_pairs", None) if tsd.phase_pairs != None: tsd.phase_pairs = eval(tsd.phase_pairs) return tsd
def test_flucstruc_phases(PfTestCase): n_ch = 10 n_samples = 5000 timebase = Timebase(np.arange(n_samples)*1.e-6) channels = ChannelList(*(Channel('ch_%d' %i, Coords('cylindrical',(1.0,i,0.0))) for i in 2*np.pi*np.arange(n_ch)/n_ch)) multichannel_data = get_multimode_test_data(channels = channels, timebase = timebase, noise = 0.5) data_reduced_time=multichannel_data.reduce_time([0,0.002]).subtract_mean().normalise(method='v',separate=True) fs_set=data_reduced_time.flucstruc() phases = [] for fs in fs_set: for j in range(0,len(fs.dphase)): phases.append(fs.dphase[j].delta)
def do_fetch(self): dtype = self.read_dtype() if self.filename.endswith('.bz2'): f = bz2.BZ2File(self.filename.replace("(shot)", str(self.shot))) data = np.fromstring(f.read(), dtype=dtype) f.close() else: data = np.fromfile(self.filename.replace("(shot)", str(self.shot)), dtype=dtype) channel_names = [i for i in dtype.names if i.startswith('channel_')] ch_generator = (named_ch(i) for i in channel_names) ch = ChannelList(*ch_generator) signal_data = np.zeros((len(channel_names), data.shape[0]), dtype=dtype[channel_names[0]]) for ch_i, ch_name in enumerate(channel_names): signal_data[ch_i, :] = data[ch_name] return TimeseriesData(timebase=Timebase(data['timebase']), signal=Signal(signal_data), channels=ch)
def fetch(self): """Fetch each channel and combine into a multichannel instance of :py:class:`~pyfusion.data.timeseries.TimeseriesData`. :rtype: :py:class:`~pyfusion.data.timeseries.TimeseriesData` """ ## initially, assume only single channel signals # this base debug breakpoint will apply to all flavours of acquisition debug_(pyfusion.DEBUG, level=3, key='entry_base_multi_fetch') ordered_channel_names = self.ordered_channel_names() data_list = [] channels = ChannelList() # empty I guess common_tb = None # common_timebase meta_dict={} group_utc = None # assume no utc, will be replaced by channels # t_min, t_max seem obsolete, and should be done in single chan fetch if hasattr(self, 't_min') and hasattr(self, 't_max'): t_range = [float(self.t_min), float(self.t_max)] else: t_range = [] for chan in ordered_channel_names: sgn = 1 if chan[0]=='-': sgn = -sgn bare_chan = (chan.split('-'))[-1] ch_data = self.acq.getdata(self.shot, bare_chan) if len(t_range) == 2: ch_data = ch_data.reduce_time(t_range) channels.append(ch_data.channels) # two tricky things here - tmp.data.channels only gets one channel hhere # Config_name for a channel is attached to the multi part - # We need to move it to the particular channel # Was channels[-1].config_name = chan # 2013 - default to something if config_name not defined if pyfusion.VERBOSE>0: print("base:multi ch_data.config_name", ch_data.config_name) if hasattr(ch_data,'config_name'): channels[-1].config_name = ch_data.config_name else: channels[-1].config_name = 'fix_me' meta_dict.update(ch_data.meta) # tack on the data utc ch_data.signal.utc = ch_data.utc # Make a common timebase and do some basic checks. if common_tb is None: common_tb = ch_data.timebase if hasattr(ch_data,'utc'): group_utc = ch_data.utc tb_chan = ch_data.channels # for the first, append the whole signal data_list.append(sgn * ch_data.signal) else: if hasattr(self, 'skip_timebase_check') and self.skip_timebase_check == 'True': # append regardless, but will have to go back over # later to check length cf common_tb if pyfusion.VERBOSE > 0: print('utcs: ******',ch_data.utc[0],group_utc[0]) if ch_data.utc[0] != group_utc[0]: dts = (ch_data.utc[0] - group_utc[0])/1e9 print('*** different start times *****\n********trace {chcnf} starts after {tbch} by {dts:.2g} s' .format(tbch = tb_chan.config_name, chcnf = ch_data.config_name, dts=dts)) # should pad this channel out with nans - for now report an error. # this won't work if combining signals on a common_tb # ch_data.timebase += -dts # not sure if + or -? # print(ch_data.timebase[0]) """ kind of works for L53_LP0701 309 13, but time error dtclock = 2e-6 nsampsdiff = int(round(dts/dtclock,0)) newlen = len(ch_data.timebase) + nsampsdiff newsig = np.array(newlen * [np.nan]) newtb = np.array(newlen * [np.nan]) newsig[nsampsdiff:] = ch_data.signal newtb[nsampsdiff:] = ch_data.timebase ch_data.signal = newsig ch_data.timebase = newtb ch_data.timebase += -dts # not sure if + or -? """ if len(ch_data.signal)<len(common_tb): common_tb = ch_data.timebase tb_chan = ch_data.channels data_list.append(ch_data.signal[0:len(common_tb)]) else: try: assert_array_almost_equal(common_tb, ch_data.timebase) data_list.append(ch_data.signal) except: print('#### matching error in {c} - perhaps timebase not the same as the previous channel'.format(c=ch_data.config_name)) raise if hasattr(self, 'skip_timebase_check') and self.skip_timebase_check == 'True': # Messy - if we ignored timebase checks, then we have to check # length and cut to size, otherwise it will be stored as a signal (and # we will end up with a signal of signals) # This code may look unpythonic, but it avoids a copy() # and may be faster than for sig in data_list.... ltb = len(common_tb) for i in range(len(data_list)): if len(data_list[i]) > ltb: # this is a replacement. data_list.insert(i,data_list.pop(i)[0:ltb]) signal = Signal(data_list) print(shape(signal)) output_data = TimeseriesData(signal=signal, timebase=common_tb, channels=channels) #output_data.meta.update({'shot':self.shot}) output_data.meta.update(meta_dict) output_data.comment = self.comment if hasattr(self, 'comment') else '' print(output_data.comment) #if not hasattr(output_data,'utc'): # output_data.utc = None # probably should try to get it # # from a channel - but how? output_data.utc = group_utc # should check that all channels have same utc debug_(pyfusion.DEBUG, level=2, key='return_base_multi_fetch') return output_data
def fetch(self): """Fetch each channel and combine into a multichannel instance of :py:class:`~pyfusion.data.timeseries.TimeseriesData`. :rtype: :py:class:`~pyfusion.data.timeseries.TimeseriesData` """ ## initially, assume only single channel signals # this base debug breakpoint will apply to all flavours of acquisition debug_(pyfusion.DEBUG, level=2, key='base_multi_fetch') ordered_channel_names = self.ordered_channel_names() data_list = [] channels = ChannelList() timebase = None meta_dict = {} if hasattr(self, 't_min') and hasattr(self, 't_max'): t_range = [float(self.t_min), float(self.t_max)] else: t_range = [] for chan in ordered_channel_names: sgn = 1 if chan[0] == '-': sgn = -sgn bare_chan = (chan.split('-'))[-1] fetcher_class = import_setting('Diagnostic', bare_chan, 'data_fetcher') tmp_data = fetcher_class(self.acq, self.shot, config_name=bare_chan).fetch() if len(t_range) == 2: tmp_data = tmp_data.reduce_time(t_range) channels.append(tmp_data.channels) # two tricky things here - tmp.data.channels only gets one channel hhere #config_name for a channel is attached to the multi part - #we need to move it to the particular channel # was channels[-1].config_name = chan channels[-1].config_name = tmp_data.config_name meta_dict.update(tmp_data.meta) #print(tmp_data.signal[-1], sgn) tmp_data.signal = sgn * tmp_data.signal #print(tmp_data.signal[-1], sgn) if timebase == None: timebase = tmp_data.timebase data_list.append(tmp_data.signal) else: if hasattr(self, 'skip_timebase_check' ) and self.skip_timebase_check == 'true': data_list.append(tmp_data.signal) else: try: assert_array_almost_equal(timebase, tmp_data.timebase) data_list.append(tmp_data.signal) except: raise signal = Signal(data_list) output_data = TimeseriesData(signal=signal, timebase=timebase, channels=channels) #output_data.meta.update({'shot':self.shot}) output_data.meta.update(meta_dict) return output_data
def get_n_channels(n_ch): """Return a list of n_ch channels.""" poloidal_coords = 2*np.pi*np.arange(n_ch)/n_ch channel_gen = (Channel('ch_%02d' %i, Coords('cylindrical', (1.0,i,0.0))) for i in poloidal_coords) return ChannelList(*channel_gen)