def test_remove_noncontiguous(self):
tb1 = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tb2 = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tb3 = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
# nonzero signal mean
tsd1 = TimeseriesData(timebase=tb1,
signal=Signal(np.arange(len(tb1))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0)))))
tsd2 = TimeseriesData(timebase=tb2,
signal=Signal(np.arange(len(tb2))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0)))))
tsd3 = TimeseriesData(timebase=tb3,
signal=Signal(np.arange(len(tb3))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0)))))
self.assertTrue(tb1.is_contiguous())
self.assertTrue(tb2.is_contiguous())
self.assertTrue(tb3.is_contiguous())
tsd2.timebase[-50:] += 1.0
self.assertFalse(tb2.is_contiguous())
ds = DataSet('ds')
for tsd in [tsd1, tsd2, tsd3]:
ds.add(tsd)
for tsd in [tsd1, tsd2, tsd3]:
self.assertTrue(tsd in ds)
filtered_ds = ds.remove_noncontiguous()
for tsd in [tsd1, tsd3]:
self.assertTrue(tsd in filtered_ds)
self.assertFalse(tsd2 in filtered_ds)
def test_n_channels(self):
test_sig_1a = Signal(np.random.rand(10))
self.assertEqual(test_sig_1a.n_channels(), 1)
test_sig_1b = Signal(np.random.rand(1,10))
self.assertEqual(test_sig_1b.n_channels(), 1)
test_sig_2 = Signal(np.random.rand(2,10))
self.assertEqual(test_sig_2.n_channels(), 2)
def test_n_samples(self):
test_sig_1a = Signal(np.random.rand(10))
self.assertEqual(test_sig_1a.n_samples(), 10)
test_sig_1b = Signal(np.random.rand(1,10))
self.assertEqual(test_sig_1b.n_samples(), 10)
test_sig_2 = Signal(np.random.rand(2,10))
self.assertEqual(test_sig_2.n_samples(), 10)
def test_reduce_time_dataset(self):
new_times = [-0.25, 0.25]
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tsd_1 = TimeseriesData(timebase=tb,
signal=Signal(np.resize(np.arange(5*len(tb)),(5,len(tb)))),
channels=get_n_channels(5))
tsd_2 = TimeseriesData(timebase=tb, signal=Signal(np.resize(np.arange(5*len(tb))+1,(5,len(tb)))),
channels=get_n_channels(5))
test_dataset = DataSet('test_dataset')
test_dataset.add(tsd_1)
test_dataset.add(tsd_2)
test_dataset.reduce_time(new_times)
def test_dataset(self):
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tsd_1 = TimeseriesData(timebase=tb,
signal=Signal(np.resize(np.arange(3*len(tb)), (3,len(tb)))),
channels=get_n_channels(3))
tsd_2 = TimeseriesData(timebase=tb,
signal=Signal(np.resize(np.arange(3*len(tb)+1),(3,len(tb)))),
channels=get_n_channels(3))
input_dataset = DataSet('test_dataset')
input_dataset.add(tsd_1)
input_dataset.add(tsd_2)
seg_dataset = input_dataset.segment(n_samples=10)
self.assertTrue(len(seg_dataset)==20)
def test_dataset(self):
ch=get_n_channels(5)
new_times = [-0.25, 0.25]
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tsd_1 = TimeseriesData(timebase=tb, signal=Signal(np.resize(np.arange(5*len(tb)),(5,len(tb)))),
channels=ch)
tsd_2 = TimeseriesData(timebase=tb,
signal=Signal(np.resize(np.arange(5*len(tb))+1, (5,len(tb)))),
channels=ch)
test_dataset = DataSet('test_ds_1')
test_dataset.add(tsd_1)
test_dataset.add(tsd_2)
self.assertTrue(tsd_1 in test_dataset)
"""
def test_multi_channel_timeseries(self):
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tsd = TimeseriesData(timebase=tb,
signal=Signal(np.resize(np.arange(3*len(tb)), (3,len(tb)))),
channels=get_n_channels(3))
seg_dataset = tsd.segment(n_samples=10)
self.assertTrue(len(seg_dataset)==10)
def do_fetch(self):
chan_name = (self.diag_name.split('-'))[-1] # remove -
filename_dict = {'diag_name': chan_name, 'shot': self.shot}
self.basename = path.join(pf.config.get('global', 'localdatapath'),
data_filename % filename_dict)
files_exist = path.exists(self.basename)
if not files_exist:
raise Exception, "file " + self.basename + " not found."
else:
signal_dict = newload(self.basename)
if ((chan_name == array(['MP5', 'HMP13', 'HMP05'])).any()): flip = -1.
else: flip = 1.
if self.diag_name[0] == '-': flip = -flip
# coords = get_coords_for_channel(**self.__dict__)
ch = Channel(self.diag_name, Coords('dummy', (0, 0, 0)))
output_data = TimeseriesData(
timebase=Timebase(signal_dict['timebase']),
signal=Signal(flip * signal_dict['signal']),
channels=ch)
output_data.meta.update({'shot': self.shot})
return output_data
def test_remove_mean_single_channel(self):
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
# nonzero signal mean
tsd = TimeseriesData(timebase=tb,
signal=Signal(np.arange(len(tb))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0)))))
filtered_tsd = tsd.subtract_mean()
assert_almost_equal(np.mean(filtered_tsd.signal), 0)
def do_fetch(self):
print self.shot, self.senal
data_dim = tjiidata.dimens(self.shot, self.senal)
if data_dim[0] < MAX_SIGNAL_LENGTH:
data_dict = tjiidata.lectur(self.shot, self.senal, data_dim[0],
data_dim[0], data_dim[1])
else:
raise ValueError, 'Not loading data to avoid segmentation fault in tjiidata.lectur'
ch = Channel(self.senal, Coords('dummy', (0, 0, 0)))
if self.invert == 'true': #yuk - TODO: use boolean type from config
s = Signal(-np.array(data_dict['y']))
else:
s = Signal(np.array(data_dict['y']))
output_data = TimeseriesData(timebase=Timebase(data_dict['x']),
signal=s,
channels=ch)
output_data.meta.update({'shot': self.shot})
return output_data
def test_reduce_time_filter_multi_channel_attached_method(self):
new_times = [-0.25, 0.25]
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tsd = TimeseriesData(timebase=tb,
signal=Signal(np.resize(np.arange(5*len(tb)),(5,len(tb)))),
channels=get_n_channels(5))
new_time_args = np.searchsorted(tb, new_times)
timebase_test = tsd.timebase[new_time_args[0]:new_time_args[1]].copy()
signal_test = tsd.signal[:,new_time_args[0]:new_time_args[1]].copy()
reduced_tsd = tsd.reduce_time(new_times)
self.assertTrue(isinstance(reduced_tsd, TimeseriesData))
assert_array_almost_equal(reduced_tsd.timebase, timebase_test)
assert_array_almost_equal(reduced_tsd.signal, signal_test)
def get_probe_angles(input_data, closed=False):
"""
return a list of thetas for a given signal (timeseries) or a string that specifies it.
get_probe_angles('W7X:W7X_MIRNOV_41_BEST_LOOP:(20180912,43)')
This is a kludgey way to read coordinates. Should be through acquisition.base or
acquisition.'device' rather than looking up config directly
"""
import pyfusion
if isinstance(input_data, str):
pieces = input_data.split(':')
if len(pieces) == 3:
dev_name, diag_name, shotstr = pieces
shot_number = eval(shotstr)
dev = pyfusion.getDevice(dev_name)
data = dev.acq.getdata(shot_number, diag_name, time_range=[0, 0.1])
else:
from pyfusion.data.timeseries import TimeseriesData, Timebase, Signal
from pyfusion.data.base import Channel, ChannelList, Coords
input_data = TimeseriesData(Timebase([0, 1]), Signal([0, 1]))
dev_name, diag_name = pieces
# channels are amongst options
opts = pyfusion.config.pf_options('Diagnostic', diag_name)
chans = [
pyfusion.config.pf_get('Diagnostic', diag_name, opt)
for opt in opts if 'channel_' in opt
]
# for now, assume config_name is some as name
input_data.channels = ChannelList(
*[Channel(ch, Coords('?', [0, 0, 0])) for ch in chans])
Phi = np.array([
2 * np.pi / 360 * float(
pyfusion.config.get(
'Diagnostic:{cn}'.format(
cn=c.config_name if c.config_name != '' else c.name),
'Coords_reduced').split(',')[0]) for c in input_data.channels
])
Theta = np.array([
2 * np.pi / 360 * float(
pyfusion.config.get(
'Diagnostic:{cn}'.format(
cn=c.config_name if c.config_name != '' else c.name),
'Coords_reduced').split(',')[1]) for c in input_data.channels
])
if closed:
Phi = np.append(Phi, Phi[0])
Theta = np.append(Theta, Theta[0])
return (dict(Theta=Theta, Phi=Phi))
def do_fetch(self):
channel_length = int(self.length)
outdata = np.zeros(1024 * 2 * 256 + 1)
## !! really should put a wrapper around gethjdata to do common stuff
# outfile is only needed if the direct passing of binary won't work
# with tempfile.NamedTemporaryFile(prefix="pyfusion_") as outfile:
# get in two steps to make debugging easier
allrets = gethjdata.gethjdata(self.shot,
channel_length,
self.path,
verbose=VERBOSE,
opt=1,
ierror=2,
isample=-1,
outdata=outdata,
outname='')
ierror, isample, getrets = allrets
if ierror != 0:
raise LookupError(
'hj Okada style data not found for {s}:{c}'.format(
s=self.shot, c=self.path))
ch = Channel(self.path, Coords('dummy', (0, 0, 0)))
# the intent statement causes the out var to be returned in the result lsit
# looks like the time,data is interleaved in a 1x256 array
# it is fed in as real*64, but returns as real*32! (as per fortran decl)
debug_(pyfusion.DEBUG,
4,
key='Heliotron_fetch',
msg='after call to getdata')
# timebase in secs (ms in raw data) - could add a preferred unit?
# this is partly allowed for in savez_compressed, newload, and
# for plotting, in the config file.
# important that the 1e-3 be inside the Timebase()
output_data = TimeseriesData(timebase=Timebase(
1e-3 * getrets[1::2][0:isample]),
signal=Signal(getrets[2::2][0:isample]),
channels=ch)
output_data.meta.update({'shot': self.shot})
if pyfusion.VERBOSE > 0: print('HJ config name', self.config_name)
output_data.config_name = self.config_name
stprms = get_static_params(shot=self.shot, signal=self.path)
if len(list(stprms)
) == 0: # maybe this should be ignored - how do we use it?
raise LookupError(
' failure to get params for {shot}:{path}'.format(
shot=self.shot, path=self.path))
output_data.params = stprms
return output_data
def do_fetch(self):
channel_length = int(self.length)
outdata=np.zeros(1024*2*256+1)
with tempfile.NamedTemporaryFile(prefix="pyfusion_") as outfile:
getrets=gethjdata.gethjdata(self.shot,channel_length,self.path,
VERBOSE, OPT,
outfile.name, outdata)
ch = Channel(self.path,
Coords('dummy', (0,0,0)))
output_data = TimeseriesData(timebase=Timebase(getrets[1::2]),
signal=Signal(getrets[2::2]), channels=ch)
output_data.meta.update({'shot':self.shot})
return output_data
def do_fetch(self):
delimiter = self.__dict__.get("delimiter", None)
data = genfromtxt(self.filename.replace("(shot)", str(self.shot)),
unpack=True,
delimiter=delimiter)
# len(data) is number of channels + 1 (timebase)
n_channels = len(data) - 1
ch_generator = (generic_ch(i) for i in range(n_channels))
ch = ChannelList(*ch_generator)
return TimeseriesData(timebase=Timebase(data[0]),
signal=Signal(data[1:]),
channels=ch)
def do_fetch(self):
print(self.pointname)
print(self.shot)
if self.NC!=None:
print(self.NC)
t_name = '{}_time'.format(self.pointname)
NC_vars = self.NC.variables.keys()
if self.pointname in NC_vars:
print('Reading cache!!!!')
t_axis = self.NC.variables[t_name].data[:].copy()
data = self.NC.variables[self.pointname].data[:].copy()
else:
tmp = self.acq.connection.get('ptdata2("{}",{})'.format(self.pointname, self.shot))
data = tmp.data()
tmp = self.acq.connection.get('dim_of(ptdata2("{}",{}))'.format(self.pointname, self.shot))
t_axis = tmp.data()
self.write_cache = True
print(t_axis)
print(data)
coords = get_coords_for_channel(**self.__dict__)
ch = Channel(self.pointname, coords)
# con=MDS.Connection('atlas.gat.com::')
# pointname = 'MPI66M067D'
# shot = 164950
# tmp = con.get('ptdata2("{}",{})'.format(pointname, shot))
# dat = tmp.data()
# tmp = con.get('dim_of(ptdata2("{}",{}))'.format(pointname, shot))
# t = tmp.data()
if self.NC!=None and self.write_cache:
print self.pointname
self.NC.createDimension(t_name, len(t_axis))
f_time = self.NC.createVariable(t_name,'d',(t_name,))
f_time[:] = +t_axis
print('Wrote time')
sig = self.NC.createVariable(self.pointname,'f',(t_name,))
sig[:] = +data
print('Wrote signal')
output_data = TimeseriesData(timebase=Timebase(t_axis),
signal=Signal(data), channels=ch)
# output_data = super(DIIIDDataFetcherPTdata, self).do_fetch()
# coords = get_coords_for_channel(**self.__dict__)
# ch = Channel(self.mds_path, coords)
# output_data.channels = ch
# output_data.meta.update({'shot':self.shot, 'kh':self.get_kh()})
# print(ch)
output_data.config_name = ch
self.fetch_mode = 'ptdata'
return output_data
def get_multimode_test_data(channels = get_n_channels(DEFAULT_N_CHANNELS),
timebase = DEFAULT_TIMEBASE,
modes = [mode_1, mode_2], noise = DEFAULT_NOISE):
"""Generate synthetic multi-channel data for testing."""
n_channels = len(channels)
data_size = (n_channels, timebase.size)
data_array = noise*2*(np.random.random(data_size)-0.5)
timebase_matrix = np.resize(timebase, data_size)
angle_matrix = np.resize(np.array([i.coords.cylindrical[1] for i in channels]),
data_size[::-1]).T
for m in modes:
data_array += m['amp']*np.cos(2*np.pi*m['freq']*timebase_matrix +
m['mode_number']*angle_matrix + m['phase'])
output = TimeseriesData(timebase=timebase, signal=Signal(data_array),
channels=channels)
return output
def testFilteredDataHistory_nocopy(self):
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
# nonzero signal mean
ch = get_n_channels(1)
tsd = TimeseriesData(timebase=tb,
signal=Signal(np.arange(len(tb))), channels=ch)
filtered_tsd = tsd.subtract_mean()
#self.assertEqual(len(filtered_tsd.history.split('\n')), 3)
self.assertEqual(len(filtered_tsd.history.split('\n')), 5) # bdb thinks extra 2 is OK
output_data = filtered_tsd.normalise(method='rms', copy=False)
#self.assertEqual(filtered_tsd.history.split('> ')[-1], "normalise(method='rms')")
self.assertEqual(filtered_tsd.history.split('> ')[-1].split('\n')[0], "normalise(method='rms')")
#self.assertEqual(output_data.history.split('> ')[-1], "normalise(method='rms')")
self.assertEqual(output_data.history.split('> ')[-1].split('\n')[0], "normalise(method='rms')")
def fetch(self, interp_if_diff = True):
"""Fetch each channel and combine into a multichannel instance
of :py:class:`~pyfusion.data.timeseries.TimeseriesData`.
:rtype: :py:class:`~pyfusion.data.timeseries.TimeseriesData`
"""
print('******** hello world ***********')
## initially, assume only single channel signals
ordered_channel_names = self.ordered_channel_names()
data_list = []
channels = ChannelList()
timebase = None
meta_dict={}
from scipy.io import netcdf
fname = '/u/haskeysr/tmp/{}.nc'.format(self.shot)
write_cache=False; read_cache=False
if os.path.exists(fname):
NC = netcdf.netcdf_file(fname,'a',version=2)
else:
NC = netcdf.netcdf_file(fname,'w',version=2)
for chan in ordered_channel_names:
fetcher_class = import_setting('Diagnostic', chan, 'data_fetcher')
tmp_data = fetcher_class(self.acq, self.shot,
config_name=chan, NC=NC).fetch()
channels.append(tmp_data.channels)
meta_dict.update(tmp_data.meta)
if timebase == None:
timebase = tmp_data.timebase
data_list.append(tmp_data.signal)
else:
try:
assert_array_almost_equal(timebase, tmp_data.timebase)
data_list.append(tmp_data.signal)
except:
if interp_if_diff:
data_list.append(np.interp(timebase, tmp_data.timebase, tmp_data.signal))
else:
raise
NC.close()
signal=Signal(data_list)
output_data = TimeseriesData(signal=signal, timebase=timebase,
channels=channels)
#output_data.meta.update({'shot':self.shot})
output_data.meta.update(meta_dict)
return output_data
def testFilteredDataHistory_copy(self):
""" make sure that _copy version does NOT alter original
"""
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
# nonzero signal mean
ch = get_n_channels(1)
tsd = TimeseriesData(timebase=tb,
signal=Signal(np.arange(len(tb))), channels=ch)
filtered_tsd = tsd.subtract_mean()
# bdb in 4 places, assume that the xtra info (norm_value) is supposed to be there.
#self.assertEqual(len(filtered_tsd.history.split('\n')), 3)
self.assertEqual(len(filtered_tsd.history.split('\n')), 5) # bdb thinks extra 2 is OK
output_data = filtered_tsd.normalise(method='rms', copy=True)
#self.assertEqual(output_data.history.split('> ')[-1], "normalise(method='rms')")
self.assertEqual(output_data.history.split('> ')[-1].split('\n')[0], "normalise(method='rms')")
#self.assertEqual(filtered_tsd.history.split('> ')[-1], "subtract_mean()")
self.assertEqual(filtered_tsd.history.split('> ')[-1].split('\n')[0], "subtract_mean()")
def do_fetch(self):
print("Shot: {}\nPoint Name: {}".format(self.shot, self.pointname))
if not hasattr(self, 'NC'):
self.NC = None
if self.NC is not None:
#print(self.NC)
t_name = '{}_time'.format(self.pointname)
NC_vars = self.NC.variables.keys()
else:
NC_vars = []
if self.pointname in NC_vars:
print(' Pointname in NC cache, Reading...\n')
t_axis = self.NC.variables[t_name].data[:].copy()
data = self.NC.variables[self.pointname].data[:].copy()
self.write_cache = False
else:
print(' Fetching from ptdata')
tmp = self.acq.connection.get('ptdata2("{}",{})'.format(
self.pointname, self.shot))
data = tmp.data()
tmp = self.acq.connection.get('dim_of(ptdata2("{}",{}))'.format(
self.pointname, self.shot))
t_axis = tmp.data()
self.write_cache = True
coords = get_coords_for_channel(**self.__dict__)
ch = Channel(self.pointname, coords)
if self.NC is not None and self.write_cache:
print("\t Writing to NC file disabled temporarily.")
#print(' Writing pointname to NC file\n')
#self.NC.createDimension(t_name, len(t_axis))
#f_time = self.NC.createVariable(t_name,'d',(t_name,))
#f_time[:] = +t_axis
# sig = self.NC.createVariable(self.pointname,'f',(t_name,))
#sig[:] = +data
output_data = TimeseriesData(timebase=Timebase(t_axis),
signal=Signal(data),
channels=ch)
output_data.config_name = ch
self.fetch_mode = 'ptdata'
return output_data
def do_fetch(self):
sig = self.conn.get(self.mds_path)
dim = self.conn.get('DIM_OF(' + self.mds_path + ')')
scl = 1.0
coords = get_coords_for_channel(**self.__dict__)
ch = Channel(self.config_name, coords)
timedata = dim.data()
output_data = TimeseriesData(timebase=Timebase(1e-9 * timedata),
signal=scl * Signal(sig),
channels=ch)
output_data.meta.update({'shot': self.shot})
if hasattr(self, 'mdsshot'): # intended for checks - not yet used.
output_data.mdsshot = self.mdsshot
output_data.config_name = self.config_name
output_data.utc = [timedata[0], timedata[-1]]
#output_data.units = dat['units'] if 'units' in dat else ''
debug_(pyfusion.DEBUG,
level=1,
key='W7M_do_fetch',
msg='entering W7X MDS do_fetch')
return (output_data)
