def test_remove_noncontiguous(self):
tb1 = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tb2 = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tb3 = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
# nonzero signal mean
tsd1 = TimeseriesData(timebase=tb1,
signal=Signal(np.arange(len(tb1))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0)))))
tsd2 = TimeseriesData(timebase=tb2,
signal=Signal(np.arange(len(tb2))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0)))))
tsd3 = TimeseriesData(timebase=tb3,
signal=Signal(np.arange(len(tb3))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0)))))
self.assertTrue(tb1.is_contiguous())
self.assertTrue(tb2.is_contiguous())
self.assertTrue(tb3.is_contiguous())
tsd2.timebase[-50:] += 1.0
self.assertFalse(tb2.is_contiguous())
ds = DataSet('ds')
for tsd in [tsd1, tsd2, tsd3]:
ds.add(tsd)
for tsd in [tsd1, tsd2, tsd3]:
self.assertTrue(tsd in ds)
filtered_ds = ds.remove_noncontiguous()
for tsd in [tsd1, tsd3]:
self.assertTrue(tsd in filtered_ds)
self.assertFalse(tsd2 in filtered_ds)
def test_reduce_time_dataset(self):
new_times = [-0.25, 0.25]
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tsd_1 = TimeseriesData(timebase=tb,
signal=Signal(np.resize(np.arange(5*len(tb)),(5,len(tb)))),
channels=get_n_channels(5))
tsd_2 = TimeseriesData(timebase=tb, signal=Signal(np.resize(np.arange(5*len(tb))+1,(5,len(tb)))),
channels=get_n_channels(5))
test_dataset = DataSet('test_dataset')
test_dataset.add(tsd_1)
test_dataset.add(tsd_2)
test_dataset.reduce_time(new_times)
def test_dataset(self):
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tsd_1 = TimeseriesData(timebase=tb,
signal=Signal(np.resize(np.arange(3*len(tb)), (3,len(tb)))),
channels=get_n_channels(3))
tsd_2 = TimeseriesData(timebase=tb,
signal=Signal(np.resize(np.arange(3*len(tb)+1),(3,len(tb)))),
channels=get_n_channels(3))
input_dataset = DataSet('test_dataset')
input_dataset.add(tsd_1)
input_dataset.add(tsd_2)
seg_dataset = input_dataset.segment(n_samples=10)
self.assertTrue(len(seg_dataset)==20)
def test_dataset(self):
ch=get_n_channels(5)
new_times = [-0.25, 0.25]
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tsd_1 = TimeseriesData(timebase=tb, signal=Signal(np.resize(np.arange(5*len(tb)),(5,len(tb)))),
channels=ch)
tsd_2 = TimeseriesData(timebase=tb,
signal=Signal(np.resize(np.arange(5*len(tb))+1, (5,len(tb)))),
channels=ch)
test_dataset = DataSet('test_ds_1')
test_dataset.add(tsd_1)
test_dataset.add(tsd_2)
self.assertTrue(tsd_1 in test_dataset)
"""
def segment(input_data, n_samples, overlap=1.0, datalist= 0):
"""Break into segments length n_samples.
Overlap of 2.0 starts a new segment halfway into previous, overlap=1 is
no overlap. overlap should divide into n_samples. Probably should
consider a nicer definition such as in pyfusion 0
if datalist = 0 returns a DataSet object otherwise, returns a OrderedDataSet object
"""
from pyfusion.data.base import DataSet, OrderedDataSet
from pyfusion.data.timeseries import TimeseriesData
if isinstance(input_data, DataSet):
output_dataset = DataSet()
for ii,data in enumerate(input_data):
try:
output_dataset.update(data.segment(n_samples))
except AttributeError:
pyfusion.logger.warning("Data filter 'segment' not applied to item in dataset")
return output_dataset
#SH modification incase ordering is important... i.e you are doing
#two processing two different arrays at the same time (in different Timeseries objects)
#and you don't want to loose the time relationship between them
if datalist:
output_data = OrderedDataSet('segmented_%s, %d samples, %.3f overlap' %(datetime.now(), n_samples, overlap))
else:
output_data = DataSet('segmented_%s, %d samples, %.3f overlap' %(datetime.now(), n_samples, overlap))
#SH : 24May2013 fixed bug here - before, the index was allowed to go past
#the length of samples, gives smalled length data towards the end - fixed to finish the
#last time we can get n_samples length
#for el in arange(0,len(input_data.timebase), n_samples/overlap):
for el in arange(0,len(input_data.timebase) - n_samples, n_samples/overlap):
if input_data.signal.ndim == 1:
tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples],
signal=input_data.signal[el:el+n_samples],
channels=input_data.channels, bypass_length_check=True)
else:
tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples],
signal=input_data.signal[:,el:el+n_samples],
channels=input_data.channels, bypass_length_check=True)
tmp_data.meta = input_data.meta.copy()
if datalist:
output_data.append(tmp_data)
else:
output_data.add(tmp_data)
return output_data
def test_multi_channel_timeseries(self):
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tsd = TimeseriesData(timebase=tb,
signal=Signal(np.resize(np.arange(3*len(tb)), (3,len(tb)))),
channels=get_n_channels(3))
seg_dataset = tsd.segment(n_samples=10)
self.assertTrue(len(seg_dataset)==10)
def do_fetch(self):
chan_name = (self.diag_name.split('-'))[-1] # remove -
filename_dict = {'diag_name': chan_name, 'shot': self.shot}
self.basename = path.join(pf.config.get('global', 'localdatapath'),
data_filename % filename_dict)
files_exist = path.exists(self.basename)
if not files_exist:
raise Exception, "file " + self.basename + " not found."
else:
signal_dict = newload(self.basename)
if ((chan_name == array(['MP5', 'HMP13', 'HMP05'])).any()): flip = -1.
else: flip = 1.
if self.diag_name[0] == '-': flip = -flip
# coords = get_coords_for_channel(**self.__dict__)
ch = Channel(self.diag_name, Coords('dummy', (0, 0, 0)))
output_data = TimeseriesData(
timebase=Timebase(signal_dict['timebase']),
signal=Signal(flip * signal_dict['signal']),
channels=ch)
output_data.meta.update({'shot': self.shot})
return output_data
def test_remove_mean_single_channel(self):
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
# nonzero signal mean
tsd = TimeseriesData(timebase=tb,
signal=Signal(np.arange(len(tb))), channels=ChannelList(Channel('ch_01',Coords('dummy',(0,0,0)))))
filtered_tsd = tsd.subtract_mean()
assert_almost_equal(np.mean(filtered_tsd.signal), 0)
def fetch_data_from_file(fetcher):
prm_dict = read_prm_file(fetcher.basename + ".prm")
bytes = int(prm_dict['DataLength(byte)'][0])
bits = int(prm_dict['Resolution(bit)'][0])
if not (prm_dict.has_key('ImageType')): #if so assume unsigned
bytes_per_sample = 2
dat_arr = Array.array('H')
offset = 2**(bits - 1)
dtype = np.dtype('uint16')
else:
if prm_dict['ImageType'][0] == 'INT16':
bytes_per_sample = 2
if prm_dict['BinaryCoding'][0] == 'offset_binary':
dat_arr = Array.array('H')
offset = 2**(bits - 1)
dtype = np.dtype('uint16')
elif prm_dict['BinaryCoding'][0] == "shifted_2's_complementary":
dat_arr = Array.array('h')
offset = 0
dtype = np.dtype('int16')
else:
raise NotImplementedError, ' binary coding ' + prm_dict[
'BinaryCoding']
fp = open(fetcher.basename + '.dat', 'rb')
dat_arr.fromfile(fp, bytes / bytes_per_sample)
fp.close()
clockHz = None
if prm_dict.has_key('SamplingClock'):
clockHz = double(prm_dict['SamplingClock'][0])
if prm_dict.has_key('SamplingInterval'):
clockHz = clockHz / double(prm_dict['SamplingInterval'][0])
if prm_dict.has_key('ClockSpeed'):
if clockHz != None:
pyfusion.utils.warn(
'Apparent duplication of clock speed information')
clockHz = double(prm_dict['ClockSpeed'][0])
clockHz = LHD_A14_clk(fetcher.shot) # see above
if clockHz != None:
timebase = arange(len(dat_arr)) / clockHz
else:
raise NotImplementedError, "timebase not recognised"
ch = Channel("%s-%s" % (fetcher.diag_name, fetcher.channel_number),
Coords('dummy', (0, 0, 0)))
if fetcher.gain != None:
gain = fetcher.gain
else:
gain = 1
output_data = TimeseriesData(timebase=Timebase(timebase),
signal=Signal(gain * dat_arr),
channels=ch)
output_data.meta.update({'shot': fetcher.shot})
return output_data
def do_fetch(self):
# TODO support non-signal datatypes
if self.fetch_mode == 'thin client':
ch = Channel(self.mds_path_components['nodepath'],
Coords('dummy', (0, 0, 0)))
data = self.acq.connection.get(
self.mds_path_components['nodepath'])
dim = self.acq.connection.get('dim_of(%s)' %
self.mds_path_components['nodepath'])
# TODO: fix this hack (same hack as when getting signal from node)
if len(data.shape) > 1:
data = np.array(data)[0, ]
if len(dim.shape) > 1:
dim = np.array(dim)[0, ]
output_data = TimeseriesData(timebase=Timebase(dim),
signal=Signal(data),
channels=ch)
output_data.meta.update({'shot': self.shot})
return output_data
elif self.fetch_mode == 'http':
data_url = self.acq.server + '/'.join([
self.mds_path_components['tree'],
str(self.shot), self.mds_path_components['tagname'],
self.mds_path_components['nodepath']
])
data = mdsweb.data_from_url(data_url)
ch = Channel(self.mds_path_components['nodepath'],
Coords('dummy', (0, 0, 0)))
t = Timebase(data.data.dim)
s = Signal(data.data.signal)
output_data = TimeseriesData(timebase=t, signal=s, channels=ch)
output_data.meta.update({'shot': self.shot})
return output_data
else:
node = self.tree.getNode(self.mds_path)
if int(node.dtype) == 195:
return get_tsd_from_node(self, node)
else:
raise Exception('Unsupported MDSplus node type')
def fetch(self):
tb = generate_timebase(t0=float(self.t0),
n_samples=int(self.n_samples),
sample_freq=float(self.sample_freq))
sig = Signal(
float(self.amplitude) * sin(2 * pi * float(self.frequency) * tb))
dummy_channel = Channel('ch_01', Coords('dummy', (0, 0, 0)))
output_data = TimeseriesData(timebase=tb,
signal=sig,
channels=ChannelList(dummy_channel))
output_data.meta.update({'shot': self.shot})
return output_data
def test_reduce_time_filter_multi_channel_attached_method(self):
new_times = [-0.25, 0.25]
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
tsd = TimeseriesData(timebase=tb,
signal=Signal(np.resize(np.arange(5*len(tb)),(5,len(tb)))),
channels=get_n_channels(5))
new_time_args = np.searchsorted(tb, new_times)
timebase_test = tsd.timebase[new_time_args[0]:new_time_args[1]].copy()
signal_test = tsd.signal[:,new_time_args[0]:new_time_args[1]].copy()
reduced_tsd = tsd.reduce_time(new_times)
self.assertTrue(isinstance(reduced_tsd, TimeseriesData))
assert_array_almost_equal(reduced_tsd.timebase, timebase_test)
assert_array_almost_equal(reduced_tsd.signal, signal_test)
def get_probe_angles(input_data, closed=False):
"""
return a list of thetas for a given signal (timeseries) or a string that specifies it.
get_probe_angles('W7X:W7X_MIRNOV_41_BEST_LOOP:(20180912,43)')
This is a kludgey way to read coordinates. Should be through acquisition.base or
acquisition.'device' rather than looking up config directly
"""
import pyfusion
if isinstance(input_data, str):
pieces = input_data.split(':')
if len(pieces) == 3:
dev_name, diag_name, shotstr = pieces
shot_number = eval(shotstr)
dev = pyfusion.getDevice(dev_name)
data = dev.acq.getdata(shot_number, diag_name, time_range=[0, 0.1])
else:
from pyfusion.data.timeseries import TimeseriesData, Timebase, Signal
from pyfusion.data.base import Channel, ChannelList, Coords
input_data = TimeseriesData(Timebase([0, 1]), Signal([0, 1]))
dev_name, diag_name = pieces
# channels are amongst options
opts = pyfusion.config.pf_options('Diagnostic', diag_name)
chans = [
pyfusion.config.pf_get('Diagnostic', diag_name, opt)
for opt in opts if 'channel_' in opt
]
# for now, assume config_name is some as name
input_data.channels = ChannelList(
*[Channel(ch, Coords('?', [0, 0, 0])) for ch in chans])
Phi = np.array([
2 * np.pi / 360 * float(
pyfusion.config.get(
'Diagnostic:{cn}'.format(
cn=c.config_name if c.config_name != '' else c.name),
'Coords_reduced').split(',')[0]) for c in input_data.channels
])
Theta = np.array([
2 * np.pi / 360 * float(
pyfusion.config.get(
'Diagnostic:{cn}'.format(
cn=c.config_name if c.config_name != '' else c.name),
'Coords_reduced').split(',')[1]) for c in input_data.channels
])
if closed:
Phi = np.append(Phi, Phi[0])
Theta = np.append(Theta, Theta[0])
return (dict(Theta=Theta, Phi=Phi))
def segment(input_data, n_samples, overlap=DEFAULT_SEGMENT_OVERLAP):
"""Break into segments length n_samples.
Overlap of 2.0 starts a new segment halfway into previous, overlap=1 is
no overlap. overlap should divide into n_samples. Probably should
consider a nicer definition such as in pyfusion 0
"""
from pyfusion.data.base import DataSet
from pyfusion.data.timeseries import TimeseriesData
if isinstance(input_data, DataSet):
output_dataset = DataSet()
for ii, data in enumerate(input_data):
try:
output_dataset.update(data.segment(n_samples))
except AttributeError:
pyfusion.logger.warning(
"Data filter 'segment' not applied to item in dataset")
return output_dataset
output_data = DataSet('segmented_%s, %d samples, %.3f overlap' %
(datetime.now(), n_samples, overlap))
for el in arange(0, len(input_data.timebase), n_samples / overlap):
if input_data.signal.ndim == 1:
tmp_data = TimeseriesData(
timebase=input_data.timebase[el:el + n_samples],
signal=input_data.signal[el:el + n_samples],
channels=input_data.channels,
bypass_length_check=True)
else:
tmp_data = TimeseriesData(
timebase=input_data.timebase[el:el + n_samples],
signal=input_data.signal[:, el:el + n_samples],
channels=input_data.channels,
bypass_length_check=True)
tmp_data.meta = input_data.meta.copy()
tmp_data.history = input_data.history # bdb - may be redundant now meta is copied
output_data.add(tmp_data)
return output_data
def do_fetch(self):
channel_length = int(self.length)
outdata = np.zeros(1024 * 2 * 256 + 1)
## !! really should put a wrapper around gethjdata to do common stuff
# outfile is only needed if the direct passing of binary won't work
# with tempfile.NamedTemporaryFile(prefix="pyfusion_") as outfile:
# get in two steps to make debugging easier
allrets = gethjdata.gethjdata(self.shot,
channel_length,
self.path,
verbose=VERBOSE,
opt=1,
ierror=2,
isample=-1,
outdata=outdata,
outname='')
ierror, isample, getrets = allrets
if ierror != 0:
raise LookupError(
'hj Okada style data not found for {s}:{c}'.format(
s=self.shot, c=self.path))
ch = Channel(self.path, Coords('dummy', (0, 0, 0)))
# the intent statement causes the out var to be returned in the result lsit
# looks like the time,data is interleaved in a 1x256 array
# it is fed in as real*64, but returns as real*32! (as per fortran decl)
debug_(pyfusion.DEBUG,
4,
key='Heliotron_fetch',
msg='after call to getdata')
# timebase in secs (ms in raw data) - could add a preferred unit?
# this is partly allowed for in savez_compressed, newload, and
# for plotting, in the config file.
# important that the 1e-3 be inside the Timebase()
output_data = TimeseriesData(timebase=Timebase(
1e-3 * getrets[1::2][0:isample]),
signal=Signal(getrets[2::2][0:isample]),
channels=ch)
output_data.meta.update({'shot': self.shot})
if pyfusion.VERBOSE > 0: print('HJ config name', self.config_name)
output_data.config_name = self.config_name
stprms = get_static_params(shot=self.shot, signal=self.path)
if len(list(stprms)
) == 0: # maybe this should be ignored - how do we use it?
raise LookupError(
' failure to get params for {shot}:{path}'.format(
shot=self.shot, path=self.path))
output_data.params = stprms
return output_data
def fetch(self, interp_if_diff=True):
"""Fetch each channel and combine into a multichannel instance
of :py:class:`~pyfusion.data.timeseries.TimeseriesData`.
:rtype: :py:class:`~pyfusion.data.timeseries.TimeseriesData`
"""
## initially, assume only single channel signals
ordered_channel_names = self.ordered_channel_names()
data_list = []
channels = ChannelList()
timebase = None
meta_dict = {}
#from scipy.io import netcdf
#home = os.environ['HOME']
#os.system('mkdir -p {}/tmp_pyfusion/'.format(home))
#fname = '{}/tmp_pyfusion/{}.nc'.format(home,self.shot)
#if os.path.exists(fname):
# NC = netcdf.netcdf_file(fname,'r',version=2)
#else:
# NC = netcdf.netcdf_file(fname,'w',version=2)
for chan in ordered_channel_names:
fetcher_class = import_setting('Diagnostic', chan, 'data_fetcher')
tmp_data = fetcher_class(self.acq, self.shot,
config_name=chan).fetch()
channels.append(tmp_data.channels)
meta_dict.update(tmp_data.meta)
if timebase is None:
timebase = tmp_data.timebase
data_list.append(tmp_data.signal)
else:
try:
assert_array_almost_equal(timebase, tmp_data.timebase)
data_list.append(tmp_data.signal)
except:
if interp_if_diff:
data_list.append(
np.interp(timebase, tmp_data.timebase,
tmp_data.signal))
else:
raise
#NC.close()
signal = Signal(data_list)
output_data = TimeseriesData(signal=signal,
timebase=timebase,
channels=channels)
#output_data.meta.update({'shot':self.shot})
output_data.meta.update(meta_dict)
return output_data
def do_fetch(self):
channel_length = int(self.length)
outdata=np.zeros(1024*2*256+1)
with tempfile.NamedTemporaryFile(prefix="pyfusion_") as outfile:
getrets=gethjdata.gethjdata(self.shot,channel_length,self.path,
VERBOSE, OPT,
outfile.name, outdata)
ch = Channel(self.path,
Coords('dummy', (0,0,0)))
output_data = TimeseriesData(timebase=Timebase(getrets[1::2]),
signal=Signal(getrets[2::2]), channels=ch)
output_data.meta.update({'shot':self.shot})
return output_data
def do_fetch(self):
delimiter = self.__dict__.get("delimiter", None)
data = genfromtxt(self.filename.replace("(shot)", str(self.shot)),
unpack=True,
delimiter=delimiter)
# len(data) is number of channels + 1 (timebase)
n_channels = len(data) - 1
ch_generator = (generic_ch(i) for i in range(n_channels))
ch = ChannelList(*ch_generator)
return TimeseriesData(timebase=Timebase(data[0]),
signal=Signal(data[1:]),
channels=ch)
def do_fetch(self):
print(self.pointname)
print(self.shot)
if self.NC!=None:
print(self.NC)
t_name = '{}_time'.format(self.pointname)
NC_vars = self.NC.variables.keys()
if self.pointname in NC_vars:
print('Reading cache!!!!')
t_axis = self.NC.variables[t_name].data[:].copy()
data = self.NC.variables[self.pointname].data[:].copy()
else:
tmp = self.acq.connection.get('ptdata2("{}",{})'.format(self.pointname, self.shot))
data = tmp.data()
tmp = self.acq.connection.get('dim_of(ptdata2("{}",{}))'.format(self.pointname, self.shot))
t_axis = tmp.data()
self.write_cache = True
print(t_axis)
print(data)
coords = get_coords_for_channel(**self.__dict__)
ch = Channel(self.pointname, coords)
# con=MDS.Connection('atlas.gat.com::')
# pointname = 'MPI66M067D'
# shot = 164950
# tmp = con.get('ptdata2("{}",{})'.format(pointname, shot))
# dat = tmp.data()
# tmp = con.get('dim_of(ptdata2("{}",{}))'.format(pointname, shot))
# t = tmp.data()
if self.NC!=None and self.write_cache:
print self.pointname
self.NC.createDimension(t_name, len(t_axis))
f_time = self.NC.createVariable(t_name,'d',(t_name,))
f_time[:] = +t_axis
print('Wrote time')
sig = self.NC.createVariable(self.pointname,'f',(t_name,))
sig[:] = +data
print('Wrote signal')
output_data = TimeseriesData(timebase=Timebase(t_axis),
signal=Signal(data), channels=ch)
# output_data = super(DIIIDDataFetcherPTdata, self).do_fetch()
# coords = get_coords_for_channel(**self.__dict__)
# ch = Channel(self.mds_path, coords)
# output_data.channels = ch
# output_data.meta.update({'shot':self.shot, 'kh':self.get_kh()})
# print(ch)
output_data.config_name = ch
self.fetch_mode = 'ptdata'
return output_data
def get_multimode_test_data(channels = get_n_channels(DEFAULT_N_CHANNELS),
timebase = DEFAULT_TIMEBASE,
modes = [mode_1, mode_2], noise = DEFAULT_NOISE):
"""Generate synthetic multi-channel data for testing."""
n_channels = len(channels)
data_size = (n_channels, timebase.size)
data_array = noise*2*(np.random.random(data_size)-0.5)
timebase_matrix = np.resize(timebase, data_size)
angle_matrix = np.resize(np.array([i.coords.cylindrical[1] for i in channels]),
data_size[::-1]).T
for m in modes:
data_array += m['amp']*np.cos(2*np.pi*m['freq']*timebase_matrix +
m['mode_number']*angle_matrix + m['phase'])
output = TimeseriesData(timebase=timebase, signal=Signal(data_array),
channels=channels)
return output
def testFilteredDataHistory_nocopy(self):
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
# nonzero signal mean
ch = get_n_channels(1)
tsd = TimeseriesData(timebase=tb,
signal=Signal(np.arange(len(tb))), channels=ch)
filtered_tsd = tsd.subtract_mean()
#self.assertEqual(len(filtered_tsd.history.split('\n')), 3)
self.assertEqual(len(filtered_tsd.history.split('\n')), 5) # bdb thinks extra 2 is OK
output_data = filtered_tsd.normalise(method='rms', copy=False)
#self.assertEqual(filtered_tsd.history.split('> ')[-1], "normalise(method='rms')")
self.assertEqual(filtered_tsd.history.split('> ')[-1].split('\n')[0], "normalise(method='rms')")
#self.assertEqual(output_data.history.split('> ')[-1], "normalise(method='rms')")
self.assertEqual(output_data.history.split('> ')[-1].split('\n')[0], "normalise(method='rms')")
def get_tsd_from_node(fetcher, node):
"""Return pyfusion TimeSeriesData corresponding to an MDSplus signal node."""
# TODO: load actual coordinates
ch = Channel(fetcher.mds_path_components['nodepath'],
Coords('dummy', (0, 0, 0)))
signal = Signal(node.data())
dim = node.dim_of().data()
# TODO: stupid hack, the test signal has dim of [[...]], real data
# has [...]. Figure out why. (...probably because original signal
# uses a build_signal function)
if len(dim) == 1:
dim = dim[0]
timebase = Timebase(dim)
output_data = TimeseriesData(timebase=timebase, signal=signal, channels=ch)
output_data.meta.update({'shot': fetcher.shot})
return output_data
def do_fetch(self):
# evaluate filename list
try:
filenames = eval(self.__dict__.get("filenames", "[]"))
except TypeError:
# assume we have been given a list of filenames as a keyword argument, rather than
# reading the config file.
filenames = self.__dict__.get("filenames")
data_array = []
channel_names = []
dtypes = []
for fn_i, fn in enumerate(filenames):
dt = eval(self.__dict__.get("dtype_%d" % (fn_i + 1), None))
dtypes.append(dt)
if fn.endswith('.bz2'):
f = bz2.BZ2File(fn.replace("(shot)", str(self.shot)))
data_array.append(np.fromstring(f.read(), dtype=dt))
f.close()
else:
data_array.append(
np.fromfile(fn.replace("(shot)", str(self.shot)),
dtype=dt))
channel_names.extend(
[i for i in dt.names if i.startswith('channel_')])
ch_generator = (named_ch(i) for i in channel_names)
ch = ChannelList(*ch_generator)
signal_data = np.zeros((len(channel_names), data_array[0].shape[0]),
dtype=dtypes[0][channel_names[0]])
sig_counter = 0
for d_i, d in enumerate(data_array):
for ch_name in dtypes[d_i].names:
if ch_name.startswith('channel_'):
signal_data[sig_counter, :] = d[ch_name]
sig_counter += 1
tsd = TimeseriesData(timebase=Timebase(data_array[0]['timebase']),
signal=Signal(signal_data),
channels=ch)
tsd.phase_pairs = self.__dict__.get("phase_pairs", None)
if tsd.phase_pairs != None:
tsd.phase_pairs = eval(tsd.phase_pairs)
return tsd
def fetch(self, interp_if_diff = True):
"""Fetch each channel and combine into a multichannel instance
of :py:class:`~pyfusion.data.timeseries.TimeseriesData`.
:rtype: :py:class:`~pyfusion.data.timeseries.TimeseriesData`
"""
print('******** hello world ***********')
## initially, assume only single channel signals
ordered_channel_names = self.ordered_channel_names()
data_list = []
channels = ChannelList()
timebase = None
meta_dict={}
from scipy.io import netcdf
fname = '/u/haskeysr/tmp/{}.nc'.format(self.shot)
write_cache=False; read_cache=False
if os.path.exists(fname):
NC = netcdf.netcdf_file(fname,'a',version=2)
else:
NC = netcdf.netcdf_file(fname,'w',version=2)
for chan in ordered_channel_names:
fetcher_class = import_setting('Diagnostic', chan, 'data_fetcher')
tmp_data = fetcher_class(self.acq, self.shot,
config_name=chan, NC=NC).fetch()
channels.append(tmp_data.channels)
meta_dict.update(tmp_data.meta)
if timebase == None:
timebase = tmp_data.timebase
data_list.append(tmp_data.signal)
else:
try:
assert_array_almost_equal(timebase, tmp_data.timebase)
data_list.append(tmp_data.signal)
except:
if interp_if_diff:
data_list.append(np.interp(timebase, tmp_data.timebase, tmp_data.signal))
else:
raise
NC.close()
signal=Signal(data_list)
output_data = TimeseriesData(signal=signal, timebase=timebase,
channels=channels)
#output_data.meta.update({'shot':self.shot})
output_data.meta.update(meta_dict)
return output_data
def testFilteredDataHistory_copy(self):
""" make sure that _copy version does NOT alter original
"""
tb = generate_timebase(t0=-0.5, n_samples=1.e2, sample_freq=1.e2)
# nonzero signal mean
ch = get_n_channels(1)
tsd = TimeseriesData(timebase=tb,
signal=Signal(np.arange(len(tb))), channels=ch)
filtered_tsd = tsd.subtract_mean()
# bdb in 4 places, assume that the xtra info (norm_value) is supposed to be there.
#self.assertEqual(len(filtered_tsd.history.split('\n')), 3)
self.assertEqual(len(filtered_tsd.history.split('\n')), 5) # bdb thinks extra 2 is OK
output_data = filtered_tsd.normalise(method='rms', copy=True)
#self.assertEqual(output_data.history.split('> ')[-1], "normalise(method='rms')")
self.assertEqual(output_data.history.split('> ')[-1].split('\n')[0], "normalise(method='rms')")
#self.assertEqual(filtered_tsd.history.split('> ')[-1], "subtract_mean()")
self.assertEqual(filtered_tsd.history.split('> ')[-1].split('\n')[0], "subtract_mean()")
def do_fetch(self):
print("Shot: {}\nPoint Name: {}".format(self.shot, self.pointname))
if not hasattr(self, 'NC'):
self.NC = None
if self.NC is not None:
#print(self.NC)
t_name = '{}_time'.format(self.pointname)
NC_vars = self.NC.variables.keys()
else:
NC_vars = []
if self.pointname in NC_vars:
print(' Pointname in NC cache, Reading...\n')
t_axis = self.NC.variables[t_name].data[:].copy()
data = self.NC.variables[self.pointname].data[:].copy()
self.write_cache = False
else:
print(' Fetching from ptdata')
tmp = self.acq.connection.get('ptdata2("{}",{})'.format(
self.pointname, self.shot))
data = tmp.data()
tmp = self.acq.connection.get('dim_of(ptdata2("{}",{}))'.format(
self.pointname, self.shot))
t_axis = tmp.data()
self.write_cache = True
coords = get_coords_for_channel(**self.__dict__)
ch = Channel(self.pointname, coords)
if self.NC is not None and self.write_cache:
print("\t Writing to NC file disabled temporarily.")
#print(' Writing pointname to NC file\n')
#self.NC.createDimension(t_name, len(t_axis))
#f_time = self.NC.createVariable(t_name,'d',(t_name,))
#f_time[:] = +t_axis
# sig = self.NC.createVariable(self.pointname,'f',(t_name,))
#sig[:] = +data
output_data = TimeseriesData(timebase=Timebase(t_axis),
signal=Signal(data),
channels=ch)
output_data.config_name = ch
self.fetch_mode = 'ptdata'
return output_data
def do_fetch(self):
print self.shot, self.senal
data_dim = tjiidata.dimens(self.shot, self.senal)
if data_dim[0] < MAX_SIGNAL_LENGTH:
data_dict = tjiidata.lectur(self.shot, self.senal, data_dim[0],
data_dim[0], data_dim[1])
else:
raise ValueError, 'Not loading data to avoid segmentation fault in tjiidata.lectur'
ch = Channel(self.senal, Coords('dummy', (0, 0, 0)))
if self.invert == 'true': #yuk - TODO: use boolean type from config
s = Signal(-np.array(data_dict['y']))
else:
s = Signal(np.array(data_dict['y']))
output_data = TimeseriesData(timebase=Timebase(data_dict['x']),
signal=s,
channels=ch)
output_data.meta.update({'shot': self.shot})
return output_data
def do_fetch(self):
sig = self.conn.get(self.mds_path)
dim = self.conn.get('DIM_OF(' + self.mds_path + ')')
scl = 1.0
coords = get_coords_for_channel(**self.__dict__)
ch = Channel(self.config_name, coords)
timedata = dim.data()
output_data = TimeseriesData(timebase=Timebase(1e-9 * timedata),
signal=scl * Signal(sig),
channels=ch)
output_data.meta.update({'shot': self.shot})
if hasattr(self, 'mdsshot'): # intended for checks - not yet used.
output_data.mdsshot = self.mdsshot
output_data.config_name = self.config_name
output_data.utc = [timedata[0], timedata[-1]]
#output_data.units = dat['units'] if 'units' in dat else ''
debug_(pyfusion.DEBUG,
level=1,
key='W7M_do_fetch',
msg='entering W7X MDS do_fetch')
return (output_data)
def fetch(self, interp_if_diff=True):
"""Fetch each channel and combine into a multichannel instance
of :py:class:`~pyfusion.data.timeseries.TimeseriesData`.
:rtype: :py:class:`~pyfusion.data.timeseries.TimeseriesData`
"""
## initially, assume only single channel signals
ordered_channel_names = self.ordered_channel_names()
data_list = []
channels = ChannelList()
timebase = None
meta_dict = {}
for chan in ordered_channel_names:
fetcher_class = import_setting('Diagnostic', chan, 'data_fetcher')
tmp_data = fetcher_class(self.acq, self.shot,
config_name=chan).fetch()
channels.append(tmp_data.channels)
meta_dict.update(tmp_data.meta)
if timebase == None:
timebase = tmp_data.timebase
data_list.append(tmp_data.signal)
else:
try:
assert_array_almost_equal(timebase, tmp_data.timebase)
data_list.append(tmp_data.signal)
except:
if interp_if_diff:
data_list.append(
np.interp(timebase, tmp_data.timebase,
tmp_data.signal))
else:
raise
signal = Signal(data_list)
output_data = TimeseriesData(signal=signal,
timebase=timebase,
channels=channels)
#output_data.meta.update({'shot':self.shot})
output_data.meta.update(meta_dict)
return output_data
def do_fetch(self):
chan_name = (self.diag_name.split('-'))[-1] # remove -
filename_dict = {'shot':self.shot, # goes with Boyd's local stg
'config_name':self.config_name}
#filename_dict = {'diag_name':self.diag_name, # goes with retrieve names
# 'channel_number':self.channel_number,
# 'shot':self.shot}
debug_(pf.DEBUG, 4, key='local_fetch')
for each_path in pf.config.get('global', 'localdatapath').split('+'):
self.basename = path.join(each_path, data_filename %filename_dict)
files_exist = path.exists(self.basename)
if files_exist: break
if not files_exist:
raise Exception("file {fn} not found. (localdatapath was {p})"
.format(fn=self.basename,
p=pf.config.get('global',
'localdatapath').split('+')))
else:
signal_dict = newload(self.basename)
if ((chan_name == array(['MP5','HMP13','HMP05'])).any()):
flip = -1.
print('flip')
else: flip = 1.
if self.diag_name[0]=='-': flip = -flip
# coords = get_coords_for_channel(**self.__dict__)
ch = Channel(self.diag_name, Coords('dummy', (0,0,0)))
output_data = TimeseriesData(timebase=Timebase(signal_dict['timebase']),
signal=Signal(flip*signal_dict['signal']), channels=ch)
# bdb - used "fetcher" instead of "self" in the "direct from LHD data" version
output_data.config_name = self.config_name # when using saved files, same as name
output_data.meta.update({'shot':self.shot})
return output_data