def downsample(input_data, skip=10, chan=None, copy=False):
""" downsample by a factor 'skip' default 10
Good example of filter that changes the size of the data.
"""
from .base import DataSet
from .timeseries import TimeseriesData
if isinstance(input_data, DataSet):
output_dataset = DataSet()
for ii, data in enumerate(input_data):
try:
output_dataset.update(data.downsample(skip))
except AttributeError:
pyfusion.logger.warning(
"Data filter 'downsample' not applied to item in dataset")
return output_dataset
# python3 check this
if input_data.signal.ndim == 1:
tmp_data = TimeseriesData(timebase=input_data.timebase[::skip],
signal=input_data.signal[::skip],
channels=input_data.channels,
bypass_length_check=True)
else:
tmp_data = TimeseriesData(timebase=input_data.timebase[::skip],
signal=input_data.signal[:, ::skip],
channels=input_data.channels,
bypass_length_check=True)
tmp_data.meta = input_data.meta.copy()
tmp_data.history = input_data.history # bdb - may be redundant now meta is copied
if hasattr(input_data, 'utc'):
tmp_data.utc = input_data.utc
return tmp_data
def segment(input_data, n_samples, overlap=DEFAULT_SEGMENT_OVERLAP):
"""Break into segments length n_samples.
Overlap of 2.0 starts a new segment halfway into previous, overlap=1 is
no overlap. overlap should divide into n_samples. Probably should
consider a nicer definition such as in pyfusion 0
"""
from pyfusion.data.base import DataSet
from pyfusion.data.timeseries import TimeseriesData
if isinstance(input_data, DataSet):
output_dataset = DataSet()
for ii,data in enumerate(input_data):
try:
output_dataset.update(data.segment(n_samples))
except AttributeError:
pyfusion.logger.warning("Data filter 'segment' not applied to item in dataset")
return output_dataset
output_data = DataSet('segmented_%s, %d samples, %.3f overlap' %(datetime.now(), n_samples, overlap))
for el in arange(0,len(input_data.timebase), n_samples/overlap):
if input_data.signal.ndim == 1:
tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples],
signal=input_data.signal[el:el+n_samples],
channels=input_data.channels, bypass_length_check=True)
else:
tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples],
signal=input_data.signal[:,el:el+n_samples],
channels=input_data.channels, bypass_length_check=True)
tmp_data.meta = input_data.meta.copy()
tmp_data.history = input_data.history # bdb - may be redundant now meta is copied
output_data.add(tmp_data)
return output_data
def segment(input_data, n_samples, overlap=DEFAULT_SEGMENT_OVERLAP):
"""Break into segments length n_samples.
Overlap of 2.0 starts a new segment halfway into previous, overlap=1 is
no overlap. overlap should divide into n_samples. Probably should
consider a nicer definition such as in pyfusion 0
n_samples < 1 implies a time interval, which is adjusted to suit fft
otherwise n_samples is taken literally and not adjusted.
fractional n_samples>1 allows the step size to be fine-tuned.
"""
from .base import DataSet
from .timeseries import TimeseriesData
if n_samples < 1:
dt = np.average(np.diff(input_data.timebase))
n_samples = next_nice_number(n_samples / dt)
if pyfusion.VERBOSE > 0:
print('used {n} sample segments'.format(n=n_samples))
if isinstance(input_data, DataSet):
output_dataset = DataSet()
for ii, data in enumerate(input_data):
try:
output_dataset.update(data.segment(n_samples))
except AttributeError:
pyfusion.logger.warning(
"Data filter 'segment' not applied to item in dataset")
return output_dataset
output_data = DataSet('segmented_%s, %d samples, %.3f overlap' %
(datetime.now(), n_samples, overlap))
# python3 check this
## for el in range(0,len(input_data.timebase), int(n_samples/overlap)):
## prior to python3 was for el in arange(0,len(input_data.timebase), n_samples/overlap):
# this FP version allows locking to a precise frequency by use of a non-integral number of samples
for el in arange(0, len(input_data.timebase), n_samples / float(overlap)):
nsint = int(n_samples)
el = int(el + 0.5)
if input_data.signal.ndim == 1:
tmp_data = TimeseriesData(timebase=input_data.timebase[el:el +
nsint],
signal=input_data.signal[el:el + nsint],
channels=input_data.channels,
bypass_length_check=True)
else:
tmp_data = TimeseriesData(
timebase=input_data.timebase[el:el + nsint],
signal=input_data.signal[:, el:el + nsint],
channels=input_data.channels,
bypass_length_check=True)
tmp_data.meta = input_data.meta.copy()
tmp_data.history = input_data.history # bdb - may be redundant now meta is copied
output_data.add(tmp_data)
return output_data
def segment(input_data, n_samples, overlap=1.0, datalist= 0):
"""Break into segments length n_samples.
Overlap of 2.0 starts a new segment halfway into previous, overlap=1 is
no overlap. overlap should divide into n_samples. Probably should
consider a nicer definition such as in pyfusion 0
if datalist = 0 returns a DataSet object otherwise, returns a OrderedDataSet object
"""
from pyfusion.data.base import DataSet, OrderedDataSet
from pyfusion.data.timeseries import TimeseriesData
if isinstance(input_data, DataSet):
output_dataset = DataSet()
for ii,data in enumerate(input_data):
try:
output_dataset.update(data.segment(n_samples))
except AttributeError:
pyfusion.logger.warning("Data filter 'segment' not applied to item in dataset")
return output_dataset
#SH modification incase ordering is important... i.e you are doing
#two processing two different arrays at the same time (in different Timeseries objects)
#and you don't want to loose the time relationship between them
if datalist:
output_data = OrderedDataSet('segmented_%s, %d samples, %.3f overlap' %(datetime.now(), n_samples, overlap))
else:
output_data = DataSet('segmented_%s, %d samples, %.3f overlap' %(datetime.now(), n_samples, overlap))
#SH : 24May2013 fixed bug here - before, the index was allowed to go past
#the length of samples, gives smalled length data towards the end - fixed to finish the
#last time we can get n_samples length
#for el in arange(0,len(input_data.timebase), n_samples/overlap):
for el in arange(0,len(input_data.timebase) - n_samples, n_samples/overlap):
if input_data.signal.ndim == 1:
tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples],
signal=input_data.signal[el:el+n_samples],
channels=input_data.channels, bypass_length_check=True)
else:
tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples],
signal=input_data.signal[:,el:el+n_samples],
channels=input_data.channels, bypass_length_check=True)
tmp_data.meta = input_data.meta.copy()
if datalist:
output_data.append(tmp_data)
else:
output_data.add(tmp_data)
return output_data
def segment(input_data, n_samples, overlap=1.0, datalist= 0):
"""Break into segments length n_samples.
Overlap of 2.0 starts a new segment halfway into previous, overlap=1 is
no overlap. overlap should divide into n_samples. Probably should
consider a nicer definition such as in pyfusion 0
if datalist = 0 returns a DataSet object otherwise, returns a OrderedDataSet object
"""
from pyfusion.data.base import DataSet, OrderedDataSet
from pyfusion.data.timeseries import TimeseriesData
if isinstance(input_data, DataSet):
output_dataset = DataSet()
for ii,data in enumerate(input_data):
try:
output_dataset.update(data.segment(n_samples))
except AttributeError:
pyfusion.logger.warning("Data filter 'segment' not applied to item in dataset")
return output_dataset
#SH modification incase ordering is important... i.e you are doing
#two processing two different arrays at the same time (in different Timeseries objects)
#and you don't want to loose the time relationship between them
if datalist:
output_data = OrderedDataSet('segmented_%s, %d samples, %.3f overlap' %(datetime.now(), n_samples, overlap))
else:
output_data = DataSet('segmented_%s, %d samples, %.3f overlap' %(datetime.now(), n_samples, overlap))
#SH : 24May2013 fixed bug here - before, the index was allowed to go past
#the length of samples, gives smalled length data towards the end - fixed to finish the
#last time we can get n_samples length
#for el in arange(0,len(input_data.timebase), n_samples/overlap):
for el in arange(0,len(input_data.timebase) - n_samples, n_samples/overlap):
if input_data.signal.ndim == 1:
tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples],
signal=input_data.signal[el:el+n_samples],
channels=input_data.channels, bypass_length_check=True)
else:
tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples],
signal=input_data.signal[:,el:el+n_samples],
channels=input_data.channels, bypass_length_check=True)
tmp_data.meta = input_data.meta.copy()
if datalist:
output_data.append(tmp_data)
else:
output_data.add(tmp_data)
return output_data
def flucstruc(input_data,
min_dphase=-pi,
group=fs_group_geometric,
method='rms',
separate=True,
label=None,
segment=0,
segment_overlap=DEFAULT_SEGMENT_OVERLAP):
"""If segment is 0, then we dont segment the data (assume already done)"""
from pyfusion.data.base import DataSet
from pyfusion.data.timeseries import FlucStruc
if label:
fs_dataset = DataSet(label)
else:
fs_dataset = DataSet('flucstrucs_%s' % datetime.now())
if segment > 0:
for seg in input_data.segment(segment, overlap=segment_overlap):
fs_dataset.update(
seg.flucstruc(min_dphase=min_dphase,
group=group,
method=method,
separate=separate,
label=label,
segment=0))
return fs_dataset
svd_data = input_data.subtract_mean().normalise(method, separate).svd()
for fs_gr in group(svd_data):
tmp = FlucStruc(svd_data,
fs_gr,
input_data.timebase,
min_dphase=min_dphase,
phase_pairs=input_data.__dict__.get(
"phase_pairs", None))
tmp.meta = input_data.meta
tmp.history = svd_data.history
tmp.scales = svd_data.scales
tmp.norm_method = svd_data.norm_method
fs_dataset.add(tmp)
return fs_dataset
def segment(input_data, n_samples, overlap=DEFAULT_SEGMENT_OVERLAP):
"""Break into segments length n_samples.
Overlap of 2.0 starts a new segment halfway into previous, overlap=1 is
no overlap. overlap should divide into n_samples. Probably should
consider a nicer definition such as in pyfusion 0
"""
from pyfusion.data.base import DataSet
from pyfusion.data.timeseries import TimeseriesData
if isinstance(input_data, DataSet):
output_dataset = DataSet()
for ii, data in enumerate(input_data):
try:
output_dataset.update(data.segment(n_samples))
except AttributeError:
pyfusion.logger.warning(
"Data filter 'segment' not applied to item in dataset")
return output_dataset
output_data = DataSet('segmented_%s, %d samples, %.3f overlap' %
(datetime.now(), n_samples, overlap))
for el in arange(0, len(input_data.timebase), n_samples / overlap):
if input_data.signal.ndim == 1:
tmp_data = TimeseriesData(
timebase=input_data.timebase[el:el + n_samples],
signal=input_data.signal[el:el + n_samples],
channels=input_data.channels,
bypass_length_check=True)
else:
tmp_data = TimeseriesData(
timebase=input_data.timebase[el:el + n_samples],
signal=input_data.signal[:, el:el + n_samples],
channels=input_data.channels,
bypass_length_check=True)
tmp_data.meta = input_data.meta.copy()
tmp_data.history = input_data.history # bdb - may be redundant now meta is copied
output_data.add(tmp_data)
return output_data
def flucstruc(input_data, min_dphase = -pi, group=fs_group_geometric, method='rms', separate=True, label=None, segment=0, segment_overlap=DEFAULT_SEGMENT_OVERLAP):
"""If segment is 0, then we dont segment the data (assume already done)"""
from pyfusion.data.base import DataSet
from pyfusion.data.timeseries import FlucStruc
if label:
fs_dataset = DataSet(label)
else:
fs_dataset = DataSet('flucstrucs_%s' %datetime.now())
if segment > 0:
for seg in input_data.segment(segment, overlap=segment_overlap):
fs_dataset.update(seg.flucstruc(min_dphase=min_dphase, group=group, method=method, separate=separate, label=label, segment=0))
return fs_dataset
svd_data = input_data.subtract_mean().normalise(method, separate).svd()
for fs_gr in group(svd_data):
tmp = FlucStruc(svd_data, fs_gr, input_data.timebase, min_dphase=min_dphase, phase_pairs=input_data.__dict__.get("phase_pairs",None))
tmp.meta = input_data.meta
tmp.history = svd_data.history
tmp.scales = svd_data.scales
fs_dataset.add(tmp)
return fs_dataset
