def downsample(input_data, skip=10, chan=None, copy=False): """ downsample by a factor 'skip' default 10 Good example of filter that changes the size of the data. """ from .base import DataSet from .timeseries import TimeseriesData if isinstance(input_data, DataSet): output_dataset = DataSet() for ii, data in enumerate(input_data): try: output_dataset.update(data.downsample(skip)) except AttributeError: pyfusion.logger.warning( "Data filter 'downsample' not applied to item in dataset") return output_dataset # python3 check this if input_data.signal.ndim == 1: tmp_data = TimeseriesData(timebase=input_data.timebase[::skip], signal=input_data.signal[::skip], channels=input_data.channels, bypass_length_check=True) else: tmp_data = TimeseriesData(timebase=input_data.timebase[::skip], signal=input_data.signal[:, ::skip], channels=input_data.channels, bypass_length_check=True) tmp_data.meta = input_data.meta.copy() tmp_data.history = input_data.history # bdb - may be redundant now meta is copied if hasattr(input_data, 'utc'): tmp_data.utc = input_data.utc return tmp_data
def segment(input_data, n_samples, overlap=DEFAULT_SEGMENT_OVERLAP): """Break into segments length n_samples. Overlap of 2.0 starts a new segment halfway into previous, overlap=1 is no overlap. overlap should divide into n_samples. Probably should consider a nicer definition such as in pyfusion 0 """ from pyfusion.data.base import DataSet from pyfusion.data.timeseries import TimeseriesData if isinstance(input_data, DataSet): output_dataset = DataSet() for ii,data in enumerate(input_data): try: output_dataset.update(data.segment(n_samples)) except AttributeError: pyfusion.logger.warning("Data filter 'segment' not applied to item in dataset") return output_dataset output_data = DataSet('segmented_%s, %d samples, %.3f overlap' %(datetime.now(), n_samples, overlap)) for el in arange(0,len(input_data.timebase), n_samples/overlap): if input_data.signal.ndim == 1: tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples], signal=input_data.signal[el:el+n_samples], channels=input_data.channels, bypass_length_check=True) else: tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples], signal=input_data.signal[:,el:el+n_samples], channels=input_data.channels, bypass_length_check=True) tmp_data.meta = input_data.meta.copy() tmp_data.history = input_data.history # bdb - may be redundant now meta is copied output_data.add(tmp_data) return output_data
def segment(input_data, n_samples, overlap=DEFAULT_SEGMENT_OVERLAP): """Break into segments length n_samples. Overlap of 2.0 starts a new segment halfway into previous, overlap=1 is no overlap. overlap should divide into n_samples. Probably should consider a nicer definition such as in pyfusion 0 n_samples < 1 implies a time interval, which is adjusted to suit fft otherwise n_samples is taken literally and not adjusted. fractional n_samples>1 allows the step size to be fine-tuned. """ from .base import DataSet from .timeseries import TimeseriesData if n_samples < 1: dt = np.average(np.diff(input_data.timebase)) n_samples = next_nice_number(n_samples / dt) if pyfusion.VERBOSE > 0: print('used {n} sample segments'.format(n=n_samples)) if isinstance(input_data, DataSet): output_dataset = DataSet() for ii, data in enumerate(input_data): try: output_dataset.update(data.segment(n_samples)) except AttributeError: pyfusion.logger.warning( "Data filter 'segment' not applied to item in dataset") return output_dataset output_data = DataSet('segmented_%s, %d samples, %.3f overlap' % (datetime.now(), n_samples, overlap)) # python3 check this ## for el in range(0,len(input_data.timebase), int(n_samples/overlap)): ## prior to python3 was for el in arange(0,len(input_data.timebase), n_samples/overlap): # this FP version allows locking to a precise frequency by use of a non-integral number of samples for el in arange(0, len(input_data.timebase), n_samples / float(overlap)): nsint = int(n_samples) el = int(el + 0.5) if input_data.signal.ndim == 1: tmp_data = TimeseriesData(timebase=input_data.timebase[el:el + nsint], signal=input_data.signal[el:el + nsint], channels=input_data.channels, bypass_length_check=True) else: tmp_data = TimeseriesData( timebase=input_data.timebase[el:el + nsint], signal=input_data.signal[:, el:el + nsint], channels=input_data.channels, bypass_length_check=True) tmp_data.meta = input_data.meta.copy() tmp_data.history = input_data.history # bdb - may be redundant now meta is copied output_data.add(tmp_data) return output_data
def segment(input_data, n_samples, overlap=1.0, datalist= 0): """Break into segments length n_samples. Overlap of 2.0 starts a new segment halfway into previous, overlap=1 is no overlap. overlap should divide into n_samples. Probably should consider a nicer definition such as in pyfusion 0 if datalist = 0 returns a DataSet object otherwise, returns a OrderedDataSet object """ from pyfusion.data.base import DataSet, OrderedDataSet from pyfusion.data.timeseries import TimeseriesData if isinstance(input_data, DataSet): output_dataset = DataSet() for ii,data in enumerate(input_data): try: output_dataset.update(data.segment(n_samples)) except AttributeError: pyfusion.logger.warning("Data filter 'segment' not applied to item in dataset") return output_dataset #SH modification incase ordering is important... i.e you are doing #two processing two different arrays at the same time (in different Timeseries objects) #and you don't want to loose the time relationship between them if datalist: output_data = OrderedDataSet('segmented_%s, %d samples, %.3f overlap' %(datetime.now(), n_samples, overlap)) else: output_data = DataSet('segmented_%s, %d samples, %.3f overlap' %(datetime.now(), n_samples, overlap)) #SH : 24May2013 fixed bug here - before, the index was allowed to go past #the length of samples, gives smalled length data towards the end - fixed to finish the #last time we can get n_samples length #for el in arange(0,len(input_data.timebase), n_samples/overlap): for el in arange(0,len(input_data.timebase) - n_samples, n_samples/overlap): if input_data.signal.ndim == 1: tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples], signal=input_data.signal[el:el+n_samples], channels=input_data.channels, bypass_length_check=True) else: tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples], signal=input_data.signal[:,el:el+n_samples], channels=input_data.channels, bypass_length_check=True) tmp_data.meta = input_data.meta.copy() if datalist: output_data.append(tmp_data) else: output_data.add(tmp_data) return output_data
def segment(input_data, n_samples, overlap=1.0, datalist= 0): """Break into segments length n_samples. Overlap of 2.0 starts a new segment halfway into previous, overlap=1 is no overlap. overlap should divide into n_samples. Probably should consider a nicer definition such as in pyfusion 0 if datalist = 0 returns a DataSet object otherwise, returns a OrderedDataSet object """ from pyfusion.data.base import DataSet, OrderedDataSet from pyfusion.data.timeseries import TimeseriesData if isinstance(input_data, DataSet): output_dataset = DataSet() for ii,data in enumerate(input_data): try: output_dataset.update(data.segment(n_samples)) except AttributeError: pyfusion.logger.warning("Data filter 'segment' not applied to item in dataset") return output_dataset #SH modification incase ordering is important... i.e you are doing #two processing two different arrays at the same time (in different Timeseries objects) #and you don't want to loose the time relationship between them if datalist: output_data = OrderedDataSet('segmented_%s, %d samples, %.3f overlap' %(datetime.now(), n_samples, overlap)) else: output_data = DataSet('segmented_%s, %d samples, %.3f overlap' %(datetime.now(), n_samples, overlap)) #SH : 24May2013 fixed bug here - before, the index was allowed to go past #the length of samples, gives smalled length data towards the end - fixed to finish the #last time we can get n_samples length #for el in arange(0,len(input_data.timebase), n_samples/overlap): for el in arange(0,len(input_data.timebase) - n_samples, n_samples/overlap): if input_data.signal.ndim == 1: tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples], signal=input_data.signal[el:el+n_samples], channels=input_data.channels, bypass_length_check=True) else: tmp_data = TimeseriesData(timebase=input_data.timebase[el:el+n_samples], signal=input_data.signal[:,el:el+n_samples], channels=input_data.channels, bypass_length_check=True) tmp_data.meta = input_data.meta.copy() if datalist: output_data.append(tmp_data) else: output_data.add(tmp_data) return output_data
def flucstruc(input_data, min_dphase=-pi, group=fs_group_geometric, method='rms', separate=True, label=None, segment=0, segment_overlap=DEFAULT_SEGMENT_OVERLAP): """If segment is 0, then we dont segment the data (assume already done)""" from pyfusion.data.base import DataSet from pyfusion.data.timeseries import FlucStruc if label: fs_dataset = DataSet(label) else: fs_dataset = DataSet('flucstrucs_%s' % datetime.now()) if segment > 0: for seg in input_data.segment(segment, overlap=segment_overlap): fs_dataset.update( seg.flucstruc(min_dphase=min_dphase, group=group, method=method, separate=separate, label=label, segment=0)) return fs_dataset svd_data = input_data.subtract_mean().normalise(method, separate).svd() for fs_gr in group(svd_data): tmp = FlucStruc(svd_data, fs_gr, input_data.timebase, min_dphase=min_dphase, phase_pairs=input_data.__dict__.get( "phase_pairs", None)) tmp.meta = input_data.meta tmp.history = svd_data.history tmp.scales = svd_data.scales tmp.norm_method = svd_data.norm_method fs_dataset.add(tmp) return fs_dataset
def segment(input_data, n_samples, overlap=DEFAULT_SEGMENT_OVERLAP): """Break into segments length n_samples. Overlap of 2.0 starts a new segment halfway into previous, overlap=1 is no overlap. overlap should divide into n_samples. Probably should consider a nicer definition such as in pyfusion 0 """ from pyfusion.data.base import DataSet from pyfusion.data.timeseries import TimeseriesData if isinstance(input_data, DataSet): output_dataset = DataSet() for ii, data in enumerate(input_data): try: output_dataset.update(data.segment(n_samples)) except AttributeError: pyfusion.logger.warning( "Data filter 'segment' not applied to item in dataset") return output_dataset output_data = DataSet('segmented_%s, %d samples, %.3f overlap' % (datetime.now(), n_samples, overlap)) for el in arange(0, len(input_data.timebase), n_samples / overlap): if input_data.signal.ndim == 1: tmp_data = TimeseriesData( timebase=input_data.timebase[el:el + n_samples], signal=input_data.signal[el:el + n_samples], channels=input_data.channels, bypass_length_check=True) else: tmp_data = TimeseriesData( timebase=input_data.timebase[el:el + n_samples], signal=input_data.signal[:, el:el + n_samples], channels=input_data.channels, bypass_length_check=True) tmp_data.meta = input_data.meta.copy() tmp_data.history = input_data.history # bdb - may be redundant now meta is copied output_data.add(tmp_data) return output_data
def flucstruc(input_data, min_dphase = -pi, group=fs_group_geometric, method='rms', separate=True, label=None, segment=0, segment_overlap=DEFAULT_SEGMENT_OVERLAP): """If segment is 0, then we dont segment the data (assume already done)""" from pyfusion.data.base import DataSet from pyfusion.data.timeseries import FlucStruc if label: fs_dataset = DataSet(label) else: fs_dataset = DataSet('flucstrucs_%s' %datetime.now()) if segment > 0: for seg in input_data.segment(segment, overlap=segment_overlap): fs_dataset.update(seg.flucstruc(min_dphase=min_dphase, group=group, method=method, separate=separate, label=label, segment=0)) return fs_dataset svd_data = input_data.subtract_mean().normalise(method, separate).svd() for fs_gr in group(svd_data): tmp = FlucStruc(svd_data, fs_gr, input_data.timebase, min_dphase=min_dphase, phase_pairs=input_data.__dict__.get("phase_pairs",None)) tmp.meta = input_data.meta tmp.history = svd_data.history tmp.scales = svd_data.scales fs_dataset.add(tmp) return fs_dataset