def create_dataset(dataset, look_back_1, look_back_2, look_forward): window_size = 40 tau = 5 dataset_padded = np.pad(dataset, ((int(window_size / 2), int(window_size / 2) - 1), (0, 0)), mode='edge') win = windows.exponential(window_size, tau=tau) win_past = windows.exponential(window_size, tau=tau) win_past[int(window_size / 2):] = 0 # Don't look to future signals = np.apply_along_axis( lambda m: convolve(m, win, mode='valid') / sum(win), axis=0, arr=dataset_padded) signals_past = np.apply_along_axis( lambda m: convolve(m, win_past, mode='valid') / sum(win_past), axis=0, arr=dataset_padded) filtered_sig_grad = np.gradient(signals, axis=0) filtered_sig_grad_past = np.gradient(signals_past, axis=0) if signals.shape != signals_past.shape != dataset.shape: raise Exception('Convolution Error') max_look_back = max([look_back_1, look_back_2]) data_x_1, data_x_2, data_y = [], [], [] for i in range(filtered_sig_grad.shape[0] - max_look_back - look_forward): a = filtered_sig_grad_past[(i + max_look_back - look_back_1):(i + max_look_back), :] data_x_1.append(a.T) a = filtered_sig_grad_past[(i + max_look_back - look_back_2):(i + max_look_back), :] data_x_2.append(a.T) data_y.append( sigmoid( np.mean(filtered_sig_grad[(i + max_look_back):(i + max_look_back + look_forward), :], axis=0))) data_x_1 = np.asarray(data_x_1) data_x_2 = np.asarray(data_x_2) data_y = np.asarray(data_y) return data_x_1, data_x_2, data_y
def test_exponential(): for k, v in exponential_data.items(): if v is None: assert_raises(ValueError, windows.exponential, *k) else: win = windows.exponential(*k) assert_allclose(win, v, rtol=1e-14)
def test_exponential(): for k, v in exponential_data.items(): if v is None: assert_raises(ValueError, windows.exponential, *k) else: win = windows.exponential(*k) assert_allclose(win, v, rtol=1e-14)
t.log(base=10) try: t.exp() except: pass else: raise Exception("This should have failed!") print(q) q.iscyclic('X') r = q.convolution_filter([0.1, 0.15, 0.5, 0.15, 0.1], axis='X') print(r) print(q.dimension_coordinate('X').bounds.array) print(r.dimension_coordinate('X').bounds.array) from scipy.signal import windows exponential_weights = windows.exponential(3) print(exponential_weights) r = q.convolution_filter(exponential_weights, axis='Y') print(r.array) r = q.derivative('X') r = q.derivative('Y', one_sided_at_boundary=True) u, v = cf.read('wind_components.nc') zeta = cf.relative_vorticity(u, v) print(zeta) print(zeta.array.round(8)) print("\n**Aggregation**\n") a = cf.read('air_temperature.nc')[0] a a_parts = [a[0, :, 0:30], a[0, :, 30:96], a[1, :, 0:30], a[1, :, 30:96]]
def pre_processing(self): """ Complete various pre-processing steps for encoded protein sequences before doing any of the DSP-related functions or transformations. Zero-pad the sequences, remove any +/- infinity or NAN values, get the approximate protein spectra and window function parameter names. Parameters ---------- :self (PyDSP object): instance of PyDSP class. Returns ------- None """ #zero-pad encoded sequences so they are all the same length self.protein_seqs = zero_padding(self.protein_seqs) #get shape parameters of proteins seqs self.num_seqs = self.protein_seqs.shape[0] self.signal_len = self.protein_seqs.shape[1] #replace any positive or negative infinity or NAN values with 0 self.protein_seqs[self.protein_seqs == -np.inf] = 0 self.protein_seqs[self.protein_seqs == np.inf] = 0 self.protein_seqs[self.protein_seqs == np.nan] = 0 #replace any NAN's with 0's #self.protein_seqs.fillna(0, inplace=True) self.protein_seqs = np.nan_to_num(self.protein_seqs) #initialise zeros array to store all protein spectra self.fft_power = np.zeros((self.num_seqs, self.signal_len)) self.fft_real = np.zeros((self.num_seqs, self.signal_len)) self.fft_imag = np.zeros((self.num_seqs, self.signal_len)) self.fft_abs = np.zeros((self.num_seqs, self.signal_len)) #list of accepted spectra, window functions and filters all_spectra = ['power', 'absolute', 'real', 'imaginary'] all_windows = [ 'hamming', 'blackman', 'blackmanharris', 'gaussian', 'bartlett', 'kaiser', 'barthann', 'bohman', 'chebwin', 'cosine', 'exponential' 'flattop', 'hann', 'boxcar', 'hanning', 'nuttall', 'parzen', 'triang', 'tukey' ] all_filters = [ 'savgol', 'medfilt', 'symiirorder1', 'lfilter', 'hilbert' ] #set required input parameters, raise error if spectrum is none if self.spectrum == None: raise ValueError( 'Invalid input Spectrum type ({}) not available in valid spectra: {}' .format(self.spectrum, all_spectra)) else: #get closest correct spectra from user input, if no close match then raise error spectra_matches = (get_close_matches(self.spectrum, all_spectra, cutoff=0.4)) if spectra_matches == []: raise ValueError( 'Invalid input Spectrum type ({}) not available in valid spectra: {}' .format(self.spectrum, all_spectra)) else: self.spectra = spectra_matches[0] #closest match in array if self.window_type == None: self.window = 1 #window = 1 is the same as applying no window else: #get closest correct window function from user input window_matches = (get_close_matches(self.window, all_windows, cutoff=0.4)) #check if sym=True or sym=False #get window function specified by window input parameter, if no match then window = 1 if window_matches != []: if window_matches[0] == 'hamming': self.window = hamming(self.signal_len, sym=True) self.window_type = "hamming" elif window_matches[0] == "blackman": self.window = blackman(self.signal_len, sym=True) self.window = "blackman" elif window_matches[0] == "blackmanharris": self.window = blackmanharris(self.signal_len, sym=True) #** self.window_type = "blackmanharris" elif window_matches[0] == "bartlett": self.window = bartlett(self.signal_len, sym=True) self.window_type = "bartlett" elif window_matches[0] == "gaussian": self.window = gaussian(self.signal_len, std=7, sym=True) self.window_type = "gaussian" elif window_matches[0] == "kaiser": self.window = kaiser(self.signal_len, beta=14, sym=True) self.window_type = "kaiser" elif window_matches[0] == "hanning": self.window = hanning(self.signal_len, sym=True) self.window_type = "hanning" elif window_matches[0] == "barthann": self.window = barthann(self.signal_len, sym=True) self.window_type = "barthann" elif window_matches[0] == "bohman": self.window = bohman(self.signal_len, sym=True) self.window_type = "bohman" elif window_matches[0] == "chebwin": self.window = chebwin(self.signal_len, sym=True) self.window_type = "chebwin" elif window_matches[0] == "cosine": self.window = cosine(self.signal_len, sym=True) self.window_type = "cosine" elif window_matches[0] == "exponential": self.window = exponential(self.signal_len, sym=True) self.window_type = "exponential" elif window_matches[0] == "flattop": self.window = flattop(self.signal_len, sym=True) self.window_type = "flattop" elif window_matches[0] == "boxcar": self.window = boxcar(self.signal_len, sym=True) self.window_type = "boxcar" elif window_matches[0] == "nuttall": self.window = nuttall(self.signal_len, sym=True) self.window_type = "nuttall" elif window_matches[0] == "parzen": self.window = parzen(self.signal_len, sym=True) self.window_type = "parzen" elif window_matches[0] == "triang": self.window = triang(self.signal_len, sym=True) self.window_type = "triang" elif window_matches[0] == "tukey": self.window = tukey(self.signal_len, sym=True) self.window_type = "tukey" else: self.window = 1 #window = 1 is the same as applying no window #calculate convolution from protein sequences if self.convolution is not None: if self.window is not None: self.convoled_seqs = signal.convolve( self.protein_seqs, self.window, mode='same') / sum( self.window) if self.filter != None: #get closest correct filter from user input filter_matches = (get_close_matches(self.filter, all_filters, cutoff=0.4)) #set filter attribute according to approximate user input if filter_matches != []: if filter_matches[0] == 'savgol': self.filter = savgol_filter(self.signal_len, self.signal_len) elif filter_matches[0] == 'medfilt': self.filter = medfilt(self.signal_len) elif filter_matches[0] == 'symiirorder1': self.filter = symiirorder1(self.signal_len, c0=1, z1=1) elif filter_matches[0] == 'lfilter': self.filter = lfilter(self.signal_len) elif filter_matches[0] == 'hilbert': self.filter = hilbert(self.signal_len) else: self.filter = "" #no filter
t.exp() # Raises Exception except: pass q, t = cf.read('file.nc') print(q) print(q.array) print(q.coordinate('X').bounds.array) q.iscyclic('X') g = q.moving_window('mean', 3, axis='X', weights=True) print(g) print(g.array) print(g.coordinate('X').bounds.array) print(q) q.iscyclic('X') r = q.convolution_filter([0.1, 0.15, 0.5, 0.15, 0.1], axis='X') print(r) print(q.dimension_coordinate('X').bounds.array) print(r.dimension_coordinate('X').bounds.array) from scipy.signal import windows exponential_window = windows.exponential(3) print(exponential_window) r = q.convolution_filter(exponential_window, axis='Y') print(r.array) r = q.derivative('X') r = q.derivative('Y', one_sided_at_boundary=True) u, v = cf.read('wind_components.nc') zeta = cf.relative_vorticity(u, v) print(zeta) print(zeta.array.round(8))
def signal_to_training( # pylint: disable=too-many-locals self, signal: Union[Dict, List[Dict]] ) -> Tuple[np.ndarray, Tuple[np.ndarray, ...], np.ndarray, Dict[str, Any]]: """ Extract training data from the dataset :param signal: List of or single dataset entry :return: Time signals, Tuple of FFTs (with different windows), peak counting vector and config data """ dict_list = list(signal) if isinstance(signal, list) else list( (signal, )) # Initialize the return values time_length = len( dict_list[0]['signal']['time']['data']) # type: ignore length = int(time_length / 2) signals = np.zeros((0, time_length)) result_r = np.zeros((0, length)) result_b = np.zeros((0, length)) result_h = np.zeros((0, length)) result_m = np.zeros((0, length)) result_p = np.zeros((0, length)) answer = np.zeros((0, length)) config = { 'SNR': [], 'count': [], 'frequencies': [], 'amplitudes': [], 'minamplitude': [], 'mindist': [] } # type: Dict[str, Any] # Calculate window functions window_bartlett = np.bartlett(time_length) window_hanning = np.hanning(time_length) window_meyer = self._meyer_wavelet(time_length) window_poisson = exponential(time_length, sym=True, tau=(time_length / 2) * (8.69 / 60.0)) # Loop all data entries for data in dict_list: time = np.asarray(data['signal']['time']['data']) signals = np.concatenate( (signals, np.reshape(time, (1, ) + time.shape))) config['SNR'].append(data['signal']['SNR']) # Assemble the FFTs fft = np.fft.fft(time)[:length] / time_length result_r = np.concatenate( (result_r, np.reshape(fft, (1, ) + fft.shape))) fft = np.fft.fft(time * window_bartlett)[:length] / time_length result_b = np.concatenate( (result_b, np.reshape(fft, (1, ) + fft.shape))) fft = np.fft.fft(time * window_hanning)[:length] / time_length result_h = np.concatenate( (result_h, np.reshape(fft, (1, ) + fft.shape))) fft = np.fft.fft(time * window_meyer)[:length] / time_length result_m = np.concatenate( (result_m, np.reshape(fft, (1, ) + fft.shape))) fft = np.fft.fft(time * window_poisson)[:length] / time_length result_p = np.concatenate( (result_p, np.reshape(fft, (1, ) + fft.shape))) # Assemble all the frequencies and amplitudes count = 0 freqs = [] ampls = [] counting = np.zeros((1, length)) for subsig in data['signal']['parts']: if subsig['signal']['type'] == 'SingleOscillation': count += 1 freq = subsig['signal']['frequency'] counting[0, int(max(0, min(length - 1, round(freq))))] += 1 freqs.append(freq) ampls.append(subsig['signal']['amplitude']) config['count'].append(count) # Sort frequencies and amplitudes by frequency np_freqs = np.asarray(freqs) sorting = np.unravel_index(np.argsort(np_freqs), np_freqs.shape) np_freqs = np_freqs[sorting] np_ampls = np.asarray(ampls)[sorting] # Assemble some statistics config['mindist'].append( 999999. if len(np_freqs) < 2 else np.min(np.diff(np_freqs))) config['minamplitude'].append( np.min(np_ampls) if len(np_ampls) > 0 else 999999.) config['frequencies'].append(np_freqs) config['amplitudes'].append(np_ampls) answer = np.concatenate((answer, counting)) # Assemble results ffts = (result_r, result_b, result_h, result_m, result_p) return signals, ffts, answer, config
def butter_lowpass_filtfilt(data, cutoff=100, fs=2000, order=5): # b, a = butter_lowpass(cutoff, fs, order=order) # y = filtfilt(b, a, data) y = exponential(data) return y