def create_dataset(dataset, look_back_1, look_back_2, look_forward):
window_size = 40
tau = 5
dataset_padded = np.pad(dataset,
((int(window_size / 2), int(window_size / 2) - 1),
(0, 0)),
mode='edge')
win = windows.exponential(window_size, tau=tau)
win_past = windows.exponential(window_size, tau=tau)
win_past[int(window_size / 2):] = 0 # Don't look to future
signals = np.apply_along_axis(
lambda m: convolve(m, win, mode='valid') / sum(win),
axis=0,
arr=dataset_padded)
signals_past = np.apply_along_axis(
lambda m: convolve(m, win_past, mode='valid') / sum(win_past),
axis=0,
arr=dataset_padded)
filtered_sig_grad = np.gradient(signals, axis=0)
filtered_sig_grad_past = np.gradient(signals_past, axis=0)
if signals.shape != signals_past.shape != dataset.shape:
raise Exception('Convolution Error')
max_look_back = max([look_back_1, look_back_2])
data_x_1, data_x_2, data_y = [], [], []
for i in range(filtered_sig_grad.shape[0] - max_look_back - look_forward):
a = filtered_sig_grad_past[(i + max_look_back -
look_back_1):(i + max_look_back), :]
data_x_1.append(a.T)
a = filtered_sig_grad_past[(i + max_look_back -
look_back_2):(i + max_look_back), :]
data_x_2.append(a.T)
data_y.append(
sigmoid(
np.mean(filtered_sig_grad[(i +
max_look_back):(i + max_look_back +
look_forward), :],
axis=0)))
data_x_1 = np.asarray(data_x_1)
data_x_2 = np.asarray(data_x_2)
data_y = np.asarray(data_y)
return data_x_1, data_x_2, data_y
def test_exponential():
for k, v in exponential_data.items():
if v is None:
assert_raises(ValueError, windows.exponential, *k)
else:
win = windows.exponential(*k)
assert_allclose(win, v, rtol=1e-14)
def test_exponential():
for k, v in exponential_data.items():
if v is None:
assert_raises(ValueError, windows.exponential, *k)
else:
win = windows.exponential(*k)
assert_allclose(win, v, rtol=1e-14)
t.log(base=10)
try:
t.exp()
except:
pass
else:
raise Exception("This should have failed!")
print(q)
q.iscyclic('X')
r = q.convolution_filter([0.1, 0.15, 0.5, 0.15, 0.1], axis='X')
print(r)
print(q.dimension_coordinate('X').bounds.array)
print(r.dimension_coordinate('X').bounds.array)
from scipy.signal import windows
exponential_weights = windows.exponential(3)
print(exponential_weights)
r = q.convolution_filter(exponential_weights, axis='Y')
print(r.array)
r = q.derivative('X')
r = q.derivative('Y', one_sided_at_boundary=True)
u, v = cf.read('wind_components.nc')
zeta = cf.relative_vorticity(u, v)
print(zeta)
print(zeta.array.round(8))
print("\n**Aggregation**\n")
a = cf.read('air_temperature.nc')[0]
a
a_parts = [a[0, :, 0:30], a[0, :, 30:96], a[1, :, 0:30], a[1, :, 30:96]]
def pre_processing(self):
"""
Complete various pre-processing steps for encoded protein sequences before
doing any of the DSP-related functions or transformations. Zero-pad
the sequences, remove any +/- infinity or NAN values, get the approximate
protein spectra and window function parameter names.
Parameters
----------
:self (PyDSP object):
instance of PyDSP class.
Returns
-------
None
"""
#zero-pad encoded sequences so they are all the same length
self.protein_seqs = zero_padding(self.protein_seqs)
#get shape parameters of proteins seqs
self.num_seqs = self.protein_seqs.shape[0]
self.signal_len = self.protein_seqs.shape[1]
#replace any positive or negative infinity or NAN values with 0
self.protein_seqs[self.protein_seqs == -np.inf] = 0
self.protein_seqs[self.protein_seqs == np.inf] = 0
self.protein_seqs[self.protein_seqs == np.nan] = 0
#replace any NAN's with 0's
#self.protein_seqs.fillna(0, inplace=True)
self.protein_seqs = np.nan_to_num(self.protein_seqs)
#initialise zeros array to store all protein spectra
self.fft_power = np.zeros((self.num_seqs, self.signal_len))
self.fft_real = np.zeros((self.num_seqs, self.signal_len))
self.fft_imag = np.zeros((self.num_seqs, self.signal_len))
self.fft_abs = np.zeros((self.num_seqs, self.signal_len))
#list of accepted spectra, window functions and filters
all_spectra = ['power', 'absolute', 'real', 'imaginary']
all_windows = [
'hamming', 'blackman', 'blackmanharris', 'gaussian', 'bartlett',
'kaiser', 'barthann', 'bohman', 'chebwin', 'cosine', 'exponential'
'flattop', 'hann', 'boxcar', 'hanning', 'nuttall', 'parzen',
'triang', 'tukey'
]
all_filters = [
'savgol', 'medfilt', 'symiirorder1', 'lfilter', 'hilbert'
]
#set required input parameters, raise error if spectrum is none
if self.spectrum == None:
raise ValueError(
'Invalid input Spectrum type ({}) not available in valid spectra: {}'
.format(self.spectrum, all_spectra))
else:
#get closest correct spectra from user input, if no close match then raise error
spectra_matches = (get_close_matches(self.spectrum,
all_spectra,
cutoff=0.4))
if spectra_matches == []:
raise ValueError(
'Invalid input Spectrum type ({}) not available in valid spectra: {}'
.format(self.spectrum, all_spectra))
else:
self.spectra = spectra_matches[0] #closest match in array
if self.window_type == None:
self.window = 1 #window = 1 is the same as applying no window
else:
#get closest correct window function from user input
window_matches = (get_close_matches(self.window,
all_windows,
cutoff=0.4))
#check if sym=True or sym=False
#get window function specified by window input parameter, if no match then window = 1
if window_matches != []:
if window_matches[0] == 'hamming':
self.window = hamming(self.signal_len, sym=True)
self.window_type = "hamming"
elif window_matches[0] == "blackman":
self.window = blackman(self.signal_len, sym=True)
self.window = "blackman"
elif window_matches[0] == "blackmanharris":
self.window = blackmanharris(self.signal_len,
sym=True) #**
self.window_type = "blackmanharris"
elif window_matches[0] == "bartlett":
self.window = bartlett(self.signal_len, sym=True)
self.window_type = "bartlett"
elif window_matches[0] == "gaussian":
self.window = gaussian(self.signal_len, std=7, sym=True)
self.window_type = "gaussian"
elif window_matches[0] == "kaiser":
self.window = kaiser(self.signal_len, beta=14, sym=True)
self.window_type = "kaiser"
elif window_matches[0] == "hanning":
self.window = hanning(self.signal_len, sym=True)
self.window_type = "hanning"
elif window_matches[0] == "barthann":
self.window = barthann(self.signal_len, sym=True)
self.window_type = "barthann"
elif window_matches[0] == "bohman":
self.window = bohman(self.signal_len, sym=True)
self.window_type = "bohman"
elif window_matches[0] == "chebwin":
self.window = chebwin(self.signal_len, sym=True)
self.window_type = "chebwin"
elif window_matches[0] == "cosine":
self.window = cosine(self.signal_len, sym=True)
self.window_type = "cosine"
elif window_matches[0] == "exponential":
self.window = exponential(self.signal_len, sym=True)
self.window_type = "exponential"
elif window_matches[0] == "flattop":
self.window = flattop(self.signal_len, sym=True)
self.window_type = "flattop"
elif window_matches[0] == "boxcar":
self.window = boxcar(self.signal_len, sym=True)
self.window_type = "boxcar"
elif window_matches[0] == "nuttall":
self.window = nuttall(self.signal_len, sym=True)
self.window_type = "nuttall"
elif window_matches[0] == "parzen":
self.window = parzen(self.signal_len, sym=True)
self.window_type = "parzen"
elif window_matches[0] == "triang":
self.window = triang(self.signal_len, sym=True)
self.window_type = "triang"
elif window_matches[0] == "tukey":
self.window = tukey(self.signal_len, sym=True)
self.window_type = "tukey"
else:
self.window = 1 #window = 1 is the same as applying no window
#calculate convolution from protein sequences
if self.convolution is not None:
if self.window is not None:
self.convoled_seqs = signal.convolve(
self.protein_seqs, self.window, mode='same') / sum(
self.window)
if self.filter != None:
#get closest correct filter from user input
filter_matches = (get_close_matches(self.filter,
all_filters,
cutoff=0.4))
#set filter attribute according to approximate user input
if filter_matches != []:
if filter_matches[0] == 'savgol':
self.filter = savgol_filter(self.signal_len,
self.signal_len)
elif filter_matches[0] == 'medfilt':
self.filter = medfilt(self.signal_len)
elif filter_matches[0] == 'symiirorder1':
self.filter = symiirorder1(self.signal_len, c0=1, z1=1)
elif filter_matches[0] == 'lfilter':
self.filter = lfilter(self.signal_len)
elif filter_matches[0] == 'hilbert':
self.filter = hilbert(self.signal_len)
else:
self.filter = "" #no filter
t.exp() # Raises Exception
except:
pass
q, t = cf.read('file.nc')
print(q)
print(q.array)
print(q.coordinate('X').bounds.array)
q.iscyclic('X')
g = q.moving_window('mean', 3, axis='X', weights=True)
print(g)
print(g.array)
print(g.coordinate('X').bounds.array)
print(q)
q.iscyclic('X')
r = q.convolution_filter([0.1, 0.15, 0.5, 0.15, 0.1], axis='X')
print(r)
print(q.dimension_coordinate('X').bounds.array)
print(r.dimension_coordinate('X').bounds.array)
from scipy.signal import windows
exponential_window = windows.exponential(3)
print(exponential_window)
r = q.convolution_filter(exponential_window, axis='Y')
print(r.array)
r = q.derivative('X')
r = q.derivative('Y', one_sided_at_boundary=True)
u, v = cf.read('wind_components.nc')
zeta = cf.relative_vorticity(u, v)
print(zeta)
print(zeta.array.round(8))
def signal_to_training( # pylint: disable=too-many-locals
self, signal: Union[Dict, List[Dict]]
) -> Tuple[np.ndarray, Tuple[np.ndarray, ...], np.ndarray, Dict[str, Any]]:
"""
Extract training data from the dataset
:param signal: List of or single dataset entry
:return: Time signals, Tuple of FFTs (with different windows), peak counting vector and config data
"""
dict_list = list(signal) if isinstance(signal, list) else list(
(signal, ))
# Initialize the return values
time_length = len(
dict_list[0]['signal']['time']['data']) # type: ignore
length = int(time_length / 2)
signals = np.zeros((0, time_length))
result_r = np.zeros((0, length))
result_b = np.zeros((0, length))
result_h = np.zeros((0, length))
result_m = np.zeros((0, length))
result_p = np.zeros((0, length))
answer = np.zeros((0, length))
config = {
'SNR': [],
'count': [],
'frequencies': [],
'amplitudes': [],
'minamplitude': [],
'mindist': []
} # type: Dict[str, Any]
# Calculate window functions
window_bartlett = np.bartlett(time_length)
window_hanning = np.hanning(time_length)
window_meyer = self._meyer_wavelet(time_length)
window_poisson = exponential(time_length,
sym=True,
tau=(time_length / 2) * (8.69 / 60.0))
# Loop all data entries
for data in dict_list:
time = np.asarray(data['signal']['time']['data'])
signals = np.concatenate(
(signals, np.reshape(time, (1, ) + time.shape)))
config['SNR'].append(data['signal']['SNR'])
# Assemble the FFTs
fft = np.fft.fft(time)[:length] / time_length
result_r = np.concatenate(
(result_r, np.reshape(fft, (1, ) + fft.shape)))
fft = np.fft.fft(time * window_bartlett)[:length] / time_length
result_b = np.concatenate(
(result_b, np.reshape(fft, (1, ) + fft.shape)))
fft = np.fft.fft(time * window_hanning)[:length] / time_length
result_h = np.concatenate(
(result_h, np.reshape(fft, (1, ) + fft.shape)))
fft = np.fft.fft(time * window_meyer)[:length] / time_length
result_m = np.concatenate(
(result_m, np.reshape(fft, (1, ) + fft.shape)))
fft = np.fft.fft(time * window_poisson)[:length] / time_length
result_p = np.concatenate(
(result_p, np.reshape(fft, (1, ) + fft.shape)))
# Assemble all the frequencies and amplitudes
count = 0
freqs = []
ampls = []
counting = np.zeros((1, length))
for subsig in data['signal']['parts']:
if subsig['signal']['type'] == 'SingleOscillation':
count += 1
freq = subsig['signal']['frequency']
counting[0, int(max(0, min(length - 1, round(freq))))] += 1
freqs.append(freq)
ampls.append(subsig['signal']['amplitude'])
config['count'].append(count)
# Sort frequencies and amplitudes by frequency
np_freqs = np.asarray(freqs)
sorting = np.unravel_index(np.argsort(np_freqs), np_freqs.shape)
np_freqs = np_freqs[sorting]
np_ampls = np.asarray(ampls)[sorting]
# Assemble some statistics
config['mindist'].append(
999999. if len(np_freqs) < 2 else np.min(np.diff(np_freqs)))
config['minamplitude'].append(
np.min(np_ampls) if len(np_ampls) > 0 else 999999.)
config['frequencies'].append(np_freqs)
config['amplitudes'].append(np_ampls)
answer = np.concatenate((answer, counting))
# Assemble results
ffts = (result_r, result_b, result_h, result_m, result_p)
return signals, ffts, answer, config
def butter_lowpass_filtfilt(data, cutoff=100, fs=2000, order=5):
# b, a = butter_lowpass(cutoff, fs, order=order)
# y = filtfilt(b, a, data)
y = exponential(data)
return y
