def __init__(self,
channels,
segment_width,
conv_filters=64,
kernel_size=3):
super(ConvNet, self).__init__()
self.scalogram = WaveletTransformTorch(dt=0.05,
dj=0.125,
cuda=torch.cuda.is_available(),
channels=channels)
self.scalogram.signal_length = segment_width # implicitely set scales
self.sequence = torch.nn.Sequential(
torch.nn.ConstantPad2d((kernel_size - 1, 0, kernel_size - 1, 0),
0),
torch.nn.Conv2d(
in_channels=channels,
out_channels=conv_filters,
kernel_size=(kernel_size, kernel_size),
),
torch.nn.ReLU(),
torch.nn.Flatten(),
torch.nn.Linear(in_features=conv_filters *
len(self.scalogram.scales) * segment_width,
out_features=1),
)
def __init__(self, window, dt=0.1, dj=0.125, f0=6):
super(Attention_WCC, self).__init__()
self.window_size = window * 3 - 2 # practical window size, keep in accordance with the other models
self.wavelet = WaveletTransformTorch(dt, dj, cuda=True)
self._set_f0(f0)
self.dt = dt
self.dj = dj
def test_torch_power_implementation():
for n_channels in range(1, 5):
# create random data
n_samples = 100
signal_length = 42
X = torch.rand(n_samples, n_channels, signal_length)
# calculate power via the original numpy route and via the direct torch implementation
wt = WaveletTransformTorch(cuda=False, channels=n_channels)
power_np = super(WaveletTransformTorch, wt).power(X.numpy())
power_torch = wt.power(X)
assert np.allclose(power_np, power_torch.numpy())
def test_cwt_torch_multi_channel():
# create random data
n_samples = 100
n_channels = 12
signal_length = 42
X = np.random.rand(n_samples, n_channels, signal_length)
# execute wavelet transformation
wa = WaveletTransformTorch(dt=1.0,
dj=0.125,
cuda=False,
channels=n_channels)
cwt = wa.cwt(X)
assert cwt.shape == (n_samples, n_channels, len(wa.scales), signal_length)
def test_cwt_scipy_vs_torch_single_channel():
# create random data
n_samples = 100
signal_length = 42
X = np.random.rand(n_samples, signal_length)
# SciPy and PyTorch based wavelet transformation
wa_scipy = WaveletTransform(dt=1.0, dj=0.125)
wa_torch = WaveletTransformTorch(dt=1.0, dj=0.125, cuda=False)
cwt_scipy = wa_scipy.cwt(X)
cwt_torch = wa_torch.cwt(X)
# ensure that the exact same scales were used
assert np.array_equal(wa_scipy.scales, wa_torch.scales)
assert np.allclose(cwt_torch, cwt_scipy, rtol=1e-5, atol=1e-6)
# test correct sizes
assert cwt_torch.shape == (n_samples, len(wa_scipy.scales), signal_length)
assert cwt_scipy.shape == (n_samples, len(wa_torch.scales), signal_length)
def test_scalogram():
# create random data
n_samples = 1
n_channels = 2
signal_length = 3
X = torch.rand(n_samples, n_channels, signal_length)
# determine scalogram for each channel
scalogram_sequential = Scalogram(signal_length, dt=1.0, dj=0.125)
scalogram_parallel = WaveletTransformTorch(dt=1.0,
dj=0.125,
cuda=False,
channels=n_channels)
power_sequential = scalogram_sequential(X)
power_parallel = scalogram_parallel.power(X)
assert np.array_equal(scalogram_sequential.wavelet.scales,
scalogram_parallel.scales)
assert torch.all(torch.eq(power_sequential, power_parallel))
class Scalogram(torch.nn.Module):
def __init__(self, seg_width, dt, dj):
super(Scalogram, self).__init__()
self.wavelet = WaveletTransformTorch(dt, dj, cuda=False)
self.wavelet.signal_length = seg_width # implicitely set num_scales
@property
def num_scales(self):
return len(self.wavelet.scales)
def forward(self, X):
# determine scalogram for each channel
scalograms = []
for channel in torch.split(X, 1, dim=1):
scalograms.append(self.wavelet.power(channel))
X = torch.stack(scalograms, dim=1)
return X
def pipeline():
phases = (
"resized",
"gray",
"optical-flow",
"optical-flow-over-resized",
"Fx",
"Fy",
"dxfx",
"dxfy",
"dyfy",
"dyfx",
"divergence",
"curl",
"shear",
)
differential_maps = (
"Fx",
"Fy",
"dxfx",
"dyfy",
"divergence",
"curl",
)
cfg = VideoConfiguration(
phases=phases,
differential_maps=differential_maps,
save_phase_as_video={phase: True
for phase in phases},
save_phase_as_ndarray={phase: True
for phase in phases},
plot_phase={phase: False
for phase in phases},
input_filepath="resources/input/animation.mp4",
# input_filepath="resources/input/pushups_6sec.mp4",
# input_filepath="resources/input/QUVARepetitionDataset/videos/078_ski_waxing.mp4",
# input_filepath="resources/input/QUVARepetitionDataset/videos/014_weights_floor.mp4",
# input_filepath="resources/input/QUVARepetitionDataset/videos/097_swimming_freestyle.mp4",
max_dimension_size=6400,
gpu=True,
read_frames=False,
)
# TODO: save & read frames
# different phases require different pix_fmt
frames = preprocess_video(cfg=cfg)
frames = {
motion: data
for motion, data in frames.items() if motion in cfg.differential_maps
}
power_maps = {}
for motion in cfg.differential_maps:
motion_map = frames[motion] # t x N x M
power_maps[motion] = np.ndarray(shape=motion_map.shape)
pixel_coordinates = list(
itertools.product(range(motion_map.shape[1]),
range(motion_map.shape[2])))
dpi = 100
dt = 1 / cfg.fps # sampling frequency
dj = 1 / 8 # scale distribution parameter
wavelet = Morlet(w0=6)
batch_size = 2048
if cfg.gpu:
wavelet_transform = WaveletTransformTorch(dt,
dj,
wavelet,
cuda=True)
signals = np.array(
[motion_map[:, x, y] for x, y in pixel_coordinates]) # N*M x t
power_spectrums = []
for batch in tqdm.tqdm(
np.array_split(signals,
len(signals) // batch_size)):
results = wavelet_transform.power(batch) # b x J x t
power_spectrums.extend(results)
power_spectrums = np.array(power_spectrums) # N*M x J x t
power_maps[motion] = np.amax(power_spectrums, axis=1) # N*M x t
reshape = (
motion_map.shape[1],
motion_map.shape[2],
power_spectrums.shape[-1],
)
power_maps[motion] = power_maps[motion].reshape(
reshape) # N x M x t
power_maps[motion] = np.moveaxis(power_maps[motion], 2,
0) # t x N x M
else:
wavelet_transform = WaveletTransform(dt, dj, wavelet)
# TODO: meshgrid ?
for (x, y) in tqdm.tqdm(pixel_coordinates):
signal = motion_map[:, x, y]
power_spectrum = wavelet_transform.power(signal) # J x t
power_maps[motion][:, x, y] = np.amax(power_spectrum,
axis=0) # t x N x M
# In case to much 0s make mean thresholding useless
# threshold = power_maps["total_mean_th"].mean()
# threshold = power_maps["total_mean_th"][power_maps["total_mean_th"] > 0].mean()
power_maps["total"] = np.sum(np.asarray(list(power_maps.values())), axis=0)
power_maps["total_mean_th"] = power_maps["total"].copy()
threshold = np.unique(power_maps["total_mean_th"]).mean()
mask = power_maps["total_mean_th"] < threshold
logging.info(
f"{round(threshold, 2)} -- {np.count_nonzero(mask)} out of {mask.size}, {round(np.count_nonzero(mask) / mask.size, 2) * 100} are below threshold"
)
power_maps["total_mean_th"][mask] = 0
logging.debug(
f"Min power in power map before thresholding: {sorted(np.unique(power_maps['total']))[:10]} and after {sorted(np.unique(power_maps['total_mean_th']))[:10]}"
)
for mmap, arr in power_maps.items():
logging.debug(
f"{mmap}, {round(np.min(arr), 2)}, {round(np.mean(arr), 2)}, {round(np.max(arr), 2)}"
)
vmax = np.max(power_maps["total"])
for motion, power_map in power_maps.items():
pixel_map = []
xv, yv = np.meshgrid(range(power_map.shape[2]),
range(power_map.shape[1]))
for i in tqdm.tqdm(range(power_map.shape[0])): # for i in t
fig, ax = plt.subplots(
figsize=(
power_map.shape[2] / dpi,
power_map.shape[1] / dpi,
),
dpi=dpi,
frameon=False,
clear=True,
tight_layout={"pad": 0},
)
ax.set_axis_off()
plt.subplots_adjust(top=1,
bottom=0,
right=1,
left=0,
hspace=0,
wspace=0)
plt.margins(0, 0)
cnt = ax.contourf(
xv,
yv,
power_map[i],
100,
cmap=plt.get_cmap("PuBu_r"),
vmin=0,
vmax=vmax,
)
# Fix for saving as PDF (aliasing)
for c in cnt.collections:
c.set_edgecolor("face")
# Save separate frames as images
# Path(
# f"resources/output/{cfg.input_filename}/{motion}/{i}.png"
# ).parent.mkdir(parents=True, exist_ok=True)
# plt.savefig(f"resources/output/{cfg.input_filename}/{motion}/{i}.png")
fig.canvas.draw()
buf = fig.canvas.tostring_rgb()
ncols, nrows = fig.canvas.get_width_height()
data = np.frombuffer(buf, dtype=np.uint8).reshape(nrows, ncols, 3)
pixel_map.append(data)
plt.close("all")
pixel_map = np.array(pixel_map)
pixel_map = np.flip(pixel_map, axis=1)
power_output_fname = (
f"resources/output/{cfg.input_filename}/_power_{motion}.mp4")
Path(power_output_fname).parent.mkdir(parents=True, exist_ok=True)
video_write(
output_fname=power_output_fname,
images=pixel_map,
framerate=cfg.fps,
)
t_min = 0
t_max = 10
t = np.linspace(t_min, t_max, (t_max - t_min) * fps)
######################################
# Generating batch of random sinusoidals
random_frequencies = np.random.uniform(0.5, 4.0, size=batch_size)
batch = np.asarray([np.sin(2 * np.pi * f * t) for f in random_frequencies])
batch += np.random.normal(0, 0.2, batch.shape) # Gaussian noise
######################################
# Performing wavelet transform
wa = WaveletTransform(dt, dj, wavelet, unbias=unbias)
wa_torch = WaveletTransformTorch(dt, dj, wavelet, unbias=unbias, cuda=cuda)
power = wa.power(batch)
batch_torch = torch.tensor(batch[:, None, :],
dtype=torch.float,
device='cuda' if cuda else 'cpu')
power_torch = wa_torch.power(batch_torch).cpu()
######################################
# Plotting
fig, ax = plt.subplots(1, 3, figsize=(12, 3))
ax = ax.flatten()
ax[0].plot(t, batch[0])
ax[0].set_title(
r'$f(t) = \sin(2\pi \cdot f t) + \mathcal{N}(\mu,\,\sigma^{2})$')
# Author: Tom Runia # Date Created: 2018-04-16 from __future__ import absolute_import from __future__ import division from __future__ import print_function import numpy as np from wavelets_pytorch.transform import WaveletTransform # SciPy version from wavelets_pytorch.transform import WaveletTransformTorch # PyTorch version dt = 0.1 # sampling frequency dj = 0.125 # scale distribution parameter batch_size = 32 # how many signals to process in parallel t = np.linspace(0., 10., int(10. / dt)) # Sinusoidals with random frequency frequencies = np.random.uniform(-0.5, 2.0, size=batch_size) batch = np.asarray([np.sin(2 * np.pi * f * t) for f in frequencies]) # Initialize wavelet filter banks (scipy and torch implementation) wa_scipy = WaveletTransform(dt, dj) wa_torch = WaveletTransformTorch(dt, dj, cuda=True) # Performing wavelet transform (and compute scalogram) cwt_scipy = wa_scipy.cwt(batch) cwt_torch = wa_torch.cwt(batch) # For plotting, see the examples/plot.py function. # ...
for run_ind in range(num_runs):
random_frequencies = np.random.uniform(-0.5, 4.0, size=batch_size)
batch = np.asarray(
[np.sin(2 * np.pi * f * t) for f in random_frequencies])
# Perform batch computation of SciPy implementation
t_start = time.time()
wa = WaveletTransform(dt, dj, unbias=unbias)
power = wa.power(batch)
runtimes_scipy[batch_ind, length_ind,
run_ind] = time.time() - t_start
#print(" Run {}/{} | SciPy: {:.2f}s".format(run_ind+1, num_runs, runtimes[batch_ind,length_ind,run_ind,0]))
# Perform batch computation of Torch implementation
t_start = time.time()
wa = WaveletTransformTorch(dt, dj, unbias=unbias)
power = wa.power(batch)
runtimes_torch[batch_ind, length_ind,
run_ind] = time.time() - t_start
#print(" Run {}/{} | Torch: {:.2f}s".format(run_ind+1, num_runs, runtimes[batch_ind,length_ind,run_ind,1]))
avg_scipy = np.mean(runtimes_scipy[batch_ind, length_ind, :])
avg_torch = np.mean(runtimes_torch[batch_ind, length_ind, :])
print(' Average SciPy: {:.2f}s'.format(avg_scipy))
print(' Average Torch: {:.2f}s'.format(avg_torch))
np.save('./runtimes_scipy.npy', runtimes_scipy)
np.save('./runtimes_torch.npy', runtimes_torch)
class Attention_WCC(nn.Module):
"""
Attention model with sliding window Wavelet coherence
"""
def get_code(self):
return 'WCC'
def __init__(self, window, dt=0.1, dj=0.125, f0=6):
super(Attention_WCC, self).__init__()
self.window_size = window * 3 - 2 # practical window size, keep in accordance with the other models
self.wavelet = WaveletTransformTorch(dt, dj, cuda=True)
self._set_f0(f0)
self.dt = dt
self.dj = dj
def _set_f0(self, f0):
# Sets the Morlet wave number, the degrees of freedom and the
# empirically derived factors for the wavelet bases C_{\delta},
# \gamma, \delta j_0 (Torrence and Compo, 1998, Table 2)
self.f0 = f0 # Wave number
self.dofmin = 2 # Minimum degrees of freedom
if self.f0 == 6:
self.cdelta = 0.776 # Reconstruction factor
self.gamma = 2.32 # Decorrelation factor for time averaging
self.deltaj0 = 0.60 # Factor for scale averaging
else:
self.cdelta = -1
self.gamma = -1
self.deltaj0 = -1
def smooth(self, W, dt, dj, scales):
"""Smoothing function used in coherence analysis.
Parameters
----------
W :
dt :
dj :
scales :
Returns
-------
T :
"""
# The smoothing is performed by using a filter given by the absolute
# value of the wavelet function at each scale, normalized to have a
# total weight of unity, according to suggestions by Torrence &
# Webster (1999) and by Grinsted et al. (2004).
m, n = W.shape
# Filter in time.
k = 2 * np.pi * fft.fftfreq(fft_kwargs(W[0, :])['n'])
k2 = k**2
snorm = scales / dt
# Smoothing by Gaussian window (absolute value of wavelet function)
# using the convolution theorem: multiplication by Gaussian curve in
# Fourier domain for each scale, outer product of scale and frequency
F = np.exp(-0.5 * (snorm[:, np.newaxis]**2) * k2) # Outer product
smooth = fft.ifft(
F * fft.fft(W, axis=1, **fft_kwargs(W[0, :])),
axis=1, # Along Fourier frequencies
**fft_kwargs(W[0, :], overwrite_x=True))
T = smooth[:, :n] # Remove possibly padded region due to FFT
if np.isreal(W).all():
T = T.real
# Filter in scale. For the Morlet wavelet it's simply a boxcar with
# 0.6 width.
wsize = self.deltaj0 / dj * 2
win = rect(np.int(np.round(wsize)), normalize=True)
T = convolve2d(T, win[:, np.newaxis], 'same') # Scales are "vertical"
return T
def sliding_window(self, x, step_size=1):
# unfold dimension to make the sliding window
return x.unfold(0, self.window_size, step_size)
def wavelet_coherence_correlation(self, y1, y2):
# Calculate the continous wavelet tranform
y1_normal = (y1 - y1.mean()) / y1.std()
y2_normal = (y2 - y2.mean()) / y2.std()
[W1, W2] = self.wavelet.cwt(np.array([y1_normal, y2_normal]))
sj = self.wavelet.scales
scales1 = np.ones([1, y1.size]) * sj[:, None]
scales2 = np.ones([1, y2.size]) * sj[:, None]
S1 = self.smooth(np.abs(W1)**2 / scales1, self.dt, self.dj, sj)
S2 = self.smooth(np.abs(W2)**2 / scales2, self.dt, self.dj, sj)
W12 = W1 * W2.conj()
scales = np.ones([1, y1.size]) * sj[:, None]
S12 = self.smooth(W12 / scales, self.dt, self.dj, sj)
WCT = np.abs(S12)**2 / (S1 * S2)
return torch.from_numpy(WCT.sum(axis=0))
def attention_generation(self, x):
result = []
for y in x:
y = y.data.cpu().numpy()
coherence = self.wavelet_coherence_correlation(y[0], y[1])
s_out = self.sliding_window(coherence)
result.append(s_out.sum(dim=1))
return torch.stack(result, dim=0)
'''
returns: attention, sectors score
'''
def forward(self, x):
attentions = self.attention_generation(x)
return Variable(attentions.float()).cuda(), None
t_min = 0
t_max = 10
t = np.linspace(t_min, t_max, (t_max - t_min) * fps)
######################################
# Generating batch of random sinusoidals
random_frequencies = np.random.uniform(0.5, 4.0, size=batch_size)
batch = np.asarray([np.sin(2 * np.pi * f * t) for f in random_frequencies])
batch += np.random.normal(0, 0.2, batch.shape) # Gaussian noise
######################################
# Performing wavelet transform
wa = WaveletTransform(dt, dj, wavelet, unbias=unbias)
wa_torch = WaveletTransformTorch(dt, dj, wavelet, unbias=unbias, cuda=True)
power = wa.power(batch)
power_torch = wa_torch.power(batch)
######################################
# Plotting
fig, ax = plt.subplots(1, 3, figsize=(12, 3))
ax = ax.flatten()
ax[0].plot(t, batch[0])
ax[0].set_title(
r'$f(t) = \sin(2\pi \cdot f t) + \mathcal{N}(\mu,\,\sigma^{2})$')
ax[0].set_xlabel('Time (s)')
# Plot scalogram for SciPy implementation
def __init__(self, seg_width, dt, dj):
super(Scalogram, self).__init__()
self.wavelet = WaveletTransformTorch(dt, dj, cuda=False)
self.wavelet.signal_length = seg_width # implicitely set num_scales