def _convolve_no_lower_bound(log_pmf1, log_pmf2, beta, enable_fast_path):
alpha = 1.0 / beta
if enable_fast_path:
true_conv_len, fft_conv_len = utils.pairwise_convolution_lengths(
len(log_pmf1), len(log_pmf2))
pmf1 = np.exp(log_pmf1)
fft1 = fft.fft(pmf1, n=fft_conv_len)
direct, bad_places = _direct_fft_conv(log_pmf1, pmf1, fft1, log_pmf2,
true_conv_len, fft_conv_len,
alpha, NEG_INF)
used_nc = _use_nc_if_better(log_pmf1, ESTIMATE_ONE_SPLIT, log_pmf2,
ESTIMATE_TWO_SPLITS, direct, bad_places,
COST_RATIO)
if used_nc:
#logging.debug('convolved without lower bound without shifting')
return direct
# shift, convolve, unshift
theta = _compute_theta(log_pmf1, log_pmf2)
s1, log_mgf1 = utils.shift(log_pmf1, theta)
s2, log_mgf2 = utils.shift(log_pmf2, theta)
convolved = _afftc_noshift(s1,
s2,
beta,
NEG_INF,
pairwise=True,
square_1=False,
enable_fast_path=enable_fast_path)[0]
return utils.unshift(convolved, theta, (log_mgf1, 1), (log_mgf2, 1))
def _convolve_no_lower_bound(log_pmf1, log_pmf2, beta,
enable_fast_path):
alpha = 1.0 / beta
if enable_fast_path:
true_conv_len, fft_conv_len = utils.pairwise_convolution_lengths(len(log_pmf1),
len(log_pmf2))
pmf1 = np.exp(log_pmf1)
fft1 = fft.fft(pmf1, n = fft_conv_len)
direct, bad_places = _direct_fft_conv(log_pmf1, pmf1, fft1, log_pmf2,
true_conv_len, fft_conv_len,
alpha, NEG_INF)
used_nc = _use_nc_if_better(log_pmf1, ESTIMATE_ONE_SPLIT,
log_pmf2, ESTIMATE_TWO_SPLITS,
direct, bad_places,
COST_RATIO)
if used_nc:
#logging.debug('convolved without lower bound without shifting')
return direct
# shift, convolve, unshift
theta = _compute_theta(log_pmf1, log_pmf2)
s1, log_mgf1 = utils.shift(log_pmf1, theta)
s2, log_mgf2 = utils.shift(log_pmf2, theta)
convolved = _afftc_noshift(s1, s2, beta, NEG_INF,
pairwise = True,
square_1 = False,
enable_fast_path = enable_fast_path)[0]
return utils.unshift(convolved, theta, (log_mgf1, 1), (log_mgf2, 1))
def encrypt(self):
print('[+] Starting encryption.')
scrambled = self.image.matrix
pbar = tqdm(total=2 * (len(self.r_vector) * len(self.c_vector)) + (len(self.r_vector) + len(self.c_vector)))
for i in range(len(self.r_vector)):
ith_row_sum = utils.sum_row(i, scrambled)
if bool(ith_row_sum % 2):
scrambled[i] = utils.shift(scrambled[i], self.r_vector[i])
else:
scrambled[i] = utils.shift(scrambled[i], -self.r_vector[i])
pbar.update(1)
for j in range(len(self.c_vector)):
# jth_column_sum = utils.sum_column(j, scrambled)
col = utils.get_column(scrambled, j)
shifted = utils.shift(col, -self.c_vector[j])
scrambled = utils.set_column(scrambled, j, shifted)
pbar.update(1)
scrambled = self.xor_scrambled(scrambled, pbar)
pbar.close()
self.image.create_image(self.encrypted, scrambled)
f = open(self.image.key_path, 'w')
f.write(str(self.r_vector) + '\n')
f.write(str(self.c_vector))
f.close()
def find_bird(bird, background):
# crop image
bird_rows, bird_cols = bird.shape
crop_bg = background[BIRD_EDGE:BIRD_EDGE + bird_rows + 10 * RESIZE, :]
res, min_val, max_val, top_left, bottom_right = find_object(bird, crop_bg)
top_left = shift(top_left, (0,BIRD_EDGE))
bottom_right = shift(bottom_right, (0,BIRD_EDGE))
return top_left, bottom_right
def Delay(self, delayProf):
"""
Compute the delayed profile composed of *self* profile and *delayProf*,
received by a node for which this *self* profile is the output profile on the sender side.
The delay profile describes the delay as a function of time for the link.
This function implements the operation:
.. math::
o[t + \delta[t]] = l[t]
Where
* :math:`\delta[t]` is the delay profile
* :math:`l[t]` is the profile transmitted into the link (*self*)
* :math:`o[t]` is the output profile received at the other end of the link
:rtype: :class:`Profile`, :math:`o[t]`
:param in delayProf: :class:`Profile` describing the delay
"""
delays = delayProf.entries['latency']
all0 = True
for time, delay in delays:
if delay != 0:
all0 = False
if all0: return copy.deepcopy(self)
datas = self.entries['data']
endTime = datas[-1][0]
times = [ x[0] for x in delays ]
times.extend( [ x[0] for x in datas ] )
times = sorted(list(set(times)))
newDatas = []
for t in times:
d = utils.get_value_at_time(datas, t)
delay = utils.get_value_at_time(delays, t, interpolate = 'latency' in self.interpolated_profiles)
newDatas.append([ t + delay, d ])
newDatas = utils.remove_degenerates(newDatas)
newDatas, remainder = utils.split(newDatas, endTime)
if remainder:
t = -remainder[0][0]
utils.shift(remainder, t)
r_slopes = utils.derive(remainder)
d_slopes = utils.derive(newDatas)
d_slopes = utils.add_values(d_slopes,r_slopes)
newDatas = utils.integrate(d_slopes, endTime)
retProf = Profile()
retProf.entries['data'] = newDatas
retProf.Derive()
return retProf
def sfft_pvalue(log_pmf, s0, L):
theta = sisfft._compute_theta(log_pmf, s0, L)
shifted, mgf = utils.shift(log_pmf, theta)
sfft_vector, fft_len = naive.power_fft(shifted, L)
error_estimate = utils.sfft_error_threshold_factor(fft_len, L)
sfft_vector[sfft_vector < np.log(error_estimate)] = utils.NEG_INF
return utils.log_sum(utils.unshift(sfft_vector, theta, (mgf, L))[s0:])
def sfft_pvalue(log_pmf, s0, L):
theta = sisfft._compute_theta(log_pmf, s0, L)
shifted, mgf = utils.shift(log_pmf, theta)
sfft_vector, fft_len = naive.power_fft(shifted, L)
error_estimate = utils.sfft_error_threshold_factor(fft_len, L)
sfft_vector[sfft_vector < np.log(error_estimate)] = utils.NEG_INF
return utils.log_sum(utils.unshift(sfft_vector, theta, (mgf, L))[s0:])
def get_im_resi(self, model_conv, im_nb, ret_all=False):
convo = fn.shift(model_conv, -self.shifts[im_nb][0], -self.shifts[im_nb][1],
interp_order=3, mode='wrap')
convo_m = fn.mean(convo, self._bshape[0], self._bshape[1])
# resi = fn.rebin(np.logical_not(self.masks[im_nb]),self._sshape)*(fn.rebin(self.images[im_nb],self._sshape) - convo)
# err = resi/fn.rebin(self.noisemaps[im_nb],self._sshape)
resi = np.logical_not(self.masks[im_nb])*(self.images[im_nb] - convo_m)
err = fn.rebin(resi/self.noisemaps[im_nb], self._sshape)/self._sfact**2.
ali_err = fn.shift(err, self.shifts[im_nb][0],
self.shifts[im_nb][1],
interp_order=3, mode='wrap')
if ret_all:
resi = fn.rebin(resi, self._sshape)/self._sfact**2.
ali_resi = fn.shift(resi, self.shifts[im_nb][0],
self.shifts[im_nb][1],
interp_order=3, mode='wrap')
return ali_err, ali_resi
# ali_err *= resi.sum()/ali_err.sum()
return ali_err
def test_base_impossible_2(self):
A = sympy.Symbol('A')
B = sympy.Symbol('B')
state_0 = Counter({A:2})
substrate = Counter({A:3})
products = Counter({B:2})
kinetic = A*(A-1)
new_state, kinetic_val = shift(state_0, substrate, products, kinetic)
self.assertEqual(new_state, None)
self.assertEqual(kinetic_val, None)
def test_base_impossible_2(self):
A = sympy.Symbol('A')
B = sympy.Symbol('B')
state_0 = Counter({A: 2})
substrate = Counter({A: 3})
products = Counter({B: 2})
kinetic = A * (A - 1)
new_state, kinetic_val = shift(state_0, substrate, products, kinetic)
self.assertEqual(new_state, None)
self.assertEqual(kinetic_val, None)
def main_rest(self, img, prev_left_fit, prev_right_fit):
height = img.shape[0]
width = img.shape[1]
warped, M = utils.perspective(utils.undistort(img, self.objpoints, self.imgpoints))
warped_edge = utils.edge(warped)
left_base, right_base = utils.base(warped_edge)
left_fit, left_fit_m = utils.fit_rest(warped_edge, prev_left_fit)
right_fit, right_fit_m = utils.fit_rest(warped_edge, prev_right_fit)
curv = utils.curvature(left_fit_m, right_fit_m, height)
shif = utils.shift(left_fit_m, right_fit_m, height, width)
return left_fit, right_fit, curv, shif, M
def escapes(self, start):
"""given a starting state it evaluate which states are reachable and the corresponding transition rate"""
start = Counter(start)
end_states = []
kinetics = []
for substrate, products, kinetic in self.reactions:
end_state, kinetic = shift(start, substrate, products, kinetic)
if kinetic and end_state is not None:
end_states.append(end_state)
kinetics.append(float(kinetic))
kinetics = np.array(kinetics)
return end_states, kinetics
def set_ini(self):
import scipy.ndimage.interpolation as inter
ini = np.array([])
for i, im in enumerate(self.images):
masked = np.logical_not(self.masks[i])*im
ali = fn.shift(masked, self.shifts[i][0]/self._sfact,
self.shifts[i][1]/self._sfact,
interp_order=3, mode='reflect')
ali_zoom = inter.zoom(ali, self._sfact)/self._sfact**2.
ini = np.append(ini, ali_zoom)
self.ini = np.median(ini.reshape((len(self.images), self._sshape[0]*self._sshape[1])),
0).reshape(self._sshape)
self.ini = np.zeros(self._sshape) # we start from 0 ...
def undo_differences(self, preds):
"""
Undoes the differences' effect on the predictions. Model that predicts
over the differenced series gets back the integrated predictions.
Parameters
----------
preds : array_like
The "raw" predictions made on stationary data.
Returns
-------
The integrated predictions or `preds` itself if d = D = 0.
"""
numel_preds = preds.size
# Handle ordinary differences' undo
if self.d != 0:
preds += sum(
shift(
data=ordinal_diff(self.the_endog, i),
crop=True)[-numel_preds:]
for i in range(self.d))
# Handle seasonal differences' undo
if self.D != 0:
ordi_diffed_endog = ordinal_diff(self.the_endog, self.d)
preds += sum(
shift(
data=seasonal_diff(
ordi_diffed_endog, i, self.seas_period
),
periods=self.seas_period, crop=True
)[-numel_preds:]
for i in range(self.D)
)
return preds
def pvalue(log_pmf, s0, L, desired_beta):
"""Compute $log((exp(log_pmf)**L)[s0:])$, such that the relative error
to the exact answer is less than or equal to $desired_beta$."""
total_len, _ = utils.iterated_convolution_lengths(len(log_pmf), L)
if s0 >= total_len:
return NEG_INF
_, p_lower_preshift, p_upper_preshift = _bounds(log_pmf, log_pmf, 0, 0.0,
s0, L, desired_beta)
sfft_good_preshift, sfft_pval_preshift = _check_sfft_pvalue(
p_lower_preshift, p_upper_preshift, desired_beta)
if sfft_good_preshift:
logging.debug(' pre-shift sfft worked %.20f', sfft_pval_preshift)
return sfft_pval_preshift
with timer('computing theta'):
theta = _compute_theta(log_pmf, s0, L)
logging.debug('raw theta %s', theta)
# TODO: too-large or negative theta causes numerical instability,
# so this is a huge hack
theta = utils.clamp(theta, 0, THETA_LIMIT)
shifted_pmf, log_mgf = utils.shift(log_pmf, theta)
beta = desired_beta / 2.0
with timer('bounds'):
log_delta, p_lower, p_upper = _bounds(log_pmf, shifted_pmf, theta,
log_mgf, s0, L, desired_beta)
sfft_good, sfft_pval = _check_sfft_pvalue(p_lower, p_upper, desired_beta)
logging.debug('theta %s, log_mgf %s, beta %s, log delta %s', theta,
log_mgf, beta, log_delta)
if sfft_good:
logging.debug(' sfft worked %.20f', sfft_pval)
return sfft_pval
delta = np.exp(log_delta)
conv = conv_power(shifted_pmf, L, beta, delta)
pval = utils.log_sum(utils.unshift(conv, theta, (log_mgf, L))[s0:])
logging.debug(' sis pvalue %.20f', pval)
return pval
def pvalue(log_pmf, s0, L, desired_beta):
"""Compute $log((exp(log_pmf)**L)[s0:])$, such that the relative error
to the exact answer is less than or equal to $desired_beta$."""
total_len, _ = utils.iterated_convolution_lengths(len(log_pmf), L)
if s0 >= total_len:
return NEG_INF
_, p_lower_preshift, p_upper_preshift = _bounds(log_pmf, log_pmf, 0, 0.0,
s0, L, desired_beta)
sfft_good_preshift, sfft_pval_preshift = _check_sfft_pvalue(p_lower_preshift,
p_upper_preshift,
desired_beta)
if sfft_good_preshift:
logging.debug(' pre-shift sfft worked %.20f', sfft_pval_preshift)
return sfft_pval_preshift
with timer('computing theta'):
theta = _compute_theta(log_pmf, s0, L)
logging.debug('raw theta %s', theta)
# TODO: too-large or negative theta causes numerical instability,
# so this is a huge hack
theta = utils.clamp(theta, 0, THETA_LIMIT)
shifted_pmf, log_mgf = utils.shift(log_pmf, theta)
beta = desired_beta / 2.0
with timer('bounds'):
log_delta, p_lower, p_upper = _bounds(log_pmf, shifted_pmf, theta, log_mgf,
s0, L, desired_beta)
sfft_good, sfft_pval = _check_sfft_pvalue(p_lower, p_upper, desired_beta)
logging.debug('theta %s, log_mgf %s, beta %s, log delta %s', theta, log_mgf, beta, log_delta)
if sfft_good:
logging.debug(' sfft worked %.20f', sfft_pval)
return sfft_pval
delta = np.exp(log_delta)
conv = conv_power(shifted_pmf, L, beta, delta)
pval = utils.log_sum(utils.unshift(conv, theta, (log_mgf, L))[s0:])
logging.debug(' sis pvalue %.20f', pval)
return pval
def find_pipes(pipe, background):
# first pipe, this takes ~2-2.5 ms to complete, so can do further cropping to reduce
# time, see TODO
res0, min_val0, max_val0, top_left0, bottom_right0 = find_object(pipe, background)
top_left0 = shift(top_left0, (0, BIRD_EDGE))
bottom_right0 = shift(bottom_right0, (0, BIRD_EDGE))
width = int(PIPE_SEPARATION_WIDTH * RESIZE)
# for finding the second pipe, looks up and down, then finds the closest match
min_x = top_left0[0] - DELTA + width
max_x = bottom_right0[0] + DELTA + width
min_y = top_left0[1] - DELTA
max_y = bottom_right0[1] + DELTA
crop_bg1 = crop(background, min_x, max_x, min_y, max_y)
if crop_bg1.shape[0] > pipe.shape[0] and crop_bg1.shape[1] > pipe.shape[1]:
res1, min_val1, max_val1, top_left1, bottom_right1 = find_object(pipe, crop_bg1)
corner = min_x, min_y
top_left1 = shift(corner, top_left1)
bottom_right1 = shift(corner, bottom_right1)
else:
max_val1 = 0
min_x = top_left0[0] - DELTA - width
max_x = bottom_right0[0] + DELTA - width
min_y = top_left0[1] - DELTA
max_y = bottom_right0[1] + DELTA
crop_bg2 = crop(background, min_x, max_x, min_y, max_y)
if crop_bg2.shape[0] > pipe.shape[0] and crop_bg2.shape[1] > pipe.shape[1]:
res2, min_val2, max_val2, top_left2, bottom_right2 = find_object(pipe, crop_bg2)
corner = min_x, min_y
top_left2 = shift(corner, top_left2)
bottom_right2 = shift(corner, bottom_right2)
else:
max_val2 = 0
if max_val1 > max_val2:
return top_left0, bottom_right0, top_left1, bottom_right1
else:
return top_left0, bottom_right0, top_left2, bottom_right2
def __init__(self, output_maps=9):
super(ICNN, self).__init__()
self.num_rows = 4
self.num_interlink_layer = 3
self.sf = 2
self.kernel_size = 5 # has to be odd (or need to change padding below)
self.last_kernel_size = 9
self.L = output_maps
self.num_channel_orignal = [
8 * i for i in range(1, self.num_rows + 1)
] # [8, 16, 24, 32]
self.num_channel_interlinked = shift(
self.num_channel_orignal, -1,
0) + self.num_channel_orignal + shift(self.num_channel_orignal, 1,
0)
# Initial batch norm
self.initial_bnorm = nn.ModuleList(
[nn.BatchNorm2d(3) for r in range(self.num_rows)])
# Input convs
self.inp_convs = nn.ModuleList([
nn.Conv2d(3,
self.num_channel_orignal[r],
self.kernel_size,
padding=self.kernel_size // 2)
for r in range(self.num_rows)
])
self.inp_bnorm = nn.ModuleList([
nn.BatchNorm2d(self.num_channel_orignal[r])
for r in range(self.num_rows)
])
# Interlinking convs
self.inter_convs_row0 = nn.ModuleList([
nn.Conv2d(self.num_channel_interlinked[0],
self.num_channel_orignal[0],
self.kernel_size,
padding=self.kernel_size // 2)
for i in range(self.num_interlink_layer)
])
self.inter_convs_row1 = nn.ModuleList([
nn.Conv2d(self.num_channel_interlinked[1],
self.num_channel_orignal[1],
self.kernel_size,
padding=self.kernel_size // 2)
for i in range(self.num_interlink_layer)
])
self.inter_convs_row2 = nn.ModuleList([
nn.Conv2d(self.num_channel_interlinked[2],
self.num_channel_orignal[2],
self.kernel_size,
padding=self.kernel_size // 2)
for i in range(self.num_interlink_layer)
])
self.inter_convs_row3 = nn.ModuleList([
nn.Conv2d(self.num_channel_interlinked[3],
self.num_channel_orignal[3],
self.kernel_size,
padding=self.kernel_size // 2)
for i in range(self.num_interlink_layer)
])
self.inter_bnorm_row0 = nn.ModuleList([
nn.BatchNorm2d(self.num_channel_orignal[0])
for i in range(self.num_interlink_layer)
])
self.inter_bnorm_row1 = nn.ModuleList([
nn.BatchNorm2d(self.num_channel_orignal[1])
for i in range(self.num_interlink_layer)
])
self.inter_bnorm_row2 = nn.ModuleList([
nn.BatchNorm2d(self.num_channel_orignal[2])
for i in range(self.num_interlink_layer)
])
self.inter_bnorm_row3 = nn.ModuleList([
nn.BatchNorm2d(self.num_channel_orignal[3])
for i in range(self.num_interlink_layer)
])
# Output convs
self.out_convs = nn.ModuleList([
nn.Conv2d(self.num_channel_orignal[r] +
self.num_channel_orignal[r + 1],
self.num_channel_orignal[r],
self.kernel_size,
padding=self.kernel_size // 2)
for r in range(1, self.num_rows - 1)
])
self.out_bnorm = nn.ModuleList([
nn.BatchNorm2d(self.num_channel_orignal[r])
for r in range(1, self.num_rows - 1)
])
self.top_conv = nn.Conv2d(self.num_channel_orignal[0] +
self.num_channel_orignal[1],
2 * self.L + 8,
self.kernel_size,
padding=self.kernel_size // 2)
self.top_bnorm = nn.BatchNorm2d(2 * self.L + 8)
# Last conv
self.last_conv1 = nn.Conv2d(2 * self.L + 8,
self.L,
self.last_kernel_size,
padding=self.last_kernel_size // 2)
def forward(ctx, input, blurKernel, weights, alpha):
"""
Wiener Filter for a batch of input images. (Filtering is taking place in the Frequency domain under the
assumption of periodic boundary conditions for the input image.)
input: (torch.(cuda.)Tensor) Input image tensor of size B x C x H x W
blurKernel: (torch.(cuda.)Tensor) PSFs tensor of size B x C x Hk x Wk
weights: (torch.(cuda.)Tensor) Regularization kernels of size D x C x Hw x Ww
alpha: (float) Regularization parameter of shape 1 x 1
returns: (torch.(cuda.)Tensor) Wiener filter output tensor B x 1 x C x H x H
output = F^H (B^H*F(input)/(|B|^2+exp(alpha)*|W|^2))
"""
assert (input.dim() < 5), "The input must be at most a 4D tensor."
while input.dim() < 4:
input = input.unsqueeze(0)
batch = input.size(0)
channels = input.size(1)
assert (blurKernel.dim() <
5), "The blurring kernel must be at most a 4D tensor."
while blurKernel.dim() < 4:
blurKernel = blurKernel.unsqueeze(0)
bshape = tuple(blurKernel.shape)
assert (bshape[0] in (1, batch) and bshape[1]
in (1, channels)), "Invalid blurring kernel dimensions."
N = alpha.size(0)
assert (alpha.dim() == 2 and alpha.size(-1) in (1, channels)), \
"Invalid dimensions for the alpha parameter. The expected shape of the " \
+ "tensor is {} x [{}|{}]".format(N, 1, channels)
alpha = alpha.exp()
assert (weights.dim() > 3 and weights.dim() < 6), "The regularization " \
+ "kernel must be a 4D or 5D tensor."
if weights.dim() < 5:
weights = weights.unsqueeze(0)
wshape = tuple(weights.shape)
assert (wshape[0] in (1, N) and wshape[2] in (1, channels)), \
"Invalid regularization kernel dimensions."
# Zero-padding of the blur kernel to match the input size
B = torch.zeros(bshape[0], bshape[1], input.size(2),
input.size(3)).type_as(blurKernel)
B[..., 0:bshape[2], 0:bshape[3]] = blurKernel
del blurKernel
# Circular shift of the zero-padded blur kernel
bs = tuple(int(i) for i in -(np.asarray(bshape[-2:]) // 2))
bs = (0, 0) + bs
B = utils.shift(B, bs, bc='circular')
# FFT of B
B = torch.rfft(B, 2)
# Zero-padding of the spatial dimensions of the weights to match the input size
G = torch.zeros(wshape[0], wshape[1], wshape[2], input.size(2),
input.size(3)).type_as(weights)
G[..., 0:wshape[3], 0:wshape[4]] = weights
del weights
# circular shift of the zero-padded weights
ws = tuple(int(i) for i in -(np.asarray(wshape[-2:]) // 2))
ws = (0, 0, 0) + ws
G = utils.shift(G, ws, bc='circular')
# FFT of G
G = torch.rfft(G, 2)
Y = cmul(conj(B), torch.rfft(input, 2)).unsqueeze(1)
ctx.intermediate_results = tuple()
if ctx.needs_input_grad[2] or ctx.needs_input_grad[3]:
ctx.intermediate_results += (alpha, B, G, Y, wshape)
elif ctx.needs_input_grad[0]:
ctx.intermediate_results += (alpha, B, G)
B = cabs(B).unsqueeze(-1)
G = cabs(G).pow(2).sum(dim=1)
G = G.mul(alpha.unsqueeze(-1).unsqueeze(-1)).unsqueeze(0).unsqueeze(-1)
G = G + B.pow(2).unsqueeze(1)
return torch.irfft(Y.div(G), 2, signal_sizes=input.shape[-2:])
ax.imshow(img[:, :, 0], cmap=plt.cm.Greys_r)
ax.axis('off')
ax.set_title('Original image')
ax = plt.subplot(2, 4, 2)
ax.imshow(rotate(img, 45)[:, :, 0], cmap=plt.cm.Greys_r)
ax.axis('off')
ax.set_title('Rotated positive')
ax = plt.subplot(2, 4, 2)
ax.imshow(rotate(img, -45)[:, :, 0], cmap=plt.cm.Greys_r)
ax.axis('off')
ax.set_title('Rotated negative')
ax = plt.subplot(2, 4, 3)
ax.imshow(shift(img, 0.2, 0.2)[:, :, 0], cmap=plt.cm.Greys_r)
ax.axis('off')
ax.set_title('Shift positive')
ax = plt.subplot(2, 4, 4)
ax.imshow(shift(img, -0.2, -0.2)[:, :, 0], cmap=plt.cm.Greys_r)
ax.axis('off')
ax.set_title('Shift negative')
ax = plt.subplot(2, 4, 5)
ax.imshow(zoom(img, 2, 2)[:, :, 0], cmap=plt.cm.Greys_r)
ax.axis('off')
ax.set_title('Zoom small')
ax = plt.subplot(2, 4, 6)
ax.imshow(zoom(img, 0.8, 0.8)[:, :, 0], cmap=plt.cm.Greys_r)
def test_roll():
x = torch.randn(3, 4, 5)
y = utils.shift(utils.shift(x, 2, 1, True), 2, 1, True)
assert (x == y).all()
# 1 - Read/synthetize data
# Synthetize data
tmax = 20
dt = 0.002
nt = math.floor(tmax / dt) + 1
t = np.arange(0, nt) * dt
f = 1.0
wavelet = utils.ricker(f, dt)
d = np.zeros(nt)
d[round(nt / 2)] = 1.0
d = np.convolve(d, wavelet, 'same') #observed data
td = 0.2 #time delay (if td > 0, syn arrive after obs)
s = utils.shift(d, dt, td) #synthetic data
s = 0.8 * s
plt.plot(t, d, 'b')
plt.plot(t, s, 'r')
plt.xlabel('Time (s)')
plt.ylabel('Amplitude')
plt.title('Observed (blue) and synthetic data (red)')
plt.savefig('Figure1.pdf')
plt.show()
# 2 - Butterwoth bandpass filter data (optional)
# 3 - Windowing
n1 = 0
n2 = nt
def gap_denoise(y,
Phi,
A,
At,
_lambda=1,
accelerate=True,
denoiser='tv',
iter_max=50,
noise_estimate=True,
sigma=None,
tv_weight=0.1,
tv_iter_max=5,
multichannel=True,
x0=None,
X_orig=None,
model=None,
show_iqa=True):
'''
Alternating direction method of multipliers (ADMM)[1]-based denoising
regularization for snapshot compressive imaging (SCI).
Parameters
----------
y : two-dimensional (2D) ndarray of ints, uints or floats
Input single measurement of the snapshot compressive imager (SCI).
Phi : three-dimensional (3D) ndarray of ints, uints or floats, omitted
Input sensing matrix of SCI with the third dimension as the
time-variant, spectral-variant, volume-variant, or angular-variant
masks, where each mask has the same pixel resolution as the snapshot
measurement.
Phi_sum : 2D ndarray,
Sum of the sensing matrix `Phi` along the third dimension.
A : function
Forward model of SCI, where multiple encoded frames are collapsed into
a single measurement.
At : function
Transpose of the forward model.
proj_meth : {'admm' or 'gap'}, optional
Projection method of the data term. Alternating direction method of
multipliers (ADMM)[1] and generalizedv alternating projection (GAP)[2]
are used, where ADMM for noisy data, especially real data and GAP for
noise-free data.
gamma : float, optional
Parameter in the ADMM projection, where more noisy measurements require
greater gamma.
denoiser : string, optional
Denoiser used as the regularization imposing on the prior term of the
reconstruction.
_lambda : float, optional
Regularization factor balancing the data term and the prior term,
where larger `_lambda` imposing more constrains on the prior term.
iter_max : int or uint, optional
Maximum number of iterations.
accelerate : boolean, optional
Enable acceleration in GAP.
noise_estimate : boolean, optional
Enable noise estimation in the denoiser.
sigma : one-dimensional (1D) ndarray of ints, uints or floats
Input noise standard deviation for the denoiser if and only if noise
estimation is disabled(i.e., noise_estimate==False). The scale of sigma
is [0, 255] regardless of the the scale of the input measurement and
masks.
tv_weight : float, optional
weight in total variation (TV) denoising.
x0 : 3D ndarray
Start point (initialized value) for the iteration process of the
reconstruction.
model : pretrained model for image/video denoising.
Returns
-------
x : 3D ndarray
Reconstructed 3D scene captured by the SCI system.
References
----------
.. [1] X. Liao, H. Li, and L. Carin, "Generalized Alternating Projection
for Weighted-$\ell_{2,1}$ Minimization with Applications to
Model-Based Compressive Sensing," SIAM Journal on Imaging Sciences,
vol. 7, no. 2, pp. 797-823, 2014.
.. [2] X. Yuan, "Generalized alternating projection based total variation
minimization for compressive sensing," in IEEE International
Conference on Image Processing (ICIP), 2016, pp. 2539-2543.
.. [3] Y. Liu, X. Yuan, J. Suo, D. Brady, and Q. Dai, "Rank Minimization
for Snapshot Compressive Imaging," IEEE Transactions on Pattern
Analysis and Machine Intelligence, doi:10.1109/TPAMI.2018.2873587,
2018.
Code credit
-----------
Xin Yuan, Bell Labs, [email protected], created Aug 7, 2018.
Yang Liu, Tsinghua University, [email protected],
updated Jan 22, 2019.
See Also
--------
admm_denoise
'''
# [0] initialization
if x0 is None:
print(At)
x0 = At(y, Phi) # default start point (initialized value)
if not isinstance(sigma, list):
sigma = [sigma]
if not isinstance(iter_max, list):
iter_max = [iter_max] * len(sigma)
y1 = np.zeros_like(y)
Phi_sum = np.sum(Phi, 2)
Phi_sum[Phi_sum == 0] = 1
# [1] start iteration for reconstruction
x = x0 # initialization
psnr_all = []
ssim_all = []
k = 0
device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')
model = net()
model.load_state_dict(torch.load(r'./check_points/deep_denoiser.pth'))
model.eval()
for q, v in model.named_parameters():
v.requires_grad = False
model = model.to(device)
for idx, nsig in enumerate(sigma): # iterate all noise levels
for it in range(iter_max[idx]):
#print('max1_{0}_{1}:'.format(idx,it),np.max(x))
yb = A(x, Phi)
if accelerate: # accelerated version of GAP
y1 = y1 + (y - yb)
x = x + _lambda * (At((y1 - yb) / Phi_sum, Phi)) # GAP_acc
else:
x = x + _lambda * (At((y - yb) / Phi_sum, Phi)) # GAP
x = shift_back(x, step=1)
# switch denoiser
if denoiser.lower() == 'tv': # total variation (TV) denoising
x = denoise_tv_chambolle(x,
nsig / 255,
n_iter_max=tv_iter_max,
multichannel=multichannel)
#x= TV_denoiser(x, tv_weight, n_iter_max=tv_iter_max)
elif denoiser.lower() == 'hsicnn':
l_ch = 10
m_ch = 10
h_ch = 10
if (k > 123 and k <= 125) or (k >= 119 and k <= 121) or (
k >= 115 and k <= 117
) or (k >= 111 and k <= 113) or (k >= 107 and k <= 109) or (
k >= 103 and k <= 105) or (k >= 99 and k <= 101) or (
k >= 95 and k <= 97) or (k >= 91 and k <= 93) or (
k >= 87 and k <= 89) or (k >= 83 and k <= 85):
tem = None
for i in range(31):
net_input = None
if i < 3:
ori_nsig = nsig
if i == 0:
net_input = np.dstack(
(x[:, :, i], x[:, :, i], x[:, :,
i], x[:, :,
i:i + 4]))
elif i == 1:
net_input = np.dstack(
(x[:, :, i - 1], x[:, :, i - 1],
x[:, :, i - 1], x[:, :, i:i + 4]))
elif i == 2:
net_input = np.dstack(
(x[:, :, i - 2], x[:, :, i - 2],
x[:, :, i - 1], x[:, :, i:i + 4]))
net_input = torch.from_numpy(
np.ascontiguousarray(net_input)).permute(
2, 0, 1).float().unsqueeze(0)
net_input = net_input.to(device)
Nsigma = torch.full((1, 1, 1, 1),
l_ch / 255.).type_as(net_input)
output = model(net_input, Nsigma)
output = output.data.squeeze().cpu().numpy()
if k < 0:
output = denoise_tv_chambolle(
x[:, :, i],
nsig / 255,
n_iter_max=tv_iter_max,
multichannel=False)
nsig = ori_nsig
if i == 0:
tem = output
else:
tem = np.dstack((tem, output))
elif i > 27:
ori_nsig = nsig
if k >= 45:
nsig /= 1
if i == 28:
net_input = np.dstack(
(x[:, :, i - 3:i + 1], x[:, :, i + 1],
x[:, :, i + 2], x[:, :, i + 2]))
elif i == 29:
net_input = np.dstack(
(x[:, :, i - 3:i + 1], x[:, :, i + 1],
x[:, :, i + 1], x[:, :, i + 1]))
elif i == 30:
net_input = np.dstack(
(x[:, :, i - 3:i + 1], x[:, :,
i], x[:, :,
i], x[:, :,
i]))
net_input = torch.from_numpy(
np.ascontiguousarray(net_input)).permute(
2, 0, 1).float().unsqueeze(0)
net_input = net_input.to(device)
Nsigma = torch.full((1, 1, 1, 1),
m_ch / 255.).type_as(net_input)
output = model(net_input, Nsigma)
output = output.data.squeeze().cpu().numpy()
if k < 0:
output = denoise_tv_chambolle(
x[:, :, i],
10 / 255,
n_iter_max=tv_iter_max,
multichannel=False)
tem = np.dstack((tem, output))
nsig = ori_nsig
else:
ori_nsig = nsig
net_input = x[:, :, i - 3:i + 4]
net_input = torch.from_numpy(
np.ascontiguousarray(net_input)).permute(
2, 0, 1).float().unsqueeze(0)
net_input = net_input.to(device)
Nsigma = torch.full((1, 1, 1, 1),
h_ch / 255.).type_as(net_input)
output = model(net_input, Nsigma)
output = output.data.squeeze().cpu().numpy()
tem = np.dstack((tem, output))
nsig = ori_nsig
#x = np.clip(tem,0,1)
x = tem
else:
x = denoise_tv_chambolle(x,
nsig / 255,
n_iter_max=tv_iter_max,
multichannel=multichannel)
#x = TV_denoiser(x, tv_weight, n_iter_max=tv_iter_max)
elif denoiser.lower() == 'bm3d':
sigma = nsig / 255
v = np.zeros((15, 15))
for x1 in range(-7, 8, 1):
for x2 in range(-7, 8, 1):
v[x1 + 7, x2 + 7] = 1 / (x1**2 + x2**2 + 1)
v = v / np.sum(v)
for i in range(28):
x[:, :, i] = bm3d_deblurring(np.atleast_3d(x[:, :, i]),
sigma, v)
else:
raise ValueError('Unsupported denoiser {}!'.format(denoiser))
# [optional] calculate image quality assessment, i.e., PSNR for
# every five iterations
if show_iqa and X_orig is not None:
ssim_all.append(calculate_ssim(X_orig, x))
psnr_all.append(psnr(X_orig, x))
if (k + 1) % 1 == 0:
if not noise_estimate and nsig is not None:
if nsig < 1:
print(
' GAP-{0} iteration {1: 3d}, sigma {2: 3g}/255, '
'PSNR {3:2.2f} dB.'.format(
denoiser.upper(), k + 1, nsig * 255,
psnr_all[k]),
'SSIM:{}'.format(ssim_all[k]))
else:
print(
' GAP-{0} iteration {1: 3d}, sigma {2: 3g}, '
'PSNR {3:2.2f} dB.'.format(
denoiser.upper(), k + 1, nsig,
psnr_all[k]),
'SSIM:{}'.format(ssim_all[k]))
else:
print(
' GAP-{0} iteration {1: 3d}, '
'PSNR {2:2.2f} dB.'.format(denoiser.upper(), k + 1,
psnr_all[k]),
'SSIM:{}'.format(ssim_all[k]))
x = shift(x, step=1)
if k == 123:
break
k = k + 1
return x, psnr_all
def scale_corners(corners):
result = []
for point in corners:
point = scale(point, 1/RESIZE)
result.append(shift(point, (60,70)))
return result
net = caffe.Net(network_proto_path, network_model_path, caffe.TEST)
transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_input_scale('data', 0.1)
transformer.set_mean('data', np.array([104, 117, 123]))
transformer.set_transpose('data', (2, 0, 1))
while True:
# Generate test images
img, bboxes = imdb.read()
jittered_bboxes = bbox_jitter(bboxes[0], 0.1, 3)
gt_bbox = bbox_transform_inv(bboxes[0])
j_box = bbox_transform_inv(jittered_bboxes[0])
test_img = img[j_box[1]:j_box[1] + j_box[3],
j_box[0]:j_box[0] + j_box[2], :]
test_img = caffe.io.resize(test_img, [48, 48, 3])
offset = net.forward_all(
data=np.asarray([transformer.preprocess('data', test_img)]))
print offset['conv6'][0, :]
print j_box
out_box = shift(j_box, offset['conv6'][0, :])
cv2.rectangle(img, (int(out_box[0]), int(out_box[1])),
(int(out_box[2]), int(out_box[3])), (0, 255, 0), 1)
cv2.rectangle(img, (int(j_box[0]), int(j_box[1])),
(int(j_box[2]), int(j_box[3])), (255, 0, 0), 1)
cv2.rectangle(img, (int(gt_bbox[0]), int(gt_bbox[1])),
(int(gt_bbox[2]), int(gt_bbox[3])), (0, 0, 255), 1)
cv2.imshow('output', img)
cv2.waitKey(0)
from utils import clip,shift import torch print(clip(torch.tensor(8.0),2)) print(shift(torch.tensor(0.00001,dtype=float))) loss_fn = torch.nn.MSELoss(reduce=True, size_average=True) #loss_fn = torch.nn.MSELoss() loss_fn = torch.nn.CrossEntropyLoss() input = torch.autograd.Variable(torch.randn(3,4)) target = torch.autograd.Variable(torch.randn(3,4)) loss = loss_fn(input, target) print(input); print(target); print(loss) print(input.size(), target.size(), loss.size())
def admm_denoise(y,
Phi,
A,
At,
_lambda=1,
gamma=0.01,
denoiser='tv',
iter_max=50,
noise_estimate=True,
sigma=None,
tv_weight=0.1,
tv_iter_max=5,
multichannel=True,
x0=None,
X_orig=None,
show_iqa=True):
'''
Alternating direction method of multipliers (ADMM)[1]-based denoising
regularization for snapshot compressive imaging (SCI).
Parameters
----------
y : two-dimensional (2D) ndarray of ints, uints or floats
Input single measurement of the snapshot compressive imager (SCI).
Phi : three-dimensional (3D) ndarray of ints, uints or floats, omitted
Input sensing matrix of SCI with the third dimension as the
time-variant, spectral-variant, volume-variant, or angular-variant
masks, where each mask has the same pixel resolution as the snapshot
measurement.
Phi_sum : 2D ndarray
Sum of the sensing matrix `Phi` along the third dimension.
A : function
Forward model of SCI, where multiple encoded frames are collapsed into
a single measurement.
At : function
Transpose of the forward model.
proj_meth : {'admm' or 'gap'}, optional
Projection method of the data term. Alternating direction method of
multipliers (ADMM)[1] and generalizedv alternating projection (GAP)[2]
are used, where ADMM for noisy data, especially real data and GAP for
noise-free data.
gamma : float, optional
Parameter in the ADMM projection, where more noisy measurements require
greater gamma.
denoiser : string, optional
Denoiser used as the regularization imposing on the prior term of the
reconstruction.
_lambda : float, optional
Regularization factor balancing the data term and the prior term,
where larger `_lambda` imposing more constrains on the prior term.
iter_max : int or uint, optional
Maximum number of iterations.
accelerate : boolean, optional
Enable acceleration in GAP.
noise_estimate : boolean, optional
Enable noise estimation in the denoiser.
sigma : one-dimensional (1D) ndarray of ints, uints or floats
Input noise standard deviation for the denoiser if and only if noise
estimation is disabled(i.e., noise_estimate==False). The scale of sigma
is [0, 255] regardless of the the scale of the input measurement and
masks.
tv_weight : float, optional
weight in total variation (TV) denoising.
x0 : 3D ndarray
Start point (initialized value) for the iteration process of the
reconstruction.
Returns
-------
x : 3D ndarray
Reconstructed 3D scene captured by the SCI system.
References
----------
.. [1] S. Boyd, N. Parikh, E. Chu, B. Peleato, and J. Eckstein,
"Distributed Optimization and Statistical Learning via the
Alternating Direction Method of Multipliers," Foundations and
Trends® in Machine Learning, vol. 3, no. 1, pp. 1-122, 2011.
.. [2] X. Yuan, "Generalized alternating projection based total variation
minimization for compressive sensing," in IEEE International
Conference on Image Processing (ICIP), 2016, pp. 2539-2543.
.. [3] Y. Liu, X. Yuan, J. Suo, D. Brady, and Q. Dai, "Rank Minimization
for Snapshot Compressive Imaging," IEEE Transactions on Pattern
Analysis and Machine Intelligence, doi:10.1109/TPAMI.2018.2873587,
2018.
Code credit
-----------
Xin Yuan, Bell Labs, [email protected], created Aug 7, 2018.
Yang Liu, Tsinghua University, [email protected],
updated Jan 22, 2019.
See Also
--------
gap_denoise
'''
# [0] initialization
if x0 is None:
x0 = At(y, Phi) # default start point (initialized value)
if not isinstance(sigma, list):
sigma = [sigma]
if not isinstance(iter_max, list):
iter_max = [iter_max] * len(sigma)
# [1] start iteration for reconstruction
x = x0 # initialization
theta = x0
Phi_sum = np.sum(Phi, 2)
Phi_sum[Phi_sum == 0] = 1
b = np.zeros_like(x0)
psnr_all = []
ssim_all = []
k = 0
device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')
model = net()
model.load_state_dict(
torch.load(
r'/home/dgl/zhengsiming/self_train/check_points/best_smallsigma.pth'
))
model.eval()
for q, v in model.named_parameters():
v.requires_grad = False
model = model.to(device)
for idx, nsig in enumerate(sigma): # iterate all noise levels
for it in range(iter_max[idx]):
# Euclidean projection
yb = A(theta + b, Phi)
x = (theta + b) + _lambda * (At(
(y - yb) / (Phi_sum + gamma), Phi)) # ADMM
x1 = shift_back(x - b, step=2)
#x1=x-b
# switch denoiser
if denoiser.lower() == 'tv': # total variation (TV) denoising
#theta = denoise_tv_chambolle(x1, nsig/255, n_iter_max=tv_iter_max, multichannel=multichannel)
theta = TV_denoiser(x1, tv_weight, n_iter_max=tv_iter_max)
elif denoiser.lower() == 'wavelet': # wavelet denoising
if noise_estimate or nsig is None: # noise estimation enabled
theta = denoise_wavelet(x1, multichannel=multichannel)
else:
theta = denoise_wavelet(x1,
sigma=nsig,
multichannel=multichannel)
elif denoiser.lower() == 'vnlnet': # Video Non-local net denoising
theta = vnlnet(np.expand_dims((x1).transpose(2, 0, 1), 3),
nsig)
theta = np.transpose(theta.squeeze(3), (1, 2, 0))
elif denoiser.lower() == 'hsicnn':
if k >= 89:
tem = None
for i in range(28):
net_input = None
if i < 3:
if i == 0:
net_input = np.dstack(
(x1[:, :, i], x1[:, :,
i], x1[:, :,
i], x1[:, :,
i:i + 4]))
elif i == 1:
net_input = np.dstack(
(x1[:, :, i - 1], x1[:, :, i - 1],
x1[:, :, i - 1], x1[:, :, i:i + 4]))
elif i == 2:
net_input = np.dstack(
(x1[:, :, i - 2], x1[:, :, i - 2],
x1[:, :, i - 1], x1[:, :, i:i + 4]))
net_input = torch.from_numpy(
np.ascontiguousarray(net_input)).permute(
2, 0, 1).float().unsqueeze(0)
net_input = net_input.to(device)
Nsigma = torch.full((1, 1, 1, 1),
10 / 255.).type_as(net_input)
output = model(net_input, Nsigma)
output = output.data.squeeze().float().cpu().numpy(
)
if i == 0:
tem = output
else:
tem = np.dstack((tem, output))
elif i > 24:
if i == 25:
net_input = np.dstack(
(x1[:, :, i - 3:i + 1], x1[:, :, i + 1],
x1[:, :, i + 2], x1[:, :, i + 2]))
elif i == 26:
net_input = np.dstack(
(x1[:, :, i - 3:i + 1], x1[:, :, i + 1],
x1[:, :, i + 1], x1[:, :, i + 1]))
elif i == 27:
net_input = np.dstack(
(x1[:, :, i - 3:i + 1], x1[:, :, i],
x1[:, :, i], x1[:, :, i]))
net_input = torch.from_numpy(
np.ascontiguousarray(net_input)).permute(
2, 0, 1).float().unsqueeze(0)
net_input = net_input.to(device)
Nsigma = torch.full((1, 1, 1, 1),
10 / 255.).type_as(net_input)
output = model(net_input, Nsigma)
output = output.data.squeeze().float().cpu().numpy(
)
tem = np.dstack((tem, output))
else:
net_input = x1[:, :, i - 3:i + 4]
net_input = torch.from_numpy(
np.ascontiguousarray(net_input)).permute(
2, 0, 1).float().unsqueeze(0)
net_input = net_input.to(device)
Nsigma = torch.full((1, 1, 1, 1),
10 / 255.).type_as(net_input)
output = model(net_input, Nsigma)
output = output.data.squeeze().float().cpu().numpy(
)
tem = np.dstack((tem, output))
theta = tem
else:
#print('theta:', np.max(theta))
theta = denoise_tv_chambolle(x1,
tv_weight,
n_iter_max=tv_iter_max,
multichannel=multichannel)
else:
raise ValueError('Unsupported denoiser {}!'.format(denoiser))
# [optional] calculate image quality assessment, i.e., PSNR for
# every five iterations
if show_iqa and X_orig is not None:
psnr_all.append(psnr(X_orig, theta))
ssim_all.append(calculate_ssim(X_orig, theta))
if (k + 1) % 1 == 0:
if not noise_estimate and nsig is not None:
if nsig < 1:
print(
' ADMM-{0} iteration {1: 3d}, sigma {2: 3g}/255, '
'PSNR {3:2.2f} dB.'.format(
denoiser.upper(), k + 1, nsig * 255,
psnr_all[k]),
'SSIM:{}'.format(ssim_all[k]))
else:
print(
' ADMM-{0} iteration {1: 3d}, sigma {2: 3g}, '
'PSNR {3:2.2f} dB.'.format(
denoiser.upper(), k + 1, nsig,
psnr_all[k]),
'SSIM:{}'.format(ssim_all[k]))
else:
print(
' ADMM-{0} iteration {1: 3d}, '
'PSNR {2: 2.2f} dB.'.format(
denoiser.upper(), k + 1, psnr_all[k]),
'SSIM:{}'.format(ssim_all[k]))
theta = shift(theta, step=2)
b = b - (x - theta) # update residual
k = k + 1
return theta, psnr_all, ssim_all
counter = 1
for Xtt in Xt:
#print('Xtt',Xtt)
observed = np.array(Xtt[0:obsLength, :])
truth = np.array(Xtt[obsLength:seqLength, :])
prediction = np.array(Xtt[obsLength:seqLength, :])
for i in range(obsLength, seqLength):
predicted_params = model.predict(np.array([observed]), verbose=0)
#print('params', predicted_params.shape, predicted_params)
pred = sample_gaussian_2d(predicted_params)
#print('truth', Xtt[i], 'pred', pred[1], pred[0])
prediction[i - obsLength, 0] = pred[1]
prediction[i - obsLength, 1] = pred[0]
squared_error += np.sum((Xtt[i] - pred[0])**2)
observed = shift(observed, -1)
#observed[obsLength-1] = np.array(Xtt[i]) #using truth
observed[obsLength - 1] = np.array([pred[1],
pred[0]]) #using predicted
#print(observed)
#print(prediction)
plt.plot(Xtt[0:obsLength, 0], Xtt[0:obsLength, 1], marker='x')
plt.plot(Xtt[obsLength:seqLength, 0],
Xtt[obsLength:seqLength, 1],
marker='x')
plt.plot(prediction[:, 0], prediction[:, 1], marker='x')
plt.xlabel('x')
plt.ylabel('y')
plt.savefig('dataset_' + str(datatest) + '_' + str(counter) + '.png')
plt.cla()
#plt.show()
def run_nn(data_name,data_set,data_end_index,fea_dict,lab_dict,arch_dict,cfg_file,processed_first,next_config_file):
# This function processes the current chunk using the information in cfg_file. In parallel, the next chunk is load into the CPU memory
# Reading chunk-specific cfg file (first argument-mandatory file)
if not(os.path.exists(cfg_file)):
sys.stderr.write('ERROR: The config file %s does not exist!\n'%(cfg_file))
sys.exit(0)
else:
config = configparser.ConfigParser()
config.read(cfg_file)
# Setting torch seed
seed=int(config['exp']['seed'])
torch.manual_seed(seed)
random.seed(seed)
np.random.seed(seed)
# Reading config parameters
output_folder=config['exp']['out_folder']
use_cuda=strtobool(config['exp']['use_cuda'])
multi_gpu=strtobool(config['exp']['multi_gpu'])
to_do=config['exp']['to_do']
info_file=config['exp']['out_info']
model=config['model']['model'].split('\n')
forward_outs=config['forward']['forward_out'].split(',')
forward_normalize_post=list(map(strtobool,config['forward']['normalize_posteriors'].split(',')))
forward_count_files=config['forward']['normalize_with_counts_from'].split(',')
require_decodings=list(map(strtobool,config['forward']['require_decoding'].split(',')))
use_cuda=strtobool(config['exp']['use_cuda'])
save_gpumem=strtobool(config['exp']['save_gpumem'])
is_production=strtobool(config['exp']['production'])
if to_do=='train':
batch_size=int(config['batches']['batch_size_train'])
if to_do=='valid':
batch_size=int(config['batches']['batch_size_valid'])
if to_do=='forward':
batch_size=1
# ***** Reading the Data********
if processed_first:
# Reading all the features and labels for this chunk
shared_list=[]
p=threading.Thread(target=read_lab_fea, args=(cfg_file,is_production,shared_list,output_folder,))
p.start()
p.join()
data_name=shared_list[0]
data_end_index=shared_list[1]
fea_dict=shared_list[2]
lab_dict=shared_list[3]
arch_dict=shared_list[4]
data_set=shared_list[5]
# converting numpy tensors into pytorch tensors and put them on GPUs if specified
if not(save_gpumem) and use_cuda:
data_set=torch.from_numpy(data_set).float().cuda()
else:
data_set=torch.from_numpy(data_set).float()
# Reading all the features and labels for the next chunk
shared_list=[]
p=threading.Thread(target=read_lab_fea, args=(next_config_file,is_production,shared_list,output_folder,))
p.start()
# Reading model and initialize networks
inp_out_dict=fea_dict
[nns,costs]=model_init(inp_out_dict,model,config,arch_dict,use_cuda,multi_gpu,to_do)
# optimizers initialization
optimizers=optimizer_init(nns,config,arch_dict)
# pre-training and multi-gpu init
for net in nns.keys():
pt_file_arch=config[arch_dict[net][0]]['arch_pretrain_file']
if pt_file_arch!='none':
checkpoint_load = torch.load(pt_file_arch)
nns[net].load_state_dict(checkpoint_load['model_par'])
optimizers[net].load_state_dict(checkpoint_load['optimizer_par'])
optimizers[net].param_groups[0]['lr']=float(config[arch_dict[net][0]]['arch_lr']) # loading lr of the cfg file for pt
if multi_gpu:
nns[net] = torch.nn.DataParallel(nns[net])
if to_do=='forward':
post_file={}
for out_id in range(len(forward_outs)):
if require_decodings[out_id]:
out_file=info_file.replace('.info','_'+forward_outs[out_id]+'_to_decode.ark')
else:
out_file=info_file.replace('.info','_'+forward_outs[out_id]+'.ark')
post_file[forward_outs[out_id]]=open_or_fd(out_file,output_folder,'wb')
# check automatically if the model is sequential
seq_model=is_sequential_dict(config,arch_dict)
# ***** Minibatch Processing loop********
if seq_model or to_do=='forward':
N_snt=len(data_name)
N_batches=int(N_snt/batch_size)
else:
N_ex_tr=data_set.shape[0]
N_batches=int(N_ex_tr/batch_size)
beg_batch=0
end_batch=batch_size
snt_index=0
beg_snt=0
start_time = time.time()
# array of sentence lengths
arr_snt_len=shift(shift(data_end_index, -1,0)-data_end_index,1,0)
arr_snt_len[0]=data_end_index[0]
loss_sum=0
err_sum=0
inp_dim=data_set.shape[1]
for i in range(N_batches):
max_len=0
if seq_model:
max_len=int(max(arr_snt_len[snt_index:snt_index+batch_size]))
inp= torch.zeros(max_len,batch_size,inp_dim).contiguous()
for k in range(batch_size):
snt_len=data_end_index[snt_index]-beg_snt
N_zeros=max_len-snt_len
# Appending a random number of initial zeros, tge others are at the end.
N_zeros_left=random.randint(0,N_zeros)
# randomizing could have a regularization effect
inp[N_zeros_left:N_zeros_left+snt_len,k,:]=data_set[beg_snt:beg_snt+snt_len,:]
beg_snt=data_end_index[snt_index]
snt_index=snt_index+1
else:
# features and labels for batch i
if to_do!='forward':
inp= data_set[beg_batch:end_batch,:].contiguous()
else:
snt_len=data_end_index[snt_index]-beg_snt
inp= data_set[beg_snt:beg_snt+snt_len,:].contiguous()
beg_snt=data_end_index[snt_index]
snt_index=snt_index+1
# use cuda
if use_cuda:
inp=inp.cuda()
if to_do=='train':
# Forward input, with autograd graph active
outs_dict=forward_model(fea_dict,lab_dict,arch_dict,model,nns,costs,inp,inp_out_dict,max_len,batch_size,to_do,forward_outs)
for opt in optimizers.keys():
optimizers[opt].zero_grad()
outs_dict['loss_final'].backward()
# Gradient Clipping (th 0.1)
#for net in nns.keys():
# torch.nn.utils.clip_grad_norm_(nns[net].parameters(), 0.1)
for opt in optimizers.keys():
if not(strtobool(config[arch_dict[opt][0]]['arch_freeze'])):
optimizers[opt].step()
else:
with torch.no_grad(): # Forward input without autograd graph (save memory)
outs_dict=forward_model(fea_dict,lab_dict,arch_dict,model,nns,costs,inp,inp_out_dict,max_len,batch_size,to_do,forward_outs)
if to_do=='forward':
for out_id in range(len(forward_outs)):
out_save=outs_dict[forward_outs[out_id]].data.cpu().numpy()
if forward_normalize_post[out_id]:
# read the config file
counts = load_counts(forward_count_files[out_id])
out_save=out_save-np.log(counts/np.sum(counts))
# save the output
write_mat(output_folder,post_file[forward_outs[out_id]], out_save, data_name[i])
else:
loss_sum=loss_sum+outs_dict['loss_final'].detach()
err_sum=err_sum+outs_dict['err_final'].detach()
# update it to the next batch
beg_batch=end_batch
end_batch=beg_batch+batch_size
# Progress bar
if to_do == 'train':
status_string="Training | (Batch "+str(i+1)+"/"+str(N_batches)+")"+" | L:" +str(round(loss_sum.cpu().item()/(i+1),3))
if i==N_batches-1:
status_string="Training | (Batch "+str(i+1)+"/"+str(N_batches)+")"
if to_do == 'valid':
status_string="Validating | (Batch "+str(i+1)+"/"+str(N_batches)+")"
if to_do == 'forward':
status_string="Forwarding | (Batch "+str(i+1)+"/"+str(N_batches)+")"
progress(i, N_batches, status=status_string)
elapsed_time_chunk=time.time() - start_time
loss_tot=loss_sum/N_batches
err_tot=err_sum/N_batches
# clearing memory
del inp, outs_dict, data_set
# save the model
if to_do=='train':
for net in nns.keys():
checkpoint={}
if multi_gpu:
checkpoint['model_par']=nns[net].module.state_dict()
else:
checkpoint['model_par']=nns[net].state_dict()
checkpoint['optimizer_par']=optimizers[net].state_dict()
out_file=info_file.replace('.info','_'+arch_dict[net][0]+'.pkl')
torch.save(checkpoint, out_file)
if to_do=='forward':
for out_name in forward_outs:
post_file[out_name].close()
# Write info file
with open(info_file, "w") as text_file:
text_file.write("[results]\n")
if to_do!='forward':
text_file.write("loss=%s\n" % loss_tot.cpu().numpy())
text_file.write("err=%s\n" % err_tot.cpu().numpy())
text_file.write("elapsed_time_chunk=%f\n" % elapsed_time_chunk)
text_file.close()
# Getting the data for the next chunk (read in parallel)
p.join()
data_name=shared_list[0]
data_end_index=shared_list[1]
fea_dict=shared_list[2]
lab_dict=shared_list[3]
arch_dict=shared_list[4]
data_set=shared_list[5]
# converting numpy tensors into pytorch tensors and put them on GPUs if specified
if not(save_gpumem) and use_cuda:
data_set=torch.from_numpy(data_set).float().cuda()
else:
data_set=torch.from_numpy(data_set).float()
return [data_name,data_set,data_end_index,fea_dict,lab_dict,arch_dict]
def run_nn_refac01(data_name,data_set,data_end_index,fea_dict,lab_dict,arch_dict,cfg_file,processed_first,next_config_file):
def _read_chunk_specific_config(cfg_file):
if not(os.path.exists(cfg_file)):
sys.stderr.write('ERROR: The config file %s does not exist!\n'%(cfg_file))
sys.exit(0)
else:
config = configparser.ConfigParser()
config.read(cfg_file)
return config
def _get_batch_size_from_config(config, to_do):
if to_do=='train':
batch_size=int(config['batches']['batch_size_train'])
elif to_do=='valid':
batch_size=int(config['batches']['batch_size_valid'])
elif to_do=='forward':
batch_size=1
return batch_size
def _initialize_random_seed(config):
seed=int(config['exp']['seed'])
torch.manual_seed(seed)
random.seed(seed)
np.random.seed(seed)
def _load_model_and_optimizer(fea_dict,model,config,arch_dict,use_cuda,multi_gpu,to_do):
inp_out_dict = fea_dict
nns, costs = model_init(inp_out_dict,model,config,arch_dict,use_cuda,multi_gpu,to_do)
optimizers = optimizer_init(nns,config,arch_dict)
for net in nns.keys():
pt_file_arch=config[arch_dict[net][0]]['arch_pretrain_file']
if pt_file_arch!='none':
if use_cuda:
checkpoint_load = torch.load(pt_file_arch)
else:
checkpoint_load = torch.load(pt_file_arch, map_location='cpu')
nns[net].load_state_dict(checkpoint_load['model_par'])
if net in optimizers:
optimizers[net].load_state_dict(checkpoint_load['optimizer_par'])
optimizers[net].param_groups[0]['lr']=float(config[arch_dict[net][0]]['arch_lr']) # loading lr of the cfg file for pt
if multi_gpu:
nns[net] = torch.nn.DataParallel(nns[net])
return nns, costs, optimizers, inp_out_dict
def _open_forward_output_files_and_get_file_handles(forward_outs, require_decodings, info_file, output_folder):
post_file={}
for out_id in range(len(forward_outs)):
if require_decodings[out_id]:
out_file=info_file.replace('.info','_'+forward_outs[out_id]+'_to_decode.ark')
else:
out_file=info_file.replace('.info','_'+forward_outs[out_id]+'.ark')
post_file[forward_outs[out_id]]=open_or_fd(out_file,output_folder,'wb')
return post_file
def _get_batch_config(data_set_input, seq_model, to_do, data_name, batch_size):
N_snt = None
N_ex_tr = None
N_batches = None
if seq_model or to_do=='forward':
N_snt=len(data_name)
N_batches=int(N_snt/batch_size)
else:
N_ex_tr=data_set_input.shape[0]
N_batches=int(N_ex_tr/batch_size)
return N_snt, N_ex_tr, N_batches
def _prepare_input(snt_index, batch_size, inp_dim, ref_dim, beg_snt_fea, beg_snt_lab, data_end_index_fea, data_end_index_lab, beg_batch, end_batch, seq_model, arr_snt_len_fea, arr_snt_len_lab, data_set_inp, data_set_ref, use_cuda):
def _zero_padding(inp, ref, max_len_fea, max_len_lab, data_end_index_fea, data_end_index_lab, data_set_inp, data_set_ref, beg_snt_fea, beg_snt_lab, snt_index, k):
def _input_and_ref_have_same_time_dimension(N_zeros_fea, N_zeros_lab):
if N_zeros_fea == N_zeros_lab:
return True
return False
snt_len_fea = data_end_index_fea[snt_index] - beg_snt_fea
snt_len_lab = data_end_index_lab[snt_index] - beg_snt_lab
N_zeros_fea = max_len_fea - snt_len_fea
N_zeros_lab = max_len_lab - snt_len_lab
if _input_and_ref_have_same_time_dimension(N_zeros_fea, N_zeros_lab):
N_zeros_fea_left = random.randint(0,N_zeros_fea)
N_zeros_lab_left = N_zeros_fea_left
else:
N_zeros_fea_left = 0
N_zeros_lab_left = 0
inp[N_zeros_fea_left:N_zeros_fea_left+snt_len_fea,k,:] = data_set_inp[beg_snt_fea:beg_snt_fea+snt_len_fea,:]
ref[N_zeros_lab_left:N_zeros_lab_left+snt_len_lab,k,:] = data_set_ref[beg_snt_lab:beg_snt_lab+snt_len_lab,:]
return inp, ref, snt_len_fea, snt_len_lab
if len(data_set_ref.shape) == 1:
data_set_ref = data_set_ref.shape.view((data_set_ref.shape[0], 1))
max_len=0
if seq_model:
max_len_fea = int(max(arr_snt_len_fea[snt_index:snt_index+batch_size]))
max_len_lab = int(max(arr_snt_len_lab[snt_index:snt_index+batch_size]))
inp = torch.zeros(max_len_fea,batch_size,inp_dim).contiguous()
ref = torch.zeros(max_len_lab,batch_size,ref_dim).contiguous()
for k in range(batch_size):
inp, ref, snt_len_fea, snt_len_lab = _zero_padding(inp, ref, max_len_fea, max_len_lab, data_end_index_fea, data_end_index_lab, data_set_inp, data_set_ref, beg_snt_fea, beg_snt_lab, snt_index, k)
beg_snt_fea = data_end_index_fea[snt_index]
beg_snt_lab = data_end_index_lab[snt_index]
snt_index = snt_index + 1
else:
if to_do != 'forward':
inp = data_set[beg_batch:end_batch,:].contiguous()
else:
snt_len_fea = data_end_index_fea[snt_index] - beg_snt_fea
snt_len_lab = data_end_index_lab[snt_index] - beg_snt_lab
inp = data_set_inp[beg_snt_fea:beg_snt_fea+snt_len_fea,:].contiguous()
ref = data_set_ref[beg_snt_lab:beg_snt_lab+snt_len_lab,:].contiguous()
beg_snt_fea = data_end_index_fea[snt_index]
beg_snt_lab = data_end_index_lab[snt_index]
snt_index = snt_index + 1
if use_cuda:
inp=inp.cuda()
ref=ref.cuda()
return inp, ref, max_len_fea, max_len_lab, snt_len_fea, snt_len_lab, beg_snt_fea, beg_snt_lab, snt_index
def _optimization_step(optimizers, outs_dict, config, arch_dict):
for opt in optimizers.keys():
optimizers[opt].zero_grad()
outs_dict['loss_final'].backward()
for opt in optimizers.keys():
if not(strtobool(config[arch_dict[opt][0]]['arch_freeze'])):
optimizers[opt].step()
def _update_progress_bar(to_do, i, N_batches, loss_sum):
if to_do == 'train':
status_string="Training | (Batch "+str(i+1)+"/"+str(N_batches)+")"+" | L:" +str(round(loss_sum.cpu().item()/(i+1),3))
if i==N_batches-1:
status_string="Training | (Batch "+str(i+1)+"/"+str(N_batches)+")"
if to_do == 'valid':
status_string="Validating | (Batch "+str(i+1)+"/"+str(N_batches)+")"
if to_do == 'forward':
status_string="Forwarding | (Batch "+str(i+1)+"/"+str(N_batches)+")"
progress(i, N_batches, status=status_string)
def _write_info_file(info_file, to_do, loss_tot, err_tot, elapsed_time_chunk):
with open(info_file, "w") as text_file:
text_file.write("[results]\n")
if to_do!='forward':
text_file.write("loss=%s\n" % loss_tot.cpu().numpy())
text_file.write("err=%s\n" % err_tot.cpu().numpy())
text_file.write("elapsed_time_chunk=%f\n" % elapsed_time_chunk)
text_file.close()
def _save_model(to_do, nns, multi_gpu, optimizers, info_file, arch_dict):
if to_do=='train':
for net in nns.keys():
checkpoint={}
if multi_gpu:
checkpoint['model_par']=nns[net].module.state_dict()
else:
checkpoint['model_par']=nns[net].state_dict()
if net in optimizers:
checkpoint['optimizer_par']=optimizers[net].state_dict()
else:
checkpoint['optimizer_par']=dict()
out_file=info_file.replace('.info','_'+arch_dict[net][0]+'.pkl')
torch.save(checkpoint, out_file)
def _get_dim_from_data_set(data_set_inp, data_set_ref):
inp_dim = data_set_inp.shape[1]
ref_dim = 1
if len(data_set_ref.shape) > 1:
ref_dim = data_set_ref.shape[1]
return inp_dim, ref_dim
from data_io import read_lab_fea_refac01 as read_lab_fea
from utils import forward_model_refac01 as forward_model
config = _read_chunk_specific_config(cfg_file)
_initialize_random_seed(config)
output_folder = config['exp']['out_folder']
use_cuda = strtobool(config['exp']['use_cuda'])
multi_gpu = strtobool(config['exp']['multi_gpu'])
to_do = config['exp']['to_do']
info_file = config['exp']['out_info']
model = config['model']['model'].split('\n')
forward_outs = config['forward']['forward_out'].split(',')
forward_normalize_post = list(map(strtobool,config['forward']['normalize_posteriors'].split(',')))
forward_count_files = config['forward']['normalize_with_counts_from'].split(',')
require_decodings = list(map(strtobool,config['forward']['require_decoding'].split(',')))
save_gpumem = strtobool(config['exp']['save_gpumem'])
is_production = strtobool(config['exp']['production'])
batch_size = _get_batch_size_from_config(config, to_do)
if processed_first:
shared_list = list()
p = read_next_chunk_into_shared_list_with_subprocess(read_lab_fea, shared_list, cfg_file, is_production, output_folder, wait_for_process=True)
data_name, data_end_index_fea, data_end_index_lab, fea_dict, lab_dict, arch_dict, data_set_dict = extract_data_from_shared_list(shared_list)
data_set_inp, data_set_ref = convert_numpy_to_torch(data_set_dict, save_gpumem, use_cuda)
else:
data_set_inp = data_set['input']
data_set_ref = data_set['ref']
data_end_index_fea = data_end_index['fea']
data_end_index_lab = data_end_index['lab']
shared_list = list()
data_loading_process = None
if not next_config_file is None:
data_loading_process = read_next_chunk_into_shared_list_with_subprocess(read_lab_fea, shared_list, next_config_file, is_production, output_folder, wait_for_process=False)
nns, costs, optimizers, inp_out_dict = _load_model_and_optimizer(fea_dict,model,config,arch_dict,use_cuda,multi_gpu,to_do)
if to_do=='forward':
post_file = _open_forward_output_files_and_get_file_handles(forward_outs, require_decodings, info_file, output_folder)
seq_model = is_sequential_dict(config,arch_dict)
N_snt, N_ex_tr, N_batches = _get_batch_config(data_set_inp, seq_model, to_do, data_name, batch_size)
beg_batch = 0
end_batch = batch_size
snt_index = 0
beg_snt_fea = 0
beg_snt_lab = 0
arr_snt_len_fea = shift(shift(data_end_index_fea, -1,0) - data_end_index_fea,1,0)
arr_snt_len_lab = shift(shift(data_end_index_lab, -1,0) - data_end_index_lab,1,0)
arr_snt_len_fea[0] = data_end_index_fea[0]
arr_snt_len_lab[0] = data_end_index_lab[0]
data_set_inp_dim, data_set_ref_dim = _get_dim_from_data_set(data_set_inp, data_set_ref)
inp_dim = data_set_inp_dim + data_set_ref_dim
loss_sum = 0
err_sum = 0
start_time = time.time()
for i in range(N_batches):
inp, ref, max_len_fea, max_len_lab, snt_len_fea, snt_len_lab, beg_snt_fea, beg_snt_lab, snt_index = _prepare_input(snt_index, batch_size, data_set_inp_dim, data_set_ref_dim, beg_snt_fea, beg_snt_lab, data_end_index_fea, data_end_index_lab, beg_batch, end_batch, seq_model, arr_snt_len_fea, arr_snt_len_lab, data_set_inp, data_set_ref, use_cuda)
if to_do=='train':
outs_dict = forward_model(fea_dict, lab_dict, arch_dict, model, nns, costs, inp, ref, inp_out_dict, max_len_fea, max_len_lab, batch_size, to_do, forward_outs)
_optimization_step(optimizers, outs_dict, config, arch_dict)
else:
with torch.no_grad():
outs_dict = forward_model(fea_dict, lab_dict, arch_dict, model, nns, costs, inp, ref, inp_out_dict, max_len_fea, max_len_lab, batch_size, to_do, forward_outs)
if to_do == 'forward':
for out_id in range(len(forward_outs)):
out_save = outs_dict[forward_outs[out_id]].data.cpu().numpy()
if forward_normalize_post[out_id]:
counts = load_counts(forward_count_files[out_id])
out_save=out_save-np.log(counts/np.sum(counts))
write_mat(output_folder,post_file[forward_outs[out_id]], out_save, data_name[i])
else:
loss_sum=loss_sum+outs_dict['loss_final'].detach()
err_sum=err_sum+outs_dict['err_final'].detach()
beg_batch=end_batch
end_batch=beg_batch+batch_size
_update_progress_bar(to_do, i, N_batches, loss_sum)
elapsed_time_chunk=time.time() - start_time
loss_tot=loss_sum/N_batches
err_tot=err_sum/N_batches
del inp, ref, outs_dict, data_set_inp_dim, data_set_ref_dim
_save_model(to_do, nns, multi_gpu, optimizers, info_file, arch_dict)
if to_do=='forward':
for out_name in forward_outs:
post_file[out_name].close()
_write_info_file(info_file, to_do, loss_tot, err_tot, elapsed_time_chunk)
if not data_loading_process is None:
data_loading_process.join()
data_name, data_end_index_fea, data_end_index_lab, fea_dict, lab_dict, arch_dict, data_set_dict = extract_data_from_shared_list(shared_list)
data_set_inp, data_set_ref = convert_numpy_to_torch(data_set_dict, save_gpumem, use_cuda)
data_set = {'input': data_set_inp, 'ref': data_set_ref}
data_end_index = {'fea': data_end_index_fea,'lab': data_end_index_lab}
return [data_name,data_set,data_end_index,fea_dict,lab_dict,arch_dict]
else:
return [None,None,None,None,None,None]
