def drawing(self):
splines = self.x
splines = [spl + eps for spl, eps in zip(splines, self.loc_noise)]
splines = [spl + eps for spl, eps in zip(splines, self.shape_noise)]
splines = apply_warp(splines, self.affine)
drawing = [get_stk_from_bspline(x) for x in splines]
return drawing
def plot_parse_progress_token(losses, states, img, nint=10, mode='motor', scale=1.4):
assert mode in ['motor', 'motor-img', 'pimg', 'heatmap']
if mode in ['pimg', 'heatmap']:
renderer = Renderer()
iters = len(losses)
intervals = get_intervals(iters, nint)
img = img.cpu()
fig, axes = plt.subplots(nrows=1, ncols=nint, figsize=(scale*nint, scale))
for i, idx in enumerate(intervals):
state = states[idx]
drawing_type = [get_stk_from_bspline(spl.cpu()) for spl in state['x']]
drawing_token = get_token_drawing(state)
if mode in ['motor', 'motor-img']:
if mode == 'motor-img':
plot_image(axes[i], img)
plot_motor_to_image(axes[i], drawing_token, lw=scale)
plot_motor_to_image(axes[i], drawing_type, lw=scale, is_type=True)
elif mode == 'pimg':
pimg = renderer(drawing_token, state['blur'], state['epsilon'])
plot_image(axes[i], pimg.cpu())
else:
pimg = renderer(drawing_token, state['blur'], state['epsilon'])
plot_image(axes[i], img)
plot_pimg_over_img(axes[i], pimg.cpu())
axes[i].set_title('%i\n%0.2f' % (idx, losses[idx].item()))
plt.subplots_adjust(wspace=0)
def get_token_drawing(state):
splines = state['x']
splines = [spl + eps for spl,eps in zip(splines, state['loc_noise'])]
splines = [spl + eps for spl,eps in zip(splines, state['shape_noise'])]
splines = apply_warp(splines, state['affine'])
drawing = [get_stk_from_bspline(x) for x in splines]
return drawing
def plot_parse(ax, parse, cpts_size=10, arrow_size=0):
strokes = [
get_stk_from_bspline(spl, 200) if len(spl) > 1 else spl
for spl in parse
]
plot_motor_to_image(ax, strokes, arrow_size=arrow_size)
for i, spl in enumerate(parse):
ax.scatter(spl[:, 0], -spl[:, 1], color=get_color(i), s=cpts_size)
def parse_score_fn(model, parses):
drawings = nested_map(lambda x: get_stk_from_bspline(x), parses)
if torch.cuda.is_available():
drawings = nested_map(lambda x: x.cuda(), drawings)
parses = nested_map(lambda x: x.cuda(), parses)
losses = model.losses_fn(
parses, drawings, filter_small=False, denormalize=True)
return -losses.cpu()
def traj_from_spline(y, neval=200):
if len(y) > 1:
y = numpy_to_torch(y).float()
x = get_stk_from_bspline(y, neval)
else:
x = y
x = torch_to_numpy(x)
return x
def transform_spline(x, start):
"""
x : (t,2) spline in normalized space, starting at 0
"""
assert len(x.shape) == 2 and x.size(1) == 2
assert start.shape == torch.Size([2])
assert torch.sum(torch.abs(x[0])) < 1e-5
x = torch.cumsum(x, dim=0)
x = get_stk_from_bspline(x, neval=200)
x = x - x[0] + start
x = convert_space(x, inverse=True)
return x
def strokes2splines(strokes, npts_ver="unif_space", nland=7):
"""
Convert from strokes to splines, with the splines "rendered" back into
the spatial coords.
INPUTS
- strokes, list of np.arrays (N,2), where each is a stroke. could
also be (N,3) where 3rd col is time, this will be removed in splines.
- npts_ver, how to decide how many pts in splines to return
--- orig, keep origianl in input strokes
--- unif_space, use npts from unif space
- nland, number of control pts
RETURNS
- splines, list of np arrays, but diff num rows compared to inputs.
NOTE: throws out time dimension
"""
from pybpl.splines import fit_bspline_to_traj, get_stk_from_bspline
from pybpl.data import unif_space
# remove time dimenision
strokes = [s[:,:2] for s in strokes]
# keep track just in case
if npts_ver=="orig":
npts = [len(s) for s in strokes]
# strokes to tenorys
strokes = [torch.tensor(s) for s in strokes]
# make uniform in space
strokes = [unif_space(s) for s in strokes]
if npts_ver=="unif_space":
npts = [len(s) for s in strokes]
# get splines
if True:
splines = [fit_bspline_to_traj(s, nland=nland) for s in strokes]
else:
assert False, "not coded"
# Option 2
# if you don't know the right number of control points, use
# this function to determine the ncpts dynamically
# (available at gns.omniglot.minimal_splines)
# NOTE: make sure you have called unif_space()
def fit_minimal_spline(stroke, thresh=0.7, max_nland=100, normalize=True):
ntraj = stroke.size(0)
max_nland = min(max_nland, ntraj+1)
# determine num control points
for nland in range(1, max_nland):
spline, residuals = fit_bspline_to_traj(stroke, nland, include_resid=True)
loss = residuals.sum()
if normalize:
loss = loss / ntraj
if loss < thresh:
return spline
return spline
# upsample pts to match original
splines = [get_stk_from_bspline(spl, neval=n)
for spl, n in zip(splines, npts)]
# return to np arrays
splines = [s.numpy() for s in splines]
return splines
def drawing(self):
return [get_stk_from_bspline(x) for x in self.x]
def splines_to_strokes(splines):
return [get_stk_from_bspline(spl) for spl in splines]