def init_env_params():
# generate individual capital grid
k = generate_grid(k_min, k_max, ngridk, tau)
# generate aggregate grid
km = generate_grid(km_min, km_max, ngridkm)
# generate idiosyncratic and aggregate shocks
emp_shocks, agg_shocks = generate_shocks0(trans_mat=prob,
N=Nagents,
T=Tperiods + Tperiods_skip)
a = np.array((1 - delta_a, 1 + delta_a))
er_b, er_g = (1 - ur_b), (1 - ur_g)
k_ss = ((1 / beta - (1 - delta)) / alpha)**(1 / (alpha - 1))
P = np.tile(prob, [ngridk * ngridkm, 1])
e = np.array((er_b, er_g))
u = 1 - e
replacement = np.array((mu, l_bar)) #replacement rate of wage
n = ngridk * ngridkm * nstates_ag * nstates_id
(k_indices, km_indices, ag,
e_i) = np.unravel_index(np.arange(n),
(ngridk, ngridkm, nstates_ag, nstates_id))
epsilon = np.arange(0, nstates_id)
Z, L, K, k_i = a[ag], e[ag], km[km_indices], k[k_indices]
irate = alpha * Z * (K / (l_bar * L))**(alpha - 1)
wage = (1 - alpha) * Z * (K / (l_bar * L))**alpha
wealth = irate * k_i + (wage * e_i) * l_bar + mu * (wage * (1 - e_i)) + (
1 - delta) * k_i - mu * (wage * (1 - L) / L) * e_i
# dictionary
env_params = {
'a': a,
'e': e,
'u': u,
'k_grid': k,
'km_grid': km,
'n': n,
'epsilon': epsilon,
'id_shocks': emp_shocks,
'agg_shocks': agg_shocks,
'ag': ag,
'K': K,
'replacement': replacement,
'P': P,
'K_ss': k_ss,
'wealth': wealth,
'B': B_init
}
return env_params
def run_generate_grid():
image_list = [line.strip()
for line in open(sys.argv[1])] # ../data/imagelist.txt
image_root = sys.argv[2] # ../data/refer360images
out_root = sys.argv[3] # ../data/grid_data_30
degree = int(sys.argv[4]) # 30
full_w = int(sys.argv[5]) # 4552
full_h = int(sys.argv[6]) # 2276
if not os.path.exists(out_root):
try:
os.makedirs(out_root)
except:
print('Cannot create folder {}'.format(out_root))
quit(1)
pbar = tqdm(image_list)
for fname in pbar:
image_path = os.path.join(image_root, fname)
pano = fname.split('/')[-1].split('.')[0]
nodes, canvas = generate_grid(degree=degree,
full_w=full_w,
full_h=full_h)
node_path = os.path.join(out_root, '{}.npy'.format(pano))
np.save(open(node_path, 'wb'), nodes)
fov_prefix = os.path.join(out_root, '{}.fov'.format(pano))
generate_fovs(image_path, node_path, fov_prefix)
def train(self, inputs, targets, lr=1, batch_size=30, epochs=100, plot=False, kernel='linear'):
self.batch_size = batch_size
# init the kernel
self.set_kernel(kernel)
# set optimization method (Gradient Descent)
self.optimization = tf.train.GradientDescentOptimizer(lr)
self.training_step = self.optimization.minimize(self.loss)
self.init = tf.global_variables_initializer()
self.session.run(self.init)
# set training data
train_inputs, train_target = inputs, targets
# performance tracking
train_loss_result, train_accuracy_result = [], []
# for each epoch
for i in range(epochs):
# generate random indexes for each batch
batch_index = np.random.choice(len(train_inputs), size=batch_size)
self.session.run(self.training_step, feed_dict={self.inputs: train_inputs[batch_index],
self.target: train_target[:, batch_index]})
# if plotting, record every epoch
if plot:
# record accuracy
train_accuracy, train_loss = self.generate_step_tracking_data(
train_inputs[batch_index], train_target[:, batch_index])
train_accuracy_result.append(train_accuracy)
train_loss_result.append(train_loss)
if (i+1) % (epochs / 5) == 0:
# if not plotting, get intermittent accuracy and loss
if not plot:
# record accuracy
train_accuracy, train_loss = self.generate_step_tracking_data(
train_inputs[batch_index], train_target[:, batch_index])
utl.print_progress(i, epochs, train_loss, train_accuracy)
# plot results
if plot:
if not self.features == 2:
print('Plotting only supported for 2 feature data sets... skipping output')
else:
utl.plot_loss(train_loss_result)
utl.plot_accuracy(train_accuracy_result)
grid = utl.generate_grid(train_inputs)
grid_predictions = self.session.run(self.prediction, feed_dict={self.inputs: train_inputs[batch_index],
self.target: train_target[:, batch_index],
self.grid: grid})
# plot the result grid
utl.plot_result(grid_predictions, inputs, targets)
# commit data points for the last support vectors used
self.support_vector_data = [train_inputs[batch_index], train_target[:, batch_index]]
def grid_example(grid_size=10,
weight_scale: float = 1.,
bias_scale: float = .1,
n_iter=2):
"""Runs the basic grid example"""
if grid_size < 2:
raise ValueError("Grid Size should be at least 2")
if weight_scale <= 0 or bias_scale <= 0:
raise ValueError("Scales should be higher than 0.0")
neighbors, Weight = generate_grid(grid_size)
Weight = Weight * weight_scale
bias = np.random.randn(grid_size * grid_size) * bias_scale
q = np.random.random((grid_size * grid_size))
junction_alg = IsingJunction(q, Weight, grid_size)
for i in range(grid_size * grid_size):
neighbors_weight = sum(
[Weight[i, neighbor] for neighbor in neighbors[i]])
bias[i] -= 1 / 2 * neighbors_weight
result_fixed, free_energy_fixed = belief_optimization(Weight,
bias,
q,
n_iter,
neighbors,
use_grad=False)
result_grad, free_energy_grad = belief_optimization(Weight,
bias,
q,
n_iter,
neighbors,
use_grad=True)
result_new_grad, free_energy_new_grad = belief_optimization(
Weight, bias, q, n_iter, neighbors, use_new_grad=True)
print(result_fixed)
print(result_grad)
print(result_new_grad)
draw_energy_comparison_plot(free_energy_grad, free_energy_new_grad,
"Gradient", "Our Version", "grad_vs_ours.pdf")
draw_energy_comparison_plot(free_energy_grad, free_energy_fixed,
"Gradient", "Fixed Point", "grad_vs_fixed.pdf")
def train_semisupervised(model,
initial_conditions,
t_final,
epochs,
train_size,
optimizer,
beta,
gamma,
known_points,
num_batches=1,
t_0_train=None,
t_final_train=None,
val_size=None,
selection=None,
perc=1.0,
treshold=float('-inf'),
additional_comment='',
verbose=True,
writer=None):
# Check if any selection criteria on samples has been passed
if selection is None:
perc = 1.0
# Move to device
model = model.to(device)
# Train mode
model.train()
# Initialize model to return
best_model = model
# Tensorboard writer
if writer is None:
writer = get_writer(t_0_train, t_final_train, initial_conditions,
int(train_size * perc), selection,
additional_comment)
# Initialize losses arrays
train_losses, val_losses, min_loss = [], [], 1
# Fetch parameters of the differential equation
t_0, params_0 = initial_conditions[0], initial_conditions[1:]
# Interval of points to use for training
if t_0_train is None or t_final_train is None:
t_0_train, t_final_train = t_0, t_final
# Points selection
grid = generate_grid(selection,
model,
initial_conditions,
t_0_train,
t_final_train,
train_size,
perc=perc)
start_time = time.time()
for epoch in tqdm(range(epochs), desc='Training', disable=not verbose):
# Generate DataLoader
batch_size = int(train_size / num_batches)
t_dataloader = generate_dataloader(grid,
t_0_train,
t_final_train,
batch_size,
perturb=True)
train_epoch_loss = 0.0
for i, t in enumerate(t_dataloader, 0):
# Network solutions
t = t.to(device)
s, i, r = model.parametric_solution(t, initial_conditions)
# Differential equation loss computation
diff_loss = sir_loss(t, s, i, r, beta=beta, gamma=gamma)
# Known points loss computation
known_loss = mse_loss(known_points, model, initial_conditions)
# Total loss is the sum of the two
batch_loss = diff_loss + known_loss
# Optimization
batch_loss.backward(retain_graph=False) # True
optimizer.step()
train_epoch_loss += batch_loss.item()
optimizer.zero_grad()
# If not specified, the validation size is the same as the train size
if not val_size:
val_size = train_size
# Once every few epochs, we run validation
if epoch % 100 == 0:
val_epoch_loss = validate(model, initial_conditions, beta, gamma,
t_final, val_size, num_batches)
# Keep the loss function history
train_losses.append(train_epoch_loss)
# val_losses.append(val_epoch_loss)
writer.add_scalar('Loss/train', train_epoch_loss, epoch)
writer.add_scalar('Loss/val', val_epoch_loss, epoch)
# Keep the best model (lowest loss) by using a deep copy
if epoch > 0.8 * epochs and train_epoch_loss < min_loss:
best_model = copy.deepcopy(model)
min_loss = train_epoch_loss
# If a treshold is passed, we stop training when it is reached. Notice default value is -inf
if train_epoch_loss < treshold:
break
final_time = time.time()
run_time = final_time - start_time
return best_model, train_losses, run_time, optimizer
def train_bundle_init(model,
initial_conditions_set,
t_final,
epochs,
train_size,
optimizer,
beta,
gamma,
decay=0,
num_batches=1,
hack_trivial=False,
t_0=0,
selection=None,
perc=1.0,
sigma=0,
treshold=float('-inf'),
additional_comment='',
verbose=True,
writer=None):
# Check if any selection criteria on samples has been passed
if selection is None:
perc = 1.0
# Move to device
model = model.to(device)
# Train mode
model.train()
# Initialize model to return
best_model = model
# Fetch parameters of the differential equation
t_0, initial_conditions_set = initial_conditions_set[
0], initial_conditions_set[1:]
# Initialize losses arrays
train_losses, val_losses, min_loss = [], [], 1
# Points selection
grid = generate_grid(selection,
model,
initial_conditions_set,
t_0,
t_final,
train_size,
perc=perc)
start_time = time.time()
# Generate initial conditions to sample from
s_0_set = torch.linspace(initial_conditions_set[0][0],
initial_conditions_set[0][1],
steps=train_size).reshape(-1, 1)
# Generate brownian noise to add stochasticity
if sigma:
brownian_noise = generate_brownian(t_0,
t_final,
train_size,
sigma=sigma)
else:
brownian_noise = None
for epoch in tqdm(range(epochs), desc='Training', disable=not verbose):
# Check if the decay should be increased or not
# decay = modularize_decay(model, t_final, initial_conditions, decay)
# Generate DataLoader
batch_size = int(train_size / num_batches)
t_dataloader = generate_dataloader(grid,
t_0,
t_final,
batch_size,
perturb=False)
train_epoch_loss = 0.0
for i, t in enumerate(t_dataloader, 0):
# Sample randomly initial conditions
rnd_init_s_0 = np.random.randint(s_0_set.shape[0], size=batch_size)
s_0 = s_0_set[rnd_init_s_0]
i_0 = 1 - s_0 # Set i_0 to be 1-s_0 to enforce that the sum of the initial conditions is 1
r_0 = 0.0 # We fix recovered people at day zero to zero
initial_conditions = [s_0, i_0, r_0]
# Network solutions
t = t.to(device)
s, i, r = model.parametric_solution(t,
initial_conditions,
beta,
gamma,
mode='bundle_init')
# Loss computation
batch_loss = sir_loss(t,
s,
i,
r,
beta=beta,
gamma=gamma,
decay=decay,
noise=brownian_noise,
sigma=sigma)
# Hack to prevent the network from solving the equations trivially
if hack_trivial:
batch_trivial_loss = trivial_loss(model,
t_final,
initial_conditions_set,
method='mse')
batch_loss = batch_loss + batch_trivial_loss
# Optimization
batch_loss.backward()
optimizer.step()
train_epoch_loss += batch_loss.item()
optimizer.zero_grad()
# Keep the loss function history
train_losses.append(train_epoch_loss)
writer.add_scalar('Loss/train', train_epoch_loss, epoch)
writer.add_scalar('Loss/Log-train', np.log(train_epoch_loss), epoch)
# Keep the best model (lowest loss) by using a deep copy
if epoch > 0.8 * epochs and train_epoch_loss < min_loss:
best_model = copy.deepcopy(model)
min_loss = train_epoch_loss
# If a treshold is passed, we stop training when it is reached. Notice default value is -inf
if train_epoch_loss < treshold:
break
# Backup save
if epoch % 1000 == 0:
# Save the model
import datetime
timestamp = datetime.datetime.now().strftime("%H-%M-%S")
torch.save(
{
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, ROOT_DIR +
'/models/SIR_bundle_total/backup_{}.pt'.format(timestamp))
final_time = time.time()
run_time = final_time - start_time
return best_model, train_losses, run_time, optimizer
def train_bundle_params(model,
initial_conditions,
t_final,
epochs,
train_size,
optimizer,
betas,
gammas,
decay=0,
num_batches=1,
hack_trivial=False,
t_0=0,
selection=None,
perc=1.0,
treshold=float('-inf'),
additional_comment='',
verbose=True,
writer=None):
# Check if any selection criteria on samples has been passed
if selection is None:
perc = 1.0
# Move to device
model = model.to(device)
# Train mode
model.train()
# Initialize model to return
best_model = model
# Tensorboard writer
if writer is None:
writer = get_writer(t_0, t_final, initial_conditions,
int(train_size * perc), selection,
additional_comment)
# Initialize losses arrays
train_losses, val_losses, min_loss = [], [], 1
# Fetch parameters of the differential equation
t_0, params_0 = initial_conditions[0], initial_conditions[1:]
# Points selection
grid = generate_grid(selection,
model,
initial_conditions,
t_0,
t_final,
train_size,
perc=perc)
start_time = time.time()
# Generate betas and gammas to sample from
betas = torch.linspace(betas[0], betas[1], steps=train_size).reshape(-1, 1)
gammas = torch.linspace(gammas[0], gammas[1],
steps=train_size).reshape(-1, 1)
for epoch in tqdm(range(epochs), desc='Training', disable=not verbose):
# Check if the decay should be increased or not
# decay = modularize_decay(model, t_final, initial_conditions, decay)
# Generate DataLoader
batch_size = int(train_size / num_batches)
t_dataloader = generate_dataloader(grid,
t_0,
t_final,
batch_size,
perturb=True)
train_epoch_loss = 0.0
for i, t in enumerate(t_dataloader, 0):
# Sample randomly beta and gamma
rnd_beta = np.random.randint(betas.shape[0], size=batch_size)
rnd_gamma = np.random.randint(gammas.shape[0], size=batch_size)
beta = betas[rnd_beta]
gamma = gammas[rnd_gamma]
# Network solutions
t = t.to(device)
s, i, r = model.parametric_solution(t,
initial_conditions,
beta,
gamma,
mode='bundle_params')
# Loss computation
batch_loss = sir_loss(t,
s,
i,
r,
beta=beta,
gamma=gamma,
decay=decay)
# Hack to prevent the network from solving the equations trivially
if hack_trivial:
batch_trivial_loss = trivial_loss(model,
t_final,
initial_conditions,
method='mse')
batch_loss = batch_loss + batch_trivial_loss
# Optimization
batch_loss.backward()
optimizer.step()
train_epoch_loss += batch_loss.item()
optimizer.zero_grad()
# Keep the loss function history
train_losses.append(train_epoch_loss)
writer.add_scalar('LogLoss/train', np.log(train_epoch_loss), epoch)
writer.add_scalar('Loss/train', train_epoch_loss, epoch)
# Keep the best model (lowest loss) by using a deep copy
if epoch > 0.8 * epochs and train_epoch_loss < min_loss:
best_model = copy.deepcopy(model)
min_loss = train_epoch_loss
# If a treshold is passed, we stop training when it is reached. Notice default value is -inf
if train_epoch_loss < treshold:
break
final_time = time.time()
run_time = final_time - start_time
return best_model, train_losses, run_time, optimizer
def train_bundle(model, initial_conditions_set, t_final, epochs, train_size, optimizer, betas, gammas, model_name,
decay=0, num_batches=1, hack_trivial=False,
treshold=float('-inf'), verbose=True, writer=None):
# Train mode
model.train()
# Initialize model to return
best_model = model
# Fetch parameters of the differential equation
t_0, initial_conditions_set = initial_conditions_set[0], initial_conditions_set[1:]
# Initialize losses arrays
train_losses, val_losses, min_loss = [], [], 1
# Points selection
grid = generate_grid(t_0, t_final, train_size)
start_time = time.time()
for epoch in tqdm(range(epochs), desc='Training', disable=not verbose):
# Generate DataLoader
batch_size = int(train_size / num_batches)
t_dataloader = generate_dataloader(grid, t_0, t_final, batch_size, perturb=True)
train_epoch_loss = 0.0
for i, t in enumerate(t_dataloader, 0):
# Sample randomly initial conditions, beta and gamma
i_0 = uniform(initial_conditions_set[0][0], initial_conditions_set[0][1], size=batch_size)
r_0 = uniform(initial_conditions_set[1][0], initial_conditions_set[1][1], size=batch_size)
beta = uniform(betas[0], betas[1], size=batch_size)
gamma = uniform(gammas[0], gammas[1], size=batch_size)
i_0 = torch.Tensor([i_0]).reshape((-1, 1))
r_0 = torch.Tensor([r_0]).reshape((-1, 1))
beta = torch.Tensor([beta]).reshape((-1, 1))
gamma = torch.Tensor([gamma]).reshape((-1, 1))
s_0 = 1 - (i_0 + r_0)
initial_conditions = [s_0, i_0, r_0]
# Network solutions
s, i, r = model.parametric_solution(t, initial_conditions, beta, gamma, mode='bundle_total')
# Loss computation
batch_loss = sir_loss(t, s, i, r, beta=beta, gamma=gamma, decay=decay)
# Hack to prevent the network from solving the equations trivially
if hack_trivial:
batch_trivial_loss = trivial_loss(i, hack_trivial)
batch_loss = batch_loss + batch_trivial_loss
# Optimization
batch_loss.backward()
optimizer.step()
train_epoch_loss += batch_loss.item()
optimizer.zero_grad()
# Keep the loss function history
train_losses.append(train_epoch_loss)
writer.add_scalar('Loss/train', train_epoch_loss, epoch)
writer.add_scalar('Loss/Log-train', np.log(train_epoch_loss), epoch)
# Keep the best model (lowest loss) by using a deep copy
if epoch > 0.8 * epochs and train_epoch_loss < min_loss:
best_model = copy.deepcopy(model)
min_loss = train_epoch_loss
# If a treshold is passed, we stop training when it is reached. Notice default value is -inf
if train_epoch_loss < treshold:
break
# Backup save
if epoch % 500 == 0 and epoch != 0:
torch.save({'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict()},
ROOT_DIR + '/models/SIR_bundle_total/{}'.format(model_name))
final_time = time.time()
run_time = final_time - start_time
return best_model, train_losses, run_time, optimizer
"-i",
"--input",
help="Folder containing an images/ folder and Calibration.xml",
required=True)
parser.add_argument(
"--rerun",
action='store_true',
help=
"Use this flag if this is the second run of the script, first run stores the point grid visible which is used from that point on"
)
args = parser.parse_args()
calibrations = read_calibration(os.path.join(args.input, "Calibration.xml"))
images = read_images(os.path.join(args.input, "images"))
if not args.rerun:
grid = generate_grid()
grid = filter_grid(grid, calibrations)
np.save('grid.npy', grid)
else:
grid = np.load('grid.npy')
draw_points_on_images(grid, calibrations, images)
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
colors = [
'#808080', '#000000', '#FF0000', '#800000', '#808000', '#008000',
'#00FFFF', '#008080', '#800080'
]
for i, calibration in enumerate(calibrations):
draw_camera_object(ax, calibration["WR"], calibration["WT"],
colors[i % len(colors)])
def train_NNLosc(initial_conditions, t_final, lam, hidden, epochs, train_size, optimizer,
num_batches=1, start_model=None, t_0_train=None, t_final_train=None, val_size=None,
selection=None, perc=1.0, treshold=float('-inf'), additional_comment='', verbose=True,
writer=None, start_epoch=0, grid=None, perturb=True, Mblock=False, explorer=None, grid_explorer=None):
if not start_model:
model = odeNet_NLosc_MM(hidden)
else:
model = start_model
if selection is None:
perc = 1.0
model = model.to(device)
model.train()
best_model = model
# Create writer if none is provided
if writer is None:
writer = get_writer(t_0_train, t_final_train, initial_conditions, int(train_size * perc),
selection, additional_comment)
train_losses = []
val_losses = []
min_loss = 1
t_0 = initial_conditions[0]
# Grid of points to use for training
if t_0_train is None or t_final_train is None:
t_0_train, t_final_train = t_0, t_final
# Points selection
if grid is None:
grid = generate_grid(selection, model, initial_conditions, t_0_train, t_final_train, train_size, perc=perc)
if Mblock:
grid = torch.cat((grid, grid_explorer))
start_time = time.time()
for epoch in tqdm(range(epochs), desc='Training Hamiltonian NN on NLosc', disable=not verbose):
# Generate DataLoader
batch_size = int(train_size / num_batches)
t_dataloader = generate_dataloader(grid, t_0_train, t_final_train, batch_size, perturb=perturb)
# Perturbing the evaluation points & forcing t[0]=t0
train_epoch_loss = 0.0
for i, data in enumerate(t_dataloader, 0):
# Network solutions
data = data.to(device)
x, px = model.parametric_solution(data, initial_conditions)
# Loss function defined by Hamilton Eqs. (symplectic): Writing explicitely the Eqs (faster)
batch_loss = hamiltonian_eq_loss(data, x, px, lam)
# Optimization
batch_loss.backward(retain_graph=True) # True
optimizer.step()
train_epoch_loss += batch_loss.item()
optimizer.zero_grad()
if not val_size:
val_size = train_size
if epoch % 500 == 0:
val_epoch_loss = validate_NNLosc(model, initial_conditions, lam, t_0, t_final, val_size, num_batches)
# Keep the loss function history
train_losses.append(train_epoch_loss)
val_losses.append(val_epoch_loss)
# Update writer
writer.add_scalar('Loss/train', train_epoch_loss, start_epoch + epoch)
writer.add_scalar('Loss/val', val_epoch_loss, start_epoch + epoch)
# Keep the best source_model (lowest loss) by using a deep copy
if epoch > 0.8 * epochs and train_epoch_loss < min_loss:
best_model = copy.deepcopy(model)
min_loss = train_epoch_loss
if train_epoch_loss < treshold:
break
final_time = time.time()
run_time = final_time - start_time
return best_model, train_losses, run_time, optimizer
def dump_mp3d_datasets(
output_root,
task='grid_fov_pretraining',
task_root='',
graph_root='',
obj_dict_file='../data/vg_object_dictionaries.top100.matterport3d.json',
image_list_file='../data/imagelist.matterport3d.txt',
degree=45):
'''Prepares and dumps dataset to an npy file.
'''
if task_root != '' and not os.path.exists(task_root):
try:
os.makedirs(task_root)
except:
print('Cannot create folder {}'.format(task_root))
quit(1)
dicts = get_mp3d_dictionaries()
vg2idx = json.load(open(obj_dict_file, 'r'))['vg2idx']
data_list = defaultdict(list)
data = []
image_list = [line.strip().split('.')[0] for line in open(image_list_file)]
pbar = tqdm(image_list)
for ii, pano in enumerate(pbar):
splits = dicts['viewpoint2split'][pano]
if task == 'grid_fov_pretraining' and task_root != '':
grid_nodes, _ = generate_grid(degree=degree)
node_path = os.path.join(graph_root, '{}.npy'.format(pano))
nodes = np.load(node_path, allow_pickle=True)[()]
for n in grid_nodes:
node = grid_nodes[n]
mdatum = {}
fov_id = node['id']
mdatum['fov_id'] = fov_id
mdatum['pano'] = pano
lat, lng = node['lat'], node['lng']
mx, my = node['x'], node['y']
mdatum['latitude'] = lat
mdatum['longitude'] = lng
mdatum['x'] = mx
mdatum['y'] = my
mdatum['refexps'] = []
fov_file = os.path.join(task_root,
'{}.fov{}.jpg'.format(pano, fov_id))
mdatum['fov_file'] = fov_file
regions, obj_list = get_objects(mx, my, nodes, vg2idx)
mdatum['regions'] = regions
mdatum['obj_list'] = obj_list
directions = [
len(obj_list['canonical'][d]) > 0
for d in ['up', 'down', 'left', 'right']
]
if any(directions):
for split in splits:
data_list[split].append(mdatum)
else:
raise NotImplementedError()
pbar.close()
for split in R2R_SPLIT_NAMES:
output_file = os.path.join(output_root,
'{}.[all].imdb.npy'.format(split))
print('Dumping {} instances to {}'.format(len(data_list[split]),
output_file))
np.save(open(output_file, 'wb'), {
'data_list': [data_list[split]],
'sentences': []
})