def MakeMade(scale,
cols_to_train,
seed,
dataset,
fixed_ordering=None,
special_orders=[],
layers=4,
residual=False,
dropout=False,
per_row_dropout=False,
prefix_dropout=False,
fixed_dropout_ratio=False,
disable_learnable_unk=False,
input_no_emb_if_leq=True,
embs_tied=False,
embed_size=32):
# TODO: if passed in a single heuristic order, be sure to InvertOrder().
num_masks = 1
if len(special_orders):
num_masks = len(special_orders)
model = MADE(
nin=len(cols_to_train),
hidden_sizes=[scale] * layers
if layers > 0 else [512, 256, 512, 128, 1024],
nout=sum([c.DistributionSize() for c in cols_to_train]),
input_bins=[c.DistributionSize() for c in cols_to_train],
input_encoding="embed",
output_encoding="embed",
seed=seed,
do_direct_io_connections=False,
natural_ordering=False if seed is not None else True,
residual_connections=residual,
embed_size=embed_size,
fixed_ordering=fixed_ordering,
dropout_p=dropout or per_row_dropout or prefix_dropout,
fixed_dropout_p=fixed_dropout_ratio,
num_masks=num_masks,
per_row_dropout_p=per_row_dropout,
prefix_dropout=prefix_dropout,
disable_learnable_unk=disable_learnable_unk,
input_no_emb_if_leq=input_no_emb_if_leq,
embs_tied=embs_tied,
).to(get_device())
if len(special_orders):
print('assigning to model.orderings:')
print(special_orders)
model.orderings = special_orders
return model
def MakeMadeDmv(cols_to_train, seed, fixed_ordering=None):
if args.inv_order:
print('Inverting order!!!!!!!!!!')
fixed_ordering = InvertOrder(fixed_ordering)
if args.special_dmv_arch:
return MADE(
nin=len(cols_to_train),
hidden_sizes=[256] * 5,
nout=sum([c.DistributionSize() for c in cols_to_train]),
input_bins=[c.DistributionSize() for c in cols_to_train],
input_encoding="embed",
output_encoding="embed",
embed_size=128,
input_no_emb_if_leq=True,
embs_tied=True,
seed=seed,
do_direct_io_connections=True, #args.direct_io,
natural_ordering=False if seed is not None else True,
residual_connections=args.residual,
fixed_ordering=fixed_ordering,
dropout_p=args.dropout,
).to(DEVICE)
hiddens = [args.fc_hiddens] * args.layers
natural_ordering = False
if args.layers == 0:
# Default ckpt.
hiddens = [512, 256, 512, 128, 1024]
natural_ordering = True
model = MADE(
nin=len(cols_to_train),
hidden_sizes=hiddens,
residual_connections=args.residual,
nout=sum([c.DistributionSize() for c in cols_to_train]),
input_bins=[c.DistributionSize() for c in cols_to_train],
input_encoding="embed"
if args.dataset in ["dmv-full", "kdd", "synthetic"] else "binary",
output_encoding="embed"
if args.dataset in ["dmv-full", "kdd", "synthetic"] else "one_hot",
seed=seed,
do_direct_io_connections=args.direct_io,
natural_ordering=False if seed is not None else True,
fixed_ordering=fixed_ordering,
dropout_p=args.dropout,
num_masks=max(1, args.special_orders),
).to(DEVICE)
# XXX this is copied from train_many_orderings
if args.special_orders > 0:
special_orders = [
# # MutInfo Max Marg
# np.array([6, 1, 4, 0, 7, 3, 5, 2, 10, 9, 8]),
# # CL Max Marg/Dom
# np.array([6, 1, 4, 0, 5, 7, 3, 2, 10, 9, 8]),
# # Random
# np.random.RandomState(0).permutation(np.arange(11)),
][:args.special_orders]
k = len(special_orders)
for i in range(k, args.special_orders):
special_orders.append(
np.random.RandomState(i - k + 1).permutation(
np.arange(len(cols_to_train))))
print('Special orders', np.array(special_orders))
if args.inv_order:
for i, order in enumerate(special_orders):
special_orders[i] = np.asarray(InvertOrder(order))
print('Inverted special orders:', special_orders)
model.orderings = special_orders
if args.use_query_order:
model.use_query_order = True
if args.use_best_order:
model.use_best_order = True
if args.use_worst_order:
model.use_worst_order = True
return model