def test_function_method_wrong_type(self):
def f():
val = torch.classes._TorchScriptTesting._Foo(5, 3)
val.increment("asdf")
return val
with self.assertRaisesRegex(RuntimeError, "Expected"):
jit.script(f)()
def init_model(model_type, hidden_size, input_size, n_layers,
output_size, dropout, device, class_task=True, script=True):
if model_type == 'subLSTM':
rnn = SubLSTM(
input_size=input_size,
hidden_size=hidden_size,
num_layers=n_layers,
fixed_forget=False,
batch_first=True,
dropout=dropout
)
if script:
rnn = jit.script(rnn)
elif model_type == 'fix-subLSTM':
rnn = SubLSTM(
input_size=input_size,
hidden_size=hidden_size,
num_layers=n_layers,
fixed_forget=True,
batch_first=True,
dropout=dropout
)
if script:
rnn = jit.script(rnn)
elif model_type == 'LSTM':
rnn = nn.LSTM(
input_size=input_size,
hidden_size=hidden_size,
num_layers=n_layers,
batch_first=True,
dropout=dropout
)
elif model_type == 'GRU':
rnn = nn.GRU(
input_size=input_size,
hidden_size=hidden_size,
num_layers=n_layers,
batch_first=True,
dropout=dropout
)
else:
raise ValueError('Unrecognized RNN type')
if class_task:
model = RNNClassifier(
rnn=rnn, rnn_output_size=hidden_size, n_classes=output_size
).to(device=device)
else:
model = RNNRegressor(
rnn=rnn, rnn_output_size=hidden_size, responses=output_size
).to(device=device)
return model
def __init__(self, options: Options, input_dim: int,
transformer_options: Tuple[int, int, int, float, str]):
super(JetEncoder, self).__init__()
self.mask_sequence_vectors = options.mask_sequence_vectors
self.embedding = jit.script(JetEmbedding(options, input_dim))
encoder_layer = self.create_encoder_layer(transformer_options)
self.encoder = jit.script(
nn.TransformerEncoder(encoder_layer, options.num_encoder_layers))
def __init__(self,
options: Options,
order: int,
transformer_options: Tuple[int, int, int, float, str] = None,
permutation_indices: List[Tuple[int, ...]] = None,
attention_dim: int = None) -> None:
super(SymmetricAttentionSplit,
self).__init__(options, order, transformer_options,
permutation_indices, attention_dim)
# Each potential jet gets its own encoder in order to extract information for attention.
self.encoders = nn.ModuleList([
jit.script(
StackedEncoder(options, options.num_jet_embedding_layers,
options.num_jet_encoder_layers,
transformer_options)) for _ in range(order)
])
# After encoding, the jets are fed into a final linear layer to extract logits.
self.linear_layers = nn.ModuleList([
nn.Linear(options.hidden_dim, self.attention_dim)
for _ in range(order)
])
# Add additional non-linearity on top of the linear layer.
self.activations = nn.ModuleList(
[nn.PReLU(self.attention_dim) for _ in range(order)])
# Operation to perform general n-dimensional attention.
self.contraction_operation = self.make_contraction()
self.reset_parameters()
def main_2():
orig_model = MyModel2()
# case 1
input1 = torch.randn(2, 3)
input1 = input1.pow(2) # always positive
traced_model1 = jit.trace(orig_model, (input1))
# case 2
input2 = torch.randn(2, 3)
input2 = -input2.pow(2) # always negative
traced_model2 = jit.trace(orig_model, (input2))
print(orig_model(input1).size(), orig_model(input2).size())
print(traced_model1(input1).size(), traced_model1(input2).size())
print(traced_model2(input1).size(), traced_model2(input2).size())
print(traced_model1.code)
print('=======')
print(traced_model2.code)
print('=======')
# scripting
scripted_model = jit.script(orig_model)
print(orig_model(input1).size(), orig_model(input2).size())
print(scripted_model(input1).size(), scripted_model(input2).size())
print(scripted_model.code)
def __init__(self,
options: Options,
order: int,
permutation_indices: List[Tuple[int, ...]],
transformer_options: Tuple[int, int, int, float, str],
softmax_output: bool = True):
super(BranchDecoder, self).__init__()
self.order = order
self.softmax_output = softmax_output
self.combinatorial_scale = options.combinatorial_scale
# Each branch has a personal encoder stack to extract particle-level data
self.encoder = jit.script(
StackedEncoder(options, options.num_branch_embedding_layers,
options.num_branch_encoder_layers,
transformer_options))
# Symmetric attention to create the output distribution
attention_layer = SymmetricAttentionSplit if options.split_symmetric_attention else SymmetricAttentionFull
self.attention = attention_layer(options, order, transformer_options,
permutation_indices)
# Optional output predicting if the particle was present or not
self.classifier = BranchClassifier(options)
self.num_targets = len(self.attention.permutation_group)
self.permutation_indices = self.attention.permutation_indices
self.padding_mask_operation = self.create_padding_mask_operation(
options.batch_size)
self.diagonal_mask_operation = self.create_diagonal_mask_operation()
self.diagonal_masks = {}
def check_point(self, is_backup: bool) -> None:
self.model = self.model.to(torch.device("cpu"))
REPO = backup(MEME_CLF_REPO) if is_backup else MEME_CLF_REPO
torch.save(self.model.features, REPO.format("reg") + "features.pt")
jit.save(
cast(ScriptModule, jit.script(self.model.features)),
REPO.format("jit") + "features.pt",
)
jit.save(
cast(ScriptModule, jit.script(self.model.dense)),
REPO.format("jit") + "dense.pt",
)
torch.save(self.model, REPO.format("reg") + self.name + ".pt")
with open(REPO.format("cp") + self.name + ".json", "w") as f:
json.dump(self.cp, f, indent=4)
self.model = self.model.to(torch.device("cuda:0"))
def _new_script_local_optimizer(optim_cls, local_params_rref, *args, **kwargs):
optim = _ScriptLocalOptimizer(optim_cls, local_params_rref, *args,
**kwargs)
with _ScriptLocalOptimizer.compile_lock:
script_optim = jit.script(optim)
return rpc.RRef(script_optim, _ScriptLocalOptimizerInterface)
def trace(self) -> ScriptModule:
"""
Create a ScriptModule from this policy.
The returned module will always have the signature `action = tm(observation, hidden)`.
For recurrent networks, it returns a stateful module that keeps the hidden states internally.
Such modules have a `reset()` method to reset the hidden states.
"""
return script(StatefulRecurrentNetwork(self))
def __init__(self): super(MyModel4, self).__init__() self.b1 = jit.script(MyModel3()) self.l1 = nn.Linear(1024, 1024) self.l2 = nn.Linear(1024, 1024) self.l3 = nn.Linear(1024, 1024)
def trace(self) -> ScriptModule:
"""
Create a ScriptModule from this policy.
The returned module will always have the signature `action = tm(observation)`.
For recurrent networks, it returns a stateful module that keeps the hidden states internally.
Such modules have a reset() method to reset the hidden states.
"""
# This does not work for recurrent policies, which is why they override this function.
return script(TracedPolicyWrapper(self))
def __init__(self, options: Options):
super(BranchClassifier, self).__init__()
self.hidden_dim = options.hidden_dim
self.num_layers = options.num_branch_classification_layers
self.hidden_layers = jit.script(
LinearStack(options, self.num_layers, self.hidden_dim,
options.skip_connections))
self.output_layer = nn.Linear(options.hidden_dim, 1)
def script(self) -> ScriptModule:
cond_lvl = "vel" if self._order == 3 else "acc"
cond_init, cond_final = to.chunk(self.conds, 2)
return script(
TraceablePolySplineTimePolicy(
spec=self.env_spec,
dt=self._dt,
t_end=self._t_end,
cond_lvl=cond_lvl,
cond_final=cond_final,
cond_init=cond_init,
t_init=self._t_init,
overtime_behavior=self._overtime_behavior,
))
def __init__(
self,
input_size: int,
n_layer: int = 6,
d_head: int = 64,
d_model: int = 256,
d_inner: int = 256,
layer_drop: float = 0.0,
dropout: float = 0.0,
dropatt: float = 0.0,
self_sup: bool = False,
max_self_sup_K: int = 30,
):
super().__init__()
assert input_size == d_model
n_head = d_model // d_head
self.transformer = jit.script(
TransformerXL(
n_layer=n_layer,
n_head=n_head,
d_model=d_model,
d_head=d_head,
d_inner=d_inner,
dropout=dropout,
dropatt=dropatt,
layer_drop=layer_drop,
))
self.max_self_sup_K = max_self_sup_K
self.self_sup = self_sup
self.classifier = nn.Sequential(
nn.Linear(d_model * 2, d_model // 2, bias=False),
nn.LayerNorm(d_model // 2),
nn.ReLU(True),
nn.Linear(d_model // 2, 1),
)
self.layer_init()
def __init__(
self,
model: DictConfig,
script: bool,
batch_size: int,
lr: float,
optimizer: DictConfig,
scheduler: DictConfig,
):
super().__init__()
self.accuracy = pl.metrics.Accuracy()
self.model = instantiate(model)
if script:
self.model = jit.script(self.model)
# These can be set by auto_scale_batch and auto_lr_find
self.batch_size = batch_size
self.lr = lr
self.optim = optimizer
self.sched = scheduler
def torchscript_predictions(self):
doc_scores = self.doc_output.torchscript_predictions()
word_scores = self.word_output.torchscript_predictions()
return jit.script(IntentSlotScores(doc_scores, word_scores))
def run(p, _log):
p = munchify(p)
torch.manual_seed(p.seed)
np.random.seed(p.seed)
random.seed(p.seed)
# setup log folder and backup source code
if p.write_logs:
setup_log_folder(p.log_folder, p.force)
save_current_script(p.log_folder)
# setup logger
log, logger = setup_logger(p.log_folder if p.write_logs else None)
log("{}".format(p))
ex.logger = logger
# import dataset
log("load datasets ...")
_module = importlib.import_module(DATASETS + "." + p.dataset_name)
train_generator = _module.create_iterator(p=p,
partition="train",
batch_size=p.train_batch_size)
eval_generator = _module.create_iterator(p=p,
partition="valid",
batch_size=p.eval_batch_size,
random=False)
test_generator = _module.create_iterator(p=p,
partition="test",
batch_size=1,
random=False)
vocab_size = len(train_generator.dataset.idx2word)
log("dataset vocab size: {}".format(vocab_size))
log("Number of train batches: {}".format(len(train_generator)))
log("Number of test batches: {}".format(len(eval_generator)))
# build model
log("load model ...")
_module = importlib.import_module(MODELS + "." + p.model_name)
p.vocab_size = vocab_size
model = _module.Model(p)
if p.jit:
log("compiling model with jit.script ...")
model = jit.script(model)
log("skipping model print ...")
#log("{}".format(model))
log("{} trainable parameters found. ".format(count_parameters(model)))
# optimizer
optimizer = torch.optim.Adam(params=model.parameters(),
lr=p.learning_rate,
betas=(p.beta1, p.beta2))
# loss
criterion = nn.CrossEntropyLoss(ignore_index=p.PAD)
# DataParallel over multiple GPUs
if p.n_gpus > 1:
if p.jit:
raise Exception(
"JIT is currently not supported for distributed training!")
log("{} GPUs detected. Using nn.DataParallel. Batch-size per GPU: {}".
format(p.n_gpus, p.train_batch_size // p.n_gpus))
model = nn.DataParallel(model)
# create trainer
log("load trainer ...")
_module = importlib.import_module(TRAINERS + "." + p.trainer_name)
trainer = _module.Trainer(model=model,
params=p,
train_generator=train_generator,
eval_generator=eval_generator,
optimizer=optimizer,
criterion=criterion,
log=log)
# begin training
trainer.train()
log("\nloading best mode from: ", trainer.best_eval_state_path)
trainer.load_state(trainer.best_eval_state_path)
log("\nfinal batch_size=1 evaluation ...")
trainer.evaluate(generator=test_generator, progress=True)
def trace(self) -> ScriptModule:
return script(
TraceableTimePolicy(self.env_spec, self._fcn_of_time, self._dt))
def test_scriptable(kernel):
jit.script(kernel)
def test_eager_vs_scripted(functional_info, sample_input):
eager = functional_info(sample_input)
scripted = jit.script(functional_info.functional)(*sample_input.args,
**sample_input.kwargs)
torch.testing.assert_close(eager, scripted)
def torchscript_predictions(self):
return jit.script(
CRFWordTaggingScores(self.target_names, jit.script(self.crf)))
def torchscript_predictions(self):
scores = [o.torchscript_predictions() for o in self.outputs.values()]
return jit.script(MultiLabelClassificationScores(scores))
def torchscript_predictions(self):
return jit.script(WordTaggingScores(self.target_names))
def test_scriptable(self, functional_info):
jit.script(functional_info.functional)
def torchscript_predictions(self):
scores = self.output.torchscript_predictions()
return jit.script(MultiLabelClassificationScores(scores))
def test_equality(f, cmp_key):
obj1 = f()
obj2 = jit.script(f)()
return (cmp_key(obj1), cmp_key(obj2))
def make_scripted(self, *p, **ks) -> IRecurrentCell:
return jit.script(self.make(*p, **ks))
args = parser.parse_args()
device = torch.device("cuda" if args.cuda else "cpu")
device = torch.device("cpu")
tgt_len = 16
B = 4
inp_data = torch.randn((tgt_len, B, args.d_model))
context = torch.randn((tgt_len, B, args.d_model))
model = jit.script(
TransformerXL(
args.n_layer,
args.n_head,
args.d_model,
args.d_head,
args.d_inner,
args.dropout,
dropatt=args.dropout,
)).to(device)
mems = model.init_mems(B)
print(sum(p.numel() for p in model.parameters()))
for _ in range(2):
out, mems, query_out = model.transformer_seq_forward(
inp_data, context, mems, torch.randint(1, (tgt_len, B)), 1)
print(out[0])
print(query_out[0])
print(torch.norm(out - query_out, dim=-1).mean())
BoxSpace(-1, 1, 4),
BoxSpace(-1, 1, 2),
),
dt=0.01,
activation_nonlin=to.sigmoid,
potentials_dyn_fcn=pd_capacity_21,
)
else:
raise NotImplementedError
# Trace the policy
# traced_net = trace(net, (to.from_numpy(net.env_spec.obs_space.sample_uniform()), net.init_hidden()))
# print(traced_net.graph)
# print(traced_net(to.from_numpy(net.env_spec.obs_space.sample_uniform()), None))
stateful_net = script(StatefulRecurrentNetwork(net))
print(stateful_net.graph)
print(stateful_net.reset.graph)
print(list(stateful_net.named_parameters()))
stateful_net.save(tmpfile)
# Load in c
cp = ControlPolicy("torch", tmpfile)
inputs = [
[1.0, 2.0, 3.0, 4.0],
[3.0, 4.0, 5.0, 6.0],
]
hid_man = net.init_hidden()
