def test_initialize(self):
sampler = ActionSequenceSampler(
create_encoder(),
is_subtype,
create_transform_input([]), transform_action_sequence,
collate,
Module(encoder_module,
DecoderModule([], [], []))
)
s = sampler.initialize(Environment())
assert 1 == len(s["action_sequence"].action_sequence)
assert s["input"] == torch.zeros((1,))
assert s["reference"] == []
def test_rule(self):
rule_prob = torch.tensor([
[[
1.0, # unknown
1.0, # close variadic field
0.2, # Root2X
0.1, # Root2Y
1.0, # X2Y_list
1.0, # Ysub2Str
]],
[[
1.0, # unknown
1.0, # close variadic field
1.0, # Root2X
1.0, # Root2Y
0.5, # X2Y_list
1.0, # Ysub2Str
]]])
token_prob = torch.tensor([[[]], [[]]])
reference_prob = torch.tensor([[[]], [[]]])
sampler = ActionSequenceSampler(
create_encoder(),
is_subtype,
create_transform_input([]), transform_action_sequence,
collate,
Module(encoder_module,
DecoderModule(rule_prob, token_prob, reference_prob))
)
s = SamplerState(0.0, sampler.initialize(Environment()))
topk_results = list(sampler.top_k_samples([s], 1))
assert 1 == len(topk_results)
assert 1 == topk_results[0].state.state["length"].item()
assert np.allclose(log(0.2), topk_results[0].state.score)
random_results = list(sampler.batch_k_samples([s], [1]))
assert 1 == len(random_results)
assert 1 == random_results[0].state.state["length"].item()
assert \
log(0.1) - 1e-5 <= random_results[0].state.score <= log(0.2) + 1e-5
all_results = list(sampler.all_samples([s]))
assert 2 == len(all_results)
assert 1 == all_results[0].state.state["length"].item()
assert np.allclose(log(0.2), all_results[0].state.score)
assert np.allclose(log(0.1), all_results[1].state.score)
all_results = list(sampler.all_samples([s], sorted=False))
assert 1 == all_results[0].state.state["length"].item()
assert \
log(0.1) - 1e-5 <= all_results[0].state.score <= log(0.2) + 1e-5
next = list(sampler.top_k_samples(
[s.state for s in topk_results], 1))[0]
assert 2 == next.state.state["length"].item()
assert np.allclose(log(0.2) + log(0.5), next.state.score)
def prepare_synthesizer(self, model, qencoder, aencoder):
transform_input = Compose(
OrderedDict([("extract_reference",
Apply(module=mlprogram.nn.Function(tokenize),
in_keys=[["text_query", "str"]],
out_key="reference")),
("encode_query",
Apply(module=EncodeWordQuery(qencoder),
in_keys=["reference"],
out_key="word_nl_query"))]))
transform_action_sequence = Compose(
OrderedDict([("add_previous_action",
Apply(
module=AddPreviousActions(aencoder,
n_dependent=1),
in_keys=["action_sequence", "reference"],
constants={"train": False},
out_key="previous_actions",
)),
("add_action",
Apply(
module=AddActions(aencoder, n_dependent=1),
in_keys=["action_sequence", "reference"],
constants={"train": False},
out_key="actions",
)), ("add_state", AddState("state")),
("add_hidden_state", AddState("hidden_state")),
("add_history", AddState("history"))]))
collate = Collate(word_nl_query=CollateOptions(True, 0, -1),
nl_query_features=CollateOptions(True, 0, -1),
reference_features=CollateOptions(True, 0, -1),
actions=CollateOptions(True, 0, -1),
previous_actions=CollateOptions(True, 0, -1),
previous_action_rules=CollateOptions(True, 0, -1),
history=CollateOptions(False, 1, 0),
hidden_state=CollateOptions(False, 0, 0),
state=CollateOptions(False, 0, 0),
ground_truth_actions=CollateOptions(True, 0, -1))
return BeamSearch(
5, 20,
ActionSequenceSampler(aencoder, is_subtype, transform_input,
transform_action_sequence, collate, model))
def prepare_synthesizer(self, model, qencoder, cencoder, aencoder):
transform_input = Compose(
OrderedDict([("extract_reference",
Apply(module=mlprogram.nn.Function(tokenize),
in_keys=[["text_query", "str"]],
out_key="reference")),
("encode_word_query",
Apply(module=EncodeWordQuery(qencoder),
in_keys=["reference"],
out_key="word_nl_query")),
("encode_char",
Apply(module=EncodeCharacterQuery(cencoder, 10),
in_keys=["reference"],
out_key="char_nl_query"))]))
transform_action_sequence = Compose(
OrderedDict([("add_previous_action",
Apply(
module=AddPreviousActions(aencoder),
in_keys=["action_sequence", "reference"],
constants={"train": False},
out_key="previous_actions",
)),
("add_previous_action_rule",
Apply(
module=AddPreviousActionRules(
aencoder,
4,
),
in_keys=["action_sequence", "reference"],
constants={"train": False},
out_key="previous_action_rules",
)),
("add_tree",
Apply(
module=AddActionSequenceAsTree(aencoder),
in_keys=["action_sequence", "reference"],
constants={"train": False},
out_key=["adjacency_matrix", "depthes"],
)),
("add_query",
Apply(
module=AddQueryForTreeGenDecoder(aencoder, 4),
in_keys=["action_sequence", "reference"],
constants={"train": False},
out_key="action_queries",
))]))
collate = Collate(word_nl_query=CollateOptions(True, 0, -1),
char_nl_query=CollateOptions(True, 0, -1),
nl_query_features=CollateOptions(True, 0, -1),
reference_features=CollateOptions(True, 0, -1),
previous_actions=CollateOptions(True, 0, -1),
previous_action_rules=CollateOptions(True, 0, -1),
depthes=CollateOptions(False, 1, 0),
adjacency_matrix=CollateOptions(False, 0, 0),
action_queries=CollateOptions(True, 0, -1),
ground_truth_actions=CollateOptions(True, 0, -1))
return BeamSearch(
5, 20,
ActionSequenceSampler(aencoder, is_subtype, transform_input,
transform_action_sequence, collate, model))
def prepare_synthesizer(self, model, encoder, interpreter, rollout=True):
collate = Collate(
test_case_tensor=CollateOptions(False, 0, 0),
input_feature=CollateOptions(False, 0, 0),
test_case_feature=CollateOptions(False, 0, 0),
reference_features=CollateOptions(True, 0, 0),
variables_tensor=CollateOptions(True, 0, 0),
previous_actions=CollateOptions(True, 0, -1),
hidden_state=CollateOptions(False, 0, 0),
state=CollateOptions(False, 0, 0),
ground_truth_actions=CollateOptions(True, 0, -1)
)
subsampler = ActionSequenceSampler(
encoder, IsSubtype(),
Sequence(OrderedDict([
("tinput",
Apply(
module=TransformInputs(),
in_keys=["test_cases"],
out_key="test_case_tensor",
)),
("tvariable",
Apply(
module=TransformVariables(),
in_keys=["variables", "test_case_tensor"],
out_key="variables_tensor"
)),
])),
Compose(OrderedDict([
("add_previous_actions",
Apply(
module=AddPreviousActions(encoder, n_dependent=1),
in_keys=["action_sequence", "reference"],
out_key="previous_actions",
constants={"train": False},
)),
("add_state", AddState("state")),
("add_hidden_state", AddState("hidden_state"))
])),
collate, model,
rng=np.random.RandomState(0))
subsampler = mlprogram.samplers.transform(
subsampler,
Parser().unparse
)
subsynthesizer = SMC(
5, 1,
subsampler,
max_try_num=1,
to_key=Pick("action_sequence"),
rng=np.random.RandomState(0)
)
sampler = SequentialProgramSampler(
subsynthesizer,
Apply(
module=TransformInputs(),
in_keys=["test_cases"],
out_key="test_case_tensor",
),
collate,
model.encode_input,
interpreter=interpreter,
expander=Expander(),
rng=np.random.RandomState(0))
if rollout:
sampler = FilteredSampler(
sampler,
metrics.use_environment(
metric=metrics.TestCaseResult(
interpreter,
metric=metrics.use_environment(
metric=metrics.Iou(),
in_keys=["actual", "expected"],
value_key="actual",
)
),
in_keys=["test_cases", "actual"],
value_key="actual"
),
1.0
)
return SMC(3, 1, sampler, rng=np.random.RandomState(0),
to_key=Pick("interpreter_state"), max_try_num=1)
else:
sampler = SamplerWithValueNetwork(
sampler,
Sequence(OrderedDict([
("tinput",
Apply(
module=TransformInputs(),
in_keys=["test_cases"],
out_key="test_case_tensor",
)),
("tvariable",
Apply(
module=TransformVariables(),
in_keys=["variables", "test_case_tensor"],
out_key="variables_tensor"
)),
])),
collate,
torch.nn.Sequential(OrderedDict([
("encoder", model.encoder),
("value", model.value),
("pick",
mlprogram.nn.Function(
Pick("value")))
])))
synthesizer = SynthesizerWithTimeout(
SMC(3, 1, sampler, rng=np.random.RandomState(0),
to_key=Pick("interpreter_state"),
max_try_num=1),
1
)
return FilteredSynthesizer(
synthesizer,
metrics.use_environment(
metric=metrics.TestCaseResult(
interpreter,
metric=metrics.use_environment(
metric=metrics.Iou(),
in_keys=["actual", "expected"],
value_key="actual",
)
),
in_keys=["test_cases", "actual"],
value_key="actual"
),
1.0
)
def test_reference(self):
torch.manual_seed(0)
rule_prob = torch.tensor([
[[
1.0, # unknown
1.0, # close variadic field
0.1, # Root2X
0.2, # Root2Y
1.0, # X2Y_list
1.0, # Ysub2Str
]],
[[
1.0, # unknown
1.0, # close variadic field
1.0, # Root2X
1.0, # Root2Y
1.0, # X2Y_list
1.0, # Ysub2Str
]],
[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]])
token_prob = torch.tensor([
[[0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0]],
[[
1.0, # Unknown
0.8, # x
0.2, # 1
]]])
reference_prob = torch.tensor(
[[[0.0, 0.0]], [[0.0, 0.0]], [[0.1, 0.1]]])
sampler = ActionSequenceSampler(
create_encoder(),
is_subtype,
create_transform_input([Token("Str", "x", "x"),
Token(None, "x", "x")]),
transform_action_sequence,
collate,
Module(encoder_module,
DecoderModule(rule_prob, token_prob, reference_prob)),
rng=np.random.RandomState(0)
)
s = SamplerState(0.0, sampler.initialize(Environment()))
results = [s.state for s in sampler.top_k_samples([s], 1)]
results = [s.state for s in sampler.top_k_samples(results, 1)]
topk_results = list(sampler.top_k_samples(results, 1))
assert 1 == len(topk_results)
assert 3 == topk_results[0].state.state["length"].item()
assert np.allclose(log(0.2) + log(1.),
topk_results[0].state.score)
random_results = list(sampler.batch_k_samples(results[:1], [1]))
assert 1 == len(random_results)
assert 3 == random_results[0].state.state["length"].item()
assert np.allclose(log(0.2) + log(1.0),
random_results[0].state.score)
all_results = list(sampler.all_samples(results))
assert 1 == len(all_results)
assert 3 == all_results[0].state.state["length"].item()
assert np.allclose(log(0.2) + log(1.),
all_results[0].state.score)
all_results = list(sampler.all_samples(results, sorted=False))
assert 1 == len(all_results)
assert 3 == all_results[0].state.state["length"].item()
assert np.allclose(log(0.2) + log(1.),
all_results[0].state.score)