def test_multiprocess(self):
accuracy = use_environment(
Accuracy(), in_keys=["actual", ["ground_truth", "expected"]],
value_key="actual"
)
dataset = ListDataset([
Environment(
{"query": "query0", "ground_truth": "c0"},
set(["ground_truth"])
),
Environment(
{"query": "query1", "ground_truth": "c0"},
set(["ground_truth"])
),
Environment(
{"query": "query2", "ground_truth": "c0"},
set(["ground_truth"])
),
])
with tempfile.TemporaryDirectory() as init_dir:
with context.Pool(2) as pool:
procs = []
for i in range(2):
p = pool.apply_async(
self._run,
args=(init_dir, dataset, {"accuracy": accuracy}, i),
)
procs.append(p)
out = [p.get() for p in procs]
r0 = out[0]
r1 = out[1]
assert r0 == r1
results = r0
assert results.metrics == {1: {"accuracy": 1.0 / 3},
3: {"accuracy": 2.0 / 3}}
assert 3 == len(results.results)
results.results[0].time = 0.0
results.results[1].time = 0.0
results.results[2].time = 0.0
results.results.sort(key=lambda x: x.sample["query"])
assert Result({"query": "query0",
"ground_truth": "c0"},
["c0", "c1", "c2"],
{1: {"accuracy": 1.0}, 3: {"accuracy": 1.0}},
True, 0.0) == results.results[0]
assert Result({"query": "query1",
"ground_truth": "c0"},
["c2", "c3", "c0"],
{1: {"accuracy": 0.0}, 3: {"accuracy": 1.0}},
True, 0.0) == results.results[1]
assert Result({"query": "query2",
"ground_truth": "c0"},
["c2", "c3", "c5"],
{1: {"accuracy": 0.0}, 3: {"accuracy": 0.0}},
True, 0.0) == results.results[2]
def test_happy_path(self):
f = UpdateInput()
entry = f(
Environment(
{"interpreter_state": BatchedState({}, {}, [], ["xxx\nyyy"])}))
assert entry["code"] == "xxx\nyyy"
state = BatchedState({}, {Diff([]): ["foo"]}, [Diff([])], ["foo"])
entry = f(Environment({"interpreter_state": state}))
assert entry["code"] == "foo"
def split(self, values: Environment) -> Sequence[Environment]:
retval: List[Environment] = []
B = None
for key, t in values.items():
if t is None:
continue
if key in self.options:
option = self.options[key]
if option.use_pad_sequence:
B = t.data.shape[1]
else:
B = t.data.shape[option.dim]
break
elif isinstance(t, list):
B = len(t)
break
assert B is not None
for _ in range(B):
retval.append(Environment())
for key, t in values.items():
if t is None:
for b in range(B):
retval[b][key] = None
continue
if key in self.options:
option = self.options[key]
if option.use_pad_sequence:
for b in range(B):
inds = torch.nonzero(t.mask[:, b], as_tuple=False)
data = t.data[:, b]
shape = data.shape[1:]
data = data[inds]
data = data.reshape(-1, *shape)
retval[b][key] = data
else:
shape = list(t.data.shape)
del shape[option.dim]
data = torch.split(t, 1, dim=option.dim)
for b in range(B):
d = data[b]
if len(shape) == 0:
d = d.reshape(())
else:
d = d.reshape(*shape)
retval[b][key] = d
elif isinstance(t, list):
for b in range(B):
retval[b][key] = t[b]
else:
logger.debug(f"{key} is invalid type: {type(t)}")
return retval
def __call__(self, entry: Environment) -> Environment:
entry = cast(Environment, entry.clone())
train = True
if "train" in entry:
train = entry["train"]
if "action_sequence" in entry:
train = entry.is_supervision("action_sequence")
if train or self.key not in entry:
entry[self.key] = self.initial
return entry
def test_to(self) -> None:
class X:
def to(self, *args, **kwargs):
self.args = (args, kwargs)
return self
e = Environment()
e["key"] = X()
e["x"] = torch.tensor(0)
e["y"] = PaddedSequenceWithMask(torch.tensor(0.0), torch.tensor(True))
e["z"] = 10
e.to(device=torch.device("cpu"))
assert e["key"].args == ((), {"device": torch.device("cpu")})
def test_download(self):
with tempfile.TemporaryDirectory() as tmpdir:
cache_path = os.path.join(tmpdir, "cache.pt")
sqlitefile = os.path.join(tmpdir, "dataset.db")
conn = sqlite3.connect(sqlitefile)
c = conn.cursor()
c.execute(
"CREATE TABLE Code(code text, error text, errorcount int)")
c.execute("INSERT INTO Code VALUES ('foo', 'bar', 1)")
c.execute("INSERT INTO Code VALUES ('foo', '', 0)")
conn.commit()
conn.close()
gzipfile = os.path.join(tmpdir, "dataset.gz")
with gzip.open(gzipfile, "wb") as file, \
open(sqlitefile, "rb") as src_file:
copyfileobj(src_file, file)
path = os.path.join(tmpdir, "dataset.zip")
with zipfile.ZipFile(path, "w") as z:
with z.open(os.path.join("prutor-deepfix-09-12-2017",
"prutor-deepfix-09-12-2017.db.gz"),
"w") as dst_file, \
open(gzipfile, "rb") as src_file:
copyfileobj(src_file, dst_file)
def get(src, dst):
copyfile(src, dst)
dataset0 = download(cache_path=cache_path, path=path, get=get)
def get2(src, dst):
raise NotImplementedError
dataset1 = download(cache_path=cache_path, path=path, get=get2)
assert 2 == len(dataset0)
assert dataset0[0] == Environment(
{
"code": "foo",
"error": "bar",
"n_error": 1
}, set(["error", "n_error"]))
assert dataset0[1] == Environment(
{
"code": "foo",
"error": "",
"n_error": 0
}, set(["error", "n_error"]))
assert list(dataset0) == list(dataset1)
def test_download(self):
values = [
"line0\n",
"x = 10\n",
"line1\n",
"if True:§ pass\n",
"line2\n",
"if True and \\True:§ pass\n",
]
with tempfile.TemporaryDirectory() as tmpdir:
cache_path = os.path.join(tmpdir, "cache.pt")
def get(path):
return values.pop(0)
dataset0 = download(get=get, cache_path=cache_path)
def get2(path):
raise NotImplementedError
dataset1 = download(get=get2, cache_path=cache_path)
train_dataset = dataset0["train"]
test_dataset = dataset0["test"]
valid_dataset = dataset0["valid"]
assert 1 == len(train_dataset)
assert list(train_dataset) == list(dataset1["train"])
assert train_dataset[0] == Environment(
{
"text_query": "line0",
"ground_truth": "x = 10"
}, set(["ground_truth"]))
assert 1 == len(test_dataset)
assert list(test_dataset) == list(dataset1["test"])
assert test_dataset[0] == Environment(
{
"text_query": "line1",
"ground_truth": "if True:\n pass"
}, set(["ground_truth"]))
assert 1 == len(valid_dataset)
assert list(valid_dataset) == list(dataset1["valid"])
assert valid_dataset[0] == Environment(
{
"text_query": "line2",
"ground_truth": "if True and True:\n pass"
}, set(["ground_truth"]))
def test_propagate_supervision(self):
apply = Apply(["x", "y"], "out", MockModule(1))
output = apply(
Environment({
"x": torch.arange(3).reshape(-1, 1),
"y": 10
}))
assert not output.is_supervision("out")
output = apply(
Environment({
"x": torch.arange(3).reshape(-1, 1),
"y": 10
}, set(["x"])))
assert output.is_supervision("out")
def test_eval(self):
entries = [Environment(
{"text_query": "foo bar", "ground_truth": "y = x + 1"},
set(["ground_truth"])
)]
dataset = ListDataset(entries)
d = get_samples(dataset, MockParser())
aencoder = ActionSequenceEncoder(d, 0)
action_sequence = GroundTruthToActionSequence(MockParser())(
"y = x + 1"
)
transform = AddActions(aencoder)
action_tensor = transform(
reference=[Token(None, "foo", "foo"), Token(None, "bar", "bar")],
action_sequence=action_sequence,
train=False
)
assert np.array_equal(
[
[2, 2, 0], [4, 3, 1], [6, 4, 2], [6, 4, 2], [5, 3, 1],
[6, 5, 5], [6, 5, 5], [5, 5, 5], [6, 4, 8], [6, 4, 8],
[5, 5, 5], [9, 6, 11], [9, 6, 11], [-1, -1, -1]
],
action_tensor.numpy()
)
def __next__(self) -> Environment:
self.n += 1
return Environment(
{
"value": torch.tensor(self.n),
"ground_truth": torch.tensor(self.n)
}, set(["ground_truth"]))
def test_constants(self):
apply = Apply([],
"out",
MockModule(1),
constants={"x": torch.arange(3).reshape(-1, 1)})
output = apply(Environment())
assert np.array_equal([[1], [2], [3]], output["out"].detach().numpy())
def test_simple_case(self):
entries = [Environment(
{"ground_truth": "y = x + 1"},
set(["ground_truth"])
)]
dataset = ListDataset(entries)
d = get_samples(dataset, MockParser())
aencoder = ActionSequenceEncoder(d, 0)
action_sequence = GroundTruthToActionSequence(MockParser())(
ground_truth="y = x + 1"
)
transform = EncodeActionSequence(aencoder)
ground_truth = transform(
action_sequence=action_sequence,
reference=[Token(None, "foo", "foo"), Token(None, "bar", "bar")],
)
assert np.array_equal(
[
[3, -1, -1], [4, -1, -1], [-1, 1, -1], [1, -1, -1],
[5, -1, -1], [-1, 2, -1], [1, -1, -1], [4, -1, -1],
[-1, 3, -1], [1, -1, -1], [6, -1, -1], [-1, 4, -1],
[1, -1, -1]
],
ground_truth.numpy()
)
def test_eval(self):
entries = [Environment(
{"text_query": "ab test", "ground_truth": "y = x + 1"},
set(["ground_truth"])
)]
dataset = ListDataset(entries)
d = get_samples(dataset, MockParserWithoutVariadicArgs())
aencoder = ActionSequenceEncoder(d, 0)
action_sequence = GroundTruthToActionSequence(MockParserWithoutVariadicArgs())(
"y = x + 1"
)
transform = AddQueryForTreeGenDecoder(aencoder, 3,)
query = transform(
reference=[Token(None, "ab", "ab"), Token(None, "test", "test")],
action_sequence=action_sequence,
train=False
)
assert np.array_equal(
[
[-1, -1, -1], [2, -1, -1], [3, 2, -1], [4, 3, 2],
[3, 2, -1], [5, 3, 2], [5, 3, 2], [4, 5, 3],
[5, 3, 2], [6, 5, 3]
],
query.numpy()
)
def test_eval(self):
entries = [Environment(
{"text_query": "ab test", "ground_truth": "y = x + 1"},
set(["ground_truth"])
)]
dataset = ListDataset(entries)
d = get_samples(dataset, MockParserWithoutVariadicArgs())
aencoder = ActionSequenceEncoder(d, 0)
action_sequence = GroundTruthToActionSequence(MockParserWithoutVariadicArgs())(
"y = x + 1"
)
transform = AddActionSequenceAsTree(aencoder,)
matrix, depth = transform(
reference=[Token(None, "ab", "ab"), Token(None, "test", "test")],
action_sequence=action_sequence,
train=False
)
assert np.array_equal(
[0, 1, 2, 3, 2, 3, 3, 4, 3, 4],
depth.numpy()
)
assert np.array_equal(
[[0, 1, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 1, 0, 1, 0, 0, 0, 0, 0],
[0, 0, 0, 1, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 1, 1, 0, 1, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 1, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],
matrix.numpy()
)
def test_n_dependent(self):
entries = [Environment(
{"text_query": "ab test", "ground_truth": "y = x + 1"},
set(["ground_truth"])
)]
dataset = ListDataset(entries)
d = get_samples(dataset, MockParserWithoutVariadicArgs())
aencoder = ActionSequenceEncoder(d, 0)
action_sequence = GroundTruthToActionSequence(MockParserWithoutVariadicArgs())(
"y = x + 1"
)
transform = AddPreviousActionRules(aencoder, 2, n_dependent=3)
prev_rule_action = transform(
reference=[Token(None, "ab", "ab"), Token(None, "test", "test")],
action_sequence=action_sequence,
train=False,
)
assert np.array_equal(
[
# str -> "y"
[[-1, -1, -1], [-1, 3, -1], [-1, -1, -1]],
# Number -> number
[[8, -1, -1], [9, -1, -1], [-1, -1, -1]],
[[-1, -1, -1], [-1, 4, -1], [-1, -1, -1]],
],
prev_rule_action.numpy()
)
def _download():
logger.info("Download DeepFix dataset")
logger.debug(f"Dataset path: {path}")
samples = []
with tempfile.TemporaryDirectory() as tmpdir:
dst = os.path.join(tmpdir, "dataset.zip")
get(path, dst)
gzipfile = os.path.join(tmpdir, "dataset.gz")
with zipfile.ZipFile(dst) as z:
with z.open(os.path.join("prutor-deepfix-09-12-2017",
"prutor-deepfix-09-12-2017.db.gz"),
"r") as file, \
open(gzipfile, "wb") as dst_file:
copyfileobj(file, dst_file)
sqlitefile = os.path.join(tmpdir, "dataset.db")
with gzip.open(gzipfile, "rb") as src_file, \
open(sqlitefile, "wb") as dst_file:
copyfileobj(src_file, dst_file)
conn = sqlite3.connect(sqlitefile)
c = conn.cursor()
for code, error, errorcount in \
c.execute("SELECT code, error, errorcount FROM Code"):
samples.append(
Environment(
{
"code": code,
"error": error,
"n_error": errorcount,
}, set(["error", "n_error"])))
return samples
def dataset():
return ListDataset([
Environment({
"query": "query",
"ground_truth": "name0"
}, set(["ground_truth"]))
])
def test_eval(self):
entries = [Environment(
{"ground_truth": "y = x + 1"},
set(["ground_truth"])
)]
dataset = ListDataset(entries)
d = get_samples(dataset, MockParser())
aencoder = ActionSequenceEncoder(d, 0)
action_sequence = GroundTruthToActionSequence(MockParser())(
"y = x + 1"
)
transform = AddPreviousActions(aencoder)
prev_action_tensor = transform(
reference=[Token(None, "foo", "foo"), Token(None, "bar", "bar")],
action_sequence=action_sequence,
train=False
)
assert np.array_equal(
[
[2, -1, -1], [3, -1, -1], [4, -1, -1], [-1, 1, -1],
[1, -1, -1], [5, -1, -1], [-1, 2, -1], [1, -1, -1],
[4, -1, -1], [-1, 3, -1], [1, -1, -1], [6, -1, -1],
[-1, 4, -1], [1, -1, -1]
],
prev_action_tensor.numpy()
)
def test_ast_set_sample(self):
asts = ["c0", "c1", "c2"]
sampler = SequentialProgramSampler(MockSynthesizer(asts),
transform_input, Collate(),
MockEncoder(), MockExpander(),
MockInterpreter())
zero = SamplerState(0, sampler.initialize([(None, None)]))
samples = list(sampler.batch_k_samples([zero], [3]))
samples.sort(key=lambda x: -x.state.score)
assert 3 == len(samples)
assert samples[0] == DuplicatedSamplerState(
SamplerState(
1,
Environment({
"test_cases": [(None, None)],
"reference": [Token(None, str(asts[0]), str(asts[0]))],
"variables": [["#" + str(asts[0])]],
"interpreter_state":
BatchedState({str(asts[0]): None},
{str(asts[0]): ["#" + str(asts[0])]},
[str(asts[0])], ["#" + str(asts[0])])
})), 1)
assert DuplicatedSamplerState(
SamplerState(
0.5,
Environment({
"test_cases": [(None, None)],
"reference": [Token(None, str(asts[1]), str(asts[1]))],
"variables": [["#" + str(asts[1])]],
"interpreter_state":
BatchedState({str(asts[1]): None},
{str(asts[1]): ["#" + str(asts[1])]},
[str(asts[1])], ["#" + str(asts[1])])
})), 1) == samples[1]
assert DuplicatedSamplerState(
SamplerState(
1.0 / 3,
Environment({
"test_cases": [(None, None)],
"reference": [Token(None, str(asts[2]), str(asts[2]))],
"variables": [["#" + str(asts[2])]],
"interpreter_state":
BatchedState({str(asts[2]): None},
{str(asts[2]): ["#" + str(asts[2])]},
[str(asts[2])], ["#" + str(asts[2])])
})), 1) == samples[2]
def create_dummy_nl2bash(path):
data = [
Environment({
"text_query": "foo",
"ground_truth": "echoa"
}, set(["ground_truth"]))
]
torch.save({"train": data, "test": data, "valid": data}, path)
def create_dummy_hearthstone(path):
data = [
Environment({
"text_query": "foo",
"ground_truth": "1 + 1"
}, set(["ground_truth"]))
]
torch.save({"train": data, "test": data, "valid": data}, path)
def to_sample(elem: Tuple[str, str]) -> Environment:
anno, code = elem
return Environment(
{
"text_query": anno,
"ground_truth": code
},
set(["ground_truth"])
)
def test_multiple_inputs(self):
apply = Apply(["x", "y"], "out", MockModule(1))
output = apply(
Environment({
"x": torch.arange(3).reshape(-1, 1),
"y": 10
}))
assert np.array_equal([[11], [12], [13]],
output["out"].detach().numpy())
def test_simple_case(self):
accuracy = use_environment(
Accuracy(), in_keys=["actual", ["ground_truth", "expected"]],
value_key="actual"
)
dataset = ListDataset([
Environment(
{"query": "query0", "ground_truth": "c0"},
set(["ground_truth"])
),
Environment(
{"query": "query1", "ground_truth": "c0"},
set(["ground_truth"])
),
Environment(
{"query": "query2", "ground_truth": "c0"},
set(["ground_truth"])
),
])
results = EvaluateSynthesizer(dataset, synthesize,
metrics={"accuracy": accuracy})()
assert results.metrics == \
{1: {"accuracy": 1.0 / 3.0}, 3: {"accuracy": 2.0 / 3.0}}
assert 3 == len(results.results)
results.results[0].time = 0.0
results.results[1].time = 0.0
results.results[2].time = 0.0
assert Result({"query": "query0",
"ground_truth": "c0"},
["c0", "c1", "c2"],
{1: {"accuracy": 1.0}, 3: {"accuracy": 1.0}},
True, 0.0) == results.results[0]
assert Result({"query": "query1",
"ground_truth": "c0"},
["c2", "c3", "c0"],
{1: {"accuracy": 0.0}, 3: {"accuracy": 1.0}},
True, 0.0) == results.results[1]
assert Result({"query": "query2",
"ground_truth": "c0"},
["c2", "c3", "c5"],
{1: {"accuracy": 0.0}, 3: {"accuracy": 0.0}},
True, 0.0) == results.results[2]
def forward(self, test_cases: List[Tuple[Input, Kind]],
actual: Code) -> float:
inputs = [input for input, _ in test_cases]
outputs = [output for _, output in test_cases]
# calc. metric
m = 0.0 # TODO reduction function is required
for actual, expected in zip(self.interpreter.eval(actual, inputs),
outputs):
m += self.metric(Environment({"expected": expected}), actual)
return m / len(outputs)
def test_happy_path(self):
to_episode = ToEpisode(Interpreter(), Expander())
episode = to_episode(
Environment(
{
"test_cases": [("xxx\nyyy", None)],
"ground_truth": Diff([Replace(0, "zzz"),
Remove(1)])
}, set(["ground_truth"])))
assert len(episode) == 2
assert episode[0]["interpreter_state"].context == ["xxx\nyyy"]
assert episode[1]["interpreter_state"].context == ["zzz\nyyy"]
def test_create_output(self):
sampler = SequentialProgramSampler(MockSynthesizer([]),
transform_input, Collate(),
MockEncoder(), MockExpander(),
MockInterpreter())
assert sampler.create_output(
None,
Environment({
"interpreter_state": BatchedState({}, {}, [], [None])
})) is None
assert sampler.create_output(
None,
Environment(
{"interpreter_state": BatchedState({}, {}, ["tmp"],
[None])})) == ("tmp", False)
assert sampler.create_output(
None,
Environment({
"interpreter_state":
BatchedState({}, {}, ["line0", "line1"], [None])
})) == ("line0\nline1", False)
def test_rule(self):
rule_prob = torch.tensor([
[[
1.0, # unknown
1.0, # close variadic field
0.2, # Root2X
0.1, # Root2Y
1.0, # X2Y_list
1.0, # Ysub2Str
]],
[[
1.0, # unknown
1.0, # close variadic field
1.0, # Root2X
1.0, # Root2Y
0.5, # X2Y_list
1.0, # Ysub2Str
]]])
token_prob = torch.tensor([[[]], [[]]])
reference_prob = torch.tensor([[[]], [[]]])
sampler = ActionSequenceSampler(
create_encoder(),
is_subtype,
create_transform_input([]), transform_action_sequence,
collate,
Module(encoder_module,
DecoderModule(rule_prob, token_prob, reference_prob))
)
s = SamplerState(0.0, sampler.initialize(Environment()))
topk_results = list(sampler.top_k_samples([s], 1))
assert 1 == len(topk_results)
assert 1 == topk_results[0].state.state["length"].item()
assert np.allclose(log(0.2), topk_results[0].state.score)
random_results = list(sampler.batch_k_samples([s], [1]))
assert 1 == len(random_results)
assert 1 == random_results[0].state.state["length"].item()
assert \
log(0.1) - 1e-5 <= random_results[0].state.score <= log(0.2) + 1e-5
all_results = list(sampler.all_samples([s]))
assert 2 == len(all_results)
assert 1 == all_results[0].state.state["length"].item()
assert np.allclose(log(0.2), all_results[0].state.score)
assert np.allclose(log(0.1), all_results[1].state.score)
all_results = list(sampler.all_samples([s], sorted=False))
assert 1 == all_results[0].state.state["length"].item()
assert \
log(0.1) - 1e-5 <= all_results[0].state.score <= log(0.2) + 1e-5
next = list(sampler.top_k_samples(
[s.state for s in topk_results], 1))[0]
assert 2 == next.state.state["length"].item()
assert np.allclose(log(0.2) + log(0.5), next.state.score)
