def test_model_eq():
m1 = clgen.Model.from_json({
"corpus": {
"language": "opencl",
"path": tests.data_path("tiny", "corpus")
},
"train_opts": {
"intermediate_checkpoints": False
}
})
m2 = clgen.Model.from_json({
"corpus": {
"language": "opencl",
"path": tests.data_path("tiny", "corpus")
},
"train_opts": {
"intermediate_checkpoints": False
}
})
m3 = clgen.Model.from_json({
"corpus": {
"language": "opencl",
"path": tests.data_path("tiny", "corpus")
},
"train_opts": {
"intermediate_checkpoints": True
}
})
assert m1 == m2
assert m2 != m3
assert m1 != False
assert m1 != 'abcdef'
def test_from_archive_path():
# delete any existing unpacked directory
fs.rm(tests.data_path("tiny", "corpus", exists=False))
c = clgen.Corpus.from_json(
{"path": tests.data_path("tiny", "corpus.tar.bz2")})
assert TINY_HASH == c.hash
def test_model_hash():
m1 = clgen.Model.from_json({
"corpus": {
"language": "opencl",
"path": tests.data_path("tiny", "corpus")
}
})
# same as m1, with explicit default opt:
m2 = clgen.Model.from_json({
"corpus": {
"language": "opencl",
"path": tests.data_path("tiny", "corpus")
},
"train_opts": {
"intermediate_checkpoints": True
}
})
# different opt value:
m3 = clgen.Model.from_json({
"corpus": {
"language": "opencl",
"path": tests.data_path("tiny", "corpus")
},
"train_opts": {
"intermediate_checkpoints": False
}
})
assert m1.hash == m2.hash
assert m2.hash != m3.hash
def test_cli_sample():
with tests.chdir(tests.data_path("pico")):
cli.main("sample model.json sampler.json".split())
cli.main("--corpus-dir model.json".split())
cli.main("--model-dir model.json".split())
cli.main("--sampler-dir model.json sampler.json".split())
cli.main("ls files model.json sampler.json".split())
def test_explore():
c = clgen.Corpus.from_json({
"language":
"opencl",
"path":
tests.data_path("tiny", "corpus", exists=False)
})
clgen.explore(c.contentcache["kernels.db"])
def test_create_db_gh():
db_path = tests.data_path("db", "tmp.db", exists=False)
fs.rm(db_path)
dbutil.create_db(db_path, github=True)
assert fs.exists(db_path)
with pytest.raises(clgen.UserError):
dbutil.create_db(db_path, github=True)
def test_model_to_json():
m1 = clgen.Model.from_json({
"corpus": {
"path": tests.data_path("tiny", "corpus")
},
"train_opts": {
"intermediate_checkpoints": True
}
})
m2 = clgen.Model.from_json(m1.to_json())
m1 == m2
def preprocess_pair(basename, preprocessor=clgen.preprocess):
gs_path = tests.data_path(os.path.join('cl',
str(basename) + '.gs'),
exists=not UPDATE_GS_FILES)
tin_path = tests.data_path(os.path.join('cl', str(basename) + '.cl'))
# Run preprocess
tin = tests.data_str(tin_path)
tout = preprocessor(tin)
if UPDATE_GS_FILES:
gs = tout
with open(gs_path, 'w') as outfile:
outfile.write(gs)
print("\n-> updated gold standard file '{}' ...".format(gs_path),
file=sys.stderr,
end=' ')
else:
gs = tests.data_str(gs_path)
return (gs, tout)
def _get_test_model():
return clgen.Model.from_json({
"corpus": {
"path": tests.data_path("tiny", "corpus"),
},
"architecture": {
"rnn_size": 8,
"num_layers": 2,
},
"train_opts": {
"epochs": 1
}
})
def get_test_model(vocab="char"):
return clgen.Model.from_json({
"corpus": {
"path": tests.data_path("tiny", "corpus"),
"vocabulary": vocab
},
"architecture": {
"rnn_size": 8,
"num_layers": 2
},
"train_opts": {
"epochs": 1
}
})
def test_insert():
db_path = tests.data_path("db", "tmp.db", exists=False)
fs.rm(db_path)
dbutil.create_db(db_path)
db = dbutil.connect(db_path)
c = db.cursor()
assert dbutil.num_rows_in(db_path, "ContentFiles") == 0
dbutil.sql_insert_dict(c, "ContentFiles", {"id": "a", "contents": "foo"})
dbutil.sql_insert_dict(c, "PreprocessedFiles", {
"id": "a",
"status": 0,
"contents": "bar"
})
dbutil.sql_insert_dict(c, "PreprocessedFiles", {
"id": "b",
"status": 1,
"contents": "car"
})
db.commit()
c = db.cursor()
assert dbutil.num_rows_in(db_path, "ContentFiles") == 1
assert dbutil.num_rows_in(db_path, "PreprocessedFiles") == 2
assert dbutil.cc(db_path, "ContentFiles", "contents") == 3
assert dbutil.cc(db_path, "ContentFiles", "id") == 1
assert dbutil.lc(db_path, "ContentFiles", "contents") == 1
dbutil.remove_bad_preprocessed(db_path)
assert dbutil.num_rows_in(db_path, "ContentFiles") == 1
# remove_bad_preprocessed doesn't actually delete any rows, just
# replaces contents
assert dbutil.num_rows_in(db_path, "PreprocessedFiles") == 2
dbutil.remove_preprocessed(db_path)
assert dbutil.num_rows_in(db_path, "ContentFiles") == 1
assert dbutil.num_rows_in(db_path, "PreprocessedFiles") == 0
def test_inline_fs_headers():
src = clgen.inline_fs_headers(tests.data_path("cl", "sample-3.cl"), [])
assert "MY_DATA_TYPE" in src
assert "__kernel void" in src
def test_cli_train():
with tests.chdir(tests.data_path("pico")):
cli.main("train model.json".split())
cli.main("--corpus-dir model.json".split())
cli.main("--model-dir model.json".split())
