def test_model_eq(): m1 = clgen.Model.from_json({ "corpus": { "language": "opencl", "path": tests.data_path("tiny", "corpus") }, "train_opts": { "intermediate_checkpoints": False } }) m2 = clgen.Model.from_json({ "corpus": { "language": "opencl", "path": tests.data_path("tiny", "corpus") }, "train_opts": { "intermediate_checkpoints": False } }) m3 = clgen.Model.from_json({ "corpus": { "language": "opencl", "path": tests.data_path("tiny", "corpus") }, "train_opts": { "intermediate_checkpoints": True } }) assert m1 == m2 assert m2 != m3 assert m1 != False assert m1 != 'abcdef'
def test_from_archive_path(): # delete any existing unpacked directory fs.rm(tests.data_path("tiny", "corpus", exists=False)) c = clgen.Corpus.from_json( {"path": tests.data_path("tiny", "corpus.tar.bz2")}) assert TINY_HASH == c.hash
def test_model_hash(): m1 = clgen.Model.from_json({ "corpus": { "language": "opencl", "path": tests.data_path("tiny", "corpus") } }) # same as m1, with explicit default opt: m2 = clgen.Model.from_json({ "corpus": { "language": "opencl", "path": tests.data_path("tiny", "corpus") }, "train_opts": { "intermediate_checkpoints": True } }) # different opt value: m3 = clgen.Model.from_json({ "corpus": { "language": "opencl", "path": tests.data_path("tiny", "corpus") }, "train_opts": { "intermediate_checkpoints": False } }) assert m1.hash == m2.hash assert m2.hash != m3.hash
def test_cli_sample(): with tests.chdir(tests.data_path("pico")): cli.main("sample model.json sampler.json".split()) cli.main("--corpus-dir model.json".split()) cli.main("--model-dir model.json".split()) cli.main("--sampler-dir model.json sampler.json".split()) cli.main("ls files model.json sampler.json".split())
def test_explore(): c = clgen.Corpus.from_json({ "language": "opencl", "path": tests.data_path("tiny", "corpus", exists=False) }) clgen.explore(c.contentcache["kernels.db"])
def test_create_db_gh(): db_path = tests.data_path("db", "tmp.db", exists=False) fs.rm(db_path) dbutil.create_db(db_path, github=True) assert fs.exists(db_path) with pytest.raises(clgen.UserError): dbutil.create_db(db_path, github=True)
def test_model_to_json(): m1 = clgen.Model.from_json({ "corpus": { "path": tests.data_path("tiny", "corpus") }, "train_opts": { "intermediate_checkpoints": True } }) m2 = clgen.Model.from_json(m1.to_json()) m1 == m2
def preprocess_pair(basename, preprocessor=clgen.preprocess): gs_path = tests.data_path(os.path.join('cl', str(basename) + '.gs'), exists=not UPDATE_GS_FILES) tin_path = tests.data_path(os.path.join('cl', str(basename) + '.cl')) # Run preprocess tin = tests.data_str(tin_path) tout = preprocessor(tin) if UPDATE_GS_FILES: gs = tout with open(gs_path, 'w') as outfile: outfile.write(gs) print("\n-> updated gold standard file '{}' ...".format(gs_path), file=sys.stderr, end=' ') else: gs = tests.data_str(gs_path) return (gs, tout)
def _get_test_model(): return clgen.Model.from_json({ "corpus": { "path": tests.data_path("tiny", "corpus"), }, "architecture": { "rnn_size": 8, "num_layers": 2, }, "train_opts": { "epochs": 1 } })
def get_test_model(vocab="char"): return clgen.Model.from_json({ "corpus": { "path": tests.data_path("tiny", "corpus"), "vocabulary": vocab }, "architecture": { "rnn_size": 8, "num_layers": 2 }, "train_opts": { "epochs": 1 } })
def test_insert(): db_path = tests.data_path("db", "tmp.db", exists=False) fs.rm(db_path) dbutil.create_db(db_path) db = dbutil.connect(db_path) c = db.cursor() assert dbutil.num_rows_in(db_path, "ContentFiles") == 0 dbutil.sql_insert_dict(c, "ContentFiles", {"id": "a", "contents": "foo"}) dbutil.sql_insert_dict(c, "PreprocessedFiles", { "id": "a", "status": 0, "contents": "bar" }) dbutil.sql_insert_dict(c, "PreprocessedFiles", { "id": "b", "status": 1, "contents": "car" }) db.commit() c = db.cursor() assert dbutil.num_rows_in(db_path, "ContentFiles") == 1 assert dbutil.num_rows_in(db_path, "PreprocessedFiles") == 2 assert dbutil.cc(db_path, "ContentFiles", "contents") == 3 assert dbutil.cc(db_path, "ContentFiles", "id") == 1 assert dbutil.lc(db_path, "ContentFiles", "contents") == 1 dbutil.remove_bad_preprocessed(db_path) assert dbutil.num_rows_in(db_path, "ContentFiles") == 1 # remove_bad_preprocessed doesn't actually delete any rows, just # replaces contents assert dbutil.num_rows_in(db_path, "PreprocessedFiles") == 2 dbutil.remove_preprocessed(db_path) assert dbutil.num_rows_in(db_path, "ContentFiles") == 1 assert dbutil.num_rows_in(db_path, "PreprocessedFiles") == 0
def test_inline_fs_headers(): src = clgen.inline_fs_headers(tests.data_path("cl", "sample-3.cl"), []) assert "MY_DATA_TYPE" in src assert "__kernel void" in src
def test_cli_train(): with tests.chdir(tests.data_path("pico")): cli.main("train model.json".split()) cli.main("--corpus-dir model.json".split()) cli.main("--model-dir model.json".split())