def main():
parser = ArgumentParser(description=__description__)
parser.add_argument("classification")
parser.add_argument("outdir")
args = parser.parse_args()
db.init("cc1")
session = db.make_session()
program_ids = [
x[0] for x in session.query(sql.distinct(CLSmithResult.program_id)) \
.filter(CLSmithResult.classification == args.classification).all()]
header = fs.read_file(dsmith.data_path("include", "clsmith.h"))
fs.mkdir(args.outdir)
for program_id in ProgressBar()(program_ids):
outpath = fs.path(args.outdir, program_id + ".cl")
if not fs.exists(outpath):
program = session.query(CLSmithProgram) \
.filter(CLSmithProgram.id == program_id).one()
pre, post = program.src.split('#include "CLSmith.h"')
inlined = pre + header + post
with open(outpath, "w") as outfile:
print(inlined, file=outfile)
def test_cp_dir():
fs.rm("/tmp/labm8")
fs.rm("/tmp/labm8.copy")
fs.mkdir("/tmp/labm8/foo/bar")
assert not fs.exists("/tmp/labm8.copy")
fs.cp("/tmp/labm8/", "/tmp/labm8.copy")
assert fs.isdir("/tmp/labm8.copy")
assert fs.isdir("/tmp/labm8.copy/foo")
assert fs.isdir("/tmp/labm8.copy/foo/bar")
def export_todir(s: session_t, table, dir: Path) -> None:
fs.mkdir(dir)
q = s.query(table)
num = s.query(sql.sql.func.count(table.id)).scalar()
for result in ProgressBar(max_value=num)(q):
buf = result.toProtobuf().SerializeToString()
checksum = crypto.sha1(buf)
with open(f"{dir}/{checksum}.pb", "wb") as f:
f.write(buf)
def __init__(self, root, escape_key=hash_key):
"""
Create filesystem cache.
Arguments:
root (str): String.
escape_key (fn, optional): Function to convert keys to file names.
"""
self.path = pathlib.Path(root)
self.escape_key = escape_key
fs.mkdir(self.path)
def train_and_save(model_desc,
platform,
source,
atomizer="CharacterAtomizer",
maxlen=1024,
n_splits=10,
split_i=0,
seed=204):
np.random.seed(seed)
name = model_desc["name"]
outpath = "models/{name}/{platform}-{source}-{atomizer}:{maxlen}-{seed}-{n_splits}-{split_i}.model".format(
**vars())
if not fs.exists(outpath):
create_fn = model_desc.get("create_model", _nop)
train_fn = model_desc.get("train_fn", _nop)
save_fn = model_desc["save_fn"]
_atomizer = globals().get(atomizer)
# load training data
data_desc = load_data_desc(platform=platform,
source=source,
max_seq_len=maxlen,
atomizer=_atomizer)
train, test = get_training_data(data_desc,
seed=seed,
split_i=split_i,
n_splits=n_splits)
# create model
model = create_fn(seed=seed, data_desc=data_desc)
# train model
train_fn(model=model,
train=train,
seed=seed,
platform=platform,
source=source)
fs.mkdir("models/{name}".format(**vars()))
save_fn(outpath, model)
print("model saved as", outpath)
# evaluate model
return load_and_test(model_desc,
platform,
source,
n_splits=n_splits,
split_i=split_i,
atomizer=atomizer,
maxlen=maxlen,
seed=seed)
def test_rm():
system.echo("Hello, world!", "/tmp/labm8.tmp")
assert fs.isfile("/tmp/labm8.tmp")
fs.rm("/tmp/labm8.tmp")
assert not fs.isfile("/tmp/labm8.tmp")
fs.rm("/tmp/labm8.tmp")
fs.rm("/tmp/labm8.tmp")
fs.rm("/tmp/labm8.dir")
fs.mkdir("/tmp/labm8.dir/foo/bar")
system.echo("Hello, world!", "/tmp/labm8.dir/foo/bar/baz")
assert fs.isfile("/tmp/labm8.dir/foo/bar/baz")
fs.rm("/tmp/labm8.dir")
assert not fs.isfile("/tmp/labm8.dir/foo/bar/baz")
assert not fs.isfile("/tmp/labm8.dir/")
def test_rmtrash():
with tempfile.NamedTemporaryFile(prefix='labm8_') as f:
assert fs.isfile(f.name)
fs.rmtrash(f.name)
assert not fs.isfile(f.name)
fs.rmtrash(f.name)
fs.rm(f.name)
with tempfile.TemporaryDirectory() as d:
fs.rm(d)
fs.mkdir(d, "foo/bar")
system.echo("Hello, world!", fs.path(d, "foo/bar/baz"))
assert fs.isfile(f, "foo/bar/baz")
fs.rmtrash(d)
assert not fs.isfile(d, "foo/bar/baz")
assert not fs.isdir(d)
def test_cp_over_dir():
fs.mkdir("/tmp/labm8.tmp.src")
system.echo("Hello, world!", "/tmp/labm8.tmp.src/foo")
fs.rm("/tmp/labm8.tmp.copy")
fs.mkdir("/tmp/labm8.tmp.copy")
assert fs.isdir("/tmp/labm8.tmp.src")
assert fs.isfile("/tmp/labm8.tmp.src/foo")
assert fs.isdir("/tmp/labm8.tmp.copy")
assert not fs.isfile("/tmp/labm8.tmp.copy/foo")
fs.cp("/tmp/labm8.tmp.src", "/tmp/labm8.tmp.copy/")
assert fs.isdir("/tmp/labm8.tmp.src")
assert fs.isfile("/tmp/labm8.tmp.src/foo")
assert fs.isdir("/tmp/labm8.tmp.copy")
assert fs.isfile("/tmp/labm8.tmp.copy/foo")
assert (fs.read("/tmp/labm8.tmp.src/foo") == fs.read(
"/tmp/labm8.tmp.copy/foo"))
def __setitem__(self, key, value):
"""
Emplace file in cache.
Arguments:
key: Key.
value (str): Path of file to insert in cache.
Raises:
ValueError: If no "value" does nto exist.
"""
if not fs.exists(value):
raise ValueError(value)
path = self.keypath(key)
fs.mkdir(self.path)
fs.mv(value, path)
def main(argv):
"""Main entry point."""
if len(argv) > 1:
raise app.UsageError("Unknown arguments: '{}'.".format(' '.join(
argv[1:])))
graph = ControlFlowGraph.GenerateRandom(
FLAGS.reachability_num_nodes,
seed=FLAGS.reachability_seed,
connections_scaling_param=FLAGS.reachability_scaling_param)
fs.mkdir(pathlib.Path(FLAGS.reachability_dot_path).parent)
with open(FLAGS.reachability_dot_path, 'w') as f:
f.write(graph.ToDot())
for node in graph.all_nodes:
s = ' '.join(child.name for child in node.children)
print(f'{node.name}: {s}')
def test_rm_glob():
fs.mkdir("/tmp/labm8.glob")
system.echo("Hello, world!", "/tmp/labm8.glob/1")
system.echo("Hello, world!", "/tmp/labm8.glob/2")
system.echo("Hello, world!", "/tmp/labm8.glob/abc")
fs.rm("/tmp/labm8.glob/a*", glob=False)
assert fs.isfile("/tmp/labm8.glob/1")
assert fs.isfile("/tmp/labm8.glob/2")
assert fs.isfile("/tmp/labm8.glob/abc")
fs.rm("/tmp/labm8.glob/a*")
assert fs.isfile("/tmp/labm8.glob/1")
assert fs.isfile("/tmp/labm8.glob/2")
assert not fs.isfile("/tmp/labm8.glob/abc")
fs.rm("/tmp/labm8.glob/*")
assert not fs.isfile("/tmp/labm8.glob/1")
assert not fs.isfile("/tmp/labm8.glob/2")
assert not fs.isfile("/tmp/labm8.glob/abc")
if __name__ == "__main__":
parser = ArgumentParser(description=__doc__)
parser.add_argument("-H",
"--hostname",
type=str,
default="cc1",
help="MySQL database hostname")
args = parser.parse_args()
db.init(args.hostname)
session = db.make_session()
clsmith_wrong_code_programs = session.query(CLSmithResult) \
.filter(CLSmithResult.classification == "w")
fs.mkdir("../data/difftest/unreduced/clsmith/wrong_code")
fs.mkdir("../data/difftest/unreduced/clsmith/wrong_code/reports")
for result in clsmith_wrong_code_programs:
vendor = vendor_str(result.testbed.platform)
with open(
f"../data/difftest/unreduced/clsmith/wrong_code/{vendor}-{result.program.id}.cl",
"w") as outfile:
print(result.program.src, file=outfile)
with open(
f"../data/difftest/unreduced/clsmith/wrong_code/reports/{vendor}-{result.id}.txt",
"w") as outfile:
print(outfile.name)
print(generate_wrong_code_report(result), file=outfile)
if __name__ == "__main__":
parser = ArgumentParser(description=__doc__)
parser.add_argument("-H",
"--hostname",
type=str,
default="cc1",
help="MySQL database hostname")
args = parser.parse_args()
db.init(args.hostname)
with Session(commit=False) as s:
# Export results
#
print("Exporting CLgen results ...")
fs.mkdir("export/clgen/result")
# Pick up where we left off
done = set([
int(fs.basename(path))
for path in Path("export/clgen/result").iterdir()
])
print(len(done), "done")
ids = set([x[0] for x in s.query(CLgenResult.id).all()])
print(len(ids), "in total")
todo = ids - done
print(len(todo), "todo")
for result_id in ProgressBar()(todo):
result = s.query(CLgenResult).filter(
CLgenResult.id == result_id).scalar()
def test_mkdir():
fs.rm("/tmp/labm8.dir")
assert not fs.isdir("/tmp/labm8.dir")
fs.mkdir("/tmp/labm8.dir")
assert fs.isdir("/tmp/labm8.dir")
#!/usr/bin/env python3.6
import sys
from phd.lib.labm8 import crypto
from phd.lib.labm8 import fs
from progressbar import ProgressBar
if __name__ == "__main__":
inpath = sys.argv[1]
outdir = sys.argv[2]
print(f"reading from {inpath} into {outdir}")
assert fs.isfile(inpath)
assert not fs.exists(outdir) or fs.isdir(outdir)
fs.mkdir(outdir)
with open(inpath) as infile:
text = infile.read()
kernels = text.split("// ==== START SAMPLE ====")
kernels = [kernel.strip() for kernel in kernels if kernel.strip()]
print(len(kernels), "kernels")
sha1s = [crypto.sha1_str(kernel) for kernel in kernels]
for kernel, sha1 in ProgressBar()(list(zip(kernels, sha1s))):
with open(f"{outdir}/{sha1}.txt", "w") as outfile:
print(kernel, file=outfile)
def test_mkdir_parents():
assert not fs.isdir("/tmp/labm8.dir/foo/bar")
fs.mkdir("/tmp/labm8.dir/foo/bar")
assert fs.isdir("/tmp/labm8.dir/foo/bar")
def test_ls_empty_dir():
fs.mkdir("/tmp/labm8.empty")
assert not fs.ls("/tmp/labm8.empty")
fs.rm("/tmp/labm8.empty")
def test_mkdir_exists():
fs.mkdir("/tmp/labm8.dir/")
assert fs.isdir("/tmp/labm8.dir/")
fs.mkdir("/tmp/labm8.dir/")
fs.mkdir("/tmp/labm8.dir/")
assert fs.isdir("/tmp/labm8.dir/")