def import_clgen_sample(session: session_t, path: Path,
cl_launchable: bool = False,
harnesses: List[cldriveParams] = [],
delete: bool = False) -> None:
src = fs.read_file(path)
hash_ = crypto.sha1_str(src)
dupe = s.query(CLgenProgram).filter(CLgenProgram.hash == hash_).first()
if dupe:
print(f"warning: ignoring duplicate file {path}")
elif not len(src):
print(f"warning: ignoring empty file {path}")
else:
program = CLgenProgram(
hash=hash_,
runtime=len(src) / CLGEN_INFERENCE_CPS,
src=src,
linecount=len(src.split('\n')),
cl_launchable=cl_launchable)
s.add(program)
s.commit()
# Make test harnesses, if required
if harnesses:
env = cldrive.make_env()
for params in harnesses:
testcase = get_or_create(
s, CLgenTestCase, program_id=program.id, params_id=params.id)
s.flush()
clgen_mkharness.mkharness(s, env, testcase)
if delete:
fs.rm(path)
def __init__(self, generator: Generators.column_t, generation_time: float,
src: str):
self.generator = generator
self.sha1 = crypto.sha1_str(src)
self.date = datetime.datetime.utcnow()
self.generation_time = generation_time
self.linecount = len(src.split("\n"))
self.charcount = len(src)
self.src = src
def from_str(session: session_t, string: str) -> 'Stderr':
string = Stderr._escape(string)
sha1 = crypto.sha1_str(string)
stderr = get_or_add(session,
Stderr,
sha1=sha1,
linecount=len(string.split("\n")),
charcount=len(string),
truncated=len(string) > Stderr.MAX_CHARS,
stderr=string[:Stderr.MAX_CHARS])
return stderr
def from_str(session: session_t, string: str) -> str:
"""
Instantiate a Stdout object
"""
# Strip the noise
string = Stdout._escape(string)
stdout = get_or_add(session,
Stdout,
sha1=crypto.sha1_str(string),
stdout=string)
return stdout
while True:
# get the next batch of programs to run
if not len(inbox):
next_batch()
# we have no programs to run
if not len(inbox):
break
# get next program to run
program = inbox.popleft()
status, runtime, stderr_ = build_with_clang(program, clang)
# create new result
hash_ = crypto.sha1_str(stderr_)
q = s.query(tables.clang_stderrs.id) \
.filter(tables.clang_stderrs.hash == hash_) \
.first()
if q:
stderr_id = q[0]
else:
stderr_id = create_stderr(s, tables, stderr_).id
result = tables.clangs(program_id=program.id,
clang=args.clang,
status=status,
runtime=runtime,
stderr_id=stderr_id)
def hash_key(key):
"""
Convert a key to a filename by hashing its value.
"""
return crypto.sha1_str(json.dumps(key, sort_keys=True))
#!/usr/bin/env python3.6
import sys
from phd.lib.labm8 import crypto
from phd.lib.labm8 import fs
from progressbar import ProgressBar
if __name__ == "__main__":
inpath = sys.argv[1]
outdir = sys.argv[2]
print(f"reading from {inpath} into {outdir}")
assert fs.isfile(inpath)
assert not fs.exists(outdir) or fs.isdir(outdir)
fs.mkdir(outdir)
with open(inpath) as infile:
text = infile.read()
kernels = text.split("// ==== START SAMPLE ====")
kernels = [kernel.strip() for kernel in kernels if kernel.strip()]
print(len(kernels), "kernels")
sha1s = [crypto.sha1_str(kernel) for kernel in kernels]
for kernel, sha1 in ProgressBar()(list(zip(kernels, sha1s))):
with open(f"{outdir}/{sha1}.txt", "w") as outfile:
print(kernel, file=outfile)
def flush():
if args.commit:
s.commit()
while len(to_del):
fs.rm(to_del.popleft())
print("Importing CLgen programs ...")
paths = [p for p in Path("export/clgen/program").iterdir()]
for i, path in enumerate(ProgressBar()(paths)):
with open(path) as infile:
data = json.loads(infile.read())
new_id = s.query(CLgenProgram.id) \
.filter(CLgenProgram.hash == crypto.sha1_str(data["src"])).scalar()
idx = CLgenProgramTranslation(
old_id=data["id"],
new_id=new_id)
s.add(idx)
to_del.append(path)
if i and not i % 1000:
flush()
flush()
PROGRAMS = dict((old_id, new_id) for old_id, new_id in
s.query(CLgenProgramTranslation.old_id,
CLgenProgramTranslation.new_id).all())
try:
runtime, status, stdout, stderr = drive_testcase(
session, testcase, env, platform_id, device_id)
# assert that executed params match expected
if stderr != '<-- UTF-ERROR -->':
verify_params(platform=args.platform, device=args.device,
optimizations=testcase.params.optimizations,
global_size=testcase.params.gsize, local_size=testcase.params.lsize,
stderr=stderr)
# create new result
stdout_ = util.escape_stdout(stdout)
stdout = get_or_create(
session, CLgenStdout,
hash=crypto.sha1_str(stdout_), stdout=stdout_)
stderr_ = util.escape_stderr(stderr)
stderr = get_or_create(
session, CLgenStderr,
hash=crypto.sha1_str(stderr_), stderr=stderr_)
session.flush()
result = CLgenResult(
testbed_id=testbed.id,
testcase_id=testcase.id,
status=status,
runtime=runtime,
stdout_id=stdout.id,
stderr_id=stderr.id,
outcome=analyze.get_cldrive_outcome(status, runtime, stderr_))
def test_sha1_empty_str():
assert ("da39a3ee5e6b4b0d3255bfef95601890afd80709" == crypto.sha1_str(""))
def test_sha1_hello_world():
assert ("0a0a9f2a6772942557ab5355d76af442f8f65e01" == crypto.sha1_str(
"Hello, World!"))