コード例 #1
0
ファイル: clgen_fetch.py プロジェクト: BeauJoh/phd
def import_clgen_sample(session: session_t, path: Path,
                        cl_launchable: bool = False,
                        harnesses: List[cldriveParams] = [],
                        delete: bool = False) -> None:
  src = fs.read_file(path)
  hash_ = crypto.sha1_str(src)

  dupe = s.query(CLgenProgram).filter(CLgenProgram.hash == hash_).first()

  if dupe:
    print(f"warning: ignoring duplicate file {path}")
  elif not len(src):
    print(f"warning: ignoring empty file {path}")
  else:
    program = CLgenProgram(
        hash=hash_,
        runtime=len(src) / CLGEN_INFERENCE_CPS,
        src=src,
        linecount=len(src.split('\n')),
        cl_launchable=cl_launchable)
    s.add(program)
    s.commit()

    # Make test harnesses, if required
    if harnesses:
      env = cldrive.make_env()
      for params in harnesses:
        testcase = get_or_create(
            s, CLgenTestCase, program_id=program.id, params_id=params.id)
        s.flush()
        clgen_mkharness.mkharness(s, env, testcase)

    if delete:
      fs.rm(path)
コード例 #2
0
ファイル: db.py プロジェクト: BeauJoh/phd
 def __init__(self, generator: Generators.column_t, generation_time: float,
              src: str):
     self.generator = generator
     self.sha1 = crypto.sha1_str(src)
     self.date = datetime.datetime.utcnow()
     self.generation_time = generation_time
     self.linecount = len(src.split("\n"))
     self.charcount = len(src)
     self.src = src
コード例 #3
0
ファイル: db.py プロジェクト: BeauJoh/phd
    def from_str(session: session_t, string: str) -> 'Stderr':
        string = Stderr._escape(string)
        sha1 = crypto.sha1_str(string)

        stderr = get_or_add(session,
                            Stderr,
                            sha1=sha1,
                            linecount=len(string.split("\n")),
                            charcount=len(string),
                            truncated=len(string) > Stderr.MAX_CHARS,
                            stderr=string[:Stderr.MAX_CHARS])
        return stderr
コード例 #4
0
ファイル: db.py プロジェクト: BeauJoh/phd
    def from_str(session: session_t, string: str) -> str:
        """
    Instantiate a Stdout object
    """
        # Strip the noise
        string = Stdout._escape(string)

        stdout = get_or_add(session,
                            Stdout,
                            sha1=crypto.sha1_str(string),
                            stdout=string)
        return stdout
コード例 #5
0
ファイル: build_a_clang.py プロジェクト: BeauJoh/phd
                while True:
                    # get the next batch of programs to run
                    if not len(inbox):
                        next_batch()
                    # we have no programs to run
                    if not len(inbox):
                        break

                    # get next program to run
                    program = inbox.popleft()

                    status, runtime, stderr_ = build_with_clang(program, clang)

                    # create new result
                    hash_ = crypto.sha1_str(stderr_)
                    q = s.query(tables.clang_stderrs.id) \
                      .filter(tables.clang_stderrs.hash == hash_) \
                      .first()

                    if q:
                        stderr_id = q[0]
                    else:
                        stderr_id = create_stderr(s, tables, stderr_).id

                    result = tables.clangs(program_id=program.id,
                                           clang=args.clang,
                                           status=status,
                                           runtime=runtime,
                                           stderr_id=stderr_id)
コード例 #6
0
ファイル: cache.py プロジェクト: BeauJoh/phd
def hash_key(key):
    """
  Convert a key to a filename by hashing its value.
  """
    return crypto.sha1_str(json.dumps(key, sort_keys=True))
コード例 #7
0
#!/usr/bin/env python3.6

import sys
from phd.lib.labm8 import crypto
from phd.lib.labm8 import fs
from progressbar import ProgressBar


if __name__ == "__main__":
  inpath = sys.argv[1]
  outdir = sys.argv[2]
  print(f"reading from {inpath} into {outdir}")

  assert fs.isfile(inpath)
  assert not fs.exists(outdir) or fs.isdir(outdir)
  fs.mkdir(outdir)

  with open(inpath) as infile:
    text = infile.read()

  kernels = text.split("// ==== START SAMPLE ====")
  kernels = [kernel.strip() for kernel in kernels if kernel.strip()]
  print(len(kernels), "kernels")

  sha1s = [crypto.sha1_str(kernel) for kernel in kernels]
  for kernel, sha1 in ProgressBar()(list(zip(kernels, sha1s))):
    with open(f"{outdir}/{sha1}.txt", "w") as outfile:
      print(kernel, file=outfile)
コード例 #8
0
ファイル: missing-stderrs.py プロジェクト: BeauJoh/phd
    def flush():
      if args.commit:
        s.commit()
        while len(to_del):
          fs.rm(to_del.popleft())


    print("Importing CLgen programs ...")
    paths = [p for p in Path("export/clgen/program").iterdir()]
    for i, path in enumerate(ProgressBar()(paths)):
      with open(path) as infile:
        data = json.loads(infile.read())

      new_id = s.query(CLgenProgram.id) \
        .filter(CLgenProgram.hash == crypto.sha1_str(data["src"])).scalar()

      idx = CLgenProgramTranslation(
          old_id=data["id"],
          new_id=new_id)
      s.add(idx)

      to_del.append(path)
      if i and not i % 1000:
        flush()
    flush()

    PROGRAMS = dict((old_id, new_id) for old_id, new_id in
                    s.query(CLgenProgramTranslation.old_id,
                            CLgenProgramTranslation.new_id).all())
コード例 #9
0
        try:
          runtime, status, stdout, stderr = drive_testcase(
              session, testcase, env, platform_id, device_id)

          # assert that executed params match expected
          if stderr != '<-- UTF-ERROR -->':
            verify_params(platform=args.platform, device=args.device,
                          optimizations=testcase.params.optimizations,
                          global_size=testcase.params.gsize, local_size=testcase.params.lsize,
                          stderr=stderr)

          # create new result
          stdout_ = util.escape_stdout(stdout)
          stdout = get_or_create(
              session, CLgenStdout,
              hash=crypto.sha1_str(stdout_), stdout=stdout_)

          stderr_ = util.escape_stderr(stderr)
          stderr = get_or_create(
              session, CLgenStderr,
              hash=crypto.sha1_str(stderr_), stderr=stderr_)
          session.flush()

          result = CLgenResult(
              testbed_id=testbed.id,
              testcase_id=testcase.id,
              status=status,
              runtime=runtime,
              stdout_id=stdout.id,
              stderr_id=stderr.id,
              outcome=analyze.get_cldrive_outcome(status, runtime, stderr_))
コード例 #10
0
def test_sha1_empty_str():
    assert ("da39a3ee5e6b4b0d3255bfef95601890afd80709" == crypto.sha1_str(""))
コード例 #11
0
def test_sha1_hello_world():
    assert ("0a0a9f2a6772942557ab5355d76af442f8f65e01" == crypto.sha1_str(
        "Hello, World!"))