def test_duplicate_testcase_testbed_ignored(session): """Test that result is ignored if testbed and testcase are not unique.""" proto = deepsmith_pb2.Result( testcase=deepsmith_pb2.Testcase( toolchain='cpp', generator=deepsmith_pb2.Generator(name='generator'), harness=deepsmith_pb2.Harness(name='harness'), inputs={ 'src': 'void main() {}', 'data': '[1,2]', }, invariant_opts={ 'config': 'opt', }, profiling_events=[ deepsmith_pb2.ProfilingEvent( client='localhost', type='generate', duration_ms=100, event_start_epoch_ms=1123123123, ), ]), testbed=deepsmith_pb2.Testbed( toolchain='cpp', name='clang', opts={'arch': 'x86_64'}, ), returncode=0, outputs={'stdout': 'Hello, world!'}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client='localhost', type='exec', duration_ms=100, event_start_epoch_ms=1123123123, ), ], outcome=deepsmith_pb2.Result.PASS, ) r1 = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto) session.add(r1) session.flush() # Attempt to add a new result which is identical to the first in all fields # except for the outputs. proto.outputs['stdout'] = '!' r2 = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto) session.add(r2) session.flush() # Check that only one result was added. assert session.query(deeplearning.deepsmith.result.Result).count() == 1 # Check that only the first result was added. r3 = session.query(deeplearning.deepsmith.result.Result).first() assert r3.outputs['stdout'] == 'Hello, world!'
def test_Generator_GetOrAdd_ToProto_equivalence(session): proto_in = deepsmith_pb2.Result( testcase=deepsmith_pb2.Testcase( toolchain="cpp", generator=deepsmith_pb2.Generator(name="generator"), harness=deepsmith_pb2.Harness(name="harness"), inputs={"src": "void main() {}", "data": "[1,2]",}, invariant_opts={"config": "opt",}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="generate", duration_ms=100, event_start_epoch_ms=1123123123, ), deepsmith_pb2.ProfilingEvent( client="localhost", type="foo", duration_ms=100, event_start_epoch_ms=1123123123, ), ], ), testbed=deepsmith_pb2.Testbed( toolchain="cpp", name="clang", opts={"arch": "x86_64", "build": "debug+assert",}, ), returncode=0, outputs={"stdout": "Hello, world!", "stderr": "",}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="exec", duration_ms=500, event_start_epoch_ms=1123123123, ), deepsmith_pb2.ProfilingEvent( client="localhost", type="overhead", duration_ms=100, event_start_epoch_ms=1123123123, ), ], outcome=deepsmith_pb2.Result.PASS, ) result = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto_in) # NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs. session.flush() proto_out = result.ToProto() assert proto_in == proto_out proto_out.ClearField("outputs") assert proto_in != proto_out # Sanity check.
def test_Testcase_GetOrAdd(session): proto = deepsmith_pb2.Testcase( toolchain="cpp", generator=deepsmith_pb2.Generator(name="generator", ), harness=deepsmith_pb2.Harness(name="harness", ), inputs={ "src": "void main() {}", "data": "[1,2]" }, invariant_opts={"config": "opt"}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="generate", duration_ms=100, event_start_epoch_ms=1021312312, ), deepsmith_pb2.ProfilingEvent( client="localhost", type="foo", duration_ms=100, event_start_epoch_ms=1230812312, ), ], ) testcase = deeplearning.deepsmith.testcase.Testcase.GetOrAdd( session, proto) # NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs. session.flush() assert testcase.toolchain.string == "cpp" assert testcase.generator.name == "generator" assert testcase.harness.name == "harness" assert len(testcase.inputset) == 2 assert len(testcase.inputs) == 2 assert testcase.inputs["src"] == "void main() {}" assert testcase.inputs["data"] == "[1,2]" assert len(testcase.invariant_optset) == 1 assert len(testcase.invariant_opts) == 1 assert testcase.invariant_opts["config"] == "opt" assert testcase.profiling_events[0].client.string == "localhost" assert testcase.profiling_events[0].type.string == "generate" assert testcase.profiling_events[0].duration_ms == 100 assert testcase.profiling_events[ 0].event_start == labdate.DatetimeFromMillisecondsTimestamp(1021312312) assert testcase.profiling_events[1].client.string == "localhost" assert testcase.profiling_events[1].type.string == "foo" assert testcase.profiling_events[1].duration_ms == 100 assert testcase.profiling_events[ 1].event_start == labdate.DatetimeFromMillisecondsTimestamp(1230812312)
def test_duplicate_testcase_testbed_ignored(session): """Test that result is ignored if testbed and testcase are not unique.""" proto = deepsmith_pb2.Result( testcase=deepsmith_pb2.Testcase( toolchain="cpp", generator=deepsmith_pb2.Generator(name="generator"), harness=deepsmith_pb2.Harness(name="harness"), inputs={"src": "void main() {}", "data": "[1,2]",}, invariant_opts={"config": "opt",}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="generate", duration_ms=100, event_start_epoch_ms=1123123123, ), ], ), testbed=deepsmith_pb2.Testbed( toolchain="cpp", name="clang", opts={"arch": "x86_64"}, ), returncode=0, outputs={"stdout": "Hello, world!"}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="exec", duration_ms=100, event_start_epoch_ms=1123123123, ), ], outcome=deepsmith_pb2.Result.PASS, ) r1 = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto) session.add(r1) session.flush() # Attempt to add a new result which is identical to the first in all fields # except for the outputs. proto.outputs["stdout"] = "!" r2 = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto) session.add(r2) session.flush() # Check that only one result was added. assert session.query(deeplearning.deepsmith.result.Result).count() == 1 # Check that only the first result was added. r3 = session.query(deeplearning.deepsmith.result.Result).first() assert r3.outputs["stdout"] == "Hello, world!"
def test_Testcase_GetOrAdd(session): proto = deepsmith_pb2.Testcase( toolchain='cpp', generator=deepsmith_pb2.Generator(name='generator', ), harness=deepsmith_pb2.Harness(name='harness', ), inputs={ 'src': 'void main() {}', 'data': '[1,2]' }, invariant_opts={'config': 'opt'}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client='localhost', type='generate', duration_ms=100, event_start_epoch_ms=1021312312, ), deepsmith_pb2.ProfilingEvent( client='localhost', type='foo', duration_ms=100, event_start_epoch_ms=1230812312, ), ]) testcase = deeplearning.deepsmith.testcase.Testcase.GetOrAdd( session, proto) # NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs. session.flush() assert testcase.toolchain.string == 'cpp' assert testcase.generator.name == 'generator' assert testcase.harness.name == 'harness' assert len(testcase.inputset) == 2 assert len(testcase.inputs) == 2 assert testcase.inputs['src'] == 'void main() {}' assert testcase.inputs['data'] == '[1,2]' assert len(testcase.invariant_optset) == 1 assert len(testcase.invariant_opts) == 1 assert testcase.invariant_opts['config'] == 'opt' assert testcase.profiling_events[0].client.string == 'localhost' assert testcase.profiling_events[0].type.string == 'generate' assert testcase.profiling_events[0].duration_ms == 100 assert (testcase.profiling_events[0].event_start == labdate.DatetimeFromMillisecondsTimestamp(1021312312)) assert testcase.profiling_events[1].client.string == 'localhost' assert testcase.profiling_events[1].type.string == 'foo' assert testcase.profiling_events[1].duration_ms == 100 assert (testcase.profiling_events[1].event_start == labdate.DatetimeFromMillisecondsTimestamp(1230812312))
def test_Generator_GetOrAdd_ToProto_equivalence(session): proto_in = deepsmith_pb2.Testcase( toolchain="cpp", generator=deepsmith_pb2.Generator(name="generator", ), harness=deepsmith_pb2.Harness(name="harness", ), inputs={ "src": "void main() {}", "data": "[1,2]" }, invariant_opts={"config": "opt"}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="generate", duration_ms=100, event_start_epoch_ms=101231231, ), ], ) testcase = deeplearning.deepsmith.testcase.Testcase.GetOrAdd( session, proto_in) # NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs. session.flush() proto_out = testcase.ToProto() assert proto_in == proto_out proto_out.ClearField("toolchain") assert proto_in != proto_out # Sanity check.
def test_Generator_GetOrAdd_ToProto_equivalence(session): proto_in = deepsmith_pb2.Testcase( toolchain='cpp', generator=deepsmith_pb2.Generator(name='generator', ), harness=deepsmith_pb2.Harness(name='harness', ), inputs={ 'src': 'void main() {}', 'data': '[1,2]' }, invariant_opts={'config': 'opt'}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client='localhost', type='generate', duration_ms=100, event_start_epoch_ms=101231231, ), ]) testcase = deeplearning.deepsmith.testcase.Testcase.GetOrAdd( session, proto_in) # NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs. session.flush() proto_out = testcase.ToProto() assert proto_in == proto_out proto_out.ClearField('toolchain') assert proto_in != proto_out # Sanity check.
def _ExportSolidityTestcases(cursor, start_id, proto_dir): batch_size = 1000 testcase_id = start_id while True: cursor.execute( """ SELECT testcases.id, programs.generator, programs.date, programs.generation_time, programs.src, testcases.harness, testcases.timeout FROM testcases LEFT JOIN programs ON testcases.program_id = programs.id WHERE testcases.id >= %s AND testcases.id NOT IN ( SELECT testcase_id FROM results ) ORDER BY testcases.id LIMIT %s """, (testcase_id, batch_size)) i = 0 for row in cursor: i += 1 (testcase_id, generator_id, program_date, program_generation_time, program_src, harness_id, harness_timeout) = row proto = deepsmith_pb2.Testcase( toolchain='solidity', generator=_GetSolidityGenerator(generator_id), harness=deepsmith_pb2.Harness( name='solc', opts={ 'timeout_seconds': str(int(harness_timeout)), 'url': 'https://github.com/ChrisCummins/dsmith/blob/5181c7c95575d428b5144a25549e5a5a55a3da31/dsmith/sol/harnesses.py#L117', }, ), inputs={ "src": program_src, }, invariant_opts={}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="cc1", type="generation", duration_ms=int(program_generation_time * 1000), event_start_epoch_ms=dateutil.MillisecondsTimestamp( program_date), ), ]) with open(proto_dir / 'sol' / 'testcases' / str(testcase_id), 'wb') as f: f.write(proto.SerializeToString()) if i < batch_size: return
def ToProto(self) -> deepsmith_pb2.ProfilingEvent: """Create protocol buffer representation. Returns: A ProfilingEvent message. """ proto = deepsmith_pb2.ProfilingEvent() return self.SetProto(proto)
def test_duplicate_testcases_ignored(session): """Test that testcases are only added if they are unique.""" proto = deepsmith_pb2.Testcase( toolchain="cpp", generator=deepsmith_pb2.Generator(name="generator"), harness=deepsmith_pb2.Harness(name="harness"), inputs={ "src": "void main() {}", "data": "[1,2]" }, invariant_opts={"config": "opt"}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="generate", duration_ms=100, event_start_epoch_ms=1021312312, ), deepsmith_pb2.ProfilingEvent( client="localhost", type="foo", duration_ms=100, event_start_epoch_ms=1230812312, ), ], ) t1 = deeplearning.deepsmith.testcase.Testcase.GetOrAdd(session, proto) session.add(t1) session.flush() # Attempt to add a new testcase which is identical to the first in all fields # except for the profiling events. proto.profiling_events[0].duration_ms = -1 t2 = deeplearning.deepsmith.testcase.Testcase.GetOrAdd(session, proto) session.add(t2) session.flush() # Check that only one testcase was added. assert session.query(deeplearning.deepsmith.testcase.Testcase).count() == 1 # Check that only the first testcase was added. t3 = session.query(deeplearning.deepsmith.testcase.Testcase).first() assert t3.profiling_events[0].duration_ms == 100 assert t3.profiling_events[1].duration_ms == 100
def test_duplicate_testcases_ignored(session): """Test that testcases are only added if they are unique.""" proto = deepsmith_pb2.Testcase( toolchain='cpp', generator=deepsmith_pb2.Generator(name='generator'), harness=deepsmith_pb2.Harness(name='harness'), inputs={ 'src': 'void main() {}', 'data': '[1,2]' }, invariant_opts={'config': 'opt'}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client='localhost', type='generate', duration_ms=100, event_start_epoch_ms=1021312312, ), deepsmith_pb2.ProfilingEvent( client='localhost', type='foo', duration_ms=100, event_start_epoch_ms=1230812312, ), ]) t1 = deeplearning.deepsmith.testcase.Testcase.GetOrAdd(session, proto) session.add(t1) session.flush() # Attempt to add a new testcase which is identical to the first in all fields # except for the profiling events. proto.profiling_events[0].duration_ms = -1 t2 = deeplearning.deepsmith.testcase.Testcase.GetOrAdd(session, proto) session.add(t2) session.flush() # Check that only one testcase was added. assert session.query(deeplearning.deepsmith.testcase.Testcase).count() == 1 # Check that only the first testcase was added. t3 = session.query(deeplearning.deepsmith.testcase.Testcase).first() assert t3.profiling_events[0].duration_ms == 100 assert t3.profiling_events[1].duration_ms == 100
def _AddRandomNewTestcase(session): deeplearning.deepsmith.testcase.Testcase.GetOrAdd( session, deepsmith_pb2.Testcase( toolchain=str(random.random()), generator=deepsmith_pb2.Generator( name=str(random.random()), opts={ str(random.random()): str(random.random()), str(random.random()): str(random.random()), str(random.random()): str(random.random()), }, ), harness=deepsmith_pb2.Harness( name=str(random.random()), opts={ str(random.random()): str(random.random()), str(random.random()): str(random.random()), str(random.random()): str(random.random()), }, ), inputs={ str(random.random()): str(random.random()), str(random.random()): str(random.random()), str(random.random()): str(random.random()), }, invariant_opts={ str(random.random()): str(random.random()), str(random.random()): str(random.random()), str(random.random()): str(random.random()), }, profiling_events=[ deepsmith_pb2.ProfilingEvent( client=str(random.random()), type=str(random.random()), duration_ms=int(random.random() * 1000), event_start_epoch_ms=int(random.random() * 1000000), ), ], ), ) session.flush()
def _AddExistingTestcase(session): deeplearning.deepsmith.testcase.Testcase.GetOrAdd( session, deepsmith_pb2.Testcase( toolchain="cpp", generator=deepsmith_pb2.Generator( name="name", opts={ "a": "a", "b": "b", "c": "c", }, ), harness=deepsmith_pb2.Harness( name="name", opts={ "a": "a", "b": "b", "c": "c", }, ), inputs={ "src": "void main() {}", "data": "[1,2]", "copt": "-DNDEBUG", }, invariant_opts={ "config": "opt", "working_dir": "/tmp", "units": "nanoseconds", }, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="generate", duration_ms=100, event_start_epoch_ms=101231231, ), ], ), ) session.flush()
def _AddExistingTestcase(session): deeplearning.deepsmith.testcase.Testcase.GetOrAdd( session, deepsmith_pb2.Testcase(toolchain='cpp', generator=deepsmith_pb2.Generator( name='name', opts={ 'a': 'a', 'b': 'b', 'c': 'c', }, ), harness=deepsmith_pb2.Harness( name='name', opts={ 'a': 'a', 'b': 'b', 'c': 'c', }, ), inputs={ 'src': 'void main() {}', 'data': '[1,2]', 'copt': '-DNDEBUG', }, invariant_opts={ 'config': 'opt', 'working_dir': '/tmp', 'units': 'nanoseconds', }, profiling_events=[ deepsmith_pb2.ProfilingEvent( client='localhost', type='generate', duration_ms=100, event_start_epoch_ms=101231231, ), ])) session.flush()
def _ExportSolidityResults(cursor, start_id, proto_dir): batch_size = 1000 result_id = start_id while True: cursor.execute( """ SELECT results.id, platforms.platform, platforms.version, platforms.host, testbeds.optimizations, programs.generator, programs.date, programs.generation_time, programs.src, testcases.harness, testcases.timeout, results.date, results.returncode, results.runtime, stdouts.stdout, stderrs.stderr FROM results LEFT JOIN testbeds ON results.testbed_id = testbeds.id LEFT JOIN platforms ON testbeds.platform_id = platforms.id LEFT JOIN testcases ON results.testcase_id = testcases.id LEFT JOIN programs ON testcases.program_id = programs.id LEFT JOIN stdouts ON results.stdout_id = stdouts.id LEFT JOIN stderrs ON results.stderr_id = stderrs.id WHERE results.id >= %s ORDER BY results.id LIMIT %s """, (result_id, batch_size)) i = 0 for row in cursor: i += 1 (result_id, platform_name, platform_version, host_os, optimizations, generator_id, program_date, program_generation_time, program_src, harness_id, harness_timeout, result_date, returncode, runtime, stdout, stderr) = row assert harness_id == 2 proto = deepsmith_pb2.Result( testcase=deepsmith_pb2.Testcase( toolchain='solidity', generator=_GetSolidityGenerator(generator_id), harness=deepsmith_pb2.Harness( name='solc', opts={ 'timeout_seconds': str(int(harness_timeout)), 'url': 'https://github.com/ChrisCummins/dsmith/blob/5181c7c95575d428b5144a25549e5a5a55a3da31/dsmith/sol/harnesses.py#L117', }, ), inputs={ "src": program_src, }, invariant_opts={}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="cc1", type="generation", duration_ms=int(program_generation_time * 1000), event_start_epoch_ms=dateutil. MillisecondsTimestamp(program_date), ), ]), testbed=deepsmith_pb2.Testbed( toolchain='solidity', name=platform_name, opts={ 'version': platform_version, 'optimizations': 'enabled' if optimizations else 'disabled', }, ), returncode=returncode, outputs={ "stdout": stdout, "stderr": stderr, }, profiling_events=[ deepsmith_pb2.ProfilingEvent( client='cc1', type="runtime", duration_ms=int(runtime * 1000), event_start_epoch_ms=dateutil.MillisecondsTimestamp( result_date), ), ], ) with open(proto_dir / 'sol' / 'results' / str(result_id), 'wb') as f: f.write(proto.SerializeToString()) if i < batch_size: return
def _ExportOpenCLResults(cursor, start_id, proto_dir): batch_size = 1000 result_id = start_id while True: cursor.execute( """ SELECT results.id, platforms.platform, platforms.device, platforms.driver, platforms.opencl, platforms.devtype, platforms.host, testbeds.optimizations, programs.generator, programs.date, programs.generation_time, programs.src, testcases.harness, testcases.timeout, results.date, results.returncode, results.runtime, stdouts.stdout, stderrs.stderr, stderrs.truncated, threads.gsize_x, threads.gsize_y, threads.gsize_z, threads.lsize_x, threads.lsize_y, threads.lsize_z, clsmith_testcase_metas.oclverified, dsmith_testcase_metas.gpuverified, dsmith_testcase_metas.oclverified, dsmith_program_metas.contains_floats, dsmith_program_metas.vector_inputs, dsmith_program_metas.compiler_warnings FROM results LEFT JOIN testbeds ON results.testbed_id = testbeds.id LEFT JOIN platforms ON testbeds.platform_id = platforms.id LEFT JOIN testcases ON results.testcase_id = testcases.id LEFT JOIN programs ON testcases.program_id = programs.id LEFT JOIN threads ON testcases.threads_id = threads.id LEFT JOIN stdouts ON results.stdout_id = stdouts.id LEFT JOIN stderrs ON results.stderr_id = stderrs.id LEFT JOIN clsmith_testcase_metas ON testcases.id=clsmith_testcase_metas.id LEFT JOIN dsmith_testcase_metas ON testcases.id=dsmith_testcase_metas.id LEFT JOIN dsmith_program_metas ON programs.id=dsmith_program_metas.id WHERE results.id >= %s ORDER BY results.id LIMIT %s """, (result_id, batch_size)) i = 0 for row in cursor: i += 1 (result_id, platform_name, device_name, driver_version, opencl_version, devtype, host_os, cl_opt, generator_id, program_date, program_generation_time, program_src, harness_id, harness_timeout, result_date, returncode, runtime, stdout, stderr, truncated_stderr, gsize_x, gsize_y, gsize_z, lsize_x, lsize_y, lsize_z, clsmith_oclverified, dsmith_gpuverified, dsmith_oclverified, dsmith_program_contains_floats, dsmith_program_vector_inputs, dsmith_program_compiler_warnings) = row inputs = { 'src': program_src, } if harness_id != -1: inputs['gsize'] = f'{gsize_x},{gsize_y},{gsize_z}' inputs['lsize'] = f'{lsize_x},{lsize_y},{lsize_z}' testbed_name = OPENCL_DEVICE_MAP[device_name] testbed_opts = {} _SetIf(testbed_opts, 'opencl_device', device_name.strip()) _SetIf(testbed_opts, 'opencl_version', opencl_version.strip()) _SetIf(testbed_opts, 'host', HOSTS_MAP.get(host_os, host_os)) if testbed_name == "clang": _SetIf(testbed_opts, 'llvm_version', driver_version.strip()) else: _SetIf(testbed_opts, 'driver_version', driver_version.strip()) _SetIf(testbed_opts, 'opencl_devtype', OPENCL_DEVTYPE_MAP.get(devtype, devtype)) _SetIf(testbed_opts, 'opencl_platform', platform_name.strip()) _SetIf(testbed_opts, 'opencl_opt', 'enabled' if cl_opt else 'disabled') invariant_opts = {} if clsmith_oclverified == 0: invariant_opts['oclverify'] = 'fail' elif clsmith_oclverified == 1: invariant_opts['oclverify'] = 'pass' elif dsmith_oclverified == 0: invariant_opts['oclverify'] = 'fail' elif dsmith_oclverified == 1: invariant_opts['oclverify'] = 'pass' if dsmith_gpuverified == 0: invariant_opts['gpuverify'] = 'fail' elif dsmith_gpuverified == 1: invariant_opts['gpuverify'] = 'pass' if dsmith_program_contains_floats == 0: invariant_opts['kernel_uses_floats'] = 'false' elif dsmith_program_contains_floats == 1: invariant_opts['kernel_uses_floats'] = 'true' if dsmith_program_vector_inputs == 0: invariant_opts['kernel_has_vector_inputs'] = 'false' elif dsmith_program_vector_inputs == 1: invariant_opts['kernel_has_vector_inputs'] = 'true' if dsmith_program_compiler_warnings == 0: invariant_opts['kernel_throws_compiler_warning'] = 'false' elif dsmith_program_compiler_warnings == 1: invariant_opts['kernel_throws_compiler_warning'] = 'true' testbed = deepsmith_pb2.Testbed( toolchain='opencl', name=testbed_name, opts=testbed_opts, ) proto = deepsmith_pb2.Result( testcase=deepsmith_pb2.Testcase( toolchain="opencl", generator=_GetOpenCLGenerator(generator_id), harness=_GetOpenCLHarness(harness_id, harness_timeout), inputs=inputs, invariant_opts=invariant_opts, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="cc1", type="generation", duration_ms=int(program_generation_time * 1000), event_start_epoch_ms=dateutil. MillisecondsTimestamp(program_date), ), ]), testbed=testbed, returncode=returncode, outputs={ "stdout": stdout, "stderr": stderr, }, profiling_events=[ deepsmith_pb2.ProfilingEvent( client={ 'Ubuntu 16.04 64bit': 'cc1', 'CentOS Linux 7.1.1503 64bit': 'fuji', 'openSUSE 13.1 64bit': 'kobol', }[host_os], type="runtime", duration_ms=int(runtime * 1000), event_start_epoch_ms=dateutil.MillisecondsTimestamp( result_date), ), ], ) with open(proto_dir / 'opencl' / 'results' / str(result_id), 'wb') as f: f.write(proto.SerializeToString()) if i < batch_size: return
def _ExportOpenCLTestcases(cursor, start_id, proto_dir): batch_size = 1000 testcase_id = start_id while True: cursor.execute( """ SELECT testcases.id, programs.generator, programs.date, programs.generation_time, programs.src, testcases.harness, testcases.timeout, threads.gsize_x, threads.gsize_y, threads.gsize_z, threads.lsize_x, threads.lsize_y, threads.lsize_z, clsmith_testcase_metas.oclverified, dsmith_testcase_metas.gpuverified, dsmith_testcase_metas.oclverified, dsmith_program_metas.contains_floats, dsmith_program_metas.vector_inputs, dsmith_program_metas.compiler_warnings FROM testcases LEFT JOIN programs ON testcases.program_id = programs.id LEFT JOIN threads ON testcases.threads_id = threads.id LEFT JOIN clsmith_testcase_metas ON testcases.id=clsmith_testcase_metas.id LEFT JOIN dsmith_testcase_metas ON testcases.id=dsmith_testcase_metas.id LEFT JOIN dsmith_program_metas ON programs.id=dsmith_program_metas.id WHERE testcases.id >= %s AND testcases.id NOT IN ( SELECT testcase_id FROM results ) ORDER BY testcases.id LIMIT %s """, (testcase_id, batch_size)) i = 0 for row in cursor: i += 1 (testcase_id, generator_id, program_date, program_generation_time, program_src, harness_id, harness_timeout, gsize_x, gsize_y, gsize_z, lsize_x, lsize_y, lsize_z, clsmith_oclverified, dsmith_gpuverified, dsmith_oclverified, dsmith_program_contains_floats, dsmith_program_vector_inputs, dsmith_program_compiler_warnings) = row inputs = { "src": program_src, } if harness_id != -1: inputs["gsize"] = f"{gsize_x},{gsize_y},{gsize_z}" inputs["lsize"] = f"{lsize_x},{lsize_y},{lsize_z}" invariant_opts = {} if clsmith_oclverified == 0: invariant_opts['oclverify'] = 'fail' elif clsmith_oclverified == 1: invariant_opts['oclverify'] = 'pass' elif dsmith_oclverified == 0: invariant_opts['oclverify'] = 'fail' elif dsmith_oclverified == 1: invariant_opts['oclverify'] = 'pass' if dsmith_gpuverified == 0: invariant_opts['gpuverify'] = 'fail' elif dsmith_gpuverified == 1: invariant_opts['gpuverify'] = 'pass' if dsmith_program_contains_floats == 0: invariant_opts['kernel_uses_floats'] = 'false' elif dsmith_program_contains_floats == 1: invariant_opts['kernel_uses_floats'] = 'true' if dsmith_program_vector_inputs == 0: invariant_opts['kernel_has_vector_inputs'] = 'false' elif dsmith_program_vector_inputs == 1: invariant_opts['kernel_has_vector_inputs'] = 'true' if dsmith_program_compiler_warnings == 0: invariant_opts['kernel_throws_compiler_warning'] = 'false' elif dsmith_program_compiler_warnings == 1: invariant_opts['kernel_throws_compiler_warning'] = 'true' proto = deepsmith_pb2.Testcase( toolchain="opencl", generator=_GetOpenCLGenerator(generator_id), harness=_GetOpenCLHarness(harness_id, harness_timeout), inputs=inputs, invariant_opts=invariant_opts, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="cc1", type="generation", duration_ms=int(program_generation_time * 1000), event_start_epoch_ms=dateutil.MillisecondsTimestamp( program_date), ), ]) with open(proto_dir / 'opencl' / 'testcases' / str(testcase_id), 'wb') as f: f.write(proto.SerializeToString()) if i < batch_size: return
def test_Generator_GetOrAdd_ToProto_equivalence(session): proto_in = deepsmith_pb2.Result( testcase=deepsmith_pb2.Testcase( toolchain='cpp', generator=deepsmith_pb2.Generator(name='generator'), harness=deepsmith_pb2.Harness(name='harness'), inputs={ 'src': 'void main() {}', 'data': '[1,2]', }, invariant_opts={ 'config': 'opt', }, profiling_events=[ deepsmith_pb2.ProfilingEvent( client='localhost', type='generate', duration_ms=100, event_start_epoch_ms=1123123123, ), deepsmith_pb2.ProfilingEvent( client='localhost', type='foo', duration_ms=100, event_start_epoch_ms=1123123123, ), ]), testbed=deepsmith_pb2.Testbed( toolchain='cpp', name='clang', opts={ 'arch': 'x86_64', 'build': 'debug+assert', }, ), returncode=0, outputs={ 'stdout': 'Hello, world!', 'stderr': '', }, profiling_events=[ deepsmith_pb2.ProfilingEvent( client='localhost', type='exec', duration_ms=500, event_start_epoch_ms=1123123123, ), deepsmith_pb2.ProfilingEvent( client='localhost', type='overhead', duration_ms=100, event_start_epoch_ms=1123123123, ), ], outcome=deepsmith_pb2.Result.PASS, ) result = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto_in) # NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs. session.flush() proto_out = result.ToProto() assert proto_in == proto_out proto_out.ClearField('outputs') assert proto_in != proto_out # Sanity check.
def _ExportOpenCLTestcases(cursor, start_id, proto_dir): batch_size = 1000 testcase_id = start_id while True: cursor.execute(""" SELECT testcases.id, programs.generator, programs.date, programs.generation_time, programs.src, testcases.harness, testcases.timeout, threads.gsize_x, threads.gsize_y, threads.gsize_z, threads.lsize_x, threads.lsize_y, threads.lsize_z FROM testcases LEFT JOIN programs on testcases.program_id = programs.id LEFT JOIN threads on testcases.threads_id = threads.id WHERE testcases.id >= %s AND testcases.id NOT IN ( SELECT testcase_id FROM results ) ORDER BY testcases.id LIMIT %s """, (testcase_id, batch_size)) i = 0 for row in cursor: i += 1 ( testcase_id, generator_id, program_date, program_generation_time, program_src, harness_id, harness_timeout, gsize_x, gsize_y, gsize_z, lsize_x, lsize_y, lsize_z ) = row inputs = { "src": program_src, } if harness_id != -1: inputs["gsize"] = f"{gsize_x},{gsize_y},{gsize_z}" inputs["lsize"] = f"{lsize_x},{lsize_y},{lsize_z}" proto = deepsmith_pb2.Testcase( toolchain="opencl", generator=_GetOpenCLGenerator(generator_id), harness=_GetOpenCLHarness(harness_id, harness_timeout), inputs=inputs, invariant_opts={}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="cc1", type="generation", duration_seconds=program_generation_time, date_epoch_seconds=int(program_date.strftime('%s')), ), ] ) with open(proto_dir / 'opencl' / 'testcases' / str(testcase_id), 'wb') as f: f.write(proto.SerializeToString()) if i < batch_size: return
def _ExportOpenCLResults(cursor, start_id, proto_dir): batch_size = 1000 result_id = start_id while True: cursor.execute(""" SELECT results.id, platforms.platform, platforms.device, platforms.driver, platforms.opencl, platforms.devtype, platforms.host, testbeds.optimizations, programs.generator, programs.date, programs.generation_time, programs.src, testcases.harness, testcases.timeout, results.date, results.returncode, results.runtime, stdouts.stdout, stderrs.stderr, stderrs.truncated, threads.gsize_x, threads.gsize_y, threads.gsize_z, threads.lsize_x, threads.lsize_y, threads.lsize_z FROM results LEFT JOIN testbeds ON results.testbed_id = testbeds.id LEFT JOIN platforms ON testbeds.platform_id = platforms.id LEFT JOIN testcases on results.testcase_id = testcases.id LEFT JOIN programs on testcases.program_id = programs.id LEFT JOIN threads on testcases.threads_id = threads.id LEFT JOIN stdouts on results.stdout_id = stdouts.id LEFT JOIN stderrs on results.stderr_id = stderrs.id WHERE results.id >= %s ORDER BY results.id LIMIT %s """, (result_id, batch_size)) i = 0 for row in cursor: i += 1 ( result_id, platform_name, device_name, driver_version, opencl_version, devtype, host_os, cl_opt, generator_id, program_date, program_generation_time, program_src, harness_id, harness_timeout, result_date, returncode, runtime, stdout, stderr, truncated_stderr, gsize_x, gsize_y, gsize_z, lsize_x, lsize_y, lsize_z ) = row inputs = { "src": program_src, } if harness_id != -1: inputs["gsize"] = f"{gsize_x},{gsize_y},{gsize_z}" inputs["lsize"] = f"{lsize_x},{lsize_y},{lsize_z}" testbed_name = OPENCL_DEVICE_MAP[device_name] testbed_opts = {} _SetIf(testbed_opts, 'opencl_device', device_name.strip()) _SetIf(testbed_opts, 'opencl_version', opencl_version.strip()) _SetIf(testbed_opts, 'host', HOSTS_MAP.get(host_os, host_os)) if testbed_name == "clang": _SetIf(testbed_opts, 'llvm_version', driver_version.strip()) else: _SetIf(testbed_opts, 'driver_version', driver_version.strip()) _SetIf(testbed_opts, 'opencl_devtype', OPENCL_DEVTYPE_MAP.get(devtype, devtype)) _SetIf(testbed_opts, 'opencl_platform', platform_name.strip()) _SetIf(testbed_opts, 'opencl_opt', 'enabled' if cl_opt else 'disabled') testbed = deepsmith_pb2.Testbed( toolchain='opencl', name=testbed_name, opts=testbed_opts, ) proto = deepsmith_pb2.Result( testcase=deepsmith_pb2.Testcase( toolchain="opencl", generator=_GetOpenCLGenerator(generator_id), harness=_GetOpenCLHarness(harness_id, harness_timeout), inputs=inputs, invariant_opts={}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="cc1", type="generation", duration_seconds=program_generation_time, date_epoch_seconds=int(program_date.strftime('%s')), ), ] ), testbed=testbed, returncode=returncode, outputs={ "stdout": stdout, "stderr": stderr, }, profiling_events=[ deepsmith_pb2.ProfilingEvent( client={ 'Ubuntu 16.04 64bit': 'cc1', 'CentOS Linux 7.1.1503 64bit': 'fuji', 'openSUSE 13.1 64bit': 'kobol', }[host_os], type="runtime", duration_seconds=runtime, date_epoch_seconds=int(result_date.strftime('%s')), ), ], ) with open(proto_dir / 'opencl' / 'results' / str(result_id), 'wb') as f: f.write(proto.SerializeToString()) if i < batch_size: return