def _GetOpenCLHarness(harness_id, timeout) -> deepsmith_pb2.Harness: if harness_id == -1: return deepsmith_pb2.Harness( name="clang", opts={ "timeout_seconds": str(int(timeout)), "url": "https://github.com/ChrisCummins/dsmith/blob/fd986a36a23b2a398f33d5b5852d930b462401b1/dsmith/opencl/harnesses.py#L292", }) elif harness_id == 0: return deepsmith_pb2.Harness( name="cl_launcher", opts={ "timeout_seconds": str(int(timeout)), "git_commit": "b637b31c31e0f90ef199ca492af05172400df050", "git_remote": "https://github.com/ChrisCummins/CLSmith.git", }) elif harness_id == 1: return deepsmith_pb2.Harness( name="cldrive", opts={ "timeout_seconds": str(int(timeout)), "git_commit": "9556e7112ba2bd6f79ee59eef74f0a2304efa007", "git_remote": "https://github.com/ChrisCummins/clgen.git", "version": "0.4.0.dev0", }) else: raise LookupError
def test_Harness_GetOrAdd_rollback(session): deeplearning.deepsmith.harness.Harness.GetOrAdd( session, deepsmith_pb2.Harness( name="name", opts={ "a": "1", "b": "2", }, )) assert session.query(deeplearning.deepsmith.harness.Harness).count() == 1 assert session.query( deeplearning.deepsmith.harness.HarnessOpt).count() == 2 assert (session.query( deeplearning.deepsmith.harness.HarnessOptSet).count() == 2) assert (session.query( deeplearning.deepsmith.harness.HarnessOptName).count() == 2) assert (session.query( deeplearning.deepsmith.harness.HarnessOptValue).count() == 2) session.rollback() assert session.query(deeplearning.deepsmith.harness.Harness).count() == 0 assert session.query( deeplearning.deepsmith.harness.HarnessOpt).count() == 0 assert (session.query( deeplearning.deepsmith.harness.HarnessOptSet).count() == 0) assert (session.query( deeplearning.deepsmith.harness.HarnessOptName).count() == 0) assert (session.query( deeplearning.deepsmith.harness.HarnessOptValue).count() == 0)
def test_GenerateTestcases(abc_instance_config, tempdir: pathlib.Path): """Run a tiny end-to-end test.""" generator_config = generator_pb2.ClgenGenerator( instance=abc_instance_config, testcase_skeleton=[ deepsmith_pb2.Testcase( toolchain="opencl", harness=deepsmith_pb2.Harness(name="cldrive", opts={"timeout_seconds": "60"}), inputs={ "gsize": "1,1,1", "lsize": "1,1,1", }, ) ], ) generator_path = tempdir / "generator.pbtxt" pbutil.ToFile(generator_config, generator_path) output_dir = tempdir / "outputs" subprocess.check_call([ str(BIN), "--generator", generator_path, "--output_directory", str(output_dir), "--num_testcases", str(3), ]) assert len(list((output_dir / "generated_testcases").iterdir())) >= 3 assert len(list((output_dir / "generated_kernels").iterdir())) >= 3 for f in (output_dir / "generated_testcases").iterdir(): assert pbutil.ProtoIsReadable(f, deepsmith_pb2.Testcase())
def test_Harness_GetOrAdd(session): proto = deepsmith_pb2.Harness( name='name', opts={ 'version': '1.0.0', 'build': 'debug+assert', } ) harness = deeplearning.deepsmith.harness.Harness.GetOrAdd( session, proto ) assert session.query( deeplearning.deepsmith.harness.HarnessOptSet).count() == 2 assert session.query(deeplearning.deepsmith.harness.HarnessOpt).count() == 2 assert session.query( deeplearning.deepsmith.harness.HarnessOptName).count() == 2 assert session.query( deeplearning.deepsmith.harness.HarnessOptValue).count() == 2 assert harness.name == 'name' assert len(harness.optset) == 2 assert len(harness.opts) == 2 assert harness.opts['version'] == '1.0.0' assert harness.opts['build'] == 'debug+assert'
def test_Generator_GetOrAdd_ToProto_equivalence(session): proto_in = deepsmith_pb2.Testcase( toolchain='cpp', generator=deepsmith_pb2.Generator(name='generator', ), harness=deepsmith_pb2.Harness(name='harness', ), inputs={ 'src': 'void main() {}', 'data': '[1,2]' }, invariant_opts={'config': 'opt'}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client='localhost', type='generate', duration_ms=100, event_start_epoch_ms=101231231, ), ]) testcase = deeplearning.deepsmith.testcase.Testcase.GetOrAdd( session, proto_in) # NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs. session.flush() proto_out = testcase.ToProto() assert proto_in == proto_out proto_out.ClearField('toolchain') assert proto_in != proto_out # Sanity check.
def test_Harness_GetOrAdd_rollback(session): deeplearning.deepsmith.harness.Harness.GetOrAdd( session, deepsmith_pb2.Harness( name='name', opts={ 'a': '1', 'b': '2', }, ) ) assert session.query(deeplearning.deepsmith.harness.Harness).count() == 1 assert session.query(deeplearning.deepsmith.harness.HarnessOpt).count() == 2 assert session.query( deeplearning.deepsmith.harness.HarnessOptSet).count() == 2 assert session.query( deeplearning.deepsmith.harness.HarnessOptName).count() == 2 assert session.query( deeplearning.deepsmith.harness.HarnessOptValue).count() == 2 session.rollback() assert session.query(deeplearning.deepsmith.harness.Harness).count() == 0 assert session.query(deeplearning.deepsmith.harness.HarnessOpt).count() == 0 assert session.query( deeplearning.deepsmith.harness.HarnessOptSet).count() == 0 assert session.query( deeplearning.deepsmith.harness.HarnessOptName).count() == 0 assert session.query( deeplearning.deepsmith.harness.HarnessOptValue).count() == 0
def test_RunTestcases_cldrive_syntax_error( cldrive_harness_config: harness_pb2.CldriveHarness, opencl_opt: bool): """Test execution of a test case with invalid syntax.""" cldrive_harness_config.opencl_opt[0] = opencl_opt harness = cldrive.CldriveHarness(cldrive_harness_config) testcases = [ deepsmith_pb2.Testcase( toolchain='opencl', harness=deepsmith_pb2.Harness(name='cldrive'), inputs={ 'src': 'kernel void A(global int* a) {\n!11@invalid syntax!', 'gsize': '1,1,1', 'lsize': '1,1,1', 'timeout_seconds': '60', }) ] results = opencl_fuzz.RunTestcases(harness, testcases) assert len(results) == 1 # Testcase.invariant_opts.driver_type field is set by cldrive harness. testcases[0].invariant_opts['driver_type'] = 'compile_only' assert testcases[0] == results[0].testcase assert results[0].testbed == cldrive.OpenClEnvironmentToTestbed( harness.envs[0]) assert results[0].outcome == deepsmith_pb2.Result.BUILD_FAILURE assert results[0].outputs['stdout'] == '' print(results[0].outputs['stderr']) opt_str = 'on' if opencl_opt else 'off' assert results[0].outputs['stderr'] == f"""\
def test_RunTestcases_cl_launcher_syntax_error( cl_launcher_harness_config: harness_pb2.ClLauncherHarness, opencl_opt: bool): """Test execution of a simple test case.""" cl_launcher_harness_config.opencl_opt[0] = opencl_opt harness = cl_launcher.ClLauncherHarness(cl_launcher_harness_config) testcases = [ deepsmith_pb2.Testcase( toolchain='opencl', harness=deepsmith_pb2.Harness(name='cl_launcher'), inputs={ 'src': '__kernel void entry(\n!11@invalid syntax!', 'gsize': '1,1,1', 'lsize': '1,1,1', 'timeout_seconds': '60', }) ] results = opencl_fuzz.RunTestcases(harness, testcases) assert len(results) == 1 print(results[0].outputs['stderr']) assert testcases[0] == results[0].testcase assert results[0].testbed == cldrive.OpenClEnvironmentToTestbed( harness.envs[0]) assert results[0].outcome == deepsmith_pb2.Result.BUILD_FAILURE assert results[0].outputs['stdout'] == '' opt_str = 'on' if opencl_opt else 'off' assert results[0].outputs['stderr'] == f"""\
def test_Generator_GetOrAdd_ToProto_equivalence(session): proto_in = deepsmith_pb2.Testcase( toolchain="cpp", generator=deepsmith_pb2.Generator(name="generator", ), harness=deepsmith_pb2.Harness(name="harness", ), inputs={ "src": "void main() {}", "data": "[1,2]" }, invariant_opts={"config": "opt"}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="generate", duration_ms=100, event_start_epoch_ms=101231231, ), ], ) testcase = deeplearning.deepsmith.testcase.Testcase.GetOrAdd( session, proto_in) # NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs. session.flush() proto_out = testcase.ToProto() assert proto_in == proto_out proto_out.ClearField("toolchain") assert proto_in != proto_out # Sanity check.
def _ExportSolidityTestcases(cursor, start_id, proto_dir): batch_size = 1000 testcase_id = start_id while True: cursor.execute( """ SELECT testcases.id, programs.generator, programs.date, programs.generation_time, programs.src, testcases.harness, testcases.timeout FROM testcases LEFT JOIN programs ON testcases.program_id = programs.id WHERE testcases.id >= %s AND testcases.id NOT IN ( SELECT testcase_id FROM results ) ORDER BY testcases.id LIMIT %s """, (testcase_id, batch_size)) i = 0 for row in cursor: i += 1 (testcase_id, generator_id, program_date, program_generation_time, program_src, harness_id, harness_timeout) = row proto = deepsmith_pb2.Testcase( toolchain='solidity', generator=_GetSolidityGenerator(generator_id), harness=deepsmith_pb2.Harness( name='solc', opts={ 'timeout_seconds': str(int(harness_timeout)), 'url': 'https://github.com/ChrisCummins/dsmith/blob/5181c7c95575d428b5144a25549e5a5a55a3da31/dsmith/sol/harnesses.py#L117', }, ), inputs={ "src": program_src, }, invariant_opts={}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="cc1", type="generation", duration_ms=int(program_generation_time * 1000), event_start_epoch_ms=dateutil.MillisecondsTimestamp( program_date), ), ]) with open(proto_dir / 'sol' / 'testcases' / str(testcase_id), 'wb') as f: f.write(proto.SerializeToString()) if i < batch_size: return
def abc_config() -> generator_pb2.ClsmithGenerator: return generator_pb2.ClsmithGenerator(testcase_skeleton=[ deepsmith_pb2.Testcase(toolchain='opencl', inputs={ 'gsize': '1,1,1', 'lsize': '1,1,1', }, harness=deepsmith_pb2.Harness( name='cl_launcher')), deepsmith_pb2.Testcase(toolchain='opencl', inputs={ 'gsize': '128,16,1', 'lsize': '8,4,1', }, harness=deepsmith_pb2.Harness( name='cl_launcher')), ])
def ToProto(self) -> deepsmith_pb2.Harness: """Create protocol buffer representation. Returns: A Harness message. """ proto = deepsmith_pb2.Harness() return self.SetProto(proto)
def dummy_result() -> deepsmith_pb2.Result: """A test fixture which returns a dummy result.""" return deepsmith_pb2.Result( testcase=deepsmith_pb2.Testcase( harness=deepsmith_pb2.Harness(name="name"), inputs={"src": "Kernel source.", "gsize": "1,1,1", "lsize": "2,2,2",}, ), outputs={"stdout": "Standard output.", "stderr": "Standard error.",}, )
def test_duplicate_testcase_testbed_ignored(session): """Test that result is ignored if testbed and testcase are not unique.""" proto = deepsmith_pb2.Result( testcase=deepsmith_pb2.Testcase( toolchain='cpp', generator=deepsmith_pb2.Generator(name='generator'), harness=deepsmith_pb2.Harness(name='harness'), inputs={ 'src': 'void main() {}', 'data': '[1,2]', }, invariant_opts={ 'config': 'opt', }, profiling_events=[ deepsmith_pb2.ProfilingEvent( client='localhost', type='generate', duration_ms=100, event_start_epoch_ms=1123123123, ), ]), testbed=deepsmith_pb2.Testbed( toolchain='cpp', name='clang', opts={'arch': 'x86_64'}, ), returncode=0, outputs={'stdout': 'Hello, world!'}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client='localhost', type='exec', duration_ms=100, event_start_epoch_ms=1123123123, ), ], outcome=deepsmith_pb2.Result.PASS, ) r1 = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto) session.add(r1) session.flush() # Attempt to add a new result which is identical to the first in all fields # except for the outputs. proto.outputs['stdout'] = '!' r2 = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto) session.add(r2) session.flush() # Check that only one result was added. assert session.query(deeplearning.deepsmith.result.Result).count() == 1 # Check that only the first result was added. r3 = session.query(deeplearning.deepsmith.result.Result).first() assert r3.outputs['stdout'] == 'Hello, world!'
def test_RunTestcases_cl_launcher_syntax_error( cl_launcher_harness_config: harness_pb2.ClLauncherHarness, opencl_opt: bool ): """Test execution of a simple test case.""" cl_launcher_harness_config.opencl_opt[0] = opencl_opt harness = cl_launcher.ClLauncherHarness(cl_launcher_harness_config) testcases = [ deepsmith_pb2.Testcase( toolchain="opencl", harness=deepsmith_pb2.Harness(name="cl_launcher"), inputs={ "src": "__kernel void entry(\n!11@invalid syntax!", "gsize": "1,1,1", "lsize": "1,1,1", "timeout_seconds": "60", }, ) ] results = opencl_fuzz.RunTestcases(harness, testcases) assert len(results) == 1 print(results[0].outputs["stderr"]) assert testcases[0] == results[0].testcase assert results[0].testbed == cldrive.OpenClEnvironmentToTestbed( harness.envs[0] ) assert results[0].outcome == deepsmith_pb2.Result.BUILD_FAILURE assert results[0].outputs["stdout"] == "" opt_str = "on" if opencl_opt else "off" assert ( results[0].outputs["stderr"] == f"""\ 3-D global size 1 = [1, 1, 1] 3-D local size 1 = [1, 1, 1] OpenCL optimizations: {opt_str} Platform: Oclgrind Device: Oclgrind Simulator 3 errors generated. Error found (callback): Oclgrind - OpenCL runtime error detected \tFunction: clBuildProgram \tError: CL_BUILD_PROGRAM_FAILURE Error building program: -11 input.cl:2:1: error: expected parameter declarator !11@invalid syntax! ^ input.cl:2:1: error: expected ')' input.cl:1:20: note: to match this '(' __kernel void entry( ^ input.cl:2:20: error: expected function body after function declarator !11@invalid syntax! ^ """ )
def abc_testcase() -> deepsmith_pb2.Testcase(): """A test fixture which returns a very simple test case.""" return deepsmith_pb2.Testcase( toolchain='opencl', harness=deepsmith_pb2.Harness(name='cl_launcher'), inputs={ 'src': CLSMITH_EXAMPLE_SRC, 'gsize': '1,1,1', 'lsize': '1,1,1', })
def abc_testcase() -> deepsmith_pb2.Testcase(): """A test fixture which returns a very simple test case.""" return deepsmith_pb2.Testcase( toolchain='opencl', harness=deepsmith_pb2.Harness(name='cldrive'), inputs={ 'src': 'kernel void A(global int* a) {a[get_global_id(0)] = 10;}', 'gsize': '1,1,1', 'lsize': '1,1,1', })
def abc_config() -> generator_pb2.ClsmithGenerator: return generator_pb2.ClsmithGenerator(testcase_skeleton=[ deepsmith_pb2.Testcase( toolchain="opencl", inputs={ "gsize": "1,1,1", "lsize": "1,1,1", }, harness=deepsmith_pb2.Harness(name="cl_launcher"), ), deepsmith_pb2.Testcase( toolchain="opencl", inputs={ "gsize": "128,16,1", "lsize": "8,4,1", }, harness=deepsmith_pb2.Harness(name="cl_launcher"), ), ])
def abc_testcase() -> deepsmith_pb2.Testcase(): """A test fixture which returns a very simple test case.""" return deepsmith_pb2.Testcase( toolchain="opencl", harness=deepsmith_pb2.Harness(name="cldrive"), inputs={ "src": "kernel void A(global int* a) {a[get_global_id(0)] = 10;}", "gsize": "1,1,1", "lsize": "1,1,1", }, )
def abc_testcase() -> deepsmith_pb2.Testcase(): """A test fixture which returns a very simple test case.""" return deepsmith_pb2.Testcase( toolchain="opencl", harness=deepsmith_pb2.Harness(name="cl_launcher"), inputs={ "src": CLSMITH_EXAMPLE_SRC, "gsize": "1,1,1", "lsize": "1,1,1", }, )
def test_Generator_GetOrAdd_ToProto_equivalence(session): proto_in = deepsmith_pb2.Result( testcase=deepsmith_pb2.Testcase( toolchain="cpp", generator=deepsmith_pb2.Generator(name="generator"), harness=deepsmith_pb2.Harness(name="harness"), inputs={"src": "void main() {}", "data": "[1,2]",}, invariant_opts={"config": "opt",}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="generate", duration_ms=100, event_start_epoch_ms=1123123123, ), deepsmith_pb2.ProfilingEvent( client="localhost", type="foo", duration_ms=100, event_start_epoch_ms=1123123123, ), ], ), testbed=deepsmith_pb2.Testbed( toolchain="cpp", name="clang", opts={"arch": "x86_64", "build": "debug+assert",}, ), returncode=0, outputs={"stdout": "Hello, world!", "stderr": "",}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="exec", duration_ms=500, event_start_epoch_ms=1123123123, ), deepsmith_pb2.ProfilingEvent( client="localhost", type="overhead", duration_ms=100, event_start_epoch_ms=1123123123, ), ], outcome=deepsmith_pb2.Result.PASS, ) result = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto_in) # NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs. session.flush() proto_out = result.ToProto() assert proto_in == proto_out proto_out.ClearField("outputs") assert proto_in != proto_out # Sanity check.
def test_RunTestcases_cldrive_syntax_error( cldrive_harness_config: harness_pb2.CldriveHarness, opencl_opt: bool ): """Test execution of a test case with invalid syntax.""" cldrive_harness_config.opencl_opt[0] = opencl_opt harness = cldrive.CldriveHarness(cldrive_harness_config) testcases = [ deepsmith_pb2.Testcase( toolchain="opencl", harness=deepsmith_pb2.Harness(name="cldrive"), inputs={ "src": "kernel void A(global int* a) {\n!11@invalid syntax!", "gsize": "1,1,1", "lsize": "1,1,1", "timeout_seconds": "60", }, ) ] results = opencl_fuzz.RunTestcases(harness, testcases) assert len(results) == 1 # Testcase.invariant_opts.driver_type field is set by cldrive harness. testcases[0].invariant_opts["driver_type"] = "compile_only" assert testcases[0] == results[0].testcase assert results[0].testbed == cldrive.OpenClEnvironmentToTestbed( harness.envs[0] ) assert results[0].outcome == deepsmith_pb2.Result.BUILD_FAILURE assert results[0].outputs["stdout"] == "" print(results[0].outputs["stderr"]) opt_str = "on" if opencl_opt else "off" assert ( results[0].outputs["stderr"] == f"""\ [cldrive] Platform: Oclgrind [cldrive] Device: Oclgrind Simulator [cldrive] OpenCL optimizations: {opt_str} 1 warning and 3 errors generated. input.cl:1:34: error: expected ';' after expression kernel void A(global int* a) {{!11@invalid syntax! ^ ; input.cl:1:34: error: expected expression input.cl:1:50: error: expected '}}' kernel void A(global int* a) {{!11@invalid syntax! ^ input.cl:1:30: note: to match this '{{' kernel void A(global int* a) {{!11@invalid syntax! ^ input.cl:1:31: warning: expression result unused kernel void A(global int* a) {{!11@invalid syntax! ^~~ clBuildProgram CL_BUILD_PROGRAM_FAILURE """ )
def test_RunTestcases_cldrive_pass( cldrive_harness_config: harness_pb2.CldriveHarness, opencl_opt: bool ): """Test execution of a simple test case.""" cldrive_harness_config.opencl_opt[0] = opencl_opt harness = cldrive.CldriveHarness(cldrive_harness_config) testcases = [ deepsmith_pb2.Testcase( toolchain="opencl", harness=deepsmith_pb2.Harness(name="cldrive"), inputs={ "src": "kernel void A(global int* a) {a[get_global_id(0)] = 10;}", "gsize": "1,1,1", "lsize": "1,1,1", "timeout_seconds": "60", }, ) ] results = opencl_fuzz.RunTestcases(harness, testcases) assert len(results) == 1 # Testcase.invariant_opts.driver_type field is set by cldrive harness. testcases[0].invariant_opts["driver_type"] = "compile_and_run" assert testcases[0] == results[0].testcase assert results[0].testbed == cldrive.OpenClEnvironmentToTestbed( harness.envs[0] ) assert results[0].outcome == deepsmith_pb2.Result.PASS assert results[0].outputs["stdout"] == ( "global int * a: 10 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 " "22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 " "46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 " "70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 " "94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 " "114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 " "132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 " "150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 " "168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 " "186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 " "204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 " "222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 " "240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255\n" ) opt_str = "on" if opencl_opt else "off" assert ( results[0].outputs["stderr"] == f"""\ [cldrive] Platform: Oclgrind [cldrive] Device: Oclgrind Simulator [cldrive] OpenCL optimizations: {opt_str} [cldrive] Kernel: "A" done. """ )
def _AddExistingHarness(session): deeplearning.deepsmith.harness.Harness.GetOrAdd( session, deepsmith_pb2.Harness( name='name', opts={ 'a': 'a', 'b': 'b', 'c': 'c', }, ) ) session.flush()
def _AddExistingHarness(session): deeplearning.deepsmith.harness.Harness.GetOrAdd( session, deepsmith_pb2.Harness( name="name", opts={ "a": "a", "b": "b", "c": "c", }, ), ) session.flush()
def dummy_result() -> deepsmith_pb2.Result: """A test fixture which returns a dummy result.""" return deepsmith_pb2.Result(testcase=deepsmith_pb2.Testcase( harness=deepsmith_pb2.Harness(name='name'), inputs={ 'src': 'Kernel source.', 'gsize': '1,1,1', 'lsize': '2,2,2', }), outputs={ 'stdout': 'Standard output.', 'stderr': 'Standard error.', })
def _AddRandomNewHarness(session): deeplearning.deepsmith.harness.Harness.GetOrAdd( session, deepsmith_pb2.Harness( name=str(random.random()), opts={ str(random.random()): str(random.random()), str(random.random()): str(random.random()), str(random.random()): str(random.random()), }, ), ) session.flush()
def test_Testcase_GetOrAdd(session): proto = deepsmith_pb2.Testcase( toolchain="cpp", generator=deepsmith_pb2.Generator(name="generator", ), harness=deepsmith_pb2.Harness(name="harness", ), inputs={ "src": "void main() {}", "data": "[1,2]" }, invariant_opts={"config": "opt"}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="generate", duration_ms=100, event_start_epoch_ms=1021312312, ), deepsmith_pb2.ProfilingEvent( client="localhost", type="foo", duration_ms=100, event_start_epoch_ms=1230812312, ), ], ) testcase = deeplearning.deepsmith.testcase.Testcase.GetOrAdd( session, proto) # NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs. session.flush() assert testcase.toolchain.string == "cpp" assert testcase.generator.name == "generator" assert testcase.harness.name == "harness" assert len(testcase.inputset) == 2 assert len(testcase.inputs) == 2 assert testcase.inputs["src"] == "void main() {}" assert testcase.inputs["data"] == "[1,2]" assert len(testcase.invariant_optset) == 1 assert len(testcase.invariant_opts) == 1 assert testcase.invariant_opts["config"] == "opt" assert testcase.profiling_events[0].client.string == "localhost" assert testcase.profiling_events[0].type.string == "generate" assert testcase.profiling_events[0].duration_ms == 100 assert testcase.profiling_events[ 0].event_start == labdate.DatetimeFromMillisecondsTimestamp(1021312312) assert testcase.profiling_events[1].client.string == "localhost" assert testcase.profiling_events[1].type.string == "foo" assert testcase.profiling_events[1].duration_ms == 100 assert testcase.profiling_events[ 1].event_start == labdate.DatetimeFromMillisecondsTimestamp(1230812312)
def test_duplicate_testcase_testbed_ignored(session): """Test that result is ignored if testbed and testcase are not unique.""" proto = deepsmith_pb2.Result( testcase=deepsmith_pb2.Testcase( toolchain="cpp", generator=deepsmith_pb2.Generator(name="generator"), harness=deepsmith_pb2.Harness(name="harness"), inputs={"src": "void main() {}", "data": "[1,2]",}, invariant_opts={"config": "opt",}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="generate", duration_ms=100, event_start_epoch_ms=1123123123, ), ], ), testbed=deepsmith_pb2.Testbed( toolchain="cpp", name="clang", opts={"arch": "x86_64"}, ), returncode=0, outputs={"stdout": "Hello, world!"}, profiling_events=[ deepsmith_pb2.ProfilingEvent( client="localhost", type="exec", duration_ms=100, event_start_epoch_ms=1123123123, ), ], outcome=deepsmith_pb2.Result.PASS, ) r1 = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto) session.add(r1) session.flush() # Attempt to add a new result which is identical to the first in all fields # except for the outputs. proto.outputs["stdout"] = "!" r2 = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto) session.add(r2) session.flush() # Check that only one result was added. assert session.query(deeplearning.deepsmith.result.Result).count() == 1 # Check that only the first result was added. r3 = session.query(deeplearning.deepsmith.result.Result).first() assert r3.outputs["stdout"] == "Hello, world!"
def OpenClSourceToTestCases(src: str) -> typing.List[deepsmith_pb2.Testcase]: """Generate DeepSmith testcases for each of the combination of gsize and lsize used in CGO'17 synthetic kernels.""" return [ deepsmith_pb2.Testcase( toolchain="opencl", harness=deepsmith_pb2.Harness(name="cldrive"), inputs={ "src": src, "gsize": f"{gsize},1,1", "lsize": f"{lsize},1,1", }, ) for lsize, gsize in LSIZE_GSIZE_PAIRS ]