def _GetOpenCLHarness(harness_id, timeout) -> deepsmith_pb2.Harness:
if harness_id == -1:
return deepsmith_pb2.Harness(
name="clang",
opts={
"timeout_seconds":
str(int(timeout)),
"url":
"https://github.com/ChrisCummins/dsmith/blob/fd986a36a23b2a398f33d5b5852d930b462401b1/dsmith/opencl/harnesses.py#L292",
})
elif harness_id == 0:
return deepsmith_pb2.Harness(
name="cl_launcher",
opts={
"timeout_seconds": str(int(timeout)),
"git_commit": "b637b31c31e0f90ef199ca492af05172400df050",
"git_remote": "https://github.com/ChrisCummins/CLSmith.git",
})
elif harness_id == 1:
return deepsmith_pb2.Harness(
name="cldrive",
opts={
"timeout_seconds": str(int(timeout)),
"git_commit": "9556e7112ba2bd6f79ee59eef74f0a2304efa007",
"git_remote": "https://github.com/ChrisCummins/clgen.git",
"version": "0.4.0.dev0",
})
else:
raise LookupError
def test_Harness_GetOrAdd_rollback(session):
deeplearning.deepsmith.harness.Harness.GetOrAdd(
session,
deepsmith_pb2.Harness(
name="name",
opts={
"a": "1",
"b": "2",
},
))
assert session.query(deeplearning.deepsmith.harness.Harness).count() == 1
assert session.query(
deeplearning.deepsmith.harness.HarnessOpt).count() == 2
assert (session.query(
deeplearning.deepsmith.harness.HarnessOptSet).count() == 2)
assert (session.query(
deeplearning.deepsmith.harness.HarnessOptName).count() == 2)
assert (session.query(
deeplearning.deepsmith.harness.HarnessOptValue).count() == 2)
session.rollback()
assert session.query(deeplearning.deepsmith.harness.Harness).count() == 0
assert session.query(
deeplearning.deepsmith.harness.HarnessOpt).count() == 0
assert (session.query(
deeplearning.deepsmith.harness.HarnessOptSet).count() == 0)
assert (session.query(
deeplearning.deepsmith.harness.HarnessOptName).count() == 0)
assert (session.query(
deeplearning.deepsmith.harness.HarnessOptValue).count() == 0)
def test_GenerateTestcases(abc_instance_config, tempdir: pathlib.Path):
"""Run a tiny end-to-end test."""
generator_config = generator_pb2.ClgenGenerator(
instance=abc_instance_config,
testcase_skeleton=[
deepsmith_pb2.Testcase(
toolchain="opencl",
harness=deepsmith_pb2.Harness(name="cldrive",
opts={"timeout_seconds": "60"}),
inputs={
"gsize": "1,1,1",
"lsize": "1,1,1",
},
)
],
)
generator_path = tempdir / "generator.pbtxt"
pbutil.ToFile(generator_config, generator_path)
output_dir = tempdir / "outputs"
subprocess.check_call([
str(BIN),
"--generator",
generator_path,
"--output_directory",
str(output_dir),
"--num_testcases",
str(3),
])
assert len(list((output_dir / "generated_testcases").iterdir())) >= 3
assert len(list((output_dir / "generated_kernels").iterdir())) >= 3
for f in (output_dir / "generated_testcases").iterdir():
assert pbutil.ProtoIsReadable(f, deepsmith_pb2.Testcase())
def test_Harness_GetOrAdd(session):
proto = deepsmith_pb2.Harness(
name='name',
opts={
'version': '1.0.0',
'build': 'debug+assert',
}
)
harness = deeplearning.deepsmith.harness.Harness.GetOrAdd(
session, proto
)
assert session.query(
deeplearning.deepsmith.harness.HarnessOptSet).count() == 2
assert session.query(deeplearning.deepsmith.harness.HarnessOpt).count() == 2
assert session.query(
deeplearning.deepsmith.harness.HarnessOptName).count() == 2
assert session.query(
deeplearning.deepsmith.harness.HarnessOptValue).count() == 2
assert harness.name == 'name'
assert len(harness.optset) == 2
assert len(harness.opts) == 2
assert harness.opts['version'] == '1.0.0'
assert harness.opts['build'] == 'debug+assert'
def test_Generator_GetOrAdd_ToProto_equivalence(session):
proto_in = deepsmith_pb2.Testcase(
toolchain='cpp',
generator=deepsmith_pb2.Generator(name='generator', ),
harness=deepsmith_pb2.Harness(name='harness', ),
inputs={
'src': 'void main() {}',
'data': '[1,2]'
},
invariant_opts={'config': 'opt'},
profiling_events=[
deepsmith_pb2.ProfilingEvent(
client='localhost',
type='generate',
duration_ms=100,
event_start_epoch_ms=101231231,
),
])
testcase = deeplearning.deepsmith.testcase.Testcase.GetOrAdd(
session, proto_in)
# NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs.
session.flush()
proto_out = testcase.ToProto()
assert proto_in == proto_out
proto_out.ClearField('toolchain')
assert proto_in != proto_out # Sanity check.
def test_Harness_GetOrAdd_rollback(session):
deeplearning.deepsmith.harness.Harness.GetOrAdd(
session,
deepsmith_pb2.Harness(
name='name',
opts={
'a': '1',
'b': '2',
},
)
)
assert session.query(deeplearning.deepsmith.harness.Harness).count() == 1
assert session.query(deeplearning.deepsmith.harness.HarnessOpt).count() == 2
assert session.query(
deeplearning.deepsmith.harness.HarnessOptSet).count() == 2
assert session.query(
deeplearning.deepsmith.harness.HarnessOptName).count() == 2
assert session.query(
deeplearning.deepsmith.harness.HarnessOptValue).count() == 2
session.rollback()
assert session.query(deeplearning.deepsmith.harness.Harness).count() == 0
assert session.query(deeplearning.deepsmith.harness.HarnessOpt).count() == 0
assert session.query(
deeplearning.deepsmith.harness.HarnessOptSet).count() == 0
assert session.query(
deeplearning.deepsmith.harness.HarnessOptName).count() == 0
assert session.query(
deeplearning.deepsmith.harness.HarnessOptValue).count() == 0
def test_RunTestcases_cldrive_syntax_error(
cldrive_harness_config: harness_pb2.CldriveHarness, opencl_opt: bool):
"""Test execution of a test case with invalid syntax."""
cldrive_harness_config.opencl_opt[0] = opencl_opt
harness = cldrive.CldriveHarness(cldrive_harness_config)
testcases = [
deepsmith_pb2.Testcase(
toolchain='opencl',
harness=deepsmith_pb2.Harness(name='cldrive'),
inputs={
'src': 'kernel void A(global int* a) {\n!11@invalid syntax!',
'gsize': '1,1,1',
'lsize': '1,1,1',
'timeout_seconds': '60',
})
]
results = opencl_fuzz.RunTestcases(harness, testcases)
assert len(results) == 1
# Testcase.invariant_opts.driver_type field is set by cldrive harness.
testcases[0].invariant_opts['driver_type'] = 'compile_only'
assert testcases[0] == results[0].testcase
assert results[0].testbed == cldrive.OpenClEnvironmentToTestbed(
harness.envs[0])
assert results[0].outcome == deepsmith_pb2.Result.BUILD_FAILURE
assert results[0].outputs['stdout'] == ''
print(results[0].outputs['stderr'])
opt_str = 'on' if opencl_opt else 'off'
assert results[0].outputs['stderr'] == f"""\
def test_RunTestcases_cl_launcher_syntax_error(
cl_launcher_harness_config: harness_pb2.ClLauncherHarness,
opencl_opt: bool):
"""Test execution of a simple test case."""
cl_launcher_harness_config.opencl_opt[0] = opencl_opt
harness = cl_launcher.ClLauncherHarness(cl_launcher_harness_config)
testcases = [
deepsmith_pb2.Testcase(
toolchain='opencl',
harness=deepsmith_pb2.Harness(name='cl_launcher'),
inputs={
'src': '__kernel void entry(\n!11@invalid syntax!',
'gsize': '1,1,1',
'lsize': '1,1,1',
'timeout_seconds': '60',
})
]
results = opencl_fuzz.RunTestcases(harness, testcases)
assert len(results) == 1
print(results[0].outputs['stderr'])
assert testcases[0] == results[0].testcase
assert results[0].testbed == cldrive.OpenClEnvironmentToTestbed(
harness.envs[0])
assert results[0].outcome == deepsmith_pb2.Result.BUILD_FAILURE
assert results[0].outputs['stdout'] == ''
opt_str = 'on' if opencl_opt else 'off'
assert results[0].outputs['stderr'] == f"""\
def test_Generator_GetOrAdd_ToProto_equivalence(session):
proto_in = deepsmith_pb2.Testcase(
toolchain="cpp",
generator=deepsmith_pb2.Generator(name="generator", ),
harness=deepsmith_pb2.Harness(name="harness", ),
inputs={
"src": "void main() {}",
"data": "[1,2]"
},
invariant_opts={"config": "opt"},
profiling_events=[
deepsmith_pb2.ProfilingEvent(
client="localhost",
type="generate",
duration_ms=100,
event_start_epoch_ms=101231231,
),
],
)
testcase = deeplearning.deepsmith.testcase.Testcase.GetOrAdd(
session, proto_in)
# NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs.
session.flush()
proto_out = testcase.ToProto()
assert proto_in == proto_out
proto_out.ClearField("toolchain")
assert proto_in != proto_out # Sanity check.
def _ExportSolidityTestcases(cursor, start_id, proto_dir):
batch_size = 1000
testcase_id = start_id
while True:
cursor.execute(
"""
SELECT
testcases.id,
programs.generator,
programs.date,
programs.generation_time,
programs.src,
testcases.harness,
testcases.timeout
FROM testcases
LEFT JOIN programs ON testcases.program_id = programs.id
WHERE testcases.id >= %s
AND testcases.id NOT IN (
SELECT testcase_id FROM
results
)
ORDER BY testcases.id
LIMIT %s
""", (testcase_id, batch_size))
i = 0
for row in cursor:
i += 1
(testcase_id, generator_id, program_date, program_generation_time,
program_src, harness_id, harness_timeout) = row
proto = deepsmith_pb2.Testcase(
toolchain='solidity',
generator=_GetSolidityGenerator(generator_id),
harness=deepsmith_pb2.Harness(
name='solc',
opts={
'timeout_seconds':
str(int(harness_timeout)),
'url':
'https://github.com/ChrisCummins/dsmith/blob/5181c7c95575d428b5144a25549e5a5a55a3da31/dsmith/sol/harnesses.py#L117',
},
),
inputs={
"src": program_src,
},
invariant_opts={},
profiling_events=[
deepsmith_pb2.ProfilingEvent(
client="cc1",
type="generation",
duration_ms=int(program_generation_time * 1000),
event_start_epoch_ms=dateutil.MillisecondsTimestamp(
program_date),
),
])
with open(proto_dir / 'sol' / 'testcases' / str(testcase_id),
'wb') as f:
f.write(proto.SerializeToString())
if i < batch_size:
return
def abc_config() -> generator_pb2.ClsmithGenerator:
return generator_pb2.ClsmithGenerator(testcase_skeleton=[
deepsmith_pb2.Testcase(toolchain='opencl',
inputs={
'gsize': '1,1,1',
'lsize': '1,1,1',
},
harness=deepsmith_pb2.Harness(
name='cl_launcher')),
deepsmith_pb2.Testcase(toolchain='opencl',
inputs={
'gsize': '128,16,1',
'lsize': '8,4,1',
},
harness=deepsmith_pb2.Harness(
name='cl_launcher')),
])
def ToProto(self) -> deepsmith_pb2.Harness:
"""Create protocol buffer representation.
Returns:
A Harness message.
"""
proto = deepsmith_pb2.Harness()
return self.SetProto(proto)
def dummy_result() -> deepsmith_pb2.Result:
"""A test fixture which returns a dummy result."""
return deepsmith_pb2.Result(
testcase=deepsmith_pb2.Testcase(
harness=deepsmith_pb2.Harness(name="name"),
inputs={"src": "Kernel source.", "gsize": "1,1,1", "lsize": "2,2,2",},
),
outputs={"stdout": "Standard output.", "stderr": "Standard error.",},
)
def test_duplicate_testcase_testbed_ignored(session):
"""Test that result is ignored if testbed and testcase are not unique."""
proto = deepsmith_pb2.Result(
testcase=deepsmith_pb2.Testcase(
toolchain='cpp',
generator=deepsmith_pb2.Generator(name='generator'),
harness=deepsmith_pb2.Harness(name='harness'),
inputs={
'src': 'void main() {}',
'data': '[1,2]',
},
invariant_opts={
'config': 'opt',
},
profiling_events=[
deepsmith_pb2.ProfilingEvent(
client='localhost',
type='generate',
duration_ms=100,
event_start_epoch_ms=1123123123,
),
]),
testbed=deepsmith_pb2.Testbed(
toolchain='cpp',
name='clang',
opts={'arch': 'x86_64'},
),
returncode=0,
outputs={'stdout': 'Hello, world!'},
profiling_events=[
deepsmith_pb2.ProfilingEvent(
client='localhost',
type='exec',
duration_ms=100,
event_start_epoch_ms=1123123123,
),
],
outcome=deepsmith_pb2.Result.PASS,
)
r1 = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto)
session.add(r1)
session.flush()
# Attempt to add a new result which is identical to the first in all fields
# except for the outputs.
proto.outputs['stdout'] = '!'
r2 = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto)
session.add(r2)
session.flush()
# Check that only one result was added.
assert session.query(deeplearning.deepsmith.result.Result).count() == 1
# Check that only the first result was added.
r3 = session.query(deeplearning.deepsmith.result.Result).first()
assert r3.outputs['stdout'] == 'Hello, world!'
def test_RunTestcases_cl_launcher_syntax_error(
cl_launcher_harness_config: harness_pb2.ClLauncherHarness, opencl_opt: bool
):
"""Test execution of a simple test case."""
cl_launcher_harness_config.opencl_opt[0] = opencl_opt
harness = cl_launcher.ClLauncherHarness(cl_launcher_harness_config)
testcases = [
deepsmith_pb2.Testcase(
toolchain="opencl",
harness=deepsmith_pb2.Harness(name="cl_launcher"),
inputs={
"src": "__kernel void entry(\n!11@invalid syntax!",
"gsize": "1,1,1",
"lsize": "1,1,1",
"timeout_seconds": "60",
},
)
]
results = opencl_fuzz.RunTestcases(harness, testcases)
assert len(results) == 1
print(results[0].outputs["stderr"])
assert testcases[0] == results[0].testcase
assert results[0].testbed == cldrive.OpenClEnvironmentToTestbed(
harness.envs[0]
)
assert results[0].outcome == deepsmith_pb2.Result.BUILD_FAILURE
assert results[0].outputs["stdout"] == ""
opt_str = "on" if opencl_opt else "off"
assert (
results[0].outputs["stderr"]
== f"""\
3-D global size 1 = [1, 1, 1]
3-D local size 1 = [1, 1, 1]
OpenCL optimizations: {opt_str}
Platform: Oclgrind
Device: Oclgrind Simulator
3 errors generated.
Error found (callback):
Oclgrind - OpenCL runtime error detected
\tFunction: clBuildProgram
\tError: CL_BUILD_PROGRAM_FAILURE
Error building program: -11
input.cl:2:1: error: expected parameter declarator
!11@invalid syntax!
^
input.cl:2:1: error: expected ')'
input.cl:1:20: note: to match this '('
__kernel void entry(
^
input.cl:2:20: error: expected function body after function declarator
!11@invalid syntax!
^
"""
)
def abc_testcase() -> deepsmith_pb2.Testcase():
"""A test fixture which returns a very simple test case."""
return deepsmith_pb2.Testcase(
toolchain='opencl',
harness=deepsmith_pb2.Harness(name='cl_launcher'),
inputs={
'src': CLSMITH_EXAMPLE_SRC,
'gsize': '1,1,1',
'lsize': '1,1,1',
})
def abc_testcase() -> deepsmith_pb2.Testcase():
"""A test fixture which returns a very simple test case."""
return deepsmith_pb2.Testcase(
toolchain='opencl',
harness=deepsmith_pb2.Harness(name='cldrive'),
inputs={
'src': 'kernel void A(global int* a) {a[get_global_id(0)] = 10;}',
'gsize': '1,1,1',
'lsize': '1,1,1',
})
def abc_config() -> generator_pb2.ClsmithGenerator:
return generator_pb2.ClsmithGenerator(testcase_skeleton=[
deepsmith_pb2.Testcase(
toolchain="opencl",
inputs={
"gsize": "1,1,1",
"lsize": "1,1,1",
},
harness=deepsmith_pb2.Harness(name="cl_launcher"),
),
deepsmith_pb2.Testcase(
toolchain="opencl",
inputs={
"gsize": "128,16,1",
"lsize": "8,4,1",
},
harness=deepsmith_pb2.Harness(name="cl_launcher"),
),
])
def abc_testcase() -> deepsmith_pb2.Testcase():
"""A test fixture which returns a very simple test case."""
return deepsmith_pb2.Testcase(
toolchain="opencl",
harness=deepsmith_pb2.Harness(name="cldrive"),
inputs={
"src": "kernel void A(global int* a) {a[get_global_id(0)] = 10;}",
"gsize": "1,1,1",
"lsize": "1,1,1",
},
)
def abc_testcase() -> deepsmith_pb2.Testcase():
"""A test fixture which returns a very simple test case."""
return deepsmith_pb2.Testcase(
toolchain="opencl",
harness=deepsmith_pb2.Harness(name="cl_launcher"),
inputs={
"src": CLSMITH_EXAMPLE_SRC,
"gsize": "1,1,1",
"lsize": "1,1,1",
},
)
def test_Generator_GetOrAdd_ToProto_equivalence(session):
proto_in = deepsmith_pb2.Result(
testcase=deepsmith_pb2.Testcase(
toolchain="cpp",
generator=deepsmith_pb2.Generator(name="generator"),
harness=deepsmith_pb2.Harness(name="harness"),
inputs={"src": "void main() {}", "data": "[1,2]",},
invariant_opts={"config": "opt",},
profiling_events=[
deepsmith_pb2.ProfilingEvent(
client="localhost",
type="generate",
duration_ms=100,
event_start_epoch_ms=1123123123,
),
deepsmith_pb2.ProfilingEvent(
client="localhost",
type="foo",
duration_ms=100,
event_start_epoch_ms=1123123123,
),
],
),
testbed=deepsmith_pb2.Testbed(
toolchain="cpp",
name="clang",
opts={"arch": "x86_64", "build": "debug+assert",},
),
returncode=0,
outputs={"stdout": "Hello, world!", "stderr": "",},
profiling_events=[
deepsmith_pb2.ProfilingEvent(
client="localhost",
type="exec",
duration_ms=500,
event_start_epoch_ms=1123123123,
),
deepsmith_pb2.ProfilingEvent(
client="localhost",
type="overhead",
duration_ms=100,
event_start_epoch_ms=1123123123,
),
],
outcome=deepsmith_pb2.Result.PASS,
)
result = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto_in)
# NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs.
session.flush()
proto_out = result.ToProto()
assert proto_in == proto_out
proto_out.ClearField("outputs")
assert proto_in != proto_out # Sanity check.
def test_RunTestcases_cldrive_syntax_error(
cldrive_harness_config: harness_pb2.CldriveHarness, opencl_opt: bool
):
"""Test execution of a test case with invalid syntax."""
cldrive_harness_config.opencl_opt[0] = opencl_opt
harness = cldrive.CldriveHarness(cldrive_harness_config)
testcases = [
deepsmith_pb2.Testcase(
toolchain="opencl",
harness=deepsmith_pb2.Harness(name="cldrive"),
inputs={
"src": "kernel void A(global int* a) {\n!11@invalid syntax!",
"gsize": "1,1,1",
"lsize": "1,1,1",
"timeout_seconds": "60",
},
)
]
results = opencl_fuzz.RunTestcases(harness, testcases)
assert len(results) == 1
# Testcase.invariant_opts.driver_type field is set by cldrive harness.
testcases[0].invariant_opts["driver_type"] = "compile_only"
assert testcases[0] == results[0].testcase
assert results[0].testbed == cldrive.OpenClEnvironmentToTestbed(
harness.envs[0]
)
assert results[0].outcome == deepsmith_pb2.Result.BUILD_FAILURE
assert results[0].outputs["stdout"] == ""
print(results[0].outputs["stderr"])
opt_str = "on" if opencl_opt else "off"
assert (
results[0].outputs["stderr"]
== f"""\
[cldrive] Platform: Oclgrind
[cldrive] Device: Oclgrind Simulator
[cldrive] OpenCL optimizations: {opt_str}
1 warning and 3 errors generated.
input.cl:1:34: error: expected ';' after expression
kernel void A(global int* a) {{!11@invalid syntax!
^
;
input.cl:1:34: error: expected expression
input.cl:1:50: error: expected '}}'
kernel void A(global int* a) {{!11@invalid syntax!
^
input.cl:1:30: note: to match this '{{'
kernel void A(global int* a) {{!11@invalid syntax!
^
input.cl:1:31: warning: expression result unused
kernel void A(global int* a) {{!11@invalid syntax!
^~~
clBuildProgram CL_BUILD_PROGRAM_FAILURE
"""
)
def test_RunTestcases_cldrive_pass(
cldrive_harness_config: harness_pb2.CldriveHarness, opencl_opt: bool
):
"""Test execution of a simple test case."""
cldrive_harness_config.opencl_opt[0] = opencl_opt
harness = cldrive.CldriveHarness(cldrive_harness_config)
testcases = [
deepsmith_pb2.Testcase(
toolchain="opencl",
harness=deepsmith_pb2.Harness(name="cldrive"),
inputs={
"src": "kernel void A(global int* a) {a[get_global_id(0)] = 10;}",
"gsize": "1,1,1",
"lsize": "1,1,1",
"timeout_seconds": "60",
},
)
]
results = opencl_fuzz.RunTestcases(harness, testcases)
assert len(results) == 1
# Testcase.invariant_opts.driver_type field is set by cldrive harness.
testcases[0].invariant_opts["driver_type"] = "compile_and_run"
assert testcases[0] == results[0].testcase
assert results[0].testbed == cldrive.OpenClEnvironmentToTestbed(
harness.envs[0]
)
assert results[0].outcome == deepsmith_pb2.Result.PASS
assert results[0].outputs["stdout"] == (
"global int * a: 10 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 "
"22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 "
"46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 "
"70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 "
"94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 "
"114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 "
"132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 "
"150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 "
"168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 "
"186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 "
"204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 "
"222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 "
"240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255\n"
)
opt_str = "on" if opencl_opt else "off"
assert (
results[0].outputs["stderr"]
== f"""\
[cldrive] Platform: Oclgrind
[cldrive] Device: Oclgrind Simulator
[cldrive] OpenCL optimizations: {opt_str}
[cldrive] Kernel: "A"
done.
"""
)
def _AddExistingHarness(session):
deeplearning.deepsmith.harness.Harness.GetOrAdd(
session,
deepsmith_pb2.Harness(
name='name',
opts={
'a': 'a',
'b': 'b',
'c': 'c',
},
)
)
session.flush()
def _AddExistingHarness(session):
deeplearning.deepsmith.harness.Harness.GetOrAdd(
session,
deepsmith_pb2.Harness(
name="name",
opts={
"a": "a",
"b": "b",
"c": "c",
},
),
)
session.flush()
def dummy_result() -> deepsmith_pb2.Result:
"""A test fixture which returns a dummy result."""
return deepsmith_pb2.Result(testcase=deepsmith_pb2.Testcase(
harness=deepsmith_pb2.Harness(name='name'),
inputs={
'src': 'Kernel source.',
'gsize': '1,1,1',
'lsize': '2,2,2',
}),
outputs={
'stdout': 'Standard output.',
'stderr': 'Standard error.',
})
def _AddRandomNewHarness(session):
deeplearning.deepsmith.harness.Harness.GetOrAdd(
session,
deepsmith_pb2.Harness(
name=str(random.random()),
opts={
str(random.random()): str(random.random()),
str(random.random()): str(random.random()),
str(random.random()): str(random.random()),
},
),
)
session.flush()
def test_Testcase_GetOrAdd(session):
proto = deepsmith_pb2.Testcase(
toolchain="cpp",
generator=deepsmith_pb2.Generator(name="generator", ),
harness=deepsmith_pb2.Harness(name="harness", ),
inputs={
"src": "void main() {}",
"data": "[1,2]"
},
invariant_opts={"config": "opt"},
profiling_events=[
deepsmith_pb2.ProfilingEvent(
client="localhost",
type="generate",
duration_ms=100,
event_start_epoch_ms=1021312312,
),
deepsmith_pb2.ProfilingEvent(
client="localhost",
type="foo",
duration_ms=100,
event_start_epoch_ms=1230812312,
),
],
)
testcase = deeplearning.deepsmith.testcase.Testcase.GetOrAdd(
session, proto)
# NOTE: We have to flush so that SQLAlchemy resolves all of the object IDs.
session.flush()
assert testcase.toolchain.string == "cpp"
assert testcase.generator.name == "generator"
assert testcase.harness.name == "harness"
assert len(testcase.inputset) == 2
assert len(testcase.inputs) == 2
assert testcase.inputs["src"] == "void main() {}"
assert testcase.inputs["data"] == "[1,2]"
assert len(testcase.invariant_optset) == 1
assert len(testcase.invariant_opts) == 1
assert testcase.invariant_opts["config"] == "opt"
assert testcase.profiling_events[0].client.string == "localhost"
assert testcase.profiling_events[0].type.string == "generate"
assert testcase.profiling_events[0].duration_ms == 100
assert testcase.profiling_events[
0].event_start == labdate.DatetimeFromMillisecondsTimestamp(1021312312)
assert testcase.profiling_events[1].client.string == "localhost"
assert testcase.profiling_events[1].type.string == "foo"
assert testcase.profiling_events[1].duration_ms == 100
assert testcase.profiling_events[
1].event_start == labdate.DatetimeFromMillisecondsTimestamp(1230812312)
def test_duplicate_testcase_testbed_ignored(session):
"""Test that result is ignored if testbed and testcase are not unique."""
proto = deepsmith_pb2.Result(
testcase=deepsmith_pb2.Testcase(
toolchain="cpp",
generator=deepsmith_pb2.Generator(name="generator"),
harness=deepsmith_pb2.Harness(name="harness"),
inputs={"src": "void main() {}", "data": "[1,2]",},
invariant_opts={"config": "opt",},
profiling_events=[
deepsmith_pb2.ProfilingEvent(
client="localhost",
type="generate",
duration_ms=100,
event_start_epoch_ms=1123123123,
),
],
),
testbed=deepsmith_pb2.Testbed(
toolchain="cpp", name="clang", opts={"arch": "x86_64"},
),
returncode=0,
outputs={"stdout": "Hello, world!"},
profiling_events=[
deepsmith_pb2.ProfilingEvent(
client="localhost",
type="exec",
duration_ms=100,
event_start_epoch_ms=1123123123,
),
],
outcome=deepsmith_pb2.Result.PASS,
)
r1 = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto)
session.add(r1)
session.flush()
# Attempt to add a new result which is identical to the first in all fields
# except for the outputs.
proto.outputs["stdout"] = "!"
r2 = deeplearning.deepsmith.result.Result.GetOrAdd(session, proto)
session.add(r2)
session.flush()
# Check that only one result was added.
assert session.query(deeplearning.deepsmith.result.Result).count() == 1
# Check that only the first result was added.
r3 = session.query(deeplearning.deepsmith.result.Result).first()
assert r3.outputs["stdout"] == "Hello, world!"
def OpenClSourceToTestCases(src: str) -> typing.List[deepsmith_pb2.Testcase]:
"""Generate DeepSmith testcases for each of the combination of gsize and
lsize used in CGO'17 synthetic kernels."""
return [
deepsmith_pb2.Testcase(
toolchain="opencl",
harness=deepsmith_pb2.Harness(name="cldrive"),
inputs={
"src": src,
"gsize": f"{gsize},1,1",
"lsize": f"{lsize},1,1",
},
) for lsize, gsize in LSIZE_GSIZE_PAIRS
]