def SubmitTestcases(
datastore_stub: datastore_pb2_grpc.DataStoreServiceStub,
testcases: typing.List[deepsmith_pb2.Testcase]) -> None:
request = services.BuildDefaultRequest(datastore_pb2.SubmitTestcasesRequest)
request.testcases.extend(testcases)
response = datastore_stub.SubmitTestcases(request)
services.AssertResponseStatus(response.status)
def GetGeneratorCapabilities(
generator_stub: generator_pb2_grpc.GeneratorServiceStub
) -> generator_pb2.GetCapabilitiesResponse:
request = services.BuildDefaultRequest(generator_pb2.GetCapabilitiesRequest)
response = generator_stub.GetGeneratorCapabilities(request)
services.AssertResponseStatus(response.status)
return response
def GetHarnessCapabilities(
harness_stub: harness_pb2_grpc.HarnessServiceStub,
) -> harness_pb2.GetHarnessCapabilitiesResponse:
request = services.BuildDefaultRequest(
harness_pb2.GetHarnessCapabilitiesRequest)
response = harness_stub.GetHarnessCapabilities(request)
services.AssertResponseStatus(response.status)
return response
def RunTestcases(
harness_stub: harness_pb2_grpc.HarnessServiceStub,
testbed: deepsmith_pb2.Testbed,
testcases: typing.List[deepsmith_pb2.Testcase],
) -> typing.List[deepsmith_pb2.Result]:
request = services.BuildDefaultRequest(harness_pb2.RunTestcasesRequest)
request.testbed.CopyFrom(testbed)
request.testcases.extend(testcases)
response = harness_stub.RunTestcases(request)
services.AssertResponseStatus(response.status)
return response.results
def GenerateTestcases(
generator_stub: generator_pb2_grpc.GeneratorServiceStub,
num_to_generate: int) -> typing.List[deepsmith_pb2.Testcase]:
logging.info(f'Generating batch of {num_to_generate} testcases')
request = services.BuildDefaultRequest(generator_pb2.GenerateTestcasesRequest)
response = generator_stub.GenerateTestcases(request)
services.AssertResponseStatus(response.status)
num_generated = len(response.testcases)
if num_generated != num_to_generate:
logging.warning(
f'Requested {num_to_generate} testcases, received {num_generated}')
return response.testcases
def GetNumberOfResultsForTestbed(
datastore_stub: datastore_pb2_grpc.DataStoreServiceStub,
harness: deepsmith_pb2.Harness, testbed: deepsmith_pb2.Testbed) -> int:
request = services.BuildDefaultRequest(datastore_pb2.GetResultsRequest)
request.return_total_matching_count = True
request.return_results = False
request.toolchain = testbed.toolchain
request.harness.CopyFrom(harness)
request.testbed.CopyFrom(testbed)
response = datastore_stub.GetTestcases(request)
services.AssertResponseStatus(response.status)
return response.total_matching_count
def GetNumberOfTestcasesInDataStore(
datastore_stub: datastore_pb2_grpc.DataStoreServiceStub,
capabilities: generator_pb2.GetCapabilitiesResponse) -> int:
request = services.BuildDefaultRequest(datastore_pb2.GetTestcasesRequest)
request.return_total_matching_count = True
request.return_testcases = False
request.mark_pending_results = False
request.include_testcases_with_results = True
request.include_testcases_with_pending_results = True
request.toolchain = capabilities.toolchain
request.generator.CopyFrom(capabilities.generator)
response = datastore_stub.GetTestcases(request)
services.AssertResponseStatus(response.status)
return response.total_matching_count
def GetTestcasesToRun(datastore_stub: datastore_pb2_grpc.DataStoreServiceStub,
harness: deepsmith_pb2.Harness,
testbed: deepsmith_pb2.Testbed,
target_total_results: int,
batch_size: int) -> typing.List[deepsmith_pb2.Testcase]:
if target_total_results >= 0:
total_results = GetNumberOfResultsForTestbed(datastore_stub, harness,
testbed)
total_testcases = GetNumberOfTestcases(datastore_stub, harness,
testbed)
batch_size = min(batch_size, total_results - total_testcases)
request = services.BuildDefaultRequest(datastore_pb2.GetTestcasesRequest)
request.toolchain = testbed.toolchain
request.harness.CopyFrom(harness)
request.testbed.CopyFrom(testbed)
request.max_num_testcases_to_return = batch_size
request.mark_results_pending.extend([testbed])
response = datastore_stub.GetTestcases(request)
services.AssertResponseStatus(response.status)
return response.testcases