def benchmarkOneQuery(request, repeatLimit=3, pageLimit=3):
"""
Repeat the query several times; perhaps don't go through *all* the
pages. Returns minimum time to run backend.searchVariants() to execute
the query (as far as pageLimit allows), *not* including JSON
processing to prepare queries or parse responses.
"""
times = []
queryString = protocol.toJson(request)
for i in range(0, repeatLimit):
resultString, elapsedTime = timeOneSearch(queryString)
accruedTime = elapsedTime
pageCount = 1
token = extractNextPageToken(resultString)
# Iterate to go beyond the first page of results.
while token is not None and pageCount < pageLimit:
pageRequest = request
pageRequest.page_token = token
pageRequestString = protocol.toJson(pageRequest)
resultString, elapsedTime = timeOneSearch(pageRequestString)
accruedTime += elapsedTime
pageCount = pageCount + 1
token = extractNextPageToken(resultString)
times.append(accruedTime)
# TODO: more sophisticated statistics. Sometimes we want min(),
# sometimes mean = sum() / len(), sometimes other measures,
# perhaps exclude outliers...
# If we compute average we should throw out at least the first one.
# return sum(times[2:])/len(times[2:])
return min(times)
def benchmarkOneQuery(request, repeatLimit=3, pageLimit=3):
"""
Repeat the query several times; perhaps don't go through *all* the
pages. Returns minimum time to run backend.searchVariants() to execute
the query (as far as pageLimit allows), *not* including JSON
processing to prepare queries or parse responses.
"""
times = []
queryString = protocol.toJson(request)
for i in range(0, repeatLimit):
resultString, elapsedTime = timeOneSearch(queryString)
accruedTime = elapsedTime
pageCount = 1
token = extractNextPageToken(resultString)
# Iterate to go beyond the first page of results.
while token is not None and pageCount < pageLimit:
pageRequest = request
pageRequest.page_token = token
pageRequestString = protocol.toJson(pageRequest)
resultString, elapsedTime = timeOneSearch(pageRequestString)
accruedTime += elapsedTime
pageCount = pageCount + 1
token = extractNextPageToken(resultString)
times.append(accruedTime)
# TODO: more sophisticated statistics. Sometimes we want min(),
# sometimes mean = sum() / len(), sometimes other measures,
# perhaps exclude outliers...
# If we compute average we should throw out at least the first one.
# return sum(times[2:])/len(times[2:])
return min(times)
def testValidateObjects(self):
# test that validation works on reference sets and references
referenceSet = self._gaObject
referenceSetPe = referenceSet.toProtocolElement()
self.assertValid(protocol.ReferenceSet,
protocol.toJson(referenceSetPe))
for gaReference in referenceSet.getReferences():
reference = protocol.toJson(gaReference.toProtocolElement())
self.assertValid(protocol.Reference, reference)
def testValidateObjects(self):
# test that validation works on reference sets and references
referenceSet = self._gaObject
referenceSetPe = referenceSet.toProtocolElement()
self.assertValid(
protocol.ReferenceSet, protocol.toJson(referenceSetPe))
for gaReference in referenceSet.getReferences():
reference = protocol.toJson(gaReference.toProtocolElement())
self.assertValid(protocol.Reference, reference)
def sendSearchRequest(self, path, request, responseClass):
"""
Sends the specified protocol request instance as JSON, and
parses the result into an instance of the specified response.
"""
response = self.sendJsonPostRequest(path, protocol.toJson(request))
self.assertEqual(200, response.status_code)
responseData = protocol.fromJson(response.data, responseClass)
self.assertTrue(
protocol.validate(protocol.toJson(responseData), responseClass))
return responseData
def sendSearchRequest(self, path, request, responseClass):
"""
Sends the specified protocol request instance as JSON, and
parses the result into an instance of the specified response.
"""
response = self.sendJsonPostRequest(path, protocol.toJson(request))
self.assertEqual(200, response.status_code)
responseData = protocol.fromJson(response.data, responseClass)
self.assertTrue(
protocol.validate(protocol.toJson(responseData), responseClass))
return responseData
def testVariantsSearch(self):
referenceName = '1'
request = protocol.SearchVariantsRequest()
request.reference_name = referenceName
request.start = 0
request.end = 0
request.variant_set_id = self.variantSet.getId()
# Request windows is too small, no results
path = '/variants/search'
responseData = self.sendSearchRequest(
path, request, protocol.SearchVariantsResponse)
self.assertEqual("", responseData.next_page_token)
self.assertEqual(0, len(responseData.variants))
# Larger request window, expect results
request.end = 2 ** 16
responseData = self.sendSearchRequest(
path, request, protocol.SearchVariantsResponse)
self.assertTrue(protocol.validate(
protocol.toJson(responseData), protocol.SearchVariantsResponse))
self.assertGreater(len(responseData.variants), 0)
# Verify all results are in the correct range, set and reference
for variant in responseData.variants:
self.assertGreaterEqual(variant.start, 0)
self.assertLessEqual(variant.end, 2 ** 16)
self.assertEqual(variant.variant_set_id, self.variantSet.getId())
self.assertEqual(variant.reference_name, referenceName)
def testVariantsValid(self):
end = datamodel.PysamDatamodelMixin.vcfMax
for reference_name in self._reference_names:
iterator = self._gaObject.getVariants(
reference_name, 0, end)
for gaVariant in iterator:
self.assertValid(protocol.Variant, protocol.toJson(gaVariant))
def testToProtocolElement(self):
dataset = datasets.Dataset('dataset1')
term = protocol.OntologyTerm()
term.term = "male genotypic sex"
term.id = "PATO:0020001"
term.source_name = "PATO"
term.source_version = pb.string("2015-11-18")
# Write out a valid input
print(protocol.toJsonDict(term))
validIndividual = protocol.Individual(name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z",
sex=term)
validIndividual.info['test'].values.add().string_value = 'test-info'
# pass through protocol creation
individual = bioMetadata.Individual(dataset, "test")
individual.populateFromJson(protocol.toJson(validIndividual))
gaIndividual = individual.toProtocolElement()
# Verify elements exist
self.assertEqual(gaIndividual.created, validIndividual.created)
self.assertEqual(gaIndividual.updated, validIndividual.updated)
# Invalid input
invalidIndividual = '{"bad:", "json"}'
individual = bioMetadata.Individual(dataset, "test")
# Should fail
self.assertRaises(exceptions.InvalidJsonException,
individual.populateFromJson, invalidIndividual)
def runListReferenceBases(self, id_, requestArgs):
"""
Runs a listReferenceBases request for the specified ID and
request arguments.
"""
compoundId = datamodel.ReferenceCompoundId.parse(id_)
referenceSet = self.getDataRepository().getReferenceSet(
compoundId.reference_set_id)
reference = referenceSet.getReference(id_)
start = _parseIntegerArgument(requestArgs, 'start', 0)
end = _parseIntegerArgument(requestArgs, 'end', reference.getLength())
if end == 0: # assume meant "get all"
end = reference.getLength()
if 'pageToken' in requestArgs:
pageTokenStr = requestArgs['pageToken']
if pageTokenStr != "":
start = _parsePageToken(pageTokenStr, 1)[0]
chunkSize = self._maxResponseLength
nextPageToken = None
if start + chunkSize < end:
end = start + chunkSize
nextPageToken = str(start + chunkSize)
sequence = reference.getBases(start, end)
# build response
response = protocol.ListReferenceBasesResponse()
response.offset = start
response.sequence = sequence
if nextPageToken is not None:
response.next_page_token = nextPageToken
return protocol.toJson(response)
def assertRequestRaises(self, exceptionClass, url, request):
"""
Verifies that the specified request returns a protocol exception
corresponding to the specified exception class.
"""
self.assertRawRequestRaises(
exceptionClass, url, protocol.toJson(request))
def runListReferenceBases(self, id_, requestArgs):
"""
Runs a listReferenceBases request for the specified ID and
request arguments.
"""
compoundId = datamodel.ReferenceCompoundId.parse(id_)
referenceSet = self.getDataRepository().getReferenceSet(
compoundId.reference_set_id)
reference = referenceSet.getReference(id_)
start = _parseIntegerArgument(requestArgs, 'start', 0)
end = _parseIntegerArgument(requestArgs, 'end', reference.getLength())
if end == 0: # assume meant "get all"
end = reference.getLength()
if 'pageToken' in requestArgs:
pageTokenStr = requestArgs['pageToken']
if pageTokenStr != "":
start = _parsePageToken(pageTokenStr, 1)[0]
chunkSize = self._maxResponseLength
nextPageToken = None
if start + chunkSize < end:
end = start + chunkSize
nextPageToken = str(start + chunkSize)
sequence = reference.getBases(start, end)
# build response
response = protocol.ListReferenceBasesResponse()
response.offset = start
response.sequence = sequence
if nextPageToken is not None:
response.next_page_token = nextPageToken
return protocol.toJson(response)
def testToProtocolElement(self):
dataset = datasets.Dataset('dataset1')
term = protocol.OntologyTerm()
term.term = "male genotypic sex"
term.id = "PATO:0020001"
term.source_name = "PATO"
term.source_version = pb.string("2015-11-18")
# Write out a valid input
print(protocol.toJsonDict(term))
validIndividual = protocol.Individual(
name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z",
sex=term)
validIndividual.info['test'].values.add().string_value = 'test-info'
# pass through protocol creation
individual = bioMetadata.Individual(
dataset, "test")
individual.populateFromJson(protocol.toJson(validIndividual))
gaIndividual = individual.toProtocolElement()
# Verify elements exist
self.assertEqual(gaIndividual.created, validIndividual.created)
self.assertEqual(gaIndividual.updated, validIndividual.updated)
# Invalid input
invalidIndividual = '{"bad:", "json"}'
individual = bioMetadata.Individual(dataset, "test")
# Should fail
self.assertRaises(
exceptions.InvalidJsonException,
individual.populateFromJson,
invalidIndividual)
def assertRequestRaises(self, exceptionClass, url, request):
"""
Verifies that the specified request returns a protocol exception
corresponding to the specified exception class.
"""
self.assertRawRequestRaises(exceptionClass, url,
protocol.toJson(request))
def setUp(self):
self.parser = cli_repomanager.RepoManager.getParser()
self.registryPath = 'a/repo/path'
self.datasetName = "datasetName"
self.filePath = 'a/file/path'
self.dirPath = 'a/dir/path/'
self.individualName = "test"
self.bioSampleName = "test"
self.individual = protocol.toJson(
protocol.Individual(name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z"))
self.bioSample = protocol.toJson(
protocol.BioSample(name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z"))
def runGetRequest(self, obj):
"""
Runs a get request by converting the specified datamodel
object into its protocol representation.
"""
protocolElement = obj.toProtocolElement()
jsonString = protocol.toJson(protocolElement)
return jsonString
def setUp(self):
self.parser = cli.RepoManager.getParser()
self.registryPath = 'a/repo/path'
self.datasetName = "datasetName"
self.filePath = 'a/file/path'
self.individualName = "test"
self.bioSampleName = "test"
self.individual = protocol.toJson(
protocol.Individual(
name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z"))
self.bioSample = protocol.toJson(
protocol.BioSample(
name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z"))
def testVariantsValid(self):
end = datamodel.PysamDatamodelMixin.vcfMax
for referenceName in self._referenceNames:
iterator = self._gaObject.getVariantAnnotations(
referenceName, 0, end)
for gaVariant, gaVariantAnnotation in iterator:
self.assertValid(protocol.VariantAnnotation,
protocol.toJson(gaVariantAnnotation))
def runGetRequest(self, obj):
"""
Runs a get request by converting the specified datamodel
object into its protocol representation.
"""
protocolElement = obj.toProtocolElement()
jsonString = protocol.toJson(protocolElement)
return jsonString
def _runSearchPageRequest(
self, protocolRequest, objectName, protocolResponseClass):
url = posixpath.join(self._urlPrefix, objectName + '/search')
data = protocol.toJson(protocolRequest)
self._logger.debug("request:{}".format(data))
response = self._session.post(
url, params=self._getHttpParameters(), data=data)
self._checkResponseStatus(response)
return self._deserializeResponse(response.text, protocolResponseClass)
def _run_list_reference_bases_page_request(self, request):
url_suffix = "listreferencebases"
url = posixpath.join(self._url_prefix, url_suffix)
response = self._session.post(url,
params=self._get_http_parameters(),
data=protocol.toJson(request))
self._check_response_status(response)
return self._deserialize_response(response.text,
protocol.ListReferenceBasesResponse)
def assertObjectNotSupported(self, response):
"""
Checks that the specified response returns a not supported 501 status
"""
self.assertEqual(501, response.status_code)
error = protocol.fromJson(response.data, protocol.GAException)
self.assertTrue(protocol.validate(protocol.toJson(error), type(error)))
self.assertGreater(error.error_code, 0)
self.assertGreater(len(error.message), 0)
def _runSearchPageRequest(
self, protocolRequest, objectName, protocolResponseClass):
url = posixpath.join(self._urlPrefix, objectName + '/search')
data = protocol.toJson(protocolRequest)
self._logger.debug("request:{}".format(data))
response = self._session.post(
url, params=self._getHttpParameters(), data=data)
self._checkResponseStatus(response)
return self._deserializeResponse(response.text, protocolResponseClass)
def assertObjectNotFound(self, response):
"""
Checks that the specified response contains a search failure.
"""
self.assertEqual(404, response.status_code)
error = protocol.fromJson(response.data, protocol.GAException)
self.assertTrue(protocol.validate(protocol.toJson(error), type(error)))
self.assertGreater(error.error_code, 0)
self.assertGreater(len(error.message), 0)
def _run_list_reference_bases_page_request(self, request):
url_suffix = "listreferencebases"
url = posixpath.join(self._url_prefix, url_suffix)
response = self._session.post(
url, params=self._get_http_parameters(),
data=protocol.toJson(request))
self._check_response_status(response)
return self._deserialize_response(
response.text, protocol.ListReferenceBasesResponse)
def _run_search_page_request(
self, protocol_request, object_name, protocol_response_class):
url = posixpath.join(self._url_prefix, object_name + '/search')
data = protocol.toJson(protocol_request)
self._logger.debug("request:{}".format(data))
response = self._session.post(
url, params=self._get_http_parameters(), data=data)
self._check_response_status(response)
return self._deserialize_response(
response.text, protocol_response_class)
def sendPostRequest(self, path, request):
"""
Sends the specified GA request object and returns the response.
"""
headers = {
'Content-type': 'application/json',
'Origin': self.exampleUrl,
}
return self.app.post(
path, headers=headers, data=protocol.toJson(request))
def runGetFeature(self, id_):
"""
Returns JSON string of the feature object corresponding to
the feature compoundID passed in.
"""
compoundId = datamodel.FeatureCompoundId.parse(id_)
dataset = self.getDataRepository().getDataset(compoundId.dataset_id)
featureSet = dataset.getFeatureSet(compoundId.feature_set_id)
gaFeature = featureSet.getFeature(compoundId)
jsonString = protocol.toJson(gaFeature)
return jsonString
def sendPostRequest(self, path, request):
"""
Sends the specified GA request object and returns the response.
"""
headers = {
'Content-type': 'application/json',
'Origin': self.exampleUrl,
}
return self.app.post(path,
headers=headers,
data=protocol.toJson(request))
def _run_search_page_request(self, protocol_request, object_name,
protocol_response_class):
url = posixpath.join(self._url_prefix, object_name + '/search')
data = protocol.toJson(protocol_request)
self._logger.debug("request:{}".format(data))
response = self._session.post(url,
params=self._get_http_parameters(),
data=data)
self._check_response_status(response)
return self._deserialize_response(response.text,
protocol_response_class)
def runGetFeature(self, id_):
"""
Returns JSON string of the feature object corresponding to
the feature compoundID passed in.
"""
compoundId = datamodel.FeatureCompoundId.parse(id_)
dataset = self.getDataRepository().getDataset(compoundId.dataset_id)
featureSet = dataset.getFeatureSet(compoundId.feature_set_id)
gaFeature = featureSet.getFeature(compoundId)
jsonString = protocol.toJson(gaFeature)
return jsonString
def testGoodMappings(self):
ontology = self._gaObject
for term in self._oboReader:
self.assertIn(term.id, ontology.getTermIds(term.name))
gaTerm = ontology.getGaTermByName(term.name)
self.assertTrue(protocol.validate(protocol.toJson(gaTerm),
OntologyTerm))
self.assertEqual(gaTerm.term, term.name)
self.assertIn(gaTerm.id, ontology.getTermIds(term.name))
self.assertEqual(
gaTerm.source_version, ontology.getSourceVersion())
self.assertEqual(gaTerm.source_name, ontology.getName())
def runGetVariant(self, id_):
"""
Returns a variant with the given id
"""
compoundId = datamodel.VariantCompoundId.parse(id_)
dataset = self.getDataRepository().getDataset(compoundId.dataset_id)
variantSet = dataset.getVariantSet(compoundId.variant_set_id)
gaVariant = variantSet.getVariant(compoundId)
# TODO variant is a special case here, as it's returning a
# protocol element rather than a datamodel object. We should
# fix this for consistency.
jsonString = protocol.toJson(gaVariant)
return jsonString
def runGetVariant(self, id_):
"""
Returns a variant with the given id
"""
compoundId = datamodel.VariantCompoundId.parse(id_)
dataset = self.getDataRepository().getDataset(compoundId.dataset_id)
variantSet = dataset.getVariantSet(compoundId.variant_set_id)
gaVariant = variantSet.getVariant(compoundId)
# TODO variant is a special case here, as it's returning a
# protocol element rather than a datamodel object. We should
# fix this for consistency.
jsonString = protocol.toJson(gaVariant)
return jsonString
def handleException(exception):
"""
Handles an exception that occurs somewhere in the process of handling
a request.
"""
serverException = exception
if not isinstance(exception, exceptions.BaseServerException):
with app.test_request_context():
app.log_exception(exception)
serverException = exceptions.getServerError(exception)
error = serverException.toProtocolElement()
responseStr = protocol.toJson(error)
return getFlaskResponse(responseStr, serverException.httpStatus)
def handleException(exception):
"""
Handles an exception that occurs somewhere in the process of handling
a request.
"""
serverException = exception
if not isinstance(exception, exceptions.BaseServerException):
with app.test_request_context():
app.log_exception(exception)
serverException = exceptions.getServerError(exception)
error = serverException.toProtocolElement()
responseStr = protocol.toJson(error)
return getFlaskResponse(responseStr, serverException.httpStatus)
def testUnsupportedReadOperations(self):
path = '/reads/search'
# unmapped Reads
request = protocol.SearchReadsRequest()
request.read_group_ids.extend([self.readGroup.getId()])
request.reference_id = ""
self.verifySearchMethodNotSupported(request, path)
# multiple ReadGroupSets set mismatch
request.read_group_ids.append(self.readGroup.getId())
request.read_group_ids.append("42")
request.reference_id = self.reference.getId()
response = self.sendJsonPostRequest(path, protocol.toJson(request))
self.assertEqual(400, response.status_code)
def testVariantAnnotationSetsSearch(self):
self.assertIsNotNone(self.variantAnnotationSet)
request = protocol.SearchVariantAnnotationSetsRequest()
request.variant_set_id = "b4d=="
path = '/variantannotationsets/search'
response = self.sendJsonPostRequest(path, protocol.toJson(request))
responseData = protocol.fromJson(response.data, protocol.GAException)
self.assertTrue(protocol.validate(protocol.toJson(responseData),
protocol.GAException))
self.assertEqual(responseData.error_code, 758389611)
self.assertEqual(responseData.message,
'No object of this type exists with id \'b4d==\'')
request.variant_set_id = self.variantSet.getId()
response = self.sendJsonPostRequest(path, protocol.toJson(request))
responseData = protocol.fromJson(response.data, protocol.
SearchVariantAnnotationSetsResponse)
self.assertTrue(protocol.validate(
protocol.toJson(responseData),
protocol.SearchVariantAnnotationSetsResponse))
self.assertGreater(len(responseData.variant_annotation_sets), 0,
"Expect some results for a known good ID")
def runListReferenceBases(self, requestJson):
"""
Runs a listReferenceBases request for the specified ID and
request arguments.
"""
# In the case when an empty post request is made to the endpoint
# we instantiate an empty ListReferenceBasesRequest.
if not requestJson:
request = protocol.ListReferenceBasesRequest()
else:
try:
request = protocol.fromJson(
requestJson,
protocol.ListReferenceBasesRequest)
except protocol.json_format.ParseError:
raise exceptions.InvalidJsonException(requestJson)
compoundId = datamodel.ReferenceCompoundId.parse(request.reference_id)
referenceSet = self.getDataRepository().getReferenceSet(
compoundId.reference_set_id)
reference = referenceSet.getReference(request.reference_id)
start = request.start
end = request.end
if end == 0: # assume meant "get all"
end = reference.getLength()
if request.page_token:
pageTokenStr = request.page_token
start = paging._parsePageToken(pageTokenStr, 1)[0]
chunkSize = self._maxResponseLength
nextPageToken = None
if start + chunkSize < end:
end = start + chunkSize
nextPageToken = str(start + chunkSize)
sequence = reference.getBases(start, end)
# build response
response = protocol.ListReferenceBasesResponse()
response.offset = start
response.sequence = sequence
if nextPageToken:
response.next_page_token = nextPageToken
return protocol.toJson(response)
def testToProtocolElement(self):
dataset = datasets.Dataset('dataset1')
# Write out a valid input
validBioSample = protocol.BioSample(name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z")
validBioSample.info['test'].values.add().string_value = 'test-info'
# pass through protocol creation
bioSample = bioMetadata.BioSample(dataset, "test")
bioSample.populateFromJson(protocol.toJson(validBioSample))
gaBioSample = bioSample.toProtocolElement()
# Verify elements exist
self.assertEqual(gaBioSample.created, validBioSample.created)
self.assertEqual(gaBioSample.updated, validBioSample.updated)
# Invalid input
invalidBioSample = '{"bad:", "json"}'
bioSample = bioMetadata.Individual(dataset, "test")
# Should fail
self.assertRaises(exceptions.InvalidJsonException,
bioSample.populateFromJson, invalidBioSample)
def runListReferenceBases(self, requestJson):
"""
Runs a listReferenceBases request for the specified ID and
request arguments.
"""
# In the case when an empty post request is made to the endpoint
# we instantiate an empty ListReferenceBasesRequest.
if not requestJson:
request = protocol.ListReferenceBasesRequest()
else:
try:
request = protocol.fromJson(requestJson,
protocol.ListReferenceBasesRequest)
except protocol.json_format.ParseError:
raise exceptions.InvalidJsonException(requestJson)
compoundId = datamodel.ReferenceCompoundId.parse(request.reference_id)
referenceSet = self.getDataRepository().getReferenceSet(
compoundId.reference_set_id)
reference = referenceSet.getReference(request.reference_id)
start = request.start
end = request.end
if end == 0: # assume meant "get all"
end = reference.getLength()
if request.page_token:
pageTokenStr = request.page_token
start = paging._parsePageToken(pageTokenStr, 1)[0]
chunkSize = self._maxResponseLength
nextPageToken = None
if start + chunkSize < end:
end = start + chunkSize
nextPageToken = str(start + chunkSize)
sequence = reference.getBases(start, end)
# build response
response = protocol.ListReferenceBasesResponse()
response.offset = start
response.sequence = sequence
if nextPageToken:
response.next_page_token = nextPageToken
return protocol.toJson(response)
def post(self, url, params=None, data=None):
self.checkSessionParameters()
assert url.startswith(self._urlPrefix)
suffix = url[len(self._urlPrefix):]
searchSuffix = "/search"
if suffix.endswith(searchSuffix):
datatype = suffix[1:-len(searchSuffix)]
assert datatype in self._searchMethodMap
method = self._searchMethodMap[datatype]
result = method(data)
else:
# ListReferenceBases is an oddball and needs to be treated
# separately.
data = json.loads(data)
args = protocol.ListReferenceBasesRequest()
args.reference_id = data.get('referenceId', "")
args.start = int(data.get('start', 0))
args.end = int(data.get('end', 0))
args.page_token = data.get('pageToken', "")
result = self._backend.runListReferenceBases(protocol.toJson(args))
return DummyResponse(result)
def post(self, url, params=None, data=None):
self.checkSessionParameters()
assert url.startswith(self._urlPrefix)
suffix = url[len(self._urlPrefix):]
searchSuffix = "/search"
if suffix.endswith(searchSuffix):
datatype = suffix[1:-len(searchSuffix)]
assert datatype in self._searchMethodMap
method = self._searchMethodMap[datatype]
result = method(data)
else:
# ListReferenceBases is an oddball and needs to be treated
# separately.
data = json.loads(data)
args = protocol.ListReferenceBasesRequest()
args.reference_id = data.get('referenceId', "")
args.start = int(data.get('start', 0))
args.end = int(data.get('end', 0))
args.page_token = data.get('pageToken', "")
result = self._backend.runListReferenceBases(
protocol.toJson(args))
return DummyResponse(result)
def testToProtocolElement(self):
dataset = datasets.Dataset('dataset1')
# Write out a valid input
validBioSample = protocol.BioSample(
name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z")
validBioSample.info['test'].values.add().string_value = 'test-info'
# pass through protocol creation
bioSample = bioMetadata.BioSample(
dataset, "test")
bioSample.populateFromJson(protocol.toJson(validBioSample))
gaBioSample = bioSample.toProtocolElement()
# Verify elements exist
self.assertEqual(gaBioSample.created, validBioSample.created)
self.assertEqual(gaBioSample.updated, validBioSample.updated)
# Invalid input
invalidBioSample = '{"bad:", "json"}'
bioSample = bioMetadata.Individual(dataset, "test")
# Should fail
self.assertRaises(
exceptions.InvalidJsonException,
bioSample.populateFromJson,
invalidBioSample)
def testProtocolElementValid(self):
self.assertValid(self.getProtocolClass(),
protocol.toJson(self._gaObject.toProtocolElement()))
def _verifyVariantCalls():
for gaCall in gaVariant.calls:
self.assertValid(protocol.Call, protocol.toJson(gaCall))
self.assertIn(gaCall.call_set_name, pyvcfCallMap)
pyvcfCall = pyvcfCallMap[gaCall.call_set_name]
self._verifyVariantCallEqual(gaCall, pyvcfCall)
def _verifyVariantCalls():
for gaCall in gaVariant.calls:
self.assertValid(protocol.Call, protocol.toJson(gaCall))
self.assertIn(gaCall.call_set_name, pyvcfCallMap)
pyvcfCall = pyvcfCallMap[gaCall.call_set_name]
self._verifyVariantCallEqual(gaCall, pyvcfCall)
class RepoManagerEndToEndTest(unittest.TestCase):
datasetName = 'datasetOne'
metadata = {'description': 'aDescription'}
individualName = "test"
bioSampleName = "test"
individual = protocol.toJson(
protocol.Individual(name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z"))
bioSample = protocol.toJson(
protocol.BioSample(name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z"))
def setUp(self):
_, self.repoFile = tempfile.mkstemp(
prefix='ga4gh_repo_manager_end2end_test')
os.unlink(self.repoFile)
def tearDown(self):
if os.path.exists(self.repoFile):
os.unlink(self.repoFile)
def _runCmd(self, cmd, *args):
command = [cmd, self.repoFile] + list(args)
cli.repo_main(command)
def testEndToEnd(self):
self._runCmd("init")
self._runCmd("add-ontology", paths.ontologyPath)
self._runCmd("add-referenceset", paths.faPath, '-n',
paths.referenceSetName)
self._runCmd("add-dataset", self.datasetName)
self._runCmd("add-biosample", self.datasetName, self.bioSampleName,
self.bioSample)
self._runCmd("add-individual", self.datasetName, self.individualName,
self.individual)
self._runCmd("add-readgroupset", self.datasetName, paths.bamPath, '-R',
paths.referenceSetName, '-n', paths.readGroupSetName)
self._runCmd("add-featureset", self.datasetName, paths.featuresPath,
'-R', paths.referenceSetName, '-O', paths.ontologyName)
# ensure we can handle trailing slashes
vcfPath = paths.vcfDirPath + '/'
self._runCmd("add-variantset", self.datasetName, vcfPath, '-R',
paths.referenceSetName)
variantAnnotationSetName = "vas"
self._runCmd("add-variantset", self.datasetName,
paths.annotatedVcfPath, '-R', paths.referenceSetName,
"-aO", paths.ontologyName, "-n", variantAnnotationSetName)
self._runCmd("verify")
self._runCmd("list")
self._runCmd("remove-variantset", self.datasetName,
paths.variantSetName, "-f")
self._runCmd("remove-variantset", self.datasetName,
variantAnnotationSetName, "-f")
self._runCmd("remove-readgroupset", self.datasetName,
paths.readGroupSetName, "-f")
self._runCmd("remove-featureset", self.datasetName,
paths.featureSetName, "-f")
self._runCmd("remove-dataset", self.datasetName, "-f")
self._runCmd("remove-referenceset", paths.referenceSetName, "-f")
self._runCmd("remove-ontology", paths.ontologyName, "-f")
def testForce(self):
datasetName = 'dataset1'
self._runCmd("init")
self._runCmd("add-dataset", datasetName)
with mock.patch('ga4gh.cli.getRawInput', lambda x: 'N'):
self._runCmd("remove-dataset", datasetName)
with mock.patch('ga4gh.cli.getRawInput', lambda x: 'y'):
self._runCmd("remove-dataset", datasetName)
with self.assertRaises(SystemExit):
self._runCmd("remove-dataset", datasetName)
def _run_search_page_request(
self, protocol_request, object_name, protocol_response_class):
search_method = self._search_method_map[object_name]
response_json = search_method(protocol.toJson(protocol_request))
return self._deserialize_response(
response_json, protocol_response_class)
def _run_list_reference_bases_page_request(self, request):
response_json = self._backend.runListReferenceBases(
protocol.toJson(request))
return self._deserialize_response(response_json,
protocol.ListReferenceBasesResponse)
def _runSearchPageRequest(
self, protocolRequest, objectName, protocolResponseClass):
searchMethod = self._searchMethodMap[objectName]
responseJson = searchMethod(protocol.toJson(protocolRequest))
return self._deserializeResponse(responseJson, protocolResponseClass)
def _run_list_reference_bases_page_request(self, request):
response_json = self._backend.runListReferenceBases(
protocol.toJson(request))
return self._deserialize_response(
response_json, protocol.ListReferenceBasesResponse)
def _runSearchPageRequest(
self, protocolRequest, objectName, protocolResponseClass):
searchMethod = self._searchMethodMap[objectName]
responseJson = searchMethod(protocol.toJson(protocolRequest))
return self._deserializeResponse(responseJson, protocolResponseClass)
def _run_search_page_request(self, protocol_request, object_name,
protocol_response_class):
search_method = self._search_method_map[object_name]
response_json = search_method(protocol.toJson(protocol_request))
return self._deserialize_response(response_json,
protocol_response_class)
def testVariantAnnotationsSearch(self):
self.assertIsNotNone(self.variantAnnotationSet)
request = protocol.SearchVariantAnnotationsRequest()
# TODO split these into separate tests, and factor out the duplicated
# code.
path = '/variantannotations/search'
request.start = 0
request.end = 1000000
request.page_size = 1
request.reference_name = "1"
request.variant_annotation_set_id = self.variantAnnotationSet.getId()
response = self.sendJsonPostRequest(path, protocol.toJson(request))
responseData = protocol.fromJson(response.data, protocol.
SearchVariantAnnotationsResponse)
self.assertGreater(len(responseData.variant_annotations), 0)
self.assertIsNotNone(
responseData.next_page_token,
"Expected more than one page of results")
request = protocol.SearchVariantAnnotationsRequest()
request.variant_annotation_set_id = self.variantAnnotationSet.getId()
request.start = 0
request.end = 10
request.reference_name = "1"
request.effects.add().id = "ThisIsNotAnEffect"
response = self.sendJsonPostRequest(path, protocol.toJson(request))
responseData = protocol.fromJson(response.data, protocol.
SearchVariantAnnotationsResponse)
self.assertEquals(
len(responseData.variant_annotations), 0,
"There should be no results for a nonsense effect")
request = protocol.SearchVariantAnnotationsRequest()
request.variant_annotation_set_id = self.variantAnnotationSet.getId()
request.start = 0
request.end = 10
request.reference_name = "1"
response = self.sendJsonPostRequest(path, protocol.toJson(request))
responseData = protocol.fromJson(response.data, protocol.
SearchVariantAnnotationsResponse)
self.assertGreater(len(responseData.variant_annotations), 0)
for ann in responseData.variant_annotations:
self.assertGreater(
len(ann.transcript_effects), 0,
("When no effects are requested ensure "
"some transcript effects are still present"))
request = protocol.SearchVariantAnnotationsRequest()
request.variant_annotation_set_id = self.variantAnnotationSet.getId()
request.start = 0
request.end = 5
request.reference_name = "1"
request.effects.add().id = "SO:0001627"
request.effects.add().id = "B4DID"
response = self.sendJsonPostRequest(path, protocol.toJson(request))
responseData = protocol.fromJson(response.data, protocol.
SearchVariantAnnotationsResponse)
responseLength = len(responseData.variant_annotations)
self.assertGreater(
responseLength, 0,
"There should be some results for a known effect")
for ann in responseData.variant_annotations:
effectPresent = False
for effect in ann.transcript_effects:
for featureType in effect.effects:
if featureType.id in map(
lambda e: e.id, request.effects):
effectPresent = True
self.assertEquals(
True, effectPresent,
"The ontology term should appear at least once")
request = protocol.SearchVariantAnnotationsRequest()
request.variant_annotation_set_id = self.variantAnnotationSet.getId()
request.start = 0
request.end = 5
request.reference_name = "1"
request.effects.add().id = "B4DID"
request.effects.add().id = "SO:0001627"
response = self.sendJsonPostRequest(path, protocol.toJson(request))
responseData = protocol.fromJson(response.data, protocol.
SearchVariantAnnotationsResponse)
self.assertEqual(
len(responseData.variant_annotations),
responseLength,
"Order shall not affect results")
for ann in responseData.variant_annotations:
effectPresent = False
for effect in ann.transcript_effects:
for featureType in effect.effects:
if featureType.id in map(
lambda e: e.id, request.effects):
effectPresent = True
self.assertEquals(
True,
effectPresent,
"The ontology term should appear at least once")
request = protocol.SearchVariantAnnotationsRequest()
request.variant_annotation_set_id = self.variantAnnotationSet.getId()
request.start = 0
request.end = 5
request.reference_name = "1"
request.effects.add().id = "SO:0001627"
response = self.sendJsonPostRequest(path, protocol.toJson(request))
responseData = protocol.fromJson(response.data, protocol.
SearchVariantAnnotationsResponse)
self.assertGreater(len(responseData.variant_annotations), 0,
"There should be some results for a good effect ID")
for ann in responseData.variant_annotations:
effectPresent = False
for effect in ann.transcript_effects:
for featureType in effect.effects:
if featureType.id in map(
lambda e: e.id, request.effects):
effectPresent = True
self.assertEquals(True, effectPresent,
"The ontology term should appear at least once")
request = protocol.SearchVariantAnnotationsRequest()
request.variant_annotation_set_id = self.variantAnnotationSet.getId()
request.start = 0
request.end = 10
request.reference_name = "1"
request.effects.add().id = "SO:0001627"
request.effects.add().id = "SO:0001791"
response = self.sendJsonPostRequest(path, protocol.toJson(request))
responseData = protocol.fromJson(response.data, protocol.
SearchVariantAnnotationsResponse)
self.assertGreater(len(responseData.variant_annotations), 0)
def testProtocolElementValid(self):
self.assertValid(
self.getProtocolClass(),
protocol.toJson(self._gaObject.toProtocolElement()))
