def benchmarkOneQuery(request, repeatLimit=3, pageLimit=3):
"""
Repeat the query several times; perhaps don't go through *all* the
pages. Returns minimum time to run backend.searchVariants() to execute
the query (as far as pageLimit allows), *not* including JSON
processing to prepare queries or parse responses.
"""
times = []
queryString = protocol.toJson(request)
for i in range(0, repeatLimit):
resultString, elapsedTime = timeOneSearch(queryString)
accruedTime = elapsedTime
pageCount = 1
token = extractNextPageToken(resultString)
# Iterate to go beyond the first page of results.
while token is not None and pageCount < pageLimit:
pageRequest = request
pageRequest.page_token = token
pageRequestString = protocol.toJson(pageRequest)
resultString, elapsedTime = timeOneSearch(pageRequestString)
accruedTime += elapsedTime
pageCount = pageCount + 1
token = extractNextPageToken(resultString)
times.append(accruedTime)
# TODO: more sophisticated statistics. Sometimes we want min(),
# sometimes mean = sum() / len(), sometimes other measures,
# perhaps exclude outliers...
# If we compute average we should throw out at least the first one.
# return sum(times[2:])/len(times[2:])
return min(times)
def handleException(exception):
"""
Handles an exception that occurs somewhere in the process of handling
a request.
"""
serverException = exception
if not isinstance(exception, exceptions.BaseServerException):
with app.test_request_context():
app.log_exception(exception)
serverException = exceptions.getServerError(exception)
error = serverException.toProtocolElement()
# If the exception is being viewed by a web browser, we can render a nicer
# view.
if flask.request and 'Accept' in flask.request.headers and \
flask.request.headers['Accept'].find('text/html') != -1:
message = "<h1>Error {}</h1><pre>{}</pre>".format(
serverException.httpStatus,
protocol.toJson(error))
if serverException.httpStatus == 401 \
or serverException.httpStatus == 403:
message += "Please try <a href=\"/login\">logging in</a>."
return message
else:
responseStr = protocol.toJson(error)
return getFlaskResponse(responseStr, serverException.httpStatus)
def handleException(exception):
"""
Handles an exception that occurs somewhere in the process of handling
a request.
"""
serverException = exception
if not isinstance(exception, exceptions.BaseServerException):
with app.test_request_context():
app.log_exception(exception)
serverException = exceptions.getServerError(exception)
error = serverException.toProtocolElement()
# If the exception is being viewed by a web browser, we can render a nicer
# view.
if flask.request and 'Accept' in flask.request.headers and \
flask.request.headers['Accept'].find('text/html') != -1:
message = "<h1>Error {}</h1><pre>{}</pre>".format(
serverException.httpStatus,
protocol.toJson(error))
if serverException.httpStatus == 401 \
or serverException.httpStatus == 403:
message += "Please try <a href=\"/login\">logging in</a>."
return message
else:
responseStr = protocol.toJson(error)
return getFlaskResponse(responseStr, serverException.httpStatus)
def benchmarkOneQuery(request, repeatLimit=3, pageLimit=3):
"""
Repeat the query several times; perhaps don't go through *all* the
pages. Returns minimum time to run backend.searchVariants() to execute
the query (as far as pageLimit allows), *not* including JSON
processing to prepare queries or parse responses.
"""
times = []
queryString = protocol.toJson(request)
for i in range(0, repeatLimit):
resultString, elapsedTime = timeOneSearch(queryString)
accruedTime = elapsedTime
pageCount = 1
token = extractNextPageToken(resultString)
# Iterate to go beyond the first page of results.
while token is not None and pageCount < pageLimit:
pageRequest = request
pageRequest.page_token = token
pageRequestString = protocol.toJson(pageRequest)
resultString, elapsedTime = timeOneSearch(pageRequestString)
accruedTime += elapsedTime
pageCount = pageCount + 1
token = extractNextPageToken(resultString)
times.append(accruedTime)
# TODO: more sophisticated statistics. Sometimes we want min(),
# sometimes mean = sum() / len(), sometimes other measures,
# perhaps exclude outliers...
# If we compute average we should throw out at least the first one.
# return sum(times[2:])/len(times[2:])
return min(times)
def testValidateObjects(self):
# test that validation works on reference sets and references
referenceSet = self._gaObject
referenceSetPe = referenceSet.toProtocolElement()
self.assertValid(protocol.ReferenceSet,
protocol.toJson(referenceSetPe))
for gaReference in referenceSet.getReferences():
reference = protocol.toJson(gaReference.toProtocolElement())
self.assertValid(protocol.Reference, reference)
def testValidateObjects(self):
# test that validation works on reference sets and references
referenceSet = self._gaObject
referenceSetPe = referenceSet.toProtocolElement()
self.assertValid(
protocol.ReferenceSet, protocol.toJson(referenceSetPe))
for gaReference in referenceSet.getReferences():
reference = protocol.toJson(gaReference.toProtocolElement())
self.assertValid(protocol.Reference, reference)
def sendSearchRequest(self, path, request, responseClass):
"""
Sends the specified protocol request instance as JSON, and
parses the result into an instance of the specified response.
"""
response = self.sendJsonPostRequest(path, protocol.toJson(request))
self.assertEqual(200, response.status_code)
responseData = protocol.fromJson(response.data, responseClass)
self.assertTrue(
protocol.validate(protocol.toJson(responseData), responseClass))
return responseData
def sendSearchRequest(self, path, request, responseClass):
"""
Sends the specified protocol request instance as JSON, and
parses the result into an instance of the specified response.
"""
response = self.sendJsonPostRequest(path, protocol.toJson(request))
self.assertEqual(200, response.status_code)
responseData = protocol.fromJson(response.data, responseClass)
self.assertTrue(
protocol.validate(protocol.toJson(responseData), responseClass))
return responseData
def sendSearchRequest(self, path, request, responseClass):
"""
Sends the specified protocol request instance as JSON, and
parses the result into an instance of the specified response.
"""
response = self.sendJsonPostRequest(path, protocol.toJson(request))
# self.assertEqual(200, response.status_code) federated search can return 404
responseData = self.deserialize(response.data, responseClass)
# responseData = protocol.fromProtobufString(response.data,
# responseClass)
self.assertTrue(
protocol.validate(protocol.toJson(responseData), responseClass))
return responseData
def _run_http_post_request(self, protocol_request, path,
protocol_response_class):
if path == "announce":
response_json = self._backend.runAddAnnouncement(
protocol.toJson(protocol_request))
return self._deserialize_response(response_json,
protocol_response_class)
elif path == "peers/list":
response_json = self._backend.runListPeers(
protocol.toJson(protocol_request))
return self._deserialize_response(response_json,
protocol_response_class)
else:
raise NotImplemented()
def sendSearchRequest(self, path, request, responseClass):
"""
Sends the specified protocol request instance as JSON, and
parses the result into an instance of the specified response.
"""
response = self.sendJsonPostRequest(path, protocol.toJson(request))
# self.assertEqual(200, response.status_code) federated search can return 404
response_json = json.loads(response.data)
response = json.dumps(response_json.get('results', {}))
responseData = protocol.deserialize(response, self.serialization,
responseClass)
self.assertTrue(
protocol.validate(protocol.toJson(responseData), responseClass))
return responseData
def testToJsonAndFromJson(self):
classes = protocol.getProtocolClasses()
for clazz in classes:
obj = clazz()
jsonStr = protocol.toJson(obj)
obj2 = protocol.fromJson(jsonStr, clazz)
self.assertTrue(obj, obj2)
def testToJsonAndFromJson(self):
classes = protocol.getProtocolClasses()
for clazz in classes:
obj = clazz()
jsonStr = protocol.toJson(obj)
obj2 = protocol.fromJson(jsonStr, clazz)
self.assertTrue(obj, obj2)
def test_search_reads(self):
run_accession = 'SRR2856889' # paired
# acc = 'SRR1482462' # unpaired
reference_name = 'chr1'
start = 9000000
end = 9100000
request = protocol.SearchReadsRequest()
request.read_group_ids.extend([run_accession])
request.reference_id = reference_name
request.start = start
request.end = end
reads_list = ncbi.search_reads(request)
response = self.client.post('/reads/search',
data=protocol.toJson(request))
response_proto = protocol.fromJson(response.get_data(),
protocol.SearchReadsResponse)
self.assertIsNotNone(response_proto, "Something should be returned")
self.assertEqual(response.status_code, 200, "A known good endpoint "
"should return success")
i = 0
for ga_alignment in response_proto.alignments:
self.assertEqual(reads_list[i], ga_alignment)
i += 1
self.assertGreater(len(response_proto.alignments), 0,
"Some alignments should be returned.")
def testToProtocolElement(self):
dataset = datasets.Dataset('dataset1')
term = protocol.OntologyTerm()
term.term = "male genotypic sex"
term.term_id = "PATO:0020001"
# Write out a valid input
print(protocol.toJsonDict(term))
validIndividual = protocol.Individual(name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z",
sex=term)
validIndividual.attributes.attr['test']. \
values.add().string_value = 'test-info'
# pass through protocol creation
individual = bioMetadata.Individual(dataset, "test")
individual.populateFromJson(protocol.toJson(validIndividual))
gaIndividual = individual.toProtocolElement()
# Verify elements exist
self.assertEqual(gaIndividual.created, validIndividual.created)
self.assertEqual(gaIndividual.updated, validIndividual.updated)
# Invalid input
invalidIndividual = '{"bad:", "json"}'
individual = bioMetadata.Individual(dataset, "test")
# Should fail
self.assertRaises(exceptions.InvalidJsonException,
individual.populateFromJson, invalidIndividual)
def testToProtocolElement(self):
dataset = datasets.Dataset('dataset1')
term = protocol.OntologyTerm()
term.term = "male genotypic sex"
term.term_id = "PATO:0020001"
# Write out a valid input
print(protocol.toJsonDict(term))
validIndividual = protocol.Individual(
name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z",
sex=term)
validIndividual.attributes.attr['test']. \
values.add().string_value = 'test-info'
# pass through protocol creation
individual = bioMetadata.Individual(
dataset, "test")
individual.populateFromJson(protocol.toJson(validIndividual))
gaIndividual = individual.toProtocolElement()
# Verify elements exist
self.assertEqual(gaIndividual.created, validIndividual.created)
self.assertEqual(gaIndividual.updated, validIndividual.updated)
# Invalid input
invalidIndividual = '{"bad:", "json"}'
individual = bioMetadata.Individual(dataset, "test")
# Should fail
self.assertRaises(
exceptions.InvalidJsonException,
individual.populateFromJson,
invalidIndividual)
def assertRequestRaises(self, exceptionClass, url, request):
"""
Verifies that the specified request returns a protocol exception
corresponding to the specified exception class.
"""
self.assertRawRequestRaises(exceptionClass, url,
protocol.toJson(request))
def testToProtocolElement(self):
dataset = datasets.Dataset('dataset1')
# Write out a valid input
validBiosample = protocol.Biosample(
name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z")
validBiosample.attributes.attr['test']. \
values.add().string_value = 'test-info'
# pass through protocol creation
biosample = bioMetadata.Biosample(
dataset, "test")
biosample.populateFromJson(protocol.toJson(validBiosample))
gaBiosample = biosample.toProtocolElement()
# Verify elements exist
self.assertEqual(gaBiosample.created, validBiosample.created)
self.assertEqual(gaBiosample.updated, validBiosample.updated)
# Invalid input
invalidBiosample = '{"bad:", "json"}'
biosample = bioMetadata.Individual(dataset, "test")
# Should fail
self.assertRaises(
exceptions.InvalidJsonException,
biosample.populateFromJson,
invalidBiosample)
def setUp(self):
self.parser = cli_repomanager.RepoManager.getParser()
self.registryPath = 'a/repo/path'
self.datasetName = "datasetName"
self.filePath = 'a/file/path'
self.dirPath = 'a/dir/path/'
self.individualName = "test"
self.biosampleName = "test"
self.individual = protocol.toJson(
protocol.Individual(name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z"))
self.biosample = protocol.toJson(
protocol.Biosample(name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z"))
def assertRequestRaises(self, exceptionClass, url, request):
"""
Verifies that the specified request returns a protocol exception
corresponding to the specified exception class.
"""
self.assertRawRequestRaises(
exceptionClass, url, protocol.toJson(request))
def testVariantsValid(self):
end = datamodel.PysamDatamodelMixin.vcfMax
for reference_name in self._reference_names:
iterator = self._gaObject.getVariants(
reference_name, 0, end)
for gaVariant in iterator:
self.assertValid(protocol.Variant, protocol.toJson(gaVariant))
def runGetRequest(self, obj):
"""
Runs a get request by converting the specified datamodel
object into its protocol representation.
"""
protocolElement = obj.toProtocolElement()
jsonString = protocol.toJson(protocolElement)
return jsonString
def getSerializedResponse(self):
"""
Returns a string version of the SearchResponse that has
been built by this SearchResponseBuilder.
"""
self._protoObject.next_page_token = pb.string(self._nextPageToken)
s = protocol.toJson(self._protoObject)
return s
def getSerializedResponse(self):
"""
Returns a string version of the SearchResponse that has
been built by this SearchResponseBuilder.
"""
self._protoObject.next_page_token = pb.string(self._nextPageToken)
s = protocol.toJson(self._protoObject)
return s
def testVariantsValid(self):
end = datamodel.PysamDatamodelMixin.vcfMax
for referenceName in self._referenceNames:
iterator = self._gaObject.getVariantAnnotations(
referenceName, 0, end)
for gaVariant, gaVariantAnnotation in iterator:
self.assertValid(protocol.VariantAnnotation,
protocol.toJson(gaVariantAnnotation))
def runGetRequest(self, obj):
"""
Runs a get request by converting the specified datamodel
object into its protocol representation.
"""
protocolElement = obj.toProtocolElement()
jsonString = protocol.toJson(protocolElement)
return jsonString
def testGoodMappings(self):
ontology = self._gaObject
for term in self._oboReader:
self.assertIn(term.id, ontology.getTermIds(term.name))
gaTerm = ontology.getGaTermByName(term.name)
self.assertTrue(protocol.validate(protocol.toJson(gaTerm),
OntologyTerm))
self.assertEqual(gaTerm.term, term.name)
self.assertIn(gaTerm.term_id, ontology.getTermIds(term.name))
def setUp(self):
self.parser = cli_repomanager.RepoManager.getParser()
self.registryPath = 'a/repo/path'
self.datasetName = "datasetName"
self.filePath = 'a/file/path'
self.dirPath = 'a/dir/path/'
self.individualName = "test"
self.biosampleName = "test"
self.individual = protocol.toJson(
protocol.Individual(
name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z"))
self.biosample = protocol.toJson(
protocol.Biosample(
name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z"))
def _run_list_reference_bases_page_request(self, request):
url_suffix = "listreferencebases"
url = posixpath.join(self._url_prefix, url_suffix)
response = self._session.post(url,
params=self._get_http_parameters(),
data=protocol.toJson(request))
self._check_response_status(response)
return self._deserialize_response(response.text,
protocol.ListReferenceBasesResponse)
def testToProtocolElement(self):
dataset = datasets.Dataset('dataset1')
validFusionDetection = protocol.FusionDetection(
name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z",
fusionDetectionId="n/a",
fusionDetectionIdTier=0,
sampleId="n/a",
sampleIdTier=0,
inHousePipeline="n/a",
inHousePipelineTier=0,
svDetection="n/a",
svDetectionTier=0,
fusionDetection="n/a",
fusionDetectionTier=0,
realignment="n/a",
realignmentTier=0,
annotation="n/a",
annotationTier=0,
genomeReference="n/a",
genomeReferenceTier=0,
geneModels="n/a",
geneModelsTier=0,
alignmentId="n/a",
alignmentIdTier=0,
site="n/a",
siteTier=0)
validFusionDetection.attributes.attr['test']. \
values.add().string_value = 'test-info'
# pass through protocol creation
fusionDetection = pipeMetadata.FusionDetection(dataset, "test")
fusionDetection.populateFromJson(protocol.toJson(validFusionDetection))
gaFusionDetection = fusionDetection.toProtocolElement()
# Verify select elements exist
self.assertEqual(gaFusionDetection.created,
validFusionDetection.created)
self.assertEqual(gaFusionDetection.updated,
validFusionDetection.updated)
self.assertEqual(gaFusionDetection.fusionDetectionId,
validFusionDetection.fusionDetectionId)
self.assertEqual(gaFusionDetection.sampleId,
validFusionDetection.sampleId)
self.assertEqual(gaFusionDetection.site, validFusionDetection.site)
self.assertEqual(gaFusionDetection.fusionDetectionTier,
validFusionDetection.fusionDetectionTier)
# Invalid input
invalidFusionDetection = '{"bad:", "json"}'
fusionDetection = pipeMetadata.FusionDetection(dataset, "test")
# Should fail
self.assertRaises(exceptions.InvalidJsonException,
fusionDetection.populateFromJson,
invalidFusionDetection)
def sendPostRequest(self, path, request):
"""
Sends the specified GA request object and returns the response.
"""
headers = {
'Content-type': 'application/json',
'Origin': self.exampleUrl,
}
return self.app.post(
path, headers=headers, data=protocol.toJson(request))
def runGetFeature(self, id_):
"""
Returns JSON string of the feature object corresponding to
the feature compoundID passed in.
"""
compoundId = datamodel.FeatureCompoundId.parse(id_)
dataset = self.getDataRepository().getDataset(compoundId.dataset_id)
featureSet = dataset.getFeatureSet(compoundId.feature_set_id)
gaFeature = featureSet.getFeature(compoundId)
jsonString = protocol.toJson(gaFeature)
return jsonString
def runGetFeature(self, id_):
"""
Returns JSON string of the feature object corresponding to
the feature compoundID passed in.
"""
compoundId = datamodel.FeatureCompoundId.parse(id_)
dataset = self.getDataRepository().getDataset(compoundId.dataset_id)
featureSet = dataset.getFeatureSet(compoundId.feature_set_id)
gaFeature = featureSet.getFeature(compoundId)
jsonString = protocol.toJson(gaFeature)
return jsonString
def _run_search_page_request(self, protocol_request, object_name,
protocol_response_class):
url = posixpath.join(self._url_prefix, object_name + '/search')
data = protocol.toJson(protocol_request)
self._logger.debug("request:{}".format(data))
response = self._session.post(url,
params=self._get_http_parameters(),
data=data)
self._check_response_status(response)
return self._deserialize_response(response.text,
protocol_response_class)
def sendPostRequest(self, path, request):
"""
Sends the specified GA request object and returns the response.
"""
headers = {
'Content-type': 'application/json',
'Origin': self.exampleUrl,
}
return self.app.post(path,
headers=headers,
data=protocol.toJson(request))
def testToProtocolElement(self):
dataset = datasets.Dataset('dataset1')
validExpressionAnalysis = protocol.ExpressionAnalysis(
name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z",
expressionAnalysisId="n/a",
expressionAnalysisIdTier=0,
sampleId="n/a",
sampleIdTier=0,
readLength="n/a",
readLengthTier=0,
reference="n/a",
referenceTier=0,
alignmentTool="n/a",
alignmentToolTier=0,
bamHandling="n/a",
bamHandlingTier=0,
expressionEstimation="n/a",
expressionEstimationTier=0,
sequencingId="n/a",
sequencingIdTier=0,
site="n/a",
siteTier=0)
validExpressionAnalysis.attributes.attr['test']. \
values.add().string_value = 'test-info'
# pass through protocol creation
expressionAnalysis = pipeMetadata.ExpressionAnalysis(dataset, "test")
expressionAnalysis.populateFromJson(
protocol.toJson(validExpressionAnalysis))
gaExpressionAnalysis = expressionAnalysis.toProtocolElement()
# Verify select elements exist
self.assertEqual(gaExpressionAnalysis.created,
validExpressionAnalysis.created)
self.assertEqual(gaExpressionAnalysis.updated,
validExpressionAnalysis.updated)
self.assertEqual(gaExpressionAnalysis.expressionAnalysisId,
validExpressionAnalysis.expressionAnalysisId)
self.assertEqual(gaExpressionAnalysis.sampleId,
validExpressionAnalysis.sampleId)
self.assertEqual(gaExpressionAnalysis.site,
validExpressionAnalysis.site)
self.assertEqual(gaExpressionAnalysis.bamHandlingTier,
validExpressionAnalysis.bamHandlingTier)
# Invalid input
invalidExpressionAnalysis = '{"bad:", "json"}'
expressionAnalysis = pipeMetadata.ExpressionAnalysis(dataset, "test")
# Should fail
self.assertRaises(exceptions.InvalidJsonException,
expressionAnalysis.populateFromJson,
invalidExpressionAnalysis)
def sendPostRequest(self, path, request, extraHeaders=None):
"""
Sends the specified GA request object and returns the response.
"""
headers = {
'Content-type': 'application/json',
'Origin': self.exampleUrl,
}
if extraHeaders:
headers.update(extraHeaders)
return self.client.post(
path, headers=headers, data=protocol.toJson(request))
def runAddAnnouncement(self, flaskrequest):
"""
Takes a flask request from the frontend and attempts to parse
into an AnnouncePeerRequest. If successful, it will log the
announcement to the `announcement` table with some other metadata
gathered from the request.
"""
announcement = {}
# We want to parse the request ourselves to collect a little more
# data about it.
try:
requestData = protocol.fromJson(
flaskrequest.get_data(), protocol.AnnouncePeerRequest)
announcement['hostname'] = flaskrequest.host_url
announcement['remote_addr'] = flaskrequest.remote_addr
announcement['user_agent'] = flaskrequest.headers.get('User-Agent')
except AttributeError:
# Sometimes in testing we will send protocol requests instead
# of flask requests and so the hostname and user agent won't
# be present.
try:
requestData = protocol.fromJson(
flaskrequest, protocol.AnnouncePeerRequest)
except Exception as e:
raise exceptions.InvalidJsonException(e)
except Exception as e:
raise exceptions.InvalidJsonException(e)
# Validate the url before accepting the announcement
peer = datamodel.peers.Peer(requestData.peer.url)
peer.setAttributesJson(protocol.toJson(
requestData.peer.attributes))
announcement['url'] = peer.getUrl()
announcement['attributes'] = peer.getAttributes()
try:
self.getDataRepository().insertAnnouncement(announcement)
except:
raise exceptions.BadRequestException(announcement['url'])
return protocol.toJson(
protocol.AnnouncePeerResponse(success=True))
def runAddAnnouncement(self, flaskrequest):
"""
Takes a flask request from the frontend and attempts to parse
into an AnnouncePeerRequest. If successful, it will log the
announcement to the `announcement` table with some other metadata
gathered from the request.
"""
announcement = {}
# We want to parse the request ourselves to collect a little more
# data about it.
try:
requestData = protocol.fromJson(
flaskrequest.get_data(), protocol.AnnouncePeerRequest)
announcement['hostname'] = flaskrequest.host_url
announcement['remote_addr'] = flaskrequest.remote_addr
announcement['user_agent'] = flaskrequest.headers.get('User-Agent')
except AttributeError:
# Sometimes in testing we will send protocol requests instead
# of flask requests and so the hostname and user agent won't
# be present.
try:
requestData = protocol.fromJson(
flaskrequest, protocol.AnnouncePeerRequest)
except Exception as e:
raise exceptions.InvalidJsonException(e)
except Exception as e:
raise exceptions.InvalidJsonException(e)
# Validate the url before accepting the announcement
peer = datamodel.peers.Peer(requestData.peer.url)
peer.setAttributesJson(protocol.toJson(
requestData.peer.attributes))
announcement['url'] = peer.getUrl()
announcement['attributes'] = peer.getAttributes()
try:
self.getDataRepository().insertAnnouncement(announcement)
except:
raise exceptions.BadRequestException(announcement['url'])
return protocol.toJson(
protocol.AnnouncePeerResponse(success=True))
def runGetVariant(self, id_):
"""
Returns a variant with the given id
"""
compoundId = datamodel.VariantCompoundId.parse(id_)
dataset = self.getDataRepository().getDataset(compoundId.dataset_id)
variantSet = dataset.getVariantSet(compoundId.variant_set_id)
gaVariant = variantSet.getVariant(compoundId)
# TODO variant is a special case here, as it's returning a
# protocol element rather than a datamodel object. We should
# fix this for consistency.
jsonString = protocol.toJson(gaVariant)
return jsonString
def testGenotypesSearchByNameError(self):
"""
Search for feature by name with a malformed regular expression.
"""
# setup phenotype query
request = protocol.SearchFeaturesRequest()
datasetName, featureSet = self.getCGDDataSetFeatureSet()
request.feature_set_id = featureSet.id
request.name = "*" # invalid regular expression
postUrl = "features/search"
response = self.sendJsonPostRequest(postUrl, protocol.toJson(request))
self.assertEqual(400, response.status_code)
def runGetVariant(self, id_):
"""
Returns a variant with the given id
"""
compoundId = datamodel.VariantCompoundId.parse(id_)
dataset = self.getDataRepository().getDataset(compoundId.dataset_id)
variantSet = dataset.getVariantSet(compoundId.variant_set_id)
gaVariant = variantSet.getVariant(compoundId)
# TODO variant is a special case here, as it's returning a
# protocol element rather than a datamodel object. We should
# fix this for consistency.
jsonString = protocol.toJson(gaVariant)
return jsonString
def testGenotypesSearchByNameError(self):
"""
Search for feature by name with a malformed regular expression.
"""
# setup phenotype query
request = protocol.SearchFeaturesRequest()
datasetName, featureSet = self.getCGDDataSetFeatureSet()
request.feature_set_id = featureSet.id
request.name = "*" # invalid regular expression
postUrl = "features/search"
response = self.sendJsonPostRequest(postUrl, protocol.toJson(request))
self.assertEqual(400, response.status_code)
def testToProtocolElement(self):
dataset = datasets.Dataset('dataset1')
validTreatment = protocol.Treatment(
name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z",
patientId="PATIENT_TEST",
patientIdTier=0,
courseNumber="n/a",
courseNumberTier=0,
therapeuticModality="n/a",
therapeuticModalityTier=0,
treatmentPlanType="n/a",
treatmentPlanTypeTier=0,
treatmentIntent="n/a",
treatmentIntentTier=0,
startDate="n/a",
startDateTier=0,
stopDate="n/a",
stopDateTier=0,
reasonForEndingTheTreatment="n/a",
reasonForEndingTheTreatmentTier=0,
responseToTreatment="n/a",
responseToTreatmentTier=0,
responseCriteriaUsed="n/a",
responseCriteriaUsedTier=0,
dateOfRecurrenceOrProgressionAfterThisTreatment="n/a",
dateOfRecurrenceOrProgressionAfterThisTreatmentTier=0,
unexpectedOrUnusualToxicityDuringTreatment="n/a",
unexpectedOrUnusualToxicityDuringTreatmentTier=0)
validTreatment.attributes.attr['test']. \
values.add().string_value = 'test-info'
# pass through protocol creation
treatment = clinMetadata.Treatment(dataset, "test")
treatment.populateFromJson(protocol.toJson(validTreatment))
gaTreatment = treatment.toProtocolElement()
# Verify select elements exist
self.assertEqual(gaTreatment.created, validTreatment.created)
self.assertEqual(gaTreatment.updated, validTreatment.updated)
self.assertEqual(gaTreatment.patientId, validTreatment.patientId)
self.assertEqual(gaTreatment.courseNumber, validTreatment.courseNumber)
self.assertEqual(gaTreatment.startDate, validTreatment.startDate)
# Invalid input
invalidTreatment = '{"bad:", "json"}'
treatment = clinMetadata.Treatment(dataset, "test")
# Should fail
self.assertRaises(exceptions.InvalidJsonException,
treatment.populateFromJson, invalidTreatment)
def testToProtocolElement(self):
dataset = datasets.Dataset('dataset1')
validSequencing = protocol.Sequencing(name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z",
sequencingId="Sequencing_TEST",
sequencingIdTier=0,
sampleId="Sample_test",
sampleIdTier=0,
dnaLibraryKit="n/a",
dnaLibraryKitTier=0,
dnaSeqPlatform="n/a",
dnaSeqPlatformTier=0,
dnaReadLength="n/a",
dnaReadLengthTier=0,
rnaLibraryKit="n/a",
rnaLibraryKitTier=0,
rnaSeqPlatform="n/a",
rnaSeqPlatformTier=0,
rnaReadLength="n/a",
rnaReadLengthTier=0,
pcrCycles="n/a",
pcrCyclesTier=0,
extractionId="n/a",
extractionIdTier=0,
site="Toronto",
siteTier=0)
validSequencing.attributes.attr['test']. \
values.add().string_value = 'test-info'
# pass through protocol creation
sequencing = pipeMetadata.Sequencing(dataset, "test")
sequencing.populateFromJson(protocol.toJson(validSequencing))
gaSequencing = sequencing.toProtocolElement()
# Verify select elements exist
self.assertEqual(gaSequencing.created, validSequencing.created)
self.assertEqual(gaSequencing.updated, validSequencing.updated)
self.assertEqual(gaSequencing.sequencingId,
validSequencing.sequencingId)
self.assertEqual(gaSequencing.sampleId, validSequencing.sampleId)
self.assertEqual(gaSequencing.site, validSequencing.site)
self.assertEqual(gaSequencing.sampleIdTier,
validSequencing.sampleIdTier)
# Invalid input
invalidSequencing = '{"bad:", "json"}'
sequencing = pipeMetadata.Sequencing(dataset, "test")
# Should fail
self.assertRaises(exceptions.InvalidJsonException,
sequencing.populateFromJson, invalidSequencing)
def testToProtocolElement(self):
dataset = datasets.Dataset('dataset1')
validSurgery = protocol.Surgery(name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z",
patientId="PATIENT_TEST",
patientIdTier=0,
startDate="n/a",
startDateTier=0,
stopDate="n/a",
stopDateTier=0,
sampleId="n/a",
sampleIdTier=0,
collectionTimePoint="n/a",
collectionTimePointTier=0,
diagnosisDate="n/a",
diagnosisDateTier=0,
site="n/a",
siteTier=0,
type="n/a",
typeTier=0,
recordingDate="n/a",
recordingDateTier=0,
treatmentPlanId="n/a",
treatmentPlanIdTier=0,
courseNumber="1",
courseNumberTier=0)
validSurgery.attributes.attr['test']. \
values.add().string_value = 'test-info'
# pass through protocol creation
surgery = clinMetadata.Surgery(dataset, "test")
surgery.populateFromJson(protocol.toJson(validSurgery))
gaSurgery = surgery.toProtocolElement()
# Verify select elements exist
self.assertEqual(gaSurgery.created, validSurgery.created)
self.assertEqual(gaSurgery.updated, validSurgery.updated)
self.assertEqual(gaSurgery.patientId, validSurgery.patientId)
self.assertEqual(gaSurgery.collectionTimePoint,
validSurgery.collectionTimePoint)
self.assertEqual(gaSurgery.recordingDate, validSurgery.recordingDate)
self.assertEqual(gaSurgery.treatmentPlanIdTier,
validSurgery.treatmentPlanIdTier)
# Invalid input
invalidSurgery = '{"bad:", "json"}'
surgery = clinMetadata.Surgery(dataset, "test")
# Should fail
self.assertRaises(exceptions.InvalidJsonException,
surgery.populateFromJson, invalidSurgery)
def testRoundTripDataset(self):
id_ = "id"
name = "name"
description = "description"
dataset = protocol.Dataset()
dataset.id = id_
dataset.name = name
dataset.description = description
jsonStr = protocol.toJson(dataset)
newDataset = protocol.fromJson(jsonStr, Dataset)
self.assertEquals(dataset.id, id_)
self.assertEquals(dataset.name, name)
self.assertEquals(dataset.description, description)
datasetDict = protocol.toJsonDict(newDataset)
self.assertEquals(datasetDict['id'], id_)
self.assertEquals(datasetDict['name'], name)
self.assertEquals(datasetDict['description'], description)
def testToProtocolElement(self):
dataset = datasets.Dataset('dataset1')
validLabtest = protocol.Labtest(
name="test",
created="2016-05-19T21:00:19Z",
updated="2016-05-19T21:00:19Z",
patientId="PATIENT_TEST",
patientIdTier=0,
startDate="n/a",
startDateTier=0,
collectionDate="n/a",
collectionDateTier=0,
endDate="n/a",
endDateTier=0,
eventType="n/a",
eventTypeTier=0,
testResults="n/a",
testResultsTier=0,
timePoint="n/a",
timePointTier=0,
recordingDate="n/a",
recordingDateTier=0,
)
validLabtest.attributes.attr['test']. \
values.add().string_value = 'test-info'
# pass through protocol creation
labtest = clinMetadata.Labtest(dataset, "test")
labtest.populateFromJson(protocol.toJson(validLabtest))
gaLabtest = labtest.toProtocolElement()
# Verify select elements exist
self.assertEqual(gaLabtest.created, validLabtest.created)
self.assertEqual(gaLabtest.updated, validLabtest.updated)
self.assertEqual(gaLabtest.patientId, validLabtest.patientId)
self.assertEqual(gaLabtest.eventType, validLabtest.eventType)
self.assertEqual(gaLabtest.collectionDate, validLabtest.collectionDate)
self.assertEqual(gaLabtest.recordingDateTier,
validLabtest.recordingDateTier)
# Invalid input
invalidLabtest = '{"bad:", "json"}'
labtest = clinMetadata.Labtest(dataset, "test")
# Should fail
self.assertRaises(exceptions.InvalidJsonException,
labtest.populateFromJson, invalidLabtest)
def runListReferenceBases(self, requestJson):
"""
Runs a listReferenceBases request for the specified ID and
request arguments.
"""
# In the case when an empty post request is made to the endpoint
# we instantiate an empty ListReferenceBasesRequest.
if not requestJson:
request = protocol.ListReferenceBasesRequest()
else:
try:
request = protocol.fromJson(
requestJson,
protocol.ListReferenceBasesRequest)
except protocol.json_format.ParseError:
raise exceptions.InvalidJsonException(requestJson)
compoundId = datamodel.ReferenceCompoundId.parse(request.reference_id)
referenceSet = self.getDataRepository().getReferenceSet(
compoundId.reference_set_id)
reference = referenceSet.getReference(request.reference_id)
start = request.start
end = request.end
if end == 0: # assume meant "get all"
end = reference.getLength()
if request.page_token:
pageTokenStr = request.page_token
start = paging._parsePageToken(pageTokenStr, 1)[0]
chunkSize = self._maxResponseLength
nextPageToken = None
if start + chunkSize < end:
end = start + chunkSize
nextPageToken = str(start + chunkSize)
sequence = reference.getBases(start, end)
# build response
response = protocol.ListReferenceBasesResponse()
response.offset = start
response.sequence = sequence
if nextPageToken:
response.next_page_token = nextPageToken
return protocol.toJson(response)
def post(self, url, params=None, data=None):
self.checkSessionParameters()
assert url.startswith(self._urlPrefix)
suffix = url[len(self._urlPrefix):]
searchSuffix = "/search"
if suffix.endswith(searchSuffix):
datatype = suffix[1:-len(searchSuffix)]
assert datatype in self._searchMethodMap
method = self._searchMethodMap[datatype]
result = method(data)
else:
# ListReferenceBases is an oddball and needs to be treated
# separately.
data = json.loads(data)
args = protocol.ListReferenceBasesRequest()
args.reference_id = data.get('referenceId', "")
args.start = int(data.get('start', 0))
args.end = int(data.get('end', 0))
args.page_token = data.get('pageToken', "")
result = self._backend.runListReferenceBases(
protocol.toJson(args))
return DummyResponse(result)
def testValidate(self):
classes = protocol.getProtocolClasses()
for clazz in classes:
obj = clazz()
jsonStr = protocol.toJson(obj)
protocol.validate(jsonStr, clazz)
def runGetInfo(self, request):
"""
Returns information about the service including protocol version.
"""
return protocol.toJson(protocol.GetInfoResponse(
protocol_version=protocol.version))
def assertAlignmentsEqual(self, gaAlignment, pysamAlignment,
readGroupInfo):
if pysamAlignment.query_qualities is None:
self.assertEqual(gaAlignment.aligned_quality, [])
else:
self.assertEqual(
gaAlignment.aligned_quality,
list(pysamAlignment.query_qualities))
self.assertEqual(
gaAlignment.aligned_sequence,
pysamAlignment.query_sequence)
if reads.SamFlags.isFlagSet(
pysamAlignment.flag, reads.SamFlags.READ_UNMAPPED):
self.assertEqual(0, gaAlignment.alignment.ByteSize())
else:
self.assertEqual(
gaAlignment.alignment.mapping_quality,
pysamAlignment.mapping_quality)
self.assertEqual(
gaAlignment.alignment.position.reference_name,
readGroupInfo.samFile.getrname(pysamAlignment.reference_id))
self.assertEqual(
gaAlignment.alignment.position.position,
pysamAlignment.reference_start)
# TODO test reverseStrand on position and on
# nextMatePosition once it has been implemented.
self.assertCigarEqual(
gaAlignment.alignment.cigar,
pysamAlignment.cigar)
self.assertFlag(
gaAlignment.duplicate_fragment,
pysamAlignment, reads.SamFlags.DUPLICATE_READ)
self.assertFlag(
gaAlignment.failed_vendor_quality_checks,
pysamAlignment, reads.SamFlags.FAILED_QUALITY_CHECK)
self.assertEqual(
gaAlignment.fragment_length,
pysamAlignment.template_length)
self.assertEqual(
gaAlignment.fragment_name,
pysamAlignment.query_name)
compoundId = datamodel.ReadAlignmentCompoundId(
self._gaObject.getCompoundId(),
pysamAlignment.query_name)
self.assertEqual(gaAlignment.id, str(compoundId))
ret = protocol.ReadAlignment()
for key, value in pysamAlignment.tags:
protocol.setAttribute(ret.attributes.attr[key].values, value)
self.assertEqual(
protocol.toJson(gaAlignment.attributes),
protocol.toJson(ret.attributes))
if reads.SamFlags.isFlagSet(
pysamAlignment.flag, reads.SamFlags.MATE_UNMAPPED):
self.assertEqual(0, gaAlignment.next_mate_position.ByteSize())
else:
self.assertEqual(
gaAlignment.next_mate_position.position,
pysamAlignment.next_reference_start)
if pysamAlignment.next_reference_id != -1:
self.assertEqual(
gaAlignment.next_mate_position.reference_name,
readGroupInfo.samFile.getrname(
pysamAlignment.next_reference_id))
else:
self.assertEqual(
gaAlignment.next_mate_position.reference_name, "")
if gaAlignment.number_reads == 1:
self.assertFlag(
False, pysamAlignment, reads.SamFlags.READ_PAIRED)
elif gaAlignment.number_reads == 2:
self.assertFlag(
True, pysamAlignment, reads.SamFlags.READ_PAIRED)
else:
# we shouldn't be setting numberReads to anything else
self.assertTrue(False)
if gaAlignment.read_number is -1:
self.assertFlag(
False, pysamAlignment, reads.SamFlags.FIRST_IN_PAIR)
self.assertFlag(
False, pysamAlignment, reads.SamFlags.SECOND_IN_PAIR)
elif gaAlignment.read_number == 0:
self.assertFlag(
True, pysamAlignment, reads.SamFlags.FIRST_IN_PAIR)
self.assertFlag(
False, pysamAlignment, reads.SamFlags.SECOND_IN_PAIR)
elif gaAlignment.read_number == 1:
self.assertFlag(
False, pysamAlignment, reads.SamFlags.FIRST_IN_PAIR)
self.assertFlag(
True, pysamAlignment, reads.SamFlags.SECOND_IN_PAIR)
elif gaAlignment.read_number == 2:
self.assertFlag(
True, pysamAlignment, reads.SamFlags.FIRST_IN_PAIR)
self.assertFlag(
True, pysamAlignment, reads.SamFlags.SECOND_IN_PAIR)
else:
# we shouldn't be setting readNumber to anything else
self.assertTrue(False)
self.assertFlag(
not gaAlignment.improper_placement,
pysamAlignment, reads.SamFlags.READ_PROPER_PAIR)
self.assertEqual(
gaAlignment.read_group_id,
readGroupInfo.id)
self.assertFlag(
gaAlignment.secondary_alignment,
pysamAlignment, reads.SamFlags.SECONDARY_ALIGNMENT)
self.assertFlag(
gaAlignment.supplementary_alignment,
pysamAlignment, reads.SamFlags.SUPPLEMENTARY_ALIGNMENT)
def _verifyVariantCalls():
for gaCall in gaVariant.calls:
self.assertValid(protocol.Call, protocol.toJson(gaCall))
self.assertIn(gaCall.call_set_name, pyvcfCallMap)
pyvcfCall = pyvcfCallMap[gaCall.call_set_name]
self._verifyVariantCallEqual(gaCall, pyvcfCall)
def testProtocolElementValid(self):
self.assertValid(
self.getProtocolClass(),
protocol.toJson(self._gaObject.toProtocolElement()))
