def scrubReadGroupSet(self, cliOutput, clientOutput):
for cliReadGroupSet, clientReadGroupSet in utils.zipLists(
cliOutput, clientOutput):
for cliReadGroup, clientReadGroup in utils.zipLists(
cliReadGroupSet['readGroups'],
clientReadGroupSet['readGroups']):
self._scrubReadGroup(cliReadGroup, clientReadGroup)
def scrubReadGroupSet(self, cliOutput, clientOutput):
for cliReadGroupSet, clientReadGroupSet in utils.zipLists(
cliOutput, clientOutput):
for cliReadGroup, clientReadGroup in utils.zipLists(
cliReadGroupSet['readGroups'],
clientReadGroupSet['readGroups']):
self._scrubReadGroup(cliReadGroup, clientReadGroup)
def testZipLists(self):
a = [1, 2]
b = [3, 4]
c = [5, 6]
d = [7, 8, 9]
e = [10]
result = utils.zipLists(a, b, c)
self.assertEqual(result, [(1, 3, 5), (2, 4, 6)])
with self.assertRaises(AssertionError):
utils.zipLists(c, d)
with self.assertRaises(AssertionError):
utils.zipLists(d, e)
def _testSearchMethod(self, repoIteratorMethod, clientSearchMethod):
repoObjs = list(repoIteratorMethod())
numRepoObjs = len(repoObjs)
pageSizes = self._getPageSizes(numRepoObjs)
for pageSize in pageSizes:
self._client.set_page_size(pageSize)
clientObjs = list(clientSearchMethod())
for repoObj, clientObj in utils.zipLists(repoObjs, clientObjs):
self._assertEqual(repoObj, clientObj)
def _testSearchMethod(self, repoIteratorMethod, clientSearchMethod):
repoObjs = list(repoIteratorMethod())
numRepoObjs = len(repoObjs)
pageSizes = self._getPageSizes(numRepoObjs)
for pageSize in pageSizes:
self._client.set_page_size(pageSize)
clientObjs = list(clientSearchMethod())
for repoObj, clientObj in utils.zipLists(repoObjs, clientObjs):
self._assertEqual(repoObj, clientObj)
def verifyObjectList(self, gaObjects, datamodelObjects, getMethod):
"""
Verifies that the specified list of protocol objects corresponds
to the specified list of datamodel objects.
"""
for gaObject, datamodelObject in utils.zipLists(
gaObjects, datamodelObjects):
self.assertEqual(gaObject, datamodelObject.toProtocolElement())
otherGaObject = getMethod(gaObject.id)
self.assertEqual(gaObject, otherGaObject)
def verifyObjectList(self, gaObjects, datamodelObjects, getMethod):
"""
Verifies that the specified list of protocol objects corresponds
to the specified list of datamodel objects.
"""
for gaObject, datamodelObject in utils.zipLists(
gaObjects, datamodelObjects):
self.assertEqual(gaObject, datamodelObject.toProtocolElement())
otherGaObject = getMethod(gaObject.id)
self.assertEqual(gaObject, otherGaObject)
def testSearchFeatures(self):
repoFeatureSets = self._repo.allFeatureSets()
for repoFeatureSet in repoFeatureSets:
repoFeatures = list(repoFeatureSet.getFeatures())
numFeatures = len(repoFeatures)
featureSetId = repoFeatureSet.getId()
pageSizes = self._getPageSizes(numFeatures)
for pageSize in pageSizes:
features = list(self._client.search_features(featureSetId))
for repoFeature, feature in utils.zipLists(
repoFeatures, features):
self.assertEqual(repoFeature, feature)
def testSearchFeatures(self):
repoFeatureSets = self._repo.allFeatureSets()
for repoFeatureSet in repoFeatureSets:
repoFeatures = list(repoFeatureSet.getFeatures())
numFeatures = len(repoFeatures)
featureSetId = repoFeatureSet.getId()
pageSizes = self._getPageSizes(numFeatures)
for pageSize in pageSizes:
features = list(self._client.search_features(featureSetId))
for repoFeature, feature in utils.zipLists(
repoFeatures, features):
self.assertEqual(repoFeature, feature)
def testSearchPhenotypes(self):
phenotypeAssociationSets = self._repo.allPhenotypeAssociationSets()
for phenotypeAssociationSet in phenotypeAssociationSets:
repoAssociations = list(phenotypeAssociationSet.getAssociations())
numAssociations = len(repoAssociations)
phenotypeAssociationSetId = phenotypeAssociationSet.getId()
pageSizes = self._getPageSizes(numAssociations)
for pageSize in pageSizes:
clientPhenotypes = list(self._client.search_phenotype
(phenotypeAssociationSetId))
for repoAssociation, clientPhenotype in utils.zipLists(
repoAssociations, clientPhenotypes):
self.assertEqual(repoAssociation.phenotype,
clientPhenotype)
def testSearchReferences(self):
repoReferenceSets = self._repo.getReferenceSets()
for repoReferenceSet in repoReferenceSets:
referenceSetId = repoReferenceSet.getId()
repoReferences = repoReferenceSet.getReferences()
numRefs = len(repoReferences)
pageSizes = self._getPageSizes(numRefs)
for pageSize in pageSizes:
self._client.set_page_size(pageSize)
references = list(
self._client.search_references(referenceSetId))
for repoReference, reference in utils.zipLists(
repoReferences, references):
self._assertEqual(repoReference, reference)
def testSearchPhenotypes(self):
phenotypeAssociationSets = self._repo.allPhenotypeAssociationSets()
for phenotypeAssociationSet in phenotypeAssociationSets:
repoAssociations = list(phenotypeAssociationSet.getAssociations())
numAssociations = len(repoAssociations)
phenotypeAssociationSetId = phenotypeAssociationSet.getId()
pageSizes = self._getPageSizes(numAssociations)
for pageSize in pageSizes:
clientPhenotypes = list(
self._client.search_phenotype(phenotypeAssociationSetId))
for repoAssociation, clientPhenotype in utils.zipLists(
repoAssociations, clientPhenotypes):
self.assertEqual(repoAssociation.phenotype,
clientPhenotype)
def testSearchReferences(self):
repoReferenceSets = self._repo.getReferenceSets()
for repoReferenceSet in repoReferenceSets:
referenceSetId = repoReferenceSet.getId()
repoReferences = repoReferenceSet.getReferences()
numRefs = len(repoReferences)
pageSizes = self._getPageSizes(numRefs)
for pageSize in pageSizes:
self._client.set_page_size(pageSize)
references = list(
self._client.search_references(referenceSetId))
for repoReference, reference in utils.zipLists(
repoReferences, references):
self._assertEqual(repoReference, reference)
def testSearchVariants(self):
for variantSet in self._repo.allVariantSets():
referenceSet = variantSet.getReferenceSet()
referenceName = referenceSet.getLocalId()
variantSetId = variantSet.getId()
repoVariants = list(
variantSet.getVariants(referenceName, 0, 2**32))
numRepoVariants = len(repoVariants)
pageSizes = self._getPageSizes(numRepoVariants)
for pageSize in pageSizes:
self._client.set_page_size(pageSize)
variants = self._client.search_variants(variantSetId)
for repoVariant, variant in utils.zipLists(
repoVariants, variants):
self._assertEqual(repoVariant, variant)
def _testSearchMethodInContainer(
self, containerAccessorMethod, clientSearchMethod,
containerIteratorMethod, equalMethod="_assertEqual"):
for container in containerIteratorMethod:
containerId = container.getId()
repoObjs = getattr(container, containerAccessorMethod)()
numRepoObjs = len(repoObjs)
pageSizes = self._getPageSizes(numRepoObjs)
for pageSize in pageSizes:
self._client.set_page_size(pageSize)
clientObjs = list(clientSearchMethod(containerId))
for repoObj, clientObj in utils.zipLists(
repoObjs, clientObjs):
assertEqual = getattr(self, equalMethod)
assertEqual(repoObj, clientObj)
def _testSearchMethodInContainer(
self, containerAccessorMethod, clientSearchMethod,
containerIteratorMethod, equalMethod="_assertEqual"):
for container in containerIteratorMethod:
containerId = container.getId()
repoObjs = getattr(container, containerAccessorMethod)()
numRepoObjs = len(repoObjs)
pageSizes = self._getPageSizes(numRepoObjs)
for pageSize in pageSizes:
self._client.set_page_size(pageSize)
clientObjs = list(clientSearchMethod(containerId))
for repoObj, clientObj in utils.zipLists(
repoObjs, clientObjs):
assertEqual = getattr(self, equalMethod)
assertEqual(repoObj, clientObj)
def testSearchVariants(self):
for variantSet in self._repo.allVariantSets():
referenceSet = variantSet.getReferenceSet()
referenceName = referenceSet.getLocalId()
variantSetId = variantSet.getId()
repoVariants = list(variantSet.getVariants(referenceName, 0,
2**32))
numRepoVariants = len(repoVariants)
pageSizes = self._getPageSizes(numRepoVariants)
for pageSize in pageSizes:
self._client.set_page_size(pageSize)
variants = self._client.search_variants(variantSetId)
for repoVariant, variant in utils.zipLists(
repoVariants, variants):
self._assertEqual(repoVariant, variant)
def testAllReads(self):
for dmDataset in self.dataRepo.getDatasets():
for dmReadGroupSet in dmDataset.getReadGroupSets():
dmReferenceSet = dmReadGroupSet.getReferenceSet()
for dmReadGroup in dmReadGroupSet.getReadGroups():
for dmReference in dmReferenceSet.getReferences():
# TODO fix these coordinates.
start = 0
end = 10
dmReads = list(dmReadGroup.getReadAlignments(
dmReference, start, end))
reads = list(self.client.search_reads(
[dmReadGroup.getId()], dmReference.getId(),
start, end))
self.assertGreater(len(reads), 0)
for dmRead, read in utils.zipLists(dmReads, reads):
self.assertEqual(dmRead, read)
def testSearchReads(self):
for repoReadGroup in self._repo.allReadGroups():
repoReferenceSet = \
repoReadGroup._parentContainer.getReferenceSet()
for repoReference in repoReferenceSet.getReferences():
referenceId = repoReference.getId()
readGroupId = repoReadGroup.getId()
repoReads = list(
repoReadGroup.getReadAlignments(repoReference))
numRepoReads = len(repoReads)
pageSizes = self._getPageSizes(numRepoReads)
for pageSize in pageSizes:
self._client.set_page_size(pageSize)
reads = list(
self._client.search_reads([readGroupId], referenceId))
for repoRead, read in utils.zipLists(repoReads, reads):
self.assertEqual(repoRead, read)
def testPrograms(self):
# test that program info is set correctly
readGroupSet = self._gaObject
for readGroup in readGroupSet.getReadGroups():
readGroupInfo = self._readGroupInfos[readGroup.getLocalId()]
gaPrograms = readGroup.getPrograms()
htslibPrograms = readGroupInfo.programs
for gaProgram, htslibProgram in utils.zipLists(
gaPrograms, htslibPrograms):
self.assertEqual(gaProgram.id, htslibProgram.get('ID'))
self.assertEqual(gaProgram.command_line,
htslibProgram.get('CL', None))
self.assertEqual(gaProgram.name, htslibProgram.get('PN', None))
self.assertEqual(gaProgram.prev_program_id,
htslibProgram.get('PP', None))
self.assertEqual(gaProgram.version,
htslibProgram.get('VN', None))
def testCompile(self):
"""
Compiles the schemas to a temporary directory and then checks
that the pb2 files in the temporary direcory are the same as the
pb2 files that are checked in.
This test prevents inadvertent mismatches between proto files and
pb2 files from being checked in.
"""
# compile the schemas to a temporary directory
scriptPath = 'scripts/process_schemas.py'
schemaVersion = '.'.join(version.version.split('.')[0:3])
schemasDir = 'src/main/proto/'
schemaDest = tempfile.mkdtemp()
cmd = "python {} {} -s {} -d {}".format(scriptPath, schemaVersion,
schemasDir, schemaDest)
utils.runCommand(cmd, silent=True)
# get the file paths of the checked in pb2 files
# (we do it in two calls to avoid the build/ tree, etc.
# in the python directory which may contain pb2 files)
pb2Patterns = ["*_pb2.py"]
checkedInDirGa4gh = 'python/ga4gh/schemas/ga4gh/'
checkedInDirGoogle = 'python/ga4gh/schemas/google/'
checkedInFilePathsGa4gh = utils.getFilePathsWithExtensionsInDirectory(
checkedInDirGa4gh, pb2Patterns)
checkedInFilePathsGoogle = utils.getFilePathsWithExtensionsInDirectory(
checkedInDirGoogle, pb2Patterns)
checkedInFilePaths = sorted(checkedInFilePathsGa4gh +
checkedInFilePathsGoogle)
# check to see that the contents of the directories are the same
tempFilePaths = utils.getFilePathsWithExtensionsInDirectory(
schemaDest, pb2Patterns)
self.assertEqual(len(checkedInFilePaths), len(tempFilePaths))
for checkedInFilePath, tempFilePath in utils.zipLists(
checkedInFilePaths, tempFilePaths):
checkedInFileShortPath = self._getDirAndFilenameOfPath(
checkedInFilePath)
tempFileShortPath = self._getDirAndFilenameOfPath(tempFilePath)
self.assertEqual(checkedInFileShortPath, tempFileShortPath)
with open(checkedInFilePath) as checkedInFile, \
open(tempFilePath) as tempFile:
for checkedInLine, tempLine in zip(checkedInFile, tempFile):
self.assertEqual(checkedInLine, tempLine)
def testCompile(self):
"""
Compiles the schemas to a temporary directory and then checks
that the pb2 files in the temporary direcory are the same as the
pb2 files that are checked in.
This test prevents inadvertent mismatches between proto files and
pb2 files from being checked in.
"""
# compile the schemas to a temporary directory
scriptPath = 'scripts/process_schemas.py'
schemaVersion = '.'.join(version.version.split('.')[0:3])
schemasDir = 'src/main/proto/'
schemaDest = tempfile.mkdtemp()
cmd = "python {} {} -s {} -d {}".format(
scriptPath, schemaVersion, schemasDir, schemaDest)
utils.runCommand(cmd, silent=True)
# get the file paths of the checked in pb2 files
# (we do it in two calls to avoid the build/ tree, etc.
# in the python directory which may contain pb2 files)
pb2Patterns = ["*_pb2.py"]
checkedInDirGa4gh = 'python/ga4gh/schemas/ga4gh/'
checkedInDirGoogle = 'python/ga4gh/schemas/google/'
checkedInFilePathsGa4gh = utils.getFilePathsWithExtensionsInDirectory(
checkedInDirGa4gh, pb2Patterns)
checkedInFilePathsGoogle = utils.getFilePathsWithExtensionsInDirectory(
checkedInDirGoogle, pb2Patterns)
checkedInFilePaths = sorted(
checkedInFilePathsGa4gh + checkedInFilePathsGoogle)
# check to see that the contents of the directories are the same
tempFilePaths = utils.getFilePathsWithExtensionsInDirectory(
schemaDest, pb2Patterns)
self.assertEqual(len(checkedInFilePaths), len(tempFilePaths))
for checkedInFilePath, tempFilePath in utils.zipLists(
checkedInFilePaths, tempFilePaths):
checkedInFileShortPath = self._getDirAndFilenameOfPath(
checkedInFilePath)
tempFileShortPath = self._getDirAndFilenameOfPath(tempFilePath)
self.assertEqual(checkedInFileShortPath, tempFileShortPath)
with open(checkedInFilePath) as checkedInFile, \
open(tempFilePath) as tempFile:
for checkedInLine, tempLine in zip(checkedInFile, tempFile):
self.assertEqual(checkedInLine, tempLine)
def testSearchVariantAnnotations(self):
for variantAnnotationSet in self._repo.allVariantAnnotationSets():
variantAnnotationSetId = variantAnnotationSet.getId()
variantSet = variantAnnotationSet._parentContainer
referenceSet = variantSet.getReferenceSet()
referenceName = referenceSet.getLocalId()
repoVariantAnnotations = list(
variantAnnotationSet.getVariantAnnotations(
referenceName, 0, 2**32))
numRepoVariantAnnotations = len(repoVariantAnnotations)
pageSizes = self._getPageSizes(numRepoVariantAnnotations)
for pageSize in pageSizes:
self._client.set_page_size(pageSize)
variantAnnotations = self._client.search_variant_annotations(
variantAnnotationSetId, referenceName=referenceName)
for repoVa, va in utils.zipLists(repoVariantAnnotations,
variantAnnotations):
self._assertEqual(repoVa, va)
def testSearchVariantAnnotations(self):
for variantAnnotationSet in self._repo.allVariantAnnotationSets():
variantAnnotationSetId = variantAnnotationSet.getId()
variantSet = variantAnnotationSet._parentContainer
referenceSet = variantSet.getReferenceSet()
referenceName = referenceSet.getLocalId()
repoVariantAnnotations = list(
variantAnnotationSet.getVariantAnnotations(
referenceName, 0, 2**32))
numRepoVariantAnnotations = len(repoVariantAnnotations)
pageSizes = self._getPageSizes(numRepoVariantAnnotations)
for pageSize in pageSizes:
self._client.set_page_size(pageSize)
variantAnnotations = self._client.search_variant_annotations(
variantAnnotationSetId, referenceName=referenceName)
for repoVa, va in utils.zipLists(
repoVariantAnnotations, variantAnnotations):
self._assertEqual(repoVa, va)
def testSearchReads(self):
for repoReadGroup in self._repo.allReadGroups():
repoReferenceSet = \
repoReadGroup._parentContainer.getReferenceSet()
for repoReference in repoReferenceSet.getReferences():
referenceId = repoReference.getId()
readGroupId = repoReadGroup.getId()
repoReads = list(
repoReadGroup.getReadAlignments(repoReference))
numRepoReads = len(repoReads)
pageSizes = self._getPageSizes(numRepoReads)
for pageSize in pageSizes:
self._client.set_page_size(pageSize)
reads = list(self._client.search_reads(
[readGroupId], referenceId))
for repoRead, read in utils.zipLists(
repoReads, reads):
self.assertEqual(repoRead, read)
def testPrograms(self):
# test that program info is set correctly
readGroupSet = self._gaObject
for readGroup in readGroupSet.getReadGroups():
readGroupInfo = self._readGroupInfos[readGroup.getLocalId()]
gaPrograms = readGroup.getPrograms()
htslibPrograms = readGroupInfo.programs
for gaProgram, htslibProgram in utils.zipLists(
gaPrograms, htslibPrograms):
self.assertEqual(
gaProgram.id, htslibProgram.get('ID'))
self.assertEqual(
gaProgram.command_line, htslibProgram.get('CL', None))
self.assertEqual(
gaProgram.name, htslibProgram.get('PN', None))
self.assertEqual(
gaProgram.prev_program_id, htslibProgram.get('PP', None))
self.assertEqual(
gaProgram.version, htslibProgram.get('VN', None))
def testAllReads(self):
for dmDataset in self.dataRepo.getDatasets():
for dmReadGroupSet in dmDataset.getReadGroupSets():
dmReferenceSet = dmReadGroupSet.getReferenceSet()
for dmReadGroup in dmReadGroupSet.getReadGroups():
for dmReference in dmReferenceSet.getReferences():
# TODO fix these coordinates.
start = 0
end = 10
dmReads = list(
dmReadGroup.getReadAlignments(
dmReference, start, end))
reads = list(
self.client.search_reads([dmReadGroup.getId()],
dmReference.getId(),
start, end))
self.assertGreater(len(reads), 0)
for dmRead, read in utils.zipLists(dmReads, reads):
self.assertEqual(dmRead, read)
def scrubReadGroup(self, cliOutput, clientOutput):
for cliReadGroup, clientReadGroup in utils.zipLists(
cliOutput, clientOutput):
self._scrubReadGroup(cliReadGroup, clientReadGroup)
def assertAlignmentListsEqual(self, gaAlignments, pysamAlignments,
readGroupInfo):
for gaAlignment, pysamAlignment in utils.zipLists(
gaAlignments, pysamAlignments):
self.assertAlignmentsEqual(gaAlignment, pysamAlignment,
readGroupInfo)
def assertAlignmentListsEqual(
self, gaAlignments, pysamAlignments, readGroupInfo):
for gaAlignment, pysamAlignment in utils.zipLists(
gaAlignments, pysamAlignments):
self.assertAlignmentsEqual(
gaAlignment, pysamAlignment, readGroupInfo)
def scrubReadGroup(self, cliOutput, clientOutput):
for cliReadGroup, clientReadGroup in utils.zipLists(
cliOutput, clientOutput):
self._scrubReadGroup(cliReadGroup, clientReadGroup)