def test_addFilters(self):
ds1 = DataSet()
filt = Filters()
filt.addRequirement(rq=[('>', '0.85')])
ds1.addFilters(filt)
self.assertEquals(str(ds1.filters), '( rq > 0.85 )')
# Or added from a source XML
ds2 = DataSet(data.getXml(16))
self.assertTrue(str(ds2.filters).startswith('( rname = E.faecalis'))
def test_addFilters(self):
ds1 = DataSet()
filt = Filters()
filt.addRequirement(rq=[('>', '0.85')])
ds1.addFilters(filt)
assert str(ds1.filters) == '( rq > 0.85 )'
# Or added from a source XML
ds2 = DataSet(data.getXml(15))
assert str(ds2.filters).startswith('( rname = E.faecalis')
def test_setFilters(self):
ds1 = DataSet()
filt = Filters()
filt.addRequirement(rq=[('>', '0.85')])
ds1.addFilters(filt)
self.assertEquals(str(ds1.filters), '( rq > 0.85 )')
# Or added from a source XML
ds2 = DataSet()
ds2.filters = ds1.filters
self.assertEquals(str(ds2.filters), '( rq > 0.85 )')
def test_addFilters(self):
ds1 = DataSet()
filt = Filters()
filt.addRequirement(rq=[('>', '0.85')])
ds1.addFilters(filt)
self.assertEquals(str(ds1.filters), '( rq > 0.85 )')
# Or added from a source XML
ds2 = DataSet(data.getXml(16))
self.assertTrue(str(ds2.filters).startswith(
'( rname = E.faecalis'))
def test_setFilters(self):
ds1 = DataSet()
filt = Filters()
filt.addRequirement(rq=[('>', '0.85')])
ds1.addFilters(filt)
self.assertEquals(str(ds1.filters), '( rq > 0.85 )')
# Or added from a source XML
ds2 = DataSet()
ds2.filters = ds1.filters
self.assertEquals(str(ds2.filters), '( rq > 0.85 )')
def test_filter_reference_contigs(self):
ds2 = ReferenceSet(data.getRef())
self.assertEqual(len(list(ds2.refNames)), 59)
filt = Filters()
filt.addRequirement(id=[('==', 'E.faecalis.1')])
ds2.addFilters(filt)
self.assertEqual(str(ds2.filters), "( id == E.faecalis.1 )")
self.assertEqual(len(ds2.refNames), 1)
self.assertEqual(len(list(ds2.records)), 1)
ds2.disableFilters()
self.assertEqual(len(list(ds2.refNames)), 59)
self.assertEqual(len(list(ds2.records)), 59)
ds2.enableFilters()
self.assertEqual(len(list(ds2.refNames)), 1)
self.assertEqual(len(list(ds2.records)), 1)
def _parseXmlFilters(element):
"""Pull filters from XML file, put them in a list of dictionaries, where
each dictionary contains a representation of a Filter tag: key, value pairs
with parameter names and value expressions.
Doctest:
>>> import xml.etree.ElementTree as ET
>>> import pbcore.data.datasets as data
>>> tree = ET.parse(data.getXml(no=8).split(':')[1])
>>> root = tree.getroot()
>>> filters = root[1]
>>> str(_parseXmlFilters(filters))
'( rq > 0.75 ) OR ( qname == 100/0/0_100 )'
"""
"""
namer = functools.partial(_namespaceTag, XMLNS)
result = []
for filtTag in filters:
#filt = Filter()
filt = {}
# This is essentially to skip the <Parameters> level
for child in filtTag:
if child.tag == namer('Parameters'):
for param in child:
filt[param.attrib['Name'].lower()] = param.attrib['Value']
result.append(filt)
return result
"""
return Filters(_eleToDictList(element))
def test_filter_reference_contigs(self):
ds2 = ReferenceSet(data.getRef())
self.assertEqual(len(list(ds2.refNames)), 59)
filt = Filters()
filt.addRequirement(id=[('==', 'E.faecalis.1')])
ds2.addFilters(filt)
self.assertEqual(str(ds2.filters),
"( id == E.faecalis.1 )")
self.assertEqual(len(ds2.refNames), 1)
self.assertEqual(len(list(ds2.records)), 1)
ds2.disableFilters()
self.assertEqual(len(list(ds2.refNames)), 59)
self.assertEqual(len(list(ds2.records)), 59)
ds2.enableFilters()
self.assertEqual(len(list(ds2.refNames)), 1)
self.assertEqual(len(list(ds2.records)), 1)
def filter_options(parser):
pbiFilterOptions = set(Filters()._pbiMappedVecAccMap().keys())
bamFilterOptions = set(Filters()._bamAccMap.keys())
parser.description = ('Add filters to an XML file. Suggested fields: '
'{f}. More expensive fields: {b}'.format(
f=sorted(list(pbiFilterOptions)),
b=sorted(
list(bamFilterOptions - pbiFilterOptions))))
#parser.add_argument("infile", type=validate_file,
parser.add_argument("infile", type=str, help="The xml file to filter")
parser.add_argument("outfile", type=str, help="The resulting xml file")
parser.add_argument("filters",
type=str,
nargs='+',
help="The values and thresholds to filter (e.g. "
"'rq>0.85')")
parser.set_defaults(func=filterXml)
def test_filter_reference_contigs(self):
ds2 = ReferenceSet(data.getRef())
assert len(list(ds2.refNames)) == 59
assert len(list(ds2.records)) == len(ds2.index)
filt = Filters()
filt.addRequirement(id=[('==', 'E.faecalis.1')])
ds2.addFilters(filt)
assert str(ds2.filters) == "( id == E.faecalis.1 )"
assert len(ds2.refNames) == 1
assert len(list(ds2.records)) == 1
assert len(list(ds2.records)) == len(ds2.index)
ds2.disableFilters()
assert len(list(ds2.refNames)) == 59
assert len(list(ds2.records)) == 59
assert len(list(ds2.records)) == len(ds2.index)
ds2.enableFilters()
assert len(list(ds2.refNames)) == 1
assert len(list(ds2.records)) == 1
assert len(list(ds2.records)) == len(ds2.index)
def filter_options(parser):
pbiFilterOptions = set(Filters()._pbiMappedVecAccMap().keys())
bamFilterOptions = set(Filters()._bamAccMap.keys())
parser.description = ('Add filters to an XML file. Suggested fields: '
'{f}. More expensive fields: {b}.\nMultiple filters '
'of different names will be ANDed together, '
'multiple filters of the same name will be ORed '
'together, duplicating existing requirements'.format(
f=sorted(list(pbiFilterOptions)),
b=sorted(
list(bamFilterOptions - pbiFilterOptions))))
# parser.add_argument("infile", type=validate_file,
parser.add_argument("infile", type=str, help="The XML file to filter")
parser.add_argument("outfile",
type=str,
help="The resulting DataSet XML file")
parser.add_argument("filters",
type=str,
nargs='+',
help=("The parameters, operators and values to filter "
"(e.g. 'rq>0.85')"))
parser.set_defaults(func=filterXml)
def test_reads_in_contig(self):
log.info("Testing reads in contigs")
ds = DataSet(data.getXml(8))
dss = ds.split(contigs=True)
self.assertEqual(len(dss), 12)
efaec1TimesFound = 0
efaec1TotFound = 0
efaec2TimesFound = 0
efaec2TotFound = 0
for ds in dss:
ef1 = len(list(ds.readsInReference('E.faecalis.1')))
ef2 = len(list(ds.readsInReference('E.faecalis.2')))
if ef1:
efaec1TimesFound += 1
efaec1TotFound += ef1
if ef2:
efaec2TimesFound += 1
efaec2TotFound += ef2
self.assertEqual(efaec1TimesFound, 1)
self.assertEqual(efaec1TotFound, 20)
self.assertEqual(efaec2TimesFound, 1)
self.assertEqual(efaec2TotFound, 3)
ds = DataSet(data.getXml(8))
filt = Filters()
filt.addRequirement(length=[('>', '100')])
ds.addFilters(filt)
dss = ds.split(contigs=True)
self.assertEqual(len(dss), 12)
efaec1TimesFound = 0
efaec1TotFound = 0
efaec2TimesFound = 0
efaec2TotFound = 0
for ds in dss:
ef1 = len(list(ds.readsInReference('E.faecalis.1')))
ef2 = len(list(ds.readsInReference('E.faecalis.2')))
if ef1:
efaec1TimesFound += 1
efaec1TotFound += ef1
if ef2:
efaec2TimesFound += 1
efaec2TotFound += ef2
self.assertEqual(efaec1TimesFound, 1)
self.assertEqual(efaec1TotFound, 20)
self.assertEqual(efaec2TimesFound, 1)
self.assertEqual(efaec2TotFound, 3)
ds = DataSet(data.getXml(8))
filt = Filters()
filt.addRequirement(length=[('>', '1000')])
ds.addFilters(filt)
dss = ds.split(contigs=True)
self.assertEqual(len(dss), 9)
efaec1TimesFound = 0
efaec1TotFound = 0
efaec2TimesFound = 0
efaec2TotFound = 0
for ds in dss:
ef1 = len(list(ds.readsInReference('E.faecalis.1')))
ef2 = len(list(ds.readsInReference('E.faecalis.2')))
if ef1:
efaec1TimesFound += 1
efaec1TotFound += ef1
if ef2:
efaec2TimesFound += 1
efaec2TotFound += ef2
self.assertEqual(efaec1TimesFound, 1)
self.assertEqual(efaec1TotFound, 20)
self.assertEqual(efaec2TimesFound, 1)
self.assertEqual(efaec2TotFound, 1)
def _parseXmlFilters(element):
"""Pull filters from XML file, put them in a list of dictionaries, where
each dictionary contains a representation of a Filter tag: key, value pairs
with parameter names and value expressions.
"""
return Filters(_eleToDictList(element))
def test_reads_in_contig(self):
log.info("Testing reads in contigs")
ds = DataSet(data.getXml(8))
dss = ds.split(contigs=True)
self.assertEqual(len(dss), 12)
efaec1TimesFound = 0
efaec1TotFound = 0
efaec2TimesFound = 0
efaec2TotFound = 0
for ds in dss:
ef1 = len(list(ds.readsInReference('E.faecalis.1')))
ef2 = len(list(ds.readsInReference('E.faecalis.2')))
if ef1:
efaec1TimesFound += 1
efaec1TotFound += ef1
if ef2:
efaec2TimesFound += 1
efaec2TotFound += ef2
self.assertEqual(efaec1TimesFound, 1)
self.assertEqual(efaec1TotFound, 20)
self.assertEqual(efaec2TimesFound, 1)
self.assertEqual(efaec2TotFound, 3)
ds = DataSet(data.getXml(8))
filt = Filters()
filt.addRequirement(length=[('>', '100')])
ds.addFilters(filt)
dss = ds.split(contigs=True)
self.assertEqual(len(dss), 12)
efaec1TimesFound = 0
efaec1TotFound = 0
efaec2TimesFound = 0
efaec2TotFound = 0
for ds in dss:
ef1 = len(list(ds.readsInReference('E.faecalis.1')))
ef2 = len(list(ds.readsInReference('E.faecalis.2')))
if ef1:
efaec1TimesFound += 1
efaec1TotFound += ef1
if ef2:
efaec2TimesFound += 1
efaec2TotFound += ef2
self.assertEqual(efaec1TimesFound, 1)
self.assertEqual(efaec1TotFound, 20)
self.assertEqual(efaec2TimesFound, 1)
self.assertEqual(efaec2TotFound, 3)
ds = DataSet(data.getXml(8))
filt = Filters()
filt.addRequirement(length=[('>', '1000')])
ds.addFilters(filt)
dss = ds.split(contigs=True)
self.assertEqual(len(dss), 9)
efaec1TimesFound = 0
efaec1TotFound = 0
efaec2TimesFound = 0
efaec2TotFound = 0
for ds in dss:
ef1 = len(list(ds.readsInReference('E.faecalis.1')))
ef2 = len(list(ds.readsInReference('E.faecalis.2')))
if ef1:
efaec1TimesFound += 1
efaec1TotFound += ef1
if ef2:
efaec2TimesFound += 1
efaec2TotFound += ef2
self.assertEqual(efaec1TimesFound, 1)
self.assertEqual(efaec1TotFound, 20)
self.assertEqual(efaec2TimesFound, 1)
self.assertEqual(efaec2TotFound, 1)