def __init__(self, **kwargs):
self.__workPath = kwargs.get("workPath", ".")
self.__workDirSuffix = kwargs.get("workDirSuffix", "marshall_")
self.__workDirPrefix = kwargs.get("workDirSuffix", "_tempdir")
#
self.__fileU = FileUtil(workPath=self.__workPath)
self.__ioU = IoUtil()
def __rebuildCache(self,
targetUrl,
mapNameL,
outDirPath,
rawDirPath,
fmt="pickle",
useCache=True):
"""Fetch the UniProt selected id mapping resource file and extract
UniProt Acc to 'mapIndex' mapping. Serialize the mapping as required.
Args:
targetUrl (str): source URL of the remote index file
mapNameL (list): list of key mapping names to extract from the index
outDirPath (str): directory path for raw and processed mapping files
fmt (str, optional): output format (pickle|json) . Defaults to "pickle".
useCache (bool, optional): use cached files. Defaults to True.
Returns:
dict: od[uniprotId] = mapped value
idmapping_selected.tab
1. UniProtKB-AC
2. UniProtKB-ID
3. GeneID (EntrezGene)
4. RefSeq
5. GI
6. PDB
7. GO
8. UniRef100
9. UniRef90
10. UniRef50
11. UniParc
12. PIR
13. NCBI-taxon
14. MIM
15. UniGene
16. PubMed
17. EMBL
18. EMBL-CDS
19. Ensembl
20. Ensembl_TRS
21. Ensembl_PRO
22. Additional PubMed
"""
startTime = time.time()
nL = mapNameL
oD = {}
try:
fileU = FileUtil()
fExt = "pic" if fmt == "pickle" else "json"
fExt = "tdd" if fmt == "tdd" else fExt
fN, _ = os.path.splitext(fileU.getFileName(targetUrl))
mapFileName = fN + "-map." + fExt
idMapPath = os.path.join(outDirPath, mapFileName)
mU = MarshalUtil()
if useCache and mU.exists(idMapPath):
logger.info("Reading cached serialized file %r", idMapPath)
if fmt in ["pickle", "json"]:
tD = mU.doImport(idMapPath, fmt=fmt)
nL = list(set(tD["idNameList"]))
oD = tD["uniprotMapD"]
logger.info("keys %r", list(oD.keys())[:10])
logger.info("nL %r", nL)
ok = True
elif fmt == "tdd":
ioU = IoUtil()
it = ioU.deserializeCsvIter(idMapPath,
delimiter="\t",
rowFormat="list",
encodingErrors="ignore")
tL = next(it, [])
nL = tL[1:]
if len(nL) == 1:
for row in it:
oD[row[0]] = row[1]
else:
for row in it:
oD[row[0]] = row[1:]
ok = True
else:
idPath = os.path.join(rawDirPath, fileU.getFileName(targetUrl))
if not fileU.exists(idPath):
logger.info(
"Fetching selected UniProt idmapping data from %r in %r",
targetUrl, outDirPath)
ok = fileU.get(targetUrl, idPath)
if not ok:
logger.error("Failed to downlowd %r", targetUrl)
return oD
else:
logger.info("Using cached mapping file %r", idPath)
# ---
ioU = IoUtil()
if fmt in ["pickle", "json"]:
if len(mapNameL) == 1:
for row in ioU.deserializeCsvIter(
idPath,
delimiter="\t",
rowFormat="list",
encodingErrors="ignore"):
oD[row[0]] = str(
row[self.__mapRecordD[mapNameL[0]] - 1])
else:
for row in ioU.deserializeCsvIter(
idPath,
delimiter="\t",
rowFormat="list",
encodingErrors="ignore"):
for mapName in mapNameL:
oD.setdefault(row[0], []).append(
str(row[self.__mapRecordD[mapName] - 1]))
logger.info("Writing serialized mapping file %r",
idMapPath)
ok = mU.doExport(idMapPath, {
"idNameList": mapNameL,
"uniprotMapD": oD
},
fmt=fmt)
elif fmt == "tdd":
#
logger.info("Writing serialized mapping file %r",
idMapPath)
fU = FileUtil()
fU.mkdirForFile(idMapPath)
colNameL = []
colNameL.append("UniProtId")
colNameL.extend(mapNameL)
with open(idMapPath, "w", encoding="utf-8") as ofh:
ofh.write("%s\n" % "\t".join(colNameL))
if len(mapNameL) == 1:
idx = self.__mapRecordD[mapNameL[0]] - 1
for row in ioU.deserializeCsvIter(
idPath,
delimiter="\t",
rowFormat="list",
encodingErrors="ignore"):
ofh.write("%s\t%s\n" % (row[0], row[idx]))
else:
idxL = [0]
idxL.extend([
self.__mapRecordD[mapName] - 1
for mapName in mapNameL
])
for row in ioU.deserializeCsvIter(
idPath,
delimiter="\t",
rowFormat="list",
encodingErrors="ignore"):
ofh.write(
"%s\n" %
"\t".join([str(row[idx]) for idx in idxL]))
#
nL, oD = self.__rebuildCache(targetUrl,
mapNameL,
outDirPath,
rawDirPath,
fmt=fmt,
useCache=True)
ok = True if nL and oD else False
logger.info("Completed reload (%r) at %s (%.4f seconds)", ok,
time.strftime("%Y %m %d %H:%M:%S", time.localtime()),
time.time() - startTime)
except Exception as e:
logger.exception("Failing with %s", str(e))
#
return nL, oD
def setUp(self):
self.__verbose = True
self.__pathPdbxDictionaryFile = os.path.join(TOPDIR, "rcsb",
"mock-data",
"dictionaries",
"mmcif_pdbx_v5_next.dic")
self.__pathJsonTestFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"dictionaries",
"vrpt_dictmap.json")
self.__pathIndexFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"MOCK_EXCHANGE_SANDBOX",
"update-lists", "all-pdb-list")
self.__pathCifFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"MOCK_BIRD_CC_REPO", "0",
"PRDCC_000010.cif")
#
self.__workPath = os.path.join(HERE, "test-output")
self.__pathSaveDictionaryFile = os.path.join(self.__workPath,
"mmcif_pdbx_v5_next.dic")
self.__pathSaveJsonTestFile = os.path.join(self.__workPath,
"json-content.json")
self.__pathSaveIndexFile = os.path.join(self.__workPath,
"all-pdb-list")
self.__pathSaveCifFile = os.path.join(self.__workPath,
"cif-content.cif")
self.__pathSavePickleFile = os.path.join(self.__workPath,
"json-content.pic")
self.__pathSaveTextFile = os.path.join(self.__workPath,
"json-content.txt")
#
#
self.__pathInsilicoFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"MOCK_EXCHANGE_SANDBOX",
"status",
"theoretical_model.tsv")
self.__pathSaveInsilicoFile = os.path.join(
self.__workPath, "saved-theoretical_model.tsv")
#
# self.__pathVariantFastaFile = os.path.join(self.__mockTopPath, 'UniProt', 'uniprot_sprot_varsplic.fasta.gz')
self.__pathFastaFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"MOCK_EXCHANGE_SANDBOX",
"sequence",
"pdb_seq_prerelease.fasta")
self.__pathSaveFastaFile = os.path.join(self.__workPath,
"test-pre-release.fasta")
#
self.__pathTaxonomyFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"NCBI", "names.dmp.gz")
self.__pathSaveTaxonomyFilePic = os.path.join(self.__workPath,
"taxonomy_names.pic")
self.__pathSaveTaxonomyFileCsv = os.path.join(self.__workPath,
"taxonomy_names.csv")
#
self.__pathSiftsFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"sifts-summary",
"pdb_chain_go.csv.gz")
#
self.__ioU = IoUtil()
self.__startTime = time.time()
logger.debug("Running tests on version %s", __version__)
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
class MarshalUtil(object):
"""Wrapper for serialization and deserialization methods."""
def __init__(self, **kwargs):
self.__workPath = kwargs.get("workPath", ".")
self.__workDirSuffix = kwargs.get("workDirSuffix", "marshall_")
self.__workDirPrefix = kwargs.get("workDirSuffix", "_tempdir")
#
self.__fileU = FileUtil(workPath=self.__workPath)
self.__ioU = IoUtil()
def doExport(self,
locator,
obj,
fmt="list",
marshalHelper=None,
numParts=None,
**kwargs):
"""Serialize the input object at locator path in specified format. The
input object is optionally preprocessed by the helper method.
Args:
locator (str): target path or URI
obj (object): data to be serialized
fmt (str, optional): format for serialization (mmcif, tdd, csv, list). Defaults to "list".
marshalHelper (method, optional): pre-processor method applied to input data object. Defaults to None.
numParts (int, optional): serialize the data in parts. Defaults to None. (json and pickle formats)
Returns:
bool: True for sucess or False otherwise
"""
try:
ret = False
localFlag = self.__fileU.isLocal(locator)
if marshalHelper:
myObj = marshalHelper(obj, **kwargs)
else:
myObj = obj
#
if localFlag and numParts and fmt in ["json", "pickle"]:
localFilePath = self.__fileU.getFilePath(locator)
ret = self.__ioU.serializeInParts(localFilePath,
myObj,
numParts,
fmt=fmt,
**kwargs)
elif localFlag:
localFilePath = self.__fileU.getFilePath(locator)
ret = self.__ioU.serialize(localFilePath,
myObj,
fmt=fmt,
workPath=self.__workPath,
**kwargs)
else:
with tempfile.TemporaryDirectory(
suffix=self.__workDirSuffix,
prefix=self.__workDirPrefix,
dir=self.__workPath) as tmpDirName:
# write a local copy then copy to destination -
#
localFilePath = os.path.join(
self.__workPath, tmpDirName,
self.__fileU.getFileName(locator))
ok1 = self.__ioU.serialize(localFilePath,
myObj,
fmt=fmt,
workPath=self.__workPath,
**kwargs)
ok2 = True
if ok1:
ok2 = self.__fileU.put(localFilePath, locator,
**kwargs)
ret = ok1 and ok2
except Exception as e:
logger.exception("Exporting locator %r failing with %s", locator,
str(e))
return ret
def doImport(self,
locator,
fmt="list",
marshalHelper=None,
numParts=None,
**kwargs):
"""Deserialize data at the target locator in specified format. The deserialized
data is optionally post-processed by the input helper method.
Args:
locator (str): path or URI to input data
fmt (str, optional): format for deserialization (mmcif, tdd, csv, list). Defaults to "list".
marshalHelper (method, optional): post-processor method applied to deserialized data object. Defaults to None.
numParts (int, optional): deserialize the data in parts. Defaults to None. (json and pickle formats)
tarMember (str, optional): name of a member of tar file bundle. Defaults to None. (tar file format)
Returns:
Any: format specific return type
"""
try:
tarMember = kwargs.get("tarMember", None)
localFlag = self.__fileU.isLocal(locator) and not tarMember
#
if localFlag and numParts and fmt in ["json", "pickle"]:
filePath = self.__fileU.getFilePath(locator)
ret = self.__ioU.deserializeInParts(filePath,
numParts,
fmt=fmt,
**kwargs)
elif localFlag:
filePath = self.__fileU.getFilePath(locator)
ret = self.__ioU.deserialize(filePath,
fmt=fmt,
workPath=self.__workPath,
**kwargs)
else:
#
if fmt == "mmcif":
ret = self.__ioU.deserialize(locator,
fmt=fmt,
workPath=self.__workPath,
**kwargs)
else:
with tempfile.TemporaryDirectory(
suffix=self.__workDirSuffix,
prefix=self.__workDirPrefix,
dir=self.__workPath) as tmpDirName:
#
# Fetch first then read a local copy -
#
if tarMember:
localFilePath = os.path.join(
self.__workPath, tmpDirName, tarMember)
else:
localFilePath = os.path.join(
self.__workPath, tmpDirName,
self.__fileU.getFileName(locator))
# --- Local copy approach ---
self.__fileU.get(locator, localFilePath, **kwargs)
ret = self.__ioU.deserialize(localFilePath,
fmt=fmt,
workPath=self.__workPath,
**kwargs)
if marshalHelper:
ret = marshalHelper(ret, **kwargs)
except Exception as e:
logger.exception("Importing locator %r failing with %s", locator,
str(e))
ret = None
return ret
def exists(self, filePath, mode=os.R_OK):
return self.__fileU.exists(filePath, mode=mode)
def mkdir(self, dirPath, mode=0o755):
return self.__fileU.mkdir(dirPath, mode=mode)
def remove(self, pth):
return self.__fileU.remove(pth)
class IoUtilTests(unittest.TestCase):
def setUp(self):
self.__verbose = True
self.__pathPdbxDictionaryFile = os.path.join(TOPDIR, "rcsb",
"mock-data",
"dictionaries",
"mmcif_pdbx_v5_next.dic")
self.__pathJsonTestFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"dictionaries",
"vrpt_dictmap.json")
self.__pathIndexFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"MOCK_EXCHANGE_SANDBOX",
"update-lists", "all-pdb-list")
self.__pathCifFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"MOCK_BIRD_CC_REPO", "0",
"PRDCC_000010.cif")
#
self.__workPath = os.path.join(HERE, "test-output")
self.__pathSaveDictionaryFile = os.path.join(self.__workPath,
"mmcif_pdbx_v5_next.dic")
self.__pathSaveJsonTestFile = os.path.join(self.__workPath,
"json-content.json")
self.__pathSaveIndexFile = os.path.join(self.__workPath,
"all-pdb-list")
self.__pathSaveCifFile = os.path.join(self.__workPath,
"cif-content.cif")
self.__pathSavePickleFile = os.path.join(self.__workPath,
"json-content.pic")
self.__pathSaveTextFile = os.path.join(self.__workPath,
"json-content.txt")
#
#
self.__pathInsilicoFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"MOCK_EXCHANGE_SANDBOX",
"status",
"theoretical_model.tsv")
self.__pathSaveInsilicoFile = os.path.join(
self.__workPath, "saved-theoretical_model.tsv")
#
# self.__pathVariantFastaFile = os.path.join(self.__mockTopPath, 'UniProt', 'uniprot_sprot_varsplic.fasta.gz')
self.__pathFastaFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"MOCK_EXCHANGE_SANDBOX",
"sequence",
"pdb_seq_prerelease.fasta")
self.__pathSaveFastaFile = os.path.join(self.__workPath,
"test-pre-release.fasta")
#
self.__pathTaxonomyFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"NCBI", "names.dmp.gz")
self.__pathSaveTaxonomyFilePic = os.path.join(self.__workPath,
"taxonomy_names.pic")
self.__pathSaveTaxonomyFileCsv = os.path.join(self.__workPath,
"taxonomy_names.csv")
#
self.__pathSiftsFile = os.path.join(TOPDIR, "rcsb", "mock-data",
"sifts-summary",
"pdb_chain_go.csv.gz")
#
self.__ioU = IoUtil()
self.__startTime = time.time()
logger.debug("Running tests on version %s", __version__)
logger.debug("Starting %s at %s", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
def tearDown(self):
endTime = time.time()
logger.debug("Completed %s at %s (%.4f seconds)", self.id(),
time.strftime("%Y %m %d %H:%M:%S", time.localtime()),
endTime - self.__startTime)
@unittest.skipIf(sys.version_info[0] < 3, "not compatible with Python 2")
def testReadCsvIter(self):
"""Test returning an iterator for a large CSV file with leading comments"""
try:
iCount = 0
for row in self.__ioU.deserializeCsvIter(self.__pathSiftsFile,
delimiter=",",
rowFormat="list",
encodingErrors="ignore"):
if len(row) < 6:
logger.error("Failing with row %r", row)
iCount += 1
self.assertGreater(iCount, 25000000)
logger.info("Row count is %d", iCount)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReadWriteInParts(self):
"""Test the case reading and writing in parts."""
try:
self.maxDiff = None
lenL = 12483
aL = [100, 200, 300, 400, 500]
dL = [aL for ii in range(lenL)]
numParts = 4
sPath = os.path.join(self.__workPath, "list-data.json")
ok = self.__ioU.serializeInParts(sPath,
dL,
numParts,
fmt="json",
indent=3)
self.assertTrue(ok)
rL = self.__ioU.deserializeInParts(sPath, numParts, fmt="json")
logger.info("Reading %d parts with total length %d", numParts,
len(rL))
self.assertEqual(dL, rL)
#
lenD = 20341
qD = OrderedDict([("a", 100), ("b", 100), ("c", 100)])
dD = OrderedDict([(str(ii), qD) for ii in range(lenD)])
numParts = 4
sPath = os.path.join(self.__workPath, "dict-data.json")
ok = self.__ioU.serializeInParts(sPath,
dD,
numParts,
fmt="json",
indent=3)
self.assertTrue(ok)
rD = self.__ioU.deserializeInParts(sPath, numParts, fmt="json")
logger.info("Reading %d parts with total length %d", numParts,
len(rD))
self.assertDictEqual(dD, rD)
#
rD = self.__ioU.deserializeInParts(sPath, None, fmt="json")
logger.info("Reading %d globbed parts with total length %d",
numParts, len(rD))
self.assertDictEqual(dD, rD)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReadDictionaryFile(self):
"""Test the case read PDBx/mmCIF dictionary text file"""
try:
cL = self.__ioU.deserialize(self.__pathPdbxDictionaryFile,
fmt="mmcif-dict")
logger.debug("Dictionary container list %d", len(cL))
self.assertGreaterEqual(len(cL), 1)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReadCifFile(self):
"""Test the case read PDBx/mmCIF text file"""
try:
cL = self.__ioU.deserialize(self.__pathCifFile, fmt="mmcif")
logger.debug("Container list %d", len(cL))
self.assertGreaterEqual(len(cL), 1)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReadListFile(self):
"""Test the case read list text file"""
try:
cL = self.__ioU.deserialize(self.__pathIndexFile, fmt="list")
logger.debug("List length %d", len(cL))
self.assertGreaterEqual(len(cL), 1000)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReadJsonFile(self):
"""Test the case read JSON file"""
try:
rObj = self.__ioU.deserialize(self.__pathJsonTestFile, fmt="json")
logger.debug("Object length %d", len(rObj))
self.assertGreaterEqual(len(rObj), 1)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReadWriteDictionaryFiles(self):
"""Test the case read and write PDBx/mmCIF dictionary text file"""
try:
cL = self.__ioU.deserialize(self.__pathPdbxDictionaryFile,
fmt="mmcif-dict")
logger.debug("Dictionary container list %d", len(cL))
self.assertGreaterEqual(len(cL), 1)
ok = self.__ioU.serialize(self.__pathSaveDictionaryFile,
cL,
fmt="mmcif-dict")
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReadWriteCifFile(self):
"""Test the case read and write PDBx/mmCIF text file"""
try:
cL = self.__ioU.deserialize(self.__pathCifFile, fmt="mmcif")
logger.debug("Container list %d", len(cL))
self.assertGreaterEqual(len(cL), 1)
ok = self.__ioU.serialize(self.__pathSaveCifFile, cL, fmt="mmcif")
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReadWriteJsonFile(self):
"""Test the case read and write JSON file"""
try:
rObj = self.__ioU.deserialize(self.__pathJsonTestFile, fmt="json")
logger.debug("Object length %d", len(rObj))
self.assertGreaterEqual(len(rObj), 1)
ok = self.__ioU.serialize(self.__pathSaveJsonTestFile,
rObj,
fmt="json")
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReadWriteListFile(self):
"""Test the case read and write list text file"""
try:
cL = self.__ioU.deserialize(self.__pathIndexFile, fmt="list")
logger.debug("List element %r length %d", cL[0], len(cL))
self.assertGreaterEqual(len(cL), 1000)
ok = self.__ioU.serialize(self.__pathSaveIndexFile, cL, fmt="list")
self.assertTrue(ok)
count = 0
for cV in cL:
fields = cV.split()
count += len(fields)
_ = count
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReadWritePickleFile(self):
"""Test the case read and write pickle file"""
try:
rObj = self.__ioU.deserialize(self.__pathJsonTestFile, fmt="json")
logger.debug("Object length %d", len(rObj))
self.assertGreaterEqual(len(rObj), 1)
ok = self.__ioU.serialize(self.__pathSavePickleFile,
rObj,
fmt="pickle")
self.assertTrue(ok)
rObjP = self.__ioU.deserialize(self.__pathSavePickleFile,
fmt="pickle")
self.assertDictEqual(rObj, rObjP)
ok = self.__ioU.serialize(self.__pathSaveTextFile,
rObj,
fmt="text-dump")
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReadWriteListWithEncodingFile(self):
"""Test the case read and write list text file with non-ascii encoding"""
try:
cL = self.__ioU.deserialize(self.__pathInsilicoFile, fmt="list")
logger.debug("Insilico List length %d", len(cL))
#
self.assertGreaterEqual(len(cL), 1450)
#
ok = self.__ioU.serialize(self.__pathSaveInsilicoFile,
cL,
fmt="list")
self.assertTrue(ok)
#
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReadWriteFastaFile(self):
"""Test the case read and write FASTA sequence file"""
try:
sD = self.__ioU.deserialize(self.__pathFastaFile,
fmt="fasta",
commentStyle="prerelease")
logger.debug("Sequence length %d", len(sD.keys()))
self.assertGreaterEqual(len(sD), 500)
ok = self.__ioU.serialize(self.__pathSaveFastaFile,
sD,
fmt="fasta")
self.assertTrue(ok)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()
def testReadWriteTaxonomyFile(self):
"""Test the case read and write taxonomy resource file"""
try:
tL = self.__ioU.deserialize(self.__pathTaxonomyFile,
fmt="tdd",
rowFormat="list")
logger.info("Taxonomy length %d", len(tL))
self.assertGreaterEqual(len(tL), 500)
tD = {}
csvL = []
for tV in tL:
if len(tV) < 7:
continue
taxId = int(tV[0])
name = tV[2]
nameType = tV[6]
csvL.append({"t": taxId, "name": name, "type": nameType})
#
if nameType in [
"scientific name", "common name", "synonym",
"genbank common name"
]:
if taxId not in tD:
tD[taxId] = {}
if nameType in ["scientific name"]:
tD[taxId]["sn"] = name
continue
if "cn" not in tD[taxId]:
tD[taxId]["cn"] = []
tD[taxId]["cn"].append(name)
else:
pass
ok = self.__ioU.serialize(self.__pathSaveTaxonomyFilePic,
tD,
fmt="pickle")
self.assertTrue(ok)
ok = self.__ioU.serialize(self.__pathSaveTaxonomyFileCsv,
csvL,
fmt="csv")
self.assertTrue(ok)
tL = self.__ioU.deserialize(self.__pathSaveTaxonomyFileCsv,
fmt="csv",
rowFormat="dict")
self.assertTrue(len(tL) > 2880000)
except Exception as e:
logger.exception("Failing with %s", str(e))
self.fail()