class ISPyBRetrieveDataCollectionExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run ispyb test with default config')
def test_execute_ISPyBRetrieveDataCollection_image(self):
referenceDataPath = self.dataPath / \
"ISPyBRetrieveDataCollection_image.json"
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
iSPyBRetrieveDataCollection = ISPyBRetrieveDataCollection(inData=inData)
iSPyBRetrieveDataCollection.execute()
outData = iSPyBRetrieveDataCollection.outData
self.assertEqual(outData['imagePrefix'], 'ref-ednatest')
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run ispyb test with default config')
def test_execute_ISPyBRetrieveDataCollection_dataCollectionId(self):
referenceDataPath = self.dataPath / \
"ISPyBRetrieveDataCollection_dataCollectionId.json"
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
iSPyBRetrieveDataCollection = ISPyBRetrieveDataCollection(inData=inData)
iSPyBRetrieveDataCollection.execute()
outData = iSPyBRetrieveDataCollection.outData
self.assertEqual(outData['imagePrefix'], 'ref-ednatest')
class DozorM2Test(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run dozor test with default config')
def test_execute_DozorM2_twoScans(self):
referenceDataPath = self.dataPath / 'inDataDozorM2_twoScans.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
dozorm2 = DozorM2(inData=inData)
dozorm2.execute()
self.assertTrue(dozorm2.isSuccess())
outData = dozorm2.outData
# self.assertEqual(len(outData['imageDozor']), 10)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run dozor test with default config')
def test_execute_DozorM2_oneScan(self):
referenceDataPath = self.dataPath / 'inDataDozorM2_oneScan.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
dozorm = DozorM2(inData=inData)
dozorm.execute()
self.assertTrue(dozorm.isSuccess())
outData = dozorm.outData
class ControlIndexingExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run indexing test with default config')
def test_execute_ControlIndexing_local_user_1(self):
referenceDataPath = self.dataPath / 'local-user_1.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlIndexing = ControlIndexing(
inData=inData,
workingDirectorySuffix='local_user_1'
)
controlIndexing.execute()
self.assertTrue(controlIndexing.isSuccess())
self.assertEqual(controlIndexing.outData["resultIndexing"]["spaceGroupNumber"], 16)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run indexing test with default config')
def test_execute_ControlIndexing_opid30a3(self):
referenceDataPath = self.dataPath / 'UPF2-UPF2__4.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlIndexing = ControlIndexing(
inData=inData,
workingDirectorySuffix='UPF2-UPF2__4'
)
controlIndexing.execute()
self.assertTrue(controlIndexing.isSuccess())
self.assertEqual(controlIndexing.outData["resultIndexing"]["spaceGroupNumber"], 16)
class MosflmTasksExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run mosflm test with default config')
def tes_execute_MosflmGeneratePredictionTask_2m_RNASE_1(self):
UtilsTest.loadTestImage('ref-2m_RNASE_1_0001.cbf')
UtilsTest.loadTestImage('ref-2m_RNASE_1_0002.cbf')
referenceDataPath = self.dataPath / 'mosflm_generatePrediction_2m_RNASE_1.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
mosflmIndexingTask = MosflmGeneratePredictionTask(inData=inData)
mosflmIndexingTask.execute()
self.assertTrue(mosflmIndexingTask.isSuccess())
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run mosflm test with default config')
def test_execute_MosflmGeneratePredictionTaskTRYP_X1_4(self):
UtilsTest.loadTestImage('ref-TRYP-X1_4_0001.cbf')
UtilsTest.loadTestImage('ref-TRYP-X1_4_0002.cbf')
UtilsTest.loadTestImage('ref-TRYP-X1_4_0003.cbf')
UtilsTest.loadTestImage('ref-TRYP-X1_4_0004.cbf')
referenceDataPath = self.dataPath / 'mosflm_generatePrediction_TRYP-X1_4.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
mosflmIndexingTask = MosflmGeneratePredictionTask(inData=inData)
mosflmIndexingTask.execute()
self.assertTrue(mosflmIndexingTask.isSuccess())
class GetListAutoprocessingResultsExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run ispyb test with default config')
@unittest.skipIf('ISPyB_token' not in os.environ,
'No ISPyB_token found in environment')
def test_execute_getListAutoprocIntegration(self):
referenceDataPath = self.dataPath / \
"GetListAutoprocessingResults.json"
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
getListAutoprocessingResults = GetListAutoprocessingResults(
inData=inData)
getListAutoprocessingResults.execute()
self.assertTrue(getListAutoprocessingResults.isSuccess())
outData = getListAutoprocessingResults.outData
self.assertEqual(2, len(outData['dataCollection']))
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run ispyb test with default config')
def test_execute_getListAutoprocIntegration_invalidToken(self):
referenceDataPath = self.dataPath / \
"GetListAutoprocessingResults.json"
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
inData['token'] = 'abcdefghijklmnopqrstuvwxyz'
getListAutoprocessingResults = GetListAutoprocessingResults(
inData=inData)
getListAutoprocessingResults.execute()
self.assertTrue(getListAutoprocessingResults.isSuccess())
outData = getListAutoprocessingResults.outData
self.assertTrue('error' in outData)
class UtilsIspybExecTest(unittest.TestCase):
@unittest.skipIf(
UtilsConfig.getSite() == "Default",
"Cannot run control dozor test with default config",
)
def test_findDataCollectionFromFileLocationAndFileName(self):
firstImagePath = "/data/id30a2/inhouse/opid30a2/20200907/RAW_DATA/MeshScan_10/mesh-opid30a2_1_0001.cbf"
dataCollection = UtilsIspyb.findDataCollectionFromFileLocationAndFileName(
firstImagePath
)
self.assertEqual(2483117, dataCollection.dataCollectionId)
@unittest.skipIf(
UtilsConfig.getSite() == "Default",
"Cannot run control dozor test with default config",
)
def test_setImageQualityIndicatorsPlot(self):
dataCollectionId = 2483117
letters = string.ascii_lowercase
filePlot = "".join(random.choice(letters) for i in range(10))
fileCsv = "".join(random.choice(letters) for i in range(10))
dataCollectionId2 = UtilsIspyb.setImageQualityIndicatorsPlot(
dataCollectionId, filePlot, fileCsv
)
self.assertEqual(dataCollectionId, dataCollectionId2)
class DozorMTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run dozor test with default config')
def test_execute_id23eh1_mesh1_dozorm(self):
referenceDataPath = self.dataPath / 'opid23eh1_mesh1_dozorm.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
dozorm = DozorM(inData=inData, workingDirectorySuffix="id23eh1_mesh1")
dozorm.execute()
self.assertTrue(dozorm.isSuccess())
outData = dozorm.outData
# self.assertEqual(len(outData['imageDozor']), 10)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run dozor test with default config')
def test_execute_id23eh1_mesh2_dozorm(self):
referenceDataPath = self.dataPath / 'opid23eh1_mesh2_dozorm.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
dozorm = DozorM(inData=inData, workingDirectorySuffix="id23eh1_mesh2")
dozorm.execute()
self.assertTrue(dozorm.isSuccess())
outData = dozorm.outData
# self.assertEqual(len(outData['imageDozor']), 10)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run dozor test with default config')
def test_execute_id23eh3_mesh1_dozorm(self):
referenceDataPath = self.dataPath / 'opid23eh1_mesh3_dozorm.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
dozorm = DozorM(inData=inData, workingDirectorySuffix="id23eh1_mesh3")
dozorm.execute()
self.assertTrue(dozorm.isSuccess())
outData = dozorm.outData
class ReadImageHeaderTasksUnitTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
UtilsTest.loadTestImage('mesh-mx415_1_0001.h5')
def test_readCBFHeader(self):
referenceDataPath = self.dataPath / 'ReadImageHeader_Pilatus2M.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
dictHeader = ReadImageHeader.readCBFHeader(inData['imagePath'][0])
self.assertEqual(dictHeader['Detector:'],
'PILATUS2 3M, S/N 24-0118, ESRF ID23')
@unittest.skipIf(
UtilsConfig.getSite() == 'Default',
'Cannot run dozor test_readEiger4mHeader with default config')
def test_readEiger4mHeader(self):
referenceDataPath = self.dataPath / 'ReadImageHeader_Eiger4M.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath(
inData['imagePath'][0])
dictHeader = ReadImageHeader.readHdf5Header(h5MasterFilePath)
self.assertEqual(dictHeader['description'], 'Dectris Eiger 4M')
@unittest.skipIf(
UtilsConfig.getSite() == 'Default',
'Cannot run dozor test_readEiger16mHeader with default config')
def test_readEiger16mHeader(self):
referenceDataPath = self.dataPath / 'ReadImageHeader_Eiger16M.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath(
inData['imagePath'][0])
dictHeader = ReadImageHeader.readHdf5Header(h5MasterFilePath)
self.assertEqual(dictHeader['description'], 'Dectris EIGER2 CdTe 16M')
class FindPipelineForMergeExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run ImageQualityIndicatorsExecTest ' +
'test with default config')
def test_execute(self):
referenceDataPath = self.dataPath / 'findDataForMerge.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
findPipelineForMerge = FindPipelineForMerge(inData=inData)
findPipelineForMerge.execute()
self.assertTrue(findPipelineForMerge.isSuccess())
outData = findPipelineForMerge.outData
self.assertTrue('schema' in outData)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run ImageQualityIndicatorsExecTest ' +
'test with default config')
def test_execute_invalidToken(self):
referenceDataPath = self.dataPath / 'findDataForMerge_invalidToken.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
findPipelineForMerge = FindPipelineForMerge(inData=inData)
findPipelineForMerge.execute()
self.assertTrue(findPipelineForMerge.isSuccess())
outData = findPipelineForMerge.outData
self.assertTrue('error' in outData)
class RetrieveAttachmentFilesExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run ispyb test with default config')
@unittest.skipIf('ISPyB_token' not in os.environ,
'No ISPyB_token found in environment')
def test_execute_retrieveAttachmentFiles(self):
referenceDataPath = self.dataPath / \
'RetrieveAttachmentFiles.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
retrieveAttachmentFiles = RetrieveAttachmentFiles(inData=inData)
retrieveAttachmentFiles.execute()
self.assertTrue(retrieveAttachmentFiles.isSuccess())
outData = retrieveAttachmentFiles.outData
self.assertEqual(1, len(outData['filePath']))
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run ispyb test with default config')
def test_execute_getListAutoprocIntegration_invalidToken(self):
referenceDataPath = self.dataPath / \
'RetrieveAttachmentFiles.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
inData['token'] = 'abcdefghijklmnopqrstuvwxyz'
retrieveAttachmentFiles = RetrieveAttachmentFiles(inData=inData)
retrieveAttachmentFiles.execute()
self.assertTrue(retrieveAttachmentFiles.isSuccess())
outData = retrieveAttachmentFiles.outData
self.assertTrue('error' in outData)
class H5ToCBFExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run h5ToCbf test with default config')
def test_execute_withImageNumber(self):
referenceDataPath = self.dataPath / 'H5ToCBF_withImageNumber.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath,
loadTestImages=False)
h5ToCBF = H5ToCBFTask(inData=inData)
h5ToCBF.execute()
self.assertTrue(h5ToCBF.isSuccess())
outData = h5ToCBF.outData
self.assertTrue(os.path.exists(outData['outputCBFFile']))
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run h5ToCbf test with default config')
def test_execute_withImageRange(self):
referenceDataPath = self.dataPath / 'H5ToCBF_withImageRange.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath,
loadTestImages=False)
h5ToCBF = H5ToCBFTask(inData=inData)
h5ToCBF.execute()
self.assertTrue(h5ToCBF.isSuccess())
outData = h5ToCBF.outData
for index in range(1, 11):
template = outData['outputCBFFileTemplate']
filePath = template.replace('######', '{0:06d}').format(index)
self.assertTrue(os.path.exists(filePath))
class CreateThumbnailUnitTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run dozor test with default config')
def test_createThumbnail_cbf(self):
referenceDataPath = self.dataPath / 'diffractionThumbnail.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
image = inData["image"][0]
workingDir = tempfile.mkdtemp(prefix="diffractionThumbnail_",
dir="/tmp")
resultPath = CreateThumbnail.createThumbnail(
image, workingDirectory=workingDir)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run dozor test with default config')
def test_createThumbnail_eiger4m(self):
referenceDataPath = self.dataPath / 'diffractionThumbnail_eiger4m.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
image = inData["image"][0]
workingDir = tempfile.mkdtemp(prefix="diffractionThumbnail_",
dir="/tmp")
resultPath = CreateThumbnail.createThumbnail(
image, workingDirectory=workingDir)
def test_getTaskConfig(self):
taskName = "ExecDozor"
dictConfig = UtilsConfig.getTaskConfig(taskName, site='esrf_id30a2')
self.assertTrue("ix_min" in dictConfig)
taskName = "ISPyB"
dictConfig = UtilsConfig.getTaskConfig(taskName, site='esrf_id30a2')
self.assertTrue("username" in dictConfig)
self.assertEqual(dictConfig["username"], os.environ["ISPyB_user"])
class ReadImageHeaderTasksExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
def test_executeReadImageHeaderTaskk(self):
referenceDataPath = self.dataPath / "ControlReadImageHeader.json"
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
readImageHeader = ReadImageHeader(inData=inData)
readImageHeader.execute()
self.assertTrue(readImageHeader.isSuccess())
def test_execute_ReadImageHeader_pilatus2m(self):
referenceDataPath = self.dataPath / "ReadImageHeader_Pilatus2M.json"
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
readImageHeader = ReadImageHeader(inData=inData)
readImageHeader.execute()
self.assertTrue(readImageHeader.isSuccess())
outData = readImageHeader.outData
self.assertIsNotNone(outData)
@unittest.skipIf(
UtilsConfig.getSite() == "Default",
"Cannot run dozor test_execute_ReadImageHeader_eiger4m with default config",
)
def test_execute_ReadImageHeader_eiger4m(self):
referenceDataPath = self.dataPath / "ReadImageHeader_Eiger4M.json"
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
readImageHeader = ReadImageHeader(inData=inData)
readImageHeader.execute()
self.assertTrue(readImageHeader.isSuccess())
outData = readImageHeader.outData
self.assertIsNotNone(outData)
@unittest.skipIf(
UtilsConfig.getSite() == "Default",
"Cannot run dozor test_execute_ReadImageHeader_eiger4m with default config",
)
def test_execute_ReadImageHeader_eiger16m(self):
referenceDataPath = self.dataPath / "ReadImageHeader_Eiger16M.json"
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
readImageHeader = ReadImageHeader(inData=inData)
readImageHeader.execute()
self.assertTrue(readImageHeader.isSuccess())
outData = readImageHeader.outData
self.assertIsNotNone(outData)
@unittest.skipIf(
UtilsConfig.getSite() == "Default",
"Cannot run dozor test_execute_ReadImageHeader_eiger4m with default config",
)
def test_execute_ReadImageHeader_eiger16m_cbf(self):
referenceDataPath = self.dataPath / "ReadImageHeader_Eiger16M_cbf.json"
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
readImageHeader = ReadImageHeader(inData=inData)
readImageHeader.execute()
self.assertTrue(readImageHeader.isSuccess())
outData = readImageHeader.outData
self.assertIsNotNone(outData)
def test_execute_ControlDozor_ispyb_h5(self):
currentSite = UtilsConfig.getSite()
referenceDataPath = self.dataPath / 'ControlDozor_ispyb_hdf5.json'
self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlDozor = ControlDozor(inData=self.inData)
controlDozor.execute()
UtilsConfig.setSite(currentSite)
self.assertTrue(controlDozor.isSuccess())
outData = controlDozor.outData
self.assertEqual(len(outData['imageQualityIndicators']), 51)
class ExecDozorUnitTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
referenceDataPath = self.dataPath / 'inDataDozor.json'
with open(str(referenceDataPath)) as f:
self.inData = json.load(f)
self.dozor = ExecDozor(inData=self.inData)
@unittest.skipIf(
UtilsConfig.getSite() == 'Default',
'Cannot run dozor test_generateCommands with default config')
def test_generateCommands(self):
dozor = ExecDozor(inData=self.inData)
strCommandText = dozor.generateCommands(self.inData)
# print(strCommandText)
self.assertTrue(strCommandText is not None)
def test_parseOutput(self):
self.dozor.startingAngle = 10.0
self.dozor.firstImageNumber = 1
self.dozor.oscillationRange = 0.1
self.dozor.overlap = 0.0
logFileName = self.dataPath / 'Dozor_v2.0.2.log'
with open(str(logFileName)) as f:
output = f.read()
result = self.dozor.parseOutput(self.inData, output)
self.assertEqual(10, len(result['imageDozor']),
"Result from 10 images")
# Log file with 'no results'
logFileName2 = self.dataPath / 'Dozor_v2.0.2_no_results.log'
with open(str(logFileName2)) as f:
output2 = f.read()
result2 = self.dozor.parseOutput(self.inData, output2)
self.assertEqual(51, len(result2['imageDozor']),
"Result from 51 images")
def test_parseDouble(self):
self.assertEqual(1.0, ExecDozor.parseDouble('1.0'), "Parsing '1.0'")
self.assertEqual(None, ExecDozor.parseDouble('****'), "Parsing '****'")
def test_generatePngPlots(self):
plotmtvFile = self.dataPath / 'dozor_rd.mtv'
tmpDir = tempfile.mkdtemp(suffix='EDTestCasePluginUnitDozor_')
listFile = ExecDozor.generatePngPlots(plotmtvFile, tmpDir)
for plotFile in listFile:
self.assertTrue(os.path.exists(plotFile))
shutil.rmtree(tmpDir)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run dozor test_getLibrary with default config')
def test_getLibrary(self):
library = self.dozor.getLibrary('cbf')
self.assertTrue('xds-zcbf.so' in library)
library = self.dozor.getLibrary('hdf5')
self.assertTrue('dectris-neggia.so' in library)
class ControlDozorExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run control dozor test with default config')
@unittest.skipIf(not os.path.exists('/data/id30a2/inhouse/opid30a2/20200907/RAW_DATA/opid30a2_1_0001.cbf'),
'Image /data/id30a2/inhouse/opid30a2/20200907/RAW_DATA/opid30a2_1_0001.cbf doesn\'t exist')
def test_execute_ControlDozor_ispyb(self):
currentSite = UtilsConfig.getSite()
referenceDataPath = self.dataPath / 'ControlDozor_ispyb.json'
self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlDozor = ControlDozor(inData=self.inData)
controlDozor.execute()
self.assertTrue(controlDozor.isSuccess())
UtilsConfig.setSite(currentSite)
outData = controlDozor.outData
self.assertEqual(len(outData['imageQualityIndicators']), 100)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run control dozor test with default config')
# @unittest.skipIf(not os.path.exists('/data/id30a2/inhouse/opid30a2/20200907/RAW_DATA/opid30a2_1_0001.cbf'),
# 'Image /data/id30a2/inhouse/opid30a2/20200907/RAW_DATA/opid30a2_1_0001.cbf doesn\'t exist')
def test_execute_ControlDozor_ispyb_id30a3(self):
currentSite = UtilsConfig.getSite()
referenceDataPath = self.dataPath / 'ControlDozor_ispyb_id30a3.json'
self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlDozor = ControlDozor(inData=self.inData)
controlDozor.execute()
self.assertTrue(controlDozor.isSuccess())
UtilsConfig.setSite(currentSite)
outData = controlDozor.outData
self.assertEqual(len(outData['imageQualityIndicators']), 4)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run control dozor test with default config')
@unittest.skipIf(not os.path.exists('/data/visitor/mx415/id30a3/20171127/' +
'RAW_DATA/mx415/1-2-2/MXPressF_01/' +
'mesh-mx415_1_1_master.h5'),
'Image /data/visitor/mx415/id30a3/20171127/RAW_DATA/mx415/' +
'1-2-2/MXPressF_01/mesh-mx415_1_1_master.h5 doesn\'t exist')
def test_execute_ControlDozor_ispyb_h5(self):
currentSite = UtilsConfig.getSite()
referenceDataPath = self.dataPath / 'ControlDozor_ispyb_hdf5.json'
self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlDozor = ControlDozor(inData=self.inData)
controlDozor.execute()
UtilsConfig.setSite(currentSite)
self.assertTrue(controlDozor.isSuccess())
outData = controlDozor.outData
self.assertEqual(len(outData['imageQualityIndicators']), 51)
class ControlDozorExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run control dozor test with default config')
def test_execute_ControlDozor(self):
referenceDataPath = self.dataPath / 'ControlDozor.json'
self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlDozor = ControlDozor(inData=self.inData)
controlDozor.execute()
self.assertTrue(controlDozor.isSuccess())
outData = controlDozor.outData
self.assertEqual(len(outData['imageQualityIndicators']), 5)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run control dozor test with default config')
def test_execute_ControlDozor_batchSize_2(self):
referenceDataPath = self.dataPath / 'ControlDozor_batchSize_2.json'
self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlDozor = ControlDozor(inData=self.inData)
controlDozor.execute()
self.assertTrue(controlDozor.isSuccess())
outData = controlDozor.outData
self.assertEqual(len(outData['imageQualityIndicators']), 5)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run control dozor test with default config')
def test_execute_ControlDozor_batchSize_2a(self):
referenceDataPath = self.dataPath / 'ControlDozor_batchSize_2a.json'
self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlDozor = ControlDozor(inData=self.inData)
controlDozor.execute()
self.assertTrue(controlDozor.isSuccess())
outData = controlDozor.outData
self.assertEqual(len(outData['imageQualityIndicators']), 4)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run control dozor test with default config')
@unittest.skipIf(
not os.path.exists('/data/visitor/mx415/id30a2/20160315/' +
'RAW_DATA/test3/mx415_1_0001.cbf'),
'Image /data/visitor/mx415/id30a2/20160315/RAW_DATA/' +
'test3/mx415_1_0001.cbf doesn\'t exist')
def test_execute_ControlDozor_wedgeNumber(self):
referenceDataPath = self.dataPath / 'ControlDozor_wedgeNumber.json'
self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlDozor = ControlDozor(inData=self.inData)
controlDozor.execute()
self.assertTrue(controlDozor.isSuccess())
outData = controlDozor.outData
self.assertEqual(len(outData['imageQualityIndicators']), 740)
class ControlIndexingExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run indexing test with default config')
def test_execute_ControlIndexing_opid30a1_4(self):
referenceDataPath = self.dataPath / 'opid30a1_4.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlIndexing = ControlIndexing(inData=inData,
workingDirectorySuffix='opid30a1_4')
controlIndexing.execute()
self.assertTrue(controlIndexing.isSuccess())
self.assertEqual(
controlIndexing.outData["resultIndexing"]["spaceGroupNumber"], 143)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run indexing test with default config')
def test_execute_ControlIndexing_TRYP_X1_4(self):
referenceDataPath = self.dataPath / 'TRYP-X1_4.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlIndexing = ControlIndexing(inData=inData,
workingDirectorySuffix='TRYP-X1_4')
controlIndexing.execute()
self.assertTrue(controlIndexing.isSuccess())
self.assertEqual(
controlIndexing.outData["resultIndexing"]["spaceGroupNumber"], 16)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run indexing test with default config')
def test_execute_ControlIndexing_adrcpt_For1_4(self):
referenceDataPath = self.dataPath / 'adrcpt-For1_4.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlIndexing = ControlIndexing(
inData=inData, workingDirectorySuffix='adrcpt-For1_4')
controlIndexing.execute()
self.assertTrue(controlIndexing.isSuccess())
self.assertEqual(
controlIndexing.outData["resultIndexing"]["spaceGroupNumber"], 75)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run indexing test with default config')
def test_execute_ControlIndexing_MWB_CD269A_07_4(self):
referenceDataPath = self.dataPath / 'MWB-CD269A_07_4.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
controlIndexing = ControlIndexing(
inData=inData, workingDirectorySuffix='MWB-CD269A_07_4')
controlIndexing.execute()
self.assertTrue(controlIndexing.isSuccess())
self.assertEqual(
controlIndexing.outData["resultIndexing"]["spaceGroupNumber"], 75)
def run(self, inData):
dictConfig = UtilsConfig.getTaskConfig('ISPyB')
username = dictConfig['username']
password = dictConfig['password']
httpAuthenticated = HttpAuthenticated(username=username,
password=password)
wdsl = dictConfig['ispyb_ws_url'] + '/ispybWS/ToolsForCollectionWebService?wsdl'
client = Client(wdsl, transport=httpAuthenticated, cache=None)
if 'image' in inData:
path = pathlib.Path(inData['image'])
indir = path.parent.as_posix()
infilename = path.name
dataCollection = client.service.findDataCollectionFromFileLocationAndFileName(
indir,
infilename
)
elif 'dataCollectionId' in inData:
dataCollectionId = inData['dataCollectionId']
dataCollection = client.service.findDataCollection(dataCollectionId)
else:
errorMessage = 'Neither image nor data collection id given as input!'
logger.error(errorMessage)
raise BaseException(errorMessage)
if dataCollection is not None:
outData = Client.dict(dataCollection)
else:
outData = {}
return outData
class ImageQualityIndicatorsExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
# self.dataPath = pathlib.Path(os.getcwd()) / 'data'
@unittest.skipIf(
UtilsConfig.getSite() == "Default",
"Cannot run ImageQualityIndicatorsExecTest " +
"test with default config",
)
@unittest.skipIf(
not os.path.exists(
"/data/scisoft/pxsoft/data/WORKFLOW_TEST_DATA/id30a2/inhouse/opid30a2"
+ "/20191129/RAW_DATA/t1/MeshScan_05/mesh-t1_1_0001.cbf"),
"Cannot find CBF file mesh-t1_1_0001.cbf",
)
def test_execute(self):
referenceDataPath = self.dataPath / "inDataImageQualityIndicatorsTask.json"
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
task = ImageQualityIndicators(inData=inData)
task.execute()
self.assertFalse(task.isFailure())
outData = task.outData
self.assertTrue("imageQualityIndicators" in outData)
# self.assertTrue('resolution_limit' in outData['crystfel_results'][0])
self.assertEqual(72, len(outData["imageQualityIndicators"]))
def init(self, inData):
self.beamline = inData.get("beamline", None)
self.doSubmit = inData.get("doSubmit", False)
self.doDozorM = inData.get("doDozorM", False)
self.doDistlSignalStrength = inData.get("doDistlSignalStrength", False)
self.isFastMesh = inData.get("fastMesh", False)
self.listImage = inData.get("image", [])
self.batchSize = inData.get("batchSize", 1)
self.doIspybUpload = inData.get("doIspybUpload", False)
self.dataCollectionId = inData.get("dataCollectionId", None)
# Configurations
self.minImageSize = UtilsConfig.get(
self, "minImageSize", defaultValue=DEFAULT_MIN_IMAGE_SIZE
)
self.waitFileTimeOut = UtilsConfig.get(
self, "waitFileTimeOut", defaultValue=DEFAULT_WAIT_FILE_TIMEOUT
)
class ImageQualityIndicatorsPilatus6MExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(
UtilsConfig.getSite() == "Default",
"Cannot run ImageQualityIndicatorsExecTest " +
"test with default config",
)
def test_execute_pilatus6m_10images_list(self):
referenceDataPath = self.dataPath / "pilatus6m_10images_list.json"
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
task = ImageQualityIndicators(inData=inData)
task.execute()
self.assertFalse(task.isFailure())
outData = task.outData
self.assertTrue("imageQualityIndicators" in outData)
@unittest.skipIf(
UtilsConfig.getSite() == "Default",
"Cannot run ImageQualityIndicatorsExecTest " +
"test with default config",
)
def test_execute_pilatus6m_10images(self):
referenceDataPath = self.dataPath / "pilatus6m_10images.json"
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
task = ImageQualityIndicators(inData=inData)
task.execute()
self.assertFalse(task.isFailure())
outData = task.outData
self.assertTrue("imageQualityIndicators" in outData)
@unittest.skipIf(
UtilsConfig.getSite() == "Default",
"Cannot run ImageQualityIndicatorsExecTest " +
"test with default config",
)
def test_execute_pilatus6m_10images_crystfel(self):
referenceDataPath = self.dataPath / "pilatus6m_10images_crystfel.json"
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
task = ImageQualityIndicators(inData=inData)
task.execute()
self.assertFalse(task.isFailure())
outData = task.outData
self.assertTrue("imageQualityIndicators" in outData)
def test_execute_ExecDozor_eiger4m(self):
currentSite = UtilsConfig.getSite()
# UtilsConfig.setSite('esrf_ispyb_valid')
referenceDataPath = self.dataPath / 'ExecDozor_eiger4m.json'
self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
dozor = ExecDozor(inData=self.inData)
dozor.execute()
self.assertTrue(dozor.isSuccess())
outData = dozor.outData
self.assertEqual(len(outData['imageDozor']), 51)
def run(self, inData):
outData = {}
hdf5File = pathlib.Path(inData['hdf5File'])
# Read the header
inDataReadImageHeader = {
"imagePath": [inData['hdf5File']],
"isFastMesh": False
}
readImageHeader = ReadImageHeader(inData=inDataReadImageHeader)
readImageHeader.execute()
if readImageHeader.isSuccess():
firstSubWedge = readImageHeader.outData["subWedge"][0]
experimentalCondition = firstSubWedge["experimentalCondition"]
detector = experimentalCondition["detector"]
beam = experimentalCondition["beam"]
goniostat = experimentalCondition["goniostat"]
beamX = detector["beamPositionX"] / 2.0 / detector["pixelSizeX"]
beamY = detector["beamPositionY"] / 2.0 / detector["pixelSizeY"]
directory = hdf5File.parent
prefix = UtilsImage.getPrefix(hdf5File)
hdf5ImageNumber = 1
if 'master' in str(hdf5File):
masterFile = hdf5File
else:
if UtilsConfig.isEMBL():
fileName = '{0}_master.h5'.format(prefix)
else:
fileName = '{0}_{1}_master.h5'.format(
prefix, hdf5ImageNumber)
masterFile = directory / fileName
image = fabio.open(str(masterFile))
image.data = image.data.reshape(2181, 2, 2074, 2).sum(3).sum(1)
cbfImage = image.convert("cbf")
pilatus_headers = fabio.cbfimage.PilatusHeader(
"Silicon sensor, thickness 0.000750 m")
pilatus_headers["Start_angle"] = goniostat["rotationAxisStart"]
pilatus_headers["Angle_increment"] = goniostat["oscillationWidth"]
pilatus_headers["Detector"] = "Eiger2 16M binned to 4M"
pilatus_headers["Pixel_size"] = (detector["pixelSizeY"] * 2,
detector["pixelSizeX"] * 2)
pilatus_headers["Exposure_time"] = beam["exposureTime"]
pilatus_headers["Wavelength"] = beam["wavelength"]
pilatus_headers[
"Detector_distance"] = detector["distance"] / 1000.0
pilatus_headers["Beam_xy"] = (beamY, beamX)
pilatus_headers["Count_cutoff"] = 1009869
cbfImage.pilatus_headers = pilatus_headers
directory = inData.get("forcedOutputDirectory",
str(self.getWorkingDirectory()))
cbfImagePath = os.path.join(
directory,
os.path.basename(inData['hdf5File']).replace(".h5", ".cbf"))
cbfImage.save(cbfImagePath)
outData["outputCBFFile"] = cbfImagePath
return outData
def getExpectedSize(self, imagePath):
# Not great but works...
expectedSize = 1000000
for beamline in [
"id23eh1", "id23eh2", "id30a1", "id30a2", "id30a3", "id30b"
]:
if beamline in imagePath:
taskConfig = UtilsConfig.getTaskConfig("ExpectedFileSize",
"esrf_" + beamline)
expectedSize = int(taskConfig["image"])
break
return expectedSize
class XDSItegrationExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run test with default config')
def test_execute_XDSIntegration(self):
referenceDataPath = self.dataPath / 'id30a1_1_fast_char.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
xdsIntegration = XDSIndexAndIntegration(inData=inData)
xdsIntegration.execute()
self.assertTrue(xdsIntegration.isSuccess())
def addFileHandler(logger):
logPath = UtilsConfig.get("Logging", "log_file_path")
if logPath is not None:
if "DATE" in logPath:
logPath = logPath.replace(
"DATE", time.strftime("%Y-%m-%d", time.localtime(time.time())))
if not os.path.exists(os.path.dirname(logPath)):
os.makedirs(os.path.dirname(logPath))
maxBytes = int(UtilsConfig.get("Logging", "log_file_maxbytes", 1e7))
backupCount = int(UtilsConfig.get("Logging", "log_file_backupCount",
0))
fileHandler = logging.handlers.RotatingFileHandler(
logPath, maxBytes=maxBytes, backupCount=backupCount)
logFileFormat = UtilsConfig.get("Logging", "log_file_format")
if logFileFormat is None:
logFileFormat = (
"%(asctime)s %(module)-20s %(funcName)-15s %(levelname)-8s %(message)s"
)
formatter = logging.Formatter(logFileFormat)
fileHandler.setFormatter(formatter)
logger.addHandler(fileHandler)
class XDSIndexingExecTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run mosflm test with default config')
def test_execute_XDSIndexing(self):
referenceDataPath = self.dataPath / 'inDataXDSIndexing.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
xdsIndexing = XDSIndexing(inData=inData)
xdsIndexing.execute()
self.assertTrue(xdsIndexing.isSuccess())
class DozorRDTest(unittest.TestCase):
def setUp(self):
self.dataPath = UtilsTest.prepareTestDataPath(__file__)
@unittest.skipIf(UtilsConfig.getSite() == 'Default',
'Cannot run dozor test with default config')
def test_execute_DozorRD(self):
referenceDataPath = self.dataPath / 'inDataDozorRD.json'
inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath)
dozorRD = DozorRD(inData=inData)
dozorRD.execute()
self.assertTrue(dozorRD.isSuccess())
outData = dozorRD.outData
