class ISPyBRetrieveDataCollectionExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run ispyb test with default config') def test_execute_ISPyBRetrieveDataCollection_image(self): referenceDataPath = self.dataPath / \ "ISPyBRetrieveDataCollection_image.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) iSPyBRetrieveDataCollection = ISPyBRetrieveDataCollection(inData=inData) iSPyBRetrieveDataCollection.execute() outData = iSPyBRetrieveDataCollection.outData self.assertEqual(outData['imagePrefix'], 'ref-ednatest') @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run ispyb test with default config') def test_execute_ISPyBRetrieveDataCollection_dataCollectionId(self): referenceDataPath = self.dataPath / \ "ISPyBRetrieveDataCollection_dataCollectionId.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) iSPyBRetrieveDataCollection = ISPyBRetrieveDataCollection(inData=inData) iSPyBRetrieveDataCollection.execute() outData = iSPyBRetrieveDataCollection.outData self.assertEqual(outData['imagePrefix'], 'ref-ednatest')
class DozorM2Test(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run dozor test with default config') def test_execute_DozorM2_twoScans(self): referenceDataPath = self.dataPath / 'inDataDozorM2_twoScans.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) dozorm2 = DozorM2(inData=inData) dozorm2.execute() self.assertTrue(dozorm2.isSuccess()) outData = dozorm2.outData # self.assertEqual(len(outData['imageDozor']), 10) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run dozor test with default config') def test_execute_DozorM2_oneScan(self): referenceDataPath = self.dataPath / 'inDataDozorM2_oneScan.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) dozorm = DozorM2(inData=inData) dozorm.execute() self.assertTrue(dozorm.isSuccess()) outData = dozorm.outData
class ControlIndexingExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run indexing test with default config') def test_execute_ControlIndexing_local_user_1(self): referenceDataPath = self.dataPath / 'local-user_1.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlIndexing = ControlIndexing( inData=inData, workingDirectorySuffix='local_user_1' ) controlIndexing.execute() self.assertTrue(controlIndexing.isSuccess()) self.assertEqual(controlIndexing.outData["resultIndexing"]["spaceGroupNumber"], 16) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run indexing test with default config') def test_execute_ControlIndexing_opid30a3(self): referenceDataPath = self.dataPath / 'UPF2-UPF2__4.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlIndexing = ControlIndexing( inData=inData, workingDirectorySuffix='UPF2-UPF2__4' ) controlIndexing.execute() self.assertTrue(controlIndexing.isSuccess()) self.assertEqual(controlIndexing.outData["resultIndexing"]["spaceGroupNumber"], 16)
class MosflmTasksExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run mosflm test with default config') def tes_execute_MosflmGeneratePredictionTask_2m_RNASE_1(self): UtilsTest.loadTestImage('ref-2m_RNASE_1_0001.cbf') UtilsTest.loadTestImage('ref-2m_RNASE_1_0002.cbf') referenceDataPath = self.dataPath / 'mosflm_generatePrediction_2m_RNASE_1.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) mosflmIndexingTask = MosflmGeneratePredictionTask(inData=inData) mosflmIndexingTask.execute() self.assertTrue(mosflmIndexingTask.isSuccess()) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run mosflm test with default config') def test_execute_MosflmGeneratePredictionTaskTRYP_X1_4(self): UtilsTest.loadTestImage('ref-TRYP-X1_4_0001.cbf') UtilsTest.loadTestImage('ref-TRYP-X1_4_0002.cbf') UtilsTest.loadTestImage('ref-TRYP-X1_4_0003.cbf') UtilsTest.loadTestImage('ref-TRYP-X1_4_0004.cbf') referenceDataPath = self.dataPath / 'mosflm_generatePrediction_TRYP-X1_4.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) mosflmIndexingTask = MosflmGeneratePredictionTask(inData=inData) mosflmIndexingTask.execute() self.assertTrue(mosflmIndexingTask.isSuccess())
class GetListAutoprocessingResultsExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run ispyb test with default config') @unittest.skipIf('ISPyB_token' not in os.environ, 'No ISPyB_token found in environment') def test_execute_getListAutoprocIntegration(self): referenceDataPath = self.dataPath / \ "GetListAutoprocessingResults.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) getListAutoprocessingResults = GetListAutoprocessingResults( inData=inData) getListAutoprocessingResults.execute() self.assertTrue(getListAutoprocessingResults.isSuccess()) outData = getListAutoprocessingResults.outData self.assertEqual(2, len(outData['dataCollection'])) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run ispyb test with default config') def test_execute_getListAutoprocIntegration_invalidToken(self): referenceDataPath = self.dataPath / \ "GetListAutoprocessingResults.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) inData['token'] = 'abcdefghijklmnopqrstuvwxyz' getListAutoprocessingResults = GetListAutoprocessingResults( inData=inData) getListAutoprocessingResults.execute() self.assertTrue(getListAutoprocessingResults.isSuccess()) outData = getListAutoprocessingResults.outData self.assertTrue('error' in outData)
class UtilsIspybExecTest(unittest.TestCase): @unittest.skipIf( UtilsConfig.getSite() == "Default", "Cannot run control dozor test with default config", ) def test_findDataCollectionFromFileLocationAndFileName(self): firstImagePath = "/data/id30a2/inhouse/opid30a2/20200907/RAW_DATA/MeshScan_10/mesh-opid30a2_1_0001.cbf" dataCollection = UtilsIspyb.findDataCollectionFromFileLocationAndFileName( firstImagePath ) self.assertEqual(2483117, dataCollection.dataCollectionId) @unittest.skipIf( UtilsConfig.getSite() == "Default", "Cannot run control dozor test with default config", ) def test_setImageQualityIndicatorsPlot(self): dataCollectionId = 2483117 letters = string.ascii_lowercase filePlot = "".join(random.choice(letters) for i in range(10)) fileCsv = "".join(random.choice(letters) for i in range(10)) dataCollectionId2 = UtilsIspyb.setImageQualityIndicatorsPlot( dataCollectionId, filePlot, fileCsv ) self.assertEqual(dataCollectionId, dataCollectionId2)
class DozorMTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run dozor test with default config') def test_execute_id23eh1_mesh1_dozorm(self): referenceDataPath = self.dataPath / 'opid23eh1_mesh1_dozorm.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) dozorm = DozorM(inData=inData, workingDirectorySuffix="id23eh1_mesh1") dozorm.execute() self.assertTrue(dozorm.isSuccess()) outData = dozorm.outData # self.assertEqual(len(outData['imageDozor']), 10) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run dozor test with default config') def test_execute_id23eh1_mesh2_dozorm(self): referenceDataPath = self.dataPath / 'opid23eh1_mesh2_dozorm.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) dozorm = DozorM(inData=inData, workingDirectorySuffix="id23eh1_mesh2") dozorm.execute() self.assertTrue(dozorm.isSuccess()) outData = dozorm.outData # self.assertEqual(len(outData['imageDozor']), 10) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run dozor test with default config') def test_execute_id23eh3_mesh1_dozorm(self): referenceDataPath = self.dataPath / 'opid23eh1_mesh3_dozorm.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) dozorm = DozorM(inData=inData, workingDirectorySuffix="id23eh1_mesh3") dozorm.execute() self.assertTrue(dozorm.isSuccess()) outData = dozorm.outData
class ReadImageHeaderTasksUnitTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) UtilsTest.loadTestImage('mesh-mx415_1_0001.h5') def test_readCBFHeader(self): referenceDataPath = self.dataPath / 'ReadImageHeader_Pilatus2M.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) dictHeader = ReadImageHeader.readCBFHeader(inData['imagePath'][0]) self.assertEqual(dictHeader['Detector:'], 'PILATUS2 3M, S/N 24-0118, ESRF ID23') @unittest.skipIf( UtilsConfig.getSite() == 'Default', 'Cannot run dozor test_readEiger4mHeader with default config') def test_readEiger4mHeader(self): referenceDataPath = self.dataPath / 'ReadImageHeader_Eiger4M.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath( inData['imagePath'][0]) dictHeader = ReadImageHeader.readHdf5Header(h5MasterFilePath) self.assertEqual(dictHeader['description'], 'Dectris Eiger 4M') @unittest.skipIf( UtilsConfig.getSite() == 'Default', 'Cannot run dozor test_readEiger16mHeader with default config') def test_readEiger16mHeader(self): referenceDataPath = self.dataPath / 'ReadImageHeader_Eiger16M.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) h5MasterFilePath, h5DataFilePath, h5FileNumber = UtilsImage.getH5FilePath( inData['imagePath'][0]) dictHeader = ReadImageHeader.readHdf5Header(h5MasterFilePath) self.assertEqual(dictHeader['description'], 'Dectris EIGER2 CdTe 16M')
class FindPipelineForMergeExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run ImageQualityIndicatorsExecTest ' + 'test with default config') def test_execute(self): referenceDataPath = self.dataPath / 'findDataForMerge.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) findPipelineForMerge = FindPipelineForMerge(inData=inData) findPipelineForMerge.execute() self.assertTrue(findPipelineForMerge.isSuccess()) outData = findPipelineForMerge.outData self.assertTrue('schema' in outData) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run ImageQualityIndicatorsExecTest ' + 'test with default config') def test_execute_invalidToken(self): referenceDataPath = self.dataPath / 'findDataForMerge_invalidToken.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) findPipelineForMerge = FindPipelineForMerge(inData=inData) findPipelineForMerge.execute() self.assertTrue(findPipelineForMerge.isSuccess()) outData = findPipelineForMerge.outData self.assertTrue('error' in outData)
class RetrieveAttachmentFilesExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run ispyb test with default config') @unittest.skipIf('ISPyB_token' not in os.environ, 'No ISPyB_token found in environment') def test_execute_retrieveAttachmentFiles(self): referenceDataPath = self.dataPath / \ 'RetrieveAttachmentFiles.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) retrieveAttachmentFiles = RetrieveAttachmentFiles(inData=inData) retrieveAttachmentFiles.execute() self.assertTrue(retrieveAttachmentFiles.isSuccess()) outData = retrieveAttachmentFiles.outData self.assertEqual(1, len(outData['filePath'])) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run ispyb test with default config') def test_execute_getListAutoprocIntegration_invalidToken(self): referenceDataPath = self.dataPath / \ 'RetrieveAttachmentFiles.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) inData['token'] = 'abcdefghijklmnopqrstuvwxyz' retrieveAttachmentFiles = RetrieveAttachmentFiles(inData=inData) retrieveAttachmentFiles.execute() self.assertTrue(retrieveAttachmentFiles.isSuccess()) outData = retrieveAttachmentFiles.outData self.assertTrue('error' in outData)
class H5ToCBFExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run h5ToCbf test with default config') def test_execute_withImageNumber(self): referenceDataPath = self.dataPath / 'H5ToCBF_withImageNumber.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath, loadTestImages=False) h5ToCBF = H5ToCBFTask(inData=inData) h5ToCBF.execute() self.assertTrue(h5ToCBF.isSuccess()) outData = h5ToCBF.outData self.assertTrue(os.path.exists(outData['outputCBFFile'])) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run h5ToCbf test with default config') def test_execute_withImageRange(self): referenceDataPath = self.dataPath / 'H5ToCBF_withImageRange.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath, loadTestImages=False) h5ToCBF = H5ToCBFTask(inData=inData) h5ToCBF.execute() self.assertTrue(h5ToCBF.isSuccess()) outData = h5ToCBF.outData for index in range(1, 11): template = outData['outputCBFFileTemplate'] filePath = template.replace('######', '{0:06d}').format(index) self.assertTrue(os.path.exists(filePath))
class CreateThumbnailUnitTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run dozor test with default config') def test_createThumbnail_cbf(self): referenceDataPath = self.dataPath / 'diffractionThumbnail.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) image = inData["image"][0] workingDir = tempfile.mkdtemp(prefix="diffractionThumbnail_", dir="/tmp") resultPath = CreateThumbnail.createThumbnail( image, workingDirectory=workingDir) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run dozor test with default config') def test_createThumbnail_eiger4m(self): referenceDataPath = self.dataPath / 'diffractionThumbnail_eiger4m.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) image = inData["image"][0] workingDir = tempfile.mkdtemp(prefix="diffractionThumbnail_", dir="/tmp") resultPath = CreateThumbnail.createThumbnail( image, workingDirectory=workingDir)
def test_getTaskConfig(self): taskName = "ExecDozor" dictConfig = UtilsConfig.getTaskConfig(taskName, site='esrf_id30a2') self.assertTrue("ix_min" in dictConfig) taskName = "ISPyB" dictConfig = UtilsConfig.getTaskConfig(taskName, site='esrf_id30a2') self.assertTrue("username" in dictConfig) self.assertEqual(dictConfig["username"], os.environ["ISPyB_user"])
class ReadImageHeaderTasksExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) def test_executeReadImageHeaderTaskk(self): referenceDataPath = self.dataPath / "ControlReadImageHeader.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) readImageHeader = ReadImageHeader(inData=inData) readImageHeader.execute() self.assertTrue(readImageHeader.isSuccess()) def test_execute_ReadImageHeader_pilatus2m(self): referenceDataPath = self.dataPath / "ReadImageHeader_Pilatus2M.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) readImageHeader = ReadImageHeader(inData=inData) readImageHeader.execute() self.assertTrue(readImageHeader.isSuccess()) outData = readImageHeader.outData self.assertIsNotNone(outData) @unittest.skipIf( UtilsConfig.getSite() == "Default", "Cannot run dozor test_execute_ReadImageHeader_eiger4m with default config", ) def test_execute_ReadImageHeader_eiger4m(self): referenceDataPath = self.dataPath / "ReadImageHeader_Eiger4M.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) readImageHeader = ReadImageHeader(inData=inData) readImageHeader.execute() self.assertTrue(readImageHeader.isSuccess()) outData = readImageHeader.outData self.assertIsNotNone(outData) @unittest.skipIf( UtilsConfig.getSite() == "Default", "Cannot run dozor test_execute_ReadImageHeader_eiger4m with default config", ) def test_execute_ReadImageHeader_eiger16m(self): referenceDataPath = self.dataPath / "ReadImageHeader_Eiger16M.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) readImageHeader = ReadImageHeader(inData=inData) readImageHeader.execute() self.assertTrue(readImageHeader.isSuccess()) outData = readImageHeader.outData self.assertIsNotNone(outData) @unittest.skipIf( UtilsConfig.getSite() == "Default", "Cannot run dozor test_execute_ReadImageHeader_eiger4m with default config", ) def test_execute_ReadImageHeader_eiger16m_cbf(self): referenceDataPath = self.dataPath / "ReadImageHeader_Eiger16M_cbf.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) readImageHeader = ReadImageHeader(inData=inData) readImageHeader.execute() self.assertTrue(readImageHeader.isSuccess()) outData = readImageHeader.outData self.assertIsNotNone(outData)
def test_execute_ControlDozor_ispyb_h5(self): currentSite = UtilsConfig.getSite() referenceDataPath = self.dataPath / 'ControlDozor_ispyb_hdf5.json' self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlDozor = ControlDozor(inData=self.inData) controlDozor.execute() UtilsConfig.setSite(currentSite) self.assertTrue(controlDozor.isSuccess()) outData = controlDozor.outData self.assertEqual(len(outData['imageQualityIndicators']), 51)
class ExecDozorUnitTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) referenceDataPath = self.dataPath / 'inDataDozor.json' with open(str(referenceDataPath)) as f: self.inData = json.load(f) self.dozor = ExecDozor(inData=self.inData) @unittest.skipIf( UtilsConfig.getSite() == 'Default', 'Cannot run dozor test_generateCommands with default config') def test_generateCommands(self): dozor = ExecDozor(inData=self.inData) strCommandText = dozor.generateCommands(self.inData) # print(strCommandText) self.assertTrue(strCommandText is not None) def test_parseOutput(self): self.dozor.startingAngle = 10.0 self.dozor.firstImageNumber = 1 self.dozor.oscillationRange = 0.1 self.dozor.overlap = 0.0 logFileName = self.dataPath / 'Dozor_v2.0.2.log' with open(str(logFileName)) as f: output = f.read() result = self.dozor.parseOutput(self.inData, output) self.assertEqual(10, len(result['imageDozor']), "Result from 10 images") # Log file with 'no results' logFileName2 = self.dataPath / 'Dozor_v2.0.2_no_results.log' with open(str(logFileName2)) as f: output2 = f.read() result2 = self.dozor.parseOutput(self.inData, output2) self.assertEqual(51, len(result2['imageDozor']), "Result from 51 images") def test_parseDouble(self): self.assertEqual(1.0, ExecDozor.parseDouble('1.0'), "Parsing '1.0'") self.assertEqual(None, ExecDozor.parseDouble('****'), "Parsing '****'") def test_generatePngPlots(self): plotmtvFile = self.dataPath / 'dozor_rd.mtv' tmpDir = tempfile.mkdtemp(suffix='EDTestCasePluginUnitDozor_') listFile = ExecDozor.generatePngPlots(plotmtvFile, tmpDir) for plotFile in listFile: self.assertTrue(os.path.exists(plotFile)) shutil.rmtree(tmpDir) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run dozor test_getLibrary with default config') def test_getLibrary(self): library = self.dozor.getLibrary('cbf') self.assertTrue('xds-zcbf.so' in library) library = self.dozor.getLibrary('hdf5') self.assertTrue('dectris-neggia.so' in library)
class ControlDozorExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run control dozor test with default config') @unittest.skipIf(not os.path.exists('/data/id30a2/inhouse/opid30a2/20200907/RAW_DATA/opid30a2_1_0001.cbf'), 'Image /data/id30a2/inhouse/opid30a2/20200907/RAW_DATA/opid30a2_1_0001.cbf doesn\'t exist') def test_execute_ControlDozor_ispyb(self): currentSite = UtilsConfig.getSite() referenceDataPath = self.dataPath / 'ControlDozor_ispyb.json' self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlDozor = ControlDozor(inData=self.inData) controlDozor.execute() self.assertTrue(controlDozor.isSuccess()) UtilsConfig.setSite(currentSite) outData = controlDozor.outData self.assertEqual(len(outData['imageQualityIndicators']), 100) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run control dozor test with default config') # @unittest.skipIf(not os.path.exists('/data/id30a2/inhouse/opid30a2/20200907/RAW_DATA/opid30a2_1_0001.cbf'), # 'Image /data/id30a2/inhouse/opid30a2/20200907/RAW_DATA/opid30a2_1_0001.cbf doesn\'t exist') def test_execute_ControlDozor_ispyb_id30a3(self): currentSite = UtilsConfig.getSite() referenceDataPath = self.dataPath / 'ControlDozor_ispyb_id30a3.json' self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlDozor = ControlDozor(inData=self.inData) controlDozor.execute() self.assertTrue(controlDozor.isSuccess()) UtilsConfig.setSite(currentSite) outData = controlDozor.outData self.assertEqual(len(outData['imageQualityIndicators']), 4) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run control dozor test with default config') @unittest.skipIf(not os.path.exists('/data/visitor/mx415/id30a3/20171127/' + 'RAW_DATA/mx415/1-2-2/MXPressF_01/' + 'mesh-mx415_1_1_master.h5'), 'Image /data/visitor/mx415/id30a3/20171127/RAW_DATA/mx415/' + '1-2-2/MXPressF_01/mesh-mx415_1_1_master.h5 doesn\'t exist') def test_execute_ControlDozor_ispyb_h5(self): currentSite = UtilsConfig.getSite() referenceDataPath = self.dataPath / 'ControlDozor_ispyb_hdf5.json' self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlDozor = ControlDozor(inData=self.inData) controlDozor.execute() UtilsConfig.setSite(currentSite) self.assertTrue(controlDozor.isSuccess()) outData = controlDozor.outData self.assertEqual(len(outData['imageQualityIndicators']), 51)
class ControlDozorExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run control dozor test with default config') def test_execute_ControlDozor(self): referenceDataPath = self.dataPath / 'ControlDozor.json' self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlDozor = ControlDozor(inData=self.inData) controlDozor.execute() self.assertTrue(controlDozor.isSuccess()) outData = controlDozor.outData self.assertEqual(len(outData['imageQualityIndicators']), 5) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run control dozor test with default config') def test_execute_ControlDozor_batchSize_2(self): referenceDataPath = self.dataPath / 'ControlDozor_batchSize_2.json' self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlDozor = ControlDozor(inData=self.inData) controlDozor.execute() self.assertTrue(controlDozor.isSuccess()) outData = controlDozor.outData self.assertEqual(len(outData['imageQualityIndicators']), 5) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run control dozor test with default config') def test_execute_ControlDozor_batchSize_2a(self): referenceDataPath = self.dataPath / 'ControlDozor_batchSize_2a.json' self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlDozor = ControlDozor(inData=self.inData) controlDozor.execute() self.assertTrue(controlDozor.isSuccess()) outData = controlDozor.outData self.assertEqual(len(outData['imageQualityIndicators']), 4) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run control dozor test with default config') @unittest.skipIf( not os.path.exists('/data/visitor/mx415/id30a2/20160315/' + 'RAW_DATA/test3/mx415_1_0001.cbf'), 'Image /data/visitor/mx415/id30a2/20160315/RAW_DATA/' + 'test3/mx415_1_0001.cbf doesn\'t exist') def test_execute_ControlDozor_wedgeNumber(self): referenceDataPath = self.dataPath / 'ControlDozor_wedgeNumber.json' self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlDozor = ControlDozor(inData=self.inData) controlDozor.execute() self.assertTrue(controlDozor.isSuccess()) outData = controlDozor.outData self.assertEqual(len(outData['imageQualityIndicators']), 740)
class ControlIndexingExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run indexing test with default config') def test_execute_ControlIndexing_opid30a1_4(self): referenceDataPath = self.dataPath / 'opid30a1_4.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlIndexing = ControlIndexing(inData=inData, workingDirectorySuffix='opid30a1_4') controlIndexing.execute() self.assertTrue(controlIndexing.isSuccess()) self.assertEqual( controlIndexing.outData["resultIndexing"]["spaceGroupNumber"], 143) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run indexing test with default config') def test_execute_ControlIndexing_TRYP_X1_4(self): referenceDataPath = self.dataPath / 'TRYP-X1_4.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlIndexing = ControlIndexing(inData=inData, workingDirectorySuffix='TRYP-X1_4') controlIndexing.execute() self.assertTrue(controlIndexing.isSuccess()) self.assertEqual( controlIndexing.outData["resultIndexing"]["spaceGroupNumber"], 16) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run indexing test with default config') def test_execute_ControlIndexing_adrcpt_For1_4(self): referenceDataPath = self.dataPath / 'adrcpt-For1_4.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlIndexing = ControlIndexing( inData=inData, workingDirectorySuffix='adrcpt-For1_4') controlIndexing.execute() self.assertTrue(controlIndexing.isSuccess()) self.assertEqual( controlIndexing.outData["resultIndexing"]["spaceGroupNumber"], 75) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run indexing test with default config') def test_execute_ControlIndexing_MWB_CD269A_07_4(self): referenceDataPath = self.dataPath / 'MWB-CD269A_07_4.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) controlIndexing = ControlIndexing( inData=inData, workingDirectorySuffix='MWB-CD269A_07_4') controlIndexing.execute() self.assertTrue(controlIndexing.isSuccess()) self.assertEqual( controlIndexing.outData["resultIndexing"]["spaceGroupNumber"], 75)
def run(self, inData): dictConfig = UtilsConfig.getTaskConfig('ISPyB') username = dictConfig['username'] password = dictConfig['password'] httpAuthenticated = HttpAuthenticated(username=username, password=password) wdsl = dictConfig['ispyb_ws_url'] + '/ispybWS/ToolsForCollectionWebService?wsdl' client = Client(wdsl, transport=httpAuthenticated, cache=None) if 'image' in inData: path = pathlib.Path(inData['image']) indir = path.parent.as_posix() infilename = path.name dataCollection = client.service.findDataCollectionFromFileLocationAndFileName( indir, infilename ) elif 'dataCollectionId' in inData: dataCollectionId = inData['dataCollectionId'] dataCollection = client.service.findDataCollection(dataCollectionId) else: errorMessage = 'Neither image nor data collection id given as input!' logger.error(errorMessage) raise BaseException(errorMessage) if dataCollection is not None: outData = Client.dict(dataCollection) else: outData = {} return outData
class ImageQualityIndicatorsExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) # self.dataPath = pathlib.Path(os.getcwd()) / 'data' @unittest.skipIf( UtilsConfig.getSite() == "Default", "Cannot run ImageQualityIndicatorsExecTest " + "test with default config", ) @unittest.skipIf( not os.path.exists( "/data/scisoft/pxsoft/data/WORKFLOW_TEST_DATA/id30a2/inhouse/opid30a2" + "/20191129/RAW_DATA/t1/MeshScan_05/mesh-t1_1_0001.cbf"), "Cannot find CBF file mesh-t1_1_0001.cbf", ) def test_execute(self): referenceDataPath = self.dataPath / "inDataImageQualityIndicatorsTask.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) task = ImageQualityIndicators(inData=inData) task.execute() self.assertFalse(task.isFailure()) outData = task.outData self.assertTrue("imageQualityIndicators" in outData) # self.assertTrue('resolution_limit' in outData['crystfel_results'][0]) self.assertEqual(72, len(outData["imageQualityIndicators"]))
def init(self, inData): self.beamline = inData.get("beamline", None) self.doSubmit = inData.get("doSubmit", False) self.doDozorM = inData.get("doDozorM", False) self.doDistlSignalStrength = inData.get("doDistlSignalStrength", False) self.isFastMesh = inData.get("fastMesh", False) self.listImage = inData.get("image", []) self.batchSize = inData.get("batchSize", 1) self.doIspybUpload = inData.get("doIspybUpload", False) self.dataCollectionId = inData.get("dataCollectionId", None) # Configurations self.minImageSize = UtilsConfig.get( self, "minImageSize", defaultValue=DEFAULT_MIN_IMAGE_SIZE ) self.waitFileTimeOut = UtilsConfig.get( self, "waitFileTimeOut", defaultValue=DEFAULT_WAIT_FILE_TIMEOUT )
class ImageQualityIndicatorsPilatus6MExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf( UtilsConfig.getSite() == "Default", "Cannot run ImageQualityIndicatorsExecTest " + "test with default config", ) def test_execute_pilatus6m_10images_list(self): referenceDataPath = self.dataPath / "pilatus6m_10images_list.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) task = ImageQualityIndicators(inData=inData) task.execute() self.assertFalse(task.isFailure()) outData = task.outData self.assertTrue("imageQualityIndicators" in outData) @unittest.skipIf( UtilsConfig.getSite() == "Default", "Cannot run ImageQualityIndicatorsExecTest " + "test with default config", ) def test_execute_pilatus6m_10images(self): referenceDataPath = self.dataPath / "pilatus6m_10images.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) task = ImageQualityIndicators(inData=inData) task.execute() self.assertFalse(task.isFailure()) outData = task.outData self.assertTrue("imageQualityIndicators" in outData) @unittest.skipIf( UtilsConfig.getSite() == "Default", "Cannot run ImageQualityIndicatorsExecTest " + "test with default config", ) def test_execute_pilatus6m_10images_crystfel(self): referenceDataPath = self.dataPath / "pilatus6m_10images_crystfel.json" inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) task = ImageQualityIndicators(inData=inData) task.execute() self.assertFalse(task.isFailure()) outData = task.outData self.assertTrue("imageQualityIndicators" in outData)
def test_execute_ExecDozor_eiger4m(self): currentSite = UtilsConfig.getSite() # UtilsConfig.setSite('esrf_ispyb_valid') referenceDataPath = self.dataPath / 'ExecDozor_eiger4m.json' self.inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) dozor = ExecDozor(inData=self.inData) dozor.execute() self.assertTrue(dozor.isSuccess()) outData = dozor.outData self.assertEqual(len(outData['imageDozor']), 51)
def run(self, inData): outData = {} hdf5File = pathlib.Path(inData['hdf5File']) # Read the header inDataReadImageHeader = { "imagePath": [inData['hdf5File']], "isFastMesh": False } readImageHeader = ReadImageHeader(inData=inDataReadImageHeader) readImageHeader.execute() if readImageHeader.isSuccess(): firstSubWedge = readImageHeader.outData["subWedge"][0] experimentalCondition = firstSubWedge["experimentalCondition"] detector = experimentalCondition["detector"] beam = experimentalCondition["beam"] goniostat = experimentalCondition["goniostat"] beamX = detector["beamPositionX"] / 2.0 / detector["pixelSizeX"] beamY = detector["beamPositionY"] / 2.0 / detector["pixelSizeY"] directory = hdf5File.parent prefix = UtilsImage.getPrefix(hdf5File) hdf5ImageNumber = 1 if 'master' in str(hdf5File): masterFile = hdf5File else: if UtilsConfig.isEMBL(): fileName = '{0}_master.h5'.format(prefix) else: fileName = '{0}_{1}_master.h5'.format( prefix, hdf5ImageNumber) masterFile = directory / fileName image = fabio.open(str(masterFile)) image.data = image.data.reshape(2181, 2, 2074, 2).sum(3).sum(1) cbfImage = image.convert("cbf") pilatus_headers = fabio.cbfimage.PilatusHeader( "Silicon sensor, thickness 0.000750 m") pilatus_headers["Start_angle"] = goniostat["rotationAxisStart"] pilatus_headers["Angle_increment"] = goniostat["oscillationWidth"] pilatus_headers["Detector"] = "Eiger2 16M binned to 4M" pilatus_headers["Pixel_size"] = (detector["pixelSizeY"] * 2, detector["pixelSizeX"] * 2) pilatus_headers["Exposure_time"] = beam["exposureTime"] pilatus_headers["Wavelength"] = beam["wavelength"] pilatus_headers[ "Detector_distance"] = detector["distance"] / 1000.0 pilatus_headers["Beam_xy"] = (beamY, beamX) pilatus_headers["Count_cutoff"] = 1009869 cbfImage.pilatus_headers = pilatus_headers directory = inData.get("forcedOutputDirectory", str(self.getWorkingDirectory())) cbfImagePath = os.path.join( directory, os.path.basename(inData['hdf5File']).replace(".h5", ".cbf")) cbfImage.save(cbfImagePath) outData["outputCBFFile"] = cbfImagePath return outData
def getExpectedSize(self, imagePath): # Not great but works... expectedSize = 1000000 for beamline in [ "id23eh1", "id23eh2", "id30a1", "id30a2", "id30a3", "id30b" ]: if beamline in imagePath: taskConfig = UtilsConfig.getTaskConfig("ExpectedFileSize", "esrf_" + beamline) expectedSize = int(taskConfig["image"]) break return expectedSize
class XDSItegrationExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run test with default config') def test_execute_XDSIntegration(self): referenceDataPath = self.dataPath / 'id30a1_1_fast_char.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) xdsIntegration = XDSIndexAndIntegration(inData=inData) xdsIntegration.execute() self.assertTrue(xdsIntegration.isSuccess())
def addFileHandler(logger): logPath = UtilsConfig.get("Logging", "log_file_path") if logPath is not None: if "DATE" in logPath: logPath = logPath.replace( "DATE", time.strftime("%Y-%m-%d", time.localtime(time.time()))) if not os.path.exists(os.path.dirname(logPath)): os.makedirs(os.path.dirname(logPath)) maxBytes = int(UtilsConfig.get("Logging", "log_file_maxbytes", 1e7)) backupCount = int(UtilsConfig.get("Logging", "log_file_backupCount", 0)) fileHandler = logging.handlers.RotatingFileHandler( logPath, maxBytes=maxBytes, backupCount=backupCount) logFileFormat = UtilsConfig.get("Logging", "log_file_format") if logFileFormat is None: logFileFormat = ( "%(asctime)s %(module)-20s %(funcName)-15s %(levelname)-8s %(message)s" ) formatter = logging.Formatter(logFileFormat) fileHandler.setFormatter(formatter) logger.addHandler(fileHandler)
class XDSIndexingExecTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run mosflm test with default config') def test_execute_XDSIndexing(self): referenceDataPath = self.dataPath / 'inDataXDSIndexing.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) xdsIndexing = XDSIndexing(inData=inData) xdsIndexing.execute() self.assertTrue(xdsIndexing.isSuccess())
class DozorRDTest(unittest.TestCase): def setUp(self): self.dataPath = UtilsTest.prepareTestDataPath(__file__) @unittest.skipIf(UtilsConfig.getSite() == 'Default', 'Cannot run dozor test with default config') def test_execute_DozorRD(self): referenceDataPath = self.dataPath / 'inDataDozorRD.json' inData = UtilsTest.loadAndSubstitueTestData(referenceDataPath) dozorRD = DozorRD(inData=inData) dozorRD.execute() self.assertTrue(dozorRD.isSuccess()) outData = dozorRD.outData