def test_HDF_Datastore_Build_with_fiducialtracks(): """The datastore build is performed successfully. """ dbName = testDataRoot / Path('database_test_files/myDB_Build_Avg.h5') if dbName.exists(): remove(str(dbName)) parser = parsers.PositionParser(positionIDs={ 1: 'prefix', 3: 'channelID', 4: 'acqID' }) readerDict = { 'FiducialTracks': readers.CSVReader(), 'AverageFiducial': readers.CSVReader() } # Directory to traverse for acquisition files searchDirectory = testDataRoot / Path('test_experiment_2') # Build datastore with db.HDFDatastore(dbName) as myDB: myDB.build(parser, searchDirectory, filenameStrings={ 'FiducialTracks': '_Fids.dat', 'AverageFiducial': '_AvgFid.dat' }, readers=readerDict, dryRun=False) # Test for existence of the data with h5py.File(str(dbName), mode='r') as hdf: key1 = 'Control/Control_1/' name1 = 'FiducialTracks_ChannelA647' name2 = 'AverageFiducial_ChannelA647' ok_(key1 + name1 in hdf) ok_(key1 + name2 in hdf) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_prefix')) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_acqID')) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_datasetType')) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_channelID')) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_dateID')) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_posID')) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_sliceID')) key2 = 'Control/Control_2/' ok_(key2 + name1 in hdf) ok_(key2 + name2 in hdf) key3 = 'shTRF2/shTRF2_1/' ok_(key3 + name1 in hdf) ok_(key3 + name2 in hdf) key4 = 'shTRF2/shTRF2_2/' ok_(key4 + name1 in hdf) ok_(key4 + name2 in hdf) # Remove test datastore file remove(str(dbName))
def test_HDF_Datastore_Query_with_Localizations(): """The datastore query is performed successfully with the datasetType. """ dbName = testDataRoot / Path('database_test_files/myDB_Build.h5') if dbName.exists(): remove(str(dbName)) parser = parsers.PositionParser(positionIDs={ 1: 'prefix', 3: 'channelID', 4: 'acqID' }) # Directory to traverse for acquisition files searchDirectory = testDataRoot / Path('test_experiment_2') # Build datastore with db.HDFDatastore(dbName) as myDB: myDB.build(parser, searchDirectory, filenameStrings={'Localizations': '_DC.dat'}, dryRun=False) results = myDB.query(datasetType='Localizations') ok_(len(results) != 0, 'Error: No dataset types found in DB.') for ds in results: assert_equal(ds.datasetType, 'Localizations') # Remove test datastore file remove(str(dbName))
def test_PositionParser_ParseFilename(): """PositionParser's full parseFilename() function works as expected. """ f = 'HeLaL_Control_1.csv' inputFile = testDataRoot / Path('parsers_test_files') \ / Path('SimpleParser/') / Path(f) parser = parsers.PositionParser(positionIDs={ 0: 'prefix', 1: None, 2: 'acqID' }) # Note: 'Control' will be dropped because it's surrounded by underscores parser.parseFilename(inputFile) assert_equal(parser.dataset.datasetIDs['acqID'], 1) assert_equal(parser.dataset.datasetIDs['prefix'], 'HeLaL') assert_equal(parser.dataset.datasetType, 'Localizations') f = 'HeLaS_Control_2.csv' inputFile = testDataRoot / Path('parsers_test_files') \ / Path('SimpleParser/') / Path(f) parser.parseFilename(inputFile) assert_equal(parser.dataset.datasetIDs['acqID'], 2) assert_equal(parser.dataset.datasetIDs['prefix'], 'HeLaS') assert_equal(parser.dataset.datasetType, 'Localizations')
def test_HDF_Datastore_WidefieldPixelSize_OMEXML_Only(): """element_size_um is correct when only OME-XML metadata is present." """ dbName = testDataRoot / Path('database_test_files/myDB_Build.h5') if dbName.exists(): remove(str(dbName)) parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 1 : 'channelID', 2 : 'acqID'}) # Directory to traverse for acquisition files searchDirectory = testDataRoot \ / Path('database_test_files/OME-TIFF_No_MM_Metadata') # Build datastore with db.HDFDatastore(dbName) as myDB: myDB.build(parser, searchDirectory, filenameStrings = {'WidefieldImage' : '2_MMStack*.ome.tif'}, dryRun = False, readTiffTags = True) # Test for existence of the data with h5py.File(str(dbName), mode = 'r') as hdf: key1 = ('Cos7/Cos7_2/WidefieldImage_ChannelA647/image_data') ok_('Cos7/Cos7_2/WidefieldImage_ChannelA647' in hdf) ok_('element_size_um' in hdf[key1].attrs) assert_equal(hdf[key1].attrs['element_size_um'][0], 1) assert_equal(hdf[key1].attrs['element_size_um'][1], 0.1) assert_equal(hdf[key1].attrs['element_size_um'][2], 0.1) # Remove test datastore file remove(str(dbName))
def test_PositionParser_parse(): """PositionParser correctly parses a number of different example filenames. """ # filename, position ids, expected result f = [('HeLa_2', { 0: 'prefix', 1: 'acqID' }, { 'prefix': 'HeLa', 'acqID': 2 }), ('HeLa_A647_2', { 0: 'prefix', 1: 'channelID', 2: 'acqID' }, { 'prefix': 'HeLa', 'channelID': 'A647', 'acqID': 2 }), ('2016-12-11_Cos7_A647_5_4_3', { 0: 'dateID', 1: 'prefix', 2: 'channelID', 3: 'posID', 4: 'sliceID', 5: 'acqID' }, { 'dateID': '2016-12-11', 'prefix': 'Cos7', 'channelID': 'A647', 'posID': 5, 'sliceID': 4, 'acqID': 3 }), ('HeLa_1_MMStack_0', { 0: 'prefix', 1: 'acqID', 2: None, 3: 'posID' }, { 'prefix': 'HeLa', 'acqID': 1, 'posID': 0 })] for currExample in f: parser = parsers.PositionParser(positionIDs=currExample[1]) idDict = parser._parse(currExample[0]) for key, value in idDict.items(): assert_equal(value, currExample[2][key])
def test_PositionParser_BadParse(): """PositionParser correctly catches errors during parsing. """ f = 'HeLaL.tif' # No acqID; file shouldn't parse parser = parsers.PositionParser(positionIDs={ 0: 'prefix', 1: None, 2: 'acqID' }) # Note: There are more position IDs than there are actual positions in f parser.parseFilename(f)
def test_HDF_Datastore_Build(): """The datastore build is performed successfully. Notes ----- This also tests that the Micro-Manager metadata is read correctly to obtain the widefield image pixel size. """ dbName = testDataRoot / Path('database_test_files/myDB_Build.h5') if dbName.exists(): remove(str(dbName)) parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 2 : 'channelID', 3 : 'acqID'}) # Directory to traverse for acquisition files searchDirectory = testDataRoot / Path('test_experiment') # Build datastore with db.HDFDatastore(dbName) as myDB: myDB.build(parser, searchDirectory, filenameStrings = {'WidefieldImage' : '.ome.tif', 'Localizations' : 'locResults.dat'}, dryRun = False, readTiffTags = True) # Test for existence of the data. # Pixel sizes should have been obtained from Micro-Manager meta data. with h5py.File(str(dbName), mode = 'r') as hdf: key1 = ('HeLaL/HeLaL_1/WidefieldImage_ChannelA647/' 'image_data') ok_('HeLaL/HeLaL_1/Localizations_ChannelA647' in hdf) ok_('HeLaL/HeLaL_1/WidefieldImage_ChannelA647' in hdf) ok_('element_size_um' in hdf[key1].attrs) assert_equal(hdf[key1].attrs['element_size_um'][0], 1) assert_equal(hdf[key1].attrs['element_size_um'][1], 0.108) assert_equal(hdf[key1].attrs['element_size_um'][2], 0.108) key2 = ('HeLaS/HeLaS_2/WidefieldImage_ChannelA647/' 'image_data') ok_('HeLaS/HeLaS_2/Localizations_ChannelA647' in hdf) ok_('HeLaS/HeLaS_2/WidefieldImage_ChannelA647' in hdf) ok_('element_size_um' in hdf[key2].attrs) assert_equal(hdf[key2].attrs['element_size_um'][0], 1) assert_equal(hdf[key2].attrs['element_size_um'][1], 0.108) assert_equal(hdf[key2].attrs['element_size_um'][2], 0.108) # Remove test datastore file remove(str(dbName))
def test_Put_Data(): """The datasetType can put its own data and datasetIDs. Notes ----- This also tests that the pixel size is correctly extracted from the Micro-Manager metadata. """ imgPath = testDataRoot / Path('database_test_files') \ / Path('Cos7_A647_WF1/') \ / Path('Cos7_A647_1_MMStack_Pos0.ome.tif') try: # Make up some dataset IDs and a dataset parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 1 : 'channelID', 2 : 'acqID'}) parser.parseFilename(str(imgPath), 'WidefieldImage') ds = parser.dataset ds.data = ds.readFromFile(str(imgPath), readTiffTags = True) pathToDB = testDataRoot # Remove datastore if it exists if exists(str(pathToDB / Path('test_db.h5'))): remove(str(pathToDB / Path('test_db.h5'))) with db.HDFDatastore(pathToDB / Path('test_db.h5')) as myDB: myDB.put(ds) key = 'Cos7/Cos7_1/WidefieldImage_ChannelA647' with h5py.File(str(pathToDB / Path('test_db.h5')), 'r') as hdf: assert_equal(hdf[key].attrs['SMLM_datasetType'], 'WidefieldImage') imgData = hdf[key + '/image_data'].value assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][0], 1) assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][1], 0) assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][2], 0) assert_equal(imgData.shape, (512, 512)) finally: # Remove the test datastore remove(str(pathToDB / Path('test_db.h5')))
def test_WidefieldImage_DatasetID_Attributes(): """Dataset IDs are written as attributes of the widefieldImage dataset. """ # Remake the datastore dbName = testDataRoot / Path('database_test_files/myDB_WF_Metadata.h5') if dbName.exists(): remove(str(dbName)) # Load the widefield image and convert it to a dataset f = 'Cos7_A647_1_MMStack_Pos0.ome.tif' inputFile = testDataRoot / Path('database_test_files') \ / Path('Cos7_A647_WF1/') / Path(f) # Set the parser to read TiffTags parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 1 : 'channelID', 2 : 'acqID'}) parser.parseFilename(str(inputFile), 'WidefieldImage') ds = parser.dataset ds.data = ds.readFromFile(str(inputFile), readTiffTags = False) # Put the widefield image into the datastore with db.HDFDatastore(dbName) as myDB: myDB.put(parser.dataset) # Check that the dataset IDs were put correctly saveKey = 'Cos7/Cos7_1/WidefieldImage_ChannelA647' with h5py.File(myDB._dsName, mode = 'r') as dbFile: ok_(dbFile[saveKey].attrs.__contains__('SMLM_prefix')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_acqID')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_datasetType')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_channelID')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_dateID')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_posID')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_sliceID')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_Version')) assert_equal(dbFile[saveKey].attrs['SMLM_prefix'], 'Cos7') assert_equal(dbFile[saveKey].attrs['SMLM_acqID'], 1) assert_equal(dbFile[saveKey].attrs['SMLM_datasetType'], 'WidefieldImage') assert_equal(dbFile[saveKey].attrs['SMLM_channelID'], 'A647') assert_equal(dbFile[saveKey].attrs['SMLM_dateID'], 'None') assert_equal(dbFile[saveKey].attrs['SMLM_posID'], 'None') assert_equal(dbFile[saveKey].attrs['SMLM_sliceID'], 'None')
def test_Get_Data(): """The datasetType can get its own data and datasetIDs. """ dsID = db.DatasetID # Load the datastore imgPath = testDataRoot / Path('database_test_files') \ / Path('Cos7_A647_WF1/') \ / Path('Cos7_A647_1_MMStack_Pos0.ome.tif') img = imread(str(imgPath)) try: # Make up some dataset IDs and a dataset parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 1 : 'channelID', 2 : 'acqID'}) parser.parseFilename(str(imgPath), 'WidefieldImage') ds = parser.dataset ds.data = ds.readFromFile(str(imgPath)) pathToDB = testDataRoot # Remove datastore if it exists if exists(str(pathToDB / Path('test_db.h5'))): remove(str(pathToDB / Path('test_db.h5'))) with db.HDFDatastore(pathToDB / Path('test_db.h5')) as myDB: myDB.put(ds, widefieldPixelSize = (0.13, 0.13)) myNewDSID = dsID('Cos7', 1, 'WidefieldImage', None, 'A647', None, None, None, None) imgDS = myDB.get(myNewDSID) ids = imgDS.datasetIDs assert_equal(ids['prefix'], 'Cos7') assert_equal(ids['acqID'], 1) assert_equal(imgDS.datasetType, 'WidefieldImage') assert_equal(ids['channelID'], 'A647') assert_equal(ids['dateID'], None) assert_equal(ids['posID'], None) assert_equal(ids['sliceID'], None) assert_equal(ids['replicateID'], None) assert_equal(imgDS.data.shape, img.shape) finally: # Remove the test datastore remove(str(pathToDB / Path('test_db.h5')))
def test_Put_Data_kwarg_WidefieldPixelSize(): """The WidefieldImage will write the correct pixel size if provided. """ # TODO: Rewrite this test to ensure that we really overwrite the metadata # pixel size. imgPath = testDataRoot / Path('database_test_files') \ / Path('Cos7_A647_WF1/') \ / Path('Cos7_A647_1_MMStack_Pos0.ome.tif') try: # Make up some dataset IDs and a dataset parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 1 : 'channelID', 2 : 'acqID'}) parser.parseFilename(str(imgPath), 'WidefieldImage') ds = parser.dataset ds.data = ds.readFromFile(str(imgPath), readTiffTags = False) pathToDB = testDataRoot # Remove datastore if it exists if exists(str(pathToDB / Path('test_db.h5'))): remove(str(pathToDB / Path('test_db.h5'))) with db.HDFDatastore(pathToDB / Path('test_db.h5')) as myDB: myDB.put(ds, widefieldPixelSize = (0.13, 0.14)) # Note that pixel sizes are saved in zyx order. # These values will be equal to 0.108, 0.108 if no widefieldPixelSize # is supplied because the default behavior is to read the MM or OME-XML # metadata. key = 'Cos7/Cos7_1/WidefieldImage_ChannelA647' with h5py.File(str(pathToDB / Path('test_db.h5')), 'r') as hdf: assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][0], 1) assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][1], 0.14) assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][2], 0.13) finally: # Remove the test datastore remove(str(pathToDB / Path('test_db.h5')))
def test_Put_WidefieldImage_TiffFile(): """Insertion of widefield image data works when parsed as a TiffFile. """ # Remake the datastore dbName = testDataRoot / Path('database_test_files/myDB_WF_Metadata.h5') if dbName.exists(): remove(str(dbName)) # Load the widefield image and convert it to an atom f = 'Cos7_A647_1_MMStack_Pos0.ome.tif' inputFile = testDataRoot / Path('database_test_files') \ / Path('Cos7_A647_WF1/') / Path(f) # Read TiffTags parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 1 : 'channelID', 2 : 'acqID'}) parser.parseFilename(str(inputFile), 'WidefieldImage') ds = parser.dataset ds.data = ds.readFromFile(str(inputFile), readTiffTags = True) # Put the widefield image into the datastore with db.HDFDatastore(dbName) as myDB: myDB.put(parser.dataset) # Check that the data was put correctly saveKey = 'Cos7/Cos7_1/WidefieldImage_ChannelA647' with h5py.File(myDB._dsName, mode = 'r') as dbFile: ok_(saveKey + '/image_data' in dbFile, 'Error: Could not find widefield image key.') # Check that metadata was correctly inserted ok_(saveKey + '/OME-XML' in dbFile, 'Error: Could not find OME-XML metadata.') ok_(saveKey + '/MM_Metadata' in dbFile, 'Error: Could not find Micro-Manager metadata.') ok_(saveKey + '/MM_Summary_Metadata' in dbFile, 'Error: Could not find Micro-Manager summary metadata.')
def test_PositionParser_GetDataset_NotInitialized(): """PositionParser returns raises a not-initialized error. """ parser = parsers.PositionParser() parser.dataset