def test_HDFDatastore__repr__(): """__repr__() returns the correct string representation. """ myDB = database.HDFDatastore('the_name', widefieldPixelSize=(0.108, 0.108)) assert_equal(myDB.__repr__(), ('HDFDatastore(\'the_name\', ' 'widefieldPixelSize = (0.1080, 0.1080))')) myDB = database.HDFDatastore('the_name') assert_equal(myDB.__repr__(), ('HDFDatastore(\'the_name\', ' 'widefieldPixelSize = None)'))
def test_HDFDatastore_Context_Manager(): """HDFDatastores work with 'with...as' statements. """ dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/' 'test_id_collection_temp.h5')) if dsName.exists(): remove(str(dsName)) temp = config.__Registered_DatasetTypes__.copy() config.__Registered_DatasetTypes__ = [ 'Localizations', 'LocMetadata', 'WidefieldImage' ] parser = parsers.SimpleParser() filenameStrings = { 'Localizations': '.csv', 'LocMetadata': '.txt', 'WidefieldImage': '.tif' } # Use the Datastore as a context manager with database.HDFDatastore(dsName) as myDS: myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False) assert_equal(len(myDS), 6) # Clean-up the file and reset the registered types config.__Registered_DatasetTypes__ = temp if dsName.exists(): remove(str(dsName))
def test_HDF_Datastore_Build_with_fiducialtracks(): """The datastore build is performed successfully. """ dbName = testDataRoot / Path('database_test_files/myDB_Build_Avg.h5') if dbName.exists(): remove(str(dbName)) parser = parsers.PositionParser(positionIDs={ 1: 'prefix', 3: 'channelID', 4: 'acqID' }) readerDict = { 'FiducialTracks': readers.CSVReader(), 'AverageFiducial': readers.CSVReader() } # Directory to traverse for acquisition files searchDirectory = testDataRoot / Path('test_experiment_2') # Build datastore with db.HDFDatastore(dbName) as myDB: myDB.build(parser, searchDirectory, filenameStrings={ 'FiducialTracks': '_Fids.dat', 'AverageFiducial': '_AvgFid.dat' }, readers=readerDict, dryRun=False) # Test for existence of the data with h5py.File(str(dbName), mode='r') as hdf: key1 = 'Control/Control_1/' name1 = 'FiducialTracks_ChannelA647' name2 = 'AverageFiducial_ChannelA647' ok_(key1 + name1 in hdf) ok_(key1 + name2 in hdf) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_prefix')) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_acqID')) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_datasetType')) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_channelID')) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_dateID')) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_posID')) ok_(hdf[key1 + name1].attrs.__contains__('SMLM_sliceID')) key2 = 'Control/Control_2/' ok_(key2 + name1 in hdf) ok_(key2 + name2 in hdf) key3 = 'shTRF2/shTRF2_1/' ok_(key3 + name1 in hdf) ok_(key3 + name2 in hdf) key4 = 'shTRF2/shTRF2_2/' ok_(key4 + name1 in hdf) ok_(key4 + name2 in hdf) # Remove test datastore file remove(str(dbName))
def test_WidefieldImage_Datastore_Query(): """The datastore query is performed successfully with the datasetType. """ dbName = testDataRoot / Path('database_test_files/myDB_Build.h5') if dbName.exists(): remove(str(dbName)) myParser = parsers.SimpleParser() # Directory to traverse for acquisition files searchDirectory = testDataRoot / Path('parsers_test_files/SimpleParser') # Build datastore with db.HDFDatastore(dbName) as myDB: myDB.build(myParser, searchDirectory, filenameStrings = {'WidefieldImage' : '.tif'}, dryRun = False) results = myDB.query(datasetType = 'WidefieldImage') ok_(len(results) != 0, 'Error: No dataset types found in DB.') # There are 2 widefield images assert_equal(len(results), 2) for ds in results: assert_equal(ds.datasetType, 'WidefieldImage') # Remove test datastore file remove(str(dbName))
def test_Put_Data(): """The datasetType can put its own data and datasetIDs. """ try: # Make up some dataset IDs and a dataset dsIDs = {} dsIDs['prefix'] = 'test_prefix' dsIDs['acqID'] = 1 ds = Localizations(datasetIDs=dsIDs) ds.data = pd.DataFrame({'A': [1, 2], 'B': [3, 4]}) pathToDB = testDataRoot # Remove datastore if it exists if exists(str(pathToDB / Path('test_db.h5'))): remove(str(pathToDB / Path('test_db.h5'))) with db.HDFDatastore(pathToDB / Path('test_db.h5')) as myDB: myDB.put(ds) key = 'test_prefix/test_prefix_1/Localizations' with h5py.File(str(pathToDB / Path('test_db.h5')), 'r') as hdf: assert_equal(hdf[key].attrs['SMLM_datasetType'], 'Localizations') df = pd.read_hdf(str(pathToDB / Path('test_db.h5')), key=key) assert_equal(df.loc[0, 'A'], 1) assert_equal(df.loc[1, 'A'], 2) assert_equal(df.loc[0, 'B'], 3) assert_equal(df.loc[1, 'B'], 4) finally: # Remove the test datastore remove(str(pathToDB / Path('test_db.h5')))
def test_HDF_Datastore_WidefieldPixelSize_OMEXML_Only(): """element_size_um is correct when only OME-XML metadata is present." """ dbName = testDataRoot / Path('database_test_files/myDB_Build.h5') if dbName.exists(): remove(str(dbName)) parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 1 : 'channelID', 2 : 'acqID'}) # Directory to traverse for acquisition files searchDirectory = testDataRoot \ / Path('database_test_files/OME-TIFF_No_MM_Metadata') # Build datastore with db.HDFDatastore(dbName) as myDB: myDB.build(parser, searchDirectory, filenameStrings = {'WidefieldImage' : '2_MMStack*.ome.tif'}, dryRun = False, readTiffTags = True) # Test for existence of the data with h5py.File(str(dbName), mode = 'r') as hdf: key1 = ('Cos7/Cos7_2/WidefieldImage_ChannelA647/image_data') ok_('Cos7/Cos7_2/WidefieldImage_ChannelA647' in hdf) ok_('element_size_um' in hdf[key1].attrs) assert_equal(hdf[key1].attrs['element_size_um'][0], 1) assert_equal(hdf[key1].attrs['element_size_um'][1], 0.1) assert_equal(hdf[key1].attrs['element_size_um'][2], 0.1) # Remove test datastore file remove(str(dbName))
def test_HDF_Datastore_Query_with_Localizations(): """The datastore query is performed successfully with the datasetType. """ dbName = testDataRoot / Path('database_test_files/myDB_Build.h5') if dbName.exists(): remove(str(dbName)) parser = parsers.PositionParser(positionIDs={ 1: 'prefix', 3: 'channelID', 4: 'acqID' }) # Directory to traverse for acquisition files searchDirectory = testDataRoot / Path('test_experiment_2') # Build datastore with db.HDFDatastore(dbName) as myDB: myDB.build(parser, searchDirectory, filenameStrings={'Localizations': '_DC.dat'}, dryRun=False) results = myDB.query(datasetType='Localizations') ok_(len(results) != 0, 'Error: No dataset types found in DB.') for ds in results: assert_equal(ds.datasetType, 'Localizations') # Remove test datastore file remove(str(dbName))
def test_HDFDatastore_Requires_Context_Manager_During_Put(): """HDFDatadatastore must be used in a with...as block when using put(). """ dbName = testDataRoot / Path('database_test_files/myDB.h5') if dbName.exists(): remove(str(dbName)) myDS = TestType.TestType(datasetIDs={ 'prefix': 'Cos7', 'acqID': 1, 'channelID': 'A647', 'posID': (0, ) }) myDS.data = data.as_matrix() myDB = database.HDFDatastore(dbName) try: # Error! myDS.build() not used inside with...as block myDB.put(myDS) except database.FileNotLocked: # Clean-up the file and reset the registered types if dbName.exists(): remove(str(dbName)) raise ( database.FileNotLocked('Error: File is not locked for writing.')) except: print("Unexpected error:", sys.exc_info()[0]) raise
def test_UnpackDatasetIDs_PrefixIsMissing(): """HDFDatastore correctly detects an acqID of None. """ myDB = database.HDFDatastore('test_db.h5') t1 = TestType.TestType(datasetIDs={'acqID': 1}) myDB._unpackDatasetIDs(t1)
def test_UnpackDatasetIDs_AcqIDIsNone(): """HDFDatastore correctly detects an acqID of None. """ myDB = database.HDFDatastore('test_db.h5') t1 = TestType.TestType(datasetIDs={'prefix': 'HeLa', 'acqID': None}) myDB._unpackDatasetIDs(t1)
def test_HDFDatastore_Context_Manager_Locks_File(): """HDFDatastores locks files for writing inside the context manager. """ dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/' 'test_id_collection_temp.h5')) if dsName.exists(): remove(str(dsName)) temp = config.__Registered_DatasetTypes__.copy() config.__Registered_DatasetTypes__ = [ 'Localizations', 'LocMetadata', 'WidefieldImage' ] parser = parsers.SimpleParser() filenameStrings = { 'Localizations': '.csv', 'LocMetadata': '.txt' } # Note that widefield images aren't included try: # Use the Datastore as a context manager with database.HDFDatastore(dsName) as myDS: myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False) with database.HDFDatastore(dsName) as badDS: # The file should be locked so that the other HDFDatastore # instance cannot perform the build inside the original's # context. # Widefield images will therefore not be included # in the build. badDS.build(parser, dsName.parent, {'WidefieldImage': '.tif'}, readTiffTags=False) except filelock.Timeout: # Clean-up the file and reset the registered types config.__Registered_DatasetTypes__ = temp if dsName.exists(): remove(str(dsName)) # rethrow the error raise (filelock.Timeout(dsName))
def test_HDFDatastore_State_Updates_Correctly(): """HDFDatastores locks files for writing inside the context manager. """ dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/' 'test_id_collection_temp.h5')) if dsName.exists(): remove(str(dsName)) temp = config.__Registered_DatasetTypes__.copy() config.__Registered_DatasetTypes__ = [ 'Localizations', 'LocMetadata', 'WidefieldImage' ] parser = parsers.SimpleParser() filenameStrings = { 'Localizations': '.csv', 'LocMetadata': '.txt' } # Note that widefield images aren't included # Use the Datastore as a context manager with database.HDFDatastore(dsName) as myDS: myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False) # Write the widefield images with a new datastore; # The state of the Datastore should contain the previously written # datasets as well with database.HDFDatastore(dsName) as newDS: newDS.build(parser, dsName.parent, {'WidefieldImage': '.tif'}, readTiffTags=False) assert_equal(len(newDS), 6) # If state wasn't updated, it would be 2! assert_equal(len(myDS), 4) # State isn't updated yet with myDS: pass assert_equal(len(myDS), 6) # State is up-to-date # Clean-up the file and reset the registered types config.__Registered_DatasetTypes__ = temp if dsName.exists(): remove(str(dsName))
def test_HDF_Datastore_Build(): """The datastore build is performed successfully. Notes ----- This also tests that the Micro-Manager metadata is read correctly to obtain the widefield image pixel size. """ dbName = testDataRoot / Path('database_test_files/myDB_Build.h5') if dbName.exists(): remove(str(dbName)) parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 2 : 'channelID', 3 : 'acqID'}) # Directory to traverse for acquisition files searchDirectory = testDataRoot / Path('test_experiment') # Build datastore with db.HDFDatastore(dbName) as myDB: myDB.build(parser, searchDirectory, filenameStrings = {'WidefieldImage' : '.ome.tif', 'Localizations' : 'locResults.dat'}, dryRun = False, readTiffTags = True) # Test for existence of the data. # Pixel sizes should have been obtained from Micro-Manager meta data. with h5py.File(str(dbName), mode = 'r') as hdf: key1 = ('HeLaL/HeLaL_1/WidefieldImage_ChannelA647/' 'image_data') ok_('HeLaL/HeLaL_1/Localizations_ChannelA647' in hdf) ok_('HeLaL/HeLaL_1/WidefieldImage_ChannelA647' in hdf) ok_('element_size_um' in hdf[key1].attrs) assert_equal(hdf[key1].attrs['element_size_um'][0], 1) assert_equal(hdf[key1].attrs['element_size_um'][1], 0.108) assert_equal(hdf[key1].attrs['element_size_um'][2], 0.108) key2 = ('HeLaS/HeLaS_2/WidefieldImage_ChannelA647/' 'image_data') ok_('HeLaS/HeLaS_2/Localizations_ChannelA647' in hdf) ok_('HeLaS/HeLaS_2/WidefieldImage_ChannelA647' in hdf) ok_('element_size_um' in hdf[key2].attrs) assert_equal(hdf[key2].attrs['element_size_um'][0], 1) assert_equal(hdf[key2].attrs['element_size_um'][1], 0.108) assert_equal(hdf[key2].attrs['element_size_um'][2], 0.108) # Remove test datastore file remove(str(dbName))
def test_HDFDatastore_Iterable(): """HDFDatastore acts as an iterable over dataset IDs. """ dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/' 'test_id_collection_temp.h5')) if dsName.exists(): remove(str(dsName)) dsID = database.DatasetID temp = config.__Registered_DatasetTypes__.copy() config.__Registered_DatasetTypes__ = [ 'Localizations', 'LocMetadata', 'WidefieldImage' ] # Create ground-truth IDs gt = [ dsID(name, acqID, dsType, attr, None, None, None, None, None) for name, acqID in [('HeLaL_Control', 1), ('HeLaS_Control', 2)] for dsType, attr in [('Localizations', None), ('LocMetadata', 'Localizations'), ('WidefieldImage', None)] ] parser = parsers.SimpleParser() filenameStrings = { 'Localizations': '.csv', 'LocMetadata': '.txt', 'WidefieldImage': '.tif' } with database.HDFDatastore(dsName) as myDS: myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False) assert_equal(len(myDS), 6) for ds in myDS: ok_(ds in gt, 'Error: DatasetID not found in Datastore') # Indexing works ok_(myDS[0] != myDS[1]) # The datastore contains all the ground truth datasets for dataset in gt: ok_(dataset in myDS) # Clean-up the file and reset the registered types config.__Registered_DatasetTypes__ = temp if dsName.exists(): remove(str(dsName))
def test_UnpackDatasetIDs_DateIsNone(): """A dateID of None will not raise an error in _unpackDatasetIDs """ myDB = database.HDFDatastore('test_db.h5') t2 = TestType.TestType( datasetIDs={ 'prefix': 'HeLa', 'acqID': 2, 'channelID': 'A647', 'dateID': None, 'posID': (0, ), 'sliceID': 1 }) myDB._unpackDatasetIDs(t2)
def test_HDFDatastore_GetWithDate(): """HDFDatastore.get() returns the correct Dataset with a dateID. """ dsName = testDataRoot / Path('database_test_files/myDB.h5') # Created in test_HDFDatastore_Put_Keys_AtomicMetadata() myDS = database.HDFDatastore(dsName) dsID = database.DatasetID # Create an ID with empty data for retrieving the dataset ds = dsID('Cos7', 1, 'TestType', None, 'A647', '2016-05-05', (1, 2), None, None) # Get the data from the datastore and compare it to the input data retrievedDataset = myDS.get(ds) ok_(array_equal(retrievedDataset.data, data))
def test_WidefieldImage_DatasetID_Attributes(): """Dataset IDs are written as attributes of the widefieldImage dataset. """ # Remake the datastore dbName = testDataRoot / Path('database_test_files/myDB_WF_Metadata.h5') if dbName.exists(): remove(str(dbName)) # Load the widefield image and convert it to a dataset f = 'Cos7_A647_1_MMStack_Pos0.ome.tif' inputFile = testDataRoot / Path('database_test_files') \ / Path('Cos7_A647_WF1/') / Path(f) # Set the parser to read TiffTags parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 1 : 'channelID', 2 : 'acqID'}) parser.parseFilename(str(inputFile), 'WidefieldImage') ds = parser.dataset ds.data = ds.readFromFile(str(inputFile), readTiffTags = False) # Put the widefield image into the datastore with db.HDFDatastore(dbName) as myDB: myDB.put(parser.dataset) # Check that the dataset IDs were put correctly saveKey = 'Cos7/Cos7_1/WidefieldImage_ChannelA647' with h5py.File(myDB._dsName, mode = 'r') as dbFile: ok_(dbFile[saveKey].attrs.__contains__('SMLM_prefix')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_acqID')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_datasetType')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_channelID')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_dateID')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_posID')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_sliceID')) ok_(dbFile[saveKey].attrs.__contains__('SMLM_Version')) assert_equal(dbFile[saveKey].attrs['SMLM_prefix'], 'Cos7') assert_equal(dbFile[saveKey].attrs['SMLM_acqID'], 1) assert_equal(dbFile[saveKey].attrs['SMLM_datasetType'], 'WidefieldImage') assert_equal(dbFile[saveKey].attrs['SMLM_channelID'], 'A647') assert_equal(dbFile[saveKey].attrs['SMLM_dateID'], 'None') assert_equal(dbFile[saveKey].attrs['SMLM_posID'], 'None') assert_equal(dbFile[saveKey].attrs['SMLM_sliceID'], 'None')
def test_Put_Data(): """The datasetType can put its own data and datasetIDs. Notes ----- This also tests that the pixel size is correctly extracted from the Micro-Manager metadata. """ imgPath = testDataRoot / Path('database_test_files') \ / Path('Cos7_A647_WF1/') \ / Path('Cos7_A647_1_MMStack_Pos0.ome.tif') try: # Make up some dataset IDs and a dataset parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 1 : 'channelID', 2 : 'acqID'}) parser.parseFilename(str(imgPath), 'WidefieldImage') ds = parser.dataset ds.data = ds.readFromFile(str(imgPath), readTiffTags = True) pathToDB = testDataRoot # Remove datastore if it exists if exists(str(pathToDB / Path('test_db.h5'))): remove(str(pathToDB / Path('test_db.h5'))) with db.HDFDatastore(pathToDB / Path('test_db.h5')) as myDB: myDB.put(ds) key = 'Cos7/Cos7_1/WidefieldImage_ChannelA647' with h5py.File(str(pathToDB / Path('test_db.h5')), 'r') as hdf: assert_equal(hdf[key].attrs['SMLM_datasetType'], 'WidefieldImage') imgData = hdf[key + '/image_data'].value assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][0], 1) assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][1], 0) assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][2], 0) assert_equal(imgData.shape, (512, 512)) finally: # Remove the test datastore remove(str(pathToDB / Path('test_db.h5')))
def test_UnpackDatasetIDs_BadDateFormat(): """The dataset raises an error when a bad date string is supplied. """ myDB = database.HDFDatastore('test_db.h5') t2 = TestType.TestType( datasetIDs={ 'prefix': 'HeLa', 'acqID': 2, 'channelID': 'A647', # Should be YYYY-MM-DD 'dateID': '2016-9-16', 'posID': (0, ), 'sliceID': 1 }) myDB._unpackDatasetIDs(t2)
def test_Get_Data(): """The datasetType can get its own data and datasetIDs. """ dsID = db.DatasetID # Load the datastore imgPath = testDataRoot / Path('database_test_files') \ / Path('Cos7_A647_WF1/') \ / Path('Cos7_A647_1_MMStack_Pos0.ome.tif') img = imread(str(imgPath)) try: # Make up some dataset IDs and a dataset parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 1 : 'channelID', 2 : 'acqID'}) parser.parseFilename(str(imgPath), 'WidefieldImage') ds = parser.dataset ds.data = ds.readFromFile(str(imgPath)) pathToDB = testDataRoot # Remove datastore if it exists if exists(str(pathToDB / Path('test_db.h5'))): remove(str(pathToDB / Path('test_db.h5'))) with db.HDFDatastore(pathToDB / Path('test_db.h5')) as myDB: myDB.put(ds, widefieldPixelSize = (0.13, 0.13)) myNewDSID = dsID('Cos7', 1, 'WidefieldImage', None, 'A647', None, None, None, None) imgDS = myDB.get(myNewDSID) ids = imgDS.datasetIDs assert_equal(ids['prefix'], 'Cos7') assert_equal(ids['acqID'], 1) assert_equal(imgDS.datasetType, 'WidefieldImage') assert_equal(ids['channelID'], 'A647') assert_equal(ids['dateID'], None) assert_equal(ids['posID'], None) assert_equal(ids['sliceID'], None) assert_equal(ids['replicateID'], None) assert_equal(imgDS.data.shape, img.shape) finally: # Remove the test datastore remove(str(pathToDB / Path('test_db.h5')))
def test_UnpackDatasetIDs(): """DatasetIDs are successfully interpreted by the Datastore. """ myDB = database.HDFDatastore('test_db.h5') dsID = database.DatasetID t1 = TestType.TestType(datasetIDs={'prefix': 'HeLa', 'acqID': 1}) t2 = TestType.TestType( datasetIDs={ 'prefix': 'HeLa', 'acqID': 2, 'channelID': 'A647', 'dateID': '2016-09-16', 'posID': (0, ), 'sliceID': 1, 'replicateID': 2 }) ids1 = myDB._unpackDatasetIDs(t1) ids2 = myDB._unpackDatasetIDs(t2) # Ground truths gt1 = dsID(prefix='HeLa', acqID=1, datasetType='TestType', attributeOf=None, channelID=None, dateID=None, posID=None, sliceID=None, replicateID=None) gt2 = dsID(prefix='HeLa', acqID=2, datasetType='TestType', attributeOf=None, channelID='A647', dateID='2016-09-16', posID=(0, ), sliceID=1, replicateID=2) assert_equal(ids1, gt1) assert_equal(ids2, gt2)
def test_HDFDatastore_genDataset(): """Empty datasets are generated properly from id tuples. """ myDB = database.HDFDatastore('test_db.h5') dsID = database.DatasetID ids = dsID('test_prefix', 2, 'TestType', None, 'A647', None, (0, ), 3, 46) ds = myDB._genDataset(ids) assert_equal(ds.datasetIDs['prefix'], 'test_prefix') assert_equal(ds.datasetIDs['acqID'], 2) assert_equal(ds.datasetIDs['channelID'], 'A647') assert_equal(ds.datasetIDs['dateID'], None) assert_equal(ds.datasetIDs['posID'], (0, )) assert_equal(ds.datasetIDs['sliceID'], 3) assert_equal(ds.datasetIDs['replicateID'], 46) assert_equal(ds.datasetType, 'TestType') assert_equal(ds.attributeOf, None) ok_(isinstance(ds, TestType.TestType))
def test_Put_Data_kwarg_WidefieldPixelSize(): """The WidefieldImage will write the correct pixel size if provided. """ # TODO: Rewrite this test to ensure that we really overwrite the metadata # pixel size. imgPath = testDataRoot / Path('database_test_files') \ / Path('Cos7_A647_WF1/') \ / Path('Cos7_A647_1_MMStack_Pos0.ome.tif') try: # Make up some dataset IDs and a dataset parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 1 : 'channelID', 2 : 'acqID'}) parser.parseFilename(str(imgPath), 'WidefieldImage') ds = parser.dataset ds.data = ds.readFromFile(str(imgPath), readTiffTags = False) pathToDB = testDataRoot # Remove datastore if it exists if exists(str(pathToDB / Path('test_db.h5'))): remove(str(pathToDB / Path('test_db.h5'))) with db.HDFDatastore(pathToDB / Path('test_db.h5')) as myDB: myDB.put(ds, widefieldPixelSize = (0.13, 0.14)) # Note that pixel sizes are saved in zyx order. # These values will be equal to 0.108, 0.108 if no widefieldPixelSize # is supplied because the default behavior is to read the MM or OME-XML # metadata. key = 'Cos7/Cos7_1/WidefieldImage_ChannelA647' with h5py.File(str(pathToDB / Path('test_db.h5')), 'r') as hdf: assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][0], 1) assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][1], 0.14) assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][2], 0.13) finally: # Remove the test datastore remove(str(pathToDB / Path('test_db.h5')))
def test_HDFDatastore_Check_Key_Existence(): """An error is raised if using a key that already exists for a dataset. """ # Remake the datastore dsName = testDataRoot / Path('database_test_files/myDB_DoubleKey.h5') if dsName.exists(): remove(str(dsName)) ds = TestType.TestType(datasetIDs={ 'prefix': 'Cos7', 'acqID': 1, 'channelID': 'A647', 'posID': (0, ) }) ds.data = data # Raises error on the second put because the key already exists. with database.HDFDatastore(dsName) as myDS: myDS.put(ds) myDS.put(ds)
def test_Get_Data(): """The datasetType can get its own data and datasetIDs. """ try: # Make up some dataset IDs and a dataset dsIDs = {} dsIDs['prefix'] = 'test_prefix' dsIDs['acqID'] = 1 ds = Localizations(datasetIDs=dsIDs) ds.data = pd.DataFrame({'A': [1, 2], 'B': [3, 4]}) pathToDB = testDataRoot # Remove datastore if it exists if exists(str(pathToDB / Path('test_db.h5'))): remove(str(pathToDB / Path('test_db.h5'))) with db.HDFDatastore(pathToDB / Path('test_db.h5')) as myDB: myDB.put(ds) # Create a new dataset containing only IDs to test getting of the data myNewDSID = db.DatasetID('test_prefix', 1, 'Localizations', None, None, None, None, None, None) myNewDS = myDB.get(myNewDSID) ids = myNewDS.datasetIDs assert_equal(ids['prefix'], 'test_prefix') assert_equal(ids['acqID'], 1) assert_equal(myNewDS.datasetType, 'Localizations') assert_equal(ids['channelID'], None) assert_equal(ids['dateID'], None) assert_equal(ids['posID'], None) assert_equal(ids['sliceID'], None) assert_equal(ids['replicateID'], None) assert_equal(myNewDS.data.loc[0, 'A'], 1) assert_equal(myNewDS.data.loc[1, 'A'], 2) assert_equal(myNewDS.data.loc[0, 'B'], 3) assert_equal(myNewDS.data.loc[1, 'B'], 4) finally: # Remove the test datastore remove(str(pathToDB / Path('test_db.h5')))
def test_Put_WidefieldImage_TiffFile(): """Insertion of widefield image data works when parsed as a TiffFile. """ # Remake the datastore dbName = testDataRoot / Path('database_test_files/myDB_WF_Metadata.h5') if dbName.exists(): remove(str(dbName)) # Load the widefield image and convert it to an atom f = 'Cos7_A647_1_MMStack_Pos0.ome.tif' inputFile = testDataRoot / Path('database_test_files') \ / Path('Cos7_A647_WF1/') / Path(f) # Read TiffTags parser = parsers.PositionParser(positionIDs = { 0 : 'prefix', 1 : 'channelID', 2 : 'acqID'}) parser.parseFilename(str(inputFile), 'WidefieldImage') ds = parser.dataset ds.data = ds.readFromFile(str(inputFile), readTiffTags = True) # Put the widefield image into the datastore with db.HDFDatastore(dbName) as myDB: myDB.put(parser.dataset) # Check that the data was put correctly saveKey = 'Cos7/Cos7_1/WidefieldImage_ChannelA647' with h5py.File(myDB._dsName, mode = 'r') as dbFile: ok_(saveKey + '/image_data' in dbFile, 'Error: Could not find widefield image key.') # Check that metadata was correctly inserted ok_(saveKey + '/OME-XML' in dbFile, 'Error: Could not find OME-XML metadata.') ok_(saveKey + '/MM_Metadata' in dbFile, 'Error: Could not find Micro-Manager metadata.') ok_(saveKey + '/MM_Summary_Metadata' in dbFile, 'Error: Could not find Micro-Manager summary metadata.')
def __init__(self, inputDatastore, pipeline, outputDirectory='processed_data', searchString='Localizations'): """Parse the input datastore by finding SMLM data files. The constructor parses the HDF datastore and creates a list of Path objects all pointing to localization datasets. Parameters ---------- inputDatastore : str or Path A string or Path to an HDF datastore. pipeline : list of Processors List of Processor objects to process the data. outputDirectory : str or Path Relative path to the folder for saving the processed results. searchString : str The dataset type to process in batch. """ # TODO: Check for file's existence self._db = dsdb.HDFDatastore(inputDatastore) try: self.datasetList = self._db.query(searchString) self.pipeline = pipeline if not self.pipeline: raise UserWarning elif not self.datasetList: raise ValueError( 'Error: No datasets containing {:s} were found.'.format( searchString)) except UserWarning: print('Warning: Pipeline contains no Processors.') self._outputDirectory = Path(outputDirectory) self._searchString = searchString
def test_HDFDatastore_Build_With_Reader(): """HDFDatastore.build() works when Reader objects are specified. """ dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/' 'test_id_collection_temp.h5')) if dsName.exists(): remove(str(dsName)) temp = config.__Registered_DatasetTypes__.copy() config.__Registered_DatasetTypes__ = [ 'Localizations', 'LocMetadata', 'WidefieldImage' ] parser = parsers.SimpleParser() filenameStrings = { 'Localizations': '.csv', 'LocMetadata': '.txt', 'WidefieldImage': '.tif' } readersDict = {'Localizations': readers.CSVReader()} # Note sep and skiprows are keyword arguments of CSVReader; readTiffTags is # a keyword argument for the WidefieldImage readfromFile() method with database.HDFDatastore(dsName) as myDS: res = myDS.build(parser, dsName.parent, filenameStrings, readers=readersDict, sep=',', skiprows=2, readTiffTags=False) config.__Registered_DatasetTypes__ = temp if dsName.exists(): remove(str(dsName)) assert_equal(len(res), 6)
def test_HDFDatastore_Requires_Context_Manager_During_Build(): """HDFDatadatastore must be used in a with...as block when using build(). """ dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/' 'test_id_collection_temp.h5')) if dsName.exists(): remove(str(dsName)) temp = config.__Registered_DatasetTypes__.copy() config.__Registered_DatasetTypes__ = [ 'Localizations', 'LocMetadata', 'WidefieldImage' ] parser = parsers.SimpleParser() filenameStrings = { 'Localizations': '.csv', 'LocMetadata': '.txt', 'WidefieldImage': '.tif' } myDS = database.HDFDatastore(dsName) try: # Error! myDS.build() not used inside with...as block myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False) except database.FileNotLocked: # Clean-up the file and reset the registered types config.__Registered_DatasetTypes__ = temp if dsName.exists(): remove(str(dsName)) raise ( database.FileNotLocked('Error: File is not locked for writing.')) except: print("Unexpected error:", sys.exc_info()[0]) raise
# -*- coding: utf-8 -*- """ Created on Thu Aug 11 14:50:05 2016 @author: douglass """ import bstore.database as db import bstore.config as cfg from pathlib import Path from bsplugins.leb import MMParser cfg.__Registered_DatasetTypes__ = [ 'Localizations', 'WidefieldImage', 'LocMetadata' ] directory = Path('./') parser = MMParser() filenameStrings = { 'Localizations': '_locResults.dat', 'LocMetadata': '_locMetadata.json', 'WidefieldImage': 'WF*ome.tif' } with db.HDFDatastore('test_experiment_db.h5') as myDS: myDS.build(parser, directory, filenameStrings, readTiffTags=True)