def test_HDFDatastore__repr__():
"""__repr__() returns the correct string representation.
"""
myDB = database.HDFDatastore('the_name', widefieldPixelSize=(0.108, 0.108))
assert_equal(myDB.__repr__(), ('HDFDatastore(\'the_name\', '
'widefieldPixelSize = (0.1080, 0.1080))'))
myDB = database.HDFDatastore('the_name')
assert_equal(myDB.__repr__(), ('HDFDatastore(\'the_name\', '
'widefieldPixelSize = None)'))
def test_HDFDatastore_Context_Manager():
"""HDFDatastores work with 'with...as' statements.
"""
dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/'
'test_id_collection_temp.h5'))
if dsName.exists():
remove(str(dsName))
temp = config.__Registered_DatasetTypes__.copy()
config.__Registered_DatasetTypes__ = [
'Localizations', 'LocMetadata', 'WidefieldImage'
]
parser = parsers.SimpleParser()
filenameStrings = {
'Localizations': '.csv',
'LocMetadata': '.txt',
'WidefieldImage': '.tif'
}
# Use the Datastore as a context manager
with database.HDFDatastore(dsName) as myDS:
myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False)
assert_equal(len(myDS), 6)
# Clean-up the file and reset the registered types
config.__Registered_DatasetTypes__ = temp
if dsName.exists():
remove(str(dsName))
def test_HDF_Datastore_Build_with_fiducialtracks():
"""The datastore build is performed successfully.
"""
dbName = testDataRoot / Path('database_test_files/myDB_Build_Avg.h5')
if dbName.exists():
remove(str(dbName))
parser = parsers.PositionParser(positionIDs={
1: 'prefix',
3: 'channelID',
4: 'acqID'
})
readerDict = {
'FiducialTracks': readers.CSVReader(),
'AverageFiducial': readers.CSVReader()
}
# Directory to traverse for acquisition files
searchDirectory = testDataRoot / Path('test_experiment_2')
# Build datastore
with db.HDFDatastore(dbName) as myDB:
myDB.build(parser,
searchDirectory,
filenameStrings={
'FiducialTracks': '_Fids.dat',
'AverageFiducial': '_AvgFid.dat'
},
readers=readerDict,
dryRun=False)
# Test for existence of the data
with h5py.File(str(dbName), mode='r') as hdf:
key1 = 'Control/Control_1/'
name1 = 'FiducialTracks_ChannelA647'
name2 = 'AverageFiducial_ChannelA647'
ok_(key1 + name1 in hdf)
ok_(key1 + name2 in hdf)
ok_(hdf[key1 + name1].attrs.__contains__('SMLM_prefix'))
ok_(hdf[key1 + name1].attrs.__contains__('SMLM_acqID'))
ok_(hdf[key1 + name1].attrs.__contains__('SMLM_datasetType'))
ok_(hdf[key1 + name1].attrs.__contains__('SMLM_channelID'))
ok_(hdf[key1 + name1].attrs.__contains__('SMLM_dateID'))
ok_(hdf[key1 + name1].attrs.__contains__('SMLM_posID'))
ok_(hdf[key1 + name1].attrs.__contains__('SMLM_sliceID'))
key2 = 'Control/Control_2/'
ok_(key2 + name1 in hdf)
ok_(key2 + name2 in hdf)
key3 = 'shTRF2/shTRF2_1/'
ok_(key3 + name1 in hdf)
ok_(key3 + name2 in hdf)
key4 = 'shTRF2/shTRF2_2/'
ok_(key4 + name1 in hdf)
ok_(key4 + name2 in hdf)
# Remove test datastore file
remove(str(dbName))
def test_WidefieldImage_Datastore_Query():
"""The datastore query is performed successfully with the datasetType.
"""
dbName = testDataRoot / Path('database_test_files/myDB_Build.h5')
if dbName.exists():
remove(str(dbName))
myParser = parsers.SimpleParser()
# Directory to traverse for acquisition files
searchDirectory = testDataRoot / Path('parsers_test_files/SimpleParser')
# Build datastore
with db.HDFDatastore(dbName) as myDB:
myDB.build(myParser, searchDirectory,
filenameStrings = {'WidefieldImage' : '.tif'},
dryRun = False)
results = myDB.query(datasetType = 'WidefieldImage')
ok_(len(results) != 0, 'Error: No dataset types found in DB.')
# There are 2 widefield images
assert_equal(len(results), 2)
for ds in results:
assert_equal(ds.datasetType, 'WidefieldImage')
# Remove test datastore file
remove(str(dbName))
def test_Put_Data():
"""The datasetType can put its own data and datasetIDs.
"""
try:
# Make up some dataset IDs and a dataset
dsIDs = {}
dsIDs['prefix'] = 'test_prefix'
dsIDs['acqID'] = 1
ds = Localizations(datasetIDs=dsIDs)
ds.data = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
pathToDB = testDataRoot
# Remove datastore if it exists
if exists(str(pathToDB / Path('test_db.h5'))):
remove(str(pathToDB / Path('test_db.h5')))
with db.HDFDatastore(pathToDB / Path('test_db.h5')) as myDB:
myDB.put(ds)
key = 'test_prefix/test_prefix_1/Localizations'
with h5py.File(str(pathToDB / Path('test_db.h5')), 'r') as hdf:
assert_equal(hdf[key].attrs['SMLM_datasetType'], 'Localizations')
df = pd.read_hdf(str(pathToDB / Path('test_db.h5')), key=key)
assert_equal(df.loc[0, 'A'], 1)
assert_equal(df.loc[1, 'A'], 2)
assert_equal(df.loc[0, 'B'], 3)
assert_equal(df.loc[1, 'B'], 4)
finally:
# Remove the test datastore
remove(str(pathToDB / Path('test_db.h5')))
def test_HDF_Datastore_WidefieldPixelSize_OMEXML_Only():
"""element_size_um is correct when only OME-XML metadata is present."
"""
dbName = testDataRoot / Path('database_test_files/myDB_Build.h5')
if dbName.exists():
remove(str(dbName))
parser = parsers.PositionParser(positionIDs = {
0 : 'prefix',
1 : 'channelID',
2 : 'acqID'})
# Directory to traverse for acquisition files
searchDirectory = testDataRoot \
/ Path('database_test_files/OME-TIFF_No_MM_Metadata')
# Build datastore
with db.HDFDatastore(dbName) as myDB:
myDB.build(parser, searchDirectory,
filenameStrings = {'WidefieldImage' : '2_MMStack*.ome.tif'},
dryRun = False, readTiffTags = True)
# Test for existence of the data
with h5py.File(str(dbName), mode = 'r') as hdf:
key1 = ('Cos7/Cos7_2/WidefieldImage_ChannelA647/image_data')
ok_('Cos7/Cos7_2/WidefieldImage_ChannelA647' in hdf)
ok_('element_size_um' in hdf[key1].attrs)
assert_equal(hdf[key1].attrs['element_size_um'][0], 1)
assert_equal(hdf[key1].attrs['element_size_um'][1], 0.1)
assert_equal(hdf[key1].attrs['element_size_um'][2], 0.1)
# Remove test datastore file
remove(str(dbName))
def test_HDF_Datastore_Query_with_Localizations():
"""The datastore query is performed successfully with the datasetType.
"""
dbName = testDataRoot / Path('database_test_files/myDB_Build.h5')
if dbName.exists():
remove(str(dbName))
parser = parsers.PositionParser(positionIDs={
1: 'prefix',
3: 'channelID',
4: 'acqID'
})
# Directory to traverse for acquisition files
searchDirectory = testDataRoot / Path('test_experiment_2')
# Build datastore
with db.HDFDatastore(dbName) as myDB:
myDB.build(parser,
searchDirectory,
filenameStrings={'Localizations': '_DC.dat'},
dryRun=False)
results = myDB.query(datasetType='Localizations')
ok_(len(results) != 0, 'Error: No dataset types found in DB.')
for ds in results:
assert_equal(ds.datasetType, 'Localizations')
# Remove test datastore file
remove(str(dbName))
def test_HDFDatastore_Requires_Context_Manager_During_Put():
"""HDFDatadatastore must be used in a with...as block when using put().
"""
dbName = testDataRoot / Path('database_test_files/myDB.h5')
if dbName.exists():
remove(str(dbName))
myDS = TestType.TestType(datasetIDs={
'prefix': 'Cos7',
'acqID': 1,
'channelID': 'A647',
'posID': (0, )
})
myDS.data = data.as_matrix()
myDB = database.HDFDatastore(dbName)
try:
# Error! myDS.build() not used inside with...as block
myDB.put(myDS)
except database.FileNotLocked:
# Clean-up the file and reset the registered types
if dbName.exists():
remove(str(dbName))
raise (
database.FileNotLocked('Error: File is not locked for writing.'))
except:
print("Unexpected error:", sys.exc_info()[0])
raise
def test_UnpackDatasetIDs_PrefixIsMissing():
"""HDFDatastore correctly detects an acqID of None.
"""
myDB = database.HDFDatastore('test_db.h5')
t1 = TestType.TestType(datasetIDs={'acqID': 1})
myDB._unpackDatasetIDs(t1)
def test_UnpackDatasetIDs_AcqIDIsNone():
"""HDFDatastore correctly detects an acqID of None.
"""
myDB = database.HDFDatastore('test_db.h5')
t1 = TestType.TestType(datasetIDs={'prefix': 'HeLa', 'acqID': None})
myDB._unpackDatasetIDs(t1)
def test_HDFDatastore_Context_Manager_Locks_File():
"""HDFDatastores locks files for writing inside the context manager.
"""
dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/'
'test_id_collection_temp.h5'))
if dsName.exists():
remove(str(dsName))
temp = config.__Registered_DatasetTypes__.copy()
config.__Registered_DatasetTypes__ = [
'Localizations', 'LocMetadata', 'WidefieldImage'
]
parser = parsers.SimpleParser()
filenameStrings = {
'Localizations': '.csv',
'LocMetadata': '.txt'
} # Note that widefield images aren't included
try:
# Use the Datastore as a context manager
with database.HDFDatastore(dsName) as myDS:
myDS.build(parser,
dsName.parent,
filenameStrings,
readTiffTags=False)
with database.HDFDatastore(dsName) as badDS:
# The file should be locked so that the other HDFDatastore
# instance cannot perform the build inside the original's
# context. # Widefield images will therefore not be included
# in the build.
badDS.build(parser,
dsName.parent, {'WidefieldImage': '.tif'},
readTiffTags=False)
except filelock.Timeout:
# Clean-up the file and reset the registered types
config.__Registered_DatasetTypes__ = temp
if dsName.exists():
remove(str(dsName))
# rethrow the error
raise (filelock.Timeout(dsName))
def test_HDFDatastore_State_Updates_Correctly():
"""HDFDatastores locks files for writing inside the context manager.
"""
dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/'
'test_id_collection_temp.h5'))
if dsName.exists():
remove(str(dsName))
temp = config.__Registered_DatasetTypes__.copy()
config.__Registered_DatasetTypes__ = [
'Localizations', 'LocMetadata', 'WidefieldImage'
]
parser = parsers.SimpleParser()
filenameStrings = {
'Localizations': '.csv',
'LocMetadata': '.txt'
} # Note that widefield images aren't included
# Use the Datastore as a context manager
with database.HDFDatastore(dsName) as myDS:
myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False)
# Write the widefield images with a new datastore;
# The state of the Datastore should contain the previously written
# datasets as well
with database.HDFDatastore(dsName) as newDS:
newDS.build(parser,
dsName.parent, {'WidefieldImage': '.tif'},
readTiffTags=False)
assert_equal(len(newDS), 6) # If state wasn't updated, it would be 2!
assert_equal(len(myDS), 4) # State isn't updated yet
with myDS:
pass
assert_equal(len(myDS), 6) # State is up-to-date
# Clean-up the file and reset the registered types
config.__Registered_DatasetTypes__ = temp
if dsName.exists():
remove(str(dsName))
def test_HDF_Datastore_Build():
"""The datastore build is performed successfully.
Notes
-----
This also tests that the Micro-Manager metadata is read correctly to obtain
the widefield image pixel size.
"""
dbName = testDataRoot / Path('database_test_files/myDB_Build.h5')
if dbName.exists():
remove(str(dbName))
parser = parsers.PositionParser(positionIDs = {
0 : 'prefix',
2 : 'channelID',
3 : 'acqID'})
# Directory to traverse for acquisition files
searchDirectory = testDataRoot / Path('test_experiment')
# Build datastore
with db.HDFDatastore(dbName) as myDB:
myDB.build(parser, searchDirectory,
filenameStrings = {'WidefieldImage' : '.ome.tif',
'Localizations' : 'locResults.dat'},
dryRun = False, readTiffTags = True)
# Test for existence of the data.
# Pixel sizes should have been obtained from Micro-Manager meta data.
with h5py.File(str(dbName), mode = 'r') as hdf:
key1 = ('HeLaL/HeLaL_1/WidefieldImage_ChannelA647/'
'image_data')
ok_('HeLaL/HeLaL_1/Localizations_ChannelA647' in hdf)
ok_('HeLaL/HeLaL_1/WidefieldImage_ChannelA647' in hdf)
ok_('element_size_um' in hdf[key1].attrs)
assert_equal(hdf[key1].attrs['element_size_um'][0], 1)
assert_equal(hdf[key1].attrs['element_size_um'][1], 0.108)
assert_equal(hdf[key1].attrs['element_size_um'][2], 0.108)
key2 = ('HeLaS/HeLaS_2/WidefieldImage_ChannelA647/'
'image_data')
ok_('HeLaS/HeLaS_2/Localizations_ChannelA647' in hdf)
ok_('HeLaS/HeLaS_2/WidefieldImage_ChannelA647' in hdf)
ok_('element_size_um' in hdf[key2].attrs)
assert_equal(hdf[key2].attrs['element_size_um'][0], 1)
assert_equal(hdf[key2].attrs['element_size_um'][1], 0.108)
assert_equal(hdf[key2].attrs['element_size_um'][2], 0.108)
# Remove test datastore file
remove(str(dbName))
def test_HDFDatastore_Iterable():
"""HDFDatastore acts as an iterable over dataset IDs.
"""
dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/'
'test_id_collection_temp.h5'))
if dsName.exists():
remove(str(dsName))
dsID = database.DatasetID
temp = config.__Registered_DatasetTypes__.copy()
config.__Registered_DatasetTypes__ = [
'Localizations', 'LocMetadata', 'WidefieldImage'
]
# Create ground-truth IDs
gt = [
dsID(name, acqID, dsType, attr, None, None, None, None, None)
for name, acqID in [('HeLaL_Control', 1), ('HeLaS_Control', 2)]
for dsType, attr in [('Localizations',
None), ('LocMetadata',
'Localizations'), ('WidefieldImage',
None)]
]
parser = parsers.SimpleParser()
filenameStrings = {
'Localizations': '.csv',
'LocMetadata': '.txt',
'WidefieldImage': '.tif'
}
with database.HDFDatastore(dsName) as myDS:
myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False)
assert_equal(len(myDS), 6)
for ds in myDS:
ok_(ds in gt, 'Error: DatasetID not found in Datastore')
# Indexing works
ok_(myDS[0] != myDS[1])
# The datastore contains all the ground truth datasets
for dataset in gt:
ok_(dataset in myDS)
# Clean-up the file and reset the registered types
config.__Registered_DatasetTypes__ = temp
if dsName.exists():
remove(str(dsName))
def test_UnpackDatasetIDs_DateIsNone():
"""A dateID of None will not raise an error in _unpackDatasetIDs
"""
myDB = database.HDFDatastore('test_db.h5')
t2 = TestType.TestType(
datasetIDs={
'prefix': 'HeLa',
'acqID': 2,
'channelID': 'A647',
'dateID': None,
'posID': (0, ),
'sliceID': 1
})
myDB._unpackDatasetIDs(t2)
def test_HDFDatastore_GetWithDate():
"""HDFDatastore.get() returns the correct Dataset with a dateID.
"""
dsName = testDataRoot / Path('database_test_files/myDB.h5')
# Created in test_HDFDatastore_Put_Keys_AtomicMetadata()
myDS = database.HDFDatastore(dsName)
dsID = database.DatasetID
# Create an ID with empty data for retrieving the dataset
ds = dsID('Cos7', 1, 'TestType', None, 'A647', '2016-05-05', (1, 2), None,
None)
# Get the data from the datastore and compare it to the input data
retrievedDataset = myDS.get(ds)
ok_(array_equal(retrievedDataset.data, data))
def test_WidefieldImage_DatasetID_Attributes():
"""Dataset IDs are written as attributes of the widefieldImage dataset.
"""
# Remake the datastore
dbName = testDataRoot / Path('database_test_files/myDB_WF_Metadata.h5')
if dbName.exists():
remove(str(dbName))
# Load the widefield image and convert it to a dataset
f = 'Cos7_A647_1_MMStack_Pos0.ome.tif'
inputFile = testDataRoot / Path('database_test_files') \
/ Path('Cos7_A647_WF1/') / Path(f)
# Set the parser to read TiffTags
parser = parsers.PositionParser(positionIDs = {
0 : 'prefix',
1 : 'channelID',
2 : 'acqID'})
parser.parseFilename(str(inputFile), 'WidefieldImage')
ds = parser.dataset
ds.data = ds.readFromFile(str(inputFile), readTiffTags = False)
# Put the widefield image into the datastore
with db.HDFDatastore(dbName) as myDB:
myDB.put(parser.dataset)
# Check that the dataset IDs were put correctly
saveKey = 'Cos7/Cos7_1/WidefieldImage_ChannelA647'
with h5py.File(myDB._dsName, mode = 'r') as dbFile:
ok_(dbFile[saveKey].attrs.__contains__('SMLM_prefix'))
ok_(dbFile[saveKey].attrs.__contains__('SMLM_acqID'))
ok_(dbFile[saveKey].attrs.__contains__('SMLM_datasetType'))
ok_(dbFile[saveKey].attrs.__contains__('SMLM_channelID'))
ok_(dbFile[saveKey].attrs.__contains__('SMLM_dateID'))
ok_(dbFile[saveKey].attrs.__contains__('SMLM_posID'))
ok_(dbFile[saveKey].attrs.__contains__('SMLM_sliceID'))
ok_(dbFile[saveKey].attrs.__contains__('SMLM_Version'))
assert_equal(dbFile[saveKey].attrs['SMLM_prefix'], 'Cos7')
assert_equal(dbFile[saveKey].attrs['SMLM_acqID'], 1)
assert_equal(dbFile[saveKey].attrs['SMLM_datasetType'],
'WidefieldImage')
assert_equal(dbFile[saveKey].attrs['SMLM_channelID'], 'A647')
assert_equal(dbFile[saveKey].attrs['SMLM_dateID'], 'None')
assert_equal(dbFile[saveKey].attrs['SMLM_posID'], 'None')
assert_equal(dbFile[saveKey].attrs['SMLM_sliceID'], 'None')
def test_Put_Data():
"""The datasetType can put its own data and datasetIDs.
Notes
-----
This also tests that the pixel size is correctly extracted from the
Micro-Manager metadata.
"""
imgPath = testDataRoot / Path('database_test_files') \
/ Path('Cos7_A647_WF1/') \
/ Path('Cos7_A647_1_MMStack_Pos0.ome.tif')
try:
# Make up some dataset IDs and a dataset
parser = parsers.PositionParser(positionIDs = {
0 : 'prefix',
1 : 'channelID',
2 : 'acqID'})
parser.parseFilename(str(imgPath), 'WidefieldImage')
ds = parser.dataset
ds.data = ds.readFromFile(str(imgPath), readTiffTags = True)
pathToDB = testDataRoot
# Remove datastore if it exists
if exists(str(pathToDB / Path('test_db.h5'))):
remove(str(pathToDB / Path('test_db.h5')))
with db.HDFDatastore(pathToDB / Path('test_db.h5')) as myDB:
myDB.put(ds)
key = 'Cos7/Cos7_1/WidefieldImage_ChannelA647'
with h5py.File(str(pathToDB / Path('test_db.h5')), 'r') as hdf:
assert_equal(hdf[key].attrs['SMLM_datasetType'], 'WidefieldImage')
imgData = hdf[key + '/image_data'].value
assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][0],
1)
assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][1],
0)
assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][2],
0)
assert_equal(imgData.shape, (512, 512))
finally:
# Remove the test datastore
remove(str(pathToDB / Path('test_db.h5')))
def test_UnpackDatasetIDs_BadDateFormat():
"""The dataset raises an error when a bad date string is supplied.
"""
myDB = database.HDFDatastore('test_db.h5')
t2 = TestType.TestType(
datasetIDs={
'prefix': 'HeLa',
'acqID': 2,
'channelID': 'A647',
# Should be YYYY-MM-DD
'dateID': '2016-9-16',
'posID': (0, ),
'sliceID': 1
})
myDB._unpackDatasetIDs(t2)
def test_Get_Data():
"""The datasetType can get its own data and datasetIDs.
"""
dsID = db.DatasetID
# Load the datastore
imgPath = testDataRoot / Path('database_test_files') \
/ Path('Cos7_A647_WF1/') \
/ Path('Cos7_A647_1_MMStack_Pos0.ome.tif')
img = imread(str(imgPath))
try:
# Make up some dataset IDs and a dataset
parser = parsers.PositionParser(positionIDs = {
0 : 'prefix',
1 : 'channelID',
2 : 'acqID'})
parser.parseFilename(str(imgPath), 'WidefieldImage')
ds = parser.dataset
ds.data = ds.readFromFile(str(imgPath))
pathToDB = testDataRoot
# Remove datastore if it exists
if exists(str(pathToDB / Path('test_db.h5'))):
remove(str(pathToDB / Path('test_db.h5')))
with db.HDFDatastore(pathToDB / Path('test_db.h5')) as myDB:
myDB.put(ds, widefieldPixelSize = (0.13, 0.13))
myNewDSID = dsID('Cos7', 1, 'WidefieldImage', None,
'A647', None, None, None, None)
imgDS = myDB.get(myNewDSID)
ids = imgDS.datasetIDs
assert_equal(ids['prefix'], 'Cos7')
assert_equal(ids['acqID'], 1)
assert_equal(imgDS.datasetType, 'WidefieldImage')
assert_equal(ids['channelID'], 'A647')
assert_equal(ids['dateID'], None)
assert_equal(ids['posID'], None)
assert_equal(ids['sliceID'], None)
assert_equal(ids['replicateID'], None)
assert_equal(imgDS.data.shape, img.shape)
finally:
# Remove the test datastore
remove(str(pathToDB / Path('test_db.h5')))
def test_UnpackDatasetIDs():
"""DatasetIDs are successfully interpreted by the Datastore.
"""
myDB = database.HDFDatastore('test_db.h5')
dsID = database.DatasetID
t1 = TestType.TestType(datasetIDs={'prefix': 'HeLa', 'acqID': 1})
t2 = TestType.TestType(
datasetIDs={
'prefix': 'HeLa',
'acqID': 2,
'channelID': 'A647',
'dateID': '2016-09-16',
'posID': (0, ),
'sliceID': 1,
'replicateID': 2
})
ids1 = myDB._unpackDatasetIDs(t1)
ids2 = myDB._unpackDatasetIDs(t2)
# Ground truths
gt1 = dsID(prefix='HeLa',
acqID=1,
datasetType='TestType',
attributeOf=None,
channelID=None,
dateID=None,
posID=None,
sliceID=None,
replicateID=None)
gt2 = dsID(prefix='HeLa',
acqID=2,
datasetType='TestType',
attributeOf=None,
channelID='A647',
dateID='2016-09-16',
posID=(0, ),
sliceID=1,
replicateID=2)
assert_equal(ids1, gt1)
assert_equal(ids2, gt2)
def test_HDFDatastore_genDataset():
"""Empty datasets are generated properly from id tuples.
"""
myDB = database.HDFDatastore('test_db.h5')
dsID = database.DatasetID
ids = dsID('test_prefix', 2, 'TestType', None, 'A647', None, (0, ), 3, 46)
ds = myDB._genDataset(ids)
assert_equal(ds.datasetIDs['prefix'], 'test_prefix')
assert_equal(ds.datasetIDs['acqID'], 2)
assert_equal(ds.datasetIDs['channelID'], 'A647')
assert_equal(ds.datasetIDs['dateID'], None)
assert_equal(ds.datasetIDs['posID'], (0, ))
assert_equal(ds.datasetIDs['sliceID'], 3)
assert_equal(ds.datasetIDs['replicateID'], 46)
assert_equal(ds.datasetType, 'TestType')
assert_equal(ds.attributeOf, None)
ok_(isinstance(ds, TestType.TestType))
def test_Put_Data_kwarg_WidefieldPixelSize():
"""The WidefieldImage will write the correct pixel size if provided.
"""
# TODO: Rewrite this test to ensure that we really overwrite the metadata
# pixel size.
imgPath = testDataRoot / Path('database_test_files') \
/ Path('Cos7_A647_WF1/') \
/ Path('Cos7_A647_1_MMStack_Pos0.ome.tif')
try:
# Make up some dataset IDs and a dataset
parser = parsers.PositionParser(positionIDs = {
0 : 'prefix',
1 : 'channelID',
2 : 'acqID'})
parser.parseFilename(str(imgPath), 'WidefieldImage')
ds = parser.dataset
ds.data = ds.readFromFile(str(imgPath), readTiffTags = False)
pathToDB = testDataRoot
# Remove datastore if it exists
if exists(str(pathToDB / Path('test_db.h5'))):
remove(str(pathToDB / Path('test_db.h5')))
with db.HDFDatastore(pathToDB / Path('test_db.h5')) as myDB:
myDB.put(ds, widefieldPixelSize = (0.13, 0.14))
# Note that pixel sizes are saved in zyx order.
# These values will be equal to 0.108, 0.108 if no widefieldPixelSize
# is supplied because the default behavior is to read the MM or OME-XML
# metadata.
key = 'Cos7/Cos7_1/WidefieldImage_ChannelA647'
with h5py.File(str(pathToDB / Path('test_db.h5')), 'r') as hdf:
assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][0],
1)
assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][1],
0.14)
assert_equal(hdf[key + '/image_data'].attrs['element_size_um'][2],
0.13)
finally:
# Remove the test datastore
remove(str(pathToDB / Path('test_db.h5')))
def test_HDFDatastore_Check_Key_Existence():
"""An error is raised if using a key that already exists for a dataset.
"""
# Remake the datastore
dsName = testDataRoot / Path('database_test_files/myDB_DoubleKey.h5')
if dsName.exists():
remove(str(dsName))
ds = TestType.TestType(datasetIDs={
'prefix': 'Cos7',
'acqID': 1,
'channelID': 'A647',
'posID': (0, )
})
ds.data = data
# Raises error on the second put because the key already exists.
with database.HDFDatastore(dsName) as myDS:
myDS.put(ds)
myDS.put(ds)
def test_Get_Data():
"""The datasetType can get its own data and datasetIDs.
"""
try:
# Make up some dataset IDs and a dataset
dsIDs = {}
dsIDs['prefix'] = 'test_prefix'
dsIDs['acqID'] = 1
ds = Localizations(datasetIDs=dsIDs)
ds.data = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
pathToDB = testDataRoot
# Remove datastore if it exists
if exists(str(pathToDB / Path('test_db.h5'))):
remove(str(pathToDB / Path('test_db.h5')))
with db.HDFDatastore(pathToDB / Path('test_db.h5')) as myDB:
myDB.put(ds)
# Create a new dataset containing only IDs to test getting of the data
myNewDSID = db.DatasetID('test_prefix', 1, 'Localizations', None, None,
None, None, None, None)
myNewDS = myDB.get(myNewDSID)
ids = myNewDS.datasetIDs
assert_equal(ids['prefix'], 'test_prefix')
assert_equal(ids['acqID'], 1)
assert_equal(myNewDS.datasetType, 'Localizations')
assert_equal(ids['channelID'], None)
assert_equal(ids['dateID'], None)
assert_equal(ids['posID'], None)
assert_equal(ids['sliceID'], None)
assert_equal(ids['replicateID'], None)
assert_equal(myNewDS.data.loc[0, 'A'], 1)
assert_equal(myNewDS.data.loc[1, 'A'], 2)
assert_equal(myNewDS.data.loc[0, 'B'], 3)
assert_equal(myNewDS.data.loc[1, 'B'], 4)
finally:
# Remove the test datastore
remove(str(pathToDB / Path('test_db.h5')))
def test_Put_WidefieldImage_TiffFile():
"""Insertion of widefield image data works when parsed as a TiffFile.
"""
# Remake the datastore
dbName = testDataRoot / Path('database_test_files/myDB_WF_Metadata.h5')
if dbName.exists():
remove(str(dbName))
# Load the widefield image and convert it to an atom
f = 'Cos7_A647_1_MMStack_Pos0.ome.tif'
inputFile = testDataRoot / Path('database_test_files') \
/ Path('Cos7_A647_WF1/') / Path(f)
# Read TiffTags
parser = parsers.PositionParser(positionIDs = {
0 : 'prefix',
1 : 'channelID',
2 : 'acqID'})
parser.parseFilename(str(inputFile), 'WidefieldImage')
ds = parser.dataset
ds.data = ds.readFromFile(str(inputFile), readTiffTags = True)
# Put the widefield image into the datastore
with db.HDFDatastore(dbName) as myDB:
myDB.put(parser.dataset)
# Check that the data was put correctly
saveKey = 'Cos7/Cos7_1/WidefieldImage_ChannelA647'
with h5py.File(myDB._dsName, mode = 'r') as dbFile:
ok_(saveKey + '/image_data' in dbFile,
'Error: Could not find widefield image key.')
# Check that metadata was correctly inserted
ok_(saveKey + '/OME-XML' in dbFile,
'Error: Could not find OME-XML metadata.')
ok_(saveKey + '/MM_Metadata' in dbFile,
'Error: Could not find Micro-Manager metadata.')
ok_(saveKey + '/MM_Summary_Metadata' in dbFile,
'Error: Could not find Micro-Manager summary metadata.')
def __init__(self,
inputDatastore,
pipeline,
outputDirectory='processed_data',
searchString='Localizations'):
"""Parse the input datastore by finding SMLM data files.
The constructor parses the HDF datastore and creates a list of Path
objects all pointing to localization datasets.
Parameters
----------
inputDatastore : str or Path
A string or Path to an HDF datastore.
pipeline : list of Processors
List of Processor objects to process the data.
outputDirectory : str or Path
Relative path to the folder for saving the processed results.
searchString : str
The dataset type to process in batch.
"""
# TODO: Check for file's existence
self._db = dsdb.HDFDatastore(inputDatastore)
try:
self.datasetList = self._db.query(searchString)
self.pipeline = pipeline
if not self.pipeline:
raise UserWarning
elif not self.datasetList:
raise ValueError(
'Error: No datasets containing {:s} were found.'.format(
searchString))
except UserWarning:
print('Warning: Pipeline contains no Processors.')
self._outputDirectory = Path(outputDirectory)
self._searchString = searchString
def test_HDFDatastore_Build_With_Reader():
"""HDFDatastore.build() works when Reader objects are specified.
"""
dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/'
'test_id_collection_temp.h5'))
if dsName.exists():
remove(str(dsName))
temp = config.__Registered_DatasetTypes__.copy()
config.__Registered_DatasetTypes__ = [
'Localizations', 'LocMetadata', 'WidefieldImage'
]
parser = parsers.SimpleParser()
filenameStrings = {
'Localizations': '.csv',
'LocMetadata': '.txt',
'WidefieldImage': '.tif'
}
readersDict = {'Localizations': readers.CSVReader()}
# Note sep and skiprows are keyword arguments of CSVReader; readTiffTags is
# a keyword argument for the WidefieldImage readfromFile() method
with database.HDFDatastore(dsName) as myDS:
res = myDS.build(parser,
dsName.parent,
filenameStrings,
readers=readersDict,
sep=',',
skiprows=2,
readTiffTags=False)
config.__Registered_DatasetTypes__ = temp
if dsName.exists():
remove(str(dsName))
assert_equal(len(res), 6)
def test_HDFDatastore_Requires_Context_Manager_During_Build():
"""HDFDatadatastore must be used in a with...as block when using build().
"""
dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/'
'test_id_collection_temp.h5'))
if dsName.exists():
remove(str(dsName))
temp = config.__Registered_DatasetTypes__.copy()
config.__Registered_DatasetTypes__ = [
'Localizations', 'LocMetadata', 'WidefieldImage'
]
parser = parsers.SimpleParser()
filenameStrings = {
'Localizations': '.csv',
'LocMetadata': '.txt',
'WidefieldImage': '.tif'
}
myDS = database.HDFDatastore(dsName)
try:
# Error! myDS.build() not used inside with...as block
myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False)
except database.FileNotLocked:
# Clean-up the file and reset the registered types
config.__Registered_DatasetTypes__ = temp
if dsName.exists():
remove(str(dsName))
raise (
database.FileNotLocked('Error: File is not locked for writing.'))
except:
print("Unexpected error:", sys.exc_info()[0])
raise
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 11 14:50:05 2016
@author: douglass
"""
import bstore.database as db
import bstore.config as cfg
from pathlib import Path
from bsplugins.leb import MMParser
cfg.__Registered_DatasetTypes__ = [
'Localizations', 'WidefieldImage', 'LocMetadata'
]
directory = Path('./')
parser = MMParser()
filenameStrings = {
'Localizations': '_locResults.dat',
'LocMetadata': '_locMetadata.json',
'WidefieldImage': 'WF*ome.tif'
}
with db.HDFDatastore('test_experiment_db.h5') as myDS:
myDS.build(parser, directory, filenameStrings, readTiffTags=True)
