def test_HDFDatastore_Context_Manager():
"""HDFDatastores work with 'with...as' statements.
"""
dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/'
'test_id_collection_temp.h5'))
if dsName.exists():
remove(str(dsName))
temp = config.__Registered_DatasetTypes__.copy()
config.__Registered_DatasetTypes__ = [
'Localizations', 'LocMetadata', 'WidefieldImage'
]
parser = parsers.SimpleParser()
filenameStrings = {
'Localizations': '.csv',
'LocMetadata': '.txt',
'WidefieldImage': '.tif'
}
# Use the Datastore as a context manager
with database.HDFDatastore(dsName) as myDS:
myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False)
assert_equal(len(myDS), 6)
# Clean-up the file and reset the registered types
config.__Registered_DatasetTypes__ = temp
if dsName.exists():
remove(str(dsName))
def test_WidefieldImage_Datastore_Query():
"""The datastore query is performed successfully with the datasetType.
"""
dbName = testDataRoot / Path('database_test_files/myDB_Build.h5')
if dbName.exists():
remove(str(dbName))
myParser = parsers.SimpleParser()
# Directory to traverse for acquisition files
searchDirectory = testDataRoot / Path('parsers_test_files/SimpleParser')
# Build datastore
with db.HDFDatastore(dbName) as myDB:
myDB.build(myParser, searchDirectory,
filenameStrings = {'WidefieldImage' : '.tif'},
dryRun = False)
results = myDB.query(datasetType = 'WidefieldImage')
ok_(len(results) != 0, 'Error: No dataset types found in DB.')
# There are 2 widefield images
assert_equal(len(results), 2)
for ds in results:
assert_equal(ds.datasetType, 'WidefieldImage')
# Remove test datastore file
remove(str(dbName))
def test_SimpleParser_BadParse():
"""SimpleParser correctly catches errors during parsing.
"""
f = 'HeLaL.tif' # No acqID; file shouldn't parse
parser = parsers.SimpleParser()
parser.parseFilename(f, datasetType='WidefieldImage')
def test_SimpleParser_ParseFilename_WidefieldImage():
"""SimpleParser correctly converts widefield image files to Datasets.
"""
# File number 1
f = 'HeLaL_Control_1.tif'
inputFile = testDataRoot / Path('parsers_test_files') \
/ Path('SimpleParser/') / Path(f)
parser = parsers.SimpleParser()
parser.parseFilename(inputFile, datasetType='WidefieldImage')
assert_equal(parser.dataset.datasetIDs['acqID'], 1)
assert_equal(parser.dataset.datasetIDs['prefix'], 'HeLaL_Control')
assert_equal(parser.dataset.datasetType, 'WidefieldImage')
f = 'HeLaS_Control_2.tif'
inputFile = testDataRoot / Path('parsers_test_files') \
/ Path('SimpleParser/') / Path(f)
parser = parsers.SimpleParser()
parser.parseFilename(inputFile, datasetType='WidefieldImage')
assert_equal(parser.dataset.datasetIDs['acqID'], 2)
assert_equal(parser.dataset.datasetIDs['prefix'], 'HeLaS_Control')
assert_equal(parser.dataset.datasetType, 'WidefieldImage')
def test_SimpleParser_ParseFilename_Localizations():
"""SimpleParser correctly converts localization files to Datasets.
"""
# File number 1
f = 'HeLaL_Control_1.csv'
inputFile = testDataRoot / Path('parsers_test_files') \
/ Path('SimpleParser/') / Path(f)
parser = parsers.SimpleParser()
parser.parseFilename(inputFile)
assert_equal(parser.dataset.datasetIDs['acqID'], 1)
assert_equal(parser.dataset.datasetIDs['prefix'], 'HeLaL_Control')
assert_equal(parser.dataset.datasetType, 'Localizations')
f = 'HeLaS_Control_2.csv'
inputFile = testDataRoot / Path('parsers_test_files') \
/ Path('SimpleParser/') / Path(f)
parser = parsers.SimpleParser()
parser.parseFilename(inputFile)
assert_equal(parser.dataset.datasetIDs['acqID'], 2)
assert_equal(parser.dataset.datasetIDs['prefix'], 'HeLaS_Control')
assert_equal(parser.dataset.datasetType, 'Localizations')
def test_HDFDatastore_Iterable():
"""HDFDatastore acts as an iterable over dataset IDs.
"""
dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/'
'test_id_collection_temp.h5'))
if dsName.exists():
remove(str(dsName))
dsID = database.DatasetID
temp = config.__Registered_DatasetTypes__.copy()
config.__Registered_DatasetTypes__ = [
'Localizations', 'LocMetadata', 'WidefieldImage'
]
# Create ground-truth IDs
gt = [
dsID(name, acqID, dsType, attr, None, None, None, None, None)
for name, acqID in [('HeLaL_Control', 1), ('HeLaS_Control', 2)]
for dsType, attr in [('Localizations',
None), ('LocMetadata',
'Localizations'), ('WidefieldImage',
None)]
]
parser = parsers.SimpleParser()
filenameStrings = {
'Localizations': '.csv',
'LocMetadata': '.txt',
'WidefieldImage': '.tif'
}
with database.HDFDatastore(dsName) as myDS:
myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False)
assert_equal(len(myDS), 6)
for ds in myDS:
ok_(ds in gt, 'Error: DatasetID not found in Datastore')
# Indexing works
ok_(myDS[0] != myDS[1])
# The datastore contains all the ground truth datasets
for dataset in gt:
ok_(dataset in myDS)
# Clean-up the file and reset the registered types
config.__Registered_DatasetTypes__ = temp
if dsName.exists():
remove(str(dsName))
def test_SimpleParser_GetDataset():
"""SimpleParser returns the correct Dataset.
"""
f = 'HeLaL_Control_1.tif'
inputFile = testDataRoot / Path('parsers_test_files') \
/ Path('SimpleParser/') / Path(f)
parser = parsers.SimpleParser()
parser.parseFilename(inputFile, datasetType='WidefieldImage')
parser.dataset.data = parser.dataset.readFromFile(inputFile)
# Test the size of the widefield image and its first value
assert_equal(parser.dataset.datasetIDs['prefix'], 'HeLaL_Control')
assert_equal(parser.dataset.data.shape, (927, 927))
def test_SimpleParser_Read_WidefieldImage():
"""SimpleParser correctly reads metadata.
"""
f = 'HeLaL_Control_1.tif'
inputFile = testDataRoot / Path('parsers_test_files') \
/ Path('SimpleParser/') / Path(f)
parser = parsers.SimpleParser()
parser.parseFilename(inputFile, datasetType='WidefieldImage')
parser.dataset.data = parser.dataset.readFromFile(inputFile)
# Test the size of the widefield image and its first value
assert_equal(parser.dataset.data.shape, (927, 927))
assert_equal(parser.dataset.data[0, 0], 102)
def test_SimpleParser_Read_LocMetadata():
"""SimpleParser correctly reads metadata.
"""
f = 'HeLaL_Control_1.txt'
inputFile = testDataRoot / Path('parsers_test_files') \
/ Path('SimpleParser/') / Path(f)
parser = parsers.SimpleParser()
parser.parseFilename(inputFile, datasetType='LocMetadata')
parser.dataset.data = parser.dataset.readFromFile(inputFile)
# Test a couple of the metadata fields
assert_equal(parser.dataset.data['StartFrame_sCMOS'], 50)
assert_equal(parser.dataset.data['Width'], 927)
def test_SimpleParser_Read_Localizations():
"""SimpleParser correctly reads localization results.
"""
f = 'HeLaL_Control_1.csv'
inputFile = testDataRoot / Path('parsers_test_files') \
/ Path('SimpleParser/') / Path(f)
parser = parsers.SimpleParser()
parser.parseFilename(inputFile, datasetType='Localizations')
parser.dataset.data = parser.dataset.readFromFile(inputFile)
# Test a couple of the localization results
assert_equal(parser.dataset.data['x'].iloc[0], 6770)
assert_equal(parser.dataset.data['intensity'].iloc[0], 4386.6)
assert_equal(parser.dataset.data['x'].iloc[1], 7958.1)
def test_HDFDatastore_Context_Manager_Locks_File():
"""HDFDatastores locks files for writing inside the context manager.
"""
dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/'
'test_id_collection_temp.h5'))
if dsName.exists():
remove(str(dsName))
temp = config.__Registered_DatasetTypes__.copy()
config.__Registered_DatasetTypes__ = [
'Localizations', 'LocMetadata', 'WidefieldImage'
]
parser = parsers.SimpleParser()
filenameStrings = {
'Localizations': '.csv',
'LocMetadata': '.txt'
} # Note that widefield images aren't included
try:
# Use the Datastore as a context manager
with database.HDFDatastore(dsName) as myDS:
myDS.build(parser,
dsName.parent,
filenameStrings,
readTiffTags=False)
with database.HDFDatastore(dsName) as badDS:
# The file should be locked so that the other HDFDatastore
# instance cannot perform the build inside the original's
# context. # Widefield images will therefore not be included
# in the build.
badDS.build(parser,
dsName.parent, {'WidefieldImage': '.tif'},
readTiffTags=False)
except filelock.Timeout:
# Clean-up the file and reset the registered types
config.__Registered_DatasetTypes__ = temp
if dsName.exists():
remove(str(dsName))
# rethrow the error
raise (filelock.Timeout(dsName))
def test_HDFDatastore_State_Updates_Correctly():
"""HDFDatastores locks files for writing inside the context manager.
"""
dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/'
'test_id_collection_temp.h5'))
if dsName.exists():
remove(str(dsName))
temp = config.__Registered_DatasetTypes__.copy()
config.__Registered_DatasetTypes__ = [
'Localizations', 'LocMetadata', 'WidefieldImage'
]
parser = parsers.SimpleParser()
filenameStrings = {
'Localizations': '.csv',
'LocMetadata': '.txt'
} # Note that widefield images aren't included
# Use the Datastore as a context manager
with database.HDFDatastore(dsName) as myDS:
myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False)
# Write the widefield images with a new datastore;
# The state of the Datastore should contain the previously written
# datasets as well
with database.HDFDatastore(dsName) as newDS:
newDS.build(parser,
dsName.parent, {'WidefieldImage': '.tif'},
readTiffTags=False)
assert_equal(len(newDS), 6) # If state wasn't updated, it would be 2!
assert_equal(len(myDS), 4) # State isn't updated yet
with myDS:
pass
assert_equal(len(myDS), 6) # State is up-to-date
# Clean-up the file and reset the registered types
config.__Registered_DatasetTypes__ = temp
if dsName.exists():
remove(str(dsName))
def test_CSVReader_Works_With_Parser():
"""CSVReader is correctly passed to readFromFile() from SimpleParser.
"""
filePath = testDataRoot / Path('readers_test_files/csv/tab_delimited/')
filename = filePath / Path('HeLaL_Control_1.csv')
# Initialize the Parser and Reader
parser = parsers.SimpleParser()
reader = readers.CSVReader()
# reader keyword argument passes the CSVReader instance;
# all other keyword arguments are passed to CSVReader's __call__ function.
parser.parseFilename(
filename, datasetType = 'Localizations', reader = reader, sep = '\t')
# Test a couple of the localization results
assert_equal(parser.dataset.data['x'].iloc[0], 6770)
assert_equal(parser.dataset.data['intensity'].iloc[0],4386.6)
assert_equal(parser.dataset.data['x'].iloc[1], 7958.1)
assert_equal(len(parser.dataset.data.columns), 9)
assert_equal(len(parser.dataset.data), 11)
def test_HDFDatastore_Build_With_Reader():
"""HDFDatastore.build() works when Reader objects are specified.
"""
dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/'
'test_id_collection_temp.h5'))
if dsName.exists():
remove(str(dsName))
temp = config.__Registered_DatasetTypes__.copy()
config.__Registered_DatasetTypes__ = [
'Localizations', 'LocMetadata', 'WidefieldImage'
]
parser = parsers.SimpleParser()
filenameStrings = {
'Localizations': '.csv',
'LocMetadata': '.txt',
'WidefieldImage': '.tif'
}
readersDict = {'Localizations': readers.CSVReader()}
# Note sep and skiprows are keyword arguments of CSVReader; readTiffTags is
# a keyword argument for the WidefieldImage readfromFile() method
with database.HDFDatastore(dsName) as myDS:
res = myDS.build(parser,
dsName.parent,
filenameStrings,
readers=readersDict,
sep=',',
skiprows=2,
readTiffTags=False)
config.__Registered_DatasetTypes__ = temp
if dsName.exists():
remove(str(dsName))
assert_equal(len(res), 6)
def test_HDFDatastore_Requires_Context_Manager_During_Build():
"""HDFDatadatastore must be used in a with...as block when using build().
"""
dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/'
'test_id_collection_temp.h5'))
if dsName.exists():
remove(str(dsName))
temp = config.__Registered_DatasetTypes__.copy()
config.__Registered_DatasetTypes__ = [
'Localizations', 'LocMetadata', 'WidefieldImage'
]
parser = parsers.SimpleParser()
filenameStrings = {
'Localizations': '.csv',
'LocMetadata': '.txt',
'WidefieldImage': '.tif'
}
myDS = database.HDFDatastore(dsName)
try:
# Error! myDS.build() not used inside with...as block
myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False)
except database.FileNotLocked:
# Clean-up the file and reset the registered types
config.__Registered_DatasetTypes__ = temp
if dsName.exists():
remove(str(dsName))
raise (
database.FileNotLocked('Error: File is not locked for writing.'))
except:
print("Unexpected error:", sys.exc_info()[0])
raise
def test_SimpleParser_GetDataset_NotInitialized():
"""SimpleParser returns raises a not-initialized error.
"""
parser = parsers.SimpleParser()
parser.dataset
def test_HDFDatastore_Persistent_State():
"""The HDFDatastore writes the state of the HDFDatastore object to file.
"""
dsName = testDataRoot / Path(('parsers_test_files/SimpleParser/'
'test_id_collection_temp.h5'))
if dsName.exists():
remove(str(dsName))
dsID = database.DatasetID
temp = config.__Registered_DatasetTypes__.copy()
config.__Registered_DatasetTypes__ = [
'Localizations', 'LocMetadata', 'WidefieldImage'
]
# Create ground-truth IDs
gt = [
dsID(name, acqID, dsType, attr, None, None, None, None, None)
for name, acqID in [('HeLaL_Control', 1), ('HeLaS_Control', 2)]
for dsType, attr in [('Localizations',
None), ('LocMetadata',
'Localizations'), ('WidefieldImage',
None)]
]
parser = parsers.SimpleParser()
filenameStrings = {
'Localizations': '.csv',
'LocMetadata': '.txt',
'WidefieldImage': '.tif'
}
with database.HDFDatastore(dsName) as myDS:
myDS.build(parser, dsName.parent, filenameStrings, readTiffTags=False)
# Open the HDF file and read the state of the HDFDatastore object
# Also, delete the old HDFDatastore object
del (myDS)
with h5py.File(str(dsName), mode='r') as file:
serialObject = file[config.__Persistence_Key__]
newDS = database.HDFDatastore('test')
newState = pickle.loads(serialObject.value)
newDS.__dict__.update(newState.__dict__)
assert_equal(len(newDS), 6)
for ds in newDS:
ok_(ds in gt, 'Error: DatasetID not found in Datastore')
# Indexing works
ok_(newDS[0] != newDS[1])
# The name of the new Dataset has not changed when reading the state
assert_equal(newDS._dsName, 'test')
# The datastore contains all the ground truth datasets
for dataset in gt:
ok_(dataset in newDS)
# Clean-up the file and reset the registered types
config.__Registered_DatasetTypes__ = temp
if dsName.exists():
remove(str(dsName))