class GitHelper(object):
repo = 'local/'
def setUp(self):
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
def git(self, command, repo=None):
repo_path = self.dir.getpath(repo or self.repo)
try:
return check_output(['git'] + command.split(),
cwd=repo_path,
stderr=STDOUT)
except CalledProcessError as e:
self.fail(e.output)
def git_rev_parse(self, label, repo=None):
return self.git('rev-parse --verify -q --short ' + label, repo).strip()
def check_tags(self, expected, repo=None):
actual = {}
for tag in self.git('tag', repo).split():
actual[tag] = self.git_rev_parse(tag, repo)
compare(expected, actual=actual)
def make_repo_with_content(self, repo):
if not os.path.exists(self.dir.getpath(repo)):
self.dir.makedir(repo)
self.git('init', repo)
self.dir.write(repo + 'a', 'some content')
self.dir.write(repo + 'b', 'other content')
self.dir.write(repo + 'c', 'more content')
self.git('add .', repo)
self.git('commit -m initial', repo)
class GitHelper(object):
repo = 'local/'
def setUp(self):
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
def git(self, command, repo=None):
repo_path = self.dir.getpath(repo or self.repo)
try:
return check_output(['git'] + command.split(), cwd=repo_path, stderr=STDOUT)
except CalledProcessError as e:
self.fail(e.output)
def git_rev_parse(self, label, repo=None):
return self.git('rev-parse --verify -q --short '+label, repo).strip()
def check_tags(self, expected, repo=None):
actual = {}
for tag in self.git('tag', repo).split():
actual[tag] = self.git_rev_parse(tag, repo)
compare(expected, actual=actual)
def make_repo_with_content(self, repo):
if not os.path.exists(self.dir.getpath(repo)):
self.dir.makedir(repo)
self.git('init', repo)
self.dir.write(repo + 'a', 'some content')
self.dir.write(repo + 'b', 'other content')
self.dir.write(repo + 'c', 'more content')
self.git('add .', repo)
self.git('commit -m initial', repo)
class TestGetLanes(unittest.TestCase):
def setUp(self):
self.tempdir = TempDirectory()
self.tempdir.write('fake_file1.txt', b'some foo thing')
self.tempdir.write('fake_tmp_files/folder/afile.txt', b'the text')
self.tempdir.write('fake_directory/fake_file2.txt', b'the text')
self.tempdir.write('fake_directory/afile.bam', b'the text')
self.tempdir.write('fake_directory/afile.sam', b'the text')
self.tempdir.write('fake_directory/afile.fastq.gz', b'the text')
self.tempdir.makedir('empty_directory')
self.tempdir_path = self.tempdir.path
def tearDown(self):
self.tempdir.cleanup()
pass
def test_get_lanes_returns_data(self):
data_list = bytes(str(self.tempdir.path + '/fake_tmp_files/folder/afile.txt\n'+self.tempdir.path + '/fake_directory/fake_file2.txt\n' + 'fake_path\n'),'ascii')
with mock.patch('archive_project.GetLanes.check_output', return_value=data_list) as co:
actual, message = get_lanes("study")
co.assert_called_once_with(['pf', 'data', '-t', 'study', '-i', 'study'])
expected = [str(self.tempdir.path + '/fake_tmp_files/folder/afile.txt'), str(self.tempdir.path + '/fake_directory/fake_file2.txt')]
self.assertEqual(expected,actual)
self.assertEqual('This path was returned by pf, but does not actually exist: fake_path\n', message)
def test_get_lanes_returns_nodata(self):
with mock.patch('archive_project.GetLanes.check_output', return_value=b'') as co:
with mock.patch('os.path.exists', return_value=True) as pe:
actual, message = get_lanes("fake_study")
self.assertEqual([],actual)
pe.assert_not_called()
self.assertEqual('Unknown study or no data associated with study: fake_study\n',message)
class Test_archivepgsql_backup_invocation(TestCase):
ARCHIVEPGSQL_PATH = os.path.join('bbpgsql', 'cmdline_scripts')
CONFIG_FILE = 'config.ini'
exe_script = 'archivepgsql'
def setUp(self):
self.setup_environment()
self.setup_config()
self.execution_sequence = 0
def setup_environment(self):
self.env = deepcopy(os.environ)
self.env['PATH'] = ''.join([
self.env['PATH'],
':',
self.ARCHIVEPGSQL_PATH])
self.tempdir = TempDirectory()
self.data_dir = self.tempdir.makedir('pgsql_data')
self.archive_dir = self.tempdir.makedir('pgsql_archive')
def setup_config(self):
self.config_path = os.path.join(self.tempdir.path, self.CONFIG_FILE)
self.config_dict = {
'General': {
'pgsql_data_directory': self.data_dir,
},
'Snapshot': {
'driver': 'memory',
},
}
write_config_to_filename(self.config_dict, self.config_path)
self.config = get_config_from_filename_and_set_up_logging(
self.config_path
)
def tearDown(self):
self.tempdir.cleanup()
@patch('bbpgsql.archive_pgsql.commit_snapshot_to_repository')
@patch('bbpgsql.archive_pgsql.create_archive')
@patch('bbpgsql.archive_pgsql.pg_stop_backup')
@patch('bbpgsql.archive_pgsql.pg_start_backup')
def test_perform_backup(self, mock_pg_start_backup, mock_pg_stop_backup,
mock_create_archive, mock_commit_snapshot_to_repository):
first_WAL = '000000D0'
second_WAL = '000000D1'
mock_pg_start_backup.return_value = first_WAL
mock_pg_stop_backup.return_value = second_WAL
archiveFile = os.path.join(self.archive_dir, 'pgsql.snapshot.tar')
tag = bbpgsql.archive_pgsql.generate_tag()
repo = get_Snapshot_repository(self.config)
bbpgsql.archive_pgsql.perform_backup(self.data_dir,
archiveFile, tag, repo)
mock_pg_start_backup.assert_called_once_with(tag)
mock_create_archive.assert_called_once_with(self.data_dir, archiveFile)
self.assertEqual(mock_pg_stop_backup.called, True)
self.assertEqual(mock_pg_stop_backup.call_count, 1)
mock_commit_snapshot_to_repository.assert_called_once_with(
repo, archiveFile, tag, first_WAL, second_WAL)
def test_create_monitor_with_watch_path(self):
wd = TempDirectory()
source_path = wd.makedir('source')
wd.makedir('build')
with chdir(wd.path):
m = create_monitor(source_path)
assert len(m.reporters) == 1
reporter = m.reporters[0]
assert reporter.watch_path == source_path
assert reporter.build_path == '{}-build'.format(os.path.realpath(source_path)) # noqa
def test_create_monitor_with_watch_and_build_path(self):
wd = TempDirectory()
source_path = wd.makedir("source")
build_path = wd.makedir("build")
with chdir(wd.path):
m = create_monitor(source_path, build_path=os.path.basename(build_path))
assert len(m.reporters) == 1
reporter = m.reporters[0]
assert reporter.watch_path == source_path
assert reporter.build_path == "{}".format(os.path.realpath(build_path))
class TestImageValidator(unittest.TestCase):
def setUp(self):
"""
Set up a folder structure containing one timepoint (0)
one channel (1) and two images in channel subfolder
"""
self.tempdir = TempDirectory()
self.temp_path = self.tempdir.path
self.tempdir.makedir('timepoint_0')
self.tempdir.makedir('timepoint_0/channel_1')
# Write images as bytes
im = np.zeros((15, 12), dtype=np.uint16)
res, im_encoded = cv2.imencode('.png', im)
im_encoded = im_encoded.tostring()
self.tempdir.write('timepoint_0/channel_1/im_0.png', im_encoded)
self.tempdir.write('timepoint_0/channel_1/im_1.png', im_encoded)
self.tempdir.listdir(recursive=True)
def tearDown(self):
"""
Tear down temporary folder and file structure
"""
TempDirectory.cleanup_all()
nose.tools.assert_equal(os.path.isdir(self.temp_path), False)
def test_image_validator(self):
"""
Test image validator on temporary folder structure
"""
meta_name = 'image_volumes_info.csv'
validator = image_validator.ImageValidator(
input_dir=self.tempdir.path,
meta_name=meta_name,
verbose=10,
)
validator.folder_validator()
# Check written metadata
metadata = pd.read_csv(os.path.join(self.tempdir.path, meta_name))
# Metadata should have 8 fields + 1 index and we have two files
nose.tools.assert_equal(metadata.shape, (2, 9))
# Metadata should contain the following column names
expected_names = [
'Unnamed: 0', 'timepoint', 'channel_num', 'sample_num',
'slice_num', 'fname', 'size_x_microns', 'size_y_microns',
'size_z_microns'
]
nose.tools.assert_equal(list(metadata), expected_names)
# Both files should have timepoint 0
nose.tools.assert_equal(np.all(metadata['timepoint'] == 0), True)
# Both file should have channel 1
nose.tools.assert_equal(np.all(metadata['channel_num'] == 1), True)
# The two image numbers should be sorted by index
nose.tools.assert_equal(metadata['sample_num'][0], 0)
nose.tools.assert_equal(metadata['sample_num'][1], 1)
def test_create_monitor_accepts_clean_kwarg(self):
wd = TempDirectory()
source_path = wd.makedir("source")
build_path = wd.makedir("build")
sub_path = os.path.join(build_path, "sub")
os.mkdir(sub_path)
with chdir(wd.path):
assert os.path.exists(build_path)
assert os.path.exists(sub_path)
create_monitor(
source_path, build_path=os.path.basename(build_path), clean=True
)
assert os.path.exists(sub_path)
assert os.path.exists(build_path)
def test_default_watcher_gets_all_files(self):
work_directory = TempDirectory()
work_directory.write("a.h", b"")
work_directory.write("a.c", b"")
work_directory.write("a.cc", b"")
work_directory.write("CMakeLists.txt", b"")
work_directory.write("blah.txt", b"")
work_directory.makedir(".git")
work_directory.makedir(".hg")
wd_len = len(work_directory.path) + 1
w = Watcher(work_directory.path, None)
watchstate = w.poll()
filelist = [f[wd_len:] for f in watchstate.inserts]
filelist.sort()
assert filelist == ["CMakeLists.txt", "a.c", "a.cc", "a.h", "blah.txt"]
def test_testlist(self):
import stat
work_directory = TempDirectory()
work_directory.makedir('test')
work_directory.write(['test', 'test_dummy.c'], b'')
build_directory = TempDirectory()
testbin_path = build_directory.write(['test_dummy'], b'')
st = os.stat(testbin_path)
os.chmod(testbin_path, st.st_mode | stat.S_IEXEC)
w = Watcher(work_directory.path, build_directory.path)
w.poll()
exefile = testbin_path + watcher.EXE_SUFFIX
testlist = [(g.source(), g.executable()) for g in w.testlist()]
assert testlist == [(os.path.join('test', 'test_dummy.c'), exefile)]
def test_testlist(self):
import stat
work_directory = TempDirectory()
work_directory.makedir("test")
work_directory.write(["test", "test_dummy.c"], b"")
build_directory = TempDirectory()
testbin_path = build_directory.write(["test_dummy"], b"")
st = os.stat(testbin_path)
os.chmod(testbin_path, st.st_mode | stat.S_IEXEC)
w = Watcher(work_directory.path, build_directory.path)
w.poll()
exefile = testbin_path + watcher.EXE_SUFFIX
testlist = [(g.source(), g.executable()) for g in w.testlist()]
assert testlist == [(os.path.join("test", "test_dummy.c"), exefile)]
def setUp(self):
self.app = Flask(__name__)
self.app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
self.app.config['USER_LIST'] = ['Test Admin']
db.init_app(self.app)
d = TempDirectory()
d.makedir('dataset')
d.write('dataset/files/testfile.jpg', 'abc')
d.write('dataset/testfile2.jpg', 'abc')
self.directory = d
self.app.config['DATASET_ROOT'] = d.path
with self.app.app_context(), TempDirectory() as d:
db.create_all()
lib.models.buildDB()
dataset = lib.models.Datasets()
dataset.name = 'Test'
dataset.path = "dataset"
db.session.add(dataset)
db.session.commit()
def setUp(self):
self.app = Flask(__name__)
self.app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
self.app.config['USER_LIST'] = ['Test Admin']
db.init_app(self.app)
d = TempDirectory()
d.makedir('dataset')
d.write('dataset/files/testfile.jpg', 'abc')
d.write('dataset/testfile2.jpg', 'abc')
self.directory = d
self.app.config['DATASET_ROOT'] = d.path
with self.app.app_context(), TempDirectory() as d:
db.create_all()
lib.models.buildDB()
dataset = lib.models.Datasets()
dataset.name = 'Test'
dataset.path = "dataset"
db.session.add(dataset)
db.session.commit()
class TestDirectoryModel(TestCase):
def setUp(self):
self.temp_dir = TempDirectory()
self.base_path = self.temp_dir.makedir(('base'))
self.base_dir = Directory(path=self.base_path)
self.base_dir.save(is_basedir=True)
self.test_dir = Directory(parent=self.base_dir, path='test')
self.test_dir.save()
def test_base_dir_absolute_path(self):
self.assertEqual(Path(self.base_path), self.base_dir.absolute_path())
def test_base_dir_relative_path(self):
self.assertEqual(Path('.'), self.base_dir.relative_path())
def test_absolute_path(self):
test_path = self.temp_dir.makedir(os.path.join(self.base_path, 'test'))
self.assertEqual(Path(test_path), self.test_dir.absolute_path())
def test_relative_path(self):
self.assertEqual(Path('test'), self.test_dir.relative_path())
def test_get_sub_path_directories(self):
paths_elements = ['test']
self.assertEqual([self.test_dir], self.base_dir.get_sub_path_directories(paths_elements))
def listdir(self):
self.assertEqual(([], [self.test_dir]), self.base_dir.listdir())
def test_exists(self):
self.assertTrue(self.base_dir.exists())
def test_get_basedir(self):
delattr(Directory, '_cached_basedir')
self.assertEqual(self.base_dir, Directory.get_basedir())
def test_use_configuration_from_root_path_when_no_other_was_found(self):
root_dir = TempDirectory()
self.tmpdirs.append(root_dir)
start_path = root_dir.makedir('some/directories/to/start/looking/for/settings')
test_file = os.path.realpath(os.path.join(root_dir.path, 'settings.ini'))
with open(test_file, 'a') as file_:
file_.write('[settings]')
self.files.append(test_file) # Required to removed it at tearDown
discovery = ConfigurationDiscovery(start_path, root_path=root_dir.path)
filenames = [cfg.filename for cfg in discovery.config_files]
self.assertEqual([test_file], filenames)
def test_default_watcher_gets_all_files(self):
work_directory = TempDirectory()
work_directory.write('a.h', b'')
work_directory.write('a.c', b'')
work_directory.write('a.cc', b'')
work_directory.write('CMakeLists.txt', b'')
work_directory.write('blah.txt', b'')
work_directory.makedir('.git')
work_directory.makedir('.hg')
wd_len = len(work_directory.path) + 1
w = Watcher(work_directory.path, None)
watchstate = w.poll()
filelist = [f[wd_len:] for f in watchstate.inserts]
filelist.sort()
assert filelist == [
'CMakeLists.txt',
'a.c',
'a.cc',
'a.h',
'blah.txt'
]
class TestXMLExporter(unittest.TestCase):
def setUp(self):
self.tmp = TempDirectory()
self.dir = self.tmp.makedir('foobar/')
self.exp = XMLExporter(self.dir)
def tearDown(self):
TempDirectory.cleanup_all()
#not responsible for creation
def DISABLEDtest_creation_of_meta_info_file(self):
filename = 'barfoo.xml'
status = self.exp._exportMetaInformationToXML('', '', filename)
self.assertTrue(os.path.isfile(self.dir+'metainformation.xml'))
self.assertTrue(status)
#not responsible for creation;
# TODO: fix
def DISABLEDtest_content_of_meta_info_file(self):
filename = 'barfoo.xml'
status = self.exp._exportMetaInformationToXML('01', '02', filename)
tree = et.parse(self.dir + 'metainformation.xml')
xml = tree.getroot()
session = xml.find('Session')
exercise = xml.find('Exercise')
annotation = xml.find('Annotationfile')
self.assertEquals('01', session.text)
self.assertEquals('02', exercise.text)
self.assertEquals(self.dir + filename, annotation.text)
def DISABLEDtest_content_of_xml_export_file(self):
src = lib.Sourcefile._make(['', '', '', '', lib.Orientation.left])
input = lib.Validationfile._make([src, 'Gold', [['Foo', 'Bar']],
[{ 'Fu': '1', 'Baz': '2'}, {'Fu': '3', 'Baz': '4'}]])
self.assertTrue(self.exp.export(input))
tree = et.parse(self.dir + 'annotation.xml')
xml = tree.getroot()
validation = xml.find('Validation')
self.assertEquals(validation.attrib, {'Type': 'Gold'})
sensor = validation.find('Sensor')
self.assertEquals(sensor.attrib, {'Orientation': 'left'})
meta = sensor.find('Meta')
self.assertEquals(meta[0].tag, 'Foo')
self.assertEquals(meta[0].text, 'Bar')
content = sensor.find('Content')
self.assertEquals(content[0].tag, 'No1')
self.assertEquals(content[0][0].tag, 'Fu')
self.assertEquals(content[0][0].text, '1')
self.assertEquals(content[0][1].tag, 'Baz')
self.assertEquals(content[0][1].text, '2')
self.assertEquals(content[1][0].tag, 'Fu')
self.assertEquals(content[1][0].text, '3')
self.assertEquals(content[1][1].tag, 'Baz')
self.assertEquals(content[1][1].text, '4')
class Test_FilesystemCommitStorage(TestCase):
def setUp(self):
self.tempdir = TempDirectory()
self.commit_storage_path = self.tempdir.makedir('commit_storage')
self.config = config()
self.config.set('WAL', 'driver', 'filesystem')
self.config.set('WAL', 'path', self.commit_storage_path)
def tearDown(self):
self.tempdir.cleanup()
def test_will_build_storage_from_config(self):
self.assertEqual(FilesystemCommitStorage,
type(get_repository_storage_from_config(self.config, 'WAL')))
def setUp(self):
data.app.config['TESTING'] = True
data.app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
data.app.config['USER_LIST'] = ['Test Admin']
db.init_app(data.app)
d = TempDirectory()
d.makedir('dataset')
d.write('dataset/files/testfile.jpg', 'abc')
d.write('dataset/testfile2.jpg', 'abc')
self.directory = d
data.app.config['DATASET_ROOT'] = d.path
with data.app.app_context(), TempDirectory() as d:
db.create_all()
lib.models.buildDB()
license = lib.models.License.query.first()
provider = lib.models.Provider.query.first()
dataset = lib.models.Datasets()
dataset.name = 'Test'
dataset.license = [license]
dataset.provider = [provider]
dataset.path = "dataset"
db.session.add(dataset)
db.session.commit()
self.app = data.app.test_client()
def test_use_configuration_from_root_path_when_no_other_was_found(self):
root_dir = TempDirectory()
self.tmpdirs.append(root_dir)
start_path = root_dir.makedir(
'some/directories/to/start/looking/for/settings')
test_file = os.path.realpath(
os.path.join(root_dir.path, 'settings.ini'))
with open(test_file, 'a') as file_:
file_.write('[settings]')
self.files.append(test_file) # Required to removed it at tearDown
discovery = ConfigurationDiscovery(start_path, root_path=root_dir.path)
filenames = [cfg.filename for cfg in discovery.config_files]
self.assertEqual([test_file], filenames)
def test_lookup_should_stop_at_root_path(self):
test_dir = TempDirectory()
self.tmpdirs.append(test_dir) # Cleanup dir at tearDown
start_path = test_dir.makedir('some/dirs/without/config')
# create a file in the test_dir
test_file = os.path.realpath(os.path.join(test_dir.path, 'settings.ini'))
with open(test_file, 'a') as file_:
file_.write('[settings]')
self.files.append(test_file) # Required to removed it at tearDown
root_dir = os.path.join(test_dir.path, 'some', 'dirs') # No settings here
discovery = ConfigurationDiscovery(start_path, root_path=root_dir)
self.assertEqual(discovery.config_files, [])
def test_lookup_should_stop_at_root_path(self):
test_dir = TempDirectory()
self.tmpdirs.append(test_dir) # Cleanup dir at tearDown
start_path = test_dir.makedir('some/dirs/without/config')
# create a file in the test_dir
test_file = os.path.realpath(
os.path.join(test_dir.path, 'settings.ini'))
with open(test_file, 'a') as file_:
file_.write('[settings]')
self.files.append(test_file) # Required to removed it at tearDown
root_dir = os.path.join(test_dir.path, 'some',
'dirs') # No settings here
discovery = ConfigurationDiscovery(start_path, root_path=root_dir)
self.assertEqual(discovery.config_files, [])
class Test_archivepgsql_BasicCommandLineOperation(TestCase):
ARCHIVEPGSQL_PATH = os.path.join('bbpgsql', 'cmdline_scripts')
CONFIG_FILE = 'config.ini'
exe_script = 'archivepgsql'
def setUp(self):
self.setup_environment()
self.setup_config()
self.cmd = [self.exe_script, '--dry-run', '--config', self.config_path]
def setup_environment(self):
self.env = deepcopy(os.environ)
self.env['PATH'] = ''.join([
self.env['PATH'],
':',
self.ARCHIVEPGSQL_PATH])
self.tempdir = TempDirectory()
self.data_dir = self.tempdir.makedir('pgsql_data')
def setup_config(self):
self.config_path = os.path.join(self.tempdir.path, self.CONFIG_FILE)
self.config_dict = {
'General': {
'pgsql_data_directory': self.data_dir,
},
'Snapshot': {
'driver': 'memory',
},
}
write_config_to_filename(self.config_dict, self.config_path)
self.config = get_config_from_filename_and_set_up_logging(
self.config_path
)
def tearDown(self):
self.tempdir.cleanup()
def test_can_execute_archivepgsql(self):
check_call(self.cmd, env=self.env, stdout=PIPE)
def test_obeys_dry_run_option(self):
proc = Popen(self.cmd, env=self.env, stdout=PIPE)
stdoutdata, stderrdata = proc.communicate()
self.assertEqual("Dry Run\n", stdoutdata)
class TestCSVExporter(unittest.TestCase):
def setUp(self):
self.tmp = TempDirectory()
self.dir = self.tmp.makedir('foobar')
self.exp = CSVExporter()
def tearDown(self):
TempDirectory.cleanup_all()
def test_creation_of_export_file(self):
filename = 'foobar.csv'
values = [lib.Sensorsegment._make([0] * 6)]
status = self.exp.export(values, self.dir+filename)
self.assertTrue(os.path.isfile(self.dir+filename))
self.assertTrue(status)
def test_content_of_export_file(self):
filename = 'foobar.csv'
values = [lib.Sensorsegment._make([0] * 6),
lib.Sensorsegment._make([1] * 6)
]
self.exp.export(values, self.dir+filename, False, False)
res = self.tmp.read(self.dir+filename)
ref = b'0,0,0,0,0,0\r\n1,1,1,1,1,1\r\n'
self.assertEqual(ref, res)
def test_content_with_header_with_indices(self):
filename = 'foobar.csv'
values = [lib.Sensorsegment._make([0] * 6)]
self.exp.export(values, self.dir+filename, True, True)
res = self.tmp.read(self.dir+filename)
ref = b'Index,accelX,accelY,accelZ,gyroX,gyroY,gyroZ\r\n1,0,0,0,0,0,0\r\n'
self.assertEqual(ref, res)
def test_content_with_header_without_indices(self):
filename = 'foobar.csv'
values = [lib.Sensorsegment._make([0] * 6)]
self.exp.export(values, self.dir+filename, True, False)
res = self.tmp.read(self.dir+filename)
ref = b'accelX,accelY,accelZ,gyroX,gyroY,gyroZ\r\n0,0,0,0,0,0\r\n'
self.assertEqual(ref, res)
def test_content_without_header_with_indices(self):
filename = 'foobar.csv'
values = [lib.Sensorsegment._make([0] * 6)]
self.exp.export(values, self.dir+filename, False, True)
res = self.tmp.read(self.dir+filename)
ref = b'1,0,0,0,0,0,0\r\n'
self.assertEqual(ref, res)
class TestFileStorage(unittest.TestCase):
"""FileStorage.
Testing storage on disk.
"""
def setUp(self):
"""Initialise fixtures."""
self._dir = TempDirectory()
self.outputdir = self._dir.makedir('output')
self.env = EnvironmentVarGuard()
self.env.set('ARCHIVE_LOCATION', self.outputdir + '/%s/')
self._store = FileStorage('archive', 'lega')
def tearDown(self):
"""Remove setup variables."""
self.env.unset('ARCHIVE_LOCATION')
self._dir.cleanup_all()
def test_location(self):
"""Test file location."""
result = self._store.location('12')
self.assertEqual(
os.path.join('/', '000', '000', '000', '000', '000', '000', '12'),
result)
def test_copy(self):
"""Test copy file."""
path = self._dir.write('output/lega/test.file',
'data1'.encode('utf-8'))
path1 = self._dir.write('output/lega/test1.file', ''.encode('utf-8'))
result = self._store.copy(open(path, 'rb'), path1)
self.assertEqual(os.stat(path).st_size, result)
def test_open(self):
"""Test open file."""
path = self._dir.write('output/lega/test.file',
'data1'.encode('utf-8'))
print(path)
with self._store.open('test.file') as resource:
self.assertEqual(BufferedReader, type(resource))
class DocumentServiceWriterTestCases(SegueApiTestCase):
def setUp(self):
super(DocumentServiceWriterTestCases, self).setUp()
self.tmp_dir = TempDirectory()
self.out_dir = self.tmp_dir.makedir("output")
self.templates = os.path.join(os.path.dirname(__file__), 'fixtures')
self.service = DocumentService(override_root=self.out_dir, template_root=self.templates, tmp_dir=self.tmp_dir.path)
def tearDown(self):
self.tmp_dir.cleanup()
@skipIf("SNAP_CI" in os.environ, "inkscape is not available in CI")
def test_svg_to_pdf(self):
result = self.service.svg_to_pdf('templates/dummy.svg', 'certificate', "ABCD", { 'XONGA': 'bir<osca' })
self.tmp_dir.check_dir('','certificate-ABCD.svg', 'output')
self.tmp_dir.check_dir('output/certificate/AB', 'certificate-ABCD.pdf')
contents_of_temp = self.tmp_dir.read("certificate-ABCD.svg")
self.assertIn("bir<osca da silva", contents_of_temp)
self.assertEquals(result, 'certificate-ABCD.pdf')
class TestTifFolderSplitter(unittest.TestCase):
@patch('concurrent.futures.ProcessPoolExecutor')
def setUp(self, MockPoolExecutor):
"""
Set up temporary test directory and mock S3 bucket connection
"""
# Magic mocking of multiprocessing
MockPoolExecutor().__enter__().map = map_mock
# Mock S3 directory for upload
self.storage_dir = "raw_frames/SMS-2010-01-01-00-00-00-0001"
# Create temporary directory and write temp image
self.tempdir = TempDirectory()
self.temp_path = self.tempdir.path
# Temporary frame
self.im = np.ones((10, 15), dtype=np.uint16)
self.im[2:5, 3:12] = 10000
# Save test tif files
self.channel_names = ['phase', 'brightfield', '666']
# Write files in dir
for c in self.channel_names:
for z in range(2):
file_name = 'img_{}_t000_p050_z00{}.tif'.format(c, z)
file_path = os.path.join(self.temp_path, file_name)
ijmeta = {"Info": json.dumps({"c": c, "z": z})}
tifffile.imsave(
file_path,
self.im + 5000 * z,
ijmetadata=ijmeta,
)
# Write external metadata in dir
self.meta_dict = {
'Summary': {
'Slices': 26,
'PixelType': 'GRAY16',
'Time': '2018-11-01 19:20:34 -0700',
'z-step_um': 0.5,
'PixelSize_um': 0,
'BitDepth': 16,
'Width': 15,
'Height': 10
},
}
self.json_filename = os.path.join(self.temp_path, 'metadata.txt')
json_ops.write_json_file(self.meta_dict, self.json_filename)
# Setup mock S3 bucket
self.mock = mock_s3()
self.mock.start()
self.conn = boto3.resource('s3', region_name='us-east-1')
self.bucket_name = 'czbiohub-imaging'
self.conn.create_bucket(Bucket=self.bucket_name)
# Instantiate file parser class
storage_class = aux_utils.get_storage_class('s3')
self.frames_inst = tif_splitter.TifFolderSplitter(
data_path=self.temp_path,
storage_dir=self.storage_dir,
storage_class=storage_class,
)
# Upload data
self.frames_inst.get_frames_and_metadata(
filename_parser='parse_sms_name', )
def tearDown(self):
"""
Tear down temporary folder and files and stop S3 mock
"""
TempDirectory.cleanup_all()
nose.tools.assert_equal(os.path.isdir(self.temp_path), False)
self.mock.stop()
def test_get_global_meta(self):
global_meta = self.frames_inst.get_global_meta()
self.assertEqual(global_meta['nbr_frames'], 6)
self.assertEqual(global_meta['bit_depth'], 'uint16')
self.assertEqual(global_meta['nbr_channels'], 3)
self.assertEqual(global_meta['nbr_slices'], 2)
self.assertEqual(global_meta['nbr_timepoints'], 1)
self.assertEqual(global_meta['nbr_positions'], 1)
self.assertEqual(global_meta['im_colors'], 1)
self.assertEqual(global_meta['im_height'], self.im.shape[0])
self.assertEqual(global_meta['im_width'], self.im.shape[1])
def test_get_global_json(self):
global_json = self.frames_inst.get_global_json()
self.assertDictEqual(global_json, self.meta_dict)
def test_get_frames_meta(self):
frames_meta = self.frames_inst.get_frames_meta()
for i, (c, z) in enumerate(itertools.product(range(3), range(2))):
# Validate file name
expected_name = 'im_c00{}_z00{}_t000_p050.png'.format(c, z)
self.assertEqual(frames_meta.loc[i, 'file_name'], expected_name)
# Validate checksum
expected_sha = meta_utils.gen_sha256(self.im + 5000 * z)
self.assertEqual(frames_meta.loc[i, 'sha256'], expected_sha)
# Validate indices
self.assertEqual(frames_meta.loc[i, 'channel_idx'], c)
self.assertEqual(frames_meta.loc[i, 'slice_idx'], z)
self.assertEqual(frames_meta.loc[i, 'time_idx'], 0)
self.assertEqual(frames_meta.loc[i, 'pos_idx'], 50)
def test_get_frames_json(self):
frames_json = self.frames_inst.get_frames_json()
# Channels will be sorted
sorted_channels = natsort.natsorted(self.channel_names)
self.assertEqual(len(frames_json), 6)
for i, (c, z) in enumerate(itertools.product(range(3), range(2))):
frame_i = frames_json[i]
meta_info = json.loads(frame_i['IJMetadata']['Info'])
self.assertDictEqual(
meta_info,
{
"c": sorted_channels[c],
"z": z
},
)
frame_i['BitsPerSample'] = 16
frame_i['ImageWidth'] = 15
frame_i['ImageLength'] = 10
def test_set_frame_info(self):
meta_dict = {
'PixelType': 'RGB',
'BitDepth': 8,
'Width': 250,
'Height': 150,
}
self.frames_inst.set_frame_info(meta_dict)
self.assertEqual(self.frames_inst.bit_depth, 'uint8')
self.assertEqual(self.frames_inst.im_colors, 3)
self.assertListEqual(self.frames_inst.frame_shape, [150, 250])
def test_set_frame_info_uint16(self):
meta_dict = {
'PixelType': 'GRAY',
'BitDepth': 16,
'Width': 250,
'Height': 150,
}
self.frames_inst.set_frame_info(meta_dict)
self.assertEqual(self.frames_inst.bit_depth, 'uint16')
self.assertEqual(self.frames_inst.im_colors, 1)
@nose.tools.raises(ValueError)
def test_set_frame_info(self):
meta_dict = {
'PixelType': 'RGB',
'BitDepth': 'float32',
'Width': 250,
'Height': 150,
}
self.frames_inst.set_frame_info(meta_dict)
def test_set_frame_info_from_file(self):
file_name = 'img_phase_t000_p050_z001.tif'
file_path = os.path.join(self.temp_path, file_name)
self.frames_inst.set_frame_info_from_file(file_path)
self.assertEqual(self.frames_inst.bit_depth, 'uint16')
self.assertEqual(self.frames_inst.im_colors, 1)
self.assertListEqual(self.frames_inst.frame_shape, [10, 15])
def test_set_frame_info_from_file_color(self):
self.tempdir.makedir('test_dir')
file_path = os.path.join(self.temp_path, 'test_dir/im_temp.tif')
im = np.zeros((10, 20, 3), dtype=np.uint8)
tifffile.imsave(file_path, im)
self.frames_inst.set_frame_info_from_file(file_path)
self.assertEqual(self.frames_inst.bit_depth, 'uint8')
self.assertEqual(self.frames_inst.im_colors, 3)
self.assertListEqual(self.frames_inst.frame_shape, [10, 20])
@nose.tools.raises(ValueError)
def test_set_frame_info_from_file_float(self):
self.tempdir.makedir('test_dir')
file_path = os.path.join(self.temp_path, 'test_dir/im_temp.tif')
im = np.zeros((10, 20, 3), dtype=np.float32)
tifffile.imsave(file_path, im)
self.frames_inst.set_frame_info_from_file(file_path)
def test_set_frame_meta(self):
parse_func = getattr(file_parsers, 'parse_sms_name')
file_name = 'im_weird_channel_with_underscores_t020_z030_p040.tif'
meta_row = self.frames_inst._set_frame_meta(parse_func, file_name)
self.assertEqual(
meta_row['channel_name'],
'weird_channel_with_underscores',
)
self.assertEqual(meta_row['channel_idx'], 3)
self.assertEqual(meta_row['time_idx'], 20)
self.assertEqual(meta_row['slice_idx'], 30)
self.assertEqual(meta_row['pos_idx'], 40)
def test_get_frames_and_metadata(self):
# Download uploaded data and compare to self.im
for i, (c, z) in enumerate(itertools.product(range(3), range(2))):
im_name = 'im_c00{}_z00{}_t000_p050.png'.format(c, z)
key = "/".join([self.storage_dir, im_name])
byte_string = self.conn.Object(self.bucket_name,
key).get()['Body'].read()
# Construct an array from the bytes and decode image
im = im_utils.deserialize_im(byte_string)
# Assert that contents are the same
self.assertEqual(im.dtype, np.uint16)
numpy.testing.assert_array_equal(im, self.im + 5000 * z)
@nose.tools.raises(AttributeError)
def test_get_frames_and_metadata_no_parser(self):
self.frames_inst.get_frames_and_metadata(
filename_parser='nonexisting_function', )
@patch('concurrent.futures.ProcessPoolExecutor')
def test_get_frames_no_metadata(self, MockPoolExecutor):
# Magic mocking of multiprocessing
MockPoolExecutor().__enter__().map = map_mock
os.remove(self.json_filename)
self.frames_inst.get_frames_and_metadata(
filename_parser='parse_sms_name', )
frames_meta = self.frames_inst.get_frames_meta()
for i, (c, z) in enumerate(itertools.product(range(3), range(2))):
# Validate file name
expected_name = 'im_c00{}_z00{}_t000_p050.png'.format(c, z)
self.assertEqual(frames_meta.loc[i, 'file_name'], expected_name)
# Validate checksum
expected_sha = meta_utils.gen_sha256(self.im + 5000 * z)
self.assertEqual(frames_meta.loc[i, 'sha256'], expected_sha)
# Validate indices
self.assertEqual(frames_meta.loc[i, 'channel_idx'], c)
self.assertEqual(frames_meta.loc[i, 'slice_idx'], z)
self.assertEqual(frames_meta.loc[i, 'time_idx'], 0)
self.assertEqual(frames_meta.loc[i, 'pos_idx'], 50)
class test_FileOs(unittest.TestCase):
"""
Ensure file object can get set and output paths to files
"""
def setUp(self):
self.d = TempDirectory()
self.base_path = self.d.path + "/test"
self.d.makedir(self.base_path)
self.file_obj = FileOs(self.base_path)
cats = [{
"name": "Zophie",
"desc": "chubby"
}, {
"name": "Pooka",
"desc": "fluffy"
}]
file_path = os.path.join(self.base_path, 'read_cats.json')
with open(file_path, mode='w', encoding='utf-8') as feedsjson:
json.dump(cats, feedsjson)
def tearDown(self):
self.d.cleanup()
def test_creates_new_file_object(self):
self.assertIsNotNone(self.file_obj)
def test_init_sets_base_path(self):
self.assertEqual(self.file_obj._base_path, self.base_path)
def test_can_get_base_path(self):
self.assertEqual(self.file_obj.get_base_path(), self.base_path)
def test_can_set_base_path(self):
new_base_path = self.d.path + "/new_path"
self.file_obj.set_base_path(new_base_path)
self.assertEqual(self.file_obj._base_path, new_base_path)
def test_can_create_file_from_absolute_path(self):
file_path = "/new_file.txt"
full_path = self.base_path + file_path
self.file_obj.set_file(file_path)
self.assertTrue(os.path.isfile(full_path))
def test_can_create_file_from_filename(self):
file_path = "new_file.txt"
full_path = os.path.join(self.base_path, file_path)
self.file_obj.set_file(file_path)
self.assertTrue(os.path.isfile(full_path))
def test_will_create_file_in_folder_not_make_file_name_plus_folder(self):
file_path = "new_file.txt"
false_full_path = self.base_path + file_path
full_path = os.path.join(self.base_path, file_path)
self.file_obj.set_file(file_path)
self.assertFalse(os.path.isfile(false_full_path))
self.assertTrue(os.path.isfile(full_path))
def test_will_set_content_in_file(self):
cats = [{
"name": "Zophie",
"desc": "chubby"
}, {
"name": "Pooka",
"desc": "fluffy"
}]
file_path = "new_file.txt"
full_path = os.path.join(self.base_path, file_path)
self.file_obj.set_json_file(file_path, cats)
with open(full_path, mode="r", encoding="utf-8") as feedsjson:
test_contents = json.load(feedsjson)
self.assertEqual(test_contents, cats)
def test_can_read_contents_of_file(self):
cats = [{
"name": "Zophie",
"desc": "chubby"
}, {
"name": "Pooka",
"desc": "fluffy"
}]
file_path = os.path.join(self.base_path, 'read_cats.json')
test_cats = self.file_obj.read_file(file_path)
self.assertEqual(test_cats, cats)
def test_can_copy_file_from_one_place_to_another(self):
source_path = 'read_cats.json'
dest_path = 'pasted_cats.json'
full_dest_path = os.path.join(self.base_path, dest_path)
self.file_obj.copy_file(source_path, dest_path)
self.assertTrue(os.path.isfile(full_dest_path))
def test_can_return_parent_folder_name(self):
full_path = 'archive/2019-01-18_13-53-00/Dvndh6DUYAAF1rM.jpg'
parent_name = self.file_obj.get_parent_folder_name(full_path)
self.assertEqual(parent_name, '2019-01-18_13-53-00')
class TestPathSource(TestCase):
def setUp(self):
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
def test_abc(self):
self.assertTrue(issubclass(Plugin, Source))
def test_schema_ok(self):
p1 = self.dir.write('foo', b'f')
p2 = self.dir.write('bar', b'b')
compare(
dict(type='paths', values=[p1, p2], repo='config'),
Plugin.schema(
dict(type='paths', values=[p1, p2], repo='config')
))
def test_schema_wrong_type(self):
text = "not a valid value for dictionary value @ data['type']"
with ShouldFailSchemaWith(text):
Plugin.schema(dict(type='bar', values=['/']))
def test_schema_extra_keys(self):
with ShouldFailSchemaWith("extra keys not allowed @ data['foo']"):
Plugin.schema(dict(type='paths', foo='bar'))
def test_name_supplied(self):
text = "not a valid value for dictionary value @ data['name']"
with ShouldFailSchemaWith(text):
Plugin.schema(dict(type='paths', name='foo'))
def test_no_paths(self):
text = "length of value must be at least 1 for dictionary value " \
"@ data['values']"
with ShouldFailSchemaWith(text):
Plugin.schema(dict(type='paths', values=[]))
def test_path_not_string(self):
text = "expected str @ data['values'][0]"
with ShouldFailSchemaWith(text):
Plugin.schema(dict(type='paths', values=[1]))
def test_path_not_starting_with_slash(self):
text = "'foo' is not an absolute path @ data['values'][0]"
try:
cwd = os.getcwd()
os.chdir(self.dir.path)
self.dir.makedir('foo')
with ShouldFailSchemaWith(text):
Plugin.schema(dict(type='paths', values=['foo']))
finally:
os.chdir(cwd)
def test_path_not_there(self):
bad_path = self.dir.getpath('bad')
text = "'%s' does not exist @ data['values'][0]" % bad_path
with ShouldFailSchemaWith(text):
Plugin.schema(dict(type='paths', values=[bad_path]))
def test_interface(self):
plugin = Plugin('source', name=None, repo='config',
values=['/foo/bar'])
compare(plugin.type, 'source')
compare(plugin.name, None)
compare(plugin.repo, 'config')
compare(plugin.source_paths, ['/foo/bar'])
class TestDataUploaderLocalStorage(db_basetest.DBBaseTest):
"""
Test the data uploader using local storage
"""
def setUp(self):
super().setUp()
# Create temporary directory and write temp image
self.tempdir = TempDirectory()
self.temp_path = self.tempdir.path
# Mock file storage
self.tempdir.makedir('storage_mount_point')
self.mount_point = os.path.join(self.temp_path, 'storage_mount_point')
self.tempdir.makedir('storage_mount_point/raw_files')
self.tempdir.makedir('storage_mount_point/raw_frames')
# Test metadata parameters
self.nbr_channels = 2
self.nbr_slices = 3
# Mock S3 dir
self.storage_dir = "raw_frames/TEST-2005-06-09-20-00-00-1000"
# Temporary file with 6 frames, tifffile stores channels first
self.im = 50 * np.ones((6, 10, 15), dtype=np.uint16)
self.im[0, :5, 3:12] = 50000
self.im[2, :5, 3:12] = 40000
self.im[4, :5, 3:12] = 30000
# Metadata
self.description = 'ImageJ=1.52e\nimages=6\nchannels=2\nslices=3\nmax=10411.0'
# Save test tif file
self.file_path = os.path.join(self.temp_path, "A1_2_PROTEIN_test.tif")
tifffile.imsave(
self.file_path,
self.im,
description=self.description,
)
self.dataset_serial = 'TEST-2005-06-09-20-00-00-1000'
# Create csv file for upload
upload_dict = {
'dataset_id': [self.dataset_serial],
'file_name': [self.file_path],
'description': ['Testing'],
'parent_dataset_id': [None],
}
upload_csv = pd.DataFrame.from_dict(upload_dict)
self.csv_path = os.path.join(self.temp_path, "test_upload.csv")
upload_csv.to_csv(self.csv_path)
self.credentials_path = os.path.join(
self.main_dir,
'db_credentials.json',
)
self.config_path = os.path.join(
self.temp_path,
'config_tif_id.json',
)
config = {
"upload_type": "frames",
"frames_format": "tif_id",
"microscope": "Leica microscope CAN bus adapter",
"filename_parser": "parse_ml_name",
"storage": "local",
"storage_access": self.mount_point
}
json_ops.write_json_file(config, self.config_path)
def tearDown(self):
"""
Rollback database session.
Tear down temporary folder and file structure, stop moto mock
"""
super().tearDown()
TempDirectory.cleanup_all()
self.assertFalse(os.path.isdir(self.temp_path))
@patch('imaging_db.database.db_operations.session_scope')
def test_upload_frames(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
data_uploader.upload_data_and_update_db(
csv=self.csv_path,
login=self.credentials_path,
config=self.config_path,
)
# Query database to find data_set and frames
datasets = self.session.query(db_ops.DataSet) \
.filter(db_ops.DataSet.dataset_serial == self.dataset_serial)
self.assertEqual(datasets.count(), 1)
dataset = datasets[0]
self.assertEqual(dataset.id, 1)
self.assertTrue(dataset.frames)
self.assertEqual(dataset.dataset_serial, self.dataset_serial)
date_time = dataset.date_time
self.assertEqual(date_time.year, 2005)
self.assertEqual(date_time.month, 6)
self.assertEqual(date_time.day, 9)
self.assertEqual(dataset.microscope,
"Leica microscope CAN bus adapter")
self.assertEqual(dataset.description, 'Testing')
# query frames_global
global_query = self.session.query(db_ops.FramesGlobal) \
.join(db_ops.DataSet) \
.filter(db_ops.DataSet.dataset_serial == self.dataset_serial)
self.assertEqual(
global_query[0].storage_dir,
self.storage_dir,
)
self.assertEqual(
global_query[0].nbr_frames,
self.nbr_channels * self.nbr_slices,
)
im_shape = self.im.shape
self.assertEqual(
global_query[0].im_width,
im_shape[2],
)
self.assertEqual(
global_query[0].im_height,
im_shape[1],
)
self.assertEqual(global_query[0].nbr_slices, self.nbr_slices)
self.assertEqual(
global_query[0].nbr_channels,
self.nbr_channels,
)
self.assertEqual(
global_query[0].nbr_positions,
1,
)
self.assertEqual(
global_query[0].nbr_timepoints,
1,
)
self.assertEqual(
global_query[0].im_colors,
1,
)
self.assertEqual(
global_query[0].bit_depth,
'uint16',
)
# query frames
frames = self.session.query(db_ops.Frames) \
.join(db_ops.FramesGlobal) \
.join(db_ops.DataSet) \
.filter(db_ops.DataSet.dataset_serial == self.dataset_serial) \
.order_by(db_ops.Frames.file_name)
# Images are separated by slice first then channel
im_order = [0, 2, 4, 1, 3, 5]
it = itertools.product(range(self.nbr_channels),
range(self.nbr_slices))
for i, (c, z) in enumerate(it):
im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z)
self.assertEqual(frames[i].file_name, im_name)
self.assertEqual(frames[i].channel_idx, c)
self.assertEqual(frames[i].slice_idx, z)
self.assertEqual(frames[i].time_idx, 0)
self.assertEqual(frames[i].pos_idx, 0)
sha256 = meta_utils.gen_sha256(self.im[im_order[i], ...])
self.assertEqual(frames[i].sha256, sha256)
# Download frames from storage and compare to originals
it = itertools.product(range(self.nbr_channels),
range(self.nbr_slices))
for i, (c, z) in enumerate(it):
im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z)
im_path = os.path.join(self.mount_point, self.storage_dir, im_name)
im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH)
nose.tools.assert_equal(im.dtype, np.uint16)
numpy.testing.assert_array_equal(im, self.im[im_order[i], ...])
@patch('imaging_db.database.db_operations.session_scope')
def test_upload_frames_already_in_db(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
data_uploader.upload_data_and_update_db(
csv=self.csv_path,
login=self.credentials_path,
config=self.config_path,
overwrite=True,
)
# Try uploading a second time
data_uploader.upload_data_and_update_db(
csv=self.csv_path,
login=self.credentials_path,
config=self.config_path,
overwrite=True,
)
@patch('imaging_db.database.db_operations.session_scope')
def test_upload_file(self, mock_session):
# Upload the same file but as file instead of frames
mock_session.return_value.__enter__.return_value = self.session
config_path = os.path.join(
self.temp_path,
'config_file.json',
)
config = {
"upload_type": "file",
"microscope": "Mass Spectrometry",
"storage": "local",
"storage_access": self.mount_point
}
json_ops.write_json_file(config, config_path)
data_uploader.upload_data_and_update_db(
csv=self.csv_path,
login=self.credentials_path,
config=config_path,
)
# Query database to find data_set and file_global
datasets = self.session.query(db_ops.DataSet) \
.filter(db_ops.DataSet.dataset_serial == self.dataset_serial)
self.assertEqual(datasets.count(), 1)
dataset = datasets[0]
self.assertEqual(dataset.id, 1)
self.assertFalse(dataset.frames)
self.assertEqual(dataset.dataset_serial, self.dataset_serial)
date_time = dataset.date_time
self.assertEqual(date_time.year, 2005)
self.assertEqual(date_time.month, 6)
self.assertEqual(date_time.day, 9)
self.assertEqual(dataset.microscope, "Mass Spectrometry")
self.assertEqual(dataset.description, 'Testing')
# query file_global
file_global = self.session.query(db_ops.FileGlobal) \
.join(db_ops.DataSet) \
.filter(db_ops.DataSet.dataset_serial == self.dataset_serial) \
.one()
expected_dir = "raw_files/TEST-2005-06-09-20-00-00-1000"
self.assertEqual(
file_global.storage_dir,
expected_dir,
)
expected_meta = {'file_origin': self.file_path}
self.assertDictEqual(file_global.metadata_json, expected_meta)
self.assertEqual(file_global.data_set, dataset)
sha256 = meta_utils.gen_sha256(self.file_path)
self.assertEqual(file_global.sha256, sha256)
# Check that file has been uploaded
file_path = os.path.join(self.mount_point, expected_dir,
'A1_2_PROTEIN_test.tif')
self.assertTrue(os.path.exists(file_path))
@patch('imaging_db.database.db_operations.session_scope')
def test_upload_file_already_in_db(self, mock_session):
# Upload the same file but as file instead of frames
mock_session.return_value.__enter__.return_value = self.session
config_path = os.path.join(
self.temp_path,
'config_file.json',
)
config = {
"upload_type": "file",
"microscope": "Mass Spectrometry",
"storage": "local",
"storage_access": self.mount_point
}
json_ops.write_json_file(config, config_path)
data_uploader.upload_data_and_update_db(
csv=self.csv_path,
login=self.credentials_path,
config=config_path,
overwrite=True,
)
# Try uploading a second time
data_uploader.upload_data_and_update_db(
csv=self.csv_path,
login=self.credentials_path,
config=config_path,
overwrite=True,
)
@patch('imaging_db.database.db_operations.session_scope')
def test_upload_ometif(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
dataset_serial = 'ISP-2005-01-01-01-00-00-0001'
# Temporary frame
im = np.ones((10, 15), dtype=np.uint16)
# Metadata
ijmeta = {
"Info":
json.dumps({
"InitialPositionList": [{
"Label": "Pos1"
}, {
"Label": "Pos3"
}]
}),
}
channel_ids = [1, 2]
im_names = ['test_Pos1.ome.tif', 'test_Pos3.ome.tif']
for i, c in enumerate(channel_ids):
mmmetadata = json.dumps({
"ChannelIndex": c,
"Slice": 20,
"FrameIndex": 30,
"PositionIndex": 40,
"Channel": 'channel_{}'.format(c),
})
extra_tags = [('MicroManagerMetadata', 's', 0, mmmetadata, True)]
# Save test ome tif file
file_path = os.path.join(self.temp_path, im_names[i])
tifffile.imsave(
file_path,
im + i * 10000,
ijmetadata=ijmeta,
extratags=extra_tags,
)
schema_file_path = os.path.realpath(
os.path.join(self.main_dir, 'metadata_schema.json'), )
# Create csv file for upload
upload_dict = {
'dataset_id': [dataset_serial],
'file_name': [self.temp_path],
'description': ['Testing'],
'positions': [[1, 3]],
'schema_filename': [schema_file_path],
}
upload_csv = pd.DataFrame.from_dict(upload_dict)
csv_path = os.path.join(self.temp_path, "test_ometif_upload.csv")
upload_csv.to_csv(csv_path)
config_path = os.path.join(
self.temp_path,
'config_ome_tiff.json',
)
config = {
"upload_type": "frames",
"frames_format": "ome_tiff",
"microscope": "",
"schema_filename": "metadata_schema.json",
"storage": "local",
"storage_access": self.mount_point
}
json_ops.write_json_file(config, config_path)
# Upload data
data_uploader.upload_data_and_update_db(
csv=csv_path,
login=self.credentials_path,
config=config_path,
)
# Query database to find data_set and frames
datasets = self.session.query(db_ops.DataSet) \
.filter(db_ops.DataSet.dataset_serial == dataset_serial)
self.assertEqual(datasets.count(), 1)
dataset = datasets[0]
self.assertEqual(dataset.id, 1)
self.assertTrue(dataset.frames)
self.assertEqual(dataset.dataset_serial, dataset_serial)
date_time = dataset.date_time
self.assertEqual(date_time.year, 2005)
self.assertEqual(date_time.month, 1)
self.assertEqual(date_time.day, 1)
self.assertEqual(dataset.description, 'Testing')
# query frames_global
global_query = self.session.query(db_ops.FramesGlobal) \
.join(db_ops.DataSet) \
.filter(db_ops.DataSet.dataset_serial == dataset_serial)
self.assertEqual(
global_query[0].storage_dir,
'raw_frames/' + dataset_serial,
)
self.assertEqual(
global_query[0].nbr_frames,
2,
)
im_shape = im.shape
self.assertEqual(
global_query[0].im_width,
im_shape[1],
)
self.assertEqual(
global_query[0].im_height,
im_shape[0],
)
self.assertEqual(global_query[0].nbr_slices, 1)
self.assertEqual(
global_query[0].nbr_channels,
2,
)
self.assertEqual(
global_query[0].nbr_positions,
1,
)
self.assertEqual(
global_query[0].nbr_timepoints,
1,
)
self.assertEqual(
global_query[0].im_colors,
1,
)
self.assertEqual(
global_query[0].bit_depth,
'uint16',
)
# query frames
frames = self.session.query(db_ops.Frames) \
.join(db_ops.FramesGlobal) \
.join(db_ops.DataSet) \
.filter(db_ops.DataSet.dataset_serial == dataset_serial) \
.order_by(db_ops.Frames.file_name)
shas = [meta_utils.gen_sha256(im), meta_utils.gen_sha256(im + 10000)]
for i, c in enumerate(channel_ids):
im_name = 'im_c00{}_z020_t030_p040.png'.format(c)
self.assertEqual(frames[i].file_name, im_name)
self.assertEqual(frames[i].channel_idx, c)
self.assertEqual(frames[i].channel_name, 'channel_{}'.format(c))
self.assertEqual(frames[i].slice_idx, 20)
self.assertEqual(frames[i].time_idx, 30)
self.assertEqual(frames[i].pos_idx, 40)
self.assertEqual(frames[i].sha256, shas[i])
# # Download frames from storage and compare to originals
for i, c in enumerate(channel_ids):
im_name = 'im_c00{}_z020_t030_p040.png'.format(c)
im_path = os.path.join(
self.mount_point,
'raw_frames',
dataset_serial,
im_name,
)
im_out = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH)
nose.tools.assert_equal(im.dtype, np.uint16)
numpy.testing.assert_array_equal(im_out, im + i * 10000)
@patch('imaging_db.database.db_operations.session_scope')
def test_upload_tiffolder(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
dataset_serial = 'SMS-2010-01-01-01-00-00-0005'
# Temporary frame
im = np.ones((10, 15), dtype=np.uint8)
# Save test tif files
self.tempdir.makedir('tiffolder')
tif_dir = os.path.join(self.temp_path, 'tiffolder')
channel_names = ['phase', 'brightfield', '666']
# Write files in dir
for c_name in channel_names:
for z in range(2):
file_name = 'img_{}_t060_p050_z00{}.tif'.format(c_name, z)
file_path = os.path.join(tif_dir, file_name)
ijmeta = {"Info": json.dumps({"c": c_name, "z": z})}
tifffile.imsave(
file_path,
im + 50 * z,
ijmetadata=ijmeta,
)
# Write external metadata in dir
self.meta_dict = {
'Summary': {
'Slices': 6,
'PixelType': 'GRAY8',
'Time': '2018-11-01 19:20:34 -0700',
'z-step_um': 0.5,
'PixelSize_um': 0,
'BitDepth': 8,
'Width': 15,
'Height': 10
},
}
self.json_filename = os.path.join(tif_dir, 'metadata.txt')
json_ops.write_json_file(self.meta_dict, self.json_filename)
# Create csv file for upload
upload_dict = {
'dataset_id': [dataset_serial],
'file_name': [tif_dir],
'description': ['Testing tifffolder upload'],
}
upload_csv = pd.DataFrame.from_dict(upload_dict)
csv_path = os.path.join(self.temp_path, "test_tiffolder_upload.csv")
upload_csv.to_csv(csv_path)
config_path = os.path.join(
self.temp_path,
'config_tiffolder.json',
)
config = {
"upload_type": "frames",
"frames_format": "tif_folder",
"microscope": "CZDRAGONFLY-PC",
"filename_parser": "parse_sms_name",
"storage": "local",
"storage_access": self.mount_point
}
json_ops.write_json_file(config, config_path)
# Upload data
data_uploader.upload_data_and_update_db(
csv=csv_path,
login=self.credentials_path,
config=config_path,
)
# Query database to find data_set and frames
datasets = self.session.query(db_ops.DataSet) \
.filter(db_ops.DataSet.dataset_serial == dataset_serial)
self.assertEqual(datasets.count(), 1)
dataset = datasets[0]
self.assertTrue(dataset.frames)
self.assertEqual(dataset.dataset_serial, dataset_serial)
date_time = dataset.date_time
self.assertEqual(date_time.year, 2010)
self.assertEqual(date_time.month, 1)
self.assertEqual(date_time.day, 1)
self.assertEqual(dataset.description, 'Testing tifffolder upload')
# query frames_global
global_query = self.session.query(db_ops.FramesGlobal) \
.join(db_ops.DataSet) \
.filter(db_ops.DataSet.dataset_serial == dataset_serial)
self.assertEqual(
global_query[0].storage_dir,
'raw_frames/' + dataset_serial,
)
self.assertEqual(
global_query[0].nbr_frames,
6,
)
self.assertEqual(
global_query[0].im_width,
15,
)
self.assertEqual(
global_query[0].im_height,
10,
)
self.assertEqual(global_query[0].nbr_slices, 2)
self.assertEqual(
global_query[0].nbr_channels,
3,
)
self.assertEqual(
global_query[0].nbr_positions,
1,
)
self.assertEqual(
global_query[0].nbr_timepoints,
1,
)
self.assertEqual(
global_query[0].im_colors,
1,
)
self.assertEqual(
global_query[0].bit_depth,
'uint8',
)
# query frames
frames = self.session.query(db_ops.Frames) \
.join(db_ops.FramesGlobal) \
.join(db_ops.DataSet) \
.filter(db_ops.DataSet.dataset_serial == dataset_serial) \
.order_by(db_ops.Frames.file_name)
# Validate content
# Channel numbers will be assigned alphabetically
channel_names.sort()
for i, (c, z) in enumerate(itertools.product(range(3), range(2))):
im_name = 'im_c00{}_z00{}_t060_p050.png'.format(c, z)
self.assertEqual(frames[i].file_name, im_name)
self.assertEqual(frames[i].channel_idx, c)
self.assertEqual(frames[i].channel_name, channel_names[c])
self.assertEqual(frames[i].slice_idx, z)
self.assertEqual(frames[i].time_idx, 60)
self.assertEqual(frames[i].pos_idx, 50)
self.assertEqual(
frames[i].sha256,
meta_utils.gen_sha256(im + 50 * z),
)
# # Download frames from storage and compare to originals
for i in range(len(channel_names)):
for z in range(2):
im_name = 'im_c00{}_z00{}_t060_p050.png'.format(i, z)
im_path = os.path.join(
self.mount_point,
'raw_frames',
dataset_serial,
im_name,
)
im_out = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH)
nose.tools.assert_equal(im.dtype, np.uint8)
numpy.testing.assert_array_equal(im_out, im + z * 50)
class TestImageInference(unittest.TestCase):
@patch('micro_dl.inference.model_inference.load_model')
def setUp(self, mock_model):
"""
Set up a directory with images
"""
mock_model.return_value = 'dummy_model'
self.tempdir = TempDirectory()
self.temp_path = self.tempdir.path
self.tempdir.makedir('image_dir')
self.tempdir.makedir('mask_dir')
self.tempdir.makedir('model_dir')
self.image_dir = os.path.join(self.temp_path, 'image_dir')
self.mask_dir = os.path.join(self.temp_path, 'mask_dir')
self.model_dir = os.path.join(self.temp_path, 'model_dir')
# Create a temp image dir
self.im = np.zeros((10, 16), dtype=np.uint8)
self.frames_meta = aux_utils.make_dataframe()
self.time_idx = 2
self.slice_idx = 3
for p in range(5):
for c in range(3):
im_name = aux_utils.get_im_name(
time_idx=self.time_idx,
channel_idx=c,
slice_idx=self.slice_idx,
pos_idx=p,
)
cv2.imwrite(os.path.join(self.image_dir, im_name),
self.im + c * 10)
self.frames_meta = self.frames_meta.append(
aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES),
ignore_index=True,
)
# Write frames meta to image dir too
self.frames_meta.to_csv(os.path.join(self.image_dir,
'frames_meta.csv'))
# Save masks and mask meta
self.mask_meta = aux_utils.make_dataframe()
self.mask_channel = 50
for p in range(5):
im_name = aux_utils.get_im_name(
time_idx=self.time_idx,
channel_idx=self.mask_channel,
slice_idx=self.slice_idx,
pos_idx=p,
)
cv2.imwrite(os.path.join(self.mask_dir, im_name), self.im + 1)
self.mask_meta = self.mask_meta.append(
aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES),
ignore_index=True,
)
# Write frames meta to mask dir too
self.mask_meta.to_csv(os.path.join(self.mask_dir, 'frames_meta.csv'))
# Setup model dir
split_samples = {
"train": [0, 1],
"val": [2],
"test": [3, 4],
}
aux_utils.write_json(
split_samples,
os.path.join(self.model_dir, 'split_samples.json'),
)
# Make configs with fields necessary for 2D segmentation inference
self.train_config = {
'network': {
'class': 'UNet2D',
'data_format': 'channels_first',
'depth': 1,
'width': 10,
'height': 10
},
'dataset': {
'split_by_column': 'pos_idx',
'input_channels': [1, 2],
'target_channels': [self.mask_channel],
'model_task': 'segmentation',
},
}
self.inference_config = {
'model_dir': self.model_dir,
'model_fname': 'dummy_weights.hdf5',
'image_dir': self.image_dir,
'data_split': 'test',
'images': {
'image_format': 'zyx',
'image_ext': '.png',
},
'metrics': {
'metrics': ['mae'],
'metrics_orientations': ['xy'],
},
'masks': {
'mask_dir': self.mask_dir,
'mask_type': 'target',
'mask_channel': 50,
}
}
# Instantiate class
self.infer_inst = image_inference.ImagePredictor(
train_config=self.train_config,
inference_config=self.inference_config,
)
def tearDown(self):
"""
Tear down temporary folder and file structure
"""
TempDirectory.cleanup_all()
self.assertEqual(os.path.isdir(self.temp_path), False)
def test_init(self):
"""
Test init of inference dataset
"""
# Check proper init
self.assertEqual(self.infer_inst.model_dir, self.model_dir)
self.assertEqual(self.infer_inst.image_dir, self.image_dir)
self.assertEqual(self.infer_inst.data_format, 'channels_first')
self.assertEqual(self.infer_inst.model, 'dummy_model')
self.assertEqual(self.infer_inst.image_format, 'zyx')
self.assertEqual(self.infer_inst.image_ext, '.png')
self.assertFalse(self.infer_inst.mask_metrics)
self.assertEqual(self.infer_inst.mask_dir, self.mask_dir)
self.assertListEqual(self.infer_inst.metrics_orientations, ['xy'])
self.assertEqual(self.infer_inst.num_overlap, 0)
self.assertIsNone(self.infer_inst.stitch_inst)
self.assertIsNone(self.infer_inst.tile_option)
self.assertIsNone(self.infer_inst.crop_shape)
def test_get_split_ids(self):
split_col, infer_ids = self.infer_inst._get_split_ids()
self.assertEqual(split_col, 'pos_idx')
self.assertListEqual(infer_ids, [3, 4])
def test_get_split_ids_no_json(self):
self.infer_inst.model_dir = self.infer_inst.image_dir
split_col, infer_ids = self.infer_inst._get_split_ids()
self.assertEqual(split_col, 'pos_idx')
self.assertListEqual(infer_ids, [0, 1, 2, 3, 4])
def test_save_pred_image(self):
im = np.zeros((1, 10, 15), dtype=np.uint8)
im[:, 5, :] = 128
self.infer_inst.save_pred_image(
predicted_image=im,
time_idx=10,
target_channel_idx=20,
pos_idx=30,
slice_idx=40,
)
pred_name = os.path.join(
self.model_dir,
'predictions/im_c020_z040_t010_p030.png',
)
im_pred = cv2.imread(pred_name, cv2.IMREAD_ANYDEPTH)
self.assertEqual(im_pred.dtype, np.uint16)
self.assertTupleEqual(im_pred.shape, (10, 15))
# Prediction intensities are maximized to range
self.assertEqual(im_pred.max(), 65535)
self.assertEqual(im_pred.min(), 0)
def test_estimate_metrics_xy(self):
target = np.ones((10, 15, 5), dtype=np.float32)
prediction = np.zeros_like(target)
prediction[:5, :, :] = 1
pred_names = ['test1', 'test2', 'test3', 'test4', 'test5']
self.infer_inst.estimate_metrics(target, prediction, pred_names, None)
metrics = self.infer_inst.df_xy
self.assertTupleEqual(metrics.shape, (5, 2))
self.assertListEqual(list(metrics), ['mae', 'pred_name'])
self.assertEqual(metrics.mae.mean(), 0.5)
def test_estimate_metrics_xy_one_name(self):
target = np.ones((10, 15, 5), dtype=np.float32)
prediction = np.zeros_like(target)
prediction[:5, :, :] = 1
self.infer_inst.estimate_metrics(target, prediction, ['test_name'],
None)
metrics = self.infer_inst.df_xy
self.assertTupleEqual(metrics.shape, (5, 2))
self.assertListEqual(list(metrics), ['mae', 'pred_name'])
self.assertEqual(metrics.mae.mean(), 0.5)
def test_estimate_metrics_xyz(self):
target = np.ones((10, 15, 5), dtype=np.float32)
prediction = np.zeros_like(target)
prediction[:5, :, :] = 1
self.infer_inst.metrics_orientations = ['xyz']
self.infer_inst.estimate_metrics(target, prediction, ['test_name'],
None)
metrics = self.infer_inst.df_xyz
self.assertTupleEqual(metrics.shape, (1, 2))
self.assertListEqual(list(metrics), ['mae', 'pred_name'])
self.assertEqual(metrics.mae[0], 0.5)
self.assertEqual(metrics.pred_name[0], 'test_name')
def test_estimate_metrics_xz(self):
target = np.ones((10, 15, 5), dtype=np.float32)
prediction = np.zeros_like(target)
prediction[:5, :, :] = 1
self.infer_inst.metrics_orientations = ['xz']
self.infer_inst.estimate_metrics(target, prediction, ['test_name'],
None)
metrics = self.infer_inst.df_xz
self.assertTupleEqual(metrics.shape, (10, 2))
self.assertListEqual(list(metrics), ['mae', 'pred_name'])
self.assertEqual(metrics.mae[0], 0.0)
self.assertEqual(metrics.mae[5], 1.0)
self.assertEqual(metrics.pred_name[9], 'test_name_xz9')
def test_estimate_metrics_yz(self):
target = np.ones((10, 15, 5), dtype=np.float32)
prediction = np.zeros_like(target)
prediction[:5, :, :] = 1
self.infer_inst.metrics_orientations = ['yz']
self.infer_inst.estimate_metrics(target, prediction, ['test_name'],
None)
metrics = self.infer_inst.df_yz
self.assertTupleEqual(metrics.shape, (15, 2))
self.assertListEqual(list(metrics), ['mae', 'pred_name'])
self.assertEqual(metrics.mae[0], 0.5)
self.assertEqual(metrics.pred_name[14], 'test_name_yz14')
def test_get_mask(self):
meta_row = dict.fromkeys(aux_utils.DF_NAMES)
meta_row['channel_idx'] = self.mask_channel
meta_row['time_idx'] = self.time_idx
meta_row['slice_idx'] = self.slice_idx
meta_row['pos_idx'] = 2
mask = self.infer_inst.get_mask(meta_row)
self.assertTupleEqual(mask.shape, (8, 16))
self.assertEqual(mask.dtype, np.uint8)
self.assertEqual(mask.max(), 1)
self.assertEqual(mask.min(), 1)
@patch('micro_dl.inference.model_inference.predict_large_image')
def test_predict_2d(self, mock_predict):
mock_predict.return_value = 1. + np.ones((1, 8, 16), dtype=np.float32)
# Predict row 0 from inference dataset iterator
pred_im, target_im, mask_im = self.infer_inst.predict_2d([0])
self.assertTupleEqual(pred_im.shape, (8, 16, 1))
self.assertEqual(pred_im.dtype, np.float32)
self.assertEqual(pred_im.max(), 2.0)
# Read saved prediction too
pred_name = os.path.join(
self.model_dir,
'predictions/im_c050_z003_t002_p003.png',
)
im_pred = cv2.imread(pred_name, cv2.IMREAD_ANYDEPTH)
self.assertEqual(im_pred.dtype, np.uint16)
self.assertTupleEqual(im_pred.shape, (8, 16))
# Check target and no mask
self.assertTupleEqual(target_im.shape, (8, 16, 1))
self.assertEqual(target_im.dtype, np.float32)
self.assertEqual(target_im.max(), 1.)
self.assertListEqual(mask_im, [])
@patch('micro_dl.inference.model_inference.predict_large_image')
def test_predict_2d_mask(self, mock_predict):
self.infer_inst.crop_shape = [6, 10]
self.infer_inst.mask_metrics = True
mock_predict.return_value = 1. + np.ones((1, 6, 10), dtype=np.float32)
# Predict row 0 from inference dataset iterator
pred_im, target_im, mask_im = self.infer_inst.predict_2d([0])
self.assertTupleEqual(pred_im.shape, (6, 10, 1))
self.assertEqual(pred_im.dtype, np.float32)
self.assertEqual(pred_im.max(), 2.0)
# Read saved prediction too
pred_name = os.path.join(
self.model_dir,
'predictions/im_c050_z003_t002_p003.png',
)
im_pred = cv2.imread(pred_name, cv2.IMREAD_ANYDEPTH)
self.assertEqual(im_pred.dtype, np.uint16)
self.assertTupleEqual(im_pred.shape, (6, 10))
# Check target and no mask
self.assertTupleEqual(target_im.shape, (6, 10, 1))
self.assertEqual(target_im.dtype, np.float32)
self.assertEqual(target_im.max(), 1.)
self.assertTupleEqual(mask_im.shape, (6, 10, 1))
self.assertEqual(mask_im.dtype, np.uint8)
@patch('micro_dl.inference.model_inference.predict_large_image')
def test_run_prediction(self, mock_predict):
mock_predict.return_value = 1. + np.ones((1, 8, 16), dtype=np.float32)
# Run prediction. Should create a metrics_xy.csv in pred dir
self.infer_inst.run_prediction()
metrics = pd.read_csv(
os.path.join(self.model_dir, 'predictions/metrics_xy.csv'))
self.assertTupleEqual(metrics.shape, (2, 2))
# MAE should be 1.
self.assertEqual(metrics.mae.mean(), 1.0)
# There should be two rows, one per test index
self.assertEqual(metrics.pred_name[0], 'im_c050_z003_t002_p003_xy0')
self.assertEqual(metrics.pred_name[1], 'im_c050_z003_t002_p004_xy0')
# There should be 2 predictions saved in pred dir
for pos in range(3, 5):
pred_name = os.path.join(
self.model_dir,
'predictions/im_c050_z003_t002_p00{}.png'.format(pos),
)
im_pred = cv2.imread(pred_name, cv2.IMREAD_ANYDEPTH)
self.assertEqual(im_pred.dtype, np.uint16)
self.assertTupleEqual(im_pred.shape, (8, 16))
class TestFolderTestCase(unittest.TestCase):
def setUp(self):
self.tempdir = TempDirectory(encoding='utf-8')
def tearDown(self):
self.tempdir.cleanup()
def test_returns_false_if_parent_attributes_not_set(self):
d = Folder(self.tempdir.path)
with self.subTest(1):
compare(self.tempdir.path, d.path)
with self.subTest(2):
compare(os.path.basename(self.tempdir.path), d.name)
with self.subTest(3):
compare(os.path.dirname(self.tempdir.path), d.parent)
def test_returns_false_if_attributes_are_not_equal(self):
temp_path = self.tempdir.path
dir_1 = Folder(temp_path)
dir_2 = Folder(self.tempdir.makedir('abc/123/JPG'))
dir_3 = Folder(self.tempdir.makedir('abc/&%^/website'))
d4 = self.tempdir.makedir('abc/123/ghi')
d5 = Folder(os.path.join(temp_path, 'abc', 'hjig'))
sorter_identity = self.tempdir.write(
os.path.join(d4, SORTER_FOLDER_IDENTITY_FILENAME), '')
dir_4 = Folder(d4)
with self.subTest(1):
compare(True, dir_1.exists)
with self.subTest(2):
compare(False, dir_1.for_sorter)
with self.subTest(3):
compare('FOLDERS', dir_1.category_folder)
with self.subTest(4):
compare(True, dir_2.for_sorter)
with self.subTest(5):
compare('image', dir_2.category_folder)
with self.subTest(6):
compare(True, dir_3.for_sorter)
with self.subTest(7):
compare(None, dir_3.category_folder)
with self.subTest(8):
compare('FOLDERS', dir_4.category_folder)
with self.subTest(9):
compare(True, os.path.isfile(sorter_identity))
with self.subTest(10):
compare(True, dir_4.for_sorter)
with self.subTest(11):
compare(False, d5.for_sorter)
def test_returns_false_if_folder_not_exists(self):
temp_path = self.tempdir.path
path = os.path.join(temp_path, 'bdk/ksks/a94')
dir_1 = Folder(path)
with self.subTest(1):
compare(False, dir_1.exists)
with self.subTest(2):
self.assertRaises(FileNotFoundError, dir_1.create)
with self.subTest(3):
compare(False, dir_1.exists)
dir_1.create(parents=True)
with self.subTest(4):
compare(True, dir_1.exists)
def test_returns_false_if_grouping_failed(self):
dir_1 = self.tempdir.makedir('abc/for/document')
dir_2 = self.tempdir.makedir('abc/for/PDF')
dir_3 = self.tempdir.makedir('abc/for/last')
dir_ = self.tempdir.makedir('one/two')
d1 = Folder(dir_1)
d2 = Folder(dir_2)
d3 = Folder(dir_3)
with self.subTest(1):
compare(dir_1, d1.path)
d1.move_to(dir_)
with self.subTest(2):
compare(os.path.join(dir_, 'document'), d1.path)
with self.subTest(3):
compare(dir_2, d2.path)
d2.move_to(dir_)
with self.subTest(4):
compare(os.path.join(dir_, 'document', 'PDF'), d2.path)
with self.subTest(5):
compare(dir_3, d3.path)
d3.move_to(dir_)
with self.subTest(6):
compare(os.path.join(dir_, 'FOLDERS', 'last'), d3.path)
def test_folder_grouping_with_files(self):
write = lambda path: self.tempdir.write(path, '')
isfile = os.path.isfile
dir_ = self.tempdir.makedir('one/two')
file_1 = write('abc/for/PDF/this long name.pdf')
file_2 = write('abc/for/DOCX/y.docx')
file_3 = write('abc/for/DOC/123 54.doc')
file_4 = write('abc/for/none/xw')
file_5 = write('abc/for/PDF/second.pdf')
dir_1 = Folder(os.path.dirname(file_1))
dir_2 = Folder(os.path.dirname(file_2))
dir_3 = Folder(os.path.dirname(file_3))
dir_4 = Folder(os.path.dirname(file_4))
dir_5 = Folder(os.path.dirname(file_5))
dir_6 = Folder(dir_)
with self.subTest(1):
compare([True, True, True, True],
[isfile(file_1), isfile(file_2), isfile(file_3), isfile(file_4)])
dir_1.move_to(dir_)
dir_2.move_to(dir_)
dir_3.move_to(dir_)
dir_4.move_to(dir_)
dir_5.move_to(dir_)
dir_6.move_to(dir_)
with self.subTest(3):
self.tempdir.compare([
'{}/'.format('document'),
'{}/'.format('FOLDERS'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('FOLDERS', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/'.format('document', 'DOCX'),
'{}/{}/'.format('document', 'DOC'),
'{}/{}/'.format('FOLDERS', 'none'),
'{}/{}/{}'.format('document', 'PDF',
'this long name.pdf'),
'{}/{}/{}'.format('document', 'PDF', 'second.pdf'),
'{}/{}/{}'.format('document', 'DOCX', 'y.docx'),
'{}/{}/{}'.format('document', 'DOC', '123 54.doc'),
'{}/{}/{}'.format('FOLDERS', 'none', 'xw'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'DOCX',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'DOC',
SORTER_FOLDER_IDENTITY_FILENAME),
], path=dir_)
with self.subTest(4):
compare(os.path.join(dir_, 'document', 'PDF'), dir_5.path)
def test_folder_grouping_with_files_and_existing_destinations(self):
temp_path = self.tempdir.path
write = lambda path: self.tempdir.write(path, '')
isfile = os.path.isfile
dir_ = self.tempdir.makedir('one/two')
self.tempdir.makedir('one/two/document/PDF/')
self.tempdir.makedir('one/two/document/DOCX')
self.tempdir.makedir('one/two/document/DOC')
file_1 = write('abc/for/PDF/this long name.pdf')
file_2 = write('abc/for/DOCX/y.docx')
file_3 = write('abc/for/DOC/123 54.doc')
file_4 = write('abc/for/none/xw')
file_5 = write('abc/for/PDF/second.pdf')
file_6 = write('abc/for/image/JPEG/abc.jpeg')
dir_1 = Folder(os.path.dirname(file_1))
dir_2 = Folder(os.path.dirname(file_2))
dir_3 = Folder(os.path.dirname(file_3))
dir_4 = Folder(os.path.dirname(file_4))
dir_5 = Folder(os.path.dirname(file_5))
dir_6 = Folder(os.path.dirname(os.path.dirname(file_6)))
dir_7 = Folder(dir_)
with self.subTest(1):
compare([True, True, True, True],
[isfile(file_1), isfile(file_2), isfile(file_3), isfile(file_4)])
with self.subTest(2):
compare('document', dir_1.category_folder)
dir_1.move_to(dir_)
dir_2.move_to(dir_)
dir_3.move_to(dir_)
dir_4.move_to(dir_)
dir_5.move_to(dir_)
dir_6.move_to(dir_)
dir_7.move_to(dir_)
with self.subTest(3):
compare(True, dir_1.for_sorter)
with self.subTest(4):
self.tempdir.compare([
'{}/'.format('document'),
'{}/'.format('image'),
'{}/'.format('FOLDERS'),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/'.format('document', 'DOCX'),
'{}/{}/'.format('document', 'DOC'),
'{}/{}/'.format('image', 'JPEG'),
'{}/{}/'.format('FOLDERS', 'none'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('image', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('FOLDERS', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'PDF',
'this long name.pdf'),
'{}/{}/{}'.format('document', 'PDF', 'second.pdf'),
'{}/{}/{}'.format('document', 'DOCX', 'y.docx'),
'{}/{}/{}'.format('document', 'DOC', '123 54.doc'),
'{}/{}/{}'.format('image', 'JPEG', 'abc.jpeg'),
'{}/{}/{}'.format('FOLDERS', 'none', 'xw'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'DOCX',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'DOC',
SORTER_FOLDER_IDENTITY_FILENAME),
], path=dir_)
with self.subTest(5):
compare(os.path.join(dir_, 'document', 'PDF'), dir_5.path)
with self.subTest(6):
compare(True, os.path.isdir(dir_1.path))
with self.subTest(7):
compare(os.path.join(temp_path, dir_, 'document', 'PDF'), dir_1.path)
with self.subTest(8):
compare(True, os.path.isfile(os.path.join(
dir_1.path, SORTER_FOLDER_IDENTITY_FILENAME)))
with self.subTest(9):
compare(False, os.path.exists(os.path.dirname(file_1)))
def test_retuns_false_folder_with_multiple_subfolders_relocation_failse(self):
temp_path = self.tempdir.path
write = lambda path: self.tempdir.write(path, '')
dir_ = self.tempdir.makedir('one/two')
self.tempdir.makedir('one/two/document/PDF/')
file_1 = write('abc/for/PDF/this long name.pdf')
write('abc/for/PDF/somefolder/another/and another/JPEG/abc.jpeg')
dir_1 = Folder(os.path.dirname(file_1))
dir_1.move_to(dir_)
with self.subTest(1):
self.tempdir.compare([
'{}/'.format('document'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}/'.format('document', 'PDF', 'somefolder'),
'{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'),
'{}/{}/{}/{}/'.format('document', 'PDF',
'somefolder', 'another'),
'{}/{}/{}/{}/{}/'.format('document', 'PDF',
'somefolder', 'another', 'and another'),
'{}/{}/{}/{}/{}/{}/'.format('document', 'PDF',
'somefolder', 'another', 'and another', 'JPEG'),
'{}/{}/{}/{}/{}/{}/{}'.format('document', 'PDF', 'somefolder',
'another', 'and another', 'JPEG', 'abc.jpeg'),
], path=dir_)
def test_returns_false_if_folder_relocation_with_ignore_file_succeeds(self):
temp_path = self.tempdir.path
write = lambda path: self.tempdir.write(path, '')
dir_ = self.tempdir.makedir('one/two')
self.tempdir.makedir('one/two/document/PDF/')
file_1 = write('abc/for/PDF/this long name.pdf')
file_2 = write(
'abc/for/PDF/somefolder/another/and another/JPEG/abc.jpeg')
file_3 = write('abc/for/PDF/somefolder/%s' % SORTER_IGNORE_FILENAME)
with self.subTest(1):
compare(True, os.path.isfile(file_3))
with self.subTest(2):
compare(True, os.path.isdir(os.path.dirname(file_2)))
dir_1 = Folder(os.path.dirname(file_1))
dir_1.move_to(dir_)
with self.subTest(3):
self.tempdir.compare([
'{}/'.format('document'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'),
], path=dir_)
def test_returns_false_if_original_folder_exists(self):
temp_path = self.tempdir.path
write = lambda path: self.tempdir.write(path, '')
isfile = os.path.isfile
dir_ = self.tempdir.makedir('one/two')
self.tempdir.makedir('one/two/document/PDF/')
file_1 = write('abc/for/PDF/this long name.pdf')
dir_1 = Folder(os.path.dirname(file_1))
dir_1.move_to(dir_)
with self.subTest(1):
self.tempdir.compare([
'{}/'.format('document'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'),
], path=dir_)
with self.subTest(2):
compare(False, os.path.isdir(os.path.dirname(file_1)))
def test_returns_false_if_original_folder_not_exists(self):
temp_path = self.tempdir.path
write = lambda path: self.tempdir.write(path, '')
isfile = os.path.isfile
dir_ = self.tempdir.makedir('one/two')
dir_2 = self.tempdir.makedir('one/two/document/PDF/')
file_1 = write('abc/for/PDF/this long name.pdf')
file_2 = write(
'abc/for/PDF/somefolder/another/and another/JPEG/abc.jpeg')
file_3 = write('abc/for/PDF/somefolder/%s' % SORTER_IGNORE_FILENAME)
with self.subTest(1):
compare(True, os.path.isfile(file_3))
with self.subTest(2):
compare(True, os.path.isdir(os.path.dirname(file_2)))
dir_1 = Folder(os.path.dirname(file_1))
dir_1.move_to(dir_)
with self.subTest(3):
self.tempdir.compare([
'{}/'.format('document'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'),
], path=dir_)
with self.subTest(4):
compare(True, os.path.isdir(dir_2))
def test_returns_false_if_grouping_in_same_folder_failed(self):
temp_path = self.tempdir.path
write = lambda path: self.tempdir.write(path, '')
isfile = os.path.isfile
dir_ = self.tempdir.makedir('one/two')
file_1 = write('one/two/PDF/this long name.pdf')
file_2 = write(
'one/two/PDF/somefolder/another/and another/JPEG/abc.jpeg')
file_3 = write('one/two/PDF/somefolder/%s' % SORTER_IGNORE_FILENAME)
with self.subTest(1):
compare(True, os.path.isfile(file_3))
with self.subTest(2):
compare(True, os.path.isdir(os.path.dirname(file_2)))
dir_1 = Folder(os.path.dirname(file_1))
dir_1.group(dir_, by_extension=True)
with self.subTest(3):
compare(True, os.path.isfile(os.path.join(temp_path, 'one',
'two', 'document', 'PDF', SORTER_FOLDER_IDENTITY_FILENAME)))
with self.subTest(4):
self.tempdir.compare([
'{}/'.format('document'),
'{}/'.format('PDF'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('document', SORTER_IGNORE_FILENAME),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'),
'{}/{}/'.format('PDF', 'somefolder'),
'{}/{}/{}'.format('PDF', 'somefolder', SORTER_IGNORE_FILENAME),
'{}/{}/{}/'.format('PDF', 'somefolder', 'another'),
'{}/{}/{}/{}/'.format('PDF', 'somefolder',
'another', 'and another'),
'{}/{}/{}/{}/{}/'.format('PDF', 'somefolder',
'another', 'and another', 'JPEG'),
'{}/{}/{}/{}/{}/{}'.format('PDF', 'somefolder',
'another', 'and another', 'JPEG', 'abc.jpeg')
], path=dir_)
with self.subTest(5):
compare(True, os.path.isdir(os.path.dirname(file_1)))
def test_watcher_with_file_filter(self):
work_directory = TempDirectory()
work_directory.write('a.h', b'')
work_directory.write('a.c', b'')
work_directory.write('a.cc', b'')
work_directory.write('CMakeLists.txt', b'')
work_directory.write('blah.txt', b'')
work_directory.makedir('blah')
work_directory.write(['blah', 'test_dummy.c'], b'')
wd_len = len(work_directory.path) + 1
w = Watcher(work_directory.path, None, [])
watchstate = w.poll()
filelist = [f[wd_len:] for f in watchstate.inserts]
filelist.sort()
assert filelist == [
'CMakeLists.txt',
'a.c',
'a.cc',
'a.h',
'blah.txt',
'blah/test_dummy.c'
]
w = Watcher(work_directory.path, None,
source_patterns=['CMakeLists.txt'])
watchstate = w.poll()
filelist = [f[wd_len:] for f in watchstate.inserts]
filelist.sort()
assert filelist == ['CMakeLists.txt']
w = Watcher(work_directory.path, None, source_patterns=['*.txt'])
watchstate = w.poll()
filelist = [f[wd_len:] for f in watchstate.inserts]
filelist.sort()
assert filelist == ['CMakeLists.txt', 'blah.txt']
w = Watcher(work_directory.path, None, source_patterns=['?.cc'])
watchstate = w.poll()
filelist = [f[wd_len:] for f in watchstate.inserts]
filelist.sort()
assert filelist == ['a.cc']
w = Watcher(work_directory.path, None, source_patterns=['blah'])
watchstate = w.poll()
filelist = [f[wd_len:] for f in watchstate.inserts]
filelist.sort()
assert filelist == ['blah.txt', 'blah/test_dummy.c']
w = Watcher(work_directory.path, None, source_patterns=['blah/'])
watchstate = w.poll()
filelist = [f[wd_len:] for f in watchstate.inserts]
filelist.sort()
assert filelist == ['blah/test_dummy.c']
w = Watcher(work_directory.path, None, source_patterns=['*.txt'],
source_exclusions=['blah.txt'])
watchstate = w.poll()
filelist = [f[wd_len:] for f in watchstate.inserts]
filelist.sort()
assert filelist == ['CMakeLists.txt']
w = Watcher(work_directory.path, None, source_patterns=['*.txt'],
source_exclusions=['blah.*'])
watchstate = w.poll()
filelist = [f[wd_len:] for f in watchstate.inserts]
filelist.sort()
assert filelist == ['CMakeLists.txt']
w = Watcher(work_directory.path, None, None,
source_exclusions=['*.txt', 'blah'])
watchstate = w.poll()
filelist = [f[wd_len:] for f in watchstate.inserts]
filelist.sort()
assert filelist == ['a.c', 'a.cc', 'a.h']
class Test_Logging_setup(TestCase):
def setUp(self):
self.default_config = bbpgsql.configuration.config()
self.td = TempDirectory()
self.logdir = self.td.makedir('log')
self.logfilepath = os.path.join(self.logdir, 'bbpgsql')
self.config_path = self.td.write('bbpgsql', config_text)
self.full_config = bbpgsql.configuration.config([self.config_path])
self.full_config.set('Logging', 'logfile', self.logfilepath)
self.config_path = self.td.write('bad_level', config_bad_values)
self.bad_config = bbpgsql.configuration.config([self.config_path])
self.config_path = self.td.write('section_only', config_only_section)
self.section_only = bbpgsql.configuration.config([self.config_path])
def tearDown(self):
self.td.cleanup()
def test_default_config_lacks_logging(self):
self.assertFalse(self.default_config.has_section('Logger'))
def test_section_only_does_no_logging(self):
log_config = bbpgsql.configuration.set_up_logging(self.section_only)
# assert only null handler
self.assertEqual(1, len(log_config['handlers']))
@patch('logging.config.dictConfig')
def test_logger_sets_up_default_config(self, mock_dictConfig):
mock_dictConfig.return_value = None
bbpgsql.configuration.set_up_logging(self.default_config)
self.assertTrue(mock_dictConfig.called)
expected = ((default_log_config, ), {})
self.assertEqual(expected, mock_dictConfig.call_args)
def test_logger_sets_log_level_from_configfile(self):
log_config = bbpgsql.configuration.set_up_logging(self.full_config)
self.assertEqual(DEBUG,
log_config['loggers']['bbpgsql']['level'])
def test_logger_rejects_bad_level_name(self):
self.assertRaises(Exception, bbpgsql.configuration.set_up_logging,
(self.bad_config))
def test_logger_gets_filename_from_config(self):
handlers, formatters = \
bbpgsql.configuration.set_up_logger_file_handler(self.full_config)
self.assertEqual(handlers,
{'file': {
'class': 'logging.handlers.TimedRotatingFileHandler',
'formatter': 'file_formatter',
'filename': self.logfilepath,
'when': 'd',
'interval': 1,
'backupCount': 14
}
}
)
self.assertEqual(formatters,
{
'file_formatter': {
'format': "%(asctime)s - %(levelname)s - %(message)s",
}
}
)
def test_logger_does_nothing_if_section_is_missing(self):
handlers, formatters = \
bbpgsql.configuration.set_up_logger_file_handler(
self.default_config)
self.assertEqual({}, handlers)
self.assertEqual({}, formatters)
def test_logger_gets_hostname_port_from_config(self):
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(
self.full_config)
self.assertEqual(handlers['syslog']['address'],
('localhost', 514))
def test_logger_does_nothing_if_syslog_host_not_defined(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'logport', '514')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
self.assertEqual({}, handlers)
self.assertEqual({}, formatters)
def test_logger_does_nothing_if_syslog_port_not_defined(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
self.assertEqual({}, handlers)
self.assertEqual({}, formatters)
def test_logger_uses_UDP_by_default(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
syslog_handler = handlers[handlers.keys()[0]]
self.assertEqual(socket.SOCK_DGRAM, syslog_handler['socktype'])
def test_logger_uses_TCP_when_tcpon_is_false(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
conf.set('Logging', 'logtcp', 'off')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
syslog_handler = handlers[handlers.keys()[0]]
self.assertEqual(socket.SOCK_DGRAM, syslog_handler['socktype'])
def test_logger_uses_TCP_when_tcpon_is_true(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
conf.set('Logging', 'logtcp', 'on')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
syslog_handler = handlers[handlers.keys()[0]]
self.assertEqual(socket.SOCK_STREAM, syslog_handler['socktype'])
def test_logger_uses_facility_user_by_default(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
syslog_handler = handlers[handlers.keys()[0]]
self.assertEqual(logging.handlers.SysLogHandler.LOG_USER,
syslog_handler['facility'])
def test_logger_uses_facility_user_when_specified(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
conf.set('Logging', 'logfacility', 'user')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
syslog_handler = handlers[handlers.keys()[0]]
self.assertEqual(logging.handlers.SysLogHandler.LOG_USER,
syslog_handler['facility'])
def test_logger_uses_facility_local0_when_specified(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
conf.set('Logging', 'logfacility', 'local0')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
syslog_handler = handlers[handlers.keys()[0]]
self.assertEqual(logging.handlers.SysLogHandler.LOG_LOCAL0,
syslog_handler['facility'])
def test_logger_raises_exception_on_invalid_facility(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
conf.set('Logging', 'logfacility', 'quidgybo')
self.assertRaises(Exception,
bbpgsql.configuration.set_up_logger_syslog_handler, conf)
@patch('bbpgsql.configuration.set_up_logger_file_handler')
@patch('bbpgsql.configuration.set_up_logger_syslog_handler')
def test_set_up_logging_calls_handler_setups(
self,
mock_set_up_logger_syslog_handler,
mock_set_up_logger_file_handler,
):
mock_set_up_logger_syslog_handler.return_value = ({}, {})
mock_set_up_logger_file_handler.return_value = ({}, {})
log_config = bbpgsql.configuration.set_up_logging(self.full_config)
from pprint import pprint
pprint(log_config)
self.assertEqual(1, mock_set_up_logger_file_handler.call_count)
self.assertEqual(1, mock_set_up_logger_syslog_handler.call_count)
@patch('bbpgsql.configuration.set_up_logger_file_handler')
@patch('bbpgsql.configuration.set_up_logger_syslog_handler')
def test_set_up_logging_does_not_setup_handlers_if_no_logging_section(
self,
mock_set_up_logger_syslog_handler,
mock_set_up_logger_file_handler,
):
bbpgsql.configuration.set_up_logging(self.default_config)
self.assertEqual(0, mock_set_up_logger_file_handler.call_count)
self.assertEqual(0, mock_set_up_logger_syslog_handler.call_count)
class TestConfigReaderTestCase(unittest.TestCase):
def setUp(self):
self.tempdir = TempDirectory()
self.file_path = os.path.join(self.tempdir.path, fake.config_file())
def tearDown(self):
self.tempdir.cleanup()
def _get_config(self):
file_path = self._get_config_file()
config = ConfigReader(file_path)
return config
def _get_config_file(self):
return fake.config_file(self.tempdir.path)
def test_returns_false_if_key_not_set(self):
file_path = self.tempdir.write(fake.config_file(), b'')
with ConfigReader(file_path) as config:
config.set('%name', '2')
config.set('num%ber', '%23')
with self.subTest(0):
def raises_error():
config.set('attr', '%(num')
self.assertRaises(ValueError, raises_error)
with self.subTest(1):
compare(config.get('%name'), 2)
with self.subTest(2):
compare(config.get('num%ber'), '%23')
def test_returns_false_if_filename_not_absolute(self):
with self.subTest(0):
config = ConfigReader(self.file_path)
self.assertTrue(os.path.isabs(config.filename))
with self.subTest(1):
config = ConfigReader()
self.assertTrue(os.path.isabs(config.filename))
with self.subTest(2):
filename = os.path.join(os.getcwd(), 'settings.ini')
config = ConfigReader()
self.assertEqual(filename, config.filename)
config.close()
try:
os.remove(config.filename)
except FileNotFoundError:
pass
def test_returns_false_if_default_name_not_match(self):
self.config = ConfigReader(self.file_path)
expected = self.file_path
self.assertEqual(self.config.filename, expected)
self.config.close()
def test_returns_false_if_name_not_changed(self):
config = self._get_config()
directory = fake.word()
test_dir = self.tempdir.makedir(directory)
path = os.path.join(test_dir, fake.config_file())
config.filename = path
self.assertEqual(config.filename, path)
config.close()
def test_returns_false_if_config_file_not_exists(self):
self.config = ConfigReader(self.file_path)
self.assertFalse(os.path.isfile(self.config.filename))
self.config.close()
def test_returns_false_if_config_file_exists(self):
config = self._get_config()
config.save()
self.assertTrue(os.path.isfile(config.filename))
config.close()
def test_returns_false_if_sections_not_exists(self):
config = self._get_config()
config.set('Sample', 'Example', 'MainSection')
config.set('Sample', 'Example', 'OtherSection')
expected = ['MainSection', 'OtherSection', 'main']
self.assertListEqual(sorted(config.sections), sorted(expected))
config.close()
def test_returns_false_if_section_not_removed(self):
config = self._get_config()
config.set('Sample', 'Example', 'MainSection')
config.set('Sample', 'Example', 'OtherSection')
config.remove_section('main')
expected = ['MainSection', 'OtherSection']
self.assertListEqual(sorted(config.sections), sorted(expected))
config.close()
def test_returns_false_if_key_not_removed(self):
config = self._get_config()
config.set('Sample', 'Example', 'MainSection')
config.set('Sample', 'Example', 'OtherSection')
config.remove_option('Sample', 'MainSection')
with self.subTest(0), self.assertRaises(MissingOptionError):
config.get('Sample', section='MainSection')
with self.subTest(1):
expected = ['MainSection', 'main', 'OtherSection']
self.assertListEqual(sorted(config.sections), sorted(expected))
config.close()
def test_returns_false_if_dict_not_returned(self):
config = self._get_config()
config.set('Sample', 'Example', 'MainSection')
config.set('Sample', 'Example', 'OtherSection')
config.set('Name', 'File', 'OtherSection')
self.assertIsInstance(config.show(output=False), dict)
config.close()
def test_returns_false_if_json_not_dumped(self):
config = self._get_config()
config.set('Sample', 'Example', 'MainSection')
config.set('Sample', 'Example', 'OtherSection')
config.set('name', 'File', 'OtherSection')
config.remove_section('main')
s_io = StringIO()
config.to_json(s_io)
s_io.seek(0)
expected = s_io.read()
self.assertEqual(config.to_json(), expected)
config.close()
def test_returns_false_if_json_file_not_created(self):
config = self._get_config()
directory = fake.word()
test_dir = self.tempdir.makedir(directory)
filename = os.path.join(test_dir, fake.file_name(extension='json'))
with open(filename, 'w') as f:
config.to_json(f)
self.assertTrue(os.path.isfile(filename))
config.close()
def test_returns_false_if_defaults_are_changed(self):
filename = self._get_config_file()
with open(filename, "w") as config_file:
config_file.write('[main]\nnewt = False\n')
config_file.close()
with ConfigReader(filename) as config:
self.assertFalse(config.get('newt'))
def test_returns_false_if_environment_variables_not_set(self):
config = self._get_config()
config.set('country', 'Kenya')
config.set('continent', 'Africa')
config.set('state', 'None')
config.to_env()
with self.subTest(0):
self.assertEqual(os.environ['MAIN_COUNTRY'], 'Kenya')
with self.subTest(1):
self.assertEqual(os.environ['MAIN_CONTINENT'], 'Africa')
with self.subTest(2):
self.assertEqual(os.environ['MAIN_STATE'], 'None')
config.close()
def test_returns_false_if_section_prepend_failed(self):
file_path_1 = self._get_config_file()
file_path_2 = self._get_config_file()
f = open(file_path_1, 'w+')
config = ConfigReader(file_path_2, f)
config.set('country', 'Kenya')
config.set('continent', 'Africa')
config.set('state', 'None')
config.set('count', '0', section='first')
config.to_env()
f.close()
with self.subTest(0):
self.assertEqual(os.environ.get('MAIN_COUNTRY'), 'Kenya')
with self.subTest(1):
self.assertEqual(os.environ.get('MAIN_CONTINENT'), 'Africa')
with self.subTest(2):
self.assertEqual(os.environ.get('MAIN_STATE'), 'None')
with self.subTest(3):
self.assertEqual(os.environ.get('FIRST_COUNT'), '0')
def test_returns_false_if_value_not_evaluated(self):
config = self._get_config()
config.set('truth', 'True')
config.set('empty', '')
config.set('count', '0', section='first')
config.set('None', 'None', section='first')
with self.subTest(0):
self.assertTrue(config.get('truth'))
with self.subTest(1):
self.assertEqual(config.get('empty'), '')
with self.subTest(2), self.assertRaises(MissingOptionError):
config.get('count')
with self.subTest(3):
self.assertEqual(config.get('count', section='first'), 0)
with self.subTest(4):
self.assertIsNone(config.get('None', default='None'))
config.close()
def test_returns_false_if_exception_not_raised(self):
config = self._get_config()
def edit_sections():
config.sections = ['new_section']
self.assertRaises(AttributeError, edit_sections)
config.close()
def test_returns_false_if_threshold_error_not_raised(self):
config = self._get_config()
def do_search(threshold):
config.search('read', threshold=threshold)
with self.subTest(0):
self.assertRaises(ThresholdError, do_search, 1.01)
with self.subTest(1):
self.assertRaises(ThresholdError, do_search, -1.0)
config.close()
def test_returns_false_if_match_not_found(self):
file_path = self.tempdir.write(fake.config_file(), b'')
with ConfigReader(file_path) as config:
with self.subTest(0):
self.assertEqual(config.get_items('main'),
OrderedDict([('reader', 'configreader')]))
with self.subTest(1):
expected = ('reader', 'configreader', 'main')
result = config.search('reader')
self.assertIn(expected, result)
def test_returns_false_if_match_not_found_2(self):
file_path = self.tempdir.write(fake.config_file(), b'[main]')
with ConfigReader(file_path) as config:
with self.subTest(0):
self.assertEqual(config.get_items('main'),
OrderedDict([('reader', 'configreader')]))
with self.subTest(1):
expected = ('reader', 'configreader', 'main')
result = config.search('reader')
self.assertIn(expected, result)
def test_returns_false_if_not_match_best(self):
config = self._get_config()
config.set('header', 'confreader')
expected = (('reader', 'configreader', 'main'), ('header',
'confreader', 'main'))
result = config.search('confgreader')
for index, item in enumerate(expected):
with self.subTest(index):
self.assertIn(item, result)
config.close()
def test_returns_false_if_exact_match_not_found(self):
config = self._get_config()
config.set('title', 'The Place')
expected = ('title', 'The Place', 'main')
result = config.search('The Place', exact_match=True)
self.assertEqual(result, expected)
config.close()
def test_returns_false_if_exact_match_found(self):
config = self._get_config()
config.set('title', 'The Place')
result = config.search('The place', exact_match=True)
self.assertEqual(result, None)
config.close()
def test_returns_false_if_exact_match_not_found_case_insensitive(self):
config = self._get_config()
config.set('title', 'The Place')
expected = ('title', 'The Place', 'main')
result = config.search('The place',
exact_match=True,
case_sensitive=False)
self.assertEqual(result, expected)
config.close()
def test_returns_false_if_path_not_changed(self):
file_path = self.tempdir.write(fake.config_file(), b'[main]')
new_path = os.path.join(self.tempdir.path, fake.config_file())
with open(file_path, 'w+') as f, ConfigReader(file_object=f) as config:
with self.subTest(0):
self.assertTrue(os.path.isfile(file_path))
with self.subTest(1):
self.assertFalse(os.path.isfile(new_path))
config.filename = new_path
with self.subTest(2):
self.assertFalse(os.path.isfile(file_path))
with self.subTest(3):
self.assertFalse(os.path.isfile(new_path))
config.save()
with self.subTest(4):
self.assertTrue(os.path.isfile(new_path))
def test_returns_false_if_old_file_object_writable(self):
file_path = self.tempdir.write(fake.config_file(), b'[main]')
with open(file_path, 'w+') as f, ConfigReader(file_object=f) as config:
new_path = os.path.join(os.path.expanduser('~'),
fake.config_file())
config.filename = new_path
self.assertRaises(ValueError, lambda: f.write(''))
def test_returns_false_if_contents_not_similar(self):
file_path = self.tempdir.write(fake.config_file(), b'[main]')
new_path = os.path.join(self.tempdir.path, fake.config_file())
with open(file_path, 'w+') as f, ConfigReader(file_object=f) as config:
f.seek(0)
expected = f.read()
config.filename = new_path
config.save()
with open(new_path) as f2:
result = f2.read()
self.assertEqual(result, expected)
def test_returns_false_if_file_object_cannot_update(self):
f = open(self.file_path, 'w')
self.assertRaises(ModeError, lambda: ConfigReader(file_object=f))
f.close()
def test_returns_false_if_contents_not_updated(self):
f = open(self.file_path, 'w+')
config = ConfigReader(file_object=f)
config.set('name', 'first')
with self.subTest(0):
with open(self.file_path) as f2:
result = f2.read()
self.assertNotEqual(result, '')
config.save()
with self.subTest(1):
with open(self.file_path) as f3:
result = f3.read()
self.assertNotEqual(result, '')
f.close()
def test_returns_false_if_file_object_and_filename_not_similar(self):
with open(self.file_path, 'w+') as f:
config = ConfigReader(file_object=f)
self.assertEqual(config.filename, f.name)
config.close()
def test_returns_false_if_changes_not_written_to_file(self):
file_path = fake.config_file(self.tempdir.path)
config = ConfigReader(file_path)
config.set('name', 'first')
with ConfigReader(file_path) as d, self.subTest(0):
with self.assertRaises(MissingOptionError):
d.get('name')
config.set('name', 'last', commit=True)
config.close()
d = ConfigReader(file_path)
with self.subTest(1):
self.assertEqual(d.get('name'), 'last')
d.close()
def test_returns_false_if_option_not_removed_from_file(self):
config = self._get_config()
config.set('name', 'first', section='two')
with self.subTest(0):
self.assertEqual(config.get('name', section='two'), 'first')
config.remove_key('name', section='two', commit=True)
with ConfigReader(self.file_path) as d, self.subTest(
1), self.assertRaises(MissingOptionError):
d.get('name', section='two')
config.close()
def test_returns_false_if_file_object_not_closed(self):
config = self._get_config()
config.close()
self.assertRaises(AttributeError, config.set, 'first', 'false')
def test_returns_false_if_items_not_match(self):
file_path_1 = self.tempdir.write(fake.config_file(), b'')
file_path_2 = self.tempdir.write(fake.config_file(), b'')
file_ = open(file_path_1, 'w+')
config = ConfigReader(filename=file_path_2, file_object=file_)
config.remove_section('main')
config.set('start', 'True', section='defaults')
config.set('value', '45', section='defaults')
items = config.get_items('main')
with self.subTest(0):
self.assertIsNone(items)
items = config.get_items('defaults')
with self.subTest(1):
self.assertIsInstance(items, OrderedDict)
with self.subTest(2):
expected = OrderedDict([('start', True), ('value', 45)])
self.assertEqual(items, expected)
config.close()
with self.subTest('Test returns false if file object still writable'):
self.assertRaises(ValueError, file_.write, '')
def test_returns_false_if_error_not_raised(self):
config = ConfigReader()
with self.subTest(0):
self.assertRaises(
SectionNameNotAllowed,
lambda: config.set('start', 'True', section='default'))
with self.subTest(1):
self.assertRaises(
SectionNameNotAllowed,
lambda: config.set('start', 'True', section='deFault'))
with self.subTest(2):
self.assertRaises(
SectionNameNotAllowed,
lambda: config.set('start', 'True', section='DEFAULT'))
def test_returns_false_if_object_not_context(self):
with ConfigReader(self.file_path) as config:
config.set('name', 'First', commit=True)
name = config.get('name')
self.assertEqual(name, 'First')
@unittest.skipIf(os.name == 'nt', 'Path for *NIX systems')
def test_returns_false_if_absolute_path_not_exist(self):
path = '/home/path/does/not/exist'
self.assertRaises(FileNotFoundError, ConfigReader, path)
@unittest.skipIf(os.name != 'nt', 'Path for Windows systems')
def test_returns_false_if_abs_path_not_exist(self):
path = 'C:\\home\\path\\does\\not\\exist'
self.assertRaises(FileNotFoundError, ConfigReader, path)
def test_returns_false_if_default_not_returned(self):
file_path = self.tempdir.write(fake.config_file(), b'')
with ConfigReader(file_path) as config:
value = config.get('members', default='10')
self.assertEqual(value, 10)
def test_returns_false_if_prepend_fails(self):
file_path = self.tempdir.write(fake.config_file(), b'')
with ConfigReader(file_path) as config:
config.set('counter', 'default', section='team')
config.set('play', '1', section='team')
config.to_env()
with self.subTest(0):
self.assertEqual(os.environ['TEAM_COUNTER'], 'default')
with self.subTest(1):
self.assertEqual(os.environ['TEAM_PLAY'], '1')
with self.subTest(2):
self.assertRaises(KeyError, lambda: os.environ['PLAY'])
config.to_env(prepend=False)
with self.subTest(3):
self.assertEqual(os.environ['PLAY'], '1')
with self.subTest(4):
self.assertEqual(os.environ['COUNTER'], 'default')
@unittest.skipIf(os.name == 'nt',
'Testing environment variable for Windows')
def test_returns_false_if_load_fails_linux(self):
file_path = self.tempdir.write(fake.config_file(), b'')
config = ConfigReader(file_path, case_sensitive=True)
config.set('states', '35', section='country')
config.set('counties', 'None', section='country')
environment = os.environ.copy()
user = '******'
environment['USER'] = user
environment['COUNTER'] = 'never'
config.to_env(environment)
config.load_env(environment)
items = config.get_items('main')
with self.subTest(0):
self.assertEqual(items['HOME'], os.path.expanduser('~'))
with self.subTest(1):
self.assertEqual(items['USER'], user)
with self.subTest(2):
self.assertEqual(items['PWD'], os.environ['PWD'])
with self.subTest(3):
self.assertRaises(KeyError, lambda: items['home'])
with self.subTest(4):
self.assertEqual(items['COUNTER'], 'never')
@unittest.skipIf(os.name == 'nt', 'Testing environment variable for Linux')
def test_returns_false_if_load_fails_case_insensitive_linux(self):
config = self._get_config()
config.set('states', '35', section='country')
config.set('counties', 'None', section='country')
config.to_env()
config.load_env()
items = config.get_items('main')
with self.subTest(0):
self.assertEqual(items['home'], os.path.expanduser('~'))
with self.subTest(1):
self.assertEqual(items['user'], os.environ['USER'])
with self.subTest(2):
self.assertEqual(items['pwd'], os.environ['PWD'])
with self.subTest(3):
def call():
return items['PWD']
self.assertRaises(KeyError, call)
@unittest.skipIf(os.name != 'nt',
'Testing environment variable for Windows')
def test_returns_false_if_load_fails_windows(self):
file_path = self.tempdir.write(fake.config_file(), b'')
config = ConfigReader(file_path, case_sensitive=True)
config.set('states', '35', section='country')
config.set('counties', 'None', section='country')
environment = os.environ.copy()
user = '******'
environment['USER'] = user
environment['COUNTER'] = 'never'
config.to_env(environment)
config.load_env(environment)
items = config.get_items('main')
with self.subTest(0):
self.assertEqual(items['APPDATA'], os.environ['APPDATA'])
with self.subTest(1):
self.assertEqual(items['USER'], user)
with self.subTest(2):
self.assertEqual(items['ALLUSERSPROFILE'],
os.environ['ALLUSERSPROFILE'])
with self.subTest(3):
self.assertRaises(KeyError, lambda: items['home'])
with self.subTest(4):
self.assertEqual(items['COUNTER'], 'never')
@unittest.skipIf(os.name != 'nt', 'Testing environment variable for Linux')
def test_returns_false_if_load_fails_case_insensitive_windows(self):
config = self._get_config()
config.set('states', '35', section='country')
config.set('counties', 'None', section='country')
config.to_env()
config.load_env()
items = config.get_items('main')
with self.subTest(0):
self.assertEqual(items['appdata'], os.environ['APPDATA'])
with self.subTest(1):
self.assertEqual(items['allusersprofile'],
os.environ['ALLUSERSPROFILE'])
with self.subTest(2):
self.assertEqual(items['homedrive'], os.environ['HOMEDRIVE'])
with self.subTest(3):
def call():
return items['PWD']
self.assertRaises(KeyError, call)
def test_returns_false_if_load_prefixed_fails(self):
config = self._get_config()
config.set('states', '35', section='countrymain')
config.set('counties', 'None', section='countrymain')
config.to_env()
config.remove_section('countrymain')
with self.subTest(0):
self.assertIsNone(config.get_items('countrymain'))
config.load_env(prefix='countrymain')
items = config.get_items('countrymain')
with self.subTest(1):
self.assertEqual(len(items), 2)
with self.subTest(2):
self.assertEqual(items['states'], 35)
with self.subTest(3):
self.assertIsNone(items['counties'])
def test_returns_false_if_variable_not_expanded(self):
config = self._get_config()
config.set('path', 'drive')
config.set('dir', '%(path)s-directory')
config.set('suffix', 'dir-%(dir)s')
with self.subTest(0):
self.assertEqual(config.get('path'), 'drive')
with self.subTest(1):
self.assertEqual(config.get('dir'), 'drive-directory')
with self.subTest(2):
self.assertEqual(config.get('suffix'), 'dir-drive-directory')
def test_returns_false_if_environment_variables_not_expanded(self):
config = self._get_config()
config.set('path', 'drive', section='test')
config.set('dir', '%(path)s-directory', section='test')
config.set('suffix', 'dir-%(dir)s', section='test')
environment = os.environ.copy()
config.to_env(environment)
with self.subTest(0):
self.assertEqual(environment['TEST_PATH'], 'drive')
with self.subTest(1):
self.assertEqual(environment['TEST_DIR'], 'drive-directory')
with self.subTest(2):
self.assertEqual(environment['TEST_SUFFIX'], 'dir-drive-directory')
def test_returns_false_if_json_not_loaded_from_file(self):
json_file = os.path.join(self.tempdir.path,
fake.file_name(extension='json'))
d = {
'name': 'plannet',
'count': 2,
'skip': False,
'handlers': [{
'name': 'handler_1'
}, {
'name': 'handler_2'
}],
'@counters': {
'start': {
'name': 'scrollers',
'count': 15
},
'mid': {
'name': 'followers',
'count': 0,
'helpers': 'available'
},
'end': {
'name': 'keepers',
'count': 5
}
}
}
try:
j = open(json_file, 'w', encoding='utf-16')
except TypeError:
j = codecs.open(json_file, 'w', encoding='utf-16')
json.dump(d, j, ensure_ascii=False)
else:
json.dump(d, j, ensure_ascii=False)
j.close()
config = self._get_config()
config.load_json(json_file, section='json_data', encoding='utf-16')
with self.subTest(0):
compare(config.get('name', section='json_data'), 'plannet')
with self.subTest(1):
self.assertFalse(config.get('skip', section='json_data'))
with self.subTest(2):
self.assertIsInstance(config.get_items('counters'), OrderedDict)
with self.subTest(3):
compare(
config.get_items('counters'),
OrderedDict([('start', {
'name': 'scrollers',
'count': 15
}),
('mid', {
'name': 'followers',
'count': 0,
'helpers': 'available'
}), ('end', {
'name': 'keepers',
'count': 5
})]))
def test_returns_false_if_option_found(self):
config = self._get_config()
config.set('path', 'drive', section='test')
with self.subTest(0):
compare(config.get('path', 'test'), 'drive')
with self.subTest(1), self.assertRaises(MissingOptionError):
config.get('busy', section='test')
config.close()
def test_returns_false_if_option_with_default_found(self):
config = self._get_config()
config.set('path', 'drive', section='test')
with self.subTest(0):
compare(config.get('path', 'test'), 'drive')
with self.subTest(1):
compare(config.get('busy', section='test', default='not really'),
'not really')
config.close()
@unittest.skipIf(os.name == 'nt',
'Testing environment variables for Linux')
def test_returns_false_if_environment_not_loaded_by_default_linux(self):
file_path = self.tempdir.write(fake.config_file(), b'')
with ConfigReader(file_path) as config:
config.set('home', '$HOME')
config.set('user', '$USER')
with self.subTest(0):
compare(config.get('home'), os.environ['HOME'])
with self.subTest(1):
compare(config.get('shell', default='$SHELL'),
os.environ['SHELL'])
@unittest.skipIf(os.name != 'nt',
'Testing environment variables for Windows')
def test_returns_false_if_environment_not_loaded_by_default_windows(self):
file_path = self.tempdir.write(fake.config_file(), b'')
with ConfigReader(file_path) as config:
config.set('app_data', '$APPDATA')
with self.subTest(0):
compare(config.get('app_data'), os.environ['APPDATA'])
with self.subTest(1):
compare(
config.get('all_users_profile',
default='$ALLUSERSPROFILE'),
os.environ['ALLUSERSPROFILE'])
def test_returns_false_if_reload_fails(self):
file_path = self.tempdir.write(
fake.config_file(),
b'[main]\nreader = configreader\nmain = False\n')
with ConfigReader(file_path) as config:
with self.subTest(0):
compare(
config.get_items('main'),
OrderedDict([('reader', 'configreader'), ('main', False)]))
config.set('found', 'True')
with self.subTest(1):
compare(
config.get_items('main'),
OrderedDict([('reader', 'configreader'), ('main', False),
('found', True)]))
config.reload()
with self.subTest(2):
compare(
config.get_items('main'),
OrderedDict([('reader', 'configreader'), ('main', False)]))
def test_returns_false_if_case_sensistivity_fails(self):
file_path = self.tempdir.write(fake.config_file(), b'')
with ConfigReader(file_path, case_sensitive=True) as config:
config.set('NAME', 'cup')
config.set('LEAGUE', '1')
compare(
config.get_items('main'),
OrderedDict([('reader', 'configreader'), ('NAME', 'cup'),
('LEAGUE', 1)]))
def test_returns_false_if_set_many_fails(self):
file_path = self._get_config_file()
config = ConfigReader(file_path)
config.set_many({'name': 'one', 'number': '2'})
with self.subTest('Test returns false if data not in default section'):
self.assertListEqual(
[config.get('name'), config.get('number')], ['one', 2])
config.reload()
with self.subTest(), self.assertRaises(MissingOptionError):
config.get('number')
data = OrderedDict([('people', 'true'), ('count', 30)])
config.set_many(data, section='demo')
with self.subTest(
'Test returns false if data not in specified section'):
self.assertDictEqual(config.get_items('demo'), data)
config.close(save=True)
with ConfigReader(file_path) as d, self.subTest():
self.assertEqual(d.get('people', section='demo'), 'true')
def test_returns_false_if_option_with_default_not_evaluated(self):
config = self._get_config()
with self.subTest(0):
res = config.get('count', 'test', default=2)
compare(res, 2)
with self.subTest(1):
compare(config.get('counter', 'test', default='20'), 20)
config.close()
def test_returns_false_if_option_with_string_default_not_evaluated(self):
config = self._get_config()
with self.subTest(0):
compare(config.get('count', 'test', default='3'), 3)
with self.subTest(1):
compare(config.get('counter', 'test', default='20'), 20)
config.close()
class Test_archivepgsql_BasicCommandLineOperation(TestCase):
ARCHIVEPGSQL_PATH = os.path.join('bbpgsql', 'cmdline_scripts')
CONFIG_FILE = 'config.ini'
exe_script = 'archivewal'
def setUp(self):
self.setup_environment()
self.setup_config()
self.cmd = [self.exe_script, '--dry-run', '--config', self.config_path]
def setup_environment(self):
self.env = deepcopy(os.environ)
self.env['PATH'] = ''.join([
self.env['PATH'],
':',
self.ARCHIVEPGSQL_PATH])
self.tempdir = TempDirectory()
def setup_config(self):
self.storage_path = self.tempdir.makedir('repo')
self.config_path = self.tempdir.getpath(self.CONFIG_FILE)
self.log_file = self.tempdir.getpath('bbpgsql.log')
self.config_dict = {
'WAL': {
'driver': 'filesystem',
'path': self.storage_path,
},
'General': {
'pgsql_data_directory': self.tempdir.path,
},
'Logging': {
'logfile': self.log_file,
'level': 'DEBUG',
},
}
write_config_to_filename(self.config_dict, self.config_path)
#print '----'
#print open(self.config_path, 'rb').read()
#print '----'
self.pg_xlog_path = 'pg_xlog'
self.tempdir.makedir(self.pg_xlog_path)
self.wal_basename = '00001'
self.wal_filename = os.path.join(self.pg_xlog_path, self.wal_basename)
self.tempdir.write(self.wal_filename, '')
print 'TEMPDIR', self.tempdir.listdir(recursive=True)
def tearDown(self):
self.tempdir.cleanup()
def test_archivewal_returns_error_with_if_less_than_one_argument(self):
proc = Popen(self.cmd, env=self.env, stdout=PIPE, stderr=STDOUT)
proc.wait()
print(proc.stdout.read())
self.assertNotEqual(0, proc.returncode)
def test_archivewal_logs_error_with_if_less_than_one_argument(self):
proc = Popen(self.cmd, env=self.env, stdout=PIPE, stderr=STDOUT)
proc.wait()
self.assertNotEqual(0, proc.returncode)
print proc.stdout.read()
log_output = open(self.log_file, 'rb').read()
print 'log_output:'
print log_output
assert 'ERROR' in log_output
def test_archivewal_success_with_file(self):
self.cmd.append(self.wal_filename)
proc = Popen(self.cmd, env=self.env, stdout=PIPE, stderr=STDOUT)
proc.wait()
print proc.stdout.read()
self.assertEqual(0, proc.returncode)
def test_archivewal_actually_archives_file(self):
self.cmd.append(self.wal_filename)
proc = Popen(self.cmd, env=self.env, stdout=PIPE, stderr=STDOUT)
proc.wait()
print proc.stdout.read()
self.assertEqual(0, proc.returncode)
archives = os.listdir(self.storage_path)
print archives
self.assertTrue(archives[0].startswith(''.join([
self.wal_basename, '_'])))
class TestFilesystemCrawler(unittest.TestCase):
filename = 'foobar.txt'
dirname = 'barfoo/'
file = ''
dir = ''
def setUp(self):
self.tmp = TempDirectory()
self.dir = self.tmp.makedir(self.dirname)
self.file = self.tmp.write(self.filename, b'foo')
def tearDown(self):
TempDirectory.cleanup_all()
def dummyFile(self, filename, expected):
f = self.tmp.write(filename, b'EOF')
crawler = Crawler()
ref = Sourcefile._make(
['', '', expected[0], expected[1], expected[2]])
res = crawler._parseFileName(f)
self.assertEquals(ref[2], res[2])
self.assertEquals(ref[3], res[3])
self.assertEquals(ref[4], res[4])
#silent skip of all non readable dirs
def DISABLEDtest_input_type(self):
crawler = Crawler()
self.assertRaises(AttributeError, crawler.crawl, '404')
#silent skip of all non readable dirs
def DISABLEtest_input_file_permissions(self):
os.chmod(self.file, 0)
crawler = Crawler()
self.assertRaises(PermissionError, crawler._parseFileName, self.file)
def test_empty_pairing(self):
crawler = Crawler()
self.assertRaises(ValueError, crawler.pair)
# Pxx_Eyy_right|left (eGaitDatabase)
def test_S_E_O_format(self):
self.dummyFile('P01_E2_right.dat', ['01', '2', Orientation.right])
# GAxxEyy_right|left (eGaitDatabase3/Controls)
def test_SE_O_format(self):
self.dummyFile('GA112030E3_left.txt', ['112030', '3', Orientation.left])
# PxxEyy_right|left (eGaitDatabase3/Geriatrics)
def test_SE_O_format2(self):
self.dummyFile('P50E6_left.txt', ['50', '6', Orientation.left])
# PatxxWald(left|right)Foot_Sensor_Date (eGaitDatabase3/GeriatricsGSTD/4MW)
def test_SO_format(self):
self.dummyFile('Pat1WaldrightFoot_1eea_20140325133006.dat',
['1Wald', '', Orientation.right])
# GAxx_(Left|Right)Foot_Sensor_Date (eGaitDatabase3/Controls/4MW)
def test_SO_format2(self):
self.dummyFile('GA214026LeftFoot_1eea_20140428110132.dat',
['214026', '', Orientation.left])
# GAstdxx_(Left|Right)Foot_Sensor_Date (eGaitDatabase3/Controls/4MW)
def test_SO_format3(self):
self.dummyFile('GAstd45jwRightFoot_1d68_20140506123332.dat',
['45jw', '', Orientation.right])
def test_correct_pairing(self):
left = self.tmp.write(self.dir + 'P01_E2_left.txt', b'')
right = self.tmp.write(self.dir + 'P01_E2_right.txt', b'')
crawler = Crawler()
crawler.crawl(self.dir)
crawler.pair()
self.assertEquals(crawler.pairedFiles[0][0].filename, 'P01_E2_left.txt')
self.assertEquals(crawler.pairedFiles[0][1].filename, 'P01_E2_right.txt')
def test_one_partner_missing_pairing(self):
left = self.tmp.write(self.dir + 'P01_E2_left.txt', b'')
crawler = Crawler()
crawler.crawl(self.dir)
try:
crawler.pair()
except ValueError:
pass
self.assertEquals(crawler.pairedFiles[0][0].filename,
crawler.pairedFiles[0][1].filename)
class TestPreprocessUtils(unittest.TestCase):
def setUp(self):
"""
Set up a directory with some images to resample
"""
self.tempdir = TempDirectory()
self.temp_path = self.tempdir.path
self.mask_dir = os.path.join(self.temp_path, 'mask_dir')
self.tempdir.makedir('mask_dir')
self.input_dir = os.path.join(self.temp_path, 'input_dir')
self.tempdir.makedir('input_dir')
self.mask_channel = 1
self.slice_idx = 7
self.time_idx = 8
# Mask meta file
self.csv_name = 'mask_image_matchup.csv'
input_meta = aux_utils.make_dataframe()
# Make input meta
for c in range(4):
for p in range(10):
im_name = aux_utils.get_im_name(
channel_idx=c,
slice_idx=self.slice_idx,
time_idx=self.time_idx,
pos_idx=p,
)
input_meta = input_meta.append(
aux_utils.parse_idx_from_name(im_name),
ignore_index=True,
)
input_meta.to_csv(
os.path.join(self.input_dir, 'frames_meta.csv'),
sep=',',
)
# Make mask meta
mask_meta = pd.DataFrame()
for p in range(10):
im_name = aux_utils.get_im_name(
channel_idx=self.mask_channel,
slice_idx=self.slice_idx,
time_idx=self.time_idx,
pos_idx=p,
)
# Indexing can be different
mask_name = 'mask_{}.png'.format(p + 1)
mask_meta = mask_meta.append(
{'mask_name': mask_name, 'file_name': im_name},
ignore_index=True,
)
mask_meta.to_csv(
os.path.join(self.mask_dir, self.csv_name),
sep=',',
)
def tearDown(self):
"""
Tear down temporary folder and file structure
"""
TempDirectory.cleanup_all()
nose.tools.assert_equal(os.path.isdir(self.temp_path), False)
def test_validate_mask_meta(self):
mask_channel = preprocess_utils.validate_mask_meta(
mask_dir=self.mask_dir,
input_dir=self.input_dir,
csv_name=self.csv_name,
mask_channel=50,
)
self.assertEqual(mask_channel, 50)
out_meta = aux_utils.read_meta(self.mask_dir)
for i, row in out_meta.iterrows():
self.assertEqual(row.slice_idx, self.slice_idx)
self.assertEqual(row.time_idx, self.time_idx)
self.assertEqual(row.channel_idx, 50)
self.assertEqual(row.pos_idx, i)
self.assertEqual(row.file_name, "mask_{}.png".format(i + 1))
def test_validate_mask_meta_no_channel(self):
mask_channel = preprocess_utils.validate_mask_meta(
mask_dir=self.mask_dir,
input_dir=self.input_dir,
csv_name=self.csv_name,
)
self.assertEqual(mask_channel, 4)
out_meta = aux_utils.read_meta(self.mask_dir)
for i, row in out_meta.iterrows():
self.assertEqual(row.slice_idx, self.slice_idx)
self.assertEqual(row.time_idx, self.time_idx)
self.assertEqual(row.channel_idx, 4)
self.assertEqual(row.pos_idx, i)
self.assertEqual(row.file_name, "mask_{}.png".format(i + 1))
@nose.tools.raises(AssertionError)
def test_validate_mask_meta_mask_channel(self):
# Same channel as image
preprocess_utils.validate_mask_meta(
mask_dir=self.mask_dir,
input_dir=self.input_dir,
csv_name=self.csv_name,
mask_channel=1,
)
@nose.tools.raises(IOError)
def test_validate_mask_meta_wrong_dir(self):
preprocess_utils.validate_mask_meta(
mask_dir=self.temp_path,
input_dir=self.input_dir,
)
class TestImageInference3D(unittest.TestCase):
@patch('micro_dl.inference.model_inference.load_model')
def setUp(self, mock_model):
"""
Set up a directory with 3D images
"""
mock_model.return_value = 'dummy_model'
self.tempdir = TempDirectory()
self.temp_path = self.tempdir.path
self.tempdir.makedir('image_dir')
self.tempdir.makedir('mask_dir')
self.tempdir.makedir('model_dir')
self.image_dir = os.path.join(self.temp_path, 'image_dir')
self.mask_dir = os.path.join(self.temp_path, 'mask_dir')
self.model_dir = os.path.join(self.temp_path, 'model_dir')
# Create a temp image dir
self.im = np.zeros((10, 10, 8), dtype=np.uint8)
self.frames_meta = aux_utils.make_dataframe()
self.time_idx = 2
self.slice_idx = 0
for p in range(5):
for c in range(3):
im_name = aux_utils.get_im_name(
time_idx=self.time_idx,
channel_idx=c,
slice_idx=self.slice_idx,
pos_idx=p,
ext='.npy',
)
np.save(os.path.join(self.image_dir, im_name),
self.im + c * 10,
allow_pickle=True,
fix_imports=True)
self.frames_meta = self.frames_meta.append(
aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES),
ignore_index=True,
)
# Write frames meta to image dir too
self.frames_meta.to_csv(os.path.join(self.image_dir,
'frames_meta.csv'))
# Save masks and mask meta
self.mask_meta = aux_utils.make_dataframe()
self.mask_channel = 50
# Mask half the image
mask = np.zeros_like(self.im)
mask[:5, ...] = 1
for p in range(5):
im_name = aux_utils.get_im_name(
time_idx=self.time_idx,
channel_idx=self.mask_channel,
slice_idx=self.slice_idx,
pos_idx=p,
ext='.npy',
)
np.save(os.path.join(self.mask_dir, im_name), mask)
self.mask_meta = self.mask_meta.append(
aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES),
ignore_index=True,
)
# Write frames meta to mask dir too
self.mask_meta.to_csv(os.path.join(self.mask_dir, 'frames_meta.csv'))
# Setup model dir
split_samples = {
"train": [0, 1],
"val": [2],
"test": [3, 4],
}
aux_utils.write_json(
split_samples,
os.path.join(self.model_dir, 'split_samples.json'),
)
# Make configs with fields necessary for 2.5D segmentation inference
self.train_config = {
'network': {
'class': 'UNet3D',
'data_format': 'channels_first',
'num_filters_per_block': [8, 16],
'depth': 5,
'width': 5,
'height': 5
},
'dataset': {
'split_by_column': 'pos_idx',
'input_channels': [1],
'target_channels': [2],
'model_task': 'regression',
},
}
self.inference_config = {
'model_dir': self.model_dir,
'model_fname': 'dummy_weights.hdf5',
'image_dir': self.image_dir,
'data_split': 'test',
'images': {
'image_format': 'zyx',
'image_ext': '.png',
},
'metrics': {
'metrics': ['mse'],
'metrics_orientations': ['xyz'],
},
'masks': {
'mask_dir': self.mask_dir,
'mask_type': 'metrics',
'mask_channel': 50,
},
'inference_3d': {
'tile_shape': [5, 5, 5],
'num_overlap': [1, 1, 1],
'overlap_operation': 'mean',
},
}
# Instantiate class
self.infer_inst = image_inference.ImagePredictor(
train_config=self.train_config,
inference_config=self.inference_config,
)
def tearDown(self):
"""
Tear down temporary folder and file structure
"""
TempDirectory.cleanup_all()
self.assertEqual(os.path.isdir(self.temp_path), False)
def test_init(self):
"""
Test init of inference dataset
"""
# Check proper init
self.assertEqual(self.infer_inst.model_dir, self.model_dir)
self.assertEqual(self.infer_inst.image_dir, self.image_dir)
self.assertEqual(self.infer_inst.data_format, 'channels_first')
self.assertEqual(self.infer_inst.model, 'dummy_model')
self.assertEqual(self.infer_inst.image_format, 'zyx')
self.assertEqual(self.infer_inst.image_ext, '.npy')
self.assertTrue(self.infer_inst.mask_metrics)
self.assertEqual(self.infer_inst.mask_dir, self.mask_dir)
self.assertListEqual(self.infer_inst.metrics_orientations, ['xyz'])
self.assertIsNone(self.infer_inst.crop_shape)
self.assertEqual(self.infer_inst.z_dim, 2)
self.assertEqual(self.infer_inst.tile_option, 'tile_xyz')
self.assertListEqual(self.infer_inst.num_overlap, [1, 1, 1])
def test_assign_3d_inference(self):
# Test other settings
self.infer_inst.params_3d = {
'num_slices': 5,
'num_overlap': 1,
'overlap_operation': 'mean',
}
self.infer_inst._assign_3d_inference()
self.assertEqual(self.infer_inst.z_dim, 2)
self.assertEqual(self.infer_inst.tile_option, 'tile_z')
self.assertEqual(self.infer_inst.num_overlap, 1)
def test_assign_3d_inference_xyz(self):
# Test other settings
self.infer_inst.params_3d = {
'num_slices': 5,
'num_overlap': 1,
'overlap_operation': 'mean',
}
self.infer_inst.image_format = 'xyz'
self.infer_inst._assign_3d_inference()
self.assertEqual(self.infer_inst.z_dim, 4)
self.assertEqual(self.infer_inst.tile_option, 'tile_z')
self.assertEqual(self.infer_inst.num_overlap, 1)
@nose.tools.raises(AssertionError)
def test_assign_3d_inference_few_slices(self):
# Test other settings
self.infer_inst.params_3d = {
'num_slices': 3,
'num_overlap': 1,
'overlap_operation': 'mean',
}
self.infer_inst._assign_3d_inference()
@nose.tools.raises(AssertionError)
def test_assign_3d_inference_not_3d(self):
self.infer_inst.params_3d = {
'num_slices': 5,
'num_overlap': 1,
'overlap_operation': 'mean',
}
self.infer_inst.config['network']['class'] = 'UNet2D'
self.infer_inst._assign_3d_inference()
def test_assign_3d_inference_on_center(self):
self.infer_inst.params_3d = {
'inf_shape': [5, 5, 5],
'num_overlap': 1,
'overlap_operation': 'mean',
}
self.infer_inst._assign_3d_inference()
self.assertEqual(self.infer_inst.tile_option, 'infer_on_center')
self.assertEqual(self.infer_inst.num_overlap, 0)
def test_get_sub_block_z(self):
# 3D image for prediction should have channel and batch dim
im = np.zeros((1, 2, 8, 10, 10), dtype=np.float32)
block = self.infer_inst._get_sub_block_z(
input_image=im,
start_z_idx=2,
end_z_idx=5,
)
self.assertTupleEqual(block.shape, (1, 2, 3, 10, 10))
def test_get_sub_block_z_channels_last(self):
self.infer_inst.data_format = 'channels_last'
im = np.zeros((1, 8, 10, 10, 2), dtype=np.float32)
block = self.infer_inst._get_sub_block_z(
input_image=im,
start_z_idx=2,
end_z_idx=5,
)
self.assertTupleEqual(block.shape, (1, 3, 10, 10, 2))
def test_get_sub_block_z_xyz(self):
self.infer_inst.image_format = 'xyz'
im = np.zeros((1, 2, 10, 10, 8), dtype=np.float32)
block = self.infer_inst._get_sub_block_z(
input_image=im,
start_z_idx=2,
end_z_idx=5,
)
self.assertTupleEqual(block.shape, (1, 2, 10, 10, 3))
def test_get_sub_block_z_xyz_channels_last(self):
self.infer_inst.image_format = 'xyz'
self.infer_inst.data_format = 'channels_last'
im = np.zeros((1, 10, 10, 8, 2), dtype=np.float32)
block = self.infer_inst._get_sub_block_z(
input_image=im,
start_z_idx=2,
end_z_idx=5,
)
self.assertTupleEqual(block.shape, (1, 10, 10, 3, 2))
@patch('micro_dl.inference.model_inference.predict_large_image')
def test_predict_sub_block_z(self, mock_predict):
mock_predict.return_value = np.zeros((1, 1, 5, 10, 10),
dtype=np.float32)
self.infer_inst.params_3d = {
'num_slices': 5,
'num_overlap': 1,
'overlap_operation': 'mean',
}
self.infer_inst.num_overlap = 1
im = np.zeros((1, 1, 8, 10, 10), dtype=np.float32)
pred_ims, start_end_idx = self.infer_inst._predict_sub_block_z(
input_image=im, )
# z = 5 slices and 1 overlap, two prediction of im can fit
self.assertEqual(len(pred_ims), 2)
self.assertEqual(len(start_end_idx), 2)
# Z locations (out of 0-8)
self.assertTupleEqual(start_end_idx[0], (0, 5))
self.assertTupleEqual(start_end_idx[1], (3, 8))
@patch('micro_dl.inference.model_inference.predict_large_image')
def test_predict_sub_block_xyz(self, mock_predict):
mock_predict.return_value = np.zeros((1, 1, 5, 5, 5), dtype=np.float32)
self.infer_inst.num_overlap = 1
im = np.zeros((1, 1, 8, 10, 10), dtype=np.float32)
pred_ims = self.infer_inst._predict_sub_block_xyz(
input_image=im,
crop_indices=[(0, 5, 0, 5, 0, 5), (3, 8, 0, 5, 0, 5)],
)
self.assertEqual(len(pred_ims), 2)
@patch('micro_dl.inference.model_inference.predict_large_image')
def test_predict_sub_block_xyz_channels_last(self, mock_predict):
mock_predict.return_value = np.zeros((1, 1, 5, 5, 5), dtype=np.float32)
self.infer_inst.num_overlap = 1
im = np.zeros((1, 8, 10, 10, 1), dtype=np.float32)
pred_ims = self.infer_inst._predict_sub_block_xyz(
input_image=im,
crop_indices=[(3, 8, 0, 5, 0, 5)],
)
self.assertEqual(len(pred_ims), 1)
@patch('micro_dl.inference.model_inference.predict_large_image')
def test_predict_3d(self, mock_predict):
mock_predict.return_value = np.zeros((1, 1, 5, 5, 5), dtype=np.float32)
# Predict row 0 from inference dataset iterator
pred_im, target_im, mask_im = self.infer_inst.predict_3d([0])
self.assertTupleEqual(pred_im.shape, (8, 8, 8))
self.assertEqual(pred_im.dtype, np.float32)
self.assertTupleEqual(target_im.shape, (8, 8, 8))
self.assertEqual(target_im.dtype, np.float32)
self.assertTupleEqual(mask_im.shape, (8, 8, 8))
self.assertEqual(mask_im.dtype, np.uint8)
# Read saved prediction, z=0 target channel=2
pred_name = os.path.join(
self.model_dir,
'predictions/im_c002_z000_t002_p003.npy',
)
im_pred = np.load(pred_name)
self.assertEqual(im_pred.dtype, np.float32)
self.assertTupleEqual(im_pred.shape, (8, 8, 8))
@patch('micro_dl.inference.model_inference.predict_large_image')
def test_predict_3d_on_center(self, mock_predict):
mock_predict.return_value = np.zeros((1, 1, 3, 3, 3), dtype=np.float32)
self.infer_inst.crop_shape = [5, 5, 5]
self.infer_inst.tile_option = 'infer_on_center'
self.infer_inst.params_3d['inf_shape'] = [3, 3, 3]
# Predict row 0 from inference dataset iterator
pred_im, target_im, mask_im = self.infer_inst.predict_3d([0])
self.assertTupleEqual(pred_im.shape, (3, 3, 3))
self.assertEqual(pred_im.dtype, np.float32)
self.assertTupleEqual(target_im.shape, (3, 3, 3))
self.assertEqual(target_im.dtype, np.float32)
self.assertTupleEqual(mask_im.shape, (3, 3, 3))
self.assertEqual(mask_im.dtype, np.uint8)
# Read saved prediction, z=0 target channel=2
pred_name = os.path.join(
self.model_dir,
'predictions/im_c002_z000_t002_p003.npy',
)
im_pred = np.load(pred_name)
self.assertEqual(im_pred.dtype, np.float32)
self.assertTupleEqual(im_pred.shape, (3, 3, 3))
@patch('micro_dl.inference.model_inference.predict_large_image')
def test_run_prediction(self, mock_predict):
mock_predict.return_value = np.zeros((1, 1, 5, 5, 5), dtype=np.float32)
# Run prediction. Should create a metrics_xy.csv in pred dir
self.infer_inst.run_prediction()
metrics = pd.read_csv(
os.path.join(self.model_dir, 'predictions/metrics_xyz.csv'))
self.assertTupleEqual(metrics.shape, (2, 4))
# MSE should be 0.
self.assertEqual(metrics.mse.mean(), 0.0)
self.assertEqual(metrics.mse_masked.mean(), 0.0)
# Mask volume fraction should be 0.5
self.assertEqual(metrics.vol_frac.mean(), 0.5)
# # There should be four rows, one per test index pos=3,4
self.assertEqual(metrics.pred_name[0], 'im_c002_z000_t002_p003')
self.assertEqual(metrics.pred_name[1], 'im_c002_z000_t002_p004')
# # There should be 2 predictions saved in pred dir, target c=2
for p in [3, 4]:
pred_name = os.path.join(
self.model_dir,
'predictions/im_c002_z000_t002_p00{}.npy'.format(p),
)
im_pred = np.load(pred_name)
self.assertEqual(im_pred.dtype, np.float32)
self.assertTupleEqual(im_pred.shape, (8, 8, 8))
class TestImageInference2p5D(unittest.TestCase):
@patch('micro_dl.inference.model_inference.load_model')
def setUp(self, mock_model):
"""
Set up a directory with images
"""
mock_model.return_value = 'dummy_model'
self.tempdir = TempDirectory()
self.temp_path = self.tempdir.path
self.tempdir.makedir('image_dir')
self.tempdir.makedir('mask_dir')
self.tempdir.makedir('model_dir')
self.image_dir = os.path.join(self.temp_path, 'image_dir')
self.mask_dir = os.path.join(self.temp_path, 'mask_dir')
self.model_dir = os.path.join(self.temp_path, 'model_dir')
# Create a temp image dir
self.im = np.zeros((10, 16), dtype=np.uint8)
self.frames_meta = aux_utils.make_dataframe()
self.time_idx = 2
for p in range(5):
for c in range(3):
for z in range(6):
im_name = aux_utils.get_im_name(
time_idx=self.time_idx,
channel_idx=c,
slice_idx=z,
pos_idx=p,
)
cv2.imwrite(os.path.join(self.image_dir, im_name),
self.im + c * 10)
self.frames_meta = self.frames_meta.append(
aux_utils.parse_idx_from_name(im_name,
aux_utils.DF_NAMES),
ignore_index=True,
)
# Write frames meta to image dir too
self.frames_meta.to_csv(os.path.join(self.image_dir,
'frames_meta.csv'))
# Save masks and mask meta
self.mask_meta = aux_utils.make_dataframe()
self.mask_channel = 50
for p in range(5):
for z in range(6):
im_name = aux_utils.get_im_name(
time_idx=self.time_idx,
channel_idx=self.mask_channel,
slice_idx=z,
pos_idx=p,
)
cv2.imwrite(os.path.join(self.mask_dir, im_name), self.im + 1)
self.mask_meta = self.mask_meta.append(
aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES),
ignore_index=True,
)
# Write frames meta to mask dir too
self.mask_meta.to_csv(os.path.join(self.mask_dir, 'frames_meta.csv'))
# Setup model dir
split_samples = {
"train": [0, 1],
"val": [2],
"test": [3, 4],
}
aux_utils.write_json(
split_samples,
os.path.join(self.model_dir, 'split_samples.json'),
)
# Make configs with fields necessary for 2.5D segmentation inference
self.train_config = {
'network': {
'class': 'UNetStackTo2D',
'data_format': 'channels_first',
'depth': 5,
'width': 10,
'height': 10
},
'dataset': {
'split_by_column': 'pos_idx',
'input_channels': [1],
'target_channels': [self.mask_channel],
'model_task': 'segmentation',
},
}
self.inference_config = {
'model_dir': self.model_dir,
'model_fname': 'dummy_weights.hdf5',
'image_dir': self.image_dir,
'data_split': 'test',
'images': {
'image_format': 'zyx',
'image_ext': '.png',
},
'metrics': {
'metrics': ['dice'],
'metrics_orientations': ['xy'],
},
'masks': {
'mask_dir': self.mask_dir,
'mask_type': 'target',
'mask_channel': 50,
}
}
# Instantiate class
self.infer_inst = image_inference.ImagePredictor(
train_config=self.train_config,
inference_config=self.inference_config,
)
def tearDown(self):
"""
Tear down temporary folder and file structure
"""
TempDirectory.cleanup_all()
self.assertEqual(os.path.isdir(self.temp_path), False)
def test_init(self):
"""
Test init of inference dataset
"""
# Check proper init
self.assertEqual(self.infer_inst.model_dir, self.model_dir)
self.assertEqual(self.infer_inst.image_dir, self.image_dir)
self.assertEqual(self.infer_inst.data_format, 'channels_first')
self.assertEqual(self.infer_inst.model, 'dummy_model')
self.assertEqual(self.infer_inst.image_format, 'zyx')
self.assertEqual(self.infer_inst.image_ext, '.png')
self.assertFalse(self.infer_inst.mask_metrics)
self.assertEqual(self.infer_inst.mask_dir, self.mask_dir)
self.assertListEqual(self.infer_inst.metrics_orientations, ['xy'])
self.assertEqual(self.infer_inst.num_overlap, 0)
self.assertIsNone(self.infer_inst.stitch_inst)
self.assertIsNone(self.infer_inst.tile_option)
self.assertIsNone(self.infer_inst.crop_shape)
@patch('micro_dl.inference.model_inference.predict_large_image')
def test_predict_2d(self, mock_predict):
mock_predict.return_value = 1. + np.ones(
(1, 1, 1, 8, 16), dtype=np.float32)
# Predict row 0 from inference dataset iterator
pred_im, target_im, mask_im = self.infer_inst.predict_2d([0])
self.assertTupleEqual(pred_im.shape, (8, 16, 1))
self.assertEqual(pred_im.dtype, np.float32)
self.assertEqual(pred_im.max(), 2.0)
# Read saved prediction, z=2 for first slice with depth=5
pred_name = os.path.join(
self.model_dir,
'predictions/im_c050_z002_t002_p003.png',
)
im_pred = cv2.imread(pred_name, cv2.IMREAD_ANYDEPTH)
self.assertEqual(im_pred.dtype, np.uint16)
self.assertTupleEqual(im_pred.shape, (8, 16))
# Check target and no mask
self.assertTupleEqual(target_im.shape, (8, 16, 1))
self.assertEqual(target_im.dtype, np.float32)
self.assertEqual(target_im.max(), 1.)
self.assertListEqual(mask_im, [])
@patch('micro_dl.inference.model_inference.predict_large_image')
def test_run_prediction(self, mock_predict):
mock_predict.return_value = 1. + np.ones(
(1, 1, 1, 8, 16), dtype=np.float32)
# Run prediction. Should create a metrics_xy.csv in pred dir
self.infer_inst.run_prediction()
metrics = pd.read_csv(
os.path.join(self.model_dir, 'predictions/metrics_xy.csv'))
self.assertTupleEqual(metrics.shape, (4, 2))
# Dice should be 1.
self.assertEqual(metrics.dice.mean(), 1.0)
# There should be four rows, one per test index pos=3,4
# depth=5 means center slices z=2,3 will be evaluated
self.assertEqual(metrics.pred_name[0], 'im_c050_z002_t002_p003_xy0')
self.assertEqual(metrics.pred_name[1], 'im_c050_z003_t002_p003_xy0')
self.assertEqual(metrics.pred_name[2], 'im_c050_z002_t002_p004_xy0')
self.assertEqual(metrics.pred_name[3], 'im_c050_z003_t002_p004_xy0')
# There should be 4 predictions saved in pred dir
for p in [3, 4]:
for z in [2, 3]:
pred_name = os.path.join(
self.model_dir,
'predictions/im_c050_z00{}_t002_p00{}.png'.format(z, p),
)
im_pred = cv2.imread(pred_name, cv2.IMREAD_ANYDEPTH)
self.assertEqual(im_pred.dtype, np.uint16)
self.assertTupleEqual(im_pred.shape, (8, 16))
class PluginWithTempDirTests(TestCase):
def setUp(self):
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
def run_actions(self, path=None, **kw):
with LogCapture() as log:
plugin = make_git_repo(path=path or self.dir.path, **kw)
with Replacer() as r:
r.replace("archivist.repos.git.datetime", test_datetime())
plugin.actions()
return log
def git(self, command, repo_path=None):
return run(["git"] + command.split(), cwd=repo_path or self.dir.path)
def make_repo_with_content(self, repo=""):
repo_path = self.dir.getpath(repo) if repo else None
self.git("init", repo_path)
self.dir.write(repo + "a", "some content")
self.dir.write(repo + "b", "other content")
self.dir.write(repo + "c", "more content")
self.git("add .", repo_path)
self.git("commit -m initial", repo_path)
return repo
def make_local_changes(self, repo=""):
self.dir.write(repo + "b", "changed content")
os.remove(self.dir.getpath(repo + "c"))
self.dir.write(repo + "d", "new content")
def status_log_entry(self, lines, repo_path=None):
return (
"archivist.repos.git",
"INFO",
"\n".join(l.format(repo=repo_path or self.dir.path) for l in lines) + "\n",
)
def check_git_log(self, lines, repo_path=None):
compare("\n".join(lines) + "\n", self.git("log --pretty=format:%s --stat", repo_path))
def get_dummy_source(self, name):
class DummySource(Source):
schema = Schema({})
def __init__(self, type, name, repo):
super(DummySource, self).__init__(type, name, repo)
def process(self, path):
pass
return DummySource("dummy", name, "repo")
def test_path_for_with_name(self):
compare(
self.dir.getpath("dummy/the_name"),
make_git_repo(path=self.dir.path).path_for(self.get_dummy_source("the_name")),
)
self.assertTrue(os.path.exists(self.dir.getpath("dummy/the_name")))
def test_path_for_no_name(self):
compare(self.dir.getpath("dummy"), make_git_repo(path=self.dir.path).path_for(self.get_dummy_source(name=None)))
self.assertTrue(os.path.exists(self.dir.getpath("dummy")))
def test_not_there(self):
repo_path = self.dir.getpath("var")
log = self.run_actions(repo_path)
log.check(("archivist.repos.git", "INFO", "creating git repo at " + repo_path))
self.assertTrue(self.dir.getpath("var/.git"))
def test_there_not_git(self):
repo_path = self.dir.makedir("var")
log = self.run_actions(repo_path)
log.check(("archivist.repos.git", "INFO", "creating git repo at " + repo_path))
self.assertTrue(self.dir.getpath("var/.git"))
def test_no_changes(self):
self.git("init")
log = self.run_actions()
log.check() # no logging
def test_just_log_changes(self):
self.make_repo_with_content()
self.make_local_changes()
log = self.run_actions(commit=False)
log.check(self.status_log_entry(["changes found in git repo at {repo}:", " M b", " D c", "?? d"]))
self.check_git_log(["initial", " a | 1 +", " b | 1 +", " c | 1 +", " 3 files changed, 3 insertions(+)"])
def test_commit_changes(self):
self.make_repo_with_content()
self.make_local_changes()
log = self.run_actions(commit=True)
log.check(
self.status_log_entry(["changes found in git repo at {repo}:", " M b", " D c", "?? d"]),
("archivist.repos.git", "INFO", "changes committed"),
)
compare("", self.git("status --porcelain"))
self.check_git_log(
[
"Recorded by archivist at 2001-01-01 00:00",
" b | 2 +-",
" c | 1 -",
" d | 1 +",
" 3 files changed, 2 insertions(+), 2 deletions(-)",
"",
"initial",
" a | 1 +",
" b | 1 +",
" c | 1 +",
" 3 files changed, 3 insertions(+)",
]
)
def test_push_changes(self):
origin_path = self.dir.makedir("origin")
self.make_repo_with_content(repo="origin/")
self.git("config --local --add receive.denyCurrentBranch ignore", origin_path)
self.git("clone -q " + origin_path + " local")
self.make_local_changes(repo="local/")
local_path = self.dir.getpath("local")
log = self.run_actions(commit=True, push=True, path=local_path)
log.check(
self.status_log_entry(
["changes found in git repo at {repo}:", " M b", " D c", "?? d"], repo_path=local_path
),
("archivist.repos.git", "INFO", "changes committed"),
("archivist.repos.git", "INFO", "changes pushed"),
)
self.check_git_log(
[
"Recorded by archivist at 2001-01-01 00:00",
" b | 2 +-",
" c | 1 -",
" d | 1 +",
" 3 files changed, 2 insertions(+), 2 deletions(-)",
"",
"initial",
" a | 1 +",
" b | 1 +",
" c | 1 +",
" 3 files changed, 3 insertions(+)",
],
repo_path=origin_path,
)
def test_push_no_changes(self):
self.git("init")
log = self.run_actions(commit=True, push=True)
log.check() # no logging
def test_default_repo_config(self):
# can't test actions due to default path
GitRepo(**GitRepo.schema(default_repo_config))
def test_init(self, tmpdir: TempDirectory):
tmpdir.makedir('foo')
Git(tmpdir.getpath('foo')).init()
assert os.path.exists(tmpdir.getpath('foo/.git'))
class TestFileTestCase(unittest.TestCase):
def setUp(self):
self.tempdir = TempDirectory(encoding='utf-8')
def tearDown(self):
self.tempdir.cleanup()
def test_returns_false_if_parent_attributes_not_set(self):
d = File(self.tempdir.path)
with self.subTest(1):
compare(self.tempdir.path, d.path)
with self.subTest(2):
compare(os.path.basename(self.tempdir.path), d.name)
with self.subTest(3):
compare(os.path.dirname(self.tempdir.path), d.parent)
def test_returns_false_if_path_change_is_relative(self):
d = File(self.tempdir.path)
with self.subTest(1):
compare(self.tempdir.path, d.path)
if os.name != 'nt':
with self.subTest(2):
compare(False, d.hidden_path)
with self.subTest(3):
compare('UNDEFINED', d.default_category)
def call(x):
d.path = x
self.assertRaises(RelativePathError, call, '.')
def test_returns_false_if_attributes_not_set(self):
file_path = self.tempdir.write('123.txt', '')
d = File(file_path)
with self.subTest(1):
compare(file_path, d.path)
if os.name != 'nt':
with self.subTest(2):
compare(False, d.hidden_path)
with self.subTest(3):
compare(os.path.basename(file_path), d.name)
with self.subTest(4):
compare(True, d.exists)
with self.subTest(5):
compare('document', d.category)
with self.subTest(6):
compare('txt', d.extension)
with self.subTest(7):
compare(d.path, str(d))
def test_returns_false_if_names_not_match(self):
initial_file = self.tempdir.write('123.txt', '')
for i in range(1, 10):
filename = '123 - dup ({}).txt'.format(str(i))
d = File(initial_file)
with self.subTest(i):
compare(filename, d.find_suitable_name(initial_file))
self.tempdir.write(os.path.join(self.tempdir.path, filename), '')
def test_returns_false_if_pathlib_methods_fail(self):
path = self.tempdir.makedir('abc/fig/')
file_ = self.tempdir.write(os.path.join(path, 'my awesome cat.txt'),
'')
d = File(file_)
with self.subTest(1):
compare('.txt', d.suffix)
with self.subTest(2):
compare('my awesome cat', d.stem)
def test_returns_false_if_file_moving_fails(self):
dir_1 = self.tempdir.makedir('abc/fig/')
dir_2 = self.tempdir.makedir('one/twp/')
dir_3 = self.tempdir.makedir('some/dir/')
file_ = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'),
'')
f = File(file_)
f.move_to(dir_2, group=False)
with self.subTest(1):
compare(os.path.join(dir_2, 'TXT', 'my awesome cat.txt'), f.path)
with self.subTest(2):
compare(True, f.exists)
f.move_to(dir_3, group=True, by_extension=True)
with self.subTest(3):
compare(
os.path.join(dir_3, 'document', 'TXT', 'my awesome cat.txt'),
f.path)
with self.subTest(4):
compare(True, f.exists)
with self.subTest(5):
compare(
False,
os.path.isfile(os.path.join(dir_2, 'TXT',
'my awesome cat.txt')))
def test_returns_false_if_grouping_by_category_fails_group_false(self):
dir_1 = self.tempdir.makedir('abc/fig/')
dir_2 = self.tempdir.makedir('one/twp/')
dir_3 = self.tempdir.makedir('some/dir/')
file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'),
'')
self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '')
self.tempdir.write(os.path.join(dir_1, '1.jpeg'), '')
f1 = File(file_1)
f1.move_to(dst_root_path=dir_2,
group=False,
by_extension=False,
group_folder_name=None) # default
with self.subTest(1):
final_path = os.path.join(dir_2, 'TXT', 'my awesome cat.txt')
compare(final_path, f1.path)
f1.move_to(dst_root_path=dir_3,
group=False,
by_extension=False,
group_folder_name=None) # default
with self.subTest(1):
final_path = os.path.join(dir_3, 'TXT', 'my awesome cat.txt')
compare(final_path, f1.path)
def test_returns_false_if_grouping_fails_group_true(self):
dir_1 = self.tempdir.makedir('abc/fig/')
dir_2 = self.tempdir.makedir('one/twp/')
self.tempdir.makedir('some/dir/')
file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'),
'')
self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '')
self.tempdir.write(os.path.join(dir_1, '1.jpeg'), '')
f1 = File(file_1)
f1.move_to(dst_root_path=dir_2,
group=True,
by_extension=False,
group_folder_name=None)
with self.subTest(1):
final_path = os.path.join(dir_2, 'document', 'my awesome cat.txt')
compare(final_path, f1.path)
def test_returns_false_if_grouping_fails_by_extension_true_group_folder_name_none(
self):
dir_1 = self.tempdir.makedir('abc/fig/')
dir_2 = self.tempdir.makedir('one/twp/')
self.tempdir.makedir('some/dir/')
file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'),
'')
self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '')
self.tempdir.write(os.path.join(dir_1, '1.jpeg'), '')
f1 = File(file_1)
f1.move_to(dst_root_path=dir_2,
group=True,
by_extension=True,
group_folder_name=None)
with self.subTest(1):
final_path = os.path.join(dir_2, 'document', 'TXT',
'my awesome cat.txt')
compare(final_path, f1.path)
def test_returns_false_if_grouping_fails_by_extension_true_group_folder_name_given(
self):
dir_1 = self.tempdir.makedir('abc/fig/')
dir_2 = self.tempdir.makedir('one/twp/')
self.tempdir.makedir('some/dir/')
file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'),
'')
self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '')
self.tempdir.write(os.path.join(dir_1, '1.jpeg'), '')
f1 = File(file_1)
f1.move_to(dst_root_path=dir_2,
group=True,
by_extension=True,
group_folder_name='sample dir')
with self.subTest(1):
final_path = os.path.join(dir_2, 'sample dir', 'TXT',
'my awesome cat.txt')
compare(final_path, f1.path)
def test_returns_false_if_grouping_fails_by_extension_false_group_folder_name_given(
self):
dir_1 = self.tempdir.makedir('abc/fig/')
dir_2 = self.tempdir.makedir('one/twp/')
dir_3 = self.tempdir.makedir('some/dir/')
file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'),
'')
file_2 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'),
'')
file_3 = self.tempdir.write(os.path.join(dir_1, '1.jpeg'), '')
f1 = File(file_1)
f1.move_to(dst_root_path=dir_2,
group=True,
by_extension=False,
group_folder_name='sample dir')
with self.subTest(1):
final_path = os.path.join(dir_2, 'sample dir',
'my awesome cat.txt')
compare(final_path, f1.path)
def test_returns_false_if_relocation_to_existing_child_folder_fails(self):
dir_1 = self.tempdir.makedir('abc/fig/')
dir_2 = self.tempdir.makedir('abc/fig/one/two')
dir_3 = self.tempdir.makedir('abc/fig/one/two/three/')
file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'),
'')
f1 = File(file_1)
f1.move_to(dst_root_path=dir_2,
group=True,
by_extension=True,
group_folder_name='sample dir')
with self.subTest(1):
final_path = os.path.join(dir_2, 'sample dir', 'TXT',
'my awesome cat.txt')
compare(final_path, f1.path)
f1.move_to(dst_root_path=dir_3,
group=True,
by_extension=True,
group_folder_name='sample dir')
with self.subTest(2):
final_path = os.path.join(dir_3, 'sample dir', 'TXT',
'my awesome cat.txt')
compare(final_path, f1.path)
def test_returns_false_if_relocation_to_non_existent_folder_fails(self):
temp_path = self.tempdir.path
dir_1 = self.tempdir.makedir('abc/fig/')
dir_2 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two')
dir_3 = os.path.join(temp_path, 'first', 'fig', 'one', 'two', 'three')
file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'),
'')
f1 = File(file_1)
f1.move_to(dst_root_path=dir_2,
group=True,
by_extension=True,
group_folder_name='sample dir')
with self.subTest(1):
final_path = os.path.join(dir_2, 'sample dir', 'TXT',
'my awesome cat.txt')
compare(final_path, f1.path)
f1.move_to(dst_root_path=dir_3,
group=True,
by_extension=True,
group_folder_name='sample dir')
with self.subTest(2):
final_path = os.path.join(dir_3, 'sample dir', 'TXT',
'my awesome cat.txt')
compare(final_path, f1.path)
def test_returns_false_if_relocation_to_parent_folder_fails(self):
temp_path = self.tempdir.path
dir_1 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two', 'three')
dir_2 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two')
dir_3 = os.path.join(temp_path, 'abc', 'fig')
self.tempdir.makedir(dir_1)
file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'),
'')
f1 = File(file_1)
f1.move_to(dst_root_path=dir_2,
group=True,
by_extension=True,
group_folder_name='sample dir')
with self.subTest(1):
final_path = os.path.join(dir_2, 'sample dir', 'TXT',
'my awesome cat.txt')
compare(final_path, f1.path)
f1.move_to(dst_root_path=dir_3,
group=True,
by_extension=True,
group_folder_name='sample dir')
with self.subTest(2):
final_path = os.path.join(dir_3, 'sample dir', 'TXT',
'my awesome cat.txt')
compare(final_path, f1.path)
def test_returns_false_if_relocation_of_file_without_extension_fails(self):
temp_path = self.tempdir.path
dir_1 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two', 'three')
dir_2 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two')
dir_3 = os.path.join(temp_path, 'abc', 'fig')
self.tempdir.makedir(dir_1)
file_1 = self.tempdir.write(
os.path.join(dir_1, '146 awesome street $# yea'), '')
f1 = File(file_1)
f1.move_to(dst_root_path=dir_2,
group=True,
by_extension=True,
group_folder_name='sample dir')
with self.subTest(1):
compare(True, f1.exists)
with self.subTest(2):
final_path = os.path.join(dir_2, 'sample dir', 'UNDEFINED',
'146 awesome street $# yea')
compare(final_path, f1.path)
f1.move_to(dst_root_path=dir_3,
group=False,
by_extension=True,
group_folder_name='sample dir')
with self.subTest(3):
final_path = os.path.join(dir_3, 'UNDEFINED',
'146 awesome street $# yea')
compare(final_path, f1.path)
def test_returns_false_if_emptynameerror_not_raised(self):
temp_path = self.tempdir.path
dir_1 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two', 'three')
dir_2 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two')
self.tempdir.makedir(dir_1)
file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'),
'')
f1 = File(file_1)
self.assertRaises(EmptyNameError,
f1.move_to,
dst_root_path=dir_2,
group=True,
by_extension=True,
group_folder_name=' ')
def test_returns_false_if_file_not_created(self):
dir_1 = self.tempdir.makedir('abc/fig/one/two/three')
file_1 = os.path.join(dir_1, 'my awesome cat.txt')
f1 = File(file_1)
with self.subTest(1):
compare(False, f1.exists)
f1.touch()
with self.subTest(2):
compare(True, f1.exists)
with self.subTest(3):
self.assertRaises(FileExistsError, f1.touch, exist_ok=False)
class TestInferenceDataSet2p5D(unittest.TestCase):
def setUp(self):
"""
Set up a directory with images
"""
self.tempdir = TempDirectory()
self.temp_path = self.tempdir.path
self.tempdir.makedir('image_dir')
self.tempdir.makedir('model_dir')
self.tempdir.makedir('mask_dir')
self.image_dir = os.path.join(self.temp_path, 'image_dir')
self.model_dir = os.path.join(self.temp_path, 'model_dir')
self.mask_dir = os.path.join(self.temp_path, 'mask_dir')
# Create a temp image dir
im = np.zeros((10, 16), dtype=np.uint8)
self.frames_meta = aux_utils.make_dataframe()
self.time_idx = 2
for p in range(5):
for z in range(4):
for c in range(3):
im_name = aux_utils.get_im_name(
time_idx=self.time_idx,
channel_idx=c,
slice_idx=z,
pos_idx=p,
)
cv2.imwrite(os.path.join(self.image_dir, im_name), im + c * 10)
self.frames_meta = self.frames_meta.append(
aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES),
ignore_index=True,
)
# Write frames meta to image dir too
self.frames_meta.to_csv(os.path.join(self.image_dir, 'frames_meta.csv'))
# Save masks and mask meta
self.mask_meta = aux_utils.make_dataframe()
self.mask_channel = 50
for p in range(5):
for z in range(4):
im_name = aux_utils.get_im_name(
time_idx=2,
channel_idx=self.mask_channel,
slice_idx=z,
pos_idx=p,
)
cv2.imwrite(os.path.join(self.mask_dir, im_name), im + 1)
self.mask_meta = self.mask_meta.append(
aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES),
ignore_index=True,
)
# Write frames meta to image dir too
self.mask_meta.to_csv(os.path.join(self.mask_dir, 'frames_meta.csv'))
# Select inference split of dataset
self.split_col_ids = ('pos_idx', [1, 3])
# Make configs with fields necessary for inference dataset
dataset_config = {
'input_channels': [2],
'target_channels': [self.mask_channel],
'model_task': 'segmentation',
}
self.network_config = {
'class': 'UNetStackTo2D',
'depth': 3,
'data_format': 'channels_first',
}
# Instantiate class
self.data_inst = inference_dataset.InferenceDataSet(
image_dir=self.image_dir,
dataset_config=dataset_config,
network_config=self.network_config,
split_col_ids=self.split_col_ids,
mask_dir=self.mask_dir,
)
def tearDown(self):
"""
Tear down temporary folder and file structure
"""
TempDirectory.cleanup_all()
self.assertEqual(os.path.isdir(self.temp_path), False)
def test_init(self):
"""
Test init of inference dataset
"""
self.assertEqual(self.data_inst.image_dir, self.image_dir)
self.assertEqual(self.data_inst.target_dir, self.mask_dir)
self.assertIsNone(self.data_inst.flat_field_dir)
self.assertEqual(self.data_inst.image_format, 'zyx')
self.assertEqual(self.data_inst.model_task, 'segmentation')
self.assertEqual(self.data_inst.depth, 3)
self.assertFalse(self.data_inst.squeeze)
self.assertFalse(self.data_inst.im_3d)
self.assertEqual(self.data_inst.data_format, 'channels_first')
self.assertListEqual(self.data_inst.input_channels, [2])
self.assertListEqual(self.data_inst.target_channels, [self.mask_channel])
# Two inference samples (pos idx 1 and 3), two slices (1, 2) each = 4
self.assertEqual(self.data_inst.num_samples, 4)
self.assertListEqual(
self.data_inst.frames_meta.pos_idx.unique().tolist(),
[1, 3],
)
# Image channels = 0, 1, 2 and target channel = 50
self.assertListEqual(
self.data_inst.frames_meta.channel_idx.unique().tolist(),
[0, 1, 2, 50])
def test_get_iteration_meta(self):
iteration_meta = self.data_inst.get_iteration_meta()
# This contains metadata for first target channel only z=1,2, p=1,3
self.assertTupleEqual(iteration_meta.shape, (4, 6))
self.assertListEqual(
iteration_meta.channel_idx.unique().tolist(),
[self.mask_channel],
)
# Contains only test indices pos 1 and 3
self.assertListEqual(
iteration_meta.pos_idx.unique().tolist(),
[1, 3],
)
# Contains two slices, 1 and 2
self.assertListEqual(
iteration_meta.slice_idx.unique().tolist(),
[1, 2],
)
def test__len__(self):
num_samples = self.data_inst.__len__()
self.assertEqual(num_samples, 4)
def test_get_image(self):
meta_row = dict.fromkeys(aux_utils.DF_NAMES)
meta_row['channel_idx'] = 2
meta_row['time_idx'] = self.time_idx
meta_row['slice_idx'] = 1
meta_row['pos_idx'] = 3
im_stack = self.data_inst._get_image(
input_dir=self.image_dir,
cur_row=meta_row,
channel_ids=[2],
depth=3,
normalize=False,
)
# Image shapes are cropped to nearest factor of two, channels first
self.assertTupleEqual(im_stack.shape, (1, 3, 8, 16))
# Channel 2 has constant values of 20
self.assertEqual(im_stack.max(), 20)
self.assertEqual(im_stack.min(), 20)
def test__getitem__(self):
# There are 2 test indices (pos 1 and 3)
input_stack, target_stack = self.data_inst.__getitem__(1)
# Cropped to factor of 2, add batch dim
self.assertTupleEqual(input_stack.shape, (1, 1, 3, 8, 16))
self.assertTupleEqual(target_stack.shape, (1, 1, 1, 8, 16))
# input stack should be normalized, not target
self.assertEqual(input_stack.max(), 0.0)
self.assertEqual(target_stack.max(), 1)
self.assertEqual(input_stack.dtype, np.float32)
self.assertEqual(target_stack.dtype, np.float32)
def test__getitem__regression(self):
dataset_config = {
'input_channels': [1, 2],
'target_channels': [0],
'model_task': 'regression',
}
# Instantiate class
data_inst = inference_dataset.InferenceDataSet(
image_dir=self.image_dir,
dataset_config=dataset_config,
network_config=self.network_config,
split_col_ids=self.split_col_ids,
)
# There are 2 test indices (pos 1 and 3)
input_stack, target_stack = data_inst.__getitem__(0)
# Cropped to factor of 2, add batch dim
self.assertTupleEqual(input_stack.shape, (1, 2, 3, 8, 16))
self.assertTupleEqual(target_stack.shape, (1, 1, 1, 8, 16))
# input stack should be normalized, not target
self.assertEqual(input_stack.max(), 0.0)
self.assertEqual(target_stack.max(), 0.0)
self.assertEqual(input_stack.dtype, np.float32)
self.assertEqual(target_stack.dtype, np.float32)
class TestDataDownloader(db_basetest.DBBaseTest):
"""
Test the data downloader with S3 storage
"""
@patch('imaging_db.database.db_operations.session_scope')
def setUp(self, mock_session):
super().setUp()
mock_session.return_value.__enter__.return_value = self.session
# Setup mock S3 bucket
self.mock = mock_s3()
self.mock.start()
self.conn = boto3.resource('s3', region_name='us-east-1')
self.bucket_name = 'czbiohub-imaging'
self.conn.create_bucket(Bucket=self.bucket_name)
# Test metadata parameters
self.nbr_channels = 2
self.nbr_slices = 3
# Mock S3 dir
self.dataset_serial = 'FRAMES-2005-06-09-20-00-00-1000'
self.frames_storage_dir = os.path.join('raw_frames',
self.dataset_serial)
# Create temporary directory and write temp image
self.tempdir = TempDirectory()
self.temp_path = self.tempdir.path
# Temporary file with 6 frames, tifffile stores channels first
self.im = 50 * np.ones((6, 10, 15), dtype=np.uint16)
self.im[0, :5, 3:12] = 50000
self.im[2, :5, 3:12] = 40000
self.im[4, :5, 3:12] = 30000
# Metadata
self.description = 'ImageJ=1.52e\nimages=6\nchannels=2\nslices=3\nmax=10411.0'
# Save test tif file
self.file_path = os.path.join(self.temp_path, "A1_2_PROTEIN_test.tif")
tifffile.imsave(
self.file_path,
self.im,
description=self.description,
)
upload_csv = pd.DataFrame(
columns=['dataset_id', 'file_name', 'description'], )
upload_csv = upload_csv.append(
{
'dataset_id': self.dataset_serial,
'file_name': self.file_path,
'description': 'Testing'
},
ignore_index=True,
)
self.csv_path_frames = os.path.join(
self.temp_path,
"test_upload_frames.csv",
)
upload_csv.to_csv(self.csv_path_frames)
self.credentials_path = os.path.join(
self.main_dir,
'db_credentials.json',
)
# Write a config file
self.config_path = os.path.join(
self.temp_path,
'config_tif_id.json',
)
config = {
"upload_type": "frames",
"frames_format": "tif_id",
"microscope": "Leica microscope CAN bus adapter",
"filename_parser": "parse_ml_name",
"storage": "s3"
}
json_ops.write_json_file(config, self.config_path)
# Upload frames
data_uploader.upload_data_and_update_db(
csv=self.csv_path_frames,
login=self.credentials_path,
config=self.config_path,
)
# Create inputs for file upload
self.dataset_serial_file = 'FILE-2005-06-01-01-00-00-1000'
self.file_storage_dir = os.path.join('raw_files',
self.dataset_serial_file)
self.csv_path_file = os.path.join(
self.temp_path,
"test_upload_file.csv",
)
# Change to unique serial
upload_csv['dataset_id'] = self.dataset_serial_file
upload_csv.to_csv(self.csv_path_file)
config_path = os.path.join(
self.temp_path,
'config_file.json',
)
config = {
"upload_type": "file",
"microscope": "Mass Spectrometry",
"storage": "s3",
}
json_ops.write_json_file(config, config_path)
# Upload file
data_uploader.upload_data_and_update_db(
csv=self.csv_path_file,
login=self.credentials_path,
config=config_path,
)
def tearDown(self):
"""
Rollback database session.
Tear down temporary folder and file structure, stop moto mock
"""
super().tearDown()
TempDirectory.cleanup_all()
self.assertFalse(os.path.isdir(self.temp_path))
self.mock.stop()
def test_parse_args(self):
with patch('argparse._sys.argv', [
'python', '--id', self.dataset_serial, '-p', '5', '-t', '0',
'-c', '1', '2', '3', '-z', '4', '5', '--dest', 'dest_path',
'--login', 'test_login.json', '--nbr_workers', '5'
]):
parsed_args = data_downloader.parse_args()
self.assertEqual(parsed_args.id, self.dataset_serial)
self.assertListEqual(parsed_args.positions, [5])
self.assertListEqual(parsed_args.times, [0])
self.assertListEqual(parsed_args.channels, ['1', '2', '3'])
self.assertListEqual(parsed_args.slices, [4, 5])
self.assertEqual(parsed_args.dest, 'dest_path')
self.assertEqual(parsed_args.login, 'test_login.json')
self.assertEqual(parsed_args.nbr_workers, 5)
def test_parse_args_defaults(self):
with patch('argparse._sys.argv', [
'python', '--id', self.dataset_serial, '--dest', 'dest_path',
'--login', 'test_login.json'
]):
parsed_args = data_downloader.parse_args()
self.assertIsNone(parsed_args.positions)
self.assertIsNone(parsed_args.times)
self.assertIsNone(parsed_args.channels)
self.assertIsNone(parsed_args.slices)
self.assertTrue(parsed_args.metadata)
self.assertTrue(parsed_args.download)
self.assertIsNone(parsed_args.nbr_workers)
@patch('imaging_db.database.db_operations.session_scope')
def test_download_frames(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial,
login=self.credentials_path,
dest=dest_dir,
storage='s3',
)
# Images are separated by slice first then channel
im_order = [0, 2, 4, 1, 3, 5]
it = itertools.product(range(self.nbr_channels),
range(self.nbr_slices))
for i, (c, z) in enumerate(it):
im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z)
im_path = os.path.join(
dest_dir,
self.dataset_serial,
im_name,
)
im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH)
numpy.testing.assert_array_equal(im, self.im[im_order[i], ...])
# Read and validate frames meta
meta_path = os.path.join(
dest_dir,
self.dataset_serial,
'frames_meta.csv',
)
frames_meta = pd.read_csv(meta_path)
for i, row in frames_meta.iterrows():
c = i // self.nbr_slices
z = i % self.nbr_slices
self.assertEqual(row.channel_idx, c)
self.assertEqual(row.slice_idx, z)
self.assertEqual(row.time_idx, 0)
self.assertEqual(row.pos_idx, 0)
im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z)
self.assertEqual(row.file_name, im_name)
sha256 = meta_utils.gen_sha256(self.im[im_order[i], ...])
self.assertEqual(row.sha256, sha256)
# Read and validate global meta
meta_path = os.path.join(
dest_dir,
self.dataset_serial,
'global_metadata.json',
)
meta_json = json_ops.read_json_file(meta_path)
self.assertEqual(meta_json['storage_dir'], self.frames_storage_dir)
self.assertEqual(meta_json['nbr_frames'], 6)
self.assertEqual(meta_json['im_width'], 15)
self.assertEqual(meta_json['im_height'], 10)
self.assertEqual(meta_json['nbr_slices'], self.nbr_slices)
self.assertEqual(meta_json['nbr_channels'], self.nbr_channels)
self.assertEqual(meta_json['im_colors'], 1)
self.assertEqual(meta_json['nbr_timepoints'], 1)
self.assertEqual(meta_json['nbr_positions'], 1)
self.assertEqual(meta_json['bit_depth'], 'uint16')
@patch('imaging_db.database.db_operations.session_scope')
def test_download_channel(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial,
login=self.credentials_path,
dest=dest_dir,
storage='s3',
channels=1,
)
download_dir = os.path.join(dest_dir, self.dataset_serial)
# Check frames_meta content
frames_meta = pd.read_csv(os.path.join(download_dir,
'frames_meta.csv'))
for i, row in frames_meta.iterrows():
self.assertEqual(row.channel_idx, 1)
im_name = 'im_c001_z00{}_t000_p000.png'.format(i)
self.assertEqual(row.file_name, im_name)
# Check downloaded images
im_order = [1, 3, 5]
for z in range(3):
im_name = 'im_c001_z00{}_t000_p000.png'.format(z)
im_path = os.path.join(download_dir, im_name)
im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH)
numpy.testing.assert_array_equal(im, self.im[im_order[i], ...])
@patch('imaging_db.database.db_operations.session_scope')
def test_download_channel_convert_str(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial,
login=self.credentials_path,
dest=dest_dir,
storage='s3',
channels='1',
)
download_dir = os.path.join(dest_dir, self.dataset_serial)
# Check frames_meta content
frames_meta = pd.read_csv(os.path.join(download_dir,
'frames_meta.csv'))
for i, row in frames_meta.iterrows():
self.assertEqual(row.channel_idx, 1)
im_name = 'im_c001_z00{}_t000_p000.png'.format(i)
self.assertEqual(row.file_name, im_name)
# Check downloaded images
im_order = [1, 3, 5]
for z in range(3):
im_name = 'im_c001_z00{}_t000_p000.png'.format(z)
im_path = os.path.join(download_dir, im_name)
im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH)
numpy.testing.assert_array_equal(im, self.im[im_order[i], ...])
@nose.tools.raises(AssertionError)
@patch('imaging_db.database.db_operations.session_scope')
def test_download_channel_name(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial,
login=self.credentials_path,
dest=dest_dir,
storage='s3',
channels='channel1',
)
@patch('imaging_db.database.db_operations.session_scope')
def test_download_pts(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial,
login=self.credentials_path,
dest=dest_dir,
storage='s3',
positions=0,
times=[0],
slices=1,
)
meta_path = os.path.join(
dest_dir,
self.dataset_serial,
'global_metadata.json',
)
frames_meta = pd.read_csv(meta_path)
for i, row in frames_meta.iterrows():
self.assertEqual(row.pos_idx, 0)
self.assertEqual(row.time_idx, 0)
self.assertEqual(row.slice_idx, 1)
@patch('imaging_db.database.db_operations.session_scope')
def test_download_file(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial_file,
login=self.credentials_path,
dest=dest_dir,
storage='s3',
metadata=False,
nbr_workers=2,
)
# See if file has been downloaded
file_path = os.path.join(
dest_dir,
self.dataset_serial_file,
'*',
)
found_file = os.path.basename(glob.glob(file_path)[0])
self.assertEqual("A1_2_PROTEIN_test.tif", found_file)
@nose.tools.raises(FileExistsError)
@patch('imaging_db.database.db_operations.session_scope')
def test_folder_exists(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
self.tempdir.makedir(
os.path.join('dest_dir', self.dataset_serial_file), )
dest_dir = os.path.join(self.temp_path, 'dest_dir')
data_downloader.download_data(
dataset_serial=self.dataset_serial_file,
login=self.credentials_path,
dest=dest_dir,
storage='s3',
nbr_workers=2,
metadata=False,
)
@nose.tools.raises(AssertionError)
@patch('imaging_db.database.db_operations.session_scope')
def test_no_download_or_meta(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
data_downloader.download_data(
dataset_serial=self.dataset_serial_file,
login=self.credentials_path,
dest=dest_dir,
storage='s3',
metadata=False,
download=False,
nbr_workers=2,
)
@nose.tools.raises(AssertionError)
@patch('imaging_db.database.db_operations.session_scope')
def test_invalid_dataset(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
self.tempdir.makedir(
os.path.join('dest_dir', self.dataset_serial_file), )
dest_dir = os.path.join(self.temp_path, 'dest_dir')
data_downloader.download_data(
dataset_serial='Not-a-serial',
login=self.credentials_path,
dest=dest_dir,
storage='s3',
metadata=False,
nbr_workers=2,
)
@nose.tools.raises(AssertionError)
@patch('imaging_db.database.db_operations.session_scope')
def test_negative_workers(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
data_downloader.download_data(
dataset_serial=self.dataset_serial_file,
login=self.credentials_path,
dest=dest_dir,
storage='s3',
metadata=False,
download=False,
nbr_workers=-2,
)
@patch('imaging_db.database.db_operations.session_scope')
def test__main__(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
with patch('argparse._sys.argv', [
'python', '--id', self.dataset_serial, '--dest', dest_dir,
'--storage', 's3', '--login', self.credentials_path
]):
runpy.run_path(
'imaging_db/cli/data_downloader.py',
run_name='__main__',
)
# Check that files are there
dest_files = os.listdir(
os.path.join(
dest_dir,
self.dataset_serial,
))
self.assertTrue('frames_meta.csv' in dest_files)
self.assertTrue('global_metadata.json' in dest_files)
for c in range(self.nbr_channels):
for z in range(self.nbr_slices):
im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z)
self.assertTrue(im_name in dest_files)
class TestDataDownloaderLocalStorage(db_basetest.DBBaseTest):
"""
Test the data downloader with local storage
"""
@patch('imaging_db.database.db_operations.session_scope')
def setUp(self, mock_session):
super().setUp()
mock_session.return_value.__enter__.return_value = self.session
# Create temporary directory and write temp image
self.tempdir = TempDirectory()
self.temp_path = self.tempdir.path
# Mock file storage
self.tempdir.makedir('storage_mount_point')
self.mount_point = os.path.join(self.temp_path, 'storage_mount_point')
self.tempdir.makedir('storage_mount_point/raw_files')
self.tempdir.makedir('storage_mount_point/raw_frames')
# Test metadata parameters
self.nbr_channels = 2
self.nbr_slices = 3
# Mock storage dir
self.dataset_serial = 'FRAMES-2005-06-09-20-00-00-1000'
self.frames_storage_dir = os.path.join('raw_frames',
self.dataset_serial)
# Temporary file with 6 frames, tifffile stores channels first
self.im = 50 * np.ones((6, 10, 15), dtype=np.uint16)
self.im[0, :5, 3:12] = 50000
self.im[2, :5, 3:12] = 40000
self.im[4, :5, 3:12] = 30000
# Metadata
self.description = 'ImageJ=1.52e\nimages=6\nchannels=2\nslices=3\nmax=10411.0'
# Save test tif file
self.file_path = os.path.join(self.temp_path, "A1_2_PROTEIN_test.tif")
tifffile.imsave(
self.file_path,
self.im,
description=self.description,
)
# Create input arguments for data upload
upload_csv = pd.DataFrame(
columns=['dataset_id', 'file_name', 'description'], )
upload_csv = upload_csv.append(
{
'dataset_id': self.dataset_serial,
'file_name': self.file_path,
'description': 'Testing'
},
ignore_index=True,
)
self.csv_path_frames = os.path.join(
self.temp_path,
"test_upload_frames.csv",
)
upload_csv.to_csv(self.csv_path_frames)
self.credentials_path = os.path.join(
self.main_dir,
'db_credentials.json',
)
self.config_path = os.path.join(
self.temp_path,
'config_tif_id.json',
)
config = {
"upload_type": "frames",
"frames_format": "tif_id",
"microscope": "Leica microscope CAN bus adapter",
"filename_parser": "parse_ml_name",
"storage": "local",
"storage_access": self.mount_point
}
json_ops.write_json_file(config, self.config_path)
# Upload frames
data_uploader.upload_data_and_update_db(
csv=self.csv_path_frames,
login=self.credentials_path,
config=self.config_path,
)
# Create input args for file upload
self.dataset_serial_file = 'FILE-2005-06-09-20-00-00-1000'
self.file_storage_dir = os.path.join('raw_files',
self.dataset_serial_file)
self.csv_path_file = os.path.join(
self.temp_path,
"test_upload_file.csv",
)
# Change to unique serial
upload_csv['dataset_id'] = self.dataset_serial_file
upload_csv.to_csv(self.csv_path_file)
config_path = os.path.join(
self.temp_path,
'config_file.json',
)
config = {
"upload_type": "file",
"microscope": "Mass Spectrometry",
"storage": "local",
"storage_access": self.mount_point
}
json_ops.write_json_file(config, config_path)
# Upload file
data_uploader.upload_data_and_update_db(
csv=self.csv_path_file,
login=self.credentials_path,
config=config_path,
)
def tearDown(self):
"""
Rollback database session.
Tear down temporary folder and file structure, stop moto mock
"""
super().tearDown()
TempDirectory.cleanup_all()
self.assertFalse(os.path.isdir(self.temp_path))
@patch('imaging_db.database.db_operations.session_scope')
def test_download_frames(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial,
login=self.credentials_path,
dest=dest_dir,
storage_access=self.mount_point,
)
# Images are separated by slice first then channel
im_order = [0, 2, 4, 1, 3, 5]
it = itertools.product(range(self.nbr_channels),
range(self.nbr_slices))
for i, (c, z) in enumerate(it):
im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z)
im_path = os.path.join(
dest_dir,
self.dataset_serial,
im_name,
)
im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH)
numpy.testing.assert_array_equal(im, self.im[im_order[i], ...])
# Read and validate frames meta
meta_path = os.path.join(
dest_dir,
self.dataset_serial,
'frames_meta.csv',
)
frames_meta = pd.read_csv(meta_path)
for i, row in frames_meta.iterrows():
c = i // self.nbr_slices
z = i % self.nbr_slices
self.assertEqual(row.channel_idx, c)
self.assertEqual(row.slice_idx, z)
self.assertEqual(row.time_idx, 0)
self.assertEqual(row.pos_idx, 0)
im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z)
self.assertEqual(row.file_name, im_name)
sha256 = meta_utils.gen_sha256(self.im[im_order[i], ...])
self.assertEqual(row.sha256, sha256)
# Read and validate global meta
meta_path = os.path.join(
dest_dir,
self.dataset_serial,
'global_metadata.json',
)
meta_json = json_ops.read_json_file(meta_path)
self.assertEqual(meta_json['storage_dir'], self.frames_storage_dir)
self.assertEqual(meta_json['nbr_frames'], 6)
self.assertEqual(meta_json['im_width'], 15)
self.assertEqual(meta_json['im_height'], 10)
self.assertEqual(meta_json['nbr_slices'], self.nbr_slices)
self.assertEqual(meta_json['nbr_channels'], self.nbr_channels)
self.assertEqual(meta_json['im_colors'], 1)
self.assertEqual(meta_json['nbr_timepoints'], 1)
self.assertEqual(meta_json['nbr_positions'], 1)
self.assertEqual(meta_json['bit_depth'], 'uint16')
@patch('imaging_db.database.db_operations.session_scope')
def test_download_channel(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial,
login=self.credentials_path,
dest=dest_dir,
storage_access=self.mount_point,
channels=1,
)
download_dir = os.path.join(dest_dir, self.dataset_serial)
# Check frames_meta content
frames_meta = pd.read_csv(os.path.join(download_dir,
'frames_meta.csv'))
for i, row in frames_meta.iterrows():
self.assertEqual(row.channel_idx, 1)
im_name = 'im_c001_z00{}_t000_p000.png'.format(i)
self.assertEqual(row.file_name, im_name)
# Check downloaded images
im_order = [1, 3, 5]
for z in range(3):
im_name = 'im_c001_z00{}_t000_p000.png'.format(z)
im_path = os.path.join(download_dir, im_name)
im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH)
numpy.testing.assert_array_equal(im, self.im[im_order[i], ...])
@nose.tools.raises(AssertionError)
@patch('imaging_db.database.db_operations.session_scope')
def test_download_channel_name(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial,
login=self.credentials_path,
dest=dest_dir,
storage_access=self.mount_point,
channels='channel1',
)
@patch('imaging_db.database.db_operations.session_scope')
def test_download_pts(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial,
login=self.credentials_path,
dest=dest_dir,
storage_access=self.mount_point,
positions=0,
times=0,
slices=1,
)
meta_path = os.path.join(
dest_dir,
self.dataset_serial,
'global_metadata.json',
)
frames_meta = pd.read_csv(meta_path)
for i, row in frames_meta.iterrows():
self.assertEqual(row.pos_idx, 0)
self.assertEqual(row.time_idx, 0)
self.assertEqual(row.slice_idx, 1)
@patch('imaging_db.database.db_operations.session_scope')
def test_download_file(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
dest_dir = os.path.join(self.temp_path, 'dest_dir')
# Download data
data_downloader.download_data(
dataset_serial=self.dataset_serial_file,
login=self.credentials_path,
dest=dest_dir,
storage_access=self.mount_point,
metadata=False,
nbr_workers=2,
)
# See if file has been downloaded
file_path = os.path.join(
dest_dir,
self.dataset_serial_file,
'*',
)
found_file = os.path.basename(glob.glob(file_path)[0])
self.assertEqual("A1_2_PROTEIN_test.tif", found_file)
@nose.tools.raises(FileExistsError)
@patch('imaging_db.database.db_operations.session_scope')
def test_folder_exists(self, mock_session):
mock_session.return_value.__enter__.return_value = self.session
# Create dest dir
self.tempdir.makedir('dest_dir')
self.tempdir.makedir(
os.path.join('dest_dir', self.dataset_serial_file), )
dest_dir = os.path.join(self.temp_path, 'dest_dir')
data_downloader.download_data(
dataset_serial=self.dataset_serial_file,
login=self.credentials_path,
dest=dest_dir,
storage_access=self.mount_point,
nbr_workers=2,
metadata=False,
)
class TestFileSplitter(unittest.TestCase):
"""
Abstract classes can be tested with Unittest's mock patch
"""
@patch.multiple(file_splitter.FileSplitter, __abstractmethods__=set())
def setUp(self):
# Setup mock local storage
# Create temporary directory and write temp image
self.tempdir = TempDirectory()
self.temp_path = self.tempdir.path
self.tempdir.makedir('storage_mount_point')
mount_point = os.path.join(self.temp_path, 'storage_mount_point')
self.test_path = "/datapath/testfile.tif"
self.storage_dir = "raw_frames/ISP-2005-06-09-20-00-00-0001"
storage_class = aux_utils.get_storage_class('local')
self.mock_inst = file_splitter.FileSplitter(
data_path=self.test_path,
storage_dir=self.storage_dir,
storage_class=storage_class,
storage_access=mount_point)
def tearDown(self):
"""
Tear down temporary folder and files and stop S3 mock
"""
TempDirectory.cleanup_all()
nose.tools.assert_equal(os.path.isdir(self.temp_path), False)
def test_init(self):
nose.tools.assert_equal(self.mock_inst.data_path, self.test_path)
nose.tools.assert_equal(self.mock_inst.storage_dir, self.storage_dir)
nose.tools.assert_equal(self.mock_inst.int2str_len, 3)
nose.tools.assert_equal(self.mock_inst.file_format, '.png')
@nose.tools.raises(AssertionError)
def test_get_imstack(self):
self.mock_inst.get_imstack()
def test_get_existing_imstack(self):
self.mock_inst.im_stack = np.zeros((5, 10))
im_stack = self.mock_inst.get_imstack()
self.assertTupleEqual(im_stack.shape, (5, 10))
@nose.tools.raises(AssertionError)
def test_get_global_meta(self):
self.mock_inst.get_global_meta()
@nose.tools.raises(AssertionError)
def test_get_global_json(self):
self.mock_inst.get_global_json()
@nose.tools.raises(AssertionError)
def test_get_frames_meta(self):
self.mock_inst.get_frames_meta()
@nose.tools.raises(AssertionError)
def test_get_frames_json(self):
self.mock_inst.get_frames_json()
def test_get_imname(self):
meta_row = {
"channel_idx": 6,
"slice_idx": 13,
"time_idx": 5,
"pos_idx": 7,
}
im_name = self.mock_inst._get_imname(meta_row=meta_row)
nose.tools.assert_equal(im_name, 'im_c006_z013_t005_p007.png')
def test_set_global_meta(self):
nbr_frames = 666
test_shape = (12, 15)
meta_row = {
"channel_idx": 6,
"slice_idx": 13,
"time_idx": 5,
"pos_idx": 7,
}
self.mock_inst.frames_meta = meta_row
self.mock_inst.frame_shape = test_shape
self.mock_inst.im_colors = 1
self.mock_inst.bit_depth = 'uint16'
# Set global meta
self.mock_inst.set_global_meta(nbr_frames=nbr_frames)
# Assert contents
meta = self.mock_inst.get_global_meta()
nose.tools.assert_equal(meta['storage_dir'], self.storage_dir)
nose.tools.assert_equal(meta['nbr_frames'], nbr_frames)
nose.tools.assert_equal(meta['im_height'], test_shape[0])
nose.tools.assert_equal(meta['im_width'], test_shape[1])
nose.tools.assert_equal(meta['im_colors'], 1)
nose.tools.assert_equal(meta['bit_depth'], 'uint16')
nose.tools.assert_equal(meta['nbr_slices'], 1)
nose.tools.assert_equal(meta['nbr_channels'], 1)
nose.tools.assert_equal(meta['nbr_timepoints'], 1)
nose.tools.assert_equal(meta['nbr_positions'], 1)
@nose.tools.raises(AssertionError)
def test_missing_meta(self):
self.mock_inst.set_global_meta(nbr_frames=666)
@nose.tools.raises(NotImplementedError)
def test_set_frame_info(self):
self.mock_inst.set_frame_info()
@nose.tools.raises(NotImplementedError)
def test_get_frames_and_metadata(self):
self.mock_inst.get_frames_and_metadata()
class TestML4IRSanity(unittest.TestCase):
"""
Datasets used in these tests are generated by sklearn.datasets.make_classification by changing its parameters.
Then queries and documents are grouped randomly (without replacement) so that each query has 2-25 documents with
one clicked document.
Baseline models (Perceptron and logistic regression are trained with sklearn)
"""
def setUp(self):
self.dir = pathlib.Path(__file__).parent
self.working_dir = TempDirectory()
self.log_dir = self.working_dir.makedir('logs')
self.working_dir.makedir('train')
self.working_dir.makedir('test')
self.working_dir.makedir('validation')
def tearDown(self):
TempDirectory.cleanup_all()
# Explicitly clear keras memory
gc.collect()
K.clear_session()
def test_linear_ml4ir_sanity_1(self):
"""
Weights of pre-trained models:
perceptron=np.array([1.87212065, -0.00305068])
log_regression=np.array([28.62071696, 1.18915853])
"""
run_sanity_test(n_features=2,
fname="dataset1.csv",
perceptron_mrr=1.0,
log_regression_mrr=1.0,
working_dir=pathlib.Path(self.working_dir.path),
log_dir=pathlib.Path(self.log_dir))
def test_linear_ml4ir_sanity_2(self):
"""
Weights of pre-trained models:
perceptron=np.array([4.50209484, -0.80280452])
log_reg=np.array([22.73585585, -3.94821153])
"""
run_sanity_test(n_features=2,
fname="dataset2.csv",
perceptron_mrr=1.0,
log_regression_mrr=1.0,
working_dir=pathlib.Path(self.working_dir.path),
log_dir=pathlib.Path(self.log_dir))
def test_linear_ml4ir_sanity_3(self):
"""
Weights of pre-trained models:
perceptron=np.array([-1.27651475, -4.07647092, 8.23950305, 0.29241316, 3.24763417])
log_reg=np.array([-1.67270377, -5.76088727, 8.36278576, -0.90878154, 3.47653204])
"""
run_sanity_test(n_features=5,
fname="dataset3.csv",
perceptron_mrr=1.0,
log_regression_mrr=1.0,
working_dir=pathlib.Path(self.working_dir.path),
log_dir=pathlib.Path(self.log_dir))
def test_linear_ml4ir_sanity_4(self):
"""
Weights of pre-trained models:
perceptron=np.array([5.10535665, 1.44131417])
log_reg=np.array([20.0954756, 4.69360163])
"""
run_sanity_test(n_features=2,
fname="dataset4.csv",
perceptron_mrr=0.975609756097561,
log_regression_mrr=0.9878048780487805,
working_dir=pathlib.Path(self.working_dir.path),
log_dir=pathlib.Path(self.log_dir))
def test_linear_ml4ir_sanity_5(self):
"""
Weights of pre-trained models:
perceptron=np.array([0.57435291, -0.99437351])
log_reg=np.array([1.15593505, -0.93317691])
"""
run_sanity_test(n_features=2,
fname="dataset5.csv",
perceptron_mrr=0.37021768881524975,
log_regression_mrr=0.4290204375570229,
working_dir=pathlib.Path(self.working_dir.path),
log_dir=pathlib.Path(self.log_dir))
def test_linear_ml4ir_sanity_6(self):
"""
Weights of pre-trained models:
perceptron=np.array(
[4.59994733, -3.56373965, -6.15935686, 0.87523846, -0.64231058, 2.15971991, 5.79875003,
-7.70152594, -0.07521741, 2.8817456])
log_reg=np.array(
[-0.38064406, -0.27970534, 0.02775136, 0.25641926, 0.15413321, 0.29194965, 0.72707686,
0.24791729, -0.39367192, 0.4882174])
"""
run_sanity_test(n_features=10,
fname="dataset6.csv",
perceptron_mrr=0.2630707011921534,
log_regression_mrr=0.29970690874564615,
working_dir=pathlib.Path(self.working_dir.path),
log_dir=pathlib.Path(self.log_dir))
def test_linear_ml4ir_sanity_7(self):
"""
Weights of pre-trained models:
perceptron=np.array([0.40127356, -0.43773627])
log_reg=np.array([4.15630544, -1.09111369])
"""
run_sanity_test(n_features=2,
fname="dataset7.csv",
perceptron_mrr=0.304679599925838,
log_regression_mrr=0.4938108725899423,
working_dir=pathlib.Path(self.working_dir.path),
log_dir=pathlib.Path(self.log_dir))
def test_linear_ml4ir_sanity_8(self):
"""
Weights of pre-trained models:
perceptron=np.array(
[2.91798129, 4.24880336, 7.42919018, 2.49609694, -0.84988373, 0.43435823, -0.18953416,
2.23129287, -0.67951411, -0.63925108])
log_reg=np.array(
[-0.14472192, -0.22594271, 0.62703883, 0.16002515, 0.17084088, -0.22872226, 0.89200279,
0.06297475, 0.70470567, -0.19396659])
"""
run_sanity_test(n_features=10,
fname="dataset8.csv",
perceptron_mrr=0.31185341006782435,
log_regression_mrr=0.3092000998257748,
working_dir=pathlib.Path(self.working_dir.path),
log_dir=pathlib.Path(self.log_dir))
class testVerify(unittest.TestCase):
"""Verify.
Testing verify functionalities.
"""
def setUp(self):
"""Initialise fixtures."""
self.env = EnvironmentVarGuard()
self._dir = TempDirectory()
self.outputdir = self._dir.makedir('output')
self.env = EnvironmentVarGuard()
self.env.set('ARCHIVE_LOCATION', self.outputdir + '/%s/')
self.env.set('LEGA_PASSWORD', 'value')
self.env.set('QUALITY_CONTROL_VERIFY_PEER', 'True')
self.env.set('QUALITY_CONTROL_VERIFY_HOSTNAME', 'False')
def tearDown(self):
"""Remove setup variables."""
self.env.unset('ARCHIVE_LOCATION')
self._dir.cleanup_all()
self.env.unset('LEGA_PASSWORD')
self.env.unset('QUALITY_CONTROL_VERIFY_PEER')
self.env.unset('QUALITY_CONTROL_VERIFY_HOSTNAME')
@tempdir()
@mock.patch('lega.verify.header_to_records')
@mock.patch('lega.verify.get_key_id')
def test_get_records(self, mock_key, mock_records, filedir):
"""Should call the url in order to provide the records."""
infile = filedir.write('infile.in',
bytearray.fromhex(pgp_data.ENC_FILE))
returned_data = KeyServerResponse(
200, io.BytesIO(pgp_data.PGP_PRIVKEY.encode()))
with PatchContextManager('lega.verify.urlopen',
returned_data) as mocked:
print(returned_data.status)
with open(infile, 'rb') as f:
get_records(f)
mocked.assert_called()
filedir.cleanup()
@tempdir()
@mock.patch('lega.verify.header_to_records')
@mock.patch('lega.verify.get_key_id')
def test_get_records_no_verify(self, mock_key, mock_records, filedir):
"""Should call the url in order to provide the records even without a verify turned off."""
self.env.set('QUALITY_CONTROL_VERIFY_PEER', 'False')
self.env.set('QUALITY_CONTROL_VERIFY_HOSTNAME', 'False')
infile = filedir.write('infile.in',
bytearray.fromhex(pgp_data.ENC_FILE))
returned_data = KeyServerResponse(
200, io.BytesIO(pgp_data.PGP_PRIVKEY.encode()))
with PatchContextManager('lega.verify.urlopen',
returned_data) as mocked:
with open(infile, 'rb') as f:
get_records(f)
mocked.assert_called()
filedir.cleanup()
@tempdir()
@mock.patch('lega.verify.header_to_records')
@mock.patch('lega.verify.get_key_id')
def test_get_records_key_error(self, mock_key, mock_records, filedir):
"""The PGP key was not found, should raise PGPKeyError error."""
infile = filedir.write('infile.in',
bytearray.fromhex(pgp_data.ENC_FILE))
with mock.patch('lega.verify.urlopen') as urlopen_mock:
urlopen_mock.side_effect = HTTPError('url', 404, 'msg', None, None)
with self.assertRaises(PGPKeyError):
with open(infile, 'rb') as f:
get_records(f)
filedir.cleanup()
@tempdir()
@mock.patch('lega.verify.header_to_records')
@mock.patch('lega.verify.get_key_id')
def test_get_records_server_error(self, mock_key, mock_records, filedir):
"""Some keyserver error occured, should raise KeyserverError error."""
infile = filedir.write('infile.in',
bytearray.fromhex(pgp_data.ENC_FILE))
with mock.patch('lega.verify.urlopen') as urlopen_mock:
urlopen_mock.side_effect = HTTPError('url', 400, 'msg', None, None)
with self.assertRaises(KeyserverError):
with open(infile, 'rb') as f:
get_records(f)
filedir.cleanup()
@tempdir()
@mock.patch('lega.verify.header_to_records')
@mock.patch('lega.verify.get_key_id')
def test_get_records_error(self, mock_key, mock_records, filedir):
"""Some general error occured, should raise Exception error."""
self.env.set('QUALITY_CONTROL_VERIFY_PEER', 'False')
self.env.set('QUALITY_CONTROL_VERIFY_HOSTNAME', 'False')
infile = filedir.write('infile.in',
bytearray.fromhex(pgp_data.ENC_FILE))
with mock.patch('lega.verify.urlopen') as urlopen_mock:
urlopen_mock.side_effect = Exception
with self.assertRaises(Exception):
with open(infile, 'rb') as f:
get_records(f)
filedir.cleanup()
@mock.patch('lega.ingest.getattr')
@mock.patch('lega.verify.get_connection')
@mock.patch('lega.verify.consume')
def test_main(self, mock_consume, mock_connection, mock_getattr):
"""Test main verify, by mocking cosume call."""
mock_consume.return_value = mock.MagicMock()
main()
mock_consume.assert_called()
@tempdir()
@mock.patch('lega.verify.db')
@mock.patch('lega.verify.body_decrypt')
@mock.patch('lega.verify.get_records')
def test_work(self, mock_records, mock_decrypt, mock_db, filedir):
"""Test verify worker, should send a messge."""
# Mocking a lot of stuff, ast it is previously tested
mock_db.status.return_value = mock.Mock()
mock_records.return_value = ['data'], 'key_id'
mock_decrypt.return_value = mock.Mock()
store = mock.MagicMock()
store.open.return_value = mock.MagicMock()
mock_broker = mock.MagicMock(name='channel')
mock_broker.channel.return_value = mock.Mock()
infile = filedir.write('infile.in', 'text'.encode("utf-8"))
data = {
'header': pgp_data.ENC_FILE,
'stable_id': '1',
'archive_path': infile,
'file_id': '123',
'org_msg': {}
}
result = work('10', store, mock_broker, data)
self.assertTrue({'status': {
'state': 'COMPLETED',
'details': '1'
}}, result)
filedir.cleanup()
