class GitHelper(object): repo = 'local/' def setUp(self): self.dir = TempDirectory() self.addCleanup(self.dir.cleanup) def git(self, command, repo=None): repo_path = self.dir.getpath(repo or self.repo) try: return check_output(['git'] + command.split(), cwd=repo_path, stderr=STDOUT) except CalledProcessError as e: self.fail(e.output) def git_rev_parse(self, label, repo=None): return self.git('rev-parse --verify -q --short ' + label, repo).strip() def check_tags(self, expected, repo=None): actual = {} for tag in self.git('tag', repo).split(): actual[tag] = self.git_rev_parse(tag, repo) compare(expected, actual=actual) def make_repo_with_content(self, repo): if not os.path.exists(self.dir.getpath(repo)): self.dir.makedir(repo) self.git('init', repo) self.dir.write(repo + 'a', 'some content') self.dir.write(repo + 'b', 'other content') self.dir.write(repo + 'c', 'more content') self.git('add .', repo) self.git('commit -m initial', repo)
class GitHelper(object): repo = 'local/' def setUp(self): self.dir = TempDirectory() self.addCleanup(self.dir.cleanup) def git(self, command, repo=None): repo_path = self.dir.getpath(repo or self.repo) try: return check_output(['git'] + command.split(), cwd=repo_path, stderr=STDOUT) except CalledProcessError as e: self.fail(e.output) def git_rev_parse(self, label, repo=None): return self.git('rev-parse --verify -q --short '+label, repo).strip() def check_tags(self, expected, repo=None): actual = {} for tag in self.git('tag', repo).split(): actual[tag] = self.git_rev_parse(tag, repo) compare(expected, actual=actual) def make_repo_with_content(self, repo): if not os.path.exists(self.dir.getpath(repo)): self.dir.makedir(repo) self.git('init', repo) self.dir.write(repo + 'a', 'some content') self.dir.write(repo + 'b', 'other content') self.dir.write(repo + 'c', 'more content') self.git('add .', repo) self.git('commit -m initial', repo)
class TestGetLanes(unittest.TestCase): def setUp(self): self.tempdir = TempDirectory() self.tempdir.write('fake_file1.txt', b'some foo thing') self.tempdir.write('fake_tmp_files/folder/afile.txt', b'the text') self.tempdir.write('fake_directory/fake_file2.txt', b'the text') self.tempdir.write('fake_directory/afile.bam', b'the text') self.tempdir.write('fake_directory/afile.sam', b'the text') self.tempdir.write('fake_directory/afile.fastq.gz', b'the text') self.tempdir.makedir('empty_directory') self.tempdir_path = self.tempdir.path def tearDown(self): self.tempdir.cleanup() pass def test_get_lanes_returns_data(self): data_list = bytes(str(self.tempdir.path + '/fake_tmp_files/folder/afile.txt\n'+self.tempdir.path + '/fake_directory/fake_file2.txt\n' + 'fake_path\n'),'ascii') with mock.patch('archive_project.GetLanes.check_output', return_value=data_list) as co: actual, message = get_lanes("study") co.assert_called_once_with(['pf', 'data', '-t', 'study', '-i', 'study']) expected = [str(self.tempdir.path + '/fake_tmp_files/folder/afile.txt'), str(self.tempdir.path + '/fake_directory/fake_file2.txt')] self.assertEqual(expected,actual) self.assertEqual('This path was returned by pf, but does not actually exist: fake_path\n', message) def test_get_lanes_returns_nodata(self): with mock.patch('archive_project.GetLanes.check_output', return_value=b'') as co: with mock.patch('os.path.exists', return_value=True) as pe: actual, message = get_lanes("fake_study") self.assertEqual([],actual) pe.assert_not_called() self.assertEqual('Unknown study or no data associated with study: fake_study\n',message)
class Test_archivepgsql_backup_invocation(TestCase): ARCHIVEPGSQL_PATH = os.path.join('bbpgsql', 'cmdline_scripts') CONFIG_FILE = 'config.ini' exe_script = 'archivepgsql' def setUp(self): self.setup_environment() self.setup_config() self.execution_sequence = 0 def setup_environment(self): self.env = deepcopy(os.environ) self.env['PATH'] = ''.join([ self.env['PATH'], ':', self.ARCHIVEPGSQL_PATH]) self.tempdir = TempDirectory() self.data_dir = self.tempdir.makedir('pgsql_data') self.archive_dir = self.tempdir.makedir('pgsql_archive') def setup_config(self): self.config_path = os.path.join(self.tempdir.path, self.CONFIG_FILE) self.config_dict = { 'General': { 'pgsql_data_directory': self.data_dir, }, 'Snapshot': { 'driver': 'memory', }, } write_config_to_filename(self.config_dict, self.config_path) self.config = get_config_from_filename_and_set_up_logging( self.config_path ) def tearDown(self): self.tempdir.cleanup() @patch('bbpgsql.archive_pgsql.commit_snapshot_to_repository') @patch('bbpgsql.archive_pgsql.create_archive') @patch('bbpgsql.archive_pgsql.pg_stop_backup') @patch('bbpgsql.archive_pgsql.pg_start_backup') def test_perform_backup(self, mock_pg_start_backup, mock_pg_stop_backup, mock_create_archive, mock_commit_snapshot_to_repository): first_WAL = '000000D0' second_WAL = '000000D1' mock_pg_start_backup.return_value = first_WAL mock_pg_stop_backup.return_value = second_WAL archiveFile = os.path.join(self.archive_dir, 'pgsql.snapshot.tar') tag = bbpgsql.archive_pgsql.generate_tag() repo = get_Snapshot_repository(self.config) bbpgsql.archive_pgsql.perform_backup(self.data_dir, archiveFile, tag, repo) mock_pg_start_backup.assert_called_once_with(tag) mock_create_archive.assert_called_once_with(self.data_dir, archiveFile) self.assertEqual(mock_pg_stop_backup.called, True) self.assertEqual(mock_pg_stop_backup.call_count, 1) mock_commit_snapshot_to_repository.assert_called_once_with( repo, archiveFile, tag, first_WAL, second_WAL)
def test_create_monitor_with_watch_path(self): wd = TempDirectory() source_path = wd.makedir('source') wd.makedir('build') with chdir(wd.path): m = create_monitor(source_path) assert len(m.reporters) == 1 reporter = m.reporters[0] assert reporter.watch_path == source_path assert reporter.build_path == '{}-build'.format(os.path.realpath(source_path)) # noqa
def test_create_monitor_with_watch_and_build_path(self): wd = TempDirectory() source_path = wd.makedir("source") build_path = wd.makedir("build") with chdir(wd.path): m = create_monitor(source_path, build_path=os.path.basename(build_path)) assert len(m.reporters) == 1 reporter = m.reporters[0] assert reporter.watch_path == source_path assert reporter.build_path == "{}".format(os.path.realpath(build_path))
class TestImageValidator(unittest.TestCase): def setUp(self): """ Set up a folder structure containing one timepoint (0) one channel (1) and two images in channel subfolder """ self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.tempdir.makedir('timepoint_0') self.tempdir.makedir('timepoint_0/channel_1') # Write images as bytes im = np.zeros((15, 12), dtype=np.uint16) res, im_encoded = cv2.imencode('.png', im) im_encoded = im_encoded.tostring() self.tempdir.write('timepoint_0/channel_1/im_0.png', im_encoded) self.tempdir.write('timepoint_0/channel_1/im_1.png', im_encoded) self.tempdir.listdir(recursive=True) def tearDown(self): """ Tear down temporary folder and file structure """ TempDirectory.cleanup_all() nose.tools.assert_equal(os.path.isdir(self.temp_path), False) def test_image_validator(self): """ Test image validator on temporary folder structure """ meta_name = 'image_volumes_info.csv' validator = image_validator.ImageValidator( input_dir=self.tempdir.path, meta_name=meta_name, verbose=10, ) validator.folder_validator() # Check written metadata metadata = pd.read_csv(os.path.join(self.tempdir.path, meta_name)) # Metadata should have 8 fields + 1 index and we have two files nose.tools.assert_equal(metadata.shape, (2, 9)) # Metadata should contain the following column names expected_names = [ 'Unnamed: 0', 'timepoint', 'channel_num', 'sample_num', 'slice_num', 'fname', 'size_x_microns', 'size_y_microns', 'size_z_microns' ] nose.tools.assert_equal(list(metadata), expected_names) # Both files should have timepoint 0 nose.tools.assert_equal(np.all(metadata['timepoint'] == 0), True) # Both file should have channel 1 nose.tools.assert_equal(np.all(metadata['channel_num'] == 1), True) # The two image numbers should be sorted by index nose.tools.assert_equal(metadata['sample_num'][0], 0) nose.tools.assert_equal(metadata['sample_num'][1], 1)
def test_create_monitor_accepts_clean_kwarg(self): wd = TempDirectory() source_path = wd.makedir("source") build_path = wd.makedir("build") sub_path = os.path.join(build_path, "sub") os.mkdir(sub_path) with chdir(wd.path): assert os.path.exists(build_path) assert os.path.exists(sub_path) create_monitor( source_path, build_path=os.path.basename(build_path), clean=True ) assert os.path.exists(sub_path) assert os.path.exists(build_path)
def test_default_watcher_gets_all_files(self): work_directory = TempDirectory() work_directory.write("a.h", b"") work_directory.write("a.c", b"") work_directory.write("a.cc", b"") work_directory.write("CMakeLists.txt", b"") work_directory.write("blah.txt", b"") work_directory.makedir(".git") work_directory.makedir(".hg") wd_len = len(work_directory.path) + 1 w = Watcher(work_directory.path, None) watchstate = w.poll() filelist = [f[wd_len:] for f in watchstate.inserts] filelist.sort() assert filelist == ["CMakeLists.txt", "a.c", "a.cc", "a.h", "blah.txt"]
def test_testlist(self): import stat work_directory = TempDirectory() work_directory.makedir('test') work_directory.write(['test', 'test_dummy.c'], b'') build_directory = TempDirectory() testbin_path = build_directory.write(['test_dummy'], b'') st = os.stat(testbin_path) os.chmod(testbin_path, st.st_mode | stat.S_IEXEC) w = Watcher(work_directory.path, build_directory.path) w.poll() exefile = testbin_path + watcher.EXE_SUFFIX testlist = [(g.source(), g.executable()) for g in w.testlist()] assert testlist == [(os.path.join('test', 'test_dummy.c'), exefile)]
def test_testlist(self): import stat work_directory = TempDirectory() work_directory.makedir("test") work_directory.write(["test", "test_dummy.c"], b"") build_directory = TempDirectory() testbin_path = build_directory.write(["test_dummy"], b"") st = os.stat(testbin_path) os.chmod(testbin_path, st.st_mode | stat.S_IEXEC) w = Watcher(work_directory.path, build_directory.path) w.poll() exefile = testbin_path + watcher.EXE_SUFFIX testlist = [(g.source(), g.executable()) for g in w.testlist()] assert testlist == [(os.path.join("test", "test_dummy.c"), exefile)]
def setUp(self): self.app = Flask(__name__) self.app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:' self.app.config['USER_LIST'] = ['Test Admin'] db.init_app(self.app) d = TempDirectory() d.makedir('dataset') d.write('dataset/files/testfile.jpg', 'abc') d.write('dataset/testfile2.jpg', 'abc') self.directory = d self.app.config['DATASET_ROOT'] = d.path with self.app.app_context(), TempDirectory() as d: db.create_all() lib.models.buildDB() dataset = lib.models.Datasets() dataset.name = 'Test' dataset.path = "dataset" db.session.add(dataset) db.session.commit()
def setUp(self): self.app = Flask(__name__) self.app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:' self.app.config['USER_LIST'] = ['Test Admin'] db.init_app(self.app) d = TempDirectory() d.makedir('dataset') d.write('dataset/files/testfile.jpg', 'abc') d.write('dataset/testfile2.jpg', 'abc') self.directory = d self.app.config['DATASET_ROOT'] = d.path with self.app.app_context(), TempDirectory() as d: db.create_all() lib.models.buildDB() dataset = lib.models.Datasets() dataset.name = 'Test' dataset.path = "dataset" db.session.add(dataset) db.session.commit()
class TestDirectoryModel(TestCase): def setUp(self): self.temp_dir = TempDirectory() self.base_path = self.temp_dir.makedir(('base')) self.base_dir = Directory(path=self.base_path) self.base_dir.save(is_basedir=True) self.test_dir = Directory(parent=self.base_dir, path='test') self.test_dir.save() def test_base_dir_absolute_path(self): self.assertEqual(Path(self.base_path), self.base_dir.absolute_path()) def test_base_dir_relative_path(self): self.assertEqual(Path('.'), self.base_dir.relative_path()) def test_absolute_path(self): test_path = self.temp_dir.makedir(os.path.join(self.base_path, 'test')) self.assertEqual(Path(test_path), self.test_dir.absolute_path()) def test_relative_path(self): self.assertEqual(Path('test'), self.test_dir.relative_path()) def test_get_sub_path_directories(self): paths_elements = ['test'] self.assertEqual([self.test_dir], self.base_dir.get_sub_path_directories(paths_elements)) def listdir(self): self.assertEqual(([], [self.test_dir]), self.base_dir.listdir()) def test_exists(self): self.assertTrue(self.base_dir.exists()) def test_get_basedir(self): delattr(Directory, '_cached_basedir') self.assertEqual(self.base_dir, Directory.get_basedir())
def test_use_configuration_from_root_path_when_no_other_was_found(self): root_dir = TempDirectory() self.tmpdirs.append(root_dir) start_path = root_dir.makedir('some/directories/to/start/looking/for/settings') test_file = os.path.realpath(os.path.join(root_dir.path, 'settings.ini')) with open(test_file, 'a') as file_: file_.write('[settings]') self.files.append(test_file) # Required to removed it at tearDown discovery = ConfigurationDiscovery(start_path, root_path=root_dir.path) filenames = [cfg.filename for cfg in discovery.config_files] self.assertEqual([test_file], filenames)
def test_default_watcher_gets_all_files(self): work_directory = TempDirectory() work_directory.write('a.h', b'') work_directory.write('a.c', b'') work_directory.write('a.cc', b'') work_directory.write('CMakeLists.txt', b'') work_directory.write('blah.txt', b'') work_directory.makedir('.git') work_directory.makedir('.hg') wd_len = len(work_directory.path) + 1 w = Watcher(work_directory.path, None) watchstate = w.poll() filelist = [f[wd_len:] for f in watchstate.inserts] filelist.sort() assert filelist == [ 'CMakeLists.txt', 'a.c', 'a.cc', 'a.h', 'blah.txt' ]
class TestXMLExporter(unittest.TestCase): def setUp(self): self.tmp = TempDirectory() self.dir = self.tmp.makedir('foobar/') self.exp = XMLExporter(self.dir) def tearDown(self): TempDirectory.cleanup_all() #not responsible for creation def DISABLEDtest_creation_of_meta_info_file(self): filename = 'barfoo.xml' status = self.exp._exportMetaInformationToXML('', '', filename) self.assertTrue(os.path.isfile(self.dir+'metainformation.xml')) self.assertTrue(status) #not responsible for creation; # TODO: fix def DISABLEDtest_content_of_meta_info_file(self): filename = 'barfoo.xml' status = self.exp._exportMetaInformationToXML('01', '02', filename) tree = et.parse(self.dir + 'metainformation.xml') xml = tree.getroot() session = xml.find('Session') exercise = xml.find('Exercise') annotation = xml.find('Annotationfile') self.assertEquals('01', session.text) self.assertEquals('02', exercise.text) self.assertEquals(self.dir + filename, annotation.text) def DISABLEDtest_content_of_xml_export_file(self): src = lib.Sourcefile._make(['', '', '', '', lib.Orientation.left]) input = lib.Validationfile._make([src, 'Gold', [['Foo', 'Bar']], [{ 'Fu': '1', 'Baz': '2'}, {'Fu': '3', 'Baz': '4'}]]) self.assertTrue(self.exp.export(input)) tree = et.parse(self.dir + 'annotation.xml') xml = tree.getroot() validation = xml.find('Validation') self.assertEquals(validation.attrib, {'Type': 'Gold'}) sensor = validation.find('Sensor') self.assertEquals(sensor.attrib, {'Orientation': 'left'}) meta = sensor.find('Meta') self.assertEquals(meta[0].tag, 'Foo') self.assertEquals(meta[0].text, 'Bar') content = sensor.find('Content') self.assertEquals(content[0].tag, 'No1') self.assertEquals(content[0][0].tag, 'Fu') self.assertEquals(content[0][0].text, '1') self.assertEquals(content[0][1].tag, 'Baz') self.assertEquals(content[0][1].text, '2') self.assertEquals(content[1][0].tag, 'Fu') self.assertEquals(content[1][0].text, '3') self.assertEquals(content[1][1].tag, 'Baz') self.assertEquals(content[1][1].text, '4')
class Test_FilesystemCommitStorage(TestCase): def setUp(self): self.tempdir = TempDirectory() self.commit_storage_path = self.tempdir.makedir('commit_storage') self.config = config() self.config.set('WAL', 'driver', 'filesystem') self.config.set('WAL', 'path', self.commit_storage_path) def tearDown(self): self.tempdir.cleanup() def test_will_build_storage_from_config(self): self.assertEqual(FilesystemCommitStorage, type(get_repository_storage_from_config(self.config, 'WAL')))
def setUp(self): data.app.config['TESTING'] = True data.app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:' data.app.config['USER_LIST'] = ['Test Admin'] db.init_app(data.app) d = TempDirectory() d.makedir('dataset') d.write('dataset/files/testfile.jpg', 'abc') d.write('dataset/testfile2.jpg', 'abc') self.directory = d data.app.config['DATASET_ROOT'] = d.path with data.app.app_context(), TempDirectory() as d: db.create_all() lib.models.buildDB() license = lib.models.License.query.first() provider = lib.models.Provider.query.first() dataset = lib.models.Datasets() dataset.name = 'Test' dataset.license = [license] dataset.provider = [provider] dataset.path = "dataset" db.session.add(dataset) db.session.commit() self.app = data.app.test_client()
def test_use_configuration_from_root_path_when_no_other_was_found(self): root_dir = TempDirectory() self.tmpdirs.append(root_dir) start_path = root_dir.makedir( 'some/directories/to/start/looking/for/settings') test_file = os.path.realpath( os.path.join(root_dir.path, 'settings.ini')) with open(test_file, 'a') as file_: file_.write('[settings]') self.files.append(test_file) # Required to removed it at tearDown discovery = ConfigurationDiscovery(start_path, root_path=root_dir.path) filenames = [cfg.filename for cfg in discovery.config_files] self.assertEqual([test_file], filenames)
def test_lookup_should_stop_at_root_path(self): test_dir = TempDirectory() self.tmpdirs.append(test_dir) # Cleanup dir at tearDown start_path = test_dir.makedir('some/dirs/without/config') # create a file in the test_dir test_file = os.path.realpath(os.path.join(test_dir.path, 'settings.ini')) with open(test_file, 'a') as file_: file_.write('[settings]') self.files.append(test_file) # Required to removed it at tearDown root_dir = os.path.join(test_dir.path, 'some', 'dirs') # No settings here discovery = ConfigurationDiscovery(start_path, root_path=root_dir) self.assertEqual(discovery.config_files, [])
def test_lookup_should_stop_at_root_path(self): test_dir = TempDirectory() self.tmpdirs.append(test_dir) # Cleanup dir at tearDown start_path = test_dir.makedir('some/dirs/without/config') # create a file in the test_dir test_file = os.path.realpath( os.path.join(test_dir.path, 'settings.ini')) with open(test_file, 'a') as file_: file_.write('[settings]') self.files.append(test_file) # Required to removed it at tearDown root_dir = os.path.join(test_dir.path, 'some', 'dirs') # No settings here discovery = ConfigurationDiscovery(start_path, root_path=root_dir) self.assertEqual(discovery.config_files, [])
class Test_archivepgsql_BasicCommandLineOperation(TestCase): ARCHIVEPGSQL_PATH = os.path.join('bbpgsql', 'cmdline_scripts') CONFIG_FILE = 'config.ini' exe_script = 'archivepgsql' def setUp(self): self.setup_environment() self.setup_config() self.cmd = [self.exe_script, '--dry-run', '--config', self.config_path] def setup_environment(self): self.env = deepcopy(os.environ) self.env['PATH'] = ''.join([ self.env['PATH'], ':', self.ARCHIVEPGSQL_PATH]) self.tempdir = TempDirectory() self.data_dir = self.tempdir.makedir('pgsql_data') def setup_config(self): self.config_path = os.path.join(self.tempdir.path, self.CONFIG_FILE) self.config_dict = { 'General': { 'pgsql_data_directory': self.data_dir, }, 'Snapshot': { 'driver': 'memory', }, } write_config_to_filename(self.config_dict, self.config_path) self.config = get_config_from_filename_and_set_up_logging( self.config_path ) def tearDown(self): self.tempdir.cleanup() def test_can_execute_archivepgsql(self): check_call(self.cmd, env=self.env, stdout=PIPE) def test_obeys_dry_run_option(self): proc = Popen(self.cmd, env=self.env, stdout=PIPE) stdoutdata, stderrdata = proc.communicate() self.assertEqual("Dry Run\n", stdoutdata)
class TestCSVExporter(unittest.TestCase): def setUp(self): self.tmp = TempDirectory() self.dir = self.tmp.makedir('foobar') self.exp = CSVExporter() def tearDown(self): TempDirectory.cleanup_all() def test_creation_of_export_file(self): filename = 'foobar.csv' values = [lib.Sensorsegment._make([0] * 6)] status = self.exp.export(values, self.dir+filename) self.assertTrue(os.path.isfile(self.dir+filename)) self.assertTrue(status) def test_content_of_export_file(self): filename = 'foobar.csv' values = [lib.Sensorsegment._make([0] * 6), lib.Sensorsegment._make([1] * 6) ] self.exp.export(values, self.dir+filename, False, False) res = self.tmp.read(self.dir+filename) ref = b'0,0,0,0,0,0\r\n1,1,1,1,1,1\r\n' self.assertEqual(ref, res) def test_content_with_header_with_indices(self): filename = 'foobar.csv' values = [lib.Sensorsegment._make([0] * 6)] self.exp.export(values, self.dir+filename, True, True) res = self.tmp.read(self.dir+filename) ref = b'Index,accelX,accelY,accelZ,gyroX,gyroY,gyroZ\r\n1,0,0,0,0,0,0\r\n' self.assertEqual(ref, res) def test_content_with_header_without_indices(self): filename = 'foobar.csv' values = [lib.Sensorsegment._make([0] * 6)] self.exp.export(values, self.dir+filename, True, False) res = self.tmp.read(self.dir+filename) ref = b'accelX,accelY,accelZ,gyroX,gyroY,gyroZ\r\n0,0,0,0,0,0\r\n' self.assertEqual(ref, res) def test_content_without_header_with_indices(self): filename = 'foobar.csv' values = [lib.Sensorsegment._make([0] * 6)] self.exp.export(values, self.dir+filename, False, True) res = self.tmp.read(self.dir+filename) ref = b'1,0,0,0,0,0,0\r\n' self.assertEqual(ref, res)
class TestFileStorage(unittest.TestCase): """FileStorage. Testing storage on disk. """ def setUp(self): """Initialise fixtures.""" self._dir = TempDirectory() self.outputdir = self._dir.makedir('output') self.env = EnvironmentVarGuard() self.env.set('ARCHIVE_LOCATION', self.outputdir + '/%s/') self._store = FileStorage('archive', 'lega') def tearDown(self): """Remove setup variables.""" self.env.unset('ARCHIVE_LOCATION') self._dir.cleanup_all() def test_location(self): """Test file location.""" result = self._store.location('12') self.assertEqual( os.path.join('/', '000', '000', '000', '000', '000', '000', '12'), result) def test_copy(self): """Test copy file.""" path = self._dir.write('output/lega/test.file', 'data1'.encode('utf-8')) path1 = self._dir.write('output/lega/test1.file', ''.encode('utf-8')) result = self._store.copy(open(path, 'rb'), path1) self.assertEqual(os.stat(path).st_size, result) def test_open(self): """Test open file.""" path = self._dir.write('output/lega/test.file', 'data1'.encode('utf-8')) print(path) with self._store.open('test.file') as resource: self.assertEqual(BufferedReader, type(resource))
class DocumentServiceWriterTestCases(SegueApiTestCase): def setUp(self): super(DocumentServiceWriterTestCases, self).setUp() self.tmp_dir = TempDirectory() self.out_dir = self.tmp_dir.makedir("output") self.templates = os.path.join(os.path.dirname(__file__), 'fixtures') self.service = DocumentService(override_root=self.out_dir, template_root=self.templates, tmp_dir=self.tmp_dir.path) def tearDown(self): self.tmp_dir.cleanup() @skipIf("SNAP_CI" in os.environ, "inkscape is not available in CI") def test_svg_to_pdf(self): result = self.service.svg_to_pdf('templates/dummy.svg', 'certificate', "ABCD", { 'XONGA': 'bir<osca' }) self.tmp_dir.check_dir('','certificate-ABCD.svg', 'output') self.tmp_dir.check_dir('output/certificate/AB', 'certificate-ABCD.pdf') contents_of_temp = self.tmp_dir.read("certificate-ABCD.svg") self.assertIn("bir<osca da silva", contents_of_temp) self.assertEquals(result, 'certificate-ABCD.pdf')
class TestTifFolderSplitter(unittest.TestCase): @patch('concurrent.futures.ProcessPoolExecutor') def setUp(self, MockPoolExecutor): """ Set up temporary test directory and mock S3 bucket connection """ # Magic mocking of multiprocessing MockPoolExecutor().__enter__().map = map_mock # Mock S3 directory for upload self.storage_dir = "raw_frames/SMS-2010-01-01-00-00-00-0001" # Create temporary directory and write temp image self.tempdir = TempDirectory() self.temp_path = self.tempdir.path # Temporary frame self.im = np.ones((10, 15), dtype=np.uint16) self.im[2:5, 3:12] = 10000 # Save test tif files self.channel_names = ['phase', 'brightfield', '666'] # Write files in dir for c in self.channel_names: for z in range(2): file_name = 'img_{}_t000_p050_z00{}.tif'.format(c, z) file_path = os.path.join(self.temp_path, file_name) ijmeta = {"Info": json.dumps({"c": c, "z": z})} tifffile.imsave( file_path, self.im + 5000 * z, ijmetadata=ijmeta, ) # Write external metadata in dir self.meta_dict = { 'Summary': { 'Slices': 26, 'PixelType': 'GRAY16', 'Time': '2018-11-01 19:20:34 -0700', 'z-step_um': 0.5, 'PixelSize_um': 0, 'BitDepth': 16, 'Width': 15, 'Height': 10 }, } self.json_filename = os.path.join(self.temp_path, 'metadata.txt') json_ops.write_json_file(self.meta_dict, self.json_filename) # Setup mock S3 bucket self.mock = mock_s3() self.mock.start() self.conn = boto3.resource('s3', region_name='us-east-1') self.bucket_name = 'czbiohub-imaging' self.conn.create_bucket(Bucket=self.bucket_name) # Instantiate file parser class storage_class = aux_utils.get_storage_class('s3') self.frames_inst = tif_splitter.TifFolderSplitter( data_path=self.temp_path, storage_dir=self.storage_dir, storage_class=storage_class, ) # Upload data self.frames_inst.get_frames_and_metadata( filename_parser='parse_sms_name', ) def tearDown(self): """ Tear down temporary folder and files and stop S3 mock """ TempDirectory.cleanup_all() nose.tools.assert_equal(os.path.isdir(self.temp_path), False) self.mock.stop() def test_get_global_meta(self): global_meta = self.frames_inst.get_global_meta() self.assertEqual(global_meta['nbr_frames'], 6) self.assertEqual(global_meta['bit_depth'], 'uint16') self.assertEqual(global_meta['nbr_channels'], 3) self.assertEqual(global_meta['nbr_slices'], 2) self.assertEqual(global_meta['nbr_timepoints'], 1) self.assertEqual(global_meta['nbr_positions'], 1) self.assertEqual(global_meta['im_colors'], 1) self.assertEqual(global_meta['im_height'], self.im.shape[0]) self.assertEqual(global_meta['im_width'], self.im.shape[1]) def test_get_global_json(self): global_json = self.frames_inst.get_global_json() self.assertDictEqual(global_json, self.meta_dict) def test_get_frames_meta(self): frames_meta = self.frames_inst.get_frames_meta() for i, (c, z) in enumerate(itertools.product(range(3), range(2))): # Validate file name expected_name = 'im_c00{}_z00{}_t000_p050.png'.format(c, z) self.assertEqual(frames_meta.loc[i, 'file_name'], expected_name) # Validate checksum expected_sha = meta_utils.gen_sha256(self.im + 5000 * z) self.assertEqual(frames_meta.loc[i, 'sha256'], expected_sha) # Validate indices self.assertEqual(frames_meta.loc[i, 'channel_idx'], c) self.assertEqual(frames_meta.loc[i, 'slice_idx'], z) self.assertEqual(frames_meta.loc[i, 'time_idx'], 0) self.assertEqual(frames_meta.loc[i, 'pos_idx'], 50) def test_get_frames_json(self): frames_json = self.frames_inst.get_frames_json() # Channels will be sorted sorted_channels = natsort.natsorted(self.channel_names) self.assertEqual(len(frames_json), 6) for i, (c, z) in enumerate(itertools.product(range(3), range(2))): frame_i = frames_json[i] meta_info = json.loads(frame_i['IJMetadata']['Info']) self.assertDictEqual( meta_info, { "c": sorted_channels[c], "z": z }, ) frame_i['BitsPerSample'] = 16 frame_i['ImageWidth'] = 15 frame_i['ImageLength'] = 10 def test_set_frame_info(self): meta_dict = { 'PixelType': 'RGB', 'BitDepth': 8, 'Width': 250, 'Height': 150, } self.frames_inst.set_frame_info(meta_dict) self.assertEqual(self.frames_inst.bit_depth, 'uint8') self.assertEqual(self.frames_inst.im_colors, 3) self.assertListEqual(self.frames_inst.frame_shape, [150, 250]) def test_set_frame_info_uint16(self): meta_dict = { 'PixelType': 'GRAY', 'BitDepth': 16, 'Width': 250, 'Height': 150, } self.frames_inst.set_frame_info(meta_dict) self.assertEqual(self.frames_inst.bit_depth, 'uint16') self.assertEqual(self.frames_inst.im_colors, 1) @nose.tools.raises(ValueError) def test_set_frame_info(self): meta_dict = { 'PixelType': 'RGB', 'BitDepth': 'float32', 'Width': 250, 'Height': 150, } self.frames_inst.set_frame_info(meta_dict) def test_set_frame_info_from_file(self): file_name = 'img_phase_t000_p050_z001.tif' file_path = os.path.join(self.temp_path, file_name) self.frames_inst.set_frame_info_from_file(file_path) self.assertEqual(self.frames_inst.bit_depth, 'uint16') self.assertEqual(self.frames_inst.im_colors, 1) self.assertListEqual(self.frames_inst.frame_shape, [10, 15]) def test_set_frame_info_from_file_color(self): self.tempdir.makedir('test_dir') file_path = os.path.join(self.temp_path, 'test_dir/im_temp.tif') im = np.zeros((10, 20, 3), dtype=np.uint8) tifffile.imsave(file_path, im) self.frames_inst.set_frame_info_from_file(file_path) self.assertEqual(self.frames_inst.bit_depth, 'uint8') self.assertEqual(self.frames_inst.im_colors, 3) self.assertListEqual(self.frames_inst.frame_shape, [10, 20]) @nose.tools.raises(ValueError) def test_set_frame_info_from_file_float(self): self.tempdir.makedir('test_dir') file_path = os.path.join(self.temp_path, 'test_dir/im_temp.tif') im = np.zeros((10, 20, 3), dtype=np.float32) tifffile.imsave(file_path, im) self.frames_inst.set_frame_info_from_file(file_path) def test_set_frame_meta(self): parse_func = getattr(file_parsers, 'parse_sms_name') file_name = 'im_weird_channel_with_underscores_t020_z030_p040.tif' meta_row = self.frames_inst._set_frame_meta(parse_func, file_name) self.assertEqual( meta_row['channel_name'], 'weird_channel_with_underscores', ) self.assertEqual(meta_row['channel_idx'], 3) self.assertEqual(meta_row['time_idx'], 20) self.assertEqual(meta_row['slice_idx'], 30) self.assertEqual(meta_row['pos_idx'], 40) def test_get_frames_and_metadata(self): # Download uploaded data and compare to self.im for i, (c, z) in enumerate(itertools.product(range(3), range(2))): im_name = 'im_c00{}_z00{}_t000_p050.png'.format(c, z) key = "/".join([self.storage_dir, im_name]) byte_string = self.conn.Object(self.bucket_name, key).get()['Body'].read() # Construct an array from the bytes and decode image im = im_utils.deserialize_im(byte_string) # Assert that contents are the same self.assertEqual(im.dtype, np.uint16) numpy.testing.assert_array_equal(im, self.im + 5000 * z) @nose.tools.raises(AttributeError) def test_get_frames_and_metadata_no_parser(self): self.frames_inst.get_frames_and_metadata( filename_parser='nonexisting_function', ) @patch('concurrent.futures.ProcessPoolExecutor') def test_get_frames_no_metadata(self, MockPoolExecutor): # Magic mocking of multiprocessing MockPoolExecutor().__enter__().map = map_mock os.remove(self.json_filename) self.frames_inst.get_frames_and_metadata( filename_parser='parse_sms_name', ) frames_meta = self.frames_inst.get_frames_meta() for i, (c, z) in enumerate(itertools.product(range(3), range(2))): # Validate file name expected_name = 'im_c00{}_z00{}_t000_p050.png'.format(c, z) self.assertEqual(frames_meta.loc[i, 'file_name'], expected_name) # Validate checksum expected_sha = meta_utils.gen_sha256(self.im + 5000 * z) self.assertEqual(frames_meta.loc[i, 'sha256'], expected_sha) # Validate indices self.assertEqual(frames_meta.loc[i, 'channel_idx'], c) self.assertEqual(frames_meta.loc[i, 'slice_idx'], z) self.assertEqual(frames_meta.loc[i, 'time_idx'], 0) self.assertEqual(frames_meta.loc[i, 'pos_idx'], 50)
class test_FileOs(unittest.TestCase): """ Ensure file object can get set and output paths to files """ def setUp(self): self.d = TempDirectory() self.base_path = self.d.path + "/test" self.d.makedir(self.base_path) self.file_obj = FileOs(self.base_path) cats = [{ "name": "Zophie", "desc": "chubby" }, { "name": "Pooka", "desc": "fluffy" }] file_path = os.path.join(self.base_path, 'read_cats.json') with open(file_path, mode='w', encoding='utf-8') as feedsjson: json.dump(cats, feedsjson) def tearDown(self): self.d.cleanup() def test_creates_new_file_object(self): self.assertIsNotNone(self.file_obj) def test_init_sets_base_path(self): self.assertEqual(self.file_obj._base_path, self.base_path) def test_can_get_base_path(self): self.assertEqual(self.file_obj.get_base_path(), self.base_path) def test_can_set_base_path(self): new_base_path = self.d.path + "/new_path" self.file_obj.set_base_path(new_base_path) self.assertEqual(self.file_obj._base_path, new_base_path) def test_can_create_file_from_absolute_path(self): file_path = "/new_file.txt" full_path = self.base_path + file_path self.file_obj.set_file(file_path) self.assertTrue(os.path.isfile(full_path)) def test_can_create_file_from_filename(self): file_path = "new_file.txt" full_path = os.path.join(self.base_path, file_path) self.file_obj.set_file(file_path) self.assertTrue(os.path.isfile(full_path)) def test_will_create_file_in_folder_not_make_file_name_plus_folder(self): file_path = "new_file.txt" false_full_path = self.base_path + file_path full_path = os.path.join(self.base_path, file_path) self.file_obj.set_file(file_path) self.assertFalse(os.path.isfile(false_full_path)) self.assertTrue(os.path.isfile(full_path)) def test_will_set_content_in_file(self): cats = [{ "name": "Zophie", "desc": "chubby" }, { "name": "Pooka", "desc": "fluffy" }] file_path = "new_file.txt" full_path = os.path.join(self.base_path, file_path) self.file_obj.set_json_file(file_path, cats) with open(full_path, mode="r", encoding="utf-8") as feedsjson: test_contents = json.load(feedsjson) self.assertEqual(test_contents, cats) def test_can_read_contents_of_file(self): cats = [{ "name": "Zophie", "desc": "chubby" }, { "name": "Pooka", "desc": "fluffy" }] file_path = os.path.join(self.base_path, 'read_cats.json') test_cats = self.file_obj.read_file(file_path) self.assertEqual(test_cats, cats) def test_can_copy_file_from_one_place_to_another(self): source_path = 'read_cats.json' dest_path = 'pasted_cats.json' full_dest_path = os.path.join(self.base_path, dest_path) self.file_obj.copy_file(source_path, dest_path) self.assertTrue(os.path.isfile(full_dest_path)) def test_can_return_parent_folder_name(self): full_path = 'archive/2019-01-18_13-53-00/Dvndh6DUYAAF1rM.jpg' parent_name = self.file_obj.get_parent_folder_name(full_path) self.assertEqual(parent_name, '2019-01-18_13-53-00')
class TestPathSource(TestCase): def setUp(self): self.dir = TempDirectory() self.addCleanup(self.dir.cleanup) def test_abc(self): self.assertTrue(issubclass(Plugin, Source)) def test_schema_ok(self): p1 = self.dir.write('foo', b'f') p2 = self.dir.write('bar', b'b') compare( dict(type='paths', values=[p1, p2], repo='config'), Plugin.schema( dict(type='paths', values=[p1, p2], repo='config') )) def test_schema_wrong_type(self): text = "not a valid value for dictionary value @ data['type']" with ShouldFailSchemaWith(text): Plugin.schema(dict(type='bar', values=['/'])) def test_schema_extra_keys(self): with ShouldFailSchemaWith("extra keys not allowed @ data['foo']"): Plugin.schema(dict(type='paths', foo='bar')) def test_name_supplied(self): text = "not a valid value for dictionary value @ data['name']" with ShouldFailSchemaWith(text): Plugin.schema(dict(type='paths', name='foo')) def test_no_paths(self): text = "length of value must be at least 1 for dictionary value " \ "@ data['values']" with ShouldFailSchemaWith(text): Plugin.schema(dict(type='paths', values=[])) def test_path_not_string(self): text = "expected str @ data['values'][0]" with ShouldFailSchemaWith(text): Plugin.schema(dict(type='paths', values=[1])) def test_path_not_starting_with_slash(self): text = "'foo' is not an absolute path @ data['values'][0]" try: cwd = os.getcwd() os.chdir(self.dir.path) self.dir.makedir('foo') with ShouldFailSchemaWith(text): Plugin.schema(dict(type='paths', values=['foo'])) finally: os.chdir(cwd) def test_path_not_there(self): bad_path = self.dir.getpath('bad') text = "'%s' does not exist @ data['values'][0]" % bad_path with ShouldFailSchemaWith(text): Plugin.schema(dict(type='paths', values=[bad_path])) def test_interface(self): plugin = Plugin('source', name=None, repo='config', values=['/foo/bar']) compare(plugin.type, 'source') compare(plugin.name, None) compare(plugin.repo, 'config') compare(plugin.source_paths, ['/foo/bar'])
class TestDataUploaderLocalStorage(db_basetest.DBBaseTest): """ Test the data uploader using local storage """ def setUp(self): super().setUp() # Create temporary directory and write temp image self.tempdir = TempDirectory() self.temp_path = self.tempdir.path # Mock file storage self.tempdir.makedir('storage_mount_point') self.mount_point = os.path.join(self.temp_path, 'storage_mount_point') self.tempdir.makedir('storage_mount_point/raw_files') self.tempdir.makedir('storage_mount_point/raw_frames') # Test metadata parameters self.nbr_channels = 2 self.nbr_slices = 3 # Mock S3 dir self.storage_dir = "raw_frames/TEST-2005-06-09-20-00-00-1000" # Temporary file with 6 frames, tifffile stores channels first self.im = 50 * np.ones((6, 10, 15), dtype=np.uint16) self.im[0, :5, 3:12] = 50000 self.im[2, :5, 3:12] = 40000 self.im[4, :5, 3:12] = 30000 # Metadata self.description = 'ImageJ=1.52e\nimages=6\nchannels=2\nslices=3\nmax=10411.0' # Save test tif file self.file_path = os.path.join(self.temp_path, "A1_2_PROTEIN_test.tif") tifffile.imsave( self.file_path, self.im, description=self.description, ) self.dataset_serial = 'TEST-2005-06-09-20-00-00-1000' # Create csv file for upload upload_dict = { 'dataset_id': [self.dataset_serial], 'file_name': [self.file_path], 'description': ['Testing'], 'parent_dataset_id': [None], } upload_csv = pd.DataFrame.from_dict(upload_dict) self.csv_path = os.path.join(self.temp_path, "test_upload.csv") upload_csv.to_csv(self.csv_path) self.credentials_path = os.path.join( self.main_dir, 'db_credentials.json', ) self.config_path = os.path.join( self.temp_path, 'config_tif_id.json', ) config = { "upload_type": "frames", "frames_format": "tif_id", "microscope": "Leica microscope CAN bus adapter", "filename_parser": "parse_ml_name", "storage": "local", "storage_access": self.mount_point } json_ops.write_json_file(config, self.config_path) def tearDown(self): """ Rollback database session. Tear down temporary folder and file structure, stop moto mock """ super().tearDown() TempDirectory.cleanup_all() self.assertFalse(os.path.isdir(self.temp_path)) @patch('imaging_db.database.db_operations.session_scope') def test_upload_frames(self, mock_session): mock_session.return_value.__enter__.return_value = self.session data_uploader.upload_data_and_update_db( csv=self.csv_path, login=self.credentials_path, config=self.config_path, ) # Query database to find data_set and frames datasets = self.session.query(db_ops.DataSet) \ .filter(db_ops.DataSet.dataset_serial == self.dataset_serial) self.assertEqual(datasets.count(), 1) dataset = datasets[0] self.assertEqual(dataset.id, 1) self.assertTrue(dataset.frames) self.assertEqual(dataset.dataset_serial, self.dataset_serial) date_time = dataset.date_time self.assertEqual(date_time.year, 2005) self.assertEqual(date_time.month, 6) self.assertEqual(date_time.day, 9) self.assertEqual(dataset.microscope, "Leica microscope CAN bus adapter") self.assertEqual(dataset.description, 'Testing') # query frames_global global_query = self.session.query(db_ops.FramesGlobal) \ .join(db_ops.DataSet) \ .filter(db_ops.DataSet.dataset_serial == self.dataset_serial) self.assertEqual( global_query[0].storage_dir, self.storage_dir, ) self.assertEqual( global_query[0].nbr_frames, self.nbr_channels * self.nbr_slices, ) im_shape = self.im.shape self.assertEqual( global_query[0].im_width, im_shape[2], ) self.assertEqual( global_query[0].im_height, im_shape[1], ) self.assertEqual(global_query[0].nbr_slices, self.nbr_slices) self.assertEqual( global_query[0].nbr_channels, self.nbr_channels, ) self.assertEqual( global_query[0].nbr_positions, 1, ) self.assertEqual( global_query[0].nbr_timepoints, 1, ) self.assertEqual( global_query[0].im_colors, 1, ) self.assertEqual( global_query[0].bit_depth, 'uint16', ) # query frames frames = self.session.query(db_ops.Frames) \ .join(db_ops.FramesGlobal) \ .join(db_ops.DataSet) \ .filter(db_ops.DataSet.dataset_serial == self.dataset_serial) \ .order_by(db_ops.Frames.file_name) # Images are separated by slice first then channel im_order = [0, 2, 4, 1, 3, 5] it = itertools.product(range(self.nbr_channels), range(self.nbr_slices)) for i, (c, z) in enumerate(it): im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z) self.assertEqual(frames[i].file_name, im_name) self.assertEqual(frames[i].channel_idx, c) self.assertEqual(frames[i].slice_idx, z) self.assertEqual(frames[i].time_idx, 0) self.assertEqual(frames[i].pos_idx, 0) sha256 = meta_utils.gen_sha256(self.im[im_order[i], ...]) self.assertEqual(frames[i].sha256, sha256) # Download frames from storage and compare to originals it = itertools.product(range(self.nbr_channels), range(self.nbr_slices)) for i, (c, z) in enumerate(it): im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z) im_path = os.path.join(self.mount_point, self.storage_dir, im_name) im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH) nose.tools.assert_equal(im.dtype, np.uint16) numpy.testing.assert_array_equal(im, self.im[im_order[i], ...]) @patch('imaging_db.database.db_operations.session_scope') def test_upload_frames_already_in_db(self, mock_session): mock_session.return_value.__enter__.return_value = self.session data_uploader.upload_data_and_update_db( csv=self.csv_path, login=self.credentials_path, config=self.config_path, overwrite=True, ) # Try uploading a second time data_uploader.upload_data_and_update_db( csv=self.csv_path, login=self.credentials_path, config=self.config_path, overwrite=True, ) @patch('imaging_db.database.db_operations.session_scope') def test_upload_file(self, mock_session): # Upload the same file but as file instead of frames mock_session.return_value.__enter__.return_value = self.session config_path = os.path.join( self.temp_path, 'config_file.json', ) config = { "upload_type": "file", "microscope": "Mass Spectrometry", "storage": "local", "storage_access": self.mount_point } json_ops.write_json_file(config, config_path) data_uploader.upload_data_and_update_db( csv=self.csv_path, login=self.credentials_path, config=config_path, ) # Query database to find data_set and file_global datasets = self.session.query(db_ops.DataSet) \ .filter(db_ops.DataSet.dataset_serial == self.dataset_serial) self.assertEqual(datasets.count(), 1) dataset = datasets[0] self.assertEqual(dataset.id, 1) self.assertFalse(dataset.frames) self.assertEqual(dataset.dataset_serial, self.dataset_serial) date_time = dataset.date_time self.assertEqual(date_time.year, 2005) self.assertEqual(date_time.month, 6) self.assertEqual(date_time.day, 9) self.assertEqual(dataset.microscope, "Mass Spectrometry") self.assertEqual(dataset.description, 'Testing') # query file_global file_global = self.session.query(db_ops.FileGlobal) \ .join(db_ops.DataSet) \ .filter(db_ops.DataSet.dataset_serial == self.dataset_serial) \ .one() expected_dir = "raw_files/TEST-2005-06-09-20-00-00-1000" self.assertEqual( file_global.storage_dir, expected_dir, ) expected_meta = {'file_origin': self.file_path} self.assertDictEqual(file_global.metadata_json, expected_meta) self.assertEqual(file_global.data_set, dataset) sha256 = meta_utils.gen_sha256(self.file_path) self.assertEqual(file_global.sha256, sha256) # Check that file has been uploaded file_path = os.path.join(self.mount_point, expected_dir, 'A1_2_PROTEIN_test.tif') self.assertTrue(os.path.exists(file_path)) @patch('imaging_db.database.db_operations.session_scope') def test_upload_file_already_in_db(self, mock_session): # Upload the same file but as file instead of frames mock_session.return_value.__enter__.return_value = self.session config_path = os.path.join( self.temp_path, 'config_file.json', ) config = { "upload_type": "file", "microscope": "Mass Spectrometry", "storage": "local", "storage_access": self.mount_point } json_ops.write_json_file(config, config_path) data_uploader.upload_data_and_update_db( csv=self.csv_path, login=self.credentials_path, config=config_path, overwrite=True, ) # Try uploading a second time data_uploader.upload_data_and_update_db( csv=self.csv_path, login=self.credentials_path, config=config_path, overwrite=True, ) @patch('imaging_db.database.db_operations.session_scope') def test_upload_ometif(self, mock_session): mock_session.return_value.__enter__.return_value = self.session dataset_serial = 'ISP-2005-01-01-01-00-00-0001' # Temporary frame im = np.ones((10, 15), dtype=np.uint16) # Metadata ijmeta = { "Info": json.dumps({ "InitialPositionList": [{ "Label": "Pos1" }, { "Label": "Pos3" }] }), } channel_ids = [1, 2] im_names = ['test_Pos1.ome.tif', 'test_Pos3.ome.tif'] for i, c in enumerate(channel_ids): mmmetadata = json.dumps({ "ChannelIndex": c, "Slice": 20, "FrameIndex": 30, "PositionIndex": 40, "Channel": 'channel_{}'.format(c), }) extra_tags = [('MicroManagerMetadata', 's', 0, mmmetadata, True)] # Save test ome tif file file_path = os.path.join(self.temp_path, im_names[i]) tifffile.imsave( file_path, im + i * 10000, ijmetadata=ijmeta, extratags=extra_tags, ) schema_file_path = os.path.realpath( os.path.join(self.main_dir, 'metadata_schema.json'), ) # Create csv file for upload upload_dict = { 'dataset_id': [dataset_serial], 'file_name': [self.temp_path], 'description': ['Testing'], 'positions': [[1, 3]], 'schema_filename': [schema_file_path], } upload_csv = pd.DataFrame.from_dict(upload_dict) csv_path = os.path.join(self.temp_path, "test_ometif_upload.csv") upload_csv.to_csv(csv_path) config_path = os.path.join( self.temp_path, 'config_ome_tiff.json', ) config = { "upload_type": "frames", "frames_format": "ome_tiff", "microscope": "", "schema_filename": "metadata_schema.json", "storage": "local", "storage_access": self.mount_point } json_ops.write_json_file(config, config_path) # Upload data data_uploader.upload_data_and_update_db( csv=csv_path, login=self.credentials_path, config=config_path, ) # Query database to find data_set and frames datasets = self.session.query(db_ops.DataSet) \ .filter(db_ops.DataSet.dataset_serial == dataset_serial) self.assertEqual(datasets.count(), 1) dataset = datasets[0] self.assertEqual(dataset.id, 1) self.assertTrue(dataset.frames) self.assertEqual(dataset.dataset_serial, dataset_serial) date_time = dataset.date_time self.assertEqual(date_time.year, 2005) self.assertEqual(date_time.month, 1) self.assertEqual(date_time.day, 1) self.assertEqual(dataset.description, 'Testing') # query frames_global global_query = self.session.query(db_ops.FramesGlobal) \ .join(db_ops.DataSet) \ .filter(db_ops.DataSet.dataset_serial == dataset_serial) self.assertEqual( global_query[0].storage_dir, 'raw_frames/' + dataset_serial, ) self.assertEqual( global_query[0].nbr_frames, 2, ) im_shape = im.shape self.assertEqual( global_query[0].im_width, im_shape[1], ) self.assertEqual( global_query[0].im_height, im_shape[0], ) self.assertEqual(global_query[0].nbr_slices, 1) self.assertEqual( global_query[0].nbr_channels, 2, ) self.assertEqual( global_query[0].nbr_positions, 1, ) self.assertEqual( global_query[0].nbr_timepoints, 1, ) self.assertEqual( global_query[0].im_colors, 1, ) self.assertEqual( global_query[0].bit_depth, 'uint16', ) # query frames frames = self.session.query(db_ops.Frames) \ .join(db_ops.FramesGlobal) \ .join(db_ops.DataSet) \ .filter(db_ops.DataSet.dataset_serial == dataset_serial) \ .order_by(db_ops.Frames.file_name) shas = [meta_utils.gen_sha256(im), meta_utils.gen_sha256(im + 10000)] for i, c in enumerate(channel_ids): im_name = 'im_c00{}_z020_t030_p040.png'.format(c) self.assertEqual(frames[i].file_name, im_name) self.assertEqual(frames[i].channel_idx, c) self.assertEqual(frames[i].channel_name, 'channel_{}'.format(c)) self.assertEqual(frames[i].slice_idx, 20) self.assertEqual(frames[i].time_idx, 30) self.assertEqual(frames[i].pos_idx, 40) self.assertEqual(frames[i].sha256, shas[i]) # # Download frames from storage and compare to originals for i, c in enumerate(channel_ids): im_name = 'im_c00{}_z020_t030_p040.png'.format(c) im_path = os.path.join( self.mount_point, 'raw_frames', dataset_serial, im_name, ) im_out = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH) nose.tools.assert_equal(im.dtype, np.uint16) numpy.testing.assert_array_equal(im_out, im + i * 10000) @patch('imaging_db.database.db_operations.session_scope') def test_upload_tiffolder(self, mock_session): mock_session.return_value.__enter__.return_value = self.session dataset_serial = 'SMS-2010-01-01-01-00-00-0005' # Temporary frame im = np.ones((10, 15), dtype=np.uint8) # Save test tif files self.tempdir.makedir('tiffolder') tif_dir = os.path.join(self.temp_path, 'tiffolder') channel_names = ['phase', 'brightfield', '666'] # Write files in dir for c_name in channel_names: for z in range(2): file_name = 'img_{}_t060_p050_z00{}.tif'.format(c_name, z) file_path = os.path.join(tif_dir, file_name) ijmeta = {"Info": json.dumps({"c": c_name, "z": z})} tifffile.imsave( file_path, im + 50 * z, ijmetadata=ijmeta, ) # Write external metadata in dir self.meta_dict = { 'Summary': { 'Slices': 6, 'PixelType': 'GRAY8', 'Time': '2018-11-01 19:20:34 -0700', 'z-step_um': 0.5, 'PixelSize_um': 0, 'BitDepth': 8, 'Width': 15, 'Height': 10 }, } self.json_filename = os.path.join(tif_dir, 'metadata.txt') json_ops.write_json_file(self.meta_dict, self.json_filename) # Create csv file for upload upload_dict = { 'dataset_id': [dataset_serial], 'file_name': [tif_dir], 'description': ['Testing tifffolder upload'], } upload_csv = pd.DataFrame.from_dict(upload_dict) csv_path = os.path.join(self.temp_path, "test_tiffolder_upload.csv") upload_csv.to_csv(csv_path) config_path = os.path.join( self.temp_path, 'config_tiffolder.json', ) config = { "upload_type": "frames", "frames_format": "tif_folder", "microscope": "CZDRAGONFLY-PC", "filename_parser": "parse_sms_name", "storage": "local", "storage_access": self.mount_point } json_ops.write_json_file(config, config_path) # Upload data data_uploader.upload_data_and_update_db( csv=csv_path, login=self.credentials_path, config=config_path, ) # Query database to find data_set and frames datasets = self.session.query(db_ops.DataSet) \ .filter(db_ops.DataSet.dataset_serial == dataset_serial) self.assertEqual(datasets.count(), 1) dataset = datasets[0] self.assertTrue(dataset.frames) self.assertEqual(dataset.dataset_serial, dataset_serial) date_time = dataset.date_time self.assertEqual(date_time.year, 2010) self.assertEqual(date_time.month, 1) self.assertEqual(date_time.day, 1) self.assertEqual(dataset.description, 'Testing tifffolder upload') # query frames_global global_query = self.session.query(db_ops.FramesGlobal) \ .join(db_ops.DataSet) \ .filter(db_ops.DataSet.dataset_serial == dataset_serial) self.assertEqual( global_query[0].storage_dir, 'raw_frames/' + dataset_serial, ) self.assertEqual( global_query[0].nbr_frames, 6, ) self.assertEqual( global_query[0].im_width, 15, ) self.assertEqual( global_query[0].im_height, 10, ) self.assertEqual(global_query[0].nbr_slices, 2) self.assertEqual( global_query[0].nbr_channels, 3, ) self.assertEqual( global_query[0].nbr_positions, 1, ) self.assertEqual( global_query[0].nbr_timepoints, 1, ) self.assertEqual( global_query[0].im_colors, 1, ) self.assertEqual( global_query[0].bit_depth, 'uint8', ) # query frames frames = self.session.query(db_ops.Frames) \ .join(db_ops.FramesGlobal) \ .join(db_ops.DataSet) \ .filter(db_ops.DataSet.dataset_serial == dataset_serial) \ .order_by(db_ops.Frames.file_name) # Validate content # Channel numbers will be assigned alphabetically channel_names.sort() for i, (c, z) in enumerate(itertools.product(range(3), range(2))): im_name = 'im_c00{}_z00{}_t060_p050.png'.format(c, z) self.assertEqual(frames[i].file_name, im_name) self.assertEqual(frames[i].channel_idx, c) self.assertEqual(frames[i].channel_name, channel_names[c]) self.assertEqual(frames[i].slice_idx, z) self.assertEqual(frames[i].time_idx, 60) self.assertEqual(frames[i].pos_idx, 50) self.assertEqual( frames[i].sha256, meta_utils.gen_sha256(im + 50 * z), ) # # Download frames from storage and compare to originals for i in range(len(channel_names)): for z in range(2): im_name = 'im_c00{}_z00{}_t060_p050.png'.format(i, z) im_path = os.path.join( self.mount_point, 'raw_frames', dataset_serial, im_name, ) im_out = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH) nose.tools.assert_equal(im.dtype, np.uint8) numpy.testing.assert_array_equal(im_out, im + z * 50)
class TestImageInference(unittest.TestCase): @patch('micro_dl.inference.model_inference.load_model') def setUp(self, mock_model): """ Set up a directory with images """ mock_model.return_value = 'dummy_model' self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.tempdir.makedir('image_dir') self.tempdir.makedir('mask_dir') self.tempdir.makedir('model_dir') self.image_dir = os.path.join(self.temp_path, 'image_dir') self.mask_dir = os.path.join(self.temp_path, 'mask_dir') self.model_dir = os.path.join(self.temp_path, 'model_dir') # Create a temp image dir self.im = np.zeros((10, 16), dtype=np.uint8) self.frames_meta = aux_utils.make_dataframe() self.time_idx = 2 self.slice_idx = 3 for p in range(5): for c in range(3): im_name = aux_utils.get_im_name( time_idx=self.time_idx, channel_idx=c, slice_idx=self.slice_idx, pos_idx=p, ) cv2.imwrite(os.path.join(self.image_dir, im_name), self.im + c * 10) self.frames_meta = self.frames_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to image dir too self.frames_meta.to_csv(os.path.join(self.image_dir, 'frames_meta.csv')) # Save masks and mask meta self.mask_meta = aux_utils.make_dataframe() self.mask_channel = 50 for p in range(5): im_name = aux_utils.get_im_name( time_idx=self.time_idx, channel_idx=self.mask_channel, slice_idx=self.slice_idx, pos_idx=p, ) cv2.imwrite(os.path.join(self.mask_dir, im_name), self.im + 1) self.mask_meta = self.mask_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to mask dir too self.mask_meta.to_csv(os.path.join(self.mask_dir, 'frames_meta.csv')) # Setup model dir split_samples = { "train": [0, 1], "val": [2], "test": [3, 4], } aux_utils.write_json( split_samples, os.path.join(self.model_dir, 'split_samples.json'), ) # Make configs with fields necessary for 2D segmentation inference self.train_config = { 'network': { 'class': 'UNet2D', 'data_format': 'channels_first', 'depth': 1, 'width': 10, 'height': 10 }, 'dataset': { 'split_by_column': 'pos_idx', 'input_channels': [1, 2], 'target_channels': [self.mask_channel], 'model_task': 'segmentation', }, } self.inference_config = { 'model_dir': self.model_dir, 'model_fname': 'dummy_weights.hdf5', 'image_dir': self.image_dir, 'data_split': 'test', 'images': { 'image_format': 'zyx', 'image_ext': '.png', }, 'metrics': { 'metrics': ['mae'], 'metrics_orientations': ['xy'], }, 'masks': { 'mask_dir': self.mask_dir, 'mask_type': 'target', 'mask_channel': 50, } } # Instantiate class self.infer_inst = image_inference.ImagePredictor( train_config=self.train_config, inference_config=self.inference_config, ) def tearDown(self): """ Tear down temporary folder and file structure """ TempDirectory.cleanup_all() self.assertEqual(os.path.isdir(self.temp_path), False) def test_init(self): """ Test init of inference dataset """ # Check proper init self.assertEqual(self.infer_inst.model_dir, self.model_dir) self.assertEqual(self.infer_inst.image_dir, self.image_dir) self.assertEqual(self.infer_inst.data_format, 'channels_first') self.assertEqual(self.infer_inst.model, 'dummy_model') self.assertEqual(self.infer_inst.image_format, 'zyx') self.assertEqual(self.infer_inst.image_ext, '.png') self.assertFalse(self.infer_inst.mask_metrics) self.assertEqual(self.infer_inst.mask_dir, self.mask_dir) self.assertListEqual(self.infer_inst.metrics_orientations, ['xy']) self.assertEqual(self.infer_inst.num_overlap, 0) self.assertIsNone(self.infer_inst.stitch_inst) self.assertIsNone(self.infer_inst.tile_option) self.assertIsNone(self.infer_inst.crop_shape) def test_get_split_ids(self): split_col, infer_ids = self.infer_inst._get_split_ids() self.assertEqual(split_col, 'pos_idx') self.assertListEqual(infer_ids, [3, 4]) def test_get_split_ids_no_json(self): self.infer_inst.model_dir = self.infer_inst.image_dir split_col, infer_ids = self.infer_inst._get_split_ids() self.assertEqual(split_col, 'pos_idx') self.assertListEqual(infer_ids, [0, 1, 2, 3, 4]) def test_save_pred_image(self): im = np.zeros((1, 10, 15), dtype=np.uint8) im[:, 5, :] = 128 self.infer_inst.save_pred_image( predicted_image=im, time_idx=10, target_channel_idx=20, pos_idx=30, slice_idx=40, ) pred_name = os.path.join( self.model_dir, 'predictions/im_c020_z040_t010_p030.png', ) im_pred = cv2.imread(pred_name, cv2.IMREAD_ANYDEPTH) self.assertEqual(im_pred.dtype, np.uint16) self.assertTupleEqual(im_pred.shape, (10, 15)) # Prediction intensities are maximized to range self.assertEqual(im_pred.max(), 65535) self.assertEqual(im_pred.min(), 0) def test_estimate_metrics_xy(self): target = np.ones((10, 15, 5), dtype=np.float32) prediction = np.zeros_like(target) prediction[:5, :, :] = 1 pred_names = ['test1', 'test2', 'test3', 'test4', 'test5'] self.infer_inst.estimate_metrics(target, prediction, pred_names, None) metrics = self.infer_inst.df_xy self.assertTupleEqual(metrics.shape, (5, 2)) self.assertListEqual(list(metrics), ['mae', 'pred_name']) self.assertEqual(metrics.mae.mean(), 0.5) def test_estimate_metrics_xy_one_name(self): target = np.ones((10, 15, 5), dtype=np.float32) prediction = np.zeros_like(target) prediction[:5, :, :] = 1 self.infer_inst.estimate_metrics(target, prediction, ['test_name'], None) metrics = self.infer_inst.df_xy self.assertTupleEqual(metrics.shape, (5, 2)) self.assertListEqual(list(metrics), ['mae', 'pred_name']) self.assertEqual(metrics.mae.mean(), 0.5) def test_estimate_metrics_xyz(self): target = np.ones((10, 15, 5), dtype=np.float32) prediction = np.zeros_like(target) prediction[:5, :, :] = 1 self.infer_inst.metrics_orientations = ['xyz'] self.infer_inst.estimate_metrics(target, prediction, ['test_name'], None) metrics = self.infer_inst.df_xyz self.assertTupleEqual(metrics.shape, (1, 2)) self.assertListEqual(list(metrics), ['mae', 'pred_name']) self.assertEqual(metrics.mae[0], 0.5) self.assertEqual(metrics.pred_name[0], 'test_name') def test_estimate_metrics_xz(self): target = np.ones((10, 15, 5), dtype=np.float32) prediction = np.zeros_like(target) prediction[:5, :, :] = 1 self.infer_inst.metrics_orientations = ['xz'] self.infer_inst.estimate_metrics(target, prediction, ['test_name'], None) metrics = self.infer_inst.df_xz self.assertTupleEqual(metrics.shape, (10, 2)) self.assertListEqual(list(metrics), ['mae', 'pred_name']) self.assertEqual(metrics.mae[0], 0.0) self.assertEqual(metrics.mae[5], 1.0) self.assertEqual(metrics.pred_name[9], 'test_name_xz9') def test_estimate_metrics_yz(self): target = np.ones((10, 15, 5), dtype=np.float32) prediction = np.zeros_like(target) prediction[:5, :, :] = 1 self.infer_inst.metrics_orientations = ['yz'] self.infer_inst.estimate_metrics(target, prediction, ['test_name'], None) metrics = self.infer_inst.df_yz self.assertTupleEqual(metrics.shape, (15, 2)) self.assertListEqual(list(metrics), ['mae', 'pred_name']) self.assertEqual(metrics.mae[0], 0.5) self.assertEqual(metrics.pred_name[14], 'test_name_yz14') def test_get_mask(self): meta_row = dict.fromkeys(aux_utils.DF_NAMES) meta_row['channel_idx'] = self.mask_channel meta_row['time_idx'] = self.time_idx meta_row['slice_idx'] = self.slice_idx meta_row['pos_idx'] = 2 mask = self.infer_inst.get_mask(meta_row) self.assertTupleEqual(mask.shape, (8, 16)) self.assertEqual(mask.dtype, np.uint8) self.assertEqual(mask.max(), 1) self.assertEqual(mask.min(), 1) @patch('micro_dl.inference.model_inference.predict_large_image') def test_predict_2d(self, mock_predict): mock_predict.return_value = 1. + np.ones((1, 8, 16), dtype=np.float32) # Predict row 0 from inference dataset iterator pred_im, target_im, mask_im = self.infer_inst.predict_2d([0]) self.assertTupleEqual(pred_im.shape, (8, 16, 1)) self.assertEqual(pred_im.dtype, np.float32) self.assertEqual(pred_im.max(), 2.0) # Read saved prediction too pred_name = os.path.join( self.model_dir, 'predictions/im_c050_z003_t002_p003.png', ) im_pred = cv2.imread(pred_name, cv2.IMREAD_ANYDEPTH) self.assertEqual(im_pred.dtype, np.uint16) self.assertTupleEqual(im_pred.shape, (8, 16)) # Check target and no mask self.assertTupleEqual(target_im.shape, (8, 16, 1)) self.assertEqual(target_im.dtype, np.float32) self.assertEqual(target_im.max(), 1.) self.assertListEqual(mask_im, []) @patch('micro_dl.inference.model_inference.predict_large_image') def test_predict_2d_mask(self, mock_predict): self.infer_inst.crop_shape = [6, 10] self.infer_inst.mask_metrics = True mock_predict.return_value = 1. + np.ones((1, 6, 10), dtype=np.float32) # Predict row 0 from inference dataset iterator pred_im, target_im, mask_im = self.infer_inst.predict_2d([0]) self.assertTupleEqual(pred_im.shape, (6, 10, 1)) self.assertEqual(pred_im.dtype, np.float32) self.assertEqual(pred_im.max(), 2.0) # Read saved prediction too pred_name = os.path.join( self.model_dir, 'predictions/im_c050_z003_t002_p003.png', ) im_pred = cv2.imread(pred_name, cv2.IMREAD_ANYDEPTH) self.assertEqual(im_pred.dtype, np.uint16) self.assertTupleEqual(im_pred.shape, (6, 10)) # Check target and no mask self.assertTupleEqual(target_im.shape, (6, 10, 1)) self.assertEqual(target_im.dtype, np.float32) self.assertEqual(target_im.max(), 1.) self.assertTupleEqual(mask_im.shape, (6, 10, 1)) self.assertEqual(mask_im.dtype, np.uint8) @patch('micro_dl.inference.model_inference.predict_large_image') def test_run_prediction(self, mock_predict): mock_predict.return_value = 1. + np.ones((1, 8, 16), dtype=np.float32) # Run prediction. Should create a metrics_xy.csv in pred dir self.infer_inst.run_prediction() metrics = pd.read_csv( os.path.join(self.model_dir, 'predictions/metrics_xy.csv')) self.assertTupleEqual(metrics.shape, (2, 2)) # MAE should be 1. self.assertEqual(metrics.mae.mean(), 1.0) # There should be two rows, one per test index self.assertEqual(metrics.pred_name[0], 'im_c050_z003_t002_p003_xy0') self.assertEqual(metrics.pred_name[1], 'im_c050_z003_t002_p004_xy0') # There should be 2 predictions saved in pred dir for pos in range(3, 5): pred_name = os.path.join( self.model_dir, 'predictions/im_c050_z003_t002_p00{}.png'.format(pos), ) im_pred = cv2.imread(pred_name, cv2.IMREAD_ANYDEPTH) self.assertEqual(im_pred.dtype, np.uint16) self.assertTupleEqual(im_pred.shape, (8, 16))
class TestFolderTestCase(unittest.TestCase): def setUp(self): self.tempdir = TempDirectory(encoding='utf-8') def tearDown(self): self.tempdir.cleanup() def test_returns_false_if_parent_attributes_not_set(self): d = Folder(self.tempdir.path) with self.subTest(1): compare(self.tempdir.path, d.path) with self.subTest(2): compare(os.path.basename(self.tempdir.path), d.name) with self.subTest(3): compare(os.path.dirname(self.tempdir.path), d.parent) def test_returns_false_if_attributes_are_not_equal(self): temp_path = self.tempdir.path dir_1 = Folder(temp_path) dir_2 = Folder(self.tempdir.makedir('abc/123/JPG')) dir_3 = Folder(self.tempdir.makedir('abc/&%^/website')) d4 = self.tempdir.makedir('abc/123/ghi') d5 = Folder(os.path.join(temp_path, 'abc', 'hjig')) sorter_identity = self.tempdir.write( os.path.join(d4, SORTER_FOLDER_IDENTITY_FILENAME), '') dir_4 = Folder(d4) with self.subTest(1): compare(True, dir_1.exists) with self.subTest(2): compare(False, dir_1.for_sorter) with self.subTest(3): compare('FOLDERS', dir_1.category_folder) with self.subTest(4): compare(True, dir_2.for_sorter) with self.subTest(5): compare('image', dir_2.category_folder) with self.subTest(6): compare(True, dir_3.for_sorter) with self.subTest(7): compare(None, dir_3.category_folder) with self.subTest(8): compare('FOLDERS', dir_4.category_folder) with self.subTest(9): compare(True, os.path.isfile(sorter_identity)) with self.subTest(10): compare(True, dir_4.for_sorter) with self.subTest(11): compare(False, d5.for_sorter) def test_returns_false_if_folder_not_exists(self): temp_path = self.tempdir.path path = os.path.join(temp_path, 'bdk/ksks/a94') dir_1 = Folder(path) with self.subTest(1): compare(False, dir_1.exists) with self.subTest(2): self.assertRaises(FileNotFoundError, dir_1.create) with self.subTest(3): compare(False, dir_1.exists) dir_1.create(parents=True) with self.subTest(4): compare(True, dir_1.exists) def test_returns_false_if_grouping_failed(self): dir_1 = self.tempdir.makedir('abc/for/document') dir_2 = self.tempdir.makedir('abc/for/PDF') dir_3 = self.tempdir.makedir('abc/for/last') dir_ = self.tempdir.makedir('one/two') d1 = Folder(dir_1) d2 = Folder(dir_2) d3 = Folder(dir_3) with self.subTest(1): compare(dir_1, d1.path) d1.move_to(dir_) with self.subTest(2): compare(os.path.join(dir_, 'document'), d1.path) with self.subTest(3): compare(dir_2, d2.path) d2.move_to(dir_) with self.subTest(4): compare(os.path.join(dir_, 'document', 'PDF'), d2.path) with self.subTest(5): compare(dir_3, d3.path) d3.move_to(dir_) with self.subTest(6): compare(os.path.join(dir_, 'FOLDERS', 'last'), d3.path) def test_folder_grouping_with_files(self): write = lambda path: self.tempdir.write(path, '') isfile = os.path.isfile dir_ = self.tempdir.makedir('one/two') file_1 = write('abc/for/PDF/this long name.pdf') file_2 = write('abc/for/DOCX/y.docx') file_3 = write('abc/for/DOC/123 54.doc') file_4 = write('abc/for/none/xw') file_5 = write('abc/for/PDF/second.pdf') dir_1 = Folder(os.path.dirname(file_1)) dir_2 = Folder(os.path.dirname(file_2)) dir_3 = Folder(os.path.dirname(file_3)) dir_4 = Folder(os.path.dirname(file_4)) dir_5 = Folder(os.path.dirname(file_5)) dir_6 = Folder(dir_) with self.subTest(1): compare([True, True, True, True], [isfile(file_1), isfile(file_2), isfile(file_3), isfile(file_4)]) dir_1.move_to(dir_) dir_2.move_to(dir_) dir_3.move_to(dir_) dir_4.move_to(dir_) dir_5.move_to(dir_) dir_6.move_to(dir_) with self.subTest(3): self.tempdir.compare([ '{}/'.format('document'), '{}/'.format('FOLDERS'), '{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}'.format('FOLDERS', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/'.format('document', 'PDF'), '{}/{}/'.format('document', 'DOCX'), '{}/{}/'.format('document', 'DOC'), '{}/{}/'.format('FOLDERS', 'none'), '{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'), '{}/{}/{}'.format('document', 'PDF', 'second.pdf'), '{}/{}/{}'.format('document', 'DOCX', 'y.docx'), '{}/{}/{}'.format('document', 'DOC', '123 54.doc'), '{}/{}/{}'.format('FOLDERS', 'none', 'xw'), '{}/{}/{}'.format('document', 'PDF', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/{}'.format('document', 'DOCX', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/{}'.format('document', 'DOC', SORTER_FOLDER_IDENTITY_FILENAME), ], path=dir_) with self.subTest(4): compare(os.path.join(dir_, 'document', 'PDF'), dir_5.path) def test_folder_grouping_with_files_and_existing_destinations(self): temp_path = self.tempdir.path write = lambda path: self.tempdir.write(path, '') isfile = os.path.isfile dir_ = self.tempdir.makedir('one/two') self.tempdir.makedir('one/two/document/PDF/') self.tempdir.makedir('one/two/document/DOCX') self.tempdir.makedir('one/two/document/DOC') file_1 = write('abc/for/PDF/this long name.pdf') file_2 = write('abc/for/DOCX/y.docx') file_3 = write('abc/for/DOC/123 54.doc') file_4 = write('abc/for/none/xw') file_5 = write('abc/for/PDF/second.pdf') file_6 = write('abc/for/image/JPEG/abc.jpeg') dir_1 = Folder(os.path.dirname(file_1)) dir_2 = Folder(os.path.dirname(file_2)) dir_3 = Folder(os.path.dirname(file_3)) dir_4 = Folder(os.path.dirname(file_4)) dir_5 = Folder(os.path.dirname(file_5)) dir_6 = Folder(os.path.dirname(os.path.dirname(file_6))) dir_7 = Folder(dir_) with self.subTest(1): compare([True, True, True, True], [isfile(file_1), isfile(file_2), isfile(file_3), isfile(file_4)]) with self.subTest(2): compare('document', dir_1.category_folder) dir_1.move_to(dir_) dir_2.move_to(dir_) dir_3.move_to(dir_) dir_4.move_to(dir_) dir_5.move_to(dir_) dir_6.move_to(dir_) dir_7.move_to(dir_) with self.subTest(3): compare(True, dir_1.for_sorter) with self.subTest(4): self.tempdir.compare([ '{}/'.format('document'), '{}/'.format('image'), '{}/'.format('FOLDERS'), '{}/{}/'.format('document', 'PDF'), '{}/{}/'.format('document', 'DOCX'), '{}/{}/'.format('document', 'DOC'), '{}/{}/'.format('image', 'JPEG'), '{}/{}/'.format('FOLDERS', 'none'), '{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}'.format('image', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}'.format('FOLDERS', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'), '{}/{}/{}'.format('document', 'PDF', 'second.pdf'), '{}/{}/{}'.format('document', 'DOCX', 'y.docx'), '{}/{}/{}'.format('document', 'DOC', '123 54.doc'), '{}/{}/{}'.format('image', 'JPEG', 'abc.jpeg'), '{}/{}/{}'.format('FOLDERS', 'none', 'xw'), '{}/{}/{}'.format('document', 'PDF', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/{}'.format('document', 'DOCX', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/{}'.format('document', 'DOC', SORTER_FOLDER_IDENTITY_FILENAME), ], path=dir_) with self.subTest(5): compare(os.path.join(dir_, 'document', 'PDF'), dir_5.path) with self.subTest(6): compare(True, os.path.isdir(dir_1.path)) with self.subTest(7): compare(os.path.join(temp_path, dir_, 'document', 'PDF'), dir_1.path) with self.subTest(8): compare(True, os.path.isfile(os.path.join( dir_1.path, SORTER_FOLDER_IDENTITY_FILENAME))) with self.subTest(9): compare(False, os.path.exists(os.path.dirname(file_1))) def test_retuns_false_folder_with_multiple_subfolders_relocation_failse(self): temp_path = self.tempdir.path write = lambda path: self.tempdir.write(path, '') dir_ = self.tempdir.makedir('one/two') self.tempdir.makedir('one/two/document/PDF/') file_1 = write('abc/for/PDF/this long name.pdf') write('abc/for/PDF/somefolder/another/and another/JPEG/abc.jpeg') dir_1 = Folder(os.path.dirname(file_1)) dir_1.move_to(dir_) with self.subTest(1): self.tempdir.compare([ '{}/'.format('document'), '{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/'.format('document', 'PDF'), '{}/{}/{}'.format('document', 'PDF', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/{}/'.format('document', 'PDF', 'somefolder'), '{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'), '{}/{}/{}/{}/'.format('document', 'PDF', 'somefolder', 'another'), '{}/{}/{}/{}/{}/'.format('document', 'PDF', 'somefolder', 'another', 'and another'), '{}/{}/{}/{}/{}/{}/'.format('document', 'PDF', 'somefolder', 'another', 'and another', 'JPEG'), '{}/{}/{}/{}/{}/{}/{}'.format('document', 'PDF', 'somefolder', 'another', 'and another', 'JPEG', 'abc.jpeg'), ], path=dir_) def test_returns_false_if_folder_relocation_with_ignore_file_succeeds(self): temp_path = self.tempdir.path write = lambda path: self.tempdir.write(path, '') dir_ = self.tempdir.makedir('one/two') self.tempdir.makedir('one/two/document/PDF/') file_1 = write('abc/for/PDF/this long name.pdf') file_2 = write( 'abc/for/PDF/somefolder/another/and another/JPEG/abc.jpeg') file_3 = write('abc/for/PDF/somefolder/%s' % SORTER_IGNORE_FILENAME) with self.subTest(1): compare(True, os.path.isfile(file_3)) with self.subTest(2): compare(True, os.path.isdir(os.path.dirname(file_2))) dir_1 = Folder(os.path.dirname(file_1)) dir_1.move_to(dir_) with self.subTest(3): self.tempdir.compare([ '{}/'.format('document'), '{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/'.format('document', 'PDF'), '{}/{}/{}'.format('document', 'PDF', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'), ], path=dir_) def test_returns_false_if_original_folder_exists(self): temp_path = self.tempdir.path write = lambda path: self.tempdir.write(path, '') isfile = os.path.isfile dir_ = self.tempdir.makedir('one/two') self.tempdir.makedir('one/two/document/PDF/') file_1 = write('abc/for/PDF/this long name.pdf') dir_1 = Folder(os.path.dirname(file_1)) dir_1.move_to(dir_) with self.subTest(1): self.tempdir.compare([ '{}/'.format('document'), '{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/'.format('document', 'PDF'), '{}/{}/{}'.format('document', 'PDF', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'), ], path=dir_) with self.subTest(2): compare(False, os.path.isdir(os.path.dirname(file_1))) def test_returns_false_if_original_folder_not_exists(self): temp_path = self.tempdir.path write = lambda path: self.tempdir.write(path, '') isfile = os.path.isfile dir_ = self.tempdir.makedir('one/two') dir_2 = self.tempdir.makedir('one/two/document/PDF/') file_1 = write('abc/for/PDF/this long name.pdf') file_2 = write( 'abc/for/PDF/somefolder/another/and another/JPEG/abc.jpeg') file_3 = write('abc/for/PDF/somefolder/%s' % SORTER_IGNORE_FILENAME) with self.subTest(1): compare(True, os.path.isfile(file_3)) with self.subTest(2): compare(True, os.path.isdir(os.path.dirname(file_2))) dir_1 = Folder(os.path.dirname(file_1)) dir_1.move_to(dir_) with self.subTest(3): self.tempdir.compare([ '{}/'.format('document'), '{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/'.format('document', 'PDF'), '{}/{}/{}'.format('document', 'PDF', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'), ], path=dir_) with self.subTest(4): compare(True, os.path.isdir(dir_2)) def test_returns_false_if_grouping_in_same_folder_failed(self): temp_path = self.tempdir.path write = lambda path: self.tempdir.write(path, '') isfile = os.path.isfile dir_ = self.tempdir.makedir('one/two') file_1 = write('one/two/PDF/this long name.pdf') file_2 = write( 'one/two/PDF/somefolder/another/and another/JPEG/abc.jpeg') file_3 = write('one/two/PDF/somefolder/%s' % SORTER_IGNORE_FILENAME) with self.subTest(1): compare(True, os.path.isfile(file_3)) with self.subTest(2): compare(True, os.path.isdir(os.path.dirname(file_2))) dir_1 = Folder(os.path.dirname(file_1)) dir_1.group(dir_, by_extension=True) with self.subTest(3): compare(True, os.path.isfile(os.path.join(temp_path, 'one', 'two', 'document', 'PDF', SORTER_FOLDER_IDENTITY_FILENAME))) with self.subTest(4): self.tempdir.compare([ '{}/'.format('document'), '{}/'.format('PDF'), '{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}'.format('document', SORTER_IGNORE_FILENAME), '{}/{}/'.format('document', 'PDF'), '{}/{}/{}'.format('document', 'PDF', SORTER_FOLDER_IDENTITY_FILENAME), '{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'), '{}/{}/'.format('PDF', 'somefolder'), '{}/{}/{}'.format('PDF', 'somefolder', SORTER_IGNORE_FILENAME), '{}/{}/{}/'.format('PDF', 'somefolder', 'another'), '{}/{}/{}/{}/'.format('PDF', 'somefolder', 'another', 'and another'), '{}/{}/{}/{}/{}/'.format('PDF', 'somefolder', 'another', 'and another', 'JPEG'), '{}/{}/{}/{}/{}/{}'.format('PDF', 'somefolder', 'another', 'and another', 'JPEG', 'abc.jpeg') ], path=dir_) with self.subTest(5): compare(True, os.path.isdir(os.path.dirname(file_1)))
def test_watcher_with_file_filter(self): work_directory = TempDirectory() work_directory.write('a.h', b'') work_directory.write('a.c', b'') work_directory.write('a.cc', b'') work_directory.write('CMakeLists.txt', b'') work_directory.write('blah.txt', b'') work_directory.makedir('blah') work_directory.write(['blah', 'test_dummy.c'], b'') wd_len = len(work_directory.path) + 1 w = Watcher(work_directory.path, None, []) watchstate = w.poll() filelist = [f[wd_len:] for f in watchstate.inserts] filelist.sort() assert filelist == [ 'CMakeLists.txt', 'a.c', 'a.cc', 'a.h', 'blah.txt', 'blah/test_dummy.c' ] w = Watcher(work_directory.path, None, source_patterns=['CMakeLists.txt']) watchstate = w.poll() filelist = [f[wd_len:] for f in watchstate.inserts] filelist.sort() assert filelist == ['CMakeLists.txt'] w = Watcher(work_directory.path, None, source_patterns=['*.txt']) watchstate = w.poll() filelist = [f[wd_len:] for f in watchstate.inserts] filelist.sort() assert filelist == ['CMakeLists.txt', 'blah.txt'] w = Watcher(work_directory.path, None, source_patterns=['?.cc']) watchstate = w.poll() filelist = [f[wd_len:] for f in watchstate.inserts] filelist.sort() assert filelist == ['a.cc'] w = Watcher(work_directory.path, None, source_patterns=['blah']) watchstate = w.poll() filelist = [f[wd_len:] for f in watchstate.inserts] filelist.sort() assert filelist == ['blah.txt', 'blah/test_dummy.c'] w = Watcher(work_directory.path, None, source_patterns=['blah/']) watchstate = w.poll() filelist = [f[wd_len:] for f in watchstate.inserts] filelist.sort() assert filelist == ['blah/test_dummy.c'] w = Watcher(work_directory.path, None, source_patterns=['*.txt'], source_exclusions=['blah.txt']) watchstate = w.poll() filelist = [f[wd_len:] for f in watchstate.inserts] filelist.sort() assert filelist == ['CMakeLists.txt'] w = Watcher(work_directory.path, None, source_patterns=['*.txt'], source_exclusions=['blah.*']) watchstate = w.poll() filelist = [f[wd_len:] for f in watchstate.inserts] filelist.sort() assert filelist == ['CMakeLists.txt'] w = Watcher(work_directory.path, None, None, source_exclusions=['*.txt', 'blah']) watchstate = w.poll() filelist = [f[wd_len:] for f in watchstate.inserts] filelist.sort() assert filelist == ['a.c', 'a.cc', 'a.h']
class Test_Logging_setup(TestCase): def setUp(self): self.default_config = bbpgsql.configuration.config() self.td = TempDirectory() self.logdir = self.td.makedir('log') self.logfilepath = os.path.join(self.logdir, 'bbpgsql') self.config_path = self.td.write('bbpgsql', config_text) self.full_config = bbpgsql.configuration.config([self.config_path]) self.full_config.set('Logging', 'logfile', self.logfilepath) self.config_path = self.td.write('bad_level', config_bad_values) self.bad_config = bbpgsql.configuration.config([self.config_path]) self.config_path = self.td.write('section_only', config_only_section) self.section_only = bbpgsql.configuration.config([self.config_path]) def tearDown(self): self.td.cleanup() def test_default_config_lacks_logging(self): self.assertFalse(self.default_config.has_section('Logger')) def test_section_only_does_no_logging(self): log_config = bbpgsql.configuration.set_up_logging(self.section_only) # assert only null handler self.assertEqual(1, len(log_config['handlers'])) @patch('logging.config.dictConfig') def test_logger_sets_up_default_config(self, mock_dictConfig): mock_dictConfig.return_value = None bbpgsql.configuration.set_up_logging(self.default_config) self.assertTrue(mock_dictConfig.called) expected = ((default_log_config, ), {}) self.assertEqual(expected, mock_dictConfig.call_args) def test_logger_sets_log_level_from_configfile(self): log_config = bbpgsql.configuration.set_up_logging(self.full_config) self.assertEqual(DEBUG, log_config['loggers']['bbpgsql']['level']) def test_logger_rejects_bad_level_name(self): self.assertRaises(Exception, bbpgsql.configuration.set_up_logging, (self.bad_config)) def test_logger_gets_filename_from_config(self): handlers, formatters = \ bbpgsql.configuration.set_up_logger_file_handler(self.full_config) self.assertEqual(handlers, {'file': { 'class': 'logging.handlers.TimedRotatingFileHandler', 'formatter': 'file_formatter', 'filename': self.logfilepath, 'when': 'd', 'interval': 1, 'backupCount': 14 } } ) self.assertEqual(formatters, { 'file_formatter': { 'format': "%(asctime)s - %(levelname)s - %(message)s", } } ) def test_logger_does_nothing_if_section_is_missing(self): handlers, formatters = \ bbpgsql.configuration.set_up_logger_file_handler( self.default_config) self.assertEqual({}, handlers) self.assertEqual({}, formatters) def test_logger_gets_hostname_port_from_config(self): handlers, formatters = \ bbpgsql.configuration.set_up_logger_syslog_handler( self.full_config) self.assertEqual(handlers['syslog']['address'], ('localhost', 514)) def test_logger_does_nothing_if_syslog_host_not_defined(self): conf = SafeConfigParser() conf.add_section('Logging') conf.set('Logging', 'logport', '514') handlers, formatters = \ bbpgsql.configuration.set_up_logger_syslog_handler(conf) self.assertEqual({}, handlers) self.assertEqual({}, formatters) def test_logger_does_nothing_if_syslog_port_not_defined(self): conf = SafeConfigParser() conf.add_section('Logging') conf.set('Logging', 'loghost', 'localhost') handlers, formatters = \ bbpgsql.configuration.set_up_logger_syslog_handler(conf) self.assertEqual({}, handlers) self.assertEqual({}, formatters) def test_logger_uses_UDP_by_default(self): conf = SafeConfigParser() conf.add_section('Logging') conf.set('Logging', 'loghost', 'localhost') conf.set('Logging', 'logport', '514') handlers, formatters = \ bbpgsql.configuration.set_up_logger_syslog_handler(conf) syslog_handler = handlers[handlers.keys()[0]] self.assertEqual(socket.SOCK_DGRAM, syslog_handler['socktype']) def test_logger_uses_TCP_when_tcpon_is_false(self): conf = SafeConfigParser() conf.add_section('Logging') conf.set('Logging', 'loghost', 'localhost') conf.set('Logging', 'logport', '514') conf.set('Logging', 'logtcp', 'off') handlers, formatters = \ bbpgsql.configuration.set_up_logger_syslog_handler(conf) syslog_handler = handlers[handlers.keys()[0]] self.assertEqual(socket.SOCK_DGRAM, syslog_handler['socktype']) def test_logger_uses_TCP_when_tcpon_is_true(self): conf = SafeConfigParser() conf.add_section('Logging') conf.set('Logging', 'loghost', 'localhost') conf.set('Logging', 'logport', '514') conf.set('Logging', 'logtcp', 'on') handlers, formatters = \ bbpgsql.configuration.set_up_logger_syslog_handler(conf) syslog_handler = handlers[handlers.keys()[0]] self.assertEqual(socket.SOCK_STREAM, syslog_handler['socktype']) def test_logger_uses_facility_user_by_default(self): conf = SafeConfigParser() conf.add_section('Logging') conf.set('Logging', 'loghost', 'localhost') conf.set('Logging', 'logport', '514') handlers, formatters = \ bbpgsql.configuration.set_up_logger_syslog_handler(conf) syslog_handler = handlers[handlers.keys()[0]] self.assertEqual(logging.handlers.SysLogHandler.LOG_USER, syslog_handler['facility']) def test_logger_uses_facility_user_when_specified(self): conf = SafeConfigParser() conf.add_section('Logging') conf.set('Logging', 'loghost', 'localhost') conf.set('Logging', 'logport', '514') conf.set('Logging', 'logfacility', 'user') handlers, formatters = \ bbpgsql.configuration.set_up_logger_syslog_handler(conf) syslog_handler = handlers[handlers.keys()[0]] self.assertEqual(logging.handlers.SysLogHandler.LOG_USER, syslog_handler['facility']) def test_logger_uses_facility_local0_when_specified(self): conf = SafeConfigParser() conf.add_section('Logging') conf.set('Logging', 'loghost', 'localhost') conf.set('Logging', 'logport', '514') conf.set('Logging', 'logfacility', 'local0') handlers, formatters = \ bbpgsql.configuration.set_up_logger_syslog_handler(conf) syslog_handler = handlers[handlers.keys()[0]] self.assertEqual(logging.handlers.SysLogHandler.LOG_LOCAL0, syslog_handler['facility']) def test_logger_raises_exception_on_invalid_facility(self): conf = SafeConfigParser() conf.add_section('Logging') conf.set('Logging', 'loghost', 'localhost') conf.set('Logging', 'logport', '514') conf.set('Logging', 'logfacility', 'quidgybo') self.assertRaises(Exception, bbpgsql.configuration.set_up_logger_syslog_handler, conf) @patch('bbpgsql.configuration.set_up_logger_file_handler') @patch('bbpgsql.configuration.set_up_logger_syslog_handler') def test_set_up_logging_calls_handler_setups( self, mock_set_up_logger_syslog_handler, mock_set_up_logger_file_handler, ): mock_set_up_logger_syslog_handler.return_value = ({}, {}) mock_set_up_logger_file_handler.return_value = ({}, {}) log_config = bbpgsql.configuration.set_up_logging(self.full_config) from pprint import pprint pprint(log_config) self.assertEqual(1, mock_set_up_logger_file_handler.call_count) self.assertEqual(1, mock_set_up_logger_syslog_handler.call_count) @patch('bbpgsql.configuration.set_up_logger_file_handler') @patch('bbpgsql.configuration.set_up_logger_syslog_handler') def test_set_up_logging_does_not_setup_handlers_if_no_logging_section( self, mock_set_up_logger_syslog_handler, mock_set_up_logger_file_handler, ): bbpgsql.configuration.set_up_logging(self.default_config) self.assertEqual(0, mock_set_up_logger_file_handler.call_count) self.assertEqual(0, mock_set_up_logger_syslog_handler.call_count)
class TestConfigReaderTestCase(unittest.TestCase): def setUp(self): self.tempdir = TempDirectory() self.file_path = os.path.join(self.tempdir.path, fake.config_file()) def tearDown(self): self.tempdir.cleanup() def _get_config(self): file_path = self._get_config_file() config = ConfigReader(file_path) return config def _get_config_file(self): return fake.config_file(self.tempdir.path) def test_returns_false_if_key_not_set(self): file_path = self.tempdir.write(fake.config_file(), b'') with ConfigReader(file_path) as config: config.set('%name', '2') config.set('num%ber', '%23') with self.subTest(0): def raises_error(): config.set('attr', '%(num') self.assertRaises(ValueError, raises_error) with self.subTest(1): compare(config.get('%name'), 2) with self.subTest(2): compare(config.get('num%ber'), '%23') def test_returns_false_if_filename_not_absolute(self): with self.subTest(0): config = ConfigReader(self.file_path) self.assertTrue(os.path.isabs(config.filename)) with self.subTest(1): config = ConfigReader() self.assertTrue(os.path.isabs(config.filename)) with self.subTest(2): filename = os.path.join(os.getcwd(), 'settings.ini') config = ConfigReader() self.assertEqual(filename, config.filename) config.close() try: os.remove(config.filename) except FileNotFoundError: pass def test_returns_false_if_default_name_not_match(self): self.config = ConfigReader(self.file_path) expected = self.file_path self.assertEqual(self.config.filename, expected) self.config.close() def test_returns_false_if_name_not_changed(self): config = self._get_config() directory = fake.word() test_dir = self.tempdir.makedir(directory) path = os.path.join(test_dir, fake.config_file()) config.filename = path self.assertEqual(config.filename, path) config.close() def test_returns_false_if_config_file_not_exists(self): self.config = ConfigReader(self.file_path) self.assertFalse(os.path.isfile(self.config.filename)) self.config.close() def test_returns_false_if_config_file_exists(self): config = self._get_config() config.save() self.assertTrue(os.path.isfile(config.filename)) config.close() def test_returns_false_if_sections_not_exists(self): config = self._get_config() config.set('Sample', 'Example', 'MainSection') config.set('Sample', 'Example', 'OtherSection') expected = ['MainSection', 'OtherSection', 'main'] self.assertListEqual(sorted(config.sections), sorted(expected)) config.close() def test_returns_false_if_section_not_removed(self): config = self._get_config() config.set('Sample', 'Example', 'MainSection') config.set('Sample', 'Example', 'OtherSection') config.remove_section('main') expected = ['MainSection', 'OtherSection'] self.assertListEqual(sorted(config.sections), sorted(expected)) config.close() def test_returns_false_if_key_not_removed(self): config = self._get_config() config.set('Sample', 'Example', 'MainSection') config.set('Sample', 'Example', 'OtherSection') config.remove_option('Sample', 'MainSection') with self.subTest(0), self.assertRaises(MissingOptionError): config.get('Sample', section='MainSection') with self.subTest(1): expected = ['MainSection', 'main', 'OtherSection'] self.assertListEqual(sorted(config.sections), sorted(expected)) config.close() def test_returns_false_if_dict_not_returned(self): config = self._get_config() config.set('Sample', 'Example', 'MainSection') config.set('Sample', 'Example', 'OtherSection') config.set('Name', 'File', 'OtherSection') self.assertIsInstance(config.show(output=False), dict) config.close() def test_returns_false_if_json_not_dumped(self): config = self._get_config() config.set('Sample', 'Example', 'MainSection') config.set('Sample', 'Example', 'OtherSection') config.set('name', 'File', 'OtherSection') config.remove_section('main') s_io = StringIO() config.to_json(s_io) s_io.seek(0) expected = s_io.read() self.assertEqual(config.to_json(), expected) config.close() def test_returns_false_if_json_file_not_created(self): config = self._get_config() directory = fake.word() test_dir = self.tempdir.makedir(directory) filename = os.path.join(test_dir, fake.file_name(extension='json')) with open(filename, 'w') as f: config.to_json(f) self.assertTrue(os.path.isfile(filename)) config.close() def test_returns_false_if_defaults_are_changed(self): filename = self._get_config_file() with open(filename, "w") as config_file: config_file.write('[main]\nnewt = False\n') config_file.close() with ConfigReader(filename) as config: self.assertFalse(config.get('newt')) def test_returns_false_if_environment_variables_not_set(self): config = self._get_config() config.set('country', 'Kenya') config.set('continent', 'Africa') config.set('state', 'None') config.to_env() with self.subTest(0): self.assertEqual(os.environ['MAIN_COUNTRY'], 'Kenya') with self.subTest(1): self.assertEqual(os.environ['MAIN_CONTINENT'], 'Africa') with self.subTest(2): self.assertEqual(os.environ['MAIN_STATE'], 'None') config.close() def test_returns_false_if_section_prepend_failed(self): file_path_1 = self._get_config_file() file_path_2 = self._get_config_file() f = open(file_path_1, 'w+') config = ConfigReader(file_path_2, f) config.set('country', 'Kenya') config.set('continent', 'Africa') config.set('state', 'None') config.set('count', '0', section='first') config.to_env() f.close() with self.subTest(0): self.assertEqual(os.environ.get('MAIN_COUNTRY'), 'Kenya') with self.subTest(1): self.assertEqual(os.environ.get('MAIN_CONTINENT'), 'Africa') with self.subTest(2): self.assertEqual(os.environ.get('MAIN_STATE'), 'None') with self.subTest(3): self.assertEqual(os.environ.get('FIRST_COUNT'), '0') def test_returns_false_if_value_not_evaluated(self): config = self._get_config() config.set('truth', 'True') config.set('empty', '') config.set('count', '0', section='first') config.set('None', 'None', section='first') with self.subTest(0): self.assertTrue(config.get('truth')) with self.subTest(1): self.assertEqual(config.get('empty'), '') with self.subTest(2), self.assertRaises(MissingOptionError): config.get('count') with self.subTest(3): self.assertEqual(config.get('count', section='first'), 0) with self.subTest(4): self.assertIsNone(config.get('None', default='None')) config.close() def test_returns_false_if_exception_not_raised(self): config = self._get_config() def edit_sections(): config.sections = ['new_section'] self.assertRaises(AttributeError, edit_sections) config.close() def test_returns_false_if_threshold_error_not_raised(self): config = self._get_config() def do_search(threshold): config.search('read', threshold=threshold) with self.subTest(0): self.assertRaises(ThresholdError, do_search, 1.01) with self.subTest(1): self.assertRaises(ThresholdError, do_search, -1.0) config.close() def test_returns_false_if_match_not_found(self): file_path = self.tempdir.write(fake.config_file(), b'') with ConfigReader(file_path) as config: with self.subTest(0): self.assertEqual(config.get_items('main'), OrderedDict([('reader', 'configreader')])) with self.subTest(1): expected = ('reader', 'configreader', 'main') result = config.search('reader') self.assertIn(expected, result) def test_returns_false_if_match_not_found_2(self): file_path = self.tempdir.write(fake.config_file(), b'[main]') with ConfigReader(file_path) as config: with self.subTest(0): self.assertEqual(config.get_items('main'), OrderedDict([('reader', 'configreader')])) with self.subTest(1): expected = ('reader', 'configreader', 'main') result = config.search('reader') self.assertIn(expected, result) def test_returns_false_if_not_match_best(self): config = self._get_config() config.set('header', 'confreader') expected = (('reader', 'configreader', 'main'), ('header', 'confreader', 'main')) result = config.search('confgreader') for index, item in enumerate(expected): with self.subTest(index): self.assertIn(item, result) config.close() def test_returns_false_if_exact_match_not_found(self): config = self._get_config() config.set('title', 'The Place') expected = ('title', 'The Place', 'main') result = config.search('The Place', exact_match=True) self.assertEqual(result, expected) config.close() def test_returns_false_if_exact_match_found(self): config = self._get_config() config.set('title', 'The Place') result = config.search('The place', exact_match=True) self.assertEqual(result, None) config.close() def test_returns_false_if_exact_match_not_found_case_insensitive(self): config = self._get_config() config.set('title', 'The Place') expected = ('title', 'The Place', 'main') result = config.search('The place', exact_match=True, case_sensitive=False) self.assertEqual(result, expected) config.close() def test_returns_false_if_path_not_changed(self): file_path = self.tempdir.write(fake.config_file(), b'[main]') new_path = os.path.join(self.tempdir.path, fake.config_file()) with open(file_path, 'w+') as f, ConfigReader(file_object=f) as config: with self.subTest(0): self.assertTrue(os.path.isfile(file_path)) with self.subTest(1): self.assertFalse(os.path.isfile(new_path)) config.filename = new_path with self.subTest(2): self.assertFalse(os.path.isfile(file_path)) with self.subTest(3): self.assertFalse(os.path.isfile(new_path)) config.save() with self.subTest(4): self.assertTrue(os.path.isfile(new_path)) def test_returns_false_if_old_file_object_writable(self): file_path = self.tempdir.write(fake.config_file(), b'[main]') with open(file_path, 'w+') as f, ConfigReader(file_object=f) as config: new_path = os.path.join(os.path.expanduser('~'), fake.config_file()) config.filename = new_path self.assertRaises(ValueError, lambda: f.write('')) def test_returns_false_if_contents_not_similar(self): file_path = self.tempdir.write(fake.config_file(), b'[main]') new_path = os.path.join(self.tempdir.path, fake.config_file()) with open(file_path, 'w+') as f, ConfigReader(file_object=f) as config: f.seek(0) expected = f.read() config.filename = new_path config.save() with open(new_path) as f2: result = f2.read() self.assertEqual(result, expected) def test_returns_false_if_file_object_cannot_update(self): f = open(self.file_path, 'w') self.assertRaises(ModeError, lambda: ConfigReader(file_object=f)) f.close() def test_returns_false_if_contents_not_updated(self): f = open(self.file_path, 'w+') config = ConfigReader(file_object=f) config.set('name', 'first') with self.subTest(0): with open(self.file_path) as f2: result = f2.read() self.assertNotEqual(result, '') config.save() with self.subTest(1): with open(self.file_path) as f3: result = f3.read() self.assertNotEqual(result, '') f.close() def test_returns_false_if_file_object_and_filename_not_similar(self): with open(self.file_path, 'w+') as f: config = ConfigReader(file_object=f) self.assertEqual(config.filename, f.name) config.close() def test_returns_false_if_changes_not_written_to_file(self): file_path = fake.config_file(self.tempdir.path) config = ConfigReader(file_path) config.set('name', 'first') with ConfigReader(file_path) as d, self.subTest(0): with self.assertRaises(MissingOptionError): d.get('name') config.set('name', 'last', commit=True) config.close() d = ConfigReader(file_path) with self.subTest(1): self.assertEqual(d.get('name'), 'last') d.close() def test_returns_false_if_option_not_removed_from_file(self): config = self._get_config() config.set('name', 'first', section='two') with self.subTest(0): self.assertEqual(config.get('name', section='two'), 'first') config.remove_key('name', section='two', commit=True) with ConfigReader(self.file_path) as d, self.subTest( 1), self.assertRaises(MissingOptionError): d.get('name', section='two') config.close() def test_returns_false_if_file_object_not_closed(self): config = self._get_config() config.close() self.assertRaises(AttributeError, config.set, 'first', 'false') def test_returns_false_if_items_not_match(self): file_path_1 = self.tempdir.write(fake.config_file(), b'') file_path_2 = self.tempdir.write(fake.config_file(), b'') file_ = open(file_path_1, 'w+') config = ConfigReader(filename=file_path_2, file_object=file_) config.remove_section('main') config.set('start', 'True', section='defaults') config.set('value', '45', section='defaults') items = config.get_items('main') with self.subTest(0): self.assertIsNone(items) items = config.get_items('defaults') with self.subTest(1): self.assertIsInstance(items, OrderedDict) with self.subTest(2): expected = OrderedDict([('start', True), ('value', 45)]) self.assertEqual(items, expected) config.close() with self.subTest('Test returns false if file object still writable'): self.assertRaises(ValueError, file_.write, '') def test_returns_false_if_error_not_raised(self): config = ConfigReader() with self.subTest(0): self.assertRaises( SectionNameNotAllowed, lambda: config.set('start', 'True', section='default')) with self.subTest(1): self.assertRaises( SectionNameNotAllowed, lambda: config.set('start', 'True', section='deFault')) with self.subTest(2): self.assertRaises( SectionNameNotAllowed, lambda: config.set('start', 'True', section='DEFAULT')) def test_returns_false_if_object_not_context(self): with ConfigReader(self.file_path) as config: config.set('name', 'First', commit=True) name = config.get('name') self.assertEqual(name, 'First') @unittest.skipIf(os.name == 'nt', 'Path for *NIX systems') def test_returns_false_if_absolute_path_not_exist(self): path = '/home/path/does/not/exist' self.assertRaises(FileNotFoundError, ConfigReader, path) @unittest.skipIf(os.name != 'nt', 'Path for Windows systems') def test_returns_false_if_abs_path_not_exist(self): path = 'C:\\home\\path\\does\\not\\exist' self.assertRaises(FileNotFoundError, ConfigReader, path) def test_returns_false_if_default_not_returned(self): file_path = self.tempdir.write(fake.config_file(), b'') with ConfigReader(file_path) as config: value = config.get('members', default='10') self.assertEqual(value, 10) def test_returns_false_if_prepend_fails(self): file_path = self.tempdir.write(fake.config_file(), b'') with ConfigReader(file_path) as config: config.set('counter', 'default', section='team') config.set('play', '1', section='team') config.to_env() with self.subTest(0): self.assertEqual(os.environ['TEAM_COUNTER'], 'default') with self.subTest(1): self.assertEqual(os.environ['TEAM_PLAY'], '1') with self.subTest(2): self.assertRaises(KeyError, lambda: os.environ['PLAY']) config.to_env(prepend=False) with self.subTest(3): self.assertEqual(os.environ['PLAY'], '1') with self.subTest(4): self.assertEqual(os.environ['COUNTER'], 'default') @unittest.skipIf(os.name == 'nt', 'Testing environment variable for Windows') def test_returns_false_if_load_fails_linux(self): file_path = self.tempdir.write(fake.config_file(), b'') config = ConfigReader(file_path, case_sensitive=True) config.set('states', '35', section='country') config.set('counties', 'None', section='country') environment = os.environ.copy() user = '******' environment['USER'] = user environment['COUNTER'] = 'never' config.to_env(environment) config.load_env(environment) items = config.get_items('main') with self.subTest(0): self.assertEqual(items['HOME'], os.path.expanduser('~')) with self.subTest(1): self.assertEqual(items['USER'], user) with self.subTest(2): self.assertEqual(items['PWD'], os.environ['PWD']) with self.subTest(3): self.assertRaises(KeyError, lambda: items['home']) with self.subTest(4): self.assertEqual(items['COUNTER'], 'never') @unittest.skipIf(os.name == 'nt', 'Testing environment variable for Linux') def test_returns_false_if_load_fails_case_insensitive_linux(self): config = self._get_config() config.set('states', '35', section='country') config.set('counties', 'None', section='country') config.to_env() config.load_env() items = config.get_items('main') with self.subTest(0): self.assertEqual(items['home'], os.path.expanduser('~')) with self.subTest(1): self.assertEqual(items['user'], os.environ['USER']) with self.subTest(2): self.assertEqual(items['pwd'], os.environ['PWD']) with self.subTest(3): def call(): return items['PWD'] self.assertRaises(KeyError, call) @unittest.skipIf(os.name != 'nt', 'Testing environment variable for Windows') def test_returns_false_if_load_fails_windows(self): file_path = self.tempdir.write(fake.config_file(), b'') config = ConfigReader(file_path, case_sensitive=True) config.set('states', '35', section='country') config.set('counties', 'None', section='country') environment = os.environ.copy() user = '******' environment['USER'] = user environment['COUNTER'] = 'never' config.to_env(environment) config.load_env(environment) items = config.get_items('main') with self.subTest(0): self.assertEqual(items['APPDATA'], os.environ['APPDATA']) with self.subTest(1): self.assertEqual(items['USER'], user) with self.subTest(2): self.assertEqual(items['ALLUSERSPROFILE'], os.environ['ALLUSERSPROFILE']) with self.subTest(3): self.assertRaises(KeyError, lambda: items['home']) with self.subTest(4): self.assertEqual(items['COUNTER'], 'never') @unittest.skipIf(os.name != 'nt', 'Testing environment variable for Linux') def test_returns_false_if_load_fails_case_insensitive_windows(self): config = self._get_config() config.set('states', '35', section='country') config.set('counties', 'None', section='country') config.to_env() config.load_env() items = config.get_items('main') with self.subTest(0): self.assertEqual(items['appdata'], os.environ['APPDATA']) with self.subTest(1): self.assertEqual(items['allusersprofile'], os.environ['ALLUSERSPROFILE']) with self.subTest(2): self.assertEqual(items['homedrive'], os.environ['HOMEDRIVE']) with self.subTest(3): def call(): return items['PWD'] self.assertRaises(KeyError, call) def test_returns_false_if_load_prefixed_fails(self): config = self._get_config() config.set('states', '35', section='countrymain') config.set('counties', 'None', section='countrymain') config.to_env() config.remove_section('countrymain') with self.subTest(0): self.assertIsNone(config.get_items('countrymain')) config.load_env(prefix='countrymain') items = config.get_items('countrymain') with self.subTest(1): self.assertEqual(len(items), 2) with self.subTest(2): self.assertEqual(items['states'], 35) with self.subTest(3): self.assertIsNone(items['counties']) def test_returns_false_if_variable_not_expanded(self): config = self._get_config() config.set('path', 'drive') config.set('dir', '%(path)s-directory') config.set('suffix', 'dir-%(dir)s') with self.subTest(0): self.assertEqual(config.get('path'), 'drive') with self.subTest(1): self.assertEqual(config.get('dir'), 'drive-directory') with self.subTest(2): self.assertEqual(config.get('suffix'), 'dir-drive-directory') def test_returns_false_if_environment_variables_not_expanded(self): config = self._get_config() config.set('path', 'drive', section='test') config.set('dir', '%(path)s-directory', section='test') config.set('suffix', 'dir-%(dir)s', section='test') environment = os.environ.copy() config.to_env(environment) with self.subTest(0): self.assertEqual(environment['TEST_PATH'], 'drive') with self.subTest(1): self.assertEqual(environment['TEST_DIR'], 'drive-directory') with self.subTest(2): self.assertEqual(environment['TEST_SUFFIX'], 'dir-drive-directory') def test_returns_false_if_json_not_loaded_from_file(self): json_file = os.path.join(self.tempdir.path, fake.file_name(extension='json')) d = { 'name': 'plannet', 'count': 2, 'skip': False, 'handlers': [{ 'name': 'handler_1' }, { 'name': 'handler_2' }], '@counters': { 'start': { 'name': 'scrollers', 'count': 15 }, 'mid': { 'name': 'followers', 'count': 0, 'helpers': 'available' }, 'end': { 'name': 'keepers', 'count': 5 } } } try: j = open(json_file, 'w', encoding='utf-16') except TypeError: j = codecs.open(json_file, 'w', encoding='utf-16') json.dump(d, j, ensure_ascii=False) else: json.dump(d, j, ensure_ascii=False) j.close() config = self._get_config() config.load_json(json_file, section='json_data', encoding='utf-16') with self.subTest(0): compare(config.get('name', section='json_data'), 'plannet') with self.subTest(1): self.assertFalse(config.get('skip', section='json_data')) with self.subTest(2): self.assertIsInstance(config.get_items('counters'), OrderedDict) with self.subTest(3): compare( config.get_items('counters'), OrderedDict([('start', { 'name': 'scrollers', 'count': 15 }), ('mid', { 'name': 'followers', 'count': 0, 'helpers': 'available' }), ('end', { 'name': 'keepers', 'count': 5 })])) def test_returns_false_if_option_found(self): config = self._get_config() config.set('path', 'drive', section='test') with self.subTest(0): compare(config.get('path', 'test'), 'drive') with self.subTest(1), self.assertRaises(MissingOptionError): config.get('busy', section='test') config.close() def test_returns_false_if_option_with_default_found(self): config = self._get_config() config.set('path', 'drive', section='test') with self.subTest(0): compare(config.get('path', 'test'), 'drive') with self.subTest(1): compare(config.get('busy', section='test', default='not really'), 'not really') config.close() @unittest.skipIf(os.name == 'nt', 'Testing environment variables for Linux') def test_returns_false_if_environment_not_loaded_by_default_linux(self): file_path = self.tempdir.write(fake.config_file(), b'') with ConfigReader(file_path) as config: config.set('home', '$HOME') config.set('user', '$USER') with self.subTest(0): compare(config.get('home'), os.environ['HOME']) with self.subTest(1): compare(config.get('shell', default='$SHELL'), os.environ['SHELL']) @unittest.skipIf(os.name != 'nt', 'Testing environment variables for Windows') def test_returns_false_if_environment_not_loaded_by_default_windows(self): file_path = self.tempdir.write(fake.config_file(), b'') with ConfigReader(file_path) as config: config.set('app_data', '$APPDATA') with self.subTest(0): compare(config.get('app_data'), os.environ['APPDATA']) with self.subTest(1): compare( config.get('all_users_profile', default='$ALLUSERSPROFILE'), os.environ['ALLUSERSPROFILE']) def test_returns_false_if_reload_fails(self): file_path = self.tempdir.write( fake.config_file(), b'[main]\nreader = configreader\nmain = False\n') with ConfigReader(file_path) as config: with self.subTest(0): compare( config.get_items('main'), OrderedDict([('reader', 'configreader'), ('main', False)])) config.set('found', 'True') with self.subTest(1): compare( config.get_items('main'), OrderedDict([('reader', 'configreader'), ('main', False), ('found', True)])) config.reload() with self.subTest(2): compare( config.get_items('main'), OrderedDict([('reader', 'configreader'), ('main', False)])) def test_returns_false_if_case_sensistivity_fails(self): file_path = self.tempdir.write(fake.config_file(), b'') with ConfigReader(file_path, case_sensitive=True) as config: config.set('NAME', 'cup') config.set('LEAGUE', '1') compare( config.get_items('main'), OrderedDict([('reader', 'configreader'), ('NAME', 'cup'), ('LEAGUE', 1)])) def test_returns_false_if_set_many_fails(self): file_path = self._get_config_file() config = ConfigReader(file_path) config.set_many({'name': 'one', 'number': '2'}) with self.subTest('Test returns false if data not in default section'): self.assertListEqual( [config.get('name'), config.get('number')], ['one', 2]) config.reload() with self.subTest(), self.assertRaises(MissingOptionError): config.get('number') data = OrderedDict([('people', 'true'), ('count', 30)]) config.set_many(data, section='demo') with self.subTest( 'Test returns false if data not in specified section'): self.assertDictEqual(config.get_items('demo'), data) config.close(save=True) with ConfigReader(file_path) as d, self.subTest(): self.assertEqual(d.get('people', section='demo'), 'true') def test_returns_false_if_option_with_default_not_evaluated(self): config = self._get_config() with self.subTest(0): res = config.get('count', 'test', default=2) compare(res, 2) with self.subTest(1): compare(config.get('counter', 'test', default='20'), 20) config.close() def test_returns_false_if_option_with_string_default_not_evaluated(self): config = self._get_config() with self.subTest(0): compare(config.get('count', 'test', default='3'), 3) with self.subTest(1): compare(config.get('counter', 'test', default='20'), 20) config.close()
class Test_archivepgsql_BasicCommandLineOperation(TestCase): ARCHIVEPGSQL_PATH = os.path.join('bbpgsql', 'cmdline_scripts') CONFIG_FILE = 'config.ini' exe_script = 'archivewal' def setUp(self): self.setup_environment() self.setup_config() self.cmd = [self.exe_script, '--dry-run', '--config', self.config_path] def setup_environment(self): self.env = deepcopy(os.environ) self.env['PATH'] = ''.join([ self.env['PATH'], ':', self.ARCHIVEPGSQL_PATH]) self.tempdir = TempDirectory() def setup_config(self): self.storage_path = self.tempdir.makedir('repo') self.config_path = self.tempdir.getpath(self.CONFIG_FILE) self.log_file = self.tempdir.getpath('bbpgsql.log') self.config_dict = { 'WAL': { 'driver': 'filesystem', 'path': self.storage_path, }, 'General': { 'pgsql_data_directory': self.tempdir.path, }, 'Logging': { 'logfile': self.log_file, 'level': 'DEBUG', }, } write_config_to_filename(self.config_dict, self.config_path) #print '----' #print open(self.config_path, 'rb').read() #print '----' self.pg_xlog_path = 'pg_xlog' self.tempdir.makedir(self.pg_xlog_path) self.wal_basename = '00001' self.wal_filename = os.path.join(self.pg_xlog_path, self.wal_basename) self.tempdir.write(self.wal_filename, '') print 'TEMPDIR', self.tempdir.listdir(recursive=True) def tearDown(self): self.tempdir.cleanup() def test_archivewal_returns_error_with_if_less_than_one_argument(self): proc = Popen(self.cmd, env=self.env, stdout=PIPE, stderr=STDOUT) proc.wait() print(proc.stdout.read()) self.assertNotEqual(0, proc.returncode) def test_archivewal_logs_error_with_if_less_than_one_argument(self): proc = Popen(self.cmd, env=self.env, stdout=PIPE, stderr=STDOUT) proc.wait() self.assertNotEqual(0, proc.returncode) print proc.stdout.read() log_output = open(self.log_file, 'rb').read() print 'log_output:' print log_output assert 'ERROR' in log_output def test_archivewal_success_with_file(self): self.cmd.append(self.wal_filename) proc = Popen(self.cmd, env=self.env, stdout=PIPE, stderr=STDOUT) proc.wait() print proc.stdout.read() self.assertEqual(0, proc.returncode) def test_archivewal_actually_archives_file(self): self.cmd.append(self.wal_filename) proc = Popen(self.cmd, env=self.env, stdout=PIPE, stderr=STDOUT) proc.wait() print proc.stdout.read() self.assertEqual(0, proc.returncode) archives = os.listdir(self.storage_path) print archives self.assertTrue(archives[0].startswith(''.join([ self.wal_basename, '_'])))
class TestFilesystemCrawler(unittest.TestCase): filename = 'foobar.txt' dirname = 'barfoo/' file = '' dir = '' def setUp(self): self.tmp = TempDirectory() self.dir = self.tmp.makedir(self.dirname) self.file = self.tmp.write(self.filename, b'foo') def tearDown(self): TempDirectory.cleanup_all() def dummyFile(self, filename, expected): f = self.tmp.write(filename, b'EOF') crawler = Crawler() ref = Sourcefile._make( ['', '', expected[0], expected[1], expected[2]]) res = crawler._parseFileName(f) self.assertEquals(ref[2], res[2]) self.assertEquals(ref[3], res[3]) self.assertEquals(ref[4], res[4]) #silent skip of all non readable dirs def DISABLEDtest_input_type(self): crawler = Crawler() self.assertRaises(AttributeError, crawler.crawl, '404') #silent skip of all non readable dirs def DISABLEtest_input_file_permissions(self): os.chmod(self.file, 0) crawler = Crawler() self.assertRaises(PermissionError, crawler._parseFileName, self.file) def test_empty_pairing(self): crawler = Crawler() self.assertRaises(ValueError, crawler.pair) # Pxx_Eyy_right|left (eGaitDatabase) def test_S_E_O_format(self): self.dummyFile('P01_E2_right.dat', ['01', '2', Orientation.right]) # GAxxEyy_right|left (eGaitDatabase3/Controls) def test_SE_O_format(self): self.dummyFile('GA112030E3_left.txt', ['112030', '3', Orientation.left]) # PxxEyy_right|left (eGaitDatabase3/Geriatrics) def test_SE_O_format2(self): self.dummyFile('P50E6_left.txt', ['50', '6', Orientation.left]) # PatxxWald(left|right)Foot_Sensor_Date (eGaitDatabase3/GeriatricsGSTD/4MW) def test_SO_format(self): self.dummyFile('Pat1WaldrightFoot_1eea_20140325133006.dat', ['1Wald', '', Orientation.right]) # GAxx_(Left|Right)Foot_Sensor_Date (eGaitDatabase3/Controls/4MW) def test_SO_format2(self): self.dummyFile('GA214026LeftFoot_1eea_20140428110132.dat', ['214026', '', Orientation.left]) # GAstdxx_(Left|Right)Foot_Sensor_Date (eGaitDatabase3/Controls/4MW) def test_SO_format3(self): self.dummyFile('GAstd45jwRightFoot_1d68_20140506123332.dat', ['45jw', '', Orientation.right]) def test_correct_pairing(self): left = self.tmp.write(self.dir + 'P01_E2_left.txt', b'') right = self.tmp.write(self.dir + 'P01_E2_right.txt', b'') crawler = Crawler() crawler.crawl(self.dir) crawler.pair() self.assertEquals(crawler.pairedFiles[0][0].filename, 'P01_E2_left.txt') self.assertEquals(crawler.pairedFiles[0][1].filename, 'P01_E2_right.txt') def test_one_partner_missing_pairing(self): left = self.tmp.write(self.dir + 'P01_E2_left.txt', b'') crawler = Crawler() crawler.crawl(self.dir) try: crawler.pair() except ValueError: pass self.assertEquals(crawler.pairedFiles[0][0].filename, crawler.pairedFiles[0][1].filename)
class TestPreprocessUtils(unittest.TestCase): def setUp(self): """ Set up a directory with some images to resample """ self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.mask_dir = os.path.join(self.temp_path, 'mask_dir') self.tempdir.makedir('mask_dir') self.input_dir = os.path.join(self.temp_path, 'input_dir') self.tempdir.makedir('input_dir') self.mask_channel = 1 self.slice_idx = 7 self.time_idx = 8 # Mask meta file self.csv_name = 'mask_image_matchup.csv' input_meta = aux_utils.make_dataframe() # Make input meta for c in range(4): for p in range(10): im_name = aux_utils.get_im_name( channel_idx=c, slice_idx=self.slice_idx, time_idx=self.time_idx, pos_idx=p, ) input_meta = input_meta.append( aux_utils.parse_idx_from_name(im_name), ignore_index=True, ) input_meta.to_csv( os.path.join(self.input_dir, 'frames_meta.csv'), sep=',', ) # Make mask meta mask_meta = pd.DataFrame() for p in range(10): im_name = aux_utils.get_im_name( channel_idx=self.mask_channel, slice_idx=self.slice_idx, time_idx=self.time_idx, pos_idx=p, ) # Indexing can be different mask_name = 'mask_{}.png'.format(p + 1) mask_meta = mask_meta.append( {'mask_name': mask_name, 'file_name': im_name}, ignore_index=True, ) mask_meta.to_csv( os.path.join(self.mask_dir, self.csv_name), sep=',', ) def tearDown(self): """ Tear down temporary folder and file structure """ TempDirectory.cleanup_all() nose.tools.assert_equal(os.path.isdir(self.temp_path), False) def test_validate_mask_meta(self): mask_channel = preprocess_utils.validate_mask_meta( mask_dir=self.mask_dir, input_dir=self.input_dir, csv_name=self.csv_name, mask_channel=50, ) self.assertEqual(mask_channel, 50) out_meta = aux_utils.read_meta(self.mask_dir) for i, row in out_meta.iterrows(): self.assertEqual(row.slice_idx, self.slice_idx) self.assertEqual(row.time_idx, self.time_idx) self.assertEqual(row.channel_idx, 50) self.assertEqual(row.pos_idx, i) self.assertEqual(row.file_name, "mask_{}.png".format(i + 1)) def test_validate_mask_meta_no_channel(self): mask_channel = preprocess_utils.validate_mask_meta( mask_dir=self.mask_dir, input_dir=self.input_dir, csv_name=self.csv_name, ) self.assertEqual(mask_channel, 4) out_meta = aux_utils.read_meta(self.mask_dir) for i, row in out_meta.iterrows(): self.assertEqual(row.slice_idx, self.slice_idx) self.assertEqual(row.time_idx, self.time_idx) self.assertEqual(row.channel_idx, 4) self.assertEqual(row.pos_idx, i) self.assertEqual(row.file_name, "mask_{}.png".format(i + 1)) @nose.tools.raises(AssertionError) def test_validate_mask_meta_mask_channel(self): # Same channel as image preprocess_utils.validate_mask_meta( mask_dir=self.mask_dir, input_dir=self.input_dir, csv_name=self.csv_name, mask_channel=1, ) @nose.tools.raises(IOError) def test_validate_mask_meta_wrong_dir(self): preprocess_utils.validate_mask_meta( mask_dir=self.temp_path, input_dir=self.input_dir, )
class TestImageInference3D(unittest.TestCase): @patch('micro_dl.inference.model_inference.load_model') def setUp(self, mock_model): """ Set up a directory with 3D images """ mock_model.return_value = 'dummy_model' self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.tempdir.makedir('image_dir') self.tempdir.makedir('mask_dir') self.tempdir.makedir('model_dir') self.image_dir = os.path.join(self.temp_path, 'image_dir') self.mask_dir = os.path.join(self.temp_path, 'mask_dir') self.model_dir = os.path.join(self.temp_path, 'model_dir') # Create a temp image dir self.im = np.zeros((10, 10, 8), dtype=np.uint8) self.frames_meta = aux_utils.make_dataframe() self.time_idx = 2 self.slice_idx = 0 for p in range(5): for c in range(3): im_name = aux_utils.get_im_name( time_idx=self.time_idx, channel_idx=c, slice_idx=self.slice_idx, pos_idx=p, ext='.npy', ) np.save(os.path.join(self.image_dir, im_name), self.im + c * 10, allow_pickle=True, fix_imports=True) self.frames_meta = self.frames_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to image dir too self.frames_meta.to_csv(os.path.join(self.image_dir, 'frames_meta.csv')) # Save masks and mask meta self.mask_meta = aux_utils.make_dataframe() self.mask_channel = 50 # Mask half the image mask = np.zeros_like(self.im) mask[:5, ...] = 1 for p in range(5): im_name = aux_utils.get_im_name( time_idx=self.time_idx, channel_idx=self.mask_channel, slice_idx=self.slice_idx, pos_idx=p, ext='.npy', ) np.save(os.path.join(self.mask_dir, im_name), mask) self.mask_meta = self.mask_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to mask dir too self.mask_meta.to_csv(os.path.join(self.mask_dir, 'frames_meta.csv')) # Setup model dir split_samples = { "train": [0, 1], "val": [2], "test": [3, 4], } aux_utils.write_json( split_samples, os.path.join(self.model_dir, 'split_samples.json'), ) # Make configs with fields necessary for 2.5D segmentation inference self.train_config = { 'network': { 'class': 'UNet3D', 'data_format': 'channels_first', 'num_filters_per_block': [8, 16], 'depth': 5, 'width': 5, 'height': 5 }, 'dataset': { 'split_by_column': 'pos_idx', 'input_channels': [1], 'target_channels': [2], 'model_task': 'regression', }, } self.inference_config = { 'model_dir': self.model_dir, 'model_fname': 'dummy_weights.hdf5', 'image_dir': self.image_dir, 'data_split': 'test', 'images': { 'image_format': 'zyx', 'image_ext': '.png', }, 'metrics': { 'metrics': ['mse'], 'metrics_orientations': ['xyz'], }, 'masks': { 'mask_dir': self.mask_dir, 'mask_type': 'metrics', 'mask_channel': 50, }, 'inference_3d': { 'tile_shape': [5, 5, 5], 'num_overlap': [1, 1, 1], 'overlap_operation': 'mean', }, } # Instantiate class self.infer_inst = image_inference.ImagePredictor( train_config=self.train_config, inference_config=self.inference_config, ) def tearDown(self): """ Tear down temporary folder and file structure """ TempDirectory.cleanup_all() self.assertEqual(os.path.isdir(self.temp_path), False) def test_init(self): """ Test init of inference dataset """ # Check proper init self.assertEqual(self.infer_inst.model_dir, self.model_dir) self.assertEqual(self.infer_inst.image_dir, self.image_dir) self.assertEqual(self.infer_inst.data_format, 'channels_first') self.assertEqual(self.infer_inst.model, 'dummy_model') self.assertEqual(self.infer_inst.image_format, 'zyx') self.assertEqual(self.infer_inst.image_ext, '.npy') self.assertTrue(self.infer_inst.mask_metrics) self.assertEqual(self.infer_inst.mask_dir, self.mask_dir) self.assertListEqual(self.infer_inst.metrics_orientations, ['xyz']) self.assertIsNone(self.infer_inst.crop_shape) self.assertEqual(self.infer_inst.z_dim, 2) self.assertEqual(self.infer_inst.tile_option, 'tile_xyz') self.assertListEqual(self.infer_inst.num_overlap, [1, 1, 1]) def test_assign_3d_inference(self): # Test other settings self.infer_inst.params_3d = { 'num_slices': 5, 'num_overlap': 1, 'overlap_operation': 'mean', } self.infer_inst._assign_3d_inference() self.assertEqual(self.infer_inst.z_dim, 2) self.assertEqual(self.infer_inst.tile_option, 'tile_z') self.assertEqual(self.infer_inst.num_overlap, 1) def test_assign_3d_inference_xyz(self): # Test other settings self.infer_inst.params_3d = { 'num_slices': 5, 'num_overlap': 1, 'overlap_operation': 'mean', } self.infer_inst.image_format = 'xyz' self.infer_inst._assign_3d_inference() self.assertEqual(self.infer_inst.z_dim, 4) self.assertEqual(self.infer_inst.tile_option, 'tile_z') self.assertEqual(self.infer_inst.num_overlap, 1) @nose.tools.raises(AssertionError) def test_assign_3d_inference_few_slices(self): # Test other settings self.infer_inst.params_3d = { 'num_slices': 3, 'num_overlap': 1, 'overlap_operation': 'mean', } self.infer_inst._assign_3d_inference() @nose.tools.raises(AssertionError) def test_assign_3d_inference_not_3d(self): self.infer_inst.params_3d = { 'num_slices': 5, 'num_overlap': 1, 'overlap_operation': 'mean', } self.infer_inst.config['network']['class'] = 'UNet2D' self.infer_inst._assign_3d_inference() def test_assign_3d_inference_on_center(self): self.infer_inst.params_3d = { 'inf_shape': [5, 5, 5], 'num_overlap': 1, 'overlap_operation': 'mean', } self.infer_inst._assign_3d_inference() self.assertEqual(self.infer_inst.tile_option, 'infer_on_center') self.assertEqual(self.infer_inst.num_overlap, 0) def test_get_sub_block_z(self): # 3D image for prediction should have channel and batch dim im = np.zeros((1, 2, 8, 10, 10), dtype=np.float32) block = self.infer_inst._get_sub_block_z( input_image=im, start_z_idx=2, end_z_idx=5, ) self.assertTupleEqual(block.shape, (1, 2, 3, 10, 10)) def test_get_sub_block_z_channels_last(self): self.infer_inst.data_format = 'channels_last' im = np.zeros((1, 8, 10, 10, 2), dtype=np.float32) block = self.infer_inst._get_sub_block_z( input_image=im, start_z_idx=2, end_z_idx=5, ) self.assertTupleEqual(block.shape, (1, 3, 10, 10, 2)) def test_get_sub_block_z_xyz(self): self.infer_inst.image_format = 'xyz' im = np.zeros((1, 2, 10, 10, 8), dtype=np.float32) block = self.infer_inst._get_sub_block_z( input_image=im, start_z_idx=2, end_z_idx=5, ) self.assertTupleEqual(block.shape, (1, 2, 10, 10, 3)) def test_get_sub_block_z_xyz_channels_last(self): self.infer_inst.image_format = 'xyz' self.infer_inst.data_format = 'channels_last' im = np.zeros((1, 10, 10, 8, 2), dtype=np.float32) block = self.infer_inst._get_sub_block_z( input_image=im, start_z_idx=2, end_z_idx=5, ) self.assertTupleEqual(block.shape, (1, 10, 10, 3, 2)) @patch('micro_dl.inference.model_inference.predict_large_image') def test_predict_sub_block_z(self, mock_predict): mock_predict.return_value = np.zeros((1, 1, 5, 10, 10), dtype=np.float32) self.infer_inst.params_3d = { 'num_slices': 5, 'num_overlap': 1, 'overlap_operation': 'mean', } self.infer_inst.num_overlap = 1 im = np.zeros((1, 1, 8, 10, 10), dtype=np.float32) pred_ims, start_end_idx = self.infer_inst._predict_sub_block_z( input_image=im, ) # z = 5 slices and 1 overlap, two prediction of im can fit self.assertEqual(len(pred_ims), 2) self.assertEqual(len(start_end_idx), 2) # Z locations (out of 0-8) self.assertTupleEqual(start_end_idx[0], (0, 5)) self.assertTupleEqual(start_end_idx[1], (3, 8)) @patch('micro_dl.inference.model_inference.predict_large_image') def test_predict_sub_block_xyz(self, mock_predict): mock_predict.return_value = np.zeros((1, 1, 5, 5, 5), dtype=np.float32) self.infer_inst.num_overlap = 1 im = np.zeros((1, 1, 8, 10, 10), dtype=np.float32) pred_ims = self.infer_inst._predict_sub_block_xyz( input_image=im, crop_indices=[(0, 5, 0, 5, 0, 5), (3, 8, 0, 5, 0, 5)], ) self.assertEqual(len(pred_ims), 2) @patch('micro_dl.inference.model_inference.predict_large_image') def test_predict_sub_block_xyz_channels_last(self, mock_predict): mock_predict.return_value = np.zeros((1, 1, 5, 5, 5), dtype=np.float32) self.infer_inst.num_overlap = 1 im = np.zeros((1, 8, 10, 10, 1), dtype=np.float32) pred_ims = self.infer_inst._predict_sub_block_xyz( input_image=im, crop_indices=[(3, 8, 0, 5, 0, 5)], ) self.assertEqual(len(pred_ims), 1) @patch('micro_dl.inference.model_inference.predict_large_image') def test_predict_3d(self, mock_predict): mock_predict.return_value = np.zeros((1, 1, 5, 5, 5), dtype=np.float32) # Predict row 0 from inference dataset iterator pred_im, target_im, mask_im = self.infer_inst.predict_3d([0]) self.assertTupleEqual(pred_im.shape, (8, 8, 8)) self.assertEqual(pred_im.dtype, np.float32) self.assertTupleEqual(target_im.shape, (8, 8, 8)) self.assertEqual(target_im.dtype, np.float32) self.assertTupleEqual(mask_im.shape, (8, 8, 8)) self.assertEqual(mask_im.dtype, np.uint8) # Read saved prediction, z=0 target channel=2 pred_name = os.path.join( self.model_dir, 'predictions/im_c002_z000_t002_p003.npy', ) im_pred = np.load(pred_name) self.assertEqual(im_pred.dtype, np.float32) self.assertTupleEqual(im_pred.shape, (8, 8, 8)) @patch('micro_dl.inference.model_inference.predict_large_image') def test_predict_3d_on_center(self, mock_predict): mock_predict.return_value = np.zeros((1, 1, 3, 3, 3), dtype=np.float32) self.infer_inst.crop_shape = [5, 5, 5] self.infer_inst.tile_option = 'infer_on_center' self.infer_inst.params_3d['inf_shape'] = [3, 3, 3] # Predict row 0 from inference dataset iterator pred_im, target_im, mask_im = self.infer_inst.predict_3d([0]) self.assertTupleEqual(pred_im.shape, (3, 3, 3)) self.assertEqual(pred_im.dtype, np.float32) self.assertTupleEqual(target_im.shape, (3, 3, 3)) self.assertEqual(target_im.dtype, np.float32) self.assertTupleEqual(mask_im.shape, (3, 3, 3)) self.assertEqual(mask_im.dtype, np.uint8) # Read saved prediction, z=0 target channel=2 pred_name = os.path.join( self.model_dir, 'predictions/im_c002_z000_t002_p003.npy', ) im_pred = np.load(pred_name) self.assertEqual(im_pred.dtype, np.float32) self.assertTupleEqual(im_pred.shape, (3, 3, 3)) @patch('micro_dl.inference.model_inference.predict_large_image') def test_run_prediction(self, mock_predict): mock_predict.return_value = np.zeros((1, 1, 5, 5, 5), dtype=np.float32) # Run prediction. Should create a metrics_xy.csv in pred dir self.infer_inst.run_prediction() metrics = pd.read_csv( os.path.join(self.model_dir, 'predictions/metrics_xyz.csv')) self.assertTupleEqual(metrics.shape, (2, 4)) # MSE should be 0. self.assertEqual(metrics.mse.mean(), 0.0) self.assertEqual(metrics.mse_masked.mean(), 0.0) # Mask volume fraction should be 0.5 self.assertEqual(metrics.vol_frac.mean(), 0.5) # # There should be four rows, one per test index pos=3,4 self.assertEqual(metrics.pred_name[0], 'im_c002_z000_t002_p003') self.assertEqual(metrics.pred_name[1], 'im_c002_z000_t002_p004') # # There should be 2 predictions saved in pred dir, target c=2 for p in [3, 4]: pred_name = os.path.join( self.model_dir, 'predictions/im_c002_z000_t002_p00{}.npy'.format(p), ) im_pred = np.load(pred_name) self.assertEqual(im_pred.dtype, np.float32) self.assertTupleEqual(im_pred.shape, (8, 8, 8))
class TestImageInference2p5D(unittest.TestCase): @patch('micro_dl.inference.model_inference.load_model') def setUp(self, mock_model): """ Set up a directory with images """ mock_model.return_value = 'dummy_model' self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.tempdir.makedir('image_dir') self.tempdir.makedir('mask_dir') self.tempdir.makedir('model_dir') self.image_dir = os.path.join(self.temp_path, 'image_dir') self.mask_dir = os.path.join(self.temp_path, 'mask_dir') self.model_dir = os.path.join(self.temp_path, 'model_dir') # Create a temp image dir self.im = np.zeros((10, 16), dtype=np.uint8) self.frames_meta = aux_utils.make_dataframe() self.time_idx = 2 for p in range(5): for c in range(3): for z in range(6): im_name = aux_utils.get_im_name( time_idx=self.time_idx, channel_idx=c, slice_idx=z, pos_idx=p, ) cv2.imwrite(os.path.join(self.image_dir, im_name), self.im + c * 10) self.frames_meta = self.frames_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to image dir too self.frames_meta.to_csv(os.path.join(self.image_dir, 'frames_meta.csv')) # Save masks and mask meta self.mask_meta = aux_utils.make_dataframe() self.mask_channel = 50 for p in range(5): for z in range(6): im_name = aux_utils.get_im_name( time_idx=self.time_idx, channel_idx=self.mask_channel, slice_idx=z, pos_idx=p, ) cv2.imwrite(os.path.join(self.mask_dir, im_name), self.im + 1) self.mask_meta = self.mask_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to mask dir too self.mask_meta.to_csv(os.path.join(self.mask_dir, 'frames_meta.csv')) # Setup model dir split_samples = { "train": [0, 1], "val": [2], "test": [3, 4], } aux_utils.write_json( split_samples, os.path.join(self.model_dir, 'split_samples.json'), ) # Make configs with fields necessary for 2.5D segmentation inference self.train_config = { 'network': { 'class': 'UNetStackTo2D', 'data_format': 'channels_first', 'depth': 5, 'width': 10, 'height': 10 }, 'dataset': { 'split_by_column': 'pos_idx', 'input_channels': [1], 'target_channels': [self.mask_channel], 'model_task': 'segmentation', }, } self.inference_config = { 'model_dir': self.model_dir, 'model_fname': 'dummy_weights.hdf5', 'image_dir': self.image_dir, 'data_split': 'test', 'images': { 'image_format': 'zyx', 'image_ext': '.png', }, 'metrics': { 'metrics': ['dice'], 'metrics_orientations': ['xy'], }, 'masks': { 'mask_dir': self.mask_dir, 'mask_type': 'target', 'mask_channel': 50, } } # Instantiate class self.infer_inst = image_inference.ImagePredictor( train_config=self.train_config, inference_config=self.inference_config, ) def tearDown(self): """ Tear down temporary folder and file structure """ TempDirectory.cleanup_all() self.assertEqual(os.path.isdir(self.temp_path), False) def test_init(self): """ Test init of inference dataset """ # Check proper init self.assertEqual(self.infer_inst.model_dir, self.model_dir) self.assertEqual(self.infer_inst.image_dir, self.image_dir) self.assertEqual(self.infer_inst.data_format, 'channels_first') self.assertEqual(self.infer_inst.model, 'dummy_model') self.assertEqual(self.infer_inst.image_format, 'zyx') self.assertEqual(self.infer_inst.image_ext, '.png') self.assertFalse(self.infer_inst.mask_metrics) self.assertEqual(self.infer_inst.mask_dir, self.mask_dir) self.assertListEqual(self.infer_inst.metrics_orientations, ['xy']) self.assertEqual(self.infer_inst.num_overlap, 0) self.assertIsNone(self.infer_inst.stitch_inst) self.assertIsNone(self.infer_inst.tile_option) self.assertIsNone(self.infer_inst.crop_shape) @patch('micro_dl.inference.model_inference.predict_large_image') def test_predict_2d(self, mock_predict): mock_predict.return_value = 1. + np.ones( (1, 1, 1, 8, 16), dtype=np.float32) # Predict row 0 from inference dataset iterator pred_im, target_im, mask_im = self.infer_inst.predict_2d([0]) self.assertTupleEqual(pred_im.shape, (8, 16, 1)) self.assertEqual(pred_im.dtype, np.float32) self.assertEqual(pred_im.max(), 2.0) # Read saved prediction, z=2 for first slice with depth=5 pred_name = os.path.join( self.model_dir, 'predictions/im_c050_z002_t002_p003.png', ) im_pred = cv2.imread(pred_name, cv2.IMREAD_ANYDEPTH) self.assertEqual(im_pred.dtype, np.uint16) self.assertTupleEqual(im_pred.shape, (8, 16)) # Check target and no mask self.assertTupleEqual(target_im.shape, (8, 16, 1)) self.assertEqual(target_im.dtype, np.float32) self.assertEqual(target_im.max(), 1.) self.assertListEqual(mask_im, []) @patch('micro_dl.inference.model_inference.predict_large_image') def test_run_prediction(self, mock_predict): mock_predict.return_value = 1. + np.ones( (1, 1, 1, 8, 16), dtype=np.float32) # Run prediction. Should create a metrics_xy.csv in pred dir self.infer_inst.run_prediction() metrics = pd.read_csv( os.path.join(self.model_dir, 'predictions/metrics_xy.csv')) self.assertTupleEqual(metrics.shape, (4, 2)) # Dice should be 1. self.assertEqual(metrics.dice.mean(), 1.0) # There should be four rows, one per test index pos=3,4 # depth=5 means center slices z=2,3 will be evaluated self.assertEqual(metrics.pred_name[0], 'im_c050_z002_t002_p003_xy0') self.assertEqual(metrics.pred_name[1], 'im_c050_z003_t002_p003_xy0') self.assertEqual(metrics.pred_name[2], 'im_c050_z002_t002_p004_xy0') self.assertEqual(metrics.pred_name[3], 'im_c050_z003_t002_p004_xy0') # There should be 4 predictions saved in pred dir for p in [3, 4]: for z in [2, 3]: pred_name = os.path.join( self.model_dir, 'predictions/im_c050_z00{}_t002_p00{}.png'.format(z, p), ) im_pred = cv2.imread(pred_name, cv2.IMREAD_ANYDEPTH) self.assertEqual(im_pred.dtype, np.uint16) self.assertTupleEqual(im_pred.shape, (8, 16))
class PluginWithTempDirTests(TestCase): def setUp(self): self.dir = TempDirectory() self.addCleanup(self.dir.cleanup) def run_actions(self, path=None, **kw): with LogCapture() as log: plugin = make_git_repo(path=path or self.dir.path, **kw) with Replacer() as r: r.replace("archivist.repos.git.datetime", test_datetime()) plugin.actions() return log def git(self, command, repo_path=None): return run(["git"] + command.split(), cwd=repo_path or self.dir.path) def make_repo_with_content(self, repo=""): repo_path = self.dir.getpath(repo) if repo else None self.git("init", repo_path) self.dir.write(repo + "a", "some content") self.dir.write(repo + "b", "other content") self.dir.write(repo + "c", "more content") self.git("add .", repo_path) self.git("commit -m initial", repo_path) return repo def make_local_changes(self, repo=""): self.dir.write(repo + "b", "changed content") os.remove(self.dir.getpath(repo + "c")) self.dir.write(repo + "d", "new content") def status_log_entry(self, lines, repo_path=None): return ( "archivist.repos.git", "INFO", "\n".join(l.format(repo=repo_path or self.dir.path) for l in lines) + "\n", ) def check_git_log(self, lines, repo_path=None): compare("\n".join(lines) + "\n", self.git("log --pretty=format:%s --stat", repo_path)) def get_dummy_source(self, name): class DummySource(Source): schema = Schema({}) def __init__(self, type, name, repo): super(DummySource, self).__init__(type, name, repo) def process(self, path): pass return DummySource("dummy", name, "repo") def test_path_for_with_name(self): compare( self.dir.getpath("dummy/the_name"), make_git_repo(path=self.dir.path).path_for(self.get_dummy_source("the_name")), ) self.assertTrue(os.path.exists(self.dir.getpath("dummy/the_name"))) def test_path_for_no_name(self): compare(self.dir.getpath("dummy"), make_git_repo(path=self.dir.path).path_for(self.get_dummy_source(name=None))) self.assertTrue(os.path.exists(self.dir.getpath("dummy"))) def test_not_there(self): repo_path = self.dir.getpath("var") log = self.run_actions(repo_path) log.check(("archivist.repos.git", "INFO", "creating git repo at " + repo_path)) self.assertTrue(self.dir.getpath("var/.git")) def test_there_not_git(self): repo_path = self.dir.makedir("var") log = self.run_actions(repo_path) log.check(("archivist.repos.git", "INFO", "creating git repo at " + repo_path)) self.assertTrue(self.dir.getpath("var/.git")) def test_no_changes(self): self.git("init") log = self.run_actions() log.check() # no logging def test_just_log_changes(self): self.make_repo_with_content() self.make_local_changes() log = self.run_actions(commit=False) log.check(self.status_log_entry(["changes found in git repo at {repo}:", " M b", " D c", "?? d"])) self.check_git_log(["initial", " a | 1 +", " b | 1 +", " c | 1 +", " 3 files changed, 3 insertions(+)"]) def test_commit_changes(self): self.make_repo_with_content() self.make_local_changes() log = self.run_actions(commit=True) log.check( self.status_log_entry(["changes found in git repo at {repo}:", " M b", " D c", "?? d"]), ("archivist.repos.git", "INFO", "changes committed"), ) compare("", self.git("status --porcelain")) self.check_git_log( [ "Recorded by archivist at 2001-01-01 00:00", " b | 2 +-", " c | 1 -", " d | 1 +", " 3 files changed, 2 insertions(+), 2 deletions(-)", "", "initial", " a | 1 +", " b | 1 +", " c | 1 +", " 3 files changed, 3 insertions(+)", ] ) def test_push_changes(self): origin_path = self.dir.makedir("origin") self.make_repo_with_content(repo="origin/") self.git("config --local --add receive.denyCurrentBranch ignore", origin_path) self.git("clone -q " + origin_path + " local") self.make_local_changes(repo="local/") local_path = self.dir.getpath("local") log = self.run_actions(commit=True, push=True, path=local_path) log.check( self.status_log_entry( ["changes found in git repo at {repo}:", " M b", " D c", "?? d"], repo_path=local_path ), ("archivist.repos.git", "INFO", "changes committed"), ("archivist.repos.git", "INFO", "changes pushed"), ) self.check_git_log( [ "Recorded by archivist at 2001-01-01 00:00", " b | 2 +-", " c | 1 -", " d | 1 +", " 3 files changed, 2 insertions(+), 2 deletions(-)", "", "initial", " a | 1 +", " b | 1 +", " c | 1 +", " 3 files changed, 3 insertions(+)", ], repo_path=origin_path, ) def test_push_no_changes(self): self.git("init") log = self.run_actions(commit=True, push=True) log.check() # no logging def test_default_repo_config(self): # can't test actions due to default path GitRepo(**GitRepo.schema(default_repo_config))
def test_init(self, tmpdir: TempDirectory): tmpdir.makedir('foo') Git(tmpdir.getpath('foo')).init() assert os.path.exists(tmpdir.getpath('foo/.git'))
class TestFileTestCase(unittest.TestCase): def setUp(self): self.tempdir = TempDirectory(encoding='utf-8') def tearDown(self): self.tempdir.cleanup() def test_returns_false_if_parent_attributes_not_set(self): d = File(self.tempdir.path) with self.subTest(1): compare(self.tempdir.path, d.path) with self.subTest(2): compare(os.path.basename(self.tempdir.path), d.name) with self.subTest(3): compare(os.path.dirname(self.tempdir.path), d.parent) def test_returns_false_if_path_change_is_relative(self): d = File(self.tempdir.path) with self.subTest(1): compare(self.tempdir.path, d.path) if os.name != 'nt': with self.subTest(2): compare(False, d.hidden_path) with self.subTest(3): compare('UNDEFINED', d.default_category) def call(x): d.path = x self.assertRaises(RelativePathError, call, '.') def test_returns_false_if_attributes_not_set(self): file_path = self.tempdir.write('123.txt', '') d = File(file_path) with self.subTest(1): compare(file_path, d.path) if os.name != 'nt': with self.subTest(2): compare(False, d.hidden_path) with self.subTest(3): compare(os.path.basename(file_path), d.name) with self.subTest(4): compare(True, d.exists) with self.subTest(5): compare('document', d.category) with self.subTest(6): compare('txt', d.extension) with self.subTest(7): compare(d.path, str(d)) def test_returns_false_if_names_not_match(self): initial_file = self.tempdir.write('123.txt', '') for i in range(1, 10): filename = '123 - dup ({}).txt'.format(str(i)) d = File(initial_file) with self.subTest(i): compare(filename, d.find_suitable_name(initial_file)) self.tempdir.write(os.path.join(self.tempdir.path, filename), '') def test_returns_false_if_pathlib_methods_fail(self): path = self.tempdir.makedir('abc/fig/') file_ = self.tempdir.write(os.path.join(path, 'my awesome cat.txt'), '') d = File(file_) with self.subTest(1): compare('.txt', d.suffix) with self.subTest(2): compare('my awesome cat', d.stem) def test_returns_false_if_file_moving_fails(self): dir_1 = self.tempdir.makedir('abc/fig/') dir_2 = self.tempdir.makedir('one/twp/') dir_3 = self.tempdir.makedir('some/dir/') file_ = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') f = File(file_) f.move_to(dir_2, group=False) with self.subTest(1): compare(os.path.join(dir_2, 'TXT', 'my awesome cat.txt'), f.path) with self.subTest(2): compare(True, f.exists) f.move_to(dir_3, group=True, by_extension=True) with self.subTest(3): compare( os.path.join(dir_3, 'document', 'TXT', 'my awesome cat.txt'), f.path) with self.subTest(4): compare(True, f.exists) with self.subTest(5): compare( False, os.path.isfile(os.path.join(dir_2, 'TXT', 'my awesome cat.txt'))) def test_returns_false_if_grouping_by_category_fails_group_false(self): dir_1 = self.tempdir.makedir('abc/fig/') dir_2 = self.tempdir.makedir('one/twp/') dir_3 = self.tempdir.makedir('some/dir/') file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') self.tempdir.write(os.path.join(dir_1, '1.jpeg'), '') f1 = File(file_1) f1.move_to(dst_root_path=dir_2, group=False, by_extension=False, group_folder_name=None) # default with self.subTest(1): final_path = os.path.join(dir_2, 'TXT', 'my awesome cat.txt') compare(final_path, f1.path) f1.move_to(dst_root_path=dir_3, group=False, by_extension=False, group_folder_name=None) # default with self.subTest(1): final_path = os.path.join(dir_3, 'TXT', 'my awesome cat.txt') compare(final_path, f1.path) def test_returns_false_if_grouping_fails_group_true(self): dir_1 = self.tempdir.makedir('abc/fig/') dir_2 = self.tempdir.makedir('one/twp/') self.tempdir.makedir('some/dir/') file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') self.tempdir.write(os.path.join(dir_1, '1.jpeg'), '') f1 = File(file_1) f1.move_to(dst_root_path=dir_2, group=True, by_extension=False, group_folder_name=None) with self.subTest(1): final_path = os.path.join(dir_2, 'document', 'my awesome cat.txt') compare(final_path, f1.path) def test_returns_false_if_grouping_fails_by_extension_true_group_folder_name_none( self): dir_1 = self.tempdir.makedir('abc/fig/') dir_2 = self.tempdir.makedir('one/twp/') self.tempdir.makedir('some/dir/') file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') self.tempdir.write(os.path.join(dir_1, '1.jpeg'), '') f1 = File(file_1) f1.move_to(dst_root_path=dir_2, group=True, by_extension=True, group_folder_name=None) with self.subTest(1): final_path = os.path.join(dir_2, 'document', 'TXT', 'my awesome cat.txt') compare(final_path, f1.path) def test_returns_false_if_grouping_fails_by_extension_true_group_folder_name_given( self): dir_1 = self.tempdir.makedir('abc/fig/') dir_2 = self.tempdir.makedir('one/twp/') self.tempdir.makedir('some/dir/') file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') self.tempdir.write(os.path.join(dir_1, '1.jpeg'), '') f1 = File(file_1) f1.move_to(dst_root_path=dir_2, group=True, by_extension=True, group_folder_name='sample dir') with self.subTest(1): final_path = os.path.join(dir_2, 'sample dir', 'TXT', 'my awesome cat.txt') compare(final_path, f1.path) def test_returns_false_if_grouping_fails_by_extension_false_group_folder_name_given( self): dir_1 = self.tempdir.makedir('abc/fig/') dir_2 = self.tempdir.makedir('one/twp/') dir_3 = self.tempdir.makedir('some/dir/') file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') file_2 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') file_3 = self.tempdir.write(os.path.join(dir_1, '1.jpeg'), '') f1 = File(file_1) f1.move_to(dst_root_path=dir_2, group=True, by_extension=False, group_folder_name='sample dir') with self.subTest(1): final_path = os.path.join(dir_2, 'sample dir', 'my awesome cat.txt') compare(final_path, f1.path) def test_returns_false_if_relocation_to_existing_child_folder_fails(self): dir_1 = self.tempdir.makedir('abc/fig/') dir_2 = self.tempdir.makedir('abc/fig/one/two') dir_3 = self.tempdir.makedir('abc/fig/one/two/three/') file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') f1 = File(file_1) f1.move_to(dst_root_path=dir_2, group=True, by_extension=True, group_folder_name='sample dir') with self.subTest(1): final_path = os.path.join(dir_2, 'sample dir', 'TXT', 'my awesome cat.txt') compare(final_path, f1.path) f1.move_to(dst_root_path=dir_3, group=True, by_extension=True, group_folder_name='sample dir') with self.subTest(2): final_path = os.path.join(dir_3, 'sample dir', 'TXT', 'my awesome cat.txt') compare(final_path, f1.path) def test_returns_false_if_relocation_to_non_existent_folder_fails(self): temp_path = self.tempdir.path dir_1 = self.tempdir.makedir('abc/fig/') dir_2 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two') dir_3 = os.path.join(temp_path, 'first', 'fig', 'one', 'two', 'three') file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') f1 = File(file_1) f1.move_to(dst_root_path=dir_2, group=True, by_extension=True, group_folder_name='sample dir') with self.subTest(1): final_path = os.path.join(dir_2, 'sample dir', 'TXT', 'my awesome cat.txt') compare(final_path, f1.path) f1.move_to(dst_root_path=dir_3, group=True, by_extension=True, group_folder_name='sample dir') with self.subTest(2): final_path = os.path.join(dir_3, 'sample dir', 'TXT', 'my awesome cat.txt') compare(final_path, f1.path) def test_returns_false_if_relocation_to_parent_folder_fails(self): temp_path = self.tempdir.path dir_1 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two', 'three') dir_2 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two') dir_3 = os.path.join(temp_path, 'abc', 'fig') self.tempdir.makedir(dir_1) file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') f1 = File(file_1) f1.move_to(dst_root_path=dir_2, group=True, by_extension=True, group_folder_name='sample dir') with self.subTest(1): final_path = os.path.join(dir_2, 'sample dir', 'TXT', 'my awesome cat.txt') compare(final_path, f1.path) f1.move_to(dst_root_path=dir_3, group=True, by_extension=True, group_folder_name='sample dir') with self.subTest(2): final_path = os.path.join(dir_3, 'sample dir', 'TXT', 'my awesome cat.txt') compare(final_path, f1.path) def test_returns_false_if_relocation_of_file_without_extension_fails(self): temp_path = self.tempdir.path dir_1 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two', 'three') dir_2 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two') dir_3 = os.path.join(temp_path, 'abc', 'fig') self.tempdir.makedir(dir_1) file_1 = self.tempdir.write( os.path.join(dir_1, '146 awesome street $# yea'), '') f1 = File(file_1) f1.move_to(dst_root_path=dir_2, group=True, by_extension=True, group_folder_name='sample dir') with self.subTest(1): compare(True, f1.exists) with self.subTest(2): final_path = os.path.join(dir_2, 'sample dir', 'UNDEFINED', '146 awesome street $# yea') compare(final_path, f1.path) f1.move_to(dst_root_path=dir_3, group=False, by_extension=True, group_folder_name='sample dir') with self.subTest(3): final_path = os.path.join(dir_3, 'UNDEFINED', '146 awesome street $# yea') compare(final_path, f1.path) def test_returns_false_if_emptynameerror_not_raised(self): temp_path = self.tempdir.path dir_1 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two', 'three') dir_2 = os.path.join(temp_path, 'abc', 'fig', 'one', 'two') self.tempdir.makedir(dir_1) file_1 = self.tempdir.write(os.path.join(dir_1, 'my awesome cat.txt'), '') f1 = File(file_1) self.assertRaises(EmptyNameError, f1.move_to, dst_root_path=dir_2, group=True, by_extension=True, group_folder_name=' ') def test_returns_false_if_file_not_created(self): dir_1 = self.tempdir.makedir('abc/fig/one/two/three') file_1 = os.path.join(dir_1, 'my awesome cat.txt') f1 = File(file_1) with self.subTest(1): compare(False, f1.exists) f1.touch() with self.subTest(2): compare(True, f1.exists) with self.subTest(3): self.assertRaises(FileExistsError, f1.touch, exist_ok=False)
class TestInferenceDataSet2p5D(unittest.TestCase): def setUp(self): """ Set up a directory with images """ self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.tempdir.makedir('image_dir') self.tempdir.makedir('model_dir') self.tempdir.makedir('mask_dir') self.image_dir = os.path.join(self.temp_path, 'image_dir') self.model_dir = os.path.join(self.temp_path, 'model_dir') self.mask_dir = os.path.join(self.temp_path, 'mask_dir') # Create a temp image dir im = np.zeros((10, 16), dtype=np.uint8) self.frames_meta = aux_utils.make_dataframe() self.time_idx = 2 for p in range(5): for z in range(4): for c in range(3): im_name = aux_utils.get_im_name( time_idx=self.time_idx, channel_idx=c, slice_idx=z, pos_idx=p, ) cv2.imwrite(os.path.join(self.image_dir, im_name), im + c * 10) self.frames_meta = self.frames_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to image dir too self.frames_meta.to_csv(os.path.join(self.image_dir, 'frames_meta.csv')) # Save masks and mask meta self.mask_meta = aux_utils.make_dataframe() self.mask_channel = 50 for p in range(5): for z in range(4): im_name = aux_utils.get_im_name( time_idx=2, channel_idx=self.mask_channel, slice_idx=z, pos_idx=p, ) cv2.imwrite(os.path.join(self.mask_dir, im_name), im + 1) self.mask_meta = self.mask_meta.append( aux_utils.parse_idx_from_name(im_name, aux_utils.DF_NAMES), ignore_index=True, ) # Write frames meta to image dir too self.mask_meta.to_csv(os.path.join(self.mask_dir, 'frames_meta.csv')) # Select inference split of dataset self.split_col_ids = ('pos_idx', [1, 3]) # Make configs with fields necessary for inference dataset dataset_config = { 'input_channels': [2], 'target_channels': [self.mask_channel], 'model_task': 'segmentation', } self.network_config = { 'class': 'UNetStackTo2D', 'depth': 3, 'data_format': 'channels_first', } # Instantiate class self.data_inst = inference_dataset.InferenceDataSet( image_dir=self.image_dir, dataset_config=dataset_config, network_config=self.network_config, split_col_ids=self.split_col_ids, mask_dir=self.mask_dir, ) def tearDown(self): """ Tear down temporary folder and file structure """ TempDirectory.cleanup_all() self.assertEqual(os.path.isdir(self.temp_path), False) def test_init(self): """ Test init of inference dataset """ self.assertEqual(self.data_inst.image_dir, self.image_dir) self.assertEqual(self.data_inst.target_dir, self.mask_dir) self.assertIsNone(self.data_inst.flat_field_dir) self.assertEqual(self.data_inst.image_format, 'zyx') self.assertEqual(self.data_inst.model_task, 'segmentation') self.assertEqual(self.data_inst.depth, 3) self.assertFalse(self.data_inst.squeeze) self.assertFalse(self.data_inst.im_3d) self.assertEqual(self.data_inst.data_format, 'channels_first') self.assertListEqual(self.data_inst.input_channels, [2]) self.assertListEqual(self.data_inst.target_channels, [self.mask_channel]) # Two inference samples (pos idx 1 and 3), two slices (1, 2) each = 4 self.assertEqual(self.data_inst.num_samples, 4) self.assertListEqual( self.data_inst.frames_meta.pos_idx.unique().tolist(), [1, 3], ) # Image channels = 0, 1, 2 and target channel = 50 self.assertListEqual( self.data_inst.frames_meta.channel_idx.unique().tolist(), [0, 1, 2, 50]) def test_get_iteration_meta(self): iteration_meta = self.data_inst.get_iteration_meta() # This contains metadata for first target channel only z=1,2, p=1,3 self.assertTupleEqual(iteration_meta.shape, (4, 6)) self.assertListEqual( iteration_meta.channel_idx.unique().tolist(), [self.mask_channel], ) # Contains only test indices pos 1 and 3 self.assertListEqual( iteration_meta.pos_idx.unique().tolist(), [1, 3], ) # Contains two slices, 1 and 2 self.assertListEqual( iteration_meta.slice_idx.unique().tolist(), [1, 2], ) def test__len__(self): num_samples = self.data_inst.__len__() self.assertEqual(num_samples, 4) def test_get_image(self): meta_row = dict.fromkeys(aux_utils.DF_NAMES) meta_row['channel_idx'] = 2 meta_row['time_idx'] = self.time_idx meta_row['slice_idx'] = 1 meta_row['pos_idx'] = 3 im_stack = self.data_inst._get_image( input_dir=self.image_dir, cur_row=meta_row, channel_ids=[2], depth=3, normalize=False, ) # Image shapes are cropped to nearest factor of two, channels first self.assertTupleEqual(im_stack.shape, (1, 3, 8, 16)) # Channel 2 has constant values of 20 self.assertEqual(im_stack.max(), 20) self.assertEqual(im_stack.min(), 20) def test__getitem__(self): # There are 2 test indices (pos 1 and 3) input_stack, target_stack = self.data_inst.__getitem__(1) # Cropped to factor of 2, add batch dim self.assertTupleEqual(input_stack.shape, (1, 1, 3, 8, 16)) self.assertTupleEqual(target_stack.shape, (1, 1, 1, 8, 16)) # input stack should be normalized, not target self.assertEqual(input_stack.max(), 0.0) self.assertEqual(target_stack.max(), 1) self.assertEqual(input_stack.dtype, np.float32) self.assertEqual(target_stack.dtype, np.float32) def test__getitem__regression(self): dataset_config = { 'input_channels': [1, 2], 'target_channels': [0], 'model_task': 'regression', } # Instantiate class data_inst = inference_dataset.InferenceDataSet( image_dir=self.image_dir, dataset_config=dataset_config, network_config=self.network_config, split_col_ids=self.split_col_ids, ) # There are 2 test indices (pos 1 and 3) input_stack, target_stack = data_inst.__getitem__(0) # Cropped to factor of 2, add batch dim self.assertTupleEqual(input_stack.shape, (1, 2, 3, 8, 16)) self.assertTupleEqual(target_stack.shape, (1, 1, 1, 8, 16)) # input stack should be normalized, not target self.assertEqual(input_stack.max(), 0.0) self.assertEqual(target_stack.max(), 0.0) self.assertEqual(input_stack.dtype, np.float32) self.assertEqual(target_stack.dtype, np.float32)
class TestDataDownloader(db_basetest.DBBaseTest): """ Test the data downloader with S3 storage """ @patch('imaging_db.database.db_operations.session_scope') def setUp(self, mock_session): super().setUp() mock_session.return_value.__enter__.return_value = self.session # Setup mock S3 bucket self.mock = mock_s3() self.mock.start() self.conn = boto3.resource('s3', region_name='us-east-1') self.bucket_name = 'czbiohub-imaging' self.conn.create_bucket(Bucket=self.bucket_name) # Test metadata parameters self.nbr_channels = 2 self.nbr_slices = 3 # Mock S3 dir self.dataset_serial = 'FRAMES-2005-06-09-20-00-00-1000' self.frames_storage_dir = os.path.join('raw_frames', self.dataset_serial) # Create temporary directory and write temp image self.tempdir = TempDirectory() self.temp_path = self.tempdir.path # Temporary file with 6 frames, tifffile stores channels first self.im = 50 * np.ones((6, 10, 15), dtype=np.uint16) self.im[0, :5, 3:12] = 50000 self.im[2, :5, 3:12] = 40000 self.im[4, :5, 3:12] = 30000 # Metadata self.description = 'ImageJ=1.52e\nimages=6\nchannels=2\nslices=3\nmax=10411.0' # Save test tif file self.file_path = os.path.join(self.temp_path, "A1_2_PROTEIN_test.tif") tifffile.imsave( self.file_path, self.im, description=self.description, ) upload_csv = pd.DataFrame( columns=['dataset_id', 'file_name', 'description'], ) upload_csv = upload_csv.append( { 'dataset_id': self.dataset_serial, 'file_name': self.file_path, 'description': 'Testing' }, ignore_index=True, ) self.csv_path_frames = os.path.join( self.temp_path, "test_upload_frames.csv", ) upload_csv.to_csv(self.csv_path_frames) self.credentials_path = os.path.join( self.main_dir, 'db_credentials.json', ) # Write a config file self.config_path = os.path.join( self.temp_path, 'config_tif_id.json', ) config = { "upload_type": "frames", "frames_format": "tif_id", "microscope": "Leica microscope CAN bus adapter", "filename_parser": "parse_ml_name", "storage": "s3" } json_ops.write_json_file(config, self.config_path) # Upload frames data_uploader.upload_data_and_update_db( csv=self.csv_path_frames, login=self.credentials_path, config=self.config_path, ) # Create inputs for file upload self.dataset_serial_file = 'FILE-2005-06-01-01-00-00-1000' self.file_storage_dir = os.path.join('raw_files', self.dataset_serial_file) self.csv_path_file = os.path.join( self.temp_path, "test_upload_file.csv", ) # Change to unique serial upload_csv['dataset_id'] = self.dataset_serial_file upload_csv.to_csv(self.csv_path_file) config_path = os.path.join( self.temp_path, 'config_file.json', ) config = { "upload_type": "file", "microscope": "Mass Spectrometry", "storage": "s3", } json_ops.write_json_file(config, config_path) # Upload file data_uploader.upload_data_and_update_db( csv=self.csv_path_file, login=self.credentials_path, config=config_path, ) def tearDown(self): """ Rollback database session. Tear down temporary folder and file structure, stop moto mock """ super().tearDown() TempDirectory.cleanup_all() self.assertFalse(os.path.isdir(self.temp_path)) self.mock.stop() def test_parse_args(self): with patch('argparse._sys.argv', [ 'python', '--id', self.dataset_serial, '-p', '5', '-t', '0', '-c', '1', '2', '3', '-z', '4', '5', '--dest', 'dest_path', '--login', 'test_login.json', '--nbr_workers', '5' ]): parsed_args = data_downloader.parse_args() self.assertEqual(parsed_args.id, self.dataset_serial) self.assertListEqual(parsed_args.positions, [5]) self.assertListEqual(parsed_args.times, [0]) self.assertListEqual(parsed_args.channels, ['1', '2', '3']) self.assertListEqual(parsed_args.slices, [4, 5]) self.assertEqual(parsed_args.dest, 'dest_path') self.assertEqual(parsed_args.login, 'test_login.json') self.assertEqual(parsed_args.nbr_workers, 5) def test_parse_args_defaults(self): with patch('argparse._sys.argv', [ 'python', '--id', self.dataset_serial, '--dest', 'dest_path', '--login', 'test_login.json' ]): parsed_args = data_downloader.parse_args() self.assertIsNone(parsed_args.positions) self.assertIsNone(parsed_args.times) self.assertIsNone(parsed_args.channels) self.assertIsNone(parsed_args.slices) self.assertTrue(parsed_args.metadata) self.assertTrue(parsed_args.download) self.assertIsNone(parsed_args.nbr_workers) @patch('imaging_db.database.db_operations.session_scope') def test_download_frames(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') # Download data data_downloader.download_data( dataset_serial=self.dataset_serial, login=self.credentials_path, dest=dest_dir, storage='s3', ) # Images are separated by slice first then channel im_order = [0, 2, 4, 1, 3, 5] it = itertools.product(range(self.nbr_channels), range(self.nbr_slices)) for i, (c, z) in enumerate(it): im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z) im_path = os.path.join( dest_dir, self.dataset_serial, im_name, ) im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH) numpy.testing.assert_array_equal(im, self.im[im_order[i], ...]) # Read and validate frames meta meta_path = os.path.join( dest_dir, self.dataset_serial, 'frames_meta.csv', ) frames_meta = pd.read_csv(meta_path) for i, row in frames_meta.iterrows(): c = i // self.nbr_slices z = i % self.nbr_slices self.assertEqual(row.channel_idx, c) self.assertEqual(row.slice_idx, z) self.assertEqual(row.time_idx, 0) self.assertEqual(row.pos_idx, 0) im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z) self.assertEqual(row.file_name, im_name) sha256 = meta_utils.gen_sha256(self.im[im_order[i], ...]) self.assertEqual(row.sha256, sha256) # Read and validate global meta meta_path = os.path.join( dest_dir, self.dataset_serial, 'global_metadata.json', ) meta_json = json_ops.read_json_file(meta_path) self.assertEqual(meta_json['storage_dir'], self.frames_storage_dir) self.assertEqual(meta_json['nbr_frames'], 6) self.assertEqual(meta_json['im_width'], 15) self.assertEqual(meta_json['im_height'], 10) self.assertEqual(meta_json['nbr_slices'], self.nbr_slices) self.assertEqual(meta_json['nbr_channels'], self.nbr_channels) self.assertEqual(meta_json['im_colors'], 1) self.assertEqual(meta_json['nbr_timepoints'], 1) self.assertEqual(meta_json['nbr_positions'], 1) self.assertEqual(meta_json['bit_depth'], 'uint16') @patch('imaging_db.database.db_operations.session_scope') def test_download_channel(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') # Download data data_downloader.download_data( dataset_serial=self.dataset_serial, login=self.credentials_path, dest=dest_dir, storage='s3', channels=1, ) download_dir = os.path.join(dest_dir, self.dataset_serial) # Check frames_meta content frames_meta = pd.read_csv(os.path.join(download_dir, 'frames_meta.csv')) for i, row in frames_meta.iterrows(): self.assertEqual(row.channel_idx, 1) im_name = 'im_c001_z00{}_t000_p000.png'.format(i) self.assertEqual(row.file_name, im_name) # Check downloaded images im_order = [1, 3, 5] for z in range(3): im_name = 'im_c001_z00{}_t000_p000.png'.format(z) im_path = os.path.join(download_dir, im_name) im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH) numpy.testing.assert_array_equal(im, self.im[im_order[i], ...]) @patch('imaging_db.database.db_operations.session_scope') def test_download_channel_convert_str(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') # Download data data_downloader.download_data( dataset_serial=self.dataset_serial, login=self.credentials_path, dest=dest_dir, storage='s3', channels='1', ) download_dir = os.path.join(dest_dir, self.dataset_serial) # Check frames_meta content frames_meta = pd.read_csv(os.path.join(download_dir, 'frames_meta.csv')) for i, row in frames_meta.iterrows(): self.assertEqual(row.channel_idx, 1) im_name = 'im_c001_z00{}_t000_p000.png'.format(i) self.assertEqual(row.file_name, im_name) # Check downloaded images im_order = [1, 3, 5] for z in range(3): im_name = 'im_c001_z00{}_t000_p000.png'.format(z) im_path = os.path.join(download_dir, im_name) im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH) numpy.testing.assert_array_equal(im, self.im[im_order[i], ...]) @nose.tools.raises(AssertionError) @patch('imaging_db.database.db_operations.session_scope') def test_download_channel_name(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') # Download data data_downloader.download_data( dataset_serial=self.dataset_serial, login=self.credentials_path, dest=dest_dir, storage='s3', channels='channel1', ) @patch('imaging_db.database.db_operations.session_scope') def test_download_pts(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') # Download data data_downloader.download_data( dataset_serial=self.dataset_serial, login=self.credentials_path, dest=dest_dir, storage='s3', positions=0, times=[0], slices=1, ) meta_path = os.path.join( dest_dir, self.dataset_serial, 'global_metadata.json', ) frames_meta = pd.read_csv(meta_path) for i, row in frames_meta.iterrows(): self.assertEqual(row.pos_idx, 0) self.assertEqual(row.time_idx, 0) self.assertEqual(row.slice_idx, 1) @patch('imaging_db.database.db_operations.session_scope') def test_download_file(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') # Download data data_downloader.download_data( dataset_serial=self.dataset_serial_file, login=self.credentials_path, dest=dest_dir, storage='s3', metadata=False, nbr_workers=2, ) # See if file has been downloaded file_path = os.path.join( dest_dir, self.dataset_serial_file, '*', ) found_file = os.path.basename(glob.glob(file_path)[0]) self.assertEqual("A1_2_PROTEIN_test.tif", found_file) @nose.tools.raises(FileExistsError) @patch('imaging_db.database.db_operations.session_scope') def test_folder_exists(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') self.tempdir.makedir( os.path.join('dest_dir', self.dataset_serial_file), ) dest_dir = os.path.join(self.temp_path, 'dest_dir') data_downloader.download_data( dataset_serial=self.dataset_serial_file, login=self.credentials_path, dest=dest_dir, storage='s3', nbr_workers=2, metadata=False, ) @nose.tools.raises(AssertionError) @patch('imaging_db.database.db_operations.session_scope') def test_no_download_or_meta(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') data_downloader.download_data( dataset_serial=self.dataset_serial_file, login=self.credentials_path, dest=dest_dir, storage='s3', metadata=False, download=False, nbr_workers=2, ) @nose.tools.raises(AssertionError) @patch('imaging_db.database.db_operations.session_scope') def test_invalid_dataset(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') self.tempdir.makedir( os.path.join('dest_dir', self.dataset_serial_file), ) dest_dir = os.path.join(self.temp_path, 'dest_dir') data_downloader.download_data( dataset_serial='Not-a-serial', login=self.credentials_path, dest=dest_dir, storage='s3', metadata=False, nbr_workers=2, ) @nose.tools.raises(AssertionError) @patch('imaging_db.database.db_operations.session_scope') def test_negative_workers(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') data_downloader.download_data( dataset_serial=self.dataset_serial_file, login=self.credentials_path, dest=dest_dir, storage='s3', metadata=False, download=False, nbr_workers=-2, ) @patch('imaging_db.database.db_operations.session_scope') def test__main__(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') with patch('argparse._sys.argv', [ 'python', '--id', self.dataset_serial, '--dest', dest_dir, '--storage', 's3', '--login', self.credentials_path ]): runpy.run_path( 'imaging_db/cli/data_downloader.py', run_name='__main__', ) # Check that files are there dest_files = os.listdir( os.path.join( dest_dir, self.dataset_serial, )) self.assertTrue('frames_meta.csv' in dest_files) self.assertTrue('global_metadata.json' in dest_files) for c in range(self.nbr_channels): for z in range(self.nbr_slices): im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z) self.assertTrue(im_name in dest_files)
class TestDataDownloaderLocalStorage(db_basetest.DBBaseTest): """ Test the data downloader with local storage """ @patch('imaging_db.database.db_operations.session_scope') def setUp(self, mock_session): super().setUp() mock_session.return_value.__enter__.return_value = self.session # Create temporary directory and write temp image self.tempdir = TempDirectory() self.temp_path = self.tempdir.path # Mock file storage self.tempdir.makedir('storage_mount_point') self.mount_point = os.path.join(self.temp_path, 'storage_mount_point') self.tempdir.makedir('storage_mount_point/raw_files') self.tempdir.makedir('storage_mount_point/raw_frames') # Test metadata parameters self.nbr_channels = 2 self.nbr_slices = 3 # Mock storage dir self.dataset_serial = 'FRAMES-2005-06-09-20-00-00-1000' self.frames_storage_dir = os.path.join('raw_frames', self.dataset_serial) # Temporary file with 6 frames, tifffile stores channels first self.im = 50 * np.ones((6, 10, 15), dtype=np.uint16) self.im[0, :5, 3:12] = 50000 self.im[2, :5, 3:12] = 40000 self.im[4, :5, 3:12] = 30000 # Metadata self.description = 'ImageJ=1.52e\nimages=6\nchannels=2\nslices=3\nmax=10411.0' # Save test tif file self.file_path = os.path.join(self.temp_path, "A1_2_PROTEIN_test.tif") tifffile.imsave( self.file_path, self.im, description=self.description, ) # Create input arguments for data upload upload_csv = pd.DataFrame( columns=['dataset_id', 'file_name', 'description'], ) upload_csv = upload_csv.append( { 'dataset_id': self.dataset_serial, 'file_name': self.file_path, 'description': 'Testing' }, ignore_index=True, ) self.csv_path_frames = os.path.join( self.temp_path, "test_upload_frames.csv", ) upload_csv.to_csv(self.csv_path_frames) self.credentials_path = os.path.join( self.main_dir, 'db_credentials.json', ) self.config_path = os.path.join( self.temp_path, 'config_tif_id.json', ) config = { "upload_type": "frames", "frames_format": "tif_id", "microscope": "Leica microscope CAN bus adapter", "filename_parser": "parse_ml_name", "storage": "local", "storage_access": self.mount_point } json_ops.write_json_file(config, self.config_path) # Upload frames data_uploader.upload_data_and_update_db( csv=self.csv_path_frames, login=self.credentials_path, config=self.config_path, ) # Create input args for file upload self.dataset_serial_file = 'FILE-2005-06-09-20-00-00-1000' self.file_storage_dir = os.path.join('raw_files', self.dataset_serial_file) self.csv_path_file = os.path.join( self.temp_path, "test_upload_file.csv", ) # Change to unique serial upload_csv['dataset_id'] = self.dataset_serial_file upload_csv.to_csv(self.csv_path_file) config_path = os.path.join( self.temp_path, 'config_file.json', ) config = { "upload_type": "file", "microscope": "Mass Spectrometry", "storage": "local", "storage_access": self.mount_point } json_ops.write_json_file(config, config_path) # Upload file data_uploader.upload_data_and_update_db( csv=self.csv_path_file, login=self.credentials_path, config=config_path, ) def tearDown(self): """ Rollback database session. Tear down temporary folder and file structure, stop moto mock """ super().tearDown() TempDirectory.cleanup_all() self.assertFalse(os.path.isdir(self.temp_path)) @patch('imaging_db.database.db_operations.session_scope') def test_download_frames(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') # Download data data_downloader.download_data( dataset_serial=self.dataset_serial, login=self.credentials_path, dest=dest_dir, storage_access=self.mount_point, ) # Images are separated by slice first then channel im_order = [0, 2, 4, 1, 3, 5] it = itertools.product(range(self.nbr_channels), range(self.nbr_slices)) for i, (c, z) in enumerate(it): im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z) im_path = os.path.join( dest_dir, self.dataset_serial, im_name, ) im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH) numpy.testing.assert_array_equal(im, self.im[im_order[i], ...]) # Read and validate frames meta meta_path = os.path.join( dest_dir, self.dataset_serial, 'frames_meta.csv', ) frames_meta = pd.read_csv(meta_path) for i, row in frames_meta.iterrows(): c = i // self.nbr_slices z = i % self.nbr_slices self.assertEqual(row.channel_idx, c) self.assertEqual(row.slice_idx, z) self.assertEqual(row.time_idx, 0) self.assertEqual(row.pos_idx, 0) im_name = 'im_c00{}_z00{}_t000_p000.png'.format(c, z) self.assertEqual(row.file_name, im_name) sha256 = meta_utils.gen_sha256(self.im[im_order[i], ...]) self.assertEqual(row.sha256, sha256) # Read and validate global meta meta_path = os.path.join( dest_dir, self.dataset_serial, 'global_metadata.json', ) meta_json = json_ops.read_json_file(meta_path) self.assertEqual(meta_json['storage_dir'], self.frames_storage_dir) self.assertEqual(meta_json['nbr_frames'], 6) self.assertEqual(meta_json['im_width'], 15) self.assertEqual(meta_json['im_height'], 10) self.assertEqual(meta_json['nbr_slices'], self.nbr_slices) self.assertEqual(meta_json['nbr_channels'], self.nbr_channels) self.assertEqual(meta_json['im_colors'], 1) self.assertEqual(meta_json['nbr_timepoints'], 1) self.assertEqual(meta_json['nbr_positions'], 1) self.assertEqual(meta_json['bit_depth'], 'uint16') @patch('imaging_db.database.db_operations.session_scope') def test_download_channel(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') # Download data data_downloader.download_data( dataset_serial=self.dataset_serial, login=self.credentials_path, dest=dest_dir, storage_access=self.mount_point, channels=1, ) download_dir = os.path.join(dest_dir, self.dataset_serial) # Check frames_meta content frames_meta = pd.read_csv(os.path.join(download_dir, 'frames_meta.csv')) for i, row in frames_meta.iterrows(): self.assertEqual(row.channel_idx, 1) im_name = 'im_c001_z00{}_t000_p000.png'.format(i) self.assertEqual(row.file_name, im_name) # Check downloaded images im_order = [1, 3, 5] for z in range(3): im_name = 'im_c001_z00{}_t000_p000.png'.format(z) im_path = os.path.join(download_dir, im_name) im = cv2.imread(im_path, cv2.IMREAD_ANYDEPTH) numpy.testing.assert_array_equal(im, self.im[im_order[i], ...]) @nose.tools.raises(AssertionError) @patch('imaging_db.database.db_operations.session_scope') def test_download_channel_name(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') # Download data data_downloader.download_data( dataset_serial=self.dataset_serial, login=self.credentials_path, dest=dest_dir, storage_access=self.mount_point, channels='channel1', ) @patch('imaging_db.database.db_operations.session_scope') def test_download_pts(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') # Download data data_downloader.download_data( dataset_serial=self.dataset_serial, login=self.credentials_path, dest=dest_dir, storage_access=self.mount_point, positions=0, times=0, slices=1, ) meta_path = os.path.join( dest_dir, self.dataset_serial, 'global_metadata.json', ) frames_meta = pd.read_csv(meta_path) for i, row in frames_meta.iterrows(): self.assertEqual(row.pos_idx, 0) self.assertEqual(row.time_idx, 0) self.assertEqual(row.slice_idx, 1) @patch('imaging_db.database.db_operations.session_scope') def test_download_file(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') dest_dir = os.path.join(self.temp_path, 'dest_dir') # Download data data_downloader.download_data( dataset_serial=self.dataset_serial_file, login=self.credentials_path, dest=dest_dir, storage_access=self.mount_point, metadata=False, nbr_workers=2, ) # See if file has been downloaded file_path = os.path.join( dest_dir, self.dataset_serial_file, '*', ) found_file = os.path.basename(glob.glob(file_path)[0]) self.assertEqual("A1_2_PROTEIN_test.tif", found_file) @nose.tools.raises(FileExistsError) @patch('imaging_db.database.db_operations.session_scope') def test_folder_exists(self, mock_session): mock_session.return_value.__enter__.return_value = self.session # Create dest dir self.tempdir.makedir('dest_dir') self.tempdir.makedir( os.path.join('dest_dir', self.dataset_serial_file), ) dest_dir = os.path.join(self.temp_path, 'dest_dir') data_downloader.download_data( dataset_serial=self.dataset_serial_file, login=self.credentials_path, dest=dest_dir, storage_access=self.mount_point, nbr_workers=2, metadata=False, )
class TestFileSplitter(unittest.TestCase): """ Abstract classes can be tested with Unittest's mock patch """ @patch.multiple(file_splitter.FileSplitter, __abstractmethods__=set()) def setUp(self): # Setup mock local storage # Create temporary directory and write temp image self.tempdir = TempDirectory() self.temp_path = self.tempdir.path self.tempdir.makedir('storage_mount_point') mount_point = os.path.join(self.temp_path, 'storage_mount_point') self.test_path = "/datapath/testfile.tif" self.storage_dir = "raw_frames/ISP-2005-06-09-20-00-00-0001" storage_class = aux_utils.get_storage_class('local') self.mock_inst = file_splitter.FileSplitter( data_path=self.test_path, storage_dir=self.storage_dir, storage_class=storage_class, storage_access=mount_point) def tearDown(self): """ Tear down temporary folder and files and stop S3 mock """ TempDirectory.cleanup_all() nose.tools.assert_equal(os.path.isdir(self.temp_path), False) def test_init(self): nose.tools.assert_equal(self.mock_inst.data_path, self.test_path) nose.tools.assert_equal(self.mock_inst.storage_dir, self.storage_dir) nose.tools.assert_equal(self.mock_inst.int2str_len, 3) nose.tools.assert_equal(self.mock_inst.file_format, '.png') @nose.tools.raises(AssertionError) def test_get_imstack(self): self.mock_inst.get_imstack() def test_get_existing_imstack(self): self.mock_inst.im_stack = np.zeros((5, 10)) im_stack = self.mock_inst.get_imstack() self.assertTupleEqual(im_stack.shape, (5, 10)) @nose.tools.raises(AssertionError) def test_get_global_meta(self): self.mock_inst.get_global_meta() @nose.tools.raises(AssertionError) def test_get_global_json(self): self.mock_inst.get_global_json() @nose.tools.raises(AssertionError) def test_get_frames_meta(self): self.mock_inst.get_frames_meta() @nose.tools.raises(AssertionError) def test_get_frames_json(self): self.mock_inst.get_frames_json() def test_get_imname(self): meta_row = { "channel_idx": 6, "slice_idx": 13, "time_idx": 5, "pos_idx": 7, } im_name = self.mock_inst._get_imname(meta_row=meta_row) nose.tools.assert_equal(im_name, 'im_c006_z013_t005_p007.png') def test_set_global_meta(self): nbr_frames = 666 test_shape = (12, 15) meta_row = { "channel_idx": 6, "slice_idx": 13, "time_idx": 5, "pos_idx": 7, } self.mock_inst.frames_meta = meta_row self.mock_inst.frame_shape = test_shape self.mock_inst.im_colors = 1 self.mock_inst.bit_depth = 'uint16' # Set global meta self.mock_inst.set_global_meta(nbr_frames=nbr_frames) # Assert contents meta = self.mock_inst.get_global_meta() nose.tools.assert_equal(meta['storage_dir'], self.storage_dir) nose.tools.assert_equal(meta['nbr_frames'], nbr_frames) nose.tools.assert_equal(meta['im_height'], test_shape[0]) nose.tools.assert_equal(meta['im_width'], test_shape[1]) nose.tools.assert_equal(meta['im_colors'], 1) nose.tools.assert_equal(meta['bit_depth'], 'uint16') nose.tools.assert_equal(meta['nbr_slices'], 1) nose.tools.assert_equal(meta['nbr_channels'], 1) nose.tools.assert_equal(meta['nbr_timepoints'], 1) nose.tools.assert_equal(meta['nbr_positions'], 1) @nose.tools.raises(AssertionError) def test_missing_meta(self): self.mock_inst.set_global_meta(nbr_frames=666) @nose.tools.raises(NotImplementedError) def test_set_frame_info(self): self.mock_inst.set_frame_info() @nose.tools.raises(NotImplementedError) def test_get_frames_and_metadata(self): self.mock_inst.get_frames_and_metadata()
class TestML4IRSanity(unittest.TestCase): """ Datasets used in these tests are generated by sklearn.datasets.make_classification by changing its parameters. Then queries and documents are grouped randomly (without replacement) so that each query has 2-25 documents with one clicked document. Baseline models (Perceptron and logistic regression are trained with sklearn) """ def setUp(self): self.dir = pathlib.Path(__file__).parent self.working_dir = TempDirectory() self.log_dir = self.working_dir.makedir('logs') self.working_dir.makedir('train') self.working_dir.makedir('test') self.working_dir.makedir('validation') def tearDown(self): TempDirectory.cleanup_all() # Explicitly clear keras memory gc.collect() K.clear_session() def test_linear_ml4ir_sanity_1(self): """ Weights of pre-trained models: perceptron=np.array([1.87212065, -0.00305068]) log_regression=np.array([28.62071696, 1.18915853]) """ run_sanity_test(n_features=2, fname="dataset1.csv", perceptron_mrr=1.0, log_regression_mrr=1.0, working_dir=pathlib.Path(self.working_dir.path), log_dir=pathlib.Path(self.log_dir)) def test_linear_ml4ir_sanity_2(self): """ Weights of pre-trained models: perceptron=np.array([4.50209484, -0.80280452]) log_reg=np.array([22.73585585, -3.94821153]) """ run_sanity_test(n_features=2, fname="dataset2.csv", perceptron_mrr=1.0, log_regression_mrr=1.0, working_dir=pathlib.Path(self.working_dir.path), log_dir=pathlib.Path(self.log_dir)) def test_linear_ml4ir_sanity_3(self): """ Weights of pre-trained models: perceptron=np.array([-1.27651475, -4.07647092, 8.23950305, 0.29241316, 3.24763417]) log_reg=np.array([-1.67270377, -5.76088727, 8.36278576, -0.90878154, 3.47653204]) """ run_sanity_test(n_features=5, fname="dataset3.csv", perceptron_mrr=1.0, log_regression_mrr=1.0, working_dir=pathlib.Path(self.working_dir.path), log_dir=pathlib.Path(self.log_dir)) def test_linear_ml4ir_sanity_4(self): """ Weights of pre-trained models: perceptron=np.array([5.10535665, 1.44131417]) log_reg=np.array([20.0954756, 4.69360163]) """ run_sanity_test(n_features=2, fname="dataset4.csv", perceptron_mrr=0.975609756097561, log_regression_mrr=0.9878048780487805, working_dir=pathlib.Path(self.working_dir.path), log_dir=pathlib.Path(self.log_dir)) def test_linear_ml4ir_sanity_5(self): """ Weights of pre-trained models: perceptron=np.array([0.57435291, -0.99437351]) log_reg=np.array([1.15593505, -0.93317691]) """ run_sanity_test(n_features=2, fname="dataset5.csv", perceptron_mrr=0.37021768881524975, log_regression_mrr=0.4290204375570229, working_dir=pathlib.Path(self.working_dir.path), log_dir=pathlib.Path(self.log_dir)) def test_linear_ml4ir_sanity_6(self): """ Weights of pre-trained models: perceptron=np.array( [4.59994733, -3.56373965, -6.15935686, 0.87523846, -0.64231058, 2.15971991, 5.79875003, -7.70152594, -0.07521741, 2.8817456]) log_reg=np.array( [-0.38064406, -0.27970534, 0.02775136, 0.25641926, 0.15413321, 0.29194965, 0.72707686, 0.24791729, -0.39367192, 0.4882174]) """ run_sanity_test(n_features=10, fname="dataset6.csv", perceptron_mrr=0.2630707011921534, log_regression_mrr=0.29970690874564615, working_dir=pathlib.Path(self.working_dir.path), log_dir=pathlib.Path(self.log_dir)) def test_linear_ml4ir_sanity_7(self): """ Weights of pre-trained models: perceptron=np.array([0.40127356, -0.43773627]) log_reg=np.array([4.15630544, -1.09111369]) """ run_sanity_test(n_features=2, fname="dataset7.csv", perceptron_mrr=0.304679599925838, log_regression_mrr=0.4938108725899423, working_dir=pathlib.Path(self.working_dir.path), log_dir=pathlib.Path(self.log_dir)) def test_linear_ml4ir_sanity_8(self): """ Weights of pre-trained models: perceptron=np.array( [2.91798129, 4.24880336, 7.42919018, 2.49609694, -0.84988373, 0.43435823, -0.18953416, 2.23129287, -0.67951411, -0.63925108]) log_reg=np.array( [-0.14472192, -0.22594271, 0.62703883, 0.16002515, 0.17084088, -0.22872226, 0.89200279, 0.06297475, 0.70470567, -0.19396659]) """ run_sanity_test(n_features=10, fname="dataset8.csv", perceptron_mrr=0.31185341006782435, log_regression_mrr=0.3092000998257748, working_dir=pathlib.Path(self.working_dir.path), log_dir=pathlib.Path(self.log_dir))
class testVerify(unittest.TestCase): """Verify. Testing verify functionalities. """ def setUp(self): """Initialise fixtures.""" self.env = EnvironmentVarGuard() self._dir = TempDirectory() self.outputdir = self._dir.makedir('output') self.env = EnvironmentVarGuard() self.env.set('ARCHIVE_LOCATION', self.outputdir + '/%s/') self.env.set('LEGA_PASSWORD', 'value') self.env.set('QUALITY_CONTROL_VERIFY_PEER', 'True') self.env.set('QUALITY_CONTROL_VERIFY_HOSTNAME', 'False') def tearDown(self): """Remove setup variables.""" self.env.unset('ARCHIVE_LOCATION') self._dir.cleanup_all() self.env.unset('LEGA_PASSWORD') self.env.unset('QUALITY_CONTROL_VERIFY_PEER') self.env.unset('QUALITY_CONTROL_VERIFY_HOSTNAME') @tempdir() @mock.patch('lega.verify.header_to_records') @mock.patch('lega.verify.get_key_id') def test_get_records(self, mock_key, mock_records, filedir): """Should call the url in order to provide the records.""" infile = filedir.write('infile.in', bytearray.fromhex(pgp_data.ENC_FILE)) returned_data = KeyServerResponse( 200, io.BytesIO(pgp_data.PGP_PRIVKEY.encode())) with PatchContextManager('lega.verify.urlopen', returned_data) as mocked: print(returned_data.status) with open(infile, 'rb') as f: get_records(f) mocked.assert_called() filedir.cleanup() @tempdir() @mock.patch('lega.verify.header_to_records') @mock.patch('lega.verify.get_key_id') def test_get_records_no_verify(self, mock_key, mock_records, filedir): """Should call the url in order to provide the records even without a verify turned off.""" self.env.set('QUALITY_CONTROL_VERIFY_PEER', 'False') self.env.set('QUALITY_CONTROL_VERIFY_HOSTNAME', 'False') infile = filedir.write('infile.in', bytearray.fromhex(pgp_data.ENC_FILE)) returned_data = KeyServerResponse( 200, io.BytesIO(pgp_data.PGP_PRIVKEY.encode())) with PatchContextManager('lega.verify.urlopen', returned_data) as mocked: with open(infile, 'rb') as f: get_records(f) mocked.assert_called() filedir.cleanup() @tempdir() @mock.patch('lega.verify.header_to_records') @mock.patch('lega.verify.get_key_id') def test_get_records_key_error(self, mock_key, mock_records, filedir): """The PGP key was not found, should raise PGPKeyError error.""" infile = filedir.write('infile.in', bytearray.fromhex(pgp_data.ENC_FILE)) with mock.patch('lega.verify.urlopen') as urlopen_mock: urlopen_mock.side_effect = HTTPError('url', 404, 'msg', None, None) with self.assertRaises(PGPKeyError): with open(infile, 'rb') as f: get_records(f) filedir.cleanup() @tempdir() @mock.patch('lega.verify.header_to_records') @mock.patch('lega.verify.get_key_id') def test_get_records_server_error(self, mock_key, mock_records, filedir): """Some keyserver error occured, should raise KeyserverError error.""" infile = filedir.write('infile.in', bytearray.fromhex(pgp_data.ENC_FILE)) with mock.patch('lega.verify.urlopen') as urlopen_mock: urlopen_mock.side_effect = HTTPError('url', 400, 'msg', None, None) with self.assertRaises(KeyserverError): with open(infile, 'rb') as f: get_records(f) filedir.cleanup() @tempdir() @mock.patch('lega.verify.header_to_records') @mock.patch('lega.verify.get_key_id') def test_get_records_error(self, mock_key, mock_records, filedir): """Some general error occured, should raise Exception error.""" self.env.set('QUALITY_CONTROL_VERIFY_PEER', 'False') self.env.set('QUALITY_CONTROL_VERIFY_HOSTNAME', 'False') infile = filedir.write('infile.in', bytearray.fromhex(pgp_data.ENC_FILE)) with mock.patch('lega.verify.urlopen') as urlopen_mock: urlopen_mock.side_effect = Exception with self.assertRaises(Exception): with open(infile, 'rb') as f: get_records(f) filedir.cleanup() @mock.patch('lega.ingest.getattr') @mock.patch('lega.verify.get_connection') @mock.patch('lega.verify.consume') def test_main(self, mock_consume, mock_connection, mock_getattr): """Test main verify, by mocking cosume call.""" mock_consume.return_value = mock.MagicMock() main() mock_consume.assert_called() @tempdir() @mock.patch('lega.verify.db') @mock.patch('lega.verify.body_decrypt') @mock.patch('lega.verify.get_records') def test_work(self, mock_records, mock_decrypt, mock_db, filedir): """Test verify worker, should send a messge.""" # Mocking a lot of stuff, ast it is previously tested mock_db.status.return_value = mock.Mock() mock_records.return_value = ['data'], 'key_id' mock_decrypt.return_value = mock.Mock() store = mock.MagicMock() store.open.return_value = mock.MagicMock() mock_broker = mock.MagicMock(name='channel') mock_broker.channel.return_value = mock.Mock() infile = filedir.write('infile.in', 'text'.encode("utf-8")) data = { 'header': pgp_data.ENC_FILE, 'stable_id': '1', 'archive_path': infile, 'file_id': '123', 'org_msg': {} } result = work('10', store, mock_broker, data) self.assertTrue({'status': { 'state': 'COMPLETED', 'details': '1' }}, result) filedir.cleanup()