def setUp(self): self.config = MSDSConfig(localdir=tempfile.mkdtemp()) self.api = MSDataSyncAPI(self.config, msds_log) self.impl = MSDSImpl(msds_log, self.api) # method stub self.api.post_sync_step = dingus.Dingus() def remove_localdir(): os.rmdir(self.config["localdir"]) self.addCleanup(remove_localdir) self.rsyncconfig = self.RemoteSyncParamsStub({ "filename1": (False, False), # this file didn't change "filename2": (False, False), # this file didn't change "filename3": (False, True), # this file changed "asdf": (False, True), # this file changed }) self.filename_id_map = { "filename1": (1, 10), "filename2": (2, 20), "filename3": (3, 30), "filename4": (4, 40), }
def test_parse_rsync_changes_cull(self): """ Check culling of empty directories from file_changes map. """ changes = { '.': (True, True), 'runs': (True, True), 'runs/2013': (True, True), 'runs/2013/7': (True, True), 'runs/2013/7/475': (True, True), 'runs/2013/7/475/solvent_475-50479.d': (True, True), 'runs/2013/7/475/sweep_475-50563.d': (True, True), 'runs/2013/7/475/sweep_475-50563.d/DATA.MS': (False, False), 'runs/2013/7/475/sweep_475-50563.d/GC.ini': (False, False), 'runs/2013/7/475/sweep_475-50563.d/PRE_POST.INI': (False, False), 'runs/2013/7/475/sweep_475-50563.d/acqmeth.txt': (False, True), 'runs/2013/7/475/sweep_475-50563.d/cnorm.ini': (False, False), 'runs/2013/7/475/sweep_475-50563.d/runstart.txt': (False, False), 'runs/2013/7/475/sweep_475-50563.d/tic_front.csv': (False, False) } culled = MSDSImpl.cull_empty_dirs(changes) culled_files = [f for f, c in culled] self.assertTrue("runs/2013" in culled_files) self.assertFalse("runs/2013/7/475/solvent_475-50479.d" in culled_files) self.assertTrue("runs/2013/7/475/sweep_475-50563.d" in culled_files) self.assertTrue("runs/2013/7/475/sweep_475-50563.d/DATA.MS" in culled_files) self.assertTrue("runs/2013/7/475/sweep_475-50563.d/acqmeth.txt" in culled_files)
def test_parse_rsync_changes_propagate(self): """ Check propagation of file changes to parent directories. """ data = """ .d ./ .d runs/ .d runs/2013/ .d runs/2013/7/ .d runs/2013/7/475/ cd+++++++++ runs/2013/7/475/solvent_475-50479.d/ .d runs/2013/7/475/sweep_475-50563.d/ <f..T...... runs/2013/7/475/sweep_475-50563.d/DATA.MS <f..T...... runs/2013/7/475/sweep_475-50563.d/GC.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/PRE_POST.INI <f+.T...... runs/2013/7/475/sweep_475-50563.d/acqmeth.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/cnorm.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/runstart.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/tic_front.csv """ changes = MSDSImpl.parse_rsync_changes(data) self.assertEqual(changes["runs/2013/7/475/solvent_475-50479.d"], (True, True), "runs/2013/7/475/solvent_475-50479.d") self.assertEqual(changes["runs/2013/7/475/sweep_475-50563.d"], (True, True), "runs/2013/7/475/sweep_475-50563.d") self.assertEqual(changes["runs/2013/7/475/sweep_475-50563.d/DATA.MS"], (False, False), "runs/2013/7/475/sweep_475-50563.d/DATA.MS") self.assertEqual(changes["runs/2013/7/475/sweep_475-50563.d/acqmeth.txt"], (False, True), "runs/2013/7/475/solvent_475-50479.d/acqmeth.txt")
def test_parse_rsync_changes_empty_dir(self): """ Check parsing of rsync output -- changed and unchanged files and directories. """ data = """ .d ./ .d runs/ .d runs/2013/ .d runs/2013/7/ .d runs/2013/7/475/ cd+++++++++ runs/2013/7/475/solvent_475-50479.d/ .d runs/2013/7/475/sweep_475-50563.d/ <f..T...... runs/2013/7/475/sweep_475-50563.d/DATA.MS <f..T...... runs/2013/7/475/sweep_475-50563.d/GC.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/PRE_POST.INI <f..T...... runs/2013/7/475/sweep_475-50563.d/acqmeth.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/cnorm.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/runstart.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/tic_front.csv """ changes = MSDSImpl.parse_rsync_changes(data) self.assertEqual(changes["runs/2013/7/475/solvent_475-50479.d"], (True, True)) self.assertEqual(changes["runs/2013/7/475/sweep_475-50563.d"], (True, False)) self.assertEqual(changes["runs/2013/7/475"], (True, True)) self.assertEqual(changes["runs"], (True, True))
def test_parse_rsync_changes_cull(self): """ Check culling of empty directories from file_changes map. """ changes = { '.': (True, True), 'runs': (True, True), 'runs/2013': (True, True), 'runs/2013/7': (True, True), 'runs/2013/7/475': (True, True), 'runs/2013/7/475/solvent_475-50479.d': (True, True), 'runs/2013/7/475/sweep_475-50563.d': (True, True), 'runs/2013/7/475/sweep_475-50563.d/DATA.MS': (False, False), 'runs/2013/7/475/sweep_475-50563.d/GC.ini': (False, False), 'runs/2013/7/475/sweep_475-50563.d/PRE_POST.INI': (False, False), 'runs/2013/7/475/sweep_475-50563.d/acqmeth.txt': (False, True), 'runs/2013/7/475/sweep_475-50563.d/cnorm.ini': (False, False), 'runs/2013/7/475/sweep_475-50563.d/runstart.txt': (False, False), 'runs/2013/7/475/sweep_475-50563.d/tic_front.csv': (False, False) } culled = MSDSImpl.cull_empty_dirs(changes) culled_files = [f for f, c in culled] self.assertTrue("runs/2013" in culled_files) self.assertFalse("runs/2013/7/475/solvent_475-50479.d" in culled_files) self.assertTrue("runs/2013/7/475/sweep_475-50563.d" in culled_files) self.assertTrue( "runs/2013/7/475/sweep_475-50563.d/DATA.MS" in culled_files) self.assertTrue( "runs/2013/7/475/sweep_475-50563.d/acqmeth.txt" in culled_files)
def test_parse_rsync_changes_propagate(self): """ Check propagation of file changes to parent directories. """ data = """ .d ./ .d runs/ .d runs/2013/ .d runs/2013/7/ .d runs/2013/7/475/ cd+++++++++ runs/2013/7/475/solvent_475-50479.d/ .d runs/2013/7/475/sweep_475-50563.d/ <f..T...... runs/2013/7/475/sweep_475-50563.d/DATA.MS <f..T...... runs/2013/7/475/sweep_475-50563.d/GC.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/PRE_POST.INI <f+.T...... runs/2013/7/475/sweep_475-50563.d/acqmeth.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/cnorm.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/runstart.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/tic_front.csv """ changes = MSDSImpl.parse_rsync_changes(data) self.assertEqual(changes["runs/2013/7/475/solvent_475-50479.d"], (True, True), "runs/2013/7/475/solvent_475-50479.d") self.assertEqual(changes["runs/2013/7/475/sweep_475-50563.d"], (True, True), "runs/2013/7/475/sweep_475-50563.d") self.assertEqual(changes["runs/2013/7/475/sweep_475-50563.d/DATA.MS"], (False, False), "runs/2013/7/475/sweep_475-50563.d/DATA.MS") self.assertEqual( changes["runs/2013/7/475/sweep_475-50563.d/acqmeth.txt"], (False, True), "runs/2013/7/475/solvent_475-50479.d/acqmeth.txt")
class MSDSImplTests(unittest.TestCase): """ These are unit tests for the `MSDSImpl` class. """ def setUp(self): self.config = MSDSConfig(localdir=tempfile.mkdtemp()) self.api = MSDataSyncAPI(self.config, msds_log) self.impl = MSDSImpl(msds_log, self.api) # method stub self.api.post_sync_step = dingus.Dingus() def remove_localdir(): os.rmdir(self.config["localdir"]) self.addCleanup(remove_localdir) self.rsyncconfig = self.RemoteSyncParamsStub({ "filename1": (False, False), # this file didn't change "filename2": (False, False), # this file didn't change "filename3": (False, True), # this file changed "asdf": (False, True), # this file changed }) self.filename_id_map = { "filename1": (1, 10), "filename2": (2, 20), "filename3": (3, 30), "filename4": (4, 40), } # serverCheckRunSampleFiles method only uses the # file_changes member of RemoteSyncParams, so stub it class RemoteSyncParamsStub(object): def __init__(self, file_changes): self.file_changes = file_changes def test1_check_run_sample_files(self): """ Check that `MSDSImpl.serverCheckRunSampleFiles()` results in an API call marking complete the correct samples. """ Server = dingus.Dingus() with dingus.patch("mdatasync_client.MSDataSyncAPI.DataSyncServer", Server): self.impl.serverCheckRunSampleFiles(self.rsyncconfig, self.filename_id_map) self.assertEqual(len(Server.return_value.calls), 1, "One DataSyncServer instance is created") runsampledict, last_error = Server.return_value.calls[0][1] self.assertEqual(len(runsampledict), 2, "Two samples are marked complete") self.assertEqual(runsampledict, { 1: [10], 2: [20] }, "The correct samples are marked complete") def create_data_file(self, *path): temp_name = os.path.join(self.config["localdir"], *path) os.makedirs(os.path.dirname(temp_name)) open(temp_name, "w").close() def cleanup(temp_name, nparents): logger.debug("removing %s" % temp_name) os.remove(temp_name) for p in range(nparents): temp_name = os.path.dirname(temp_name) logger.debug("rmdir %s" % temp_name) os.rmdir(temp_name) self.addCleanup(cleanup, temp_name, len(path) - 1) def test2_check_run_sample_files_temp_files(self): """ Sample is not marked incomplete if there is a TEMP file in its directory. """ # create a file called TEMP within a directory called # filename2 somewhere within the data dir self.create_data_file("test", "dir", "filename2", "TEMP") Server = dingus.Dingus() with dingus.patch("mdatasync_client.MSDataSyncAPI.DataSyncServer", Server): self.impl.serverCheckRunSampleFiles(self.rsyncconfig, self.filename_id_map) self.assertEqual(len(Server.return_value.calls), 1, "One DataSyncServer instance is created") runsampledict, last_error = Server.return_value.calls[0][1] self.assertEqual(len(runsampledict), 1, "One sample is marked complete") self.assertEqual(runsampledict, { 1: [10] }, "The correct sample is marked complete") def test_parse_rsync_changes_empty_dir(self): """ Check parsing of rsync output -- changed and unchanged files and directories. """ data = """ .d ./ .d runs/ .d runs/2013/ .d runs/2013/7/ .d runs/2013/7/475/ cd+++++++++ runs/2013/7/475/solvent_475-50479.d/ .d runs/2013/7/475/sweep_475-50563.d/ <f..T...... runs/2013/7/475/sweep_475-50563.d/DATA.MS <f..T...... runs/2013/7/475/sweep_475-50563.d/GC.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/PRE_POST.INI <f..T...... runs/2013/7/475/sweep_475-50563.d/acqmeth.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/cnorm.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/runstart.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/tic_front.csv """ changes = MSDSImpl.parse_rsync_changes(data) self.assertEqual(changes["runs/2013/7/475/solvent_475-50479.d"], (True, True)) self.assertEqual(changes["runs/2013/7/475/sweep_475-50563.d"], (True, False)) self.assertEqual(changes["runs/2013/7/475"], (True, True)) self.assertEqual(changes["runs"], (True, True)) # self.assertEqual(changes["."], # (True, True)) def test_parse_rsync_changes_propagate(self): """ Check propagation of file changes to parent directories. """ data = """ .d ./ .d runs/ .d runs/2013/ .d runs/2013/7/ .d runs/2013/7/475/ cd+++++++++ runs/2013/7/475/solvent_475-50479.d/ .d runs/2013/7/475/sweep_475-50563.d/ <f..T...... runs/2013/7/475/sweep_475-50563.d/DATA.MS <f..T...... runs/2013/7/475/sweep_475-50563.d/GC.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/PRE_POST.INI <f+.T...... runs/2013/7/475/sweep_475-50563.d/acqmeth.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/cnorm.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/runstart.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/tic_front.csv """ changes = MSDSImpl.parse_rsync_changes(data) self.assertEqual(changes["runs/2013/7/475/solvent_475-50479.d"], (True, True), "runs/2013/7/475/solvent_475-50479.d") self.assertEqual(changes["runs/2013/7/475/sweep_475-50563.d"], (True, True), "runs/2013/7/475/sweep_475-50563.d") self.assertEqual(changes["runs/2013/7/475/sweep_475-50563.d/DATA.MS"], (False, False), "runs/2013/7/475/sweep_475-50563.d/DATA.MS") self.assertEqual(changes["runs/2013/7/475/sweep_475-50563.d/acqmeth.txt"], (False, True), "runs/2013/7/475/solvent_475-50479.d/acqmeth.txt") def test_parse_rsync_changes_cull(self): """ Check culling of empty directories from file_changes map. """ changes = { '.': (True, True), 'runs': (True, True), 'runs/2013': (True, True), 'runs/2013/7': (True, True), 'runs/2013/7/475': (True, True), 'runs/2013/7/475/solvent_475-50479.d': (True, True), 'runs/2013/7/475/sweep_475-50563.d': (True, True), 'runs/2013/7/475/sweep_475-50563.d/DATA.MS': (False, False), 'runs/2013/7/475/sweep_475-50563.d/GC.ini': (False, False), 'runs/2013/7/475/sweep_475-50563.d/PRE_POST.INI': (False, False), 'runs/2013/7/475/sweep_475-50563.d/acqmeth.txt': (False, True), 'runs/2013/7/475/sweep_475-50563.d/cnorm.ini': (False, False), 'runs/2013/7/475/sweep_475-50563.d/runstart.txt': (False, False), 'runs/2013/7/475/sweep_475-50563.d/tic_front.csv': (False, False) } culled = MSDSImpl.cull_empty_dirs(changes) culled_files = [f for f, c in culled] self.assertTrue("runs/2013" in culled_files) self.assertFalse("runs/2013/7/475/solvent_475-50479.d" in culled_files) self.assertTrue("runs/2013/7/475/sweep_475-50563.d" in culled_files) self.assertTrue("runs/2013/7/475/sweep_475-50563.d/DATA.MS" in culled_files) self.assertTrue("runs/2013/7/475/sweep_475-50563.d/acqmeth.txt" in culled_files)
class MSDSImplTests(unittest.TestCase): """ These are unit tests for the `MSDSImpl` class. """ def setUp(self): self.config = MSDSConfig(localdir=tempfile.mkdtemp()) self.api = MSDataSyncAPI(self.config, msds_log) self.impl = MSDSImpl(msds_log, self.api) # method stub self.api.post_sync_step = dingus.Dingus() def remove_localdir(): os.rmdir(self.config["localdir"]) self.addCleanup(remove_localdir) self.rsyncconfig = self.RemoteSyncParamsStub({ "filename1": (False, False), # this file didn't change "filename2": (False, False), # this file didn't change "filename3": (False, True), # this file changed "asdf": (False, True), # this file changed }) self.filename_id_map = { "filename1": (1, 10), "filename2": (2, 20), "filename3": (3, 30), "filename4": (4, 40), } # serverCheckRunSampleFiles method only uses the # file_changes member of RemoteSyncParams, so stub it class RemoteSyncParamsStub(object): def __init__(self, file_changes): self.file_changes = file_changes def test1_check_run_sample_files(self): """ Check that `MSDSImpl.serverCheckRunSampleFiles()` results in an API call marking complete the correct samples. """ Server = dingus.Dingus() with dingus.patch("mdatasync_client.MSDataSyncAPI.DataSyncServer", Server): self.impl.serverCheckRunSampleFiles(self.rsyncconfig, self.filename_id_map) self.assertEqual(len(Server.return_value.calls), 1, "One DataSyncServer instance is created") runsampledict, last_error = Server.return_value.calls[0][1] self.assertEqual(len(runsampledict), 2, "Two samples are marked complete") self.assertEqual(runsampledict, { 1: [10], 2: [20] }, "The correct samples are marked complete") def create_data_file(self, *path): temp_name = os.path.join(self.config["localdir"], *path) os.makedirs(os.path.dirname(temp_name)) open(temp_name, "w").close() def cleanup(temp_name, nparents): logger.debug("removing %s" % temp_name) os.remove(temp_name) for p in range(nparents): temp_name = os.path.dirname(temp_name) logger.debug("rmdir %s" % temp_name) os.rmdir(temp_name) self.addCleanup(cleanup, temp_name, len(path) - 1) def test2_check_run_sample_files_temp_files(self): """ Sample is not marked incomplete if there is a TEMP file in its directory. """ # create a file called TEMP within a directory called # filename2 somewhere within the data dir self.create_data_file("test", "dir", "filename2", "TEMP") Server = dingus.Dingus() with dingus.patch("mdatasync_client.MSDataSyncAPI.DataSyncServer", Server): self.impl.serverCheckRunSampleFiles(self.rsyncconfig, self.filename_id_map) self.assertEqual(len(Server.return_value.calls), 1, "One DataSyncServer instance is created") runsampledict, last_error = Server.return_value.calls[0][1] self.assertEqual(len(runsampledict), 1, "One sample is marked complete") self.assertEqual(runsampledict, {1: [10]}, "The correct sample is marked complete") def test_parse_rsync_changes_empty_dir(self): """ Check parsing of rsync output -- changed and unchanged files and directories. """ data = """ .d ./ .d runs/ .d runs/2013/ .d runs/2013/7/ .d runs/2013/7/475/ cd+++++++++ runs/2013/7/475/solvent_475-50479.d/ .d runs/2013/7/475/sweep_475-50563.d/ <f..T...... runs/2013/7/475/sweep_475-50563.d/DATA.MS <f..T...... runs/2013/7/475/sweep_475-50563.d/GC.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/PRE_POST.INI <f..T...... runs/2013/7/475/sweep_475-50563.d/acqmeth.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/cnorm.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/runstart.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/tic_front.csv """ changes = MSDSImpl.parse_rsync_changes(data) self.assertEqual(changes["runs/2013/7/475/solvent_475-50479.d"], (True, True)) self.assertEqual(changes["runs/2013/7/475/sweep_475-50563.d"], (True, False)) self.assertEqual(changes["runs/2013/7/475"], (True, True)) self.assertEqual(changes["runs"], (True, True)) # self.assertEqual(changes["."], # (True, True)) def test_parse_rsync_changes_propagate(self): """ Check propagation of file changes to parent directories. """ data = """ .d ./ .d runs/ .d runs/2013/ .d runs/2013/7/ .d runs/2013/7/475/ cd+++++++++ runs/2013/7/475/solvent_475-50479.d/ .d runs/2013/7/475/sweep_475-50563.d/ <f..T...... runs/2013/7/475/sweep_475-50563.d/DATA.MS <f..T...... runs/2013/7/475/sweep_475-50563.d/GC.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/PRE_POST.INI <f+.T...... runs/2013/7/475/sweep_475-50563.d/acqmeth.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/cnorm.ini <f..T...... runs/2013/7/475/sweep_475-50563.d/runstart.txt <f..T...... runs/2013/7/475/sweep_475-50563.d/tic_front.csv """ changes = MSDSImpl.parse_rsync_changes(data) self.assertEqual(changes["runs/2013/7/475/solvent_475-50479.d"], (True, True), "runs/2013/7/475/solvent_475-50479.d") self.assertEqual(changes["runs/2013/7/475/sweep_475-50563.d"], (True, True), "runs/2013/7/475/sweep_475-50563.d") self.assertEqual(changes["runs/2013/7/475/sweep_475-50563.d/DATA.MS"], (False, False), "runs/2013/7/475/sweep_475-50563.d/DATA.MS") self.assertEqual( changes["runs/2013/7/475/sweep_475-50563.d/acqmeth.txt"], (False, True), "runs/2013/7/475/solvent_475-50479.d/acqmeth.txt") def test_parse_rsync_changes_cull(self): """ Check culling of empty directories from file_changes map. """ changes = { '.': (True, True), 'runs': (True, True), 'runs/2013': (True, True), 'runs/2013/7': (True, True), 'runs/2013/7/475': (True, True), 'runs/2013/7/475/solvent_475-50479.d': (True, True), 'runs/2013/7/475/sweep_475-50563.d': (True, True), 'runs/2013/7/475/sweep_475-50563.d/DATA.MS': (False, False), 'runs/2013/7/475/sweep_475-50563.d/GC.ini': (False, False), 'runs/2013/7/475/sweep_475-50563.d/PRE_POST.INI': (False, False), 'runs/2013/7/475/sweep_475-50563.d/acqmeth.txt': (False, True), 'runs/2013/7/475/sweep_475-50563.d/cnorm.ini': (False, False), 'runs/2013/7/475/sweep_475-50563.d/runstart.txt': (False, False), 'runs/2013/7/475/sweep_475-50563.d/tic_front.csv': (False, False) } culled = MSDSImpl.cull_empty_dirs(changes) culled_files = [f for f, c in culled] self.assertTrue("runs/2013" in culled_files) self.assertFalse("runs/2013/7/475/solvent_475-50479.d" in culled_files) self.assertTrue("runs/2013/7/475/sweep_475-50563.d" in culled_files) self.assertTrue( "runs/2013/7/475/sweep_475-50563.d/DATA.MS" in culled_files) self.assertTrue( "runs/2013/7/475/sweep_475-50563.d/acqmeth.txt" in culled_files)