def test_version_warning(version): """ This is to test version warnings or even errors (eventually) """ remoteA = 'A' remoteB = 'B' set_debug(False) print(remoteA,remoteB) test = testutils.Tester('ver',remoteA,remoteB) ## Config if version: test.config._syncrclone_version = version test.write_config() # Setup test.write_pre('A/file00.txt','0') test.setup() # This has a sync that will throw the warnings stdout = ''.join(test.synclogs[-1]) if version and version.startswith('20200825.0'): assert 'WARNING Previous behavior of conflict_mode changed. Please update your config' in stdout else: assert 'WARNING Previous behavior of conflict_mode changed. Please update your config' not in stdout # if different_version_match os.chdir(PWD0)
def test_dry_run(): remoteA = 'A' remoteB = 'B' set_debug(False) test = testutils.Tester('backups',remoteA,remoteB) test.config.renamesA = 'mtime' test.write_config() # Expected conflicts test.write_pre('A/mod','0') # 1 test.write_pre('A/move','01') # 2 test.write_pre('A/movemod','012') # 2 test.write_pre('A/del','0123') # 1 test.setup() test.write_post('A/mod','A') shutil.move('A/move','A/moved') shutil.move('A/movemod','A/movedmod');test.write_post('A/movedmod','ABC') os.remove('A/del') test.write_post('A/new','01234') # 1 presync_compare = test.compare_tree() assert len(presync_compare) print('-='*40);print('=-'*40) test.sync(['--dry-run']) assert presync_compare == test.compare_tree() os.chdir(PWD0)
def test_logs(): remoteA = 'cryptA:' remoteB = 'B' set_debug(False) test = testutils.Tester('logs',remoteA,remoteB) test.config.renamesA = 'mtime' test.config.log_dest = 'logs' test.write_config() test.write_pre('A/file','0') test.setup() time.sleep(1.1) # To make sure we don't overwrite logs test.write_post('A/fileA','A') os.remove('A/file') test.write_post('B/fileB','BB') print('-='*40);print('=-'*40) test.sync() stdout = ''.join(test.synclogs[-1]) assert test.compare_tree() == set() # Will includes logs logs = sorted(glob.glob('A/logs/*')) assert len(logs) == 2, "should have two. setup + sync" # We know from all of the other tests that stdout (above) works otherwise # they would fail. So just check that it's the same with open(logs[-1]) as f: f.read().strip() == stdout.strip() os.chdir(PWD0)
def test_reuse_hash(): remoteA = 'A' remoteB = 'B' set_debug(True) print(remoteA,remoteB) test = testutils.Tester('reusehash',remoteA,remoteB) ## Config test.config.reuse_hashesA = True test.config.reuse_hashesB = False test.config.compare = 'hash' test.config.filter_flags = ['--filter','- .*','--filter','- .**/'] test.write_config() # Setup test.write_pre('A/file00.txt','0') test.setup() test.write_post('A/fileA1.txt','A1') test.write_post('B/fileB1.txt','B1') print('-='*40);print('=-'*40) test.sync(['--debug']) stdout = ''.join(test.synclogs[-1]) assert "A: Updated 1. Fetching hashes for 1" in stdout assert "B: Updated 1. Fetching hashes for 1" not in stdout os.chdir(PWD0)
def test_backups(backup): remoteA = 'A' remoteB = 'B' set_debug(False) test = testutils.Tester('backups',remoteA,remoteB) test.config.conflict_mode = 'newer' test.write_config() test.write_pre('A/ModifiedOnA.txt','A') test.write_pre('A/DeletedOnA.txt','A') test.write_pre('A/ModifiedOnB.txt','B') test.write_pre('A/DeletedOnB.txt','B') test.setup() os.remove('A/DeletedOnA.txt') os.remove('B/DeletedOnB.txt') test.write_post('A/ModifiedOnA.txt','A1') test.write_post('B/ModifiedOnB.txt','B1') print('-='*40);print('=-'*40) if backup: test.sync() elif backup is None: # Same as False but set it in the configs test.config.backup = False test.write_config() test.sync() else: test.sync(['--no-backup']) # Compare diffs = test.compare_tree() assert diffs == set() assert exists('A/ModifiedOnA.txt') and test.read('A/ModifiedOnA.txt') == 'A1' assert exists('A/ModifiedOnB.txt') and test.read('A/ModifiedOnB.txt') == 'B1' backedA = glob.glob('A/.*/backups/A_*/*') backedB = glob.glob('B/.*/backups/B_*/*') if backup: assert len(backedA) == len(backedB) == len(backedB) == len(backedB) == 2 # The B files were modified so these should all read B assert all(test.read(f) == 'B' for f in backedA) # not B1 assert all(file.endswith('B.txt') for file in backedA) # The A files were modified so these should all read A assert all(test.read(f) == 'A' for f in backedB) # not A1 assert all(file.endswith('A.txt') for file in backedB) else: # False and None assert len(backedA) == len(backedB) == len(backedB) == len(backedB) == 0 os.chdir(PWD0)
def test_legacy_filelist(legA, legB): remoteA = 'A' remoteB = 'B' set_debug(False) test = testutils.Tester('legacy_list', remoteA, remoteB) test.config.name = 'legacytest' test.write_config() test.write_pre('A/fileADEL.txt', 'ADEL') test.write_pre('A/fileSTAY.txt', 'STAY') test.write_pre('A/fileBDEL.txt', 'BDEL') test.setup() os.remove('A/fileADEL.txt' ) # If we do not have the previous list, then it will copy back! os.remove('B/fileBDEL.txt' ) # If we do not have the previous list, then it will copy back! # Convert the lists def xz2zipjson(xz, zj): HEADER = b'zipjson\x00\x00' with lzma.open(xz) as file: files = json.load(file) with open(zj, 'wb') as file: file.write(HEADER + zlib.compress( json.dumps(files, ensure_ascii=False).encode('utf8'))) os.unlink(xz) if legA: xz2zipjson('A/.syncrclone/A-legacytest_fl.json.xz', 'A/.syncrclone/A-legacytest_fl.zipjson') if legB: xz2zipjson('B/.syncrclone/B-legacytest_fl.json.xz', 'B/.syncrclone/B-legacytest_fl.zipjson') print('-=' * 40) print('=-' * 40) test.sync() # Compare diffs = test.compare_tree() assert not diffs assert not exists('A/fileADEL.txt') assert not exists('B/fileBDEL.txt') #test.sync() # Just to see if the log changed in manual testing os.chdir(PWD0)
def test_conflict_resolution(conflict_mode): remoteA = 'A' remoteB = 'B' set_debug(False) test = testutils.Tester('conflicts', remoteA, remoteB) ## Config test.config.conflict_mode = conflict_mode test.write_config() test.write_pre('A/file.txt', '0') test.setup() test.write_post('A/file.txt', 'A') test.write_post('B/file.txt', 'Bb', add_dt=20) # newer and larger print('-=' * 40) print('=-' * 40) test.sync() stdout = ''.join(test.synclogs[-1]) diffs = test.compare_tree() assert diffs == set() files = [os.path.relpath(f, 'A/') for f in testutils.tree('A/')] A = exists('A/file.txt') and test.read('A/file.txt') == 'A' B = exists('A/file.txt') and test.read('A/file.txt') == 'Bb' tA = any(file.endswith('.A') for file in files) tB = any(file.endswith('.B') for file in files) if conflict_mode in ['A', 'older', 'smaller']: assert (A, B, tA, tB) == (True, False, False, False), f"{conflict_mode} {(A,B,tA,tB)}" elif conflict_mode in ['B', 'newer', 'larger']: assert (A, B, tA, tB) == (False, True, False, False), f"{conflict_mode} {(A,B,tA,tB)}" elif conflict_mode in ['newer_tag']: assert (A, B, tA, tB) == (False, True, True, False), f"{conflict_mode} {(A,B,tA,tB)}" elif conflict_mode in ['tag']: assert (A, B, tA, tB) == (False, False, True, True), f"{conflict_mode} {(A,B,tA,tB)}" else: raise ValueError('Not studied') # Should not be here os.chdir(PWD0)
def test_conflict_resolution(conflict_mode,tag_conflict): remoteA = 'A' remoteB = 'B' set_debug(False) test = testutils.Tester('conflicts',remoteA,remoteB) ## Config test.config.conflict_mode = conflict_mode test.config.tag_conflict = tag_conflict test.write_config() test.write_pre('A/file.txt','0') test.setup() test.write_post('A/file.txt','A') test.write_post('B/file.txt','Bb',add_dt=20) # newer and larger print('-='*40);print('=-'*40) test.sync(['--debug']) stdout = ''.join(test.synclogs[-1]) diffs = test.compare_tree() assert diffs == set() files = [os.path.relpath(f,'A/') for f in testutils.tree('A/')] A = exists('A/file.txt') and test.read('A/file.txt') == 'A' B = exists('A/file.txt') and test.read('A/file.txt') == 'Bb' tA = any(file.endswith('.A.txt') for file in files) tB = any(file.endswith('.B.txt') for file in files) if conflict_mode in ['A','older','smaller']: res = (A,B) == (True,False) tag = (tA,tB) == (False,True) if tag_conflict else (False,False) elif conflict_mode in ['B','newer','larger']: res = (A,B) == False,True tag = (tA,tB) == (True,False) if tag_conflict else (False,False) elif conflict_mode in ['tag']: res = (A,B) == (False,False) tag = (tA,tB) == (True,True) else: raise ValueError('Not studied') # Should not be here assert res, f'Wrong res {A,B}, mode {conflict_mode,tag_conflict}' assert tag, f'Wrong tag {tA,tB} mode {conflict_mode,tag_conflict}' os.chdir(PWD0)
def test_no_hashes(): remoteA = 'A' remoteB = 'cryptB:' # Crypt does not have hashes set_debug(False) print(remoteA,remoteB) test = testutils.Tester('nocommon',remoteA,remoteB) ## Config test.config.compare = 'hash' test.config.hash_fail_fallback = 'mtime' test.write_config() # Setup test.write_pre('A/file00.txt','0') test.setup() # This has a sync that will throw the warnings stdout = ''.join(test.synclogs[-1]) # This will have to change if I change the verbage assert "WARNING No common hashes found and/or one or both remotes do not provide hashes. Falling back to 'mtime'" in stdout os.chdir(PWD0)
def test_three_way(): """ Test three way. In order to make this work within my own testing framework, I have to switch the configs manually as opposed to having multiple configurations. It isn't ideal but works for testing """ set_debug(False) test = testutils.Tester('three','A','B') # Just use simple comparisons test.config.renamesA = test.config.renamesB = 'hash' test.config.name = 'AB' test.write_config() test.write_pre('A/file1.txt','file1') test.write_pre('A/file2.txt','file1') ## Run test.setup() test.write_post('A/file1.txt','mod',mode='at') test.write_post('B/file3.txt','file3') test.sync() # Modify it to sync A <--> C test.config.remoteB = 'C' test.config.name = 'AC' test.write_config() test.write_pre('C/fileC.txt','this is on C') # This *just* makes sure that we don't have a false positive and we # are hacking it to compare C assert {('missing_inA', 'fileC.txt'), ('missing_inB', 'file1.txt'), ('missing_inB', 'file2.txt'), ('missing_inB', 'file3.txt')} == test.compare_tree(A='A',B='C') test.sync() assert test.compare_tree(A='A',B='C') == set() assert test.compare_tree(A='A',B='B') == {('missing_inB', 'fileC.txt')} # Should still miss that test.config.remoteB = 'B' test.config.name = 'AB' test.write_config() test.sync() assert test.compare_tree(A='A',B='C') == set() assert test.compare_tree(A='A',B='B') == set() # Change it again but this time with B <--> C test.config.remoteB = 'C' test.config.remoteA = 'B' test.config.name = 'BC' test.write_config() test.sync() # Shouldn't do anything test.write_post('B/file3.txt','file3 modified') test.sync() assert test.compare_tree(A='B',B='C') == set() assert test.read('C/file3.txt') == 'file3 modified' assert test.compare_tree(A='A',B='C') == {('disagree', 'file3.txt')} os.chdir(PWD0)
def test_main(remoteA,renamesA,remoteB,renamesB,compare,interactive=False): """ Main test with default settings (if the defaults change, this will need to be updated. A few minor changes from the defaults are also made More edge cases and specific settings are played with later """ set_debug(False) print(remoteA,remoteB) test = testutils.Tester('main',remoteA,remoteB) ## Config test.config.reuse_hashesA = False # Also shouldn't compute them test.config.renamesA = renamesA test.config.reuse_hashesB = True test.config.renamesB = renamesB test.config.rclone_flags = ['--fast-list'] # Will be ignored if not supported but good to have otherwise test.config.filter_flags = ['--filter','+ /yes/**', '--filter','- *.no'] test.config.compare = compare test.config.conflict_mode = 'newer_tag' # Deprecated. Update in the future test.config.log_dest = 'logs/' test.write_config() ## Initial files test.write_pre('A/leave_alone.txt','do not touch') # We use newer_tag so we can confirm that these are *NOT* considered # conflicts. They should be backed up *and* test.write_pre('A/EditOnA.txt','Edit on A') test.write_pre('A/EditOnB.txt','Edit on B') # Give it extra so size is unique test.write_pre('A/MoveOnA.txt','Move on A' + randstr(52)) test.write_pre('A/MoveOnB.txt','Move on B' + randstr(100)) test.write_pre('A/MoveOnAB.txt','Move on Both' + randstr(74)) test.write_pre('A/MoveEditOnA.txt','Move and Edit on A') test.write_pre('A/MoveEditOnB.txt','Move and Edit on B') test.write_pre('A/EditOnBoth_Anewer.txt','A will be newer') test.write_pre('A/EditOnBoth_Bnewer.txt','B will be newer') test.write_pre('A/MoveEditOnBoth_Bnewer.txt','Will move and edit on both sides' + randstr(10)) test.write_pre('A/delA.txt','delete on A') test.write_pre('A/delB.txt','delete on B') test.write_pre('A/delA modB.txt','delA but mod on B') test.write_pre('A/delB modA.txt','delB but mod on A') test.write_pre('A/unic°de and space$.txt','UTF8') test.write_pre('A/common_contentAfter0.txt','abc xyz') test.write_pre('A/common_contentAfter1.txt','abc xy') test.write_pre('A/common_contentBefore0.txt','ABC XYZ') test.write_pre('A/common_contentBefore1.txt','ABC XYZ') ## Run test.setup() ## Modify test.write_post('A/EditOnA.txt','Edited on A',mode='at') test.write_post('B/EditOnB.txt','Edited on B',mode='at') test.move('A/MoveOnA.txt','A/sub/MovedOnA.txt') # move and rename test.move('B/MoveOnB.txt','B/sub2/MoveOnB.txt') # just move test.move('A/MoveOnAB.txt','A/MovedOnAB.txt') test.move('B/MoveOnAB.txt','B/MovedOnAB.txt') test.move('A/MoveEditOnA.txt','A/MovedEditOnA.txt') test.move('B/MoveEditOnB.txt','B/MovedEditOnB.txt') test.write_post('A/MovedEditOnA.txt','Move and Edit on A',mode='at') test.write_post('B/MovedEditOnB.txt','Move and Edit on B',mode='at') # recall when comparing by size, When comparing by size, older --> smaller, newer --> larger test.write_post('A/EditOnBoth_Anewer.txt','AAAa',mode='at',add_dt=50) # larger too test.write_post('A/EditOnBoth_Bnewer.txt','AAA',mode='at',add_dt=0) test.write_post('B/EditOnBoth_Anewer.txt','BBB',mode='at',add_dt=0) test.write_post('B/EditOnBoth_Bnewer.txt','BBBb',mode='at',add_dt=50) # larger too test.move('A/MoveEditOnBoth_Bnewer.txt','A/MovedEditedOnBoth_Bnewer.txt') test.move('B/MoveEditOnBoth_Bnewer.txt','B/MovedEditedOnBoth_Bnewer.txt') test.write_post('A/MovedEditedOnBoth_Bnewer.txt','A', mode='at') test.write_post('B/MovedEditedOnBoth_Bnewer.txt','BB', mode='at',add_dt=50) # larger too os.remove('A/delA.txt') os.remove('B/delB.txt') test.write_post('A/delB modA.txt','mod on A',mode='at') os.remove('B/delB modA.txt') test.write_post('B/delA modB.txt','mod on B',mode='at') os.remove('A/delA modB.txt') test.write_post('A/newA.txt','New on A') test.write_post('B/newB.txt','New on B') test.write_post('A/newA.no','New on A and no') # use new to test exclusions too test.write_post('B/newB.no','New on B and no') test.write_post('A/yes/newA.yes.no','New on A and no but yes') test.write_post('B/yes/newB.yes.no','New on B and no but yes') test.write_post('B/unic°de and space$.txt','works',mode='at') # These don't need to be tested other than not showing a diff test.write_post('A/common_contentAfter1.txt','abc xyz') test.write_post('B/common_contentBefore1.txt','ABC XYZW') print('-='*40) print('=-'*40) args = ['--interactive'] if interactive else [] obj = test.sync(args) ## Confirm! print('-'*100) diffs = test.compare_tree() # Exclusions except when filters to allow! assert {('missing_inA', 'newB.no'), ('missing_inB', 'newA.no')} == diffs stdout = ''.join(test.synclogs[-1]) # Check on A from now on! # Edits -- Should *NOT* tag but *should* backup assert test.read('A/EditOnA.txt') == 'Edit on AEdited on A',"mod did not propogate" assert test.read('A/EditOnB.txt') == 'Edit on BEdited on B',"mod did not propogate" assert not exists('A/EditOnA.txt.*'),'Should NOT have been tagged' assert not exists('A/EditOnB.txt.*'),'Should NOT have been tagged' assert test.globread('B/.syncrclone/backups/B_*/EditOnA.txt') == 'Edit on A','not backed up' assert test.globread('A/.syncrclone/backups/A_*/EditOnB.txt') == 'Edit on B','not backed up' # Moves w/o edit assert not exists('A/MoveOnB.txt') assert exists('A/sub2/MoveOnB.txt') assert "Move on A: 'MoveOnB.txt' --> 'sub2/MoveOnB.txt'" in stdout assert not exists('A/MoveOnA.txt') assert exists('A/sub/MovedOnA.txt') assert "Move on B: 'MoveOnA.txt' --> 'sub/MovedOnA.txt'" in stdout assert not exists('A/MoveOnAB.txt') assert exists('A/MovedOnAB.txt') assert not "move A: 'MoveOnAB.txt' --> 'MovedOnAB.txt'" in stdout assert not "move B: 'MoveOnAB.txt' --> 'MovedOnAB.txt'" in stdout # moves with edit -- should not have been tracked assert not exists('A/MoveEditOnA.txt') assert exists('A/MovedEditOnA.txt') assert 'MovedEditOnA.txt: Copied' in stdout,'Not copied. May be rclone log change' assert not exists('A/MoveEditOnB.txt') assert exists('A/MovedEditOnB.txt') assert 'MovedEditOnB.txt: Copied' in stdout,'Not copied. May be rclone log change' assert test.read('A/EditOnBoth_Anewer.txt') == 'A will be newerAAAa' assert test.read('A/EditOnBoth_Bnewer.txt') == 'B will be newerBBBb' assert exists('A/EditOnBoth_Anewer.*.B.txt'), "Not tagged" assert exists('A/EditOnBoth_Bnewer.*.A.txt'), "Not tagged" assert test.read('B/MovedEditedOnBoth_Bnewer.txt').endswith('BB') assert not exists('A/delA.txt') assert not exists('A/delB.txt') assert exists('A/.syncrclone/backups/A_*/delB.txt'), "did not backup" assert exists('B/.syncrclone/backups/B_*/delA.txt'), "did not backup" assert exists('A/delB modA.txt') # Should not have been deleted assert "DELETE CONFLICT: File 'delB modA.txt' deleted on B but modified on A. Transfering" in stdout assert exists('A/delA modB.txt') assert "DELETE CONFLICT: File 'delA modB.txt' deleted on A but modified on B. Transfering" in stdout assert exists('A/newA.txt') assert exists('A/newB.txt') assert exists('A/yes/newA.yes.no') assert exists('A/yes/newB.yes.no') assert test.read('A/unic°de and space$.txt') == 'UTF8works' assert exists('A/.syncrclone/backups/A_*/uni*.txt'),'did not back up' os.chdir(PWD0)
def test_move_attribs(attrib): """ Test moves over various conditions for each type of move tracking including modified sizes. 7 files: 1,2,3,C,D,4,5 all have different mtimes and are moved - 1,2 are unique content and size - 3,C are unique content but same size - D,4 are the same content and size - 5 is unique content and size but it's modified See Truth Table in the code Only test with local A (as it supports inode). B doesn't matter since moves are only tracked on A. """ remoteA = 'A' remoteB = 'B' print(attrib) set_debug(True) print(remoteA,remoteB) test = testutils.Tester('renames',remoteA,remoteB) ## Config test.config.reuse_hashesA = False test.config.renamesA = attrib if attrib == 'size': # Presumably we do not have mtime test.config.compare = 'size' test.config.dt = 0.001 test.write_config() # Setup test.write_pre('A/file1.txt','1') test.write_pre('A/file2.txt','12') test.write_pre('A/file3.txt','123') test.write_pre('A/fileC.txt','ABC') test.write_pre('A/fileD.txt','ABCD') test.write_pre('A/file4.txt','ABCD') test.write_pre('A/file5.txt','12345') test.setup() for c in '123CD45': shutil.move(f'A/file{c}.txt',f'A/file{c}_moved.txt') test.write_post('A/file5_moved.txt','12345') # Changes mtime but same content print('-='*40) print('=-'*40) test.sync() stdout = ''.join(test.synclogs[-1]) ## Truth Table if not attrib: notmoved = '123CD45' moved = too_many = '' elif attrib == 'size': # Size alone won't care about the mod. Note that compare is 'size' for # this one too since that is likely all you would have moved = '125' too_many = '3CD4' notmoved = '' elif attrib == 'mtime': moved = '1234CD' too_many = '' notmoved = '5' elif attrib == 'inode': moved = '123CD4' too_many = '' notmoved = '5' elif attrib == 'hash': moved = '123C5' too_many = 'D4' notmoved = '' for c in moved: assert f"Move on B: 'file{c}.txt' --> 'file{c}_moved.txt'" in stdout,f"{attrib} file{c} didn't move" for c in too_many: assert f"Too many possible previous files for 'file{c}_moved.txt' on A" in stdout, f"{attrib} file{c} failed multiple" for c in notmoved: assert f"Move on B: 'file{c}.txt' --> 'file{c}_moved.txt'" not in stdout,f"{attrib} file{c} moved" os.chdir(PWD0)