def test_version_warning(version):
"""
This is to test version warnings or even errors (eventually)
"""
remoteA = 'A'
remoteB = 'B'
set_debug(False)
print(remoteA,remoteB)
test = testutils.Tester('ver',remoteA,remoteB)
## Config
if version:
test.config._syncrclone_version = version
test.write_config()
# Setup
test.write_pre('A/file00.txt','0')
test.setup() # This has a sync that will throw the warnings
stdout = ''.join(test.synclogs[-1])
if version and version.startswith('20200825.0'):
assert 'WARNING Previous behavior of conflict_mode changed. Please update your config' in stdout
else:
assert 'WARNING Previous behavior of conflict_mode changed. Please update your config' not in stdout
# if different_version_match
os.chdir(PWD0)
def test_dry_run():
remoteA = 'A'
remoteB = 'B'
set_debug(False)
test = testutils.Tester('backups',remoteA,remoteB)
test.config.renamesA = 'mtime'
test.write_config()
# Expected conflicts
test.write_pre('A/mod','0') # 1
test.write_pre('A/move','01') # 2
test.write_pre('A/movemod','012') # 2
test.write_pre('A/del','0123') # 1
test.setup()
test.write_post('A/mod','A')
shutil.move('A/move','A/moved')
shutil.move('A/movemod','A/movedmod');test.write_post('A/movedmod','ABC')
os.remove('A/del')
test.write_post('A/new','01234') # 1
presync_compare = test.compare_tree()
assert len(presync_compare)
print('-='*40);print('=-'*40)
test.sync(['--dry-run'])
assert presync_compare == test.compare_tree()
os.chdir(PWD0)
def test_logs():
remoteA = 'cryptA:'
remoteB = 'B'
set_debug(False)
test = testutils.Tester('logs',remoteA,remoteB)
test.config.renamesA = 'mtime'
test.config.log_dest = 'logs'
test.write_config()
test.write_pre('A/file','0')
test.setup()
time.sleep(1.1) # To make sure we don't overwrite logs
test.write_post('A/fileA','A')
os.remove('A/file')
test.write_post('B/fileB','BB')
print('-='*40);print('=-'*40)
test.sync()
stdout = ''.join(test.synclogs[-1])
assert test.compare_tree() == set() # Will includes logs
logs = sorted(glob.glob('A/logs/*'))
assert len(logs) == 2, "should have two. setup + sync"
# We know from all of the other tests that stdout (above) works otherwise
# they would fail. So just check that it's the same
with open(logs[-1]) as f:
f.read().strip() == stdout.strip()
os.chdir(PWD0)
def test_reuse_hash():
remoteA = 'A'
remoteB = 'B'
set_debug(True)
print(remoteA,remoteB)
test = testutils.Tester('reusehash',remoteA,remoteB)
## Config
test.config.reuse_hashesA = True
test.config.reuse_hashesB = False
test.config.compare = 'hash'
test.config.filter_flags = ['--filter','- .*','--filter','- .**/']
test.write_config()
# Setup
test.write_pre('A/file00.txt','0')
test.setup()
test.write_post('A/fileA1.txt','A1')
test.write_post('B/fileB1.txt','B1')
print('-='*40);print('=-'*40)
test.sync(['--debug'])
stdout = ''.join(test.synclogs[-1])
assert "A: Updated 1. Fetching hashes for 1" in stdout
assert "B: Updated 1. Fetching hashes for 1" not in stdout
os.chdir(PWD0)
def test_backups(backup):
remoteA = 'A'
remoteB = 'B'
set_debug(False)
test = testutils.Tester('backups',remoteA,remoteB)
test.config.conflict_mode = 'newer'
test.write_config()
test.write_pre('A/ModifiedOnA.txt','A')
test.write_pre('A/DeletedOnA.txt','A')
test.write_pre('A/ModifiedOnB.txt','B')
test.write_pre('A/DeletedOnB.txt','B')
test.setup()
os.remove('A/DeletedOnA.txt')
os.remove('B/DeletedOnB.txt')
test.write_post('A/ModifiedOnA.txt','A1')
test.write_post('B/ModifiedOnB.txt','B1')
print('-='*40);print('=-'*40)
if backup:
test.sync()
elif backup is None: # Same as False but set it in the configs
test.config.backup = False
test.write_config()
test.sync()
else:
test.sync(['--no-backup'])
# Compare
diffs = test.compare_tree()
assert diffs == set()
assert exists('A/ModifiedOnA.txt') and test.read('A/ModifiedOnA.txt') == 'A1'
assert exists('A/ModifiedOnB.txt') and test.read('A/ModifiedOnB.txt') == 'B1'
backedA = glob.glob('A/.*/backups/A_*/*')
backedB = glob.glob('B/.*/backups/B_*/*')
if backup:
assert len(backedA) == len(backedB) == len(backedB) == len(backedB) == 2
# The B files were modified so these should all read B
assert all(test.read(f) == 'B' for f in backedA) # not B1
assert all(file.endswith('B.txt') for file in backedA)
# The A files were modified so these should all read A
assert all(test.read(f) == 'A' for f in backedB) # not A1
assert all(file.endswith('A.txt') for file in backedB)
else: # False and None
assert len(backedA) == len(backedB) == len(backedB) == len(backedB) == 0
os.chdir(PWD0)
def test_legacy_filelist(legA, legB):
remoteA = 'A'
remoteB = 'B'
set_debug(False)
test = testutils.Tester('legacy_list', remoteA, remoteB)
test.config.name = 'legacytest'
test.write_config()
test.write_pre('A/fileADEL.txt', 'ADEL')
test.write_pre('A/fileSTAY.txt', 'STAY')
test.write_pre('A/fileBDEL.txt', 'BDEL')
test.setup()
os.remove('A/fileADEL.txt'
) # If we do not have the previous list, then it will copy back!
os.remove('B/fileBDEL.txt'
) # If we do not have the previous list, then it will copy back!
# Convert the lists
def xz2zipjson(xz, zj):
HEADER = b'zipjson\x00\x00'
with lzma.open(xz) as file:
files = json.load(file)
with open(zj, 'wb') as file:
file.write(HEADER + zlib.compress(
json.dumps(files, ensure_ascii=False).encode('utf8')))
os.unlink(xz)
if legA:
xz2zipjson('A/.syncrclone/A-legacytest_fl.json.xz',
'A/.syncrclone/A-legacytest_fl.zipjson')
if legB:
xz2zipjson('B/.syncrclone/B-legacytest_fl.json.xz',
'B/.syncrclone/B-legacytest_fl.zipjson')
print('-=' * 40)
print('=-' * 40)
test.sync()
# Compare
diffs = test.compare_tree()
assert not diffs
assert not exists('A/fileADEL.txt')
assert not exists('B/fileBDEL.txt')
#test.sync() # Just to see if the log changed in manual testing
os.chdir(PWD0)
def test_conflict_resolution(conflict_mode):
remoteA = 'A'
remoteB = 'B'
set_debug(False)
test = testutils.Tester('conflicts', remoteA, remoteB)
## Config
test.config.conflict_mode = conflict_mode
test.write_config()
test.write_pre('A/file.txt', '0')
test.setup()
test.write_post('A/file.txt', 'A')
test.write_post('B/file.txt', 'Bb', add_dt=20) # newer and larger
print('-=' * 40)
print('=-' * 40)
test.sync()
stdout = ''.join(test.synclogs[-1])
diffs = test.compare_tree()
assert diffs == set()
files = [os.path.relpath(f, 'A/') for f in testutils.tree('A/')]
A = exists('A/file.txt') and test.read('A/file.txt') == 'A'
B = exists('A/file.txt') and test.read('A/file.txt') == 'Bb'
tA = any(file.endswith('.A') for file in files)
tB = any(file.endswith('.B') for file in files)
if conflict_mode in ['A', 'older', 'smaller']:
assert (A, B, tA, tB) == (True, False, False,
False), f"{conflict_mode} {(A,B,tA,tB)}"
elif conflict_mode in ['B', 'newer', 'larger']:
assert (A, B, tA, tB) == (False, True, False,
False), f"{conflict_mode} {(A,B,tA,tB)}"
elif conflict_mode in ['newer_tag']:
assert (A, B, tA, tB) == (False, True, True,
False), f"{conflict_mode} {(A,B,tA,tB)}"
elif conflict_mode in ['tag']:
assert (A, B, tA, tB) == (False, False, True,
True), f"{conflict_mode} {(A,B,tA,tB)}"
else:
raise ValueError('Not studied') # Should not be here
os.chdir(PWD0)
def test_conflict_resolution(conflict_mode,tag_conflict):
remoteA = 'A'
remoteB = 'B'
set_debug(False)
test = testutils.Tester('conflicts',remoteA,remoteB)
## Config
test.config.conflict_mode = conflict_mode
test.config.tag_conflict = tag_conflict
test.write_config()
test.write_pre('A/file.txt','0')
test.setup()
test.write_post('A/file.txt','A')
test.write_post('B/file.txt','Bb',add_dt=20) # newer and larger
print('-='*40);print('=-'*40)
test.sync(['--debug'])
stdout = ''.join(test.synclogs[-1])
diffs = test.compare_tree()
assert diffs == set()
files = [os.path.relpath(f,'A/') for f in testutils.tree('A/')]
A = exists('A/file.txt') and test.read('A/file.txt') == 'A'
B = exists('A/file.txt') and test.read('A/file.txt') == 'Bb'
tA = any(file.endswith('.A.txt') for file in files)
tB = any(file.endswith('.B.txt') for file in files)
if conflict_mode in ['A','older','smaller']:
res = (A,B) == (True,False)
tag = (tA,tB) == (False,True) if tag_conflict else (False,False)
elif conflict_mode in ['B','newer','larger']:
res = (A,B) == False,True
tag = (tA,tB) == (True,False) if tag_conflict else (False,False)
elif conflict_mode in ['tag']:
res = (A,B) == (False,False)
tag = (tA,tB) == (True,True)
else:
raise ValueError('Not studied') # Should not be here
assert res, f'Wrong res {A,B}, mode {conflict_mode,tag_conflict}'
assert tag, f'Wrong tag {tA,tB} mode {conflict_mode,tag_conflict}'
os.chdir(PWD0)
def test_no_hashes():
remoteA = 'A'
remoteB = 'cryptB:' # Crypt does not have hashes
set_debug(False)
print(remoteA,remoteB)
test = testutils.Tester('nocommon',remoteA,remoteB)
## Config
test.config.compare = 'hash'
test.config.hash_fail_fallback = 'mtime'
test.write_config()
# Setup
test.write_pre('A/file00.txt','0')
test.setup() # This has a sync that will throw the warnings
stdout = ''.join(test.synclogs[-1])
# This will have to change if I change the verbage
assert "WARNING No common hashes found and/or one or both remotes do not provide hashes. Falling back to 'mtime'" in stdout
os.chdir(PWD0)
def test_three_way():
"""
Test three way. In order to make this work within my own testing framework,
I have to switch the configs manually as opposed to having multiple
configurations. It isn't ideal but works for testing
"""
set_debug(False)
test = testutils.Tester('three','A','B')
# Just use simple comparisons
test.config.renamesA = test.config.renamesB = 'hash'
test.config.name = 'AB'
test.write_config()
test.write_pre('A/file1.txt','file1')
test.write_pre('A/file2.txt','file1')
## Run
test.setup()
test.write_post('A/file1.txt','mod',mode='at')
test.write_post('B/file3.txt','file3')
test.sync()
# Modify it to sync A <--> C
test.config.remoteB = 'C'
test.config.name = 'AC'
test.write_config()
test.write_pre('C/fileC.txt','this is on C')
# This *just* makes sure that we don't have a false positive and we
# are hacking it to compare C
assert {('missing_inA', 'fileC.txt'),
('missing_inB', 'file1.txt'),
('missing_inB', 'file2.txt'),
('missing_inB', 'file3.txt')} == test.compare_tree(A='A',B='C')
test.sync()
assert test.compare_tree(A='A',B='C') == set()
assert test.compare_tree(A='A',B='B') == {('missing_inB', 'fileC.txt')} # Should still miss that
test.config.remoteB = 'B'
test.config.name = 'AB'
test.write_config()
test.sync()
assert test.compare_tree(A='A',B='C') == set()
assert test.compare_tree(A='A',B='B') == set()
# Change it again but this time with B <--> C
test.config.remoteB = 'C'
test.config.remoteA = 'B'
test.config.name = 'BC'
test.write_config()
test.sync() # Shouldn't do anything
test.write_post('B/file3.txt','file3 modified')
test.sync()
assert test.compare_tree(A='B',B='C') == set()
assert test.read('C/file3.txt') == 'file3 modified'
assert test.compare_tree(A='A',B='C') == {('disagree', 'file3.txt')}
os.chdir(PWD0)
def test_main(remoteA,renamesA,remoteB,renamesB,compare,interactive=False):
"""
Main test with default settings (if the defaults change, this will need to
be updated. A few minor changes from the defaults are also made
More edge cases and specific settings are played with later
"""
set_debug(False)
print(remoteA,remoteB)
test = testutils.Tester('main',remoteA,remoteB)
## Config
test.config.reuse_hashesA = False # Also shouldn't compute them
test.config.renamesA = renamesA
test.config.reuse_hashesB = True
test.config.renamesB = renamesB
test.config.rclone_flags = ['--fast-list'] # Will be ignored if not supported but good to have otherwise
test.config.filter_flags = ['--filter','+ /yes/**',
'--filter','- *.no']
test.config.compare = compare
test.config.conflict_mode = 'newer_tag' # Deprecated. Update in the future
test.config.log_dest = 'logs/'
test.write_config()
## Initial files
test.write_pre('A/leave_alone.txt','do not touch')
# We use newer_tag so we can confirm that these are *NOT* considered
# conflicts. They should be backed up *and*
test.write_pre('A/EditOnA.txt','Edit on A')
test.write_pre('A/EditOnB.txt','Edit on B')
# Give it extra so size is unique
test.write_pre('A/MoveOnA.txt','Move on A' + randstr(52))
test.write_pre('A/MoveOnB.txt','Move on B' + randstr(100))
test.write_pre('A/MoveOnAB.txt','Move on Both' + randstr(74))
test.write_pre('A/MoveEditOnA.txt','Move and Edit on A')
test.write_pre('A/MoveEditOnB.txt','Move and Edit on B')
test.write_pre('A/EditOnBoth_Anewer.txt','A will be newer')
test.write_pre('A/EditOnBoth_Bnewer.txt','B will be newer')
test.write_pre('A/MoveEditOnBoth_Bnewer.txt','Will move and edit on both sides' + randstr(10))
test.write_pre('A/delA.txt','delete on A')
test.write_pre('A/delB.txt','delete on B')
test.write_pre('A/delA modB.txt','delA but mod on B')
test.write_pre('A/delB modA.txt','delB but mod on A')
test.write_pre('A/unic°de and space$.txt','UTF8')
test.write_pre('A/common_contentAfter0.txt','abc xyz')
test.write_pre('A/common_contentAfter1.txt','abc xy')
test.write_pre('A/common_contentBefore0.txt','ABC XYZ')
test.write_pre('A/common_contentBefore1.txt','ABC XYZ')
## Run
test.setup()
## Modify
test.write_post('A/EditOnA.txt','Edited on A',mode='at')
test.write_post('B/EditOnB.txt','Edited on B',mode='at')
test.move('A/MoveOnA.txt','A/sub/MovedOnA.txt') # move and rename
test.move('B/MoveOnB.txt','B/sub2/MoveOnB.txt') # just move
test.move('A/MoveOnAB.txt','A/MovedOnAB.txt')
test.move('B/MoveOnAB.txt','B/MovedOnAB.txt')
test.move('A/MoveEditOnA.txt','A/MovedEditOnA.txt')
test.move('B/MoveEditOnB.txt','B/MovedEditOnB.txt')
test.write_post('A/MovedEditOnA.txt','Move and Edit on A',mode='at')
test.write_post('B/MovedEditOnB.txt','Move and Edit on B',mode='at')
# recall when comparing by size, When comparing by size, older --> smaller, newer --> larger
test.write_post('A/EditOnBoth_Anewer.txt','AAAa',mode='at',add_dt=50) # larger too
test.write_post('A/EditOnBoth_Bnewer.txt','AAA',mode='at',add_dt=0)
test.write_post('B/EditOnBoth_Anewer.txt','BBB',mode='at',add_dt=0)
test.write_post('B/EditOnBoth_Bnewer.txt','BBBb',mode='at',add_dt=50) # larger too
test.move('A/MoveEditOnBoth_Bnewer.txt','A/MovedEditedOnBoth_Bnewer.txt')
test.move('B/MoveEditOnBoth_Bnewer.txt','B/MovedEditedOnBoth_Bnewer.txt')
test.write_post('A/MovedEditedOnBoth_Bnewer.txt','A', mode='at')
test.write_post('B/MovedEditedOnBoth_Bnewer.txt','BB', mode='at',add_dt=50) # larger too
os.remove('A/delA.txt')
os.remove('B/delB.txt')
test.write_post('A/delB modA.txt','mod on A',mode='at')
os.remove('B/delB modA.txt')
test.write_post('B/delA modB.txt','mod on B',mode='at')
os.remove('A/delA modB.txt')
test.write_post('A/newA.txt','New on A')
test.write_post('B/newB.txt','New on B')
test.write_post('A/newA.no','New on A and no') # use new to test exclusions too
test.write_post('B/newB.no','New on B and no')
test.write_post('A/yes/newA.yes.no','New on A and no but yes')
test.write_post('B/yes/newB.yes.no','New on B and no but yes')
test.write_post('B/unic°de and space$.txt','works',mode='at')
# These don't need to be tested other than not showing a diff
test.write_post('A/common_contentAfter1.txt','abc xyz')
test.write_post('B/common_contentBefore1.txt','ABC XYZW')
print('-='*40)
print('=-'*40)
args = ['--interactive'] if interactive else []
obj = test.sync(args)
## Confirm!
print('-'*100)
diffs = test.compare_tree()
# Exclusions except when filters to allow!
assert {('missing_inA', 'newB.no'), ('missing_inB', 'newA.no')} == diffs
stdout = ''.join(test.synclogs[-1])
# Check on A from now on!
# Edits -- Should *NOT* tag but *should* backup
assert test.read('A/EditOnA.txt') == 'Edit on AEdited on A',"mod did not propogate"
assert test.read('A/EditOnB.txt') == 'Edit on BEdited on B',"mod did not propogate"
assert not exists('A/EditOnA.txt.*'),'Should NOT have been tagged'
assert not exists('A/EditOnB.txt.*'),'Should NOT have been tagged'
assert test.globread('B/.syncrclone/backups/B_*/EditOnA.txt') == 'Edit on A','not backed up'
assert test.globread('A/.syncrclone/backups/A_*/EditOnB.txt') == 'Edit on B','not backed up'
# Moves w/o edit
assert not exists('A/MoveOnB.txt')
assert exists('A/sub2/MoveOnB.txt')
assert "Move on A: 'MoveOnB.txt' --> 'sub2/MoveOnB.txt'" in stdout
assert not exists('A/MoveOnA.txt')
assert exists('A/sub/MovedOnA.txt')
assert "Move on B: 'MoveOnA.txt' --> 'sub/MovedOnA.txt'" in stdout
assert not exists('A/MoveOnAB.txt')
assert exists('A/MovedOnAB.txt')
assert not "move A: 'MoveOnAB.txt' --> 'MovedOnAB.txt'" in stdout
assert not "move B: 'MoveOnAB.txt' --> 'MovedOnAB.txt'" in stdout
# moves with edit -- should not have been tracked
assert not exists('A/MoveEditOnA.txt')
assert exists('A/MovedEditOnA.txt')
assert 'MovedEditOnA.txt: Copied' in stdout,'Not copied. May be rclone log change'
assert not exists('A/MoveEditOnB.txt')
assert exists('A/MovedEditOnB.txt')
assert 'MovedEditOnB.txt: Copied' in stdout,'Not copied. May be rclone log change'
assert test.read('A/EditOnBoth_Anewer.txt') == 'A will be newerAAAa'
assert test.read('A/EditOnBoth_Bnewer.txt') == 'B will be newerBBBb'
assert exists('A/EditOnBoth_Anewer.*.B.txt'), "Not tagged"
assert exists('A/EditOnBoth_Bnewer.*.A.txt'), "Not tagged"
assert test.read('B/MovedEditedOnBoth_Bnewer.txt').endswith('BB')
assert not exists('A/delA.txt')
assert not exists('A/delB.txt')
assert exists('A/.syncrclone/backups/A_*/delB.txt'), "did not backup"
assert exists('B/.syncrclone/backups/B_*/delA.txt'), "did not backup"
assert exists('A/delB modA.txt') # Should not have been deleted
assert "DELETE CONFLICT: File 'delB modA.txt' deleted on B but modified on A. Transfering" in stdout
assert exists('A/delA modB.txt')
assert "DELETE CONFLICT: File 'delA modB.txt' deleted on A but modified on B. Transfering" in stdout
assert exists('A/newA.txt')
assert exists('A/newB.txt')
assert exists('A/yes/newA.yes.no')
assert exists('A/yes/newB.yes.no')
assert test.read('A/unic°de and space$.txt') == 'UTF8works'
assert exists('A/.syncrclone/backups/A_*/uni*.txt'),'did not back up'
os.chdir(PWD0)
def test_move_attribs(attrib):
"""
Test moves over various conditions for each type of move tracking
including modified sizes.
7 files: 1,2,3,C,D,4,5 all have different mtimes and are moved
- 1,2 are unique content and size
- 3,C are unique content but same size
- D,4 are the same content and size
- 5 is unique content and size but it's modified
See Truth Table in the code
Only test with local A (as it supports inode). B doesn't matter since
moves are only tracked on A.
"""
remoteA = 'A'
remoteB = 'B'
print(attrib)
set_debug(True)
print(remoteA,remoteB)
test = testutils.Tester('renames',remoteA,remoteB)
## Config
test.config.reuse_hashesA = False
test.config.renamesA = attrib
if attrib == 'size': # Presumably we do not have mtime
test.config.compare = 'size'
test.config.dt = 0.001
test.write_config()
# Setup
test.write_pre('A/file1.txt','1')
test.write_pre('A/file2.txt','12')
test.write_pre('A/file3.txt','123')
test.write_pre('A/fileC.txt','ABC')
test.write_pre('A/fileD.txt','ABCD')
test.write_pre('A/file4.txt','ABCD')
test.write_pre('A/file5.txt','12345')
test.setup()
for c in '123CD45':
shutil.move(f'A/file{c}.txt',f'A/file{c}_moved.txt')
test.write_post('A/file5_moved.txt','12345') # Changes mtime but same content
print('-='*40)
print('=-'*40)
test.sync()
stdout = ''.join(test.synclogs[-1])
## Truth Table
if not attrib:
notmoved = '123CD45'
moved = too_many = ''
elif attrib == 'size':
# Size alone won't care about the mod. Note that compare is 'size' for
# this one too since that is likely all you would have
moved = '125'
too_many = '3CD4'
notmoved = ''
elif attrib == 'mtime':
moved = '1234CD'
too_many = ''
notmoved = '5'
elif attrib == 'inode':
moved = '123CD4'
too_many = ''
notmoved = '5'
elif attrib == 'hash':
moved = '123C5'
too_many = 'D4'
notmoved = ''
for c in moved:
assert f"Move on B: 'file{c}.txt' --> 'file{c}_moved.txt'" in stdout,f"{attrib} file{c} didn't move"
for c in too_many:
assert f"Too many possible previous files for 'file{c}_moved.txt' on A" in stdout, f"{attrib} file{c} failed multiple"
for c in notmoved:
assert f"Move on B: 'file{c}.txt' --> 'file{c}_moved.txt'" not in stdout,f"{attrib} file{c} moved"
os.chdir(PWD0)
