def test_syncfile_01( testdir ):
"""
Attempt to sync a source file that doesn't exist
Should throw an error
source NO
"""
testdir.reset()
syncopts = syncopts_defaults.copy()
syncopts[ 'tmpbase' ] = os.path.abspath( testdir.psconfig.TMP_DIR )
f = testdir.objects.values()[0][0]
src = fsitem.FSItem( '{0}xyz'.format( f.path ) )
tgt = fsitem.FSItem( src.absname.replace( testdir.source, testdir.target ) )
with pytest.raises( pylut.SyncError ) as einfo:
pylut.syncfile( src_path=src, tgt_path=tgt, **syncopts )
assert 'No such file or directory' in einfo.value.reason
def test_syncfile_01(testdir):
"""
Attempt to sync a source file that doesn't exist
Should throw an error
source NO
"""
testdir.reset()
syncopts = syncopts_defaults.copy()
syncopts['tmpbase'] = os.path.abspath(testdir.psconfig.TMP_DIR)
f = testdir.objects.values()[0][0]
src = fsitem.FSItem('{0}xyz'.format(f.path))
tgt = fsitem.FSItem(src.absname.replace(testdir.source, testdir.target))
with pytest.raises(pylut.SyncError) as einfo:
pylut.syncfile(src_path=src, tgt_path=tgt, **syncopts)
assert 'No such file or directory' in einfo.value.reason
def test_syncfile_07(testdir):
"""
Testing existing tmp mismatch, keep tmp
tmp MISMATCH
tgt NO
Expect tmp to get unlinked, then test is same as test_syncfile_02
Verify tmp has new FID
test-pair = 11
"""
testdir.reset()
testdir.mk_all_tgtdirs()
syncopts = syncopts_defaults.copy()
syncopts.update(tmpbase=os.path.abspath(testdir.psconfig.TMP_DIR))
for f in testdir.files:
src = fsitem.FSItem(f.path)
tgt = fsitem.FSItem(src.absname.replace(testdir.source,
testdir.target))
# initial sync to create tmp
syncopts.update(keeptmp=True)
tmp, action = pylut.syncfile(src_path=src, tgt_path=tgt, **syncopts)
# delete tgt
os.unlink(str(tgt))
assert os.path.lexists(str(tgt)) == False
tgt.update()
# save current tmp FID
tmp_fid_orig = tmp.inode()
# change src file
if src.is_regular():
# sleep is long, faster to just change the file data
change = random.randint(1, 1024)
_mkregfile(src, size=src.size + change)
else:
# no choice but sleep for non-regular files
time.sleep(1)
_touch(src)
src.update()
# sync again, expect a new tmp file
tmp, action = pylut.syncfile(src_path=src, tgt_path=tgt, **syncopts)
# expect tmp is new, so ensure metadata is up to date
tmp.update()
# verify src and tgt are in sync
assert _files_match(src, tgt, syncopts)
# verify tmp and target are the same file
assert _files_equal(tgt, tmp)
# verify tmp has different FID
assert tmp_fid_orig != tmp.inode()
def test_syncfile_07( testdir ):
"""
Testing existing tmp mismatch, keep tmp
tmp MISMATCH
tgt NO
Expect tmp to get unlinked, then test is same as test_syncfile_02
Verify tmp has new FID
test-pair = 11
"""
testdir.reset()
testdir.mk_all_tgtdirs()
syncopts = syncopts_defaults.copy()
syncopts.update( tmpbase=os.path.abspath( testdir.psconfig.TMP_DIR ) )
for f in testdir.files:
src = fsitem.FSItem( f.path )
tgt = fsitem.FSItem( src.absname.replace( testdir.source, testdir.target ) )
# initial sync to create tmp
syncopts.update( keeptmp=True )
tmp, action = pylut.syncfile( src_path=src, tgt_path=tgt, **syncopts )
# delete tgt
os.unlink( str( tgt ) )
assert os.path.lexists( str( tgt ) ) == False
tgt.update()
# save current tmp FID
tmp_fid_orig = tmp.inode()
# change src file
if src.is_regular():
# sleep is long, faster to just change the file data
change = random.randint( 1, 1024 )
_mkregfile( src, size=src.size + change )
else:
# no choice but sleep for non-regular files
time.sleep( 1 )
_touch( src )
src.update()
# sync again, expect a new tmp file
tmp, action = pylut.syncfile( src_path=src, tgt_path=tgt, **syncopts )
# expect tmp is new, so ensure metadata is up to date
tmp.update()
# verify src and tgt are in sync
assert _files_match( src, tgt, syncopts )
# verify tmp and target are the same file
assert _files_equal( tgt, tmp )
# verify tmp has different FID
assert tmp_fid_orig != tmp.inode()
def test_syncfile_02( testdir ):
"""
Initial sync, keep tmp
tmp NO
target NO
keeptmp YES
test-pair = 03
"""
testdir.reset()
testdir.mk_all_tgtdirs()
syncopts = syncopts_defaults.copy()
syncopts.update( keeptmp=True,
tmpbase=os.path.abspath( testdir.psconfig.TMP_DIR )
)
for f in testdir.files:
src = fsitem.FSItem( f.path )
tgt = fsitem.FSItem( src.absname.replace( testdir.source, testdir.target ) )
pylut.syncfile( src_path=src, tgt_path=tgt, **syncopts )
assert _files_match( src, tgt, syncopts )
def test_syncfile_02(testdir):
"""
Initial sync, keep tmp
tmp NO
target NO
keeptmp YES
test-pair = 03
"""
testdir.reset()
testdir.mk_all_tgtdirs()
syncopts = syncopts_defaults.copy()
syncopts.update(keeptmp=True,
tmpbase=os.path.abspath(testdir.psconfig.TMP_DIR))
for f in testdir.files:
src = fsitem.FSItem(f.path)
tgt = fsitem.FSItem(src.absname.replace(testdir.source,
testdir.target))
pylut.syncfile(src_path=src, tgt_path=tgt, **syncopts)
assert _files_match(src, tgt, syncopts)
def test_syncfile_06(testdir):
"""
Testing existing tmp ok, keep tmp
tmp OK
tgt NO
expect tgt file hardlink to be created
tmp file should remain untouched
test-pair = 10
"""
testdir.reset()
testdir.mk_all_tgtdirs()
syncopts = syncopts_defaults.copy()
syncopts.update(tmpbase=os.path.abspath(testdir.psconfig.TMP_DIR))
for f in testdir.files:
src = fsitem.FSItem(f.path)
tgt = fsitem.FSItem(src.absname.replace(testdir.source,
testdir.target))
# initial sync to create tmp
syncopts.update(keeptmp=True)
tmp, action = pylut.syncfile(src_path=src, tgt_path=tgt, **syncopts)
# delete tgt
os.unlink(str(tgt))
assert os.path.lexists(str(tgt)) == False
tgt.update()
# save original tmp FID
tmp_fid_orig = tmp.inode()
tmp.update()
# sync again, should be fast since valid tmp already exists
# starttime = time.time()
tmp, action = pylut.syncfile(src_path=src, tgt_path=tgt, **syncopts)
# endtime = time.time()
# verify src and tgt are in sync
assert _files_match(src, tgt, syncopts)
# verify tmp and target are the same file
assert _files_equal(tgt, tmp)
# # check that elapsed time was <1 second
# elapsedtime = endtime - starttime
# assert elapsedtime < 1
# verify tmp has same FID as before
assert tmp_fid_orig == tmp.inode()
def test_syncfile_06( testdir ):
"""
Testing existing tmp ok, keep tmp
tmp OK
tgt NO
expect tgt file hardlink to be created
tmp file should remain untouched
test-pair = 10
"""
testdir.reset()
testdir.mk_all_tgtdirs()
syncopts = syncopts_defaults.copy()
syncopts.update( tmpbase=os.path.abspath( testdir.psconfig.TMP_DIR ) )
for f in testdir.files:
src = fsitem.FSItem( f.path )
tgt = fsitem.FSItem( src.absname.replace( testdir.source, testdir.target ) )
# initial sync to create tmp
syncopts.update( keeptmp=True )
tmp, action = pylut.syncfile( src_path=src, tgt_path=tgt, **syncopts )
# delete tgt
os.unlink( str( tgt ) )
assert os.path.lexists( str( tgt ) ) == False
tgt.update()
# save original tmp FID
tmp_fid_orig = tmp.inode()
tmp.update()
# sync again, should be fast since valid tmp already exists
# starttime = time.time()
tmp, action = pylut.syncfile( src_path=src, tgt_path=tgt, **syncopts )
# endtime = time.time()
# verify src and tgt are in sync
assert _files_match( src, tgt, syncopts )
# verify tmp and target are the same file
assert _files_equal( tgt, tmp )
# # check that elapsed time was <1 second
# elapsedtime = endtime - starttime
# assert elapsedtime < 1
# verify tmp has same FID as before
assert tmp_fid_orig == tmp.inode()
def test_syncfile_04(testdir):
"""
Testing existing target ok, keep tmp
tmp NO
tgt OK
expect tmp file hardlink to be created, tgt file should remain untouched
test-pair = 08
"""
testdir.reset()
testdir.mk_all_tgtdirs()
syncopts = syncopts_defaults.copy()
syncopts.update(tmpbase=os.path.abspath(testdir.psconfig.TMP_DIR))
for f in testdir.files:
src = fsitem.FSItem(f.path)
tgt = fsitem.FSItem(src.absname.replace(testdir.source,
testdir.target))
# make initial sync so tgt exists, don't keep tmpfile
syncopts.update(keeptmp=False)
tmp, action = pylut.syncfile(src_path=src, tgt_path=tgt, **syncopts)
# save original tgt FID
tgt_fid_orig = tgt.inode()
tgt.update()
# verify tmp does not exist
assert os.path.lexists(str(tmp)) == False
# sync again, keep tmpfile this time
syncopts.update(keeptmp=True)
# starttime = time.time()
tmp, action = pylut.syncfile(src_path=src, tgt_path=tgt, **syncopts)
# endtime = time.time()
# # check that elapsed time was <1 second
# elapsedtime = endtime - starttime
# assert elapsedtime < 1
# verify tmp and target are the same file
assert _files_equal(tgt, tmp)
# verify tgt has same FID as before
assert tgt_fid_orig == tgt.inode()
# verify src and tgt are in sync
assert _files_match(src, tgt, syncopts)
def test_syncfile_04( testdir ):
"""
Testing existing target ok, keep tmp
tmp NO
tgt OK
expect tmp file hardlink to be created, tgt file should remain untouched
test-pair = 08
"""
testdir.reset()
testdir.mk_all_tgtdirs()
syncopts = syncopts_defaults.copy()
syncopts.update( tmpbase=os.path.abspath( testdir.psconfig.TMP_DIR ) )
for f in testdir.files:
src = fsitem.FSItem( f.path )
tgt = fsitem.FSItem( src.absname.replace( testdir.source, testdir.target ) )
# make initial sync so tgt exists, don't keep tmpfile
syncopts.update( keeptmp=False )
tmp, action = pylut.syncfile( src_path=src, tgt_path=tgt, **syncopts )
# save original tgt FID
tgt_fid_orig = tgt.inode()
tgt.update()
# verify tmp does not exist
assert os.path.lexists( str( tmp ) ) == False
# sync again, keep tmpfile this time
syncopts.update( keeptmp=True )
# starttime = time.time()
tmp, action = pylut.syncfile( src_path=src, tgt_path=tgt, **syncopts )
# endtime = time.time()
# # check that elapsed time was <1 second
# elapsedtime = endtime - starttime
# assert elapsedtime < 1
# verify tmp and target are the same file
assert _files_equal( tgt, tmp )
# verify tgt has same FID as before
assert tgt_fid_orig == tgt.inode()
# verify src and tgt are in sync
assert _files_match( src, tgt, syncopts )
def _sync_a_file( src, tgt, rsyncopts, ftype ):
"""
Common code for syncing any file. This function will be called by one of
the sync_file or sync_hardlink Celery Task functions.
:param src FSItem: src file
:param tgt FSItem: tgt file
:param rsyncopts dict: options passed to pylut.syncdir & pylut.syncfile
:param ftype string: file type that is being sync'd (ie: file or hardlink)
:return: None
"""
rsyncopts.update( tmpbase = os.path.join( tgt.mountpoint, psynctmpdir ),
keeptmp = True,
)
synctype = 'SYNC' + ftype.upper()
logr.info( synctype = synctype,
msgtype = 'start',
src = str( src ),
tgt = str( tgt ),
size = src.size )
try:
tmpfn, action_type = pylut.syncfile( src, tgt, **rsyncopts )
except ( pylut.PylutError ) as e:
logr.warning( synctype = synctype,
msgtype = 'error',
src = str( src ),
tgt = str( tgt ),
error = str( e ) )
return
msg_parts = {}
if tmpfn.nlink < 3 and rsyncopts[ 'pre_checksums' ]:
msg_parts.update( src_chksum = src.checksum(),
tgt_chksum = tgt.checksum() )
sync_action = 'None'
if action_type[ 'data_copy' ]:
sync_action = 'data_copy'
if rsyncopts[ 'post_checksums' ] and not rsyncopts[ 'pre_checksums' ]:
# do post checksums only if pre_checksums haven't done it already
msg_parts.update( src_chksum = src.checksum(),
tgt_chksum = tgt.checksum() )
elif action_type[ 'meta_update' ]:
sync_action = 'meta_update'
msg_parts.update( synctype = synctype,
msgtype = 'end',
src = str( src ),
tgt = str( tgt ),
action = str( sync_action ) )
logr.info( **msg_parts )
import pylut import fsitem src_path=fsitem.FSItem( '/u/staff/aloftus/lustre_version.pbs' ) tgt_path=fsitem.FSItem( '/projects/test/psynctest/lustre_version.pbs' ) pylut.syncfile( src_path, tgt_path )