def _compare_checksums(self, src_path):
# type: (str) -> bool
'''compare checksum of src_path and dest: self.name
Return True if the same'''
try:
f1 = open(src_path, 'rb')
except IOError as err:
error('failed to open %s : %s' % (src_path, err.strerror))
# return True because we can't fix an error in src_path
return True
sum1 = hashlib.md5()
sum2 = hashlib.md5()
with f1:
try:
f2 = open(self.name, 'rb')
except IOError as err:
error('failed to open %s : %s' % (self.name, err.strerror))
return False
with f2:
ended = False
while not ended and (sum1.digest() == sum2.digest()):
try:
data1 = f1.read(IO_SIZE)
except IOError as err:
error('failed to read file %s: %s' % (src_path,
err.strerror))
return False
if not data1:
ended = True
else:
sum1.update(data1)
try:
data2 = f2.read(IO_SIZE)
except IOError as err:
error('failed to read file %s: %s' % (self.name,
err.strerror))
return False
if not data2:
ended = True
else:
sum2.update(data2)
if sum1.digest() != sum2.digest():
if synctool.lib.DRY_RUN:
stdout('%s mismatch (MD5 checksum)' % self.name)
else:
stdout('%s updated (MD5 mismatch)' % self.name)
unix_out('# updating file %s' % self.name)
terse(synctool.lib.TERSE_SYNC, self.name)
return False
return True
def compare(self, _src_path, dest_stat):
# type: (str, SyncStat) -> bool
'''see if devs are the same'''
if not self.exists:
return False
# dest_stat is a SyncStat object and it's useless here
# I need a real, fresh statbuf that includes st_rdev field
try:
dest_stat = os.lstat(self.name)
except OSError as err:
error('error checking %s : %s' % (self.name, err.strerror))
return False
# Note: mypy triggers false errors here
# Also, no luck with Union[SyncStat, posix.stat_result]
# In any case, for VNodeChrDev and VNodeBlkDev,
# the self.src_stat is of type posix.stat_result
src_major = os.major(self.src_stat.st_rdev) # type: ignore
src_minor = os.minor(self.src_stat.st_rdev) # type: ignore
dest_major = os.major(dest_stat.st_rdev) # type: ignore
dest_minor = os.minor(dest_stat.st_rdev) # type: ignore
if src_major != dest_major or src_minor != dest_minor:
stdout('%s should have major,minor %d,%d but has %d,%d' %
(self.name, src_major, src_minor, dest_major, dest_minor))
unix_out('# updating major,minor %s' % self.name)
terse(synctool.lib.TERSE_SYNC, self.name)
return False
return True
def check_purge_timestamp(self):
# type: () -> bool
'''check timestamp between src and dest
Returns True if same, False if not
'''
# This is only used for purge/
# check() has already determined that the files are the same
# Now only check the timestamp ...
if synctool.param.SYNC_TIMES:
# this was already handled by check() and fix()
return True
# set times, but not for symlinks, directories
if (not self.src_stat.is_link() and not self.src_stat.is_dir() and
self.src_stat.mtime != self.dest_stat.mtime):
stdout('%s mismatch (only timestamp)' % self.dest_path)
terse(synctool.lib.TERSE_WARNING,
'%s (only timestamp)' % self.dest_path)
vnode = self.vnode_obj()
# leave the atime intact
vnode.stat.atime = self.dest_stat.atime
vnode.set_times()
return False
return True
def _single_purge_callback(obj, pre_dict, post_dict):
# type: (SyncObject, Dict[str, str], Dict[str, str]) -> Tuple[bool, bool]
'''do purge function for single files'''
# The same as _single_overlay_callback(), except that
# purge entries may differ in timestamp. synctool has to report
# this because pure rsync will as well (which is bloody annoying)
#
# For normal synctool overlay/, it's regarded as not important
# and synctool will not complain about it
#
# This actually leaves a final wart; synctool --single may create
# purge entries that rsync will complain about and sync again
# Anyway, I don't think it's a big deal, and that's what you get
# when you mix up synctool and rsync
go_on = True
updated = False
if _match_single(obj.dest_path):
_, updated = _overlay_callback(obj, pre_dict, post_dict)
if not updated:
if obj.check_purge_timestamp():
stdout('%s is up to date' % obj.dest_path)
terse(synctool.lib.TERSE_OK, obj.dest_path)
unix_out('# %s is up to date\n' % obj.dest_path)
# else: pass
if not SINGLE_FILES:
return False, updated
return go_on, updated
def _single_overlay_callback(obj, post_dict, updated, *args):
'''do overlay function for single files'''
if obj.ov_type == synctool.overlay.OV_TEMPLATE:
return generate_template(obj, post_dict), False
go_on = True
if _match_single(obj.dest_path):
_, updated = _overlay_callback(obj, post_dict, False, *args)
if not updated:
stdout('%s is up to date' % obj.dest_path)
terse(synctool.lib.TERSE_OK, obj.dest_path)
unix_out('# %s is up to date\n' % obj.dest_path)
else:
# register .post on the parent dir, if it has a .post script
obj.dest_path = os.path.dirname(obj.dest_path)
obj.dest_stat = synctool.syncstat.SyncStat(obj.dest_path)
if obj.dest_path in post_dict:
changed_dict = args[0]
changed_dict[obj.dest_path] = (obj, post_dict[obj.dest_path])
if not SINGLE_FILES:
return False, updated
return go_on, updated
def _split_extension(filename, src_dir):
'''filename in the overlay tree, without leading path
src_dir is passed for the purpose of printing error messages
Returns tuple: SyncObject, importance'''
(name, ext) = os.path.splitext(filename)
if not ext:
return SyncObject(filename, name, OV_NO_EXT), _group_all()
if ext == '.post':
(name2, ext) = os.path.splitext(name)
if ext == '._template':
# it's a generic template generator
return SyncObject(filename, name, OV_TEMPLATE_POST), _group_all()
# it's a generic .post script
return SyncObject(filename, name, OV_POST), _group_all()
if ext[:2] != '._':
return SyncObject(filename, filename, OV_NO_EXT), _group_all()
ext = ext[2:]
if not ext:
return SyncObject(filename, filename, OV_NO_EXT), _group_all()
if ext == 'template':
return SyncObject(filename, name, OV_TEMPLATE), _group_all()
try:
importance = synctool.param.MY_GROUPS.index(ext)
except ValueError:
if not ext in synctool.param.ALL_GROUPS:
src_path = os.path.join(src_dir, filename)
if synctool.param.TERSE:
terse(synctool.lib.TERSE_ERROR, 'invalid group on %s' %
src_path)
else:
stderr('unknown group on %s, skipped' % prettypath(src_path))
return None, -1
# it is not one of my groups
verbose('skipping %s, it is not one of my groups' %
prettypath(os.path.join(src_dir, filename)))
return None, -1
(name2, ext) = os.path.splitext(name)
if ext == '.post':
_, ext = os.path.splitext(name2)
if ext == '._template':
# it's a group-specific template generator
return (SyncObject(filename, name2, OV_TEMPLATE_POST), importance)
# register group-specific .post script
return SyncObject(filename, name2, OV_POST), importance
elif ext == '._template':
return SyncObject(filename, name2, OV_TEMPLATE), importance
return SyncObject(filename, name), importance
def compare(self, _src_path, dest_stat):
# type: (str, SyncStat) -> bool
'''see if devs are the same'''
if not self.exists:
return False
# dest_stat is a SyncStat object and it's useless here
# I need a real, fresh statbuf that includes st_rdev field
try:
dest_stat = os.lstat(self.name)
except OSError as err:
error('error checking %s : %s' % (self.name, err.strerror))
return False
src_major = os.major(self.src_stat.st_rdev) # type: ignore
src_minor = os.minor(self.src_stat.st_rdev) # type: ignore
dest_major = os.major(dest_stat.st_rdev) # type: ignore
dest_minor = os.minor(dest_stat.st_rdev) # type: ignore
if src_major != dest_major or src_minor != dest_minor:
stdout('%s should have major,minor %d,%d but has %d,%d' %
(self.name, src_major, src_minor, dest_major, dest_minor))
unix_out('# updating major,minor %s' % self.name)
terse(synctool.lib.TERSE_SYNC, self.name)
return False
return True
def check_purge_timestamp(self):
'''check timestamp between src and dest
Returns True if same, False if not
'''
# This is only used for purge/
# check() has already determined that the files are the same
# Now only check the timestamp ...
if synctool.param.SYNC_TIMES:
# this was already handled by check() and fix()
return True
# set times, but not for symlinks, directories
if (not self.src_stat.is_link() and not self.src_stat.is_dir() and
self.src_stat.mtime != self.dest_stat.mtime):
stdout('%s mismatch (only timestamp)' % self.dest_path)
terse(synctool.lib.TERSE_WARNING,
'%s (only timestamp)' % self.dest_path)
vnode = self.vnode_obj()
# leave the atime intact
vnode.stat.atime = self.dest_stat.atime
vnode.set_times()
return False
return True
def compare(self, src_path, dest_stat):
'''see if devs are the same'''
if not self.exists:
return False
# dest_stat is a SyncStat object and it's useless here
# I need a real, fresh statbuf that includes st_rdev field
try:
dest_stat = os.lstat(self.name)
except OSError as err:
error('error checking %s : %s' % (self.name, err.strerror))
return False
src_major = os.major(self.src_stat.st_rdev)
src_minor = os.minor(self.src_stat.st_rdev)
dest_major = os.major(dest_stat.st_rdev)
dest_minor = os.minor(dest_stat.st_rdev)
if src_major != dest_major or src_minor != dest_minor:
stdout('%s should have major,minor %d,%d but has %d,%d' %
(self.name, src_major, src_minor, dest_major, dest_minor))
unix_out('# updating major,minor %s' % self.name)
terse(synctool.lib.TERSE_SYNC, self.name)
return False
return True
def check(self):
'''check differences between src and dest,
Return a FIX_xxx code
'''
# src_path is under $overlay/
# dest_path is in the filesystem
vnode = None
if not self.dest_stat.exists():
stdout('%s does not exist' % self.dest_path)
return SyncObject.FIX_CREATE
src_type = self.src_stat.filetype()
dest_type = self.dest_stat.filetype()
if src_type != dest_type:
# entry is of a different file type
vnode = self.vnode_obj()
stdout('%s should be a %s' % (self.dest_path, vnode.typename()))
terse(synctool.lib.TERSE_WARNING, 'wrong type %s' % self.dest_path)
return SyncObject.FIX_TYPE
vnode = self.vnode_obj()
if not vnode.compare(self.src_path, self.dest_stat):
# content is different; change the entire object
log('updating %s' % self.dest_path)
return SyncObject.FIX_UPDATE
# check ownership and permissions
# rectify if needed
fix_action = 0
if ((self.src_stat.uid != self.dest_stat.uid)
or (self.src_stat.gid != self.dest_stat.gid)):
stdout('%s should have owner %s.%s (%d.%d), '
'but has %s.%s (%d.%d)' %
(self.dest_path, self.src_stat.ascii_uid(),
self.src_stat.ascii_gid(), self.src_stat.uid,
self.src_stat.gid, self.dest_stat.ascii_uid(),
self.dest_stat.ascii_gid(), self.dest_stat.uid,
self.dest_stat.gid))
terse(
synctool.lib.TERSE_OWNER,
'%s.%s %s' % (self.src_stat.ascii_uid(),
self.src_stat.ascii_gid(), self.dest_path))
fix_action = SyncObject.FIX_OWNER
if self.src_stat.mode != self.dest_stat.mode:
stdout('%s should have mode %04o, but has %04o' %
(self.dest_path, self.src_stat.mode & 07777,
self.dest_stat.mode & 07777))
terse(synctool.lib.TERSE_MODE,
'%04o %s' % (self.src_stat.mode & 07777, self.dest_path))
fix_action |= SyncObject.FIX_MODE
return fix_action
def copy_stat(self):
'''set access and mod times'''
if not synctool.lib.DRY_RUN and synctool.param.SYNC_TIMES:
try:
verbose('copystat: %s => %s' % (self.src_path, self.name))
shutil.copystat(self.src_path, self.name)
except OSError as err:
error('failed to set utime on %s : %s' % (self.name,
err.strerror))
terse(synctool.lib.TERSE_FAIL, 'utime %s' % self.name)
def copy_stat(self):
'''set access and mod times'''
if not synctool.lib.DRY_RUN and synctool.param.SYNC_TIMES:
try:
verbose('copystat: %s => %s' % (self.src_path, self.name))
shutil.copystat(self.src_path, self.name)
except OSError as err:
error('failed to set utime on %s : %s' %
(self.name, err.strerror))
terse(synctool.lib.TERSE_FAIL, 'utime %s' % self.name)
def set_times(self, atime, mtime):
'''set access and mod times'''
# only used for purge --single
if not synctool.lib.DRY_RUN:
try:
os.utime(self.name, (atime, mtime))
except OSError as err:
error('failed to set utime on %s : %s' % (self.name,
err.strerror))
terse(synctool.lib.TERSE_FAIL, 'utime %s' % self.name)
def set_times(self, atime, mtime):
'''set access and mod times'''
# only used for purge --single
if not synctool.lib.DRY_RUN:
try:
os.utime(self.name, (atime, mtime))
except OSError as err:
error('failed to set utime on %s : %s' %
(self.name, err.strerror))
terse(synctool.lib.TERSE_FAIL, 'utime %s' % self.name)
def create(self):
'''make a fifo'''
verbose(dryrun_msg(' os.mkfifo(%s)' % self.name))
unix_out('mkfifo %s' % self.name)
terse(synctool.lib.TERSE_NEW, self.name)
if not synctool.lib.DRY_RUN:
try:
os.mkfifo(self.name, self.stat.mode & 0777)
except OSError as err:
error('failed to create fifo %s : %s' % (self.name,
err.strerror))
terse(TERSE_FAIL, 'fifo %s' % self.name)
def set_permissions(self):
'''set access permission bits equal to source'''
verbose(dryrun_msg(' os.chmod(%s, %04o)' %
(self.name, self.stat.mode & 07777)))
unix_out('chmod 0%o %s' % (self.stat.mode & 07777, self.name))
if not synctool.lib.DRY_RUN:
try:
os.chmod(self.name, self.stat.mode & 07777)
except OSError as err:
error('failed to chmod %04o %s : %s' %
(self.stat.mode & 07777, self.name, err.strerror))
terse(synctool.lib.TERSE_FAIL, 'mode %s' % self.name)
def upload(up):
# type: (UploadFile) -> None
'''copy a file from a node into the overlay/ tree'''
# Note: this global is only needed because of callback fn ...
global GLOBAL_UPLOAD_FILE
if up.filename[0] != os.sep:
error('the filename to upload must be an absolute path')
sys.exit(-1)
if up.suffix and up.suffix not in synctool.param.ALL_GROUPS:
error("no such group '%s'" % up.suffix)
sys.exit(-1)
if up.overlay and up.overlay not in synctool.param.ALL_GROUPS:
error("no such group '%s'" % up.overlay)
sys.exit(-1)
if up.purge and up.purge not in synctool.param.ALL_GROUPS:
error("no such group '%s'" % up.purge)
sys.exit(-1)
if synctool.lib.DRY_RUN and not synctool.lib.QUIET:
stdout('DRY RUN, not uploading any files')
terse(synctool.lib.TERSE_DRYRUN, 'not uploading any files')
if up.purge != None:
rsync_upload(up)
return
# pretend that the current node is now the given node;
# this is needed for find() to find the best reference for the file
orig_nodename = synctool.param.NODENAME
synctool.param.NODENAME = up.node
synctool.config.insert_group(up.node, up.node)
orig_my_groups = synctool.param.MY_GROUPS[:]
synctool.param.MY_GROUPS = synctool.config.get_my_groups()
# see if file is already in the repository
# Note: ugly global is needed because of callback function
GLOBAL_UPLOAD_FILE = up
synctool.overlay.visit(synctool.param.OVERLAY_DIR, _upload_callback)
up = GLOBAL_UPLOAD_FILE
synctool.param.NODENAME = orig_nodename
synctool.param.MY_GROUPS = orig_my_groups
rsync_upload(up)
def set_permissions(self):
# type: () -> None
'''set access permission bits equal to source'''
verbose(dryrun_msg(' os.chmod(%s, %04o)' %
(self.name, self.stat.mode & 07777)))
unix_out('chmod 0%o %s' % (self.stat.mode & 07777, self.name))
if not synctool.lib.DRY_RUN:
try:
os.chmod(self.name, self.stat.mode & 07777)
except OSError as err:
error('failed to chmod %04o %s : %s' %
(self.stat.mode & 07777, self.name, err.strerror))
terse(TERSE_FAIL, 'mode %s' % self.name)
def create(self):
'''create symbolic link'''
verbose(dryrun_msg(' os.symlink(%s, %s)' % (self.oldpath, self.name)))
unix_out('ln -s %s %s' % (self.oldpath, self.name))
terse(synctool.lib.TERSE_LINK, self.name)
if not synctool.lib.DRY_RUN:
try:
os.symlink(self.oldpath, self.name)
except OSError as err:
error('failed to create symlink %s -> %s : %s' %
(self.name, self.oldpath, err.strerror))
terse(synctool.lib.TERSE_FAIL, 'link %s' % self.name)
def compare(self, src_path, dest_stat):
'''see if files are the same
Return True if the same'''
if self.stat.size != dest_stat.size:
if synctool.lib.DRY_RUN:
stdout('%s mismatch (file size)' % self.name)
else:
stdout('%s updated (file size mismatch)' % self.name)
terse(synctool.lib.TERSE_SYNC, self.name)
unix_out('# updating file %s' % self.name)
return False
return self._compare_checksums(src_path)
def create(self):
# type: () -> None
'''make a fifo'''
verbose(dryrun_msg(' os.mkfifo(%s)' % self.name))
unix_out('mkfifo %s' % self.name)
terse(synctool.lib.TERSE_NEW, self.name)
if not synctool.lib.DRY_RUN:
try:
os.mkfifo(self.name, self.stat.mode & 0777)
except OSError as err:
error('failed to create fifo %s : %s' % (self.name,
err.strerror))
terse(TERSE_FAIL, 'fifo %s' % self.name)
def create(self):
'''create symbolic link'''
verbose(dryrun_msg(' os.symlink(%s, %s)' % (self.oldpath,
self.name)))
unix_out('ln -s %s %s' % (self.oldpath, self.name))
terse(synctool.lib.TERSE_LINK, self.name)
if not synctool.lib.DRY_RUN:
try:
os.symlink(self.oldpath, self.name)
except OSError as err:
error('failed to create symlink %s -> %s : %s' %
(self.name, self.oldpath, err.strerror))
terse(TERSE_FAIL, 'link %s' % self.name)
def upload(up):
'''copy a file from a node into the overlay/ tree'''
# Note: this global is only needed because of callback fn ...
global GLOBAL_UPLOAD_FILE
if up.filename[0] != os.sep:
error('the filename to upload must be an absolute path')
sys.exit(-1)
if up.suffix and up.suffix not in synctool.param.ALL_GROUPS:
error("no such group '%s'" % up.suffix)
sys.exit(-1)
if up.overlay and up.overlay not in synctool.param.ALL_GROUPS:
error("no such group '%s'" % up.overlay)
sys.exit(-1)
if up.purge and up.purge not in synctool.param.ALL_GROUPS:
error("no such group '%s'" % up.purge)
sys.exit(-1)
if synctool.lib.DRY_RUN and not synctool.lib.QUIET:
stdout('DRY RUN, not uploading any files')
terse(synctool.lib.TERSE_DRYRUN, 'not uploading any files')
if up.purge != None:
rsync_upload(up)
return
# pretend that the current node is now the given node;
# this is needed for find() to find the best reference for the file
orig_nodename = synctool.param.NODENAME
synctool.param.NODENAME = up.node
synctool.config.insert_group(up.node, up.node)
orig_my_groups = synctool.param.MY_GROUPS[:]
synctool.param.MY_GROUPS = synctool.config.get_my_groups()
# see if file is already in the repository
# Note: ugly global is needed because of callback function
GLOBAL_UPLOAD_FILE = up
synctool.overlay.visit(synctool.param.OVERLAY_DIR, _upload_callback)
up = GLOBAL_UPLOAD_FILE
synctool.param.NODENAME = orig_nodename
synctool.param.MY_GROUPS = orig_my_groups
rsync_upload(up)
def compare(self, src_path, dest_stat):
'''see if files are the same
Return True if the same
'''
if self.stat.size != dest_stat.size:
if synctool.lib.DRY_RUN:
stdout('%s mismatch (file size)' % self.name)
else:
stdout('%s updated (file size mismatch)' % self.name)
terse(synctool.lib.TERSE_SYNC, self.name)
unix_out('# updating file %s' % self.name)
return False
return self._compare_checksums(src_path)
def copy_stat(self):
'''set access and mod times'''
# FIXME change this to os.utime()
# FIXME but see above ... method VNode.set_times()
# FIXME agree to copy mtime, but atime should be kept intact?
if not synctool.lib.DRY_RUN and synctool.param.SYNC_TIMES:
try:
verbose('copystat: %s => %s' % (self.src_path, self.name))
shutil.copystat(self.src_path, self.name)
except OSError as err:
error('failed to set utime on %s : %s' % (self.name,
err.strerror))
terse(synctool.lib.TERSE_FAIL, 'utime %s' % self.name)
def set_owner(self):
'''set ownership equal to source'''
verbose(dryrun_msg(' os.chown(%s, %d, %d)' %
(self.name, self.stat.uid, self.stat.gid)))
unix_out('chown %s.%s %s' % (self.stat.ascii_uid(),
self.stat.ascii_gid(), self.name))
if not synctool.lib.DRY_RUN:
try:
os.chown(self.name, self.stat.uid, self.stat.gid)
except OSError as err:
error('failed to chown %s.%s %s : %s' %
(self.stat.ascii_uid(), self.stat.ascii_gid(),
self.name, err.strerror))
terse(synctool.lib.TERSE_FAIL, 'owner %s' % self.name)
def move_saved(self):
'''move existing entry to .saved'''
verbose(dryrun_msg('saving %s as %s.saved' % (self.name, self.name)))
unix_out('mv %s %s.saved' % (self.name, self.name))
if not synctool.lib.DRY_RUN:
verbose(' os.rename(%s, %s.saved)' % (self.name, self.name))
try:
os.rename(self.name, '%s.saved' % self.name)
except OSError as err:
stderr('failed to save %s as %s.saved : %s' % (self.name,
self.name,
err.strerror))
terse(synctool.lib.TERSE_FAIL, 'save %s.saved' % self.name)
def set_owner(self):
# type: () -> None
'''set ownership equal to source'''
verbose(dryrun_msg(' os.chown(%s, %d, %d)' %
(self.name, self.stat.uid, self.stat.gid)))
unix_out('chown %s.%s %s' % (self.stat.ascii_uid(),
self.stat.ascii_gid(), self.name))
if not synctool.lib.DRY_RUN:
try:
os.chown(self.name, self.stat.uid, self.stat.gid)
except OSError as err:
error('failed to chown %s.%s %s : %s' %
(self.stat.ascii_uid(), self.stat.ascii_gid(),
self.name, err.strerror))
terse(TERSE_FAIL, 'owner %s' % self.name)
def set_times(self, atime, mtime):
'''set access and mod times'''
# only used for purge --single
# FIXME change to: def set_times(self): set equal to source
# FIXME agree to copy mtime, but atime should be kept intact?
# FIXME change set_times() to set_mtime() ?
if not synctool.lib.DRY_RUN:
try:
os.utime(self.name, (atime, mtime))
except OSError as err:
error('failed to set utime on %s : %s' % (self.name,
err.strerror))
terse(synctool.lib.TERSE_FAIL, 'utime %s' % self.name)
def create(self):
'''copy file'''
if not self.exists:
terse(synctool.lib.TERSE_NEW, self.name)
verbose(dryrun_msg(' copy %s %s' % (self.src_path, self.name)))
unix_out('cp %s %s' % (self.src_path, self.name))
if not synctool.lib.DRY_RUN:
try:
# copy file
shutil.copy(self.src_path, self.name)
except (OSError, IOError) as err:
error('failed to copy %s to %s: %s' %
(prettypath(self.src_path), self.name, err.strerror))
terse(TERSE_FAIL, self.name)
def set_times(self):
'''set access and modification times'''
# only mtime is shown
verbose(dryrun_msg(' os.utime(%s, %s)' %
(self.name, print_timestamp(self.stat.mtime))))
# print timestamp in other format
dt = datetime.datetime.fromtimestamp(self.stat.mtime)
time_str = dt.strftime('%Y%m%d%H%M.%S')
unix_out('touch -t %s %s' % (time_str, self.name))
if not synctool.lib.DRY_RUN:
try:
os.utime(self.name, (self.stat.atime, self.stat.mtime))
except OSError as err:
error('failed to set utime on %s : %s' % (self.name,
err.strerror))
terse(TERSE_FAIL, 'utime %s' % self.name)
def create(self):
# type: () -> None
'''copy file'''
if not self.exists:
terse(synctool.lib.TERSE_NEW, self.name)
verbose(dryrun_msg(' copy %s %s' % (self.src_path, self.name)))
unix_out('cp %s %s' % (self.src_path, self.name))
if not synctool.lib.DRY_RUN:
try:
# copy file
shutil.copy(self.src_path, self.name)
except (OSError, IOError) as err:
error('failed to copy %s to %s: %s' %
(prettypath(self.src_path), self.name, err.strerror))
terse(TERSE_FAIL, self.name)
def check_purge_timestamp(self):
'''check timestamp between src and dest
Returns True if same, False if not
'''
# This is only used for purge/
# check() has already determined that the files are the same
# Now only check the timestamp ...
# FIXME have SyncStat time fields
# Note that SyncStat objects do not know the timestamps;
# they are not cached only to save memory
# So now we have to os.stat() again to get the times; it is
# not a big problem because this func is used for purge_single only
# src_path is under $purge/
# dest_path is in the filesystem
try:
src_stat = os.lstat(self.src_path)
except OSError as err:
error('stat(%s) failed: %s' % (self.src_path, err.strerror))
return False
try:
dest_stat = os.lstat(self.dest_path)
except OSError as err:
error('stat(%s) failed: %s' % (self.dest_path, err.strerror))
return False
# FIXME set_times() should not be called for symlinks
if src_stat.st_mtime > dest_stat.st_mtime:
stdout('%s mismatch (only timestamp)' % self.dest_path)
terse(synctool.lib.TERSE_WARNING,
'%s (only timestamp)' % self.dest_path)
verbose(dryrun_msg(' os.utime(%s, %s)'
'' % (self.dest_path,
time.ctime(src_stat.st_mtime))))
unix_out('touch -r %s %s' % (self.src_path, self.dest_path))
vnode = self.vnode_obj()
vnode.set_times(src_stat.st_atime, src_stat.st_mtime)
return False
return True
def set_permissions(self):
'''set permissions of symlink (if possible)'''
# check if this platform supports lchmod()
# Linux does not have lchmod: its symlinks are always mode 0777
if not hasattr(os, 'lchmod'):
return
verbose(dryrun_msg(' os.lchmod(%s, %04o)' %
(self.name, self.stat.mode & 07777)))
unix_out('lchmod 0%o %s' % (self.stat.mode & 07777, self.name))
if not synctool.lib.DRY_RUN:
try:
os.lchmod(self.name, self.stat.mode & 07777)
except OSError as err:
error('failed to lchmod %04o %s : %s' %
(self.stat.mode & 07777, self.name, err.strerror))
terse(synctool.lib.TERSE_FAIL, 'mode %s' % self.name)
def set_times(self):
# type: () -> None
'''set access and modification times'''
# only mtime is shown
verbose(dryrun_msg(' os.utime(%s, %s)' %
(self.name, print_timestamp(self.stat.mtime))))
# print timestamp in other format
dt = datetime.datetime.fromtimestamp(self.stat.mtime)
time_str = dt.strftime('%Y%m%d%H%M.%S')
unix_out('touch -t %s %s' % (time_str, self.name))
if not synctool.lib.DRY_RUN:
try:
os.utime(self.name, (self.stat.atime, self.stat.mtime))
except OSError as err:
error('failed to set utime on %s : %s' % (self.name,
err.strerror))
terse(TERSE_FAIL, 'utime %s' % self.name)
def create(self):
'''make a block device file'''
major = os.major(self.src_stat.st_rdev)
minor = os.minor(self.src_stat.st_rdev)
verbose(dryrun_msg(' os.mknod(%s, BLK %d,%d)' % (self.name, major,
minor)))
unix_out('mknod %s b %d %d' % (self.name, major, minor))
terse(synctool.lib.TERSE_NEW, self.name)
if not synctool.lib.DRY_RUN:
try:
os.mknod(self.name,
(self.src_stat.st_mode & 0777) | stat.S_IFBLK,
os.makedev(major, minor))
except OSError as err:
error('failed to create device %s : %s' % (self.name,
err.strerror))
terse(TERSE_FAIL, 'device %s' % self.name)
def check_purge_timestamp(self):
'''check timestamp between src and dest
Returns True if same, False if not
'''
# This is only used for purge/
# check() has already determined that the files are the same
# Now only check the timestamp ...
# Note that SyncStat objects do not know the timestamps;
# they are not cached only to save memory
# So now we have to os.stat() again to get the times; it is
# not a big problem because this func is used for purge_single only
# src_path is under $purge/
# dest_path is in the filesystem
try:
src_stat = os.lstat(self.src_path)
except OSError as err:
error('stat(%s) failed: %s' % (self.src_path, err.strerror))
return False
try:
dest_stat = os.lstat(self.dest_path)
except OSError as err:
error('stat(%s) failed: %s' % (self.dest_path, err.strerror))
return False
if src_stat.st_mtime > dest_stat.st_mtime:
stdout('%s mismatch (only timestamp)' % self.dest_path)
terse(synctool.lib.TERSE_WARNING,
'%s (only timestamp)' % self.dest_path)
verbose(
dryrun_msg(' os.utime(%s, %s)'
'' %
(self.dest_path, time.ctime(src_stat.st_mtime))))
unix_out('touch -r %s %s' % (self.src_path, self.dest_path))
vnode = self.vnode_obj()
vnode.set_times(src_stat.st_atime, src_stat.st_mtime)
return False
return True
def compare(self, src_path, dest_stat):
'''compare symbolic links'''
if not self.exists:
return False
try:
link_to = os.readlink(self.name)
except OSError as err:
error('failed to read symlink %s : %s' % (self.name, err.strerror))
return False
if self.oldpath != link_to:
stdout('%s should point to %s, but points to %s' %
(self.name, self.oldpath, link_to))
terse(synctool.lib.TERSE_LINK, self.name)
return False
return True
def create(self):
# type: () -> None
'''make a character device file'''
major = os.major(self.src_stat.st_rdev) # type: ignore
minor = os.minor(self.src_stat.st_rdev) # type: ignore
verbose(dryrun_msg(' os.mknod(%s, CHR %d,%d)' % (self.name, major,
minor)))
unix_out('mknod %s c %d %d' % (self.name, major, minor))
terse(synctool.lib.TERSE_NEW, self.name)
if not synctool.lib.DRY_RUN:
try:
os.mknod(self.name,
(self.src_stat.st_mode & 0777) | stat.S_IFCHR,
os.makedev(major, minor))
except OSError as err:
error('failed to create device %s : %s' % (self.name,
err.strerror))
terse(TERSE_FAIL, 'device %s' % self.name)
def move_saved(self):
'''move existing entry to .saved'''
# do not save files that already are .saved
_, ext = os.path.splitext(self.name)
if ext == '.saved':
return
verbose(dryrun_msg('saving %s as %s.saved' % (self.name, self.name)))
unix_out('mv %s %s.saved' % (self.name, self.name))
if not synctool.lib.DRY_RUN:
verbose(' os.rename(%s, %s.saved)' % (self.name, self.name))
try:
os.rename(self.name, '%s.saved' % self.name)
except OSError as err:
error('failed to save %s as %s.saved : %s' %
(self.name, self.name, err.strerror))
terse(synctool.lib.TERSE_FAIL, 'save %s.saved' % self.name)
def set_owner(self):
'''set ownership of symlink'''
if not hasattr(os, 'lchown'):
# you never know
return
verbose(dryrun_msg(' os.lchown(%s, %d, %d)' %
(self.name, self.stat.uid, self.stat.gid)))
unix_out('lchown %s.%s %s' % (self.stat.ascii_uid(),
self.stat.ascii_gid(), self.name))
if not synctool.lib.DRY_RUN:
try:
os.lchown(self.name, self.stat.uid, self.stat.gid)
except OSError as err:
error('failed to lchown %s.%s %s : %s' %
(self.stat.ascii_uid(), self.stat.ascii_gid(),
self.name, err.strerror))
terse(synctool.lib.TERSE_FAIL, 'owner %s' % self.name)
def create(self):
# type: () -> None
'''make a block device file'''
major = os.major(self.src_stat.st_rdev) # type: ignore
minor = os.minor(self.src_stat.st_rdev) # type: ignore
verbose(dryrun_msg(' os.mknod(%s, BLK %d,%d)' % (self.name, major,
minor)))
unix_out('mknod %s b %d %d' % (self.name, major, minor))
terse(synctool.lib.TERSE_NEW, self.name)
if not synctool.lib.DRY_RUN:
try:
os.mknod(self.name,
(self.src_stat.st_mode & 0777) | stat.S_IFBLK,
os.makedev(major, minor))
except OSError as err:
error('failed to create device %s : %s' % (self.name,
err.strerror))
terse(TERSE_FAIL, 'device %s' % self.name)
def create(self):
'''create directory'''
if synctool.lib.path_exists(self.name):
# it can happen that the dir already exists
# due to recursion in visit() + VNode.mkdir_basepath()
# So this is double checked for dirs that did not exist
return
verbose(dryrun_msg(' os.mkdir(%s)' % self.name))
unix_out('mkdir %s' % self.name)
terse(synctool.lib.TERSE_MKDIR, self.name)
if not synctool.lib.DRY_RUN:
try:
os.mkdir(self.name, self.stat.mode & 07777)
except OSError as err:
error('failed to make directory %s : %s' % (self.name,
err.strerror))
terse(TERSE_FAIL, 'mkdir %s' % self.name)
def set_permissions(self):
# type: () -> None
'''set permissions of symlink (if possible)'''
# check if this platform supports lchmod()
# Linux does not have lchmod: its symlinks are always mode 0777
if not hasattr(os, 'lchmod'):
return
verbose(dryrun_msg(' os.lchmod(%s, %04o)' %
(self.name, self.stat.mode & 07777)))
unix_out('lchmod 0%o %s' % (self.stat.mode & 07777, self.name))
if not synctool.lib.DRY_RUN:
try:
os.lchmod(self.name, self.stat.mode & 07777)
except OSError as err:
error('failed to lchmod %04o %s : %s' %
(self.stat.mode & 07777, self.name, err.strerror))
terse(TERSE_FAIL, 'mode %s' % self.name)
def set_owner(self):
# type: () -> None
'''set ownership of symlink'''
if not hasattr(os, 'lchown'):
# you never know
return
verbose(dryrun_msg(' os.lchown(%s, %d, %d)' %
(self.name, self.stat.uid, self.stat.gid)))
unix_out('lchown %s.%s %s' % (self.stat.ascii_uid(),
self.stat.ascii_gid(), self.name))
if not synctool.lib.DRY_RUN:
try:
os.lchown(self.name, self.stat.uid, self.stat.gid)
except OSError as err:
error('failed to lchown %s.%s %s : %s' %
(self.stat.ascii_uid(), self.stat.ascii_gid(),
self.name, err.strerror))
terse(TERSE_FAIL, 'owner %s' % self.name)
def create(self):
# type: () -> None
'''create directory'''
if synctool.lib.path_exists(self.name):
# it can happen that the dir already exists
# due to recursion in visit() + VNode.mkdir_basepath()
# So this is double checked for dirs that did not exist
return
verbose(dryrun_msg(' os.mkdir(%s)' % self.name))
unix_out('mkdir %s' % self.name)
terse(synctool.lib.TERSE_MKDIR, self.name)
if not synctool.lib.DRY_RUN:
try:
os.mkdir(self.name, self.stat.mode & 07777)
except OSError as err:
error('failed to make directory %s : %s' % (self.name,
err.strerror))
terse(TERSE_FAIL, 'mkdir %s' % self.name)
def compare(self, src_path, dest_stat):
'''compare symbolic links'''
if not self.exists:
return False
try:
link_to = os.readlink(self.name)
except OSError as err:
error('failed to read symlink %s : %s' % (self.name,
err.strerror))
return False
if self.oldpath != link_to:
stdout('%s should point to %s, but points to %s' %
(self.name, self.oldpath, link_to))
terse(synctool.lib.TERSE_LINK, self.name)
return False
return True
def harddelete(self):
'''delete directory'''
if synctool.lib.DRY_RUN:
not_str = 'not '
else:
not_str = ''
stdout('%sremoving %s' % (not_str, self.name + os.sep))
unix_out('rmdir %s' % self.name)
terse(synctool.lib.TERSE_DELETE, self.name + os.sep)
if not synctool.lib.DRY_RUN:
verbose(' os.rmdir(%s)' % self.name)
try:
os.rmdir(self.name)
except OSError:
# probably directory not empty
# refuse to delete dir, just move it aside
verbose('refusing to delete directory %s' % self.name)
self.move_saved()
def move_saved(self):
'''move existing entry to .saved'''
# do not save files that already are .saved
_, ext = os.path.splitext(self.name)
if ext == '.saved':
return
verbose(dryrun_msg('saving %s as %s.saved' % (self.name, self.name)))
unix_out('mv %s %s.saved' % (self.name, self.name))
if not synctool.lib.DRY_RUN:
verbose(' os.rename(%s, %s.saved)' % (self.name, self.name))
try:
os.rename(self.name, '%s.saved' % self.name)
except OSError as err:
error('failed to save %s as %s.saved : %s' % (self.name,
self.name,
err.strerror))
terse(synctool.lib.TERSE_FAIL, 'save %s.saved' % self.name)
def create(self):
'''copy file'''
if not self.exists:
terse(synctool.lib.TERSE_NEW, self.name)
verbose(dryrun_msg(' copy %s %s' % (self.src_path, self.name)))
unix_out('cp %s %s' % (self.src_path, self.name))
if not synctool.lib.DRY_RUN:
try:
# copy file
shutil.copy(self.src_path, self.name)
if synctool.param.SYNC_TIMES:
shutil.copystat(self.src_path, self.name)
except IOError as err:
error('failed to copy %s to %s: %s' %
(prettypath(self.src_path), self.name, err.strerror))
terse(synctool.lib.TERSE_FAIL, self.name)
def harddelete(self):
# type: () -> None
'''delete directory'''
if synctool.lib.DRY_RUN:
not_str = 'not '
else:
not_str = ''
stdout('%sremoving %s' % (not_str, self.name + os.sep))
unix_out('rmdir %s' % self.name)
terse(synctool.lib.TERSE_DELETE, self.name + os.sep)
if not synctool.lib.DRY_RUN:
verbose(' os.rmdir(%s)' % self.name)
try:
os.rmdir(self.name)
except OSError:
# probably directory not empty
# refuse to delete dir, just move it aside
verbose('refusing to delete directory %s' % self.name)
self.move_saved()
def harddelete(self):
'''delete existing entry'''
if synctool.lib.DRY_RUN:
not_str = 'not '
else:
not_str = ''
stdout('%sdeleting %s' % (not_str, self.name))
unix_out('rm %s' % self.name)
terse(synctool.lib.TERSE_DELETE, self.name)
if not synctool.lib.DRY_RUN:
verbose(' os.unlink(%s)' % self.name)
try:
os.unlink(self.name)
except OSError as err:
error('failed to delete %s : %s' % (self.name, err.strerror))
terse(synctool.lib.TERSE_FAIL, 'delete %s' % self.name)
else:
log('deleted %s' % self.name)
def harddelete(self):
'''delete existing entry'''
if synctool.lib.DRY_RUN:
not_str = 'not '
else:
not_str = ''
stdout('%sdeleting %s' % (not_str, self.name))
unix_out('rm %s' % self.name)
terse(synctool.lib.TERSE_DELETE, self.name)
if not synctool.lib.DRY_RUN:
verbose(' os.unlink(%s)' % self.name)
try:
os.unlink(self.name)
except OSError as err:
error('failed to delete %s : %s' % (self.name, err.strerror))
terse(synctool.lib.TERSE_FAIL, 'delete %s' % self.name)
else:
log('deleted %s' % self.name)
def _single_overlay_callback(obj, pre_dict, post_dict):
'''do overlay function for single files'''
if not SINGLE_FILES:
# proceed quickly
return True, False
if obj.ov_type == synctool.overlay.OV_TEMPLATE:
return generate_template(obj, post_dict), False
go_on = True
updated = False
if _match_single(obj.dest_path):
_, updated = _overlay_callback(obj, pre_dict, post_dict)
if not updated:
stdout('%s is up to date' % obj.dest_path)
terse(synctool.lib.TERSE_OK, obj.dest_path)
unix_out('# %s is up to date\n' % obj.dest_path)
return go_on, updated
def create(self):
'''make a character device file'''
major = os.major(self.src_stat.st_rdev)
minor = os.minor(self.src_stat.st_rdev)
verbose(
dryrun_msg(' os.mknod(%s, CHR %d,%d)' %
(self.name, major, minor)))
unix_out('mknod %s c %d %d' % (self.name, major, minor))
terse(synctool.lib.TERSE_NEW, self.name)
if not synctool.lib.DRY_RUN:
try:
os.mknod(self.name,
(self.src_stat.st_mode & 0777) | stat.S_IFCHR,
os.makedev(major, minor))
except OSError as err:
error('failed to create device %s : %s' %
(self.name, err.strerror))
terse(synctool.lib.TERSE_FAIL, 'device %s' % self.name)
def vnode_dest_obj(self):
# type: () -> VNode
'''create vnode object for this SyncObject's destination'''
exists = self.dest_stat.exists()
if self.dest_stat.is_file():
return VNodeFile(self.dest_path, self.src_stat, exists,
self.src_path)
if self.dest_stat.is_dir():
return VNodeDir(self.dest_path, self.src_stat, exists)
if self.dest_stat.is_link():
try:
oldpath = os.readlink(self.src_path)
except OSError as err:
error('failed to read symlink %s : %s' % (self.print_src(),
err.strerror))
terse(TERSE_FAIL, self.src_path)
return None
return VNodeLink(self.dest_path, self.src_stat, exists, oldpath)
if self.dest_stat.is_fifo():
return VNodeFifo(self.dest_path, self.src_stat, exists)
if self.dest_stat.is_chardev():
return VNodeChrDev(self.dest_path, self.src_stat, exists,
os.stat(self.src_path))
if self.dest_stat.is_blockdev():
return VNodeBlkDev(self.dest_path, self.src_stat, exists,
os.stat(self.src_path))
# error, can not handle file type of src_path
return None
def main():
# type: () -> None
'''run the program'''
param.init()
action = get_options()
config.init_mynodename()
if not param.NODENAME:
error('unable to determine my nodename (hostname: %s)' %
param.HOSTNAME)
stderr('please check %s' % param.CONF_FILE)
sys.exit(-1)
if param.NODENAME not in param.NODES:
error("unknown node '%s'" % param.NODENAME)
stderr('please check %s' % param.CONF_FILE)
sys.exit(-1)
if param.NODENAME in param.IGNORE_GROUPS:
# this is only a warning ...
# you can still run synctool-pkg on the client by hand
warning('node %s is disabled in %s' %
(param.NODENAME, param.CONF_FILE))
if synctool.lib.UNIX_CMD:
t = time.localtime(time.time())
unix_out('#')
unix_out('# script generated by synctool on '
'%04d/%02d/%02d %02d:%02d:%02d' %
(t[0], t[1], t[2], t[3], t[4], t[5]))
unix_out('#')
unix_out('# my hostname: %s' % param.HOSTNAME)
unix_out('# SYNCTOOL_NODE=%s' % param.NODENAME)
unix_out('# SYNCTOOL_ROOT=%s' % param.ROOTDIR)
unix_out('#')
if not synctool.lib.DRY_RUN:
unix_out('# NOTE: --fix specified, applying updates')
unix_out('#')
unix_out('')
else:
if not synctool.lib.MASTERLOG:
# only print this when running stand-alone
if not synctool.lib.QUIET:
if synctool.lib.DRY_RUN:
stdout('DRY RUN, not doing any updates')
terse(synctool.lib.TERSE_DRYRUN, 'not doing any updates')
else:
stdout('--fix specified, applying changes')
terse(synctool.lib.TERSE_FIXING, ' applying changes')
else:
if synctool.lib.DRY_RUN:
verbose('DRY RUN, not doing any updates')
else:
verbose('--fix specified, applying changes')
verbose('my nodename: %s' % param.NODENAME)
verbose('my hostname: %s' % param.HOSTNAME)
verbose('rootdir: %s' % param.ROOTDIR)
os.environ['SYNCTOOL_NODE'] = param.NODENAME
os.environ['SYNCTOOL_ROOT'] = param.ROOTDIR
unix_out('umask 077')
unix_out('')
os.umask(077)
if action == ACTION_DIFF:
diff_files()
elif action == ACTION_REFERENCE:
reference_files()
elif action == ACTION_ERASE_SAVED:
if SINGLE_FILES:
single_erase_saved()
else:
erase_saved()
elif SINGLE_FILES:
single_files()
else:
purge_files()
overlay_files()
delete_files()
unix_out('# EOB')
