def full_syncher(oneshot=False):
while True:
# Backup suffix
backupsuffix = "-" + time.strftime("%Y%m%d_%H%M")
# If oneshot, run full sync immediately
if oneshot:
log(utils.INFO, "B",
"INITIAL SYNC\n" + "Please wait: this can take a long time")
else:
time.sleep(3600)
log(utils.DEBUG2, "B", "TIMED FULL SYNC: Waking up")
# Check if it's time of a scheduled fullsync
hour = int(time.strftime("%H"))
if hour not in schedules['fullsync']['hours']:
log(utils.DEBUG2, "B", "TIMED FULL SYNC: not now. Sleeping...")
continue
log(utils.INFO, "B", "TIMED FULL SYNC: Starting")
# Add required options
rsync_options = ["-AX"]
rsync_options.append("--max-size=1024G")
# If use_backupdir is enabled, add the relevant options
if config.use_backupdir:
rsync_options.append("-b")
rsync_options.append("--backup-dir=" + config.backupdir)
rsync_options.append("--suffix=" + backupsuffix)
# Proceed
ldirs = "/"
rdirs = "/"
# Acquire lock
locks['global'].acquire(False)
# First sync, from L to R
success = True # Be optimistic ;)
excludelist = utils.gen_exclude(options.rsync_excludes)
cmd = (["rsync", "-airu"] + options.rsync_extra + rsync_options +
["-u", "--files-from=-"] + excludelist +
[options.srcroot, options.dsthost + ":" + options.dstroot])
log(utils.INFO, "B", "Timed full sync from L to R started")
(process, output, error) = execute(cmd, "L", ldirs, timeout=False)
if process.returncode not in utils.RSYNC_SUCCESS:
success = False
# Second sync, from R to L
if config.acl_from_left_only:
try:
rsync_options.remove("-AX")
except:
pass
cmd = (["rsync", "-airu"] + options.rsync_extra + rsync_options +
["-u", "--files-from=-"] + excludelist +
[options.dsthost + ":" + options.dstroot, options.srcroot])
log(utils.INFO, "B", "Timed full sync from R to L started")
(process, output, error) = execute(cmd, "R", rdirs, timeout=False)
if process.returncode not in utils.RSYNC_SUCCESS:
success = False
log(utils.INFO, "B", "TIMED FULL SYNC: Ending")
# Release lock
locks['global'].release()
# If oneshot, return now
if oneshot:
return success
def rsync(action,
recurse=config.rsync_event_recurse,
acl=False,
warn=True,
updateonly=False):
log(utils.DEBUG1,
action['source'],
"RSYNC action",
eventid=action['eventid'])
# Options selection
rsync_options = []
if action['backfired']:
rsync_options.append("--existing")
if recurse or action['recurse']:
rsync_options.append("-r")
if acl and (action['source'] == "L" or not config.acl_from_left_only):
rsync_options.append("-AX")
if action['flags'] == utils.FFORCE and config.maxsize:
rsync_options.append(config.maxsize)
if action['updateonly'] or updateonly:
rsync_options.append("-u")
# Command selection
if action['source'] == "L":
left = options.srcroot
right = options.dsthost + ":" + options.dstroot
filelist = action['filelist'].replace(left, "")
else:
left = options.dsthost + ":" + options.dstroot
right = options.srcroot
filelist = action['filelist'].replace(right, "")
# Filelist mangling to remove duplicate
if len(filelist) == 0:
filelist = "/"
else:
fileset = set(utils.deconcat(filelist))
filelist = "\n".join(fileset)
# Generating exclude list
excludelist = utils.gen_exclude(options.rsync_excludes)
# Execute and report
log(utils.DEBUG2,
action['source'],
"Preparing to sync: \n" + filelist,
eventid=action['eventid'])
cmd = (["rsync", "-ai"] + options.rsync_extra + rsync_options +
["--files-from=-"] + excludelist + [left, right])
(process, output, error) = execute(cmd,
action['source'],
filelist,
warn=warn,
eventid=action['eventid'])
if process.returncode == utils.RSYNC_TERMINATED:
log(utils.INFO,
action['source'],
"Rescheduling files: \n" + filelist,
eventid=action['eventid'])
actions.append(action)
def rsync(action, recurse=config.rsync_event_recurse, acl=False, warn=True,
updateonly=False):
log(utils.DEBUG1, action['source'], "RSYNC action",
eventid=action['eventid'])
# Options selection
rsync_options = []
if action['backfired']:
rsync_options.append("--existing")
if recurse or action['recurse']:
rsync_options.append("-r")
if acl and (action['source'] == "L" or not config.acl_from_left_only):
rsync_options.append("-AX")
if action['flags'] == utils.FFORCE and config.maxsize:
rsync_options.append(config.maxsize)
if action['updateonly'] or updateonly:
rsync_options.append("-u")
# Command selection
if action['source'] == "L":
left = options.srcroot
right = options.dsthost + ":" + options.dstroot
filelist = action['filelist'].replace(left, "")
else:
left = options.dsthost + ":" + options.dstroot
right = options.srcroot
filelist = action['filelist'].replace(right, "")
# Filelist mangling to remove duplicate
if len(filelist) == 0:
filelist = "/"
else:
fileset = set(utils.deconcat(filelist))
filelist = "\n".join(fileset)
# Generating exclude list
excludelist = utils.gen_exclude(options.rsync_excludes)
# Execute and report
log(utils.DEBUG2, action['source'], "Preparing to sync: \n" + filelist,
eventid=action['eventid'])
cmd = (["rsync", "-ai"] + options.rsync_extra + rsync_options +
["--files-from=-"] + excludelist + [left, right])
(process, output, error) = execute(cmd, action['source'], filelist,
warn=warn, eventid=action['eventid'])
if process.returncode == utils.RSYNC_TERMINATED:
log(utils.INFO, action['source'], "Rescheduling files: \n" + filelist,
eventid=action['eventid'])
actions.append(action)
def check(src, dst, filelist="", checksum=False):
# If checksum is enabled, continue only with a filelist
if not filelist and checksum:
return 0, ""
# Check via rsync
excludelist = utils.gen_exclude(options.rsync_excludes)
try:
excludelist.remove("--exclude=*"+config.safesuffix)
except:
pass
rsync_args = ["-anui"]
# Set filesize limit
# For lite or modified_only checks, use the default from config.py
# For full check or when using cheksum, increse size limit
if options.modified_only or options.lite:
pass
else:
rsync_args.append("--max-size=1024G")
# Enable checksum
if checksum:
rsync_args.append("--checksum")
# Link disabling
if options.nolinks:
try:
options.extra.remove("-L")
except:
pass
rsync_args.append("--no-l")
# Pass filelist
if filelist:
rsync_args.append("--files-from=-")
# Construct command and execute
cmd = (["rsync"] + options.extra + rsync_args + ["-n"] +
excludelist + [src, dst])
(process, output, error) = execute(cmd, filelist)
# Return
return process.returncode, output
def check(src, dst, filelist="", checksum=False):
# If checksum is enabled, continue only with a filelist
if not filelist and checksum:
return 0, ""
# Check via rsync
excludelist = utils.gen_exclude(options.rsync_excludes)
try:
excludelist.remove("--exclude=*" + config.safesuffix)
except:
pass
rsync_args = ["-anui"]
# Set filesize limit
# For lite or modified_only checks, use the default from config.py
# For full check or when using cheksum, increse size limit
if options.modified_only or options.lite:
pass
else:
rsync_args.append("--max-size=1024G")
# Enable checksum
if checksum:
rsync_args.append("--checksum")
# Link disabling
if options.nolinks:
try:
options.extra.remove("-L")
except:
pass
rsync_args.append("--no-l")
# Pass filelist
if filelist:
rsync_args.append("--files-from=-")
# Construct command and execute
cmd = (["rsync"] + options.extra + rsync_args + ["-n"] + excludelist +
[src, dst])
(process, output, error) = execute(cmd, filelist)
# Return
return process.returncode, output
def full_syncher(oneshot=False):
while True:
# Backup suffix
backupsuffix = "-"+time.strftime("%Y%m%d_%H%M")
# If oneshot, run full sync immediately
if oneshot:
log(utils.INFO, "B", "INITIAL SYNC\n" +
"Please wait: this can take a long time")
else:
time.sleep(3600)
log(utils.DEBUG2, "B", "TIMED FULL SYNC: Waking up")
# Check if it's time of a scheduled fullsync
hour = int(time.strftime("%H"))
if hour not in schedules['fullsync']['hours']:
log(utils.DEBUG2, "B", "TIMED FULL SYNC: not now. Sleeping...")
continue
log(utils.INFO, "B", "TIMED FULL SYNC: Starting")
# Add required options
rsync_options = ["-AX"]
rsync_options.append("--max-size=1024G")
# If use_backupdir is enabled, add the relevant options
if config.use_backupdir:
rsync_options.append("-b")
rsync_options.append("--backup-dir="+config.backupdir)
rsync_options.append("--suffix="+backupsuffix)
# Proceed
ldirs = "/"
rdirs = "/"
# Acquire lock
locks['global'].acquire(False)
# First sync, from L to R
success = True # Be optimistic ;)
excludelist = utils.gen_exclude(options.rsync_excludes)
cmd = (["rsync", "-airu"] + options.rsync_extra + rsync_options +
["-u", "--files-from=-"] + excludelist +
[options.srcroot, options.dsthost + ":" + options.dstroot])
log(utils.INFO, "B",
"Timed full sync from L to R started")
(process, output, error) = execute(cmd, "L", ldirs, timeout=False)
if process.returncode not in utils.RSYNC_SUCCESS:
success = False
# Second sync, from R to L
if config.acl_from_left_only:
try:
rsync_options.remove("-AX")
except:
pass
cmd = (["rsync", "-airu"] + options.rsync_extra + rsync_options +
["-u", "--files-from=-"] + excludelist +
[options.dsthost + ":" + options.dstroot, options.srcroot])
log(utils.INFO, "B",
"Timed full sync from R to L started")
(process, output, error) = execute(cmd, "R", rdirs, timeout=False)
if process.returncode not in utils.RSYNC_SUCCESS:
success = False
log(utils.INFO, "B", "TIMED FULL SYNC: Ending")
# Release lock
locks['global'].release()
# If oneshot, return now
if oneshot:
return success
