def parse_options():
parser = optparse.OptionParser()
parser.add_option("-d", "--debug", dest="debug", type="int",
help="Enable debugger", action="store",
default=config.debug)
parser.add_option("-t", "--tempfiles", dest="tempfiles", action="store",
help="Tempfile list (regex)", default=config.tempfiles)
parser.add_option("-e", "--excludes", dest="excludes", action="store",
help="Excluded files (regex)", default=config.excludes)
parser.add_option("-k", "--kill", dest="kill", action="store_true",
help="Kill other inotify processes", default=False)
parser.add_option("-i", "--interval", dest="interval", action="store",
type="int",
help="Interval between event collection/notification",
default=config.event_interval)
parser.add_option("-T", "--translate", dest="translate",
help="Translate/replace path element", action="store",
default=config.translate)
parser.add_option("--srcroot", dest="srcroot",
action="store", default=None)
(options, args) = parser.parse_args()
# Define base dirs
options.srcroot = utils.normalize_dir(options.srcroot)
options.psyncdir = utils.normalize_dir(options.srcroot+config.psyncdir)
return (options, args)
def parse_options():
parser = optparse.OptionParser()
parser.add_option("-d", "--debug", dest="debug", type="int",
help="Enable debugger", action="store",
default=config.debug)
parser.add_option("-r", "--remote-host", dest="dsthost",
help="Remote host", action="store", default=None)
parser.add_option("-t", "--tempfiles", dest="tempfiles",
help="Tempfile list (regex)", action="store",
default=config.tempfiles)
parser.add_option("-e", "--excludes", dest="excludes", action="store",
help="Excluded files (regex)", default=config.excludes)
parser.add_option("--rsync-excludes", dest="rsync_excludes",
help="Rsync exclusion list", action="append",
default=config.rsync_excludes)
parser.add_option("-x", "--extra", dest="rsync_extra",
help="Extra rsync options", action="append",
default=config.rsync_extra)
parser.add_option("-b", "--banned", dest="banned",
help="Exit immediately if BANNED program is running",
action="store", default=config.banned)
parser.add_option("-T", "--translate", dest="translate",
help="Translate/replace path element", action="store",
default=config.translate)
parser.add_option("-n", "--dry-run", dest="dryrun",
help="Simulate sync and log, but do nothing",
action="store_true", default=config.dryrun)
parser.add_option("-s", "--sync-only", dest="synconly",
help="First sync only, then exit", action="store_true",
default=config.first_sync_only)
parser.add_option("-k", "--skip-initial-sync", dest="skipsync",
help="Skip initial sync", action="store_true",
default=config.skip_initial_sync)
parser.add_option("-f", "--force", dest="force", action="count",
help="Force delete/move commands",
default=config.force)
parser.add_option("--srcroot", dest="srcroot", action="store",
default=None)
parser.add_option("--dstroot", dest="dstroot", action="store",
default=None)
(options, args) = parser.parse_args()
# Let rsync be more verbose based of debug level
if options.debug:
options.rsync_extra.append("-v")
if options.debug > 1:
options.rsync_extra.append("--stats")
# If dryrun, increase debug level
if options.dryrun:
options.debug = 2
options.rsync_extra.append("-n")
# Normalize directories
options.srcroot = utils.normalize_dir(args[0])
options.dstroot = utils.normalize_dir(args[1])
# Define left and right private .psync dirs
options.lpsyncdir = utils.normalize_dir(options.srcroot + config.psyncdir)
options.rpsyncdir = utils.normalize_dir(options.dstroot + config.psyncdir)
return (options, args)
def parse_options():
parser = optparse.OptionParser()
parser.add_option("-d",
"--debug",
dest="debug",
type="int",
help="Enable debugger",
action="store",
default=config.debug)
parser.add_option("-t",
"--tempfiles",
dest="tempfiles",
action="store",
help="Tempfile list (regex)",
default=config.tempfiles)
parser.add_option("-e",
"--excludes",
dest="excludes",
action="store",
help="Excluded files (regex)",
default=config.excludes)
parser.add_option("-k",
"--kill",
dest="kill",
action="store_true",
help="Kill other inotify processes",
default=False)
parser.add_option("-i",
"--interval",
dest="interval",
action="store",
type="int",
help="Interval between event collection/notification",
default=config.event_interval)
parser.add_option("-T",
"--translate",
dest="translate",
help="Translate/replace path element",
action="store",
default=config.translate)
parser.add_option("--srcroot",
dest="srcroot",
action="store",
default=None)
(options, args) = parser.parse_args()
# Define base dirs
options.srcroot = utils.normalize_dir(options.srcroot)
options.psyncdir = utils.normalize_dir(options.srcroot + config.psyncdir)
return (options, args)
def parse_options():
parser = optparse.OptionParser()
parser.add_option("-r", "--remote-host", dest="dsthost",
help="Remote host", action="store", default=None)
parser.add_option("-e", "--exclude", dest="rsync_excludes",
help="Files to exclude", action="append",
default=config.rsync_excludes)
parser.add_option("-x", "--extra", dest="extra",
help="Extra rsync options", action="append",
default=config.rsync_extra)
parser.add_option("-d", "--debug", dest="debug", help="Debug",
action="store", default=config.debug)
parser.add_option("-l", "--lite", dest="lite", help="Relaxed checks",
action="store_true", default=False)
parser.add_option("-c", "--checksum", dest="checksum",
help="Compute checksum for changed files",
action="store_true", default=False)
parser.add_option("-m", "--modified_only", dest="modified_only",
help="Consider only modified files, ignoring new files",
action="store_true", default=False)
parser.add_option("-X", "--nolinks", dest="nolinks",
help="Exclude links from comparison",
action="store_true", default=False)
parser.add_option("-b", "--backup", dest="backup", help="Do backups",
action="store_true", default=False)
parser.add_option("--srcroot", dest="srcroot", action="store",
default=None)
parser.add_option("--dstroot", dest="dstroot", action="store",
default=None)
(options, args) = parser.parse_args()
# Checksum or modified_only automatically disables lite check
if options.checksum or options.modified_only:
options.lite = False
# srcroot and dstroot
options.srcroot = utils.normalize_dir(args[0])
options.dstroot = utils.normalize_dir(args[1])
return (options, args)
def parse_options():
parser = optparse.OptionParser()
parser.add_option("-d",
"--debug",
dest="debug",
type="int",
help="Enable debugger",
action="store",
default=config.debug)
parser.add_option("-r",
"--remote-host",
dest="dsthost",
help="Remote host",
action="store",
default=None)
parser.add_option("-t",
"--tempfiles",
dest="tempfiles",
help="Tempfile list (regex)",
action="store",
default=config.tempfiles)
parser.add_option("-e",
"--excludes",
dest="excludes",
action="store",
help="Excluded files (regex)",
default=config.excludes)
parser.add_option("--rsync-excludes",
dest="rsync_excludes",
help="Rsync exclusion list",
action="append",
default=config.rsync_excludes)
parser.add_option("-x",
"--extra",
dest="rsync_extra",
help="Extra rsync options",
action="append",
default=config.rsync_extra)
parser.add_option("-b",
"--banned",
dest="banned",
help="Exit immediately if BANNED program is running",
action="store",
default=config.banned)
parser.add_option("-T",
"--translate",
dest="translate",
help="Translate/replace path element",
action="store",
default=config.translate)
parser.add_option("-n",
"--dry-run",
dest="dryrun",
help="Simulate sync and log, but do nothing",
action="store_true",
default=config.dryrun)
parser.add_option("-s",
"--sync-only",
dest="synconly",
help="First sync only, then exit",
action="store_true",
default=config.first_sync_only)
parser.add_option("-k",
"--skip-initial-sync",
dest="skipsync",
help="Skip initial sync",
action="store_true",
default=config.skip_initial_sync)
parser.add_option("-f",
"--force",
dest="force",
action="count",
help="Force delete/move commands",
default=config.force)
parser.add_option("--srcroot",
dest="srcroot",
action="store",
default=None)
parser.add_option("--dstroot",
dest="dstroot",
action="store",
default=None)
(options, args) = parser.parse_args()
# Let rsync be more verbose based of debug level
if options.debug:
options.rsync_extra.append("-v")
if options.debug > 1:
options.rsync_extra.append("--stats")
# If dryrun, increase debug level
if options.dryrun:
options.debug = 2
options.rsync_extra.append("-n")
# Normalize directories
options.srcroot = utils.normalize_dir(args[0])
options.dstroot = utils.normalize_dir(args[1])
# Define left and right private .psync dirs
options.lpsyncdir = utils.normalize_dir(options.srcroot + config.psyncdir)
options.rpsyncdir = utils.normalize_dir(options.dstroot + config.psyncdir)
return (options, args)
def reader(process, source="B"):
rogue = 0
while True:
# Select variables based on event source
if source == "L":
psyncdir = options.lpsyncdir
else:
psyncdir = options.rpsyncdir
# Read line
line = process.stdout.readline()
line = line.strip(" \n")
# If it is a log, print it
match = re.match("^\[(.*?)\] \[(.*?):(.*?)\] \[(.*?)\]", line)
if match:
severity = match.group(3)
line = line[len(match.group()) + 1:]
# If it is an heartbeat-related log line, take note and continue
if line.find(psyncdir + config.heartfile) >= 0:
beat_inotify(source)
log(utils.DEBUG3, source, line, 1)
else:
# Otherwise, simply print it
log(severity, source, line, 1)
continue
# If HEART, take note and continue
if line.find(psyncdir + config.heartfile) >= 0:
beat_inotify(source)
log(utils.DEBUG2, source, "heartbeat")
continue
# Check if connected
if not are_ready():
if len(line) > 0:
log(utils.ERROR, source,
"Not connected, ignoring event: " + line)
continue
# Be sure to process a good formed line
nfields = 6
match = re.match("^(RSYNC|MOVE|DELETE|NONE)", line, flags=re.I)
if not match or line.count(config.separator) != nfields:
log(utils.WARNING, source,
"Rogue line (n." + str(rogue) + "): " + line)
rogue = rogue + 1
if rogue >= 5:
return
else:
continue
else:
rogue = 0
entry = utils.deconcat(line, config.separator)
method = entry[0]
itemtype = entry[1]
parent = utils.normalize_dir(entry[2])
srcfile = entry[3]
dstfile = entry[4]
flags = entry[5]
checksum = entry[6]
# Validate checksum
computed = line[:-len(config.separator + checksum)]
computed = hashlib.md5(computed).hexdigest()
if checksum != computed:
log(utils.ERROR, source,
"Ignoring event due to invalid checksum for line: " + line)
log(utils.ERROR, source,
"Received: " + checksum + " - Computed: " + computed)
continue
else:
log(
utils.DEBUG2, source, "Checksum ok. Received: " + checksum +
" - Computed: " + computed)
# Beat the heart
beat_inotify(source)
# If method is NONE, continue reading
if method == "NONE":
log(utils.INFO, source, "Ignoring event NONE for file: " + srcfile)
continue
# Parse event
log(utils.DEBUG1, source, "Read event: " + line)
# Pending checks
if method in config.pending_events:
backfired = check_pendings(source, srcfile, method)
else:
backfired = False
if backfired:
if method == "RSYNC" and config.rsync_style > 1:
log(
utils.DEBUG1, source, "Ignoring backfired event " +
method + config.separator + srcfile)
continue
# If batched rsync is true, continue to the next event
if config.rsync_style == 3:
continue
# Normalize dir
if itemtype == "DIR":
srcfile = utils.normalize_dir(srcfile)
if method == "MOVE":
dstfile = utils.normalize_dir(dstfile)
# Build filelist
try:
prev = actions.pop()
# Is mergeable?
if (
# is not a symlink
not os.path.islink(srcfile) and
# source and method are same than previous
source == prev['source'] and method == prev['method']
and ((
# method is rsync and other options are the same
prev['method'] == "RSYNC" and prev['backfired']
== backfired and prev['flags'] == flags) or (
# method is delete
prev['method'] == "DELETE"))):
filelist = utils.concat(prev['filelist'], srcfile)
state['current_merges'] = state['current_merges'] + 1
else:
state['current_merges'] = 0
filelist = srcfile
actions.append(prev)
except:
state['current_merges'] = 0
filelist = srcfile
log(utils.DEBUG1, source,
"Current merges: " + str(state['current_merges']))
entry = {
'source': source,
'method': method,
'itemtype': itemtype,
'filelist': filelist,
'dstfile': dstfile,
'eventid': checksum[-5:],
'backfired': backfired,
'flags': flags,
'recurse': False,
'updateonly': False
}
actions.append(entry)
def unregister_dir(source, dirname):
mirror = mirror = get_mirror(source)
dirs[source].pop(utils.normalize_dir(dirname), None)
dirs[mirror].pop(utils.normalize_dir(dirname), None)
log(utils.DEBUG2, source,
"Unregistering directory from later check: " + dirname)
def register_dir(source, dirname):
dirs[source][utils.normalize_dir(dirname)] = True
log(utils.DEBUG2, source,
"Registering directory for later check: " + dirname)
def parse_line(line):
line = line.rstrip("\n")
# Check if it's an inotify logline
if inotifylog(line):
return
# Check for safety
if not safeline(line):
return
log(utils.DEBUG2, "Raw EVENT: "+line)
# Translate and re-check for safety
if options.translate:
translated, original, line = translate(line)
if not safeline(line):
return
else:
translated = False
# If safe, go ahead
event, dirname, filename, dstfile, end = utils.deconcat(line,
config.separator,
False)
# Item identification
dirname = utils.normalize_dir(dirname)
if event.find(",ISDIR") >= 0:
itemtype = "DIR"
filename = utils.normalize_dir(filename)
dstfile = utils.normalize_dir(dstfile)
else:
itemtype = "FILE"
event = utils.deconcat(event, ",")[0]
# Flags - by default, they are empty
flags = utils.FNORMAL
# Select sync method and skip unwanted events
# On directories, CREATE is skipped to avoid backfire from rsync
# On files, CREATE is skipped because we want to sync only
# closed/CLOSE_WRITE (ie: complete) files.
# To expand: when files are CREATED but not CLOSED, the mtime
# attribute can be 'wrong' (ie: newer) then what it should be
# Example: a file which need 60 seconds to be uploaded, will have
# a constantly-changing mtime until the upload complete, when the mtime
# will be rolled back to the original value.
# This behavior is application dependent, but we can't risk: a wrong
# mtime can led to wrong replication direction and truncated file.
if event == "CREATE":
log(utils.DEBUG2, "Skipping uninteresting event for "+filename)
return
if event.find("SELF") >= 0:
log(utils.DEBUG2, "Skipping uninteresting event for "+filename)
return
# Method selection
if event == "ATTRIB" or event == "CLOSE_WRITE" or event == "MODIFY":
method = "RSYNC"
# MOVE handling
elif event == "MOVED_FROM" or event == "MOVED_TO":
return
elif event == "MOVE":
method = "MOVE"
# DELETE and undefined method
elif event == "DELETE":
method = "DELETE"
else:
log(utils.DEBUG2, "Skipping uninteresting event for "+filename)
return
# If event if for tempfile, ignore it
if re.search(options.tempfiles, dstfile, re.I):
log(utils.DEBUG2, "Skipping event for tempfile "+dstfile)
return
else:
# If source was a tempfile but destination is a normal file, use RSYNC
if re.search(options.tempfiles, filename, re.I):
method = "RSYNC"
filename = dstfile
flags = utils.FFORCE
log(utils.DEBUG2, "Changing method from MOVE to RSYNC " +
"for tempfile " + filename)
# If event is from/to excluded files, ignore it
if (re.search(options.excludes, filename.rstrip("/"), re.I) or
re.search(options.excludes, dstfile.rstrip("/"), re.I)):
log(utils.DEBUG2, "Skipping event for excluded path "+filename)
return
# Be EXTRA CAREFUL to skip the safesuffix
if (re.search(config.safesuffix, filename.rstrip("/"), re.I) or
re.search(config.safesuffix, dstfile.rstrip("/"), re.I)):
log(utils.DEBUG2, "Skipping event for excluded path "+filename)
return
# If it was a translated line, only allow RSYNC method
if translated and not method == "RSYNC":
log(utils.DEBUG2, "Skipping non-rsync method for translated line")
return
# Construct action
entry = {'method':method, 'itemtype':itemtype, 'dir':dirname,
'file':filename, 'dstfile':dstfile, 'timestamp':time.time(),
'flags':flags}
# Rsync checks
if method == "RSYNC":
if not rsync_early_checks(entry):
return
# Move checks
if method == "MOVE":
if not move_early_checks(entry):
return
# Coalesce and append actions
try:
prev = actions.pop()
except:
prev = False
if prev:
if (method == "RSYNC" and prev['method'] == "DELETE" and
filename == prev['file']):
pass
else:
actions.append(prev)
actions.append(entry)
def parse_options():
parser = optparse.OptionParser()
parser.add_option("-r",
"--remote-host",
dest="dsthost",
help="Remote host",
action="store",
default=None)
parser.add_option("-e",
"--exclude",
dest="rsync_excludes",
help="Files to exclude",
action="append",
default=config.rsync_excludes)
parser.add_option("-x",
"--extra",
dest="extra",
help="Extra rsync options",
action="append",
default=config.rsync_extra)
parser.add_option("-d",
"--debug",
dest="debug",
help="Debug",
action="store",
default=config.debug)
parser.add_option("-l",
"--lite",
dest="lite",
help="Relaxed checks",
action="store_true",
default=False)
parser.add_option("-c",
"--checksum",
dest="checksum",
help="Compute checksum for changed files",
action="store_true",
default=False)
parser.add_option("-m",
"--modified_only",
dest="modified_only",
help="Consider only modified files, ignoring new files",
action="store_true",
default=False)
parser.add_option("-X",
"--nolinks",
dest="nolinks",
help="Exclude links from comparison",
action="store_true",
default=False)
parser.add_option("-b",
"--backup",
dest="backup",
help="Do backups",
action="store_true",
default=False)
parser.add_option("--srcroot",
dest="srcroot",
action="store",
default=None)
parser.add_option("--dstroot",
dest="dstroot",
action="store",
default=None)
(options, args) = parser.parse_args()
# Checksum or modified_only automatically disables lite check
if options.checksum or options.modified_only:
options.lite = False
# srcroot and dstroot
options.srcroot = utils.normalize_dir(args[0])
options.dstroot = utils.normalize_dir(args[1])
return (options, args)
def parse_line(line):
line = line.rstrip("\n")
# Check if it's an inotify logline
if inotifylog(line):
return
# Check for safety
if not safeline(line):
return
log(utils.DEBUG2, "Raw EVENT: " + line)
# Translate and re-check for safety
if options.translate:
translated, original, line = translate(line)
if not safeline(line):
return
else:
translated = False
# If safe, go ahead
event, dirname, filename, dstfile, end = utils.deconcat(
line, config.separator, False)
# Item identification
dirname = utils.normalize_dir(dirname)
if event.find(",ISDIR") >= 0:
itemtype = "DIR"
filename = utils.normalize_dir(filename)
dstfile = utils.normalize_dir(dstfile)
else:
itemtype = "FILE"
event = utils.deconcat(event, ",")[0]
# Flags - by default, they are empty
flags = utils.FNORMAL
# Select sync method and skip unwanted events
# On directories, CREATE is skipped to avoid backfire from rsync
# On files, CREATE is skipped because we want to sync only
# closed/CLOSE_WRITE (ie: complete) files.
# To expand: when files are CREATED but not CLOSED, the mtime
# attribute can be 'wrong' (ie: newer) then what it should be
# Example: a file which need 60 seconds to be uploaded, will have
# a constantly-changing mtime until the upload complete, when the mtime
# will be rolled back to the original value.
# This behavior is application dependent, but we can't risk: a wrong
# mtime can led to wrong replication direction and truncated file.
if event == "CREATE":
log(utils.DEBUG2, "Skipping uninteresting event for " + filename)
return
if event.find("SELF") >= 0:
log(utils.DEBUG2, "Skipping uninteresting event for " + filename)
return
# Method selection
if event == "ATTRIB" or event == "CLOSE_WRITE" or event == "MODIFY":
method = "RSYNC"
# MOVE handling
elif event == "MOVED_FROM" or event == "MOVED_TO":
return
elif event == "MOVE":
method = "MOVE"
# DELETE and undefined method
elif event == "DELETE":
method = "DELETE"
else:
log(utils.DEBUG2, "Skipping uninteresting event for " + filename)
return
# If event if for tempfile, ignore it
if re.search(options.tempfiles, dstfile, re.I):
log(utils.DEBUG2, "Skipping event for tempfile " + dstfile)
return
else:
# If source was a tempfile but destination is a normal file, use RSYNC
if re.search(options.tempfiles, filename, re.I):
method = "RSYNC"
filename = dstfile
flags = utils.FFORCE
log(
utils.DEBUG2, "Changing method from MOVE to RSYNC " +
"for tempfile " + filename)
# If event is from/to excluded files, ignore it
if (re.search(options.excludes, filename.rstrip("/"), re.I)
or re.search(options.excludes, dstfile.rstrip("/"), re.I)):
log(utils.DEBUG2, "Skipping event for excluded path " + filename)
return
# Be EXTRA CAREFUL to skip the safesuffix
if (re.search(config.safesuffix, filename.rstrip("/"), re.I)
or re.search(config.safesuffix, dstfile.rstrip("/"), re.I)):
log(utils.DEBUG2, "Skipping event for excluded path " + filename)
return
# If it was a translated line, only allow RSYNC method
if translated and not method == "RSYNC":
log(utils.DEBUG2, "Skipping non-rsync method for translated line")
return
# Construct action
entry = {
'method': method,
'itemtype': itemtype,
'dir': dirname,
'file': filename,
'dstfile': dstfile,
'timestamp': time.time(),
'flags': flags
}
# Rsync checks
if method == "RSYNC":
if not rsync_early_checks(entry):
return
# Move checks
if method == "MOVE":
if not move_early_checks(entry):
return
# Coalesce and append actions
try:
prev = actions.pop()
except:
prev = False
if prev:
if (method == "RSYNC" and prev['method'] == "DELETE"
and filename == prev['file']):
pass
else:
actions.append(prev)
actions.append(entry)
def touch(filename):
fd = open(filename, "w")
fd.close()
def create_psyncdir():
if not os.path.exists(options.psyncdir):
os.makedirs(options.psyncdir)
time.sleep(1)
# Parse options
(options, args) = parse_options()
heartfile = options.psyncdir + config.heartfile
backupdir = utils.normalize_dir(options.srcroot + config.backupdir)
# Prepare system
prepare_system()
# Launch pipe to inotify
inotify = launch_inotify()
# Read events as fast as possible
producer = threading.Thread(name="producer", target=read_inotify)
producer.daemon = True
producer.start()
# Analyze and coalesce changes
consumer = threading.Thread(name="consumer", target=dequeue)
consumer.daemon = True
consumer.start()
# Main thread
while True:
def reader(process, source="B"):
rogue = 0
while True:
# Select variables based on event source
if source == "L":
psyncdir = options.lpsyncdir
else:
psyncdir = options.rpsyncdir
# Read line
line = process.stdout.readline()
line = line.strip(" \n")
# If it is a log, print it
match = re.match("^\[(.*?)\] \[(.*?):(.*?)\] \[(.*?)\]", line)
if match:
severity = match.group(3)
line = line[len(match.group()) + 1:]
# If it is an heartbeat-related log line, take note and continue
if line.find(psyncdir + config.heartfile) >= 0:
beat_inotify(source)
log(utils.DEBUG3, source, line, 1)
else:
# Otherwise, simply print it
log(severity, source, line, 1)
continue
# If HEART, take note and continue
if line.find(psyncdir + config.heartfile) >= 0:
beat_inotify(source)
log(utils.DEBUG2, source, "heartbeat")
continue
# Check if connected
if not are_ready():
if len(line) > 0:
log(utils.ERROR, source,
"Not connected, ignoring event: " + line)
continue
# Be sure to process a good formed line
nfields = 6
match = re.match("^(RSYNC|MOVE|DELETE|NONE)", line, flags=re.I)
if not match or line.count(config.separator) != nfields:
log(utils.WARNING, source,
"Rogue line (n." + str(rogue) + "): " + line)
rogue = rogue + 1
if rogue >= 5:
return
else:
continue
else:
rogue = 0
entry = utils.deconcat(line, config.separator)
method = entry[0]
itemtype = entry[1]
parent = utils.normalize_dir(entry[2])
srcfile = entry[3]
dstfile = entry[4]
flags = entry[5]
checksum = entry[6]
# Validate checksum
computed = line[:-len(config.separator + checksum)]
computed = hashlib.md5(computed).hexdigest()
if checksum != computed:
log(utils.ERROR, source,
"Ignoring event due to invalid checksum for line: " + line)
log(utils.ERROR, source,
"Received: " + checksum + " - Computed: " + computed)
continue
else:
log(utils.DEBUG2, source, "Checksum ok. Received: " +
checksum + " - Computed: " + computed)
# Beat the heart
beat_inotify(source)
# If method is NONE, continue reading
if method == "NONE":
log(utils.INFO, source, "Ignoring event NONE for file: " + srcfile)
continue
# Parse event
log(utils.DEBUG1, source, "Read event: " + line)
# Pending checks
if method in config.pending_events:
backfired = check_pendings(source, srcfile, method)
else:
backfired = False
if backfired:
if method == "RSYNC" and config.rsync_style > 1:
log(utils.DEBUG1, source, "Ignoring backfired event "+method+
config.separator + srcfile)
continue
# If batched rsync is true, continue to the next event
if config.rsync_style == 3:
continue
# Normalize dir
if itemtype == "DIR":
srcfile = utils.normalize_dir(srcfile)
if method == "MOVE":
dstfile = utils.normalize_dir(dstfile)
# Build filelist
try:
prev = actions.pop()
# Is mergeable?
if (
# is not a symlink
not os.path.islink(srcfile) and
# source and method are same than previous
source == prev['source'] and method == prev['method'] and (
(
# method is rsync and other options are the same
prev['method'] == "RSYNC" and
prev['backfired'] == backfired and
prev['flags'] == flags
) or (
# method is delete
prev['method'] == "DELETE"
)
)
):
filelist = utils.concat(prev['filelist'], srcfile)
state['current_merges'] = state['current_merges'] + 1
else:
state['current_merges'] = 0
filelist = srcfile
actions.append(prev)
except:
state['current_merges'] = 0
filelist = srcfile
log(utils.DEBUG1, source,
"Current merges: " + str(state['current_merges']))
entry = {'source': source, 'method': method, 'itemtype': itemtype,
'filelist': filelist, 'dstfile': dstfile,
'eventid': checksum[-5:], 'backfired': backfired,
'flags': flags, 'recurse': False, 'updateonly': False}
actions.append(entry)
def unregister_dir(source, dirname):
mirror = mirror = get_mirror(source)
dirs[source].pop(utils.normalize_dir(dirname), None)
dirs[mirror].pop(utils.normalize_dir(dirname), None)
log(utils.DEBUG2, source,
"Unregistering directory from later check: " + dirname)
def register_dir(source, dirname):
dirs[source][utils.normalize_dir(dirname)] = True
log(utils.DEBUG2, source,
"Registering directory for later check: " + dirname)
(lprocess, loutput, lerror) = dosidebackup(lchanged, dst)
(rprocess, routput, rerror) = dosidebackup(rchanged, src)
if not lprocess and not rprocess:
return
print
print "Doing backups..."
print loutput
print
print routput
print "...done"
# Initial values and options parsing
error = 0
(options, args) = parse_options()
(src, dst) = (options.srcroot, options.dsthost+":"+options.dstroot)
backupdir = utils.normalize_dir(src+config.backupdir)
# Find changed files with rsync
(lcheck, loutput) = check(src, dst)
(rcheck, routput) = check(dst, src)
if lcheck or rcheck:
error = 1
quit(error)
# Parse rsync output
(lcount, lchanged, lalert) = parse_output(loutput, strip=options.checksum)
(rcount, rchanged, ralert) = parse_output(routput, strip=options.checksum)
# If checksum, do the second pass
if options.checksum:
# Only check the specified files
(lcheck, loutput) = check(src, dst, filelist=lchanged, checksum=True)
(rcheck, routput) = check(dst, src, filelist=rchanged, checksum=True)
if lcheck or rcheck:
actions.append(prev)
actions.append(entry)
def touch(filename):
fd = open(filename, "w")
fd.close()
def create_psyncdir():
if not os.path.exists(options.psyncdir):
os.makedirs(options.psyncdir)
time.sleep(1)
# Parse options
(options, args) = parse_options()
heartfile = options.psyncdir+config.heartfile
backupdir = utils.normalize_dir(options.srcroot+config.backupdir)
# Prepare system
prepare_system()
# Launch pipe to inotify
inotify = launch_inotify()
# Read events as fast as possible
producer = threading.Thread(name="producer", target=read_inotify)
producer.daemon = True
producer.start()
# Analyze and coalesce changes
consumer = threading.Thread(name="consumer", target=dequeue)
consumer.daemon = True
consumer.start()
# Main thread
while True:
(rprocess, routput, rerror) = dosidebackup(rchanged, src)
if not lprocess and not rprocess:
return
print
print "Doing backups..."
print loutput
print
print routput
print "...done"
# Initial values and options parsing
error = 0
(options, args) = parse_options()
(src, dst) = (options.srcroot, options.dsthost + ":" + options.dstroot)
backupdir = utils.normalize_dir(src + config.backupdir)
# Find changed files with rsync
(lcheck, loutput) = check(src, dst)
(rcheck, routput) = check(dst, src)
if lcheck or rcheck:
error = 1
quit(error)
# Parse rsync output
(lcount, lchanged, lalert) = parse_output(loutput, strip=options.checksum)
(rcount, rchanged, ralert) = parse_output(routput, strip=options.checksum)
# If checksum, do the second pass
if options.checksum:
# Only check the specified files
(lcheck, loutput) = check(src, dst, filelist=lchanged, checksum=True)
(rcheck, routput) = check(dst, src, filelist=rchanged, checksum=True)
if lcheck or rcheck:
