def parse_options(): parser = optparse.OptionParser() parser.add_option("-d", "--debug", dest="debug", type="int", help="Enable debugger", action="store", default=config.debug) parser.add_option("-t", "--tempfiles", dest="tempfiles", action="store", help="Tempfile list (regex)", default=config.tempfiles) parser.add_option("-e", "--excludes", dest="excludes", action="store", help="Excluded files (regex)", default=config.excludes) parser.add_option("-k", "--kill", dest="kill", action="store_true", help="Kill other inotify processes", default=False) parser.add_option("-i", "--interval", dest="interval", action="store", type="int", help="Interval between event collection/notification", default=config.event_interval) parser.add_option("-T", "--translate", dest="translate", help="Translate/replace path element", action="store", default=config.translate) parser.add_option("--srcroot", dest="srcroot", action="store", default=None) (options, args) = parser.parse_args() # Define base dirs options.srcroot = utils.normalize_dir(options.srcroot) options.psyncdir = utils.normalize_dir(options.srcroot+config.psyncdir) return (options, args)
def parse_options(): parser = optparse.OptionParser() parser.add_option("-d", "--debug", dest="debug", type="int", help="Enable debugger", action="store", default=config.debug) parser.add_option("-r", "--remote-host", dest="dsthost", help="Remote host", action="store", default=None) parser.add_option("-t", "--tempfiles", dest="tempfiles", help="Tempfile list (regex)", action="store", default=config.tempfiles) parser.add_option("-e", "--excludes", dest="excludes", action="store", help="Excluded files (regex)", default=config.excludes) parser.add_option("--rsync-excludes", dest="rsync_excludes", help="Rsync exclusion list", action="append", default=config.rsync_excludes) parser.add_option("-x", "--extra", dest="rsync_extra", help="Extra rsync options", action="append", default=config.rsync_extra) parser.add_option("-b", "--banned", dest="banned", help="Exit immediately if BANNED program is running", action="store", default=config.banned) parser.add_option("-T", "--translate", dest="translate", help="Translate/replace path element", action="store", default=config.translate) parser.add_option("-n", "--dry-run", dest="dryrun", help="Simulate sync and log, but do nothing", action="store_true", default=config.dryrun) parser.add_option("-s", "--sync-only", dest="synconly", help="First sync only, then exit", action="store_true", default=config.first_sync_only) parser.add_option("-k", "--skip-initial-sync", dest="skipsync", help="Skip initial sync", action="store_true", default=config.skip_initial_sync) parser.add_option("-f", "--force", dest="force", action="count", help="Force delete/move commands", default=config.force) parser.add_option("--srcroot", dest="srcroot", action="store", default=None) parser.add_option("--dstroot", dest="dstroot", action="store", default=None) (options, args) = parser.parse_args() # Let rsync be more verbose based of debug level if options.debug: options.rsync_extra.append("-v") if options.debug > 1: options.rsync_extra.append("--stats") # If dryrun, increase debug level if options.dryrun: options.debug = 2 options.rsync_extra.append("-n") # Normalize directories options.srcroot = utils.normalize_dir(args[0]) options.dstroot = utils.normalize_dir(args[1]) # Define left and right private .psync dirs options.lpsyncdir = utils.normalize_dir(options.srcroot + config.psyncdir) options.rpsyncdir = utils.normalize_dir(options.dstroot + config.psyncdir) return (options, args)
def parse_options(): parser = optparse.OptionParser() parser.add_option("-d", "--debug", dest="debug", type="int", help="Enable debugger", action="store", default=config.debug) parser.add_option("-t", "--tempfiles", dest="tempfiles", action="store", help="Tempfile list (regex)", default=config.tempfiles) parser.add_option("-e", "--excludes", dest="excludes", action="store", help="Excluded files (regex)", default=config.excludes) parser.add_option("-k", "--kill", dest="kill", action="store_true", help="Kill other inotify processes", default=False) parser.add_option("-i", "--interval", dest="interval", action="store", type="int", help="Interval between event collection/notification", default=config.event_interval) parser.add_option("-T", "--translate", dest="translate", help="Translate/replace path element", action="store", default=config.translate) parser.add_option("--srcroot", dest="srcroot", action="store", default=None) (options, args) = parser.parse_args() # Define base dirs options.srcroot = utils.normalize_dir(options.srcroot) options.psyncdir = utils.normalize_dir(options.srcroot + config.psyncdir) return (options, args)
def parse_options(): parser = optparse.OptionParser() parser.add_option("-r", "--remote-host", dest="dsthost", help="Remote host", action="store", default=None) parser.add_option("-e", "--exclude", dest="rsync_excludes", help="Files to exclude", action="append", default=config.rsync_excludes) parser.add_option("-x", "--extra", dest="extra", help="Extra rsync options", action="append", default=config.rsync_extra) parser.add_option("-d", "--debug", dest="debug", help="Debug", action="store", default=config.debug) parser.add_option("-l", "--lite", dest="lite", help="Relaxed checks", action="store_true", default=False) parser.add_option("-c", "--checksum", dest="checksum", help="Compute checksum for changed files", action="store_true", default=False) parser.add_option("-m", "--modified_only", dest="modified_only", help="Consider only modified files, ignoring new files", action="store_true", default=False) parser.add_option("-X", "--nolinks", dest="nolinks", help="Exclude links from comparison", action="store_true", default=False) parser.add_option("-b", "--backup", dest="backup", help="Do backups", action="store_true", default=False) parser.add_option("--srcroot", dest="srcroot", action="store", default=None) parser.add_option("--dstroot", dest="dstroot", action="store", default=None) (options, args) = parser.parse_args() # Checksum or modified_only automatically disables lite check if options.checksum or options.modified_only: options.lite = False # srcroot and dstroot options.srcroot = utils.normalize_dir(args[0]) options.dstroot = utils.normalize_dir(args[1]) return (options, args)
def reader(process, source="B"): rogue = 0 while True: # Select variables based on event source if source == "L": psyncdir = options.lpsyncdir else: psyncdir = options.rpsyncdir # Read line line = process.stdout.readline() line = line.strip(" \n") # If it is a log, print it match = re.match("^\[(.*?)\] \[(.*?):(.*?)\] \[(.*?)\]", line) if match: severity = match.group(3) line = line[len(match.group()) + 1:] # If it is an heartbeat-related log line, take note and continue if line.find(psyncdir + config.heartfile) >= 0: beat_inotify(source) log(utils.DEBUG3, source, line, 1) else: # Otherwise, simply print it log(severity, source, line, 1) continue # If HEART, take note and continue if line.find(psyncdir + config.heartfile) >= 0: beat_inotify(source) log(utils.DEBUG2, source, "heartbeat") continue # Check if connected if not are_ready(): if len(line) > 0: log(utils.ERROR, source, "Not connected, ignoring event: " + line) continue # Be sure to process a good formed line nfields = 6 match = re.match("^(RSYNC|MOVE|DELETE|NONE)", line, flags=re.I) if not match or line.count(config.separator) != nfields: log(utils.WARNING, source, "Rogue line (n." + str(rogue) + "): " + line) rogue = rogue + 1 if rogue >= 5: return else: continue else: rogue = 0 entry = utils.deconcat(line, config.separator) method = entry[0] itemtype = entry[1] parent = utils.normalize_dir(entry[2]) srcfile = entry[3] dstfile = entry[4] flags = entry[5] checksum = entry[6] # Validate checksum computed = line[:-len(config.separator + checksum)] computed = hashlib.md5(computed).hexdigest() if checksum != computed: log(utils.ERROR, source, "Ignoring event due to invalid checksum for line: " + line) log(utils.ERROR, source, "Received: " + checksum + " - Computed: " + computed) continue else: log( utils.DEBUG2, source, "Checksum ok. Received: " + checksum + " - Computed: " + computed) # Beat the heart beat_inotify(source) # If method is NONE, continue reading if method == "NONE": log(utils.INFO, source, "Ignoring event NONE for file: " + srcfile) continue # Parse event log(utils.DEBUG1, source, "Read event: " + line) # Pending checks if method in config.pending_events: backfired = check_pendings(source, srcfile, method) else: backfired = False if backfired: if method == "RSYNC" and config.rsync_style > 1: log( utils.DEBUG1, source, "Ignoring backfired event " + method + config.separator + srcfile) continue # If batched rsync is true, continue to the next event if config.rsync_style == 3: continue # Normalize dir if itemtype == "DIR": srcfile = utils.normalize_dir(srcfile) if method == "MOVE": dstfile = utils.normalize_dir(dstfile) # Build filelist try: prev = actions.pop() # Is mergeable? if ( # is not a symlink not os.path.islink(srcfile) and # source and method are same than previous source == prev['source'] and method == prev['method'] and (( # method is rsync and other options are the same prev['method'] == "RSYNC" and prev['backfired'] == backfired and prev['flags'] == flags) or ( # method is delete prev['method'] == "DELETE"))): filelist = utils.concat(prev['filelist'], srcfile) state['current_merges'] = state['current_merges'] + 1 else: state['current_merges'] = 0 filelist = srcfile actions.append(prev) except: state['current_merges'] = 0 filelist = srcfile log(utils.DEBUG1, source, "Current merges: " + str(state['current_merges'])) entry = { 'source': source, 'method': method, 'itemtype': itemtype, 'filelist': filelist, 'dstfile': dstfile, 'eventid': checksum[-5:], 'backfired': backfired, 'flags': flags, 'recurse': False, 'updateonly': False } actions.append(entry)
def unregister_dir(source, dirname): mirror = mirror = get_mirror(source) dirs[source].pop(utils.normalize_dir(dirname), None) dirs[mirror].pop(utils.normalize_dir(dirname), None) log(utils.DEBUG2, source, "Unregistering directory from later check: " + dirname)
def register_dir(source, dirname): dirs[source][utils.normalize_dir(dirname)] = True log(utils.DEBUG2, source, "Registering directory for later check: " + dirname)
def parse_line(line): line = line.rstrip("\n") # Check if it's an inotify logline if inotifylog(line): return # Check for safety if not safeline(line): return log(utils.DEBUG2, "Raw EVENT: "+line) # Translate and re-check for safety if options.translate: translated, original, line = translate(line) if not safeline(line): return else: translated = False # If safe, go ahead event, dirname, filename, dstfile, end = utils.deconcat(line, config.separator, False) # Item identification dirname = utils.normalize_dir(dirname) if event.find(",ISDIR") >= 0: itemtype = "DIR" filename = utils.normalize_dir(filename) dstfile = utils.normalize_dir(dstfile) else: itemtype = "FILE" event = utils.deconcat(event, ",")[0] # Flags - by default, they are empty flags = utils.FNORMAL # Select sync method and skip unwanted events # On directories, CREATE is skipped to avoid backfire from rsync # On files, CREATE is skipped because we want to sync only # closed/CLOSE_WRITE (ie: complete) files. # To expand: when files are CREATED but not CLOSED, the mtime # attribute can be 'wrong' (ie: newer) then what it should be # Example: a file which need 60 seconds to be uploaded, will have # a constantly-changing mtime until the upload complete, when the mtime # will be rolled back to the original value. # This behavior is application dependent, but we can't risk: a wrong # mtime can led to wrong replication direction and truncated file. if event == "CREATE": log(utils.DEBUG2, "Skipping uninteresting event for "+filename) return if event.find("SELF") >= 0: log(utils.DEBUG2, "Skipping uninteresting event for "+filename) return # Method selection if event == "ATTRIB" or event == "CLOSE_WRITE" or event == "MODIFY": method = "RSYNC" # MOVE handling elif event == "MOVED_FROM" or event == "MOVED_TO": return elif event == "MOVE": method = "MOVE" # DELETE and undefined method elif event == "DELETE": method = "DELETE" else: log(utils.DEBUG2, "Skipping uninteresting event for "+filename) return # If event if for tempfile, ignore it if re.search(options.tempfiles, dstfile, re.I): log(utils.DEBUG2, "Skipping event for tempfile "+dstfile) return else: # If source was a tempfile but destination is a normal file, use RSYNC if re.search(options.tempfiles, filename, re.I): method = "RSYNC" filename = dstfile flags = utils.FFORCE log(utils.DEBUG2, "Changing method from MOVE to RSYNC " + "for tempfile " + filename) # If event is from/to excluded files, ignore it if (re.search(options.excludes, filename.rstrip("/"), re.I) or re.search(options.excludes, dstfile.rstrip("/"), re.I)): log(utils.DEBUG2, "Skipping event for excluded path "+filename) return # Be EXTRA CAREFUL to skip the safesuffix if (re.search(config.safesuffix, filename.rstrip("/"), re.I) or re.search(config.safesuffix, dstfile.rstrip("/"), re.I)): log(utils.DEBUG2, "Skipping event for excluded path "+filename) return # If it was a translated line, only allow RSYNC method if translated and not method == "RSYNC": log(utils.DEBUG2, "Skipping non-rsync method for translated line") return # Construct action entry = {'method':method, 'itemtype':itemtype, 'dir':dirname, 'file':filename, 'dstfile':dstfile, 'timestamp':time.time(), 'flags':flags} # Rsync checks if method == "RSYNC": if not rsync_early_checks(entry): return # Move checks if method == "MOVE": if not move_early_checks(entry): return # Coalesce and append actions try: prev = actions.pop() except: prev = False if prev: if (method == "RSYNC" and prev['method'] == "DELETE" and filename == prev['file']): pass else: actions.append(prev) actions.append(entry)
def parse_line(line): line = line.rstrip("\n") # Check if it's an inotify logline if inotifylog(line): return # Check for safety if not safeline(line): return log(utils.DEBUG2, "Raw EVENT: " + line) # Translate and re-check for safety if options.translate: translated, original, line = translate(line) if not safeline(line): return else: translated = False # If safe, go ahead event, dirname, filename, dstfile, end = utils.deconcat( line, config.separator, False) # Item identification dirname = utils.normalize_dir(dirname) if event.find(",ISDIR") >= 0: itemtype = "DIR" filename = utils.normalize_dir(filename) dstfile = utils.normalize_dir(dstfile) else: itemtype = "FILE" event = utils.deconcat(event, ",")[0] # Flags - by default, they are empty flags = utils.FNORMAL # Select sync method and skip unwanted events # On directories, CREATE is skipped to avoid backfire from rsync # On files, CREATE is skipped because we want to sync only # closed/CLOSE_WRITE (ie: complete) files. # To expand: when files are CREATED but not CLOSED, the mtime # attribute can be 'wrong' (ie: newer) then what it should be # Example: a file which need 60 seconds to be uploaded, will have # a constantly-changing mtime until the upload complete, when the mtime # will be rolled back to the original value. # This behavior is application dependent, but we can't risk: a wrong # mtime can led to wrong replication direction and truncated file. if event == "CREATE": log(utils.DEBUG2, "Skipping uninteresting event for " + filename) return if event.find("SELF") >= 0: log(utils.DEBUG2, "Skipping uninteresting event for " + filename) return # Method selection if event == "ATTRIB" or event == "CLOSE_WRITE" or event == "MODIFY": method = "RSYNC" # MOVE handling elif event == "MOVED_FROM" or event == "MOVED_TO": return elif event == "MOVE": method = "MOVE" # DELETE and undefined method elif event == "DELETE": method = "DELETE" else: log(utils.DEBUG2, "Skipping uninteresting event for " + filename) return # If event if for tempfile, ignore it if re.search(options.tempfiles, dstfile, re.I): log(utils.DEBUG2, "Skipping event for tempfile " + dstfile) return else: # If source was a tempfile but destination is a normal file, use RSYNC if re.search(options.tempfiles, filename, re.I): method = "RSYNC" filename = dstfile flags = utils.FFORCE log( utils.DEBUG2, "Changing method from MOVE to RSYNC " + "for tempfile " + filename) # If event is from/to excluded files, ignore it if (re.search(options.excludes, filename.rstrip("/"), re.I) or re.search(options.excludes, dstfile.rstrip("/"), re.I)): log(utils.DEBUG2, "Skipping event for excluded path " + filename) return # Be EXTRA CAREFUL to skip the safesuffix if (re.search(config.safesuffix, filename.rstrip("/"), re.I) or re.search(config.safesuffix, dstfile.rstrip("/"), re.I)): log(utils.DEBUG2, "Skipping event for excluded path " + filename) return # If it was a translated line, only allow RSYNC method if translated and not method == "RSYNC": log(utils.DEBUG2, "Skipping non-rsync method for translated line") return # Construct action entry = { 'method': method, 'itemtype': itemtype, 'dir': dirname, 'file': filename, 'dstfile': dstfile, 'timestamp': time.time(), 'flags': flags } # Rsync checks if method == "RSYNC": if not rsync_early_checks(entry): return # Move checks if method == "MOVE": if not move_early_checks(entry): return # Coalesce and append actions try: prev = actions.pop() except: prev = False if prev: if (method == "RSYNC" and prev['method'] == "DELETE" and filename == prev['file']): pass else: actions.append(prev) actions.append(entry)
def touch(filename): fd = open(filename, "w") fd.close() def create_psyncdir(): if not os.path.exists(options.psyncdir): os.makedirs(options.psyncdir) time.sleep(1) # Parse options (options, args) = parse_options() heartfile = options.psyncdir + config.heartfile backupdir = utils.normalize_dir(options.srcroot + config.backupdir) # Prepare system prepare_system() # Launch pipe to inotify inotify = launch_inotify() # Read events as fast as possible producer = threading.Thread(name="producer", target=read_inotify) producer.daemon = True producer.start() # Analyze and coalesce changes consumer = threading.Thread(name="consumer", target=dequeue) consumer.daemon = True consumer.start() # Main thread while True:
def reader(process, source="B"): rogue = 0 while True: # Select variables based on event source if source == "L": psyncdir = options.lpsyncdir else: psyncdir = options.rpsyncdir # Read line line = process.stdout.readline() line = line.strip(" \n") # If it is a log, print it match = re.match("^\[(.*?)\] \[(.*?):(.*?)\] \[(.*?)\]", line) if match: severity = match.group(3) line = line[len(match.group()) + 1:] # If it is an heartbeat-related log line, take note and continue if line.find(psyncdir + config.heartfile) >= 0: beat_inotify(source) log(utils.DEBUG3, source, line, 1) else: # Otherwise, simply print it log(severity, source, line, 1) continue # If HEART, take note and continue if line.find(psyncdir + config.heartfile) >= 0: beat_inotify(source) log(utils.DEBUG2, source, "heartbeat") continue # Check if connected if not are_ready(): if len(line) > 0: log(utils.ERROR, source, "Not connected, ignoring event: " + line) continue # Be sure to process a good formed line nfields = 6 match = re.match("^(RSYNC|MOVE|DELETE|NONE)", line, flags=re.I) if not match or line.count(config.separator) != nfields: log(utils.WARNING, source, "Rogue line (n." + str(rogue) + "): " + line) rogue = rogue + 1 if rogue >= 5: return else: continue else: rogue = 0 entry = utils.deconcat(line, config.separator) method = entry[0] itemtype = entry[1] parent = utils.normalize_dir(entry[2]) srcfile = entry[3] dstfile = entry[4] flags = entry[5] checksum = entry[6] # Validate checksum computed = line[:-len(config.separator + checksum)] computed = hashlib.md5(computed).hexdigest() if checksum != computed: log(utils.ERROR, source, "Ignoring event due to invalid checksum for line: " + line) log(utils.ERROR, source, "Received: " + checksum + " - Computed: " + computed) continue else: log(utils.DEBUG2, source, "Checksum ok. Received: " + checksum + " - Computed: " + computed) # Beat the heart beat_inotify(source) # If method is NONE, continue reading if method == "NONE": log(utils.INFO, source, "Ignoring event NONE for file: " + srcfile) continue # Parse event log(utils.DEBUG1, source, "Read event: " + line) # Pending checks if method in config.pending_events: backfired = check_pendings(source, srcfile, method) else: backfired = False if backfired: if method == "RSYNC" and config.rsync_style > 1: log(utils.DEBUG1, source, "Ignoring backfired event "+method+ config.separator + srcfile) continue # If batched rsync is true, continue to the next event if config.rsync_style == 3: continue # Normalize dir if itemtype == "DIR": srcfile = utils.normalize_dir(srcfile) if method == "MOVE": dstfile = utils.normalize_dir(dstfile) # Build filelist try: prev = actions.pop() # Is mergeable? if ( # is not a symlink not os.path.islink(srcfile) and # source and method are same than previous source == prev['source'] and method == prev['method'] and ( ( # method is rsync and other options are the same prev['method'] == "RSYNC" and prev['backfired'] == backfired and prev['flags'] == flags ) or ( # method is delete prev['method'] == "DELETE" ) ) ): filelist = utils.concat(prev['filelist'], srcfile) state['current_merges'] = state['current_merges'] + 1 else: state['current_merges'] = 0 filelist = srcfile actions.append(prev) except: state['current_merges'] = 0 filelist = srcfile log(utils.DEBUG1, source, "Current merges: " + str(state['current_merges'])) entry = {'source': source, 'method': method, 'itemtype': itemtype, 'filelist': filelist, 'dstfile': dstfile, 'eventid': checksum[-5:], 'backfired': backfired, 'flags': flags, 'recurse': False, 'updateonly': False} actions.append(entry)
(lprocess, loutput, lerror) = dosidebackup(lchanged, dst) (rprocess, routput, rerror) = dosidebackup(rchanged, src) if not lprocess and not rprocess: return print print "Doing backups..." print loutput print print routput print "...done" # Initial values and options parsing error = 0 (options, args) = parse_options() (src, dst) = (options.srcroot, options.dsthost+":"+options.dstroot) backupdir = utils.normalize_dir(src+config.backupdir) # Find changed files with rsync (lcheck, loutput) = check(src, dst) (rcheck, routput) = check(dst, src) if lcheck or rcheck: error = 1 quit(error) # Parse rsync output (lcount, lchanged, lalert) = parse_output(loutput, strip=options.checksum) (rcount, rchanged, ralert) = parse_output(routput, strip=options.checksum) # If checksum, do the second pass if options.checksum: # Only check the specified files (lcheck, loutput) = check(src, dst, filelist=lchanged, checksum=True) (rcheck, routput) = check(dst, src, filelist=rchanged, checksum=True) if lcheck or rcheck:
actions.append(prev) actions.append(entry) def touch(filename): fd = open(filename, "w") fd.close() def create_psyncdir(): if not os.path.exists(options.psyncdir): os.makedirs(options.psyncdir) time.sleep(1) # Parse options (options, args) = parse_options() heartfile = options.psyncdir+config.heartfile backupdir = utils.normalize_dir(options.srcroot+config.backupdir) # Prepare system prepare_system() # Launch pipe to inotify inotify = launch_inotify() # Read events as fast as possible producer = threading.Thread(name="producer", target=read_inotify) producer.daemon = True producer.start() # Analyze and coalesce changes consumer = threading.Thread(name="consumer", target=dequeue) consumer.daemon = True consumer.start() # Main thread while True:
(rprocess, routput, rerror) = dosidebackup(rchanged, src) if not lprocess and not rprocess: return print print "Doing backups..." print loutput print print routput print "...done" # Initial values and options parsing error = 0 (options, args) = parse_options() (src, dst) = (options.srcroot, options.dsthost + ":" + options.dstroot) backupdir = utils.normalize_dir(src + config.backupdir) # Find changed files with rsync (lcheck, loutput) = check(src, dst) (rcheck, routput) = check(dst, src) if lcheck or rcheck: error = 1 quit(error) # Parse rsync output (lcount, lchanged, lalert) = parse_output(loutput, strip=options.checksum) (rcount, rchanged, ralert) = parse_output(routput, strip=options.checksum) # If checksum, do the second pass if options.checksum: # Only check the specified files (lcheck, loutput) = check(src, dst, filelist=lchanged, checksum=True) (rcheck, routput) = check(dst, src, filelist=rchanged, checksum=True) if lcheck or rcheck: