def process_default(self, event): global FilesActionMap global logger # sanity check if event.mask not in InotifyMask: logger.warn('Got unexpected/unhandled event %d. Ignoring.' % event.mask) return # map and process individual actions FilesActionMap = read_atomic(FILES_STATUS_FILE) action = InotifyMask[event.mask] FilesActionMap[event.pathname] = (action, time.time()) write_atomic(FILES_STATUS_FILE, FilesActionMap) logger.debug('Pending monitor actions %s.' % FilesActionMap)
def check_updated(poporig): """Check if sync queue has been updated in meantime. Returns True if yes, False otherwise. """ global FilesSyncQueue global logger FilesSyncQueue = read_atomic(FILES_SYNC_FILE) popnew = FilesSyncQueue.popleft() if poporig == popnew: write_atomic(FILES_SYNC_FILE, FilesSyncQueue) else: logger.warn('Oops, left side of file sync queue %s has changed. ' 'This should never happen.' % FilesSyncQueue) return True return False
def check_updated(monitor_action, monitor_timestamp, myfile): """Check if action map has been updated in the meantime. Returns True if yes, False otherwise. """ global FilesActionMap global logger # reread status and check if there are newer changes FilesActionMap = read_atomic(FILES_STATUS_FILE) _, monitor_timestampnew = FilesActionMap[myfile] if monitor_timestampnew == monitor_timestamp: # remove from action map if there are no changes del FilesActionMap[myfile] write_atomic(FILES_STATUS_FILE, FilesActionMap) else: return True return False
def main(argv): global FilesSyncQueue global logger global foreground # parse argv parse_argv(argv, globals()) # daemonize daemonize(SYNCER_PID, foreground) # initialize logging logger = setup_logging(argv[0], CONSOLE_LOG_LEVEL, FILE_LOG_LEVEL, LOG_FORMAT, SYNCER_LOG, DATE_FORMAT) # sanity check if not os.path.isdir(WATCH_DIR): logger.critical('Watched directory %s does not exist. ' 'Bailing out.' % WATCH_DIR) sys.exit(1) # if FilesSyncQueue is nonexistant or damaged, truncate it try: FilesSyncQueue = read_atomic(FILES_SYNC_FILE) except (IOError, AttributeError, EOFError): logger.warn('Unusable file sync queue file %s. Recreating.' % FILES_SYNC_FILE) pass write_atomic(FILES_SYNC_FILE, FilesSyncQueue) # start main loop logger.debug('File sync service starting... Entering wait loop.') while True: while decisionlogic(): pass time.sleep(SLEEP_TIME)
def decisionlogic(): """Main decision/syncing loop. Returns False if no more actions to perform. """ global FilesSyncQueue global logger # reread fresh status on every run FilesSyncQueue = read_atomic(FILES_SYNC_FILE) # ignore if no actions pending if len(FilesSyncQueue) == 0: return False # pop the necessary values from sync queue poporig = FilesSyncQueue.popleft() myfile, action, myperm = poporig # check if file exists at all when resyncing and forget action if not if action == 'sync' and not os.path.exists(myfile): logger.info('Tried to sync nonexisting file %s. Ignoring.' % myfile) check_updated(poporig) return # get relative path of file and relative path of directories above _, relpath = myfile.split('%s/' % WATCH_DIR) # return values retval = None # execute pre-command on remote directory (usually mkdir) if action == 'sync': # only if synced file in remote subdirectory if relpath.find('/') != -1: relpathdir, _ = relpath.rsplit('/', 1) logger.debug('Executing pre_command: %s.' % PRE_COMMAND % relpathdir) retval = run_with_timeout(PRE_COMMAND % relpathdir, shell=True, timeout=600) logger.debug('Executing sync_command: %s.' % SYNC_COMMAND % (myfile, relpath)) retval = run_with_timeout(SYNC_COMMAND % (myfile, relpath), shell=True, timeout=3600) # most fatal error, log stdout and stderr too if retval[0] != 0 and retval[0] != -9: logger.critical('Fatal error %d when syncing remote file %s. ' 'STDOUT: %s. STDERR: %s.' % (retval[0], myfile, retval[1], retval[2])) #sys.exit(1) # remove remote file elif action == 'remove': logger.debug('Executing remove_command: %s.' % REMOVE_COMMAND % relpath) retval = run_with_timeout(REMOVE_COMMAND % relpath, shell=True, timeout=600) # make remote directory with given permissions elif action == 'make_dir': logger.debug('Executing make_dir_command: %s.' % MAKE_DIR_COMMAND % (myperm, relpath)) retval = run_with_timeout(MAKE_DIR_COMMAND % (myperm, relpath), shell=True, timeout=600) # remove remote directory elif action == 'remove_dir': logger.debug('Executing remove_dir_command: %s.' % REMOVE_DIR_COMMAND % relpath) retval = run_with_timeout(REMOVE_DIR_COMMAND % relpath, shell=True, timeout=600) # change permissions of a remote file or directory elif action == 'change_perm': logger.debug('Executing change_perm: %s.' % CHMOD_COMMAND % (myperm, relpath)) retval = run_with_timeout(CHMOD_COMMAND % (myperm, relpath), shell=True, timeout=600) if retval and retval[0] != 0: # remote command timeouted, print reasons in 1/stdout and 2/stderr # and sleep for extended time logger.warn('Remote action %s on file %s failed. Sleeping.' % (action, relpath)) time.sleep(TIMEOUT_SLEEP_TIME) return # final pop and delete after all is done check_updated(poporig) return True
def decisionlogic(): """Main decision/summing loop. Returns False if no more actions to perform. """ global FilesActionMap global FilesHashMap global FilesSyncQueue global logger # reread fresh status on every run FilesActionMap = read_atomic(FILES_STATUS_FILE) # ignore if no actions pending if len(FilesActionMap.keys()) == 0: return False # random choice to avoid checksumming the same file over and over if # it changes often myfile = random.choice(FilesActionMap.keys()) monitor_action, monitor_timestamp = FilesActionMap[myfile] # by default don't resync nor remote remove files sync_action = None myperm = None # file is freshly created or changed if monitor_action == 'changed' or monitor_action == 'created' or \ monitor_action == 'attrib': # calculate checksum try: mysha1sum = sha1sum(myfile) except (IOError, OSError): logger.info('Could not checksum file %s. Ignoring.' % myfile) check_updated(None, monitor_timestamp, myfile) return True # get permissions try: myperm = oct(stat.S_IMODE(os.stat(myfile)[stat.ST_MODE])) except (IOError, OSError): logger.info('Could not get permissions for file %s. Ignoring.' % myfile) check_updated(None, monitor_timestamp, myfile) return True # already known file if myfile in FilesHashMap: mysha1sumold, mypermold = FilesHashMap[myfile] # if checksum is different, resync is mandatory if mysha1sumold != mysha1sum: sync_action = 'sync' # else if just mode changed, change remote mode elif myperm != mypermold: sync_action = 'change_perm' # first time seen file (no checksum and no mode) else: sync_action = 'sync' FilesHashMap[myfile] = mysha1sum, myperm # deleted file elif monitor_action == 'deleted': if myfile in FilesHashMap: del FilesHashMap[myfile] sync_action = 'remove' # created directory elif monitor_action == 'created_dir': try: myperm = oct(stat.S_IMODE(os.stat(myfile)[stat.ST_MODE])) except (IOError, OSError): logger.info('Could not get permissions for directory %s. ' 'Ignoring.' % myfile) check_updated(None, monitor_timestamp, myfile) return True sync_action = 'make_dir' # deleted directory elif monitor_action == 'deleted_dir': sync_action = 'remove_dir' # permissions change for directory elif monitor_action == 'attrib_dir': # get permissions try: myperm = oct(stat.S_IMODE(os.stat(myfile)[stat.ST_MODE])) except (IOError, OSError): logger.info('Could not get permissions for directory %s. ' 'Ignoring.' % myfile) check_updated(None, monitor_timestamp, myfile) return True sync_action = 'change_perm' # write hash file.. write_atomic(FILES_HASH_FILE, FilesHashMap) logger.debug('Hash file status: %s.' % FilesHashMap) # check if file/directory has been updated in the meantime check_updated(sync_action, monitor_timestamp, myfile) # resync or remove remote files logger.debug('Pending action %s for file %s.' % (sync_action, myfile)) if sync_action: FilesSyncQueue = read_atomic(FILES_SYNC_FILE) FilesSyncQueue.append((myfile, sync_action, myperm)) write_atomic(FILES_SYNC_FILE, FilesSyncQueue) logger.debug('Pending sync queue: %s.' % FilesSyncQueue) return True
def main(argv): global FilesActionMap global FilesHashMap global FilesSyncQueue global logger global foreground # parse argv parse_argv(argv, globals()) # daemonize daemonize(SUMMER_PID, foreground) # initialize logging logger = setup_logging(argv[0], CONSOLE_LOG_LEVEL, FILE_LOG_LEVEL, LOG_FORMAT, SUMMER_LOG, DATE_FORMAT) # sanity check if not os.path.isdir(WATCH_DIR): logger.critical('Watched directory %s does not exist. ' 'Bailing out.' % WATCH_DIR) sys.exit(1) # if FilesActionMap is nonexistant or damaged, truncate it try: FilesActionMap = read_atomic(FILES_STATUS_FILE) except (IOError, AttributeError, EOFError): logger.warn('Unusable action map status file %s. Recreating.' % FILES_STATUS_FILE) pass write_atomic(FILES_STATUS_FILE, FilesActionMap) # if FilesHashMap is nonexistant or damaged, truncate it try: FilesHashMap = read_atomic(FILES_HASH_FILE) except (IOError, AttributeError, EOFError): logger.warn('Unusable hash map file %s. Recreating.' % FILES_HASH_FILE) pass write_atomic(FILES_HASH_FILE, FilesHashMap) # if FilesSyncQueue is nonexistant or damaged, truncate it try: FilesSyncQueue = read_atomic(FILES_SYNC_FILE) except (IOError, AttributeError, EOFError): logger.warn('Unusable sync queue file %s. Recreating.' % FILES_SYNC_FILE) pass write_atomic(FILES_SYNC_FILE, FilesSyncQueue) # clear non-existant files from checksum map, most probably due to # changes when monitor was inactive for path in FilesHashMap.keys(): if not os.path.exists(path): logger.warn('File %s is in hash map, but not on disk. ' 'Deleting from map and trying to delete remotely.' % path) # remove from hash file FilesHashMap = read_atomic(FILES_HASH_FILE) del FilesHashMap[path] write_atomic(FILES_HASH_FILE, FilesHashMap) # enqueue to remove remotely FilesSyncQueue = read_atomic(FILES_SYNC_FILE) FilesSyncQueue.append((path, 'remove', 0)) write_atomic(FILES_SYNC_FILE, FilesSyncQueue) # start main loop logger.debug('Checksumming service starting... Entering wait loop.') while True: while decisionlogic(): pass time.sleep(SLEEP_TIME)
def main(argv): global FilesActionMap global logger global foreground # parse argv parse_argv(argv, globals()) # daemonize daemonize(MONITOR_PID, foreground) # initialize logging logger = setup_logging(argv[0], CONSOLE_LOG_LEVEL, FILE_LOG_LEVEL, LOG_FORMAT, MONITOR_LOG, DATE_FORMAT) # sanity check if not os.path.isdir(WATCH_DIR): logger.critical('Watched directory %s does not exist. ' 'Bailing out.' % WATCH_DIR) sys.exit(1) # if FilesActionMap is nonexistant or damaged, truncate it try: FilesActionMap = read_atomic(FILES_STATUS_FILE) except (IOError, AttributeError, EOFError): logger.warn('Unusable action map status file %s. Recreating.' % FILES_STATUS_FILE) pass write_atomic(FILES_STATUS_FILE, FilesActionMap) # initial recursive walk (initial events) for root, dirs, files in os.walk(WATCH_DIR): for name in files: path = os.path.join(root, name) FilesActionMap[path] = ('created', time.time()) for name in dirs: path = os.path.join(root, name) FilesActionMap[path] = ('created_dir', time.time()) write_atomic(FILES_STATUS_FILE, FilesActionMap) logger.debug('Initial events %s. Commiting.' % FilesActionMap) # start inotify monitor watch_manager = pyinotify.WatchManager() handler = ProcessEventHandler() notifier = pyinotify.Notifier(watch_manager, default_proc_fun=handler, read_freq=SLEEP_TIME) # try coalescing events if possible try: notifier.coalesce_events() logger.debug('Successfuly enabled events coalescing. Good.') except AttributeError: pass # catch only create/delete/modify/attrib events; don't monitor # IN_MODIFY, instead use IN_CLOSE_WRITE when file has been written to # and finally closed; and monitor IN_MOVED_TO when using temporary # files for atomicity as well as IN_MOVED_FROM when file is moved from # watched path event_mask = pyinotify.IN_CREATE|pyinotify.IN_DELETE|\ pyinotify.IN_CLOSE_WRITE|pyinotify.IN_ATTRIB|\ pyinotify.IN_MOVED_TO|pyinotify.IN_MOVED_FROM|\ pyinotify.IN_ISDIR|pyinotify.IN_UNMOUNT|\ pyinotify.IN_Q_OVERFLOW watch_manager.add_watch(WATCH_DIR, event_mask, rec=True, auto_add=True) # enter loop logger.debug('Inotify handler starting... Entering notify loop.') notifier.loop()