class FileEvent:
def __init__(self, eventHandler):
self.logger = logging.getLogger('FileEvent')
self.wm = WatchManager()
self.watches = dict()
# Set the flags of the events that are to be listened to
FLAGS = EventsCodes.ALL_FLAGS
self.mask = FLAGS['IN_CREATE'] | FLAGS['IN_DELETE'] | FLAGS['IN_MODIFY'] | FLAGS['IN_DELETE_SELF']
# Set-up notifier
self.notifier = ThreadedNotifier(self.wm, EventProcessor(eventHandler))
def startNotifyLoop(self):
self.notifier.start()
def stopNotifyLoop(self):
self.notifier.stop()
def addWatches(self, paths, mask=None):
added_watches = dict()
for path in paths:
added_watches.update(self.addWatch(path, mask))
return added_watches
# Also monitors all sub-directories of the given directory and automatically adds newly
# created directories to watch.
# TODO should be able to add files as well, but doesn't work atm
def addWatch(self, path, mask=None):
if mask is None:
mask = self.mask
added_watches = self.wm.add_watch(path, mask, rec=True, auto_add=True)
self.watches.update(added_watches)
return added_watches
def removeWatch(self, path):
watch_descriptor = self.wm.get_wd(path)
if watch_descriptor is not None:
result = self.wm.rm_watch(watch_descriptor, rec=True)
# Remove the no longer active watches from the current watches dictionary
for key, value in self.watches.items():
if value in result:
del self.watches[key]
else:
result = None
return result
def getWatches(self):
return self.watches
class INotifyDriver(Component):
channel = "inotify"
def __init__(self, freq=1, timeout=1, channel=channel):
super(INotifyDriver, self).__init__(channel=channel)
self._freq = freq
self._wm = WatchManager()
self._notifier = Notifier(self._wm, self._process, timeout=timeout)
def _sleep(self, rtime):
# Only consider sleeping if _freq is > 0
if self._freq > 0:
ctime = time.time()
s = self._freq - (ctime - rtime)
if s > 0:
time.sleep(s)
def __tick__(self):
self._notifier.process_events()
rtime = time.time()
if self._notifier.check_events():
self._sleep(rtime)
self._notifier.read_events()
def _process(self, event):
dir = event.dir
mask = event.mask
path = event.path
name = event.name
pathname = event.pathname
#print dir, mask, path, name, pathname
for k, v in EVENT_MAP.iteritems():
if mask & k:
e = v(name, path, pathname, dir)
c = e.name.lower()
self.push(e, c)
def add(self, path, mask=None, recursive=True):
mask = mask or MASK
self._wm.add_watch(path, mask, rec=recursive)
def remove(self, path, recursive=False):
wd = self._wm.get_wd(path)
if wd:
self._wm.rm_watch(wd, rec=recursive)
class INotifyDriver(Component):
channel = "inotify"
def __init__(self, freq=1, timeout=1, channel=channel):
super(INotifyDriver, self).__init__(channel=channel)
self._freq = freq
self._wm = WatchManager()
self._notifier = Notifier(self._wm, self._process, timeout=timeout)
def _sleep(self, rtime):
# Only consider sleeping if _freq is > 0
if self._freq > 0:
ctime = time.time()
s = self._freq - (ctime - rtime)
if s > 0:
time.sleep(s)
def __tick__(self):
self._notifier.process_events()
rtime = time.time()
if self._notifier.check_events():
self._sleep(rtime)
self._notifier.read_events()
def _process(self, event):
dir = event.dir
mask = event.mask
path = event.path
name = event.name
pathname = event.pathname
for k, v in EVENT_MAP.iteritems():
if mask & k:
e = v(name, path, pathname, dir)
c = e.name.lower()
self.push(e, c)
def add(self, path, mask=None, recursive=False):
mask = mask or MASK
self._wm.add_watch(path, mask, rec=recursive)
def remove(self, path, recursive=False):
wd = self._wm.get_wd(path)
if wd:
self._wm.rm_watch(wd, rec=recursive)
class LinuxFileSysMonitor(UFSMonitor):
"""File system monitor thread"""
def __init__(self, name=None):
super(LinuxFileSysMonitor, self).__init__(name)
self.defaultMask = IN_DELETE | IN_CREATE | IN_MODIFY | IN_MOVED_TO | IN_MOVED_FROM
self.wm = WatchManager()
self.__lock = threading.Lock()
def addWatch(self, path, mask=0):
"""Add watch for path"""
if mask == 0:
mask = self.defaultMask
super(LinuxFileSysMonitor, self).addWatch(path, mask)
self.wm.add_watch(path, mask, auto_add=True, rec=True)
def rmWatch(self, path):
"""Remove watch for path"""
super(LinuxFileSysMonitor, self).rmWatch(path)
wd = self.wm.get_wd(path)
self.wm.rm_watch(wd, rec=True)
def run(self):
"""Thread entry"""
super(LinuxFileSysMonitor, self).run()
self.notifier = Notifier(self.wm, EventHandler(None, fsMonitor = self))
while self.isRunning:
self.processMsg(1)
if self.notifier.check_events(1000):
self.notifier.read_events()
self.notifier.process_events()
def stop(self):
"""Stop watch"""
super(LinuxFileSysMonitor, self).stop()
self.notifier.stop()
class LinuxFileSysMonitor(UFSMonitor):
"""File system monitor thread"""
def __init__(self, name=None):
super(LinuxFileSysMonitor, self).__init__(name)
self.defaultMask = IN_DELETE | IN_CREATE | IN_MODIFY | IN_MOVED_TO | IN_MOVED_FROM
self.wm = WatchManager()
self.__lock = threading.Lock()
def addWatch(self, path, mask=0):
"""Add watch for path"""
if mask == 0:
mask = self.defaultMask
super(LinuxFileSysMonitor, self).addWatch(path, mask)
self.wm.add_watch(path, mask, auto_add=True, rec=True)
def rmWatch(self, path):
"""Remove watch for path"""
super(LinuxFileSysMonitor, self).rmWatch(path)
wd = self.wm.get_wd(path)
self.wm.rm_watch(wd, rec=True)
def run(self):
"""Thread entry"""
super(LinuxFileSysMonitor, self).run()
self.notifier = Notifier(self.wm, EventHandler(None, fsMonitor=self))
while self.isRunning:
self.processMsg(1)
if self.notifier.check_events(1000):
self.notifier.read_events()
self.notifier.process_events()
def stop(self):
"""Stop watch"""
super(LinuxFileSysMonitor, self).stop()
self.notifier.stop()
class FileMonitor(threading.Thread):
def __init__(self, listener):
threading.Thread.__init__(self)
self.__p = Processor(self, listener)
self.manager = WatchManager()
self.notifier = Notifier(self.manager, self.__p)
self.event = threading.Event()
self.setDaemon(True)
self.watches = []
self.__isSuspended = False
def suspend(self):
self.__isSuspended = True
def unsuspend(self):
t = threading.Thread(target=self.__unsuspend)
t.start()
def __unsuspend(self):
time.sleep(1.5)
self.__isSuspended = False
for watch in self.watches:
if not os.path.exists(watch):
_logger.debug("Removed stale watch on %s", watch)
self.watches.remove(watch)
def is_suspended(self):
return self.__isSuspended
def has_watch(self, path):
return path in self.watches
def add_watch(self, path):
_logger.debug("Adding watch for %s", path)
self.manager.add_watch(path, MASK, self.__p)
self.watches.append(path)
def remove_watch(self, path):
_logger.debug("Removing watch for %s", path)
wd = self.manager.get_wd(path)
self.manager.rm_watch(wd, True)
self.watches.remove(path)
for i in range(len(self.watches)):
try:
if self.watches[i].startswith(path):
self.watches.remove(self.watches[i])
except IndexError:
break
def run(self):
while not self.event.isSet():
self.notifier.process_events()
if self.notifier.check_events(1000):
self.notifier.read_events()
_logger.info("Shutting down file monitor")
self.notifier.stop()
def stop(self):
self.event.set()
self.join()
class FilesystemMonitor(object):
"""
FileMonitor Class keeps track of all files down a tree starting at the root
"""
def __init__(self, searcher):
self.searcher = searcher
self._thread_pool = ThreadPool(THREAD_POOL_WORKS)
# Add a watch to the root of the dir
self.watch_manager = WatchManager()
self.notifier = ThreadedNotifier(self.watch_manager, FileProcessEvent(self))
self.notifier.start()
self._build_exclude_list()
def _build_exclude_list(self):
log.info("[FileMonitor] Set Regexs for Ignore List")
self._exclude_regexs = []
# Complie Ignore list in to a list of regexs
for ignore in self.searcher.configuration.get_value("EXCLUDE_LIST"):
ignore = ignore.strip()
ignore = ignore.replace(".", "\.")
ignore = ignore.replace("*", ".*")
ignore = "^"+ignore+"$"
log.debug("[FileMonitor] Ignore Regex = %s" % ignore)
self._exclude_regexs.append(re.compile(ignore))
def change_root(self, previous_root):
self._thread_pool.clearTasks()
wd = self.watch_manager.get_wd(previous_root)
if wd:
self.watch_manager.rm_watch(wd, rec=True)
self.searcher.clear_database()
self.add_directory(self.searcher.current_root)
def add_directory(self, path):
"""
Starts a WalkDirectoryThread to add the directory
"""
basename = os.path.basename(path)
if self.validate(basename):
self.watch_manager.add_watch(path, EVENT_MASK)
self._thread_pool.queueTask(self.walk_directory, path)
def add_file(self, path, name):
"""
Add a single file to the databse
"""
if self.validate(name):
self.searcher.add_file(path, name)
def remove_file(self, path, name):
self.searcher.remove_file(path, name)
def remove_directory(self, path):
self.searcher.remove_directory(path)
def walk_directory(self, root):
"""
From a give root of a tree this method will walk through ever branch
and return a generator.
"""
if os.path.isdir(root):
names = os.listdir(root)
for name in names:
try:
file_stat = os.lstat(os.path.join(root, name))
except os.error:
continue
if stat.S_ISDIR(file_stat.st_mode):
self.add_directory(os.path.join(root, name))
else:
if not stat.S_ISLNK(file_stat.st_mode):
self.add_file(root, name)
def finish(self):
wd = self.watch_manager.get_wd(self.searcher.current_root)
self.watch_manager.rm_watch(wd, rec=True)
self.notifier.stop()
self._thread_pool.joinAll(waitForTasks=False)
def validate(self, name):
# Check to make sure the file not in the ignore list
for ignore_re in self._exclude_regexs:
if ignore_re.match(name):
log.debug("[WalkDirectoryThread] ##### Ignored %s #####", name)
return False
log.debug("[WalkDirectoryThread] # Passed %s", name)
return True
class Notify(BaseComponent):
channel = "notify"
def __init__(self, channel=channel):
super(Notify, self).__init__(channel=channel)
self._poller = None
self._wm = WatchManager()
self._notifier = Notifier(self._wm, self._on_process_events)
def _on_process_events(self, event):
dir = event.dir
mask = event.mask
path = event.path
name = event.name
pathname = event.pathname
for k, v in EVENT_MAP.items():
if mask & k:
self.fire(v(name, path, pathname, dir))
def add_path(self, path, mask=None, recursive=False):
mask = mask or MASK
self._wm.add_watch(path, mask, rec=recursive)
def remove_path(self, path, recursive=False):
wd = self._wm.get_wd(path)
if wd:
self._wm.rm_watch(wd, rec=recursive)
@handler("ready")
def _on_ready(self, component):
self._poller.addReader(self, self._notifier._fd)
@handler("registered", channel="*")
def _on_registered(self, component, manager):
if self._poller is None:
if isinstance(component, BasePoller):
self._poller = component
self.fire(ready(self))
else:
if component is not self:
return
component = findcmp(self.root, BasePoller)
if component is not None:
self._poller = component
self.fire(ready(self))
else:
self._poller = Poller().register(self)
self.fire(ready(self))
@handler("started", channel="*", priority=1)
def _on_started(self, event, component):
if self._poller is None:
self._poller = Poller().register(self)
self.fire(ready(self))
event.stop()
@handler("_read", priority=1)
def __on_read(self, fd):
self._notifier.read_events()
self._notifier.process_events()
class Notify(BaseComponent):
channel = "notify"
def __init__(self, channel=channel):
super(Notify, self).__init__(channel=channel)
self._poller = None
self._wm = WatchManager()
self._notifier = Notifier(self._wm, self._on_process_events)
def _on_process_events(self, event):
dir = event.dir
mask = event.mask
path = event.path
name = event.name
pathname = event.pathname
for k, v in EVENT_MAP.items():
if mask & k:
self.fire(v(name, path, pathname, dir))
def add_path(self, path, mask=None, recursive=False):
mask = mask or MASK
self._wm.add_watch(path, mask, rec=recursive)
def remove_path(self, path, recursive=False):
wd = self._wm.get_wd(path)
if wd:
self._wm.rm_watch(wd, rec=recursive)
@handler("ready")
def _on_ready(self, component):
self._poller.addReader(self, self._notifier._fd)
@handler("registered", channel="*")
def _on_registered(self, component, manager):
if self._poller is None:
if isinstance(component, BasePoller):
self._poller = component
self.fire(ready(self))
else:
if component is not self:
return
component = findcmp(self.root, BasePoller)
if component is not None:
self._poller = component
self.fire(ready(self))
else:
self._poller = Poller().register(self)
self.fire(ready(self))
@handler("started", channel="*", priority=1)
def _on_started(self, event, component):
if self._poller is None:
self._poller = Poller().register(self)
self.fire(ready(self))
event.stop()
@handler("_read", priority=1)
def __on_read(self, fd):
self._notifier.read_events()
self._notifier.process_events()
def process_IN_IGNORED(self, event):
if settings.debug:
print event
else:
pass
if __name__ == '__main__':
mask = EventsCodes.IN_DELETE | EventsCodes.IN_CREATE | EventsCodes.IN_MODIFY | \
EventsCodes.IN_MOVED_FROM | EventsCodes.IN_MOVED_TO
repo = subversion.Manager(settings.repo_url)
wm = WatchManager()
notifier = ThreadedNotifier(wm, EventProcessor())
notifier.start()
wm.add_watch(settings.root, mask, rec=True, auto_add=True)
wm.rm_watch(wm.get_wd(settings.root + ".svn"), rec=True)
print 'start monitoring %s' % settings.root
while True:
try:
time.sleep(0.01)
except KeyboardInterrupt:
# ...until c^c signal
print 'stop monitoring...'
notifier.stop()
break
except Exception, err:
print err
class FilesystemMonitor(object):
"""
FileMonitor Class keeps track of all files down a tree starting at the root
"""
def __init__(self, searcher):
self.searcher = searcher
self._thread_pool = ThreadPool(THREAD_POOL_WORKS)
# Add a watch to the root of the dir
self.watch_manager = WatchManager()
self.notifier = ThreadedNotifier(self.watch_manager,
FileProcessEvent(self))
self.notifier.start()
self._build_exclude_list()
def _build_exclude_list(self):
log.info("[FileMonitor] Set Regexs for Ignore List")
self._exclude_regexs = []
# Complie Ignore list in to a list of regexs
for ignore in self.searcher.configuration.exclude_list:
ignore = ignore.strip()
ignore = ignore.replace(".", "\.")
ignore = ignore.replace("*", ".*")
ignore = "^" + ignore + "$"
log.debug("[FileMonitor] Ignore Regex = %s" % ignore)
self._exclude_regexs.append(re.compile(ignore))
def change_root(self, previous_root):
self._thread_pool.clearTasks()
wd = self.watch_manager.get_wd(previous_root)
if wd:
self.watch_manager.rm_watch(wd, rec=True)
self.searcher.clear_database()
self.add_directory(self.searcher.current_root)
def add_directory(self, path):
"""
Starts a WalkDirectoryThread to add the directory
"""
basename = os.path.basename(path)
if self.validate(basename):
self.watch_manager.add_watch(path, EVENT_MASK)
self._thread_pool.queueTask(self.walk_directory, path)
def add_file(self, path, name):
"""
Add a single file to the databse
"""
if self.validate(name):
self.searcher.add_file(path, name)
def remove_file(self, path, name):
self.searcher.remove_file(path, name)
def remove_directory(self, path):
self.searcher.remove_directory(path)
def walk_directory(self, root):
"""
From a give root of a tree this method will walk through ever branch
and return a generator.
"""
if os.path.isdir(root):
names = os.listdir(root)
for name in names:
try:
file_stat = os.lstat(os.path.join(root, name))
except os.error:
continue
if stat.S_ISDIR(file_stat.st_mode):
self.add_directory(os.path.join(root, name))
else:
if not stat.S_ISLNK(file_stat.st_mode):
self.add_file(root, name)
def finish(self):
wd = self.watch_manager.get_wd(self.searcher.current_root)
self.watch_manager.rm_watch(wd, rec=True)
self.notifier.stop()
self._thread_pool.joinAll(waitForTasks=False)
def validate(self, name):
# Check to make sure the file not in the ignore list
for ignore_re in self._exclude_regexs:
if ignore_re.match(name):
log.debug("[WalkDirectoryThread] ##### Ignored %s #####", name)
return False
log.debug("[WalkDirectoryThread] # Passed %s", name)
return True
