class DirectoryWatcher(WatchManager):
def __init__(self):
WatchManager.__init__(self)
def start(self):
self.inotify = ThreadedNotifier(self)
self.inotify.start()
self.inotify.join()
def stop(self):
self.inotify.stop()
def add_monitor_path(self, path):
if path is None:
Logger.error("FS: unable to monitor None directory")
return False
exclude1 = "^%s/conf.Windows*"%(path)
exclude2 = "^%s/conf.Linux*"%(path)
exc_filter = ExcludeFilter([exclude1, exclude2])
try:
self.add_watch(path=path, mask=Rec.mask, proc_fun=Rec(), rec=True, auto_add=True, exclude_filter=exc_filter)
except WatchManagerError, e:
Logger.error("FS: unable to monitor directory %s, %s"%(path, str(e)))
return False
return False
class Notify():
def __init__(self):
self.wm = WatchManager()
self.pe = ProcessNotifyEvents()
self.notifier = ThreadedNotifier(self.wm, self.pe)
self.notifier.start()
self.path = None
#thread.start_new_thread(self.jobTask, (self,))
def setNotify(self, path, cbfun):
#print 'setnotify ' + path
if self.path:
self.wm.rm_watch(list(self.wdd.values()))
self.path = path
self.pe.cbfun = cbfun # ugly...
#print sys.getfilesystemencoding()
self.wdd = self.wm.add_watch(self.path,
pyinotify.IN_CREATE |
pyinotify.IN_DELETE |
pyinotify.IN_MOVED_TO |
pyinotify.IN_MOVED_FROM |
pyinotify.IN_MODIFY)
def stop(self):
if self.path:
self.wm.rm_watch(list(self.wdd.values()))
self.notifier.stop()
def notifyThread(self):
while 1:
notifier.process_events()
if notifier.check_events():
notifier.read_events()
def test_gutils_netcdf_to_erddap_watch(self):
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
# Convert ASCII data to NetCDF
processor = Netcdf2ErddapProcessor(
deployments_path=resource('slocum'),
erddap_content_path=erddap_content_path,
erddap_flag_path=erddap_flag_path
)
notifier = ThreadedNotifier(wm, processor, read_freq=5)
notifier.coalesce_events()
notifier.start()
wdd = wm.add_watch(
netcdf_path,
mask,
rec=True,
auto_add=True
)
# Wait 5 seconds for the watch to start
time.sleep(5)
orig_netcdf = resource('profile.nc')
dummy_netcdf = os.path.join(netcdf_path, 'profile.nc')
shutil.copy(orig_netcdf, dummy_netcdf)
wait_for_files(erddap_content_path, 1)
wait_for_files(erddap_flag_path, 1)
wm.rm_watch(wdd.values(), rec=True)
notifier.stop()
class DirectoryWatcher(WatchManager):
def __init__(self):
WatchManager.__init__(self)
def start(self):
self.inotify = ThreadedNotifier(self)
self.inotify.start()
self.inotify.join()
def stop(self):
self.inotify.stop()
def add_monitor_path(self, path):
if path is None:
Logger.error("FS: unable to monitor None directory")
return False
exclude1 = "^%s/conf.Windows*" % (path)
exclude2 = "^%s/conf.Linux*" % (path)
exc_filter = ExcludeFilter([exclude1, exclude2])
try:
self.add_watch(path=path,
mask=Rec.mask,
proc_fun=Rec(),
rec=True,
auto_add=True,
exclude_filter=exc_filter)
except WatchManagerError, e:
Logger.error("FS: unable to monitor directory %s, %s" %
(path, str(e)))
return False
return False
def test_gutils_binary_to_ascii_watch(self):
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
# Convert binary data to ASCII
processor = Slocum2AsciiProcessor(
deployments_path=resource('slocum'),
)
notifier = ThreadedNotifier(wm, processor)
notifier.coalesce_events()
notifier.start()
wdd = wm.add_watch(
binary_path,
mask,
rec=True,
auto_add=True
)
# Wait 5 seconds for the watch to start
time.sleep(5)
gpath = os.path.join(original_binary, '*.*bd')
# Sort the files so the .cac files are generated in the right order
for g in sorted(glob(gpath)):
shutil.copy2(g, binary_path)
wait_for_files(ascii_path, 32)
wm.rm_watch(wdd.values(), rec=True)
notifier.stop()
def test_gutils_ascii_to_netcdf_watch(self):
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
# Convert ASCII data to NetCDF
processor = Slocum2NetcdfProcessor(deployments_path=resource('slocum'),
subset=False,
template='trajectory',
profile_id_type=2,
tsint=10,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1)
notifier = ThreadedNotifier(wm, processor)
notifier.coalesce_events()
notifier.start()
wdd = wm.add_watch(ascii_path, mask, rec=True, auto_add=True)
# Wait 5 seconds for the watch to start
time.sleep(5)
# Make the ASCII we are watching for
merger = SlocumMerger(original_binary,
ascii_path,
globs=['*.tbd', '*.sbd'])
merger.convert()
wait_for_files(netcdf_path, 230)
wm.rm_watch(wdd.values(), rec=True)
notifier.stop()
class PhotoWatcher(ProcessEvent):
MASK = (EventsCodes.ALL_FLAGS['IN_DELETE'] |
EventsCodes.ALL_FLAGS['IN_CLOSE_WRITE'] |
EventsCodes.ALL_FLAGS['IN_MOVED_FROM'] |
EventsCodes.ALL_FLAGS['IN_MOVED_TO'])
def __init__(self, db, walker, root):
self.root = root
self.db = db
self.walker = walker
self.wm = WatchManager()
self.wdds = []
def Watch(self):
self.notifier = ThreadedNotifier(self.wm, self)
self.notifier.start()
self.wdds.append(self.wm.add_watch(self.root, self.MASK, rec=True))
# add soft link sub-folders
for dirname, dirnames, _filenames in os.walk(self.root, followlinks=True):
for d in dirnames:
path = os.path.join(dirname, d)
if os.path.islink(path):
self.wdds.append(
self.wm.add_watch(os.path.realpath(path), self.MASK, rec=True))
def Stop(self):
self.notifier.stop()
def process_IN_DELETE(self, event):
self.db.DeletePhoto(os.path.join(event.path, event.name))
def process_IN_MOVED_FROM(self, event):
self.process_IN_DELETE(event)
def process_IN_MOVED_TO(self, event):
full_path = os.path.join(event.path, event.name)
try:
meta = self.walker.ReadMetadata(full_path)
except Exception:
return
self.db.StorePhoto(full_path, meta)
def process_IN_CLOSE_WRITE(self, event):
full_path = os.path.join(event.path, event.name)
try:
meta = self.walker.ReadMetadata(full_path)
except Exception:
return
if self.db.HasPhoto(full_path):
self.db.UpdatePhoto(full_path, meta)
else:
self.db.StorePhoto(full_path, meta)
class FileEvent:
def __init__(self, eventHandler):
self.logger = logging.getLogger('FileEvent')
self.wm = WatchManager()
self.watches = dict()
# Set the flags of the events that are to be listened to
FLAGS = EventsCodes.ALL_FLAGS
self.mask = FLAGS['IN_CREATE'] | FLAGS['IN_DELETE'] | FLAGS['IN_MODIFY'] | FLAGS['IN_DELETE_SELF']
# Set-up notifier
self.notifier = ThreadedNotifier(self.wm, EventProcessor(eventHandler))
def startNotifyLoop(self):
self.notifier.start()
def stopNotifyLoop(self):
self.notifier.stop()
def addWatches(self, paths, mask=None):
added_watches = dict()
for path in paths:
added_watches.update(self.addWatch(path, mask))
return added_watches
# Also monitors all sub-directories of the given directory and automatically adds newly
# created directories to watch.
# TODO should be able to add files as well, but doesn't work atm
def addWatch(self, path, mask=None):
if mask is None:
mask = self.mask
added_watches = self.wm.add_watch(path, mask, rec=True, auto_add=True)
self.watches.update(added_watches)
return added_watches
def removeWatch(self, path):
watch_descriptor = self.wm.get_wd(path)
if watch_descriptor is not None:
result = self.wm.rm_watch(watch_descriptor, rec=True)
# Remove the no longer active watches from the current watches dictionary
for key, value in self.watches.items():
if value in result:
del self.watches[key]
else:
result = None
return result
def getWatches(self):
return self.watches
def main():
# Add a dummy message to the queue:
add_msg( MsgTypes.Initialise )
# Setup the HTTP server:
SocketServer.TCPServer.allow_reuse_address = True
HOST, PORT = "localhost", 9000
server = ThreadedHTTPServer((HOST, PORT), MyTCPHandler)
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
# Setup the heartbeat:
heartbeat_thread = threading.Thread(target=heartbeat_messages)
heartbeat_thread.daemon = True
heartbeat_thread.start()
# Setup pyinotify so files are being watched:
wm = WatchManager()
notifier = ThreadedNotifier(wm, PTmp())
notifier.start()
mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_ATTRIB'] |EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events
#mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events
wdd = wm.add_watch('../sentry.py', mask, rec=True)
wdd = wm.add_watch('../', mask, rec=True)
try:
while True:
time.sleep(5)
except:
# Turn off pyinotify:
notifier.stop()
print 'Exception caught: shutting down connections'
add_msg(MsgTypes.Shutdown)
time.sleep(0.5)
print 'Terminating...'
raise
class FileWatcher(Thread):
def __init__(self, holder, uri, schedule_reader):
Thread.__init__(self)
self._loop = True
self._error_event = Event()
self._wm = WatchManager()
self._holder = holder
self._uri = uri
self._schedule_reader = schedule_reader
self._notifier = ThreadedNotifier(self._wm,
_EventHandler(self._holder,
self._uri,
self._schedule_reader,
self._error_event))
self._path, self._pattern = os.path.split(urlparse(uri).path)
def start(self):
"""Start the file watcher
"""
self._notifier.start()
self._wm.add_watch(self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY)
Thread.start(self)
def run(self):
while self._loop:
if self._error_event.wait(1):
self._error_event.clear()
self._notifier.stop()
del self._notifier
self._notifier = ThreadedNotifier(
self._wm,
_EventHandler(self._holder,
self._uri,
self._schedule_reader,
self._error_event))
self._notifier.start()
self._wm.add_watch(
self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY)
def stop(self):
"""Stop the file watcher
"""
self._notifier.stop()
self._loop = False
class _PathWatcher(Queue): # iNotify watcher for directory
'''
iNotify watcher object for monitor of changes in directory.
'''
FLAGS = IN_MODIFY | IN_DELETE | IN_CREATE | IN_MOVED_FROM | IN_MOVED_TO | IN_ATTRIB
def __init__(self, path, exclude=None):
class _EH(ProcessEvent):
def process_default(self, event):
_handleEvent(event)
Queue.__init__(self)
self._path = path
self.exclude = exclude or []
_handleEvent = self.put
self._wm = WatchManager()
self._iNotifier = ThreadedNotifier(self._wm, _EH(), timeout=10)
self._iNotifier.start()
self.started = False
self._watch = []
def start(self, exclude=None):
if not self.started:
# Add watch and start watching
# Update exclude filter if it provided in call of start method
self.exclude = exclude or self.exclude
self._watch = self._wm.add_watch(self._path,
self.FLAGS,
exclude_filter=ExcludeFilter(
self.exclude),
auto_add=True,
rec=True,
do_glob=False)
self.started = True
def stop(self):
if self.started:
# Remove watch and stop watching
self._wm.rm_watch(self._watch[self._path], rec=True)
self.started = False
def exit(self):
self.stop()
self._iNotifier.stop()
def lastwatch(paths, settings, dry_run=False):
flags = EventsCodes.FLAG_COLLECTIONS.get('OP_FLAGS', None)
if flags:
mask = flags.get('IN_OPEN') | flags.get('IN_CLOSE_NOWRITE')
mask |= flags.get('IN_CREATE') | flags.get('IN_MOVED_TO')
else:
mask = EventsCodes.IN_OPEN | EventsCodes.IN_CLOSE_NOWRITE
mask |= EventsCodes.IN_CREATE | EventsCodes.IN_MOVED_TO
wm = WatchManager()
handler = Handler(settings, dry_run=dry_run)
watcher = ThreadedNotifier(wm, handler)
watcher.start()
try:
for path in paths:
path = os.path.realpath(path)
sys.stdout.write(_("Indexing %s for watching...") % path)
sys.stdout.flush()
wm.add_watch(path, mask, rec=True, auto_add=True)
sys.stdout.write(_(" done.") + "\n")
print _("You have successfully launched Lastwatch.")
print "\n".join(wrap(_("The directories you have specified will be "
"monitored as long as this process is running, "
"the flowers are blooming and the earth "
"revolves around the sun..."), 80))
# flowers to devhell ;-)
handler.set_active()
while True:
time.sleep(1)
except KeyboardInterrupt:
watcher.stop()
print _("LastWatch stopped.")
return
except Exception, err:
print err
class FilesystemWatcher:
# Mask for FS events we are interested in - currently only care about
# writable file streams that have been closed (i.e. a pdf has finished
# uploading)
mask = EventsCodes.ALL_FLAGS['IN_CLOSE_WRITE']
# This is the main watch manager provided by pyinotify
# we use this to manage subscriptions
_wm = WatchManager()
# List of watched directories
wdd = {}
# create a thread-safe queue for papers that must be processed.
paper_queue = Queue()
def __init__(self, logger):
self.logger = logger
self.notifier = ThreadedNotifier(
self._wm, PaperProcesser(self.logger, self.paper_queue))
def watch_directory(self, path):
"""set up threaded directory watcher at given path"""
self._wm.add_watch(path, self.mask, rec=True)
#add all files in the given directory to queue
for root, dirs, files in os.walk(path):
for file in files:
if file.endswith("pdf") or file.endswith("xml"):
self.logger.info("Adding %s to queue", file)
self.paper_queue.put(("QUEUE", os.path.join(root, file)))
def start(self):
self.notifier.start()
def stop(self):
self.notifier.stop()
class Watch(object): def __init__(self, path, callback, ignore_modifications=False, latency=None): self.event_mask = EVENT_MASK if ignore_modifications else EVENT_MASK_WITH_MODIFICATIONS self._dir_queue = queue.Queue() self._root = os.path.realpath(path) self._watch_manager = WatchManager() self._processor = FileProcessEvent(directory_queue=self._dir_queue, root=self._root, callback=callback, latency=latency) self._notifier = ThreadedNotifier(self._watch_manager, self._processor) self._notifier.name = "[inotify] notifier" self._notifier.daemon = True self._notifier.start() self._watch_manager.add_watch(path, self.event_mask, rec=True, auto_add=True) def stop(self): self._notifier.stop() self._processor.stop()
class FileWatcher(Thread):
def __init__(self, holder, uri, schedule_reader):
Thread.__init__(self)
self._loop = True
self._error_event = Event()
self._wm = WatchManager()
self._holder = holder
self._uri = uri
self._schedule_reader = schedule_reader
self._notifier = ThreadedNotifier(
self._wm,
_EventHandler(self._holder, self._uri, self._schedule_reader,
self._error_event))
self._path, self._pattern = os.path.split(urlparse(uri).path)
def start(self):
"""Start the file watcher
"""
self._notifier.start()
self._wm.add_watch(self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY)
Thread.start(self)
def run(self):
while self._loop:
if self._error_event.wait(1):
self._error_event.clear()
self._notifier.stop()
del self._notifier
self._notifier = ThreadedNotifier(
self._wm,
_EventHandler(self._holder, self._uri,
self._schedule_reader, self._error_event))
self._notifier.start()
self._wm.add_watch(self._path,
IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY)
def stop(self):
"""Stop the file watcher
"""
self._notifier.stop()
self._loop = False
class Watcher(object):
"""
Watching on the fly
"""
def __init__(self, gdr, workq, src, dst):
self.gdr = gdr
self.workq = workq
self.src = src
self.dst = dst
def loop(self):
wm = WatchManager()
handler = EventHandler(self.workq, self.src, self.dst)
self.notifier = ThreadedNotifier(wm, handler)
self.notifier.start()
mask = IN_CREATE | IN_MODIFY
wm.add_watch(self.src, mask, rec=self.gdr.rec)
def stop(self):
self.notifier.stop()
class Watcher(object):
"""
Watching on the fly
"""
def __init__(self, gdr, workq, src, dst):
self.gdr = gdr
self.workq = workq
self.src = src
self.dst = dst
def loop(self):
wm = WatchManager()
handler = EventHandler(self.workq, self.src, self.dst)
self.notifier = ThreadedNotifier(wm, handler)
self.notifier.start()
mask = IN_CREATE | IN_MODIFY
wm.add_watch(self.src, mask, rec=self.gdr.rec)
def stop(self):
self.notifier.stop()
def main_loop():
# Setup pyinotify so files are being watched:
wm = WatchManager()
notifier = ThreadedNotifier(wm, PTmp())
notifier.start()
mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_ATTRIB'] |EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events
#mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events
#wdd = wm.add_watch('../sentry.py', mask, rec=True)
#wdd = wm.add_watch('../', mask, rec=True)
try:
while True:
print '\rConnections to %d clients. (%s)' % (len(open_handles), time_string()),
sys.stdout.flush()
update_subscribers(msg_type=MsgTypes.Heartbeat)
time.sleep(5)
except:
notifier.stop()
raise
class Observer:
"""Monitor files and notify the main program when changes happen"""
def __init__(self, makeRunView):
self.wm = WatchManager()
self.eh = EventHandler(makeRunView)
self.notifier = ThreadedNotifier(self.wm, self.eh)
self.notifier.start()
# Watched events
self.mask = IN_DELETE | IN_CREATE | IN_CLOSE_WRITE
def kill(self):
status = self.notifier.stop()
logging.debug("Observer shut down")
return status
def addFile(self, fname):
wdd = self.wm.add_watch(fname, self.mask, rec=True)
class FSMonitorInotify(FSMonitor):
"""inotify support for FSMonitor"""
EVENTMAPPING = {
FSMonitor.CREATED: pyinotify.IN_CREATE,
FSMonitor.MODIFIED: pyinotify.IN_MODIFY | pyinotify.IN_ATTRIB,
FSMonitor.DELETED: pyinotify.IN_DELETE,
FSMonitor.MONITORED_DIR_MOVED: pyinotify.IN_MOVE_SELF,
FSMonitor.DROPPED_EVENTS: pyinotify.IN_Q_OVERFLOW,
}
def __init__(self,
callback,
persistent=False,
trigger_events_for_initial_scan=False,
ignored_dirs=[],
dbfile="fsmonitor.db"):
FSMonitor.__init__(self, callback, persistent,
trigger_events_for_initial_scan, ignored_dirs,
dbfile)
self.wm = None
self.notifier = None
def __fsmonitor_event_to_inotify_event(self, event_mask):
"""map an FSMonitor event to an inotify event"""
inotify_event_mask = 0
for fsmonitor_event_mask in self.__class__.EVENTMAPPING.keys():
if event_mask & fsmonitor_event_mask:
inotify_event_mask = inotify_event_mask | self.__class__.EVENTMAPPING[
fsmonitor_event_mask]
return inotify_event_mask
def __add_dir(self, path, event_mask):
"""override of FSMonitor.__add_dir()"""
# Perform an initial scan of the directory structure. If this has
# already been done, then it will return immediately.
if self.persistent:
if self.trigger_events_for_initial_scan:
FSMonitor.generate_missed_events(self, path, event_mask)
else:
self.pathscanner.initial_scan(path)
event_mask_inotify = self.__fsmonitor_event_to_inotify_event(
event_mask)
# Use the inotify API to monitor a directory.
wdd = self.wm.add_watch(path,
event_mask_inotify,
proc_fun=self.process_event,
rec=True,
auto_add=True)
if wdd is None:
raise MonitorError, "Could not monitor %s" % path
return None
else:
self.monitored_paths[path] = MonitoredPath(path, event_mask, wdd)
self.monitored_paths[path].monitoring = True
# Generate the missed events.
if self.persistent:
FSMonitor.generate_missed_events(self, path)
return self.monitored_paths[path]
def __remove_dir(self, path):
"""override of FSMonitor.__remove_dir()"""
if path in self.monitored_paths.keys():
wd = self.monitored_paths[path].fsmonitor_ref
# TODO: figure out why this won't work, it seems this fails due to
# a bug in pyinotify?
#self.wm.rm_watch(wd, rec=True, quiet=True)
del self.monitored_paths[path]
def run(self):
# Setup. Ensure that this isn't interleaved with any other thread, so
# that the DB setup continues as expected.
self.lock.acquire()
FSMonitor.setup(self)
self.process_event = FSMonitorInotifyProcessEvent(self)
self.lock.release()
# Set up inotify.
self.wm = WatchManager()
self.notifier = ThreadedNotifier(self.wm, self.process_event)
self.notifier.start()
while not self.die:
self.__process_queues()
time.sleep(0.5)
self.notifier.stop()
def stop(self):
"""override of FSMonitor.stop()"""
# Let the thread know it should die.
self.lock.acquire()
self.die = True
self.lock.release()
# Stop monitoring each monitored path.
for path in self.monitored_paths.keys():
self.__remove_dir(path)
def __process_queues(self):
# Process add queue.
self.lock.acquire()
if not self.add_queue.empty():
(path, event_mask) = self.add_queue.get()
self.lock.release()
self.__add_dir(path, event_mask)
else:
self.lock.release()
# Process remove queue.
self.lock.acquire()
if not self.remove_queue.empty():
path = self.add_queue.get()
self.lock.release()
self.__remove_dir(path)
else:
self.lock.release()
self._dispatch_download(file_path, file_name)
else:
print "Skipping %s" % file_name
def _process_directory(self, dir_path):
candidates = os.listdir(dir_path)
print candidates
for candidate in candidates:
if candidate.split('.')[-1].lower() in allowed_extensions:
self._dispatch_download(dir_path, candidate)
else:
print "Skipping %s" % candidate
print "SDownloader is now starting..."
wm = WatchManager()
wm.add_watch(target_dir, mask, rec=True, auto_add=True)
print "Sdownloader is now watching %s" % target_dir
notifier = ThreadedNotifier(wm, MyWatcher())
notifier.start()
while True:
try:
time.sleep(0.5)
except (Exception, KeyboardInterrupt):
notifier.stop()
break
class FilesystemMonitor(object):
"""
FileMonitor Class keeps track of all files down a tree starting at the root
"""
def __init__(self, searcher):
self.searcher = searcher
self._thread_pool = ThreadPool(THREAD_POOL_WORKS)
# Add a watch to the root of the dir
self.watch_manager = WatchManager()
self.notifier = ThreadedNotifier(self.watch_manager,
FileProcessEvent(self))
self.notifier.start()
self._build_exclude_list()
def _build_exclude_list(self):
log.info("[FileMonitor] Set Regexs for Ignore List")
self._exclude_regexs = []
# Complie Ignore list in to a list of regexs
for ignore in self.searcher.configuration.exclude_list:
ignore = ignore.strip()
ignore = ignore.replace(".", "\.")
ignore = ignore.replace("*", ".*")
ignore = "^" + ignore + "$"
log.debug("[FileMonitor] Ignore Regex = %s" % ignore)
self._exclude_regexs.append(re.compile(ignore))
def change_root(self, previous_root):
self._thread_pool.clearTasks()
wd = self.watch_manager.get_wd(previous_root)
if wd:
self.watch_manager.rm_watch(wd, rec=True)
self.searcher.clear_database()
self.add_directory(self.searcher.current_root)
def add_directory(self, path):
"""
Starts a WalkDirectoryThread to add the directory
"""
basename = os.path.basename(path)
if self.validate(basename):
self.watch_manager.add_watch(path, EVENT_MASK)
self._thread_pool.queueTask(self.walk_directory, path)
def add_file(self, path, name):
"""
Add a single file to the databse
"""
if self.validate(name):
self.searcher.add_file(path, name)
def remove_file(self, path, name):
self.searcher.remove_file(path, name)
def remove_directory(self, path):
self.searcher.remove_directory(path)
def walk_directory(self, root):
"""
From a give root of a tree this method will walk through ever branch
and return a generator.
"""
if os.path.isdir(root):
names = os.listdir(root)
for name in names:
try:
file_stat = os.lstat(os.path.join(root, name))
except os.error:
continue
if stat.S_ISDIR(file_stat.st_mode):
self.add_directory(os.path.join(root, name))
else:
if not stat.S_ISLNK(file_stat.st_mode):
self.add_file(root, name)
def finish(self):
wd = self.watch_manager.get_wd(self.searcher.current_root)
self.watch_manager.rm_watch(wd, rec=True)
self.notifier.stop()
self._thread_pool.joinAll(waitForTasks=False)
def validate(self, name):
# Check to make sure the file not in the ignore list
for ignore_re in self._exclude_regexs:
if ignore_re.match(name):
log.debug("[WalkDirectoryThread] ##### Ignored %s #####", name)
return False
log.debug("[WalkDirectoryThread] # Passed %s", name)
return True
def stop(self, *args, **kwargs):
self._default_proc_fun.stop()
ThreadedNotifier.stop(self, *args, **kwargs)
class TargetWatcher(object):
def __init__(self, configuration, builder, watch_index):
self._builder = builder
self._root = configuration.GetExpandedDir("projects", "root_dir")
self._batch_timeout = float(
configuration.Get("file_watcher", "event_batch_timeout_ms")) / 1000
self._moddef_filename = configuration.Get(
"general", "module_definition_filename")
self.wm = WatchManager()
self.watch_index = watch_index
self.watched_module_definitions = collections.defaultdict(dict)
mask = (EventsCodes.ALL_FLAGS['IN_DELETE']
| EventsCodes.ALL_FLAGS['IN_CREATE']
| EventsCodes.ALL_FLAGS['IN_MODIFY'])
handler = functools.partial(TargetWatcher.ProcessEvent, self)
self.events_queue = queue.Queue()
self.acc_thread = threading.Thread(target=functools.partial(
TargetWatcher.AccumulationThreadProc, self),
daemon=True)
self.acc_thread.start()
self.notifier = ThreadedNotifier(self.wm, handler)
self.notifier.start()
self.watch = self.wm.add_watch(self._root,
mask,
rec=True,
auto_add=True)
self.modification_handlers = []
def _GetAllModuleDefinitionsForTarget(self, target_name):
prefix = ""
module_definition_dirs = [""]
for path_element in target_name.split("/"):
if prefix:
prefix = prefix + "/" + path_element
else:
prefix = path_element
module_definition_dirs.append(prefix)
return module_definition_dirs
def AddModificationHandler(self, handler):
self.modification_handlers.append(handler)
def ProcessEventsBatch(self, batch):
modified_targets = set()
modified_module_definitions = set()
root_prefix_len = len(self._root)
for event in batch:
rel_path = event.pathname[root_prefix_len + 1:]
if rel_path.endswith(self._moddef_filename):
conf_dir = rel_path[:-len(self._moddef_filename) - 1]
modified_module_definitions.update(
set(self.watched_module_definitions[conf_dir].values()))
else:
found_targets = self.watch_index.GetMatchingTargets(rel_path)
modified_targets.update(found_targets.values())
if modified_module_definitions or modified_targets:
self.ModificationsFound(modified_module_definitions,
modified_targets)
def ModificationsFound(self, modified_module_definitions,
modified_targets):
logging.info("Files modified: module definitions %s, other %s",
modified_module_definitions, modified_targets)
for handler in self.modification_handlers:
handler(modified_module_definitions, modified_targets)
def AccumulationThreadProc(self):
event_buffer = []
while True:
try:
if event_buffer:
item = self.events_queue.get(block=True,
timeout=self._batch_timeout)
else:
item = self.events_queue.get(block=True)
event_buffer.append(item)
except queue.Empty as e:
try:
self.ProcessEventsBatch(event_buffer[:])
except e:
logging.exception("Uncaught change event processing error")
event_buffer = []
def Join(self):
self.notifier.stop()
def ProcessEvent(self, event):
self.events_queue.put(event)
def _RefreshGlobs(self, target):
watched_globs = self._builder.GetWatchableSources(target.GetName())
target_dir = os.path.dirname(target.GetName())
rel_globs = [
os.path.join(target_dir, glob_p) for glob_p in watched_globs
]
self.watch_index.LoadGlobsForTarget(target, rel_globs)
def ReloadTarget(self, target):
self._RefreshGlobs(target)
def AddTarget(self, target):
self._RefreshGlobs(target)
for prefix in self._GetAllModuleDefinitionsForTarget(target.GetName()):
self.watched_module_definitions[prefix][target.GetName()] = target
def RemoveTarget(self, target):
del self.watched[target.GetName()]
for prefix in self._GetAllModuleDefinitionsForTarget(target.GetName()):
del self.watched_module_definitions[prefix][target.GetName()]
def main(**options):
"""
Main function. It will create instances of Odoo Server, EventHandler, ThreadedNotifier and
ProcessPool.
"""
logger = logging.getLogger(PROGNAME)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s: %(message)s")
logfile = options["logfile"]
if logfile is None:
streamH = logging.StreamHandler(sys.stdout)
else:
streamH = logging.FileHandler(logfile)
streamH.setFormatter(formatter)
logger.addHandler(streamH)
logger.info("Started")
odoo = OdooInstance(
logger,
db=options["db"],
user=options["user"],
password=options["password"],
host=options["host"],
port=options["port"],
)
odoo.connect_to_odoo()
exit_code = 0
if odoo.is_connected:
count = 0
while True:
count += 1
try:
event_handler = EventHandler(
odoo=odoo,
logger=logger,
dir_to_watch=options["dir_to_watch"],
regex=options["regex"],
delimiter=options["delimiter"],
)
# start loop handler
masks = IN_CLOSE_WRITE | IN_DELETE | IN_DELETE_SELF
wm = WatchManager()
wm.add_watch(event_handler.dir_to_watch, masks)
notifier = ThreadedNotifier(wm, default_proc_fun=event_handler)
notifier.start()
pool = ProcessPool(logger, odoo)
pool.loop()
except (KeyboardInterrupt, EOFError):
logger.info("\nReceived signal to exit! Goodbye")
break
except:
logger.error("ERROR! \n%s" % traceback.print_exception(*sys.exc_info()))
finally:
if notifier:
notifier.stop()
if count >= 50:
logger.error("ERROR! Too many tries, aborting!")
break
else:
logger.critical("Can't connect to odoo")
exit_code = 1
exit(exit_code)
class FilesystemMonitor(object):
"""
FileMonitor Class keeps track of all files down a tree starting at the root
"""
def __init__(self, searcher):
self.searcher = searcher
self._thread_pool = ThreadPool(THREAD_POOL_WORKS)
# Add a watch to the root of the dir
self.watch_manager = WatchManager()
self.notifier = ThreadedNotifier(self.watch_manager, FileProcessEvent(self))
self.notifier.start()
self._build_exclude_list()
def _build_exclude_list(self):
log.info("[FileMonitor] Set Regexs for Ignore List")
self._exclude_regexs = []
# Complie Ignore list in to a list of regexs
for ignore in self.searcher.configuration.get_value("EXCLUDE_LIST"):
ignore = ignore.strip()
ignore = ignore.replace(".", "\.")
ignore = ignore.replace("*", ".*")
ignore = "^"+ignore+"$"
log.debug("[FileMonitor] Ignore Regex = %s" % ignore)
self._exclude_regexs.append(re.compile(ignore))
def change_root(self, previous_root):
self._thread_pool.clearTasks()
wd = self.watch_manager.get_wd(previous_root)
if wd:
self.watch_manager.rm_watch(wd, rec=True)
self.searcher.clear_database()
self.add_directory(self.searcher.current_root)
def add_directory(self, path):
"""
Starts a WalkDirectoryThread to add the directory
"""
basename = os.path.basename(path)
if self.validate(basename):
self.watch_manager.add_watch(path, EVENT_MASK)
self._thread_pool.queueTask(self.walk_directory, path)
def add_file(self, path, name):
"""
Add a single file to the databse
"""
if self.validate(name):
self.searcher.add_file(path, name)
def remove_file(self, path, name):
self.searcher.remove_file(path, name)
def remove_directory(self, path):
self.searcher.remove_directory(path)
def walk_directory(self, root):
"""
From a give root of a tree this method will walk through ever branch
and return a generator.
"""
if os.path.isdir(root):
names = os.listdir(root)
for name in names:
try:
file_stat = os.lstat(os.path.join(root, name))
except os.error:
continue
if stat.S_ISDIR(file_stat.st_mode):
self.add_directory(os.path.join(root, name))
else:
if not stat.S_ISLNK(file_stat.st_mode):
self.add_file(root, name)
def finish(self):
wd = self.watch_manager.get_wd(self.searcher.current_root)
self.watch_manager.rm_watch(wd, rec=True)
self.notifier.stop()
self._thread_pool.joinAll(waitForTasks=False)
def validate(self, name):
# Check to make sure the file not in the ignore list
for ignore_re in self._exclude_regexs:
if ignore_re.match(name):
log.debug("[WalkDirectoryThread] ##### Ignored %s #####", name)
return False
log.debug("[WalkDirectoryThread] # Passed %s", name)
return True
class FileInputWatcher:
'''controlling class for file monitoring'''
def __init__(self, dir_to_watch, queue):
print 'FileInputWatcher Initialized'
# SBB20090903 Adding debugging capability, not processing multiple file drops into multiple directories.
if settings.DEBUG:
print '*************Debugging On*************'
self.queue = queue
self.dir_to_watch = dir_to_watch
# make a notifier (nothing)
self.notifier1 = None
self.notifier2 = None
def monitor(self):
'''The command to start monitoring a directory or set of them.'''
print 'Monitoring Directories: %s' % self.dir_to_watch
print "Watching started at %s" % (time.asctime())
if osiswin32:
print 'Watching win32 OS'
return self.watch_win32(self.dir_to_watch)
else:
print 'Watching POSIX OS'
#if settings.DEBUG:
#print 'sending to self.watch_posix_start()'
self.watch_posix_start()
#if settings.DEBUG:
#print "It returned from self.watch_posix_start()!"
#print "self.watch_posix_start() returned with value", result
return True
def stop_monitoring(self):
'''os independent method to stop monitoring, but only posix uses it.'''
#print "self.notifier1.started", self.notifier1.__getattribute__('started')
if isinstance(self.notifier1, ThreadedNotifier):
#if settings.DEBUG:
#print "self.notifier1 is an instance"
if isinstance(self.notifier2, ThreadedNotifier):
#if settings.DEBUG:
#print "self.notifier2 is an instance"
self.watch_posix_stop()
else:
if settings.DEBUG:
print "notifiers were not instantiated, so not calling self.watch_posix_stop() again"
print 'Done Monitoring'
def watch_win32(self, dir_to_watch):
'''os-specific watch command'''
# Loop forever, listing any file changes. The WaitFor... will
# time out every half a second allowing for keyboard interrupts
# to terminate the loop.
files_added = []
old_path_contents = []
new_path_contents = []
cnt = 0
try:
while 1:
cnt += 1
#print 'Watching %s' % cnt
#old_path_contents = os.listdir(dirToWatch)
for item in dir_to_watch:
change_handle = win32file.FindFirstChangeNotification(item, 0, win32con.FILE_NOTIFY_CHANGE_FILE_NAME)
old_path_contents.append(os.listdir(item))
result = win32event.WaitForSingleObject(change_handle, 500)
# If the WaitFor... returned because of a notification (as
# opposed to timing out or some error) then look for the
# changes in the directory contents.
if result == win32con.WAIT_OBJECT_0:
#new_path_contents = os.listdir(dirToWatch)
# build the new list with all the files from all dirs
for item in dir_to_watch:
new_path_contents.append(os.listdir(item))
files_added = [f for f in new_path_contents if not f in old_path_contents]
#files_deleted = [f for f in old_path_contents if not f in new_path_contents]
if files_added:
print
print time.asctime ()
print "Added:", files_added or "Nothing"
return files_added
#print "Deleted:", files_deleted or "Nothing"
win32file.FindNextChangeNotification(change_handle)
except KeyboardInterrupt:
return []
#finally:
# win32file.FindCloseChangeNotification(change_handle)
def watch_posix_start(self):
'''os-specific command to watch'''
# test to see if we already have a notifier object, if not, make it, otherwise we are already watching a set of folders
if self.notifier1 == None and self.notifier2 == None:
try:
pyinotify.compatibility_mode()
print 'pyinotify running in compatibility mode'
except:
print 'pyinotify running in standard mode'
try:
#mask = EventsCodes.IN_CREATE |EventsCodes.IN_MOVED_TO
mask = pyinotify.ALL_EVENTS
#ECJ20100831 Reason why we have two threads: it never returns control ever to the main loop if only one thread, so no ctrl+c
#The second thread is a dummy, so it performs switching/relinquishing control
#Eventually, we want to watch many folders, so that is why we are using the ThreadedNotifier, versus the recommended Notifier.
#Even then, Notifier is still probably the way to go, but we'll use this as long as it works, because then you don't have to poll/While loop
# Thread #1
watch_manager1 = WatchManager()
self.notifier1 = ThreadedNotifier(watch_manager1, EventHandler(self.queue))
self.notifier1.start()
print 'Starting the threaded notifier on ', self.dir_to_watch
watch_manager1.add_watch(self.dir_to_watch, mask)
# Thread #2
watch_manager2 = WatchManager()
self.notifier2 = ThreadedNotifier(watch_manager2, EventHandlerDummy(self.queue))
self.notifier2.start()
#just watch any place, but don't remove this or Ctrl+C will not work
watch_manager2.add_watch(settings.BASE_PATH, mask)
if settings.DEBUG:
print "both notifiers started"
except KeyboardInterrupt:
print "Keyboard Interrupt in notifier"
self.notifier1.stop()
self.notifier2.stop()
return
except NameError:
self.notifier1.stop()
self.notifier2.stop()
return ['POSIX Watch Error']
except:
print "General exception caught within notifier while loop, stopping both notifiers now"
self.notifier1.stop()
self.notifier2.stop()
# SBB20090903 Turn on verbose mode
self.notifier1.VERBOSE = settings.DEBUG
print "returning to calling function"
return True
def watch_posix_stop(self):
'os specific call to stop monitoring'
print 'Stopping the threaded notifiers.'
self.notifier1.stop()
print "stopped self.notifier1.stop()"
self.notifier2.stop()
print "stopped self.notifier2.stop()"
return
def stop(self, *args, **kwargs):
self._default_proc_fun.stop()
ThreadedNotifier.stop(self, *args, **kwargs)
if watch:
# set up a directory watch
wm = WatchManager()
mask = pyinotify.IN_CLOSE_WRITE
notifier = ThreadedNotifier(wm, FileCloseCb())
notifier.start()
wdd = wm.add_watch(inputDir, mask, rec=False)
# process user input
run = True
print 'Type "QUIT" to exit'
while (run):
command = raw_input("> ")
if command == 'QUIT':
run = False
notifier.stop()
else:
# anonymize all files in the inputDir
for f in os.listdir(inputDir):
if f.endswith(".ds"):
while threading.active_count() > threads:
time.sleep(5)
try:
print f
#anonymizeFile(inputDir, f)
t = FS_Walk_Anonymizer(inputDir, f)
t.start()
except:
logging.debug(
"ERROR: [{}] Unable to start the anonymization thread for {}"
.format(
class ModuleWatcher(ProcessEvent):
"""Automatically reload any modules or packages as they change"""
def __init__(self, serverhandler):
self.wm = WatchManager()
self.notifier = ThreadedNotifier(self.wm, self)
self.moduleMap = {}
self.watchDescriptorMap = {}
self.serverhandler = serverhandler
self.shuttingDown = False
flags = EventsCodes.ALL_FLAGS
path, watchDescriptor = list(
self.wm.add_watch("commands/", flags["IN_CREATE"] | flags["IN_DELETE"]).items()
)[0]
self.watchDescriptorMap[path] = watchDescriptor
def _watch_file(self, file_name, modulepath, module):
"""Add a watch for a specific file, and map said file to a module name"""
file_name = os.path.realpath(file_name)
self.moduleMap[file_name] = modulepath, module
flags = EventsCodes.ALL_FLAGS
a = self.wm.add_watch(file_name, flags["IN_MODIFY"])
if a == {}:
return
path, watchDescriptor = list(a.items())[0]
self.watchDescriptorMap[path] = watchDescriptor
def watch_module(self, name):
"""Load a module, determine which files it uses, and watch them"""
if imp.is_builtin(name) in [-1, 1]:
# Pretty pointless to watch built-in modules
return
f, pathname, description = imp.find_module(name)
try:
mod = imp.load_module(name, f, pathname, description)
if f:
self._watch_file(f.name, name, mod)
else:
for root, dirs, files in os.walk(pathname):
for filename in files:
fpath = os.path.join(root, filename)
if fpath.endswith(".py"):
self._watch_file(fpath, name, mod)
finally:
if f:
f.close()
def start_watching(self):
"""Start the pyinotify watch thread"""
if self.notifier is not None:
self.notifier.start()
def stop_watching(self):
"""Stop the pyinotify watch thread"""
self.shuttingDown = True
if self.notifier is not None:
self.notifier.stop()
def process_default(self, event):
if event.maskname == "IN_IGNORED": # TODO: Watch IN_IGNORED properly.
self.process_IN_IGNORED(event)
else:
print(event)
def process_IN_CREATE(self, event):
"""A file has been created."""
if event.name.endswith(".py"):
self.watch_module("commands/" + event.name[:-3])
modpath, modname = self.moduleMap[event.pathname]
f, pathname, description = imp.find_module(modpath)
try:
module = imp.load_module(modpath, f, pathname, description)
print(
"Loaded "
+ event.pathname
+ " which provides the following command(s): "
+ ", ".join(module.getCommandNames())
)
for command in module.getCommandNames():
self.serverhandler.commandMap[command] = module
finally:
if f:
f.close()
def process_IN_DELETE(self, event):
"""A file has been deleted."""
if (event.name.endswith(".py") or event.name.endswith(".pyc")) and event.pathname in self.moduleMap:
commands = self.moduleMap[event.pathname][1].getCommandNames()
print("Unloaded " + event.name + " which provided the following command(s): " + ", ".join(commands))
del self.watchDescriptorMap[event.pathname]
for command in commands:
del self.serverhandler.commandMap[command]
del self.moduleMap[event.pathname]
def process_IN_IGNORED(self, event):
if not os.path.exists(event.path):
return
elif not self.shuttingDown:
if event.path.endswith(".py"):
modname = os.path.relpath(event.path)[:-3]
elif event.path.endswith(".pyc"):
modname = os.path.relpath(event.path)[:-4]
else:
return
try:
imp.find_module(modname)
except ImportError as ie:
print("While trying to load " + modname + ", received this error: " + ie)
return
self.process_IN_MODIFY(event)
del self.watchDescriptorMap[event.path]
del self.moduleMap[event.path]
self.watch_module(modname)
def process_IN_MODIFY(self, event):
"""A file has been changed"""
modpath, modname = self.moduleMap[event.path]
f, pathname, description = imp.find_module(modpath)
try:
module = imp.load_module(modpath, f, pathname, description)
print(
"Reloaded "
+ event.path
+ " which provides the following command(s): "
+ ", ".join(module.getCommandNames())
)
for command in module.getCommandNames():
self.serverhandler.commandMap[command] = module
finally:
if f:
f.close()
class Server(Command):
description = 'Run the calibre server in development mode conveniently'
MONOCLE_PATH = '../monocle'
def rebuild_monocole(self):
subprocess.check_call(['sprocketize', '-C', self.MONOCLE_PATH,
'-I', 'src', 'src/monocle.js'],
stdout=open('resources/content_server/read/monocle.js', 'wb'))
def launch_server(self):
print 'Starting server...\n'
with self.lock:
self.rebuild_monocole()
self.server_proc = p = subprocess.Popen(['calibre-server', '--develop'],
stderr=subprocess.STDOUT, stdout=self.server_log)
time.sleep(0.2)
if p.poll() is not None:
print 'Starting server failed'
raise SystemExit(1)
return p
def kill_server(self):
print 'Killing server...\n'
if self.server_proc is not None:
with self.lock:
if self.server_proc.poll() is None:
self.server_proc.terminate()
while self.server_proc.poll() is None:
time.sleep(0.1)
def watch(self):
if wm is not None:
self.notifier = ThreadedNotifier(wm, ProcessEvents(self))
self.notifier.start()
self.wdd = wm.add_watch(os.path.abspath('src'), mask, rec=True)
def reload_browser(self, delay=0.1):
time.sleep(delay)
try:
t = telnetlib.Telnet('localhost', 4242)
t.read_until("repl>")
t.write('BrowserReload();')
t.read_until("repl>")
t.close()
except:
print 'Failed to reload browser'
import traceback
traceback.print_exc()
def run(self, opts):
self.lock = RLock()
tdir = tempfile.gettempdir()
logf = os.path.join(tdir, 'calibre-server.log')
self.server_log = open(logf, 'ab')
self.prompt = 'Press Enter to kill/restart server. Ctrl+C to quit: '
print 'Server log available at:', logf
print
self.watch()
first = True
while True:
self.launch_server()
if not first:
self.reload_browser()
first = False
try:
raw_input(self.prompt)
except:
print
self.kill_server()
break
else:
self.kill_server()
print
if hasattr(self, 'notifier'):
self.notifier.stop()
class Repo(ProcessEvent):
def __init__(self, filename=None, config={}, **kwargs):
"""
Load a repository metadata structure
filename:
the name of a .chunker file to load, containing
either full state, or a useful subset
**kwargs:
a basic .chunker data structure
config:
optional dictionary of extra info. Keys:
username - for change log
hostname - for change log
"""
self.notifier = None
if not filename and not kwargs:
raise Exception("Repo has no initialisation data")
struct = {}
if filename and os.path.exists(filename):
try:
data = gzip.open(filename).read()
except:
data = open(filename).read()
struct.update(json.loads(data))
struct.update(kwargs)
self.config = config
self.name = struct.get("name") or os.path.basename(struct.get("root")) or os.path.splitext(os.path.basename(filename or ""))[0]
self.root = struct.get("root") or os.path.join(os.path.expanduser("~/Downloads"), self.name)
self.type = struct.get("type", "share") # static / share
self.uuid = struct.get("uuid", sha256(uuid.uuid4()))
self.key = struct.get("key", None) # for encrypting / decrypting chunks
self.peers = struct.get("peers", [])
self.files = dict([
(filename, File.from_struct(self, filename, data))
for filename, data
in struct.get("files", {}).items()
])
# if we're creating a new static chunkfile, then add our local files to the chunkfile
# should this be in start()?
if (self.type == "static" and not self.files):
self.__add_local_files()
def to_struct(self, state=False):
"""
Serialise the repository into a JSON-compatible dictionary
"""
data = {
"name": self.name,
"type": self.type,
"uuid": self.uuid,
"key": self.key,
"files": dict([
(filename, file.to_struct(state=state))
for filename, file
in self.files.items()
])
}
if state:
data.update({
"peers": [
peer.to_struct(state=state)
for peer
in self.peers
],
"root": self.root,
})
return data
def __repr__(self):
return "Repo(%r, %r, %r, %r)" % (self.type, self.uuid, self.root, self.name)
def save_state(self):
"""
Save the repository state to the default state location
(eg ~/.config/chunker/<uuid>.state on unix)
"""
self.save(get_config_path(self.uuid + ".state"), state=True, compress=True)
def remove_state(self):
p = get_config_path(self.uuid + ".state")
if os.path.exists(p):
os.unlink(p)
def save(self, filename=None, state=False, compress=False):
"""
Export the repository state (ie, write Repo.to_struct() to a JSON file)
filename:
where to save the state to
state:
whether to save active state, eg which chunks are currently downloaded
True -> useful for an app to exit and re-open on the same PC later
False -> useful for exporting the minimal amount of info to get a
new node to join the swarm
compress:
whether or not to run the data through gzip (disabling this can make
debugging easier)
"""
struct = self.to_struct(state=state)
if compress:
fp = gzip.open(filename, "w")
data = json.dumps(struct)
else:
fp = open(filename, "w")
data = json.dumps(struct, indent=4)
fp.write(data)
fp.close()
def log(self, msg):
log.info("[%s] %s" % (self.name, msg))
###################################################################
# Metadata
###################################################################
def __add_local_files(self):
for dirpath, dirnames, filenames in os.walk(self.root):
for filename in filenames:
path = os.path.join(dirpath, filename)
relpath = self.__relpath(path)
# look for
# - files that we haven't seen before
# - files with newer timestamps than our latest known version
#
# note that if a file has new content, but the timestamp is
# unchanged since we last saw it, we won't add a new version,
# but rather treat the current version as corrupt
if (
relpath not in self.files or
ts_round(os.stat(path).st_mtime) > self.files[relpath].timestamp
):
self.update(relpath, {
"versions": [{
"timestamp": ts_round(os.stat(path).st_mtime),
"chunks": None,
}]
})
for file in self.files.values():
# "not supposed to be deleted, but it is" -> it has been
# deleted while we weren't looking.
if not file.deleted and not os.path.exists(file.fullpath):
# We don't know when it was deleted, so add the deletion
# tag as just after the last modification (ie, mark that
# version as deleted, but any newer remote version takes
# precedence)
self.update(file.filename, {
"versions": [{
"timestamp": ts_round(file.timestamp + 1),
"chunks": [],
"deleted": True,
}]
})
def update(self, filename, filedata):
"""
Update the repository with new metadata for a named file
"""
file = File.from_struct(self, filename, filedata)
if file.filename not in self.files:
self.files[file.filename] = file
else:
self.files[file.filename].versions.extend(file.versions)
self.files[file.filename].versions.sort()
if file.deleted:
file.log("deleted")
if os.path.exists(file.fullpath):
os.unlink(file.fullpath)
else:
if os.path.exists(file.fullpath):
file.log("updated")
else:
with open(file.fullpath, "a"):
if file.is_complete():
# mark as finished already
os.utime(file.fullpath, (file.timestamp, file.timestamp))
else:
# mark as incomplete
os.utime(file.fullpath, (0, 0))
file.log("created")
self.save_state()
###################################################################
# Networking
###################################################################
def add_peer(self, peer):
if peer not in self.peers:
self.log("Found new peer: %r" % (peer, ))
self.peers.append(peer)
###################################################################
# Chunks
###################################################################
def get_missing_chunks(self):
"""
Get a list of missing chunks
"""
l = []
for file in self.files.values():
l.extend(file.get_missing_chunks())
return l
def get_known_chunks(self):
"""
Get a list of known chunks
"""
l = []
for file in self.files.values():
l.extend(file.get_known_chunks())
return l
def add_chunk(self, chunk_id, data):
"""
Notify the repository that a new chunk is available
(probably freshly downloaded from the network)
"""
self.log("Trying to insert chunk %s into files" % chunk_id)
for chunk in self.missing_chunks:
if chunk.id == chunk_id:
chunk.save_data(data)
def self_heal(self, known_chunks=None, missing_chunks=None):
"""
Try to use known chunks to fill in gaps
"""
if known_chunks is None:
known_chunks = self.get_known_chunks()
if missing_chunks is None:
missing_chunks = self.get_missing_chunks()
heal(known_chunks, missing_chunks)
###################################################################
# Crypto
###################################################################
def encrypt(self, data):
if self.key:
c = AES.new(self.key)
data = c.encrypt(data)
return data
def decrypt(self, data):
if self.key:
c = AES.new(self.key)
data = c.decrypt(data)
return data
###################################################################
# File system monitoring
###################################################################
def start(self):
"""
Start monitoring for file changes
"""
if self.type == "share":
self.log("Checking for files updated while we were offline")
self.__add_local_files()
self.log("Watching %s for file changes" % self.root)
watcher = WatchManager()
watcher.add_watch(self.root, ALL_EVENTS, rec=True, auto_add=True)
self.notifier = ThreadedNotifier(watcher, self)
self.notifier.daemon = True
self.notifier.start()
else:
self.log("Not watching %s for file changes" % self.root)
# self.self_heal()
def netcomms():
while True:
# select()'ing three empty lists is an error on windows
if not self.peers:
sleep(5)
continue
rs, ws, xs = select(self.peers, self.peers, [], 0)
for r in rs:
packet = r.recv()
r.last_pong = time()
for w in ws:
if w.last_ping < time() - 30 and w.last_pong < time() - 30:
data = json.dumps({"cmd": "get-status", "since": w.last_update})
w.send(data)
w.last_ping = time()
for peer in self.peers:
if peer.last_pong < time() - 300:
log.info("Peer no longer reachable - %r" % peer)
peer.last_pong = time() + 10000
# if there was nothing to do, sleep for a bit
# (if there was something to do, immediately go back for more)
if not rs:
sleep(1)
nc = Thread(target=netcomms, name="NetComms[%s]" % self.name)
nc.daemon = True
nc.start()
def stop(self):
"""
Stop monitoring for file changes
"""
if self.notifier:
self.log("No longer watching %s for file changes" % self.root)
self.notifier.stop()
self.notifier = None
def __relpath(self, path):
base = os.path.abspath(self.root)
path = os.path.abspath(path)
return path[len(base)+1:]
def process_IN_CREATE(self, event):
if os.path.isdir(event.pathname):
return
path = self.__relpath(event.pathname)
self.update(path, filedata={
"versions": [{
"timestamp": int(os.stat(event.pathname).st_mtime),
"chunks": None,
"username": self.config.get("username"),
"hostname": self.config.get("hostname"),
}]
})
# def process_IN_MODIFY(self, event):
# if os.path.isdir(event.pathname):
# return
# path = self.__relpath(event.pathname)
# self.update(path, filedata={
# "versions": [{
# "deleted": False,
# "timestamp": int(os.stat(event.pathname).st_mtime),
# "chunks": None,
# "username": self.config.get("username"),
# "hostname": self.config.get("hostname"),
# }]
# })
def process_IN_DELETE(self, event):
if os.path.isdir(event.pathname):
return
path = self.__relpath(event.pathname)
self.update(path, filedata={
"versions": [{
"deleted": True,
"timestamp": int(time()),
"chunks": [],
"username": self.config.get("username"),
"hostname": self.config.get("hostname"),
}]
})
class ExcelFileWatcher(object):
class Process(ProcessEvent):
#def __init__(self, options):
# self.regex = re.compile(options.regex)
# self.script = options.script
def __init__(self, folderpath, event_callback):
#self.regex = re.compile(options.regex)
#self.script = options.script
self.event_callback = event_callback
self.folderpath = folderpath
def process_IN_CREATE(self, event):
target = os.path.join(event.path, event.name)
print "create event happened!"
if os.path.isdir(target):
raise Reload()
def process_IN_DELETE(self, event):
print "delete event happened"
raise Reload()
def process_IN_CLOSE_WRITE(self, event):
target = os.path.join(event.path, event.name)
#print "target = ", target
#print 'event.path = ', event.path
#print 'event.name = ', event.name
self.event_callback(event.path, event.name)
#if self.regex.match(target):
# args = self.script.replace('$f', target).split()
# os.system("clear")
# sys.stdout.write("executing script: " + " ".join(args) + "\n")
# subprocess.call(args)
# sys.stdout.write("------------------------\n")
def __init__(self, folderpath, event_callback):
self.event_callback = event_callback
self.folderpath = folderpath
def start_watch_loop(self):
wm = WatchManager()
process = self.Process(self.folderpath, self.event_callback) #options)
self.notifier = ThreadedNotifier(wm, process) #Notifier(wm, process)
#notifier = Notifier(wm)
mask = IN_DELETE | IN_CREATE | IN_CLOSE_WRITE
#wdd = wm.add_watch(options.directory, mask, rec=True)
#wm.add_watch('./excelsrc', mask, rec=True)
wm.add_watch('./' + self.folderpath, mask, rec=True)
self.notifier.start()
"""
while True:
wm = WatchManager()
process = self.Process(self.folderpath, self.event_callback) #options)
notifier = ThreadedNotifier(wm, process) #Notifier(wm, process)
#notifier = Notifier(wm)
mask = IN_DELETE | IN_CREATE | IN_CLOSE_WRITE
#wdd = wm.add_watch(options.directory, mask, rec=True)
#wm.add_watch('./excelsrc', mask, rec=True)
wm.add_watch('./'+self.folderpath, mask, rec=True)
try:
while True:
print '.'
notifier.process_events()
print '+'
if notifier.check_events():
notifier.read_events()
print '-'
except Reload:
pass
except KeyboardInterrupt:
notifier.stop()
break
"""
def stop_watch_loop(self):
self.notifier.stop()
class FileWatcher(ProcessEvent):
""" FileWatcher -> Starts an INotify thread to watch a directory for
file changes.
"""
def __init__(self):
""" FileWatcher(directory) -> Watch the directory for changes.
"""
if not pyinotify:
raise Exception("pyinotify is not loaded.")
super(FileWatcher, self).__init__()
self._file_callback_dict = {}
self._watch_manager = WatchManager()
self._events_mask = EventsCodes.ALL_FLAGS['IN_MODIFY']
self._notifier = ThreadedNotifier(self._watch_manager, self)
self._notifier.setDaemon(True)
@classmethod
def check(cls):
""" Returns true if pyinotify is loaded otherwise false.
"""
return pyinotify
def is_running(self):
""" Returns a boolean indecating the state of the notifier.
"""
return self._notifier.isAlive()
def start(self):
""" Start the notifier thread.
"""
self._notifier.start()
def stop(self):
""" Stop the notifier thread.
"""
self._notifier.stop()
def add_directory(self, directory):
""" add_directory(directory) -> Add a directory to watch.
"""
dir_watch = self._watch_manager.add_watch(directory,
self._events_mask,
rec=True)
def remove_directory(self, directory):
""" remove_directory(directory) -> Remove a directory from the watch.
"""
self._watch_manager.rm_watch(directory, rec=True)
def has_file(self, filename):
""" Returns a boolean indecating if the file is being watched.
"""
return filename in self._file_callback_dict
def add_file(self, filename, callback, *user_data):
""" add_file(filename, callback, *user_data) -> Add a file to watch
with its callback and optional user_data.
"""
self._file_callback_dict[filename] = (callback, user_data)
def remove_file(self, filename):
""" remove_file(filename) -> Remove the file from the watch list.
"""
return self._file_callback_dict.pop(filename, None)
def process_IN_MODIFY(self, event):
""" Process modify events.
"""
filename = os.path.join(event.path, event.name)
callback_data = self._file_callback_dict.get(filename, ())
if callback_data:
callback = callback_data[0]
glib.idle_add(callback, callback_data[1:])
class INotifier(object):
"""
Class providing an easy wrapper for listing to INotify kernel events.
"""
Write, Delete = range(2)
def __init__(self, path):
"""
Initialisation.
@param path (str) Full path to watch.
"""
self.path = path
if os.path.isdir(self.path):
self.path_dir = self.path
self.path_file = None
else:
self.path_dir = os.path.dirname(self.path)
self.path_file = os.path.basename(self.path)
self.callbacks = {}
wm = WatchManager()
wm.add_watch(self.path_dir, pyinotify.ALL_EVENTS)
self.inotifier = ThreadedNotifier(wm, self.__processINotify)
self.inotifier.start()
def __del__(self):
"""
Destruction. Unload (i.e. stop) the inotifier.
"""
self.unload()
def unload(self):
"""
Call this to stop listening for events. Should be called on shutdown.
"""
self.inotifier.stop()
def __processINotify(self, event):
"""
Called when an INotify was received. Call the applicable callback method based on the event type.
"""
if event.mask == pyinotify.IN_DELETE or event.mask == pyinotify.IN_MOVED_FROM:
if self.path_file == None or (self.path_file and event.name == self.path_file):
for c in self.__getCallbacks(INotifier.Delete):
c(event)
elif event.mask == pyinotify.IN_CLOSE_WRITE or event.mask == pyinotify.IN_MOVED_TO:
if self.path_file == None or (self.path_file and event.name == self.path_file):
for c in self.__getCallbacks(INotifier.Write):
c(event)
def __getCallbacks(self, type):
"""
Get the current callback methods for the given type.
@param type (INotifier.Write or INotifier.Delete) Type of callback get.
"""
return self.callbacks.get(type, [])
def addCallback(self, type, callback):
"""
Add a callback function to handle an event.
@param type (INotifier.Write or INotifier.Delete) Type of callback to add. Write is called on file writes,
Delete on file deletion.
@param callback (method) Method to call, should accept one parameter: the pyinotify
event.
"""
if not type in self.callbacks:
self.callbacks[type] = set()
self.callbacks[type].add(callback)
class FileWatcher(ProcessEvent):
""" FileWatcher -> Starts an INotify thread to watch a directory for
file changes.
"""
def __init__(self):
""" FileWatcher(directory) -> Watch the directory for changes.
"""
if not pyinotify:
raise Exception("pyinotify is not loaded.")
super(FileWatcher, self).__init__()
self._file_callback_dict = {}
self._watch_manager = WatchManager()
self._events_mask = EventsCodes.ALL_FLAGS['IN_MODIFY']
self._notifier = ThreadedNotifier(self._watch_manager, self)
self._notifier.setDaemon(True)
@classmethod
def check(cls):
""" Returns true if pyinotify is loaded otherwise false.
"""
return pyinotify
def is_running(self):
""" Returns a boolean indecating the state of the notifier.
"""
return self._notifier.isAlive()
def start(self):
""" Start the notifier thread.
"""
self._notifier.start()
def stop(self):
""" Stop the notifier thread.
"""
self._notifier.stop()
def add_directory(self, directory):
""" add_directory(directory) -> Add a directory to watch.
"""
dir_watch = self._watch_manager.add_watch(directory, self._events_mask, rec=True)
def remove_directory(self, directory):
""" remove_directory(directory) -> Remove a directory from the watch.
"""
self._watch_manager.rm_watch(directory, rec=True)
def has_file(self, filename):
""" Returns a boolean indecating if the file is being watched.
"""
return filename in self._file_callback_dict
def add_file(self, filename, callback, *user_data):
""" add_file(filename, callback, *user_data) -> Add a file to watch
with its callback and optional user_data.
"""
self._file_callback_dict[filename] = (callback, user_data)
def remove_file(self, filename):
""" remove_file(filename) -> Remove the file from the watch list.
"""
return self._file_callback_dict.pop(filename, None)
def process_IN_MODIFY(self, event):
""" Process modify events.
"""
filename = os.path.join(event.path, event.name)
callback_data = self._file_callback_dict.get(filename, ())
if callback_data:
callback = callback_data[0]
glib.idle_add(callback, callback_data[1:])
class AutoLibraryUpdate ( EventPlugin ):
# Tell QL about our plugin:
PLUGIN_ID = "Automatic library update"
PLUGIN_NAME = "Automatic library update"
PLUGIN_DESC = "Keep your quodlibet library up-to-date with inotify."
PLUGIN_VERSION = "0.1"
library = None
wm = None
tn = None
event_handler = None
running = False
# Set everything up:
def __init__( self ):
from quodlibet.library import library as library
self.library = library
# enabled hook, get everything cooking:
def enabled( self ):
if not self.running :
self.wm = WatchManager()
# the event handler will do the actual event processing,
# see the ALE class below for details
self.event_handler = self.ALE( self )
# we'll use a threaded notifier flagged as a daemon thread
# so that it will stop when QL does
self.tn = ThreadedNotifier( self.wm, self.event_handler )
self.tn.daemon = True
self.tn.start()
# mask for watched events:
FLAGS=EventsCodes.ALL_FLAGS
mask = FLAGS['IN_DELETE'] | FLAGS['IN_CLOSE_WRITE']\
| FLAGS['IN_MOVED_FROM'] | FLAGS['IN_MOVED_TO']
# watch paths in scan_list:
for path in self.scan_list():
log ( 'Adding watch: for ' + path )
self.wm.add_watch( path, mask, rec=True )
self.running = True
# disable hook, stop the notifier:
def disabled( self ):
if self.running:
self.running = False
self.tn.stop()
# find list of directories to scan
def scan_list( self ):
return config.get( "settings", "scan").split(":")
# auto-library-event class, handles the events
class ALE( ProcessEvent ):
def __init__( self, alu ):
self._alu = alu
log("alu : %s" % alu )
# process close-write event ( copy, new file etc )
def process_IN_CLOSE_WRITE(self, event):
glib.idle_add( self.add_event, event )
# process moved-to event:
def process_IN_MOVED_TO( self, event ):
log( 'moved event' )
glib.idle_add( self.add_event, event )
# general we think we added something callback:
def add_event ( self, event ):
lib = self._alu.library
path = os.path.join(event.path, event.name)
if event.dir:
for path, dnames, fnames in os.walk(path):
for filename in fnames:
fullfilename = os.path.join(path, filename)
lib.add_filename(fullfilename)
else:
item = lib.add_filename ( path )
log( '%s added to library' % item )
return False
# process delete event
def process_IN_DELETE(self, event):
glib.idle_add( self.check_event, event )
# process the moved-from event:
def process_IN_MOVED_FROM( self, event ):
glib.idle_add( self.check_event, event )
# general "check to see if it's still there" callback:
def check_event( self, event ):
lib = self._alu.library
path = os.path.join(event.path, event.name)
log( 'check (%s) event for %s' % (event.path, path) )
if event.dir:
log( 'scan? %s ' % path )
path_re = re.compile('^' + path)
to_reload = []
for i in lib._contents:
if path_re.match( i ):
try:
item = lib.__getitem__( i )
except:
pass
if item:
to_reload.append(item)
for item in to_reload:
lib.reload( item )
else:
item = lib.__getitem__( path )
log( 'removing %s from library' % item )
if item:
lib.reload(item)
return False
class Root(ProcessEvent):
watchThread = None
notifier = None
scanThread = None
continueScanning = True
peers = []
def __init__(self, path):
self.path = path;
self.setStatus("???")
self.peers.append(Peer("dummy", self.path))
self.watchManager = WatchManager()
self.watchMask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE']
self.contents = {}
self.load()
self.startWatching()
self.startRescan()
self.save()
def setStatus(self, msg):
print "root: " + self.path + " status: " + msg
self.status = msg;
def save(self):
print "Saving to: ", self.getContentsFilepath()
try:
handle = file(self.getContentsFilepath(), 'w+')
handle.write(json.dumps(self.contents, sort_keys=True, indent=4))
handle.close()
except Exception as e:
print e
return self.getContentsFilepath()
def load(self):
try:
handle = file(self.getContentsFilepath(), 'r')
jsonRaw = handle.read()
jsonContents = json.loads(jsonRaw)
print "Contents of json load:" + str(len(jsonContents)) + " items"
except Exception as e:
print "Load failed:" + str(e)
return
self.contents = jsonContents
def getContentsFilepath(self):
contentsDir = os.path.join(os.path.join(self.path, ".ovress"))
if not os.path.isdir(contentsDir):
os.mkdir(contentsDir)
return os.path.join(contentsDir, "contents.json")
def startWatching(self):
if self.watchThread == None or not self.watchThread.is_alive():
self.watchThread = Thread(target = self.doWatch, name = "watchThread")
self.watchThread.start()
def doWatch(self):
print("watching path: " + str(self.path));
self.wdd = self.watchManager.add_watch(self.path, self.watchMask, rec=True)
self.notifier = ThreadedNotifier(self.watchManager, default_proc_fun=self)
self.notifier.start()
def startRescan(self):
if self.scanThread == None or not self.scanThread.is_alive():
self.scanThread = Thread(target = self.doScan, name = "scanThread")
self.scanThread.start()
def doScan(self):
dateStarted = datetime.datetime.now()
countCached = 0
countScanned = 0
countNew = 0
self.setStatus("scanning")
for dirname, subdirs, fileList in os.walk(self.path):
for filename in fileList:
absolutePath = os.path.join(dirname, filename)
relativePath = absolutePath.replace(self.path, "")
generateMetadata = True
try:
fileMetadata = self.contents[relativePath]
if (fileMetadata['refreshed'] + 3600) > time():
self.setStatus("Using cached scan of: " + relativePath)
generateMetadata = False
except KeyError:
pass
if generateMetadata:
fileMetadata = self.getFileMetadata(absolutePath)
self.contents[relativePath] = fileMetadata
countScanned = countScanned + 1
countNew = countNew + 1
sleep(0.1)
if countNew == Config.instance.saveInterval:
countNew = 0
self.save()
self.setStatus("Scanned: " + str(countScanned) + ". Cached: " + str(countCached))
for peer in self.peers:
peer.onScanFile(relativePath, fileMetadata)
if not self.continueScanning: return
self.setStatus("complete")
def getFileMetadata(self, path):
exists = False;
filesize = 0;
filetype = '?';
md5sum = ""
if os.path.exists(path):
exists = True
filesize = os.path.getsize(path)
md5sum = md5.md5(path).hexdigest()
return {
"exists": exists,
"filesize": filesize,
"type": filetype,
"md5": md5sum,
"refreshed": time()
}
def getContents(self):
return self.contents
def process_IN_CREATE(self, event):
fullPath = os.path.join(event.path, event.name)
print "Create %s" % fullPath
self.contents[fullPath] = self.getFileMetadata(fullPath)
def process_IN_DELETE(self, event):
fullPath = os.path.join(event.path, event.name)
print "Delete %s" % fullPath
self.contents[fullPath] = self.getFileMetadata(fullPath)
def stop(self):
self.continueScanning = False
if self.notifier is not None:
self.notifier.stop();
self.setStatus("Stopped");
def getPeers(self):
return self.peers
def to_JSON(self):
return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4)
class FSMonitorInotify(FSMonitor):
"""inotify support for FSMonitor"""
EVENTMAPPING = {
FSMonitor.CREATED : pyinotify.IN_CREATE,
FSMonitor.MODIFIED : pyinotify.IN_MODIFY | pyinotify.IN_ATTRIB,
FSMonitor.DELETED : pyinotify.IN_DELETE,
FSMonitor.MONITORED_DIR_MOVED : pyinotify.IN_MOVE_SELF,
FSMonitor.DROPPED_EVENTS : pyinotify.IN_Q_OVERFLOW,
}
def __init__(self, callback, persistent=False, trigger_events_for_initial_scan=False, ignored_dirs=[], dbfile="fsmonitor.db"):
FSMonitor.__init__(self, callback, persistent, trigger_events_for_initial_scan, ignored_dirs, dbfile)
self.wm = None
self.notifier = None
def __fsmonitor_event_to_inotify_event(self, event_mask):
"""map an FSMonitor event to an inotify event"""
inotify_event_mask = 0
for fsmonitor_event_mask in self.__class__.EVENTMAPPING.keys():
if event_mask & fsmonitor_event_mask:
inotify_event_mask = inotify_event_mask | self.__class__.EVENTMAPPING[fsmonitor_event_mask]
return inotify_event_mask
def __add_dir(self, path, event_mask):
"""override of FSMonitor.__add_dir()"""
# Perform an initial scan of the directory structure. If this has
# already been done, then it will return immediately.
if self.persistent:
if self.trigger_events_for_initial_scan:
FSMonitor.generate_missed_events(self, path, event_mask)
else:
self.pathscanner.initial_scan(path)
event_mask_inotify = self.__fsmonitor_event_to_inotify_event(event_mask)
# Use the inotify API to monitor a directory.
wdd = self.wm.add_watch(path, event_mask_inotify, proc_fun=self.process_event, rec=True, auto_add=True)
if wdd is None:
raise MonitorError, "Could not monitor %s" % path
return None
else:
self.monitored_paths[path] = MonitoredPath(path, event_mask, wdd)
self.monitored_paths[path].monitoring = True
# Generate the missed events.
if self.persistent:
FSMonitor.generate_missed_events(self, path)
return self.monitored_paths[path]
def __remove_dir(self, path):
"""override of FSMonitor.__remove_dir()"""
if path in self.monitored_paths.keys():
wd = self.monitored_paths[path].fsmonitor_ref
# TODO: figure out why this won't work, it seems this fails due to
# a bug in pyinotify?
#self.wm.rm_watch(wd, rec=True, quiet=True)
del self.monitored_paths[path]
def run(self):
# Setup. Ensure that this isn't interleaved with any other thread, so
# that the DB setup continues as expected.
self.lock.acquire()
FSMonitor.setup(self)
self.process_event = FSMonitorInotifyProcessEvent(self)
self.lock.release()
# Set up inotify.
self.wm = WatchManager()
self.notifier = ThreadedNotifier(self.wm, self.process_event)
self.notifier.start()
while not self.die:
self.__process_queues()
time.sleep(0.5)
self.notifier.stop()
def stop(self):
"""override of FSMonitor.stop()"""
# Let the thread know it should die.
self.lock.acquire()
self.die = True
self.lock.release()
# Stop monitoring each monitored path.
for path in self.monitored_paths.keys():
self.__remove_dir(path)
def __process_queues(self):
# Process add queue.
self.lock.acquire()
if not self.add_queue.empty():
(path, event_mask) = self.add_queue.get()
self.lock.release()
self.__add_dir(path, event_mask)
else:
self.lock.release()
# Process remove queue.
self.lock.acquire()
if not self.remove_queue.empty():
path = self.add_queue.get()
self.lock.release()
self.__remove_dir(path)
else:
self.lock.release()
def run_pipeline(conf_info):
"""
Run the pipeline and monitor status.
"""
# Fail if the run_path doesn't actually exist
if not os.path.exists(conf_info.run_path):
LOGGER.error('Run path does not exist: %s' \
% (conf_info.run_path))
return False
# Change cwd to run_path
stdout_filepath = os.path.join(conf_info.analysis_dir, 'pipeline_run_stdout.txt')
stderr_filepath = os.path.join(conf_info.analysis_dir, 'pipeline_run_stderr.txt')
# Create status object
conf_info.createStatusObject()
# Monitor file creation
wm = WatchManager()
mask = EventsCodes.IN_DELETE | EventsCodes.IN_CREATE
event = RunEvent(conf_info)
notifier = ThreadedNotifier(wm, event)
notifier.start()
wdd = wm.add_watch(conf_info.run_path, mask, rec=True)
# Log pipeline starting
LOGGER.info('STARTING PIPELINE @ %s' % (time.ctime()))
# Start the pipeline (and hide!)
#pipe = subprocess.Popen(['make',
# '-j8',
# 'recursive'],
# stdout=subprocess.PIPE,
# stderr=subprocess.PIPE)
fout = open(stdout_filepath, 'w')
ferr = open(stderr_filepath, 'w')
pipe = subprocess.Popen(['make',
'--directory=%s' % (conf_info.run_path),
'-j8',
'recursive'],
stdout=fout,
stderr=ferr)
#shell=True)
# Wait for run to finish
retcode = pipe.wait()
# Clean up
notifier.stop()
fout.close()
ferr.close()
# Process stderr
ferr = open(stderr_filepath, 'r')
run_failed_stderr = False
for line in ferr:
err_status = pipeline_stderr_handler(line, conf_info)
if err_status == RUN_FAILED:
run_failed_stderr = True
ferr.close()
# Finished file check!
print('RUN SUCCESS CHECK:')
for key, value in event.run_status_dict.items():
print(' %s: %s' % (key, value))
dstatus = event.run_status_dict
# Success or failure check
status = (retcode == 0) and \
run_failed_stderr is False and \
dstatus['firecrest'] is True and \
dstatus['bustard'] is True and \
dstatus['gerald'] is True
return status