class DirectoryWatcher(WatchManager):
def __init__(self):
WatchManager.__init__(self)
def start(self):
self.inotify = ThreadedNotifier(self)
self.inotify.start()
self.inotify.join()
def stop(self):
self.inotify.stop()
def add_monitor_path(self, path):
if path is None:
Logger.error("FS: unable to monitor None directory")
return False
exclude1 = "^%s/conf.Windows*"%(path)
exclude2 = "^%s/conf.Linux*"%(path)
exc_filter = ExcludeFilter([exclude1, exclude2])
try:
self.add_watch(path=path, mask=Rec.mask, proc_fun=Rec(), rec=True, auto_add=True, exclude_filter=exc_filter)
except WatchManagerError, e:
Logger.error("FS: unable to monitor directory %s, %s"%(path, str(e)))
return False
return False
class Notify():
def __init__(self):
self.wm = WatchManager()
self.pe = ProcessNotifyEvents()
self.notifier = ThreadedNotifier(self.wm, self.pe)
self.notifier.start()
self.path = None
#thread.start_new_thread(self.jobTask, (self,))
def setNotify(self, path, cbfun):
#print 'setnotify ' + path
if self.path:
self.wm.rm_watch(list(self.wdd.values()))
self.path = path
self.pe.cbfun = cbfun # ugly...
#print sys.getfilesystemencoding()
self.wdd = self.wm.add_watch(self.path,
pyinotify.IN_CREATE |
pyinotify.IN_DELETE |
pyinotify.IN_MOVED_TO |
pyinotify.IN_MOVED_FROM |
pyinotify.IN_MODIFY)
def stop(self):
if self.path:
self.wm.rm_watch(list(self.wdd.values()))
self.notifier.stop()
def notifyThread(self):
while 1:
notifier.process_events()
if notifier.check_events():
notifier.read_events()
def test_gutils_netcdf_to_erddap_watch(self):
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
# Convert ASCII data to NetCDF
processor = Netcdf2ErddapProcessor(
deployments_path=resource('slocum'),
erddap_content_path=erddap_content_path,
erddap_flag_path=erddap_flag_path
)
notifier = ThreadedNotifier(wm, processor, read_freq=5)
notifier.coalesce_events()
notifier.start()
wdd = wm.add_watch(
netcdf_path,
mask,
rec=True,
auto_add=True
)
# Wait 5 seconds for the watch to start
time.sleep(5)
orig_netcdf = resource('profile.nc')
dummy_netcdf = os.path.join(netcdf_path, 'profile.nc')
shutil.copy(orig_netcdf, dummy_netcdf)
wait_for_files(erddap_content_path, 1)
wait_for_files(erddap_flag_path, 1)
wm.rm_watch(wdd.values(), rec=True)
notifier.stop()
def test_gutils_ascii_to_netcdf_watch(self):
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
# Convert ASCII data to NetCDF
processor = Slocum2NetcdfProcessor(deployments_path=resource('slocum'),
subset=False,
template='trajectory',
profile_id_type=2,
tsint=10,
filter_distance=1,
filter_points=5,
filter_time=10,
filter_z=1)
notifier = ThreadedNotifier(wm, processor)
notifier.coalesce_events()
notifier.start()
wdd = wm.add_watch(ascii_path, mask, rec=True, auto_add=True)
# Wait 5 seconds for the watch to start
time.sleep(5)
# Make the ASCII we are watching for
merger = SlocumMerger(original_binary,
ascii_path,
globs=['*.tbd', '*.sbd'])
merger.convert()
wait_for_files(netcdf_path, 230)
wm.rm_watch(wdd.values(), rec=True)
notifier.stop()
def _watch(self):
wm = WatchManager()
wm2 = WatchManager()
clusterNotifier = ThreadedNotifier(wm, \
ClustersDefinitionsChangeHandler(\
masterCallback=self.callback))
suiteNotifier = ThreadedNotifier(wm2, \
SuiteDefinitionsChangeHandler(\
masterCallback=self.callback))
clusterNotifier.start()
suiteNotifier.start()
local_path = ''
if self.config.has_option(self.repo, 'local_path'):
local_path = self.config.get(self.repo, 'local_path')
else:
LOGGER.error('No local path defined for repository %s' % self.repo)
if not self.config.has_option(self.repo, 'cluster_defs_path'):
clustdir = local_path + os.sep + 'clusters'
else:
clustdir = local_path + os.sep + self.config.get(self.repo, 'cluster_defs_path')
if not self.config.has_option(self.repo, 'suite_defs_path'):
suitedir = local_path + os.sep + 'test-suites'
else:
suitedir = local_path + os.sep + self.config.get(self.repo, 'suite_defs_path')
try:
wm.add_watch(clustdir, self.mask, rec=True, quiet=False)
wm2.add_watch(suitedir, self.mask, rec=True, quiet=False)
except WatchManagerError, e:
LOGGER.error(e)
def start_watching():
class OnEvent(ProcessEvent):
def __init__(self):
ProcessEvent(self)
self.timer = None
def process_default(self, event):
accepted = [
pyinotify.IN_MODIFY, pyinotify.IN_MOVE_SELF,
pyinotify.IN_MOVED_FROM, pyinotify.IN_MOVED_TO
]
x = [x for x in accepted if event.mask & x == event.mask]
if len(x) > 0:
print "default: %-20s %s" % (os.path.join(
event.path, event.name), event.maskname)
if self.timer:
self.timer.cancel()
self.timer = None
self.timer = threading.Timer(0.2, do_post)
self.timer.start()
wm = WatchManager()
mask = pyinotify.ALL_EVENTS
proc = OnEvent()
notifier = ThreadedNotifier(wm, proc)
notifier.start()
wdd = wm.add_watch('.git', mask, rec=True, auto_add=True)
class DirectoryWatcher(WatchManager):
def __init__(self):
WatchManager.__init__(self)
def start(self):
self.inotify = ThreadedNotifier(self)
self.inotify.start()
self.inotify.join()
def stop(self):
self.inotify.stop()
def add_monitor_path(self, path):
if path is None:
Logger.error("FS: unable to monitor None directory")
return False
exclude1 = "^%s/conf.Windows*" % (path)
exclude2 = "^%s/conf.Linux*" % (path)
exc_filter = ExcludeFilter([exclude1, exclude2])
try:
self.add_watch(path=path,
mask=Rec.mask,
proc_fun=Rec(),
rec=True,
auto_add=True,
exclude_filter=exc_filter)
except WatchManagerError, e:
Logger.error("FS: unable to monitor directory %s, %s" %
(path, str(e)))
return False
return False
def test_gutils_binary_to_ascii_watch(self):
wm = WatchManager()
mask = IN_MOVED_TO | IN_CLOSE_WRITE
# Convert binary data to ASCII
processor = Slocum2AsciiProcessor(
deployments_path=resource('slocum'),
)
notifier = ThreadedNotifier(wm, processor)
notifier.coalesce_events()
notifier.start()
wdd = wm.add_watch(
binary_path,
mask,
rec=True,
auto_add=True
)
# Wait 5 seconds for the watch to start
time.sleep(5)
gpath = os.path.join(original_binary, '*.*bd')
# Sort the files so the .cac files are generated in the right order
for g in sorted(glob(gpath)):
shutil.copy2(g, binary_path)
wait_for_files(ascii_path, 32)
wm.rm_watch(wdd.values(), rec=True)
notifier.stop()
def watch(self): mask = pyinotify.IN_MODIFY | pyinotify.IN_CREATE | pyinotify.IN_MOVED_TO self.__wm = WatchManager() self.__wm.add_watch(os.path.join(self.directory, *self.branch_dir), mask, rec = True) notifier = ThreadedNotifier(self.__wm, self.process_event) notifier.start()
def start_watching_disk_file(filename, handler):
logger.info("start watching %s" % filename)
wm = WatchManager()
notifier = ThreadedNotifier(wm, PTmp(wm, filename, handler))
notifier.start()
notifiers[filename] = notifier
if os.path.exists(filename):
handler.on_create(filename)
class PhotoWatcher(ProcessEvent):
MASK = (EventsCodes.ALL_FLAGS['IN_DELETE'] |
EventsCodes.ALL_FLAGS['IN_CLOSE_WRITE'] |
EventsCodes.ALL_FLAGS['IN_MOVED_FROM'] |
EventsCodes.ALL_FLAGS['IN_MOVED_TO'])
def __init__(self, db, walker, root):
self.root = root
self.db = db
self.walker = walker
self.wm = WatchManager()
self.wdds = []
def Watch(self):
self.notifier = ThreadedNotifier(self.wm, self)
self.notifier.start()
self.wdds.append(self.wm.add_watch(self.root, self.MASK, rec=True))
# add soft link sub-folders
for dirname, dirnames, _filenames in os.walk(self.root, followlinks=True):
for d in dirnames:
path = os.path.join(dirname, d)
if os.path.islink(path):
self.wdds.append(
self.wm.add_watch(os.path.realpath(path), self.MASK, rec=True))
def Stop(self):
self.notifier.stop()
def process_IN_DELETE(self, event):
self.db.DeletePhoto(os.path.join(event.path, event.name))
def process_IN_MOVED_FROM(self, event):
self.process_IN_DELETE(event)
def process_IN_MOVED_TO(self, event):
full_path = os.path.join(event.path, event.name)
try:
meta = self.walker.ReadMetadata(full_path)
except Exception:
return
self.db.StorePhoto(full_path, meta)
def process_IN_CLOSE_WRITE(self, event):
full_path = os.path.join(event.path, event.name)
try:
meta = self.walker.ReadMetadata(full_path)
except Exception:
return
if self.db.HasPhoto(full_path):
self.db.UpdatePhoto(full_path, meta)
else:
self.db.StorePhoto(full_path, meta)
class FileEvent:
def __init__(self, eventHandler):
self.logger = logging.getLogger('FileEvent')
self.wm = WatchManager()
self.watches = dict()
# Set the flags of the events that are to be listened to
FLAGS = EventsCodes.ALL_FLAGS
self.mask = FLAGS['IN_CREATE'] | FLAGS['IN_DELETE'] | FLAGS['IN_MODIFY'] | FLAGS['IN_DELETE_SELF']
# Set-up notifier
self.notifier = ThreadedNotifier(self.wm, EventProcessor(eventHandler))
def startNotifyLoop(self):
self.notifier.start()
def stopNotifyLoop(self):
self.notifier.stop()
def addWatches(self, paths, mask=None):
added_watches = dict()
for path in paths:
added_watches.update(self.addWatch(path, mask))
return added_watches
# Also monitors all sub-directories of the given directory and automatically adds newly
# created directories to watch.
# TODO should be able to add files as well, but doesn't work atm
def addWatch(self, path, mask=None):
if mask is None:
mask = self.mask
added_watches = self.wm.add_watch(path, mask, rec=True, auto_add=True)
self.watches.update(added_watches)
return added_watches
def removeWatch(self, path):
watch_descriptor = self.wm.get_wd(path)
if watch_descriptor is not None:
result = self.wm.rm_watch(watch_descriptor, rec=True)
# Remove the no longer active watches from the current watches dictionary
for key, value in self.watches.items():
if value in result:
del self.watches[key]
else:
result = None
return result
def getWatches(self):
return self.watches
def schedule(self, name, event_handler, *paths):
"""Schedules monitoring."""
#from pyinotify import PrintAllEvents
#dispatcher = PrintAllEvents()
dispatcher = _ProcessEventDispatcher(event_handler=event_handler)
notifier = ThreadedNotifier(self.wm, dispatcher)
self.notifiers.add(notifier)
for path in paths:
if not isinstance(path, str):
raise TypeError(
"Path must be string, not '%s'." % type(path).__name__)
descriptors = self.wm.add_watch(path, ALL_EVENTS, rec=True, auto_add=True)
self.name_to_rule[name] = _Rule(name, notifier, descriptors)
notifier.start()
def live(path, tv):
chromedriver = os.path.join(os.path.split(os.path.realpath(__file__))[0],\
'chromedriver')
os.environ['webdriver.chrome.driver'] = chromedriver
browsers = [ getattr(webdriver, browser.title())() \
for browser in [s.lower() for s in tv]]
wm = WatchManager()
notifier = ThreadedNotifier(wm, ProcessDir(browsers))
notifier.start()
print('watching %s' % os.path.abspath(path))
mask = IN_MODIFY
wdd = wm.add_watch(path, mask, rec=True)
def live(path, tv):
chromedriver = os.path.join(os.path.split(os.path.realpath(__file__))[0],\
'chromedriver')
os.environ['webdriver.chrome.driver'] = chromedriver
browsers = [ getattr(webdriver, browser.title())() \
for browser in [s.lower() for s in tv]]
wm = WatchManager()
notifier = ThreadedNotifier(wm, ProcessDir(browsers))
notifier.start()
print('watching %s' % os.path.abspath(path))
mask = IN_MODIFY
wdd = wm.add_watch(path, mask, rec=True)
def setup_inotify(self):
if pyinotify is None:
return False
watch_manager = WatchManager()
result = watch_manager.add_watch(self.__status_file, IN_MODIFY)[self.__status_file] > 0
if result:
global notifier
def notify_cb(event):
glib.idle_add(self.check_status_cb)
notifier = ThreadedNotifier(watch_manager, notify_cb)
notifier.start()
return result
def get_threaded_notifier(paths,db_file,flush_interval,verbose,fs_encodings=[]): wm = WatchManager() eventhandler = FileStatEventHandler(wm,db_file,flush_interval,verbose,fs_encodings) notifier = ThreadedNotifier(wm, eventhandler) notifier.start() for p in paths: if verbose: print ' Monitoring "%s" ... ' % p, wdd = wm.add_watch(p, mask, rec=True,auto_add=False) if verbose: print '[done]' return eventhandler,notifier
def run(self):
logging.info( 'Thead Watcher' )
wm = WatchManager()
notifier = ThreadedNotifier(wm, PTmp())
notifier.start()
wdd = wm.add_watch( account_dir, 4095, rec=True, auto_add=True) #ALL_EVENTS=>4095
logging.info('Add watch on folder ' + account_dir);
while True:
try:
notifier.process_events()
if notifier.check_events():
notifier.read_events()
except KeyboardInterrupt:
break
class Observer:
"""Monitor files and notify the main program when changes happen"""
def __init__(self, makeRunView):
self.wm = WatchManager()
self.eh = EventHandler(makeRunView)
self.notifier = ThreadedNotifier(self.wm, self.eh)
self.notifier.start()
# Watched events
self.mask = IN_DELETE | IN_CREATE | IN_CLOSE_WRITE
def kill(self):
status = self.notifier.stop()
logging.debug("Observer shut down")
return status
def addFile(self, fname):
wdd = self.wm.add_watch(fname, self.mask, rec=True)
def setup_inotify(self):
if pyinotify is None:
return False
watch_manager = WatchManager()
result = watch_manager.add_watch(self.__status_file,
IN_MODIFY)[self.__status_file] > 0
if result:
global notifier
def notify_cb(event):
glib.idle_add(self.check_status_cb)
notifier = ThreadedNotifier(watch_manager, notify_cb)
notifier.start()
return result
def main():
# Add a dummy message to the queue:
add_msg( MsgTypes.Initialise )
# Setup the HTTP server:
SocketServer.TCPServer.allow_reuse_address = True
HOST, PORT = "localhost", 9000
server = ThreadedHTTPServer((HOST, PORT), MyTCPHandler)
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
# Setup the heartbeat:
heartbeat_thread = threading.Thread(target=heartbeat_messages)
heartbeat_thread.daemon = True
heartbeat_thread.start()
# Setup pyinotify so files are being watched:
wm = WatchManager()
notifier = ThreadedNotifier(wm, PTmp())
notifier.start()
mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_ATTRIB'] |EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events
#mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events
wdd = wm.add_watch('../sentry.py', mask, rec=True)
wdd = wm.add_watch('../', mask, rec=True)
try:
while True:
time.sleep(5)
except:
# Turn off pyinotify:
notifier.stop()
print 'Exception caught: shutting down connections'
add_msg(MsgTypes.Shutdown)
time.sleep(0.5)
print 'Terminating...'
raise
class FileWatcher(Thread):
def __init__(self, holder, uri, schedule_reader):
Thread.__init__(self)
self._loop = True
self._error_event = Event()
self._wm = WatchManager()
self._holder = holder
self._uri = uri
self._schedule_reader = schedule_reader
self._notifier = ThreadedNotifier(self._wm,
_EventHandler(self._holder,
self._uri,
self._schedule_reader,
self._error_event))
self._path, self._pattern = os.path.split(urlparse(uri).path)
def start(self):
"""Start the file watcher
"""
self._notifier.start()
self._wm.add_watch(self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY)
Thread.start(self)
def run(self):
while self._loop:
if self._error_event.wait(1):
self._error_event.clear()
self._notifier.stop()
del self._notifier
self._notifier = ThreadedNotifier(
self._wm,
_EventHandler(self._holder,
self._uri,
self._schedule_reader,
self._error_event))
self._notifier.start()
self._wm.add_watch(
self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY)
def stop(self):
"""Stop the file watcher
"""
self._notifier.stop()
self._loop = False
class _PathWatcher(Queue): # iNotify watcher for directory
'''
iNotify watcher object for monitor of changes in directory.
'''
FLAGS = IN_MODIFY | IN_DELETE | IN_CREATE | IN_MOVED_FROM | IN_MOVED_TO | IN_ATTRIB
def __init__(self, path, exclude=None):
class _EH(ProcessEvent):
def process_default(self, event):
_handleEvent(event)
Queue.__init__(self)
self._path = path
self.exclude = exclude or []
_handleEvent = self.put
self._wm = WatchManager()
self._iNotifier = ThreadedNotifier(self._wm, _EH(), timeout=10)
self._iNotifier.start()
self.started = False
self._watch = []
def start(self, exclude=None):
if not self.started:
# Add watch and start watching
# Update exclude filter if it provided in call of start method
self.exclude = exclude or self.exclude
self._watch = self._wm.add_watch(self._path,
self.FLAGS,
exclude_filter=ExcludeFilter(
self.exclude),
auto_add=True,
rec=True,
do_glob=False)
self.started = True
def stop(self):
if self.started:
# Remove watch and stop watching
self._wm.rm_watch(self._watch[self._path], rec=True)
self.started = False
def exit(self):
self.stop()
self._iNotifier.stop()
def lastwatch(paths, settings, dry_run=False):
flags = EventsCodes.FLAG_COLLECTIONS.get('OP_FLAGS', None)
if flags:
mask = flags.get('IN_OPEN') | flags.get('IN_CLOSE_NOWRITE')
mask |= flags.get('IN_CREATE') | flags.get('IN_MOVED_TO')
else:
mask = EventsCodes.IN_OPEN | EventsCodes.IN_CLOSE_NOWRITE
mask |= EventsCodes.IN_CREATE | EventsCodes.IN_MOVED_TO
wm = WatchManager()
handler = Handler(settings, dry_run=dry_run)
watcher = ThreadedNotifier(wm, handler)
watcher.start()
try:
for path in paths:
path = os.path.realpath(path)
sys.stdout.write(_("Indexing %s for watching...") % path)
sys.stdout.flush()
wm.add_watch(path, mask, rec=True, auto_add=True)
sys.stdout.write(_(" done.") + "\n")
print _("You have successfully launched Lastwatch.")
print "\n".join(wrap(_("The directories you have specified will be "
"monitored as long as this process is running, "
"the flowers are blooming and the earth "
"revolves around the sun..."), 80))
# flowers to devhell ;-)
handler.set_active()
while True:
time.sleep(1)
except KeyboardInterrupt:
watcher.stop()
print _("LastWatch stopped.")
return
except Exception, err:
print err
def start_daemon(path, dbpath, md_queue, fd_queue, condition):
""" installs a subtree listener and wait for events """
os.nice(19)
wm_auto = WatchManager()
subtreemask = IN_CLOSE_WRITE | IN_DELETE | IN_MOVED_FROM | IN_MOVED_TO | IN_CREATE | IN_ISDIR
excludefilter = ExcludeFilter(["(.*)sqlite"])
notifier_sb = ThreadedNotifier(wm_auto, SubtreeListener(dbpath, md_queue, fd_queue, condition))
notifier_sb.start()
THREADS.append(notifier_sb)
wdd_sb = wm_auto.add_watch(path, subtreemask, auto_add=True, rec=True, exclude_filter=excludefilter)
THREADS.append(PollAnalyzer(condition, dbpath))
THREADS[-1].start()
# THREADS.append(CatalogThreadingTCPServer(("localhost", 8080), CatalogHTTPRequestHandler, dbpath))
# THREADS[-1].serve_forever()
THREADS[-1].join()
def create_watcher():
from pyinotify import WatchManager, Notifier, ThreadedNotifier, \
EventsCodes, ProcessEvent, IN_CLOSE_WRITE
wm = WatchManager()
mask = IN_CLOSE_WRITE #| EventsCodes.IN_CREATE # watched events
class PTmp(ProcessEvent):
def process_IN_CLOSE_WRITE(self, event):
def inner(): on_reload_event(event)
gdb.post_event(inner)
notifier = ThreadedNotifier(wm, PTmp())
wdd = wm.add_watch(WORKING_DIR, mask, rec=True)
notifier.daemon = True # Then our atexit function will work
notifier.start()
def on_exited(*e): notifier.stop()
import atexit
atexit.register(on_exited)
return (notifier, wdd)
class FilesystemWatcher:
# Mask for FS events we are interested in - currently only care about
# writable file streams that have been closed (i.e. a pdf has finished
# uploading)
mask = EventsCodes.ALL_FLAGS['IN_CLOSE_WRITE']
# This is the main watch manager provided by pyinotify
# we use this to manage subscriptions
_wm = WatchManager()
# List of watched directories
wdd = {}
# create a thread-safe queue for papers that must be processed.
paper_queue = Queue()
def __init__(self, logger):
self.logger = logger
self.notifier = ThreadedNotifier(
self._wm, PaperProcesser(self.logger, self.paper_queue))
def watch_directory(self, path):
"""set up threaded directory watcher at given path"""
self._wm.add_watch(path, self.mask, rec=True)
#add all files in the given directory to queue
for root, dirs, files in os.walk(path):
for file in files:
if file.endswith("pdf") or file.endswith("xml"):
self.logger.info("Adding %s to queue", file)
self.paper_queue.put(("QUEUE", os.path.join(root, file)))
def start(self):
self.notifier.start()
def stop(self):
self.notifier.stop()
class Watch(object): def __init__(self, path, callback, ignore_modifications=False, latency=None): self.event_mask = EVENT_MASK if ignore_modifications else EVENT_MASK_WITH_MODIFICATIONS self._dir_queue = queue.Queue() self._root = os.path.realpath(path) self._watch_manager = WatchManager() self._processor = FileProcessEvent(directory_queue=self._dir_queue, root=self._root, callback=callback, latency=latency) self._notifier = ThreadedNotifier(self._watch_manager, self._processor) self._notifier.name = "[inotify] notifier" self._notifier.daemon = True self._notifier.start() self._watch_manager.add_watch(path, self.event_mask, rec=True, auto_add=True) def stop(self): self._notifier.stop() self._processor.stop()
class FileWatcher(Thread):
def __init__(self, holder, uri, schedule_reader):
Thread.__init__(self)
self._loop = True
self._error_event = Event()
self._wm = WatchManager()
self._holder = holder
self._uri = uri
self._schedule_reader = schedule_reader
self._notifier = ThreadedNotifier(
self._wm,
_EventHandler(self._holder, self._uri, self._schedule_reader,
self._error_event))
self._path, self._pattern = os.path.split(urlparse(uri).path)
def start(self):
"""Start the file watcher
"""
self._notifier.start()
self._wm.add_watch(self._path, IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY)
Thread.start(self)
def run(self):
while self._loop:
if self._error_event.wait(1):
self._error_event.clear()
self._notifier.stop()
del self._notifier
self._notifier = ThreadedNotifier(
self._wm,
_EventHandler(self._holder, self._uri,
self._schedule_reader, self._error_event))
self._notifier.start()
self._wm.add_watch(self._path,
IN_OPEN | IN_CLOSE_WRITE | IN_MODIFY)
def stop(self):
"""Stop the file watcher
"""
self._notifier.stop()
self._loop = False
def monitor(filename, server):
"""Monitor the config file for changes and update the server's values when they do.
Using pyinotify, it checks the config file for changes and tells the
server to make changes when they occur. Starts and returns the
ThreadedNotifier that is created.
Keyword arguments:
filename -- the name of the file you want to monitor. Note that it won't actually monitor this file, but the directory it resides in.
server -- the EmailServer currently running, to pass the new values to when they're changed.
"""
logging.info('Monitoring {0} for changes'.format(filename))
wm = WatchManager()
notifier = ThreadedNotifier(wm, PClose(server, filename))
notifier.name = 'MonitorThread'
# we actually watch the folder to the file, otherwise the handle is lost every time the file is modified.
wm.add_watch(os.path.split(filename)[0], IN_CLOSE_WRITE, rec=True)
notifier.start()
return notifier
class Watcher(object):
"""
Watching on the fly
"""
def __init__(self, gdr, workq, src, dst):
self.gdr = gdr
self.workq = workq
self.src = src
self.dst = dst
def loop(self):
wm = WatchManager()
handler = EventHandler(self.workq, self.src, self.dst)
self.notifier = ThreadedNotifier(wm, handler)
self.notifier.start()
mask = IN_CREATE | IN_MODIFY
wm.add_watch(self.src, mask, rec=self.gdr.rec)
def stop(self):
self.notifier.stop()
class Watcher(object):
"""
Watching on the fly
"""
def __init__(self, gdr, workq, src, dst):
self.gdr = gdr
self.workq = workq
self.src = src
self.dst = dst
def loop(self):
wm = WatchManager()
handler = EventHandler(self.workq, self.src, self.dst)
self.notifier = ThreadedNotifier(wm, handler)
self.notifier.start()
mask = IN_CREATE | IN_MODIFY
wm.add_watch(self.src, mask, rec=self.gdr.rec)
def stop(self):
self.notifier.stop()
def start_watching():
class OnEvent(ProcessEvent):
def __init__(self):
ProcessEvent(self)
self.timer = None
def process_default(self, event):
accepted = [pyinotify.IN_MODIFY, pyinotify.IN_MOVE_SELF, pyinotify.IN_MOVED_FROM, pyinotify.IN_MOVED_TO]
x = [x for x in accepted if event.mask & x == event.mask]
if len(x) > 0:
print "default: %-20s %s" % (os.path.join(event.path, event.name), event.maskname)
if self.timer:
self.timer.cancel()
self.timer = None
self.timer = threading.Timer(0.2, do_post)
self.timer.start()
wm = WatchManager()
mask = pyinotify.ALL_EVENTS
proc = OnEvent()
notifier = ThreadedNotifier(wm, proc)
notifier.start()
wdd = wm.add_watch('.git', mask, rec=True, auto_add=True)
def main_loop():
# Setup pyinotify so files are being watched:
wm = WatchManager()
notifier = ThreadedNotifier(wm, PTmp())
notifier.start()
mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_ATTRIB'] |EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events
#mask = EventsCodes.ALL_FLAGS['IN_DELETE'] | EventsCodes.ALL_FLAGS['IN_CREATE'] | EventsCodes.ALL_FLAGS['IN_MODIFY']# watched events
#wdd = wm.add_watch('../sentry.py', mask, rec=True)
#wdd = wm.add_watch('../', mask, rec=True)
try:
while True:
print '\rConnections to %d clients. (%s)' % (len(open_handles), time_string()),
sys.stdout.flush()
update_subscribers(msg_type=MsgTypes.Heartbeat)
time.sleep(5)
except:
notifier.stop()
raise
class ExcelFileWatcher(object):
class Process(ProcessEvent):
#def __init__(self, options):
# self.regex = re.compile(options.regex)
# self.script = options.script
def __init__(self, folderpath, event_callback):
#self.regex = re.compile(options.regex)
#self.script = options.script
self.event_callback = event_callback
self.folderpath = folderpath
def process_IN_CREATE(self, event):
target = os.path.join(event.path, event.name)
print "create event happened!"
if os.path.isdir(target):
raise Reload()
def process_IN_DELETE(self, event):
print "delete event happened"
raise Reload()
def process_IN_CLOSE_WRITE(self, event):
target = os.path.join(event.path, event.name)
#print "target = ", target
#print 'event.path = ', event.path
#print 'event.name = ', event.name
self.event_callback(event.path, event.name)
#if self.regex.match(target):
# args = self.script.replace('$f', target).split()
# os.system("clear")
# sys.stdout.write("executing script: " + " ".join(args) + "\n")
# subprocess.call(args)
# sys.stdout.write("------------------------\n")
def __init__(self, folderpath, event_callback):
self.event_callback = event_callback
self.folderpath = folderpath
def start_watch_loop(self):
wm = WatchManager()
process = self.Process(self.folderpath, self.event_callback) #options)
self.notifier = ThreadedNotifier(wm, process) #Notifier(wm, process)
#notifier = Notifier(wm)
mask = IN_DELETE | IN_CREATE | IN_CLOSE_WRITE
#wdd = wm.add_watch(options.directory, mask, rec=True)
#wm.add_watch('./excelsrc', mask, rec=True)
wm.add_watch('./' + self.folderpath, mask, rec=True)
self.notifier.start()
"""
while True:
wm = WatchManager()
process = self.Process(self.folderpath, self.event_callback) #options)
notifier = ThreadedNotifier(wm, process) #Notifier(wm, process)
#notifier = Notifier(wm)
mask = IN_DELETE | IN_CREATE | IN_CLOSE_WRITE
#wdd = wm.add_watch(options.directory, mask, rec=True)
#wm.add_watch('./excelsrc', mask, rec=True)
wm.add_watch('./'+self.folderpath, mask, rec=True)
try:
while True:
print '.'
notifier.process_events()
print '+'
if notifier.check_events():
notifier.read_events()
print '-'
except Reload:
pass
except KeyboardInterrupt:
notifier.stop()
break
"""
def stop_watch_loop(self):
self.notifier.stop()
self._dispatch_download(file_path, file_name)
else:
print "Skipping %s" % file_name
def _process_directory(self, dir_path):
candidates = os.listdir(dir_path)
print candidates
for candidate in candidates:
if candidate.split('.')[-1].lower() in allowed_extensions:
self._dispatch_download(dir_path, candidate)
else:
print "Skipping %s" % candidate
print "SDownloader is now starting..."
wm = WatchManager()
wm.add_watch(target_dir, mask, rec=True, auto_add=True)
print "Sdownloader is now watching %s" % target_dir
notifier = ThreadedNotifier(wm, MyWatcher())
notifier.start()
while True:
try:
time.sleep(0.5)
except (Exception, KeyboardInterrupt):
notifier.stop()
break
class _Core():
"""
the core functionaliy for persy
"""
def init(self, config, log):
"""
initializes the git binding
"""
self.config = config
self.log = log
self.worker = None
self.notifier = None
self.running = False
#initialzing the git binding
#===========================
#if persy is interrupted while git was running, a git.lockfile may be present. we have to remove it!
if os.path.exists(self.config.getAttribute('GIT_LOCKFILE')):
try:
os.remove(self.config.getAttribute('GIT_LOCKFILE'))
except Exception as err:
log.warn(str(err))
else:
log.warn(_("removed git lock file"))
#init pug
os.popen("touch %s"%self.config.getAttribute('LOGFILE_GIT'))
std = open(self.config.getAttribute('LOGFILE_GIT'), 'a')
stdin = None #default stdin
stdout = std #default stdout
stderr = std #default stderr
cwd = os.environ["HOME"]
self.vcs = pug.PuG(cwd=cwd, GIT_WORK_TREE=config['general']['gitworktree'], GIT_DIR=config['general']['gitdir'], stdin=stdin, stdout=subprocess2.PIPE, stderr=stderr)
def init_local(self):
"""
initialises the local repository
"""
if not self.config.has_key('general') or not self.config['general'].has_key('name') or not self.config['general']['name'] :
self.log.critical(_('username not set, cannot create git repository. use "persy --config --name=NAME" to set one'), verbose=True)
sys.exit(-1)
if not self.config.has_key('general') or not self.config['general'].has_key('mail') or not self.config['general']['mail']:
self.log.critical(_('mail not set, cannot create git repository. use "persy --config --mail=MAIL" to set one'), verbose=True)
sys.exit(-1)
self.log.info(_("initialising local repository..."), verbose=True)
try:
self.vcs.init()
self.vcs.config('user.name',self.config['general']['name'])
self.vcs.config('user.email',self.config['general']['mail'])
self.vcsignore()
except Exception as err:
self.log.critical(str(err), verbose=True)
else:
self.log.info(_("done"), verbose=True)
def vcslog(self):
"""
runs the vcs(git)-log command and writs the output to stdout
"""
self.vcs.log(stdout=sys.stdout, stdin=sys.stdin, stderr=sys.stderr)
def vcsstatus(self):
"""
runs the vcs(git)-status command
"""
self.vcs.status(stdout=sys.stdout, stdin=sys.stdin, stderr=sys.stderr)
def initRemote(self):
"""
initialises the remote repository and adds it to the synchronization pack
tries to connect to the remote server and creates the destination folder and initializes the remote git repository
"""
self.log.info(_("initialising and adding remote repository..."), verbose=True)
try:
client = paramiko.SSHClient()
client.load_system_host_keys()
client.connect(self.config['remote']['hostname'],
username=self.config['remote']['username'],
port=int(self.config['remote']['port']))
# the follow commands are executet on a remote host. we can not know the path to git,
# mkdir and cd so we will not replace them with a absolute path
stdin1, stdout1, stderr1 = client.exec_command("mkdir -m 700 %s"%self.config['remote']['path'])
stdin2, stdout2, stderr2 = client.exec_command("cd %s && git --bare init"%self.config['remote']['path'])
client.close()
if stderr1:
self.log.warn(_("error creating dir, maybe it exists already?"), verbose=True)
elif stderr2:
self.log.critical(_("error on remote git init"), verbose=True)
elif not self.config['remote']['use_remote']:
#no errors:so we are save to use the remote
self.config['remote']['use_remote'] = True
self.config.write()
self.vcs.remoteAdd(self.config.getAttribute('SERVER_NICK'),"ssh://%s@%s/%s"%(self.config['remote']['username'],self.config['remote']['hostname'],self.config['remote']['path']))
except Exception as e:
self.log.critical(str(e), verbose=True)
else:
self.log.info(_("done"), verbose=True)
def syncWithRemote(self):
"""
Syncs with a remote server
adds a new remote repository with the information from config to the local git repository and performs a pull
"""
# i dont use clone because of massive errors when using it
# the best way iś to add the remote server and pull from it
self.log.debug('persy-core-sync called')
#check to only do this if all of the remote watched directoris do NOT exist
for f in self.config['local']['watched']:
if os.path.exists(f):
self.log.critical(_("%f does already exist but it should not! Please remove it and try it again."))
return False
try:
self.log.debug('persy-core-sync remote add')
self.vcs.remoteAdd(self.config.getAttribute('SERVER_NICK'),"ssh://%s@%s/%s"%(self.config['remote']['username'],self.config['remote']['hostname'],self.config['remote']['path']))
self.log.debug('persy-core-sync pull')
self.vcs.pull(self.config.getAttribute('SERVER_NICK'),self.config.getAttribute('BRANCH'))
except Exception as e:
self.log.critical(str(e))
self.log.debug('persy-core-sync ende')
self.config['remote']['use_remote'] = True
self.config.write()
def isInSyncWithRemote(self):
"""
returns true if it is already in sync
"in sync" means: git remote returns an entry for an "origin" repository.
"""
try:
self.vcs.command('git', params=['remote'])
except Exception as e:
self.log.critical(str(e))
std = self.vcs.getLastOutput().strip("\n ")
if std == 'origin':
return True
return False
def vcsignore(self):
"""
creates a file for ignoring unwatched directories so they dont appear in the status (and cant be removed exidently with "git clean")
list every file in /home/USER.
add every file and folder (if not already done) to .vcsignore if they are not part WATCHED
"""
current = os.listdir(self.config.getAttribute('USERHOME'))
for f in self.config['local']['watched']:
#if not f.startswith(USERHOME):
if f.startswith(self.config.getAttribute('USERHOME')): #if absolute path
#strip dir stuff, the +1 is for the file seperator
f = f[len(self.config.getAttribute('USERHOME'))+1:]
elif f.startswith('~/'):
f = f[2:] #strip the ~/
elif f.startswith('./'):
f = f[2:] #strip the ./
elif f.startswith('/'):
#continue #savetycheck
#i assume if it still starts with /, its outside of /home
continue
if os.sep in f:
f = f[:f.index(os.sep)]
if f in current:
current.remove(f)
if self.config['local']['maxfilesize'] and self.config['local']['maxfilesize'] > 0:
#add files to the ignorelist if they are larger than maxfilesize
callcmd = []
callcmd.append('find')
callcmd.append(os.path.join(self.config.getAttribute('USERHOME'),f))
callcmd.append('-type')
callcmd.append('f')
callcmd.append('-size')
callcmd.append("+" + str(self.config['local']['maxfilesize']) + "k")
(stdoutdata, stderrdata) = subprocess.Popen(callcmd, stdout=subprocess.PIPE).communicate()
for entry in stdoutdata.split("\n"):
current.append(entry)
for entry in self.config['local']['exclude']:
current.append(entry)
with open(os.path.join(self.config.getAttribute('PERSY_DIR'),self.config.getAttribute('GITIGNOREFILE')), "w+") as f:
for c in current:
f.write(c+"\n")
def optimize(self):
"""
tries to optimize the local repository.
executes the git-gc command.
"""
self.log.info('starting optimization', verbose=True)
class Starter(Thread):
def __init__(self,vcs, log):
Thread.__init__(self)
self.vcs = vcs
self.log = log
def run(self):
try:
self.log.debug('git gc')
self.vcs.gc()
except Exception as e:
self.log.warn(str(e), verbose=True)
try:
Starter(self.vcs, self.log).start()
except Exception as e:
self.log.warn(str(e), verbose=True)
def browse(self):
"""
starts the default git browser
"""
class Starter(Thread):
def __init__(self,vcs, config):
Thread.__init__(self)
self.vcs = vcs
self.config = config
def run(self):
self.vcs.command(self.config['general']['prefgitbrowser'], stdout=sys.stdout, stdin=sys.stdin, stderr=sys.stderr)
Starter(self.vcs, self.config).start()
def persy_stop(self):
"""
Stops persys working thread and the notifier.
"""
self.log.info("stop working")
self.log.setStop()
self.running = False
if self.worker:
try:
self.worker.stop()
self.worker.join()
except RuntimeError:
pass
if self.notifier:
try:
self.notifier.stop()
except RuntimeError:
pass
except OSError:
pass
except KeyError:
pass
def persy_start(self):
"""
initializes the worker thread and notifier
"""
self.log.info("start working")
self.log.setStart()
self.running = True
FLAGS=EventsCodes.ALL_FLAGS
mask = FLAGS['IN_MODIFY'] | FLAGS['IN_DELETE_SELF']|FLAGS['IN_DELETE'] | FLAGS['IN_CREATE'] | FLAGS['IN_CLOSE_WRITE'] | FLAGS['IN_MOVE_SELF'] | FLAGS['IN_MOVED_TO'] | FLAGS['IN_MOVED_FROM'] # watched events
wm = WatchManager()
#addin the watched directories
for watch in self.config['local']['watched']:
wdd = wm.add_watch("%s"%(watch), mask, rec=True, auto_add=True)
#watch for changes of the configurationfile
if self.config['general']['autoshare']:
wdd = wm.add_watch(self.config.getAttribute('CONFIGFILE'), mask, rec=True, auto_add=True)
self.log.debug("init the syncer")
self.worker = TheSyncer(self, self.config, self.log, self.config['remote']['sleep'], self.config['local']['sleep'])
self.log.debug("init the filesystem notifier")
self.notifier = ThreadedNotifier(wm, FileChangeHandler(self.log, self.worker.newEvent))
self.log.resetError()
self.log.debug("starting syncer")
self.worker.start()
self.notifier.start()
#if self.statusIcon:
# statusIcon.set_from_file(config.getAttribute('ICON_OK'))#from_stock(gtk.STOCK_HOME)
def setonetimesync(self):
"""
if the worker is running, execute the setonetimesync function of the worker
"""
if self.worker:
try:
self.worker.setonetimesync()
except RuntimeError:
pass
def isLocalInitialized(self):
"""
returns true if the local GIT_DIR exists
"""
return os.path.exists(self.config.getAttribute('GIT_DIR'))
def git_add(self, item):
"""
adds an item to git
"""
self.vcs.add(item)
def git_commit(self, message):
"""
executes a commit with "message" as the commit message
"""
self.vcs.commit(message)
def git_pull(self, nickname, branch):
"""
executes a pull from "branch" on "nickname"
"""
self.vcs.pull(nickname, branch)
def git_push(self, nickname, branch):
"""
executes a push to "branch" on "nickname"
"""
self.vcs.push(nickname,branch)
def git_svn_pull(self):
self.vcs.svn_pull()
def git_svn_push(self):
self.vcs.svn_push()
def git_get_submodules(self):
"""
returns all submodules in watched directories
"""
return self.vcs.get_submodules(include_dir = self.config['local']['watched'])
class FSMonitorInotify(FSMonitor):
"""inotify support for FSMonitor"""
EVENTMAPPING = {
FSMonitor.CREATED: pyinotify.IN_CREATE,
FSMonitor.MODIFIED: pyinotify.IN_MODIFY | pyinotify.IN_ATTRIB,
FSMonitor.DELETED: pyinotify.IN_DELETE,
FSMonitor.MONITORED_DIR_MOVED: pyinotify.IN_MOVE_SELF,
FSMonitor.DROPPED_EVENTS: pyinotify.IN_Q_OVERFLOW,
}
def __init__(self,
callback,
persistent=False,
trigger_events_for_initial_scan=False,
ignored_dirs=[],
dbfile="fsmonitor.db"):
FSMonitor.__init__(self, callback, persistent,
trigger_events_for_initial_scan, ignored_dirs,
dbfile)
self.wm = None
self.notifier = None
def __fsmonitor_event_to_inotify_event(self, event_mask):
"""map an FSMonitor event to an inotify event"""
inotify_event_mask = 0
for fsmonitor_event_mask in self.__class__.EVENTMAPPING.keys():
if event_mask & fsmonitor_event_mask:
inotify_event_mask = inotify_event_mask | self.__class__.EVENTMAPPING[
fsmonitor_event_mask]
return inotify_event_mask
def __add_dir(self, path, event_mask):
"""override of FSMonitor.__add_dir()"""
# Perform an initial scan of the directory structure. If this has
# already been done, then it will return immediately.
if self.persistent:
if self.trigger_events_for_initial_scan:
FSMonitor.generate_missed_events(self, path, event_mask)
else:
self.pathscanner.initial_scan(path)
event_mask_inotify = self.__fsmonitor_event_to_inotify_event(
event_mask)
# Use the inotify API to monitor a directory.
wdd = self.wm.add_watch(path,
event_mask_inotify,
proc_fun=self.process_event,
rec=True,
auto_add=True)
if wdd is None:
raise MonitorError, "Could not monitor %s" % path
return None
else:
self.monitored_paths[path] = MonitoredPath(path, event_mask, wdd)
self.monitored_paths[path].monitoring = True
# Generate the missed events.
if self.persistent:
FSMonitor.generate_missed_events(self, path)
return self.monitored_paths[path]
def __remove_dir(self, path):
"""override of FSMonitor.__remove_dir()"""
if path in self.monitored_paths.keys():
wd = self.monitored_paths[path].fsmonitor_ref
# TODO: figure out why this won't work, it seems this fails due to
# a bug in pyinotify?
#self.wm.rm_watch(wd, rec=True, quiet=True)
del self.monitored_paths[path]
def run(self):
# Setup. Ensure that this isn't interleaved with any other thread, so
# that the DB setup continues as expected.
self.lock.acquire()
FSMonitor.setup(self)
self.process_event = FSMonitorInotifyProcessEvent(self)
self.lock.release()
# Set up inotify.
self.wm = WatchManager()
self.notifier = ThreadedNotifier(self.wm, self.process_event)
self.notifier.start()
while not self.die:
self.__process_queues()
time.sleep(0.5)
self.notifier.stop()
def stop(self):
"""override of FSMonitor.stop()"""
# Let the thread know it should die.
self.lock.acquire()
self.die = True
self.lock.release()
# Stop monitoring each monitored path.
for path in self.monitored_paths.keys():
self.__remove_dir(path)
def __process_queues(self):
# Process add queue.
self.lock.acquire()
if not self.add_queue.empty():
(path, event_mask) = self.add_queue.get()
self.lock.release()
self.__add_dir(path, event_mask)
else:
self.lock.release()
# Process remove queue.
self.lock.acquire()
if not self.remove_queue.empty():
path = self.add_queue.get()
self.lock.release()
self.__remove_dir(path)
else:
self.lock.release()
class TargetWatcher(object):
def __init__(self, configuration, builder, watch_index):
self._builder = builder
self._root = configuration.GetExpandedDir("projects", "root_dir")
self._batch_timeout = float(
configuration.Get("file_watcher", "event_batch_timeout_ms")) / 1000
self._moddef_filename = configuration.Get(
"general", "module_definition_filename")
self.wm = WatchManager()
self.watch_index = watch_index
self.watched_module_definitions = collections.defaultdict(dict)
mask = (EventsCodes.ALL_FLAGS['IN_DELETE']
| EventsCodes.ALL_FLAGS['IN_CREATE']
| EventsCodes.ALL_FLAGS['IN_MODIFY'])
handler = functools.partial(TargetWatcher.ProcessEvent, self)
self.events_queue = queue.Queue()
self.acc_thread = threading.Thread(target=functools.partial(
TargetWatcher.AccumulationThreadProc, self),
daemon=True)
self.acc_thread.start()
self.notifier = ThreadedNotifier(self.wm, handler)
self.notifier.start()
self.watch = self.wm.add_watch(self._root,
mask,
rec=True,
auto_add=True)
self.modification_handlers = []
def _GetAllModuleDefinitionsForTarget(self, target_name):
prefix = ""
module_definition_dirs = [""]
for path_element in target_name.split("/"):
if prefix:
prefix = prefix + "/" + path_element
else:
prefix = path_element
module_definition_dirs.append(prefix)
return module_definition_dirs
def AddModificationHandler(self, handler):
self.modification_handlers.append(handler)
def ProcessEventsBatch(self, batch):
modified_targets = set()
modified_module_definitions = set()
root_prefix_len = len(self._root)
for event in batch:
rel_path = event.pathname[root_prefix_len + 1:]
if rel_path.endswith(self._moddef_filename):
conf_dir = rel_path[:-len(self._moddef_filename) - 1]
modified_module_definitions.update(
set(self.watched_module_definitions[conf_dir].values()))
else:
found_targets = self.watch_index.GetMatchingTargets(rel_path)
modified_targets.update(found_targets.values())
if modified_module_definitions or modified_targets:
self.ModificationsFound(modified_module_definitions,
modified_targets)
def ModificationsFound(self, modified_module_definitions,
modified_targets):
logging.info("Files modified: module definitions %s, other %s",
modified_module_definitions, modified_targets)
for handler in self.modification_handlers:
handler(modified_module_definitions, modified_targets)
def AccumulationThreadProc(self):
event_buffer = []
while True:
try:
if event_buffer:
item = self.events_queue.get(block=True,
timeout=self._batch_timeout)
else:
item = self.events_queue.get(block=True)
event_buffer.append(item)
except queue.Empty as e:
try:
self.ProcessEventsBatch(event_buffer[:])
except e:
logging.exception("Uncaught change event processing error")
event_buffer = []
def Join(self):
self.notifier.stop()
def ProcessEvent(self, event):
self.events_queue.put(event)
def _RefreshGlobs(self, target):
watched_globs = self._builder.GetWatchableSources(target.GetName())
target_dir = os.path.dirname(target.GetName())
rel_globs = [
os.path.join(target_dir, glob_p) for glob_p in watched_globs
]
self.watch_index.LoadGlobsForTarget(target, rel_globs)
def ReloadTarget(self, target):
self._RefreshGlobs(target)
def AddTarget(self, target):
self._RefreshGlobs(target)
for prefix in self._GetAllModuleDefinitionsForTarget(target.GetName()):
self.watched_module_definitions[prefix][target.GetName()] = target
def RemoveTarget(self, target):
del self.watched[target.GetName()]
for prefix in self._GetAllModuleDefinitionsForTarget(target.GetName()):
del self.watched_module_definitions[prefix][target.GetName()]
class Repo(ProcessEvent):
def __init__(self, filename=None, config={}, **kwargs):
"""
Load a repository metadata structure
filename:
the name of a .chunker file to load, containing
either full state, or a useful subset
**kwargs:
a basic .chunker data structure
config:
optional dictionary of extra info. Keys:
username - for change log
hostname - for change log
"""
self.notifier = None
if not filename and not kwargs:
raise Exception("Repo has no initialisation data")
struct = {}
if filename and os.path.exists(filename):
try:
data = gzip.open(filename).read()
except:
data = open(filename).read()
struct.update(json.loads(data))
struct.update(kwargs)
self.config = config
self.name = struct.get("name") or os.path.basename(struct.get("root")) or os.path.splitext(os.path.basename(filename or ""))[0]
self.root = struct.get("root") or os.path.join(os.path.expanduser("~/Downloads"), self.name)
self.type = struct.get("type", "share") # static / share
self.uuid = struct.get("uuid", sha256(uuid.uuid4()))
self.key = struct.get("key", None) # for encrypting / decrypting chunks
self.peers = struct.get("peers", [])
self.files = dict([
(filename, File.from_struct(self, filename, data))
for filename, data
in struct.get("files", {}).items()
])
# if we're creating a new static chunkfile, then add our local files to the chunkfile
# should this be in start()?
if (self.type == "static" and not self.files):
self.__add_local_files()
def to_struct(self, state=False):
"""
Serialise the repository into a JSON-compatible dictionary
"""
data = {
"name": self.name,
"type": self.type,
"uuid": self.uuid,
"key": self.key,
"files": dict([
(filename, file.to_struct(state=state))
for filename, file
in self.files.items()
])
}
if state:
data.update({
"peers": [
peer.to_struct(state=state)
for peer
in self.peers
],
"root": self.root,
})
return data
def __repr__(self):
return "Repo(%r, %r, %r, %r)" % (self.type, self.uuid, self.root, self.name)
def save_state(self):
"""
Save the repository state to the default state location
(eg ~/.config/chunker/<uuid>.state on unix)
"""
self.save(get_config_path(self.uuid + ".state"), state=True, compress=True)
def remove_state(self):
p = get_config_path(self.uuid + ".state")
if os.path.exists(p):
os.unlink(p)
def save(self, filename=None, state=False, compress=False):
"""
Export the repository state (ie, write Repo.to_struct() to a JSON file)
filename:
where to save the state to
state:
whether to save active state, eg which chunks are currently downloaded
True -> useful for an app to exit and re-open on the same PC later
False -> useful for exporting the minimal amount of info to get a
new node to join the swarm
compress:
whether or not to run the data through gzip (disabling this can make
debugging easier)
"""
struct = self.to_struct(state=state)
if compress:
fp = gzip.open(filename, "w")
data = json.dumps(struct)
else:
fp = open(filename, "w")
data = json.dumps(struct, indent=4)
fp.write(data)
fp.close()
def log(self, msg):
log.info("[%s] %s" % (self.name, msg))
###################################################################
# Metadata
###################################################################
def __add_local_files(self):
for dirpath, dirnames, filenames in os.walk(self.root):
for filename in filenames:
path = os.path.join(dirpath, filename)
relpath = self.__relpath(path)
# look for
# - files that we haven't seen before
# - files with newer timestamps than our latest known version
#
# note that if a file has new content, but the timestamp is
# unchanged since we last saw it, we won't add a new version,
# but rather treat the current version as corrupt
if (
relpath not in self.files or
ts_round(os.stat(path).st_mtime) > self.files[relpath].timestamp
):
self.update(relpath, {
"versions": [{
"timestamp": ts_round(os.stat(path).st_mtime),
"chunks": None,
}]
})
for file in self.files.values():
# "not supposed to be deleted, but it is" -> it has been
# deleted while we weren't looking.
if not file.deleted and not os.path.exists(file.fullpath):
# We don't know when it was deleted, so add the deletion
# tag as just after the last modification (ie, mark that
# version as deleted, but any newer remote version takes
# precedence)
self.update(file.filename, {
"versions": [{
"timestamp": ts_round(file.timestamp + 1),
"chunks": [],
"deleted": True,
}]
})
def update(self, filename, filedata):
"""
Update the repository with new metadata for a named file
"""
file = File.from_struct(self, filename, filedata)
if file.filename not in self.files:
self.files[file.filename] = file
else:
self.files[file.filename].versions.extend(file.versions)
self.files[file.filename].versions.sort()
if file.deleted:
file.log("deleted")
if os.path.exists(file.fullpath):
os.unlink(file.fullpath)
else:
if os.path.exists(file.fullpath):
file.log("updated")
else:
with open(file.fullpath, "a"):
if file.is_complete():
# mark as finished already
os.utime(file.fullpath, (file.timestamp, file.timestamp))
else:
# mark as incomplete
os.utime(file.fullpath, (0, 0))
file.log("created")
self.save_state()
###################################################################
# Networking
###################################################################
def add_peer(self, peer):
if peer not in self.peers:
self.log("Found new peer: %r" % (peer, ))
self.peers.append(peer)
###################################################################
# Chunks
###################################################################
def get_missing_chunks(self):
"""
Get a list of missing chunks
"""
l = []
for file in self.files.values():
l.extend(file.get_missing_chunks())
return l
def get_known_chunks(self):
"""
Get a list of known chunks
"""
l = []
for file in self.files.values():
l.extend(file.get_known_chunks())
return l
def add_chunk(self, chunk_id, data):
"""
Notify the repository that a new chunk is available
(probably freshly downloaded from the network)
"""
self.log("Trying to insert chunk %s into files" % chunk_id)
for chunk in self.missing_chunks:
if chunk.id == chunk_id:
chunk.save_data(data)
def self_heal(self, known_chunks=None, missing_chunks=None):
"""
Try to use known chunks to fill in gaps
"""
if known_chunks is None:
known_chunks = self.get_known_chunks()
if missing_chunks is None:
missing_chunks = self.get_missing_chunks()
heal(known_chunks, missing_chunks)
###################################################################
# Crypto
###################################################################
def encrypt(self, data):
if self.key:
c = AES.new(self.key)
data = c.encrypt(data)
return data
def decrypt(self, data):
if self.key:
c = AES.new(self.key)
data = c.decrypt(data)
return data
###################################################################
# File system monitoring
###################################################################
def start(self):
"""
Start monitoring for file changes
"""
if self.type == "share":
self.log("Checking for files updated while we were offline")
self.__add_local_files()
self.log("Watching %s for file changes" % self.root)
watcher = WatchManager()
watcher.add_watch(self.root, ALL_EVENTS, rec=True, auto_add=True)
self.notifier = ThreadedNotifier(watcher, self)
self.notifier.daemon = True
self.notifier.start()
else:
self.log("Not watching %s for file changes" % self.root)
# self.self_heal()
def netcomms():
while True:
# select()'ing three empty lists is an error on windows
if not self.peers:
sleep(5)
continue
rs, ws, xs = select(self.peers, self.peers, [], 0)
for r in rs:
packet = r.recv()
r.last_pong = time()
for w in ws:
if w.last_ping < time() - 30 and w.last_pong < time() - 30:
data = json.dumps({"cmd": "get-status", "since": w.last_update})
w.send(data)
w.last_ping = time()
for peer in self.peers:
if peer.last_pong < time() - 300:
log.info("Peer no longer reachable - %r" % peer)
peer.last_pong = time() + 10000
# if there was nothing to do, sleep for a bit
# (if there was something to do, immediately go back for more)
if not rs:
sleep(1)
nc = Thread(target=netcomms, name="NetComms[%s]" % self.name)
nc.daemon = True
nc.start()
def stop(self):
"""
Stop monitoring for file changes
"""
if self.notifier:
self.log("No longer watching %s for file changes" % self.root)
self.notifier.stop()
self.notifier = None
def __relpath(self, path):
base = os.path.abspath(self.root)
path = os.path.abspath(path)
return path[len(base)+1:]
def process_IN_CREATE(self, event):
if os.path.isdir(event.pathname):
return
path = self.__relpath(event.pathname)
self.update(path, filedata={
"versions": [{
"timestamp": int(os.stat(event.pathname).st_mtime),
"chunks": None,
"username": self.config.get("username"),
"hostname": self.config.get("hostname"),
}]
})
# def process_IN_MODIFY(self, event):
# if os.path.isdir(event.pathname):
# return
# path = self.__relpath(event.pathname)
# self.update(path, filedata={
# "versions": [{
# "deleted": False,
# "timestamp": int(os.stat(event.pathname).st_mtime),
# "chunks": None,
# "username": self.config.get("username"),
# "hostname": self.config.get("hostname"),
# }]
# })
def process_IN_DELETE(self, event):
if os.path.isdir(event.pathname):
return
path = self.__relpath(event.pathname)
self.update(path, filedata={
"versions": [{
"deleted": True,
"timestamp": int(time()),
"chunks": [],
"username": self.config.get("username"),
"hostname": self.config.get("hostname"),
}]
})
def __init__(self, parent=None, infile=None):
QtGui.QWidget.__init__(self, parent)
#set up the main UI
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.dbglvl = dbglvl
# self.libdir='/home/gilbank/Proj/spup/GUI/libs/'
# self.libdir='libs/'
self.libdir = '/usr/local/lib/qlgui/'
self.settingsdir = '/home/ccd/.spup/'
self.spup = True # else work with old spectrograph data
self.ui.logView.setReadOnly(True)
# Install the custom output stream
sys.stdout = EmittingStream(textWritten=self.normalOutputWritten)
# set some default size values. use to try to set sliders, etc. to sensible ranges:
if (self.spup):
self.sx = 2148
self.sy = 256
# self.sy = 128 # 1x2 binning
else:
self.sx = 2098
self.sy = 512
self.xbin = 1
# self.ybin=1
self.ybin = 2
self.sx = self.sx / self.xbin
self.xy = self.sy / self.ybin
# -- update sliders to double sliders:
geom = self.ui.SB1Slider.geometry()
self.ui.SB1Slider = QxtSpanSlider(self.ui.centralWidget)
self.ui.SB1Slider.setGeometry(geom)
#self.ui.SB1Slider.setOrientation(QtCore.Qt.Horizontal)
"""
self.SB1Slider = QxtSpanSlider(self.centralWidget)
self.SB1Slider.setGeometry(QtCore.QRect(1100, 520, 160, 23))
self.SB1Slider.setOrientation(QtCore.Qt.Horizontal)
self.SB1Slider.setObjectName(_fromUtf8("SB1Slider"))
"""
if (self.spup):
self.ui.SB1Slider.setRange(0, 132) #***
self.ui.SB1Slider.setSpan(80, 120) #***
else:
self.ui.SB1Slider.setRange(0, 132)
self.ui.SB1Slider.setSpan(80, 120)
# --
geom = self.ui.SB2Slider.geometry()
self.ui.SB2Slider = QxtSpanSlider(self.ui.centralWidget)
self.ui.SB2Slider.setGeometry(geom)
#self.ui.SB1Slider.setOrientation(QtCore.Qt.Horizontal)
"""
self.SB1Slider = QxtSpanSlider(self.centralWidget)
self.SB1Slider.setGeometry(QtCore.QRect(1100, 520, 160, 23))
self.SB1Slider.setOrientation(QtCore.Qt.Horizontal)
self.SB1Slider.setObjectName(_fromUtf8("SB1Slider"))
"""
if (self.spup):
self.ui.SB2Slider.setRange(0, 132) #***
self.ui.SB2Slider.setSpan(50, 10) #***
else:
self.ui.SB2Slider.setRange(0, 132)
self.ui.SB2Slider.setSpan(50, 10)
geom = self.ui.extractWinSlider.geometry()
self.ui.extractWinSlider = QxtSpanSlider(self.ui.centralWidget)
self.ui.extractWinSlider.setGeometry(geom)
"""
self.extractWin_x0=self.ui.extractWinSlider.lowerValue
self.extractWin_x1=self.ui.extractWinSlider.upperValue
self.SB1_x0=self.ui.SB1Slider.lowerValue
self.SB1_x1=self.ui.SB1Slider.upperValue
self.SB2_x0=self.ui.SB2Slider.lowerValue
self.SB2_x1=self.ui.SB2Slider.upperValue
"""
# self.mklog()
self.ui.lineEditz0.setText('0.')
self.ui.lineEditz1.setText('10000.')
self.extractionUnits = ''
# header keyword:
if (self.spup):
self.imtype = 'EXPTYPE'
self.sciKW = 'SCIENCE' # keyword for science
# self.arcKW='arc'
self.arcKW = 'ARC'
else:
self.imtype = 'IMAGETYP'
self.sciKW = 'object' # keyword for science
self.arcKW = 'COMPARISON'
pars = {}
pars['smoothPix'] = 0
pars['SkySub'] = False
pars['useSB1'] = True
pars['useSB2'] = True
pars['LockDisplays'] = False
# pars['showExtWin'] = True
# pars['showSideBands'] = True
# pars['dataDir'] = '/home/gilbank/Proj/CassSpect/data/Deatrick/'
# pars['dataDir'] = '/home/ccd/cassspectr/cassspectr/'
try:
f = open('/home/ccd/.spup/datadir.json', 'r')
datadir = json.load(f)
pars['dataDir'] = datadir
f.close()
except ValueError as V:
print '%s\ncould not open datadir.json' % V
self.showDialogSetDatadir()
except:
self.showDialogSetDatadir()
# pars['outDataDir'] = '/home/ccd/cassspectr/cassspectr/'
pars['outDataDir'] = pars['dataDir']
pars['currFrame'] = 'none'
pars['obsType'] = 'none'
pars['currArc'] = 'none'
pars['currFlat'] = 'none'
pars['currFluxCal'] = 'none'
pars['currBias'] = 'none'
pars['scaling'] = 'ZScale'
pars['z0'] = np.NAN
pars['z1'] = np.NAN
if (self.spup):
pars['extractWin_x0'] = 120.0 #**
pars['extractWin_x1'] = 140.0 #**
else:
pars['extractWin_x0'] = 50.0
pars['extractWin_x1'] = 70.0
pars['SBoffset'] = 5
pars['SBwidth'] = 15
self.extractWin_x0 = pars['extractWin_x0']
self.extractWin_x1 = pars['extractWin_x1']
pars['autoExtraction'] = True
# pars['isScience']=True
self.pars = pars
# check if qlsettings.json exists and, if so, reload:
self.loadPars()
# Decide extraction type:
self.ui.lineEditDataDir.setText(self.pars['dataDir'])
self.ui.lineEditOutDataDir.setText(self.pars['outDataDir'])
# self.imagetype='science'
self.imagetype = 'none'
self.wlc = False
# -- Initialise values:
self.autoExtraction = True
self.lamp = None
self.grating = None
self.parentDataDir = None #'/home/gilbank/Proj/CassSpect/data/Deatrick/'
self.imagetype = ""
self.lastWLCFile = None
self.wlc = False
self.latestFrame = None ##self.parentDataDir+'a3000142.fits'
self.scaledImage = None
# disable unimplemented options:
# self.ui.useSB1Check.setEnabled(False)
# self.ui.useSB2Check.setEnabled(False)
self.ui.checkBoxLockDisplays.setEnabled(False)
self.ui.checkBoxWrite1DFITS.setEnabled(False)
self.ui.pushButCurrArc.setEnabled(False)
# -- Connect signals and slots
QtCore.QObject.connect(self.ui.actionQuit,
QtCore.SIGNAL(_fromUtf8("activated()")),
self.closeEvent)
QtCore.QObject.connect(self.ui.pushButPan,
QtCore.SIGNAL(_fromUtf8("pressed()")),
self.doPanAll)
QtCore.QObject.connect(self.ui.pushButZoom,
QtCore.SIGNAL(_fromUtf8("pressed()")),
self.doZoomAll)
QtCore.QObject.connect(self.ui.pushButResetView,
QtCore.SIGNAL(_fromUtf8("pressed()")),
self.resetZoomAll)
QtCore.QObject.connect(self.ui.pushButSmoothObj,
QtCore.SIGNAL(_fromUtf8("pressed()")),
self.display1D)
QtCore.QObject.connect(self.ui.pushButNoSmooth,
QtCore.SIGNAL(_fromUtf8("pressed()")),
self.noSmoothing)
QtCore.QObject.connect(self.ui.pushButCurrFrame,
QtCore.SIGNAL(_fromUtf8("pressed()")),
self.loadFrame)
# QtCore.QObject.connect(self.ui.pushButCurrFrame, QtCore.SIGNAL(_fromUtf8("pressed()")), self.loadAndUpdate)
QtCore.QObject.connect(self.ui.checkBoxShowExtractWin,
QtCore.SIGNAL(_fromUtf8("toggled(bool)")),
self.display2D)
QtCore.QObject.connect(self.ui.checkBoxShowSB,
QtCore.SIGNAL(_fromUtf8("toggled(bool)")),
self.display2D)
QtCore.QObject.connect(self.ui.zScaleUpdateBut,
QtCore.SIGNAL(_fromUtf8("clicked()")),
self.updatezScaling)
QtCore.QObject.connect(self.ui.pushButAutoExtract,
QtCore.SIGNAL(_fromUtf8("clicked()")),
self.updateAutoExtraction)
# QtCore.QObject.connect(self.ui.useSB1Check, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.updateAll)
QtCore.QObject.connect(self.ui.useSB1Check,
QtCore.SIGNAL(_fromUtf8("toggled(bool)")),
self.sideBandToggled)
QtCore.QObject.connect(self.ui.useSB2Check,
QtCore.SIGNAL(_fromUtf8("toggled(bool)")),
self.sideBandToggled)
# QtCore.QObject.connect(self.ui.useSB2Check, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.updateAll)
QtCore.QObject.connect(self.ui.SB1Slider,
QtCore.SIGNAL(_fromUtf8("sliderReleased()")),
self.readExtractionSliders)
QtCore.QObject.connect(self.ui.SB2Slider,
QtCore.SIGNAL(_fromUtf8("sliderReleased()")),
self.readExtractionSliders)
QtCore.QObject.connect(self.ui.extractWinSlider,
QtCore.SIGNAL(_fromUtf8("sliderReleased()")),
self.readExtractionSliders)
QtCore.QObject.connect(
self.ui.comboBoxColorMap,
QtCore.SIGNAL(_fromUtf8("currentIndexChanged(int)")),
self.display2D)
QtCore.QObject.connect(self.ui.pushButChooseDataDir,
QtCore.SIGNAL(_fromUtf8("clicked()")),
self.showDialogSetDatadir)
QtCore.QObject.connect(self.ui.pushButChooseOutDataDir,
QtCore.SIGNAL(_fromUtf8("pressed()")),
self.showDialogSetOutDatadir)
QtCore.QObject.connect(self.ui.pushButCurrFrameBrowse,
QtCore.SIGNAL(_fromUtf8("pressed()")),
self.showDialogSetCurrFrame)
QtCore.QObject.connect(self.ui.pushButIgnoreWLC,
QtCore.SIGNAL(_fromUtf8("clicked()")),
self.ignoreWLC)
# QtCore.QObject.connect(self.ui.pushButIgnoreWLC, QtCore.SIGNAL(_fromUtf8("clicked()")), self.ignoreWLC)
QtCore.QObject.connect(self.ui.pushButIgnoreWLClibArc,
QtCore.SIGNAL(_fromUtf8("clicked()")),
self.useLibArc)
QtCore.QObject.connect(self.ui.useSB1Check,
QtCore.SIGNAL(_fromUtf8("toggled(bool)")),
self.updateAll)
QtCore.QObject.connect(self.ui.useSB2Check,
QtCore.SIGNAL(_fromUtf8("toggled(bool)")),
self.updateAll)
# QtCore.QObject.connect(self.ui.raw2DGraph,QtCore.SIGNAL("MOUSE_MOVED"),self.display2Dlabel)
QtCore.QObject.connect(self.ui.raw2DGraph,
QtCore.SIGNAL(_fromUtf8("enterEvent()")),
self.display2Dlabel)
# ---- Set up logging
# see: https://docs.python.org/2/howto/logging-cookbook.html
logfile = 'QL%s.log' % (time.strftime("%Y%m%d", time.gmtime()))
logging.basicConfig(filename=logfile,
level=logging.DEBUG,
format='%(asctime)s %(message)s')
# logFormatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s', datefmt='%m-%d %H:%M:%S')
logFormatter = logging.Formatter(
'%(asctime)s %(levelname)-8s %(message)s', datefmt='%H:%M:%S')
self.logBuffer = logBuffer()
self.logBuffer.bufferMessage.connect(self.on_logBuffer_bufferMessage)
### logFormatter = logging.Formatter('%(levelname)s: %(message)s')
logHandler = logging.StreamHandler(self.logBuffer)
logHandler.setFormatter(logFormatter)
# add some colours:
# logging.addLevelName( logging.WARNING, "\033[1;31m%s\033[1;0m" % logging.getLevelName(logging.WARNING))
# logging.addLevelName( logging.ERROR, "\033[1;41m%s\033[1;0m" % logging.getLevelName(logging.ERROR))
# logging.addLevelName( logging.INFO, "\x1b[31m%s" % logging.getLevelName(logging.INFO)) #**
self.logger = logging.getLogger()
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logHandler)
self.logtext = ''
#self.ui.scrollAreaWidgetContents = QtGui.QTextEdit()
self.logger.info('GUI started')
self.makeDummy()
#-- Monitor data directory for new images
wm = WatchManager()
notifier = ThreadedNotifier(wm, self.process_IN_CREATE)
notifier.start()
self.notifier = notifier
mask = EventsCodes.ALL_FLAGS['IN_CREATE']
wdd = wm.add_watch(self.pars['dataDir'], mask, rec=False)
#--
# read values from GUI:
self.pars['showSideBands'] = self.ui.checkBoxShowSB.isChecked()
self.pars['smoothPix'] = self.ui.spinBoxSmoothWidth.value()
self.logger.debug(self.ui.checkBoxShowSB.isChecked())
# self.profileGraph = matplotlibWidgetwBar(self.centralWidget)
# self.profPlot = matplotlibWidget(self.ui.centralWidget)
### sc = matplotlibWidgetwBar(self.ui.profileGraph)
# **** clunky, but effective!
#profWin = matplotlibWidgetwBar(self.ui.centralWidget)
profWin = matplotlibWidget(self.ui.centralWidget)
#self.profPlot.setGeometry(QtCore.QRect(1100, 140, 330, 271))
self.profWin = profWin
# imname = 'a3000134.fits'
# self.logger.info(imname)
# self.image = self.pars['dataDir']+imname
# data2D,hdr = fits.getdata(self.image, header=True)
# self.data2D = data2D
# self.pars['currFrame'] = imname
# self.pars['obsType'] = hdr[self.imtype]
# print self.pars['obsType']
# self.ui.lineEditCurrFrame.setText(pars['currFrame'])
self.plotProfile()
#self.profWin.mpl_toolbar.zoom()
#graph2D = matplotlibWidget(self.ui.centralWidget)
graph2D = matplotlibWidgetwBar(self.ui.centralWidget)
self.graph2D = graph2D
self.display2D()
specWin = matplotlibWidget(self.ui.centralWidget)
self.specWin = specWin
# self.logger.debug('**',self.sky1d)
self.display1D()
"""
def main(**options):
"""
Main function. It will create instances of Odoo Server, EventHandler, ThreadedNotifier and
ProcessPool.
"""
logger = logging.getLogger(PROGNAME)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s: %(message)s")
logfile = options["logfile"]
if logfile is None:
streamH = logging.StreamHandler(sys.stdout)
else:
streamH = logging.FileHandler(logfile)
streamH.setFormatter(formatter)
logger.addHandler(streamH)
logger.info("Started")
odoo = OdooInstance(
logger,
db=options["db"],
user=options["user"],
password=options["password"],
host=options["host"],
port=options["port"],
)
odoo.connect_to_odoo()
exit_code = 0
if odoo.is_connected:
count = 0
while True:
count += 1
try:
event_handler = EventHandler(
odoo=odoo,
logger=logger,
dir_to_watch=options["dir_to_watch"],
regex=options["regex"],
delimiter=options["delimiter"],
)
# start loop handler
masks = IN_CLOSE_WRITE | IN_DELETE | IN_DELETE_SELF
wm = WatchManager()
wm.add_watch(event_handler.dir_to_watch, masks)
notifier = ThreadedNotifier(wm, default_proc_fun=event_handler)
notifier.start()
pool = ProcessPool(logger, odoo)
pool.loop()
except (KeyboardInterrupt, EOFError):
logger.info("\nReceived signal to exit! Goodbye")
break
except:
logger.error("ERROR! \n%s" % traceback.print_exception(*sys.exc_info()))
finally:
if notifier:
notifier.stop()
if count >= 50:
logger.error("ERROR! Too many tries, aborting!")
break
else:
logger.critical("Can't connect to odoo")
exit_code = 1
exit(exit_code)
class FilesystemMonitor(object):
"""
FileMonitor Class keeps track of all files down a tree starting at the root
"""
def __init__(self, searcher):
self.searcher = searcher
self._thread_pool = ThreadPool(THREAD_POOL_WORKS)
# Add a watch to the root of the dir
self.watch_manager = WatchManager()
self.notifier = ThreadedNotifier(self.watch_manager, FileProcessEvent(self))
self.notifier.start()
self._build_exclude_list()
def _build_exclude_list(self):
log.info("[FileMonitor] Set Regexs for Ignore List")
self._exclude_regexs = []
# Complie Ignore list in to a list of regexs
for ignore in self.searcher.configuration.get_value("EXCLUDE_LIST"):
ignore = ignore.strip()
ignore = ignore.replace(".", "\.")
ignore = ignore.replace("*", ".*")
ignore = "^"+ignore+"$"
log.debug("[FileMonitor] Ignore Regex = %s" % ignore)
self._exclude_regexs.append(re.compile(ignore))
def change_root(self, previous_root):
self._thread_pool.clearTasks()
wd = self.watch_manager.get_wd(previous_root)
if wd:
self.watch_manager.rm_watch(wd, rec=True)
self.searcher.clear_database()
self.add_directory(self.searcher.current_root)
def add_directory(self, path):
"""
Starts a WalkDirectoryThread to add the directory
"""
basename = os.path.basename(path)
if self.validate(basename):
self.watch_manager.add_watch(path, EVENT_MASK)
self._thread_pool.queueTask(self.walk_directory, path)
def add_file(self, path, name):
"""
Add a single file to the databse
"""
if self.validate(name):
self.searcher.add_file(path, name)
def remove_file(self, path, name):
self.searcher.remove_file(path, name)
def remove_directory(self, path):
self.searcher.remove_directory(path)
def walk_directory(self, root):
"""
From a give root of a tree this method will walk through ever branch
and return a generator.
"""
if os.path.isdir(root):
names = os.listdir(root)
for name in names:
try:
file_stat = os.lstat(os.path.join(root, name))
except os.error:
continue
if stat.S_ISDIR(file_stat.st_mode):
self.add_directory(os.path.join(root, name))
else:
if not stat.S_ISLNK(file_stat.st_mode):
self.add_file(root, name)
def finish(self):
wd = self.watch_manager.get_wd(self.searcher.current_root)
self.watch_manager.rm_watch(wd, rec=True)
self.notifier.stop()
self._thread_pool.joinAll(waitForTasks=False)
def validate(self, name):
# Check to make sure the file not in the ignore list
for ignore_re in self._exclude_regexs:
if ignore_re.match(name):
log.debug("[WalkDirectoryThread] ##### Ignored %s #####", name)
return False
log.debug("[WalkDirectoryThread] # Passed %s", name)
return True
class INotifier(object):
"""
Class providing an easy wrapper for listing to INotify kernel events.
"""
Write, Delete = range(2)
def __init__(self, path):
"""
Initialisation.
@param path (str) Full path to watch.
"""
self.path = path
if os.path.isdir(self.path):
self.path_dir = self.path
self.path_file = None
else:
self.path_dir = os.path.dirname(self.path)
self.path_file = os.path.basename(self.path)
self.callbacks = {}
wm = WatchManager()
wm.add_watch(self.path_dir, pyinotify.ALL_EVENTS)
self.inotifier = ThreadedNotifier(wm, self.__processINotify)
self.inotifier.start()
def __del__(self):
"""
Destruction. Unload (i.e. stop) the inotifier.
"""
self.unload()
def unload(self):
"""
Call this to stop listening for events. Should be called on shutdown.
"""
self.inotifier.stop()
def __processINotify(self, event):
"""
Called when an INotify was received. Call the applicable callback method based on the event type.
"""
if event.mask == pyinotify.IN_DELETE or event.mask == pyinotify.IN_MOVED_FROM:
if self.path_file == None or (self.path_file and event.name == self.path_file):
for c in self.__getCallbacks(INotifier.Delete):
c(event)
elif event.mask == pyinotify.IN_CLOSE_WRITE or event.mask == pyinotify.IN_MOVED_TO:
if self.path_file == None or (self.path_file and event.name == self.path_file):
for c in self.__getCallbacks(INotifier.Write):
c(event)
def __getCallbacks(self, type):
"""
Get the current callback methods for the given type.
@param type (INotifier.Write or INotifier.Delete) Type of callback get.
"""
return self.callbacks.get(type, [])
def addCallback(self, type, callback):
"""
Add a callback function to handle an event.
@param type (INotifier.Write or INotifier.Delete) Type of callback to add. Write is called on file writes,
Delete on file deletion.
@param callback (method) Method to call, should accept one parameter: the pyinotify
event.
"""
if not type in self.callbacks:
self.callbacks[type] = set()
self.callbacks[type].add(callback)
class Server(Command):
description = 'Run the calibre server in development mode conveniently'
MONOCLE_PATH = '../monocle'
def rebuild_monocole(self):
subprocess.check_call(['sprocketize', '-C', self.MONOCLE_PATH,
'-I', 'src', 'src/monocle.js'],
stdout=open('resources/content_server/read/monocle.js', 'wb'))
def launch_server(self):
print 'Starting server...\n'
with self.lock:
self.rebuild_monocole()
self.server_proc = p = subprocess.Popen(['calibre-server', '--develop'],
stderr=subprocess.STDOUT, stdout=self.server_log)
time.sleep(0.2)
if p.poll() is not None:
print 'Starting server failed'
raise SystemExit(1)
return p
def kill_server(self):
print 'Killing server...\n'
if self.server_proc is not None:
with self.lock:
if self.server_proc.poll() is None:
self.server_proc.terminate()
while self.server_proc.poll() is None:
time.sleep(0.1)
def watch(self):
if wm is not None:
self.notifier = ThreadedNotifier(wm, ProcessEvents(self))
self.notifier.start()
self.wdd = wm.add_watch(os.path.abspath('src'), mask, rec=True)
def reload_browser(self, delay=0.1):
time.sleep(delay)
try:
t = telnetlib.Telnet('localhost', 4242)
t.read_until("repl>")
t.write('BrowserReload();')
t.read_until("repl>")
t.close()
except:
print 'Failed to reload browser'
import traceback
traceback.print_exc()
def run(self, opts):
self.lock = RLock()
tdir = tempfile.gettempdir()
logf = os.path.join(tdir, 'calibre-server.log')
self.server_log = open(logf, 'ab')
self.prompt = 'Press Enter to kill/restart server. Ctrl+C to quit: '
print 'Server log available at:', logf
print
self.watch()
first = True
while True:
self.launch_server()
if not first:
self.reload_browser()
first = False
try:
raw_input(self.prompt)
except:
print
self.kill_server()
break
else:
self.kill_server()
print
if hasattr(self, 'notifier'):
self.notifier.stop()
def __init__(self, parent = None):
QtGui.QWidget.__init__(self, parent)
self.canvas = MplCanvas()
self.vbl = QtGui.QVBoxLayout()
self.vbl.addWidget(self.canvas)
#--
self.mpl_toolbar = NavigationToolbar(self.canvas, None)
self.vbl.addWidget(self.mpl_toolbar)
#--
self.setLayout(self.vbl)
self.extractWin_x0=self.extractWinSlider.lowerValue
self.extractWin_x1=self.extractWinSlider.upperValue
self.SB1_x0=self.SB1Slider.lowerValue
self.SB1_x1=self.SB1Slider.upperValue
self.SB2_x0=self.SB2Slider.lowerValue
self.SB2_x1=self.SB2Slider.upperValue
self.autoExtraction=True
self.lamp=None
self.grating=None
# self.raw2DGraph = QtGui.QGraphicsView(self.centralWidget)
# self.raw2DGraph = QtGui.QWidget(self.centralWidget)
self.raw2DGraph = matplotlibWidget(self.centralWidget)
self.raw2DGraph.setGeometry(QtCore.QRect(20, 40, 1061, 81))
self.raw2DGraph.setObjectName(_fromUtf8("raw2DGraph"))
self.skysub2DView = matplotlibWidget(self.centralWidget)
self.skysub2DView.setGeometry(QtCore.QRect(20, 160, 1061, 81))
self.skysub2DView.setObjectName(_fromUtf8("skysub2DView"))
self.useSB1Check = QtGui.QCheckBox(self.centralWidget)
self.useSB1Check.setGeometry(QtCore.QRect(1110, 560, 85, 21))
self.useSB1Check.setChecked(True)
self.useSB1Check.setObjectName(_fromUtf8("useSB1Check"))
self.useSB2Check = QtGui.QCheckBox(self.centralWidget)
self.useSB2Check.setGeometry(QtCore.QRect(1260, 560, 85, 21))
self.useSB2Check.setChecked(True)
self.useSB2Check.setObjectName(_fromUtf8("useSB2Check"))
# ----
# ---- 1D extracted spec
self.spec1DView = matplotlibWidgetwBar(self.centralWidget)
self.spec1DView.setGeometry(QtCore.QRect(20, 290, 1061, 381))
self.spec1DView.setObjectName(_fromUtf8("spec1DView"))
# ----
# ---- 2D spec related stuff
self.z0Fill = QtGui.QLineEdit(self.centralWidget)
self.z0Fill.setGeometry(QtCore.QRect(1140, 50, 113, 23))
self.z0Fill.setObjectName(_fromUtf8("z0Fill"))
self.z1Fill = QtGui.QLineEdit(self.centralWidget)
self.z1Fill.setGeometry(QtCore.QRect(1290, 50, 113, 23))
self.z1Fill.setObjectName(_fromUtf8("z1Fill"))
self.ScalingCombo = QtGui.QComboBox(self.centralWidget)
self.ScalingCombo.setGeometry(QtCore.QRect(1160, 80, 78, 24))
self.ScalingCombo.setObjectName(_fromUtf8("ScalingCombo"))
self.ScalingCombo.addItem(_fromUtf8(""))
self.ScalingCombo.addItem(_fromUtf8(""))
self.ScalingCombo.addItem(_fromUtf8(""))
self.zScaleUpdateBut = QtGui.QPushButton(self.centralWidget)
self.zScaleUpdateBut.setGeometry(QtCore.QRect(1280, 80, 99, 24))
self.zScaleUpdateBut.setObjectName(_fromUtf8("zScaleUpdateBut"))
# ----
# ---- log window and file loaders
# self.logView = QtGui.QListView(self.centralWidget)
self.logView = QtGui.QTextEdit(self.centralWidget)
self.logView.setReadOnly(True)
# self.logView.setGeometry(QtCore.QRect(20, 710, 531, 192))
self.logView.setGeometry(QtCore.QRect(20, 710, 731, 192))
self.logView.setObjectName(_fromUtf8("logView"))
self.CurrFrameBut = QtGui.QPushButton(self.centralWidget)
#self.CurrFrameBut.setGeometry(QtCore.QRect(740, 720, 99, 24))
self.CurrFrameBut.setGeometry(QtCore.QRect(940, 720, 99, 24))
self.CurrFrameBut.setObjectName(_fromUtf8("CurrFrameBut"))
self.currFrameFill = QtGui.QLineEdit(self.centralWidget)
#self.currFrameFill.setGeometry(QtCore.QRect(600, 720, 131, 23))
self.currFrameFill.setGeometry(QtCore.QRect(800, 720, 131, 23))
self.currFrameFill.setObjectName(_fromUtf8("currFrameFill"))
self.curArcBut = QtGui.QPushButton(self.centralWidget)
self.curArcBut.setEnabled(False)
self.curArcBut.setGeometry(QtCore.QRect(940, 770, 99, 24))
self.curArcBut.setObjectName(_fromUtf8("curArcBut"))
self.currArcFill = QtGui.QLineEdit(self.centralWidget)
self.currArcFill.setGeometry(QtCore.QRect(800, 770, 131, 23))
self.currArcFill.setObjectName(_fromUtf8("currArcFill"))
self.curFlatBut = QtGui.QPushButton(self.centralWidget)
self.curFlatBut.setEnabled(False)
self.curFlatBut.setGeometry(QtCore.QRect(940, 820, 99, 24))
self.curFlatBut.setObjectName(_fromUtf8("curFlatBut"))
self.currFlatFill = QtGui.QLineEdit(self.centralWidget)
self.currFlatFill.setGeometry(QtCore.QRect(800, 820, 131, 23))
self.currFlatFill.setObjectName(_fromUtf8("currFlatFill"))
self.curFluxCalBut = QtGui.QPushButton(self.centralWidget)
self.curFluxCalBut.setEnabled(False)
self.curFluxCalBut.setGeometry(QtCore.QRect(940, 870, 99, 24))
self.curFluxCalBut.setObjectName(_fromUtf8("curFluxCalBut"))
self.currFluxCalFill = QtGui.QLineEdit(self.centralWidget)
self.currFluxCalFill.setGeometry(QtCore.QRect(800, 870, 131, 23))
self.currFluxCalFill.setObjectName(_fromUtf8("currFluxCalFill"))
# -- temp stuff fold old spectrograph only (no grating keyword in header)
self.gratingCombo = QtGui.QComboBox(self.centralWidget)
self.gratingCombo.setGeometry(QtCore.QRect(1150, 790, 171, 24))
self.gratingCombo.setObjectName(_fromUtf8("gratingCombo"))
self.gratingCombo.addItem(_fromUtf8(""))
self.gratingCombo.addItem(_fromUtf8(""))
self.gratingCombo.addItem(_fromUtf8(""))
self.label_15 = QtGui.QLabel(self.centralWidget)
self.label_15.setGeometry(QtCore.QRect(1130, 760, 141, 16))
self.label_15.setObjectName(_fromUtf8("label_15"))
# --
self.parentDataDir=None #'/home/gilbank/Proj/CassSpect/data/Deatrick/'
self.imagetype=""
self.lastWLCFile=None
self.wlc=False
self.latestFrame=None ##self.parentDataDir+'a3000142.fits'
#self.ReadOutDelay = 2.0 # seconds # not used now
# ----
self.widget = QtGui.QWidget(self.centralWidget)
"""
self.widget.setGeometry(QtCore.QRect(740, 130, 333, 23))
self.widget.setObjectName(_fromUtf8("widget"))
"""
self.horizontalLayout = QtGui.QHBoxLayout(self.widget)
self.horizontalLayout.setMargin(0)
self.horizontalLayout.setObjectName(_fromUtf8("horizontalLayout"))
self.showSBCheck = QtGui.QCheckBox(self.widget)
self.showSBCheck.setChecked(True)
self.showSBCheck.setObjectName(_fromUtf8("showSBCheck"))
self.horizontalLayout.addWidget(self.showSBCheck)
self.showEWCheck = QtGui.QCheckBox(self.widget)
self.showEWCheck.setChecked(True)
self.showEWCheck.setObjectName(_fromUtf8("showEWCheck"))
self.horizontalLayout.addWidget(self.showEWCheck)
MainWindow.setCentralWidget(self.centralWidget)
self.menuBar = QtGui.QMenuBar(MainWindow)
self.menuBar.setGeometry(QtCore.QRect(0, 0, 1437, 21))
self.menuBar.setObjectName(_fromUtf8("menuBar"))
self.menuFile = QtGui.QMenu(self.menuBar)
self.menuFile.setObjectName(_fromUtf8("menuFile"))
MainWindow.setMenuBar(self.menuBar)
self.mainToolBar = QtGui.QToolBar(MainWindow)
self.mainToolBar.setObjectName(_fromUtf8("mainToolBar"))
MainWindow.addToolBar(QtCore.Qt.TopToolBarArea, self.mainToolBar)
self.statusBar = QtGui.QStatusBar(MainWindow)
self.statusBar.setObjectName(_fromUtf8("statusBar"))
MainWindow.setStatusBar(self.statusBar)
self.actionSet_data_dir = QtGui.QAction(MainWindow)
self.actionSet_data_dir.setObjectName(_fromUtf8("actionSet_data_dir"))
self.actionQuit = QtGui.QAction(MainWindow)
self.actionQuit.setObjectName(_fromUtf8("actionQuit"))
self.actionLoadData = QtGui.QAction(MainWindow)
self.actionLoadData.setObjectName(_fromUtf8("actionLoadData"))
self.menuFile.addAction(self.actionSet_data_dir)
self.menuFile.addAction(self.actionLoadData)
self.menuFile.addAction(self.actionQuit)
self.menuBar.addAction(self.menuFile.menuAction())
self.scaledImage=None
self.retranslateUi(MainWindow)
QtCore.QObject.connect(self.showEWCheck, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.displayRaw2D)
QtCore.QObject.connect(self.showSBCheck, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.displayRaw2D)
QtCore.QObject.connect(self.zScaleUpdateBut, QtCore.SIGNAL(_fromUtf8("clicked()")), self.updatezScaling)
QtCore.QObject.connect(self.autoExtractBut, QtCore.SIGNAL(_fromUtf8("clicked()")), self.updateAutoExtraction)
#QtCore.QObject.connect(self.showSBCheck, QtCore.SIGNAL(_fromUtf8("clicked()")), MainWindow.show)
QtCore.QObject.connect(self.SB1Slider, QtCore.SIGNAL(_fromUtf8("sliderReleased()")), self.readExtractionSliders)
QtCore.QObject.connect(self.SB2Slider, QtCore.SIGNAL(_fromUtf8("sliderReleased()")), self.readExtractionSliders)
QtCore.QObject.connect(self.extractWinSlider, QtCore.SIGNAL(_fromUtf8("sliderReleased()")), self.readExtractionSliders)
QtCore.QObject.connect(self.useSB1Check, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.updateAll)
QtCore.QObject.connect(self.useSB2Check, QtCore.SIGNAL(_fromUtf8("toggled(bool)")), self.updateAll)
QtCore.QObject.connect(self.actionLoadData, QtCore.SIGNAL(_fromUtf8("activated()")), self.showDialog)
# QtCore.QObject.connect(self.CurrFrameBut, QtCore.SIGNAL(_fromUtf8("clicked()")), self.showDialog)
QtCore.QObject.connect(self.actionSet_data_dir, QtCore.SIGNAL(_fromUtf8("activated()")), self.showDialogSetDatadir)
QtCore.QObject.connect(self.actionQuit, QtCore.SIGNAL(_fromUtf8("activated()")), self.closeEvent)
QtCore.QMetaObject.connectSlotsByName(MainWindow)
if(self.parentDataDir==None):
self.showDialogSetDatadir()
# -- pyinotify code for monitoring dir for new files:
wm = WatchManager()
# notifier = ThreadedNotifier(wm, PTmp())
notifier = ThreadedNotifier(wm, self.process_IN_CREATE)
# notifier = ThreadedNotifier(wm, self.processMonitorData)
# notifier = Notifier(wm, PTmp())
notifier.start()
mask = EventsCodes.ALL_FLAGS['IN_CREATE']
wdd = wm.add_watch(self.parentDataDir, mask, rec=False)
_redis = redis.Redis(host='localhost', port=6379, db=0)
#TODO: put into config file
_queue_name = "rsync_files_changed"
def process_IN_MOVED_TO(self, event):
if os.path.basename(event.name)[0] != ".":
self._redis.lpush(self._queue_name,
os.path.join(event.path, event.name))
path = "."
wm = WatchManager()
mask = IN_MOVED_TO
notifier = ThreadedNotifier(wm, FileRsyncWatcher())
wm.add_watch(path, mask, auto_add=True, rec=True)
notifier.start()
def fun_quit(a, b):
global flag_run
flag_run = False
notifier.stop()
signal.signal(signal.SIGINT, fun_quit)
signal.signal(signal.SIGTERM, fun_quit)
while flag_run:
time.sleep(1)
# print len(list_file_newer)#, #list_file_newer
class FilesystemMonitor(object):
"""
FileMonitor Class keeps track of all files down a tree starting at the root
"""
def __init__(self, searcher):
self.searcher = searcher
self._thread_pool = ThreadPool(THREAD_POOL_WORKS)
# Add a watch to the root of the dir
self.watch_manager = WatchManager()
self.notifier = ThreadedNotifier(self.watch_manager,
FileProcessEvent(self))
self.notifier.start()
self._build_exclude_list()
def _build_exclude_list(self):
log.info("[FileMonitor] Set Regexs for Ignore List")
self._exclude_regexs = []
# Complie Ignore list in to a list of regexs
for ignore in self.searcher.configuration.exclude_list:
ignore = ignore.strip()
ignore = ignore.replace(".", "\.")
ignore = ignore.replace("*", ".*")
ignore = "^" + ignore + "$"
log.debug("[FileMonitor] Ignore Regex = %s" % ignore)
self._exclude_regexs.append(re.compile(ignore))
def change_root(self, previous_root):
self._thread_pool.clearTasks()
wd = self.watch_manager.get_wd(previous_root)
if wd:
self.watch_manager.rm_watch(wd, rec=True)
self.searcher.clear_database()
self.add_directory(self.searcher.current_root)
def add_directory(self, path):
"""
Starts a WalkDirectoryThread to add the directory
"""
basename = os.path.basename(path)
if self.validate(basename):
self.watch_manager.add_watch(path, EVENT_MASK)
self._thread_pool.queueTask(self.walk_directory, path)
def add_file(self, path, name):
"""
Add a single file to the databse
"""
if self.validate(name):
self.searcher.add_file(path, name)
def remove_file(self, path, name):
self.searcher.remove_file(path, name)
def remove_directory(self, path):
self.searcher.remove_directory(path)
def walk_directory(self, root):
"""
From a give root of a tree this method will walk through ever branch
and return a generator.
"""
if os.path.isdir(root):
names = os.listdir(root)
for name in names:
try:
file_stat = os.lstat(os.path.join(root, name))
except os.error:
continue
if stat.S_ISDIR(file_stat.st_mode):
self.add_directory(os.path.join(root, name))
else:
if not stat.S_ISLNK(file_stat.st_mode):
self.add_file(root, name)
def finish(self):
wd = self.watch_manager.get_wd(self.searcher.current_root)
self.watch_manager.rm_watch(wd, rec=True)
self.notifier.stop()
self._thread_pool.joinAll(waitForTasks=False)
def validate(self, name):
# Check to make sure the file not in the ignore list
for ignore_re in self._exclude_regexs:
if ignore_re.match(name):
log.debug("[WalkDirectoryThread] ##### Ignored %s #####", name)
return False
log.debug("[WalkDirectoryThread] # Passed %s", name)
return True
class FileInputWatcher:
'''controlling class for file monitoring'''
def __init__(self, dir_to_watch, queue):
print 'FileInputWatcher Initialized'
# SBB20090903 Adding debugging capability, not processing multiple file drops into multiple directories.
if settings.DEBUG:
print '*************Debugging On*************'
self.queue = queue
self.dir_to_watch = dir_to_watch
# make a notifier (nothing)
self.notifier1 = None
self.notifier2 = None
def monitor(self):
'''The command to start monitoring a directory or set of them.'''
print 'Monitoring Directories: %s' % self.dir_to_watch
print "Watching started at %s" % (time.asctime())
if osiswin32:
print 'Watching win32 OS'
return self.watch_win32(self.dir_to_watch)
else:
print 'Watching POSIX OS'
#if settings.DEBUG:
#print 'sending to self.watch_posix_start()'
self.watch_posix_start()
#if settings.DEBUG:
#print "It returned from self.watch_posix_start()!"
#print "self.watch_posix_start() returned with value", result
return True
def stop_monitoring(self):
'''os independent method to stop monitoring, but only posix uses it.'''
#print "self.notifier1.started", self.notifier1.__getattribute__('started')
if isinstance(self.notifier1, ThreadedNotifier):
#if settings.DEBUG:
#print "self.notifier1 is an instance"
if isinstance(self.notifier2, ThreadedNotifier):
#if settings.DEBUG:
#print "self.notifier2 is an instance"
self.watch_posix_stop()
else:
if settings.DEBUG:
print "notifiers were not instantiated, so not calling self.watch_posix_stop() again"
print 'Done Monitoring'
def watch_win32(self, dir_to_watch):
'''os-specific watch command'''
# Loop forever, listing any file changes. The WaitFor... will
# time out every half a second allowing for keyboard interrupts
# to terminate the loop.
files_added = []
old_path_contents = []
new_path_contents = []
cnt = 0
try:
while 1:
cnt += 1
#print 'Watching %s' % cnt
#old_path_contents = os.listdir(dirToWatch)
for item in dir_to_watch:
change_handle = win32file.FindFirstChangeNotification(item, 0, win32con.FILE_NOTIFY_CHANGE_FILE_NAME)
old_path_contents.append(os.listdir(item))
result = win32event.WaitForSingleObject(change_handle, 500)
# If the WaitFor... returned because of a notification (as
# opposed to timing out or some error) then look for the
# changes in the directory contents.
if result == win32con.WAIT_OBJECT_0:
#new_path_contents = os.listdir(dirToWatch)
# build the new list with all the files from all dirs
for item in dir_to_watch:
new_path_contents.append(os.listdir(item))
files_added = [f for f in new_path_contents if not f in old_path_contents]
#files_deleted = [f for f in old_path_contents if not f in new_path_contents]
if files_added:
print
print time.asctime ()
print "Added:", files_added or "Nothing"
return files_added
#print "Deleted:", files_deleted or "Nothing"
win32file.FindNextChangeNotification(change_handle)
except KeyboardInterrupt:
return []
#finally:
# win32file.FindCloseChangeNotification(change_handle)
def watch_posix_start(self):
'''os-specific command to watch'''
# test to see if we already have a notifier object, if not, make it, otherwise we are already watching a set of folders
if self.notifier1 == None and self.notifier2 == None:
try:
pyinotify.compatibility_mode()
print 'pyinotify running in compatibility mode'
except:
print 'pyinotify running in standard mode'
try:
#mask = EventsCodes.IN_CREATE |EventsCodes.IN_MOVED_TO
mask = pyinotify.ALL_EVENTS
#ECJ20100831 Reason why we have two threads: it never returns control ever to the main loop if only one thread, so no ctrl+c
#The second thread is a dummy, so it performs switching/relinquishing control
#Eventually, we want to watch many folders, so that is why we are using the ThreadedNotifier, versus the recommended Notifier.
#Even then, Notifier is still probably the way to go, but we'll use this as long as it works, because then you don't have to poll/While loop
# Thread #1
watch_manager1 = WatchManager()
self.notifier1 = ThreadedNotifier(watch_manager1, EventHandler(self.queue))
self.notifier1.start()
print 'Starting the threaded notifier on ', self.dir_to_watch
watch_manager1.add_watch(self.dir_to_watch, mask)
# Thread #2
watch_manager2 = WatchManager()
self.notifier2 = ThreadedNotifier(watch_manager2, EventHandlerDummy(self.queue))
self.notifier2.start()
#just watch any place, but don't remove this or Ctrl+C will not work
watch_manager2.add_watch(settings.BASE_PATH, mask)
if settings.DEBUG:
print "both notifiers started"
except KeyboardInterrupt:
print "Keyboard Interrupt in notifier"
self.notifier1.stop()
self.notifier2.stop()
return
except NameError:
self.notifier1.stop()
self.notifier2.stop()
return ['POSIX Watch Error']
except:
print "General exception caught within notifier while loop, stopping both notifiers now"
self.notifier1.stop()
self.notifier2.stop()
# SBB20090903 Turn on verbose mode
self.notifier1.VERBOSE = settings.DEBUG
print "returning to calling function"
return True
def watch_posix_stop(self):
'os specific call to stop monitoring'
print 'Stopping the threaded notifiers.'
self.notifier1.stop()
print "stopped self.notifier1.stop()"
self.notifier2.stop()
print "stopped self.notifier2.stop()"
return
try:
if pattern.match(i) and datetime.datetime.strptime(
i[:16], "%Y-%m-%d_%H-%M") >= now:
timedict[i[:16]] = "%s/%s" % (AUTODIR, i)
except Exception, e:
logging.error(e)
# 3. crea e avvia il thread Pusher
pusher = Pusher("localhost")
pusher.start()
# crea e avvia il ThreadedNotifier
wm_drop = WatchManager()
dropboxmask = IN_MOVED_TO
notifier_drop = ThreadedNotifier(wm_drop, Dropboxer())
notifier_drop.start()
wm_auto = WatchManager()
autodirmask = IN_CLOSE_WRITE | IN_DELETE
notifier_auto = ThreadedNotifier(wm_auto, Autodir())
notifier_auto.start()
wdd_dropbox = wm_drop.add_watch(AUTOSHAREDIR, dropboxmask, rec=True)
wdd_autodir = wm_auto.add_watch(AUTODIR, autodirmask, rec=True)
if __name__ == "__main__":
daemonize('/dev/null', '/tmp/auto.log', '/tmp/auto.log')
main()
def run():
if not client_status.is_registered:
print 'Sending info about new client...'
status, content = api.hi(platform.uname())
print content
if status == 200:
client_status.is_registered = True
client_status.save()
else:
exit('Aborted')
if os.path.exists(CRASH_PATH):
crash = os.stat(CRASH_PATH)
if crash.st_size:
with open(CRASH_PATH) as f:
crash_info = f.read()
status = api.report_crash(crash_info, crash.st_mtime)
if status == 200:
log.info('Crash reported')
os.remove(CRASH_PATH)
context = DaemonContext(pidfile=PIDLockFile(PIDFILE_PATH),
signal_map={signal.SIGTERM: on_stop},
stderr=open(CRASH_PATH, 'w'))
context.files_preserve = map(
lambda h: h.stream,
filter(lambda h: isinstance(h, FileHandler), log.logger.handlers))
print 'Starting daemon'
with context:
log.info('Daemon started')
log.info('Build filesystem image')
basepath = '/'
root = FSNode(basepath, ignore=IGNORE_PATHS)
root_d = root.as_dict()
root_str = json.dumps(root_d)
h = sha(root_str).hexdigest()
if not client_status.fshash or client_status.fshash != h:
status, content = api.set_fs(root_str)
if status == 200:
client_status.fshash = h
client_status.save()
log.info('Filesystem image updated')
else:
log.error('Filesystem image update failed')
log.info('Create watch manager')
wm = WatchManager()
changelog = []
global notifier
notifier = ThreadedNotifier(wm, FSEvent(changelog=changelog))
notifier.start()
mask = IN_CREATE | IN_DELETE | IN_MOVED_FROM | IN_MOVED_TO
log.info('Start watching filesystem changes')
for item in os.listdir(basepath):
path = os.path.join(basepath, item)
if item in IGNORE_PATHS or os.path.islink(path):
continue
wm.add_watch(path, mask, rec=True)
actions = [
Action(FS_UPLOAD_PERIOD, upload_fs, changelog),
Action(LOG_UPLOAD_PERIOD, upload_log),
Action(RESTORE_CHECK_PERIOD, restore)
]
backup_action = lambda: Action(backup.get_next(), make_backup)
def on_schedule_update(actions=actions):
actions[-1] = backup_action()
update_schedule_action = lambda: Action(SCHEDULE_UPDATE_PERIOD,
update_schedule,
on_update=on_schedule_update)
if update_schedule() or client_status.schedule:
actions.append(update_schedule_action())
actions.append(backup_action())
else:
def on_schedule_download(actions=actions):
actions[-1] = update_schedule_action()
actions.append(backup_action())
actions.append(
Action(SCHEDULE_UPDATE_PERIOD,
update_schedule,
on_update=on_schedule_download,
on_404=True))
log.info('Start main loop')
while True:
action = min(actions)
log.info('Next action is %s' % action)
time.sleep(action.time_left())
action()
handler = ScoreHandler()
handlerloc = LocHandler()
notifier = ThreadedNotifier(wm, handler)
notifierloc = ThreadedNotifier(wmloc, handlerloc)
wm.add_watch(FILE100, IN_CLOSE_WRITE)
wm.add_watch(FILE111, IN_CLOSE_WRITE)
wm.add_watch(FILE120p3, IN_CLOSE_WRITE)
wm.add_watch(FILE120p4, IN_CLOSE_WRITE)
wm.add_watch(FILE120p6, IN_CLOSE_WRITE)
wm.add_watch(FILE120p14, IN_CLOSE_WRITE)
wm.add_watch(FILE120p16, IN_CLOSE_WRITE)
wm.add_watch(FILE120p21, IN_CLOSE_WRITE)
wm.add_watch(FILEr48, IN_CLOSE_WRITE)
wm.add_watch(FILEr59, IN_CLOSE_WRITE)
wmloc.add_watch(LOCDIR, IN_CLOSE_WRITE)
notifier.start()
notifierloc.start()
if ANCRSS == True:
c.execute_delayed(60, check_rss)
if READTWIT == True:
c.execute_delayed(30, check_tweets)
irc.process_forever()
class FSMonitorInotify(FSMonitor):
"""inotify support for FSMonitor"""
EVENTMAPPING = {
FSMonitor.CREATED : pyinotify.IN_CREATE,
FSMonitor.MODIFIED : pyinotify.IN_MODIFY | pyinotify.IN_ATTRIB,
FSMonitor.DELETED : pyinotify.IN_DELETE,
FSMonitor.MONITORED_DIR_MOVED : pyinotify.IN_MOVE_SELF,
FSMonitor.DROPPED_EVENTS : pyinotify.IN_Q_OVERFLOW,
}
def __init__(self, callback, persistent=False, trigger_events_for_initial_scan=False, ignored_dirs=[], dbfile="fsmonitor.db"):
FSMonitor.__init__(self, callback, persistent, trigger_events_for_initial_scan, ignored_dirs, dbfile)
self.wm = None
self.notifier = None
def __fsmonitor_event_to_inotify_event(self, event_mask):
"""map an FSMonitor event to an inotify event"""
inotify_event_mask = 0
for fsmonitor_event_mask in self.__class__.EVENTMAPPING.keys():
if event_mask & fsmonitor_event_mask:
inotify_event_mask = inotify_event_mask | self.__class__.EVENTMAPPING[fsmonitor_event_mask]
return inotify_event_mask
def __add_dir(self, path, event_mask):
"""override of FSMonitor.__add_dir()"""
# Perform an initial scan of the directory structure. If this has
# already been done, then it will return immediately.
if self.persistent:
if self.trigger_events_for_initial_scan:
FSMonitor.generate_missed_events(self, path, event_mask)
else:
self.pathscanner.initial_scan(path)
event_mask_inotify = self.__fsmonitor_event_to_inotify_event(event_mask)
# Use the inotify API to monitor a directory.
wdd = self.wm.add_watch(path, event_mask_inotify, proc_fun=self.process_event, rec=True, auto_add=True)
if wdd is None:
raise MonitorError, "Could not monitor %s" % path
return None
else:
self.monitored_paths[path] = MonitoredPath(path, event_mask, wdd)
self.monitored_paths[path].monitoring = True
# Generate the missed events.
if self.persistent:
FSMonitor.generate_missed_events(self, path)
return self.monitored_paths[path]
def __remove_dir(self, path):
"""override of FSMonitor.__remove_dir()"""
if path in self.monitored_paths.keys():
wd = self.monitored_paths[path].fsmonitor_ref
# TODO: figure out why this won't work, it seems this fails due to
# a bug in pyinotify?
#self.wm.rm_watch(wd, rec=True, quiet=True)
del self.monitored_paths[path]
def run(self):
# Setup. Ensure that this isn't interleaved with any other thread, so
# that the DB setup continues as expected.
self.lock.acquire()
FSMonitor.setup(self)
self.process_event = FSMonitorInotifyProcessEvent(self)
self.lock.release()
# Set up inotify.
self.wm = WatchManager()
self.notifier = ThreadedNotifier(self.wm, self.process_event)
self.notifier.start()
while not self.die:
self.__process_queues()
time.sleep(0.5)
self.notifier.stop()
def stop(self):
"""override of FSMonitor.stop()"""
# Let the thread know it should die.
self.lock.acquire()
self.die = True
self.lock.release()
# Stop monitoring each monitored path.
for path in self.monitored_paths.keys():
self.__remove_dir(path)
def __process_queues(self):
# Process add queue.
self.lock.acquire()
if not self.add_queue.empty():
(path, event_mask) = self.add_queue.get()
self.lock.release()
self.__add_dir(path, event_mask)
else:
self.lock.release()
# Process remove queue.
self.lock.acquire()
if not self.remove_queue.empty():
path = self.add_queue.get()
self.lock.release()
self.__remove_dir(path)
else:
self.lock.release()
class FileWatcher(ProcessEvent):
""" FileWatcher -> Starts an INotify thread to watch a directory for
file changes.
"""
def __init__(self):
""" FileWatcher(directory) -> Watch the directory for changes.
"""
if not pyinotify:
raise Exception("pyinotify is not loaded.")
super(FileWatcher, self).__init__()
self._file_callback_dict = {}
self._watch_manager = WatchManager()
self._events_mask = EventsCodes.ALL_FLAGS['IN_MODIFY']
self._notifier = ThreadedNotifier(self._watch_manager, self)
self._notifier.setDaemon(True)
@classmethod
def check(cls):
""" Returns true if pyinotify is loaded otherwise false.
"""
return pyinotify
def is_running(self):
""" Returns a boolean indecating the state of the notifier.
"""
return self._notifier.isAlive()
def start(self):
""" Start the notifier thread.
"""
self._notifier.start()
def stop(self):
""" Stop the notifier thread.
"""
self._notifier.stop()
def add_directory(self, directory):
""" add_directory(directory) -> Add a directory to watch.
"""
dir_watch = self._watch_manager.add_watch(directory,
self._events_mask,
rec=True)
def remove_directory(self, directory):
""" remove_directory(directory) -> Remove a directory from the watch.
"""
self._watch_manager.rm_watch(directory, rec=True)
def has_file(self, filename):
""" Returns a boolean indecating if the file is being watched.
"""
return filename in self._file_callback_dict
def add_file(self, filename, callback, *user_data):
""" add_file(filename, callback, *user_data) -> Add a file to watch
with its callback and optional user_data.
"""
self._file_callback_dict[filename] = (callback, user_data)
def remove_file(self, filename):
""" remove_file(filename) -> Remove the file from the watch list.
"""
return self._file_callback_dict.pop(filename, None)
def process_IN_MODIFY(self, event):
""" Process modify events.
"""
filename = os.path.join(event.path, event.name)
callback_data = self._file_callback_dict.get(filename, ())
if callback_data:
callback = callback_data[0]
glib.idle_add(callback, callback_data[1:])
