def _assert_action_after_watcher(self, process_asap, assertions_cb):
events = []
def callback(event):
events.append(event)
import os
import time
from fsevents import Stream
from fsevents import Observer
observer = Observer(process_asap=process_asap)
observer.start()
stream = Stream(callback, self.tempdir, file_events=True)
observer.schedule(stream)
# add single file
del events[:]
f = open(os.path.join(self.tempdir, "test"), "w")
f.write("abc")
f.flush()
f.close()
time.sleep(0.2)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
os.unlink(f.name)
assertions_cb(events, f)
def test_single_file_added_with_observer_unscheduled(self):
events = []
def callback(*args):
events.append(args)
f, path = self._make_temporary()
from fsevents import Stream
stream = Stream(callback, path)
from fsevents import Observer
observer = Observer()
observer.start()
import time
while not observer.isAlive():
time.sleep(0.1)
observer.schedule(stream)
observer.unschedule(stream)
# add single file
del events[:]
f.close()
time.sleep(0.1)
# stop and join observer
observer.stop()
observer.join()
self.assertEqual(events, [])
def test_existing_directories_are_not_reported(self):
import os
from fsevents import Stream, Observer
events = []
def callback(event):
events.append(event)
stream = Stream(callback, self.tempdir, file_events=True)
new1 = os.path.join(self.tempdir, "newdir1")
new2 = os.path.join(self.tempdir, "newdir2")
os.mkdir(new1)
observer = Observer()
observer.schedule(stream)
observer.start()
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
time.sleep(1)
os.mkdir(new2)
try:
time.sleep(1.1)
observer.stop()
observer.unschedule(stream)
observer.join()
from fsevents import IN_CREATE
self.assertEqual(len(events), 1)
self.assertEqual(events[0].mask, IN_CREATE)
self.assertEqual(events[0].name, os.path.realpath(new2))
finally:
os.rmdir(new1)
os.rmdir(new2)
def test_start_then_watch(self):
events = []
def callback(*args):
events.append(args)
f, path = self._make_temporary()
from fsevents import Stream
stream = Stream(callback, path)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
f.close()
time.sleep(0.2)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
self.assertEqual(events, [(path, self.create_and_remove_mask)])
def test_start_no_watch(self):
events = []
def callback(*args):
events.append(args)
from fsevents import Observer
observer = Observer()
f, path = self._make_temporary()
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
f.close()
time.sleep(0.2)
# stop and join observer
observer.stop()
observer.join()
self.assertEqual(events, [])
def test_single_file_added_multiple_streams(self):
events = []
def callback(*args):
events.append(args)
f, path = self._make_temporary()
from fsevents import Stream
stream1 = Stream(callback, path)
stream2 = Stream(callback, path)
from fsevents import Observer
observer = Observer()
observer.schedule(stream1)
observer.schedule(stream2)
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
time.sleep(0.1)
del events[:]
f.close()
time.sleep(0.2)
# stop and join observer
observer.stop()
observer.unschedule(stream1)
observer.unschedule(stream2)
observer.join()
self.assertEquals(events[0][0], path)
self.assertEquals(events[1][0], path)
def test_start_then_watch(self):
events = []
def callback(*args):
events.append(args)
f, path = self._make_temporary()
from fsevents import Stream
stream = Stream(callback, path)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
f.close()
time.sleep(0.2)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
self.assertEqual(events, [(path, self.create_and_remove_mask)])
def test_single_file_added_with_observer_unscheduled(self):
events = []
def callback(*args):
events.append(args)
f, path = self._make_temporary()
from fsevents import Stream
stream = Stream(callback, path)
from fsevents import Observer
observer = Observer()
observer.start()
import time
while not observer.isAlive():
time.sleep(0.1)
observer.schedule(stream)
observer.unschedule(stream)
# add single file
del events[:]
f.close()
time.sleep(0.1)
# stop and join observer
observer.stop()
observer.join()
self.assertEqual(events, [])
def test_existing_directories_are_not_reported(self):
import os
from fsevents import Stream, Observer
events = []
def callback(event):
events.append(event)
stream = Stream(callback, self.tempdir, file_events=True)
new1 = os.path.join(self.tempdir, "newdir1")
new2 = os.path.join(self.tempdir, "newdir2")
os.mkdir(new1)
observer = Observer()
observer.schedule(stream)
observer.start()
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
time.sleep(1)
os.mkdir(new2)
try:
time.sleep(1.1)
observer.stop()
observer.unschedule(stream)
observer.join()
from fsevents import IN_CREATE
self.assertEqual(len(events), 1)
self.assertEqual(events[0].mask, IN_CREATE)
self.assertEqual(events[0].name, os.path.realpath(new2))
finally:
os.rmdir(new1)
os.rmdir(new2)
def test_single_file_added_multiple_streams(self):
events = []
def callback(*args):
events.append(args)
f, path = self._make_temporary()
from fsevents import Stream
stream1 = Stream(callback, path)
stream2 = Stream(callback, path)
from fsevents import Observer
observer = Observer()
observer.schedule(stream1)
observer.schedule(stream2)
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
time.sleep(0.1)
del events[:]
f.close()
time.sleep(0.2)
# stop and join observer
observer.stop()
observer.unschedule(stream1)
observer.unschedule(stream2)
observer.join()
self.assertEquals(events, [(path, 0), (path, 0)])
def main():
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('path')
arg_parser.add_argument('-c', '--cmd', required=True, nargs='+', help='Console command for execute')
arg_parser.add_argument('-d', '--delay', type=int, default=5, nargs='?', help='Synchronization delay in seconds')
arg_parser.add_argument('-v', '--verbose', action='store_true', help='verbose flag')
try:
args = arg_parser.parse_args()
path = args.path
if not os.path.isdir(path):
raise IOError('Path `%s` is not a directory' % path)
event_handler = EventHandler()
timer = TimerInterval(callback_func, args.delay, event_handler=event_handler, cmd=args.cmd, verbose=args.verbose)
stream = Stream(event_handler.callback_handler, path, file_events=True)
observer = Observer()
observer.schedule(stream)
observer.start()
timer.start()
except IOError as e:
print e
except KeyboardInterrupt:
observer.unschedule(stream)
observer.stop()
timer.stop()
def test_single_file_added(self):
events = []
def callback(*args):
events.append(args)
f, path = self._make_temporary()
from fsevents import Observer
observer = Observer(callback, file_events=True)
observer.start()
observer.schedule(path)
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
f.close()
time.sleep(1.1)
# stop and join observer
observer.stop()
observer.join()
self.assertEquals(events[0][0], path)
def run():
observer = Observer()
observer.start()
# start foreman
forerunner = Forerunner()
forerunner.start()
# call callback when files under the cwd change
stream = Stream(forerunner.callback, './', file_events=True)
observer.schedule(stream)
while True:
try:
sleep(1)
except KeyboardInterrupt:
observer.unschedule(stream)
observer.stop()
observer.join()
foreman = forerunner.foreman
for i in range(1, 10):
if foreman.poll() is not None:
break
print "sleeping... %i" % i
sleep(1)
if foreman.poll() is None:
foreman.kill()
sleep(1)
break
class Dmon:
def __init__(self, dir, handlers):
self._handlers = handlers
self._dir = dir
def _call_handler(self, method, fname):
for handler in self._handlers:
getattr(handler, method)(fname)
def _run_pyinotify(self):
from lib import pyinotify
class PyInotifyEventProcessor(pyinotify.ProcessEvent):
def __init__(self, c_handler):
self._call = c_handler
def process_IN_CREATE(self, event):
self._call('on_create', event.pathname)
def process_IN_DELETE(self, event):
self._call('on_delete', event.pathname)
def process_IN_MODIFY(self, event):
self._call('on_update', event.pathname)
wm = pyinotify.WatchManager()
self._observer = pyinotify.Notifier(wm, PyInotifyEventProcessor(self._call_handler))
mask = pyinotify.ALL_EVENTS
wm.add_watch(self._dir, mask, rec=True)
while True:
self._observer.process_events()
if self._observer.check_events():
self._observer.read_events()
time.sleep(1)
def _run_macfse(self):
from lib import fsevents
from fsevents import Stream
from fsevents import Observer
def macfse_callback(event):
if event.mask in [256, 128]:
self._call_handler('on_create', event.name)
elif event.mask in [512, 64]:
self._call_handler('on_delete', event.name)
elif event.mask == 2:
self._call_handler('on_update', event.name)
self._observer = Observer()
self._stream = Stream(macfse_callback, self._dir, file_events=True)
self._observer.schedule(self._stream)
self._observer.start()
def start(self):
if 'darwin' in sys.platform.lower():
self._run_macfse()
elif 'linux' in sys.platform.lower():
self._run_pyinotify()
def stop(self):
if 'darwin' in sys.platform.lower():
self._observer.unschedule(self._stream)
self._observer.stop()
elif 'linux' in sys.platform.lower():
self._observer.stop()
def test_since_stream(self):
events = []
def callback(*args):
events.append(args)
# two files in same directory
import os
path1 = os.path.realpath(self._make_tempdir()) + '/'
f = self._make_temporary(path1)[0]
g = self._make_temporary(path1)[0]
from fsevents import Stream, FS_FLAGHISTORYDONE
stream = Stream(callback, path1, ids = True)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
#create one file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
f.close()
time.sleep(0.2)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
self.assertEqual(len(events),1)
self.assertEqual(events[0][:-1], (path1, self.create_and_remove_mask))
#create a second file
g.close()
stream = Stream(callback, path1, since = events[0][2])
del events[:]
# new observer
observer = Observer()
observer.schedule(stream)
observer.start()
time.sleep(0.2)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
self.assertEqual(len(events),2)
#FIXME: why do events arrive here in reversed order?
self.assertEqual(events[1], (path1, self.create_and_remove_mask))
self.assertEqual(events[0], (path1[:-1], FS_FLAGHISTORYDONE))
def watch_local_files():
print "Watching for changes to: {}".format(", ".join(TEMPLATES.keys()))
observer = Observer()
stream = Stream(event_callback, os.getcwd(), file_events=True)
observer.schedule(stream)
try:
observer.start()
while True:
time.sleep(86400)
except KeyboardInterrupt:
observer.stop()
def watch_local_files():
print "Watching for changes to: {}".format(", ".join(TEMPLATES.keys()))
observer = Observer()
stream = Stream(event_callback, os.getcwd(), file_events=True)
observer.schedule(stream)
try:
observer.start()
while True:
time.sleep(86400)
except KeyboardInterrupt:
observer.stop()
def main():
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('path')
arg_parser.add_argument('-c',
'--cmd',
required=True,
nargs='+',
help='Console command for execute')
arg_parser.add_argument('-d',
'--delay',
type=int,
default=5,
nargs='?',
help='Synchronization delay in seconds')
arg_parser.add_argument('-f',
'--files',
action='store_true',
help="show changed files snapshot")
arg_parser.add_argument('-v',
'--verbose',
action='store_true',
help='increase verbosity')
try:
args = arg_parser.parse_args()
path = args.path
if not os.path.isdir(path):
raise IOError('Path `%s` is not a directory' % path)
event_handler = EventHandler()
callback_params = {
'event_handler': event_handler,
'cmd': args.cmd,
'verbose': args.verbose,
'show_files': args.files
}
timer = TimerInterval(callback_func, args.delay, **callback_params)
stream = Stream(event_handler.callback_handler, path, file_events=True)
observer = Observer()
observer.schedule(stream)
observer.start()
timer.start()
except IOError as e:
print e
except KeyboardInterrupt:
observer.unschedule(stream)
observer.stop()
timer.stop()
def loop(self):
observer = Observer()
observer.schedule(self.stream)
observer.daemon = True
observer.start()
try:
while True:
# Cannot use observer.join() as it is not interrupted by
# Ctrl-C
time.sleep(10000)
finally:
observer.unschedule(self.stream)
observer.stop()
def loop(self):
observer = Observer()
observer.schedule(self.stream)
observer.daemon = True
observer.start()
try:
while True:
# Cannot use observer.join() as it is not interrupted by
# Ctrl-C
time.sleep(10000)
finally:
observer.unschedule(self.stream)
observer.stop()
def test_multiple_files_added(self):
events = []
def callback(*args):
events.append(args)
from fsevents import Observer
observer = Observer()
from fsevents import Stream
observer.start()
# wait until activation
import time
while not observer.isAlive():
time.sleep(0.1)
time.sleep(0.1)
# two files in same directory
import os
path1 = os.path.realpath(self._make_tempdir()) + '/'
f = self._make_temporary(path1)[0]
g = self._make_temporary(path1)[0]
# one file in a separate directory
path2 = os.path.realpath(self._make_tempdir()) + '/'
h = self._make_temporary(path2)[0]
stream = Stream(callback, path1, path2)
observer.schedule(stream)
try:
del events[:]
f.close()
g.close()
h.close()
time.sleep(0.2)
self.assertEqual(
sorted(events),
sorted([(path1, self.create_and_remove_mask),
(path2, self.create_and_remove_mask)]))
finally:
f.close()
g.close()
h.close()
os.rmdir(path1)
os.rmdir(path2)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
def test_file_moved_to_not_watched_path(self):
import os
events = []
def callback(event):
events.append(event)
from fsevents import Stream
not_watched_path = os.path.realpath(self._make_tempdir()) + '/'
stream = Stream(callback, self.tempdir, file_events=True)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
time.sleep(2.1)
src_name = os.path.join(self.tempdir, "test")
dst_name = os.path.join(not_watched_path, "test")
f = open(src_name, "w")
f.write("abc")
f.close()
time.sleep(1.0)
os.rename(src_name, dst_name)
time.sleep(0.1)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
os.unlink(dst_name)
os.rmdir(not_watched_path)
from fsevents import IN_CREATE, IN_MODIFY, IN_DELETE
self.assertEquals(len(events), 3)
# assert the creation events
self.assertEquals(events[0].mask, IN_CREATE)
self.assertEquals(events[0].name, os.path.realpath(src_name))
self.assertEquals(events[1].mask, IN_MODIFY)
self.assertEquals(events[1].name, os.path.realpath(src_name))
self.assertEquals(events[2].mask, IN_DELETE)
self.assertEquals(events[2].name, os.path.realpath(src_name))
def start_scanner(path):
"""
watch for file events in the supplied path
"""
try:
observer = Observer()
observer.start()
stream = Stream(file_modified, path, file_events=True)
observer.schedule(stream)
print "Watching for changes. Press Ctrl-C to stop."
while 1:
pass
except (KeyboardInterrupt, OSError, IOError):
observer.unschedule(stream)
observer.stop()
def main():
logging.basicConfig(level=logging.INFO)
load_config()
observer = Observer()
observer.setDaemon(True)
observer.start()
observer.schedule(Stream(callback, ".", file_events=True))
try:
while True:
time.sleep(10)
except KeyboardInterrupt:
pass
finally:
observer.stop()
print("")
def start_scanner(path):
"""
watch for file events in the supplied path
"""
try:
observer = Observer()
observer.start()
stream = Stream(file_modified, path, file_events=True)
observer.schedule(stream)
print "Watching for changes. Press Ctrl-C to stop."
while 1:
pass
except (KeyboardInterrupt, OSError, IOError):
observer.unschedule(stream)
observer.stop()
def main():
logging.basicConfig(level=logging.INFO)
load_config()
observer = Observer()
observer.setDaemon(True)
observer.start()
observer.schedule(Stream(callback, ".", file_events=True))
try:
while True:
time.sleep(10)
except KeyboardInterrupt:
pass
finally:
observer.stop()
print("")
def test_single_file_created_and_modified(self):
events = []
def callback(event):
events.append(event)
from fsevents import Stream
stream = Stream(callback, self.tempdir, file_events=True)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
time.sleep(2.1)
import os
f = open(os.path.join(self.tempdir, "test"), "w")
f.write("abc")
f.flush()
time.sleep(1.0)
f.write("def")
f.flush()
f.close()
time.sleep(0.1)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
os.unlink(f.name)
from fsevents import IN_CREATE, IN_MODIFY
self.assertEquals(len(events), 3)
# assert events related to the creations
self.assertEquals(events[0].mask, IN_CREATE)
self.assertEquals(events[0].name, os.path.realpath(f.name))
self.assertEquals(events[1].mask, IN_MODIFY)
self.assertEquals(events[1].name, os.path.realpath(f.name))
# assert events related to the modify
self.assertEquals(events[2].mask, IN_MODIFY)
self.assertEquals(events[2].name, os.path.realpath(f.name))
def test_multiple_files_added(self):
events = []
def callback(*args):
events.append(args)
from fsevents import Observer
observer = Observer(callback, file_events=True)
observer.start()
# wait until activation
import time
while not observer.isAlive():
time.sleep(0.1)
time.sleep(0.1)
# two files in same directory
import os
path1 = os.path.realpath(self._make_tempdir()) + '/'
f = self._make_temporary(path1)[0]
g = self._make_temporary(path1)[0]
# one file in a separate directory
path2 = os.path.realpath(self._make_tempdir()) + '/'
h = self._make_temporary(path2)[0]
observer.schedule(path1)
observer.schedule(path2)
try:
del events[:]
f.close()
g.close()
h.close()
time.sleep(0.2)
events = [e[0] for e in events]
self.assertEqual(sorted(events), sorted([path1, path2]))
finally:
f.close()
g.close()
h.close()
os.rmdir(path1)
os.rmdir(path2)
# stop and join observer
observer.stop()
observer.unschedule(path1)
observer.unschedule(path2)
observer.join()
def loop(self):
observer = Observer()
observer.schedule(self.stream)
observer.daemon = True
observer.start()
try:
while True:
try:
# Cannot use blocking get() as it is not interrupted by
# Ctrl-C
if self.wait_queue.get(10000) is True:
self.force_restart()
except Empty:
pass
finally:
observer.unschedule(self.stream)
observer.stop()
def main():
signal.signal(signal.SIGINT, signal_handler)
parser = argparse.ArgumentParser(description='Sync files to a remote server of scp.')
parser.add_argument('path', type=str, metavar=('local', 'remote'), nargs=2,
help='path for the monitor')
args = parser.parse_args()
print args.path
observer = Observer()
stream = Stream(lambda x, y: callback(x,y, args.path[1]), args.path[0])
observer.schedule(stream)
try:
observer.start()
while True: # instead of this infinite loop, you can do
pass # whatever processing you wanted
except KeyboardInterrupt:
observer.stop()
def main():
parser = argparse.ArgumentParser(description="Refresh browser tabs when local files change.")
parser.add_argument("path", help="The directory to watch for changes.")
parser.add_argument("keyword", help="Tabs with this keyword in their URL will be refreshed.")
parser.add_argument("-b", "--browser", help="Which browser to refresh.", choices=sorted(browser_reloaders.keys()), default="chrome")
args = parser.parse_args()
print("Watching {} for changes...".format(os.path.abspath(args.path)))
event_callback_partial = partial(event_callback, browsers=(args.browser,), keyword=args.keyword)
observer = Observer()
stream = Stream(event_callback_partial, sys.argv[1], file_events=True)
observer.schedule(stream)
try:
observer.start()
while True:
time.sleep(86400)
except KeyboardInterrupt:
observer.stop()
def loop(self):
observer = Observer()
observer.schedule(self.stream)
observer.daemon = True
observer.start()
try:
while True:
try:
# Cannot use blocking get() as it is not interrupted by
# Ctrl-C
if self.wait_queue.get(10000) is True:
self.force_restart()
except Empty:
pass
finally:
observer.unschedule(self.stream)
observer.stop()
def test_single_file_created_and_modified(self):
events = []
def callback(event):
events.append(event)
from fsevents import Stream
stream = Stream(callback, self.tempdir, file_events=True)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
time.sleep(2.1)
import os
f = open(os.path.join(self.tempdir, "test"), "w")
f.write("abc")
f.flush()
time.sleep(1.0)
f.write("def")
f.flush()
f.close()
time.sleep(0.1)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
os.unlink(f.name)
from fsevents import IN_CREATE, IN_MODIFY
self.assertEqual(len(events), 2)
self.assertEqual(events[0].mask, IN_CREATE)
self.assertEqual(events[0].name, os.path.realpath(f.name))
self.assertEqual(events[1].mask, IN_MODIFY)
self.assertEqual(events[1].name, os.path.realpath(f.name))
class MtimeFileWatcher(object):
SUPPORTS_MULTIPLE_DIRECTORIES = True
def __init__(self, directories, **kwargs):
self._changes = _changes = []
# Path to current module
module_dir = directories[0]
# Paths to watch
paths = [module_dir]
# Explicitly adding paths outside of module dir.
for path in NON_MODULE_DIRS:
paths.append(path)
self.observer = Observer()
def callback(event, mask=None):
# Get extension
try:
ext = os.path.splitext(event.name)[1]
except IndexError:
ext = None
# Add to changes if we're watching a file with this extension.
if ext in WATCHED_EXTENSIONS:
_changes.append(event.name)
self.stream = Stream(callback, file_events=True, *paths)
def start(self):
self.observer.schedule(self.stream)
self.observer.start()
def changes(self, timeout=None):
time.sleep(0.1)
changed = set(self._changes)
del self._changes[:]
return changed
def quit(self):
self.observer.unschedule(self.stream)
self.observer.stop()
self.observer.join()
class Monitor:
def __init__(self):
self.obs = Observer()
self.stream = Stream(self.callback, SOURCE_PATH, file_events=True)
def callback(self, event): #file event callback function
if event.name.endswith(VALID_SUFFIX) and os.path.isfile(event.name):
sc.upload_track(event.name)
def start(self):
print "Starting monitor..."
self.obs.start()
self.obs.schedule(self.stream)
def stop(self):
print "Stopping monitor..."
self.obs.unschedule(self.stream)
self.obs.stop()
def test_single_file_moved(self):
events = []
def callback(event):
events.append(event)
import os
f = open(os.path.join(self.tempdir, "test"), "w")
f.write("abc")
f.flush()
f.close()
from fsevents import Stream
stream = Stream(callback, self.tempdir, file_events=True)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
time.sleep(2.1)
new = "%s.new" % f.name
os.rename(f.name, new)
time.sleep(0.1)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
os.unlink(new)
from fsevents import IN_MOVED_FROM
from fsevents import IN_MOVED_TO
self.assertEqual(len(events), 2)
self.assertEqual(events[0].mask, IN_MOVED_FROM)
self.assertEqual(events[0].name, os.path.realpath(f.name))
self.assertEqual(events[1].mask, IN_MOVED_TO)
self.assertEqual(events[1].name, os.path.realpath(new))
self.assertEqual(events[0].cookie, events[1].cookie)
def test_single_file_added_to_subdirectory(self):
events = []
def callback(*args):
events.append(args)
import os
directory = self._make_tempdir()
subdirectory = os.path.realpath(os.path.join(directory,
'subdir')) + '/'
os.mkdir(subdirectory)
import time
time.sleep(0.1)
try:
from fsevents import Stream
stream = Stream(callback, directory)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
# add single file
while not observer.isAlive():
time.sleep(0.1)
del events[:]
f = open(os.path.join(subdirectory, "test"), "w")
f.write("abc")
f.close()
time.sleep(0.2)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
self.assertEqual(len(events), 1)
self.assertEqual(events, [(subdirectory, self.modified_mask)])
finally:
os.unlink(f.name)
os.rmdir(subdirectory)
os.rmdir(directory)
class Monitor: def __init__( self ): self.obs = Observer() self.stream = Stream ( self.callback, SOURCE_PATH, file_events=True ) def callback( self, event ): #file event callback function if event.name.endswith( VALID_SUFFIX ) and os.path.isfile( event.name ): sc.upload_track ( event.name ) def start( self ): print "Starting monitor..." self.obs.start() self.obs.schedule( self.stream ) def stop( self ): print "Stopping monitor..." self.obs.unschedule( self.stream ) self.obs.stop()
def test_single_file_moved(self):
events = []
def callback(event):
events.append(event)
import os
f = open(os.path.join(self.tempdir, "test"), "w")
f.write("abc")
f.flush()
f.close()
from fsevents import Stream
stream = Stream(callback, self.tempdir, file_events=True)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
time.sleep(2.1)
new = "%s.new" % f.name
os.rename(f.name, new)
time.sleep(0.1)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
os.unlink(new)
from fsevents import IN_MOVED_FROM
from fsevents import IN_MOVED_TO
self.assertEqual(len(events), 2)
self.assertEqual(events[0].mask, IN_MOVED_FROM)
self.assertEqual(events[0].name, os.path.realpath(f.name))
self.assertEqual(events[1].mask, IN_MOVED_TO)
self.assertEqual(events[1].name, os.path.realpath(new))
self.assertEqual(events[0].cookie, events[1].cookie)
def test_single_directory_deleted(self):
events = []
def callback(event):
events.append(event)
import os
new1 = os.path.join(self.tempdir, "newdir1")
new2 = os.path.join(self.tempdir, "newdir2")
try:
os.mkdir(new1)
os.mkdir(new2)
import time
time.sleep(0.2)
from fsevents import Stream
stream = Stream(callback, self.tempdir, file_events=True)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
time.sleep(0.1)
os.rmdir(new2)
time.sleep(1.0)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
from fsevents import IN_DELETE
self.assertEqual(len(events), 1)
self.assertEqual(events[0].mask, IN_DELETE)
self.assertEqual(events[0].name, os.path.realpath(new2))
finally:
os.rmdir(new1)
class FileSystemNotifier(object):
def __init__(self, target_dirs, callback):
self.target_dirs = target_dirs
self.callback = callback
def start(self):
self.stream = Stream(self.directory_changed, *self.target_dirs)
self.observer = Observer()
self.observer.schedule(self.stream)
self.observer.daemon = True # Kill observer when main thread killed.
self.observer.start() # start observer in the other thread.
def directory_changed(self, subpath, mask):
logging.debug('Directory changed: %s, %s' % (subpath, mask))
self.callback(subpath, mask)
def shutdown(self):
self.observer.unschedule(self.stream)
self.observer.stop()
def test_single_file_added_to_subdirectory(self):
events = []
def callback(*args):
events.append(args)
import os
directory = self._make_tempdir()
subdirectory = os.path.realpath(os.path.join(directory, 'subdir')) + '/'
os.mkdir(subdirectory)
import time
time.sleep(0.1)
try:
from fsevents import Stream
stream = Stream(callback, directory)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
# add single file
while not observer.isAlive():
time.sleep(0.1)
del events[:]
f = open(os.path.join(subdirectory, "test"), "w")
f.write("abc")
f.close()
time.sleep(0.2)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
self.assertEqual(len(events), 1)
self.assertEqual(events, [(subdirectory, self.modified_mask)])
finally:
os.unlink(f.name)
os.rmdir(subdirectory)
os.rmdir(directory)
def test_single_directory_deleted(self):
events = []
def callback(event):
events.append(event)
import os
new1 = os.path.join(self.tempdir, "newdir1")
new2 = os.path.join(self.tempdir, "newdir2")
try:
os.mkdir(new1)
os.mkdir(new2)
import time
time.sleep(0.2)
from fsevents import Stream
stream = Stream(callback, self.tempdir, file_events=True)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
time.sleep(0.1)
os.rmdir(new2)
time.sleep(1.0)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
from fsevents import IN_DELETE
self.assertEqual(len(events), 1)
self.assertEqual(events[0].mask, IN_DELETE)
self.assertEqual(events[0].name, os.path.realpath(new2))
finally:
os.rmdir(new1)
class FileSystemNotifier(object):
def __init__(self, target_dirs, callback):
self.target_dirs = target_dirs
self.callback = callback
def start(self):
self.stream = Stream(self.directory_changed, *self.target_dirs)
self.observer = Observer()
self.observer.schedule(self.stream)
self.observer.daemon = True # Kill observer when main thread killed.
self.observer.start() # start observer in the other thread.
def directory_changed(self, subpath, mask):
logging.debug('Directory changed: %s, %s' % (subpath, mask))
self.callback(subpath, mask)
def shutdown(self):
self.observer.unschedule(self.stream)
self.observer.stop()
def main():
signal.signal(signal.SIGINT, signal_handler)
parser = argparse.ArgumentParser(
description='Sync files to a remote server of scp.')
parser.add_argument('path',
type=str,
metavar=('local', 'remote'),
nargs=2,
help='path for the monitor')
args = parser.parse_args()
print args.path
observer = Observer()
stream = Stream(lambda x, y: callback(x, y, args.path[1]), args.path[0])
observer.schedule(stream)
try:
observer.start()
while True: # instead of this infinite loop, you can do
pass # whatever processing you wanted
except KeyboardInterrupt:
observer.stop()
def test_fileevent_stream(self):
events = []
def callback(*args):
events.append(args)
# two files in same directory
import os
path1 = os.path.realpath(self._make_tempdir()) + '/'
f = self._make_temporary(path1)[0]
g = self._make_temporary(path1)[0]
from fsevents import Stream, FS_CFLAGFILEEVENTS, FS_ITEMISDIR
stream = Stream(callback, path1, flags=FS_CFLAGFILEEVENTS)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
#create two files (here in the same directory)
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
f.close()
g.close()
time.sleep(0.2)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
import os
self.assertEqual(len(events),3)
self.assertEqual(events, [(path1[:-1], self.create_and_remove_mask|FS_ITEMISDIR),
(f.name, self.create_and_remove_mask),
(g.name, self.create_and_remove_mask)])
def test_existing_directories_are_not_reported(self):
import os
from fsevents import Observer
import time
events = []
def callback(*args):
events.append(args)
new1 = os.path.join(self.tempdir, "newdir1")
new2 = os.path.join(self.tempdir, "newdir2")
os.mkdir(new1)
time.sleep(1.0)
observer = Observer(callback, file_events=True)
observer.start()
observer.schedule(self.tempdir)
while not observer.isAlive():
time.sleep(0.1)
del events[:]
time.sleep(1)
os.mkdir(new2)
try:
time.sleep(1.1)
observer.stop()
observer.unschedule(self.tempdir)
observer.join()
from fsevents import FSE_CREATED_FLAG
from fsevents import FSE_ITEM_IS_DIR_FLAG
flags = FSE_CREATED_FLAG | FSE_ITEM_IS_DIR_FLAG
self.assertEquals(len(events), 1)
self.assertEquals(events[0][0], os.path.realpath(new2))
self.assertEquals(events[0][1], flags)
finally:
os.rmdir(new1)
os.rmdir(new2)
def main():
if len(sys.argv) != 3:
print "Usage: %s FilePath SetID" % sys.argv[0]
sys.exit(1)
path = sys.argv[1]
if not os.path.exists(path):
print "Path does not exist"
sys.exit(1)
global driver
driver = setupDriver(sys.argv[2])
observer = Observer()
stream = Stream(callback, os.path.abspath(path), file_events=True)
observer.schedule(stream)
observer.start()
print "Starting observation..."
print "Ctrl-C to exit."
try:
# yields to allow Ctrl-C to be captured
while True:
# sleep saves CPU cycles
time.sleep(1)
except KeyboardInterrupt:
print "\nQuitting program gracefully"
observer.unschedule(stream)
observer.stop()
try:
driver.quit()
except URLError:
# Some URLError
pass
sys.exit(1)
def test_single_file_deleted(self):
events = []
def callback(event):
events.append(event)
import os
f = open(os.path.join(self.tempdir, "test"), "w")
f.write("abc")
f.flush()
f.close()
from fsevents import Stream
stream = Stream(callback, self.tempdir, file_events=True)
from fsevents import Observer
observer = Observer()
observer.schedule(stream)
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
time.sleep(2.1)
os.unlink(f.name)
time.sleep(0.1)
# stop and join observer
observer.stop()
observer.unschedule(stream)
observer.join()
from fsevents import IN_DELETE
self.assertEqual(len(events), 1)
self.assertEqual(events[0].mask, IN_DELETE)
self.assertEqual(events[0].name, os.path.realpath(f.name))
def test_start_no_watch(self):
events = []
def callback(*args):
events.append(args)
from fsevents import Observer
observer = Observer()
f, path = self._make_temporary()
observer.start()
# add single file
import time
while not observer.isAlive():
time.sleep(0.1)
del events[:]
f.close()
time.sleep(0.2)
# stop and join observer
observer.stop()
observer.join()
self.assertEqual(events, [])
syncer.lock()
syncer.sync(config["path_source"], args.from_server)
syncer.release()
else:
logging.critical("--init needs --from-server or --from-local")
syncer.cleanup(True)
sys.exit(5)
# CTRL+Z will force a full sync :
signal.signal(signal.SIGTSTP, syncer.sig_handler)
observer.start()
logging.info("------- FS WATCHING %s -------" % config["path_source"])
logging.info("(CTRL+z to force a full sync)")
logging.debug("Interval %ssec", args.interval)
os.chdir(config["path_source"])
stream = Stream(syncer.callback, config["path_source"])
try:
observer.schedule(stream)
observer.join()
logging.info("Schedule finished")
except KeyboardInterrupt:
logging.warning("CTRL+c")
finally:
logging.info("Stopping observer...")
observer.stop()
logging.debug("Cleanup threads ...")
syncer.cleanup(True)
logging.info("Finished.")
class FSEventsListener(object):
def __init__(self, path, f, ignore=None):
logging.info("Using FSEvents")
self.path = path
self.f = f
self.ignore = ignore
self.observer = Observer()
self.observer.daemon = True
self._streams = []
self._streams.append(createStream(self.path, path, self._update))
self._streamed_folders = [self.path]
def add_stream(p):
if p in self._streamed_folders:
return
self._streams.append(createStream(p, file_path, self._update))
self._streamed_folders.append(p)
# Start an extra listener for all symlinks
for file_path in fileList(self.path, folders=True):
if os.path.islink(file_path):
if os.path.isdir(file_path):
add_stream(os.path.realpath(file_path))
else:
add_stream(os.path.dirname(os.path.realpath(file_path)))
def run(self):
self.resume()
self.observer.start()
def pause(self):
logging.debug("MacListener.PAUSE")
for stream in self._streams:
self.observer.unschedule(stream)
def resume(self):
logging.debug("MacListener.RESUME")
for stream in self._streams:
self.observer.schedule(stream)
def stop():
self.observer.stop()
def _update(self, event):
path = event.name
if self.ignore and self.ignore(path):
return
logging.debug("MacListener.update %s", event)
result = {
'added': [],
'deleted': [],
'changed': [],
}
if os.path.exists(path):
seconds_since_created = int(time.time()) - get_creation_time(
os.path.realpath(path))
if seconds_since_created < 1.0:
result["added"].append(path)
else:
result["changed"].append(path)
else:
result["deleted"].append(path)
self.f(result)
class MacOSFileBaseSystem(FilesystemBasic):
"""MacOS specific Filesystem."""
is_mac = True
"""Used for testing to ensure that the Filesystem does subclass this class."""
observer = None
"""The observer which is instantiated in start_events."""
@property
def real_root(self):
"""Return the realpath of the root.
fs_events paths are realpaths. i.e.: without symlinks.
"""
return os.path.realpath(self.root)
def fsevent_handler(self, path, mask, _id):
"""Handle events from fsevents, get the current directory content and update the model.
Args:
:param path: absolute path which caused this event.
:param mask: bit mask of attributes set for this event.
:param _id: internal id of the event.
:type path: string
:type mask: int
:type _id: int
..Note: The flags on these events are badly/not documented. Log the events to make
sure which flags are actualy set. Do not trust your intuition.
After preliminary checks, the path is handed to self._update, to trigger the necessary
events.
"""
event = FSEvent(path=path, mask=mask, event_id=_id)
if file_ignored(event.path):
logger.debug('Event ignored for %s', path)
return
cc_path = event.cc_path(self.real_root)
if cc_path == ['.']:
cc_path = []
logger.info('update for %s', cc_path)
try:
self._update(cc_path)
except BaseException:
logger.debug('got exception while processing FSEvents %s',
exc_info=True)
def _update(self, cc_path):
"""Inspect the directory to detect what has changed since the last call to `_update`.
Args:
:param cc_path: path of directory to update.
:type cc_path: list of strings.
"""
logger.info('_update for %s for tree %s', cc_path, self.model)
# ignore event if parent directory no longer exists on the fs
parent_folder = cc_path_to_fs(cc_path[:-1], self.real_root)
if not os.path.exists(parent_folder):
logger.debug('Event ignored: parent folder no longer exist')
return
with TreeToSyncEngineEngineAdapter(node=self.model,
storage_id=self.storage_id,
sync_engine=self._event_sink):
# Ensure that the path exists in the model.
parent = self.model
for idx, name in enumerate(cc_path):
if parent.has_child(name):
parent = parent.get_node([name])
else:
partial_cc_path = cc_path[:idx + 1]
parent = parent.add_child(
name, props=self.get_props(partial_cc_path))
directory = cc_path_to_fs(cc_path, self.real_root)
new_inodes = {
props['_inode']: (name, props)
for name, props in self.get_tree_children(cc_path)
}
old_inodes = {
node.props['_inode']: node
for node in parent.children
}
new_inodes_set = new_inodes.keys()
old_inodes_set = old_inodes.keys()
inode_intersection = new_inodes_set & old_inodes_set
removed_inodes = old_inodes_set - new_inodes_set
added_inodes = new_inodes_set - old_inodes_set
for inode in inode_intersection:
old_node = old_inodes[inode]
new_node_name, new_node_props = new_inodes[inode]
old_node.props.update(new_node_props)
old_node.name = new_node_name
for inode in removed_inodes:
# TODO: might be moved to a different dir, might be deleted
old_inodes[inode].delete()
for inode in added_inodes:
new_node_name, new_node_props = new_inodes[inode]
new_node = parent.add_child(new_node_name, new_node_props)
if new_node_props[jars.IS_DIR]:
self._update(new_node.path)
def start_events(self):
"""Setup the observer."""
self.get_tree(cached=False)
if self.observer is None:
self.observer = Observer()
stream = Stream(self.fsevent_handler, self.real_root, ids=True)
self.observer.schedule(stream)
if not self.observer.is_alive():
self.observer.start()
def stop_events(self, *args, **kwargs):
"""Call stop() on the observer."""
self.update()
if self.observer is not None:
self.observer.stop()
def clear_model(self):
"""Reset the model to only contain one root node."""
self.model = Node(None)
def get_tree(self, cached=False):
"""Return a deep copy of the internal model."""
if cached:
return copy.deepcopy(self.model)
else:
return super().get_tree(cached=False)
def update(self):
"""Update the internal model, by walking the directory structure of the root."""
self.model = self.get_tree()
def get_internal_model(self):
"""Return the current internal model, used in testing."""
return self.model
class sftpwatch:
def __init__(self,
ignore=[regexp(r'.*\.git.*'),
regexp(r'.*\.DS_Store$')],
mapping=[],
host=None,
user=None,
password=None,
rootdir=None):
self.ignore = ignore
self.PATH_MAPPING = mapping
self.host = host
self.user = user
self.password = password
self.root = rootdir
self.observer = None
self.stream = None
self.sf = None
self.connected = False
self.exit = False
try:
self.connect()
except Exception as e:
logging.critical("SSH Could not connect!")
logging.critical(str(e))
exit(1)
def connect(self):
self.sf = self.getsftp(args.host, args.user, args.password)
def watch(self):
self.observer = Observer()
self.observer.start()
self.stream = Stream(self.file_event_callback,
self.root,
file_events=True)
self.observer.schedule(self.stream)
def ssh_connect(self, hostname, username, password):
"""Connect to SSH server and return (authenticated) transport socket"""
host_keys = paramiko.util.load_host_keys(
expanduser('~/.ssh/known_hosts'))
if hostname in host_keys:
hostkeytype = host_keys[hostname].keys()[0]
hostkey = host_keys[hostname][hostkeytype]
logging.debug('Using host key of type %s' % hostkeytype)
else:
logging.critical("Host key not found")
exit(0)
t = paramiko.Transport((hostname, 22))
t.set_keepalive(30)
t.connect(hostkey, username, password)
return t
def getsftp(self, hostname, username, password):
"""Return a ready-to-roll paramiko sftp object"""
t = self.ssh_connect(hostname, username, password)
return paramiko.SFTPClient.from_transport(t)
def transfer_file(self, localpath, remotepath):
"""Transfer file over sftp"""
with self.sf.open(remotepath, 'wb') as destination:
with open(localpath, 'rb') as source:
total = 0
while True:
data = source.read(8192)
if not data:
return total
destination.write(data)
total += len(data)
def file_event_callback(self, event):
"""Respond to file events"""
# Make sure we have sftp connectivity
while not self.exit:
try:
assert not self.sf == None
self.sf.stat("/")
break
except (OSError, AssertionError) as e:
logging.warning("Attempting to connect...")
try:
self.connect()
break
except Exception as ee:
logging.warning("Could not Connect.")
logging.error("Error was: %s" % str(ee))
logging.warning("Trying again in 5 seconds...")
sleep(5)
if self.exit:
return
# check ignored
for expr in self.ignore:
if not expr.match(event.name) == None:
return
# Determine file path relative to our root
filePath = event.name.replace(self.root, "")
logging.debug("Path from basedir: %s" % filePath)
# Apply directory mapping
for mapping in self.PATH_MAPPING:
localMapPath, remoteMapPath = mapping
if filePath[0:len(localMapPath)] == localMapPath:
logging.debug("Using mapping: %s" % (str(mapping)))
filePath = remoteMapPath + "/" + filePath[len(localMapPath):]
break
filePath = normpath(filePath)
# Ensure path starts with /
if filePath[0] != "/":
filePath = "/" + filePath
if event.mask & (fsevents.IN_MODIFY | fsevents.IN_CREATE
| fsevents.IN_MOVED_TO):
logging.debug("\nFile was modified: %s" % event.name)
logging.debug("Remote path: %s" % filePath)
# Ensure directory exists
path_dirs = dirname(filePath).split("/")
for i in range(1, len(path_dirs)):
pathSegment = "/".join(path_dirs[0:i + 1])
logging.debug("stat %s" % pathSegment)
try:
self.sf.stat(pathSegment)
except IOError as e:
logging.info("Creating %s" % pathSegment)
self.sf.mkdir(pathSegment)
# If file, upload it
if isfile(event.name) or islink(event.name):
tries = 0
while True:
try:
bytesSent = self.transfer_file(event.name, filePath)
break
except IOError as ioe:
logging.error("Unable to upload file: %s" % str(ioe))
# reconnect to SSH on error
#sf.close()
#sf = getsftp(args.host, args.user, args.password)
tries += 1
sleep(0.5)
if tries > 5:
return False
logging.info(
"%s: sent %s KB to %s" %
(event.name, max(1, int(bytesSent / 1024)), filePath))
else:
logging.info("Not a file: %s" % event.name)
if event.mask & (fsevents.IN_MOVED_FROM | fsevents.IN_DELETE):
logging.info("removing %s" % filePath)
# Just delete it
try:
self.sf.remove(filePath)
except:
# Silently fail so we don't delete unexpected stuff
pass
"""
We can respond to:
done IN_MOVED_FROM - path is old file path
done IN_MOVED_TO - path is new file path
done IN_MODIFY - file was edited
done IN_CREATE - file was created
done IN_DELETE - file was deleted
IN_ATTRIB - attributes modified - ignore for now
"""
#self.sf.close()
def signal_handler(self, signal, frame):
logging.info('Cleaning up....')
self.exit = True
self.observer.unschedule(self.stream)
self.observer.stop()
