class WatchFolder:
"""
Creates and manages watch folder instances.
"""
def __init__(self):
self._watch_folder_instances = {}
self._watch_folder_observer = Observer()
self._watch_folder_observer.start()
def add_folder_path(self, folder_path):
"""
Sets up a watch folder instance using the folder path.
:param folder_path: The absolute path of the folder.
"""
watch_folder_instance = WatchFolderInstance(folder_path)
self._watch_folder_instances[folder_path] = watch_folder_instance
watch_folder_instance.watch = self._watch_folder_observer.schedule(
watch_folder_instance.event_handler, folder_path, recursive=False)
def get_instance(self, folder_path):
"""
Returns the watch folder instance for the folder path.
:param folder_path: The absolute path of the folder.
"""
try:
return self._watch_folder_instances[folder_path].queue.get()
except KeyError:
return None
def is_instance_empty(self, folder_path):
"""
Checks if the watch folder instance for the folder path has found any new files.
:param folder_path: The absolute path of the folder.
"""
try:
watch_folder_instance = self._watch_folder_instances[folder_path]
return watch_folder_instance.queue.empty()
except KeyError:
return True
def stop_and_remove_instance(self, folder_path):
"""
Stops the watch folder instance for the folder path and removes it.
:param folder_path: The absolute path of the folder.
"""
try:
watch_folder_instance = self._watch_folder_instances[folder_path]
watch_folder_instance.queue.put(False)
self._watch_folder_observer.unschedule(watch_folder_instance.watch)
return True
except KeyError:
logging.exception(
'--- FAILED TO STOP AND REMOVE WATCH FOLDER INSTANCE ---')
return False
class FileMonitor(object):
def __init__(self):
self.watches = []
self.observer = Observer()
self.observer.start()
def addWatch(self, path, filename=None, recursive=False):
if filename is None:
self.watches.append(self.observer.schedule(MyEventHandler(), path, recursive))
return
if not path.endswith('/'):
full_path = path + '/' + filename
else:
full_path = path + filename
self.watches.append(self.observer.schedule(MyEventHandler([full_path,]), path, recursive))
def removeWatch(self, path):
if not path.endswith('/'):
path += '/'
for watch in self.watches:
if watch.path == path:
self.observer.unschedule(watch)
return True
return False
def removeAllWatches(self):
self.observer.unschedule_all()
class FileMonitor(object):
def __init__(self):
self.watches = []
self.observer = Observer()
self.observer.start()
def add_watch(self, path, filename=None, recursive=False):
# if no filename is provided, just watch the directory
if filename is None:
self.watches.append(
self.observer.schedule(FSEventHandler(), path, recursive))
return
# if we're monitoring a specific file, we have to pass the full path to the event handler
if not path.endswith('/'):
full_path = path + '/' + filename
else:
full_path = path + filename
self.watches.append(
self.observer.schedule(FSEventHandler([
full_path,
]), path, recursive))
def remove_watch(self, path):
if not path.endswith('/'):
path += '/'
for watch in self.watches:
if watch.path == path:
self.observer.unschedule(watch)
return True
return False
def remove_all_watches(self):
self.observer.unschedule_all()
def run_watcher(self):
# Create observer
observer = Observer()
# Start observer
observer.start()
# Run change check in a loop
while not self._watcher_stop:
try:
# 2KGRW
# Schedule a watch
# self: `FileSystemEventHandler` instance
# self._watched_path: File path to watch
# recursive: Whether recursive
self.watch_obj = observer.schedule(self,
self._watched_path,
recursive=True)
# print(self.watch_obj)
# If have error
except Exception as e:
print('erro aqui óh: \n {e}')
# Sleep before next check
time.sleep(self._interval)
if self._watcher_stop:
observer.unschedule(self.watch_obj)
class EntryWatcher:
def __init__(self, navigator: 'Navigator'):
self.navigator = navigator
self.observer = Observer()
self.observer.start()
self.entry = None
self.watch = None
def on_update(self, _):
if self.watch and self.entry == self.navigator.current_entry:
return
if self.watch:
self.unregister(self.watch)
self.watch = None
self.entry = self.navigator.current_entry
self.watch = self.register()
def unregister(self, watch):
if self.watch:
self.observer.unschedule(watch)
def register(self):
entry = self.navigator.current_entry
if entry.is_dir():
path = entry.path
handler = DirHandler(self.navigator)
logger.debug('Watching {} for changes'.format(entry.path))
else:
path = entry.dir
handler = FileHandler(self.navigator, entry.path)
watch = self.observer.schedule(handler, path, recursive=False)
return watch
def unschedule(self, path):
'''
@summary: 監視リストから指定したファイルパスを監視するwatchを除去する
整理はしていない
@warning: Linuxのみwatchスレッドが死んでいない テスト&要修正
'''
path = abspath(path)
Observer.unschedule(self, self.watch_path[path])
del self.watch_path[path]
logger.debug("unschedule %s" % path)
def main(self):
eventos = Copiar_JDF_to()
observador = Observer()
observador.schedule(eventos, self.pasta_monitorada , recursive=False)
observador.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observador.unschedule(eventos)
observador.stop()
observador.join()
print ("FIM")
def test_watchdog_only_recursive(tmpdir):
from robocorp_ls_core import watchdog_wrapper
watchdog_wrapper._import_watchdog()
import watchdog
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
import os.path
class Handler(FileSystemEventHandler):
def __init__(self):
FileSystemEventHandler.__init__(self)
self.changes = []
def on_any_event(self, event):
print(event.src_path)
self.changes.append(os.path.basename(event.src_path))
handler = Handler()
observer = Observer()
watches = []
watches.append(observer.schedule(handler, str(tmpdir), recursive=True))
try:
observer.start()
time.sleep(0.1)
tmpdir.join("my0.txt").write("foo")
tmpdir.join("dir_rec").mkdir()
tmpdir.join("dir_rec").join("my1.txt").write("foo")
expected = {"dir_rec", "my0.txt", "my1.txt"}
timeout_at = time.time() + 5
while not expected.issubset(
handler.changes) and time.time() < timeout_at:
time.sleep(0.2)
if not expected.issubset(handler.changes):
raise AssertionError(
f"Did not find expected changes. Found: {handler.changes}")
finally:
for watch in watches:
observer.unschedule(watch)
observer.stop()
class FileWatch(FileSystemEventHandler):
def __init__(self, origin, watch_path, event_func):
FileSystemEventHandler.__init__(self)
self.origin = origin
self.watch_path = watch_path
self.event_func = event_func
self.observer = Observer()
self.watch = None
self.mutex = threading.Lock()
def get_lock(self):
return self.mutex
def start(self):
dir = os.path.abspath(os.path.dirname(self.watch_path))
self.origin.dbg('FileWatch: scheduling watch for directory %s' % dir)
self.watch = self.observer.schedule(self, dir, recursive=False)
self.observer.start()
def stop(self):
if self.watch:
self.origin.dbg('FileWatch: unscheduling watch %r' % self.watch)
self.observer.unschedule(self.watch)
self.watch = None
if self.observer.is_alive():
self.observer.stop()
self.observer.join()
def __del__(self):
self.stop()
self.observer = None
# Override from FileSystemEventHandler
def on_any_event(self, event):
if event.is_directory:
return None
if os.path.abspath(event.src_path) != os.path.abspath(self.watch_path):
return None
self.origin.dbg('FileWatch: received event %r' % event)
try:
self.mutex.acquire()
self.event_func(event)
finally:
self.mutex.release()
def main():
print('Starting vagrant rsync auto service.')
#INITIALIZE
vagrant('global-status', _bg=True, _out=vagrant_status_out).wait()
last = time.time()
handler = VagrantEventHandler()
observer = Observer()
watchers = {}
#START WATCHING
try:
observer.start()
while True:
vagrant_dirs = set(vagrant_rsync_dirs.keys())
watched_dirs = set(watchers.keys())
for d in (vagrant_dirs - watched_dirs):
watchers[d] = observer.schedule(handler, d, recursive=True)
print('Observer is watching {}'.format(d))
for d in (watched_dirs - vagrant_dirs):
w = watchers.pop(d)
observer.unschedule(w)
print('Observer has stopped watching {}'.format(d))
time.sleep(SLEEP_INTERVAL)
passed = time.time() - last
if passed > CHECK_INTERVAL:
vagrant('global-status', _bg=True, _out=vagrant_status_out)
last = time.time()
#SHUT DOWN
except KeyboardInterrupt:
print(' << (*) >> ')
observer.stop()
observer.join()
print('Goodbye :-)')
def test_watchdog_recursive():
""" See https://github.com/gorakhargosh/watchdog/issues/706
"""
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
import os.path
class Handler(FileSystemEventHandler):
def __init__(self):
FileSystemEventHandler.__init__(self)
self.changes = []
def on_any_event(self, event):
self.changes.append(os.path.basename(event.src_path))
handler = Handler()
observer = Observer()
watches = []
watches.append(observer.schedule(handler, str(p('')), recursive=True))
try:
observer.start()
time.sleep(0.1)
touch(p('my0.txt'))
mkdir(p('dir_rec'))
touch(p('dir_rec', 'my1.txt'))
expected = {"dir_rec", "my0.txt", "my1.txt"}
timeout_at = time.time() + 5
while not expected.issubset(
handler.changes) and time.time() < timeout_at:
time.sleep(0.2)
assert expected.issubset(
handler.changes
), "Did not find expected changes. Found: {}".format(handler.changes)
finally:
for watch in watches:
observer.unschedule(watch)
observer.stop()
observer.join(1)
class IPCom:
"""IPC async thread"""
INSTANCE: Optional["IPCom"] = None
def __init__(self):
self.observer = Observer()
self.observer.start()
self.pid = os.getpid()
def fswatch(self, watcher: FileSystemEventHandler, path: Path, recursive=False):
if not self.observer.is_alive():
# Just in case...
self.observer.start()
return self.observer.schedule(
watcher, str(path.absolute()), recursive=recursive
)
def fsunwatch(self, watcher):
self.observer.unschedule(watcher)
class Rec2DB(object):
def __init__(self, sensor):
super(Rec2DB, self).__init__()
self._sensor = sensor
self._handler = Handler(sensor)
self._observer = Observer()
self._watcher = None
def loop(self):
self._observer.start()
folder = "/tmp/" + self._sensor
os.makedirs(folder, exist_ok=True)
self._watcher = self._observer.schedule(self._handler,
folder,
recursive=True)
self._observer.join()
def stop(self):
if self._watcher: self._observer.unschedule(self._watcher)
self._observer.stop()
class Rainmaker():
def __init__(self):
self.event_handlers = {}
self.observer = Observer()
self.observer.start()
def add_watch(self,watch_path,rec_flag=True):
event_handler = RainmakerEventHandler( watch_path )
self.event_handlers[watch_path] = event_handler
self.observer.schedule( event_handler, watch_path, recursive = rec_flag)
def remove_watch(self, k):
eh = self.event_handlers.pop(k)
self.observer.unschedule(eh)
def shutdown(self):
self.log.info( "Shutting down FSwatcher")
self.observer.stop()
self.observer.unschedule_all()
self.observer.join()
class Rec2DB(object):
def __init__(self, sensor):
super(Rec2DB, self).__init__()
self._sensor = sensor
self._handler = Handler(sensor)
self._observer = Observer()
self._watcher = None
def loop(self):
self._observer.start()
folder = os.path.join(os.path.realpath(storage), self._sensor)
print("Observing " + folder, flush=True)
os.makedirs(folder, exist_ok=True)
self._watcher = self._observer.schedule(self._handler,
folder,
recursive=True)
self._observer.join()
def stop(self):
if self._watcher: self._observer.unschedule(self._watcher)
self._observer.stop()
class FileDaemon:
def __init__(self, msg_identifier, send_config):
#Attributes
self.target_dir = send_config["PATH_BASE"]
self._monitor_flag_ = threading.Event()
self.watch = None
#Components
self._logger_ = EventLogger()
self._event_handler_ = SyncEventHandler(msg_identifier, send_config)
self._observer_ = Observer()
"""
Public methods
"""
def initialize(self):
"""
Initializes components, starts logging
"""
#Prep for monitor
self._monitor_flag_.clear()
#Initialize components
logfile = os.path.dirname(self.target_dir[:-1]) + SLASH + "daemon.log"
self._logger_.init_session(logfile)
self._event_handler_.initialize()
self._observer_.start()
def start(self):
"""
Starts/resumes observing a target directory. Wraps up the monitor method
which is used as the target for a separate thread.
"""
if self._monitor_flag_.is_set():
return #Already running
else:
self._monitor_flag_.set()
threading.Thread(target=self._monitor_).start()
def full_sync(self):
"""
Handle for the directory sync method of event_handler which
allows controller to request a sync of entire directory
"""
self._event_handler_.dir_sync(self.target_dir)
def pause(self):
"""
Pause observation with intent to resume
"""
self._monitor_flag_.clear()
def stop(self):
"""
End observation with no intent to resume
"""
self._monitor_flag_.clear()
time.sleep(2) #Wait for local activity to settle
self._observer_.stop() #stop observing
self._observer_.join() #wait for all threads to be done
def is_alive(self):
return self._monitor_flag_.is_set()
"""
Protected methods
"""
def _monitor_(self):
"""
Function run on separate thread which acts as parent to observer thread(s).
Used to control operation flow of observer using monitor_flag
"""
self._logger_.log("INFO","Scheduling observation of " + self.target_dir + " tree...")
self.__watch__ = self._observer_.schedule(self._event_handler_, self.target_dir, recursive=True)
while (self._monitor_flag_.is_set()):
time.sleep(1)
self._observer_.unschedule(self.__watch__)
class FileWatcher:
class Handler(PatternMatchingEventHandler):
def __init__(self, watcher):
super(FileWatcher.Handler, self).__init__(patterns = ["*.pdf"], ignore_directories = True)
self.watcher = watcher
def on_any_event(self, event):
log.debug("Trace received %s event - %s.", event.event_type, event.src_path)
def on_moved(self, event):
# file is downloaded with .crdownload suffix and once downloaded renamed to target name
# captured as a move to the expected target suffix
log.debug("Received moved event - %s -> %s.", event.src_path, event.dest_path)
self.watcher.notify(Path(event.dest_path))
def __init__(self, watchdir = None):
if watchdir is None:
self.watched_dir = tempfile.TemporaryDirectory()
self.watched_dir_path = Path(self.watched_dir.name)
else:
self.watched_dir = str(watchdir)
self.watched_dir_path = watchdir
log.debug("temp dir for download is [{!s}]".format(self.watched_dir))
self.new_file = None
self.observer = Observer()
self.observer.start()
self.watch = None
def get_dir(self):
return self.watched_dir_path
def notify(self, f):
self.new_file = f
def _new_handler(self):
return FileWatcher.Handler(self)
def _init_observer(self):
if self.watch is None:
event_handler = self._new_handler()
self.new_file = None
self.watch = self.observer.schedule(event_handler, str(self.watched_dir_path), recursive=False)
log.debug('start waiting for download')
def start(self):
self._init_observer()
def wait_move_file(self, target, timeout=40):
self._init_observer()
try:
while self.new_file is None and timeout >= 0:
time.sleep(1)
timeout -= 1
self.observer.unschedule(self.watch)
if self.new_file is not None:
if not target.is_file():
copyfile(str(self.new_file), str(target))
self.new_file.unlink()
log.info("File downloaded : %s", str(target))
else:
log.warning("File already existed dowloaded file is trashed")
else:
log.error("download failed with timeout")
except:
self.observer.unschedule(self.watch)
log.exception("error while waiting for file download")
self.watch = None
return self.new_file is not None
def run_watcher(self):
"""
Watcher thread's function.
:return:
None.
"""
# Create observer
observer = Observer()
# Start observer
observer.start()
# Dict that maps file path to `watch object`
watche_obj_map = {}
# Run change check in a loop
while not self._watcher_to_stop:
# Get current watch paths
old_watch_path_s = set(watche_obj_map)
# Get new watch paths
new_watch_path_s = self._find_watch_paths()
# For each new watch path
for new_watch_path in new_watch_path_s:
# Remove from the old watch paths if exists
old_watch_path_s.discard(new_watch_path)
# If the new watch path was not watched
if new_watch_path not in watche_obj_map:
try:
# Schedule a watch
watch_obj = observer.schedule(
# 2KGRW
# `FileSystemEventHandler` instance
self,
# File path to watch
new_watch_path,
# Whether recursive
recursive=True,
)
# Store the watch obj
watche_obj_map[new_watch_path] = watch_obj
# If have error
except OSError:
# Set the watch object be None
watche_obj_map[new_watch_path] = None
# For each old watch path that is not in the new watch paths
for old_watch_path in old_watch_path_s:
# Get watch object
watch_obj = watche_obj_map.pop(old_watch_path, None)
# If have watch object
if watch_obj is not None:
# Unschedule the watch
observer.unschedule(watch_obj)
# Store new watch paths
self._watch_paths = new_watch_path_s
# Sleep before next check
time.sleep(self._interval)
class ModuleFileWatcher(EditorModule):
def __init__(self):
super(ModuleFileWatcher, self).__init__()
self.watches = {}
def getName(self):
return 'filewatcher'
def getDependency(self):
return []
def onLoad(self):
self.observer = Observer()
self.observer.start()
signals.connect('file.moved', self.onFileMoved)
signals.connect('file.added', self.onFileCreated)
signals.connect('file.removed', self.onFileDeleted)
signals.connect('file.modified', self.onFileModified)
def onStart(self):
self.assetWatcher = self.startWatch(
self.getProject().getAssetPath(),
ignorePatterns=['*/.git', '*/.*', '*/_gii'])
def startWatch(self, path, **options):
path = os.path.realpath(path)
if self.watches.get(path):
logging.warning('already watching: %s' % path)
return self.watches[path]
logging.info('start watching: %s' % path)
ignorePatterns = ['*/.git', '*/.*', '*/_gii'] + options.get(
'ignorePatterns', [])
handler = FileWatcherEventHandler(
options.get('patterns', None), ignorePatterns,
options.get('ignoreDirectories', False),
options.get('caseSensitive', True))
watch = self.observer.schedule(handler, path,
options.get('recursive', True))
self.watches[path] = watch
return watch
def onStop(self):
# print 'stop file watcher'
self.observer.stop()
self.observer.join(0.5)
# print 'stopped file watcher'
def stopWatch(self, path):
path = os.path.realpath(path)
watch = self.watches.get(path, None)
if not watch: return
self.observer.unschedule(watch)
self.watches[path] = None
def stopAllWatches(self):
# logging.info('stop all file watchers')
self.observer.unschedule_all()
self.watches = {}
def onFileMoved(self, path, newpath):
# print('asset moved:',path, newpath)
app.getAssetLibrary().scheduleScanProject()
pass
def onFileCreated(self, path):
# print('asset created:',path)
app.getAssetLibrary().scheduleScanProject()
pass
def onFileModified(self, path):
# print('asset modified:',path)
app.getAssetLibrary().scheduleScanProject()
pass
def onFileDeleted(self, path):
# print('asset deleted:',path)
app.getAssetLibrary().scheduleScanProject()
pass
class WatchdogReloaderLoop(ReloaderLoop):
def __init__(self, *args, **kwargs):
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
super().__init__(*args, **kwargs)
trigger_reload = self.trigger_reload
class EventHandler(PatternMatchingEventHandler):
def on_any_event(self, event):
trigger_reload(event.src_path)
reloader_name = Observer.__name__.lower()
if reloader_name.endswith("observer"):
reloader_name = reloader_name[:-8]
self.name = f"watchdog ({reloader_name})"
self.observer = Observer()
# Extra patterns can be non-Python files, match them in addition
# to all Python files in default and extra directories. Ignore
# __pycache__ since a change there will always have a change to
# the source file (or initial pyc file) as well. Ignore Git and
# Mercurial internal changes.
extra_patterns = [p for p in self.extra_files if not os.path.isdir(p)]
self.event_handler = EventHandler(
patterns=["*.py", "*.pyc", "*.zip", *extra_patterns],
ignore_patterns=[
"*/__pycache__/*",
"*/.git/*",
"*/.hg/*",
*self.exclude_patterns,
],
)
self.should_reload = False
def trigger_reload(self, filename):
# This is called inside an event handler, which means throwing
# SystemExit has no effect.
# https://github.com/gorakhargosh/watchdog/issues/294
self.should_reload = True
self.log_reload(filename)
def __enter__(self):
self.watches = {}
self.observer.start()
return super().__enter__()
def __exit__(self, exc_type, exc_val, exc_tb):
self.observer.stop()
self.observer.join()
def run(self):
while not self.should_reload:
self.run_step()
time.sleep(self.interval)
sys.exit(3)
def run_step(self):
to_delete = set(self.watches)
for path in _find_watchdog_paths(self.extra_files,
self.exclude_patterns):
if path not in self.watches:
try:
self.watches[path] = self.observer.schedule(
self.event_handler, path, recursive=True)
except OSError:
# Clear this path from list of watches We don't want
# the same error message showing again in the next
# iteration.
self.watches[path] = None
to_delete.discard(path)
for path in to_delete:
watch = self.watches.pop(path, None)
if watch is not None:
self.observer.unschedule(watch)
class CollectionScanner(object):
def __init__(self):
self.fswatcher = CollectionEventHandler(self)
self.observer = Observer()
self.watches = {}
self.scanner_pool = Pool(processes=4)
signal.signal(signal.SIGINT, self.quit)
signal.signal(signal.SIGTERM, self.quit)
def scan_directory(self, directory, full_scan):
self.scanner_pool.apply_async(start_scanrunner, [directory, full_scan, self.last_shutdown_time])
def add_directory(self, directory, full_scan=False):
# start a full scan if required
# otherwise do an incremental
# and schedule a watch.
self.scan_directory(directory, full_scan)
#start_scanrunner(directory, full_scan, self.last_shutdown_time)
#TODO(nikhil) fix me
self.watches[directory] = True
self.watches[directory] = self.observer.schedule(self.fswatcher, path=directory, recursive=True)
def remove_directory(self, directory):
try:
self.observer.unschedule(self.watches[directory])
del self.watches[directory]
except KeyError:
pass
# also remove all tracks within that directory from DB
tracks = Track.select(Track.q.url.startswith('file://'+directory))
for track in tracks:
track.destroySelf()
def configuration_changed(self):
# check if collection dirs have
# been changed since we last started
# if yes, we will do a full rescan
# otherwise, an incremental scan.
config = GlobalConfig()
paths_saved_at = 0
last_scan = 0
self.last_shutdown_time = 0
try:
paths_saved_at = int(config['collection']['paths_saved_at'])
except KeyError:
#TODO(nikhil) test this behaviour
pass
try:
last_scan = int(config['collection']['last_scan'])
except KeyError:
last_scan = paths_saved_at - 1
try:
self.last_shutdown_time = int(config['last_shutdown_time'])
except KeyError:
pass
collection_directories = set()
try:
collection_directories = set(config['collection']['paths'])
except KeyError:
pass
full_scan = False
if last_scan < paths_saved_at:
full_scan = True
if full_scan:
try:
# for a full scan, first wipe all tables
Artist.deleteMany(None)
Album.deleteMany(None)
Genre.deleteMany(None)
Composer.deleteMany(None)
Track.deleteMany(None)
TrackStatistics.deleteMany(None)
AlbumStatistics.deleteMany(None)
ArtistStatistics.deleteMany(None)
GenreStatistics.deleteMany(None)
except OperationalError:
pass
# first remove watches on
# any directories that have been
# removed from the collection directories
existing_directories = set(self.watches.keys())
for dir in existing_directories.difference(collection_directories):
self.remove_directory(dir)
for dir in collection_directories:
if dir in self.watches:
# directory is already being watched
# do nothing
pass
else:
self.add_directory(dir, full_scan)
def start(self):
self.configuration_changed()
# Note: a 'first' scan is an
# incremental scan behaving like a full
# scan, so we don't have to explicitly
# handle that case.
# finally put a watch on the config file itself
self.observer.schedule(ConfigWatcher(self), path=USERDIR)
self.observer.start()
def quit(self, signum, frame):
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# stop watching file before we make any changes
self.observer.unschedule_all()
self.observer.stop()
# close all update threads
# save current time
config = GlobalConfig()
try:
config['collection']
except KeyError:
config['collection'] = {}
now = int(time.time())
config['collection']['last_scan'] = now
config.save()
self.scanner_pool.close()
self.scanner_pool.terminate()
self.observer.join()
self.scanner_pool.join()
sys.exit(0)
class ConfigWatcher(object):
@staticmethod
def get_config():
config_name = 'config.ini'
default_home = get_default_home()
path = os.path.join(os.path.dirname(sys.executable), config_name)
if os.path.exists(path):
return path
user_ini = os.path.expanduser(os.path.join(default_home, config_name))
if os.path.exists(user_ini):
return user_ini
return None
config_ini = get_config.__func__()
class ConfigModifiedEventHandler(FileSystemEventHandler):
def on_modified(self, event):
if isinstance(event, FileModifiedEvent) and not event.is_directory and \
event.src_path == ConfigWatcher.config_ini:
import ConfigParser
config = ConfigParser.RawConfigParser()
config.read(ConfigWatcher.config_ini)
try:
new_upload_rate = config.getint(ConfigParser.DEFAULTSECT, 'upload-rate')
except ConfigParser.NoOptionError as e:
new_upload_rate = -1
try:
new_download_rate = config.getint(ConfigParser.DEFAULTSECT, 'download-rate')
except ConfigParser.NoOptionError as e:
new_download_rate = -1
self.set_rate_limit_changed(new_upload_rate, new_download_rate)
@staticmethod
def set_rate_limit_changed(new_upload_rate, new_download_rate):
try:
current_upload_rate = int(Manager.get()._dao.get_config('upload_rate', -1))
except ValueError:
current_upload_rate = -1
try:
current_download_rate = int(Manager.get()._dao.get_config('download_rate', -1))
except ValueError:
current_download_rate = -1
change = not (new_upload_rate == current_upload_rate and new_download_rate == current_download_rate)
if not change:
return
slower = new_upload_rate < current_upload_rate or new_download_rate < current_download_rate
if slower:
# change processors first, then update the rate(s)
ConfigWatcher.ConfigModifiedEventHandler.change_processors(new_upload_rate, new_download_rate)
if new_upload_rate != current_upload_rate:
BaseAutomationClient.set_upload_rate_limit(new_upload_rate)
Manager.get()._dao.update_config('upload_rate', new_upload_rate)
log.trace('update upload rate: %s', str(BaseAutomationClient.upload_token_bucket))
if new_download_rate != current_download_rate:
BaseAutomationClient.set_download_rate_limit(new_download_rate)
Manager.get()._dao.update_config('download_rate', new_download_rate)
log.trace('update download rate: %s', str(BaseAutomationClient.download_token_bucket))
if change and new_upload_rate > current_upload_rate and new_download_rate > current_download_rate:
# changed rate(s) first, then change processors
ConfigWatcher.ConfigModifiedEventHandler.change_processors(new_upload_rate, new_download_rate)
@staticmethod
def change_processors(upload_rate, download_rate):
max_processors = get_number_of_processors(upload_rate, download_rate)
for engine in Manager.get().get_engines().values():
try:
engine.get_queue_manager().restart(max_processors)
log.trace('update engine %s processors: %d local, %d remote, %d generic',
engine.get_uid(), max_processors[0], max_processors[1], max_processors[2])
except Exception as e:
log.error('failed to update engine %s processors: %s', engine.get_uid(), str(e))
def __init__(self):
self.watch = None
self.observer = Observer()
def setup_watchdog(self):
if not ConfigWatcher.config_ini:
return
event_handler = ConfigWatcher.ConfigModifiedEventHandler()
self.watch = self.observer.schedule(event_handler, os.path.dirname(ConfigWatcher.config_ini), recursive=False)
self.observer.start()
def unset_watcher(self):
self.observer.unschedule(self.watch)
class FileSync(TimerWorker):
###########################
# START Class Inheritance #
def init_args(self,
dimensigon: 'Dimensigon',
file_sync_period=defaults.FILE_SYNC_PERIOD,
file_watches_refresh_period=FILE_WATCHES_REFRESH_PERIOD,
max_allowed_errors=MAX_ALLOWED_ERRORS,
retry_blacklist=RETRY_BLACKLIST):
self.dm = dimensigon
# Multiprocessing
self.queue = MPQueue()
# Parameters
self.INTERVAL_SECS = file_sync_period
self.file_watches_refresh_period = file_watches_refresh_period
self.max_allowed_errors = max_allowed_errors
self.retry_blacklist = retry_blacklist
# internals
self._changed_files: t.Set[Id] = set(
) # list of changed files to be sent
self._changed_servers: t.Dict[Id, t.List[Id]] = dict()
self._file2watch: t.Dict[Id, ObservedWatch] = {}
self._last_file_updated = None
self._blacklist: t.Dict[t.Tuple[Id, Id], BlacklistEntry] = {}
self._blacklist_log: t.Dict[t.Tuple[Id, Id], BlacklistEntry] = {}
self.session = None
self._server = None
self._loop = None
# log variables
self._mapper: t.Dict[Id, t.List[_PygtailBuffer]] = {}
def startup(self):
self._executor = ThreadPoolExecutor(
max_workers=max(os.cpu_count(), 4),
thread_name_prefix="FileSyncThreadPool")
self._observer = Observer()
self.session = self._create_session()
self._observer.start()
self._set_initial_modifications()
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
self.dispatcher.listen([NewEvent, AliveEvent],
lambda x: self.add(None, x.args[0]))
def shutdown(self):
self.session.close()
self._observer.stop()
self._executor.shutdown()
def main_func(self):
# collect new File events
while True:
item = self.queue.safe_get()
if item:
self._add(*item)
else:
break
self._set_watchers()
self._sync_files()
# send log data
self._send_new_data()
# END Class Inheritance #
#########################
############################
# INIT Interface functions #
def add(self,
file: t.Union[File, Id] = None,
server: t.Union[Server, Id] = None):
if isinstance(file, File):
file_id = file.id
else:
file_id = file
if isinstance(server, Server):
server_id = server.id
else:
server_id = server
try:
self.queue.safe_put((file_id, server_id), timeout=2)
except queue.Full:
self.logger.warning("Queue is full. Try increasing its size")
# END Interface functions #
############################
##############################
# INNER methods & attributes #
def _create_session(self):
self.Session = sessionmaker(bind=self.dm.engine)
return self.Session()
@property
def server(self) -> Server:
if self._server is None:
self._server = self.session.query(Server).filter_by(
_me=1, deleted=0).one_or_none()
return self._server
@property
def my_files_query(self):
return self.session.query(File).filter_by(deleted=0).filter_by(
src_server_id=self.server.id) if self.session else None
# return File.query.filter_by(src_server_id=self._server_id)
@property
def file_query(self):
return self.session.query(File).filter_by(
deleted=0) if self.session else None
# return File.query
def get_file(self, file_id):
return self.file_query.filter_by(
id=file_id).one_or_none() if self.session else None
# return self.file_query.filter_by(id=file_id).one_or_none()
def _add(self, file_id: Id = None, server_id: Id = None):
if server_id:
if file_id is None:
# new server alive, send all files
for fsa in self.session.query(FileServerAssociation).filter_by(
dst_server_id=server_id,
deleted=0).join(File).filter_by(
src_server_id=self.server.id).all():
if not (getattr(fsa.file, 'deleted', True)
or fsa.destination_server.deleted):
mtime = os.stat(fsa.target).st_mtime_ns
if fsa.l_mtime != mtime:
self._add(fsa.file.id, fsa.destination_server.id)
elif file_id not in self._changed_files:
# send data to a specific server
if file_id not in self._changed_servers:
self._changed_servers[file_id] = [server_id]
else:
if server_id not in self._changed_servers[file_id]:
self._changed_servers[file_id].append(server_id)
else:
self._changed_files.add(file_id)
if file_id in self._changed_servers:
self._changed_servers.pop(file_id)
@staticmethod
def _read_file(file, compress=True):
with open(file, 'rb') as fd:
if compress:
return base64.b64encode(zlib.compress(
fd.read())).decode('utf-8')
else:
return base64.b64encode(fd.read()).decode('utf-8')
async def _send_file(self, file: File, servers: t.List[Id] = None):
try:
content = await self._loop.run_in_executor(self._executor,
self._read_file,
file.target)
except Exception as e:
self.logger.exception(
f"Unable to get content from file {file.target}.")
return
if servers:
server_ids = servers
fsas = [
fsa for fsa in file.destinations
if fsa.destination_server.id in server_ids
]
else:
server_ids = [
fsa.destination_server.id for fsa in file.destinations
]
fsas = file.destinations
with self.dm.flask_app.app_context():
auth = get_root_auth()
alive = self.dm.cluster_manager.get_alive()
tasks = [
ntwrk.async_post(fsa.destination_server,
view_or_url='api_1_0.file_sync',
view_data={'file_id': file.id},
json=dict(file=fsa.target,
data=content,
force=True),
auth=auth) for fsa in fsas
if fsa.destination_server.id in alive
]
skipped = [
fsa.destination_server.name for fsa in fsas
if fsa.destination_server.id not in alive
]
if skipped:
self.logger.debug(
f"Following servers are skipped because we do not see them alive: {', '.join(skipped)}"
)
if tasks:
self.logger.debug(
f"Syncing file {file} with the following servers: {', '.join([fsa.destination_server.name for fsa in fsas if fsa.destination_server.id in alive])}."
)
resp = await asyncio.gather(*tasks)
for resp, fsa in zip(resp, fsas):
if not resp.ok:
self.logger.warning(
f"Unable to send file {file.target} to {fsa.destination_server}. Reason: {resp}"
)
if (file.id, fsa.destination_server.id
) not in self._blacklist:
bl = BlacklistEntry()
self._blacklist[(file.id,
fsa.destination_server.id)] = bl
else:
bl = self._blacklist.get(
(file.id, fsa.destination_server.id))
bl.retries += 1
if bl.retries >= self.max_allowed_errors:
self.logger.debug(
f"Adding server {fsa.destination_server} to the blacklist."
)
bl.blacklisted = time.time()
else:
if (file.id,
fsa.destination_server.id) in self._blacklist:
self._blacklist.pop(
(file.id, fsa.destination_server.id), None)
fsa.l_mtime = file.l_mtime
try:
self.session.commit()
except:
self.session.rollback()
def _sync_files(self):
coros = []
for file_id in self._changed_files:
f = self.get_file(file_id)
if f:
try:
f.l_mtime = os.stat(f.target).st_mtime_ns
except FileNotFoundError:
pass
else:
coros.append(self._send_file(f))
if coros:
try:
self.session.commit()
except:
self.session.rollback()
for file_id, server_id in list(self._blacklist.keys()):
file = self.get_file(file_id)
dest = self.session.query(Server).filter_by(deleted=0).filter_by(
id=server_id).count()
bl = self._blacklist[(file_id, server_id)]
if file and dest: # file and server in the black list may be deleted
# if server_id in getattr(getattr(self.app, 'cluster_manager', None), 'cluster', [server_id]):
if bl.retries < self.max_allowed_errors or time.time(
) - bl.blacklisted > self.retry_blacklist:
if file_id not in self._changed_files:
coros.append(self._send_file(file, [server_id]))
else:
self._blacklist.pop((file_id, server_id), None)
for file_id, server_ids in self._changed_servers.items():
file = self.get_file(file_id)
if file:
coros.append(self._send_file(file, server_ids))
if coros:
try:
self._loop.run_until_complete(
asyncio.gather(*coros, return_exceptions=False))
except Exception:
self.logger.exception("Error while trying to send data.")
self._changed_files.clear()
self._changed_servers.clear()
def _schedule_file(self, file_id, target=None):
if isinstance(file_id, File):
target = file_id.target
file_id = file_id.id
assert target is not None
weh = WatchdogEventHandler(file_id, self, patterns=[target])
try:
watch = self._observer.schedule(weh,
os.path.dirname(target),
recursive=False)
except FileNotFoundError:
pass
else:
self._file2watch.update({file_id: watch})
def _set_initial_modifications(self):
for file in self.my_files_query.all():
self._schedule_file(file)
try:
if os.path.exists(file.target):
mtime = os.stat(file.target).st_mtime_ns
if mtime != file.l_mtime:
self._add(file.id, None)
else:
for fsa in file.destinations:
if fsa.l_mtime != mtime:
self._add(file.id, fsa.destination_server.id)
except:
pass
def _set_watchers(self, force=False):
if self._last_file_updated is None or time.time(
) - self._last_file_updated > self.file_watches_refresh_period or force:
self._last_file_updated = time.time()
id2target = {
f.id: f.target
for f in self.my_files_query.options(
orm.load_only("id", "target")).all()
}
files_from_db = set(id2target.keys())
files_already_watching = set(self._file2watch.keys())
file_watches_to_remove = files_already_watching - files_from_db
file_watches_to_add = files_from_db - files_already_watching
if file_watches_to_remove:
self.logger.debug(
f"Unscheduling the following files: {file_watches_to_remove}"
)
for file_id in file_watches_to_remove:
self._observer.unschedule(self._file2watch[file_id])
self._file2watch.pop(file_id)
if file_watches_to_add:
self.logger.debug(
f"Scheduling the following files: {file_watches_to_add}")
for file_id in file_watches_to_add:
self._schedule_file(file_id, id2target[file_id])
# add for sending file for first time
self._add(file_id, None)
@property
def my_logs(self):
return self.session.query(Log).filter_by(
source_server=Server.get_current(session=self.session)).all()
def update_mapper(self):
logs = self.my_logs
id2log = {log.id: log for log in logs}
# remove logs
for log_id in list(self._mapper.keys()):
if log_id not in id2log:
del self._mapper[log_id]
# add new logs
for log in logs:
if log.id not in self._mapper:
self._mapper[log.id] = []
self.update_pytail_objects(log, self._mapper[log.id])
def update_pytail_objects(self, log: Log, pytail_list: t.List):
if os.path.isfile(log.target):
if len(pytail_list) == 0:
filename = '.' + os.path.basename(log.target) + '.offset'
path = os.path.dirname(log.target)
offset_file = self.dm.config.path(defaults.OFFSET_DIR,
remove_root(path), filename)
if not os.path.exists(offset_file):
_log_logger.debug(f"creating offset file {offset_file}")
os.makedirs(os.path.dirname(offset_file), exist_ok=True)
pytail_list.append(
_PygtailBuffer(file=log.target,
offset_mode='manual',
offset_file=offset_file))
else:
for folder, dirnames, filenames in os.walk(log.target):
for filename in filenames:
if log._re_include.search(
filename) and not log._re_exclude.search(filename):
file = os.path.join(folder, filename)
offset_file = os.path.join(folder,
'.' + filename + '.offset')
if not any(map(lambda p: p.file == file, pytail_list)):
pytail_list.append(
_PygtailBuffer(file=file,
offset_mode='manual',
offset_file=offset_file))
if not log.recursive:
break
new_dirnames = []
for dirname in dirnames:
if log._re_include.search(
dirname) and not log._re_exclude.search(dirname):
new_dirnames.append(dirname)
dirnames[:] = new_dirnames
def _send_new_data(self):
self.update_mapper()
tasks = OrderedDict()
for log_id, pb in self._mapper.items():
log = self.session.query(Log).get(log_id)
for pytail in pb:
data = pytail.fetch()
data = data.encode() if isinstance(data, str) else data
if data and log.destination_server.id in self.dm.cluster_manager.get_alive(
):
if log.mode == Mode.MIRROR:
file = pytail.file
elif log.mode == Mode.REPO_ROOT:
path_to_remove = os.path.dirname(log.target)
relative = os.path.relpath(pytail.file, path_to_remove)
file = os.path.join('{LOG_REPO}', relative)
elif log.mode == Mode.FOLDER:
path_to_remove = os.path.dirname(log.target)
relative = os.path.relpath(pytail.file, path_to_remove)
file = os.path.join(log.dest_folder, relative)
else:
def get_root(dirname):
new_dirname = os.path.dirname(dirname)
if new_dirname == dirname:
return dirname
else:
return get_root(new_dirname)
relative = os.path.relpath(pytail.file,
get_root(pytail.file))
file = os.path.join('{LOG_REPO}', relative)
with self.dm.flask_app.app_context():
auth = get_root_auth()
task = ntwrk.async_post(
log.destination_server,
'api_1_0.logresource',
view_data={'log_id': str(log_id)},
json={
"file":
file,
'data':
base64.b64encode(
zlib.compress(data)).decode('ascii'),
"compress":
True
},
auth=auth)
tasks[task] = (pytail, log)
_log_logger.debug(
f"Task sending data from '{pytail.file}' to '{log.destination_server}' prepared"
)
if tasks:
with self.dm.flask_app.app_context():
responses = asyncio.run(asyncio.gather(*list(tasks.keys())))
for task, resp in zip(tasks.keys(), responses):
pytail, log = tasks[task]
if resp.ok:
pytail.update_offset_file()
_log_logger.debug(f"Updated offset from '{pytail.file}'")
if log.id not in self._blacklist:
self._blacklist_log.pop(log.id, None)
else:
_log_logger.error(
f"Unable to send log information from '{pytail.file}' to '{log.destination_server}'. Error: {resp}"
)
if log.id not in self._blacklist:
bl = BlacklistEntry()
self._blacklist_log[log.id] = bl
else:
bl = self._blacklist_log.get(log.id)
bl.retries += 1
if bl.retries >= self.max_allowed_errors:
_log_logger.debug(
f"Adding server {log.destination_server.id} to the blacklist."
)
bl.blacklisted = time.time()
class Feeder():
def __init__(self):
logger.debug("Initializing Feeder")
self.office = list(map(float, os.environ["OFFICE"].split(",")))
self.alg_id = None
self.recording_volume = os.environ["STORAGE_VOLUME"]
self.every_nth_frame = int(os.environ["EVERY_NTH_FRAME"])
#Hosts
self.dbhost = os.environ["DBHOST"]
self.vahost = "http://localhost:8080/pipelines"
self.mqtthost = os.environ["MQTTHOST"]
#Clients
self.db_alg = DBIngest(host=self.dbhost,
index="algorithms",
office=self.office)
self.db_inf = DBIngest(host=self.dbhost,
index="analytics",
office=self.office)
self.db_sensors = DBQuery(host=self.dbhost,
index="sensors",
office=self.office)
self.mqttclient = None
self.mqtttopic = None
self.observer = Observer()
self.batchsize = 300
self.inference_cache = []
self._threadflag = False
def start(self):
logger.info(" ### Starting Feeder ### ")
logger.debug("Waiting for VA startup")
r = requests.Response()
r.status_code = 400
while r.status_code != 200 and r.status_code != 201:
try:
r = requests.get(self.vahost)
except Exception as e:
r = requests.Response()
r.status_code = 400
time.sleep(10)
# Register Algorithm
logger.debug("Registering as algorithm in the DB")
while True:
try:
self.alg_id = self.db_alg.ingest({
"name": "object_detection",
"office": {
"lat": self.office[0],
"lon": self.office[1]
},
"status": "idle",
"skip": self.every_nth_frame,
})["_id"]
break
except Exception as e:
logger.debug("Register algo exception: " + str(e))
time.sleep(10)
self.mqtttopic = "smtc_va_inferences_" + self.alg_id
camera_monitor_thread = Thread(target=self.monitor_cameras,
daemon=True)
logger.debug("Starting working threads")
self._threadflag = True
self.startmqtt()
self.observer.start()
camera_monitor_thread.start()
logger.debug("Waiting for interrupt...")
camera_monitor_thread.join()
self.observer.join()
def stop(self):
logger.info(" ### Stopping Feeder ### ")
self._threadflag = False
logger.debug("Unregistering algorithm from DB")
self.db_alg.delete(self.alg_id)
self.mqttclient.loop_stop()
self.observer.stop()
def startmqtt(self):
self.mqttclient = mqtt.Client("feeder_" + self.alg_id)
self.mqttclient.connect(self.mqtthost)
self.mqttclient.on_message = self.mqtt_handler
self.mqttclient.loop_start()
self.mqttclient.subscribe(self.mqtttopic)
def mqtt_handler(self, client, userdata, message):
m_in = json.loads(str(message.payload.decode("utf-8", "ignore")))
for tag in m_in["tags"]:
m_in[tag] = m_in["tags"][tag]
del m_in["tags"]
m_in["time"] = m_in["real_base"] + m_in["timestamp"]
# convert to milliseconds
m_in["time"] = int(m_in["time"] / 1000000)
self.inference_cache.append(m_in)
if len(self.inference_cache) >= self.batchsize:
try:
self.db_inf.ingest_bulk(self.inference_cache[:self.batchsize])
self.inference_cache = self.inference_cache[self.batchsize:]
except Exception as e:
logger.debug("Ingest Error: " + str(e))
def monitor_cameras(self):
logger.debug("Starting Sensor Monitor Thread")
while self._threadflag:
logger.debug("Searching for sensors...")
try:
for sensor in self.db_sensors.search(
"sensor:'camera' and status:'idle' and office:[" +
str(self.office[0]) + "," + str(self.office[1]) + "]"):
logger.debug(sensor)
try:
fswatch = None
logger.debug("Sensor found! " + sensor["_id"])
logger.debug("Setting sensor " + sensor["_id"] +
" to streaming")
r = self.db_sensors.update(sensor["_id"],
{"status": "streaming"},
version=sensor["_version"])
logger.debug(
"Setting algorithm to streaming from sensor " +
sensor["_id"])
r = self.db_alg.update(self.alg_id, {
"source": sensor["_id"],
"status": "processing"
})
# Attempt to POST to VA service
jsonData = {
"source": {
"uri": sensor["_source"]["url"],
"type": "uri"
},
"tags": {
"algorithm": self.alg_id,
"sensor": sensor["_id"],
"office": {
"lat": self.office[0],
"lon": self.office[1],
},
},
"parameters": {
"every-nth-frame": self.every_nth_frame,
"recording_prefix":
"recordings/" + sensor["_id"],
"method": "mqtt",
"address": self.mqtthost,
"clientid": self.alg_id,
"topic": self.mqtttopic
},
}
folderpath = os.path.join(
os.path.realpath(self.recording_volume),
sensor["_id"])
if not os.path.exists(folderpath):
os.makedirs(folderpath)
logger.debug("Adding folder watch for " + folderpath)
filehandler = FSHandler(
sensor=sensor["_id"],
office=self.office,
dbhost=self.dbhost,
rec_volume=self.recording_volume)
fswatch = self.observer.schedule(filehandler,
folderpath,
recursive=True)
try:
logger.info("Posting Request to VA Service")
r = requests.post(self.vahost +
"/object_detection/2",
json=jsonData,
timeout=10)
r.raise_for_status()
pipeline_id = None
if r.status_code == 200:
logger.debug("Started pipeline " + r.text)
pipeline_id = int(r.text)
while r.status_code == 200:
logger.debug("Querying status of pipeline")
r = requests.get(self.vahost +
"/object_detection/2/" +
str(pipeline_id) + "/status",
timeout=10)
r.raise_for_status()
jsonValue = r.json()
if "avg_pipeline_latency" not in jsonValue:
jsonValue["avg_pipeline_latency"] = 0
state = jsonValue["state"]
try:
logger.debug("fps: ")
logger.debug(str(jsonValue))
except:
logger.debug("error")
logger.debug("Pipeline state is " + str(state))
if state == "COMPLETED" or state == "ABORTED" or state == "ERROR":
logger.debug("Pipeline ended")
break
self.db_alg.update(
self.alg_id, {
"performance":
jsonValue["avg_fps"],
"latency":
jsonValue["avg_pipeline_latency"] *
1000
})
time.sleep(10)
logger.debug("Setting sensor " + sensor["_id"] +
" to disconnected")
r = self.db_sensors.update(
sensor["_id"], {"status": "disconnected"})
except requests.exceptions.RequestException as e:
logger.error(
"Feeder: Request to VA Service Failed: " +
str(e))
logger.debug("Setting sensor " + sensor["_id"] +
" to idle")
r = self.db_sensors.update(sensor["_id"],
{"status": "idle"})
except Exception as e:
logger.error("Feeder Exception: " + str(e))
if fswatch:
self.observer.unschedule(fswatch)
del (filehandler)
logger.debug("Setting algorithm to idle")
r = self.db_alg.update(self.alg_id, {"status": "idle"})
break
except Exception as e:
print(e, flush=True)
time.sleep(5)
logger.debug("Sensor monitor thread done")
class INotify(PollMixin):
"""
I am a prototype INotify, made to work on Mac OS X (Darwin)
using the Watchdog python library. This is actually a simplified subset
of the twisted Linux INotify class because we do not utilize the watch mask
and only implement the following methods:
- watch
- startReading
- stopReading
- wait_until_stopped
- set_pending_delay
"""
def __init__(self):
self._pending_delay = 1.0
self.recursive_includes_new_subdirectories = False
self._callbacks = {}
self._watches = {}
self._state = NOT_STARTED
self._observer = Observer(timeout=self._pending_delay)
def set_pending_delay(self, delay):
Message.log(message_type=u"watchdog:inotify:set-pending-delay",
delay=delay)
assert self._state != STARTED
self._pending_delay = delay
def startReading(self):
with start_action(action_type=u"watchdog:inotify:start-reading"):
assert self._state != STARTED
try:
# XXX twisted.internet.inotify doesn't require watches to
# be set before startReading is called.
# _assert(len(self._callbacks) != 0, "no watch set")
self._observer.start()
self._state = STARTED
except:
self._state = STOPPED
raise
def stopReading(self):
with start_action(action_type=u"watchdog:inotify:stop-reading"):
if self._state != STOPPED:
self._state = STOPPING
self._observer.unschedule_all()
self._observer.stop()
self._observer.join()
self._state = STOPPED
def loseConnection(self):
return self.stopReading()
def wait_until_stopped(self):
return self.poll(lambda: self._state == STOPPED)
def _isWatched(self, path_u):
return path_u in self._callbacks.keys()
def ignore(self, path):
path_u = path.path
self._observer.unschedule(self._watches[path_u])
del self._callbacks[path_u]
del self._watches[path_u]
def watch(self,
path,
mask=IN_WATCH_MASK,
autoAdd=False,
callbacks=None,
recursive=False):
precondition(isinstance(autoAdd, bool), autoAdd=autoAdd)
precondition(isinstance(recursive, bool), recursive=recursive)
assert autoAdd == False
path_u = path.path
if not isinstance(path_u, unicode):
path_u = path_u.decode('utf-8')
_assert(isinstance(path_u, unicode), path_u=path_u)
if path_u not in self._callbacks.keys():
self._callbacks[path_u] = callbacks or []
self._watches[path_u] = self._observer.schedule(
INotifyEventHandler(path_u, mask, self._callbacks[path_u],
self._pending_delay),
path=path_u,
recursive=False,
)
class FolderWatcher:
def __init__(self):
self.__observer = Observer()
self.__handlers: typing.Dict[str, typing.Union[
SimpleAsyncEventHandler, FileEventToHABAppEvent]] = {}
self.__watches = {}
self.__started = False
def start(self, shutdown_helper):
from ..shutdown_helper import ShutdownHelper
assert isinstance(shutdown_helper, ShutdownHelper)
# we shall only start once!
assert self.__started is False
self.__started = True
# start watching the folders
self.__observer.start()
# register for proper shutdown
shutdown_helper.register_func(self.__observer.stop)
shutdown_helper.register_func(self.__observer.join, last=True)
return None
def watch_folder(self,
folder: pathlib.Path,
file_ending: str,
target_func,
watch_subfolders=False,
worker_factory=None) -> SimpleAsyncEventHandler:
assert isinstance(folder, pathlib.Path), type(folder)
assert folder.is_dir(), folder
folder_str = str(folder)
assert folder_str not in self.__watches, folder_str
self.__handlers[folder_str] = handler = SimpleAsyncEventHandler(
target_func=target_func,
file_ending=file_ending,
worker_factory=worker_factory)
self.__watches[folder_str] = self.__observer.schedule(
handler, path=folder_str, recursive=watch_subfolders)
return handler
def watch_folder_habapp_events(self,
folder: pathlib.Path,
file_ending: str,
habapp_topic: str,
watch_subfolders: bool = False):
assert isinstance(folder, pathlib.Path), type(folder)
assert folder.is_dir(), folder
folder_str = str(folder)
assert folder_str not in self.__watches, folder_str
self.__handlers[folder_str] = handler = FileEventToHABAppEvent(
folder=folder,
habapp_topic=habapp_topic,
file_ending=file_ending,
recursive=watch_subfolders)
self.__watches[folder_str] = self.__observer.schedule(
handler, path=folder_str, recursive=watch_subfolders)
return handler
def unwatch_folder(self, folder):
if isinstance(folder, pathlib.Path):
folder = str(folder)
assert isinstance(folder, str), type(folder)
self.__handlers.pop(folder)
self.__observer.unschedule(self.__watches.pop(folder))
def get_handler(
self, folder
) -> typing.Union[SimpleAsyncEventHandler, FileEventToHABAppEvent]:
if isinstance(folder, pathlib.Path):
folder = str(folder)
assert isinstance(folder, str), type(folder)
return self.__handlers[folder]
class ChangesWatcher(FileSystemEventHandler):
def __init__(self, application):
self.task_map = {} # path: ChangesKeeper
self.application = application
self.observer = Observer()
self.changes = []
self.changes_timer = None
self.observer.start()
def add_watch(self, path, mute_list=None):
if path in self.task_map:
return False
else:
mute_list = (mute_list or []) + DEFAULT_MUTE_LIST
keeper = ChangesKeeper(path, mute_list)
self.task_map[path] = keeper
if os.path.exists(path):
self.observer.schedule(self, path, recursive=True)
return True
return False
def remove_watch(self, path):
if path in self.task_map:
keeper = self.task_map[path]
watch = keeper.watch
self.observer.unschedule(watch)
return True
else:
return False
def get_changes_since(self, timestamp, parent_path=None):
ret = []
for change in self.changes:
if change.timestamp > timestamp and (
not parent_path
or path_is_parent(parent_path, change.path)):
ret.append(change)
return ret
def add_pure_change(self, change):
""" 监测change的类型，并添加非垃圾change和不在黑名单中的change
"""
# 如果是黑名单及黑名单子目录的change，则跳过
for mute_list in [
keeper.mute_list for keeper in self.task_map.values()
]:
for mute_path in mute_list:
if path_is_parent(mute_path, change.path):
print '...', change.type, change.path
return
# 寻找当前change对应的垃圾change，找到后删除；未找到则添加当前change
trash_changes = self.find_related_trash_changes(change)
if trash_changes:
for change in trash_changes:
self.changes.remove(change)
print '- ', change.type, change.path
else:
self.changes.append(change)
print '+ ', change.type, change.path
self.compile_if_needed(change)
ioloop.IOLoop.instance().add_callback(
lambda: self.remove_outdated_changes(30))
def compile_if_needed(self, change):
if change.type == EVENT_TYPE_DELETED:
return
input_path = normalize_path(change.path)
base_path, ext = os.path.splitext(input_path)
ext = ext.lower()
if ext not in ['.less', '.coffee']:
return
project = self.application.find_project(input_path)
if not project:
return
os.chdir(APP_FOLDER)
begin_time = time.time()
if ext == '.less':
if project.compileLess:
output_path = base_path + '.css'
run_cmd('%s bundled/less/bin/lessc %s %s' %
(NODE_BIN_PATH, input_path, output_path))
print '.less ->- .css', change.path, time.time(
) - begin_time, 'seconds'
else:
print '.less -X- .css', change.path, '(OFF by settings)'
elif ext == '.coffee':
if project.compileCoffee:
run_cmd('%s bundled/coffee/bin/coffee --compile %s' %
(NODE_BIN_PATH, input_path))
print '.coffee ->- .js', change.path, time.time(
) - begin_time, 'seconds'
else:
print '.coffee -X- .js', change.path, '(OFF by settings)'
def check_folder_change(self, folder_path):
if sys.platform.startswith('win') or \
not os.path.isdir(folder_path):
return
now = time.time() - 0.5 # 0.5秒内的都算修改
for filename in os.listdir(folder_path):
file_path = os.path.join(folder_path, filename)
if not os.path.isfile(file_path):
continue
modified_time = os.path.getmtime(file_path)
if modified_time > now:
self.on_any_event(FileModifiedEvent(file_path))
def on_any_event(self, event):
if event.is_directory:
self.check_folder_change(event.src_path)
return
# 暂停文件变更的上报, 以免中途编译占用太长时间，而将事件提前返回
loop = ioloop.IOLoop.instance()
if self.changes_timer:
ioloop.IOLoop.instance().remove_timeout(self.changes_timer)
now = time.time()
if event.event_type == EVENT_TYPE_MOVED:
self.add_pure_change(
Change(
dict(timestamp=now,
path=normalize_path(event.src_path),
type=EVENT_TYPE_DELETED)))
self.add_pure_change(
Change(
dict(timestamp=now,
path=normalize_path(event.dest_path),
type=EVENT_TYPE_CREATED)))
else:
self.add_pure_change(
Change(
dict(timestamp=now,
path=normalize_path(event.src_path),
type=event.event_type)))
# 延迟0.1秒上报变更，防止有些事件连续发生时错过
self.changes_timer = loop.add_timeout(
time.time() + 0.1, self.application.project_file_changed)
def find_related_trash_changes(self, change):
""" 寻找当前change之前短时间内的一些垃圾change
有些编辑器喜欢用 改名->写入->改回名 的方式来保存文件，所以不能直接将change上报，需要进行一定的过滤
"""
trash_changes = []
for old_change in self.changes[::-1]:
if old_change.path != change.path:
continue
if change.timestamp > old_change.timestamp + 1:
break
if change.type == EVENT_TYPE_DELETED:
# 如果当前change类型是DELETED，那么返回所有该文件的事件，直到CREATED事件为止
trash_changes.append(old_change)
if old_change.type == EVENT_TYPE_CREATED:
return trash_changes
elif change.type == EVENT_TYPE_CREATED:
# 如果当前change类型是CREATED，那么返回所有该文件的事件，直到DELETED事件为止
trash_changes.append(old_change)
if old_change.type == EVENT_TYPE_DELETED:
return trash_changes
return []
def remove_outdated_changes(self, seconds):
for change in self.changes[:]:
if change.timestamp - time.time() > seconds:
self.changes.remove(change)
class KicadProject(FileSystemEventHandler):
def __init__(self, project_file):
self.on_change_hook = None
self.files = {}
self.folders = []
self.extensions = ['bom', 'net', 'sch', 'kicad_pcb']
self.project_file = project_file
self.root_path = os.path.dirname(self.project_file)
# observer to trigger event on resource change
self.enabled = True
self.observer = Observer()
self.watch = self.observer.schedule(self,
self.root_path,
recursive=True)
self.observer.start()
def on_any_event(self, event):
if self.enabled == False:
return
self.enabled = False
for extension in self.extensions:
if hasattr(event, 'dest_path') and os.path.isfile(
event.dest_path) and os.path.basename(
event.dest_path).startswith(
'.') == False and event.dest_path.endswith(
'.' + extension):
print("Something happend with %s" % (event.dest_path))
path = os.path.relpath(event.dest_path, self.root_path)
if self.on_change_hook:
wx.CallAfter(self.on_change_hook, path)
elif hasattr(event, 'src_path') and os.path.isfile(
event.src_path
) and os.path.basename(event.src_path).startswith(
'.') == False and event.src_path.endswith('.' + extension):
print("Something happend with %s" % (event.src_path))
path = os.path.relpath(event.src_path, self.root_path)
if self.on_change_hook:
wx.CallAfter(self.on_change_hook, path)
self.enabled = True
# - on_moved(self, event)
# - on_created(self, event)
# - on_deleted(self, event)
# - on_modified(self, event)
def Enabled(self, enabled=True):
if enabled:
if self.watch is None:
self.watch = self.observer.schedule(self,
self.root_path,
recursive=True)
else:
if self.watch:
self.observer.unschedule(self.watch)
self.watch = None
def Load(self):
"""
fill cache files from disk
"""
self.files, self.folders = self.GetFiles()
def GetFiles(self):
"""
Recurse all folders and return kicad files and folders path
"""
basepath = os.path.normpath(os.path.abspath(self.root_path))
to_explore = [basepath]
files = []
folders = []
while len(to_explore) > 0:
path = to_explore.pop()
if os.path.exists(path):
for file in glob(os.path.join(path, "*")):
if os.path.isfile(file):
for ext in self.extensions:
if os.path.normpath(
os.path.abspath(file)).endswith(ext):
files.append(
os.path.relpath(
os.path.normpath(
os.path.abspath(file)), basepath))
else:
folder = file
if folder != '/':
folders.append(
os.path.relpath(
os.path.normpath(os.path.abspath(folder)),
basepath))
if os.path.normpath(os.path.abspath(
folder)) != os.path.normpath(
os.path.abspath(path)):
to_explore.append(folder)
return files, folders
class ModuleFileWatcher( EditorModule ):
def __init__(self):
super(ModuleFileWatcher, self).__init__()
self.watches={}
def getName(self):
return 'filewatcher'
def getDependency(self):
return []
def onLoad(self):
self.observer=Observer()
self.observer.start()
signals.connect( 'file.moved', self.onFileMoved )
signals.connect( 'file.added', self.onFileCreated )
signals.connect( 'file.removed', self.onFileDeleted )
signals.connect( 'file.modified', self.onFileModified )
def onStart( self ):
self.assetWatcher=self.startWatch(
self.getProject().getAssetPath(),
ignorePatterns = ['*/.git','*/.*','*/_gii']
)
def startWatch(self, path, **options):
path = os.path.realpath(path)
if self.watches.get(path):
logging.warning( 'already watching: %s' % path )
return self.watches[path]
logging.info ( 'start watching: %s' % path )
ignorePatterns = ['*/.git','*/.*','*/_gii'] + options.get('ignorePatterns',[])
handler = FileWatcherEventHandler(
options.get( 'patterns', None ),
ignorePatterns,
options.get( 'ignoreDirectories', False ),
options.get( 'caseSensitive', True )
)
watch = self.observer.schedule( handler, path, options.get( 'recursive', True ) )
self.watches[ path ] = watch
return watch
def onStop( self ):
# print 'stop file watcher'
self.observer.stop()
self.observer.join( 0.5 )
# print 'stopped file watcher'
def stopWatch(self, path):
path = os.path.realpath(path)
watch = self.watches.get(path, None)
if not watch: return
self.observer.unschedule(watch)
self.watches[path] = None
def stopAllWatches(self):
# logging.info('stop all file watchers')
self.observer.unschedule_all()
self.watches = {}
def onFileMoved(self, path, newpath):
# print('asset moved:',path, newpath)
app.getAssetLibrary().scheduleScanProject()
pass
def onFileCreated(self, path):
# print('asset created:',path)
app.getAssetLibrary().scheduleScanProject()
pass
def onFileModified(self, path):
# print('asset modified:',path)
app.getAssetLibrary().scheduleScanProject()
pass
def onFileDeleted(self, path):
# print('asset deleted:',path)
app.getAssetLibrary().scheduleScanProject()
pass
class WatchdogTests(unittest.TestCase):
def setUp(self):
self.events = queue.Queue()
self.observer = Observer()
self.handler = EventHandler(self.events, channel_id=5)
self.watch = self.observer.schedule(event_handler=self.handler, path=TMP_DIR)
self.observer.start()
def testEventChannelID(self):
with open(TMP_FILE, "w") as local_file:
local_file.write(string_common)
event = self.events.get(block=True, timeout=1)
self.assertEqual(event.channel_id, 5)
# Modifying the file should add a new task to the queue
def testTaskInsertion(self):
new_string = "This is a whole new line in file "+filename
with open (TMP_FILE, "w") as local_file:
local_file.write(new_string)
while True:
try:
task = self.events.get(block=True, timeout=1)
except queue.Empty:
break
self.assertEqual(task.src_path,local_dir+filename)
self.assertEqual(task.dest_path, remote_dir+filename)
# It should be possible to add a new Handler to an already scheduled watch
def testHandlerAdding(self):
fake_handle="I am a fake handler, don't mind me!"
class FakeHandler(EventHandler):
def handle_modification(self, event):
self.tasks.put(fake_handle)
fake_queue = queue.Queue()
fake_handler = FakeHandler(fake_queue)
self.observer.add_handler_for_watch(fake_handler, self.watch)
new_string = "This is a whole new line in file " + filename
with open(local_dir + filename, "w") as local_file:
local_file.write(new_string)
while True:
try:
task = fake_queue.get(block=True, timeout=1)
except queue.Empty:
break
self.assertEqual(task, fake_handle)
# A Handler blocking on one event (ex: inserting into a busy queue)
# should not prevent handling of further events
def testHandlerHanging(self):
class HangingHandler(EventHandler):
def on_any_event(self, event):
print("Handler hanging...")
time.sleep(1)
print("Handler dispatching")
super().on_any_event(event)
self.observer.remove_handler_for_watch(self.handler, self.watch)
slow_handler = HangingHandler(self.events)
self.observer.add_handler_for_watch(slow_handler, self.watch)
for i in range(5):
with open(os.path.join(TMP_DIR, "f"+str(i)), "w") as local_file:
local_file.write("Write #"+str(i))
time.sleep(0.1)
time.sleep(6)
num = len(empty_tasks(self.events))
print("[Hanging] "+str(num)+" events")
self.assertTrue(num >= 5)
# Scheduling a new watch while another one is running
# In this test, each write should set off 2 events (open+close) as seen on the next test
# However watchdog is nice enough to avoid adding similar events to its internal queue
def testNewScheduling(self):
self.assertTrue(self.events.empty()) # Queue starts off
with open(remote_dir + filename, "w") as remote_file:
remote_file.write("Going once")
self.assertRaises(queue.Empty, self.events.get, timeout=1) # No Handler is watching the file yet
handler2 = EventHandler(self.events)
self.observer.schedule(event_handler=handler2, path=remote_dir, recursive=True)
with open(remote_dir + filename, "w") as remote_file:
remote_file.write("Going twice")
l1 = empty_tasks(self.events) # Now it should work
self.assertEqual(len(l1), 1) # Single event
for task in l1:
self.assertEqual(task.src_path, remote_dir + filename)
self.assertEqual(task.dest_path, local_dir + filename)
with open(local_dir + filename, "w") as local_file:
local_file.write("Going thrice") # Writing to the local file still works
l2 = empty_tasks(self.events)
self.assertEqual(len(l2), 1) # Single event
for task in l2:
self.assertEqual(task.src_path, local_dir + filename)
self.assertEqual(task.dest_path, remote_dir + filename)
# It should be possible to remove a scheduled watch
def testWatchRemoval(self):
handler2 = EventHandler(self.events)
watch2 = self.observer.schedule(event_handler=handler2, path=remote_dir, recursive=True)
for client in [ {"path":local_dir+filename, "watch":self.watch},
{"path":remote_dir+filename, "watch":watch2} ]:
with open(client["path"], "w") as file:
file.write("This will make an event")
time.sleep(0.5)
task = self.events.get(timeout=1)
self.assertEquals(task.src_path, client["path"])
self.observer.unschedule(client["watch"])
with open(local_dir+filename, "w") as local_file:
local_file.write("This won't")
self.assertRaises(queue.Empty, self.events.get, timeout=1)
# Each open() and each close() should produce an event
# They are sometimes squashed into a single event if done
# Quickly enough (i.e. "with open(file) as f: f.write()")
def testEventsPerWrite(self):
local_file = open(local_dir+filename, "w")
self.assertTrue(len(empty_tasks(self.events)) == 1) # Opening sets off an event
local_file.write("First")
self.assertTrue(len(empty_tasks(self.events)) == 0) # Writing doesn't set off an event
local_file.write("Second")
self.assertTrue(len(empty_tasks(self.events)) == 0) # Writing doesn't set off an event
local_file.close()
self.assertTrue(len(empty_tasks(self.events)) == 1) # Closing sets off an event
def testEventsForBigFileCopy(self):
self.observer.schedule(self.handler, TMP_DIR, recursive=True)
try:
os.mkdir(TMP_DIR)
except FileExistsError:
pass
try:
shutil.copy(BIG_FILE, BIG_IN)
except OSError:
self.fail()
num = empty_tasks(self.events)
print("Used "+str(len(num))+" events")
class FileObserver:
"""
A class that will observe some file system paths for any change.
"""
def __init__(self, on_change):
"""
Initialize the file observer
Parameters
----------
on_change:
Reference to the function that will be called if there is a change in aby of the observed paths
"""
self._observed_paths = {}
self._observed_watches = {}
self._watch_dog_observed_paths = {}
self._observer = Observer()
self._code_change_handler = PatternMatchingEventHandler(
patterns=["*"], ignore_patterns=[], ignore_directories=False)
self._code_change_handler.on_modified = self.on_change
self._input_on_change = on_change
self._lock = threading.Lock()
def on_change(self, event):
"""
It got executed once there is a change in one of the paths that watchdog is observing.
This method will check if any of the input paths is really changed, and based on that it will
invoke the input on_change function with the changed paths
Parameters
----------
event: watchdog.events.FileSystemEventHandler
Determines that there is a change happened to some file/dir in the observed paths
"""
event_path = event.src_path
observed_paths = []
for watchdog_path, child_observed_paths in self._watch_dog_observed_paths.items(
):
if event_path.startswith(watchdog_path):
observed_paths += child_observed_paths
if not observed_paths:
return
changed_paths = []
for path in observed_paths:
path_obj = Path(path)
# The path got deleted
if not path_obj.exists():
self._observed_paths.pop(path, None)
changed_paths += [path]
else:
new_checksum = calculate_checksum(path)
if new_checksum != self._observed_paths.get(path, None):
changed_paths += [path]
self._observed_paths[path] = new_checksum
if changed_paths:
self._input_on_change(changed_paths)
def watch(self, path):
"""
Start watching the input path. File Observer will keep track of the input path with its hash, to check it later
if it got really changed or not.
File Observer will send the parent path to watchdog for to be observed to avoid the missing events if the input
paths got deleted.
Parameters
----------
path: str
The file/dir path to be observed
Raises
------
FileObserverException:
if the input path is not exist
"""
path_obj = Path(path)
if not path_obj.exists():
raise FileObserverException("Can not observe non exist path")
self._observed_paths[path] = calculate_checksum(path)
# Watchdog will observe the path's parent path to make sure not missing the event if the path itself got deleted
parent_path = str(path_obj.parent)
child_paths = self._watch_dog_observed_paths.get(parent_path, [])
first_time = not bool(child_paths)
if path not in child_paths:
child_paths += [path]
self._watch_dog_observed_paths[parent_path] = child_paths
if first_time:
self._observed_watches[parent_path] = self._observer.schedule(
self._code_change_handler, parent_path, recursive=True)
def unwatch(self, path):
"""
Remove the input path form the observed paths, and stop watching this path.
Parameters
----------
path: str
The file/dir path to be unobserved
"""
path_obj = Path(path)
if not path_obj.exists():
raise FileObserverException("Can not unwatch non exist path")
parent_path = str(path_obj.parent)
child_paths = self._watch_dog_observed_paths.get(parent_path, [])
if path in child_paths:
child_paths.remove(path)
self._observed_paths.pop(path, None)
if not child_paths:
self._watch_dog_observed_paths.pop(parent_path, None)
if self._observed_watches[parent_path]:
self._observer.unschedule(self._observed_watches[parent_path])
self._observed_watches.pop(parent_path, None)
def start(self):
"""
Start Observing.
"""
with self._lock:
if not self._observer.is_alive():
self._observer.start()
def stop(self):
"""
Stop Observing.
"""
with self._lock:
if self._observer.is_alive():
self._observer.stop()
class Render:
def __init__(self, modules, dest="dist"):
self.modules = [i for i in modules if i]
assert all(isinstance(i, Module) for i in self.modules)
self.watchers = {}
self.options = {}
self.observer = None
self.dest = dest
def start(self, watch=False):
self.run_once()
if not watch:
return
if not self.watchers:
print("Nothing to do!")
return
self.observer = Observer()
for path in self.watchers:
for index in range(len(self.watchers[path])):
handler = WatchdogHandler(self, path, index)
handler.watcher = self.observer.schedule(handler,
path,
recursive=True)
self.observer.start()
print(':: Watchdog started')
while True:
time.sleep(1)
def handle(self, path, index, event, watcher):
module, options = self.watchers[path][index]
self.watchers[path].pop(index)
self.observer.unschedule(watcher)
new_watchers = []
def watch(path, **options):
self.watchers[path].insert(index, (module, options))
new_watchers.append(path)
self.watch = watch
module.handle(self, path, event, **options)
if not new_watchers:
print("Watcher unregistered!")
else:
for n, path in enumerate(new_watchers):
handler = WatchdogHandler(self, path, index + n)
handler.watcher = self.observer.schedule(handler,
path,
recursive=True)
def run_once(self):
self.watchers = {}
for i in self.modules:
def watch(path, **options):
if path not in self.watchers:
self.watchers[path] = []
self.watchers[path].append((i, options))
self.watch = watch
i.run(self)
def abort(self):
print()
print("Aborted!")
quit()
class GoSyncModel(object):
def __init__(self):
self.calculatingDriveUsage = False
self.driveAudioUsage = 0
self.driveMoviesUsage = 0
self.drivePhotoUsage = 0
self.driveDocumentUsage = 0
self.driveOthersUsage = 0
self.totalFilesToCheck = 0
self.savedTotalSize = 0
self.fcount = 0
self.updates_done = 0
self.config_path = os.path.join(os.environ['HOME'], ".gosync")
self.credential_file = os.path.join(self.config_path, "credentials.json")
self.settings_file = os.path.join(self.config_path, "settings.yaml")
self.base_mirror_directory = os.path.join(os.environ['HOME'], "Google Drive")
self.client_secret_file = os.path.join(os.environ['HOME'], '.gosync', 'client_secrets.json')
self.sync_selection = []
self.config_file = os.path.join(os.environ['HOME'], '.gosync', 'gosyncrc')
self.config_dict = {}
self.account_dict = {}
self.drive_usage_dict = {}
self.config=None
if not os.path.exists(self.config_path):
os.mkdir(self.config_path, 0755)
raise ClientSecretsNotFound()
if not os.path.exists(self.base_mirror_directory):
os.mkdir(self.base_mirror_directory, 0755)
if not os.path.exists(self.client_secret_file):
raise ClientSecretsNotFound()
if not os.path.exists(self.settings_file) or \
not os.path.isfile(self.settings_file):
sfile = open(self.settings_file, 'w')
sfile.write("save_credentials: False")
sfile.write("\n")
sfile.write("save_credentials_file: ")
sfile.write(self.credential_file)
sfile.write("\n")
sfile.write('client_config_file: ' + self.client_secret_file + "\n")
sfile.write("save_credentials_backend: file\n")
sfile.close()
self.observer = Observer()
self.DoAuthenticate()
self.about_drive = self.authToken.service.about().get().execute()
self.user_email = self.about_drive['user']['emailAddress']
self.mirror_directory = os.path.join(self.base_mirror_directory, self.user_email)
if not os.path.exists(self.mirror_directory):
os.mkdir(self.mirror_directory, 0755)
self.tree_pickle_file = os.path.join(self.config_path, 'gtree-' + self.user_email + '.pick')
if not os.path.exists(self.config_file):
self.CreateDefaultConfigFile()
try:
self.LoadConfig()
except:
raise
self.iobserv_handle = self.observer.schedule(FileModificationNotifyHandler(self),
self.mirror_directory, recursive=True)
self.sync_lock = threading.Lock()
self.sync_thread = threading.Thread(target=self.run)
self.usage_calc_thread = threading.Thread(target=self.calculateUsage)
self.sync_thread.daemon = True
self.usage_calc_thread.daemon = True
self.syncRunning = threading.Event()
self.syncRunning.clear()
self.usageCalculateEvent = threading.Event()
self.usageCalculateEvent.set()
self.logger = logging.getLogger(APP_NAME)
self.logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(os.path.join(os.environ['HOME'], 'GoSync.log'))
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
self.logger.addHandler(fh)
if not os.path.exists(self.tree_pickle_file):
self.driveTree = GoogleDriveTree()
else:
self.driveTree = pickle.load(open(self.tree_pickle_file, "rb"))
def SetTheBallRolling(self):
self.sync_thread.start()
self.usage_calc_thread.start()
self.observer.start()
def IsUserLoggedIn(self):
return self.is_logged_in
def HashOfFile(self, abs_filepath):
data = open(abs_filepath, "r").read()
return hashlib.md5(data).hexdigest()
def CreateDefaultConfigFile(self):
f = open(self.config_file, 'w')
self.config_dict['Sync Selection'] = [['root', '']]
self.account_dict[self.user_email] = self.config_dict
json.dump(self.account_dict, f)
f.close()
def LoadConfig(self):
try:
f = open(self.config_file, 'r')
try:
self.config = json.load(f)
try:
self.config_dict = self.config[self.user_email]
self.sync_selection = self.config_dict['Sync Selection']
print self.config_dict['Drive Usage']
try:
self.drive_usage_dict = self.config_dict['Drive Usage']
self.totalFilesToCheck = self.drive_usage_dict['Total Files']
self.savedTotalSize = self.drive_usage_dict['Total Size']
self.driveAudioUsage = self.drive_usage_dict['Audio Size']
self.driveMoviesUsage = self.drive_usage_dict['Movies Size']
self.driveDocumentUsage = self.drive_usage_dict['Document Size']
self.drivePhotoUsage = self.drive_usage_dict['Photo Size']
self.driveOthersUsage = self.drive_usage_dict['Others Size']
except:
pass
except:
pass
f.close()
except:
raise ConfigLoadFailed()
except:
raise ConfigLoadFailed()
def SaveConfig(self):
f = open(self.config_file, 'w')
f.truncate()
if not self.sync_selection:
self.config_dict['Sync Selection'] = [['root', '']]
self.account_dict[self.user_email] = self.config_dict
json.dump(self.account_dict, f)
f.close()
def DoAuthenticate(self):
try:
self.authToken = GoogleAuth(self.settings_file)
self.authToken.LocalWebserverAuth()
self.drive = GoogleDrive(self.authToken)
self.is_logged_in = True
except:
dial = wx.MessageDialog(None, "Authentication Rejected!\n",
'Information', wx.ID_OK | wx.ICON_EXCLAMATION)
dial.ShowModal()
self.is_logged_in = False
pass
def DoUnAuthenticate(self):
self.do_sync = False
self.observer.unschedule(self.iobserv_handle)
self.iobserv_handle = None
os.remove(self.credential_file)
self.is_logged_in = False
def DriveInfo(self):
return self.about_drive
def PathLeaf(self, path):
head, tail = ntpath.split(path)
return tail or ntpath.basename(head)
def GetFolderOnDrive(self, folder_name, parent='root'):
"""
Return the folder with name in "folder_name" in the parent folder
mentioned in parent.
"""
self.logger.debug("GetFolderOnDrive: searching %s on %s... " % (folder_name, parent))
file_list = self.drive.ListFile({'q': "'%s' in parents and trashed=false" % parent}).GetList()
for f in file_list:
if f['title'] == folder_name and f['mimeType']=='application/vnd.google-apps.folder':
self.logger.debug("Found!\n")
return f
return None
def LocateFolderOnDrive(self, folder_path):
"""
Locate and return the directory in the path. The complete path
is walked and the last directory is returned. An exception is raised
if the path walking fails at any stage.
"""
dir_list = folder_path.split(os.sep)
croot = 'root'
for dir1 in dir_list:
try:
folder = self.GetFolderOnDrive(dir1, croot)
if not folder:
raise FolderNotFound()
except:
raise
croot = folder['id']
return folder
def LocateFileInFolder(self, filename, parent='root'):
try:
file_list = self.MakeFileListQuery({'q': "'%s' in parents and trashed=false" % parent})
for f in file_list:
if f['title'] == filename:
return f
raise FileNotFound()
except:
raise FileNotFound()
def LocateFileOnDrive(self, abs_filepath):
dirpath = os.path.dirname(abs_filepath)
filename = self.PathLeaf(abs_filepath)
if dirpath != '':
try:
f = self.LocateFolderOnDrive(dirpath)
try:
fil = self.LocateFileInFolder(filename, f['id'])
return fil
except FileNotFound:
self.logger.debug("LocateFileOnDrive: File not found.\n")
raise
except FileListQueryFailed:
self.logger.debug("LocateFileOnDrive: File list query failed\n")
raise
except FolderNotFound:
self.logger.debug("LocateFileOnDrive: Folder not found\n")
raise
except FileListQueryFailed:
self.logger.debug("LocateFileOnDrive: %s folder not found\n" % dirpath)
raise
else:
try:
fil = self.LocateFileInFolder(filename)
return fil
except FileNotFound:
self.logger.debug("LocateFileOnDrive: File not found.\n")
raise
except FileListQueryFailed:
self.logger.debug("LocateFileOnDrive: File list query failed.\n")
raise
except:
self.logger.error("LocateFileOnDrive: Unknown error in locating file in drive\n")
raise
def CreateDirectoryInParent(self, dirname, parent_id='root'):
upfile = self.drive.CreateFile({'title': dirname,
'mimeType': "application/vnd.google-apps.folder",
"parents": [{"kind": "drive#fileLink", "id": parent_id}]})
upfile.Upload()
def CreateDirectoryByPath(self, dirpath):
self.logger.debug("create directory: %s\n" % dirpath)
drivepath = dirpath.split(self.mirror_directory+'/')[1]
basepath = os.path.dirname(drivepath)
dirname = self.PathLeaf(dirpath)
try:
f = self.LocateFolderOnDrive(drivepath)
return
except FolderNotFound:
if basepath == '':
self.CreateDirectoryInParent(dirname)
else:
try:
parent_folder = self.LocateFolderOnDrive(basepath)
self.CreateDirectoryInParent(dirname, parent_folder['id'])
except:
errorMsg = "Failed to locate directory path %s on drive.\n" % basepath
self.logger.error(errorMsg)
dial = wx.MessageDialog(None, errorMsg, 'Directory Not Found',
wx.ID_OK | wx.ICON_EXCLAMATION)
dial.ShowModal()
return
except FileListQueryFailed:
errorMsg = "Server Query Failed!\n"
self.logger.error(errorMsg)
dial = wx.MessageDialog(None, errorMsg, 'Directory Not Found',
wx.ID_OK | wx.ICON_EXCLAMATION)
dial.ShowModal()
return
def CreateRegularFile(self, file_path, parent='root', uploaded=False):
self.logger.debug("Create file %s\n" % file_path)
filename = self.PathLeaf(file_path)
upfile = self.drive.CreateFile({'title': filename,
"parents": [{"kind": "drive#fileLink", "id": parent}]})
upfile.SetContentFile(file_path)
upfile.Upload()
def UploadFile(self, file_path):
if os.path.isfile(file_path):
drivepath = file_path.split(self.mirror_directory+'/')[1]
self.logger.debug("file: %s drivepath is %s\n" % (file_path, drivepath))
try:
f = self.LocateFileOnDrive(drivepath)
self.logger.debug('Found file %s on remote (dpath: %s)\n' % (f['title'], drivepath))
newfile = False
self.logger.debug('Checking if they are same... ')
if f['md5Checksum'] == self.HashOfFile(file_path):
self.logger.debug('yes\n')
return
else:
self.logger.debug('no\n')
except (FileNotFound, FolderNotFound):
self.logger.debug("A new file!\n")
newfile = True
dirpath = os.path.dirname(drivepath)
if dirpath == '':
self.logger.debug('Creating %s file in root\n' % file_path)
self.CreateRegularFile(file_path, 'root', newfile)
else:
try:
f = self.LocateFolderOnDrive(dirpath)
self.CreateRegularFile(file_path, f['id'], newfile)
except FolderNotFound:
# We are coming from premise that upload comes as part
# of observer. So before notification of this file's
# creation happens, a notification of its parent directory
# must have come first.
# So,
# Folder not found? That cannot happen. Can it?
raise RegularFileUploadFailed()
else:
self.CreateDirectoryByPath(file_path)
def UploadObservedFile(self, file_path):
self.sync_lock.acquire()
self.UploadFile(file_path)
self.sync_lock.release()
def RenameFile(self, file_object, new_title):
try:
file = {'title': new_title}
updated_file = self.authToken.service.files().patch(fileId=file_object['id'],
body=file, fields='title').execute()
return updated_file
except errors.HttpError, error:
self.logger.error('An error occurred while renaming file: %s' % error)
return None
except:
class _MultiFileWatcher(object):
"""Watches multiple files."""
_singleton = None
@classmethod
def get_singleton(cls):
"""Return the singleton _MultiFileWatcher object.
Instantiates one if necessary.
"""
if cls._singleton is None:
LOGGER.debug("No singleton. Registering one.")
_MultiFileWatcher()
return _MultiFileWatcher._singleton
# Don't allow constructor to be called more than once.
def __new__(cls):
"""Constructor."""
if _MultiFileWatcher._singleton is not None:
raise RuntimeError("Use .get_singleton() instead")
return super(_MultiFileWatcher, cls).__new__(cls)
def __init__(self):
"""Constructor."""
_MultiFileWatcher._singleton = self
# Map of folder_to_watch -> _FolderEventHandler.
self._folder_handlers = {}
# Used for mutation of _folder_handlers dict
self._lock = threading.Lock()
# The Observer object from the Watchdog module. Since this class is
# only instantiated once, we only have a single Observer in Streamlit,
# and it's in charge of watching all paths we're interested in.
self._observer = Observer()
self._observer.start() # Start observer thread.
def watch_file(self, file_path, callback):
"""Start watching a file.
Parameters
----------
file_path : str
The full path of the file to watch.
callback : callable
The function to execute when the file is changed.
"""
folder_path = os.path.abspath(os.path.dirname(file_path))
with self._lock:
folder_handler = self._folder_handlers.get(folder_path)
if folder_handler is None:
folder_handler = _FolderEventHandler()
self._folder_handlers[folder_path] = folder_handler
folder_handler.watch = self._observer.schedule(
folder_handler, folder_path, recursive=False
)
folder_handler.add_file_change_listener(file_path, callback)
def stop_watching_file(self, file_path, callback):
"""Stop watching a file.
Parameters
----------
file_path : str
The full path of the file to stop watching.
callback : callable
The function to execute when the file is changed.
"""
folder_path = os.path.abspath(os.path.dirname(file_path))
with self._lock:
folder_handler = self._folder_handlers.get(folder_path)
if folder_handler is None:
LOGGER.debug(
"Cannot stop watching path, because it is already not being "
"watched. %s",
folder_path,
)
return
folder_handler.remove_file_change_listener(file_path, callback)
if not folder_handler.is_watching_files():
# Sometimes watchdog's FileSystemEventHandler does not have
# a .watch property. It's unclear why -- may be due to a
# race condition.
if hasattr(folder_handler, "watch"):
self._observer.unschedule(folder_handler.watch)
del self._folder_handlers[folder_path]
def close(self):
with self._lock:
"""Close this _MultiFileWatcher object forever."""
if len(self._folder_handlers) != 0:
self._folder_handlers = {}
LOGGER.debug(
"Stopping observer thread even though there is a non-zero "
"number of event observers!"
)
else:
LOGGER.debug("Stopping observer thread")
self._observer.stop()
self._observer.join(timeout=5)
class Rainmaker():
def __init__(self,config=None, auto_start=True ):
self.log=logging.getLogger('main')
self.event_handlers = []
self.config = config if config else RainmakerConfig()
self.profiles = self.config.profiles
self.msg_q = Queue()
self.observer = Observer()
self.observer.start()
if not auto_start:
return
for k in self.profiles:
if self.profiles[k]['auto_start']==True:
self.add_watch(k)
if not self.event_handlers:
self.log.warn('No running profiles')
def add_watch(self,key):
if not key in self.profiles:
self.log.error('unknown profile %s' % key)
self.log.info('Starting profile: %s' % key)
profile = self.profiles[key]
profile['local_root'] = os.path.abspath(os.path.expanduser(profile['local_root']))
profile.subst_all()
if not os.path.isdir(profile['local_root']):
self.log.info('creating dir: %s' % profile['local_root'])
os.mkdir(profile['local_root'])
patterns=['*.unison.tmp']
event_handler = RainmakerEventHandler()
event_handler.init2( self.config, profile, self.msg_q, key )
self.event_handlers.append( event_handler )
rec_flag = True
if profile.has_key('recursive'):
rec_flag = bool(profile['recursive'])
self.observer.schedule( event_handler, profile['local_root'], recursive = rec_flag)
logging.info('Started profile: %s' % key)
if profile['cmds']['startup'] != '':
event_handler.startup_cmd()
def remove_watch(self, k):
for eh in self.event_handlers:
if eh.name == k:
self.log.info('Stopping profile: %s' % k)
self.observer.unschedule(eh)
break
def messages(self):
messages = []
try:
while True:
messages.append( self.msg_q.get_nowait() )
except Empty:
pass
return messages
def shutdown(self):
self.log.info( "Shutting down FSwatcher")
self.observer.stop()
self.observer.unschedule_all()
self.observer.join()
self.log.info("Shutting down thread and Fork pool")
for eh in self.event_handlers:
self.log.info('Stopping profile: %s' % eh.name)
eh.stop()
class INotify(PollMixin):
"""
I am a prototype INotify, made to work on Mac OS X (Darwin)
using the Watchdog python library. This is actually a simplified subset
of the twisted Linux INotify class because we do not utilize the watch mask
and only implement the following methods:
- watch
- startReading
- stopReading
- wait_until_stopped
- set_pending_delay
"""
def __init__(self):
self._pending_delay = 1.0
self.recursive_includes_new_subdirectories = False
self._callbacks = {}
self._watches = {}
self._state = NOT_STARTED
self._observer = Observer(timeout=self._pending_delay)
def set_pending_delay(self, delay):
Message.log(message_type=u"watchdog:inotify:set-pending-delay", delay=delay)
assert self._state != STARTED
self._pending_delay = delay
def startReading(self):
with start_action(action_type=u"watchdog:inotify:start-reading"):
assert self._state != STARTED
try:
# XXX twisted.internet.inotify doesn't require watches to
# be set before startReading is called.
# _assert(len(self._callbacks) != 0, "no watch set")
self._observer.start()
self._state = STARTED
except:
self._state = STOPPED
raise
def stopReading(self):
with start_action(action_type=u"watchdog:inotify:stop-reading"):
if self._state != STOPPED:
self._state = STOPPING
self._observer.unschedule_all()
self._observer.stop()
self._observer.join()
self._state = STOPPED
def wait_until_stopped(self):
return self.poll(lambda: self._state == STOPPED)
def _isWatched(self, path_u):
return path_u in self._callbacks.keys()
def ignore(self, path):
path_u = path.path
self._observer.unschedule(self._watches[path_u])
del self._callbacks[path_u]
del self._watches[path_u]
def watch(self, path, mask=IN_WATCH_MASK, autoAdd=False, callbacks=None, recursive=False):
precondition(isinstance(autoAdd, bool), autoAdd=autoAdd)
precondition(isinstance(recursive, bool), recursive=recursive)
assert autoAdd == False
path_u = path.path
if not isinstance(path_u, unicode):
path_u = path_u.decode('utf-8')
_assert(isinstance(path_u, unicode), path_u=path_u)
if path_u not in self._callbacks.keys():
self._callbacks[path_u] = callbacks or []
self._watches[path_u] = self._observer.schedule(
INotifyEventHandler(path_u, mask, self._callbacks[path_u], self._pending_delay),
path=path_u,
recursive=False,
)
if replica in triggered_reps:
reportRecursiveChanges("", triggered_reps[replica])
del triggered_reps[replica]
sendCmd("DONE", [])
_debug_triggers()
elif cmd == "RESET":
# Stop observing replica.
[replica] = args
if not replica in replicas:
warn("unknown replica: " + replica)
continue
watch = replicas[replica]["watch"]
if watch is not None:
observer.unschedule(watch)
del replicas[replica]
if replica in triggered_reps:
del triggered_reps[replica]
_debug_triggers()
else:
sendError("unexpected root cmd: " + cmd)
if __name__ == '__main__':
try:
main()
finally:
for replica in replicas:
observer.unschedule(replicas[replica]["watch"])
observer.stop()
observer.join()
class GoSyncModel(object):
def __init__(self):
self.calculatingDriveUsage = False
self.driveAudioUsage = 0
self.driveMoviesUsage = 0
self.drivePhotoUsage = 0
self.driveDocumentUsage = 0
self.driveOthersUsage = 0
self.totalFilesToCheck = 0
self.savedTotalSize = 0
self.fcount = 0
self.updates_done = 0
self.config_path = os.path.join(os.environ['HOME'], ".gosync")
if not os.path.exists(self.config_path):
os.mkdir(self.config_path, 0755)
raise ClientSecretsNotFound()
self.usersettings = parseUserSettings(self.config_path)
self.credential_file = os.path.join(self.config_path, "credentials.json")
self.settings_file = os.path.join(self.config_path, "settings.yaml")
# self.base_mirror_directory = os.path.join(os.environ['HOME'], "Google Drive")
self.base_mirror_directory = self.usersettings.syncFolder;
self.client_secret_file = os.path.join(os.environ['HOME'], '.gosync', 'client_secrets.json')
self.sync_selection = []
self.config_file = os.path.join(os.environ['HOME'], '.gosync', 'gosyncrc')
self.config_dict = {}
self.account_dict = {}
self.drive_usage_dict = {}
self.config=None
if not os.path.exists(self.base_mirror_directory):
os.mkdir(self.base_mirror_directory, 0755)
if not os.path.exists(self.client_secret_file):
raise ClientSecretsNotFound()
if not os.path.exists(self.settings_file) or \
not os.path.isfile(self.settings_file):
sfile = open(self.settings_file, 'w')
sfile.write("save_credentials: {}".format(self.usersettings.saveCredentials))
sfile.write("\n")
sfile.write("save_credentials_file: ")
sfile.write(self.credential_file)
sfile.write("\n")
sfile.write('client_config_file: ' + self.client_secret_file + "\n")
sfile.write("save_credentials_backend: file\n")
sfile.close()
self.observer = Observer()
self.DoAuthenticate()
self.about_drive = self.authToken.service.about().get().execute()
self.user_email = self.about_drive['user']['emailAddress']
self.mirror_directory = os.path.join(self.base_mirror_directory, self.user_email)
if not os.path.exists(self.mirror_directory):
os.mkdir(self.mirror_directory, 0755)
self.tree_pickle_file = os.path.join(self.config_path, 'gtree-' + self.user_email + '.pick')
if not os.path.exists(self.config_file):
self.CreateDefaultConfigFile()
try:
self.LoadConfig()
except:
raise
self.iobserv_handle = self.observer.schedule(FileModificationNotifyHandler(self),
self.mirror_directory, recursive=True)
self.driveTree = None
self.sync_lock = threading.Lock()
self.sync_thread = threading.Thread(target=self.run)
self.usage_calc_thread = threading.Thread(target=self.calculateUsage)
self.sync_thread.daemon = True
self.usage_calc_thread.daemon = True
self.syncRunning = threading.Event()
self.syncRunning.clear()
self.usageCalculateEvent = threading.Event()
self.usageCalculateEvent.set()
self.logger = logging.getLogger(APP_NAME)
self.logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(os.path.join(os.environ['HOME'], 'GoSync.log'))
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
self.logger.addHandler(fh)
if not os.path.exists(self.tree_pickle_file):
self.driveTree = GoogleDriveTree()
else:
self.driveTree = pickle.load(open(self.tree_pickle_file, "rb"))
def SetTheBallRolling(self):
self.sync_thread.start()
self.usage_calc_thread.start()
self.observer.start()
def IsUserLoggedIn(self):
return self.is_logged_in
def HashOfFile(self, abs_filepath):
data = open(abs_filepath, "r").read()
return hashlib.md5(data).hexdigest()
def CreateDefaultConfigFile(self):
f = open(self.config_file, 'w')
self.config_dict['Sync Selection'] = [['root', '']]
self.account_dict[self.user_email] = self.config_dict
json.dump(self.account_dict, f)
f.close()
def LoadConfig(self):
try:
f = open(self.config_file, 'r')
try:
self.config = json.load(f)
try:
self.config_dict = self.config[self.user_email]
self.sync_selection = self.config_dict['Sync Selection']
print self.config_dict['Drive Usage']
try:
self.drive_usage_dict = self.config_dict['Drive Usage']
self.totalFilesToCheck = self.drive_usage_dict['Total Files']
self.savedTotalSize = self.drive_usage_dict['Total Size']
self.driveAudioUsage = self.drive_usage_dict['Audio Size']
self.driveMoviesUsage = self.drive_usage_dict['Movies Size']
self.driveDocumentUsage = self.drive_usage_dict['Document Size']
self.drivePhotoUsage = self.drive_usage_dict['Photo Size']
self.driveOthersUsage = self.drive_usage_dict['Others Size']
except:
pass
except:
pass
f.close()
except:
raise ConfigLoadFailed()
except:
raise ConfigLoadFailed()
def SaveConfig(self):
f = open(self.config_file, 'w')
f.truncate()
if not self.sync_selection:
self.config_dict['Sync Selection'] = [['root', '']]
self.account_dict[self.user_email] = self.config_dict
json.dump(self.account_dict, f)
f.close()
def DoAuthenticate(self):
try:
if not os.path.exists(self.credential_file):
self.authToken = GoogleAuth(self.settings_file)
else:
self.authToken = GoogleAuth();
self.authToken.LoadCredentialsFile(self.credential_file)
if (self.authToken.credentials is None):
self.authToken.LocalWebserverAuth()
elif self.authToken.access_token_expired :
self.authToken.Refresh()
else:
self.authToken.Authorize()
self.drive = GoogleDrive(self.authToken)
self.is_logged_in = True
except:
dial = wx.MessageDialog(None, "Authentication Rejected!\n",
'Information', wx.ID_OK | wx.ICON_EXCLAMATION)
dial.ShowModal()
self.is_logged_in = False
pass
def DoUnAuthenticate(self):
self.do_sync = False
self.observer.unschedule(self.iobserv_handle)
self.iobserv_handle = None
os.remove(self.credential_file)
self.is_logged_in = False
def DriveInfo(self):
return self.about_drive
def PathLeaf(self, path):
head, tail = ntpath.split(path)
return tail or ntpath.basename(head)
def GetFolderOnDrive(self, folder_name, parent='root'):
"""
Return the folder with name in "folder_name" in the parent folder
mentioned in parent.
"""
self.logger.debug("GetFolderOnDrive: searching %s on %s... " % (folder_name, parent))
file_list = self.drive.ListFile({'q': "'%s' in parents and trashed=false" % parent}).GetList()
for f in file_list:
if f['title'] == folder_name and f['mimeType']=='application/vnd.google-apps.folder':
self.logger.debug("Found!\n")
return f
return None
def LocateFolderOnDrive(self, folder_path):
"""
Locate and return the directory in the path. The complete path
is walked and the last directory is returned. An exception is raised
if the path walking fails at any stage.
"""
dir_list = folder_path.split(os.sep)
croot = 'root'
for dir1 in dir_list:
try:
folder = self.GetFolderOnDrive(dir1, croot)
if not folder:
raise FolderNotFound()
except:
raise
croot = folder['id']
return folder
def LocateFileInFolder(self, filename, parent='root'):
try:
file_list = self.MakeFileListQuery({'q': "'%s' in parents and trashed=false" % parent})
for f in file_list:
if f['title'] == filename:
return f
raise FileNotFound()
except:
raise FileNotFound()
def LocateFileOnDrive(self, abs_filepath):
dirpath = os.path.dirname(abs_filepath)
filename = self.PathLeaf(abs_filepath)
if dirpath != '':
try:
f = self.LocateFolderOnDrive(dirpath)
try:
fil = self.LocateFileInFolder(filename, f['id'])
return fil
except FileNotFound:
self.logger.debug("LocateFileOnDrive: File not found.\n")
raise
except FileListQueryFailed:
self.logger.debug("LocateFileOnDrive: File list query failed\n")
raise
except FolderNotFound:
self.logger.debug("LocateFileOnDrive: Folder not found\n")
raise
except FileListQueryFailed:
self.logger.debug("LocateFileOnDrive: %s folder not found\n" % dirpath)
raise
else:
try:
fil = self.LocateFileInFolder(filename)
return fil
except FileNotFound:
self.logger.debug("LocateFileOnDrive: File not found.\n")
raise
except FileListQueryFailed:
self.logger.debug("LocateFileOnDrive: File list query failed.\n")
raise
except:
self.logger.error("LocateFileOnDrive: Unknown error in locating file in drive\n")
raise
def CreateDirectoryInParent(self, dirname, parent_id='root'):
upfile = self.drive.CreateFile({'title': dirname,
'mimeType': "application/vnd.google-apps.folder",
"parents": [{"kind": "drive#fileLink", "id": parent_id}]})
upfile.Upload()
def CreateDirectoryByPath(self, dirpath):
self.logger.debug("create directory: %s\n" % dirpath)
drivepath = dirpath.split(self.mirror_directory+'/')[1]
basepath = os.path.dirname(drivepath)
dirname = self.PathLeaf(dirpath)
try:
f = self.LocateFolderOnDrive(drivepath)
return
except FolderNotFound:
if basepath == '':
self.CreateDirectoryInParent(dirname)
else:
try:
parent_folder = self.LocateFolderOnDrive(basepath)
self.CreateDirectoryInParent(dirname, parent_folder['id'])
except:
errorMsg = "Failed to locate directory path %s on drive.\n" % basepath
self.logger.error(errorMsg)
dial = wx.MessageDialog(None, errorMsg, 'Directory Not Found',
wx.ID_OK | wx.ICON_EXCLAMATION)
dial.ShowModal()
return
except FileListQueryFailed:
errorMsg = "Server Query Failed!\n"
self.logger.error(errorMsg)
dial = wx.MessageDialog(None, errorMsg, 'Directory Not Found',
wx.ID_OK | wx.ICON_EXCLAMATION)
dial.ShowModal()
return
def CreateRegularFile(self, file_path, parent='root', uploaded=False):
self.logger.debug("Create file %s\n" % file_path)
filename = self.PathLeaf(file_path)
upfile = self.drive.CreateFile({'title': filename,
"parents": [{"kind": "drive#fileLink", "id": parent}]})
upfile.SetContentFile(file_path)
upfile.Upload()
def UploadFile(self, file_path):
if os.path.isfile(file_path):
drivepath = file_path.split(self.mirror_directory+'/')[1]
self.logger.debug("file: %s drivepath is %s\n" % (file_path, drivepath))
try:
f = self.LocateFileOnDrive(drivepath)
self.logger.debug('Found file %s on remote (dpath: %s)\n' % (f['title'], drivepath))
newfile = False
self.logger.debug('Checking if they are same... ')
if f['md5Checksum'] == self.HashOfFile(file_path):
self.logger.debug('yes\n')
return
else:
self.logger.debug('no\n')
except (FileNotFound, FolderNotFound):
self.logger.debug("A new file!\n")
newfile = True
dirpath = os.path.dirname(drivepath)
if dirpath == '':
self.logger.debug('Creating %s file in root\n' % file_path)
self.CreateRegularFile(file_path, 'root', newfile)
else:
try:
f = self.LocateFolderOnDrive(dirpath)
self.CreateRegularFile(file_path, f['id'], newfile)
except FolderNotFound:
# We are coming from premise that upload comes as part
# of observer. So before notification of this file's
# creation happens, a notification of its parent directory
# must have come first.
# So,
# Folder not found? That cannot happen. Can it?
raise RegularFileUploadFailed()
else:
self.CreateDirectoryByPath(file_path)
def UploadObservedFile(self, file_path):
self.sync_lock.acquire()
self.UploadFile(file_path)
self.sync_lock.release()
def RenameFile(self, file_object, new_title):
try:
file = {'title': new_title}
updated_file = self.authToken.service.files().patch(fileId=file_object['id'],
body=file, fields='title').execute()
return updated_file
except errors.HttpError, error:
self.logger.error('An error occurred while renaming file: %s' % error)
return None
except:
class watcher(FileSystemEventHandler):
def __init__(self, message=None):
super(watcher, self).__init__() # Initialize the base class(es)
self.directories = {}
self.watchers = []
self.q = SimpleQueue()
self.msg = message if message else consoleMessage
from watchdog.observers import Observer
self.observer = Observer()
self.msg(f"Starting Observer()")
self.observer.start()
def _addWatchItem(self, item):
filespec = Path(item).resolve()
if filespec.parent not in self.directories:
self.msg(f"Creating watchlist() for [{filespec.parent}]")
self.directories[filespec.parent] = watchlist(filespec)
else:
self.directories[filespec.parent].addItem(filespec.name)
def _start(self, directory):
self.msg(f"Scheduling watcher for [{directory}]")
self.watchers.append(
self.observer.schedule(self, str(directory), recursive=False))
def _stop(self, watcher):
if watcher in self.watchers:
#//TODO.py: This won't work as-is. If needed, should make a small class to hold the
# watcher stuff: directory, the watcher instance, etc.
self.msg(f"UnScheduling watcher for [{directory}]")
self.observer.unschedule(watcher)
def on_modified(self, event):
if event.is_directory: return None
spec = Path(event.src_path).resolve()
#path = spec.parent
#name = spec.name
if spec.parent in self.directories:
if self.directories[spec.parent].exists(spec.name):
self.msg(
f"File {event.src_path} was modified. Adding to queue...")
self.q.put(event.src_path)
else:
self.msg(
f"File {event.src_path} was modified, but I don't care about it..."
)
else:
self.msg(
f"Got notification of change to {spec.name} in directory {spec.parent} --> {event.src_path}"
)
def begin(self, filelist):
# hang on to the filelist so .reset() can efficiently determine a course of action
self.filelist = filelist
# first, create all the watchdir() for each unique directory
for item in filelist:
self._addWatchItem(item)
# now, schedule an event handler(watchdog.watch) for each folder
for dir in self.directories:
self._start(dir)
def reset(self, filelist):
# if the filelist didn't change (true most of the time) ignore the reset
if set(self.filelist) == set(filelist):
self.msg("reset: no change to filelist, ignoring reset...")
return
# get list of directories being monitored
self.msg("reset: shutting down directory observers...")
self.observer.unschedule_all()
self.directories = {}
self.watchers = []
self.begin(filelist)
def end(self):
self.observer.unschedule_all()
self.observer.stop()
self.msg("\nObserver Stopped")
self.observer.join()
def look(self):
return True if self.q.qsize() else False
def get(self):
return self.q.get() if self.q.qsize() else ""
def dump(self):
for i in self.directories:
self.msg(f"Folder: {i}")
self.directories[i].dump()
class Schedule:
def __init__(self, config):
self.observer = Observer()
self.handlers = {}
self.watchers = {}
self.counter = Counter()
self.notifier = Notifier(config)
self.offset_db = OffsetPersistence(config)
def __make_key(self, filename):
return path.abspath(urlsafe_b64decode(filename).decode())
def add_watcher(self, filename):
#如果监控的文件都在同一个目录下，下面这种方法只会启动一个inotify来进行监控这个目录下的所有文件，handler.start实际上调用了monitor的start
filename = self.__make_key(filename)
if handler.filename not in self.handlers.keys():
handler = Watcher(filename, counter=self.counter, notifier=self.notifier, offset_db=self.offset_db)
if path.dirname(handler.filename) not in self.watchers.keys():
self.watchers[path.dirname(handler.filename)] = self.observer.schedule(handler, path.dirname(handler.filename), recursive=False)
else:
watch = self.watchers[path.dirname(handler.filename)]
self.observer.add_handler_for_watch(handler, watch)
self.handlers[handler.filename] = handler
handler.start()
def remove_watcher(self, filename):
#判断key是否在watchers里面，有则从observer中剔除，然后从watchers字典内删除，并把handlers剔除并stop
key = self.__make_key(filename)
handler = self.handlers.pop(key)
if handler is not None:
watch = self.watchers[path.dirname(key)]
self.observer.remove_handler_for_watch(handler, watch)
handler.stop()
if not self.observer._handlers[watch]:
self.observer.unschedule(watch)
self.watchers.pop(path.dirname(handler.filename))
#添加和移除monitor，实质上就是调用Monitor类的add和remove方法来操作
def add_monitor(self, filename, name, src):
key = self.__make_key(filename)
handler = self.handlers.get(key)
if handler is None:
logging.warning('watcher {0} not found, auto add it'.format(filename))
handler.add_watcher(filename)
handler = self.handlers.get(key)
handler.monitor.add(filename, name, src)
def remove_monitor(self, filename, name):
key = self.__make_key(filename)
handler = self.handlers.get(key)
if handler is None:
logging.warning('watcher {0} not found'.format(filename))
return
handler.monitor.remove(name)
def start(self):
self.observer.start()
self.notifier.start()
def join(self):
self.observer.join()
def stop(self):
self.observer.stop()
for handler in self.handlers.values():
handler.stop()
self.notifier.stop()
self.offset_db.close()
class _MultiPathWatcher(object):
"""Watches multiple paths."""
_singleton: Optional["_MultiPathWatcher"] = None
@classmethod
def get_singleton(cls) -> "_MultiPathWatcher":
"""Return the singleton _MultiPathWatcher object.
Instantiates one if necessary.
"""
if cls._singleton is None:
LOGGER.debug("No singleton. Registering one.")
_MultiPathWatcher()
return cast("_MultiPathWatcher", _MultiPathWatcher._singleton)
# Don't allow constructor to be called more than once.
def __new__(cls) -> "_MultiPathWatcher":
"""Constructor."""
if _MultiPathWatcher._singleton is not None:
raise RuntimeError("Use .get_singleton() instead")
return super(_MultiPathWatcher, cls).__new__(cls)
def __init__(self) -> None:
"""Constructor."""
_MultiPathWatcher._singleton = self
# Map of folder_to_watch -> _FolderEventHandler.
self._folder_handlers: Dict[str, _FolderEventHandler] = {}
# Used for mutation of _folder_handlers dict
self._lock = threading.Lock()
# The Observer object from the Watchdog module. Since this class is
# only instantiated once, we only have a single Observer in Streamlit,
# and it's in charge of watching all paths we're interested in.
self._observer = Observer()
self._observer.start() # Start observer thread.
def __repr__(self) -> str:
return repr_(self)
def watch_path(
self,
path: str,
callback: Callable[[str], None],
*, # keyword-only arguments:
glob_pattern: Optional[str] = None,
allow_nonexistent: bool = False,
) -> None:
"""Start watching a path."""
folder_path = os.path.abspath(os.path.dirname(path))
with self._lock:
folder_handler = self._folder_handlers.get(folder_path)
if folder_handler is None:
folder_handler = _FolderEventHandler()
self._folder_handlers[folder_path] = folder_handler
folder_handler.watch = self._observer.schedule(folder_handler,
folder_path,
recursive=True)
folder_handler.add_path_change_listener(
path,
callback,
glob_pattern=glob_pattern,
allow_nonexistent=allow_nonexistent,
)
def stop_watching_path(self, path: str, callback: Callable[[str],
None]) -> None:
"""Stop watching a path."""
folder_path = os.path.abspath(os.path.dirname(path))
with self._lock:
folder_handler = self._folder_handlers.get(folder_path)
if folder_handler is None:
LOGGER.debug(
"Cannot stop watching path, because it is already not being "
"watched. %s",
folder_path,
)
return
folder_handler.remove_path_change_listener(path, callback)
if not folder_handler.is_watching_paths():
# Sometimes watchdog's FileSystemEventHandler does not have
# a .watch property. It's unclear why -- may be due to a
# race condition.
if hasattr(folder_handler, "watch"):
self._observer.unschedule(folder_handler.watch)
del self._folder_handlers[folder_path]
def close(self) -> None:
with self._lock:
"""Close this _MultiPathWatcher object forever."""
if len(self._folder_handlers) != 0:
self._folder_handlers = {}
LOGGER.debug(
"Stopping observer thread even though there is a non-zero "
"number of event observers!")
else:
LOGGER.debug("Stopping observer thread")
self._observer.stop()
self._observer.join(timeout=5)
class ChangesWatcher(FileSystemEventHandler):
def __init__(self, application):
self.task_map = {} # path: ChangesKeeper
self.application = application
self.observer = Observer()
self.changes = []
self.changes_timer = None
self.observer.start()
def add_watch(self, path, mute_list=None):
if path in self.task_map:
return False
else:
mute_list = (mute_list or []) + DEFAULT_MUTE_LIST
keeper = ChangesKeeper(path, mute_list)
self.task_map[path] = keeper
if os.path.exists(path):
self.observer.schedule(self, path, recursive=True)
return True
return False
def remove_watch(self, path):
if path in self.task_map:
keeper = self.task_map[path]
watch = keeper.watch
self.observer.unschedule(watch)
return True
else:
return False
def get_changes_since(self, timestamp, parent_path=None):
ret = []
for change in self.changes:
if change.timestamp > timestamp and (not parent_path or path_is_parent(parent_path, change.path)):
ret.append(change)
return ret
def add_pure_change(self, change):
""" 监测change的类型，并添加非垃圾change和不在黑名单中的change
"""
# 如果是黑名单及黑名单子目录的change，则跳过
for mute_list in [keeper.mute_list for keeper in self.task_map.values()]:
for mute_path in mute_list:
if path_is_parent(mute_path, change.path):
print '...', change.type, change.path
return
# 寻找当前change对应的垃圾change，找到后删除；未找到则添加当前change
trash_changes = self.find_related_trash_changes(change)
if trash_changes:
for change in trash_changes:
self.changes.remove(change)
print '- ', change.type, change.path
else:
self.changes.append(change)
print '+ ', change.type, change.path
self.compile_if_needed(change)
ioloop.IOLoop.instance().add_callback(lambda: self.remove_outdated_changes(30))
def compile_if_needed(self, change):
if change.type == EVENT_TYPE_DELETED:
return
input_path = normalize_path(change.path)
base_path, ext = os.path.splitext(input_path)
ext = ext.lower()
if ext not in['.less', '.coffee']:
return
project = self.application.find_project(input_path)
if not project:
return
os.chdir(APP_FOLDER)
begin_time = time.time()
if ext == '.less':
if project.compileLess:
output_path = base_path + '.css'
run_cmd('%s bundled/less/bin/lessc %s %s' % (NODE_BIN_PATH, input_path, output_path))
print '.less ->- .css', change.path, time.time() - begin_time, 'seconds'
else:
print '.less -X- .css', change.path, '(OFF by settings)'
elif ext == '.coffee':
if project.compileCoffee:
run_cmd('%s bundled/coffee/bin/coffee --compile %s' % (NODE_BIN_PATH, input_path))
print '.coffee ->- .js', change.path, time.time() - begin_time, 'seconds'
else:
print '.coffee -X- .js', change.path, '(OFF by settings)'
def check_folder_change(self, folder_path):
if sys.platform.startswith('win') or \
not os.path.isdir(folder_path):
return
now = time.time() - 0.5 # 0.5秒内的都算修改
for filename in os.listdir(folder_path):
file_path = os.path.join(folder_path, filename)
if not os.path.isfile(file_path):
continue
modified_time = os.path.getmtime(file_path)
if modified_time > now:
self.on_any_event(FileModifiedEvent(file_path))
def on_any_event(self, event):
if event.is_directory:
self.check_folder_change(event.src_path)
return
# 暂停文件变更的上报, 以免中途编译占用太长时间，而将事件提前返回
loop = ioloop.IOLoop.instance()
if self.changes_timer:
ioloop.IOLoop.instance().remove_timeout(self.changes_timer)
now = time.time()
if event.event_type == EVENT_TYPE_MOVED:
self.add_pure_change(Change(dict(
timestamp=now,
path=normalize_path(event.src_path),
type=EVENT_TYPE_DELETED
)))
self.add_pure_change(Change(dict(
timestamp=now,
path=normalize_path(event.dest_path),
type=EVENT_TYPE_CREATED
)))
else:
self.add_pure_change(Change(dict(
timestamp=now,
path=normalize_path(event.src_path),
type=event.event_type
)))
# 延迟0.1秒上报变更，防止有些事件连续发生时错过
self.changes_timer = loop.add_timeout(time.time() + 0.1, self.application.project_file_changed)
def find_related_trash_changes(self, change):
""" 寻找当前change之前短时间内的一些垃圾change
有些编辑器喜欢用 改名->写入->改回名 的方式来保存文件，所以不能直接将change上报，需要进行一定的过滤
"""
trash_changes = []
for old_change in self.changes[::-1]:
if old_change.path != change.path:
continue
if change.timestamp > old_change.timestamp + 1:
break
if change.type == EVENT_TYPE_DELETED:
# 如果当前change类型是DELETED，那么返回所有该文件的事件，直到CREATED事件为止
trash_changes.append(old_change)
if old_change.type == EVENT_TYPE_CREATED:
return trash_changes
elif change.type == EVENT_TYPE_CREATED:
# 如果当前change类型是CREATED，那么返回所有该文件的事件，直到DELETED事件为止
trash_changes.append(old_change)
if old_change.type == EVENT_TYPE_DELETED:
return trash_changes
return []
def remove_outdated_changes(self, seconds):
for change in self.changes[:]:
if change.timestamp - time.time() > seconds:
self.changes.remove(change)
