class Crafter(object):
"""
Setup and manage watchdog daemon.
"""
def __init__(self, base_dir):
# Cache paths
src_dir = os.path.join(base_dir, 'src')
dest_dir = os.path.join(base_dir, 'preview')
# Init handler
self.handler = ObserverHandler(src_dir, dest_dir)
# New observer class
self.observer = Observer()
self.observer.schedule(self.handler, path=src_dir, recursive=True)
def craft(self):
"""
Start watching src directory
"""
self.observer.start()
def shutdown(self):
"""
Properly shutdown watchdog daemon
"""
self.observer.stop()
self.observer.join()
def watch(self):
"""
Start watching
"""
logger.info('Watching directory %s' % self.directory)
# Set up handler for when we see new files
callback = self.callback
class NewFileEventHandler(FileSystemEventHandler):
def on_created(self, event):
if not event.is_directory:
logger.info('Detected new file: %s' % event.src_path)
callback(event.src_path)
event_handler = NewFileEventHandler()
# Use polling observer (rather than filesystem-specific observers),
# because it is more reliable.
observer = PollingObserver(timeout=self.sleep_time)
# Start the observer
observer.schedule(event_handler, self.directory, recursive=False)
observer.start()
# Wait while the observer is running
try:
while True:
sleep(self.sleep_time)
# Exit gracefully
except KeyboardInterrupt:
logger.info('Detected interrupt. Stopping observer.')
observer.stop()
observer.join()
def main():
if int(os.getuid()) != 0:
raise SystemExit("ERROR: this script should be run as root")
parser = ArgumentParser(description='Watch a directory and install the code')
args = parser.parse_args()
current_path = Path(__file__).resolve().parent
setup_path = Path(current_path, '..').resolve()
git_path = Path(current_path, '..', '..').resolve()
observer = Observer()
observer.schedule(FileHandler(setup_path, 'site'), str(Path(git_path, 'site')), True)
observer.schedule(FileHandler(setup_path, 'bin'), str(Path(git_path, 'bin')), True)
observer.schedule(FileHandler(setup_path, 'bin'), str(Path(git_path, 'sbin')), True)
observer.start()
try:
print("Watching the following folders for change:")
print(" - site")
print(" - bin")
print(" - sbin")
print()
input("~~Hit enter to exit~~\n")
finally:
observer.stop()
observer.join()
def watch_assets(options):
"""
Watch for changes to asset files, and regenerate js/css
"""
# Don't watch assets when performing a dry run
if tasks.environment.dry_run:
return
observer = PollingObserver()
CoffeeScriptWatcher().register(observer)
SassWatcher().register(observer)
XModuleSassWatcher().register(observer)
XModuleAssetsWatcher().register(observer)
print("Starting asset watcher...")
observer.start()
if not getattr(options, 'background', False):
# when running as a separate process, the main thread needs to loop
# in order to allow for shutdown by contrl-c
try:
while True:
observer.join(2)
except KeyboardInterrupt:
observer.stop()
print("\nStopped asset watcher.")
def watch(self):
observer = PollingObserver(0.1)
observer.schedule(self, self.path)
observer.start()
self.file = open(self.path + "\Power.log", "r")
self.on_modified(None)
self.watching = True
def watch_record(indexer, use_polling=False):
"""
Start watching `cfstore.record_path`.
:type indexer: rash.indexer.Indexer
"""
if use_polling:
from watchdog.observers.polling import PollingObserver as Observer
Observer # fool pyflakes
else:
from watchdog.observers import Observer
event_handler = RecordHandler(indexer)
observer = Observer()
observer.schedule(event_handler, path=indexer.record_path, recursive=True)
indexer.logger.debug("Start observer.")
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
indexer.logger.debug("Got KeyboardInterrupt. Stopping observer.")
observer.stop()
indexer.logger.debug("Joining observer.")
observer.join()
indexer.logger.debug("Finish watching record.")
def watch(directory=None, auto_clear=False, beep_on_failure=True,
onpass=None, onfail=None, poll=False, extensions=[]):
"""
Starts a server to render the specified file or directory
containing a README.
"""
if directory and not os.path.isdir(directory):
raise ValueError('Directory not found: ' + directory)
directory = os.path.abspath(directory or '')
# Initial run
event_handler = ChangeHandler(directory, auto_clear, beep_on_failure,
onpass, onfail, extensions)
event_handler.run()
# Setup watchdog
if poll:
observer = PollingObserver()
else:
observer = Observer()
observer.schedule(event_handler, path=directory, recursive=True)
observer.start()
# Watch and run tests until interrupted by user
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def main():
"""Script entry point."""
from watchdog.observers.polling import PollingObserver
from .parser import AAConfigParser
from .tricks import AutoRunTrick
parser = _create_main_argparser()
args = parser.parse_args()
configm = _apply_main_args(args)
# The reason to use PollingObserver() is it's os-independent. And it's
# more reliable.
observer = PollingObserver()
parser = AAConfigParser(configm)
handler_for_watch = parser.schedule_with(observer, AutoRunTrick)
handlers = set.union(*tuple(handler_for_watch.values()))
for handler in handlers:
handler.start()
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
for handler in handlers:
handler.stop()
class Watcher:
def __init__(self):
# self.observer = Observer() # Use this if SS_DIR is local
self.observer = PollingObserver() # Use this if SS_DIR is remote mount
def run(self):
event_handler = file_changed()
self.observer.schedule(event_handler, SS_DIR, recursive=True)
self.observer.start()
loop1()
def watch(script, callback):
if script in _observers:
raise RuntimeError("Script already observed")
script_dir = os.path.dirname(os.path.abspath(script.filename))
script_name = os.path.basename(script.filename)
event_handler = _ScriptModificationHandler(callback, filename=script_name)
observer = Observer()
observer.schedule(event_handler, script_dir)
observer.start()
_observers[script] = observer
def __init__(self, index, interface='wlan0'):
self.interface = interface
self.run_state = '/run/network/ifstate.{interface}'.format(interface=interface)
self.configurations = ['', interface] + rospy.get_param("~wlan_interfaces", [])
rospy.loginfo("Loaded wlan configs %s", self.configurations)
super(WifiUI, self).__init__(index, len(self.configurations))
self.config = get_configuration(self.run_state, self.configurations)
rospy.loginfo("Start observingchanges in %s", os.path.dirname(self.run_state))
observer = Observer()
observer.schedule(self, os.path.dirname(self.run_state), recursive=True)
observer.start()
def watch(self):
observer = PollingObserver()
observer.schedule(self.pickup_event_processor, path=self.pickup_dir)
observer.start()
try:
while self.keep_running:
sleep(3)
except KeyboardInterrupt:
observer.stop()
observer.join()
def serve(config, options=None):
"""
Start the devserver, and rebuild the docs whenever any changes take effect.
"""
# Create a temporary build directory, and set some options to serve it
tempdir = tempfile.mkdtemp()
options['site_dir'] = tempdir
# Only use user-friendly URLs when running the live server
options['use_directory_urls'] = True
# Perform the initial build
config = load_config(options=options)
build(config, live_server=True)
# Note: We pass any command-line options through so that we
# can re-apply them if the config file is reloaded.
event_handler = BuildEventHandler(options)
config_event_handler = ConfigEventHandler(options)
# We could have used `Observer()`, which can be faster, but
# `PollingObserver()` works more universally.
observer = PollingObserver()
observer.schedule(event_handler, config['docs_dir'], recursive=True)
for theme_dir in config['theme_dir']:
if not os.path.exists(theme_dir):
continue
observer.schedule(event_handler, theme_dir, recursive=True)
observer.schedule(config_event_handler, '.')
observer.start()
class TCPServer(socketserver.TCPServer):
allow_reuse_address = True
class DocsDirectoryHandler(FixedDirectoryHandler):
base_dir = config['site_dir']
host, port = config['dev_addr'].split(':', 1)
server = TCPServer((host, int(port)), DocsDirectoryHandler)
print('Running at: http://%s:%s/' % (host, port))
print('Live reload enabled.')
print('Hold ctrl+c to quit.')
try:
server.serve_forever()
except KeyboardInterrupt:
print('Stopping server...')
# Clean up
observer.stop()
observer.join()
shutil.rmtree(tempdir)
print('Quit complete')
def watch(
directories=[],
ignore=[],
auto_clear=False,
beep_on_failure=True,
onpass=None,
onfail=None,
runner=None,
beforerun=None,
onexit=None,
poll=False,
extensions=[],
args=[],
spool=True,
verbose=False,
quiet=False,
):
if not directories:
directories = ["."]
directories = [os.path.abspath(directory) for directory in directories]
for directory in directories:
if not os.path.isdir(directory):
raise ValueError("Directory not found: " + directory)
if ignore:
recursive_dirs, non_recursive_dirs = split_recursive(directories, ignore)
else:
recursive_dirs = directories
non_recursive_dirs = []
# Initial run
event_handler = ChangeHandler(
auto_clear, beep_on_failure, onpass, onfail, runner, beforerun, extensions, args, spool, verbose, quiet
)
event_handler.run()
# Setup watchdog
observer = PollingObserver() if poll else Observer()
for directory in recursive_dirs:
observer.schedule(event_handler, path=directory, recursive=True)
for directory in non_recursive_dirs:
observer.schedule(event_handler, path=directory, recursive=False)
# Watch and run tests until interrupted by user
try:
observer.start()
while True:
time.sleep(1)
observer.join()
except KeyboardInterrupt:
observer.stop()
if onexit:
os.system(onexit)
class SuiteTable(object):
def __init__(self, dbfile=":memory:", poll=False):
self.db = sqlite3.connect(dbfile, check_same_thread=False)
self.log = logging.getLogger(__name__)
self._create_db()
# set up watchdog observer to monitor changes to
# keyword files (or more correctly, to directories
# of keyword files)
self.observer = PollingObserver() if poll else Observer()
self.observer.start()
def _create_db(self):
if not self._table_exists("collection_table"):
self.db.execute("""
CREATE TABLE collection_table
(collection_id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT COLLATE NOCASE,
type COLLATE NOCASE,
version TEXT,
scope TEXT,
namedargs TEXT,
path TEXT,
doc TEXT,
doc_format TEXT)
""")
self.db.execute("""
CREATE INDEX collection_index
ON collection_table (name)
""")
if not self._table_exists("keyword_table"):
self.db.execute("""
CREATE TABLE keyword_table
(keyword_id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT COLLATE NOCASE,
collection_id INTEGER,
doc TEXT,
args TEXT)
""")
self.db.execute("""
CREATE INDEX keyword_index
ON keyword_table (name)
""")
def _table_exists(self, name):
cursor = self.db.execute("""
SELECT name FROM sqlite_master
WHERE type='table' AND name='%s'
""" % name)
return len(cursor.fetchall()) > 0
class OSFileSystem(FileSystem):
"""
This class represents a file system implemented by the python os module.
"""
def __init__(self, instance=os, root="."):
FileSystem.__init__(self, instance)
self.root = os.path.normpath(root)
self.eventQueue = EventQueue()
self.eventHandler = EventHandler(self.eventQueue)
self.observer = Observer()
self.observer.schedule(self.eventHandler, path=self.root, recursive=True)
self.observer.start()
def join_path(self, path, *largs):
return os.path.join(path, *largs)
def get_relative_path(self, path):
if path.startswith(self.root):
return path.split(self.root + os.path.sep, 1)[1]
else:
return path
def open(self, path, mode="rb", buffering=None):
return open(path, mode)
def mkdirs(self, path, mode=511):
return os.makedirs(path, mode)
def blockchecksums(self, path):
return blockchecksums(path)
def delta(self, path, checksums):
return delta(path, checksums)
def patch(self, path, delta):
patched = patch(path, delta)
self.instance.remove(path)
return self.instance.rename(patched, path)
def poll(self):
r = []
while True:
try:
r.append(self.eventQueue.get_nowait())
except Empty:
break
return r
def watch():
# Users expect an implicit push
push(watch=True)
# Start the observer
observer = PollingObserver()
observer.event_queue.max_size = 1
observer.schedule(EventHandler(), os.getcwd(), recursive=True)
observer.start()
puts(colored.yellow('Watching for changes... (ctrl-c to stop)'))
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
# Block until the thread terminates
observer.join()
def main():
handler = ChangeHandler()
directory = "./"
observer = PollingObserver(0.35)
# Poll every 0.35 seconds
if not os.path.exists(directory):
os.makedirs(directory)
observer.schedule(handler, directory, recursive=True)
# Only search in the LaTeX directory
observer.start()
try:
while True:
time.sleep(60 * 5)
# Sleep for 5 minutes (time doesn't really matter)
except KeyboardInterrupt:
observer.stop()
observer.join()
def watch_assets(options):
"""
Watch for changes to asset files, and regenerate js/css
"""
# Don't watch assets when performing a dry run
if tasks.environment.dry_run:
return
themes = get_parsed_option(options, 'themes')
theme_dirs = get_parsed_option(options, 'theme_dirs', [])
# wait comes in as a list of strings, define the default value similarly for convenience.
default_wait = [unicode(DEFAULT_OBSERVER_TIMEOUT)]
wait = float(get_parsed_option(options, 'wait', default_wait)[0])
if not theme_dirs and themes:
# We can not add theme sass watchers without knowing the directory that contains the themes.
raise ValueError('theme-dirs must be provided for watching theme sass.')
else:
theme_dirs = [path(_dir) for _dir in theme_dirs]
sass_directories = get_watcher_dirs(theme_dirs, themes)
observer = PollingObserver(timeout=wait)
CoffeeScriptWatcher().register(observer)
SassWatcher().register(observer, sass_directories)
XModuleSassWatcher().register(observer, ['common/lib/xmodule/'])
XModuleAssetsWatcher().register(observer)
print("Starting asset watcher...")
observer.start()
# Run the Webpack file system watcher too
execute_webpack_watch(settings=Env.DEVSTACK_SETTINGS)
if not getattr(options, 'background', False):
# when running as a separate process, the main thread needs to loop
# in order to allow for shutdown by control-c
try:
while True:
observer.join(2)
except KeyboardInterrupt:
observer.stop()
print("\nStopped asset watcher.")
def run_watch(self):
if self.poll:
from watchdog.observers.polling import PollingObserver as Observer
else:
from watchdog.observers import Observer
event_handler = RoninEventHandler(self)
observer = Observer()
observer.schedule(event_handler, self.source, recursive=True)
observer.start()
try:
logger.info("Watching directory: '{0}' for changes (poll={1})".format(self.source, self.poll))
while True:
time.sleep(1)
except KeyboardInterrupt:
logger.info("Stopping watcher...")
observer.stop()
observer.join()
def folderObserver(pathStructure, dbPath):
logging = DefaultLogger()
if pathStructure == None or pathStructure['inBox'] == None:
message = 'Watch: Unable to run as pathStructure is undefined'
logging.debug(message)
return
event_handler = singleFileWatcher(pathStructure, dbPath)
observer = PollingObserver()
observer.schedule(event_handler, pathStructure['inBox'], recursive=False)
observer.start()
try:
while True and observer.is_alive():
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def server(self, args):
server = Process(target=self._server)
server.start()
event_handler = PatternMatchingEventHandler(ignore_patterns=self.WATCH_EXCLUDE)
event_handler.on_modified = lambda event : self._build()
observer = Observer()
observer.schedule(event_handler, self.BASE_DIR, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except (KeyboardInterrupt, SystemExit):
server.terminate()
observer.stop()
observer.join()
self.logger.info("Clossing")
def watch_assets(options):
"""
Watch for changes to asset files, and regenerate js/css
"""
# Don't watch assets when performing a dry run
if tasks.environment.dry_run:
return
themes = getattr(options, 'themes', None)
theme_dirs = getattr(options, 'theme-dirs', [])
if not theme_dirs and themes:
# We can not add theme sass watchers without knowing the directory that contains the themes.
raise ValueError('theme-dirs must be provided for watching theme sass.')
else:
theme_dirs = [path(_dir) for _dir in theme_dirs]
if isinstance(themes, basestring):
themes = themes.split(',')
else:
themes = themes if isinstance(themes, list) else [themes]
sass_directories = get_watcher_dirs(theme_dirs, themes)
observer = PollingObserver()
CoffeeScriptWatcher().register(observer)
SassWatcher().register(observer, sass_directories)
XModuleSassWatcher().register(observer, ['common/lib/xmodule/'])
XModuleAssetsWatcher().register(observer)
print("Starting asset watcher...")
observer.start()
if not getattr(options, 'background', False):
# when running as a separate process, the main thread needs to loop
# in order to allow for shutdown by contrl-c
try:
while True:
observer.join(2)
except KeyboardInterrupt:
observer.stop()
print("\nStopped asset watcher.")
def watch_assets(options):
"""
Watch for changes to asset files, and regenerate js/css
"""
# Don't watch assets when performing a dry run
if tasks.environment.dry_run:
return
themes = get_parsed_option(options, 'themes')
theme_dirs = get_parsed_option(options, 'theme_dirs', [])
if not theme_dirs and themes:
# We can not add theme sass watchers without knowing the directory that contains the themes.
raise ValueError('theme-dirs must be provided for watching theme sass.')
else:
theme_dirs = [path(_dir) for _dir in theme_dirs]
sass_directories = get_watcher_dirs(theme_dirs, themes)
observer = PollingObserver()
CoffeeScriptWatcher().register(observer)
SassWatcher().register(observer, sass_directories)
XModuleSassWatcher().register(observer, ['common/lib/xmodule/'])
XModuleAssetsWatcher().register(observer)
print("Starting asset watcher...")
observer.start()
# We only want Webpack to re-run on changes to its own entry points, not all JS files, so we use its own watcher
# instead of subclassing from Watchdog like the other watchers do
execute_webpack_watch(settings=Env.DEVSTACK_SETTINGS)
if not getattr(options, 'background', False):
# when running as a separate process, the main thread needs to loop
# in order to allow for shutdown by contrl-c
try:
while True:
observer.join(2)
except KeyboardInterrupt:
observer.stop()
print("\nStopped asset watcher.")
def __init__(self, contentdir=''):
"""
@param contentDirs are the dirs where we will load wiki files from & parse
"""
self.file_observers = []
self.spacehandler = SpaceHandler(self)
self.contentdir = contentdir if contentdir.endswith('/') else '%s/' % contentdir
if not j.system.fs.exists(contentdir):
print "Contentdir %s was not found .. creating it." % contentdir
j.system.fs.createDir(contentdir)
if contentdir.strip():
# Watch the contentdir for changes
observer = Observer()
self.file_observers.append(observer)
j.core.portal.active.watchedspaces.append(contentdir)
print('Monitoring', contentdir)
observer.schedule(self.spacehandler, contentdir, recursive=True)
observer.start()
class FileMonitor(FileSystemEventHandler):
def __init__(self, file, action):
self.path,self.filename = os.path.split(file)
self.action = action
self.observer = None
def start(self):
self.observer = Observer()
self.observer.schedule(self, self.path, recursive=False)
self.observer.start()
def stop(self):
if self.observer is not None:
self.observer.stop()
def join(self):
if self.observer is not None:
self.observer.join()
def on_modified(self, event):
try:
if os.path.samefile(event.src_path,self.filename):
self.action()
except OSError as e:
print 'Exception on file check', e
def watch_assets(options):
"""
Watch for changes to asset files, and regenerate js/css
"""
# Don't watch assets when performing a dry run
if tasks.environment.dry_run:
return
themes = get_parsed_option(options, 'themes')
theme_dirs = get_parsed_option(options, 'theme_dirs', [])
if not theme_dirs and themes:
# We can not add theme sass watchers without knowing the directory that contains the themes.
raise ValueError('theme-dirs must be provided for watching theme sass.')
else:
theme_dirs = [path(_dir) for _dir in theme_dirs]
sass_directories = get_watcher_dirs(theme_dirs, themes)
observer = PollingObserver()
CoffeeScriptWatcher().register(observer)
SassWatcher().register(observer, sass_directories)
XModuleSassWatcher().register(observer, ['common/lib/xmodule/'])
XModuleAssetsWatcher().register(observer)
print("Starting asset watcher...")
observer.start()
if not getattr(options, 'background', False):
# when running as a separate process, the main thread needs to loop
# in order to allow for shutdown by contrl-c
try:
while True:
observer.join(2)
except KeyboardInterrupt:
observer.stop()
print("\nStopped asset watcher.")
def read_logs(api_url, logs):
observers = []
for log in logs:
path = log['path']
dir_name, log_name = os.path.split(path)
print('Listening for changes on {} in {}'.format(log_name, dir_name))
if log['os_log']:
event_handler = LinuxLogEventHandler(api_url, log['regex'],
log_name)
else:
event_handler = TextLogEventHandler(api_url, log['regex'],
log_name)
interval = log['interval']
if interval > 0:
observer = PollingObserver(interval)
# Initialize the offset to the end of the file:
event_handler.offset = sum(1 for _ in open(path))
else:
observer = Observer()
observer.schedule(event_handler, dir_name, recursive=False)
observer.start()
observers.append(observer)
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
for observer in observers:
observer.stop()
for observer in observers:
observer.join()
def test_watch_new_file(self):
tmp_dir = tempfile.mkdtemp()
created_file_count = defaultdict(lambda: 0)
def _increment_counter(name: str, dir: str):
created_file_count[os.path.join(dir, name)] += 1
observer = PollingObserver()
file_event_handler = FileFinallyCreatedEventHandler(
['*.txt'], lambda name, dir: 'exclude' not in name,
_increment_counter, 1)
observer.schedule(file_event_handler, tmp_dir, recursive=True)
observer.start()
test_files = [
os.path.join(tmp_dir, x) for x in
['test_0.txt', 'test_1.txt', 'java.script', 'exclude.txt']
]
for test_file_path in test_files:
with open(test_file_path, 'w') as f:
f.write('test ' + test_file_path)
time.sleep(2)
file_event_handler.pending_watch_created_files()
for test_file_path in test_files:
if test_file_path.endswith(
'.txt') and 'exclude' not in test_file_path:
self.assertEqual(1, created_file_count[test_file_path])
else:
self.assertEqual(0, created_file_count[test_file_path])
observer.stop()
observer.join()
shutil.rmtree(tmp_dir, ignore_errors=True)
def main():
ocrmypdf.configure_logging(verbosity=ocrmypdf.Verbosity.default,
manage_root_logger=True)
log.setLevel(LOGLEVEL)
log.info(f"Starting OCRmyPDF watcher with config:\n"
f"Input Directory: {INPUT_DIRECTORY}\n"
f"Output Directory: {OUTPUT_DIRECTORY}\n"
f"Output Directory Year & Month: {OUTPUT_DIRECTORY_YEAR_MONTH}")
log.debug(f"INPUT_DIRECTORY: {INPUT_DIRECTORY}\n"
f"OUTPUT_DIRECTORY: {OUTPUT_DIRECTORY}\n"
f"OUTPUT_DIRECTORY_YEAR_MONTH: {OUTPUT_DIRECTORY_YEAR_MONTH}\n"
f"ON_SUCCESS_DELETE: {ON_SUCCESS_DELETE}\n"
f"DESKEW: {DESKEW}\n"
f"ARGS: {OCR_JSON_SETTINGS}\n"
f"POLL_NEW_FILE_SECONDS: {POLL_NEW_FILE_SECONDS}\n"
f"USE_POLLING: {USE_POLLING}\n"
f"LOGLEVEL: {LOGLEVEL}\n")
if 'input_file' in OCR_JSON_SETTINGS or 'output_file' in OCR_JSON_SETTINGS:
log.error(
'OCR_JSON_SETTINGS should not specify input file or output file')
sys.exit(1)
handler = HandleObserverEvent(patterns=PATTERNS)
if USE_POLLING:
observer = PollingObserver()
else:
observer = Observer()
observer.schedule(handler, INPUT_DIRECTORY, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
class FileWatcher:
def __init__(self, pool, query, src_path, patterns=None, ignore_directories=False, recursive=True, timeout=1, key=1):
if patterns is None:
patterns = ["*.txt"]
self.src_path = src_path
self.recursive = recursive
self.event_observer = PollingObserver(timeout=timeout)
self.event_handler = InsertToSQL(pool, query, patterns=patterns, ignore_directories=ignore_directories, key=key)
def bark(self):
self.start()
try:
while True:
# Main Loop.
# Watchdog is polling every TIMEOUT seconds
pass
except KeyboardInterrupt:
self.stop()
def start(self):
# Schedule observer
self.event_observer.schedule(self.event_handler, self.src_path, recursive=self.recursive)
# Start watchdog thread; can give it name with observer.set_name()
self.event_observer.start()
def stop(self):
self.event_observer.stop()
self.event_observer.join()
class Command(BaseCommand):
"""
On every iteration of an infinite loop, consume what we can from the
consumption directory.
"""
# This is here primarily for the tests and is irrelevant in production.
stop_flag = False
def __init__(self, *args, **kwargs):
self.logger = logging.getLogger(__name__)
BaseCommand.__init__(self, *args, **kwargs)
self.observer = None
def add_arguments(self, parser):
parser.add_argument("directory",
default=settings.CONSUMPTION_DIR,
nargs="?",
help="The consumption directory.")
parser.add_argument("--oneshot",
action="store_true",
help="Run only once.")
def handle(self, *args, **options):
directory = options["directory"]
recursive = settings.CONSUMER_RECURSIVE
if not directory:
raise CommandError("CONSUMPTION_DIR does not appear to be set.")
if not os.path.isdir(directory):
raise CommandError(
f"Consumption directory {directory} does not exist")
if recursive:
for dirpath, _, filenames in os.walk(directory):
for filename in filenames:
filepath = os.path.join(dirpath, filename)
_consume(filepath)
else:
for entry in os.scandir(directory):
_consume(entry.path)
if options["oneshot"]:
return
if settings.CONSUMER_POLLING == 0 and INotify:
self.handle_inotify(directory, recursive)
else:
self.handle_polling(directory, recursive)
logger.debug("Consumer exiting.")
def handle_polling(self, directory, recursive):
logging.getLogger(__name__).info(
f"Polling directory for changes: {directory}")
self.observer = PollingObserver(timeout=settings.CONSUMER_POLLING)
self.observer.schedule(Handler(), directory, recursive=recursive)
self.observer.start()
try:
while self.observer.is_alive():
self.observer.join(1)
if self.stop_flag:
self.observer.stop()
except KeyboardInterrupt:
self.observer.stop()
self.observer.join()
def handle_inotify(self, directory, recursive):
logging.getLogger(__name__).info(
f"Using inotify to watch directory for changes: {directory}")
inotify = INotify()
inotify_flags = flags.CLOSE_WRITE | flags.MOVED_TO
if recursive:
descriptor = inotify.add_watch_recursive(directory, inotify_flags)
else:
descriptor = inotify.add_watch(directory, inotify_flags)
try:
while not self.stop_flag:
for event in inotify.read(timeout=1000):
if recursive:
path = inotify.get_path(event.wd)
else:
path = directory
filepath = os.path.join(path, event.name)
_consume(filepath)
except KeyboardInterrupt:
pass
inotify.rm_watch(descriptor)
inotify.close()
class FolderScanner(FileSystemEventHandler, InputScanner, QObject):
"""
Watches file changes (creation, move) in a specific filesystem folder
the watched directory is retrieved from user config on scanner startup
"""
@log
def __init__(self):
FileSystemEventHandler.__init__(self)
InputScanner.__init__(self)
QObject.__init__(self)
self._observer = None
@log
def start(self):
"""
Starts scanning scan folder for new files
"""
try:
scan_folder_path = config.get_scan_folder_path()
self._observer = PollingObserver()
self._observer.schedule(self, scan_folder_path, recursive=False)
self._observer.start()
except OSError as os_error:
raise ScannerStartError(os_error)
@log
def stop(self):
"""
Stops scanning scan folder for new files
"""
if self._observer is not None:
self._observer.stop()
self._observer = None
@log
def on_moved(self, event):
if event.event_type == 'moved':
image_path = event.dest_path
_LOGGER.debug(f"File move detected : {image_path}")
self.broadcast_image(read_disk_image(Path(image_path)))
@log
def on_created(self, event):
if event.event_type == 'created':
file_is_incomplete = True
last_file_size = -1
image_path = event.src_path
_LOGGER.debug(
f"File creation detected : {image_path}. Waiting until file is complete and readable ..."
)
while file_is_incomplete:
info = QFileInfo(image_path)
size = info.size()
_LOGGER.debug(f"File {image_path}'s size = {size}")
if size == last_file_size:
file_is_incomplete = False
_LOGGER.debug(f"File {image_path} is ready to be read")
last_file_size = size
time.sleep(_DEFAULT_SCAN_FILE_SIZE_RETRY_PERIOD_IN_SEC)
self.broadcast_image(read_disk_image(Path(image_path)))
def main():
print 'Starting main...'
log.info('--------------------------------------------------------------------')
log.info('HealthPro CSV Ingester service started.')
log.info('Details about database from config file: Server: {}, DB Table: {}, '\
''.format(db_info['host'], healthpro_table_name))
observer = PollingObserver(timeout=5) # check every 5 seconds
try:
if not do_startup_checks():
raise Exception('One or more startup checks failed')
class FSEHandler(FileSystemEventHandler):
# Here we define what we'd like to do when certain filesystem
# events take place -- e.g., when a new CSV appears in the watched
# directory.
# Uncomment on_any_event for extra logging.
#def on_any_event(self, event):
# log.info('FSEHandler->on_any_event: event_type=[' \
# '{}], src_path=[{}]'.format(event.event_type, event.src_path))
def on_deleted(self, event):
# Our forked Watchdog (v0.8.3.1) emits this event when inbox folder
# unmounts (or otherwise is not available).
# We log and send an email only once. It very well might remount without
# us needing to do anything.
global inbox_gone_flag
if event.src_path == inbox_dir:
if not inbox_gone_flag:
inbox_gone_flag = True
msg = event.src_path + ' is gone!'
log.error(msg)
send_notice_email(msg)
def on_created(self, event):
# In on_deleted above, we set the inbox_gone_flag. But if a file appears
# we know the inbox is back and all is well; so unset it.
global inbox_gone_flag
if inbox_gone_flag:
inbox_gone_flag = False
log.info('FSEHandler->on_created: a new file has appeared: '
+ event.src_path)
process_file(event.src_path)
observe_subdirs_flag = False
observer.schedule(FSEHandler(), inbox_dir, observe_subdirs_flag)
observer.start()
log.info('Waiting for activity...')
print 'Service started.'
try:
while True:
observer.join(10)
if observer.is_alive():
continue
else:
raise Exception('Observer thread has stopped.')
except KeyboardInterrupt:
print '\nKeyboard interrupt caught. Quitting.'
observer.stop()
sys.exit(0)
observer.join()
except Exception, ex:
print '\nError caught. Quitting. Check log.'
log.error(str(ex))
send_error_email('An error occurred in main(). Please check.')
observer.stop()
sys.exit(1)
#!/usr/bin/env python
import sys, os, re
import time
import datetime
import logging
from watchdog.observers.polling import PollingObserver as Observer
from watchdog.events import FileSystemEventHandler
import threading, thread
import h5py
from Bio import SeqIO
from StringIO import StringIO
import MySQLdb
import subprocess
import string
import configargparse
from warnings import filterwarnings
import socket
import hashlib
import xmltodict
parser = configargparse.ArgParser(
description=
'eboladb_squiggle_align: A program providing the ability to determine which region of the ebola genome and individual read is derived from.'
)
parser.add(
'-fasta',
'--reference_fasta_file',
type=str,
dest='fasta',
required=True,
default=None,
class Evaluator(object):
''' An object that encapsulates model evaluation '''
def __init__(self, config, model, dataloader, device):
self.model = model
self.config = config
self.device = device
self.dataloader = dataloader
self.should_exit = False
signal.signal(signal.SIGHUP, self.on_training_complete)
self.observer = None
if 'cuda' in device.type:
self.model = nn.DataParallel(model.cuda())
self.modules = {'model': model}
@property
def dataset(self):
''' Get the dataset '''
return self.dataloader.dataset
def evaluate(self, batch):
''' Runs one evaluation step '''
with torch.no_grad():
self.model.eval()
_, nll = self.model(batch)
#pdb.set_trace()
# nn.DataParallel wants to gather rather than doing a reduce_add, so the output here
# will be a tensor of values that must be summed
nll = nll.sum()
# need to use .item() which converts to Python scalar
# because as a Tensor it accumulates gradients
return nll.item(), torch.sum(batch['target_lens']).item()
def evaluate_epoch(self, epoch, experiment, verbose=0):
''' Evaluate a single epoch '''
neg_log_likelihood = metrics.Metric('nll', metrics.format_float)
#pdb.set_trace()
def get_description():
mode_name = 'Test' if self.dataset.split == 'test' else 'Validate'
description = f'{mode_name} #{epoch}'
if verbose > 0:
description += f' {neg_log_likelihood}'
if verbose > 1:
description += f' [{profile.mem_stat_string(["allocated"])}]'
return description
batches = tqdm(
self.dataloader,
unit='batch',
dynamic_ncols=True,
desc=get_description(),
file=sys.stdout # needed to make tqdm_wrap_stdout work
)
with tqdm_wrap_stdout():
for batch in batches:
# run the data through the model
batches.set_description_str(get_description())
nll, length = self.evaluate(batch)
if length:
neg_log_likelihood.update(nll / length)
experiment.log_metric('nll', neg_log_likelihood.average)
return neg_log_likelihood.average
def on_new_checkpoint(self, path, experiment, verbose=0):
''' Upon receiving a new checkpoint path '''
epoch, step = restore(path,
self.modules,
num_checkpoints=self.config.average_checkpoints,
map_location=self.device.type)
experiment.set_step(step)
self.evaluate_epoch(epoch, experiment, verbose)
def on_training_complete(self, signum, frame): # pylint:disable=unused-argument
''' Received a SIGHUP indicating the training session has ended '''
self.should_exit = True
def shutdown(self):
''' Shutdown the current observer '''
if self.observer:
self.observer.stop()
self.observer.join()
def watch(self, experiment, verbose=0):
''' Watch for a new checkpoint and run an evaluation step '''
# Use a polling observer because slurm doesn't seem to correctly handle inotify events :/
self.observer = PollingObserver() if self.config.polling else Observer(
)
event_handler = CheckpointEventHandler(self.on_new_checkpoint,
experiment, verbose)
self.observer.schedule(event_handler, path=self.config.watch_directory)
self.observer.start()
while not self.should_exit:
time.sleep(1)
atexit.register(self.shutdown)
def __call__(self, epoch, experiment, verbose=0):
''' Validate the model and store off the stats '''
enter_mode = experiment.validate
if self.dataset.split == 'test':
enter_mode = experiment.test
with ExitStack() as stack:
stack.enter_context(enter_mode())
stack.enter_context(torch.no_grad())
if self.config.watch_directory:
self.watch(experiment, verbose)
else:
return self.evaluate_epoch(epoch, experiment, verbose)
# file modifications on VM share (nfs) always triggering create event
# so don't need modified event
#def on_modified(self, event):
# if os.path.isfile(event.src_path):
# print "modified, touching {}".format(event.src_path)
# _retouch_file(event.src_path)
if __name__ == "__main__":
path = sys.argv[1] if len(sys.argv) > 1 else '.'
# default polling_timeout to 3 seconds
polling_timeout = int(sys.argv[2]) if len(sys.argv) > 2 else 3
logger.info(
'Starting watching and touching - path - {}, polling timeout - {} ...'.
format(path, polling_timeout))
event_handler = PollingTouchFileHandler()
#observer = Observer()
observer = PollingObserver(timeout=polling_timeout)
observer.schedule(event_handler, path, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def main(argv):
"""
Build the docs and serve them with an HTTP server.
"""
parser = argparse.ArgumentParser(
description='Build and serve HTML Sphinx docs')
parser.add_argument('--port',
help='Serve on this port, default 8000',
type=int,
default=8000)
parser.add_argument(
'--source',
help='Directory of source Sphinx (reStructuredText) docs',
type=os.path.realpath,
default='docs/source')
parser.add_argument('--destination',
help='Where to build the HTML output',
type=os.path.realpath,
default='docs/build/html')
parser.add_argument('--doctrees',
help='Where the doctrees are built',
type=os.path.realpath,
default='docs/build/doctrees')
options = parser.parse_args(argv)
bound_build_docs = partial(build_docs, options.source, options.destination,
options.doctrees)
# Do the initial build
bound_build_docs()
# Watch the source directory for changes, build docs again if detected
observer = Observer()
observer.schedule(BuildDocsHandler(bound_build_docs),
path=options.source,
recursive=True)
observer.start()
# Set the root for the request handler, overriding Python stdlib current
# working directory.
DocsHTTPRequestHandler._root = options.destination
server = SocketServer.TCPServer(('', options.port), DocsHTTPRequestHandler)
try:
logger.info('Serving on localhost:{}'.format(options.port))
server.serve_forever()
except KeyboardInterrupt:
sys.stdout.write('\n')
logger.info('(stopping server)')
observer.stop()
finally:
observer.join()
logging.info('Server stopped, exiting')
sys.exit(0)
class ProbeEvaluator(object):
''' An object that encapsulates model evaluation '''
def __init__(self, config, model, dataloader, device):
self.model = model
self.config = config
self.device = device
self.dataloader = dataloader
self.should_exit = False
signal.signal(signal.SIGHUP, self.on_training_complete)
self.observer = None
if 'cuda' in device.type:
self.model = nn.DataParallel(model.cuda())
self.modules = {
'model': model
}
# stats
self.stats = {model_stat: {stats_type: {'mean': torch.zeros((model.num_layers, model.num_heads),
dtype=torch.float32).to(device),
'var': torch.zeros((model.num_layers, model.num_heads),
dtype=torch.float32).to(device)}
for stats_type in STATS_TYPES}
for model_stat in MODEL_STATS}
self.count = {model_stat: 0 for model_stat in MODEL_STATS}
@property
def dataset(self):
''' Get the dataset '''
return self.dataloader.dataset
def evaluate(self, batch):
''' Runs one evaluation step '''
with torch.no_grad():
self.model.eval()
# _, nll = self.model(batch)
result = self.model(batch)
nll = result['nll']
# stats
encoder_stats = probe(result['encoder_attn_weights_tensor'])
decoder_stats = probe(result['decoder_attn_weights_tensor'])
enc_dec_stats = probe(result['enc_dec_attn_weights_tensor'])
stats = {'encoder_stats': {stats_type: encoder_stats[stats_type].view(self.model.num_layers,
self.model.num_heads,
-1)
for stats_type in STATS_TYPES},
'decoder_stats': {stats_type: decoder_stats[stats_type].view(self.model.num_layers,
self.model.num_heads,
-1)
for stats_type in STATS_TYPES},
'enc_dec_stats': {stats_type: enc_dec_stats[stats_type].view(self.model.num_layers,
self.model.num_heads,
-1)
for stats_type in STATS_TYPES}}
# nn.DataParallel wants to gather rather than doing a reduce_add, so the output here
# will be a tensor of values that must be summed
nll = nll.sum()
# need to use .item() which converts to Python scalar
# because as a Tensor it accumulates gradients
return nll.item(), torch.sum(batch['target_lens']).item(), stats
def evaluate_epoch(self, epoch, experiment, stats_file, verbose=0):
''' Evaluate a single epoch '''
neg_log_likelihood = metrics.Metric('nll', metrics.format_float)
def get_description():
mode_name = 'Test' if self.dataset.split == 'test' else 'Validate'
description = f'{mode_name} #{epoch}'
if verbose > 0:
description += f' {neg_log_likelihood}'
if verbose > 1:
description += f' [{profile.mem_stat_string(["allocated"])}]'
return description
batches = tqdm(
self.dataloader,
unit='batch',
dynamic_ncols=True,
desc=get_description(),
file=sys.stdout # needed to make tqdm_wrap_stdout work
)
with tqdm_wrap_stdout():
for batch in batches:
# run the data through the model
batches.set_description_str(get_description())
nll, length, stats = self.evaluate(batch)
self.update_stats(stats, self.stats, self.count)
if length:
neg_log_likelihood.update(nll / length)
experiment.log_metric('nll', neg_log_likelihood.average)
self.save_stats(stats_file)
return neg_log_likelihood.average
def update_stats(self, stats, self_stats, self_count):
''' Update stats after each batch '''
for model_stat in stats:
current_count = stats[model_stat][STATS_TYPES[0]].size()[-1]
old_count = self_count[model_stat]
new_count = old_count + current_count
for stat_type in stats[model_stat]:
old_mean = self_stats[model_stat][stat_type]['mean']
current_mean = stats[model_stat][stat_type].mean(dim=-1)
new_mean = (old_mean * self_count[model_stat] + stats[model_stat][stat_type].sum(dim=-1)) / new_count
old_var = self_stats[model_stat][stat_type]['var']
current_var = stats[model_stat][stat_type].var(
dim=-1) # torch.sum((stats[model_stat][stat_type] - new_mean.unsqueeze(-1)) ** 2, dim=-1) / (current_count - 1)
new_var = (old_count * (old_var + (old_mean - new_mean) ** 2)
+ current_count * (current_var + (current_mean - new_mean) ** 2)) / new_count
self_stats[model_stat][stat_type]['mean'] = new_mean
self_stats[model_stat][stat_type]['var'] = new_var
self_count[model_stat] = new_count
def save_stats(self, stats_file):
''' Save stats to file '''
stats = {'stats': self.stats, 'count': self.count}
pickle.dump(stats, stats_file, protocol=pickle.HIGHEST_PROTOCOL)
def on_new_checkpoint(self, path, experiment, verbose=0):
''' Upon receiving a new checkpoint path '''
epoch, step = restore(
path,
self.modules,
num_checkpoints=self.config.average_checkpoints,
map_location=self.device.type
)
experiment.set_step(step)
self.evaluate_epoch(epoch, experiment, verbose)
def on_training_complete(self, signum, frame): # pylint:disable=unused-argument
''' Received a SIGHUP indicating the training session has ended '''
self.should_exit = True
def shutdown(self):
''' Shutdown the current observer '''
if self.observer:
self.observer.stop()
self.observer.join()
def watch(self, experiment, verbose=0):
''' Watch for a new checkpoint and run an evaluation step '''
# Use a polling observer because slurm doesn't seem to correctly handle inotify events :/
self.observer = PollingObserver() if self.config.polling else Observer()
event_handler = CheckpointEventHandler(self.on_new_checkpoint, experiment, verbose)
self.observer.schedule(event_handler, path=self.config.watch_directory)
self.observer.start()
while not self.should_exit:
time.sleep(1)
atexit.register(self.shutdown)
def __call__(self, epoch, experiment, verbose=0):
''' Validate the model and store off the stats '''
enter_mode = experiment.validate
if self.dataset.split == 'test':
enter_mode = experiment.test
with ExitStack() as stack:
stack.enter_context(enter_mode())
stack.enter_context(torch.no_grad())
stats_filename = self.config.stats_filename or f'stats_probe_evaluate.pickle'
stats_path = os.path.join(self.config.stats_directory, stats_filename)
stats_file = stack.enter_context(open(stats_path, 'wb'))
if self.config.watch_directory:
self.watch(experiment, verbose)
else:
return self.evaluate_epoch(epoch, experiment, stats_file, verbose)
class LivePremailer():
def __init__(self):
self.observer_paths = {HERE}
self.bsync_params = {
'server': None,
'directory': None,
'reloadDelay': 1000,
'online': 'true',
'logLevel': 'silent',
'files': HERE,
}
def append_arguments(self, parser):
parser.add_argument('--devpostfix',
nargs='?',
default='_dev',
help='Postfix which should be used to search\
dev templates')
parser.add_argument('--livepostfix',
nargs='?',
default='_live',
help='Postfix which should be used to name\
files with live preview')
parser.add_argument('--loadhistory',
action='store_true',
help='lpremailer will load all paths located\
in {} file to memory and rerender them\
everytime change in any file occurs'.format(
HISTORY_FILENAME))
parser.add_argument('--savehistory',
action='store_true',
help='lpremailer will save all dev file paths\
recorded during development in {} file'.
format(HISTORY_FILENAME))
parser.add_argument('--astext',
action='store_true',
help='lpremailer will save all dev files\
as simple txt messages')
def parse_args(self):
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers()
runserver_help = 'Runs server for live premailer'
sub_parser = subparsers.add_parser(PARSER_RUN, help=runserver_help)
sub_parser.set_defaults(which=PARSER_RUN)
sub_parser.add_argument('--staticdir',
nargs='?',
help='Path to directory where static folder\
is located')
self.append_arguments(sub_parser)
init_help = 'Create json files for htmls with provided\
postfix in current directory'
sub_parser = subparsers.add_parser(PARSER_INIT, help=init_help)
sub_parser.set_defaults(which=PARSER_INIT)
sub_parser.add_argument('--force',
action='store_true',
help='Overwrites json files')
self.append_arguments(sub_parser)
self.args = parser.parse_args()
def json_files(self):
handler = RenderHandler(self.args)
if self.args.which == PARSER_INIT:
JsonGenerator(handler, HERE).generate()
sys.exit(1)
def start_observer(self):
self.observer = PollingObserver()
self.observer.should_keep_running()
self.observer.handler = RenderHandler(self.args)
for path in self.observer_paths:
self.observer.schedule(self.observer.handler, path, recursive=True)
self.observer.start()
def update_params(self):
if not os.path.exists(self.args.staticdir):
logging.warning('Static files won\'t be maintained/served.')
return
files = '!**/*.less,!**/*.sass,!**/*.scss'
files = (self.bsync_params['files'], self.args.staticdir, files)
self.bsync_params['files'] = ','.join(files)
self.bsync_params['ss'] = self.args.staticdir
self.observer_paths.add(self.args.staticdir)
def bsync_command(self):
return 'browser-sync start {}'\
.format(parse_params(self.bsync_params))
def run_bsync(self):
self.bsync = subprocess.Popen(self.bsync_command(), shell=True)
def run(self):
self.parse_args()
self.json_files()
self.update_params()
self.start_observer()
self.run_bsync()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
if self.args.savehistory:
self.observer.handler.save_history()
self.observer.stop()
self.bsync.kill()
self.observer.join()
def watch(entries=[], ignore=[], extensions=[], beep_on_failure=True,
auto_clear=False, wait=False, beforerun=None, afterrun=None,
onpass=None, onfail=None, onexit=None, runner=None, spool=None,
poll=False, verbose=False, quiet=False, pytest_args=[]):
argv = _get_pytest_runner(runner) + (pytest_args or [])
if not entries:
entries = ['.']
files = []
directories = []
for entry in entries:
entry = os.path.abspath(entry)
if os.path.isfile(entry):
files.append(entry)
elif os.path.isdir(entry):
directories.append(entry)
else:
raise ValueError('Directory not found: ' + entry)
# Setup event handler
event_listener = EventListener(extensions)
# Setup watchdog
observer = PollingObserver() if poll else Observer()
for file in files:
single_file_listener = EventSingleFileListener(
file, event_queue=event_listener.event_queue)
observer.schedule(
single_file_listener, path=os.path.dirname(file), recursive=False)
recursedirs, norecursedirs = _split_recursive(directories, ignore)
for directory in recursedirs:
observer.schedule(event_listener, path=directory, recursive=True)
for directory in norecursedirs:
observer.schedule(event_listener, path=directory, recursive=False)
observer.start()
# Watch and run tests until interrupted by user
events = []
while True:
try:
# Prepare next run
if auto_clear:
clear()
elif not quiet:
print()
# Show event summary
if not quiet:
_show_summary(argv, events, verbose)
# Run custom command
run_hook(beforerun)
# Run tests
p = subprocess.Popen(argv, shell=is_windows)
try:
while True:
# Check for completion
exit_code = p.poll()
if exit_code is not None:
break
# Interrupt the current test run on filesystem event
if not wait and not event_listener.event_queue.empty():
send_keyboard_interrupt(p)
exit_code = p.wait()
break
# Allow user to initiate a keyboard interrupt
time.sleep(0.1)
except KeyboardInterrupt:
# Wait for current test run cleanup
run_hook(afterrun, p.wait())
# Exit, since this keyboard interrupt was user-initiated
break
# Run custom command
run_hook(afterrun, exit_code)
# Run dependent commands
if exit_code in [EXIT_OK, EXIT_NOTESTSCOLLECTED]:
run_hook(onpass)
else:
if beep_on_failure:
beep()
run_hook(onfail)
# Wait for a filesystem event
while event_listener.event_queue.empty():
time.sleep(0.1)
# Collect events for summary of next run
events = dequeue_all(event_listener.event_queue, spool)
except KeyboardInterrupt:
break
except Exception as ex:
print(format_exc() if verbose else 'Error: {}'.format(ex))
break
# Stop and wait for observer
observer.stop()
observer.join()
# Run exit script
run_hook(onexit)
class DirWatcher(object):
def __init__(self, settings, api, file_pusher):
self._api = api
self._file_count = 0
self._dir = settings.files_dir
self._settings = settings
self._user_file_policies = {"end": set(), "live": set(), "now": set()}
self._file_pusher = file_pusher
self._file_event_handlers = {}
self._file_observer = PollingObserver()
self._file_observer.schedule(self._per_file_event_handler(),
self._dir,
recursive=True)
self._file_observer.start()
logger.info("watching files in: %s", settings.files_dir)
@property
def emitter(self):
try:
return next(iter(self._file_observer.emitters))
except StopIteration:
return None
def update_policy(self, path, policy):
self._user_file_policies[policy].add(path)
for src_path in glob.glob(os.path.join(self._dir, path)):
save_name = os.path.relpath(src_path, self._dir)
self._get_file_event_handler(src_path,
save_name).on_modified(force=True)
def _per_file_event_handler(self):
"""Create a Watchdog file event handler that does different things for every file
"""
file_event_handler = PatternMatchingEventHandler()
file_event_handler.on_created = self._on_file_created
file_event_handler.on_modified = self._on_file_modified
file_event_handler.on_moved = self._on_file_moved
file_event_handler._patterns = [
os.path.join(self._dir, os.path.normpath("*"))
]
# Ignore hidden files/folders
# TODO: what other files should we skip?
file_event_handler._ignore_patterns = [
"*.tmp",
"*.wandb",
"wandb-summary.json",
os.path.join(self._dir, ".*"),
os.path.join(self._dir, "*/.*"),
]
# TODO: pipe in actual settings
for glb in self._settings.ignore_globs:
file_event_handler._ignore_patterns.append(
os.path.join(self._dir, glb))
return file_event_handler
def _on_file_created(self, event):
logger.info("file/dir created: %s", event.src_path)
if os.path.isdir(event.src_path):
return None
self._file_count += 1
# We do the directory scan less often as it grows
if self._file_count % 100 == 0:
emitter = self.emitter
if emitter:
emitter._timeout = int(self._file_count / 100) + 1
save_name = os.path.relpath(event.src_path, self._dir)
self._get_file_event_handler(event.src_path, save_name).on_modified()
def _on_file_modified(self, event):
logger.info("file/dir modified: %s", event.src_path)
if os.path.isdir(event.src_path):
return None
save_name = os.path.relpath(event.src_path, self._dir)
self._get_file_event_handler(event.src_path, save_name).on_modified()
def _on_file_moved(self, event):
# TODO: test me...
logger.info("file/dir moved: %s -> %s", event.src_path,
event.dest_path)
if os.path.isdir(event.dest_path):
return None
old_save_name = os.path.relpath(event.src_path, self._dir)
new_save_name = os.path.relpath(event.dest_path, self._dir)
# We have to move the existing file handler to the new name
handler = self._get_file_event_handler(event.src_path, old_save_name)
self._file_event_handlers[new_save_name] = handler
del self._file_event_handlers[old_save_name]
handler.on_renamed(event.dest_path, new_save_name)
def _get_file_event_handler(self, file_path, save_name):
"""Get or create an event handler for a particular file.
file_path: the file's actual path
save_name: its path relative to the run directory (aka the watch directory)
"""
if save_name not in self._file_event_handlers:
# TODO: we can use PolicyIgnore if there are files we never want to sync
if 'tfevents' in save_name or 'graph.pbtxt' in save_name:
self._file_event_handlers[save_name] = PolicyLive(
file_path, save_name, self._api, self._file_pusher)
else:
Handler = PolicyEnd
for policy, globs in six.iteritems(self._user_file_policies):
if policy == "end":
continue
# Convert set to list to avoid RuntimeError's
# TODO: we may need to add locks
for g in list(globs):
paths = glob.glob(os.path.join(self._dir, g))
if any(save_name in p for p in paths):
if policy == "live":
Handler = PolicyLive
elif policy == "now":
Handler = PolicyNow
self._file_event_handlers[save_name] = Handler(
file_path, save_name, self._api, self._file_pusher)
return self._file_event_handlers[save_name]
def finish(self):
logger.info("shutting down directory watcher")
try:
# avoid hanging if we crashed before the observer was started
if self._file_observer.is_alive():
# rather unfortunatly we need to manually do a final scan of the dir
# with `queue_events`, then iterate through all events before stopping
# the observer to catch all files written. First we need to prevent the
# existing thread from consuming our final events, then we process them
self._file_observer._timeout = 0
self._file_observer._stopped_event.set()
self._file_observer.join()
self.emitter.queue_events(0)
while True:
try:
self._file_observer.dispatch_events(
self._file_observer.event_queue, 0)
except queue.Empty:
break
# Calling stop unschedules any inflight events so we handled them above
self._file_observer.stop()
# TODO: py2 TypeError: PyCObject_AsVoidPtr called with null pointer
except TypeError:
pass
# TODO: py3 SystemError: <built-in function stop> returned an error
except SystemError:
pass
# Ensure we've at least noticed every file in the run directory. Sometimes
# we miss things because asynchronously watching filesystems isn't reliable.
logger.info("scan: %s", self._dir)
for dirpath, _, filenames in os.walk(self._dir):
for fname in filenames:
file_path = os.path.join(dirpath, fname)
save_name = os.path.relpath(file_path, self._dir)
logger.info("scan save: %s %s", file_path, save_name)
self._get_file_event_handler(file_path, save_name).finish()
class MonitorFile(PatternMatchingEventHandler):
"""Monitor files and create Beergarden events
This is a wrapper around a watchdog PollingObserver. PollingObserver is used instead
of Observer because Observer throws events on each file transaction.
Note that the events generated are NOT watchdog events, they are whatever
Beergarden events are specified during initialization.
"""
def __init__(
self,
path: str,
create_event: Event = None,
modify_event: Event = None,
moved_event: Event = None,
deleted_event: Event = None,
):
super().__init__(patterns=[path], ignore_directories=True)
self._path = path
self._observer = PollingObserver()
self.create_event = create_event
self.modify_event = modify_event
self.moved_event = moved_event
self.deleted_event = deleted_event
def start(self):
self._observer.schedule(self, Path(self._path).parent, recursive=False)
self._observer.start()
def stop(self):
if self._observer.is_alive():
self._observer.stop()
self._observer.join()
def on_created(self, _):
"""Callback invoked when the file is created
When a user VIM edits a file it DELETES, then CREATES the file, this
captures that case
"""
if self.create_event:
publish(self.create_event)
def on_modified(self, _):
"""Callback invoked when the file is modified
This captures all other modification events that occur against the file
"""
if self.modify_event:
publish(self.modify_event)
def on_moved(self, _):
"""Callback invoked when the file is moved
This captures if the file is moved into or from the directory
"""
if self.moved_event:
publish(self.moved_event)
def on_deleted(self, _):
"""Callback invoked when the file is deleted
This captures if the file was deleted (be warned that VIM does this by
default during write actions)
"""
if self.deleted_event:
publish(self.deleted_event)
class ScanFolder:
'Class defining a scan folder'
def __init__(self):
self.FourDFlowScanNameC1 = ""
self.FourDFlowScanNameC2 = ""
self.DicomNamesC1 = []
self.DicomNamesC2 = []
self.Candidate_Num1 = 0
self.Candidate_Num2 = 0
self.finalPath1 = ""
self.finalPath2 = ""
self.Patients = []
self.path = r'/mnt/data_imaging/jetsonTest'
#self.path = r'/mnt/data_imaging/dicom'
self.documents = dict(
) # key = document label value = Document reference
self.patterns = "*"
self.ignore_patterns = ""
self.ignore_directories = False
self.case_sensitive = True
self.my_event_handler = watchdog.events.PatternMatchingEventHandler(
self.patterns, self.ignore_patterns, self.ignore_directories,
self.case_sensitive)
#self.my_event_handler.on_any_event = self.on_any_event
#self.my_event_handler.on_deleted = self.on_deleted
self.my_event_handler.on_created = self.on_created
self.observer = Observer()
self.observer.schedule(self.my_event_handler,
self.path,
recursive=True)
self.observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
self.observer.stop()
self.observer.join()
# def on_any_event(self, event):
# print(event.src_path, event.event_type)
# print("self even")
#def stop(self):
# self.observer.stop()
# self.observer.join()
def on_deleted(self, event):
print(f"what the f**k! Someone deleted {event.src_path}!")
#print(self.FourDFlowScanNameC1)
#self.FourDFlowScanNameC1 = "dogs"
def on_created(self, event):
#print(f"hey, {event.src_path} has been created!")
fullFilePath = {event.src_path}
parts = str({event.src_path}.pop()).split(
'/') #change to be / for jetson and \ for my desktop
#print(parts)
#print(len(parts))
##need to try catch this in case of weird inputs?
fileName = parts[-1]
scanName = parts[-2]
patientName = parts[-3]
#if "fileName" doesn't have a '.extension' then it is a folder OR use os.path.isfile() / os.path.isdir()
fileNameParts = fileName.split('.')
#print(fileNameParts)
numParts = len(fileNameParts)
#print(numParts)
#print("The scan name")
#print(scanName)
if (numParts >= 1):
#pattern matching block, swap from 4D to 4D and Aorta
pattern = "4D"
#print("our matching result :" + str( re.search(pattern, scanName)))
result = re.search(pattern, scanName)
pattern2 = "[aA][oO][rR][tT][aA]"
result2 = re.search(pattern2, scanName)
if (result and result2):
patientFound = False
for patient in self.Patients:
pName = patient.getPatientName()
if (pName == patientName):
patientFound = True
patient.takeInScan(scanName, fullFilePath)
if (patient.pushToJetson()):
print("Let's push to Jetson!")
flow = patient.getFlow()
flow = Path(flow).parent
print(flow)
mag = patient.getMag()
mag = Path(mag).parent
cmd = [
"python", "./mrStructWrite.py",
"--flow=" + str(flow), "--mag=" + str(mag)
]
#subprocess.call(["python /home/condauser/Documents/ao_seg/4dflow_dicom_to_array.py --flow=",str(flow)," --mag=",str(mag)])
subprocess.call(cmd)
#cmdstr = re.sub(r"(",r"\(","python /home/condauser/Documents/ao_seg/4dflow_dicom_to_array.py --flow="+str(flow)+" --mag="+str(mag))
#print(cmdstr)
#os.system(cmdstr)
#delete the patient
if (not patientFound):
#print("Adding patient " + str(patientName))
self.Patients.append(Patient(patientName))
self.Patients[-1].takeInScan(scanName, fullFilePath)
def main():
print 'Starting main...'
log.info('--------------------------------------------------------------------')
log.info('HealthPro CSV Ingester service started.')
log.info('Details about database from config file: Server: {}, DB Table: {}. '\
''.format(db_info['host'], healthpro_table_name))
observer = PollingObserver(timeout=5) # check every 5 seconds
try:
if not do_startup_checks():
raise Exception('One or more startup checks failed')
class FSEHandler(FileSystemEventHandler):
# Here we define what we'd like to do when certain filesystem
# events take place -- e.g., when a new CSV appears in the watched
# directory.
# Uncomment on_any_event for extra logging.
#def on_any_event(self, event):
# log.info('FSEHandler->on_any_event: event_type=[' \
# '{}], src_path=[{}]'.format(event.event_type, event.src_path))
def on_deleted(self, event):
# We log and send an email only once.
global inbox_gone_flag
if event.src_path == inbox_dir:
if not inbox_gone_flag:
inbox_gone_flag = True
msg = event.src_path + ' is gone! Please see this SOP: https://nexus.weill.cornell.edu/pages/viewpage.action?pageId=110266114'
log.error(msg)
send_notice_email(msg)
def on_created(self, event):
# In on_deleted above, we set the inbox_gone_flag. But if a file appears
# we know the inbox is back and all is well; so unset it.
global inbox_gone_flag
if inbox_gone_flag:
inbox_gone_flag = False
log.info('FSEHandler->on_created: a new file has appeared: '
+ event.src_path)
process_file(event.src_path)
observe_subdirs_flag = False
observer.schedule(FSEHandler(), inbox_dir, observe_subdirs_flag)
observer.start()
log.info('Waiting for activity...')
print 'Service started.'
if cfg.get('start-telemetry-ping-listener') == 'yes':
start_telemetry_ping_listener()
try:
while True:
observer.join(10)
if observer.is_alive():
continue
else:
raise Exception('Observer thread has stopped.')
except KeyboardInterrupt:
print '\nKeyboard interrupt caught. Quitting.'
observer.stop()
sys.exit(0)
observer.join()
except Exception, ex:
print '\nError caught. Quitting. Check log.'
log.error(str(ex))
send_error_email('An error occurred in main(). Please check.')
observer.stop()
sys.exit(1)
class MyGui:
def __init__(self):
def get_pref_DIRECTORY_TO_WATCH():
folder_selected = filedialog.askdirectory()
DIR_W_Path.set(folder_selected)
def get_pref_DIRECTORY_TO_MOVE():
folder_selected = filedialog.askdirectory()
DIR_M_Path.set(folder_selected)
def get_pref_DIRECTORY_UNMATCHED():
folder_selected = filedialog.askdirectory()
DIR_U_Path.set(folder_selected)
def pref_set():
global DIRECTORY_UNMATCHED
DIRECTORY_UNMATCHED = DIR_U_Path.get().replace("/", "\\")
global DIRECTORY_TO_MOVE
DIRECTORY_TO_MOVE = DIR_M_Path.get().replace("/", "\\")
global DIRECTORY_TO_WATCH
DIRECTORY_TO_WATCH = DIR_W_Path.get().replace("/", "\\")
global pref_ID
pref_ID = UI_pref_ID.get()
global pref_DryRun
pref_DryRun = UI_pref_DryRun.get()
global pref_StripSymbol
pref_StripSymbol = UI_pref_StripSymbol.get()
if ((DIRECTORY_TO_WATCH != "") and (DIRECTORY_TO_MOVE != "")
and (DIRECTORY_UNMATCHED != "")
and (os.path.exists(DIRECTORY_TO_WATCH))
and (os.path.exists(DIRECTORY_TO_MOVE))
and (os.path.exists(DIRECTORY_UNMATCHED))):
self.but.config(state="normal", text="Start Watchdog")
else:
self.but.config(state="disabled", text="Set Preferences First")
root.title('Porndog - Adult Scene Renamer')
root.iconbitmap('.\\icon.ico')
root.geometry("500x500")
Watchdog_Preferences_Label = Label(
root, text="Watchdog Preferences - Directories")
Watchdog_Preferences_Label.place(x=192, y=5)
DIR_W_Path = tk.StringVar()
DIR_W_Label = Label(root, text="Active Directory - ")
DIR_W_Label.place(x=80, y=28)
self.DIR_W_TextField = Entry(root, textvariable=DIR_W_Path, width=35)
self.DIR_W_TextField.place(x=180, y=30)
self.DIR_W_Button = ttk.Button(root,
text="Browse Folder",
command=get_pref_DIRECTORY_TO_WATCH)
self.DIR_W_Button.place(x=400, y=28)
DIR_M_Path = tk.StringVar()
DIR_M_Label = Label(root, text="Move Directory - ")
DIR_M_Label.place(x=82, y=58)
self.DIR_M_TextField = Entry(root, textvariable=DIR_M_Path, width=35)
self.DIR_M_TextField.place(x=180, y=60)
self.DIR_M_Button = ttk.Button(root,
text="Browse Folder",
command=get_pref_DIRECTORY_TO_MOVE)
self.DIR_M_Button.place(x=400, y=58)
DIR_U_Path = tk.StringVar()
DIR_U_Label = Label(root, text="Unmatched Directory - ")
DIR_U_Label.place(x=50, y=88)
self.DIR_U_TextField = Entry(root, textvariable=DIR_U_Path, width=35)
self.DIR_U_TextField.place(x=180, y=90)
self.DIR_U_Button = ttk.Button(root,
text="Browse Folder",
command=get_pref_DIRECTORY_UNMATCHED)
self.DIR_U_Button.place(x=400, y=88)
UI_FilenamePref_Label = Label(root, text="Filename Preferences")
UI_FilenamePref_Label.place(x=225, y=120)
UI_pref_ID = BooleanVar()
self.UI_Checkbutton_ID = Checkbutton(
root, text="Prefer Scene ID over Scene Title", variable=UI_pref_ID)
self.UI_Checkbutton_ID.place(x=185, y=150)
UI_pref_StripSymbol = tk.StringVar()
UI_pref_StripSymbol_Label = Label(root, text="Strip Symbol - ")
UI_pref_StripSymbol_Label.place(x=95, y=180)
self.UI_pref_StripSymbol_TextField = Entry(
root, textvariable=UI_pref_StripSymbol, width=35)
self.UI_pref_StripSymbol_TextField.place(x=180, y=180)
UI_OtherPref_Label = Label(root, text="Other Preferences")
UI_OtherPref_Label.place(x=228, y=210)
UI_pref_DryRun = BooleanVar()
self.UI_Checkbutton_DryRun = Checkbutton(root,
text="Dry Run",
variable=UI_pref_DryRun)
self.UI_Checkbutton_DryRun.place(x=240, y=240)
self.SET_BUTTON = tk.Button(root,
text="Set Preferences",
command=pref_set)
self.SET_BUTTON.place(x=230, y=270)
self.but = tk.Button(root,
text="Start Watchdog",
command=self.start_observer)
self.but.place(x=230, y=300)
self.but.config(state="disabled", text="Set Preferences First")
self.but2 = tk.Button(root,
text="Stop Watchdog",
command=self.stop_observer)
self.but2.place(x=230, y=330)
self.but2.config(state="disabled", text="Stop Watchdog")
def start_observer(self):
loggerwatchdog.info(
"******************** Pre-initialization ********************")
loggerwatchdog.info("Watchdog will be active to this directory: " +
DIRECTORY_TO_WATCH)
loggerwatchdog.info(
"Watchdog will move the files to this directory: " +
DIRECTORY_TO_MOVE)
loggerwatchdog.info(
"Watchdog will move unmatched files to this directory: " +
DIRECTORY_UNMATCHED)
loggerwatchdog.info("Preferred ID is set to: " + str(pref_ID))
loggerwatchdog.info("Dry Run is set to: " + str(pref_DryRun))
if (pref_StripSymbol != ""):
loggerwatchdog.info("Your strip symbol is: " + (pref_StripSymbol))
else:
loggerwatchdog.info("You haven't set a Strip Symbol.")
loggerwatchdog.info(
"******************** Pre-initialization ********************")
self.DIR_W_TextField.config(state="disabled")
self.DIR_W_Button.config(state="disabled")
self.DIR_M_TextField.config(state="disabled")
self.DIR_M_Button.config(state="disabled")
self.DIR_U_TextField.config(state="disabled")
self.DIR_U_Button.config(state="disabled")
self.UI_Checkbutton_ID.config(state="disabled")
self.UI_pref_StripSymbol_TextField.config(state="disabled")
self.UI_Checkbutton_DryRun.config(state="disabled")
self.SET_BUTTON.config(state="disabled")
self.but.config(state="disabled", text="Watchdog Initiated")
self.but2.config(state="normal", text="Stop Watchdog")
self.observer = Observer()
self.observer.schedule(event_handler,
DIRECTORY_TO_WATCH,
recursive=True)
self.observer.start()
loggerwatchdog.info(
"******************** Watchdog initiated ********************")
def stop_observer(self):
self.observer.stop()
self.observer.join()
self.observer = None
self.DIR_W_TextField.config(state="normal")
self.DIR_W_Button.config(state="normal")
self.DIR_M_TextField.config(state="normal")
self.DIR_M_Button.config(state="normal")
self.DIR_U_TextField.config(state="normal")
self.DIR_U_Button.config(state="normal")
self.UI_Checkbutton_ID.config(state="normal")
self.UI_pref_StripSymbol_TextField.config(state="normal")
self.UI_Checkbutton_DryRun.config(state="normal")
self.SET_BUTTON.config(state="normal")
self.but.config(state="disabled", text="Set Preferences First")
self.but2.config(state="disabled", text="Stop Watchdog")
[i for i in name.split() if not i.isdigit()])
item["mac"] = newName.replace("/mnt/jarvis", "/Volumes")
item["category"] = category
item["path"] = newName.replace("/mnt/jarvis/Library",
"Y:").replace("/", "\\")
with open('/home/dyang/PWP-Lib-Search/public/plants.json',
'w') as publicJson:
json.dump(plantData, publicJson)
with open('/home/dyang/PWP-Lib-Search/build/plants.json',
'w') as buildJson:
json.dump(plantData, buildJson)
logging.info("rename file success: " + newName)
event_handler.on_created = on_created
event_handler.on_deleted = on_deleted
event_handler.on_moved = on_moved
nWatch = PollingObserver(timeout=60)
targetPath = str(path)
nWatch.schedule(event_handler, targetPath, recursive=False)
nWatch.start()
try:
while True:
time.sleep(1000)
except KeyboardInterrupt:
nWatch.stop()
nWatch.join()
def main():
logging.basicConfig(level=LOGGING_LEVEL,
format=LOGGING_FORMAT,
datefmt=LOGGING_FORMAT_DATE)
# readability
global path, tag, channel, stream_id, filenames_previous, connection, channel, args
args = parse_args()
path = args.path[0]
tag = args.tag
rabbitmq_host = args.host
stream_id = create_stream_id(tag)
filenames_previous = set()
# if multiple file endings
include = args.include.split(',')
# add star before the file ending if missing
include = [
ext.lower() if ext.startswith('*') else "*" + ext.lower()
for ext in include
]
logging.info(
f'starting with args {[path, include, tag, rabbitmq_host]} (excl. creds)'
)
# create the event handler
my_event_handler = PatternMatchingEventHandler(include,
ignore_patterns="",
ignore_directories=True,
case_sensitive=False)
my_event_handler.on_created = on_created
# my_observer = Observer() # only on local file systems (inotify)
my_observer = PollingObserver(timeout=20)
my_observer.schedule(my_event_handler, path, recursive=False)
# TODO: make queue persistent between rabbitMQ restarts?
# connect to the RabbitMQ server
logging.info(
f'starting with args {[path, include, tag, rabbitmq_host]} (excl. creds)'
)
connection = pika.BlockingConnection(
pika.ConnectionParameters(args.host,
credentials=PlainCredentials(
args.username, args.password),
heartbeat=0))
channel = connection.channel()
channel.queue_declare(queue='files')
logging.info('connected to AMPQ.')
# trigger event for all existing files?
existing_file_event = DummyEvent()
for existing_file in os.listdir(path): # list dir
if os.path.isfile(os.path.join(path, existing_file)): # only files
if f'*{os.path.splitext(existing_file)[-1].lower()}' in include: # if extension is matching
existing_file_event.src_path = os.path.join(
path, existing_file
) # add property to dummy object to have the same structure as a watchdog even object
on_created(
existing_file_event) # manually trigger creation function
# start monitoring
my_observer.start()
logging.info('started monitoring directory.')
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
my_observer.stop()
my_observer.join()
connection.close()
class LocalComputeLogSubscriptionManager:
def __init__(self, manager):
self._manager = manager
self._subscriptions = defaultdict(list)
self._watchers = {}
self._observer = None
def _watch_key(self, run_id, key):
return "{}:{}".format(run_id, key)
def add_subscription(self, subscription):
check.inst_param(subscription, "subscription", ComputeLogSubscription)
if self._manager.is_watch_completed(subscription.run_id,
subscription.key):
subscription.fetch()
subscription.complete()
else:
watch_key = self._watch_key(subscription.run_id, subscription.key)
self._subscriptions[watch_key].append(subscription)
self.watch(subscription.run_id, subscription.key)
def remove_all_subscriptions(self, run_id, step_key):
watch_key = self._watch_key(run_id, step_key)
for subscription in self._subscriptions.pop(watch_key, []):
subscription.complete()
def watch(self, run_id, step_key):
watch_key = self._watch_key(run_id, step_key)
if watch_key in self._watchers:
return
update_paths = [
self._manager.get_local_path(run_id, step_key,
ComputeIOType.STDOUT),
self._manager.get_local_path(run_id, step_key,
ComputeIOType.STDERR),
]
complete_paths = [
self._manager.complete_artifact_path(run_id, step_key)
]
directory = os.path.dirname(
self._manager.get_local_path(run_id, step_key,
ComputeIOType.STDERR))
if not self._observer:
self._observer = PollingObserver(self._manager.polling_timeout)
self._observer.start()
ensure_dir(directory)
self._watchers[watch_key] = self._observer.schedule(
LocalComputeLogFilesystemEventHandler(self, run_id, step_key,
update_paths,
complete_paths),
str(directory),
)
def notify_subscriptions(self, run_id, step_key):
watch_key = self._watch_key(run_id, step_key)
for subscription in self._subscriptions[watch_key]:
subscription.fetch()
def unwatch(self, run_id, step_key, handler):
watch_key = self._watch_key(run_id, step_key)
if watch_key in self._watchers:
self._observer.remove_handler_for_watch(handler,
self._watchers[watch_key])
del self._watchers[watch_key]
def dispose(self):
if self._observer:
self._observer.stop()
self._observer.join(15)
def run(self):
if not self._allowRoot:
self._check_for_root()
global app
global babel
global printer
global printerProfileManager
global fileManager
global slicingManager
global analysisQueue
global userManager
global eventManager
global loginManager
global pluginManager
global appSessionManager
global pluginLifecycleManager
global preemptiveCache
global debug
from tornado.ioloop import IOLoop
from tornado.web import Application, RequestHandler
import sys
debug = self._debug
# first initialize the settings singleton and make sure it uses given configfile and basedir if available
s = settings(init=True, basedir=self._basedir, configfile=self._configfile)
# then monkey patch a bunch of stuff
util.tornado.fix_ioloop_scheduling()
util.flask.enable_additional_translations(additional_folders=[s.getBaseFolder("translations")])
# setup app
self._setup_app(app)
# setup i18n
self._setup_i18n(app)
# then initialize logging
self._setup_logging(self._debug, self._logConf)
self._logger = logging.getLogger(__name__)
def exception_logger(exc_type, exc_value, exc_tb):
self._logger.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_tb))
sys.excepthook = exception_logger
self._logger.info("Starting OctoPrint %s" % DISPLAY_VERSION)
# start the intermediary server
self._start_intermediary_server(s)
# then initialize the plugin manager
pluginManager = octoprint.plugin.plugin_manager(init=True)
printerProfileManager = PrinterProfileManager()
eventManager = events.eventManager()
analysisQueue = octoprint.filemanager.analysis.AnalysisQueue()
slicingManager = octoprint.slicing.SlicingManager(s.getBaseFolder("slicingProfiles"), printerProfileManager)
storage_managers = dict()
storage_managers[octoprint.filemanager.FileDestinations.LOCAL] = octoprint.filemanager.storage.LocalFileStorage(s.getBaseFolder("uploads"))
fileManager = octoprint.filemanager.FileManager(analysisQueue, slicingManager, printerProfileManager, initial_storage_managers=storage_managers)
printer = Printer(fileManager, analysisQueue, printerProfileManager)
appSessionManager = util.flask.AppSessionManager()
pluginLifecycleManager = LifecycleManager(pluginManager)
preemptiveCache = PreemptiveCache(os.path.join(s.getBaseFolder("data"), "preemptive_cache_config.yaml"))
# ... and initialize all plugins
def octoprint_plugin_inject_factory(name, implementation):
"""Factory for injections for all OctoPrintPlugins"""
if not isinstance(implementation, octoprint.plugin.OctoPrintPlugin):
# we only care about OctoPrintPlugins
return None
return dict(
plugin_manager=pluginManager,
printer_profile_manager=printerProfileManager,
event_bus=eventManager,
analysis_queue=analysisQueue,
slicing_manager=slicingManager,
file_manager=fileManager,
printer=printer,
app_session_manager=appSessionManager,
plugin_lifecycle_manager=pluginLifecycleManager,
data_folder=os.path.join(settings().getBaseFolder("data"), name),
preemptive_cache=preemptiveCache
)
def settings_plugin_inject_factory(name, implementation):
"""Factory for additional injections depending on plugin type"""
if not isinstance(implementation, octoprint.plugin.SettingsPlugin):
# we only care about SettingsPlugins
return None
# SettingsPlugin instnances get a PluginSettings instance injected
default_settings = implementation.get_settings_defaults()
get_preprocessors, set_preprocessors = implementation.get_settings_preprocessors()
plugin_settings = octoprint.plugin.plugin_settings(name,
defaults=default_settings,
get_preprocessors=get_preprocessors,
set_preprocessors=set_preprocessors)
return dict(settings=plugin_settings)
def settings_plugin_config_migration_and_cleanup(name, implementation):
"""Take care of migrating and cleaning up any old settings"""
if not isinstance(implementation, octoprint.plugin.SettingsPlugin):
return
settings_version = implementation.get_settings_version()
settings_migrator = implementation.on_settings_migrate
if settings_version is not None and settings_migrator is not None:
stored_version = implementation._settings.get_int([octoprint.plugin.SettingsPlugin.config_version_key])
if stored_version is None or stored_version < settings_version:
settings_migrator(settings_version, stored_version)
implementation._settings.set_int([octoprint.plugin.SettingsPlugin.config_version_key], settings_version)
implementation.on_settings_cleanup()
implementation._settings.save()
implementation.on_settings_initialized()
pluginManager.implementation_inject_factories=[octoprint_plugin_inject_factory, settings_plugin_inject_factory]
pluginManager.initialize_implementations()
settingsPlugins = pluginManager.get_implementations(octoprint.plugin.SettingsPlugin)
for implementation in settingsPlugins:
try:
settings_plugin_config_migration_and_cleanup(implementation._identifier, implementation)
except:
self._logger.exception("Error while trying to migrate settings for plugin {}, ignoring it".format(implementation._identifier))
pluginManager.implementation_post_inits=[settings_plugin_config_migration_and_cleanup]
pluginManager.log_all_plugins()
# initialize file manager and register it for changes in the registered plugins
fileManager.initialize()
pluginLifecycleManager.add_callback(["enabled", "disabled"], lambda name, plugin: fileManager.reload_plugins())
# initialize slicing manager and register it for changes in the registered plugins
slicingManager.initialize()
pluginLifecycleManager.add_callback(["enabled", "disabled"], lambda name, plugin: slicingManager.reload_slicers())
# setup jinja2
self._setup_jinja2()
# make sure plugin lifecycle events relevant for jinja2 are taken care of
def template_enabled(name, plugin):
if plugin.implementation is None or not isinstance(plugin.implementation, octoprint.plugin.TemplatePlugin):
return
self._register_additional_template_plugin(plugin.implementation)
def template_disabled(name, plugin):
if plugin.implementation is None or not isinstance(plugin.implementation, octoprint.plugin.TemplatePlugin):
return
self._unregister_additional_template_plugin(plugin.implementation)
pluginLifecycleManager.add_callback("enabled", template_enabled)
pluginLifecycleManager.add_callback("disabled", template_disabled)
# setup assets
self._setup_assets()
# configure timelapse
octoprint.timelapse.configure_timelapse()
# setup command triggers
events.CommandTrigger(printer)
if self._debug:
events.DebugEventListener()
# setup access control
userManagerName = s.get(["accessControl", "userManager"])
try:
clazz = octoprint.util.get_class(userManagerName)
userManager = clazz()
except AttributeError as e:
self._logger.exception("Could not instantiate user manager {}, falling back to FilebasedUserManager!".format(userManagerName))
userManager = octoprint.users.FilebasedUserManager()
finally:
userManager.enabled = s.getBoolean(["accessControl", "enabled"])
loginManager = LoginManager()
loginManager.session_protection = "strong"
loginManager.user_callback = load_user
if not userManager.enabled:
loginManager.anonymous_user = users.DummyUser
principals.identity_loaders.appendleft(users.dummy_identity_loader)
loginManager.init_app(app)
# register API blueprint
self._setup_blueprints()
## Tornado initialization starts here
if self._host is None:
self._host = s.get(["server", "host"])
if self._port is None:
self._port = s.getInt(["server", "port"])
ioloop = IOLoop()
ioloop.install()
self._router = SockJSRouter(self._create_socket_connection, "/sockjs")
upload_suffixes = dict(name=s.get(["server", "uploads", "nameSuffix"]), path=s.get(["server", "uploads", "pathSuffix"]))
def mime_type_guesser(path):
from octoprint.filemanager import get_mime_type
return get_mime_type(path)
download_handler_kwargs = dict(
as_attachment=True,
allow_client_caching=False
)
additional_mime_types=dict(mime_type_guesser=mime_type_guesser)
admin_validator = dict(access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.admin_validator))
no_hidden_files_validator = dict(path_validation=util.tornado.path_validation_factory(lambda path: not octoprint.util.is_hidden_path(path), status_code=404))
def joined_dict(*dicts):
if not len(dicts):
return dict()
joined = dict()
for d in dicts:
joined.update(d)
return joined
server_routes = self._router.urls + [
# various downloads
(r"/downloads/timelapse/([^/]*\.mp[g4])", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("timelapse")), download_handler_kwargs, no_hidden_files_validator)),
(r"/downloads/files/local/(.*)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("uploads")), download_handler_kwargs, no_hidden_files_validator, additional_mime_types)),
(r"/downloads/logs/([^/]*)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("logs")), download_handler_kwargs, admin_validator)),
# camera snapshot
(r"/downloads/camera/current", util.tornado.UrlProxyHandler, dict(url=s.get(["webcam", "snapshot"]), as_attachment=True, access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.user_validator))),
# generated webassets
(r"/static/webassets/(.*)", util.tornado.LargeResponseHandler, dict(path=os.path.join(s.getBaseFolder("generated"), "webassets"))),
# online indicators - text file with "online" as content and a transparent gif
(r"/online.txt", util.tornado.StaticDataHandler, dict(data="online\n")),
(r"/online.gif", util.tornado.StaticDataHandler, dict(data=bytes(base64.b64decode("R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7")), content_type="image/gif"))
]
# fetch additional routes from plugins
for name, hook in pluginManager.get_hooks("octoprint.server.http.routes").items():
try:
result = hook(list(server_routes))
except:
self._logger.exception("There was an error while retrieving additional server routes from plugin hook {name}".format(**locals()))
else:
if isinstance(result, (list, tuple)):
for entry in result:
if not isinstance(entry, tuple) or not len(entry) == 3:
continue
if not isinstance(entry[0], basestring):
continue
if not isinstance(entry[2], dict):
continue
route, handler, kwargs = entry
route = r"/plugin/{name}/{route}".format(name=name, route=route if not route.startswith("/") else route[1:])
self._logger.debug("Adding additional route {route} handled by handler {handler} and with additional arguments {kwargs!r}".format(**locals()))
server_routes.append((route, handler, kwargs))
server_routes.append((r".*", util.tornado.UploadStorageFallbackHandler, dict(fallback=util.tornado.WsgiInputContainer(app.wsgi_app), file_prefix="octoprint-file-upload-", file_suffix=".tmp", suffixes=upload_suffixes)))
self._tornado_app = Application(server_routes)
max_body_sizes = [
("POST", r"/api/files/([^/]*)", s.getInt(["server", "uploads", "maxSize"])),
("POST", r"/api/languages", 5 * 1024 * 1024)
]
# allow plugins to extend allowed maximum body sizes
for name, hook in pluginManager.get_hooks("octoprint.server.http.bodysize").items():
try:
result = hook(list(max_body_sizes))
except:
self._logger.exception("There was an error while retrieving additional upload sizes from plugin hook {name}".format(**locals()))
else:
if isinstance(result, (list, tuple)):
for entry in result:
if not isinstance(entry, tuple) or not len(entry) == 3:
continue
if not entry[0] in util.tornado.UploadStorageFallbackHandler.BODY_METHODS:
continue
if not isinstance(entry[2], int):
continue
method, route, size = entry
route = r"/plugin/{name}/{route}".format(name=name, route=route if not route.startswith("/") else route[1:])
self._logger.debug("Adding maximum body size of {size}B for {method} requests to {route})".format(**locals()))
max_body_sizes.append((method, route, size))
self._stop_intermediary_server()
# initialize and bind the server
self._server = util.tornado.CustomHTTPServer(self._tornado_app, max_body_sizes=max_body_sizes, default_max_body_size=s.getInt(["server", "maxSize"]))
self._server.listen(self._port, address=self._host)
eventManager.fire(events.Events.STARTUP)
# auto connect
if s.getBoolean(["serial", "autoconnect"]):
(port, baudrate) = s.get(["serial", "port"]), s.getInt(["serial", "baudrate"])
printer_profile = printerProfileManager.get_default()
connectionOptions = get_connection_options()
if port in connectionOptions["ports"]:
printer.connect(port=port, baudrate=baudrate, profile=printer_profile["id"] if "id" in printer_profile else "_default")
# start up watchdogs
if s.getBoolean(["feature", "pollWatched"]):
# use less performant polling observer if explicitely configured
observer = PollingObserver()
else:
# use os default
observer = Observer()
observer.schedule(util.watchdog.GcodeWatchdogHandler(fileManager, printer), s.getBaseFolder("watched"))
observer.start()
# run our startup plugins
octoprint.plugin.call_plugin(octoprint.plugin.StartupPlugin,
"on_startup",
args=(self._host, self._port))
def call_on_startup(name, plugin):
implementation = plugin.get_implementation(octoprint.plugin.StartupPlugin)
if implementation is None:
return
implementation.on_startup(self._host, self._port)
pluginLifecycleManager.add_callback("enabled", call_on_startup)
# prepare our after startup function
def on_after_startup():
self._logger.info("Listening on http://%s:%d" % (self._host, self._port))
# now this is somewhat ugly, but the issue is the following: startup plugins might want to do things for
# which they need the server to be already alive (e.g. for being able to resolve urls, such as favicons
# or service xmls or the like). While they are working though the ioloop would block. Therefore we'll
# create a single use thread in which to perform our after-startup-tasks, start that and hand back
# control to the ioloop
def work():
octoprint.plugin.call_plugin(octoprint.plugin.StartupPlugin,
"on_after_startup")
def call_on_after_startup(name, plugin):
implementation = plugin.get_implementation(octoprint.plugin.StartupPlugin)
if implementation is None:
return
implementation.on_after_startup()
pluginLifecycleManager.add_callback("enabled", call_on_after_startup)
# when we are through with that we also run our preemptive cache
if settings().getBoolean(["devel", "cache", "preemptive"]):
self._execute_preemptive_flask_caching(preemptiveCache)
import threading
threading.Thread(target=work).start()
ioloop.add_callback(on_after_startup)
# prepare our shutdown function
def on_shutdown():
# will be called on clean system exit and shutdown the watchdog observer and call the on_shutdown methods
# on all registered ShutdownPlugins
self._logger.info("Shutting down...")
observer.stop()
observer.join()
octoprint.plugin.call_plugin(octoprint.plugin.ShutdownPlugin,
"on_shutdown")
if self._octoprint_daemon is not None:
self._logger.info("Cleaning up daemon pidfile")
self._octoprint_daemon.terminated()
self._logger.info("Goodbye!")
atexit.register(on_shutdown)
def sigterm_handler(*args, **kwargs):
# will stop tornado on SIGTERM, making the program exit cleanly
def shutdown_tornado():
ioloop.stop()
ioloop.add_callback_from_signal(shutdown_tornado)
signal.signal(signal.SIGTERM, sigterm_handler)
try:
# this is the main loop - as long as tornado is running, OctoPrint is running
ioloop.start()
except (KeyboardInterrupt, SystemExit):
pass
except:
self._logger.fatal("Now that is embarrassing... Something really really went wrong here. Please report this including the stacktrace below in OctoPrint's bugtracker. Thanks!")
self._logger.exception("Stacktrace follows:")
def run(parser, args):
args.tomlfile = args.toml
args.toml = toml.load(args.toml)
print(args)
# TODO: Move logging config to separate configuration file
# set up logging to file
logging.basicConfig(
level=logging.DEBUG,
format='%(levelname)s::%(asctime)s::%(name)s::%(message)s',
filename=args.log_file,
filemode='w')
# define a Handler that writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-15s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger('').addHandler(console)
# Start by logging sys.argv and the parameters used
logger = logging.getLogger("Manager")
logger.info(" ".join(sys.argv))
print_args(args, logger=logger)
logger.info("Initialising iterAlign.")
logger.info("Setting up FastQ monitoring.")
#### Check if a run is active - if not, wait.
args.simulation = True
connection = None
if args.watch is None:
args.simulation = False
logger.info("Creating rpc connection for device {}.".format(
args.device))
try:
connection, messageport = get_rpc_connection(args.device)
except ValueError as e:
print(e)
sys.exit(1)
send_message(connection, "Iteralign Connected to MinKNOW",
Severity.WARN)
logger.info("Loaded RPC")
while parse_message(connection.acquisition.current_status()
)['status'] != "PROCESSING":
time.sleep(1)
#### Check if we know where data is being written to , if not... wait
args.watch = parse_message(connection.acquisition.get_acquisition_info(
))['config_summary']['reads_directory']
else:
messageport = ""
event_handler = FastqHandler(args, logging, messageport, connection)
# This block handles the fastq
observer = Observer()
observer.schedule(event_handler, path=args.watch, recursive=True)
observer.daemon = True
try:
observer.start()
logger.info("FastQ Monitoring Running.")
while 1:
time.sleep(1)
except KeyboardInterrupt:
logger.info("Exiting - Will take a few seconds to clean up!")
observer.stop()
observer.join()
os._exit(0)
class AiidaLabAppWatch:
"""Watch to monitor the app installation status.
Create a watch instance to monitor the installation status of an
AiiDAlab app. This is achieved by monitoring the app repository
for existance and changes.
Arguments:
app (AiidaLabApp):
The AiidaLab app to monitor.
"""
class AppPathFileSystemEventHandler(FileSystemEventHandler):
"""Internal event handeler for app path file system events."""
def __init__(self, app):
self.app = app
def on_any_event(self, event):
"""Refresh app for any event."""
self.app.refresh_async()
def __init__(self, app):
self.app = app
self._started = False
self._monitor_thread = None
self._observer = None
def __repr__(self):
return f"<{type(self).__name__}(app={self.app!r})>"
def _start_observer(self):
"""Start the directory observer thread.
The ._observer thread is controlled by the ._monitor_thread.
"""
assert os.path.isdir(self.app.path)
assert self._observer is None or not self._observer.isAlive()
event_handler = self.AppPathFileSystemEventHandler(self.app)
self._observer = Observer()
self._observer.schedule(event_handler, self.app.path, recursive=True)
try:
self._observer.start()
except OSError as error:
if error.errno in (errno.ENOSPC,
errno.EMFILE) and 'inotify' in str(error):
# We reached the inotify watch limit, using polling-based fallback observer.
self._observer = PollingObserver()
self._observer.schedule(event_handler,
self.app.path,
recursive=True)
self._observer.start()
else: # reraise unrelated error
raise error
def _stop_observer(self):
"""Stop the directory observer thread.
The ._observer thread is controlled by the ._monitor_thread.
"""
assert self._observer is not None
self._observer.stop()
def start(self):
"""Watch the app repository for file system events.
The app state is refreshed automatically for all events.
"""
if self._started:
raise RuntimeError(
f"Instances of {type(self).__name__} can only be started once."
)
if self._monitor_thread is None:
def check_path_exists_changed():
is_dir = os.path.isdir(self.app.path)
while not self._monitor_thread.stop_flag:
switched = is_dir != os.path.isdir(self.app.path)
if switched:
is_dir = not is_dir
self.app.refresh()
if is_dir:
if self._observer is None or not self._observer.isAlive(
):
self._start_observer()
elif self._observer and self._observer.isAlive():
self._stop_observer()
sleep(1)
# stop-flag set, stopping observer...
if self._observer:
self._observer.stop()
self._monitor_thread = Thread(target=check_path_exists_changed)
self._monitor_thread.stop_flag = False
self._monitor_thread.start()
self._started = True
def stop(self):
"""Stop watching the app repository for file system events."""
if self._monitor_thread is not None:
self._monitor_thread.stop_flag = True
def is_alive(self):
"""Return True if this watch is still alive."""
return self._monitor_thread and self._monitor_thread.is_alive()
def join(self, timeout=None):
"""Join the watch after stopping.
This function will timeout if a timeout argument is provided. Use the
is_alive() function to determien whether the watch was stopped within
the given timout.
"""
if self._monitor_thread is not None:
self._monitor_thread.join(timeout=timeout)
def main(argv):
"""
Build the docs and serve them with an HTTP server.
"""
parser = argparse.ArgumentParser(description='Build and serve HTML Sphinx docs')
parser.add_argument(
'--port',
help='Serve on this port, default 8000',
type=int,
default=8000)
parser.add_argument(
'--source',
help='Directory of source Sphinx (reStructuredText) docs',
type=os.path.realpath,
default='docs/source')
parser.add_argument(
'--destination',
help='Where to build the HTML output',
type=os.path.realpath,
default='docs/build/html')
parser.add_argument(
'--doctrees',
help='Where the doctrees are built',
type=os.path.realpath,
default='docs/build/doctrees')
options = parser.parse_args(argv)
bound_build_docs = partial(build_docs, options.source, options.destination, options.doctrees)
# Do the initial build
bound_build_docs()
# Watch the source directory for changes, build docs again if detected
observer = Observer()
observer.schedule(
BuildDocsHandler(bound_build_docs),
path=options.source, recursive=True)
observer.start()
# Set the root for the request handler, overriding Python stdlib current
# working directory.
DocsHTTPRequestHandler._root = options.destination
server = SocketServer.TCPServer(
('', options.port),
DocsHTTPRequestHandler)
try:
logger.info('Serving on localhost:{}'.format(options.port))
server.serve_forever()
except KeyboardInterrupt:
sys.stdout.write('\n')
logger.info('(stopping server)')
observer.stop()
finally:
observer.join()
logging.info('Server stopped, exiting')
sys.exit(0)
class AutoOcrScheduler(object):
SINGLE_FOLDER = 'single_folder'
MIRROR_TREE = 'mirror_tree'
OUTPUT_MODES = [SINGLE_FOLDER, MIRROR_TREE]
def __init__(
self,
config_dir,
input_dir,
output_dir,
output_mode,
success_action=OcrTask.ON_SUCCESS_DO_NOTHING,
archive_dir=None,
notify_url='',
process_existing_files=False,
run_scheduler=True,
polling_observer=False,
):
self.logger = logger.getChild('scheduler')
self.config_dir = local.path(config_dir)
self.input_dir = local.path(input_dir)
self.output_dir = local.path(output_dir)
if self.input_dir == self.output_dir:
raise AutoOcrSchedulerError('Invalid configuration. Input and output directories must not be the same to avoid recursive OCR invocation!')
self.output_mode = output_mode.lower()
if self.output_mode not in AutoOcrScheduler.OUTPUT_MODES:
raise AutoOcrSchedulerError('Invalid output mode: {}. Must be one of: {}'.format(self.output_mode, ', '.join(AutoOcrScheduler.OUTPUT_MODES)))
self.success_action = success_action.lower()
if self.success_action not in OcrTask.SUCCESS_ACTIONS:
raise AutoOcrSchedulerError('Invalid success action: {}. Must be one of {}'.format(self.success_action, ', '.join(OcrTask.SUCCESS_ACTIONS)))
self.archive_dir = local.path(archive_dir) if archive_dir else None
if self.success_action == OcrTask.ON_SUCCESS_ARCHIVE and not self.archive_dir:
raise AutoOcrSchedulerError('Archive directory required for success action {}'.format(self.success_action))
self.notify_url = notify_url
self.current_tasks = {}
self.walk_existing_task = None
self.current_outputs = set()
# Create a Threadpool to run OCR tasks on
self.threadpool = ThreadPoolExecutor(max_workers=3)
# Wire up an AutoOcrWatchdogHandler
watchdog_handler = AutoOcrWatchdogHandler(self.on_file_touched, self.on_file_deleted)
# Schedule watchdog to observe the input directory
if run_scheduler:
self.observer = PollingObserver() if polling_observer else Observer()
self.observer.schedule(watchdog_handler, self.input_dir, recursive=True)
self.observer.start()
self.logger.warning('Watching %s', self.input_dir)
else:
self.observer = None
self.logger.warning('Not watching %s', self.input_dir)
# Process existing files in input directory, if requested
if process_existing_files:
self.walk_existing_task = self.threadpool.submit(self.walk_existing_files)
def shutdown(self):
# Shut down the feed of incoming watchdog events
if self.observer:
self.logger.debug('Shutting down filesystem watchdog...')
self.observer.unschedule_all()
self.observer.stop()
# Cancel all outstanding cancelable tasks
if self.walk_existing_task:
self.logger.debug('Canceling walk existing files task...')
self.walk_existing_task.cancel()
self.logger.debug('Canceling all %d in-flight tasks...', len(self.current_tasks))
tasks = [task for _, task in self.current_tasks.items()]
for task in tasks:
task.cancel()
# Wait for the threadpool to clean up
if self.threadpool:
self.logger.debug('Shutting down threadpool...')
self.threadpool.shutdown()
self.threadpool = None
# Wait for the watchdog to clean up
if self.observer:
self.logger.debug('Cleaning up filesystem watchdog...')
self.observer.join()
self.observer = None
def __enter__(self):
return self
def __exit__(self, *args):
self.shutdown()
return False
def _map_output_path(self, input_path):
if self.output_mode == AutoOcrScheduler.MIRROR_TREE:
return self.output_dir / (input_path - self.input_dir)
else:
assert self.output_mode == AutoOcrScheduler.SINGLE_FOLDER
output_path = self.output_dir / (input_path.name)
unique = 1
if output_path.exists() or output_path in self.current_outputs:
suffix = '.{}.{}{}'.format(datetime.now().strftime('%Y%m%d'), unique, output_path.suffix)
output_path = output_path.with_suffix(suffix)
while output_path.exists() or output_path in self.current_outputs:
unique = unique + 1
output_path = output_path.with_suffix('.{}{}'.format(unique, output_path.suffix), depth=2)
return output_path
def _map_archive_path(self, input_path):
return self.archive_dir / (input_path - self.input_dir)
def _get_config_path(self, input_path):
assert (input_path - self.input_dir)[0] != '..'
config_path = input_path.parent / 'ocr.config'
while True:
if config_path.exists():
return config_path
if config_path.parent == self.input_dir:
break
config_path = config_path.parent.parent / 'ocr.config'
config_path = self.config_dir / 'ocr.config'
if config_path.exists():
return config_path
return None
def queue_path(self, path):
output_path = self._map_output_path(path)
config_file = self._get_config_path(path)
archive_file = self._map_archive_path(path)
task = OcrTask(path,
output_path,
self.threadpool.submit,
self.on_task_done,
config_file=config_file,
success_action=self.success_action,
archive_path=archive_file,
notify_url=self.notify_url)
self.current_tasks[path] = task
self.current_outputs.add(output_path)
def walk_existing_files(self):
self.logger.debug('Enumerating existing input files...')
def keep_file(file):
return any([fnmatch.fnmatch(file, pattern) for pattern in AutoOcrWatchdogHandler.MATCH_PATTERNS])
for file in self.input_dir.walk(filter=keep_file):
self.on_file_touched(file)
self.walk_existing_task = None
def on_file_touched(self, path):
if path in self.current_tasks:
self.current_tasks[path].touch()
else:
self.queue_path(path)
def on_file_deleted(self, path):
if path in self.current_tasks:
self.current_tasks[path].cancel()
def on_task_done(self, task):
self.current_outputs.remove(task.output_path)
del self.current_tasks[task.input_path]
def wait_for_idle(self):
if self.walk_existing_task:
self.logger.debug('Waiting for walk existing files to complete...')
concurrent.futures.wait([self.walk_existing_task])
while self.current_tasks:
self.logger.debug('Waiting for %d tasks to complete...', len(self.current_tasks))
concurrent.futures.wait([task.future for _, task in self.current_tasks.items()])
class KeywordTable(object):
"""A SQLite database of keywords"""
def __init__(self, dbfile=":memory:", poll=False):
self.db = sqlite3.connect(dbfile, check_same_thread=False)
self.log = logging.getLogger(__name__)
self._create_db()
# self.log.warning("I'm warnin' ya!")
# set up watchdog observer to monitor changes to
# keyword files (or more correctly, to directories
# of keyword files)
self.observer = PollingObserver() if poll else Observer()
self.observer.start()
def add(self, name, monitor=True):
"""Add a folder, library (.py) or resource file (.robot, .tsv, .txt, .resource) to the database
"""
if os.path.isdir(name):
if (not os.path.basename(name).startswith(".")):
self.add_folder(name)
elif os.path.isfile(name):
if ((self._looks_like_resource_file(name))
or (self._looks_like_libdoc_file(name))
or (self._looks_like_library_file(name))):
self.add_file(name)
else:
# let's hope it's a library name!
self.add_library(name)
def on_change(self, path, event_type):
"""Respond to changes in the file system
This method will be given the path to a file that
has changed on disk. We need to reload the keywords
from that file
"""
# I can do all this work in a sql statement, but
# for debugging it's easier to do it in stages.
sql = """SELECT collection_id
FROM collection_table
WHERE path == ?
"""
cursor = self._execute(sql, (path, ))
results = cursor.fetchall()
# there should always be exactly one result, but
# there's no harm in using a loop to process the
# single result
for result in results:
collection_id = result[0]
# remove all keywords in this collection
sql = """DELETE from keyword_table
WHERE collection_id == ?
"""
cursor = self._execute(sql, (collection_id, ))
self._load_keywords(collection_id, path=path)
def _load_keywords(self, collection_id, path=None, libdoc=None):
"""Load a collection of keywords
One of path or libdoc needs to be passed in...
"""
if libdoc is None and path is None:
raise (
Exception("You must provide either a path or libdoc argument"))
if libdoc is None:
libdoc = LibraryDocumentation(path)
if len(libdoc.keywords) > 0:
for keyword in libdoc.keywords:
self._add_keyword(collection_id, keyword.name, keyword.doc,
keyword.args)
def add_file(self, path):
"""Add a resource file or library file to the database"""
libdoc = LibraryDocumentation(path)
if len(libdoc.keywords) > 0:
# if libdoc.doc.startswith("Documentation for resource file"):
# bah! The file doesn't have an file-level documentation
# and libdoc substitutes some placeholder text.
# libdoc.doc = ""
collection_id = self.add_collection(path, libdoc.name, libdoc.type,
libdoc.doc, libdoc.version,
libdoc.scope)
self._load_keywords(collection_id, libdoc=libdoc)
def add_library(self, name):
"""Add a library to the database
This method is for adding a library by name (eg: "BuiltIn")
rather than by a file.
"""
libdoc = LibraryDocumentation(name)
if len(libdoc.keywords) > 0:
# FIXME: figure out the path to the library file
collection_id = self.add_collection(None, libdoc.name, libdoc.type,
libdoc.doc, libdoc.version,
libdoc.scope)
self._load_keywords(collection_id, libdoc=libdoc)
def add_folder(self, dirname, watch=True):
"""Recursively add all files in a folder to the database
By "all files" I mean, "all files that are resource files
or library files". It will silently ignore files that don't
look like they belong in the database. Pity the fool who
uses non-standard suffixes.
N.B. folders with names that begin with '." will be skipped
"""
ignore_file = os.path.join(dirname, ".rfhubignore")
exclude_patterns = []
try:
with open(ignore_file, "r") as f:
exclude_patterns = []
for line in f.readlines():
line = line.strip()
if (re.match(r'^\s*#', line)): continue
if len(line.strip()) > 0:
exclude_patterns.append(line)
except:
# should probably warn the user?
pass
for filename in os.listdir(dirname):
path = os.path.join(dirname, filename)
(basename, ext) = os.path.splitext(filename.lower())
try:
if (os.path.isdir(path)):
if (not basename.startswith(".")):
if os.access(path, os.R_OK):
self.add_folder(path, watch=False)
else:
if (ext in (".xml", ".robot", ".txt", ".py", ".tsv",
".resource")):
if os.access(path, os.R_OK):
self.add(path)
except Exception as e:
# I really need to get the logging situation figured out.
print("bummer:", str(e))
# FIXME:
# instead of passing a flag around, I should just keep track
# of which folders we're watching, and don't add wathers for
# any subfolders. That will work better in the case where
# the user accidentally starts up the hub giving the same
# folder, or a folder and it's children, on the command line...
if watch:
# add watcher on normalized path
dirname = os.path.abspath(dirname)
event_handler = WatchdogHandler(self, dirname)
self.observer.schedule(event_handler, dirname, recursive=True)
def add_collection(self,
path,
c_name,
c_type,
c_doc,
c_version="unknown",
c_scope="",
c_namedargs="yes",
c_doc_format="ROBOT"):
"""Insert data into the collection table"""
if path is not None:
# We want to store the normalized form of the path in the
# database
path = os.path.abspath(path)
cursor = self.db.cursor()
cursor.execute(
"""
INSERT INTO collection_table
(name, type, version, scope, namedargs, path, doc, doc_format)
VALUES
(?,?,?,?,?,?,?,?)
""", (c_name, c_type, c_version, c_scope, c_namedargs, path, c_doc,
c_doc_format))
collection_id = cursor.lastrowid
return collection_id
def add_installed_libraries(
self,
extra_libs=["SeleniumLibrary", "SudsLibrary", "RequestsLibrary"]):
"""Add any installed libraries that we can find
We do this by looking in the `libraries` folder where
robot is installed. If you have libraries installed
in a non-standard place, this won't pick them up.
"""
libdir = os.path.dirname(robot.libraries.__file__)
loaded = []
for filename in os.listdir(libdir):
if filename.endswith(".py") or filename.endswith(".pyc"):
libname, ext = os.path.splitext(filename)
if (libname.lower() not in loaded
and not self._should_ignore(libname)):
try:
self.add(libname)
loaded.append(libname.lower())
except Exception as e:
# need a better way to log this...
self.log.debug("unable to add library: " + str(e))
# I hate how I implemented this, but I don't think there's
# any way to find out which installed python packages are
# robot libraries.
for library in extra_libs:
if (library.lower() not in loaded
and not self._should_ignore(library)):
try:
self.add(library)
loaded.append(library.lower())
except Exception as e:
self.log.debug("unable to add external library %s: %s" % \
(library, str(e)))
def get_collection(self, collection_id):
"""Get a specific collection"""
sql = """SELECT collection.collection_id, collection.type,
collection.name, collection.path,
collection.doc,
collection.version, collection.scope,
collection.namedargs,
collection.doc_format
FROM collection_table as collection
WHERE collection_id == ? OR collection.name like ?
"""
cursor = self._execute(sql, (collection_id, collection_id))
# need to handle the case where we get more than one result...
sql_result = cursor.fetchone()
return {
"collection_id": sql_result[0],
"type": sql_result[1],
"name": sql_result[2],
"path": sql_result[3],
"doc": sql_result[4],
"version": sql_result[5],
"scope": sql_result[6],
"namedargs": sql_result[7],
"doc_format": sql_result[8]
}
return sql_result
def get_collections(self, pattern="*", libtype="*"):
"""Returns a list of collection name/summary tuples"""
sql = """SELECT collection.collection_id, collection.name, collection.doc,
collection.type, collection.path
FROM collection_table as collection
WHERE name like ?
AND type like ?
ORDER BY collection.name
"""
cursor = self._execute(
sql, (self._glob_to_sql(pattern), self._glob_to_sql(libtype)))
sql_result = cursor.fetchall()
return [{
"collection_id": result[0],
"name": result[1],
"synopsis": result[2].split("\n")[0],
"type": result[3],
"path": result[4]
} for result in sql_result]
def get_keyword_data(self, collection_id):
sql = """SELECT keyword.keyword_id, keyword.name, keyword.args, keyword.doc
FROM keyword_table as keyword
WHERE keyword.collection_id == ?
ORDER BY keyword.name
"""
cursor = self._execute(sql, (collection_id, ))
return cursor.fetchall()
def get_keyword(self, collection_id, name):
"""Get a specific keyword from a library"""
sql = """SELECT keyword.name, keyword.args, keyword.doc
FROM keyword_table as keyword
WHERE keyword.collection_id == ?
AND keyword.name like ?
"""
cursor = self._execute(sql, (collection_id, name))
# We're going to assume no library has duplicate keywords
# While that in theory _could_ happen, it never _should_,
# and you get what you deserve if it does.
row = cursor.fetchone()
if row is not None:
return {
"name": row[0],
"args": json.loads(row[1]),
"doc": row[2],
"collection_id": collection_id
}
return {}
def get_keyword_hierarchy(self, pattern="*"):
"""Returns all keywords that match a glob-style pattern
The result is a list of dictionaries, sorted by collection
name.
The pattern matching is insensitive to case. The function
returns a list of (library_name, keyword_name,
keyword_synopsis tuples) sorted by keyword name
"""
sql = """SELECT collection.collection_id, collection.name, collection.path,
keyword.name, keyword.doc
FROM collection_table as collection
JOIN keyword_table as keyword
WHERE collection.collection_id == keyword.collection_id
AND keyword.name like ?
ORDER by collection.name, collection.collection_id, keyword.name
"""
cursor = self._execute(sql, (self._glob_to_sql(pattern), ))
libraries = []
current_library = None
for row in cursor.fetchall():
(c_id, c_name, c_path, k_name, k_doc) = row
if c_id != current_library:
current_library = c_id
libraries.append({
"name": c_name,
"collection_id": c_id,
"keywords": [],
"path": c_path
})
libraries[-1]["keywords"].append({"name": k_name, "doc": k_doc})
return libraries
def search(self, pattern="*", mode="both"):
"""Perform a pattern-based search on keyword names and documentation
The pattern matching is insensitive to case. The function
returns a list of tuples of the form library_id, library_name,
keyword_name, keyword_synopsis, sorted by library id,
library name, and then keyword name
If a pattern begins with "name:", only the keyword names will
be searched. Otherwise, the pattern is searched for in both
the name and keyword documentation.
You can limit the search to a single library by specifying
"in:" followed by the name of the library or resource
file. For example, "screenshot in:SeleniumLibrary" will only
search for the word 'screenshot' in the SeleniumLibrary.
"""
pattern = self._glob_to_sql(pattern)
COND = "(keyword.name like ? OR keyword.doc like ?)"
args = [pattern, pattern]
if mode == "name":
COND = "(keyword.name like ?)"
args = [
pattern,
]
sql = """SELECT collection.collection_id, collection.name, keyword.name, keyword.doc
FROM collection_table as collection
JOIN keyword_table as keyword
WHERE collection.collection_id == keyword.collection_id
AND %s
ORDER by collection.collection_id, collection.name, keyword.name
""" % COND
cursor = self._execute(sql, args)
result = [(row[0], row[1], row[2], row[3].strip().split("\n")[0])
for row in cursor.fetchall()]
return list(set(result))
def get_keywords(self, pattern="*"):
"""Returns all keywords that match a glob-style pattern
The pattern matching is insensitive to case. The function
returns a list of (library_name, keyword_name,
keyword_synopsis tuples) sorted by keyword name
"""
sql = """SELECT collection.collection_id, collection.name,
keyword.name, keyword.doc, keyword.args
FROM collection_table as collection
JOIN keyword_table as keyword
WHERE collection.collection_id == keyword.collection_id
AND keyword.name like ?
ORDER by collection.name, keyword.name
"""
pattern = self._glob_to_sql(pattern)
cursor = self._execute(sql, (pattern, ))
result = [(row[0], row[1], row[2], row[3], row[4])
for row in cursor.fetchall()]
return list(sorted(set(result), key=itemgetter(2)))
def reset(self):
"""Remove all data from the database, but leave the tables intact"""
self._execute("DELETE FROM collection_table")
self._execute("DELETE FROM keyword_table")
def _looks_like_library_file(self, name):
return name.endswith(".py")
def _looks_like_libdoc_file(self, name):
"""Return true if an xml file looks like a libdoc file"""
# inefficient since we end up reading the file twice,
# but it's fast enough for our purposes, and prevents
# us from doing a full parse of files that are obviously
# not libdoc files
if name.lower().endswith(".xml"):
with open(name, "r") as f:
# read the first few lines; if we don't see
# what looks like libdoc data, return false
data = f.read(200)
index = data.lower().find("<keywordspec ")
if index > 0:
return True
return False
def _looks_like_resource_file(self, name):
"""Return true if the file has a keyword table but not a testcase table"""
# inefficient since we end up reading the file twice,
# but it's fast enough for our purposes, and prevents
# us from doing a full parse of files that are obviously
# not robot files
if (re.search(r'__init__.(txt|robot|html|tsv)$', name)):
# These are initialize files, not resource files
return False
found_keyword_table = False
if (name.lower().endswith(".robot") or name.lower().endswith(".txt")
or name.lower().endswith(".tsv")
or name.lower().endswith(".resource")):
with open(name, "r") as f:
data = f.read()
for match in re.finditer(
r'^\*+\s*(Test Cases?|(?:User )?Keywords?)', data,
re.MULTILINE | re.IGNORECASE):
if (re.match(r'Test Cases?', match.group(1),
re.IGNORECASE)):
# if there's a test case table, it's not a keyword file
return False
if (not found_keyword_table
and re.match(r'(User )?Keywords?', match.group(1),
re.IGNORECASE)):
found_keyword_table = True
return found_keyword_table
def _should_ignore(self, name):
"""Return True if a given library name should be ignored
This is necessary because not all files we find in the library
folder are libraries. I wish there was a public robot API
for "give me a list of installed libraries"...
"""
_name = name.lower()
return (_name.startswith("deprecated") or _name.startswith("_")
or _name in ("remote", "reserved", "dialogs_py", "dialogs_ipy",
"dialogs_jy"))
def _execute(self, *args):
"""Execute an SQL query
This exists because I think it's tedious to get a cursor and
then use a cursor.
"""
cursor = self.db.cursor()
cursor.execute(*args)
return cursor
def _add_keyword(self, collection_id, name, doc, args):
"""Insert data into the keyword table
'args' should be a list, but since we can't store a list in an
sqlite database we'll make it json we can can convert it back
to a list later.
"""
argstring = json.dumps(args.argument_names)
self.db.execute(
"""
INSERT INTO keyword_table
(collection_id, name, doc, args)
VALUES
(?,?,?,?)
""", (collection_id, name, doc, argstring))
def _create_db(self):
if not self._table_exists("collection_table"):
self.db.execute("""
CREATE TABLE collection_table
(collection_id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT COLLATE NOCASE,
type COLLATE NOCASE,
version TEXT,
scope TEXT,
namedargs TEXT,
path TEXT,
doc TEXT,
doc_format TEXT)
""")
self.db.execute("""
CREATE INDEX collection_index
ON collection_table (name)
""")
if not self._table_exists("keyword_table"):
self.db.execute("""
CREATE TABLE keyword_table
(keyword_id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT COLLATE NOCASE,
collection_id INTEGER,
doc TEXT,
args TEXT)
""")
self.db.execute("""
CREATE INDEX keyword_index
ON keyword_table (name)
""")
def _glob_to_sql(self, string):
"""Convert glob-like wildcards to SQL wildcards
* becomes %
? becomes _
% becomes \%
\\ remains \\
\* remains \*
\? remains \?
This also adds a leading and trailing %, unless the pattern begins with
^ or ends with $
"""
# What's with the chr(1) and chr(2) nonsense? It's a trick to
# hide \* and \? from the * and ? substitutions. This trick
# depends on the substitutiones being done in order. chr(1)
# and chr(2) were picked because I know those characters
# almost certainly won't be in the input string
table = ((r'\\', chr(1)), (r'\*', chr(2)), (r'\?', chr(3)),
(r'%', r'\%'), (r'?', '_'), (r'*', '%'), (chr(1), r'\\'),
(chr(2), r'\*'), (chr(3), r'\?'))
for (a, b) in table:
string = string.replace(a, b)
string = string[1:] if string.startswith("^") else "%" + string
string = string[:-1] if string.endswith("$") else string + "%"
return string
def _table_exists(self, name):
cursor = self.db.execute("""
SELECT name FROM sqlite_master
WHERE type='table' AND name='%s'
""" % name)
return len(cursor.fetchall()) > 0
def tail_like(path):
observer = PollingObserver()
handler = TailHandler(path)
observer.schedule(handler, dirname(path))
observer.start()
current_path = path
# 末尾へ seek
current_pos = os.stat(path)[6]
current_file = open(path, 'r')
try:
buffer = []
old_unix_time = 0
while True:
current_file.seek(current_pos)
data = ''
for block in iter(lambda: current_file.read(32), ''):
data += block
current_pos = current_file.tell()
if current_path != handler.path:
print('change path')
current_path = handler.path
current_pos = os.stat(current_path)[6]
current_file = open(current_path, 'r')
if data == '':
continue
rows = data.split('\n')
for row in rows:
columns = row.split(',')
unix_time = columns[0]
if row == '':
pass
elif unix_time != '' and old_unix_time == 0:
pass
elif float(unix_time) > old_unix_time:
send_data(buffer)
old_unix_time = float(unix_time)
# print(buffer)
buffer = []
buffer.append(MakeTrajectoryData(columns))
else:
buffer.append(MakeTrajectoryData(columns))
if unix_time != '':
old_unix_time = float(unix_time)
time.sleep(READ_INTERVAL)
except KeyboardInterrupt:
observer.stop()
finally:
handler.close()
observer.join()
def start(self):
gevent.spawn(self._cleanScheduledJobs, 3600*24)
observer = Observer()
handler = JumpscriptHandler(self)
observer.schedule(handler, "jumpscripts", recursive=True)
observer.start()
def run(parser, args):
# new code block: change the reference path within the args.toml file into the args.mindex path
d = toml.load(args.toml)
print(d["conditions"]["reference"])
args.tomlfile = args.toml
args.toml = toml.load(args.toml)
print(args)
# TODO: Move logging config to separate configuration file
# set up logging to file
logging.basicConfig(level=logging.DEBUG,
format='%(levelname)s::%(asctime)s::%(name)s::%(message)s',
filename=args.log_file,
filemode='w')
# define a Handler that writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-15s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger('').addHandler(console)
# Start by logging sys.argv and the parameters used
logger = logging.getLogger("Manager")
logger.info(" ".join(sys.argv))
print_args(args, logger=logger)
logger.info("Initialising iterAlign.")
logger.info("Setting up FastQ monitoring.")
#### Check if a run is active - if not, wait.
args.simulation = True
connection = None
#set default message severity level.
severity = 2
if args.watch is None:
args.simulation = False
logger.info("Creating rpc connection for device {}.".format(args.device))
try:
connection, messageport = get_rpc_connection(args.device)
except ValueError as e:
print(e)
sys.exit(1)
#send_message_port("Iteralign Connected to MinKNOW", args.host, messageport)
send_message(connection, "Iteralign Connected to MinKNOW.", Severity.WARN)
logger.info("Loaded RPC")
while parse_message(connection.acquisition.current_status())['status'] != "PROCESSING":
time.sleep(1)
### Check if we know where data is being written to , if not... wait
args.watch = parse_message(connection.acquisition.get_acquisition_info())['config_summary'][
'reads_directory']
else:
messageport = ""
event_handler = FastqHandler(args, logging, messageport, connection)
# This block handles the fastq
observer = Observer()
observer.schedule(event_handler, path=args.watch, recursive=True)
observer.daemon = True
try:
observer.start()
logger.info("FastQ Monitoring Running.")
while 1:
time.sleep(1)
except KeyboardInterrupt:
logger.info("Exiting - Will take a few seconds to clean up!")
observer.stop()
observer.join()
if args.keepfiles:
logging.info("The 'keepfiles' argument was set, files generated by classifier have been retained")
else:
if os.path.isdir(args.path):
for path, dirs, files in os.walk(args.path):
for f in files:
if f.startswith(args.prefix):
os.unlink(f)
logging.info("file removed: {}".format(f))
if os.path.isdir("./"):
for path, dirs, files in os.walk("./"):
for f in files:
if f.endswith(args.creport):
os.unlink(f)
logging.info("file removed: {}".format(f))
logging.info("All files generated by classifier have been removed.")
os._exit(0)
'event_type': event.event_type
}
if event.event_type != events.EVENT_TYPE_DELETED:
data['type'] = magic.from_file(event.src_path, mime=True)
requests.post(
'http://%s:%s/items' % (APP_HOST, APP_PORT),
data=data
)
def on_modified(self, event):
self.process(event)
def on_created(self, event):
self.process(event)
def on_deleted(self, event):
self.process(event)
if __name__ == '__main__':
args = sys.argv[1:]
observer = Observer()
observer.schedule(MyHandler(), path=args[0] if args else '../watch_here')
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
class Config:
def __init__(self, args, callback=None):
self._watchdog_observer = None
self._path = args.config
self._callback = callback
self._config = {}
# Load from file
try:
self.load_config_file()
except FileNotFoundError:
logger.info("No configuration file, creating a new one")
self._config = CONF_SCHEMA({})
# Overwrite with command line arguments
args_keys = vars(args)
for key in args_keys:
if self._config.get(key) != args_keys[key]:
self._config[key] = args_keys[key]
self.save_config_file()
self._watchdog_observer = Observer()
watchdog_event_handler = FileSystemEventHandler()
watchdog_event_handler.on_modified = lambda event: self.load_config_file(
)
self._watchdog_observer.schedule(watchdog_event_handler, self._path)
self._watchdog_observer.start()
def load_config_file(self):
"""Load configuration from yaml file."""
with open(self._path, "r") as infile:
logger.debug("Loading configuration from <%s>", self._path)
try:
configuration = yaml.safe_load(infile)
if not configuration:
logger.warning(
"Could not load a configuration from %s, creating a new one",
self._path,
)
configuration = {}
self._config = CONF_SCHEMA(configuration)
self._callback()
except AttributeError:
# No callback configured
pass
except vol.MultipleInvalid as error:
logger.error("In configuration file %s: %s", self._path, error)
quit(1)
def save_config_file(self):
"""Save configuration back to yaml file."""
try:
with open(self._path, "w", encoding="utf8") as outfile:
cfg = self._config.pop(
CONF_CONFIG) # temporary displace config file
yaml.dump(self._config,
outfile,
default_flow_style=False,
allow_unicode=True)
except Exception as err:
logger.error("Could not save configuration: %s", err)
finally:
self._config[CONF_CONFIG] = cfg # restore
def __del__(self):
"""Release watchdog."""
if self._watchdog_observer:
self._watchdog_observer.stop()
self._watchdog_observer.join()
if self._config != {}:
self.save_config_file()
def __repr__(self):
return self._config
@property
def mqtt_conf(self):
return (
self._config[CONF_MQTT_SERVER],
self._config[CONF_MQTT_PORT],
self._config[CONF_MQTT_BASE_TOPIC],
)
@property
def dali_driver(self):
return self._config[CONF_DALI_DRIVER]
@property
def ha_discovery_prefix(self):
return self._config[CONF_HA_DISCOVERY_PREFIX]
@property
def log_level(self):
return self._config[CONF_LOG_LEVEL]
@property
def log_color(self):
return self._config[CONF_LOG_COLOR]
@property
def devices_names_file(self):
return self._config[CONF_DEVICES_NAMES_FILE]
