class Crafter(object):
"""
Setup and manage watchdog daemon.
"""
def __init__(self, base_dir):
# Cache paths
src_dir = os.path.join(base_dir, 'src')
dest_dir = os.path.join(base_dir, 'preview')
# Init handler
self.handler = ObserverHandler(src_dir, dest_dir)
# New observer class
self.observer = Observer()
self.observer.schedule(self.handler, path=src_dir, recursive=True)
def craft(self):
"""
Start watching src directory
"""
self.observer.start()
def shutdown(self):
"""
Properly shutdown watchdog daemon
"""
self.observer.stop()
self.observer.join()
def backend_monitor(API, config, logger):
logger.info("Start backend monitor")
if platform.system() == 'Windows':
observer = PollingObserver()
else:
observer = Observer()
for project_key in config['projects']['project_list']:
event_handler = FileSystemMonitor(
API, config, project_key, logger,
load_list_last_crawl(config, project_key))
list_last_crawl = load_list_last_crawl(config, project_key)
observer.schedule(event_handler,
config[project_key]['rootdir'],
recursive=True)
observer.start()
try:
while True:
# check the consistency between list_last_crawl and the current list in event_handler every 30s
time.sleep(30)
for project_key in config['projects']['project_list']:
if set(list_last_crawl) != event_handler.set_last_crawl:
log_full_run_filelist(dirs,
list(event_handler.set_last_crawl),
config[project_key]['name'])
list_last_crawl = load_list_last_crawl(config, project_key)
except KeyboardInterrupt:
observer.stop()
observer.join()
return
class Watcher:
DIRECTORY_TO_WATCH = os.path.join(Config.BASE_DIR, Config.QUEUE_LOCATION)
def __init__(self):
self.observer = PollingObserver()
def run(self):
event_handler = Handler()
self.observer.schedule(event_handler,
self.DIRECTORY_TO_WATCH,
recursive=True)
self.observer.start()
print(self.DIRECTORY_TO_WATCH)
logging.debug(self.DIRECTORY_TO_WATCH)
try:
while True:
time.sleep(5)
logging.debug("Watcher: Sleeping")
print("Watcher: Sleeping")
except:
self.observer.stop()
logging.error("-1 - Error")
print("-1 - Error")
self.observer.join()
def watch(directory=None, auto_clear=False, beep_on_failure=True,
onpass=None, onfail=None, poll=False, extensions=[]):
"""
Starts a server to render the specified file or directory
containing a README.
"""
if directory and not os.path.isdir(directory):
raise ValueError('Directory not found: ' + directory)
directory = os.path.abspath(directory or '')
# Initial run
event_handler = ChangeHandler(directory, auto_clear, beep_on_failure,
onpass, onfail, extensions)
event_handler.run()
# Setup watchdog
if poll:
observer = PollingObserver()
else:
observer = Observer()
observer.schedule(event_handler, path=directory, recursive=True)
observer.start()
# Watch and run tests until interrupted by user
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def watch_assets(options):
"""
Watch for changes to asset files, and regenerate js/css
"""
# Don't watch assets when performing a dry run
if tasks.environment.dry_run:
return
observer = PollingObserver()
CoffeeScriptWatcher().register(observer)
SassWatcher().register(observer)
XModuleSassWatcher().register(observer)
XModuleAssetsWatcher().register(observer)
print("Starting asset watcher...")
observer.start()
if not getattr(options, 'background', False):
# when running as a separate process, the main thread needs to loop
# in order to allow for shutdown by contrl-c
try:
while True:
observer.join(2)
except KeyboardInterrupt:
observer.stop()
print("\nStopped asset watcher.")
class Watcher:
def __init__(self):
logging.basicConfig(level=logging.DEBUG)
self.src_path = config.src_path
self.event_observer = PollingObserver()
def run(self):
event_handler = FileSystemHandler()
self.event_observer.schedule(event_handler, self.src_path)
self.event_observer.start()
logging.info('Initialized watcher on folder %s', self.src_path)
# process files already in src folder
for file in os.listdir(self.src_path):
filename = os.path.join(self.src_path, file)
event = FileCreatedEvent(filename)
event_handler.on_any_event(event)
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
self.stop()
def stop(self):
self.event_observer.stop()
self.event_observer.join()
class AIOWatchdog(object):
def __init__(self,
path=".",
recursive=True,
event_handler=None,
observer=None):
if observer is None:
self._observer = Observer()
else:
self._observer = observer
evh = event_handler or AIOEventHandler()
if isinstance(path, list):
for _path in path:
self._observer.schedule(evh, _path, recursive)
else:
self._observer.schedule(evh, path, recursive)
def start(self):
self._observer.start()
def stop(self):
self._observer.stop()
self._observer.join()
def main():
if int(os.getuid()) != 0:
raise SystemExit("ERROR: this script should be run as root")
parser = ArgumentParser(
description='Watch a directory and install the code')
args = parser.parse_args()
current_path = Path(__file__).resolve().parent
setup_path = Path(current_path, '..').resolve()
git_path = Path(current_path, '..', '..').resolve()
observer = Observer()
observer.schedule(FileHandler(setup_path, 'site'),
str(Path(git_path, 'site')), True)
observer.schedule(FileHandler(setup_path, 'bin'),
str(Path(git_path, 'bin')), True)
observer.schedule(FileHandler(setup_path, 'bin'),
str(Path(git_path, 'sbin')), True)
observer.start()
try:
print("Watching the following folders for change:")
print(" - site")
print(" - bin")
print(" - sbin")
print()
input("~~Hit enter to exit~~\n")
finally:
observer.stop()
observer.join()
def watch_record(indexer, use_polling=False):
"""
Start watching `cfstore.record_path`.
:type indexer: rash.indexer.Indexer
"""
if use_polling:
from watchdog.observers.polling import PollingObserver as Observer
Observer # fool pyflakes
else:
from watchdog.observers import Observer
event_handler = RecordHandler(indexer)
observer = Observer()
observer.schedule(event_handler, path=indexer.record_path, recursive=True)
indexer.logger.debug("Start observer.")
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
indexer.logger.debug("Got KeyboardInterrupt. Stopping observer.")
observer.stop()
indexer.logger.debug("Joining observer.")
observer.join()
indexer.logger.debug("Finish watching record.")
def main():
"""Script entry point."""
from watchdog.observers.polling import PollingObserver
from .parser import AAConfigParser
from .tricks import AutoRunTrick
parser = _create_main_argparser()
args = parser.parse_args()
configm = _apply_main_args(args)
# The reason to use PollingObserver() is it's os-independent. And it's
# more reliable.
observer = PollingObserver()
parser = AAConfigParser(configm)
handler_for_watch = parser.schedule_with(observer, AutoRunTrick)
handlers = set.union(*tuple(handler_for_watch.values()))
for handler in handlers:
handler.start()
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
for handler in handlers:
handler.stop()
def main():
if int(os.getuid()) != 0:
raise SystemExit("ERROR: this script should be run as root")
parser = ArgumentParser(description='Watch a directory and install the code')
args = parser.parse_args()
current_path = Path(__file__).resolve().parent
setup_path = Path(current_path, '..').resolve()
git_path = Path(current_path, '..', '..').resolve()
observer = Observer()
observer.schedule(FileHandler(setup_path, 'site'), str(Path(git_path, 'site')), True)
observer.schedule(FileHandler(setup_path, 'bin'), str(Path(git_path, 'bin')), True)
observer.schedule(FileHandler(setup_path, 'bin'), str(Path(git_path, 'sbin')), True)
observer.start()
try:
print("Watching the following folders for change:")
print(" - site")
print(" - bin")
print(" - sbin")
print()
input("~~Hit enter to exit~~\n")
finally:
observer.stop()
observer.join()
def watch_assets(options):
"""
Watch for changes to asset files, and regenerate js/css
"""
# Don't watch assets when performing a dry run
if tasks.environment.dry_run:
return
observer = PollingObserver()
CoffeeScriptWatcher().register(observer)
SassWatcher().register(observer)
XModuleSassWatcher().register(observer)
XModuleAssetsWatcher().register(observer)
print("Starting asset watcher...")
observer.start()
if not getattr(options, 'background', False):
# when running as a separate process, the main thread needs to loop
# in order to allow for shutdown by contrl-c
try:
while True:
observer.join(2)
except KeyboardInterrupt:
observer.stop()
print("\nStopped asset watcher.")
def main():
"""Script entry point."""
from watchdog.observers.polling import PollingObserver
from .parser import AAConfigParser
from .tricks import AutoRunTrick
parser = _create_main_argparser()
args = parser.parse_args()
configm = _apply_main_args(args)
# The reason to use PollingObserver() is it's os-independent. And it's
# more reliable.
observer = PollingObserver()
parser = AAConfigParser(configm)
handler_for_watch = parser.schedule_with(observer, AutoRunTrick)
handlers = set.union(*tuple(handler_for_watch.values()))
for handler in handlers:
handler.start()
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
for handler in handlers:
handler.stop()
def start_watch():
event_handler = when_file_chanage(kill_progress)
observer = Observer(timeout=1)
observer.schedule(event_handler, path=os.getcwd(), recursive=True)
observer.start()
global p, job_name
cmd = ["uwsgi", "--json", job_name]
p = subprocess.Popen(cmd, stderr=subprocess.PIPE)
return_code = p.poll()
while return_code is None:
if not observer.is_alive():
kill_progress()
break
return_code = p.poll()
line = p.stderr.readline().strip().decode("utf-8")
if len(line) != 0:
print(line)
sys.stderr.flush()
time.sleep(0.01)
while len(line) != 0:
line = p.stderr.readline().strip().decode("utf-8")
print(line)
sys.stderr.flush()
observer.stop()
return return_code
class DirWatcher(threading.Thread):
def __init__(self, dirpath, scanq, options):
super().__init__()
self.__dirpath = dirpath
self.__handler = WatchdogHandler(scanq, options)
self.__observer = Observer()
if options.poll:
self.__observer = PollingObserver()
self.__options = options
self.__scanq = scanq
self.__stop = False
def run(self):
self.__observer.schedule(self.__handler,
self.__dirpath,
recursive=True)
self.__observer.start()
try:
while True:
if self.__stop:
return
pass
except KeyboardInterrupt:
return
def stop(self):
self.__observer.stop()
self.__observer.join()
self.__stop = True
def watch(self):
"""
Start watching
"""
logger.info('Watching directory %s' % self.directory)
# Set up handler for when we see new files
callback = self.callback
class NewFileEventHandler(FileSystemEventHandler):
def on_created(self, event):
if not event.is_directory:
logger.info('Detected new file: %s' % event.src_path)
callback(event.src_path)
event_handler = NewFileEventHandler()
# Use polling observer (rather than filesystem-specific observers),
# because it is more reliable.
observer = PollingObserver(timeout=self.sleep_time)
# Start the observer
observer.schedule(event_handler, self.directory, recursive=False)
observer.start()
# Wait while the observer is running
try:
while True:
sleep(self.sleep_time)
# Exit gracefully
except KeyboardInterrupt:
logger.info('Detected interrupt. Stopping observer.')
observer.stop()
observer.join()
def watch(self):
"""
Start watching
"""
logger.info('Watching directory %s' % self.directory)
# Set up handler for when we see new files
callback = self.callback
class NewFileEventHandler(FileSystemEventHandler):
def on_created(self, event):
if not event.is_directory:
logger.info('Detected new file: %s' % event.src_path)
callback(event.src_path)
event_handler = NewFileEventHandler()
# Use polling observer (rather than filesystem-specific observers),
# because it is more reliable.
observer = PollingObserver(timeout=self.sleep_time)
# Start the observer
observer.schedule(event_handler, self.directory, recursive=False)
observer.start()
# Wait while the observer is running
try:
while True:
sleep(self.sleep_time)
# Exit gracefully
except KeyboardInterrupt:
logger.info('Detected interrupt. Stopping observer.')
observer.stop()
observer.join()
def watch_record(indexer, use_polling=False):
"""
Start watching `cfstore.record_path`.
:type indexer: rash.indexer.Indexer
"""
if use_polling:
from watchdog.observers.polling import PollingObserver as Observer
Observer # fool pyflakes
else:
from watchdog.observers import Observer
event_handler = RecordHandler(indexer)
observer = Observer()
observer.schedule(event_handler, path=indexer.record_path, recursive=True)
indexer.logger.debug('Start observer.')
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
indexer.logger.debug('Got KeyboardInterrupt. Stopping observer.')
observer.stop()
indexer.logger.debug('Joining observer.')
observer.join()
indexer.logger.debug('Finish watching record.')
def start(self):
path = self.config.get('watchdog', 'path')
patterns = self.config.get('watchdog', 'patterns').split(';')
ignore_directories = self.config.getboolean('watchdog',
'ignore_directories')
ignore_patterns = self.config.get('watchdog',
'ignore_patterns').split(';')
case_sensitive = self.config.getboolean('watchdog', 'case_sensitive')
recursive = self.config.getboolean('watchdog', 'recursive')
event_handler = PatternMatchingEventHandler(
patterns=patterns,
ignore_patterns=ignore_patterns,
ignore_directories=ignore_directories,
case_sensitive=case_sensitive)
event_handler.on_created = self.on_created
# event_handler.on_modified = self.on_modified
observer = Observer()
observer.schedule(path=path,
event_handler=event_handler,
recursive=recursive)
observer.start()
self.logger.info('WatchDog Observer for HCS/AFS/AAS is startting.....')
self.logger.info('patterns=%s' % patterns)
self.logger.info('path=%s' % path)
try:
while observer.is_alive():
time.sleep(1)
except (KeyboardInterrupt):
observer.stop()
self.logger.debug('WatchDog Observer is stoped.')
observer.join()
class LocalComputeLogSubscriptionManager(object):
def __init__(self, manager):
self._manager = manager
self._subscriptions = defaultdict(list)
self._watchers = {}
self._observer = PollingObserver(WATCHDOG_POLLING_TIMEOUT)
self._observer.start()
def _key(self, run_id, key):
return "{}:{}".format(run_id, key)
def add_subscription(self, subscription):
check.inst_param(subscription, "subscription", ComputeLogSubscription)
key = self._key(subscription.run_id, subscription.key)
self._subscriptions[key].append(subscription)
self.watch(subscription.run_id, subscription.key)
def remove_all_subscriptions(self, run_id, key):
key = self._key(run_id, key)
for subscription in self._subscriptions.pop(key, []):
subscription.complete()
def watch(self, run_id, key):
key = self._key(run_id, key)
if key in self._watchers:
return
update_paths = [
self._manager.get_local_path(run_id, key, ComputeIOType.STDOUT),
self._manager.get_local_path(run_id, key, ComputeIOType.STDERR),
]
complete_paths = [self._manager.complete_artifact_path(run_id, key)]
directory = os.path.dirname(
self._manager.get_local_path(run_id, key, ComputeIOType.STDERR))
ensure_dir(directory)
self._watchers[key] = self._observer.schedule(
LocalComputeLogFilesystemEventHandler(self, run_id, key,
update_paths,
complete_paths),
str(directory),
)
def notify_subscriptions(self, run_id, key):
key = self._key(run_id, key)
for subscription in self._subscriptions[key]:
subscription.fetch()
def unwatch(self, run_id, key, handler):
key = self._key(run_id, key)
if key in self._watchers:
self._observer.remove_handler_for_watch(handler,
self._watchers[key])
del self._watchers[key]
def dispose(self):
self._observer.stop()
def main(argv):
args = Args(sys.argv[1:])
# now that logging is setup log our settings
args.log_settings()
src_path = args.path
# Did we ask for debug?
if args.debug:
set_debug()
if args.globus_debug:
set_gt_debug()
# using globus, init to prompt for endpoiont activation etc
GlobusTransfer(args.source, args.destination, args.destination_dir,
src_path)
event_handler = Handler(args)
if args.prepopulate:
event_handler.prepopulate()
# observer = watchdog.observers.Observer()
observer = PollingObserver()
observer.schedule(event_handler, path=src_path, recursive=True)
# setup signal handler
def dump_status(event_handler, signalNumber, frame, details=False):
"""Dump the Handler() status when USR1 is recived."""
event_handler.status(details=details)
signal.signal(signal.SIGUSR1, partial(dump_status, event_handler))
signal.signal(signal.SIGUSR2,
partial(dump_status, event_handler, details=True))
observer.start()
s = sched.scheduler(time.time, time.sleep)
logger.info("Starting Main Event Loop")
try:
while True:
logger.info(
f"Starting iteration {event_handler.iteration} will sleep {args.sleep} seconds"
)
s.enter(
args.sleep,
1,
event_handler.new_iteration,
argument=(args.dwell_time, ),
)
s.run()
except KeyboardInterrupt:
observer.stop()
observer.join()
def _start_watch_mode(self, args: ConvertModeArguments) -> None:
"""Starts and runs the watch mode until canceled
Arguments:
args {ConvertModeArguments} -- The arguments for convert mode
"""
# Use custom event handler
event_handler = self._create_watch_handler(
args['in_path'], args['out_path'],
)
# Init the observer
observer = Observer()
observer.schedule(event_handler, args['in_path'], recursive=True)
self._logger.debug(f"Starting watch mode for: {args['in_path']}")
if self._visual:
print(
colored('Starting watcher for:', 'blue', attrs=['bold']),
colored(f"{os.path.abspath(args['in_path'])}", 'blue'),
)
else:
self._logger.info(f"Starting watch mode for: {args['in_path']}")
# Start
observer.start()
# Keep the process running while
# the watcher watches (until KeyboardInterrupt)
try:
while True:
# Pretty spinner =)
spinner_text = colored(
'Watching files', 'blue',
) + colored(' (use Ctrl+c to exit)', 'red')
with yaspin(
Spinners.bouncingBar,
text=spinner_text,
color='blue',
):
time.sleep(1)
except KeyboardInterrupt:
self._logger.debug('Got a KeyboardInterrupt, stopping watcher.')
observer.stop()
observer.join()
self._logger.debug(f"Stoped watching {args['in_path']}")
if self._visual:
print(
colored('Stoped watcher for:', 'blue', attrs=['bold']),
colored(f"{os.path.abspath(args['in_path'])}", 'blue'),
)
else:
self._logger.info(f"Stoped watching {args['in_path']}")
class PosixObserver(BaseObserver):
"""
Use the Watchdog module to observe filesystem events.
"""
def monitor(self):
# Set up the event handler
event_handler = MyEventHandler(patterns=['*'],
ignore_patterns=['version.py'],
ignore_directories=True)
event_handler.setup(self)
# Extract the set of directories to listen to
listen_dirs = self.get_dirs_to_monitor()
# Create an observer and schedule each of the directories
self.observer = Observer()
logger.debug("Starting observer: %s" % RippleConfig().monitor)
if RippleConfig().monitor == "poll":
self.observer = PollingObserver()
for d in listen_dirs:
# Put this in a try so it doesn't crash if the dir doesnt exist
if os.path.isdir(d):
logger.info("Monitoring: %s" % d)
self.observer.schedule(event_handler, d, recursive=True)
else:
logger.error("Directory does not exist: %s" % d)
try:
self.observer.start()
while True:
time.sleep(1)
except KeyboardInterrupt:
self.stop_monitoring()
self.observer.join()
def get_dirs_to_monitor(self):
"""
Work out which directories to monitor.
"""
rules = RippleConfig().rules
listen_dirs = []
for rule in rules:
if rule['trigger']['monitor'] == 'filesystem':
listen_dirs.append(rule['trigger']['parameters']['directory'])
listen_dirs = list(set(listen_dirs))
logger.debug("Monitoring dirs: %s" % listen_dirs)
return listen_dirs
def stop_monitoring(self):
"""
Terminate the monitor
"""
logger.debug("Terminating POSIX monitor.")
self.observer.stop()
def watch(
directories=[],
ignore=[],
auto_clear=False,
beep_on_failure=True,
onpass=None,
onfail=None,
runner=None,
beforerun=None,
onexit=None,
poll=False,
extensions=[],
args=[],
spool=True,
verbose=False,
quiet=False,
):
if not directories:
directories = ["."]
directories = [os.path.abspath(directory) for directory in directories]
for directory in directories:
if not os.path.isdir(directory):
raise ValueError("Directory not found: " + directory)
if ignore:
recursive_dirs, non_recursive_dirs = split_recursive(directories, ignore)
else:
recursive_dirs = directories
non_recursive_dirs = []
# Initial run
event_handler = ChangeHandler(
auto_clear, beep_on_failure, onpass, onfail, runner, beforerun, extensions, args, spool, verbose, quiet
)
event_handler.run()
# Setup watchdog
observer = PollingObserver() if poll else Observer()
for directory in recursive_dirs:
observer.schedule(event_handler, path=directory, recursive=True)
for directory in non_recursive_dirs:
observer.schedule(event_handler, path=directory, recursive=False)
# Watch and run tests until interrupted by user
try:
observer.start()
while True:
time.sleep(1)
observer.join()
except KeyboardInterrupt:
observer.stop()
if onexit:
os.system(onexit)
def watch(self):
observer = PollingObserver()
observer.schedule(self.pickup_event_processor, path=self.pickup_dir)
observer.start()
try:
while self.keep_running:
sleep(3)
except KeyboardInterrupt:
observer.stop()
observer.join()
def serve(config, options=None):
"""
Start the devserver, and rebuild the docs whenever any changes take effect.
"""
# Create a temporary build directory, and set some options to serve it
tempdir = tempfile.mkdtemp()
options['site_dir'] = tempdir
# Only use user-friendly URLs when running the live server
options['use_directory_urls'] = True
# Perform the initial build
config = load_config(options=options)
build(config, live_server=True)
# Note: We pass any command-line options through so that we
# can re-apply them if the config file is reloaded.
event_handler = BuildEventHandler(options)
config_event_handler = ConfigEventHandler(options)
# We could have used `Observer()`, which can be faster, but
# `PollingObserver()` works more universally.
observer = PollingObserver()
observer.schedule(event_handler, config['docs_dir'], recursive=True)
for theme_dir in config['theme_dir']:
if not os.path.exists(theme_dir):
continue
observer.schedule(event_handler, theme_dir, recursive=True)
observer.schedule(config_event_handler, '.')
observer.start()
class TCPServer(socketserver.TCPServer):
allow_reuse_address = True
class DocsDirectoryHandler(FixedDirectoryHandler):
base_dir = config['site_dir']
host, port = config['dev_addr'].split(':', 1)
server = TCPServer((host, int(port)), DocsDirectoryHandler)
print('Running at: http://%s:%s/' % (host, port))
print('Live reload enabled.')
print('Hold ctrl+c to quit.')
try:
server.serve_forever()
except KeyboardInterrupt:
print('Stopping server...')
# Clean up
observer.stop()
observer.join()
shutil.rmtree(tempdir)
print('Quit complete')
def watch(self):
observer = PollingObserver()
observer.schedule(self.pickup_event_processor, path=self.pickup_dir)
observer.start()
try:
while self.keep_running:
sleep(3)
except KeyboardInterrupt:
observer.stop()
observer.join()
class ImageWatcher:
""" Responsible for managing events in images
on a particular path.
"""
def __init__(self, src_path):
""" Initializes attributes and start thread for the
monitoring
Arguments:
src_path {str} -- directory path to be monitored
"""
self.__src_path = src_path
self.__event_handler = ImageHandler()
self.__event_observer = PollingObserver()
self.__stop_thread = False
self.__process = Thread(target=self.__run)
self.__process.start()
def __run(self):
""" Sets the background monitoring. """
self.__start()
try:
while True:
if self.__stop_thread:
break
time.sleep(1)
except Exception:
logger.exception("Image watcher interrupted")
finally:
self.__stop()
def __start(self):
""" Starts monitoring. """
self.__schedule()
self.__event_observer.start()
logger.debug('Image watcher started!')
def __stop(self):
""" Breaks monitoring. """
self.__event_observer.stop()
self.__event_observer.join()
logger.debug('Image watcher stoped!')
def __schedule(self):
""" Schedules the event handler. """
self.__event_observer.schedule(self.__event_handler,
self.__src_path,
recursive=True)
def close(self):
""" Sets flag to stop monitoring. """
self.__stop_thread = True
def watch(directories=[],
ignore=[],
auto_clear=False,
beep_on_failure=True,
onpass=None,
onfail=None,
runner=None,
beforerun=None,
onexit=None,
poll=False,
extensions=[],
args=[],
spool=True,
verbose=False,
quiet=False):
if not directories:
directories = ['.']
directories = [os.path.abspath(directory) for directory in directories]
for directory in directories:
if not os.path.isdir(directory):
raise ValueError('Directory not found: ' + directory)
if ignore:
recursive_dirs, non_recursive_dirs = split_recursive(
directories, ignore)
else:
recursive_dirs = directories
non_recursive_dirs = []
# Initial run
event_handler = ChangeHandler(auto_clear, beep_on_failure, onpass, onfail,
runner, beforerun, extensions, args, spool,
verbose, quiet)
event_handler.run()
# Setup watchdog
observer = PollingObserver() if poll else Observer()
for directory in recursive_dirs:
observer.schedule(event_handler, path=directory, recursive=True)
for directory in non_recursive_dirs:
observer.schedule(event_handler, path=directory, recursive=False)
# Watch and run tests until interrupted by user
try:
observer.start()
while True:
time.sleep(1)
observer.join()
except KeyboardInterrupt:
observer.stop()
if onexit:
os.system(onexit)
def tail_like(path):
observer = PollingObserver()
handler = TailHandler(path)
observer.schedule(handler, dirname(path))
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
finally:
handler.close()
observer.join()
def watch_file(directory):
event_handler = MyHandler()
observer = PollingObserver()
observer.schedule(event_handler, path=directory, recursive=False)
observer.start()
print 'watching for changes to', directory
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def compile_contracts(watch, contracts):
"""
Compile project contracts, storing their output in `./build/contracts.json`
Call bare to compile all contracts or specify contract names or file paths
to restrict to only compiling those contracts.
Pass in a file path and a contract name separated by a colon(":") to
specify only named contracts in the specified file.
"""
project_dir = os.getcwd()
click.echo("============ Compiling ==============")
click.echo("> Loading contracts from: {0}".format(
get_contracts_dir(project_dir)))
result = compile_and_write_contracts(project_dir, *contracts)
contract_source_paths, compiled_sources, output_file_path = result
click.echo("> Found {0} contract source files".format(
len(contract_source_paths)))
for path in contract_source_paths:
click.echo("- {0}".format(os.path.basename(path)))
click.echo("")
click.echo("> Compiled {0} contracts".format(len(compiled_sources)))
for contract_name in sorted(compiled_sources.keys()):
click.echo("- {0}".format(contract_name))
click.echo("")
click.echo("> Outfile: {0}".format(output_file_path))
if watch:
# The path to watch
watch_path = utils.get_contracts_dir(project_dir)
click.echo("============ Watching ==============")
event_handler = ContractChangedEventHandler(
project_dir=project_dir,
contract_filters=contracts,
)
observer = PollingObserver()
observer.schedule(event_handler, watch_path, recursive=True)
observer.start()
try:
while observer.is_alive():
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def start_watch_loop(self, *args, **options):
self.set_options(**options)
callback = partial(self.watch_handle, *args, **options)
handler = ChangeDebounceHandler(callback)
observer = Observer()
for path in self.collect_watch_paths():
observer.schedule(handler, path, recursive=True)
observer.start()
try:
while True:
handler.process()
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def watcher(path=None):
if path is None:
path = str(Path.home())
event_handler = MyHandler()
print(f"Monitoring directory: {path}")
observer = PollingObserver()
observer.schedule(event_handler, path=path, recursive=False)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
class PhoenixAdultRenamer():
def __init__(self):
for file in searching_all_files(input_path):
work_with_file(file)
self.observer = Observer()
def run(self):
self.observer.schedule(WatchFileHandler(), input_path, recursive=True)
self.observer.start()
while True:
time.sleep(1)
def stop(self):
self.observer.stop()
self.observer.join()
def watch():
# Users expect an implicit push
push(watch=True)
# Start the observer
observer = PollingObserver()
observer.event_queue.max_size = 1
observer.schedule(EventHandler(), os.getcwd(), recursive=True)
observer.start()
puts(colored.yellow('Watching for changes... (ctrl-c to stop)'))
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
# Block until the thread terminates
observer.join()
def main():
handler = ChangeHandler()
directory = "./"
observer = PollingObserver(0.35)
# Poll every 0.35 seconds
if not os.path.exists(directory):
os.makedirs(directory)
observer.schedule(handler, directory, recursive=True)
# Only search in the LaTeX directory
observer.start()
try:
while True:
time.sleep(60 * 5)
# Sleep for 5 minutes (time doesn't really matter)
except KeyboardInterrupt:
observer.stop()
observer.join()
def watch_assets(options):
"""
Watch for changes to asset files, and regenerate js/css
"""
# Don't watch assets when performing a dry run
if tasks.environment.dry_run:
return
themes = get_parsed_option(options, 'themes')
theme_dirs = get_parsed_option(options, 'theme_dirs', [])
# wait comes in as a list of strings, define the default value similarly for convenience.
default_wait = [unicode(DEFAULT_OBSERVER_TIMEOUT)]
wait = float(get_parsed_option(options, 'wait', default_wait)[0])
if not theme_dirs and themes:
# We can not add theme sass watchers without knowing the directory that contains the themes.
raise ValueError(
'theme-dirs must be provided for watching theme sass.')
else:
theme_dirs = [path(_dir) for _dir in theme_dirs]
sass_directories = get_watcher_dirs(theme_dirs, themes)
observer = PollingObserver(timeout=wait)
CoffeeScriptWatcher().register(observer)
SassWatcher().register(observer, sass_directories)
XModuleSassWatcher().register(observer, ['common/lib/xmodule/'])
XModuleAssetsWatcher().register(observer)
print("Starting asset watcher...")
observer.start()
# Run the Webpack file system watcher too
execute_webpack_watch(settings=Env.DEVSTACK_SETTINGS)
if not getattr(options, 'background', False):
# when running as a separate process, the main thread needs to loop
# in order to allow for shutdown by control-c
try:
while True:
observer.join(2)
except KeyboardInterrupt:
observer.stop()
print("\nStopped asset watcher.")
def watchJsonFile():
class MyEventHandler(PatternMatchingEventHandler):
def on_modified(self, event):
super(MyEventHandler, self).on_modified(event)
# FIXME python recognizes change, needs to take change and push to led lights
updateColors()
printData()
event_handler = MyEventHandler(patterns='dat*.json')
my_observer = Observer()
my_observer.schedule(event_handler, ".", recursive=True)
my_observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
my_observer.stop()
my_observer.join()
def start_toml_observer(path, transfer=None, dry_run=False, import_db=None):
toml_handler = TomlCreatedEventHandler(transfer=transfer,
dry_run=dry_run,
import_db=import_db)
# We use the polling observer as inotify
# does not see remote file creation events
observer = Observer()
observer.schedule(toml_handler, path, recursive=True)
print("Starting observer")
observer.start()
print("Observer started")
try:
while True:
time.sleep(1)
finally:
observer.stop()
observer.join()
def watch_assets(options):
"""
Watch for changes to asset files, and regenerate js/css
"""
# Don't watch assets when performing a dry run
if tasks.environment.dry_run:
return
themes = get_parsed_option(options, 'themes')
theme_dirs = get_parsed_option(options, 'theme_dirs', [])
# wait comes in as a list of strings, define the default value similarly for convenience.
default_wait = [unicode(DEFAULT_OBSERVER_TIMEOUT)]
wait = float(get_parsed_option(options, 'wait', default_wait)[0])
if not theme_dirs and themes:
# We can not add theme sass watchers without knowing the directory that contains the themes.
raise ValueError('theme-dirs must be provided for watching theme sass.')
else:
theme_dirs = [path(_dir) for _dir in theme_dirs]
sass_directories = get_watcher_dirs(theme_dirs, themes)
observer = PollingObserver(timeout=wait)
CoffeeScriptWatcher().register(observer)
SassWatcher().register(observer, sass_directories)
XModuleSassWatcher().register(observer, ['common/lib/xmodule/'])
XModuleAssetsWatcher().register(observer)
print("Starting asset watcher...")
observer.start()
# Run the Webpack file system watcher too
execute_webpack_watch(settings=Env.DEVSTACK_SETTINGS)
if not getattr(options, 'background', False):
# when running as a separate process, the main thread needs to loop
# in order to allow for shutdown by control-c
try:
while True:
observer.join(2)
except KeyboardInterrupt:
observer.stop()
print("\nStopped asset watcher.")
def run_watch(self):
if self.poll:
from watchdog.observers.polling import PollingObserver as Observer
else:
from watchdog.observers import Observer
event_handler = RoninEventHandler(self)
observer = Observer()
observer.schedule(event_handler, self.source, recursive=True)
observer.start()
try:
logger.info("Watching directory: '{0}' for changes (poll={1})".format(self.source, self.poll))
while True:
time.sleep(1)
except KeyboardInterrupt:
logger.info("Stopping watcher...")
observer.stop()
observer.join()
def server(self, args):
server = Process(target=self._server)
server.start()
event_handler = PatternMatchingEventHandler(ignore_patterns=self.WATCH_EXCLUDE)
event_handler.on_modified = lambda event : self._build()
observer = Observer()
observer.schedule(event_handler, self.BASE_DIR, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except (KeyboardInterrupt, SystemExit):
server.terminate()
observer.stop()
observer.join()
self.logger.info("Clossing")
def folderObserver(pathStructure, dbPath):
logging = DefaultLogger()
if pathStructure == None or pathStructure['inBox'] == None:
message = 'Watch: Unable to run as pathStructure is undefined'
logging.debug(message)
return
event_handler = singleFileWatcher(pathStructure, dbPath)
observer = PollingObserver()
observer.schedule(event_handler, pathStructure['inBox'], recursive=False)
observer.start()
try:
while True and observer.is_alive():
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def watch_assets(options):
"""
Watch for changes to asset files, and regenerate js/css
"""
# Don't watch assets when performing a dry run
if tasks.environment.dry_run:
return
themes = get_parsed_option(options, 'themes')
theme_dirs = get_parsed_option(options, 'theme_dirs', [])
if not theme_dirs and themes:
# We can not add theme sass watchers without knowing the directory that contains the themes.
raise ValueError('theme-dirs must be provided for watching theme sass.')
else:
theme_dirs = [path(_dir) for _dir in theme_dirs]
sass_directories = get_watcher_dirs(theme_dirs, themes)
observer = PollingObserver()
CoffeeScriptWatcher().register(observer)
SassWatcher().register(observer, sass_directories)
XModuleSassWatcher().register(observer, ['common/lib/xmodule/'])
XModuleAssetsWatcher().register(observer)
print("Starting asset watcher...")
observer.start()
# We only want Webpack to re-run on changes to its own entry points, not all JS files, so we use its own watcher
# instead of subclassing from Watchdog like the other watchers do
execute_webpack_watch(settings=Env.DEVSTACK_SETTINGS)
if not getattr(options, 'background', False):
# when running as a separate process, the main thread needs to loop
# in order to allow for shutdown by contrl-c
try:
while True:
observer.join(2)
except KeyboardInterrupt:
observer.stop()
print("\nStopped asset watcher.")
def watch_assets(options):
"""
Watch for changes to asset files, and regenerate js/css
"""
# Don't watch assets when performing a dry run
if tasks.environment.dry_run:
return
themes = getattr(options, 'themes', None)
theme_dirs = getattr(options, 'theme-dirs', [])
if not theme_dirs and themes:
# We can not add theme sass watchers without knowing the directory that contains the themes.
raise ValueError('theme-dirs must be provided for watching theme sass.')
else:
theme_dirs = [path(_dir) for _dir in theme_dirs]
if isinstance(themes, basestring):
themes = themes.split(',')
else:
themes = themes if isinstance(themes, list) else [themes]
sass_directories = get_watcher_dirs(theme_dirs, themes)
observer = PollingObserver()
CoffeeScriptWatcher().register(observer)
SassWatcher().register(observer, sass_directories)
XModuleSassWatcher().register(observer, ['common/lib/xmodule/'])
XModuleAssetsWatcher().register(observer)
print("Starting asset watcher...")
observer.start()
if not getattr(options, 'background', False):
# when running as a separate process, the main thread needs to loop
# in order to allow for shutdown by contrl-c
try:
while True:
observer.join(2)
except KeyboardInterrupt:
observer.stop()
print("\nStopped asset watcher.")
class FileMonitor(FileSystemEventHandler):
def __init__(self, file, action):
self.path,self.filename = os.path.split(file)
self.action = action
self.observer = None
def start(self):
self.observer = Observer()
self.observer.schedule(self, self.path, recursive=False)
self.observer.start()
def stop(self):
if self.observer is not None:
self.observer.stop()
def join(self):
if self.observer is not None:
self.observer.join()
def on_modified(self, event):
try:
if os.path.samefile(event.src_path,self.filename):
self.action()
except OSError as e:
print 'Exception on file check', e
class Monitor(object):
"""Monitoring your project files.
**Parameters**
:param basedir: The directory to start watching from.
Defaults to `'.'`.
**Attributes**
:ivar observer: The watchdog.Observer instance.
:ivar handlers: All the attached handlers.
:ivar running: Boolean flag, to control the running while loop.
**Basic Usage** ::
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# -*- filename: monitor.py
from pandora import Monitor
import logging
monitor = Monitor()
logging.basicConfig(
level=logging.INFO,
format='%(asctime)-15s %(levelname)-6s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logger = logging.getLogger(__name__)
@monitor.listen(['monitor.py'], on_startup=False)
def reload(event=None, source=None):
logger.info('Should reload the monitor.')
monitor.restart_process()
monitor.run()
"""
def __init__(self, basedir='.', polling=False):
"""Initialize the Monitor."""
if not polling:
self.observer = Observer()
else:
self.observer = PollingObserver()
self.handlers = []
self.basedir = basedir
self.running = False
def stop(self):
"""Stop the observer manually."""
self.running = False
self.observer.stop()
self.observer.join()
def run(self, delay=1):
"""Start the observer and run the handlers when files change.
This will catch the `KeyboardInterrupt` (`control-c`) exception. It
will stop and join the observer process. After this your script
continues.
"""
self.observer.start()
self.running = True
tpl = 'Waiting for changes [{:4s}]'
items = [tpl.format('.' * i) for i in range(1, 5)]
spinner = itertools.cycle(items)
try:
while self.running:
now = dt.datetime.now()
sys.stdout.write(now.strftime('%H:%M:%S - '))
sys.stdout.write(next(spinner))
sys.stdout.flush()
time.sleep(delay)
sys.stdout.write('\b' * (len(items[0]) + 11))
sys.stdout.flush()
for handler in self.handlers:
handler.run()
except KeyboardInterrupt: # pragma: no cover
self.observer.stop()
self.observer.join()
def listen(self, patterns, on_startup=False):
"""A decorator to attach a handler to the observer.
The patterns are looped through, and the `basedir` is prepended
to each pattern. If the patterns starts with an exclamation mark (!),
the pattern is marked as an `exclude` pattern.
**Parameters**
:param patterns: A list of patterns to match files agains.
Example: `['*.py', '*.ini', '!monitor.py']`.
:param on_startup: Should the handler be ran as soon as the
monitor is ran?
Defaults to `False`.
:type patterns: list
:type on_startup: bool
:return: A wrapper function, to be used as decorator.
:rtype: callable
"""
include = []
exclude = []
for pattern in patterns:
if pattern[0] == '!':
pattern = pattern[1:]
into = exclude
else:
into = include
into.append(path.join(self.basedir, pattern))
if not len(include):
include = None
if not len(exclude):
exclude = None
def wrapper(callback):
handler = Handler(include, exclude, callback, on_startup)
self.handlers.append(handler)
self.observer.schedule(handler, self.basedir, recursive=True)
return callback
return wrapper
def restart_process(self): # pragma: no cover
"""Restart the current python process.
This is particulairy handy if your monitor `.py` file has changed. It
simply calls :meth:`stop` and restarts itself.
"""
from os import execl
import sys
self.stop()
execl(sys.executable, sys.executable, *sys.argv)
def main(argv):
"""
Build the docs and serve them with an HTTP server.
"""
parser = argparse.ArgumentParser(description='Build and serve HTML Sphinx docs')
parser.add_argument(
'--port',
help='Serve on this port, default 8000',
type=int,
default=8000)
parser.add_argument(
'--source',
help='Directory of source Sphinx (reStructuredText) docs',
type=os.path.realpath,
default='docs/source')
parser.add_argument(
'--destination',
help='Where to build the HTML output',
type=os.path.realpath,
default='docs/build/html')
parser.add_argument(
'--doctrees',
help='Where the doctrees are built',
type=os.path.realpath,
default='docs/build/doctrees')
options = parser.parse_args(argv)
bound_build_docs = partial(build_docs, options.source, options.destination, options.doctrees)
# Do the initial build
bound_build_docs()
# Watch the source directory for changes, build docs again if detected
observer = Observer()
observer.schedule(
BuildDocsHandler(bound_build_docs),
path=options.source, recursive=True)
observer.start()
# Set the root for the request handler, overriding Python stdlib current
# working directory.
DocsHTTPRequestHandler._root = options.destination
server = SocketServer.TCPServer(
('', options.port),
DocsHTTPRequestHandler)
try:
logger.info('Serving on localhost:{}'.format(options.port))
server.serve_forever()
except KeyboardInterrupt:
sys.stdout.write('\n')
logger.info('(stopping server)')
observer.stop()
finally:
observer.join()
logging.info('Server stopped, exiting')
sys.exit(0)
def watch(directories=[], ignore=[], extensions=[], beep_on_failure=True,
auto_clear=False, wait=False, beforerun=None, afterrun=None,
onpass=None, onfail=None, onexit=None, runner=None, spool=None,
poll=False, verbose=False, quiet=False, pytest_args=[]):
argv = (runner or 'py.test').split(' ') + (pytest_args or [])
if not directories:
directories = ['.']
directories = [os.path.abspath(directory) for directory in directories]
for directory in directories:
if not os.path.isdir(directory):
raise ValueError('Directory not found: ' + directory)
# Setup event handler
event_listener = EventListener(extensions)
# Setup watchdog
observer = PollingObserver() if poll else Observer()
recursedirs, norecursedirs = _split_recursive(directories, ignore)
for directory in recursedirs:
observer.schedule(event_listener, path=directory, recursive=True)
for directory in norecursedirs:
observer.schedule(event_listener, path=directory, recursive=False)
observer.start()
# Watch and run tests until interrupted by user
events = []
while True:
try:
# Prepare next run
if auto_clear:
clear()
elif not quiet:
print()
# Show event summary
if not quiet:
_show_summary(argv, events, verbose)
# Run custom command
run_hook(beforerun)
# Run tests
p = subprocess.Popen(argv, shell=is_windows)
try:
while True:
# Check for completion
exit_code = p.poll()
if exit_code is not None:
break
# Interrupt the current test run on filesystem event
if not wait and not event_listener.event_queue.empty():
send_keyboard_interrupt(p)
exit_code = p.wait()
break
# Allow user to initiate a keyboard interrupt
sleep(0.1)
except KeyboardInterrupt:
# Wait for current test run cleanup
run_hook(afterrun, p.wait())
# Exit, since this keyboard interrupt was user-initiated
break
# Run custom command
run_hook(afterrun, exit_code)
# Run dependent commands
if exit_code in [EXIT_OK, EXIT_NOTESTSCOLLECTED]:
run_hook(onpass)
else:
if beep_on_failure:
beep()
run_hook(onfail)
# Wait for a filesystem event
while event_listener.event_queue.empty():
sleep(0.1)
# Collect events for summary of next run
events = dequeue_all(event_listener.event_queue, spool)
except KeyboardInterrupt:
break
except Exception as ex:
print(format_exc() if verbose else 'Error: {}'.format(ex))
break
# Stop and wait for observer
observer.stop()
observer.join()
# Run exit script
run_hook(onexit)
#!/usr/bin/env python
import sys, os, re
import time
import datetime
import logging
from watchdog.observers.polling import PollingObserver as Observer
from watchdog.events import FileSystemEventHandler
import threading, thread
import h5py
from Bio import SeqIO
from StringIO import StringIO
import MySQLdb
import subprocess
import string
import configargparse
from warnings import filterwarnings
import socket
import hashlib
import xmltodict
parser = configargparse.ArgParser(description='eboladb_squiggle_align: A program providing the ability to determine which region of the ebola genome and individual read is derived from.')
parser.add('-fasta', '--reference_fasta_file', type=str, dest='fasta', required=True, default=None, help="The fasta format file for the reference sequence for your organism.")
parser.add('-ids', nargs = '*', dest='ids',required=True, help = 'A list of start and stop positions for each amplicon from the reference genome')
parser.add('-w', '--watch-dir', type=str, required=True, default=None, help="The path to the folder containing the downloads directory with fast5 reads to analyse - e.g. C:\data\minion\downloads (for windows).", dest='watchdir')
args = parser.parse_args()
#########################################################
if __name__ == "__main__":
try:
def main():
global CONFIG_DIR_PATH
global server_com
config, user_exists = load_config()
CONFIG_DIR_PATH = config['dir_path']
args = args_parse_init(
stdout_level=config['stdout_log_level'],
file_level=config['file_log_level'],
)
report_file = config['crash_repo_path']
if args.no_repo:
report_file = False
logger_init(
crash_repo_path=report_file,
stdout_level=args.std_log_level,
file_level=args.file_log_level,
disabled=args.no_log,
)
snapshot_manager = DirSnapshotManager(
snapshot_file_path=config['snapshot_file_path'],
)
server_com = ServerCommunicator(
server_url=config['server_url'],
username=None,
password=None,
snapshot_manager=snapshot_manager)
client_command = {
"create_user": server_com.create_user,
"activate_user": server_com.activate_user,
"delete_user": server_com.delete_user,
"reset_password": server_com.reset_password,
"set_password": server_com.set_password,
"add_share": server_com.add_share,
"remove_share": server_com.remove_share,
"remove_beneficiary": server_com.remove_beneficiary,
"get_shares_list": server_com.get_shares_list
}
sock_server = CmdMessageServer(
config['host'],
int(config['port']),
client_command)
while not user_exists:
asyncore.poll(timeout=1.0)
config, user_exists = load_config()
server_com.username = config['username']
server_com.password = config['password']
server_com.auth = HTTPBasicAuth(server_com.username, server_com.password)
event_handler = DirectoryEventHandler(server_com, snapshot_manager)
file_system_op = FileSystemOperator(event_handler, server_com, snapshot_manager)
executer = CommandExecuter(file_system_op, server_com)
server_com.setExecuter(executer)
observer = Observer()
observer.schedule(event_handler, config['dir_path'], recursive=True)
observer.start()
last_synk_time = 0
try:
while True:
asyncore.poll(timeout=5.0)
if (time.time() - last_synk_time) >= 5.0:
last_synk_time = time.time()
server_com.synchronize(file_system_op)
except KeyboardInterrupt:
observer.stop()
observer.join()
def run():
parser = argparse.ArgumentParser(
description='Poll a directory for changes and re-touch changed paths '
'so that inotify-incapable mounts (like CIFS) receive inotify '
'events anyway.')
parser.add_argument('-i', '--polling-interval',
default=1.0,
help="Polling interval in seconds",
type=float,
dest='interval'
)
parser.add_argument('-l', '--log-level',
default=11,
help="Logger verbosity level",
type=int,
dest='loglevel'
)
parser.add_argument("-r", "--simulate-rm",
default=False,
action='store_true',
dest='simulate_rm',
help="Simulate rm operations by natively flashing a path in/out of "
"existance. Only use if you find your tools get confused when a file "
"disapeared from under them."
)
parser.add_argument("-m", "--simulate-mv",
default=False,
action='store_true',
dest='simulate_mv',
help="Simulate mv operations by natively moving a path back and forth."
" Only use if you find your tools require specific handling of"
" move events."
)
parser.add_argument('-w', '--watchdir',
default=".",
required=False,
help="the directory to watch for changes",
dest="watchdir"
)
args = parser.parse_args()
args.watchdir = os.path.realpath(os.path.abspath(args.watchdir))
logging.basicConfig(level=args.loglevel, format="%(message)s (%(levelname)s)\n")
logger.info("Watching %r", args.watchdir)
polling_handler = PollingHandler(args)
native_handler = NativeHandler(polling_handler, args)
polling_observer = PollingObserver()
native_observer = Observer()
native_observer.schedule(native_handler, path=args.watchdir, recursive=True)
native_observer.start()
polling_observer.schedule(polling_handler, path=args.watchdir, recursive=True)
polling_observer.start()
try:
while True:
time.sleep(args.interval)
except KeyboardInterrupt:
logger.info("Shutdown")
native_observer.stop()
polling_observer.stop()
native_observer.join()
polling_observer.join()
'event_type': event.event_type
}
if event.event_type != events.EVENT_TYPE_DELETED:
data['type'] = magic.from_file(event.src_path, mime=True)
requests.post(
'http://%s:%s/items' % (APP_HOST, APP_PORT),
data=data
)
def on_modified(self, event):
self.process(event)
def on_created(self, event):
self.process(event)
def on_deleted(self, event):
self.process(event)
if __name__ == '__main__':
args = sys.argv[1:]
observer = Observer()
observer.schedule(MyHandler(), path=args[0] if args else '../watch_here')
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
#!/usr/bin/env python
import sys, os, re
import time
import datetime
import logging
from watchdog.observers.polling import PollingObserver as Observer
from watchdog.events import FileSystemEventHandler
import threading
import h5py
from Bio import SeqIO
from StringIO import StringIO
import MySQLdb
import subprocess
import string
import numpy
import configargparse
## minup: a program to process & upload MinION fast5 files in to the minoTour website in real-time or post-run.
## written & designed by Martin J. Blythe, Fei Sang & Matt W. Loose. DeepSeq, The University of Nottingham 2014. UK
## linux version
config_file = script_dir = os.path.dirname(os.path.realpath('__file__')) +'/'+'minup_posix.config'
#print config_file
parser = configargparse.ArgParser(description='minup: A program to analyse minION fast5 files in real-time or post-run.', default_config_files=[config_file])
parser.add('-dbh', '--mysql-host', type=str, dest='dbhost', required=False, default='localhost', help="The location of the MySQL database. default is 'localhost' .")
parser.add('-dbu', '--mysql-username', type=str, dest='dbusername', required=True, default=None, help="The MySQL username with create & write privileges on MinoTour.")
parser.add('-dbp', '--mysql-port', type=int, dest='dbport', required=False, default=3306, help="The MySQL port number, else the default port '3306' is used")
parser.add('-pw', '--mysql-password', type=str, dest='dbpass', required=False, default=None, help="The password for the MySQL username with permission to upload to MinoTour.")
parser.add('-f', '--align-ref-fasta', type=str, required=False, default=False, help="The path to the reference fasta file to upload a Last alignment. This is sufficient to enable aligning provided LastAl and LastDB are in the path. Leaving this entry blank will upload the data without any alignment.", dest='ref_fasta')
parser.add('-w', '--watch-dir', type=str, required=True, default=None, help="The path to the folder containing the downloads directory with fast5 reads to analyse - e.g. C:\data\minion\downloads (for windows).", dest='watchdir')
parser.add('-n', '--aligning-threads', type=str, required=False, help="The number of threads to use for aligning", default=2, dest='threads')
parser.add('-u', '--minotour-username', type=str, required=True, help="The MinoTour username with permissions to upload data.", default=False, dest='minotourusername')
class DFDash:
LOGFILE_NAME = "gamelog.txt"
DB_NAME = "DFDash.db"
class WebApplication:
def __init__(self, port=8080):
self._app = Flask(self.__class__.__name__)
def run(self):
self._app.run()
def __init__(self, df, port=8080, db=None, limit=None):
"""
:type df: unicode
:type port: int
"""
self.df = df
self.port = port
self.line_count = 0
self.line_limit = limit
self._log_path = os.path.join(df, DFDash.LOGFILE_NAME)
self._log_offset = 0
self._observer = PollingObserver()
self._db_lock = threading.Lock()
self._db_path = db or os.path.join(self.df, DFDash.DB_NAME)
if not os.path.exists(os.path.dirname(self._db_path)):
os.makedirs(os.path.dirname(self._db_path))
self.connect_db().ensure_db_initialized()
self._log_file = None
self.open_log_file()
def run(self):
db = self.connect_db()
self._on_log_change(db)
self.watch_log()
try:
while True:
time.sleep(1)
except BaseException:
self.stop_watching_log()
raise
def open_log_file(self):
print("Opening log file: {}".format(self._log_path))
self._log_file = codecs.open(self._log_path, "r", "cp437")
offset = self.fetch_seek()
print("Seeking to offset {}".format(offset))
self._log_file.seek(offset)
def fetch_seek(self):
self._db_lock.acquire()
db = self.connect_db()
offset = db.config_get("gamelog_seek")
db.close()
self._db_lock.release()
if offset is None:
offset = 0
else:
offset = int(offset)
self._log_offset = offset
return self._log_offset
def store_seek(self, db):
offset = self._log_offset
print("Storing log file offset: {}k".format(offset/1024))
db.config_put("gamelog_seek", offset)
def stop_watching_log(self):
self._observer.stop()
self._observer.join()
def connect_db(self):
db = DB(self._db_path)
try:
db.execute("SELECT instr('foo', 'f')")
except OperationalError as soe:
if soe.message == "no such function: instr":
def instr(a, b):
"""
:type a: str | unicode
:type b: str | unicode
:rtype: int
"""
if None in (a, b):
return None
return a.find(b) + 1
db._db.create_function("instr", 2, instr)
else:
raise
return db
def watch_log(self):
self._observer.schedule(
EventHandler(self._on_log_event,
self.connect_db),
bytes(self.df))
self._observer.start()
def _on_log_event(self, event, db):
"""
:type event: FileSystemEvent
"""
if event.is_directory:
return
if event.src_path != self._log_path:
return
event_type = event.event_type
if event_type == "modified":
self._on_log_change(db, print_events=True)
def _on_log_change(self, db, print_events=False):
last_line = None
while True:
line = self._log_file.readline()
line_length = len(line)
line = line.rstrip()
if line == "":
break
if re.match(r'^x[0-9]+$', line):
if last_line is None:
print("Got line '{}' but last_line is None :S".format(line))
self.line_count += 1
self._log_offset += line_length
continue
else:
#print("Got line '{}', repeating last line".format(line))
line = last_line
events = Event.from_text(line)
for event in events:
commit = False
if print_events:
print("{}: {}".format(event.event_type, event.message))
if self.line_count is not None and self.line_count % COMMIT_EVERY == 0:
commit = True
event.put(db, commit)
last_line = line
self.line_count += 1
self._log_offset += line_length
if self.line_count % COMMIT_EVERY == 0:
self.store_seek(db)
print("{}...".format(self.line_count))
if self.line_count == self.line_limit:
self._log_offset += line_length
self.store_seek(db)
raise UserWarning("Limit reached.")
self.store_seek(db)
print("Death causes: {!r}".format(Stats.deaths(db)))
