def start_watch(cmd, debug):
global p
event_handler = when_file_chanage(kill_progress, harakiri)
observer = Observer(timeout=1)
observer.schedule(event_handler, path=os.getcwd(), recursive=True)
observer.start()
p = subprocess.Popen(cmd, stderr=subprocess.PIPE)
return_code = p.poll()
while return_code is None:
sleep_time = 1
if debug:
sleep_time = 0.05
time.sleep(sleep_time)
if not observer.is_alive():
kill_progress()
break
return_code = p.poll()
if debug:
line_byte = p.stderr.readline()
line = line_byte.decode("UTF-8")
line = line.strip()
if len(line) != 0:
print(line)
sys.stderr.flush()
while len(line) != 0:
line_byte = p.stderr.readline()
line = line_byte.decode("UTF-8")
line = line.strip()
print(line)
sys.stderr.flush()
observer.stop()
return return_code
class ObserverBase:
'''
create doc content observer
'''
def __init__(self, event_handler):
self.event_handler = event_handler
self.path = event_handler.path
self._init_observer()
self._init_section()
def _init_section(self):
pass
def _create_items(self, tree_elements):
pass
def _init_observer(self):
self.go_recursively = True
self.observer = Observer()
self.observer.schedule(self.event_handler,
self.path,
recursive=self.go_recursively)
def start_observer(self):
self.observer.start()
def stop_observer(self):
if self.observer.is_alive():
self.observer.stop()
self.observer.join()
class MyEventHandler(FileSystemEventHandler):
def __init__(self, path, alert_file):
self.file_name = alert_file
self.path = path
self.observer = Observer()
self.text_operations = text_operations.TextOperations(self.file_name)
self.solution = solution.Solution()
self.end_program = end_program.EndProgram()
self.today_events = []
def on_modified(self, event):
if not event.is_directory and event.src_path.endswith(self.file_name):
new_event = self.text_operations.new_alert()
if new_event:
self.today_events.append(new_event)
def observe(self):
self.observer.schedule(self, self.path, recursive=False)
self.observer.start()
try:
while self.observer.is_alive():
self.observer.join(1)
except KeyboardInterrupt:
self.observer.stop()
self.end_program.check_firewall()
self.end_program.make_email_message(self.today_events)
print('Bye!')
def wait_on_entry_calc(self, key):
if self.separate_files:
entry = self._get_cache_by_key(key)
filename = f'{self._cache_fname()}_{hashlib.sha256(pickle.dumps(key)).hexdigest()}'
else:
with self.lock:
self._reload_cache()
entry = self._get_cache()[key]
filename = self._cache_fname()
if not entry['being_calculated']:
return entry['value']
event_handler = _PickleCore.CacheChangeHandler(filename=filename,
core=self,
key=key)
observer = Observer()
event_handler.inject_observer(observer)
observer.schedule(event_handler,
path=self.expended_cache_dir,
recursive=True)
observer.start()
time_spent = 0
while observer.is_alive():
observer.join(timeout=1.0)
time_spent += 1
if 0 < self.wait_for_calc_timeout < time_spent:
raise RecalculationNeeded()
# print("Returned value: {}".format(event_handler.value))
return event_handler.value
def watch(watch_dir='/tmp/wt_screens',
archive_dir='/tmp/wt_archive',
error_dir='/tmp/wt_archive/errors'):
"""Watch a directory and process files going into it"""
observer = Observer()
handler = EventHandler(watch_dir,
archive_dir=archive_dir,
error_dir=error_dir)
print('Starting watch on {}, archive to {}, errors to {}'.format(
watch_dir, archive_dir, error_dir))
observer.schedule(handler, watch_dir, recursive=False)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
print('Stopping directory watch as requested...')
observer.stop()
observer.join()
finally:
if observer.is_alive():
print('Stopping directory watch on unexpected event...')
observer.stop()
observer.join()
class Watcher:
def __init__(self, klasor: Klasor):
self.watchHandler = WatchHandler()
self.observer = None
self.klasor = klasor
self.thread = None
# self.watchHandler.den.alt():
# print("aha")
def Start(self):
self.observer = Observer()
self.observer.schedule(self.watchHandler,
path=str(self.klasor),
recursive=False)
self.observer.start()
print("izleme başladı")
sys.stderr.write("hede")
# sys.stdout.write("eauiea")
hata = sys.stderr
print(hata)
def Stop(self):
self.observer.stop()
self.observer.join()
print("watcher is closed")
print(self.observer.is_alive())
def main() -> None:
"""The main function."""
args = parse_args()
# Check the target
target = args.target
if not os.path.isfile(target):
raise Exception('Target `%s` does not exist' % target)
# Maxmimum number of tasks. Two 2 tasks are required as a minimum: one for
# running afl-showmap, and another for updating the coverage bitmap
max_task = args.jobs
if max_task == 0:
max_task = multiprocessing.cpu_count()
# Check afl-showmap
if not which('afl-showmap'):
raise Exception('Could not find afl-showmap. Check PATH')
# Wait for fuzzer_stats to exist
out_dir = args.out_dir
fuzzer_stats_path = out_dir / 'fuzzer_stats'
if not fuzzer_stats_path.exists():
raise Exception('No fuzzer_stats in `%s`' % out_dir)
# Open CSV plot_data
csv_path = args.csv
if csv_path:
csv_path = Path(csv_path)
with open(csv_path, 'w') as outf:
CsvDictWriter(outf, fieldnames=CSV_FIELDNAMES).writeheader()
with Executor(max_workers=max_task) as executor, \
open(out_dir / 'blackbox.tar.gz', 'wb') as tar_file:
# The coverage bitmap
cov_bitmap = [255] * MAP_SIZE
cov_queue = Queue(max_task)
# Thread responsible for deduplicating entries in the output directory
# and logging coverage to a CSV
cov_thread = Thread(target=remove_duplicate_testcases,
args=(cov_bitmap, cov_queue, tar_file, csv_path))
cov_thread.daemon = True
cov_thread.start()
# Start the monitor
handler = TestCaseHandler(executor, cov_queue, target, afl_stats,
args.timeout)
observer = Observer()
observer.schedule(handler, out_dir / 'queue' / '.blackbox')
observer.start()
# Continue until interrupted
try:
while observer.is_alive():
observer.join(1)
except KeyboardInterrupt:
print('\nCtrl-C detected, goodbye')
observer.stop()
observer.join()
class FileSystemWatchDog(WatchDog):
def __init__(self, watchedFolder: str, watchedExtensions: List[str] = None, recursive=False) -> None:
super().__init__(watchedFolder, watchedExtensions, recursive)
self.observer = None
def run(self):
self.observer = Observer()
self.observer.schedule(WatchDogFileSystemEventHandler(self), self.watchedFolder, recursive=self.recursive)
self.observer.start()
self.observer.join()
@property
def running(self):
return self.observer and self.observer.is_alive()
def stop(self):
if self.observer:
self.observer.stop()
def processFiles(self, fileHandler):
for root, _, filenames in os.walk(self.watchedFolder):
for filename in filenames:
if self.checkExtension(filename):
fileHandler(os.path.join(root, filename))
if not self.recursive:
return
def copyFile(self, srcFilename, destFilename):
shutil.copy(srcFilename, destFilename)
def renameFile(self, srcFilename, destFilename):
os.rename(srcFilename, destFilename)
def process(self, received_file_queue, exception_handler):
"""
The file watcher process.
:param received_file_queue: queue transfering the detected changed files
:type received_file_queue: multiprocessing.queues.Queue
:param exception_handler: exception handler callback method receiving all exception from the process
:type exception_handler: method
"""
filesystem_observer = Observer()
try:
# required for correct handling of Crtl + C in a multiprocess environment
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
self._processed_files_lock = threading.Lock()
self._received_file_queue = received_file_queue
if not os.path.isdir(self._data_directory):
raise IOError("Data directory \"%s\" not found." % self._data_directory)
filesystem_observer.schedule(self, self._data_directory, recursive=True)
filesystem_observer.start()
# wait until stop is signalled
self._exception_event.wait()
except Exception as e:
exception_handler(DelayedException(e))
finally:
if filesystem_observer.is_alive():
filesystem_observer.stop()
filesystem_observer.join()
def sneakernet_import_thread():
"""Add block data dir & confed paths to fs observer to watch for new bls"""
observer = Observer()
for path in watch_paths:
observer.schedule(_Importer(), path, recursive=True)
observer.start()
while observer.is_alive():
# call import func with timeout
observer.join(60)
def auto_refresher():
observer = Observer()
observer.schedule(Refresher(),
identify_home(),
recursive=False)
observer.start()
while observer.is_alive():
# call import func with timeout
observer.join(120)
def replicate_files_on_change(
src_dir,
copy_file,
timeout=None,
use_gitignore=True,
debugging=False,
observer_up_event=None,
terminate_event=None,
):
"""Wait for changes to files in src_dir and copy with copy_file().
If provided, the timeout indicates when to return after that many seconds of no change.
"""
print("debug: replicate on change start")
src_dir = os.path.abspath(src_dir)
if use_gitignore:
spec = get_pathspec(src_dir, use_gitignore)
event_handler = GitIgnoreCopyFileEventHandler(copy_file,
spec,
debugging=debugging)
else:
event_handler = CopyFileEventHandler(copy_file, debugging=debugging)
observer = Observer()
observer.schedule(event_handler, src_dir, recursive=True)
if debugging:
print("Starting observer")
observer.start()
if observer_up_event is not None:
while not observer.is_alive():
pass
# wait for event listeners to settle
time.sleep(0.5)
observer_up_event.set()
print("notified observer up")
try:
while True:
time.sleep(0.5)
if timeout:
raise_if_timeout(event_handler.last_event_timestamp, timeout)
if terminate_event and terminate_event.is_set():
raise ConditionalTermination("Termination condition was set.")
except (KeyboardInterrupt, NoChangeTimeout, ConditionalTermination) as e:
if debugging:
print(f"Exitting on {type(e).__name__}: {e}")
finally:
now = time.time()
# still, dispatch the existing change events before we part ways
while time.time(
) - now < TAIL_TIMEOUT and not observer.event_queue.empty():
print("flushing events")
observer.dispatch_events(observer.event_queue, observer.timeout)
left = now + TAIL_TIMEOUT - time.time()
observer.stop()
observer.join(timeout=max(0, left))
print(f"debug: finished replicate on change with {left} left")
def auto_refresher():
class Refresher(FileSystemEventHandler):
@staticmethod
def on_modified(event):
self.amount = _read_data_file(self.data_file)
observer = Observer()
observer.schedule(Refresher(), usage_file)
observer.start()
while observer.is_alive():
# call import func with timeout
observer.join(120)
def run_watch(handler, path):
obsrv = Observer()
obsrv.schedule(handler, path, recursive=True)
obsrv.start()
print('start watching: %s' % path)
try:
while obsrv.is_alive():
obsrv.join(1)
except KeyboardInterrupt:
obsrv.stop()
print('pywatcher KeyboardInterrupt')
print('pywatcher stop.')
obsrv.join()
def main() -> None:
"""The main function."""
args = parse_args()
# Check the target
target = args.target
if not os.path.isfile(target):
raise Exception('Target `%s` does not exist' % target)
# Maxmimum number of tasks. Two 2 tasks are required as a minimum: one for
# running afl-showmap, and another for updating the coverage bitmap
max_task = args.j
if max_task == 0:
max_task = multiprocessing.cpu_count()
elif max_task <= 2:
max_task = 2
# Check afl-showmap
if not which('afl-showmap'):
raise Exception('Could not find afl-showmap. Check PATH')
# Wait for fuzzer_stats to exist
out_dir = Path(args.out_dir)
fuzzer_stats_path = out_dir / 'fuzzer_stats'
while not fuzzer_stats_path.exists():
sleep(1)
with open(fuzzer_stats_path, 'r') as inf:
afl_stats = FuzzerStats(inf)
# Open CSV plot_data
csv_path = args.csv
if csv_path:
csv_path = Path(csv_path)
with open(csv_path, 'w') as outf:
CsvDictWriter(outf, fieldnames=CSV_FIELDNAMES).writeheader()
# Start the watchdog
handler = TestCaseHandler(max_task - 1, target, afl_stats, csv_path)
observer = Observer()
observer.schedule(handler, out_dir / 'queue' / '.blackbox')
observer.start()
# Continue until interrupted
try:
while observer.is_alive():
observer.join(1)
except KeyboardInterrupt:
print('\nCtrl-C detected, goodbye')
observer.stop()
observer.join()
os.kill(0, 9)
async def watch(path, **kwargs):
"""Watch on files of in a directory and log new lines"""
handler = WatcherHandler(**kwargs)
handler.read_initial_size(path)
observer = Observer()
observer.schedule(handler, path=path, recursive=True)
observer.start()
try:
while observer.is_alive():
await asyncio.sleep(0.1)
finally:
observer.stop()
observer.join()
def main():
global args
args = parser.parse_args()
validate_args(args)
if args.enable == 0 or args.disable:
info('functionality disabled, waiting for termination signal')
signal.signal(signal.SIGINT, handle_term)
signal.signal(signal.SIGTERM, handle_term)
signal.pause()
if args.recognize_regex or args.ignore_regex:
try:
event_handler = RestartRegexMatchingEventHandler(
regexes=args.recognize_regex or ['.*'],
ignore_regexes=args.ignore_regex +
['.*/\..*'] if args.ignore_hidden else [],
ignore_directories=args.ignore_directories,
case_sensitive=args.case_sensitive)
except re.error as exc:
error('regex: ' + str(exc))
else:
event_handler = RestartPatternMatchingEventHandler(
patterns=args.recognize,
ignore_patterns=args.ignore +
['*/.*'] if args.ignore_hidden else [],
ignore_directories=args.ignore_directories,
case_sensitive=args.case_sensitive)
observer = Observer()
observer.schedule(event_handler, args.path, recursive=args.recursive)
try:
global rpc
rpc = childutils.getRPCInterface(os.environ)
except KeyError as exc:
error('missing environment variable ' + str(exc))
info('watching ' + args.path)
try:
observer.start()
except OSError as exc:
error(str(exc))
signal.signal(signal.SIGINT, handle_term)
signal.signal(signal.SIGTERM, handle_term)
while observer.is_alive():
observer.join(1)
def sharpei(
local: str,
remote: str,
initial_sync: bool,
):
"""
Sync LOCAL to REMOTE
LOCAL is a local file or directory
REMOTE is an s3 target of format s3://bucket/[key / key prefix]
"""
local = path.abspath(local)
click.echo(
click.style("Sharpei", fg="cyan", bold=True) +
f" {__version__} watching {local} for changes")
# Break up s3 string
match = re.fullmatch(_s3_path_re, remote)
if not match:
raise click.ClickException(
"Invalid remote path or prefix, must be of format s3://bucket/[path]"
)
s3_bucket, s3_prefix = match.groups()
event_handler = SharpeiEventHandler(
base_path=local,
bucket=s3_bucket,
base_key=s3_prefix or "",
s3_args={}, # TODO
)
if initial_sync:
if path.isfile(local):
initial_sync_event = FileCreatedEvent(local)
else:
initial_sync_event = DirCreatedEvent(local)
initial_sync_event.is_synthetic = True
event_handler.on_created(initial_sync_event)
observer = Observer()
observer.schedule(event_handler, local, recursive=True)
observer.start()
try:
while observer.is_alive():
observer.join(1)
except KeyboardInterrupt:
click.echo(f"Sharpei exiting.")
observer.stop()
observer.join()
class Monitor():
def __init__(self, target_path: str, recursive: bool = False):
self.path = target_path
self.recursive = recursive
self.observer = Observer()
event_handler = FileEventHandler(self.path)
self.observer.schedule(event_handler, self.path, self.recursive)
self.observer.start()
print("Start monitor " + self.path)
try:
while self.observer.is_alive():
self.observer.join(1)
except KeyboardInterrupt:
self.observer.stop()
self.observer.join()
def do_watch(fn, path, pattern, ignore_pattern=None, outfile=None):
outfile = outfile and os.path.normpath(outfile)
try:
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
from watchdog.events import LoggingEventHandler
except ImportError:
msg = """\
watch dog is not found.
please run `pip install "swagger_bundler[watch]`
"""
highlight.show_on_warning(msg)
sys.exit(1)
class _CallbackHandler(PatternMatchingEventHandler):
def process(self, event):
sys.stderr.write("event detect: event_type={}, src={}\n".format(event.event_type, event.src_path))
try:
return fn()
except:
logger.warn("exception", exc_info=True)
def on_any_event(self, event):
if outfile is None or outfile != os.path.normpath(event.src_path):
self.process(event)
observer = Observer()
patterns = [pattern]
ignore_patterns = []
if ignore_pattern is not None:
ignore_patterns.append(ignore_pattern)
sys.stderr.write("watch starting patterns={}, ignore_patterns={}\n".format(patterns, ignore_patterns))
callback_handler = _CallbackHandler(patterns=patterns, ignore_patterns=ignore_patterns)
observer.schedule(LoggingEventHandler(), path=path, recursive=True)
observer.schedule(callback_handler, path=path, recursive=True)
observer.start()
try:
while True:
time.sleep(0.5)
except KeyboardInterrupt:
observer.stop()
try:
while observer.is_alive():
observer.join(1)
except Exception as e:
sys.stderr.write(str(e))
class DirectoryWatcher:
"""A directory watcher class."""
def __init__(
self,
directory: Path,
on_modified: Callable[[Path], None],
) -> None:
"""
Initialize a watcher which observes modifications in `directory`.
on_modified: A callable which is invoked with the path of modified
files within `directory`.
"""
self.on_modified = on_modified
self.watched_directory = str(directory)
self.observer = Observer()
def start(self) -> None:
"""Start watching the specified directory for file modifications."""
event_handler = DirectoryEventHandler(self.on_modified)
self.observer.schedule(
event_handler,
self.watched_directory,
recursive=True,
)
try:
self.observer.start()
except BaseException as e:
logger = logging.getLogger(__name__)
logger.exception(
'Could not start filesystem watcher.\n'
f'Error message: "{e}".\n'
'Set logging level to DEBUG for full stack trace.',
exc_info=logger.getEffectiveLevel() <= logging.DEBUG,
)
def stop(self) -> None:
"""Stop watching the directory."""
if self.observer.is_alive():
try:
self.observer.stop()
self.observer.join()
except (RuntimeError, SystemError):
# TODO: Understand exactly what join() does, and why
# it sometimes throws a RuntimeError
# Also find out why MacOS throws SystemError
pass
class FileWatch(FileSystemEventHandler):
def __init__(self, origin, watch_path, event_func):
FileSystemEventHandler.__init__(self)
self.origin = origin
self.watch_path = watch_path
self.event_func = event_func
self.observer = Observer()
self.watch = None
self.mutex = threading.Lock()
def get_lock(self):
return self.mutex
def start(self):
dir = os.path.abspath(os.path.dirname(self.watch_path))
self.origin.dbg('FileWatch: scheduling watch for directory %s' % dir)
self.watch = self.observer.schedule(self, dir, recursive=False)
self.observer.start()
def stop(self):
if self.watch:
self.origin.dbg('FileWatch: unscheduling watch %r' % self.watch)
self.observer.unschedule(self.watch)
self.watch = None
if self.observer.is_alive():
self.observer.stop()
self.observer.join()
def __del__(self):
self.stop()
self.observer = None
# Override from FileSystemEventHandler
def on_any_event(self, event):
if event.is_directory:
return None
if os.path.abspath(event.src_path) != os.path.abspath(self.watch_path):
return None
self.origin.dbg('FileWatch: received event %r' % event)
try:
self.mutex.acquire()
self.event_func(event)
finally:
self.mutex.release()
class FileSystemObserver(object):
def __init__(self, test_output_dir):
self.test_output_dir = test_output_dir
# Start observing output dir
self.done_event = Event()
self.event_handler = OutputEventHandler(self.done_event)
self.observer = Observer()
self.observer.schedule(self.event_handler,
self.test_output_dir,
recursive=True)
self.observer.start()
def get_output_dir(self):
return self.test_output_dir
def restart_observer_if_needed(self):
if self.observer.is_alive():
return
self.observer = Observer()
self.done_event.clear()
self.observer.schedule(self.event_handler,
self.test_output_dir,
recursive=True)
self.observer.start()
def wait_for_output(self, timeout_seconds, output_validator, max_files):
logging.info('Waiting up to %d seconds for %d test outputs...',
timeout_seconds, max_files)
self.restart_observer_if_needed()
wait_start_time = time.perf_counter()
for i in range(0, max_files):
# Note: The timing on Event.wait() is inaccurate
self.done_event.wait(timeout_seconds)
self.done_event.clear()
current_time = time.perf_counter()
if timeout_seconds < (current_time - wait_start_time
) or output_validator.validate():
break
self.observer.stop()
self.observer.join()
def main():
""":"""
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
# TODO (jam) set this path to whatever dir the user sets in config
path = sys.argv[1] if len(sys.argv) > 1 else "."
event_handler = LoggingEventHandler()
observer = Observer()
observer.schedule(event_handler, path, recursive=True)
observer.start()
try:
while observer.is_alive():
observer.join(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
class IPCom:
"""IPC async thread"""
INSTANCE: Optional["IPCom"] = None
def __init__(self):
self.observer = Observer()
self.observer.start()
self.pid = os.getpid()
def fswatch(self, watcher: FileSystemEventHandler, path: Path, recursive=False):
if not self.observer.is_alive():
# Just in case...
self.observer.start()
return self.observer.schedule(
watcher, str(path.absolute()), recursive=recursive
)
def fsunwatch(self, watcher):
self.observer.unschedule(watcher)
def wait_on_entry_calc(self, key):
with self.lock:
self._reload_cache()
entry = self._get_cache()[key]
if not entry['being_calculated']:
return entry['value']
event_handler = _PickleCore.CacheChangeHandler(
filename=self._cache_fname(), core=self, key=key)
observer = Observer()
event_handler.inject_observer(observer)
observer.schedule(event_handler,
path=self.expended_cache_dir,
recursive=True)
observer.start()
observer.join(timeout=1.0)
if observer.is_alive():
# print('Timedout waiting. Starting again...')
return self.wait_on_entry_calc(key)
# print("Returned value: {}".format(event_handler.value))
return event_handler.value
class Target(FileSystemEventHandler):
def __init__(self, watchDir):
self.observer = None
self.watchDir = watchDir
def run(self):
self.observer = Observer()
self.observer.schedule(self, self.watchDir, recursive=True)
self.observer.start()
try:
while True:
time.sleep(1)
if not self.status():
q.put(self.status())
break
except KeyboardInterrupt:
self.observer.stop()
self.stop()
self.observer.join()
def stop(self):
self.observer.stop()
def status(self):
return self.observer.is_alive()
def on_moved(self, event):
pass
def on_created(self, event):
pass
def on_deleted(self, event):
pass
def on_modified(self, event):
pass
def do_rsync_auto(config, servers, additional_args=None, command=None, run_once=False,
burst_limit=0, verbose=True, local_path=None, remote_path=None):
"""Launch rsync-auto for providers."""
local_path = base_rsync.get_local_path(config, local_path)
remote_path = base_rsync.get_remote_path(config, remote_path)
logging.info(click.style('Doing an initial rsync...', bold=True))
base_rsync.do_rsync(config, servers, additional_args=additional_args,
verbose=verbose, local_path=local_path, remote_path=remote_path)
if command and run_once:
logging.info(click.style('Launching run-once command...', bold=True))
for server in servers:
Thread(
target=base_ssh.do_ssh,
args=(config, [server]),
kwargs={
'command': command,
'verbose': verbose,
},
).start()
command = None
_show_monitoring_message(config)
handler = RsyncHandler(config, servers, additional_args=additional_args, command=command,
burst_limit=burst_limit, run_once=run_once, verbose=verbose,
local_path=local_path, remote_path=remote_path)
observer = Observer()
observer.schedule(handler, path=local_path, recursive=True)
observer.start()
try:
while observer.is_alive():
sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
class DirectoryWatcher:
"""A directory watcher class."""
def __init__(
self,
directory: Path,
on_modified: Callable[[Path], None],
) -> None:
"""
Initialize a watcher which observes modifications in `directory`.
on_modified: A callable which is invoked with the path of modified
files within `directory`.
"""
self.on_modified = on_modified
self.watched_directory = str(directory)
self.observer = Observer()
def start(self) -> None:
"""Start watching the specified directory for file modifications."""
event_handler = DirectoryEventHandler(self.on_modified)
self.observer.schedule(
event_handler,
self.watched_directory,
recursive=True,
)
self.observer.start()
def stop(self) -> None:
"""Stop watching the directory."""
if self.observer.is_alive():
try:
self.observer.stop()
self.observer.join()
except (RuntimeError, SystemError):
# TODO: Understand exactly what join() does, and why
# it sometimes throws a RuntimeError
# Also find out why MacOS throws SystemError
pass
async def _start_monitoring(self, category_name):
observer = None
try:
category = self.bot.config.pics_categories[category_name]
directory = category['send_directory']
event_handler = _FileSystemEventHandler(
directory, self.monitoring_events[category_name],
self.bot.loop)
observer = Observer()
observer.schedule(event_handler, directory)
observer.start()
await self.to_close.wait()
observer.stop()
observer.join()
except asyncio.CancelledError:
if observer is not None and observer.is_alive():
observer.stop()
observer.join()
def start_observer(self, app):
restart_timeout = self._config.get('restart')
lock = threading.Lock()
source = app._observer_source()
app._beforeObserve()
producer = consumer = observer = None
def _default_except_handler(timeout=None):
if consumer is not None and consumer.is_alive():
consumer.stop()
observer = None
if timeout:
time.sleep(timeout)
observer_found = None
producer = LogProducer(app,
lock,
source=source,
logger=self._logger,
watch_everything=self.watch_everything)
consumer = LogConsumer(args=(app, producer, lock, self._logger,
self.consumer_sleep))
consumer.start()
try:
observer = Observer(timeout=self.observer_timeout)
observer.schedule(producer, source, recursive=True)
observer.start()
observer_found = {}
while not self.stop_requested:
if restart_timeout and producer.timestamp is not None:
timestamp = getToday() - producer.timestamp
if timestamp.total_seconds() > restart_timeout:
self._out('restart on producer.timestamp: %s' %
restart_timeout,
is_error=True)
self.restart_requested = True
time.sleep(5)
if app is None or app.is_dead(force=0):
self._out('app is dead', is_error=True)
self.restart_requested = True
app = None
if not observer.is_alive():
self._out('observer is lifeless', is_error=True)
self.restart_requested = True
if not consumer.is_alive():
self._out('cunsumer is lifeless', is_error=True)
self.restart_requested = True
if self.restart_requested:
break
except (OSError, FileNotFoundError) as ex:
self.restart_requested = True
self._out('!!! Observer Exception: %s' % ex, is_error=True)
_default_except_handler(15)
except Exception as ex:
self.restart_requested = True
self._out('!!! Unexpected Exception: %s' % ex, is_error=True)
_default_except_handler(30)
#raise
else:
self._out('==> Stop, stop_requested: %s' % self.stop_requested)
finally:
if producer is not None:
producer.stop()
if observer is not None and observer.is_alive():
observer.stop()
observer.join()
if consumer is not None and consumer.is_alive():
observer_found = consumer.stop()
consumer.join()
if observer_found:
self._found.update(observer_found)
consumer = None
producer = None
observer = None
class FileModifyWatcher(object):
"""Use inotify to watch file-system for file modifications
Usage:
1) subclass the method handle_event, action to be performed
2) create an object passing a list of files to be watched
3) call the loop method
"""
supported_platforms = ('Darwin', 'Linux')
def __init__(self, path_list):
"""@param file_list (list-str): files to be watched"""
self.file_list = set()
self.watch_dirs = set() # all dirs to be watched
self.notify_dirs = set() # dirs that generate notification whatever file
for filename in path_list:
path = os.path.abspath(filename)
if os.path.isfile(path):
self.file_list.add(path)
self.watch_dirs.add(os.path.dirname(path))
else:
self.notify_dirs.add(path)
self.watch_dirs.add(path)
self.platform = get_platform_system()
if self.platform not in self.supported_platforms:
msg = "Unsupported platform '%s'\n" % self.platform
msg += ("'auto' command is supported only on %s" %
(self.supported_platforms,))
raise Exception(msg)
self.observer = Observer()
self.observer.start()
while not self.observer.is_alive():
time.sleep(0.1)
def __del__(self):
self.observer.stop()
self.observer.join()
def _handle(self, event):
"""calls platform specific handler"""
filename = event.src_path
if (filename in self.file_list or
os.path.dirname(filename) in self.notify_dirs):
self.handle_event(event)
def handle_event(self, event):
"""this should be sub-classed """
raise NotImplementedError
def _loop(self):
handler = self._handle
observer = self.observer
class EventHandler(FileSystemEventHandler):
def on_modified(self, event):
try:
handler(event)
except KeyboardInterrupt:
pass
event_handler = EventHandler()
for watch_this in self.watch_dirs:
observer.schedule(event_handler, watch_this, recursive=False)
def loop(self):
"""Infinite loop watching for file modifications
"""
self._loop()
class Merger:
def __init__(self, ledger_file, rules_file, yes_append, csv_files):
self._ledger_file = ledger_file
self._rules_file = rules_file
self._yes_append = yes_append
self._csv_files = csv_files
self._observer = None
def on_rules_modified(event):
print()
self.reload()
self.print_transactions()
self._prompt_append(wait=False)
self._event_handler = PatternMatchingEventHandler([rules_file])
self._event_handler.on_modified = on_rules_modified
self._observer = Observer()
self._observer.schedule(self._event_handler,
os.path.dirname(rules_file))
def __del__(self):
if self._observer and self._observer.is_alive():
self._observer.stop()
self._observer.join()
def _prompt_append(self, wait=True):
click.secho('\nLedger file: %s' % self._ledger_file, fg='yellow')
click.secho('Append these transactions to Ledger file?',
fg='yellow', nl=False)
if wait:
return click.confirm('')
else:
click.secho(' [y/N]: ', nl=False)
return
def reload(self):
print('\rReloading...\n')
rules = Rules(self._rules_file)
self.transactions = []
self.unknown = []
for csv_file in self._csv_files:
click.secho('>>> {}'.format(csv_file), fg='blue')
ts = []
self.transactions.append((csv_file, ts))
for t in get_transactions(csv_file, rules, self._ledger_file):
print(t)
print()
ts.append(t)
if t.account2 == 'expenses:unknown':
self.unknown.append(t)
def print_transactions(self):
if not any(x for (_, x) in self.transactions):
click.secho('No new transactions found')
return False
if self.unknown:
click.secho('{} transactions with unknown account:'
.format(len(self.unknown)), fg='yellow')
for t in self.unknown:
click.secho(' {}: {} (£{:0.2f})'
.format(t.date, t.description, t.amount))
def main(self):
self.reload()
if self.print_transactions() is False:
return
self._observer.start()
if self._ledger_file and (self._yes_append or self._prompt_append()):
print('Appending...', end=' ')
now = datetime.now().replace(microsecond=0)
with open(self._ledger_file, 'ta') as f:
for filename, txns in self.transactions:
print('; Converted from {}'.format(filename), file=f)
print('; [{}]\n'.format(now), file=f)
for t in txns:
print(str(t) + '\n', file=f)
print('done')
class FileMonitor(FileSystemEventHandler):
def __init__(self):
self._observer = Observer()
self._event_callback = {
'modified': [],
'moved': [],
'created': [],
'deleted': [],
}
def start(self, directory, recursive=False):
self._observer.schedule(self, directory, recursive)
self._observer.start()
def reset(self, directory, recursive=False):
if self._observer.is_alive():
self._observer.unschedule_all()
self._observer.stop()
self._observer.join()
self._observer.schedule(self, directory, recursive)
self._observer.start()
def stop(self):
self._observer.unschedule_all()
self._observer.stop()
self._observer.join()
def bind(self, event, func):
if event in self._event_callback:
self._event_callback[event].append(func)
return True
return False
def on_moved(self, event):
"""Called when a file or a directory is moved or renamed.
:param event:
Event representing file/directory movement.
:type event:
:class:`DirMovedEvent` or :class:`FileMovedEvent`
"""
print( 'moved', event.src_path, event.dest_path )
for cb in self._event_callback['moved']:
cb(event)
def on_created(self, event):
"""Called when a file or directory is created.
:param event:
Event representing file/directory creation.
:type event:
:class:`DirCreatedEvent` or :class:`FileCreatedEvent`
"""
print( 'created', event.src_path )
for cb in self._event_callback['created']:
cb(event)
def on_deleted(self, event):
"""Called when a file or directory is deleted.
:param event:
Event representing file/directory deletion.
:type event:
:class:`DirDeletedEvent` or :class:`FileDeletedEvent`
"""
print( 'deleted', event.src_path )
for cb in self._event_callback['deleted']:
cb(event)
def on_modified(self, event):
"""Called when a file or directory is modified.
:param event:
Event representing file/directory modification.
:type event:
:class:`DirModifiedEvent` or :class:`FileModifiedEvent`
"""
print( 'modified', event.src_path )
for cb in self._event_callback['modified']:
cb(event)
class APIDocumentObserver(with_metaclass(SingletonMeta, object)):
"""
``APIDocumentObserver`` is observer of API Documents.
"""
def __init__(self, doc_path=None, doc_index_path=None, doc_file_path_list=None):
"""
Construct method of ``APIDocumentObserver``.
:param doc_path: API Document Directory path
:param doc_index_path: API Document index path
:param doc_file_path_list: Document file list ``ORDER`` in ``index.json``
:return: APIDocumentObserver Singleton instance
"""
from watchdog.observers import Observer
from .document_trace_handler import DocumentTraceHandler
from .document_trace_file import DocumentTraceFile
if doc_path:
if doc_file_path_list and isinstance(doc_file_path_list, list):
tracing_files = [DocumentTraceFile(tracing_file_path=f) for f in doc_file_path_list]
else:
tracing_files = list()
if doc_index_path:
tracing_files.append(DocumentTraceFile(tracing_file_path=doc_index_path, is_index_file=True))
event_handler = DocumentTraceHandler(tracing_files=tracing_files)
self.observer = Observer()
self.observer.schedule(event_handler, doc_path, recursive=True)
else:
raise Exception("doc_path is None.")
def start_watch(self):
"""
Start watch docs
"""
if self.observer:
self.observer.start()
logger.info("Start watchdocs..")
else:
pass
@property
def is_started(self):
"""
After run ``start_watch()``, ``is_started`` is True, or False.
:return: True | False
"""
if self.observer:
return self.observer.is_alive()
return False
def stop_watch(self):
"""
Stop watch docs
"""
if self.observer and self.observer.is_alive():
self.observer.stop()
self.observer.join()
logger.info("Stop watchdocs..")
class Client(LoggerMixin, Process, FileSystemEventHandler):
"""
Responsible for Listener and Processor components. Provides
functions to start/stop both itself and its components. In addition, it is
also capable of receiving file-system events via the 'watchdog' library.
General procedure:
1. Starts both the Process and Listener components.
2. Listen and act upon exit/error notifications from components
3. Listen for file-system events and acts accordingly.
"""
def __init__(self, dsn, watch_path=None, failover_files=None):
"""
To make the client listen for Postgres 'recovery.conf, recovery.done'
events::
from hermes.client import Client
dsn = {'database': 'example_db',
'host': '127.0.0.1',
'port': 5432,
'user': '******',
'password': '******'}
watch_path = '/var/lib/postgresql/9.4/main/'
failover_files = ['recovery.done', 'recovery.conf']
client = Client(dsn, watch_path, failover_files)
# Add processor and listener
...
# Start the client
client.start()
Or, if you decide you don't want to use a file watcher, then you
can omit those parameters. However, the Client will still perform
master/slave checks if a problem is encountered::
from hermes.client import Client
dsn = {'database': 'example_db',
'host': '127.0.0.1',
'port': 5432,
'user': '******',
'password': '******'}
client = Client(dsn)
# Add processor and listener
...
# Start the client
client.start()
:param dsn: A Postgres-compatible DSN dictionary
:param watch_path: The directory to monitor for filechanges. If None,
then file monitoring is disabled.
:param failover_files: A list of files which, when modified, will
cause the client to call :func:`~execute_role_based_procedure`
"""
super(Client, self).__init__()
self.directory_observer = Observer()
self._processor = None
self._listener = None
self._watch_path = watch_path
self._failover_files = failover_files
self.master_pg_conn = PostgresConnector(dsn)
self._should_run = False
self._child_interrupted = False
self._exception_raised = False
self._exit_queue = Queue(1)
def add_processor(self, processor):
"""
:param processor: A :class:`~hermes.components.Component` object which
will receive notifications and run the
:func:`~hermes.components.Component.execute` method.
:raises: :class:`~hermes.exceptions.InvalidConfigurationException` if
the provided processor is not a subclass of
:class:`~hermes.components.Component`
"""
if not isinstance(processor, Component):
raise InvalidConfigurationException(
"Processor must of type Component"
)
self._processor = processor
def add_listener(self, listener):
"""
:param listener: A :class:`~hermes.components.Component` object which
will listen for notifications from Postgres and pass an event down
a queue.
:raises: :class:`~hermes.exceptions.InvalidConfigurationException` if
the provided listener is not a subclass of
:class:`~hermes.components.Component`
"""
if not isinstance(listener, Component):
raise InvalidConfigurationException(
"Listener must of type Component"
)
self._listener = listener
def _validate_components(self):
"""
Checks through a set of validation procedures to ensure the client is
configured properly.
:raises: :class:`~hermes.exceptions.InvalidConfigurationException`
"""
if not self._processor:
raise InvalidConfigurationException("A processor must be defined")
if not self._listener:
raise InvalidConfigurationException("A listener must be defined")
if self._processor.error_queue is not self._listener.error_queue:
raise InvalidConfigurationException(
"A processor and listener's error queue must be the same"
)
def start(self):
"""
Starts the Client, its Components and the directory observer
:raises: :class:`~hermes.exceptions.InvalidConfigurationException`
"""
signal(SIGINT, self._handle_terminate)
signal(SIGTERM, self._handle_terminate)
self._validate_components()
super(Client, self).start()
def run(self):
"""
Performs a :func:`~select.select` on the components' error queue.
When a notification is detected, the client will log the message and
then calculate if the Postgres server is still a Master - if not, the
components are shutdown.
"""
super(Client, self).run()
self._start_observer()
signal(SIGCHLD, self._handle_sigchld)
self._should_run = True
self.execute_role_based_procedure()
while self._should_run:
self._exception_raised = self._child_interrupted = False
try:
exit_pipe = self._exit_queue._reader
ready_pipes, _, _ = select.select(
(exit_pipe, ), (), ()
)
if exit_pipe in ready_pipes:
self.terminate()
except select.error:
if not self._child_interrupted and not self._exception_raised:
self._should_run = False
def _start_components(self, restart=False):
"""
Starts the Processor and Listener if the client is not running
"""
if not self._processor.is_alive():
if restart and self._processor.ident:
self._processor.join()
self._processor.start()
if not self._listener.is_alive():
if restart and self._listener.ident:
self._listener.join()
self._listener.start()
def _stop_components(self):
"""
Stops the Processor and Listener if the client is running
"""
if self._listener and self._listener.ident and self._listener.is_alive():
self._listener.terminate()
self._listener.join()
if self._processor and self._processor.ident and self._processor.is_alive():
self._processor.terminate()
self._processor.join()
def _start_observer(self):
"""
Schedules the observer using 'settings.WATCH_PATH'
"""
if self._watch_path:
self.directory_observer.schedule(
self, self._watch_path, recursive=False
)
self.directory_observer.start()
def _stop_observer(self):
"""
Stops the observer if it is 'alive'
"""
if self._watch_path and self.directory_observer:
if self.directory_observer.is_alive():
self.directory_observer.stop()
def on_any_event(self, event):
"""
Listens to an event passed by 'watchdog' and checks the current
master/slave status
:param event: A :class:`~watchdog.events.FileSystemEvent`
object passed by 'watchdog' indicating an event change within the
specified directory.
"""
file_name = event.src_path.split('/')[-1]
if file_name in self._failover_files:
self.execute_role_based_procedure()
def execute_role_based_procedure(self):
"""
Starts or stops components based on the role (Master/Slave) of the
Postgres host.
Implements a `binary exponential backoff
<http://en.wikipedia.org/wiki/Exponential_backoff
#Binary_exponential_backoff_.2F_truncated_exponential_backoff>`_
up to 32 seconds if it encounters a FATAL connection error.
"""
backoff = 0
while True:
try:
server_is_master = self.master_pg_conn.is_server_master()
if server_is_master:
self.log.warning('Server is a master, starting components')
self._start_components(restart=True)
else:
self.log.warning('Server is a slave, stopping components')
self._stop_components()
break
except OperationalError:
self._stop_components()
self.log.warning(
'Cannot connect to the DB, maybe it has been shutdown?',
exc_info=True
)
if backoff: # pragma: no cover
backoff <<= 1
if backoff > 32:
backoff = 1
else:
backoff = 1
sleep(backoff)
def _handle_sigchld(self, sig, frame):
"""
A child process dying, and the client not shutting down, indicates
a process has been shut down by some external caller.
We must check both the processor and listener for 'liveness' and
start those which have failed.
"""
if sig == SIGCHLD and self._should_run and not self._exception_raised:
try:
expected, action = self._processor.error_queue.get_nowait()
self._exception_raised = True
if expected:
if action == TERMINATE:
self.execute_role_based_procedure()
else:
self.log.critical(
'An unexpected error was raised - shutting down'
)
self._shutdown()
except Empty:
self._child_interrupted = True
self._start_components(restart=True)
def _handle_terminate(self, sig, frame):
"""
Handles SIGINT and SIGTERM signals.
If called from another process then puts to the exit queue, else
calls _shutdown.
"""
if self.ident != os.getpid():
self._exit_queue.put_nowait(True)
else:
self._shutdown()
def _shutdown(self):
"""
Shuts down the Client:
* Sets '_should_run' to False.
* Stops the components.
* Stops the observer.
"""
self.log.warning('Shutting down...')
self._should_run = False
self._stop_components()
self._stop_observer()
class LocalSyncWorker(ConsolidatedEventHandler, metaclass=Singleton):
def __init__(self):
super().__init__()
self.ignore = threading.Event()
try:
user = get_current_user()
except NoResultFound:
# TODO: This only happens when a user logs out and the db has
# been cleared. The app tries to run again without a user being
# being set. This error doesn't disrupt user experience, but we
# might consider tracking down the specific source and preventing
# it from happening.
raise
self.folder = user.folder
self.observer = Observer()
self.observer.schedule(self, self.folder, recursive=True)
def start(self):
logger.debug('Starting watchdog observer')
self.observer.start()
def stop(self):
logger.debug('Stopping LocalSyncWorker')
# observer is actually a separate child thread and must be join()ed
self.observer.stop()
self.join()
def is_alive(self):
return self.observer.is_alive()
def join(self):
self.observer.join()
logger.debug('LocalSyncWorker Stopped')
def dispatch(self, event):
if self.ignore.is_set():
return logger.debug('Ignoring event {}'.format(event))
super().dispatch(event)
def on_moved(self, event):
logger.info('Move event for {}: from {} to {}'.format('directory' if event.is_directory else 'file', event.src_path, event.dest_path))
if is_ignored(event.dest_path):
logger.info('Removing {} {} from DB, it was moved to an ignored file'.format('directory' if event.is_directory else 'file', event.src_path, event.dest_path))
context = OperationContext(local=Path(event.src_path), check_is_folder=False)
return self.put_event(operations.DatabaseDelete(context))
# Note: OperationContext should extrapolate all attributes from what it is given
if event.is_directory:
try:
# TODO: avoid a lazy context load in this case to catch the NodeNotFound exception?
_ = OperationContext(local=Path(event.src_path), is_folder=True).remote
return self.put_event(operations.RemoteMoveFolder(
OperationContext(local=Path(event.src_path), is_folder=True),
OperationContext(local=Path(event.dest_path), is_folder=True),
))
except NodeNotFound:
return self.put_event(operations.RemoteCreateFolder(
OperationContext(local=Path(event.dest_path), is_folder=True),
))
try:
# TODO: avoid a lazy context load in this case to catch the NodeNotFound exception?
_ = OperationContext(local=Path(event.src_path)).remote # noqa
return self.put_event(operations.RemoteMoveFile(
OperationContext(local=Path(event.src_path)),
OperationContext(local=Path(event.dest_path)),
))
except NodeNotFound:
return self.put_event(operations.RemoteCreateFile(
OperationContext(local=Path(event.dest_path)),
))
def on_created(self, event):
logger.info('Creation event for {}: {}'.format('directory' if event.is_directory else 'file', event.src_path))
node = utils.extract_node(event.src_path)
path = Path(event.src_path)
# If the file exists in the database, this is a modification
# This logic may not be the most correct, #TODO re-evaluate
if utils.local_to_db(path, node):
return self.on_modified(event)
context = OperationContext(local=path, node=node)
if event.is_directory:
return self.put_event(operations.RemoteCreateFolder(context))
return self.put_event(operations.RemoteCreateFile(context))
def on_deleted(self, event, *args, is_folder=False, **kwargs):
logger.info('Deletion event for {}: {}'.format('directory' if event.is_directory else 'file', event.src_path))
# A hack: override checking if the passed path is a directory. Since Windows
# emits folder deletion events as file deletes we need to ignore whether or not
# a delete event is for a folder. Since the RemoteDelete operation works identically
# for files and folders we can get away with this here.
context = OperationContext(local=Path(event.src_path), check_is_folder=False)
context.db # noqa
return self.put_event(operations.RemoteDelete(context))
def on_modified(self, event):
logger.info('Modification event for {}: {}'.format('directory' if event.is_directory else 'file', event.src_path))
node = utils.extract_node(event.src_path)
path = Path(event.src_path)
# If the file does not exist in the database, this may be a create
if not utils.local_to_db(path, node):
# for e in self._create_cache:
# if e.src_path == event.src_path:
# logging.warning('Found a duplicate create event {}. Ignoring...'.format(event))
# return
return self.on_created(event)
context = OperationContext(local=Path(event.src_path))
if event.is_directory:
# FIXME: This branch should never be reached, due to a check in dispatch method
logger.error("Received unexpected modification event for folder: {}".format(event.src_path))
return self.put_event(operations.RemoteCreateFolder(context))
return self.put_event(operations.RemoteUpdateFile(context))
def put_event(self, event):
OperationWorker().put(event)
