def start_watch():
event_handler = when_file_chanage(kill_progress)
observer = Observer(timeout=1)
observer.schedule(event_handler, path=os.getcwd(), recursive=True)
observer.start()
global p, job_name
cmd = ["uwsgi", "--json", job_name]
p = subprocess.Popen(cmd, stderr=subprocess.PIPE)
return_code = p.poll()
while return_code is None:
if not observer.is_alive():
kill_progress()
break
return_code = p.poll()
line = p.stderr.readline().strip().decode("utf-8")
if len(line) != 0:
print(line)
sys.stderr.flush()
time.sleep(0.01)
while len(line) != 0:
line = p.stderr.readline().strip().decode("utf-8")
print(line)
sys.stderr.flush()
observer.stop()
return return_code
def start(self):
path = self.config.get('watchdog', 'path')
patterns = self.config.get('watchdog', 'patterns').split(';')
ignore_directories = self.config.getboolean('watchdog',
'ignore_directories')
ignore_patterns = self.config.get('watchdog',
'ignore_patterns').split(';')
case_sensitive = self.config.getboolean('watchdog', 'case_sensitive')
recursive = self.config.getboolean('watchdog', 'recursive')
event_handler = PatternMatchingEventHandler(
patterns=patterns,
ignore_patterns=ignore_patterns,
ignore_directories=ignore_directories,
case_sensitive=case_sensitive)
event_handler.on_created = self.on_created
# event_handler.on_modified = self.on_modified
observer = Observer()
observer.schedule(path=path,
event_handler=event_handler,
recursive=recursive)
observer.start()
self.logger.info('WatchDog Observer for HCS/AFS/AAS is startting.....')
self.logger.info('patterns=%s' % patterns)
self.logger.info('path=%s' % path)
try:
while observer.is_alive():
time.sleep(1)
except (KeyboardInterrupt):
observer.stop()
self.logger.debug('WatchDog Observer is stoped.')
observer.join()
def compile_contracts(watch, contracts):
"""
Compile project contracts, storing their output in `./build/contracts.json`
Call bare to compile all contracts or specify contract names or file paths
to restrict to only compiling those contracts.
Pass in a file path and a contract name separated by a colon(":") to
specify only named contracts in the specified file.
"""
project_dir = os.getcwd()
click.echo("============ Compiling ==============")
click.echo("> Loading contracts from: {0}".format(
get_contracts_dir(project_dir)))
result = compile_and_write_contracts(project_dir, *contracts)
contract_source_paths, compiled_sources, output_file_path = result
click.echo("> Found {0} contract source files".format(
len(contract_source_paths)))
for path in contract_source_paths:
click.echo("- {0}".format(os.path.basename(path)))
click.echo("")
click.echo("> Compiled {0} contracts".format(len(compiled_sources)))
for contract_name in sorted(compiled_sources.keys()):
click.echo("- {0}".format(contract_name))
click.echo("")
click.echo("> Outfile: {0}".format(output_file_path))
if watch:
# The path to watch
watch_path = utils.get_contracts_dir(project_dir)
click.echo("============ Watching ==============")
event_handler = ContractChangedEventHandler(
project_dir=project_dir,
contract_filters=contracts,
)
observer = PollingObserver()
observer.schedule(event_handler, watch_path, recursive=True)
observer.start()
try:
while observer.is_alive():
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def folderObserver(pathStructure, dbPath):
logging = DefaultLogger()
if pathStructure == None or pathStructure['inBox'] == None:
message = 'Watch: Unable to run as pathStructure is undefined'
logging.debug(message)
return
event_handler = singleFileWatcher(pathStructure, dbPath)
observer = PollingObserver()
observer.schedule(event_handler, pathStructure['inBox'], recursive=False)
observer.start()
try:
while True and observer.is_alive():
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def folderObserver(pathStructure, dbPath):
logging = DefaultLogger()
if pathStructure == None or pathStructure['inBox'] == None:
message = 'Watch: Unable to run as pathStructure is undefined'
logging.debug(message)
return
event_handler = singleFileWatcher(pathStructure, dbPath)
observer = PollingObserver()
observer.schedule(event_handler, pathStructure['inBox'], recursive=False)
observer.start()
try:
while True and observer.is_alive():
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
class DirWatcher(object):
def __init__(self, settings, api, file_pusher):
self._api = api
self._file_count = 0
self._dir = settings.files_dir
self._settings = settings
self._user_file_policies = {"end": set(), "live": set(), "now": set()}
self._file_pusher = file_pusher
self._file_event_handlers = {}
self._file_observer = PollingObserver()
self._file_observer.schedule(self._per_file_event_handler(),
self._dir,
recursive=True)
self._file_observer.start()
logger.info("watching files in: %s", settings.files_dir)
@property
def emitter(self):
try:
return next(iter(self._file_observer.emitters))
except StopIteration:
return None
def update_policy(self, path, policy):
self._user_file_policies[policy].add(path)
for src_path in glob.glob(os.path.join(self._dir, path)):
save_name = os.path.relpath(src_path, self._dir)
self._get_file_event_handler(src_path,
save_name).on_modified(force=True)
def _per_file_event_handler(self):
"""Create a Watchdog file event handler that does different things for every file
"""
file_event_handler = PatternMatchingEventHandler()
file_event_handler.on_created = self._on_file_created
file_event_handler.on_modified = self._on_file_modified
file_event_handler.on_moved = self._on_file_moved
file_event_handler._patterns = [
os.path.join(self._dir, os.path.normpath("*"))
]
# Ignore hidden files/folders
# TODO: what other files should we skip?
file_event_handler._ignore_patterns = [
"*.tmp",
"*.wandb",
"wandb-summary.json",
os.path.join(self._dir, ".*"),
os.path.join(self._dir, "*/.*"),
]
# TODO: pipe in actual settings
for glb in self._settings.ignore_globs:
file_event_handler._ignore_patterns.append(
os.path.join(self._dir, glb))
return file_event_handler
def _on_file_created(self, event):
logger.info("file/dir created: %s", event.src_path)
if os.path.isdir(event.src_path):
return None
self._file_count += 1
# We do the directory scan less often as it grows
if self._file_count % 100 == 0:
emitter = self.emitter
if emitter:
emitter._timeout = int(self._file_count / 100) + 1
save_name = os.path.relpath(event.src_path, self._dir)
self._get_file_event_handler(event.src_path, save_name).on_modified()
def _on_file_modified(self, event):
logger.info("file/dir modified: %s", event.src_path)
if os.path.isdir(event.src_path):
return None
save_name = os.path.relpath(event.src_path, self._dir)
self._get_file_event_handler(event.src_path, save_name).on_modified()
def _on_file_moved(self, event):
# TODO: test me...
logger.info("file/dir moved: %s -> %s", event.src_path,
event.dest_path)
if os.path.isdir(event.dest_path):
return None
old_save_name = os.path.relpath(event.src_path, self._dir)
new_save_name = os.path.relpath(event.dest_path, self._dir)
# We have to move the existing file handler to the new name
handler = self._get_file_event_handler(event.src_path, old_save_name)
self._file_event_handlers[new_save_name] = handler
del self._file_event_handlers[old_save_name]
handler.on_renamed(event.dest_path, new_save_name)
def _get_file_event_handler(self, file_path, save_name):
"""Get or create an event handler for a particular file.
file_path: the file's actual path
save_name: its path relative to the run directory (aka the watch directory)
"""
if save_name not in self._file_event_handlers:
# TODO: we can use PolicyIgnore if there are files we never want to sync
if 'tfevents' in save_name or 'graph.pbtxt' in save_name:
self._file_event_handlers[save_name] = PolicyLive(
file_path, save_name, self._api, self._file_pusher)
else:
Handler = PolicyEnd
for policy, globs in six.iteritems(self._user_file_policies):
if policy == "end":
continue
# Convert set to list to avoid RuntimeError's
# TODO: we may need to add locks
for g in list(globs):
paths = glob.glob(os.path.join(self._dir, g))
if any(save_name in p for p in paths):
if policy == "live":
Handler = PolicyLive
elif policy == "now":
Handler = PolicyNow
self._file_event_handlers[save_name] = Handler(
file_path, save_name, self._api, self._file_pusher)
return self._file_event_handlers[save_name]
def finish(self):
logger.info("shutting down directory watcher")
try:
# avoid hanging if we crashed before the observer was started
if self._file_observer.is_alive():
# rather unfortunatly we need to manually do a final scan of the dir
# with `queue_events`, then iterate through all events before stopping
# the observer to catch all files written. First we need to prevent the
# existing thread from consuming our final events, then we process them
self._file_observer._timeout = 0
self._file_observer._stopped_event.set()
self._file_observer.join()
self.emitter.queue_events(0)
while True:
try:
self._file_observer.dispatch_events(
self._file_observer.event_queue, 0)
except queue.Empty:
break
# Calling stop unschedules any inflight events so we handled them above
self._file_observer.stop()
# TODO: py2 TypeError: PyCObject_AsVoidPtr called with null pointer
except TypeError:
pass
# TODO: py3 SystemError: <built-in function stop> returned an error
except SystemError:
pass
# Ensure we've at least noticed every file in the run directory. Sometimes
# we miss things because asynchronously watching filesystems isn't reliable.
logger.info("scan: %s", self._dir)
for dirpath, _, filenames in os.walk(self._dir):
for fname in filenames:
file_path = os.path.join(dirpath, fname)
save_name = os.path.relpath(file_path, self._dir)
logger.info("scan save: %s %s", file_path, save_name)
self._get_file_event_handler(file_path, save_name).finish()
class Command(BaseCommand):
"""
On every iteration of an infinite loop, consume what we can from the
consumption directory.
"""
# This is here primarily for the tests and is irrelevant in production.
stop_flag = False
def __init__(self, *args, **kwargs):
self.logger = logging.getLogger(__name__)
BaseCommand.__init__(self, *args, **kwargs)
self.observer = None
def add_arguments(self, parser):
parser.add_argument("directory",
default=settings.CONSUMPTION_DIR,
nargs="?",
help="The consumption directory.")
parser.add_argument("--oneshot",
action="store_true",
help="Run only once.")
def handle(self, *args, **options):
directory = options["directory"]
recursive = settings.CONSUMER_RECURSIVE
if not directory:
raise CommandError("CONSUMPTION_DIR does not appear to be set.")
if not os.path.isdir(directory):
raise CommandError(
f"Consumption directory {directory} does not exist")
if recursive:
for dirpath, _, filenames in os.walk(directory):
for filename in filenames:
filepath = os.path.join(dirpath, filename)
_consume(filepath)
else:
for entry in os.scandir(directory):
_consume(entry.path)
if options["oneshot"]:
return
if settings.CONSUMER_POLLING == 0 and INotify:
self.handle_inotify(directory, recursive)
else:
self.handle_polling(directory, recursive)
logger.debug("Consumer exiting.")
def handle_polling(self, directory, recursive):
logging.getLogger(__name__).info(
f"Polling directory for changes: {directory}")
self.observer = PollingObserver(timeout=settings.CONSUMER_POLLING)
self.observer.schedule(Handler(), directory, recursive=recursive)
self.observer.start()
try:
while self.observer.is_alive():
self.observer.join(1)
if self.stop_flag:
self.observer.stop()
except KeyboardInterrupt:
self.observer.stop()
self.observer.join()
def handle_inotify(self, directory, recursive):
logging.getLogger(__name__).info(
f"Using inotify to watch directory for changes: {directory}")
inotify = INotify()
inotify_flags = flags.CLOSE_WRITE | flags.MOVED_TO
if recursive:
descriptor = inotify.add_watch_recursive(directory, inotify_flags)
else:
descriptor = inotify.add_watch(directory, inotify_flags)
try:
while not self.stop_flag:
for event in inotify.read(timeout=1000):
if recursive:
path = inotify.get_path(event.wd)
else:
path = directory
filepath = os.path.join(path, event.name)
_consume(filepath)
except KeyboardInterrupt:
pass
inotify.rm_watch(descriptor)
inotify.close()
def main():
print 'Starting main...'
log.info('--------------------------------------------------------------------')
log.info('HealthPro CSV Ingester service started.')
log.info('Details about database from config file: Server: {}, DB Table: {}, '\
''.format(db_info['host'], healthpro_table_name))
observer = PollingObserver(timeout=5) # check every 5 seconds
try:
if not do_startup_checks():
raise Exception('One or more startup checks failed')
class FSEHandler(FileSystemEventHandler):
# Here we define what we'd like to do when certain filesystem
# events take place -- e.g., when a new CSV appears in the watched
# directory.
# Uncomment on_any_event for extra logging.
#def on_any_event(self, event):
# log.info('FSEHandler->on_any_event: event_type=[' \
# '{}], src_path=[{}]'.format(event.event_type, event.src_path))
def on_deleted(self, event):
# Our forked Watchdog (v0.8.3.1) emits this event when inbox folder
# unmounts (or otherwise is not available).
# We log and send an email only once. It very well might remount without
# us needing to do anything.
global inbox_gone_flag
if event.src_path == inbox_dir:
if not inbox_gone_flag:
inbox_gone_flag = True
msg = event.src_path + ' is gone!'
log.error(msg)
send_notice_email(msg)
def on_created(self, event):
# In on_deleted above, we set the inbox_gone_flag. But if a file appears
# we know the inbox is back and all is well; so unset it.
global inbox_gone_flag
if inbox_gone_flag:
inbox_gone_flag = False
log.info('FSEHandler->on_created: a new file has appeared: '
+ event.src_path)
process_file(event.src_path)
observe_subdirs_flag = False
observer.schedule(FSEHandler(), inbox_dir, observe_subdirs_flag)
observer.start()
log.info('Waiting for activity...')
print 'Service started.'
try:
while True:
observer.join(10)
if observer.is_alive():
continue
else:
raise Exception('Observer thread has stopped.')
except KeyboardInterrupt:
print '\nKeyboard interrupt caught. Quitting.'
observer.stop()
sys.exit(0)
observer.join()
except Exception, ex:
print '\nError caught. Quitting. Check log.'
log.error(str(ex))
send_error_email('An error occurred in main(). Please check.')
observer.stop()
sys.exit(1)
class MonitorFile(PatternMatchingEventHandler):
"""Monitor files and create Beergarden events
This is a wrapper around a watchdog PollingObserver. PollingObserver is used instead
of Observer because Observer throws events on each file transaction.
Note that the events generated are NOT watchdog events, they are whatever
Beergarden events are specified during initialization.
"""
def __init__(
self,
path: str,
create_event: Event = None,
modify_event: Event = None,
moved_event: Event = None,
deleted_event: Event = None,
):
super().__init__(patterns=[path], ignore_directories=True)
self._path = path
self._observer = PollingObserver()
self.create_event = create_event
self.modify_event = modify_event
self.moved_event = moved_event
self.deleted_event = deleted_event
def start(self):
self._observer.schedule(self, Path(self._path).parent, recursive=False)
self._observer.start()
def stop(self):
if self._observer.is_alive():
self._observer.stop()
self._observer.join()
def on_created(self, _):
"""Callback invoked when the file is created
When a user VIM edits a file it DELETES, then CREATES the file, this
captures that case
"""
if self.create_event:
publish(self.create_event)
def on_modified(self, _):
"""Callback invoked when the file is modified
This captures all other modification events that occur against the file
"""
if self.modify_event:
publish(self.modify_event)
def on_moved(self, _):
"""Callback invoked when the file is moved
This captures if the file is moved into or from the directory
"""
if self.moved_event:
publish(self.moved_event)
def on_deleted(self, _):
"""Callback invoked when the file is deleted
This captures if the file was deleted (be warned that VIM does this by
default during write actions)
"""
if self.deleted_event:
publish(self.deleted_event)
def main():
print 'Starting main...'
log.info('--------------------------------------------------------------------')
log.info('HealthPro CSV Ingester service started.')
log.info('Details about database from config file: Server: {}, DB Table: {}. '\
''.format(db_info['host'], healthpro_table_name))
observer = PollingObserver(timeout=5) # check every 5 seconds
try:
if not do_startup_checks():
raise Exception('One or more startup checks failed')
class FSEHandler(FileSystemEventHandler):
# Here we define what we'd like to do when certain filesystem
# events take place -- e.g., when a new CSV appears in the watched
# directory.
# Uncomment on_any_event for extra logging.
#def on_any_event(self, event):
# log.info('FSEHandler->on_any_event: event_type=[' \
# '{}], src_path=[{}]'.format(event.event_type, event.src_path))
def on_deleted(self, event):
# We log and send an email only once.
global inbox_gone_flag
if event.src_path == inbox_dir:
if not inbox_gone_flag:
inbox_gone_flag = True
msg = event.src_path + ' is gone! Please see this SOP: https://nexus.weill.cornell.edu/pages/viewpage.action?pageId=110266114'
log.error(msg)
send_notice_email(msg)
def on_created(self, event):
# In on_deleted above, we set the inbox_gone_flag. But if a file appears
# we know the inbox is back and all is well; so unset it.
global inbox_gone_flag
if inbox_gone_flag:
inbox_gone_flag = False
log.info('FSEHandler->on_created: a new file has appeared: '
+ event.src_path)
process_file(event.src_path)
observe_subdirs_flag = False
observer.schedule(FSEHandler(), inbox_dir, observe_subdirs_flag)
observer.start()
log.info('Waiting for activity...')
print 'Service started.'
if cfg.get('start-telemetry-ping-listener') == 'yes':
start_telemetry_ping_listener()
try:
while True:
observer.join(10)
if observer.is_alive():
continue
else:
raise Exception('Observer thread has stopped.')
except KeyboardInterrupt:
print '\nKeyboard interrupt caught. Quitting.'
observer.stop()
sys.exit(0)
observer.join()
except Exception, ex:
print '\nError caught. Quitting. Check log.'
log.error(str(ex))
send_error_email('An error occurred in main(). Please check.')
observer.stop()
sys.exit(1)
datefmt='%Y-%m-%d %H:%M:%S',
stream=sys.stdout)
GEOSERVER_DATA_DIR = getenv('GEOSERVER_DATA_DIR', '/etc/geoserver')
POLLING_INTERVAL = int(getenv('POLLING_INTERVAL', '5'))
FILE_BLACKLIST = getenv('FILE_BLACKLIST', '.log')
def sig_handler(signum, frame):
"""See signal callback registration: :py:func:`signal.signal`.
This callback performs a clean shutdown when a SIGINT/SIGTERM is received.
"""
logging.info('Received stop signal from signal: %i' % signum)
observer.stop()
# Wire in signal handlers for SIGINT/SIGTERM
signal.signal(signal.SIGINT, sig_handler)
signal.signal(signal.SIGTERM, sig_handler)
# Initialize OS agnostic PollingObserver and handle schedule withGeoServerFileSystemEventHandler.
# This plays nicely with the lack of inotify support over NFS.
event_handler = GeoServerFileSystemEventHandler(POLLING_INTERVAL, FILE_BLACKLIST)
observer = PollingObserver(POLLING_INTERVAL)
observer.schedule(event_handler, GEOSERVER_DATA_DIR, recursive=True)
observer.start()
# Join for a second and then check thread life.
# This ensures exceptions within thread bubble up and cause process shutdown.
while observer.is_alive():
observer.join(1)
GEOSERVER_DATA_DIR = getenv('GEOSERVER_DATA_DIR', '/etc/geoserver')
POLLING_INTERVAL = int(getenv('POLLING_INTERVAL', '5'))
FILE_BLACKLIST = getenv('FILE_BLACKLIST', '.log')
def sig_handler(signum, frame):
"""See signal callback registration: :py:func:`signal.signal`.
This callback performs a clean shutdown when a SIGINT/SIGTERM is received.
"""
logging.info('Received stop signal from signal: %i' % signum)
observer.stop()
# Wire in signal handlers for SIGINT/SIGTERM
signal.signal(signal.SIGINT, sig_handler)
signal.signal(signal.SIGTERM, sig_handler)
# Initialize OS agnostic PollingObserver and handle schedule withGeoServerFileSystemEventHandler.
# This plays nicely with the lack of inotify support over NFS.
event_handler = GeoServerFileSystemEventHandler(POLLING_INTERVAL,
FILE_BLACKLIST)
observer = PollingObserver(POLLING_INTERVAL)
observer.schedule(event_handler, GEOSERVER_DATA_DIR, recursive=True)
observer.start()
# Join for a second and then check thread life.
# This ensures exceptions within thread bubble up and cause process shutdown.
while observer.is_alive():
observer.join(1)
class ConfigsManager(PublisherSubscriberComponent):
"""
This class reads all configurations and sends them over to the "config"
topic in Rabbit MQ. Updated configs are sent as well
"""
def __init__(self,
name: str,
logger: logging.Logger,
config_directory: str,
rabbit_ip: str,
file_patterns: Optional[List[str]] = None,
ignore_file_patterns: Optional[List[str]] = None,
ignore_directories: bool = True,
case_sensitive: bool = False):
"""
Constructs the ConfigsManager instance
:param config_directory: The root config directory to watch.
This is searched recursively.
:param file_patterns: The file patterns in the directory to watch.
Defaults to all ini files
:param ignore_file_patterns: Any file patterns to ignore.
Defaults to None
:param ignore_directories: Whether changes in directories should be
ignored. Default: True
:param case_sensitive: Whether the patterns in `file_patterns` and
`ignore_file_patterns` are case sensitive. Defaults to False
"""
if not file_patterns:
file_patterns = ['*.ini']
self._name = name
self._config_directory = config_directory
self._file_patterns = file_patterns
self._watching = False
self._connected_to_rabbit = False
logger.debug("Creating config RabbitMQ connection")
rabbitmq = RabbitMQApi(logger.getChild("config_{}".format(
RabbitMQApi.__name__)),
host=rabbit_ip)
super().__init__(logger, rabbitmq)
self._logger.debug("Creating heartbeat RabbitMQ connection")
self._heartbeat_rabbit = RabbitMQApi(logger.getChild(
"heartbeat_{}".format(RabbitMQApi.__name__)),
host=rabbit_ip)
self._event_handler = ConfigFileEventHandler(
self._logger.getChild(ConfigFileEventHandler.__name__),
self._on_event_thrown, file_patterns, ignore_file_patterns,
ignore_directories, case_sensitive)
self._observer = PollingObserver()
self._observer.schedule(self._event_handler,
config_directory,
recursive=True)
def __str__(self) -> str:
return self.name
@property
def name(self) -> str:
return self._name
def _initialise_rabbitmq(self) -> None:
while True:
try:
self._connect_to_rabbit()
self._logger.info("Connected to Rabbit")
self.rabbitmq.confirm_delivery()
self._logger.info("Enabled delivery confirmation on configs"
"RabbitMQ channel")
self.rabbitmq.exchange_declare(CONFIG_EXCHANGE, 'topic', False,
True, False, False)
self._logger.info("Declared %s exchange in Rabbit",
CONFIG_EXCHANGE)
self._heartbeat_rabbit.confirm_delivery()
self._logger.info("Enabled delivery confirmation on heartbeat"
"RabbitMQ channel")
self._heartbeat_rabbit.exchange_declare(
HEALTH_CHECK_EXCHANGE, 'topic', False, True, False, False)
self._logger.info("Declared %s exchange in Rabbit",
HEALTH_CHECK_EXCHANGE)
self._logger.info(
"Creating and binding queue '%s' to exchange '%s' with "
"routing key '%s", CONFIG_PING_QUEUE,
HEALTH_CHECK_EXCHANGE, _HEARTBEAT_ROUTING_KEY)
self._heartbeat_rabbit.queue_declare(CONFIG_PING_QUEUE, False,
True, False, False)
self._logger.debug("Declared '%s' queue", CONFIG_PING_QUEUE)
self._heartbeat_rabbit.queue_bind(CONFIG_PING_QUEUE,
HEALTH_CHECK_EXCHANGE,
'ping')
self._logger.debug("Bound queue '%s' to exchange '%s'",
CONFIG_PING_QUEUE, HEALTH_CHECK_EXCHANGE)
# Pre-fetch count is set to 300
prefetch_count = round(300)
self._heartbeat_rabbit.basic_qos(prefetch_count=prefetch_count)
self._logger.debug("Declaring consuming intentions")
self._heartbeat_rabbit.basic_consume(CONFIG_PING_QUEUE,
self._process_ping, True,
False, None)
break
except (ConnectionNotInitialisedException,
AMQPConnectionError) as connection_error:
# Should be impossible, but since exchange_declare can throw
# it we shall ensure to log that the error passed through here
# too.
self._logger.error(
"Something went wrong that meant a connection was not made"
)
self._logger.error(connection_error.message)
raise connection_error
except AMQPChannelError:
# This error would have already been logged by the RabbitMQ
# logger and handled by RabbitMQ. As a result we don't need to
# anything here, just re-try.
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
def _connect_to_rabbit(self) -> None:
if not self._connected_to_rabbit:
self._logger.info("Connecting to the config RabbitMQ")
self.rabbitmq.connect_till_successful()
self._logger.info("Connected to config RabbitMQ")
self._logger.info("Connecting to the heartbeat RabbitMQ")
self._heartbeat_rabbit.connect_till_successful()
self._logger.info("Connected to heartbeat RabbitMQ")
self._connected_to_rabbit = True
else:
self._logger.info(
"Already connected to RabbitMQ, will not connect again")
def disconnect_from_rabbit(self) -> None:
if self._connected_to_rabbit:
self._logger.info("Disconnecting from the config RabbitMQ")
self.rabbitmq.disconnect_till_successful()
self._logger.info("Disconnected from the config RabbitMQ")
self._logger.info("Disconnecting from the heartbeat RabbitMQ")
self._heartbeat_rabbit.disconnect_till_successful()
self._logger.info("Disconnected from the heartbeat RabbitMQ")
self._connected_to_rabbit = False
else:
self._logger.info("Already disconnected from RabbitMQ")
def _send_heartbeat(self, data_to_send: dict) -> None:
self._logger.debug("Sending heartbeat to the %s exchange",
HEALTH_CHECK_EXCHANGE)
self._logger.debug("Sending %s", data_to_send)
self._heartbeat_rabbit.basic_publish_confirm(
exchange=HEALTH_CHECK_EXCHANGE,
routing_key=_HEARTBEAT_ROUTING_KEY,
body=data_to_send,
is_body_dict=True,
properties=pika.BasicProperties(delivery_mode=2),
mandatory=True)
self._logger.debug("Sent heartbeat to %s exchange",
HEALTH_CHECK_EXCHANGE)
def _process_ping(self, ch: BlockingChannel,
method: pika.spec.Basic.Deliver,
properties: pika.spec.BasicProperties,
body: bytes) -> None:
self._logger.debug("Received %s. Let's pong", body)
try:
heartbeat = {
'component_name': self.name,
'is_alive': self._observer.is_alive(),
'timestamp': datetime.now().timestamp(),
}
self._send_heartbeat(heartbeat)
except MessageWasNotDeliveredException as e:
# Log the message and do not raise it as heartbeats must be
# real-time
self._logger.error("Error when sending heartbeat")
self._logger.exception(e)
def _send_data(self, config: Dict[str, Any], route_key: str) -> None:
self._logger.debug("Sending %s with routing key %s", config, route_key)
while True:
try:
self._logger.debug(
"Attempting to send config with routing key %s", route_key)
# We need to definitely send this
self.rabbitmq.basic_publish_confirm(
CONFIG_EXCHANGE,
route_key,
config,
mandatory=True,
is_body_dict=True,
properties=BasicProperties(delivery_mode=2))
self._logger.info("Configuration update sent")
break
except MessageWasNotDeliveredException as mwnde:
self._logger.error(
"Config was not successfully sent with "
"routing key %s", route_key)
self._logger.exception(mwnde)
self._logger.info(
"Will attempt sending the config again with "
"routing key %s", route_key)
self.rabbitmq.connection.sleep(10)
except (ConnectionNotInitialisedException,
AMQPConnectionError) as connection_error:
# If the connection is not initialised or there is a connection
# error, we need to restart the connection and try it again
self._logger.error("There has been a connection error")
self._logger.exception(connection_error)
self._logger.info("Restarting the connection")
self._connected_to_rabbit = False
# Wait some time before reconnecting and then retrying
time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
self._connect_to_rabbit()
self._logger.info(
"Connection restored, will attempt sending "
"the config with routing key %s", route_key)
except AMQPChannelError:
# This error would have already been logged by the RabbitMQ
# logger and handled by RabbitMQ. Since a new channel is created
# we need to re-initialise RabbitMQ
self._initialise_rabbitmq()
def _on_event_thrown(self, event: FileSystemEvent) -> None:
"""
When an event is thrown, it reads the config and sends it as a dict via
rabbitmq to the config exchange of type topic
with the routing key determined by the relative file path.
:param event: The event passed by watchdog
:return None
"""
self._logger.debug("Event thrown: %s", event)
self._logger.info("Detected a config %s in %s", event.event_type,
event.src_path)
if event.event_type == "deleted":
self._logger.debug("Creating empty dict")
config_dict = {}
else:
config = ConfigParser()
self._logger.debug("Reading configuration")
try:
config.read(event.src_path)
except (DuplicateSectionError, DuplicateOptionError,
InterpolationError, ParsingError) as e:
self._logger.error(e.message)
# When the config is invalid, we do nothing and discard this
# event.
return None
self._logger.debug("Config read successfully")
config_dict = {key: dict(config[key]) for key in config}
self._logger.debug("Config converted to dict: %s", config_dict)
# Since the watcher is configured to watch files in
# self._config_directory we only need check that (for get_routing_key)
config_folder = os.path.normpath(self._config_directory)
key = routing_key.get_routing_key(event.src_path, config_folder)
self._logger.debug("Sending config %s to RabbitMQ with routing key %s",
config_dict, key)
self._send_data(config_dict, key)
@property
def config_directory(self) -> str:
return self._config_directory
@property
def watching(self) -> bool:
return self._watching
@property
def connected_to_rabbit(self) -> bool:
return self._connected_to_rabbit
def start(self) -> None:
"""
This method is used to start rabbit and the observer and begin watching
the config files. It also sends the configuration files for the first
time
:return None
"""
log_and_print("{} started.".format(self), self._logger)
self._initialise_rabbitmq()
def do_first_run_event(name: str) -> None:
event = FileSystemEvent(name)
event.event_type = _FIRST_RUN_EVENT
self._on_event_thrown(event)
self._logger.info("Throwing first run event for all config files")
self.foreach_config_file(do_first_run_event)
if not self._watching:
self._logger.info("Starting config file observer")
self._observer.start()
self._watching = True
else:
self._logger.info("File observer is already running")
self._logger.debug("Config file observer started")
self._connect_to_rabbit()
self._listen_for_data()
def _listen_for_data(self) -> None:
self._logger.info("Starting the config ping listener")
self._heartbeat_rabbit.start_consuming()
def _on_terminate(self, signum: int, stack: FrameType) -> None:
"""
This method is used to stop the observer and join the threads
"""
log_and_print(
"{} is terminating. Connections with RabbitMQ will be "
"closed, and afterwards the process will exit.".format(self),
self._logger)
if self._watching:
self._logger.info("Stopping config file observer")
self._observer.stop()
self._observer.join()
self._watching = False
self._logger.debug("Config file observer stopped")
else:
self._logger.info("Config file observer already stopped")
self.disconnect_from_rabbit()
log_and_print("{} terminated.".format(self), self._logger)
sys.exit()
def foreach_config_file(self, callback: Callable[[str], None]) -> None:
"""
Runs a function over all the files being watched by this class
:param callback: The function to watch. Must accept a string for the
file path as {config_directory} + {file path}
:return: Nothing
"""
for root, dirs, files in os.walk(self.config_directory):
for name in files:
if any([
fnmatch.fnmatch(name, pattern)
for pattern in self._file_patterns
]):
callback(os.path.join(root, name))
