def run(args): definition = load_definition(args.definition_path) clock = Clock(args.once) tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() zettarepl = Zettarepl(scheduler, local_shell) zettarepl.set_tasks(definition.tasks) zettarepl.run()
def run(args): try: definition = Definition.from_data(yaml.load(args.definition_path)) except yaml.YAMLError as e: sys.stderr.write(f"Definition syntax error: {e!s}\n") sys.exit(1) except jsonschema.exceptions.ValidationError as e: sys.stderr.write(f"Definition validation error: {e!s}\n") sys.exit(1) except ValueError as e: sys.stderr.write(f"{e!s}\n") sys.exit(1) clock = Clock(args.once) tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() zettarepl = Zettarepl(scheduler, local_shell) zettarepl.set_tasks(definition.tasks) zettarepl.run()
class ZettareplProcess: def __init__(self, definition, debug_level, log_handler, command_queue, observer_queue): self.definition = definition self.debug_level = debug_level self.log_handler = log_handler self.command_queue = command_queue self.observer_queue = observer_queue self.zettarepl = None self.vmware_contexts = {} def __call__(self): setproctitle.setproctitle('middlewared (zettarepl)') osc.die_with_parent() if logging.getLevelName(self.debug_level) == logging.TRACE: # If we want TRACE then we want all debug from zettarepl default_level = logging.DEBUG elif logging.getLevelName(self.debug_level) == logging.DEBUG: # Regular development level. We don't need verbose debug from zettarepl default_level = logging.INFO else: default_level = logging.getLevelName(self.debug_level) setup_logging("", "DEBUG", self.log_handler) oqlh = ObserverQueueLoggingHandler(self.observer_queue) oqlh.setFormatter( logging.Formatter( '[%(asctime)s] %(levelname)-8s [%(threadName)s] [%(name)s] %(message)s', '%Y/%m/%d %H:%M:%S')) logging.getLogger("zettarepl").addHandler(oqlh) for handler in logging.getLogger("zettarepl").handlers: handler.addFilter(LongStringsFilter()) handler.addFilter(ReplicationTaskLoggingLevelFilter(default_level)) c = Client('ws+unix:///var/run/middlewared-internal.sock', py_exceptions=True) c.subscribe('core.reconfigure_logging', lambda *args, **kwargs: reconfigure_logging()) definition = Definition.from_data(self.definition, raise_on_error=False) self.observer_queue.put(DefinitionErrors(definition.errors)) clock = Clock() tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() self.zettarepl = Zettarepl(scheduler, local_shell) self.zettarepl.set_observer(self._observer) self.zettarepl.set_tasks(definition.tasks) start_daemon_thread(target=self._process_command_queue) while True: try: self.zettarepl.run() except Exception: logging.getLogger("zettarepl").error("Unhandled exception", exc_info=True) time.sleep(10) def _observer(self, message): self.observer_queue.put(message) logger = logging.getLogger("middlewared.plugins.zettarepl") try: if isinstance( message, (PeriodicSnapshotTaskStart, PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)): task_id = int(message.task_id.split("_")[-1]) if isinstance(message, PeriodicSnapshotTaskStart): with Client() as c: context = c.call("vmware.periodic_snapshot_task_begin", task_id, job=True) self.vmware_contexts[task_id] = context if context and context["vmsynced"]: # If there were no failures and we successfully took some VMWare snapshots # set the ZFS property to show the snapshot has consistent VM snapshots # inside it. return message.response( properties={"freenas:vmsynced": "Y"}) if isinstance( message, (PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)): context = self.vmware_contexts.pop(task_id, None) if context: with Client() as c: c.call("vmware.periodic_snapshot_task_end", context, job=True) except ClientException as e: if e.error: logger.error( "Unhandled exception in ZettareplProcess._observer: %r", e.error) if e.trace: logger.error( "Unhandled exception in ZettareplProcess._observer:\n%s", e.trace["formatted"]) except Exception: logger.error("Unhandled exception in ZettareplProcess._observer", exc_info=True) def _process_command_queue(self): logger = logging.getLogger("middlewared.plugins.zettarepl") while self.zettarepl is not None: command, args = self.command_queue.get() if command == "timezone": self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone( args) if command == "tasks": definition = Definition.from_data(args, raise_on_error=False) self.observer_queue.put(DefinitionErrors(definition.errors)) self.zettarepl.set_tasks(definition.tasks) if command == "run_task": class_name, task_id = args for task in self.zettarepl.tasks: if task.__class__.__name__ == class_name and task.id == task_id: logger.debug("Running task %r", task) self.zettarepl.scheduler.interrupt([task]) break else: logger.warning("Task %s(%r) not found", class_name, task_id) self.observer_queue.put( ReplicationTaskError(task_id, "Task not found"))
class ZettareplProcess: def __init__(self, definition, debug_level, log_handler, command_queue, observer_queue): self.definition = definition self.debug_level = debug_level self.log_handler = log_handler self.command_queue = command_queue self.observer_queue = observer_queue self.zettarepl = None self.vmware_contexts = {} def __call__(self): setproctitle.setproctitle('middlewared (zettarepl)') start_daemon_thread(target=watch_parent) if logging.getLevelName(self.debug_level) == logging.TRACE: # If we want TRACE then we want all debug from zettarepl debug_level = "DEBUG" elif logging.getLevelName(self.debug_level) == logging.DEBUG: # Regular development level. We don't need verbose debug from zettarepl debug_level = "INFO" else: debug_level = self.debug_level setup_logging("", debug_level, self.log_handler) for handler in logging.getLogger().handlers: handler.addFilter(LongStringsFilter()) definition = Definition.from_data(self.definition) clock = Clock() tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() self.zettarepl = Zettarepl(scheduler, local_shell) self.zettarepl.set_observer(self._observer) self.zettarepl.set_tasks(definition.tasks) start_daemon_thread(target=self._process_command_queue) while True: try: self.zettarepl.run() except Exception: logging.getLogger("zettarepl").error("Unhandled exception", exc_info=True) time.sleep(10) def _observer(self, message): self.observer_queue.put(message) logger = logging.getLogger("middlewared.plugins.zettarepl") try: if isinstance( message, (PeriodicSnapshotTaskStart, PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)): task_id = int(message.task_id.split("_")[-1]) if isinstance(message, PeriodicSnapshotTaskStart): with Client(py_exceptions=True) as c: context = c.call("vmware.periodic_snapshot_task_begin", task_id) self.vmware_contexts[task_id] = context if context and context["vmsynced"]: # If there were no failures and we successfully took some VMWare snapshots # set the ZFS property to show the snapshot has consistent VM snapshots # inside it. return message.response( properties={"freenas:vmsynced": "Y"}) if isinstance( message, (PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)): context = self.vmware_contexts.pop(task_id, None) if context: with Client(py_exceptions=True) as c: c.call("vmware.periodic_snapshot_task_end", context) except Exception: logger.error("Unhandled exception in ZettareplProcess._observer", exc_info=True) def _process_command_queue(self): logger = logging.getLogger("middlewared.plugins.zettarepl") while self.zettarepl is not None: command, args = self.command_queue.get() if command == "timezone": self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone( args) if command == "tasks": self.zettarepl.set_tasks(Definition.from_data(args).tasks) if command == "run_task": class_name, task_id = args for task in self.zettarepl.tasks: if task.__class__.__name__ == class_name and task.id == task_id: logger.debug("Running task %r", task) self.zettarepl.scheduler.interrupt([task]) break else: logger.warning("Task %s(%r) not found", class_name, task_id)