def run(args):
definition = load_definition(args.definition_path)
clock = Clock(args.once)
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
zettarepl = Zettarepl(scheduler, local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl.run()
def run(args):
try:
definition = Definition.from_data(yaml.load(args.definition_path))
except yaml.YAMLError as e:
sys.stderr.write(f"Definition syntax error: {e!s}\n")
sys.exit(1)
except jsonschema.exceptions.ValidationError as e:
sys.stderr.write(f"Definition validation error: {e!s}\n")
sys.exit(1)
except ValueError as e:
sys.stderr.write(f"{e!s}\n")
sys.exit(1)
clock = Clock(args.once)
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
zettarepl = Zettarepl(scheduler, local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl.run()
class ZettareplProcess:
def __init__(self, definition, debug_level, log_handler, command_queue,
observer_queue):
self.definition = definition
self.debug_level = debug_level
self.log_handler = log_handler
self.command_queue = command_queue
self.observer_queue = observer_queue
self.zettarepl = None
self.vmware_contexts = {}
def __call__(self):
setproctitle.setproctitle('middlewared (zettarepl)')
osc.die_with_parent()
if logging.getLevelName(self.debug_level) == logging.TRACE:
# If we want TRACE then we want all debug from zettarepl
default_level = logging.DEBUG
elif logging.getLevelName(self.debug_level) == logging.DEBUG:
# Regular development level. We don't need verbose debug from zettarepl
default_level = logging.INFO
else:
default_level = logging.getLevelName(self.debug_level)
setup_logging("", "DEBUG", self.log_handler)
oqlh = ObserverQueueLoggingHandler(self.observer_queue)
oqlh.setFormatter(
logging.Formatter(
'[%(asctime)s] %(levelname)-8s [%(threadName)s] [%(name)s] %(message)s',
'%Y/%m/%d %H:%M:%S'))
logging.getLogger("zettarepl").addHandler(oqlh)
for handler in logging.getLogger("zettarepl").handlers:
handler.addFilter(LongStringsFilter())
handler.addFilter(ReplicationTaskLoggingLevelFilter(default_level))
c = Client('ws+unix:///var/run/middlewared-internal.sock',
py_exceptions=True)
c.subscribe('core.reconfigure_logging',
lambda *args, **kwargs: reconfigure_logging())
definition = Definition.from_data(self.definition,
raise_on_error=False)
self.observer_queue.put(DefinitionErrors(definition.errors))
clock = Clock()
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
self.zettarepl = Zettarepl(scheduler, local_shell)
self.zettarepl.set_observer(self._observer)
self.zettarepl.set_tasks(definition.tasks)
start_daemon_thread(target=self._process_command_queue)
while True:
try:
self.zettarepl.run()
except Exception:
logging.getLogger("zettarepl").error("Unhandled exception",
exc_info=True)
time.sleep(10)
def _observer(self, message):
self.observer_queue.put(message)
logger = logging.getLogger("middlewared.plugins.zettarepl")
try:
if isinstance(
message,
(PeriodicSnapshotTaskStart, PeriodicSnapshotTaskSuccess,
PeriodicSnapshotTaskError)):
task_id = int(message.task_id.split("_")[-1])
if isinstance(message, PeriodicSnapshotTaskStart):
with Client() as c:
context = c.call("vmware.periodic_snapshot_task_begin",
task_id,
job=True)
self.vmware_contexts[task_id] = context
if context and context["vmsynced"]:
# If there were no failures and we successfully took some VMWare snapshots
# set the ZFS property to show the snapshot has consistent VM snapshots
# inside it.
return message.response(
properties={"freenas:vmsynced": "Y"})
if isinstance(
message,
(PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)):
context = self.vmware_contexts.pop(task_id, None)
if context:
with Client() as c:
c.call("vmware.periodic_snapshot_task_end",
context,
job=True)
except ClientException as e:
if e.error:
logger.error(
"Unhandled exception in ZettareplProcess._observer: %r",
e.error)
if e.trace:
logger.error(
"Unhandled exception in ZettareplProcess._observer:\n%s",
e.trace["formatted"])
except Exception:
logger.error("Unhandled exception in ZettareplProcess._observer",
exc_info=True)
def _process_command_queue(self):
logger = logging.getLogger("middlewared.plugins.zettarepl")
while self.zettarepl is not None:
command, args = self.command_queue.get()
if command == "timezone":
self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone(
args)
if command == "tasks":
definition = Definition.from_data(args, raise_on_error=False)
self.observer_queue.put(DefinitionErrors(definition.errors))
self.zettarepl.set_tasks(definition.tasks)
if command == "run_task":
class_name, task_id = args
for task in self.zettarepl.tasks:
if task.__class__.__name__ == class_name and task.id == task_id:
logger.debug("Running task %r", task)
self.zettarepl.scheduler.interrupt([task])
break
else:
logger.warning("Task %s(%r) not found", class_name,
task_id)
self.observer_queue.put(
ReplicationTaskError(task_id, "Task not found"))
class ZettareplProcess:
def __init__(self, definition, debug_level, log_handler, command_queue,
observer_queue):
self.definition = definition
self.debug_level = debug_level
self.log_handler = log_handler
self.command_queue = command_queue
self.observer_queue = observer_queue
self.zettarepl = None
self.vmware_contexts = {}
def __call__(self):
setproctitle.setproctitle('middlewared (zettarepl)')
start_daemon_thread(target=watch_parent)
if logging.getLevelName(self.debug_level) == logging.TRACE:
# If we want TRACE then we want all debug from zettarepl
debug_level = "DEBUG"
elif logging.getLevelName(self.debug_level) == logging.DEBUG:
# Regular development level. We don't need verbose debug from zettarepl
debug_level = "INFO"
else:
debug_level = self.debug_level
setup_logging("", debug_level, self.log_handler)
for handler in logging.getLogger().handlers:
handler.addFilter(LongStringsFilter())
definition = Definition.from_data(self.definition)
clock = Clock()
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
self.zettarepl = Zettarepl(scheduler, local_shell)
self.zettarepl.set_observer(self._observer)
self.zettarepl.set_tasks(definition.tasks)
start_daemon_thread(target=self._process_command_queue)
while True:
try:
self.zettarepl.run()
except Exception:
logging.getLogger("zettarepl").error("Unhandled exception",
exc_info=True)
time.sleep(10)
def _observer(self, message):
self.observer_queue.put(message)
logger = logging.getLogger("middlewared.plugins.zettarepl")
try:
if isinstance(
message,
(PeriodicSnapshotTaskStart, PeriodicSnapshotTaskSuccess,
PeriodicSnapshotTaskError)):
task_id = int(message.task_id.split("_")[-1])
if isinstance(message, PeriodicSnapshotTaskStart):
with Client(py_exceptions=True) as c:
context = c.call("vmware.periodic_snapshot_task_begin",
task_id)
self.vmware_contexts[task_id] = context
if context and context["vmsynced"]:
# If there were no failures and we successfully took some VMWare snapshots
# set the ZFS property to show the snapshot has consistent VM snapshots
# inside it.
return message.response(
properties={"freenas:vmsynced": "Y"})
if isinstance(
message,
(PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)):
context = self.vmware_contexts.pop(task_id, None)
if context:
with Client(py_exceptions=True) as c:
c.call("vmware.periodic_snapshot_task_end",
context)
except Exception:
logger.error("Unhandled exception in ZettareplProcess._observer",
exc_info=True)
def _process_command_queue(self):
logger = logging.getLogger("middlewared.plugins.zettarepl")
while self.zettarepl is not None:
command, args = self.command_queue.get()
if command == "timezone":
self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone(
args)
if command == "tasks":
self.zettarepl.set_tasks(Definition.from_data(args).tasks)
if command == "run_task":
class_name, task_id = args
for task in self.zettarepl.tasks:
if task.__class__.__name__ == class_name and task.id == task_id:
logger.debug("Running task %r", task)
self.zettarepl.scheduler.interrupt([task])
break
else:
logger.warning("Task %s(%r) not found", class_name,
task_id)
