def create_zettarepl(definition):
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock(return_value=None)
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
return zettarepl
class ZettareplProcess:
def __init__(self, definition, debug_level, log_handler, command_queue,
observer_queue):
self.definition = definition
self.debug_level = debug_level
self.log_handler = log_handler
self.command_queue = command_queue
self.observer_queue = observer_queue
self.zettarepl = None
self.vmware_contexts = {}
def __call__(self):
setproctitle.setproctitle('middlewared (zettarepl)')
osc.die_with_parent()
if logging.getLevelName(self.debug_level) == logging.TRACE:
# If we want TRACE then we want all debug from zettarepl
default_level = logging.DEBUG
elif logging.getLevelName(self.debug_level) == logging.DEBUG:
# Regular development level. We don't need verbose debug from zettarepl
default_level = logging.INFO
else:
default_level = logging.getLevelName(self.debug_level)
setup_logging("", "DEBUG", self.log_handler)
oqlh = ObserverQueueLoggingHandler(self.observer_queue)
oqlh.setFormatter(
logging.Formatter(
'[%(asctime)s] %(levelname)-8s [%(threadName)s] [%(name)s] %(message)s',
'%Y/%m/%d %H:%M:%S'))
logging.getLogger("zettarepl").addHandler(oqlh)
for handler in logging.getLogger("zettarepl").handlers:
handler.addFilter(LongStringsFilter())
handler.addFilter(ReplicationTaskLoggingLevelFilter(default_level))
c = Client('ws+unix:///var/run/middlewared-internal.sock',
py_exceptions=True)
c.subscribe('core.reconfigure_logging',
lambda *args, **kwargs: reconfigure_logging())
definition = Definition.from_data(self.definition,
raise_on_error=False)
self.observer_queue.put(DefinitionErrors(definition.errors))
clock = Clock()
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
self.zettarepl = Zettarepl(scheduler, local_shell)
self.zettarepl.set_observer(self._observer)
self.zettarepl.set_tasks(definition.tasks)
start_daemon_thread(target=self._process_command_queue)
while True:
try:
self.zettarepl.run()
except Exception:
logging.getLogger("zettarepl").error("Unhandled exception",
exc_info=True)
time.sleep(10)
def _observer(self, message):
self.observer_queue.put(message)
logger = logging.getLogger("middlewared.plugins.zettarepl")
try:
if isinstance(
message,
(PeriodicSnapshotTaskStart, PeriodicSnapshotTaskSuccess,
PeriodicSnapshotTaskError)):
task_id = int(message.task_id.split("_")[-1])
if isinstance(message, PeriodicSnapshotTaskStart):
with Client() as c:
context = c.call("vmware.periodic_snapshot_task_begin",
task_id,
job=True)
self.vmware_contexts[task_id] = context
if context and context["vmsynced"]:
# If there were no failures and we successfully took some VMWare snapshots
# set the ZFS property to show the snapshot has consistent VM snapshots
# inside it.
return message.response(
properties={"freenas:vmsynced": "Y"})
if isinstance(
message,
(PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)):
context = self.vmware_contexts.pop(task_id, None)
if context:
with Client() as c:
c.call("vmware.periodic_snapshot_task_end",
context,
job=True)
except ClientException as e:
if e.error:
logger.error(
"Unhandled exception in ZettareplProcess._observer: %r",
e.error)
if e.trace:
logger.error(
"Unhandled exception in ZettareplProcess._observer:\n%s",
e.trace["formatted"])
except Exception:
logger.error("Unhandled exception in ZettareplProcess._observer",
exc_info=True)
def _process_command_queue(self):
logger = logging.getLogger("middlewared.plugins.zettarepl")
while self.zettarepl is not None:
command, args = self.command_queue.get()
if command == "timezone":
self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone(
args)
if command == "tasks":
definition = Definition.from_data(args, raise_on_error=False)
self.observer_queue.put(DefinitionErrors(definition.errors))
self.zettarepl.set_tasks(definition.tasks)
if command == "run_task":
class_name, task_id = args
for task in self.zettarepl.tasks:
if task.__class__.__name__ == class_name and task.id == task_id:
logger.debug("Running task %r", task)
self.zettarepl.scheduler.interrupt([task])
break
else:
logger.warning("Task %s(%r) not found", class_name,
task_id)
self.observer_queue.put(
ReplicationTaskError(task_id, "Task not found"))
class ZettareplProcess:
def __init__(self, definition, debug_level, log_handler, command_queue,
observer_queue):
self.definition = definition
self.debug_level = debug_level
self.log_handler = log_handler
self.command_queue = command_queue
self.observer_queue = observer_queue
self.zettarepl = None
self.vmware_contexts = {}
def __call__(self):
setproctitle.setproctitle('middlewared (zettarepl)')
start_daemon_thread(target=watch_parent)
if logging.getLevelName(self.debug_level) == logging.TRACE:
# If we want TRACE then we want all debug from zettarepl
debug_level = "DEBUG"
elif logging.getLevelName(self.debug_level) == logging.DEBUG:
# Regular development level. We don't need verbose debug from zettarepl
debug_level = "INFO"
else:
debug_level = self.debug_level
setup_logging("", debug_level, self.log_handler)
for handler in logging.getLogger().handlers:
handler.addFilter(LongStringsFilter())
definition = Definition.from_data(self.definition)
clock = Clock()
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
self.zettarepl = Zettarepl(scheduler, local_shell)
self.zettarepl.set_observer(self._observer)
self.zettarepl.set_tasks(definition.tasks)
start_daemon_thread(target=self._process_command_queue)
while True:
try:
self.zettarepl.run()
except Exception:
logging.getLogger("zettarepl").error("Unhandled exception",
exc_info=True)
time.sleep(10)
def _observer(self, message):
self.observer_queue.put(message)
logger = logging.getLogger("middlewared.plugins.zettarepl")
try:
if isinstance(
message,
(PeriodicSnapshotTaskStart, PeriodicSnapshotTaskSuccess,
PeriodicSnapshotTaskError)):
task_id = int(message.task_id.split("_")[-1])
if isinstance(message, PeriodicSnapshotTaskStart):
with Client(py_exceptions=True) as c:
context = c.call("vmware.periodic_snapshot_task_begin",
task_id)
self.vmware_contexts[task_id] = context
if context and context["vmsynced"]:
# If there were no failures and we successfully took some VMWare snapshots
# set the ZFS property to show the snapshot has consistent VM snapshots
# inside it.
return message.response(
properties={"freenas:vmsynced": "Y"})
if isinstance(
message,
(PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)):
context = self.vmware_contexts.pop(task_id, None)
if context:
with Client(py_exceptions=True) as c:
c.call("vmware.periodic_snapshot_task_end",
context)
except Exception:
logger.error("Unhandled exception in ZettareplProcess._observer",
exc_info=True)
def _process_command_queue(self):
logger = logging.getLogger("middlewared.plugins.zettarepl")
while self.zettarepl is not None:
command, args = self.command_queue.get()
if command == "timezone":
self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone(
args)
if command == "tasks":
self.zettarepl.set_tasks(Definition.from_data(args).tasks)
if command == "run_task":
class_name, task_id = args
for task in self.zettarepl.tasks:
if task.__class__.__name__ == class_name and task.id == task_id:
logger.debug("Running task %r", task)
self.zettarepl.scheduler.interrupt([task])
break
else:
logger.warning("Task %s(%r) not found", class_name,
task_id)
def test_push_replication(dst_parent_is_readonly, dst_exists, transport,
properties, compression):
if transport["type"] != "ssh" and compression:
return
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst_parent", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs set test:property=test-value data/src",
shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst_parent", shell=True)
if dst_exists:
subprocess.check_call("zfs create data/dst_parent/dst", shell=True)
if dst_parent_is_readonly:
subprocess.check_call("zfs set readonly=on data/dst_parent",
shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst_parent/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition["replication-tasks"]["src"]["properties"] = properties
if compression:
definition["replication-tasks"]["src"]["compression"] = compression
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock()
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 2
assert (("test-value" in subprocess.check_output(
"zfs get test:property data/dst_parent/dst",
shell=True,
encoding="utf-8")) == properties)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 3
def test_snapshot_gone(transport):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 2
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock()
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
deleted = False
def resume_replications_mock(*args, **kwargs):
nonlocal deleted
if not deleted:
# Snapshots are already listed, and now we remove one of them to simulate PULL replication
# from remote system that has `allow_empty_snapshots: false`. Only do this once.
subprocess.check_call("zfs destroy data/src@2018-10-01_01-00",
shell=True)
deleted = True
return resume_replications(*args, **kwargs)
with patch("zettarepl.replication.run.resume_replications",
resume_replications_mock):
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst", False)) == 1
def test_push_replication(dst_parent_is_readonly, dst_exists, transport, replicate, properties, encrypted,
has_encrypted_child, dst_parent_encrypted):
if replicate and not properties:
return
if encrypted and has_encrypted_child:
# If parent is encrypted, child is also encrypted
return
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst_parent", shell=True)
create_dataset("data/src", encrypted)
subprocess.check_call("zfs set test:property=test-value data/src", shell=True)
if has_encrypted_child:
create_dataset("data/src/child", True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True)
create_dataset("data/dst_parent", dst_parent_encrypted)
if dst_exists:
subprocess.check_call("zfs create data/dst_parent/dst", shell=True)
if dst_parent_is_readonly:
subprocess.check_call("zfs set readonly=on data/dst_parent", shell=True)
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst_parent/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition["replication-tasks"]["src"]["replicate"] = replicate
definition["replication-tasks"]["src"]["properties"] = properties
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock()
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
if dst_exists and properties and encrypted and not dst_parent_encrypted:
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskError), error
assert error.error == ("Unable to send encrypted dataset 'data/src' to existing unencrypted or unrelated "
"dataset 'data/dst_parent/dst'")
return
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 2
assert (
("test-value" in subprocess.check_output("zfs get test:property data/dst_parent/dst",
shell=True, encoding="utf-8")) ==
properties
)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 3
