def test__can_run_in_parallel__different_direction(t1, t2, can):
t1.direction = ReplicationDirection.PUSH
t2.direction = ReplicationDirection.PULL
zettarepl = Zettarepl(Mock(), Mock())
assert zettarepl._replication_tasks_can_run_in_parallel(t1, t2) == can
def __call__(self):
if logging.getLevelName(self.debug_level) == logging.TRACE:
# If we want TRACE then we want all debug from zettarepl
debug_level = "DEBUG"
elif logging.getLevelName(self.debug_level) == logging.DEBUG:
# Regular development level. We don't need verbose debug from zettarepl
debug_level = "INFO"
else:
debug_level = self.debug_level
setup_logging("zettarepl", debug_level, self.log_handler)
definition = Definition.from_data(self.definition)
clock = Clock()
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
self.zettarepl = Zettarepl(scheduler, local_shell)
self.zettarepl.set_observer(self._observer)
self.zettarepl.set_tasks(definition.tasks)
start_daemon_thread(target=self._process_command_queue)
while True:
try:
self.zettarepl.run()
except Exception:
logging.getLogger("zettarepl").error("Unhandled exception",
exc_info=True)
time.sleep(10)
def test__can_run_in_parallel__same_direction(t1, t2, can):
with patch("zettarepl.zettarepl.are_same_host", Mock(return_value=True)):
t1.direction = t2.direction = None
zettarepl = Zettarepl(Mock(), Mock())
assert zettarepl._replication_tasks_can_run_in_parallel(t1, t2) == can
def test_push_replication(transport, properties):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs set test:property=test-value data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition["replication-tasks"]["src"]["properties"] = properties
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 2
assert (
("test-value" in subprocess.check_output("zfs get test:property data/dst", shell=True, encoding="utf-8")) ==
properties
)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 3
def test__run_periodic_snapshot_tasks__recursive():
with patch("zettarepl.zettarepl.create_snapshot") as create_snapshot:
with patch("zettarepl.zettarepl.get_empty_snapshots_for_deletion", Mock(return_value=[])):
zettarepl = Zettarepl(Mock(), Mock())
zettarepl._run_periodic_snapshot_tasks(
datetime(2018, 9, 1, 15, 11),
[
Mock(dataset="data", recursive=False, naming_schema="snap-%Y-%m-%d_%H-%M"),
Mock(dataset="data", recursive=True, naming_schema="snap-%Y-%m-%d_%H-%M"),
]
)
create_snapshot.assert_called_once_with(ANY, Snapshot("data", "snap-2018-09-01_15-11"), True, ANY)
def __call__(self):
setproctitle.setproctitle('middlewared (zettarepl)')
osc.die_with_parent()
if logging.getLevelName(self.debug_level) == logging.TRACE:
# If we want TRACE then we want all debug from zettarepl
default_level = logging.DEBUG
elif logging.getLevelName(self.debug_level) == logging.DEBUG:
# Regular development level. We don't need verbose debug from zettarepl
default_level = logging.INFO
else:
default_level = logging.getLevelName(self.debug_level)
setup_logging("", "DEBUG", self.log_handler)
oqlh = ObserverQueueLoggingHandler(self.observer_queue)
oqlh.setFormatter(
logging.Formatter(
'[%(asctime)s] %(levelname)-8s [%(threadName)s] [%(name)s] %(message)s',
'%Y/%m/%d %H:%M:%S'))
logging.getLogger("zettarepl").addHandler(oqlh)
for handler in logging.getLogger("zettarepl").handlers:
handler.addFilter(LongStringsFilter())
handler.addFilter(ReplicationTaskLoggingLevelFilter(default_level))
c = Client('ws+unix:///var/run/middlewared-internal.sock',
py_exceptions=True)
c.subscribe(
'core.reconfigure_logging',
lambda *args, **kwargs: reconfigure_logging('zettarepl_file'))
definition = Definition.from_data(self.definition,
raise_on_error=False)
self.observer_queue.put(DefinitionErrors(definition.errors))
clock = Clock()
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
self.zettarepl = Zettarepl(scheduler, local_shell)
self.zettarepl.set_observer(self._observer)
self.zettarepl.set_tasks(definition.tasks)
start_daemon_thread(target=self._process_command_queue)
while True:
try:
self.zettarepl.run()
except Exception:
logging.getLogger("zettarepl").error("Unhandled exception",
exc_info=True)
time.sleep(10)
def test__replication_tasks_for_periodic_snapshot_tasks():
zettarepl = Zettarepl(Mock(), Mock())
pst1 = Mock()
pst2 = Mock()
pst3 = Mock()
rt1 = Mock(spec=ReplicationTask)
rt1.periodic_snapshot_tasks = [pst1, pst2]
rt2 = Mock(spec=ReplicationTask)
rt2.periodic_snapshot_tasks = []
assert zettarepl._replication_tasks_for_periodic_snapshot_tasks([rt1, rt2], [pst1, pst3]) == [rt1]
def test__transport_for_replication_tasks():
zettarepl = Zettarepl(Mock(), Mock())
t1 = Mock()
t2 = Mock()
rt1 = Mock(transport=t1)
rt2 = Mock(transport=t2)
rt3 = Mock(transport=t1)
assert sorted(zettarepl._transport_for_replication_tasks([rt1, rt2, rt3]),
key=lambda t: [t1, t2].index(t[0])) == [
(t1, [rt1, rt3]),
(t2, [rt2]),
]
def test_push_replication():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
definition = Definition.from_data(yaml.load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
""")))
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 2
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 3
def test__run_periodic_snapshot_tasks__alphabetical():
with patch("zettarepl.zettarepl.create_snapshot") as create_snapshot:
with patch("zettarepl.zettarepl.get_empty_snapshots_for_deletion", Mock(return_value=[])):
zettarepl = Zettarepl(Mock(), Mock())
zettarepl._run_periodic_snapshot_tasks(
datetime(2018, 9, 1, 15, 11),
[
Mock(dataset="data", recursive=False, naming_schema="snap-%Y-%m-%d_%H-%M-2d"),
Mock(dataset="data", recursive=False, naming_schema="snap-%Y-%m-%d_%H-%M-1w"),
]
)
assert create_snapshot.call_count == 2
create_snapshot.assert_has_calls([
call(ANY, Snapshot("data", "snap-2018-09-01_15-11-1w"), False, ANY),
call(ANY, Snapshot("data", "snap-2018-09-01_15-11-2d"), False, ANY),
])
def create_zettarepl(definition):
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock(return_value=None)
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
return zettarepl
def test_replication_resume(caplog, transport):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("dd if=/dev/zero of=/mnt/data/src/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("(zfs send data/src@2018-10-01_01-00 | throttle -b 102400 | zfs recv -s -F data/dst) & "
"sleep 1; killall zfs", shell=True)
assert "receive_resume_token\t1-" in subprocess.check_output("zfs get -H receive_resume_token data/dst",
shell=True, encoding="utf-8")
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition = Definition.from_data(definition)
caplog.set_level(logging.INFO)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert any(
"Resuming replication for destination dataset" in record.message
for record in caplog.get_records("call")
)
assert len(list_snapshots(local_shell, "data/dst", False)) == 1
def test_does_not_remove_the_last_snapshot_left():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2020-05-07_00-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2020-05-23_00-00", shell=True)
data = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: false
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "*"
hour: "*"
day-of-month: "*"
month: "*"
day-of-week: "*"
lifetime: P30D
"""))
definition = Definition.from_data(data)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl._run_local_retention(datetime(2020, 6, 25, 0, 0))
assert list_snapshots(local_shell, "data/src",
False) == [Snapshot("data/src", "2020-05-23_00-00")]
def run(args):
definition = load_definition(args.definition_path)
clock = Clock(args.once)
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
zettarepl = Zettarepl(scheduler, local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl.run()
def test_zvol_replication(as_root):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
if as_root:
subprocess.check_call("zfs create -V 1M data/src", shell=True)
else:
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create -V 1M data/src/zvol", shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00",
shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00",
shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert len(list_snapshots(local_shell, "data/dst", False)) == 2
if not as_root:
assert len(list_snapshots(local_shell, "data/dst/zvol", False)) == 2
def test_source_retention_multiple_sources():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/a", shell=True)
subprocess.check_call("zfs create data/src/b", shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
subprocess.check_call("zfs snapshot -r data/dst@2018-10-01_00-00", shell=True)
subprocess.check_call("zfs snapshot -r data/dst@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot -r data/dst@2018-10-01_02-00", shell=True)
definition = Definition.from_data(yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: [data/src/a, data/src/b]
target-dataset: data/dst
recursive: false
periodic-snapshot-tasks:
- src
auto: true
retention-policy: source
hold-pending-snapshots: true
""")))
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl._run_remote_retention(datetime(2018, 10, 1, 3, 0))
assert list_snapshots(local_shell, "data/dst/a", False) == [Snapshot("data/dst/a", "2018-10-01_02-00")]
assert list_snapshots(local_shell, "data/dst/b", False) == [Snapshot("data/dst/b", "2018-10-01_02-00")]
def test_push_remote_retention(retention_policy, remains):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_00-00", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_03-00", shell=True)
data = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
"""))
data["replication-tasks"]["src"].update(**retention_policy)
definition = Definition.from_data(data)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl._run_remote_retention(datetime(2018, 10, 1, 3, 0))
assert list_snapshots(local_shell, "data/dst", False) == remains
def test_hold_pending_snapshots(hold_pending_snapshots, remains):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_00-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_00-00", shell=True)
definition = Definition.from_data(
yaml.load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
- id: src
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
- id: src
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: source
hold-pending-snapshots: """ + yaml.dump(hold_pending_snapshots) +
"""
""")))
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl._run_local_retention(datetime(2018, 10, 1, 3, 0))
assert list_snapshots(local_shell, "data/src", False) == remains
def test_hold_pending_snapshots(retention_policy, remains):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_00-00", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs snapshot data/dst@2018-10-01_03-00", shell=True)
data = yaml.load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
- id: src
direction: pull
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
naming-schema: "%Y-%m-%d_%H-%M"
recursive: true
auto: true
"""))
data["replication-tasks"][0].update(**retention_policy)
definition = Definition.from_data(data)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl._run_local_retention(datetime(2018, 10, 1, 3, 0))
assert list_snapshots(local_shell, "data/dst", False) == remains
def run(args):
try:
definition = Definition.from_data(yaml.load(args.definition_path))
except yaml.YAMLError as e:
sys.stderr.write(f"Definition syntax error: {e!s}\n")
sys.exit(1)
except jsonschema.exceptions.ValidationError as e:
sys.stderr.write(f"Definition validation error: {e!s}\n")
sys.exit(1)
except ValueError as e:
sys.stderr.write(f"{e!s}\n")
sys.exit(1)
clock = Clock(args.once)
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
zettarepl = Zettarepl(scheduler, local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl.run()
def test_does_not_remove_the_last_snapshot_left(snapshots__removal_dates__result):
snapshots, removal_dates, result = snapshots__removal_dates__result
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/src2", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/child", shell=True)
subprocess.check_call("zfs create data/src2", shell=True)
for snapshot in snapshots:
subprocess.check_call(f"zfs snapshot {snapshot}", shell=True)
data = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: false
naming-schema: "%Y-%m-%d-%H-%M"
schedule:
minute: "*"
hour: "*"
day-of-month: "*"
month: "*"
day-of-week: "*"
lifetime: P30D
"""))
definition = Definition.from_data(data)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell, use_removal_dates=True)
zettarepl.set_tasks(definition.tasks)
with patch("zettarepl.zettarepl.get_removal_dates", Mock(return_value=removal_dates)):
zettarepl._run_local_retention(datetime(2021, 4, 19, 17, 0))
assert list_snapshots(local_shell, "data/src", False) + list_snapshots(local_shell, "data/src2", False) == [
snapshots[i] for i in result
]
def test_pull_replication():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
definition = Definition.from_data(
yaml.load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
- id: src
direction: pull
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
naming-schema:
- "%Y-%m-%d_%H-%M"
auto: true
retention-policy: none
""")))
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl.set_tasks(definition.tasks)
zettarepl._run_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
assert len(list_snapshots(local_shell, "data/dst", False)) == 2
class ZettareplProcess:
def __init__(self, definition, debug_level, log_handler, command_queue,
observer_queue):
self.definition = definition
self.debug_level = debug_level
self.log_handler = log_handler
self.command_queue = command_queue
self.observer_queue = observer_queue
self.zettarepl = None
self.vmware_contexts = {}
def __call__(self):
setproctitle.setproctitle('middlewared (zettarepl)')
start_daemon_thread(target=watch_parent)
if logging.getLevelName(self.debug_level) == logging.TRACE:
# If we want TRACE then we want all debug from zettarepl
debug_level = "DEBUG"
elif logging.getLevelName(self.debug_level) == logging.DEBUG:
# Regular development level. We don't need verbose debug from zettarepl
debug_level = "INFO"
else:
debug_level = self.debug_level
setup_logging("", debug_level, self.log_handler)
for handler in logging.getLogger().handlers:
handler.addFilter(LongStringsFilter())
definition = Definition.from_data(self.definition)
clock = Clock()
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
self.zettarepl = Zettarepl(scheduler, local_shell)
self.zettarepl.set_observer(self._observer)
self.zettarepl.set_tasks(definition.tasks)
start_daemon_thread(target=self._process_command_queue)
while True:
try:
self.zettarepl.run()
except Exception:
logging.getLogger("zettarepl").error("Unhandled exception",
exc_info=True)
time.sleep(10)
def _observer(self, message):
self.observer_queue.put(message)
logger = logging.getLogger("middlewared.plugins.zettarepl")
try:
if isinstance(
message,
(PeriodicSnapshotTaskStart, PeriodicSnapshotTaskSuccess,
PeriodicSnapshotTaskError)):
task_id = int(message.task_id.split("_")[-1])
if isinstance(message, PeriodicSnapshotTaskStart):
with Client(py_exceptions=True) as c:
context = c.call("vmware.periodic_snapshot_task_begin",
task_id)
self.vmware_contexts[task_id] = context
if context and context["vmsynced"]:
# If there were no failures and we successfully took some VMWare snapshots
# set the ZFS property to show the snapshot has consistent VM snapshots
# inside it.
return message.response(
properties={"freenas:vmsynced": "Y"})
if isinstance(
message,
(PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)):
context = self.vmware_contexts.pop(task_id, None)
if context:
with Client(py_exceptions=True) as c:
c.call("vmware.periodic_snapshot_task_end",
context)
except Exception:
logger.error("Unhandled exception in ZettareplProcess._observer",
exc_info=True)
def _process_command_queue(self):
logger = logging.getLogger("middlewared.plugins.zettarepl")
while self.zettarepl is not None:
command, args = self.command_queue.get()
if command == "timezone":
self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone(
args)
if command == "tasks":
self.zettarepl.set_tasks(Definition.from_data(args).tasks)
if command == "run_task":
class_name, task_id = args
for task in self.zettarepl.tasks:
if task.__class__.__name__ == class_name and task.id == task_id:
logger.debug("Running task %r", task)
self.zettarepl.scheduler.interrupt([task])
break
else:
logger.warning("Task %s(%r) not found", class_name,
task_id)
def test_push_replication(dst_parent_is_readonly, dst_exists, transport,
properties, compression):
if transport["type"] != "ssh" and compression:
return
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst_parent", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs set test:property=test-value data/src",
shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
subprocess.check_call("zfs create data/dst_parent", shell=True)
if dst_exists:
subprocess.check_call("zfs create data/dst_parent/dst", shell=True)
if dst_parent_is_readonly:
subprocess.check_call("zfs set readonly=on data/dst_parent",
shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst_parent/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition["replication-tasks"]["src"]["properties"] = properties
if compression:
definition["replication-tasks"]["src"]["compression"] = compression
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock()
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 2
assert (("test-value" in subprocess.check_output(
"zfs get test:property data/dst_parent/dst",
shell=True,
encoding="utf-8")) == properties)
subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 3
def test_preserves_clone_origin():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/iocage", shell=True)
subprocess.check_call("zfs create data/src/iocage/child", shell=True)
subprocess.check_call("zfs create data/src/iocage/child/dataset",
shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/iocage/child/dataset/blob bs=1M count=1",
shell=True)
subprocess.check_call("zfs snapshot -r data/src@2019-11-08_14-00",
shell=True)
subprocess.check_call("zfs create data/src/iocage/another", shell=True)
subprocess.check_call("zfs create data/src/iocage/another/child",
shell=True)
subprocess.check_call(
"zfs clone data/src/iocage/child/dataset@2019-11-08_14-00 "
"data/src/iocage/another/child/clone",
shell=True)
subprocess.check_call("zfs snapshot -r data/src@2019-11-08_15-00",
shell=True)
assert (subprocess.check_output(
"zfs get -H origin data/src/iocage/another/child/clone",
encoding="utf-8",
shell=True).split("\n")[0].split("\t")[2] ==
"data/src/iocage/child/dataset@2019-11-08_14-00")
assert int(
subprocess.check_output(
"zfs get -H -p used data/src/iocage/another/child/clone",
encoding="utf-8",
shell=True).split("\n")[0].split("\t")[2]) < 2e6
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
replication-tasks:
src:
direction: push
transport:
type: local
source-dataset: data/src
target-dataset: data/dst
recursive: true
properties: true
replicate: true
also-include-naming-schema:
- "%Y-%m-%d_%H-%M"
auto: false
retention-policy: none
retries: 1
"""))
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
assert (subprocess.check_output(
"zfs get -H origin data/dst/iocage/another/child/clone",
encoding="utf-8",
shell=True).split("\n")[0].split("\t")[2] ==
"data/dst/iocage/child/dataset@2019-11-08_14-00")
assert int(
subprocess.check_output(
"zfs get -H -p used data/dst/iocage/another/child/clone",
encoding="utf-8",
shell=True).split("\n")[0].split("\t")[2]) < 2e6
def test_parallel_replication():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/a", shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/a/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/a@2018-10-01_01-00",
shell=True)
subprocess.check_call("zfs create data/src/b", shell=True)
subprocess.check_call(
"dd if=/dev/urandom of=/mnt/data/src/b/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/b@2018-10-01_01-00",
shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
definition = Definition.from_data(
yaml.load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src-a:
dataset: data/src/a
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
src-b:
dataset: data/src/b
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src-a:
direction: push
transport:
type: ssh
hostname: localhost
private-key: |
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEA6+D7AWdNnM8T5f2P1j5VIwVABjugtL252iEhhGWTNaR2duCK
kuZmG3+o55b0vo2mUpjWt+CKsLkVL/e/JgZqzlHYm+MhRs4q7zODo/IEtx/uAVKM
zqWS0Zs9NRdU1UnhsETjrhhQhNFwx7MUYlB2s8+mAduLbeqRuVIKsXzg5Rz+m3VL
Wl4R82A10oZl0UPyIcEHAtHMgMVyGzQcNUsxsp/oN40nnkXiXHaXtSMxjEtOPLno
t21bJ0ZV8RFmXtJqHXgyTTM5maJM3JwqhMHD2tcHodqCcnxvuuWv31pAB+HKQ8IM
dORYnhZqqs/Bt80gRLQuJBpNeX2/cKPDDMCRnQIDAQABAoIBAQCil6+N9R5rw9Ys
iA85GDhpbnoGkd2iGNHeiU3oTHgf1uEN6pO61PR3ahUMpmLIYy3N66q+jxoq3Tm8
meL6HBxNYd+U/Qh4HS89OV45iV80t97ArJ2A6GL+9ypGyXFhoI7giWwEGqCOHSzH
iyq25k4cfjspNqOyval7fBEA7Vq8smAMDJQE7WIJWzqrTbVAmVf9ho4r5dYxYBNW
fXWo84DU8K+p0mE0BTokqqMWhKiA5JJG7OZB/iyeW2BWFOdASXvQmh1hRwMzpU4q
BcZ7cJHz248SNSGMe5R3w7SmLO7PRr1/QkktJNdFmT7o/RGmQh8+KHql6r/vIzMM
ci60OAxlAoGBAPYsZJZF3HK70fK3kARSzOD1LEVBDTCLnpVVzMSp6thG8cQqfCI5
pCfT/NcUsCAP6J+yl6dqdtonXISmGolI1s1KCBihs5D4jEdjbg9KbKh68AsHXaD3
v5L3POJ9hQnI6zJdvCfxniHdUArfyYhqsp1bnCn+85g4ed7BzDqMX2IDAoGBAPVL
Y45rALw7lsjxJndyFdffJtyAeuwxgJNwWGuY21xhwqPbuwsgLHsGerHNKB5QAJT8
JOlrcrfC13s6Tt4wmIy/o2h1p9tMaitmVR6pJzEfHyJhSRTbeFybQ9yqlKHuk2tI
jcUZV/59cyRrjhPKWoVym3Fh/P7D1t1kfdTvBrvfAoGAUH0rVkb5UTo/5xBFsmQw
QM1o8CvY2CqOa11mWlcERjrMCcuqUrZuCeeyH9DP1WveL3kBROf2fFWqVmTJAGIk
eXLfOs6EG75of17vOWioJl4r5i8+WccniDH2YkeQHCbpX8puHtFNVt05spSBHG1m
gTTW1pRZqUet8TuEPxBuj2kCgYAVjCrRruqgnmdvfWeQpI/wp6SlSBAEQZD24q6R
vRq/8cKEXGAA6TGfGQGcLtZwWzzB2ahwbMTmCZKeO5AECqbL7mWvXm6BYCQPbeza
Raews/grL/qYf3MCR41djAqEcw22Jeh2QPSu4VxE/cG8UVFEWb335tCvnIp6ZkJ7
ewfPZwKBgEnc8HH1aq8IJ6vRBePNu6M9ON6PB9qW+ZHHcy47bcGogvYRQk1Ng77G
LdZpyjWzzmb0Z4kjEYcrlGdbNQf9iaT0r+SJPzwBDG15+fRqK7EJI00UhjB0T67M
otrkElxOBGqHSOl0jfUBrpSkSHiy0kDc3/cTAWKn0gowaznSlR9N
-----END RSA PRIVATE KEY-----
host-key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKwIWcodi3rHl0zS3VaddXnwfgIcpp1ECR9KhaB1cyIspgcHOA98wxY8onw+zHxpMUB4pve+t416FFLSkmlJ2f4="
source-dataset: data/src/a
target-dataset: data/dst/a
recursive: true
periodic-snapshot-tasks:
- src-a
auto: true
retention-policy: none
speed-limit: 100000
src-b:
direction: push
transport:
type: ssh
hostname: localhost
private-key: |
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEA6+D7AWdNnM8T5f2P1j5VIwVABjugtL252iEhhGWTNaR2duCK
kuZmG3+o55b0vo2mUpjWt+CKsLkVL/e/JgZqzlHYm+MhRs4q7zODo/IEtx/uAVKM
zqWS0Zs9NRdU1UnhsETjrhhQhNFwx7MUYlB2s8+mAduLbeqRuVIKsXzg5Rz+m3VL
Wl4R82A10oZl0UPyIcEHAtHMgMVyGzQcNUsxsp/oN40nnkXiXHaXtSMxjEtOPLno
t21bJ0ZV8RFmXtJqHXgyTTM5maJM3JwqhMHD2tcHodqCcnxvuuWv31pAB+HKQ8IM
dORYnhZqqs/Bt80gRLQuJBpNeX2/cKPDDMCRnQIDAQABAoIBAQCil6+N9R5rw9Ys
iA85GDhpbnoGkd2iGNHeiU3oTHgf1uEN6pO61PR3ahUMpmLIYy3N66q+jxoq3Tm8
meL6HBxNYd+U/Qh4HS89OV45iV80t97ArJ2A6GL+9ypGyXFhoI7giWwEGqCOHSzH
iyq25k4cfjspNqOyval7fBEA7Vq8smAMDJQE7WIJWzqrTbVAmVf9ho4r5dYxYBNW
fXWo84DU8K+p0mE0BTokqqMWhKiA5JJG7OZB/iyeW2BWFOdASXvQmh1hRwMzpU4q
BcZ7cJHz248SNSGMe5R3w7SmLO7PRr1/QkktJNdFmT7o/RGmQh8+KHql6r/vIzMM
ci60OAxlAoGBAPYsZJZF3HK70fK3kARSzOD1LEVBDTCLnpVVzMSp6thG8cQqfCI5
pCfT/NcUsCAP6J+yl6dqdtonXISmGolI1s1KCBihs5D4jEdjbg9KbKh68AsHXaD3
v5L3POJ9hQnI6zJdvCfxniHdUArfyYhqsp1bnCn+85g4ed7BzDqMX2IDAoGBAPVL
Y45rALw7lsjxJndyFdffJtyAeuwxgJNwWGuY21xhwqPbuwsgLHsGerHNKB5QAJT8
JOlrcrfC13s6Tt4wmIy/o2h1p9tMaitmVR6pJzEfHyJhSRTbeFybQ9yqlKHuk2tI
jcUZV/59cyRrjhPKWoVym3Fh/P7D1t1kfdTvBrvfAoGAUH0rVkb5UTo/5xBFsmQw
QM1o8CvY2CqOa11mWlcERjrMCcuqUrZuCeeyH9DP1WveL3kBROf2fFWqVmTJAGIk
eXLfOs6EG75of17vOWioJl4r5i8+WccniDH2YkeQHCbpX8puHtFNVt05spSBHG1m
gTTW1pRZqUet8TuEPxBuj2kCgYAVjCrRruqgnmdvfWeQpI/wp6SlSBAEQZD24q6R
vRq/8cKEXGAA6TGfGQGcLtZwWzzB2ahwbMTmCZKeO5AECqbL7mWvXm6BYCQPbeza
Raews/grL/qYf3MCR41djAqEcw22Jeh2QPSu4VxE/cG8UVFEWb335tCvnIp6ZkJ7
ewfPZwKBgEnc8HH1aq8IJ6vRBePNu6M9ON6PB9qW+ZHHcy47bcGogvYRQk1Ng77G
LdZpyjWzzmb0Z4kjEYcrlGdbNQf9iaT0r+SJPzwBDG15+fRqK7EJI00UhjB0T67M
otrkElxOBGqHSOl0jfUBrpSkSHiy0kDc3/cTAWKn0gowaznSlR9N
-----END RSA PRIVATE KEY-----
host-key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKwIWcodi3rHl0zS3VaddXnwfgIcpp1ECR9KhaB1cyIspgcHOA98wxY8onw+zHxpMUB4pve+t416FFLSkmlJ2f4="
source-dataset: data/src/b
target-dataset: data/dst/b
recursive: true
periodic-snapshot-tasks:
- src-b
auto: true
retention-policy: none
speed-limit: 100000
""")))
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 10 <= end - start <= 15
zettarepl._spawn_retention.assert_called_once()
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
zettarepl._replication_tasks_can_run_in_parallel = Mock(return_value=False)
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 20 <= end - start <= 25
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
def test_snapshot_gone(transport):
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True)
definition = yaml.safe_load(
textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 2
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock()
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
deleted = False
def resume_replications_mock(*args, **kwargs):
nonlocal deleted
if not deleted:
# Snapshots are already listed, and now we remove one of them to simulate PULL replication
# from remote system that has `allow_empty_snapshots: false`. Only do this once.
subprocess.check_call("zfs destroy data/src@2018-10-01_01-00",
shell=True)
deleted = True
return resume_replications(*args, **kwargs)
with patch("zettarepl.replication.run.resume_replications",
resume_replications_mock):
zettarepl._spawn_replication_tasks(
select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst", False)) == 1
def test_push_replication(dst_parent_is_readonly, dst_exists, transport, replicate, properties, encrypted,
has_encrypted_child, dst_parent_encrypted):
if replicate and not properties:
return
if encrypted and has_encrypted_child:
# If parent is encrypted, child is also encrypted
return
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs destroy -r data/dst_parent", shell=True)
create_dataset("data/src", encrypted)
subprocess.check_call("zfs set test:property=test-value data/src", shell=True)
if has_encrypted_child:
create_dataset("data/src/child", True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True)
create_dataset("data/dst_parent", dst_parent_encrypted)
if dst_exists:
subprocess.check_call("zfs create data/dst_parent/dst", shell=True)
if dst_parent_is_readonly:
subprocess.check_call("zfs set readonly=on data/dst_parent", shell=True)
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src:
dataset: data/src
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src:
direction: push
source-dataset: data/src
target-dataset: data/dst_parent/dst
recursive: true
periodic-snapshot-tasks:
- src
auto: true
retention-policy: none
retries: 1
"""))
definition["replication-tasks"]["src"]["transport"] = transport
definition["replication-tasks"]["src"]["replicate"] = replicate
definition["replication-tasks"]["src"]["properties"] = properties
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
observer = Mock()
zettarepl.set_observer(observer)
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
if dst_exists and properties and encrypted and not dst_parent_encrypted:
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskError), error
assert error.error == ("Unable to send encrypted dataset 'data/src' to existing unencrypted or unrelated "
"dataset 'data/dst_parent/dst'")
return
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 2
assert (
("test-value" in subprocess.check_output("zfs get test:property data/dst_parent/dst",
shell=True, encoding="utf-8")) ==
properties
)
subprocess.check_call("zfs snapshot -r data/src@2018-10-01_03-00", shell=True)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
wait_replication_tasks_to_complete(zettarepl)
error = observer.call_args_list[-1][0][0]
assert isinstance(error, ReplicationTaskSuccess), error
assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 3
class ZettareplProcess:
def __init__(self, definition, debug_level, log_handler, command_queue,
observer_queue):
self.definition = definition
self.debug_level = debug_level
self.log_handler = log_handler
self.command_queue = command_queue
self.observer_queue = observer_queue
self.zettarepl = None
self.vmware_contexts = {}
def __call__(self):
setproctitle.setproctitle('middlewared (zettarepl)')
osc.die_with_parent()
if logging.getLevelName(self.debug_level) == logging.TRACE:
# If we want TRACE then we want all debug from zettarepl
default_level = logging.DEBUG
elif logging.getLevelName(self.debug_level) == logging.DEBUG:
# Regular development level. We don't need verbose debug from zettarepl
default_level = logging.INFO
else:
default_level = logging.getLevelName(self.debug_level)
setup_logging("", "DEBUG", self.log_handler)
oqlh = ObserverQueueLoggingHandler(self.observer_queue)
oqlh.setFormatter(
logging.Formatter(
'[%(asctime)s] %(levelname)-8s [%(threadName)s] [%(name)s] %(message)s',
'%Y/%m/%d %H:%M:%S'))
logging.getLogger("zettarepl").addHandler(oqlh)
for handler in logging.getLogger("zettarepl").handlers:
handler.addFilter(LongStringsFilter())
handler.addFilter(ReplicationTaskLoggingLevelFilter(default_level))
c = Client('ws+unix:///var/run/middlewared-internal.sock',
py_exceptions=True)
c.subscribe('core.reconfigure_logging',
lambda *args, **kwargs: reconfigure_logging())
definition = Definition.from_data(self.definition,
raise_on_error=False)
self.observer_queue.put(DefinitionErrors(definition.errors))
clock = Clock()
tz_clock = TzClock(definition.timezone, clock.now)
scheduler = Scheduler(clock, tz_clock)
local_shell = LocalShell()
self.zettarepl = Zettarepl(scheduler, local_shell)
self.zettarepl.set_observer(self._observer)
self.zettarepl.set_tasks(definition.tasks)
start_daemon_thread(target=self._process_command_queue)
while True:
try:
self.zettarepl.run()
except Exception:
logging.getLogger("zettarepl").error("Unhandled exception",
exc_info=True)
time.sleep(10)
def _observer(self, message):
self.observer_queue.put(message)
logger = logging.getLogger("middlewared.plugins.zettarepl")
try:
if isinstance(
message,
(PeriodicSnapshotTaskStart, PeriodicSnapshotTaskSuccess,
PeriodicSnapshotTaskError)):
task_id = int(message.task_id.split("_")[-1])
if isinstance(message, PeriodicSnapshotTaskStart):
with Client() as c:
context = c.call("vmware.periodic_snapshot_task_begin",
task_id,
job=True)
self.vmware_contexts[task_id] = context
if context and context["vmsynced"]:
# If there were no failures and we successfully took some VMWare snapshots
# set the ZFS property to show the snapshot has consistent VM snapshots
# inside it.
return message.response(
properties={"freenas:vmsynced": "Y"})
if isinstance(
message,
(PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)):
context = self.vmware_contexts.pop(task_id, None)
if context:
with Client() as c:
c.call("vmware.periodic_snapshot_task_end",
context,
job=True)
except ClientException as e:
if e.error:
logger.error(
"Unhandled exception in ZettareplProcess._observer: %r",
e.error)
if e.trace:
logger.error(
"Unhandled exception in ZettareplProcess._observer:\n%s",
e.trace["formatted"])
except Exception:
logger.error("Unhandled exception in ZettareplProcess._observer",
exc_info=True)
def _process_command_queue(self):
logger = logging.getLogger("middlewared.plugins.zettarepl")
while self.zettarepl is not None:
command, args = self.command_queue.get()
if command == "timezone":
self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone(
args)
if command == "tasks":
definition = Definition.from_data(args, raise_on_error=False)
self.observer_queue.put(DefinitionErrors(definition.errors))
self.zettarepl.set_tasks(definition.tasks)
if command == "run_task":
class_name, task_id = args
for task in self.zettarepl.tasks:
if task.__class__.__name__ == class_name and task.id == task_id:
logger.debug("Running task %r", task)
self.zettarepl.scheduler.interrupt([task])
break
else:
logger.warning("Task %s(%r) not found", class_name,
task_id)
self.observer_queue.put(
ReplicationTaskError(task_id, "Task not found"))
def test_parallel_replication():
subprocess.call("zfs destroy -r data/src", shell=True)
subprocess.call("zfs receive -A data/dst", shell=True)
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/src", shell=True)
subprocess.check_call("zfs create data/src/a", shell=True)
subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/a/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/a@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs create data/src/b", shell=True)
subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/b/blob bs=1M count=1", shell=True)
subprocess.check_call("zfs snapshot data/src/b@2018-10-01_01-00", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
definition = yaml.safe_load(textwrap.dedent("""\
timezone: "UTC"
periodic-snapshot-tasks:
src-a:
dataset: data/src/a
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
src-b:
dataset: data/src/b
recursive: true
lifetime: PT1H
naming-schema: "%Y-%m-%d_%H-%M"
schedule:
minute: "0"
replication-tasks:
src-a:
direction: push
transport:
type: ssh
hostname: localhost
source-dataset: data/src/a
target-dataset: data/dst/a
recursive: true
periodic-snapshot-tasks:
- src-a
auto: true
retention-policy: none
speed-limit: 100000
src-b:
direction: push
transport:
type: ssh
hostname: localhost
source-dataset: data/src/b
target-dataset: data/dst/b
recursive: true
periodic-snapshot-tasks:
- src-b
auto: true
retention-policy: none
speed-limit: 100000
"""))
set_localhost_transport_options(definition["replication-tasks"]["src-a"]["transport"])
set_localhost_transport_options(definition["replication-tasks"]["src-b"]["transport"])
definition = Definition.from_data(definition)
local_shell = LocalShell()
zettarepl = Zettarepl(Mock(), local_shell)
zettarepl._spawn_retention = Mock()
zettarepl.set_tasks(definition.tasks)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 10 <= end - start <= 15
zettarepl._spawn_retention.assert_called_once()
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
subprocess.call("zfs destroy -r data/dst", shell=True)
subprocess.check_call("zfs create data/dst", shell=True)
subprocess.check_call("zfs create data/dst/a", shell=True)
subprocess.check_call("zfs create data/dst/b", shell=True)
zettarepl._replication_tasks_can_run_in_parallel = Mock(return_value=False)
zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks))
start = time.monotonic()
wait_replication_tasks_to_complete(zettarepl)
end = time.monotonic()
assert 20 <= end - start <= 25
assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1
assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
