async def rsync_task_extend(self, data, context):
data['extra'] = shlex.split(data['extra'].replace('"', r'"\"').replace("'", r'"\"'))
for field in ('mode', 'direction'):
data[field] = data[field].upper()
Cron.convert_db_format_to_schedule(data)
data['job'] = context['jobs'].get(data['id'])
return data
async def rsync_task_extend(self, data, context):
data['extra'] = list(filter(None, re.split(r"\s+", data["extra"])))
for field in ('mode', 'direction'):
data[field] = data[field].upper()
Cron.convert_db_format_to_schedule(data)
data['job'] = context['jobs'].get(data['id'])
return data
async def extend(self, cloud_sync):
cloud_sync["credentials"] = cloud_sync.pop("credential")
cloud_sync["encryption_password"] = await self.middleware.call(
"pwenc.decrypt", cloud_sync["encryption_password"])
cloud_sync["encryption_salt"] = await self.middleware.call(
"pwenc.decrypt", cloud_sync["encryption_salt"])
Cron.convert_db_format_to_schedule(cloud_sync)
return cloud_sync
async def smart_test_extend(self, data):
disks = data.pop('disks')
data['disks'] = [disk['disk_identifier'] for disk in disks]
test_type = {
'L': 'LONG',
'S': 'SHORT',
'C': 'CONVEYANCE',
'O': 'OFFLINE',
}
data['type'] = test_type[data.pop('type')]
Cron.convert_db_format_to_schedule(data)
return data
async def smart_test_extend(self, data):
disks = data.pop('disks')
data['disks'] = [disk['disk_identifier'] for disk in disks]
test_type = {
'L': 'LONG',
'S': 'SHORT',
'C': 'CONVEYANCE',
'O': 'OFFLINE',
}
data['type'] = test_type[data.pop('type')]
Cron.convert_db_format_to_schedule(data)
return data
async def _extend(self, cloud_sync):
cloud_sync["credentials"] = cloud_sync.pop("credential")
if "encryption_password" in cloud_sync:
cloud_sync["encryption_password"] = await self.middleware.call(
"notifier.pwenc_decrypt", cloud_sync["encryption_password"])
if "encryption_salt" in cloud_sync:
cloud_sync["encryption_salt"] = await self.middleware.call(
"notifier.pwenc_decrypt", cloud_sync["encryption_salt"])
Cron.convert_db_format_to_schedule(cloud_sync)
return cloud_sync
async def rsync_task_extend(self, data, context):
try:
data['extra'] = shlex.split(data['extra'].replace('"',
r'"\"').replace(
"'", r'"\"'))
except ValueError:
# This is to handle the case where the extra value is misconfigured for old cases
# Moving on, we are going to verify that it can be split successfully using shlex
data['extra'] = data['extra'].split()
for field in ('mode', 'direction'):
data[field] = data[field].upper()
Cron.convert_db_format_to_schedule(data)
data['job'] = context['jobs'].get(data['id'])
return data
async def extend(self, data, context):
Cron.convert_db_format_to_schedule(data, begin_end=True)
data['legacy'] = self._is_legacy(data,
context['legacy_replication_tasks'])
data['vmware_sync'] = any((vmware['filesystem'] == data['dataset'] or (
data['recursive']
and is_child(vmware['filesystem'], data['dataset'])))
for vmware in context['vmware'])
data['state'] = context['state'].get(
f'periodic_snapshot_task_{data["id"]}', {
'state': 'UNKNOWN',
})
return data
async def extend(self, data, context):
Cron.convert_db_format_to_schedule(data, begin_end=True)
data['vmware_sync'] = any((vmware['filesystem'] == data['dataset'] or (
data['recursive']
and is_child(vmware['filesystem'], data['dataset'])))
for vmware in context['vmware'])
if 'error' in context['state']:
data['state'] = context['state']['error']
else:
data['state'] = context['state']['tasks'].get(
f'periodic_snapshot_task_{data["id"]}', {
'state': 'PENDING',
})
return data
async def extend(self, data, context):
Cron.convert_db_format_to_schedule(data, begin_end=True)
data['legacy'] = self._is_legacy(data, context['legacy_replication_tasks'])
data['vmware_sync'] = any(
(
vmware['filesystem'] == data['dataset'] or
(data['recursive'] and is_child(vmware['filesystem'], data['dataset']))
)
for vmware in context['vmware']
)
data['state'] = context['state'].get(f'periodic_snapshot_task_{data["id"]}', {
'state': 'UNKNOWN',
})
return data
async def extend(self, data, context):
data["periodic_snapshot_tasks"] = [{
k.replace("task_", ""): v
for k, v in task.items()
} for task in data["periodic_snapshot_tasks"]]
for task in data["periodic_snapshot_tasks"]:
Cron.convert_db_format_to_schedule(task, begin_end=True)
if data["direction"] == "PUSH":
data["also_include_naming_schema"] = data["naming_schema"]
data["naming_schema"] = []
if data["direction"] == "PULL":
data["also_include_naming_schema"] = []
Cron.convert_db_format_to_schedule(data,
"schedule",
key_prefix="schedule_",
begin_end=True)
Cron.convert_db_format_to_schedule(data,
"restrict_schedule",
key_prefix="restrict_schedule_",
begin_end=True)
if "error" in context["state"]:
data["state"] = context["state"]["error"]
else:
data["state"] = context["state"]["tasks"].get(
f"replication_task_{data['id']}", {
"state": "PENDING",
})
data["job"] = data["state"].pop("job", None)
return data
async def extend(self, data, context):
data["periodic_snapshot_tasks"] = [{
k.replace("task_", ""): v
for k, v in task.items()
} for task in data["periodic_snapshot_tasks"]]
for task in data["periodic_snapshot_tasks"]:
Cron.convert_db_format_to_schedule(task, begin_end=True)
if data["direction"] == "PUSH":
data["also_include_naming_schema"] = data["naming_schema"]
data["naming_schema"] = []
if data["direction"] == "PULL":
data["also_include_naming_schema"] = []
Cron.convert_db_format_to_schedule(data,
"schedule",
key_prefix="schedule_",
begin_end=True)
Cron.convert_db_format_to_schedule(data,
"restrict_schedule",
key_prefix="restrict_schedule_",
begin_end=True)
if data["transport"] == "LEGACY":
if data["id"] in context["legacy_result"]:
legacy_result = context["legacy_result"][data["id"]]
msg = legacy_result.get("msg")
if msg == "Running":
state = "RUNNING"
elif msg in ["Succeeded", "Up to date"]:
state = "FINISHED"
else:
state = "ERROR"
data["state"] = {
"datetime": context["legacy_result_datetime"],
"state": state,
"last_snapshot": legacy_result.get("last_snapshot"),
}
if state == "ERROR":
data["state"]["error"] = msg
else:
data["state"] = {
"state": "PENDING",
}
else:
data["state"] = context["state"].get(
f"replication_task_{data['id']}", {
"state": "PENDING",
})
data["job"] = data["state"].pop("job", None)
return data
async def extend(self, data, context):
data["periodic_snapshot_tasks"] = [
{k.replace("task_", ""): v for k, v in task.items()}
for task in data["periodic_snapshot_tasks"]
]
for task in data["periodic_snapshot_tasks"]:
Cron.convert_db_format_to_schedule(task, begin_end=True)
if data["direction"] == "PUSH":
data["also_include_naming_schema"] = data["naming_schema"]
data["naming_schema"] = []
if data["direction"] == "PULL":
data["also_include_naming_schema"] = []
Cron.convert_db_format_to_schedule(data, "schedule", key_prefix="schedule_", begin_end=True)
Cron.convert_db_format_to_schedule(data, "restrict_schedule", key_prefix="restrict_schedule_", begin_end=True)
if data["transport"] == "LEGACY":
if data["id"] in context["legacy_result"]:
legacy_result = context["legacy_result"][data["id"]]
msg = legacy_result.get("msg")
if msg == "Running":
state = "RUNNING"
elif msg in ["Succeeded", "Up to date"]:
state = "FINISHED"
else:
state = "ERROR"
data["state"] = {
"datetime": context["legacy_result_datetime"],
"state": state,
"last_snapshot": legacy_result.get("last_snapshot"),
}
if state == "ERROR":
data["state"]["error"] = msg
else:
data["state"] = {
"state": "PENDING",
}
else:
data["state"] = context["state"].get(f"replication_task_{data['id']}", {
"state": "PENDING",
})
return data
def test__cron__convert_db_format_to_schedule(data_dict, begin_end, result):
Cron.convert_db_format_to_schedule(data_dict, "schedule", "cron_", begin_end)
assert data_dict == result
async def rsync_task_extend(self, data):
data['extra'] = list(filter(None, re.split(r"\s+", data["extra"])))
for field in ('mode', 'direction'):
data[field] = data[field].upper()
Cron.convert_db_format_to_schedule(data)
return data
async def pool_scrub_extend(self, data):
data['pool'] = data.pop('volume')
data['pool'] = data['pool']['id']
Cron.convert_db_format_to_schedule(data)
return data
async def extend(self, cloud_sync):
cloud_sync["credentials"] = cloud_sync.pop("credential")
Cron.convert_db_format_to_schedule(cloud_sync)
return cloud_sync
async def extend(self, data, context):
data["periodic_snapshot_tasks"] = [
{k.replace("task_", ""): v for k, v in task.items()}
for task in data["periodic_snapshot_tasks"]
]
for task in data["periodic_snapshot_tasks"]:
Cron.convert_db_format_to_schedule(task, begin_end=True)
if data["direction"] == "PUSH":
data["also_include_naming_schema"] = data["naming_schema"]
data["naming_schema"] = []
if data["direction"] == "PULL":
data["also_include_naming_schema"] = []
Cron.convert_db_format_to_schedule(data, "schedule", key_prefix="schedule_", begin_end=True)
Cron.convert_db_format_to_schedule(data, "restrict_schedule", key_prefix="restrict_schedule_", begin_end=True)
if data["transport"] == "LEGACY":
if data["id"] in context["legacy_result"]:
legacy_result = context["legacy_result"][data["id"]]
msg = legacy_result.get("msg")
if msg == "Running":
state = "RUNNING"
elif msg in ["Succeeded", "Up to date"]:
state = "FINISHED"
else:
state = "ERROR"
data["state"] = {
"datetime": context["legacy_result_datetime"],
"state": state,
"last_snapshot": legacy_result.get("last_snapshot"),
}
if state == "ERROR":
data["state"]["error"] = msg
else:
data["state"] = {
"state": "PENDING",
}
progressfile = f"/tmp/.repl_progress_{data['id']}"
if os.path.exists(progressfile):
with open(progressfile, "r") as f:
pid = f.read().strip()
try:
pid = int(pid)
except ValueError:
pass
else:
try:
title = " ".join(await self.middleware.run_in_thread(
lambda: psutil.Process(pid).cmdline()
))
except psutil.NoSuchProcess:
pass
else:
m = self.ZFS_SEND_REGEX.match(title)
if m:
dataset, snapshot = m.group("snapshot").split("@")
data["state"]["progress"] = {
"dataset": dataset,
"snapshot": snapshot,
"current": int(m.group("current")),
"total": int(m.group("total")),
}
else:
data["state"] = context["state"].get(f"replication_task_{data['id']}", {
"state": "PENDING",
})
data["job"] = data["state"].pop("job", None)
return data
async def rsync_task_extend(self, data):
data['extra'] = list(filter(None, re.split(r"\s+", data["extra"])))
Cron.convert_db_format_to_schedule(data)
return data
def cron_extend(self, data):
Cron.convert_db_format_to_schedule(data)
return data
def test__cron__convert_db_format_to_schedule(data_dict, begin_end, result):
Cron.convert_db_format_to_schedule(data_dict, "schedule", "cron_", begin_end)
assert data_dict == result
async def pool_scrub_extend(self, data):
data['pool'] = data.pop('volume')
data['pool'] = data['pool']['id']
Cron.convert_db_format_to_schedule(data)
return data
def cron_extend(self, data):
Cron.convert_db_format_to_schedule(data)
return data
async def rsync_task_extend(self, data):
data['extra'] = list(filter(None, re.split(r"\s+", data["extra"])))
Cron.convert_db_format_to_schedule(data)
return data