async def do_update(self, id, data):
"""
Update a kerberos realm by id. This will be automatically populated during the
domain join process in an Active Directory environment. Kerberos realm names
are case-sensitive, but convention is to only use upper-case.
"""
old = await self._get_instance(id)
new = old.copy()
new.update(data)
verrors = ValidationErrors()
verrors.add_child('kerberos_realm_update', await self._validate(new))
if verrors:
raise verrors
data = await self.kerberos_compress(new)
await self.middleware.call(
'datastore.update',
self._config.datastore,
id,
new,
{'prefix': self._config.datastore_prefix}
)
await self.middleware.call('etc.generate', 'kerberos')
return await self._get_instance(id)
async def do_create(self, data):
"""
Create an entry in the idmap backend table.
`unix_primary_group` If True, the primary group membership is fetched from the LDAP attributes (gidNumber).
If False, the primary group membership is calculated via the "primaryGroupID" LDAP attribute.
`unix_nss_info` if True winbind will retrieve the login shell and home directory from the LDAP attributes.
If False or if the AD LDAP entry lacks the SFU attributes the smb4.conf parameters `template shell` and `template homedir` are used.
`schema_mode` Defines the schema that idmap_ad should use when querying Active Directory regarding user and group information.
This can be either the RFC2307 schema support included in Windows 2003 R2 or the Service for Unix (SFU) schema.
For SFU 3.0 or 3.5 please choose "SFU", for SFU 2.0 please choose "SFU20". The behavior of primary group membership is
controlled by the unix_primary_group option.
"""
verrors = ValidationErrors()
data = await self.middleware.call('idmap.common_backend_compress', data)
verrors.add_child('idmap_ad_create', await self.middleware.call('idmap._common_validate', data))
if verrors:
raise verrors
data["id"] = await self.middleware.call(
"datastore.insert", self._config.datastore, data,
{
"prefix": self._config.datastore_prefix
},
)
return await self._get_instance(data['id'])
async def do_update(self, id, data):
"""
Update kerberos keytab by id.
"""
old = await self._get_instance(id)
new = old.copy()
new.update(data)
verrors = ValidationErrors()
verrors.add_child('kerberos_principal_update', await self._validate(new))
if verrors:
raise verrors
data = await self.kerberos_keytab_compress(data)
await self.middleware.call(
'datastore.update',
self._config.datastore,
id,
new,
{'prefix': self._config.datastore_prefix}
)
await self.middleware.call('etc.generate', 'kerberos')
return await self._get_instance(id)
async def do_create(self, data):
"""
Create a new kerberos realm. This will be automatically populated during the
domain join process in an Active Directory environment. Kerberos realm names
are case-sensitive, but convention is to only use upper-case.
Entries for kdc, admin_server, and kpasswd_server are not required.
If they are unpopulated, then kerberos will use DNS srv records to
discover the correct servers. The option to hard-code them is provided
due to AD site discovery. Kerberos has no concept of Active Directory
sites. This means that middleware performs the site discovery and
sets the kerberos configuration based on the AD site.
"""
verrors = ValidationErrors()
verrors.add_child('kerberos_realm_create', await self._validate(data))
if verrors:
raise verrors
data = await self.kerberos_compress(data)
data["id"] = await self.middleware.call(
"datastore.insert", self._config.datastore, data,
{
"prefix": self._config.datastore_prefix
},
)
await self.middleware.call('etc.generate', 'kerberos')
await self.middleware.call('service.restart', 'cron')
return await self._get_instance(data['id'])
async def do_create(self, data):
"""
Create a kerberos keytab. Uploaded keytab files will be merged with the system
keytab under /etc/krb5.keytab.
`file` b64encoded kerberos keytab
`name` name for kerberos keytab
"""
verrors = ValidationErrors()
verrors.add_child('kerberos_principal_create', await self._validate(data))
if verrors:
raise verrors
data = await self.kerberos_keytab_compress(data)
data["id"] = await self.middleware.call(
"datastore.insert", self._config.datastore, data,
{
"prefix": self._config.datastore_prefix
},
)
await self.middleware.call('etc.generate', 'kerberos')
return await self._get_instance(data['id'])
def _validate(self, schema_name, data):
verrors = ValidationErrors()
if data["provider"] not in REMOTES:
verrors.add(f"{schema_name}.provider", "Invalid provider")
else:
provider = REMOTES[data["provider"]]
attributes_verrors = validate_attributes(provider.credentials_schema, data)
verrors.add_child(f"{schema_name}.attributes", attributes_verrors)
if verrors:
raise verrors
async def _validate(self, schema_name, data, id=None):
verrors = ValidationErrors()
await self._ensure_unique(verrors, schema_name, "name", data["name"], id)
if data["provider"] not in REMOTES:
verrors.add(f"{schema_name}.provider", "Invalid provider")
else:
provider = REMOTES[data["provider"]]
attributes_verrors = validate_attributes(provider.credentials_schema, data)
verrors.add_child(f"{schema_name}.attributes", attributes_verrors)
if verrors:
raise verrors
async def _validate(self, service, schema_name):
verrors = ValidationErrors()
factory = ALERT_SERVICES_FACTORIES.get(service["type"])
if factory is None:
verrors.add(f"{schema_name}.type", "This field has invalid value")
raise verrors
try:
factory.validate(service.get('attributes', {}))
except ValidationErrors as e:
verrors.add_child(f"{schema_name}.attributes", e)
if verrors:
raise verrors
async def _validate(self, schema_name, data, id=None):
verrors = ValidationErrors()
await self._ensure_unique(verrors, schema_name, "name", data["name"],
id)
if data["provider"] not in REMOTES:
verrors.add(f"{schema_name}.provider", "Invalid provider")
else:
provider = REMOTES[data["provider"]]
attributes_verrors = validate_attributes(
provider.credentials_schema, data)
verrors.add_child(f"{schema_name}.attributes", attributes_verrors)
if verrors:
raise verrors
async def do_update(self, id, data):
"""
Update a domain by id.
"""
old = await self._get_instance(id)
new = old.copy()
new.update(data)
verrors = ValidationErrors()
verrors.add_child('idmap_domain_update', await self._validate(new))
if verrors:
raise verrors
await self.middleware.call('datastore.update', self._config.datastore,
id, new,
{'prefix': self._config.datastore_prefix})
return await self._get_instance(id)
async def _validate(self, service, schema_name):
verrors = ValidationErrors()
factory = ALERT_SERVICES_FACTORIES.get(service["type"])
if factory is None:
verrors.add(f"{schema_name}.type", "This field has invalid value")
try:
factory.validate(service["attributes"])
except ValidationErrors as e:
verrors.add_child(f"{schema_name}.attributes", e)
validate_settings(verrors, f"{schema_name}.settings",
service["settings"])
if verrors:
raise verrors
async def do_create(self, data):
"""
Create an entry in the idmap_rid backend table.
"""
verrors = ValidationErrors()
verrors.add_child('idmap_rid_create', await self.middleware.call('idmap._common_validate', data))
if verrors:
raise verrors
data = await self.middleware.call('idmap.common_backend_compress', data)
data["id"] = await self.middleware.call(
"datastore.insert", self._config.datastore, data,
{
"prefix": self._config.datastore_prefix
},
)
return await self._get_instance(data['id'])
async def run_onetime(self, job, data):
"""
Run replication task without creating it.
"""
data["name"] = f"Temporary replication task for job {job.id}"
data["schedule"] = None
data["only_matching_schedule"] = False
data["auto"] = False
data["enabled"] = True
verrors = ValidationErrors()
verrors.add_child("replication_run_onetime", await self._validate(data))
if verrors:
raise verrors
await self.middleware.call("zettarepl.run_onetime_replication_task", job, data)
async def do_create(self, data):
"""
Create an entry in the idmap backend table.
`script` full path to the script or program that generates the mappings.
"""
verrors = ValidationErrors()
verrors.add_child('idmap_script_create', await self.middleware.call('idmap._common_validate', data))
if verrors:
raise verrors
data = await self.middleware.call('idmap.common_backend_compress', data)
data["id"] = await self.middleware.call(
"datastore.insert", self._config.datastore, data,
{
"prefix": self._config.datastore_prefix
},
)
return await self._get_instance(data['id'])
async def do_create(self, data):
"""
Create a kerberos keytab. Uploaded keytab files will be merged with the system
keytab under /etc/krb5.keytab.
`file` b64encoded kerberos keytab
`name` name for kerberos keytab
"""
verrors = ValidationErrors()
verrors.add_child('kerberos_principal_create', await self._validate(data))
if verrors:
raise verrors
id = await super().do_create(data)
await self.middleware.call('etc.generate', 'kerberos')
return await self._get_instance(id)
async def do_update(self, id, data):
"""
Update kerberos keytab by id.
"""
old = await self._get_instance(id)
new = old.copy()
new.update(data)
verrors = ValidationErrors()
verrors.add_child('kerberos_principal_update', await self._validate(new))
if verrors:
raise verrors
await super().do_update(id, new)
await self.middleware.call('etc.generate', 'kerberos')
return await self._get_instance(id)
async def do_update(self, id, data):
"""
Update an entry in the idmap backend table by id.
"""
old = await self._get_instance(id)
new = old.copy()
new.update(data)
verrors = ValidationErrors()
verrors.add_child(
'idmap_ad_update', await
self.middleware.call('idmap._common_validate', 'ad', new))
if verrors:
raise verrors
new = await self.middleware.call('idmap.common_backend_compress', new)
await self.middleware.call('datastore.update', self._config.datastore,
id, new,
{'prefix': self._config.datastore_prefix})
return await self._get_instance(id)
async def do_create(self, data):
"""
Create an entry in the idmap_rfc2307 backend table.
`ldap_server` defines the type of LDAP server to use. This can either be an LDAP server provided
by the Active Directory Domain (ad) or a stand-alone LDAP server.
`bind_path_user` specfies the search base where user objects can be found in the LDAP server.
`bind_path_group` specifies the search base where group objects can be found in the LDAP server.
`user_cn` query cn attribute instead of uid attribute for the user name in LDAP.
`realm` append @realm to cn for groups (and users if user_cn is set) in LDAP queries.
`ldmap_domain` when using the LDAP server in the Active Directory server, this allows one to
specify the domain where to access the Active Directory server. This allows using trust relationships
while keeping all RFC 2307 records in one place. This parameter is optional, the default is to access
the AD server in the current domain to query LDAP records.
`ldap_url` when using a stand-alone LDAP server, this parameter specifies the LDAP URL for accessing the LDAP server.
`ldap_user_dn` defines the user DN to be used for authentication.
`realm` defines the realm to use in the user and group names. This is only required when using cn_realm together with
a stand-alone ldap server.
"""
verrors = ValidationErrors()
verrors.add_child(
'idmap_rfc2307_create', await
self.middleware.call('idmap._common_validate', 'rfc2307', data))
if verrors:
raise verrors
data = await self.middleware.call('idmap.common_backend_compress',
data)
data["id"] = await self.middleware.call(
"datastore.insert",
self._config.datastore,
data,
{"prefix": self._config.datastore_prefix},
)
return await self._get_instance(data['id'])
async def do_create(self, data):
"""
Create an entry in the idmap_rid backend table.
"""
verrors = ValidationErrors()
verrors.add_child(
'idmap_rid_create', await
self.middleware.call('idmap._common_validate', 'rid', data))
if verrors:
raise verrors
data = await self.middleware.call('idmap.common_backend_compress',
data)
data["id"] = await self.middleware.call(
"datastore.insert",
self._config.datastore,
data,
{"prefix": self._config.datastore_prefix},
)
return await self._get_instance(data['id'])
async def do_update(self, id, data):
"""
Update an entry in the idmap backend table by id.
"""
old = await self._get_instance(id)
new = old.copy()
new.update(data)
verrors = ValidationErrors()
verrors.add_child('idmap_ad_update', await self.middleware.call('idmap._common_validate', new))
if verrors:
raise verrors
new = await self.middleware.call('idmap.common_backend_compress', new)
await self.middleware.call(
'datastore.update',
self._config.datastore,
id,
new,
{'prefix': self._config.datastore_prefix}
)
return await self._get_instance(id)
async def do_update(self, id, data):
"""
Update a domain by id.
"""
old = await self._get_instance(id)
new = old.copy()
new.update(data)
verrors = ValidationErrors()
verrors.add_child('idmap_domain_update', await self._validate(new))
if verrors:
raise verrors
await self.middleware.call(
'datastore.update',
self._config.datastore,
id,
new,
{'prefix': self._config.datastore_prefix}
)
return await self._get_instance(id)
async def do_create(self, data):
"""
Create an entry in the idmap backend table.
`script` full path to the script or program that generates the mappings.
"""
verrors = ValidationErrors()
verrors.add_child(
'idmap_script_create', await
self.middleware.call('idmap._common_validate', 'script', data))
if verrors:
raise verrors
data = await self.middleware.call('idmap.common_backend_compress',
data)
data["id"] = await self.middleware.call(
"datastore.insert",
self._config.datastore,
data,
{"prefix": self._config.datastore_prefix},
)
return await self._get_instance(data['id'])
async def do_create(self, data):
"""
Create an entry in the idmap_rfc2307 backend table.
`ldap_server` defines the type of LDAP server to use. This can either be an LDAP server provided
by the Active Directory Domain (ad) or a stand-alone LDAP server.
`bind_path_user` specfies the search base where user objects can be found in the LDAP server.
`bind_path_group` specifies the search base where group objects can be found in the LDAP server.
`user_cn` query cn attribute instead of uid attribute for the user name in LDAP.
`realm` append @realm to cn for groups (and users if user_cn is set) in LDAP queries.
`ldmap_domain` when using the LDAP server in the Active Directory server, this allows one to
specify the domain where to access the Active Directory server. This allows using trust relationships
while keeping all RFC 2307 records in one place. This parameter is optional, the default is to access
the AD server in the current domain to query LDAP records.
`ldap_url` when using a stand-alone LDAP server, this parameter specifies the LDAP URL for accessing the LDAP server.
`ldap_user_dn` defines the user DN to be used for authentication.
`realm` defines the realm to use in the user and group names. This is only required when using cn_realm together with
a stand-alone ldap server.
"""
verrors = ValidationErrors()
verrors.add_child('idmap_rfc2307_create', await self.middleware.call('idmap._common_validate', data))
if verrors:
raise verrors
data = await self.middleware.call('idmap.common_backend_compress', data)
data["id"] = await self.middleware.call(
"datastore.insert", self._config.datastore, data,
{
"prefix": self._config.datastore_prefix
},
)
return await self._get_instance(data['id'])
async def do_update(self, data):
"""
`appdefaults_aux` add parameters to "appdefaults" section of the krb5.conf file.
`libdefaults_aux` add parameters to "libdefaults" section of the krb5.conf file.
"""
verrors = ValidationErrors()
old = await self.config()
new = old.copy()
new.update(data)
verrors.add_child(
'kerberos_settings_update', await
self._validate_appdefaults(new['appdefaults_aux']))
verrors.add_child(
'kerberos_settings_update', await
self._validate_libdefaults(new['libdefaults_aux']))
verrors.check()
await super().do_update(data)
await self.middleware.call('etc.generate', 'kerberos')
return await self.config()
async def do_create(self, data):
"""
Create a new IDMAP domain. These domains must be unique. This table
will be automatically populated after joining an Active Directory domain
if "allow trusted domains" is set to True in the AD service configuration.
There are three default system domains: DS_TYPE_ACTIVEDIRECTORY, DS_TYPE_LDAP, DS_TYPE_DEFAULT_DOMAIN.
The system domains correspond with the idmap settings under Active Directory, LDAP, and SMB
respectively.
`name` the pre-windows 2000 domain name.
`DNS_domain_name` DNS name of the domain.
"""
verrors = ValidationErrors()
verrors.add_child('idmap_domain_create', await self._validate(data))
if verrors:
raise verrors
data["id"] = await self.middleware.call(
"datastore.insert", self._config.datastore, data,
{
"prefix": self._config.datastore_prefix
},
)
return await self._get_instance(data['id'])
async def do_update(self, id, data):
"""
Update kerberos keytab by id.
"""
old = await self._get_instance(id)
new = old.copy()
new.update(data)
verrors = ValidationErrors()
verrors.add_child('kerberos_principal_update', await
self._validate(new))
if verrors:
raise verrors
new = await self.kerberos_keytab_compress(data)
await self.middleware.call('datastore.update', self._config.datastore,
id, new,
{'prefix': self._config.datastore_prefix})
await self.middleware.call('etc.generate', 'kerberos')
return await self._get_instance(id)
async def do_create(self, data):
"""
Create a kerberos keytab. Uploaded keytab files will be merged with the system
keytab under /etc/krb5.keytab.
`file` b64encoded kerberos keytab
`name` name for kerberos keytab
"""
verrors = ValidationErrors()
verrors.add_child('kerberos_principal_create', await self._validate(data))
if verrors:
raise verrors
data["id"] = await self.middleware.call(
"datastore.insert", self._config.datastore, data,
{
"prefix": self._config.datastore_prefix
},
)
await self.middleware.call('etc.generate', 'kerberos')
return await self._get_instance(data['id'])
async def do_update(self, id, data):
"""
Update a kerberos realm by id. This will be automatically populated during the
domain join process in an Active Directory environment. Kerberos realm names
are case-sensitive, but convention is to only use upper-case.
"""
old = await self._get_instance(id)
new = old.copy()
new.update(data)
verrors = ValidationErrors()
verrors.add_child('kerberos_realm_update', await self._validate(new))
if verrors:
raise verrors
data = await self.kerberos_compress(new)
await self.middleware.call('datastore.update', self._config.datastore,
id, new,
{'prefix': self._config.datastore_prefix})
await self.middleware.call('etc.generate', 'kerberos')
return await self._get_instance(id)
async def _validate(self, data, id=None):
verrors = ValidationErrors()
await self._ensure_unique(verrors, "", "name", data["name"], id)
# Direction
snapshot_tasks = []
if data["direction"] == "PUSH":
e, snapshot_tasks = await self._query_periodic_snapshot_tasks(
data["periodic_snapshot_tasks"])
verrors.add_child("periodic_snapshot_tasks", e)
if data["naming_schema"]:
verrors.add("naming_schema",
"This field has no sense for push replication")
if not snapshot_tasks and not data["also_include_naming_schema"]:
verrors.add(
"periodic_snapshot_tasks",
"You must at least either bind a periodic snapshot task or provide "
"\"Also Include Naming Schema\" for push replication task")
if data["schedule"] is None and data[
"auto"] and not data["periodic_snapshot_tasks"]:
verrors.add(
"auto",
"Push replication that runs automatically must be either "
"bound to a periodic snapshot task or have a schedule")
if data["direction"] == "PULL":
if data["schedule"] is None and data["auto"]:
verrors.add(
"auto",
"Pull replication that runs automatically must have a schedule"
)
if data["periodic_snapshot_tasks"]:
verrors.add(
"periodic_snapshot_tasks",
"Pull replication can't be bound to a periodic snapshot task"
)
if not data["naming_schema"]:
verrors.add("naming_schema",
"Naming schema is required for pull replication")
if data["also_include_naming_schema"]:
verrors.add("also_include_naming_schema",
"This field has no sense for pull replication")
if data["hold_pending_snapshots"]:
verrors.add(
"hold_pending_snapshots",
"Pull replication tasks can't hold pending snapshots because "
"they don't do source retention")
# Transport
if data["transport"] == "SSH+NETCAT":
if data["netcat_active_side"] is None:
verrors.add(
"netcat_active_side",
"You must choose active side for SSH+netcat replication")
if data["netcat_active_side_port_min"] is not None and data[
"netcat_active_side_port_max"] is not None:
if data["netcat_active_side_port_min"] > data[
"netcat_active_side_port_max"]:
verrors.add(
"netcat_active_side_port_max",
"Please specify value greater or equal than netcat_active_side_port_min"
)
if data["compression"] is not None:
verrors.add(
"compression",
"Compression is not supported for SSH+netcat replication")
if data["speed_limit"] is not None:
verrors.add(
"speed_limit",
"Speed limit is not supported for SSH+netcat replication")
else:
if data["netcat_active_side"] is not None:
verrors.add(
"netcat_active_side",
"This field only has sense for SSH+netcat replication")
for k in [
"netcat_active_side_listen_address",
"netcat_active_side_port_min",
"netcat_active_side_port_max",
"netcat_passive_side_connect_address"
]:
if data[k] is not None:
verrors.add(
k,
"This field only has sense for SSH+netcat replication")
if data["transport"] == "LOCAL":
if data["ssh_credentials"] is not None:
verrors.add(
"ssh_credentials",
"Remote credentials have no sense for local replication")
if data["compression"] is not None:
verrors.add("compression",
"Compression has no sense for local replication")
if data["speed_limit"] is not None:
verrors.add("speed_limit",
"Speed limit has no sense for local replication")
else:
if data["ssh_credentials"] is None:
verrors.add(
"ssh_credentials",
"SSH Credentials are required for non-local replication")
else:
try:
await self.middleware.call(
"keychaincredential.get_of_type",
data["ssh_credentials"], "SSH_CREDENTIALS")
except CallError as e:
verrors.add("ssh_credentials", str(e))
# Common for all directions and transports
for i, source_dataset in enumerate(data["source_datasets"]):
for snapshot_task in snapshot_tasks:
if is_child(source_dataset, snapshot_task["dataset"]):
if data["recursive"]:
for exclude in snapshot_task["exclude"]:
if is_child(exclude, source_dataset
) and exclude not in data["exclude"]:
verrors.add(
"exclude",
f"You should exclude {exclude!r} as bound periodic snapshot "
f"task dataset {snapshot_task['dataset']!r} does"
)
else:
if source_dataset in snapshot_task["exclude"]:
verrors.add(
f"source_datasets.{i}",
f"Dataset {source_dataset!r} is excluded by bound "
f"periodic snapshot task for dataset "
f"{snapshot_task['dataset']!r}")
if not data["recursive"] and data["exclude"]:
verrors.add(
"exclude",
"Excluding child datasets is only supported for recursive replication"
)
for i, v in enumerate(data["exclude"]):
if not any(
v.startswith(ds + "/") for ds in data["source_datasets"]):
verrors.add(
f"exclude.{i}",
"This dataset is not a child of any of source datasets")
if data["replicate"]:
if not data["recursive"]:
verrors.add(
"recursive",
"This option is required for full filesystem replication")
if data["exclude"]:
verrors.add(
"exclude",
"This option is not supported for full filesystem replication"
)
if not data["properties"]:
verrors.add(
"properties",
"This option is required for full filesystem replication")
if data["encryption"]:
for k in [
"encryption_key", "encryption_key_format",
"encryption_key_location"
]:
if data[k] is None:
verrors.add(
k,
"This property is required when remote dataset encryption is enabled"
)
if data["schedule"]:
if not data["auto"]:
verrors.add(
"schedule",
"You can't have schedule for replication that does not run automatically"
)
else:
if data["only_matching_schedule"]:
verrors.add(
"only_matching_schedule",
"You can't have only-matching-schedule without schedule")
if data["retention_policy"] == "CUSTOM":
if data["lifetime_value"] is None:
verrors.add(
"lifetime_value",
"This field is required for custom retention policy")
if data["lifetime_unit"] is None:
verrors.add(
"lifetime_value",
"This field is required for custom retention policy")
else:
if data["lifetime_value"] is not None:
verrors.add(
"lifetime_value",
"This field has no sense for specified retention policy")
if data["lifetime_unit"] is not None:
verrors.add(
"lifetime_unit",
"This field has no sense for specified retention policy")
if data["enabled"]:
for i, snapshot_task in enumerate(snapshot_tasks):
if not snapshot_task["enabled"]:
verrors.add(
f"periodic_snapshot_tasks.{i}",
"You can't bind disabled periodic snapshot task to enabled replication task"
)
return verrors
async def do_create(self, data):
"""
Create a Replication Task
Create a Replication Task that will push or pull ZFS snapshots to or from remote host..
* `name` specifies a name for replication task
* `direction` specifies whether task will `PUSH` or `PULL` snapshots
* `transport` is a method of snapshots transfer:
* `SSH` transfers snapshots via SSH connection. This method is supported everywhere but does not achieve
great performance
`ssh_credentials` is a required field for this transport (Keychain Credential ID of type `SSH_CREDENTIALS`)
* `SSH+NETCAT` uses unencrypted connection for data transfer. This can only be used in trusted networks
and requires a port (specified by range from `netcat_active_side_port_min` to `netcat_active_side_port_max`)
to be open on `netcat_active_side`
`ssh_credentials` is also required for control connection
* `LOCAL` replicates to or from localhost
* `source_datasets` is a non-empty list of datasets to replicate snapshots from
* `target_dataset` is a dataset to put snapshots into. It must exist on target side
* `recursive` and `exclude` have the same meaning as for Periodic Snapshot Task
* `properties` control whether we should send dataset properties along with snapshots
* `periodic_snapshot_tasks` is a list of periodic snapshot task IDs that are sources of snapshots for this
replication task. Only push replication tasks can be bound to periodic snapshot tasks.
* `naming_schema` is a list of naming schemas for pull replication
* `also_include_naming_schema` is a list of naming schemas for push replication
* `auto` allows replication to run automatically on schedule or after bound periodic snapshot task
* `schedule` is a schedule to run replication task. Only `auto` replication tasks without bound periodic
snapshot tasks can have a schedule
* `restrict_schedule` restricts when replication task with bound periodic snapshot tasks runs. For example,
you can have periodic snapshot tasks that run every 15 minutes, but only run replication task every hour.
* Enabling `only_matching_schedule` will only replicate snapshots that match `schedule` or
`restrict_schedule`
* `allow_from_scratch` will destroy all snapshots on target side and replicate everything from scratch if none
of the snapshots on target side matches source snapshots
* `readonly` controls destination datasets readonly property:
* `SET` will set all destination datasets to readonly=on after finishing the replication
* `REQUIRE` will require all existing destination datasets to have readonly=on property
* `IGNORE` will avoid this kind of behavior
* `hold_pending_snapshots` will prevent source snapshots from being deleted by retention of replication fails
for some reason
* `retention_policy` specifies how to delete old snapshots on target side:
* `SOURCE` deletes snapshots that are absent on source side
* `CUSTOM` deletes snapshots that are older than `lifetime_value` and `lifetime_unit`
* `NONE` does not delete any snapshots
* `compression` compresses SSH stream. Available only for SSH transport
* `speed_limit` limits speed of SSH stream. Available only for SSH transport
* `large_block`, `embed` and `compressed` are various ZFS stream flag documented in `man zfs send`
* `retries` specifies number of retries before considering replication failed
.. examples(websocket)::
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "replication.create",
"params": [{
"name": "Work Backup",
"direction": "PUSH",
"transport": "SSH",
"ssh_credentials": [12],
"source_datasets", ["data/work"],
"target_dataset": "repl/work",
"recursive": true,
"periodic_snapshot_tasks": [5],
"auto": true,
"restrict_schedule": {
"minute": "0",
"hour": "*/2",
"dom": "*",
"month": "*",
"dow": "1,2,3,4,5",
"begin": "09:00",
"end": "18:00"
},
"only_matching_schedule": true,
"retention_policy": "CUSTOM",
"lifetime_value": 1,
"lifetime_unit": "WEEK",
}]
}
"""
verrors = ValidationErrors()
verrors.add_child("replication_create", await self._validate(data))
if verrors:
raise verrors
periodic_snapshot_tasks = data["periodic_snapshot_tasks"]
await self.compress(data)
id = await self.middleware.call(
"datastore.insert", self._config.datastore, data,
{"prefix": self._config.datastore_prefix})
await self._set_periodic_snapshot_tasks(id, periodic_snapshot_tasks)
await self.middleware.call("zettarepl.update_tasks")
return await self._get_instance(id)
async def do_update(self, id, data):
"""
Update a Replication Task with specific `id`
See the documentation for `create` method for information on payload contents
.. examples(websocket)::
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "replication.update",
"params": [
7,
{
"name": "Work Backup",
"direction": "PUSH",
"transport": "SSH",
"ssh_credentials": [12],
"source_datasets", ["data/work"],
"target_dataset": "repl/work",
"recursive": true,
"periodic_snapshot_tasks": [5],
"auto": true,
"restrict_schedule": {
"minute": "0",
"hour": "*/2",
"dom": "*",
"month": "*",
"dow": "1,2,3,4,5",
"begin": "09:00",
"end": "18:00"
},
"only_matching_schedule": true,
"retention_policy": "CUSTOM",
"lifetime_value": 1,
"lifetime_unit": "WEEK",
}
]
}
"""
old = await self._get_instance(id)
new = old.copy()
if new["ssh_credentials"]:
new["ssh_credentials"] = new["ssh_credentials"]["id"]
new["periodic_snapshot_tasks"] = [
task["id"] for task in new["periodic_snapshot_tasks"]
]
new.update(data)
verrors = ValidationErrors()
verrors.add_child("replication_update", await self._validate(new, id))
if verrors:
raise verrors
periodic_snapshot_tasks = new["periodic_snapshot_tasks"]
await self.compress(new)
new.pop("state", None)
new.pop("job", None)
await self.middleware.call("datastore.update", self._config.datastore,
id, new,
{'prefix': self._config.datastore_prefix})
await self._set_periodic_snapshot_tasks(id, periodic_snapshot_tasks)
await self.middleware.call("zettarepl.update_tasks")
return await self._get_instance(id)
async def do_update(self, id, data):
"""
Update a Periodic Snapshot Task with specific `id`
See the documentation for `create` method for information on payload contents
.. examples(websocket)::
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "pool.snapshottask.update",
"params": [
1,
{
"dataset": "data/work",
"recursive": true,
"exclude": ["data/work/temp"],
"lifetime_value": 2,
"lifetime_unit": "WEEK",
"naming_schema": "auto_%Y-%m-%d_%H-%M",
"schedule": {
"minute": "0",
"hour": "*",
"dom": "*",
"month": "*",
"dow": "1,2,3,4,5",
"begin": "09:00",
"end": "18:00"
}
}
]
}
"""
old = await self._get_instance(id)
new = old.copy()
new.update(data)
verrors = ValidationErrors()
verrors.add_child('periodic_snapshot_update', await self._validate(new))
if not data['enabled']:
for replication_task in await self.middleware.call('replication.query', [['enabled', '=', True]]):
if any(periodic_snapshot_task['id'] == id
for periodic_snapshot_task in replication_task['periodic_snapshot_tasks']):
verrors.add(
'periodic_snapshot_update.enabled',
(f'You can\'t disable this periodic snapshot task because it is bound to enabled replication '
f'task {replication_task["id"]!r}')
)
break
if verrors:
raise verrors
legacy_replication_tasks = await self._legacy_replication_tasks()
if self._is_legacy(new, legacy_replication_tasks):
verrors.add_child(f'periodic_snapshot_update', self._validate_legacy(new))
else:
if self._is_legacy(old, legacy_replication_tasks):
verrors.add(
'periodic_snapshot_update.naming_schema',
('This snapshot task is being used in legacy replication task. You must use naming schema '
f'{self._legacy_naming_schema(new)!r}. Please upgrade your replication tasks to edit this field.')
)
if verrors:
raise verrors
Cron.convert_schedule_to_db_format(new, begin_end=True)
for key in ('legacy', 'vmware_sync', 'state'):
new.pop(key, None)
await self.middleware.call(
'datastore.update',
self._config.datastore,
id,
new,
{'prefix': self._config.datastore_prefix}
)
await self.middleware.call('service.restart', 'cron')
await self.middleware.call('zettarepl.update_tasks')
return await self._get_instance(id)
async def attach(self, job, oid, options):
"""
For TrueNAS Core/Enterprise platform, if the `oid` pool is passphrase GELI encrypted, `passphrase`
must be specified for this operation to succeed.
`target_vdev` is the GUID of the vdev where the disk needs to be attached. In case of STRIPED vdev, this
is the STRIPED disk GUID which will be converted to mirror. If `target_vdev` is mirror, it will be converted
into a n-way mirror.
"""
pool = await self.middleware.call('pool.get_instance', oid)
verrors = ValidationErrors()
if not pool['is_decrypted']:
verrors.add('oid', 'Pool must be unlocked for this action.')
verrors.check()
topology = pool['topology']
topology_type = vdev = None
for i in topology:
for v in topology[i]:
if v['guid'] == options['target_vdev']:
topology_type = i
vdev = v
break
if topology_type:
break
else:
verrors.add('pool_attach.target_vdev', 'Unable to locate VDEV')
verrors.check()
if topology_type in ('cache', 'spares'):
verrors.add('pool_attach.target_vdev',
f'Attaching disks to {topology_type} not allowed.')
elif topology_type == 'data':
# We would like to make sure here that we don't have inconsistent vdev types across data
if vdev['type'] not in ('DISK', 'MIRROR'):
verrors.add(
'pool_attach.target_vdev',
f'Attaching disk to {vdev["type"]} vdev is not allowed.')
# Let's validate new disk now
verrors.add_child(
'pool_attach',
await self.middleware.call('disk.check_disks_availability',
[options['new_disk']],
options['allow_duplicate_serials']),
)
verrors.check()
guid = vdev['guid'] if vdev['type'] == 'DISK' else vdev['children'][0][
'guid']
disks = {
options['new_disk']: {
'create_swap': topology_type == 'data',
'vdev': []
}
}
await self.middleware.call('pool.format_disks', job, disks)
devname = disks[options['new_disk']]['vdev'][0]
extend_job = await self.middleware.call('zfs.pool.extend',
pool['name'], None,
[{
'target': guid,
'type': 'DISK',
'path': devname
}])
await job.wrap(extend_job)
asyncio.ensure_future(self.middleware.call('disk.swaps_configure'))
async def do_create(self, data):
"""
Create a Replication Task
Create a Replication Task that will push or pull ZFS snapshots to or from remote host..
* `name` specifies a name for replication task
* `direction` specifies whether task will `PUSH` or `PULL` snapshots
* `transport` is a method of snapshots transfer:
* `SSH` transfers snapshots via SSH connection. This method is supported everywhere but does not achieve
great performance
`ssh_credentials` is a required field for this transport (Keychain Credential ID of type `SSH_CREDENTIALS`)
* `SSH+NETCAT` uses unencrypted connection for data transfer. This can only be used in trusted networks
and requires a port (specified by range from `netcat_active_side_port_min` to `netcat_active_side_port_max`)
to be open on `netcat_active_side`
`ssh_credentials` is also required for control connection
* `LOCAL` replicates to or from localhost
* `LEGACY` uses legacy replication engine prior to FreeNAS 11.3
* `source_datasets` is a non-empty list of datasets to replicate snapshots from
* `target_dataset` is a dataset to put snapshots into. It must exist on target side
* `recursive` and `exclude` have the same meaning as for Periodic Snapshot Task
* `periodic_snapshot_tasks` is a list of periodic snapshot task IDs that are sources of snapshots for this
replication task. Only push replication tasks can be bound to periodic snapshot tasks.
* `naming_schema` is a list of naming schemas for pull replication
* `also_include_naming_schema` is a list of naming schemas for push replication
* `auto` allows replication to run automatically on schedule or after bound periodic snapshot task
* `schedule` is a schedule to run replication task. Only `auto` replication tasks without bound periodic
snapshot tasks can have a schedule
* `restrict_schedule` restricts when replication task with bound periodic snapshot tasks runs. For example,
you can have periodic snapshot tasks that run every 15 minutes, but only run replication task every hour.
* Enabling `only_matching_schedule` will only replicate snapshots that match `schedule` or
`restrict_schedule`
* `allow_from_scratch` will destroy all snapshots on target side and replicate everything from scratch if none
of the snapshots on target side matches source snapshots
* `hold_pending_snapshots` will prevent source snapshots from being deleted by retention of replication fails
for some reason
* `retention_policy` specifies how to delete old snapshots on target side:
* `SOURCE` deletes snapshots that are absent on source side
* `CUSTOM` deletes snapshots that are older than `lifetime_value` and `lifetime_unit`
* `NONE` does not delete any snapshots
* `compression` compresses SSH stream. Available only for SSH transport
* `speed_limit` limits speed of SSH stream. Available only for SSH transport
* `dedup`, `large_block`, `embed` and `compressed` are various ZFS stream flag documented in `man zfs send`
* `retries` specifies number of retries before considering replication failed
.. examples(websocket)::
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "replication.create",
"params": [{
"name": "Work Backup",
"direction": "PUSH",
"transport": "SSH",
"ssh_credentials": [12],
"source_datasets", ["data/work"],
"target_dataset": "repl/work",
"recursive": true,
"periodic_snapshot_tasks": [5],
"auto": true,
"restrict_schedule": {
"minute": "0",
"hour": "*/2",
"dom": "*",
"month": "*",
"dow": "1,2,3,4,5",
"begin": "09:00",
"end": "18:00"
},
"only_matching_schedule": true,
"retention_policy": "CUSTOM",
"lifetime_value": 1,
"lifetime_unit": "WEEK",
}]
}
"""
verrors = ValidationErrors()
verrors.add_child("replication_create", await self._validate(data))
if verrors:
raise verrors
periodic_snapshot_tasks = data["periodic_snapshot_tasks"]
await self.compress(data)
id = await self.middleware.call(
"datastore.insert",
self._config.datastore,
data,
{"prefix": self._config.datastore_prefix}
)
await self._set_periodic_snapshot_tasks(id, periodic_snapshot_tasks)
await self.middleware.call("service.restart", "cron")
await self.middleware.call("zettarepl.update_tasks")
return await self._get_instance(id)
async def do_create(self, data):
"""
Create a Periodic Snapshot Task
Create a Periodic Snapshot Task that will take snapshots of specified `dataset` at specified `schedule`.
Recursive snapshots can be created if `recursive` flag is enabled. You can `exclude` specific child datasets
from snapshot.
Snapshots will be automatically destroyed after a certain amount of time, specified by
`lifetime_value` and `lifetime_unit`.
Snapshots will be named according to `naming_schema` which is a `strftime`-like template for snapshot name
and must contain `%Y`, `%m`, `%d`, `%H` and `%M`.
.. examples(websocket)::
Create a recursive Periodic Snapshot Task for dataset `data/work` excluding `data/work/temp`. Snapshots
will be created on weekdays every hour from 09:00 to 18:00 and will be stored for two weeks.
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "pool.snapshottask.create",
"params": [{
"dataset": "data/work",
"recursive": true,
"exclude": ["data/work/temp"],
"lifetime_value": 2,
"lifetime_unit": "WEEK",
"naming_schema": "auto_%Y-%m-%d_%H-%M",
"schedule": {
"minute": "0",
"hour": "*",
"dom": "*",
"month": "*",
"dow": "1,2,3,4,5",
"begin": "09:00",
"end": "18:00"
}
}]
}
"""
verrors = ValidationErrors()
verrors.add_child('periodic_snapshot_create', await self._validate(data))
if verrors:
raise verrors
if self._is_legacy(data, await self._legacy_replication_tasks()):
verrors.add_child('periodic_snapshot_create', self._validate_legacy(data))
if verrors:
raise verrors
Cron.convert_schedule_to_db_format(data, begin_end=True)
data['id'] = await self.middleware.call(
'datastore.insert',
self._config.datastore,
data,
{'prefix': self._config.datastore_prefix}
)
await self.middleware.call('service.restart', 'cron')
await self.middleware.call('zettarepl.update_tasks')
return await self._get_instance(data['id'])
async def do_update(self, id, data):
"""
Update a Replication Task with specific `id`
See the documentation for `create` method for information on payload contents
.. examples(websocket)::
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "replication.update",
"params": [
7,
{
"name": "Work Backup",
"direction": "PUSH",
"transport": "SSH",
"ssh_credentials": [12],
"source_datasets", ["data/work"],
"target_dataset": "repl/work",
"recursive": true,
"periodic_snapshot_tasks": [5],
"auto": true,
"restrict_schedule": {
"minute": "0",
"hour": "*/2",
"dom": "*",
"month": "*",
"dow": "1,2,3,4,5",
"begin": "09:00",
"end": "18:00"
},
"only_matching_schedule": true,
"retention_policy": "CUSTOM",
"lifetime_value": 1,
"lifetime_unit": "WEEK",
}
]
}
"""
old = await self._get_instance(id)
new = old.copy()
if new["ssh_credentials"]:
new["ssh_credentials"] = new["ssh_credentials"]["id"]
new["periodic_snapshot_tasks"] = [task["id"] for task in new["periodic_snapshot_tasks"]]
new.update(data)
verrors = ValidationErrors()
verrors.add_child("replication_update", await self._validate(new))
if verrors:
raise verrors
periodic_snapshot_tasks = new["periodic_snapshot_tasks"]
await self.compress(new)
new.pop('state', None)
await self.middleware.call(
"datastore.update",
self._config.datastore,
id,
new,
{'prefix': self._config.datastore_prefix}
)
await self._set_periodic_snapshot_tasks(id, periodic_snapshot_tasks)
await self.middleware.call("service.restart", "cron")
await self.middleware.call("zettarepl.update_tasks")
return await self._get_instance(id)
async def _validate(self, data):
verrors = ValidationErrors()
# Direction
snapshot_tasks = []
if data["direction"] == "PUSH":
e, snapshot_tasks = await self._query_periodic_snapshot_tasks(data["periodic_snapshot_tasks"])
verrors.add_child("periodic_snapshot_tasks", e)
if data["naming_schema"]:
verrors.add("naming_schema", "This field has no sense for push replication")
if data["transport"] != "LEGACY" and not snapshot_tasks and not data["also_include_naming_schema"]:
verrors.add(
"periodic_snapshot_tasks", "You must at least either bind a periodic snapshot task or provide "
"\"Also Include Naming Schema\" for push replication task"
)
if data["schedule"]:
if data["periodic_snapshot_tasks"]:
verrors.add("schedule", "Push replication can't be bound to periodic snapshot task and have "
"schedule at the same time")
else:
if data["auto"] and not data["periodic_snapshot_tasks"] and data["transport"] != "LEGACY":
verrors.add("auto", "Push replication that runs automatically must be either "
"bound to periodic snapshot task or have schedule")
if data["direction"] == "PULL":
if data["schedule"]:
pass
else:
if data["auto"]:
verrors.add("auto", "Pull replication that runs automatically must have schedule")
if data["periodic_snapshot_tasks"]:
verrors.add("periodic_snapshot_tasks", "Pull replication can't be bound to periodic snapshot task")
if not data["naming_schema"]:
verrors.add("naming_schema", "Naming schema is required for pull replication")
if data["also_include_naming_schema"]:
verrors.add("also_include_naming_schema", "This field has no sense for pull replication")
if data["hold_pending_snapshots"]:
verrors.add("hold_pending_snapshots", "Pull replication tasks can't hold pending snapshots because "
"they don't do source retention")
# Transport
if data["transport"] == "SSH+NETCAT":
if data["netcat_active_side"] is None:
verrors.add("netcat_active_side", "You must choose active side for SSH+netcat replication")
if data["netcat_active_side_port_min"] is not None and data["netcat_active_side_port_max"] is not None:
if data["netcat_active_side_port_min"] > data["netcat_active_side_port_max"]:
verrors.add("netcat_active_side_port_max",
"Please specify value greater or equal than netcat_active_side_port_min")
if data["compression"] is not None:
verrors.add("compression", "Compression is not supported for SSH+netcat replication")
if data["speed_limit"] is not None:
verrors.add("speed_limit", "Speed limit is not supported for SSH+netcat replication")
else:
if data["netcat_active_side"] is not None:
verrors.add("netcat_active_side", "This field only has sense for SSH+netcat replication")
for k in ["netcat_active_side_listen_address", "netcat_active_side_port_min", "netcat_active_side_port_max",
"netcat_passive_side_connect_address"]:
if data[k] is not None:
verrors.add(k, "This field only has sense for SSH+netcat replication")
if data["transport"] == "LOCAL":
if data["ssh_credentials"] is not None:
verrors.add("ssh_credentials", "Remote credentials have no sense for local replication")
if data["compression"] is not None:
verrors.add("compression", "Compression has no sense for local replication")
if data["speed_limit"] is not None:
verrors.add("speed_limit", "Speed limit has no sense for local replication")
else:
if data["ssh_credentials"] is None:
verrors.add("ssh_credentials", "SSH Credentials are required for non-local replication")
else:
try:
await self.middleware.call("keychaincredential.get_of_type", data["ssh_credentials"],
"SSH_CREDENTIALS")
except CallError as e:
verrors.add("ssh_credentials", str(e))
if data["transport"] == "LEGACY":
for should_be_true in ["auto", "allow_from_scratch"]:
if not data[should_be_true]:
verrors.add(should_be_true, "Legacy replication does not support disabling this option")
for should_be_false in ["exclude", "periodic_snapshot_tasks", "naming_schema", "also_include_naming_schema",
"only_matching_schedule", "dedup", "large_block", "embed", "compressed"]:
if data[should_be_false]:
verrors.add(should_be_false, "Legacy replication does not support this option")
if data["direction"] != "PUSH":
verrors.add("direction", "Only push application is allowed for Legacy transport")
if len(data["source_datasets"]) != 1:
verrors.add("source_datasets", "You can only have one source dataset for legacy replication")
if data["retries"] != 1:
verrors.add("retries", "This value should be 1 for legacy replication")
# Common for all directions and transports
for i, source_dataset in enumerate(data["source_datasets"]):
for snapshot_task in snapshot_tasks:
if is_child(source_dataset, snapshot_task["dataset"]):
if data["recursive"]:
for exclude in snapshot_task["exclude"]:
if exclude not in data["exclude"]:
verrors.add("exclude", f"You should exclude {exclude!r} as bound periodic snapshot "
f"task dataset {snapshot_task['dataset']!r} does")
else:
if source_dataset in snapshot_task["exclude"]:
verrors.add(f"source_datasets.{i}", f"Dataset {source_dataset!r} is excluded by bound "
f"periodic snapshot task for dataset "
f"{snapshot_task['dataset']!r}")
if not data["recursive"] and data["exclude"]:
verrors.add("exclude", "Excluding child datasets is only supported for recursive replication")
for i, v in enumerate(data["exclude"]):
if not any(v.startswith(ds + "/") for ds in data["source_datasets"]):
verrors.add(f"exclude.{i}", "This dataset is not a child of any of source datasets")
if data["schedule"]:
if not data["auto"]:
verrors.add("schedule", "You can't have schedule for replication that does not run automatically")
else:
if data["only_matching_schedule"]:
verrors.add("only_matching_schedule", "You can't have only-matching-schedule without schedule")
if data["retention_policy"] == "CUSTOM":
if data["lifetime_value"] is None:
verrors.add("lifetime_value", "This field is required for custom retention policy")
if data["lifetime_unit"] is None:
verrors.add("lifetime_value", "This field is required for custom retention policy")
else:
if data["lifetime_value"] is not None:
verrors.add("lifetime_value", "This field has no sense for specified retention policy")
if data["lifetime_unit"] is not None:
verrors.add("lifetime_unit", "This field has no sense for specified retention policy")
if data["enabled"]:
for i, snapshot_task in enumerate(snapshot_tasks):
if not snapshot_task["enabled"]:
verrors.add(
f"periodic_snapshot_tasks.{i}",
"You can't bind disabled periodic snapshot task to enabled replication task"
)
return verrors
async def do_create(self, data):
"""
Create a Periodic Snapshot Task
Create a Periodic Snapshot Task that will take snapshots of specified `dataset` at specified `schedule`.
Recursive snapshots can be created if `recursive` flag is enabled. You can `exclude` specific child datasets
or zvols from the snapshot.
Snapshots will be automatically destroyed after a certain amount of time, specified by
`lifetime_value` and `lifetime_unit`.
If multiple periodic tasks create snapshots at the same time (for example hourly and daily at 00:00) the snapshot
will be kept until the last of these tasks reaches its expiry time.
Snapshots will be named according to `naming_schema` which is a `strftime`-like template for snapshot name
and must contain `%Y`, `%m`, `%d`, `%H` and `%M`.
.. examples(websocket)::
Create a recursive Periodic Snapshot Task for dataset `data/work` excluding `data/work/temp`. Snapshots
will be created on weekdays every hour from 09:00 to 18:00 and will be stored for two weeks.
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "pool.snapshottask.create",
"params": [{
"dataset": "data/work",
"recursive": true,
"exclude": ["data/work/temp"],
"lifetime_value": 2,
"lifetime_unit": "WEEK",
"naming_schema": "auto_%Y-%m-%d_%H-%M",
"schedule": {
"minute": "0",
"hour": "*",
"dom": "*",
"month": "*",
"dow": "1,2,3,4,5",
"begin": "09:00",
"end": "18:00"
}
}]
}
"""
verrors = ValidationErrors()
verrors.add_child('periodic_snapshot_create', await
self._validate(data))
if verrors:
raise verrors
Cron.convert_schedule_to_db_format(data, begin_end=True)
data['id'] = await self.middleware.call(
'datastore.insert', self._config.datastore, data,
{'prefix': self._config.datastore_prefix})
await self.middleware.call('zettarepl.update_tasks')
return await self._get_instance(data['id'])
async def do_update(self, id, data):
"""
Update a Periodic Snapshot Task with specific `id`
See the documentation for `create` method for information on payload contents
.. examples(websocket)::
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "pool.snapshottask.update",
"params": [
1,
{
"dataset": "data/work",
"recursive": true,
"exclude": ["data/work/temp"],
"lifetime_value": 2,
"lifetime_unit": "WEEK",
"naming_schema": "auto_%Y-%m-%d_%H-%M",
"schedule": {
"minute": "0",
"hour": "*",
"dom": "*",
"month": "*",
"dow": "1,2,3,4,5",
"begin": "09:00",
"end": "18:00"
}
}
]
}
"""
old = await self._get_instance(id)
new = old.copy()
new.update(data)
verrors = ValidationErrors()
verrors.add_child('periodic_snapshot_update', await
self._validate(new))
if not new['enabled']:
for replication_task in await self.middleware.call(
'replication.query', [['enabled', '=', True]]):
if any(periodic_snapshot_task['id'] == id
for periodic_snapshot_task in
replication_task['periodic_snapshot_tasks']):
verrors.add('periodic_snapshot_update.enabled', (
f'You can\'t disable this periodic snapshot task because it is bound to enabled replication '
f'task {replication_task["id"]!r}'))
break
if verrors:
raise verrors
Cron.convert_schedule_to_db_format(new, begin_end=True)
for key in ('vmware_sync', 'state'):
new.pop(key, None)
await self.middleware.call('datastore.update', self._config.datastore,
id, new,
{'prefix': self._config.datastore_prefix})
await self.middleware.call('zettarepl.update_tasks')
return await self._get_instance(id)
async def _validate(self, data, id=None):
verrors = ValidationErrors()
await self._ensure_unique(verrors, "", "name", data["name"], id)
# Direction
snapshot_tasks = []
if data["direction"] == "PUSH":
e, snapshot_tasks = await self._query_periodic_snapshot_tasks(data["periodic_snapshot_tasks"])
verrors.add_child("periodic_snapshot_tasks", e)
if data["naming_schema"]:
verrors.add("naming_schema", "This field has no sense for push replication")
if data["transport"] != "LEGACY" and not snapshot_tasks and not data["also_include_naming_schema"]:
verrors.add(
"periodic_snapshot_tasks", "You must at least either bind a periodic snapshot task or provide "
"\"Also Include Naming Schema\" for push replication task"
)
if data["schedule"]:
if data["periodic_snapshot_tasks"]:
verrors.add("schedule", "Push replication can't be bound to periodic snapshot task and have "
"schedule at the same time")
else:
if data["auto"] and not data["periodic_snapshot_tasks"] and data["transport"] != "LEGACY":
verrors.add("auto", "Push replication that runs automatically must be either "
"bound to periodic snapshot task or have schedule")
if data["direction"] == "PULL":
if data["schedule"]:
pass
else:
if data["auto"]:
verrors.add("auto", "Pull replication that runs automatically must have schedule")
if data["periodic_snapshot_tasks"]:
verrors.add("periodic_snapshot_tasks", "Pull replication can't be bound to periodic snapshot task")
if not data["naming_schema"]:
verrors.add("naming_schema", "Naming schema is required for pull replication")
if data["also_include_naming_schema"]:
verrors.add("also_include_naming_schema", "This field has no sense for pull replication")
if data["hold_pending_snapshots"]:
verrors.add("hold_pending_snapshots", "Pull replication tasks can't hold pending snapshots because "
"they don't do source retention")
# Transport
if data["transport"] == "SSH+NETCAT":
if data["netcat_active_side"] is None:
verrors.add("netcat_active_side", "You must choose active side for SSH+netcat replication")
if data["netcat_active_side_port_min"] is not None and data["netcat_active_side_port_max"] is not None:
if data["netcat_active_side_port_min"] > data["netcat_active_side_port_max"]:
verrors.add("netcat_active_side_port_max",
"Please specify value greater or equal than netcat_active_side_port_min")
if data["compression"] is not None:
verrors.add("compression", "Compression is not supported for SSH+netcat replication")
if data["speed_limit"] is not None:
verrors.add("speed_limit", "Speed limit is not supported for SSH+netcat replication")
else:
if data["netcat_active_side"] is not None:
verrors.add("netcat_active_side", "This field only has sense for SSH+netcat replication")
for k in ["netcat_active_side_listen_address", "netcat_active_side_port_min", "netcat_active_side_port_max",
"netcat_passive_side_connect_address"]:
if data[k] is not None:
verrors.add(k, "This field only has sense for SSH+netcat replication")
if data["transport"] == "LOCAL":
if data["ssh_credentials"] is not None:
verrors.add("ssh_credentials", "Remote credentials have no sense for local replication")
if data["compression"] is not None:
verrors.add("compression", "Compression has no sense for local replication")
if data["speed_limit"] is not None:
verrors.add("speed_limit", "Speed limit has no sense for local replication")
else:
if data["ssh_credentials"] is None:
verrors.add("ssh_credentials", "SSH Credentials are required for non-local replication")
else:
try:
await self.middleware.call("keychaincredential.get_of_type", data["ssh_credentials"],
"SSH_CREDENTIALS")
except CallError as e:
verrors.add("ssh_credentials", str(e))
if data["transport"] == "LEGACY":
for should_be_true in ["auto", "allow_from_scratch"]:
if not data[should_be_true]:
verrors.add(should_be_true, "Legacy replication does not support disabling this option")
for should_be_false in ["exclude", "periodic_snapshot_tasks", "naming_schema", "also_include_naming_schema",
"only_matching_schedule", "dedup", "large_block", "embed", "compressed"]:
if data[should_be_false]:
verrors.add(should_be_false, "Legacy replication does not support this option")
if data["direction"] != "PUSH":
verrors.add("direction", "Only push application is allowed for Legacy transport")
if len(data["source_datasets"]) != 1:
verrors.add("source_datasets", "You can only have one source dataset for legacy replication")
if os.path.basename(data["target_dataset"]) != os.path.basename(data["source_datasets"][0]):
verrors.add(
"target_dataset",
"Target dataset basename should be same as source dataset basename for Legacy transport",
)
if data["retention_policy"] not in ["SOURCE", "NONE"]:
verrors.add("retention_policy", "Only \"source\" and \"none\" retention policies are supported by "
"legacy replication")
if data["retries"] != 1:
verrors.add("retries", "This value should be 1 for legacy replication")
# Common for all directions and transports
for i, source_dataset in enumerate(data["source_datasets"]):
for snapshot_task in snapshot_tasks:
if is_child(source_dataset, snapshot_task["dataset"]):
if data["recursive"]:
for exclude in snapshot_task["exclude"]:
if exclude not in data["exclude"]:
verrors.add("exclude", f"You should exclude {exclude!r} as bound periodic snapshot "
f"task dataset {snapshot_task['dataset']!r} does")
else:
if source_dataset in snapshot_task["exclude"]:
verrors.add(f"source_datasets.{i}", f"Dataset {source_dataset!r} is excluded by bound "
f"periodic snapshot task for dataset "
f"{snapshot_task['dataset']!r}")
if not data["recursive"] and data["exclude"]:
verrors.add("exclude", "Excluding child datasets is only supported for recursive replication")
for i, v in enumerate(data["exclude"]):
if not any(v.startswith(ds + "/") for ds in data["source_datasets"]):
verrors.add(f"exclude.{i}", "This dataset is not a child of any of source datasets")
if data["schedule"]:
if not data["auto"]:
verrors.add("schedule", "You can't have schedule for replication that does not run automatically")
else:
if data["only_matching_schedule"]:
verrors.add("only_matching_schedule", "You can't have only-matching-schedule without schedule")
if data["retention_policy"] == "CUSTOM":
if data["lifetime_value"] is None:
verrors.add("lifetime_value", "This field is required for custom retention policy")
if data["lifetime_unit"] is None:
verrors.add("lifetime_value", "This field is required for custom retention policy")
else:
if data["lifetime_value"] is not None:
verrors.add("lifetime_value", "This field has no sense for specified retention policy")
if data["lifetime_unit"] is not None:
verrors.add("lifetime_unit", "This field has no sense for specified retention policy")
if data["enabled"]:
for i, snapshot_task in enumerate(snapshot_tasks):
if not snapshot_task["enabled"]:
verrors.add(
f"periodic_snapshot_tasks.{i}",
"You can't bind disabled periodic snapshot task to enabled replication task"
)
return verrors
