def _is_legacy(self, data, legacy_replication_tasks): if data['naming_schema'] == self._legacy_naming_schema(data): for replication_task in legacy_replication_tasks: if ( data['dataset'] == replication_task['source_datasets'][0] or (data['recursive'] and is_child(replication_task['source_datasets'][0], data['dataset'])) or (replication_task['recursive'] and is_child(data['dataset'], replication_task['source_datasets'][0])) ): return True return False
async def query(self, path, enabled): results = [] for replication in await self.middleware.call('replication.query', [['enabled', '=', enabled]]): if replication['direction'] == 'PUSH': if any(is_child(os.path.join('/mnt', source_dataset), path) for source_dataset in replication['source_datasets']): results.append(replication) if replication['direction'] == 'PULL': if is_child(os.path.join('/mnt', replication['target_dataset']), path): results.append(replication) return results
async def query(self, path, enabled, options=None): vms_attached = [] ignored_vms = { vm['id']: vm for vm in await self.middleware.call('vm.query', [( 'status.state', '!=' if enabled else '=', 'RUNNING')]) } for device in await self.middleware.call('datastore.query', 'vm.device'): if (device['dtype'] not in ('DISK', 'RAW', 'CDROM')) or device['vm']['id'] in ignored_vms: continue disk = device['attributes'].get('path') if not disk: continue if disk.startswith('/dev/zvol'): disk = os.path.join('/mnt', zvol_path_to_name(disk)) if is_child(disk, path): vm = { 'id': device['vm'].get('id'), 'name': device['vm'].get('name'), } if vm not in vms_attached: vms_attached.append(vm) return vms_attached
def statfs(self, path): """ Return stats from the filesystem of a given path. Raises: CallError(ENOENT) - Path not found """ try: st = os.statvfs(path) except FileNotFoundError: raise CallError('Path not found.', errno.ENOENT) for partition in sorted(psutil.disk_partitions(), key=lambda p: len(p.mountpoint), reverse=True): if is_child(os.path.realpath(path), partition.mountpoint): break else: raise CallError('Unable to find mountpoint.') return { 'flags': [], 'fstype': partition.fstype, 'source': partition.device, 'dest': partition.mountpoint, 'blocksize': st.f_frsize, 'total_blocks': st.f_blocks, 'free_blocks': st.f_bfree, 'avail_blocks': st.f_bavail, 'files': st.f_files, 'free_files': st.f_ffree, 'name_max': st.f_namemax, 'fsid': [], 'total_bytes': st.f_blocks * st.f_frsize, 'free_bytes': st.f_bfree * st.f_frsize, 'avail_bytes': st.f_bavail * st.f_frsize, }
async def query(self, path, enabled): results = [] for nfs in await self.middleware.call('sharing.nfs.query', [['enabled', '=', enabled]]): if any(is_child(nfs_path, path) for nfs_path in nfs['paths']): results.append(nfs) return results
async def query(self, path, enabled, options=None): results = [] for task in await self.middleware.call('pool.snapshottask.query', [['enabled', '=', enabled]]): if is_child(os.path.join('/mnt', task['dataset']), path): results.append(task) return results
async def query(self, path, enabled): results = [] for afp in await self.middleware.call('sharing.afp.query', [['enabled', '=', enabled]]): if is_child(afp['path'], path): results.append(afp) return results
async def query(self, path, enabled): results = [] for share in await self.middleware.call('sharing.webdav.query', [['enabled', '=', enabled]]): if is_child(share['path'], path): results.append(share) return results
async def extend(self, data, context): Cron.convert_db_format_to_schedule(data, begin_end=True) data['vmware_sync'] = any((vmware['filesystem'] == data['dataset'] or ( data['recursive'] and is_child(vmware['filesystem'], data['dataset']))) for vmware in context['vmware']) if 'error' in context['state']: data['state'] = context['state']['error'] else: data['state'] = context['state']['tasks'].get( f'periodic_snapshot_task_{data["id"]}', { 'state': 'PENDING', }) return data
async def extend(self, data, context): Cron.convert_db_format_to_schedule(data, begin_end=True) data['legacy'] = self._is_legacy(data, context['legacy_replication_tasks']) data['vmware_sync'] = any((vmware['filesystem'] == data['dataset'] or ( data['recursive'] and is_child(vmware['filesystem'], data['dataset']))) for vmware in context['vmware']) data['state'] = context['state'].get( f'periodic_snapshot_task_{data["id"]}', { 'state': 'UNKNOWN', }) return data
async def query(self, path, enabled, options=None): results = [] s3_config = await self.middleware.call('s3.config') if not s3_config['storage_path'] or not os.path.exists( s3_config['storage_path']): return [] s3_ds = await self.middleware.call('zfs.dataset.path_to_dataset', s3_config['storage_path']) if s3_ds is None: return [] if is_child(os.path.join('/mnt', s3_ds), path): results.append({'id': s3_ds}) return results
async def extend(self, data, context): Cron.convert_db_format_to_schedule(data, begin_end=True) data['legacy'] = self._is_legacy(data, context['legacy_replication_tasks']) data['vmware_sync'] = any( ( vmware['filesystem'] == data['dataset'] or (data['recursive'] and is_child(vmware['filesystem'], data['dataset'])) ) for vmware in context['vmware'] ) data['state'] = context['state'].get(f'periodic_snapshot_task_{data["id"]}', { 'state': 'UNKNOWN', }) return data
async def query(self, path, enabled, options=None): chart_releases_attached = [] for release in await self.middleware.call( 'chart.release.query', [], {'extra': { 'retrieve_resources': True }}): if not release['resources']['host_path_volumes'] or ( release['status'] == 'STOPPED' if enabled else release['status'] != 'STOPPED'): continue if any( is_child(p['host_path']['path'], path) for p in release['resources']['host_path_volumes']): chart_releases_attached.append({ 'id': release['name'], 'name': release['name'], }) return chart_releases_attached
async def _validate(self, data, id=None): verrors = ValidationErrors() await self._ensure_unique(verrors, "", "name", data["name"], id) # Direction snapshot_tasks = [] if data["direction"] == "PUSH": e, snapshot_tasks = await self._query_periodic_snapshot_tasks( data["periodic_snapshot_tasks"]) verrors.add_child("periodic_snapshot_tasks", e) if data["naming_schema"]: verrors.add("naming_schema", "This field has no sense for push replication") if not snapshot_tasks and not data["also_include_naming_schema"]: verrors.add( "periodic_snapshot_tasks", "You must at least either bind a periodic snapshot task or provide " "\"Also Include Naming Schema\" for push replication task") if data["schedule"] is None and data[ "auto"] and not data["periodic_snapshot_tasks"]: verrors.add( "auto", "Push replication that runs automatically must be either " "bound to a periodic snapshot task or have a schedule") if data["direction"] == "PULL": if data["schedule"] is None and data["auto"]: verrors.add( "auto", "Pull replication that runs automatically must have a schedule" ) if data["periodic_snapshot_tasks"]: verrors.add( "periodic_snapshot_tasks", "Pull replication can't be bound to a periodic snapshot task" ) if not data["naming_schema"]: verrors.add("naming_schema", "Naming schema is required for pull replication") if data["also_include_naming_schema"]: verrors.add("also_include_naming_schema", "This field has no sense for pull replication") if data["hold_pending_snapshots"]: verrors.add( "hold_pending_snapshots", "Pull replication tasks can't hold pending snapshots because " "they don't do source retention") # Transport if data["transport"] == "SSH+NETCAT": if data["netcat_active_side"] is None: verrors.add( "netcat_active_side", "You must choose active side for SSH+netcat replication") if data["netcat_active_side_port_min"] is not None and data[ "netcat_active_side_port_max"] is not None: if data["netcat_active_side_port_min"] > data[ "netcat_active_side_port_max"]: verrors.add( "netcat_active_side_port_max", "Please specify value greater or equal than netcat_active_side_port_min" ) if data["compression"] is not None: verrors.add( "compression", "Compression is not supported for SSH+netcat replication") if data["speed_limit"] is not None: verrors.add( "speed_limit", "Speed limit is not supported for SSH+netcat replication") else: if data["netcat_active_side"] is not None: verrors.add( "netcat_active_side", "This field only has sense for SSH+netcat replication") for k in [ "netcat_active_side_listen_address", "netcat_active_side_port_min", "netcat_active_side_port_max", "netcat_passive_side_connect_address" ]: if data[k] is not None: verrors.add( k, "This field only has sense for SSH+netcat replication") if data["transport"] == "LOCAL": if data["ssh_credentials"] is not None: verrors.add( "ssh_credentials", "Remote credentials have no sense for local replication") if data["compression"] is not None: verrors.add("compression", "Compression has no sense for local replication") if data["speed_limit"] is not None: verrors.add("speed_limit", "Speed limit has no sense for local replication") else: if data["ssh_credentials"] is None: verrors.add( "ssh_credentials", "SSH Credentials are required for non-local replication") else: try: await self.middleware.call( "keychaincredential.get_of_type", data["ssh_credentials"], "SSH_CREDENTIALS") except CallError as e: verrors.add("ssh_credentials", str(e)) # Common for all directions and transports for i, source_dataset in enumerate(data["source_datasets"]): for snapshot_task in snapshot_tasks: if is_child(source_dataset, snapshot_task["dataset"]): if data["recursive"]: for exclude in snapshot_task["exclude"]: if is_child(exclude, source_dataset ) and exclude not in data["exclude"]: verrors.add( "exclude", f"You should exclude {exclude!r} as bound periodic snapshot " f"task dataset {snapshot_task['dataset']!r} does" ) else: if source_dataset in snapshot_task["exclude"]: verrors.add( f"source_datasets.{i}", f"Dataset {source_dataset!r} is excluded by bound " f"periodic snapshot task for dataset " f"{snapshot_task['dataset']!r}") if not data["recursive"] and data["exclude"]: verrors.add( "exclude", "Excluding child datasets is only supported for recursive replication" ) for i, v in enumerate(data["exclude"]): if not any( v.startswith(ds + "/") for ds in data["source_datasets"]): verrors.add( f"exclude.{i}", "This dataset is not a child of any of source datasets") if data["replicate"]: if not data["recursive"]: verrors.add( "recursive", "This option is required for full filesystem replication") if data["exclude"]: verrors.add( "exclude", "This option is not supported for full filesystem replication" ) if not data["properties"]: verrors.add( "properties", "This option is required for full filesystem replication") if data["encryption"]: for k in [ "encryption_key", "encryption_key_format", "encryption_key_location" ]: if data[k] is None: verrors.add( k, "This property is required when remote dataset encryption is enabled" ) if data["schedule"]: if not data["auto"]: verrors.add( "schedule", "You can't have schedule for replication that does not run automatically" ) else: if data["only_matching_schedule"]: verrors.add( "only_matching_schedule", "You can't have only-matching-schedule without schedule") if data["retention_policy"] == "CUSTOM": if data["lifetime_value"] is None: verrors.add( "lifetime_value", "This field is required for custom retention policy") if data["lifetime_unit"] is None: verrors.add( "lifetime_value", "This field is required for custom retention policy") else: if data["lifetime_value"] is not None: verrors.add( "lifetime_value", "This field has no sense for specified retention policy") if data["lifetime_unit"] is not None: verrors.add( "lifetime_unit", "This field has no sense for specified retention policy") if data["enabled"]: for i, snapshot_task in enumerate(snapshot_tasks): if not snapshot_task["enabled"]: verrors.add( f"periodic_snapshot_tasks.{i}", "You can't bind disabled periodic snapshot task to enabled replication task" ) return verrors
async def is_child_of_path(self, resource, path): return any( is_child(nfs_path, path) for nfs_path in resource[self.path_field])
async def _validate(self, data, id=None): verrors = ValidationErrors() await self._ensure_unique(verrors, "", "name", data["name"], id) # Direction snapshot_tasks = [] if data["direction"] == "PUSH": e, snapshot_tasks = await self._query_periodic_snapshot_tasks(data["periodic_snapshot_tasks"]) verrors.add_child("periodic_snapshot_tasks", e) if data["naming_schema"]: verrors.add("naming_schema", "This field has no sense for push replication") if data["transport"] != "LEGACY" and not snapshot_tasks and not data["also_include_naming_schema"]: verrors.add( "periodic_snapshot_tasks", "You must at least either bind a periodic snapshot task or provide " "\"Also Include Naming Schema\" for push replication task" ) if data["schedule"]: if data["periodic_snapshot_tasks"]: verrors.add("schedule", "Push replication can't be bound to periodic snapshot task and have " "schedule at the same time") else: if data["auto"] and not data["periodic_snapshot_tasks"] and data["transport"] != "LEGACY": verrors.add("auto", "Push replication that runs automatically must be either " "bound to periodic snapshot task or have schedule") if data["direction"] == "PULL": if data["schedule"]: pass else: if data["auto"]: verrors.add("auto", "Pull replication that runs automatically must have schedule") if data["periodic_snapshot_tasks"]: verrors.add("periodic_snapshot_tasks", "Pull replication can't be bound to periodic snapshot task") if not data["naming_schema"]: verrors.add("naming_schema", "Naming schema is required for pull replication") if data["also_include_naming_schema"]: verrors.add("also_include_naming_schema", "This field has no sense for pull replication") if data["hold_pending_snapshots"]: verrors.add("hold_pending_snapshots", "Pull replication tasks can't hold pending snapshots because " "they don't do source retention") # Transport if data["transport"] == "SSH+NETCAT": if data["netcat_active_side"] is None: verrors.add("netcat_active_side", "You must choose active side for SSH+netcat replication") if data["netcat_active_side_port_min"] is not None and data["netcat_active_side_port_max"] is not None: if data["netcat_active_side_port_min"] > data["netcat_active_side_port_max"]: verrors.add("netcat_active_side_port_max", "Please specify value greater or equal than netcat_active_side_port_min") if data["compression"] is not None: verrors.add("compression", "Compression is not supported for SSH+netcat replication") if data["speed_limit"] is not None: verrors.add("speed_limit", "Speed limit is not supported for SSH+netcat replication") else: if data["netcat_active_side"] is not None: verrors.add("netcat_active_side", "This field only has sense for SSH+netcat replication") for k in ["netcat_active_side_listen_address", "netcat_active_side_port_min", "netcat_active_side_port_max", "netcat_passive_side_connect_address"]: if data[k] is not None: verrors.add(k, "This field only has sense for SSH+netcat replication") if data["transport"] == "LOCAL": if data["ssh_credentials"] is not None: verrors.add("ssh_credentials", "Remote credentials have no sense for local replication") if data["compression"] is not None: verrors.add("compression", "Compression has no sense for local replication") if data["speed_limit"] is not None: verrors.add("speed_limit", "Speed limit has no sense for local replication") else: if data["ssh_credentials"] is None: verrors.add("ssh_credentials", "SSH Credentials are required for non-local replication") else: try: await self.middleware.call("keychaincredential.get_of_type", data["ssh_credentials"], "SSH_CREDENTIALS") except CallError as e: verrors.add("ssh_credentials", str(e)) if data["transport"] == "LEGACY": for should_be_true in ["auto", "allow_from_scratch"]: if not data[should_be_true]: verrors.add(should_be_true, "Legacy replication does not support disabling this option") for should_be_false in ["exclude", "periodic_snapshot_tasks", "naming_schema", "also_include_naming_schema", "only_matching_schedule", "dedup", "large_block", "embed", "compressed"]: if data[should_be_false]: verrors.add(should_be_false, "Legacy replication does not support this option") if data["direction"] != "PUSH": verrors.add("direction", "Only push application is allowed for Legacy transport") if len(data["source_datasets"]) != 1: verrors.add("source_datasets", "You can only have one source dataset for legacy replication") if os.path.basename(data["target_dataset"]) != os.path.basename(data["source_datasets"][0]): verrors.add( "target_dataset", "Target dataset basename should be same as source dataset basename for Legacy transport", ) if data["retention_policy"] not in ["SOURCE", "NONE"]: verrors.add("retention_policy", "Only \"source\" and \"none\" retention policies are supported by " "legacy replication") if data["retries"] != 1: verrors.add("retries", "This value should be 1 for legacy replication") # Common for all directions and transports for i, source_dataset in enumerate(data["source_datasets"]): for snapshot_task in snapshot_tasks: if is_child(source_dataset, snapshot_task["dataset"]): if data["recursive"]: for exclude in snapshot_task["exclude"]: if exclude not in data["exclude"]: verrors.add("exclude", f"You should exclude {exclude!r} as bound periodic snapshot " f"task dataset {snapshot_task['dataset']!r} does") else: if source_dataset in snapshot_task["exclude"]: verrors.add(f"source_datasets.{i}", f"Dataset {source_dataset!r} is excluded by bound " f"periodic snapshot task for dataset " f"{snapshot_task['dataset']!r}") if not data["recursive"] and data["exclude"]: verrors.add("exclude", "Excluding child datasets is only supported for recursive replication") for i, v in enumerate(data["exclude"]): if not any(v.startswith(ds + "/") for ds in data["source_datasets"]): verrors.add(f"exclude.{i}", "This dataset is not a child of any of source datasets") if data["schedule"]: if not data["auto"]: verrors.add("schedule", "You can't have schedule for replication that does not run automatically") else: if data["only_matching_schedule"]: verrors.add("only_matching_schedule", "You can't have only-matching-schedule without schedule") if data["retention_policy"] == "CUSTOM": if data["lifetime_value"] is None: verrors.add("lifetime_value", "This field is required for custom retention policy") if data["lifetime_unit"] is None: verrors.add("lifetime_value", "This field is required for custom retention policy") else: if data["lifetime_value"] is not None: verrors.add("lifetime_value", "This field has no sense for specified retention policy") if data["lifetime_unit"] is not None: verrors.add("lifetime_unit", "This field has no sense for specified retention policy") if data["enabled"]: for i, snapshot_task in enumerate(snapshot_tasks): if not snapshot_task["enabled"]: verrors.add( f"periodic_snapshot_tasks.{i}", "You can't bind disabled periodic snapshot task to enabled replication task" ) return verrors
async def _validate(self, data): verrors = ValidationErrors() # Direction snapshot_tasks = [] if data["direction"] == "PUSH": e, snapshot_tasks = await self._query_periodic_snapshot_tasks(data["periodic_snapshot_tasks"]) verrors.add_child("periodic_snapshot_tasks", e) if data["naming_schema"]: verrors.add("naming_schema", "This field has no sense for push replication") if data["transport"] != "LEGACY" and not snapshot_tasks and not data["also_include_naming_schema"]: verrors.add( "periodic_snapshot_tasks", "You must at least either bind a periodic snapshot task or provide " "\"Also Include Naming Schema\" for push replication task" ) if data["schedule"]: if data["periodic_snapshot_tasks"]: verrors.add("schedule", "Push replication can't be bound to periodic snapshot task and have " "schedule at the same time") else: if data["auto"] and not data["periodic_snapshot_tasks"] and data["transport"] != "LEGACY": verrors.add("auto", "Push replication that runs automatically must be either " "bound to periodic snapshot task or have schedule") if data["direction"] == "PULL": if data["schedule"]: pass else: if data["auto"]: verrors.add("auto", "Pull replication that runs automatically must have schedule") if data["periodic_snapshot_tasks"]: verrors.add("periodic_snapshot_tasks", "Pull replication can't be bound to periodic snapshot task") if not data["naming_schema"]: verrors.add("naming_schema", "Naming schema is required for pull replication") if data["also_include_naming_schema"]: verrors.add("also_include_naming_schema", "This field has no sense for pull replication") if data["hold_pending_snapshots"]: verrors.add("hold_pending_snapshots", "Pull replication tasks can't hold pending snapshots because " "they don't do source retention") # Transport if data["transport"] == "SSH+NETCAT": if data["netcat_active_side"] is None: verrors.add("netcat_active_side", "You must choose active side for SSH+netcat replication") if data["netcat_active_side_port_min"] is not None and data["netcat_active_side_port_max"] is not None: if data["netcat_active_side_port_min"] > data["netcat_active_side_port_max"]: verrors.add("netcat_active_side_port_max", "Please specify value greater or equal than netcat_active_side_port_min") if data["compression"] is not None: verrors.add("compression", "Compression is not supported for SSH+netcat replication") if data["speed_limit"] is not None: verrors.add("speed_limit", "Speed limit is not supported for SSH+netcat replication") else: if data["netcat_active_side"] is not None: verrors.add("netcat_active_side", "This field only has sense for SSH+netcat replication") for k in ["netcat_active_side_listen_address", "netcat_active_side_port_min", "netcat_active_side_port_max", "netcat_passive_side_connect_address"]: if data[k] is not None: verrors.add(k, "This field only has sense for SSH+netcat replication") if data["transport"] == "LOCAL": if data["ssh_credentials"] is not None: verrors.add("ssh_credentials", "Remote credentials have no sense for local replication") if data["compression"] is not None: verrors.add("compression", "Compression has no sense for local replication") if data["speed_limit"] is not None: verrors.add("speed_limit", "Speed limit has no sense for local replication") else: if data["ssh_credentials"] is None: verrors.add("ssh_credentials", "SSH Credentials are required for non-local replication") else: try: await self.middleware.call("keychaincredential.get_of_type", data["ssh_credentials"], "SSH_CREDENTIALS") except CallError as e: verrors.add("ssh_credentials", str(e)) if data["transport"] == "LEGACY": for should_be_true in ["auto", "allow_from_scratch"]: if not data[should_be_true]: verrors.add(should_be_true, "Legacy replication does not support disabling this option") for should_be_false in ["exclude", "periodic_snapshot_tasks", "naming_schema", "also_include_naming_schema", "only_matching_schedule", "dedup", "large_block", "embed", "compressed"]: if data[should_be_false]: verrors.add(should_be_false, "Legacy replication does not support this option") if data["direction"] != "PUSH": verrors.add("direction", "Only push application is allowed for Legacy transport") if len(data["source_datasets"]) != 1: verrors.add("source_datasets", "You can only have one source dataset for legacy replication") if data["retries"] != 1: verrors.add("retries", "This value should be 1 for legacy replication") # Common for all directions and transports for i, source_dataset in enumerate(data["source_datasets"]): for snapshot_task in snapshot_tasks: if is_child(source_dataset, snapshot_task["dataset"]): if data["recursive"]: for exclude in snapshot_task["exclude"]: if exclude not in data["exclude"]: verrors.add("exclude", f"You should exclude {exclude!r} as bound periodic snapshot " f"task dataset {snapshot_task['dataset']!r} does") else: if source_dataset in snapshot_task["exclude"]: verrors.add(f"source_datasets.{i}", f"Dataset {source_dataset!r} is excluded by bound " f"periodic snapshot task for dataset " f"{snapshot_task['dataset']!r}") if not data["recursive"] and data["exclude"]: verrors.add("exclude", "Excluding child datasets is only supported for recursive replication") for i, v in enumerate(data["exclude"]): if not any(v.startswith(ds + "/") for ds in data["source_datasets"]): verrors.add(f"exclude.{i}", "This dataset is not a child of any of source datasets") if data["schedule"]: if not data["auto"]: verrors.add("schedule", "You can't have schedule for replication that does not run automatically") else: if data["only_matching_schedule"]: verrors.add("only_matching_schedule", "You can't have only-matching-schedule without schedule") if data["retention_policy"] == "CUSTOM": if data["lifetime_value"] is None: verrors.add("lifetime_value", "This field is required for custom retention policy") if data["lifetime_unit"] is None: verrors.add("lifetime_value", "This field is required for custom retention policy") else: if data["lifetime_value"] is not None: verrors.add("lifetime_value", "This field has no sense for specified retention policy") if data["lifetime_unit"] is not None: verrors.add("lifetime_unit", "This field has no sense for specified retention policy") if data["enabled"]: for i, snapshot_task in enumerate(snapshot_tasks): if not snapshot_task["enabled"]: verrors.add( f"periodic_snapshot_tasks.{i}", "You can't bind disabled periodic snapshot task to enabled replication task" ) return verrors
async def is_child_of_path(self, resource, path): return is_child(resource[self.path_field], path)