class CloudSyncService(TaskPathService): local_fs_lock_manager = FsLockManager() remote_fs_lock_manager = FsLockManager() share_task_type = 'CloudSync' class Config: datastore = "tasks.cloudsync" datastore_extend = "cloudsync.extend" @filterable async def query(self, filters=None, options=None): """ Query all Cloud Sync Tasks with `query-filters` and `query-options`. """ tasks_or_task = await super().query(filters, options) jobs = {} for j in await self.middleware.call( "core.get_jobs", [('OR', [("method", "=", "cloudsync.sync"), ("method", "=", "cloudsync.restore")])], {"order_by": ["id"]}): try: task_id = int(j["arguments"][0]) except (IndexError, ValueError): continue if task_id in jobs and jobs[task_id]["state"] == "RUNNING": continue jobs[task_id] = j if isinstance(tasks_or_task, list): for task in tasks_or_task: task["job"] = jobs.get(task["id"]) else: tasks_or_task["job"] = jobs.get(tasks_or_task["id"]) return tasks_or_task @private async def extend(self, cloud_sync): cloud_sync["credentials"] = cloud_sync.pop("credential") Cron.convert_db_format_to_schedule(cloud_sync) return cloud_sync @private async def _compress(self, cloud_sync): cloud_sync["credential"] = cloud_sync.pop("credentials") Cron.convert_schedule_to_db_format(cloud_sync) cloud_sync.pop('job', None) cloud_sync.pop(self.locked_field, None) return cloud_sync @private async def _get_credentials(self, credentials_id): try: return await self.middleware.call("datastore.query", "system.cloudcredentials", [("id", "=", credentials_id)], {"get": True}) except IndexError: return None @private async def _basic_validate(self, verrors, name, data): if data["encryption"]: if not data["encryption_password"]: verrors.add( f"{name}.encryption_password", "This field is required when encryption is enabled") credentials = await self._get_credentials(data["credentials"]) if not credentials: verrors.add(f"{name}.credentials", "Invalid credentials") try: shlex.split(data["args"]) except ValueError as e: verrors.add(f"{name}.args", f"Parse error: {e.args[0]}") if verrors: raise verrors provider = REMOTES[credentials["provider"]] schema = [] if provider.buckets: schema.append(Str("bucket", required=True, empty=False)) schema.append(Str("folder", required=True)) schema.extend(provider.task_schema) schema.extend(self.common_task_schema(provider)) attributes_verrors = validate_attributes(schema, data, additional_attrs=True) if not attributes_verrors: await provider.pre_save_task(data, credentials, verrors) verrors.add_child(f"{name}.attributes", attributes_verrors) @private async def _validate(self, verrors, name, data): await self._basic_validate(verrors, name, data) for i, (limit1, limit2) in enumerate(zip(data["bwlimit"], data["bwlimit"][1:])): if limit1["time"] >= limit2["time"]: verrors.add( f"{name}.bwlimit.{i + 1}.time", f"Invalid time order: {limit1['time']}, {limit2['time']}") await self.validate_path_field(data, name, verrors) if data["snapshot"]: if data["direction"] != "PUSH": verrors.add(f"{name}.snapshot", "This option can only be enabled for PUSH tasks") if data["transfer_mode"] == "MOVE": verrors.add( f"{name}.snapshot", "This option can not be used for MOVE transfer mode") if await self.middleware.call( "pool.dataset.query", [["name", "^", os.path.relpath(data["path"], "/mnt") + "/"], ["type", "=", "FILESYSTEM"]]): verrors.add( f"{name}.snapshot", "This option is only available for datasets that have no further " "nesting") @private async def _validate_folder(self, verrors, name, data): if data["direction"] == "PULL": folder = data["attributes"]["folder"].rstrip("/") if folder: folder_parent = os.path.normpath(os.path.join(folder, "..")) if folder_parent == ".": folder_parent = "" folder_basename = os.path.basename(folder) ls = await self.list_directory( dict( credentials=data["credentials"], encryption=data["encryption"], filename_encryption=data["filename_encryption"], encryption_password=data["encryption_password"], encryption_salt=data["encryption_salt"], attributes=dict(data["attributes"], folder=folder_parent), args=data["args"], )) for item in ls: if item["Name"] == folder_basename: if not item["IsDir"]: verrors.add(f"{name}.attributes.folder", "This is not a directory") break else: verrors.add(f"{name}.attributes.folder", "Directory does not exist") if data["direction"] == "PUSH": credentials = await self._get_credentials(data["credentials"]) provider = REMOTES[credentials["provider"]] if provider.readonly: verrors.add(f"{name}.direction", "This remote is read-only") @accepts( Dict( "cloud_sync_create", Str("description", default=""), Str("direction", enum=["PUSH", "PULL"], required=True), Str("transfer_mode", enum=["SYNC", "COPY", "MOVE"], required=True), Str("path", required=True), Int("credentials", required=True), Bool("encryption", default=False), Bool("filename_encryption", default=False), Str("encryption_password", default=""), Str("encryption_salt", default=""), Cron("schedule", defaults={"minute": "00"}, required=True), Bool("follow_symlinks", default=False), Int("transfers", null=True, default=None, validators=[Range(min=1)]), List("bwlimit", default=[], items=[ Dict( "cloud_sync_bwlimit", Str("time", validators=[Time()]), Int("bandwidth", validators=[Range(min=1)], null=True)) ]), List("exclude", default=[], items=[Str("path", empty=False)]), Dict("attributes", additional_attrs=True, required=True), Bool("snapshot", default=False), Str("pre_script", default="", max_length=None), Str("post_script", default="", max_length=None), Str("args", default="", max_length=None), Bool("enabled", default=True), register=True, )) async def do_create(self, cloud_sync): """ Creates a new cloud_sync entry. .. examples(websocket):: Create a new cloud_sync using amazon s3 attributes, which is supposed to run every hour. :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "cloudsync.create", "params": [{ "description": "s3 sync", "path": "/mnt/tank", "credentials": 1, "minute": "00", "hour": "*", "daymonth": "*", "month": "*", "attributes": { "bucket": "mybucket", "folder": "" }, "enabled": true }] } """ verrors = ValidationErrors() await self._validate(verrors, "cloud_sync", cloud_sync) if verrors: raise verrors await self._validate_folder(verrors, "cloud_sync", cloud_sync) if verrors: raise verrors cloud_sync = await self._compress(cloud_sync) cloud_sync["id"] = await self.middleware.call("datastore.insert", "tasks.cloudsync", cloud_sync) await self.middleware.call("service.restart", "cron") cloud_sync = await self.extend(cloud_sync) return cloud_sync @accepts(Int("id"), Patch("cloud_sync_create", "cloud_sync_update", ("attr", { "update": True }))) async def do_update(self, id, data): """ Updates the cloud_sync entry `id` with `data`. """ cloud_sync = await self.get_instance(id) # credentials is a foreign key for now if cloud_sync["credentials"]: cloud_sync["credentials"] = cloud_sync["credentials"]["id"] cloud_sync.update(data) verrors = ValidationErrors() await self._validate(verrors, "cloud_sync_update", cloud_sync) if verrors: raise verrors await self._validate_folder(verrors, "cloud_sync_update", cloud_sync) if verrors: raise verrors cloud_sync = await self._compress(cloud_sync) await self.middleware.call("datastore.update", "tasks.cloudsync", id, cloud_sync) await self.middleware.call("service.restart", "cron") return await self.get_instance(id) @accepts(Int("id")) async def do_delete(self, id): """ Deletes cloud_sync entry `id`. """ await self.middleware.call("cloudsync.abort", id) await self.middleware.call("datastore.delete", "tasks.cloudsync", id) await self.middleware.call("alert.oneshot_delete", "CloudSyncTaskFailed", id) await self.middleware.call("service.restart", "cron") @accepts(Int("credentials_id")) async def list_buckets(self, credentials_id): credentials = await self._get_credentials(credentials_id) if not credentials: raise CallError("Invalid credentials") provider = REMOTES[credentials["provider"]] if not provider.buckets: raise CallError("This provider does not use buckets") return await self.ls({"credentials": credentials}, "") @accepts( Dict( "cloud_sync_ls", Int("credentials", required=True), Bool("encryption", default=False), Bool("filename_encryption", default=False), Str("encryption_password", default=""), Str("encryption_salt", default=""), Dict("attributes", required=True, additional_attrs=True), Str("args", default=""), )) async def list_directory(self, cloud_sync): """ List contents of a remote bucket / directory. If remote supports buckets, path is constructed by two keys "bucket"/"folder" in `attributes`. If remote does not support buckets, path is constructed using "folder" key only in `attributes`. "folder" is directory name and "bucket" is bucket name for remote. Path examples: S3 Service `bucketname/directory/name` Dropbox Service `directory/name` `credentials` is a valid id of a Cloud Sync Credential which will be used to connect to the provider. """ verrors = ValidationErrors() await self._basic_validate(verrors, "cloud_sync", dict(cloud_sync)) if verrors: raise verrors credentials = await self._get_credentials(cloud_sync["credentials"]) path = get_remote_path(REMOTES[credentials["provider"]], cloud_sync["attributes"]) return await self.ls(dict(cloud_sync, credentials=credentials), path) @private async def ls(self, config, path): await self.middleware.call("network.general.will_perform_activity", "cloud_sync") decrypt_filenames = config.get("encryption") and config.get( "filename_encryption") async with RcloneConfig(config) as config: proc = await run([ "rclone", "--config", config.config_path, "lsjson", "remote:" + path ], check=False, encoding="utf8", errors="ignore") if proc.returncode == 0: result = json.loads(proc.stdout) if decrypt_filenames: if result: decrypted_names = {} proc = await run(([ "rclone", "--config", config.config_path, "cryptdecode", "encrypted:" ] + [item["Name"] for item in result]), check=False, encoding="utf8", errors="ignore") for line in proc.stdout.splitlines(): try: encrypted, decrypted = line.rstrip( "\r\n").split(" \t ", 1) except ValueError: continue if decrypted != "Failed to decrypt": decrypted_names[encrypted] = decrypted for item in result: if item["Name"] in decrypted_names: item["Decrypted"] = decrypted_names[ item["Name"]] return result else: raise CallError( proc.stderr, extra={"excerpt": lsjson_error_excerpt(proc.stderr)}) @item_method @accepts(Int("id"), Dict( "cloud_sync_sync_options", Bool("dry_run", default=False), register=True, )) @job(lock=lambda args: "cloud_sync:{}".format(args[-1]), lock_queue_size=1, logs=True) async def sync(self, job, id, options): """ Run the cloud_sync job `id`, syncing the local data to remote. """ cloud_sync = await self.get_instance(id) if cloud_sync['locked']: await self.middleware.call('cloudsync.generate_locked_alert', id) return await self._sync(cloud_sync, options, job) @accepts( Patch("cloud_sync_create", "cloud_sync_sync_onetime"), Patch("cloud_sync_sync_options", "cloud_sync_sync_onetime_options"), ) @job(logs=True) async def sync_onetime(self, job, cloud_sync, options): """ Run cloud sync task without creating it. """ verrors = ValidationErrors() await self._validate(verrors, "cloud_sync_sync_onetime", cloud_sync) if verrors: raise verrors await self._validate_folder(verrors, "cloud_sync_sync_onetime", cloud_sync) if verrors: raise verrors cloud_sync["credentials"] = await self._get_credentials( cloud_sync["credentials"]) await self._sync(cloud_sync, options, job) async def _sync(self, cloud_sync, options, job): credentials = cloud_sync["credentials"] local_path = cloud_sync["path"] local_direction = FsLockDirection.READ if cloud_sync[ "direction"] == "PUSH" else FsLockDirection.WRITE remote_path = get_remote_path(REMOTES[credentials["provider"]], cloud_sync["attributes"]) remote_direction = FsLockDirection.READ if cloud_sync[ "direction"] == "PULL" else FsLockDirection.WRITE directions = { FsLockDirection.READ: "reading", FsLockDirection.WRITE: "writing", } job.set_progress( 0, f"Locking local path {local_path!r} for {directions[local_direction]}" ) async with self.local_fs_lock_manager.lock(local_path, local_direction): job.set_progress( 0, f"Locking remote path {remote_path!r} for {directions[remote_direction]}" ) async with self.remote_fs_lock_manager.lock( f"{credentials['id']}/{remote_path}", remote_direction): job.set_progress(0, "Starting") try: await rclone(self.middleware, job, cloud_sync, options["dry_run"]) if "id" in cloud_sync: await self.middleware.call("alert.oneshot_delete", "CloudSyncTaskFailed", cloud_sync["id"]) except Exception: if "id" in cloud_sync: await self.middleware.call( "alert.oneshot_create", "CloudSyncTaskFailed", { "id": cloud_sync["id"], "name": cloud_sync["description"], }) raise @item_method @accepts(Int("id")) async def abort(self, id): """ Aborts cloud sync task. """ cloud_sync = await self._get_instance(id) if cloud_sync["job"] is None: return False if cloud_sync["job"]["state"] not in ["WAITING", "RUNNING"]: return False await self.middleware.call("core.job_abort", cloud_sync["job"]["id"]) return True @accepts() async def providers(self): """ Returns a list of dictionaries of supported providers for Cloud Sync Tasks. `credentials_schema` is JSON schema for credentials attributes. `task_schema` is JSON schema for task attributes. `buckets` is a boolean value which is set to "true" if provider supports buckets. Example of a single provider: [ { "name": "AMAZON_CLOUD_DRIVE", "title": "Amazon Cloud Drive", "credentials_schema": [ { "property": "client_id", "schema": { "title": "Amazon Application Client ID", "_required_": true, "type": "string" } }, { "property": "client_secret", "schema": { "title": "Application Key", "_required_": true, "type": "string" } } ], "credentials_oauth": null, "buckets": false, "bucket_title": "Bucket", "task_schema": [] } ] """ return sorted([{ "name": provider.name, "title": provider.title, "credentials_schema": [{ "property": field.name, "schema": field.to_json_schema() } for field in provider.credentials_schema], "credentials_oauth": f"{OAUTH_URL}/{provider.name.lower()}" if provider.credentials_oauth else None, "buckets": provider.buckets, "bucket_title": provider.bucket_title, "task_schema": [{ "property": field.name, "schema": field.to_json_schema() } for field in provider.task_schema + self.common_task_schema(provider)], } for provider in REMOTES.values()], key=lambda provider: provider["title"].lower()) def common_task_schema(self, provider): schema = [] if provider.fast_list: schema.append( Bool("fast_list", default=False, title="Use --fast-list", description=textwrap.dedent("""\ Use fewer transactions in exchange for more RAM. This may also speed up or slow down your transfer. See [rclone documentation](https://rclone.org/docs/#fast-list) for more details. """).rstrip())) return schema
class PeriodicSnapshotTaskService(CRUDService): class Config: datastore = 'storage.task' datastore_prefix = 'task_' datastore_extend = 'pool.snapshottask.periodic_snapshot_extend' namespace = 'pool.snapshottask' @private def periodic_snapshot_extend(self, data): data['begin'] = str(data['begin']) data['end'] = str(data['end']) data['ret_unit'] = data['ret_unit'].upper() data['dow'] = [ int(day) for day in data.pop('byweekday').split(',') if day ] data.pop('repeat_unit', None) return data @private async def common_validation(self, data, schema_name): verrors = ValidationErrors() if not data['dow']: verrors.add(f'{schema_name}.dow', 'At least one day must be chosen') data['ret_unit'] = data['ret_unit'].lower() data['begin'] = time( *[int(value) for value in data['begin'].split(':')]) data['end'] = time(*[int(value) for value in data['end'].split(':')]) data['byweekday'] = ','.join([str(day) for day in data.pop('dow')]) return data, verrors @accepts( Dict('periodic_snapshot_create', Bool('enabled', default=True), Bool('recursive', default=False), Int('interval', enum=[ 5, 10, 15, 30, 60, 120, 180, 240, 360, 720, 1440, 10080, 20160, 40320 ], required=True), Int('ret_count', required=True), List('dow', items=[Int('day', validators=[Range(min=1, max=7)])], required=True), Str('begin', validators=[Time()], required=True), Str('end', validators=[Time()], required=True), Str('filesystem', required=True), Str('ret_unit', enum=['HOUR', 'DAY', 'WEEK', 'MONTH', 'YEAR'], required=True), register=True)) async def do_create(self, data): data, verrors = await self.common_validation( data, 'periodic_snapshot_create') if data['filesystem'] not in ( await self.middleware.call('pool.filesystem_choices')): verrors.add('periodic_snapshot_create.filesystem', 'Invalid ZFS filesystem') if verrors: raise verrors data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self.middleware.call('service.restart', 'cron') return await self._get_instance(data['id']) @accepts(Int('id', required=True), Patch('periodic_snapshot_create', 'periodic_snapshot_update', ('attr', { 'update': True }))) async def do_update(self, id, data): old = await self._get_instance(id) new = old.copy() new.update(data) new, verrors = await self.common_validation( new, 'periodic_snapshot_update') if old['filesystem'] != new['filesystem']: if new['filesystem'] not in ( await self.middleware.call('pool.filesystem_choices')): verrors.add('periodic_snapshot_update.filesystem', 'Invalid ZFS filesystem') if verrors: raise verrors await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) await self.middleware.call('service.restart', 'cron') return await self._get_instance(id) @accepts(Int('id')) async def do_delete(self, id): response = await self.middleware.call('datastore.delete', self._config.datastore, id) await self.middleware.call('service.restart', 'cron') return response
class PoolResilverService(ConfigService): class Config: namespace = 'pool.resilver' datastore = 'storage.resilver' datastore_extend = 'pool.resilver.resilver_extend' async def resilver_extend(self, data): data['begin'] = data['begin'].strftime('%H:%M') data['end'] = data['end'].strftime('%H:%M') data['weekday'] = [int(v) for v in data['weekday'].split(',')] return data async def validate_fields_and_update(self, data, schema): verrors = ValidationErrors() begin = data.get('begin') if begin: data['begin'] = time(int(begin.split(':')[0]), int(begin.split(':')[1])) end = data.get('end') if end: data['end'] = time(int(end.split(':')[0]), int(end.split(':')[1])) weekdays = data.get('weekday') if weekdays: if len([day for day in weekdays if day not in range(1, 8)]) > 0: verrors.add( f'{schema}.weekday', 'The week days should be in range of 1-7 inclusive') else: data['weekday'] = ','.join([str(day) for day in weekdays]) return verrors, data @accepts( Dict('pool_resilver', Str('begin', validators=[Time()]), Str('end', validators=[Time()]), Bool('enabled'), List('weekday', items=[Int('weekday')]))) async def do_update(self, data): config = await self.config() original_config = config.copy() config.update(data) verrors, new_config = await self.validate_fields_and_update( config, 'pool_resilver_update') if verrors: raise verrors # before checking if any changes have been made, original_config needs to be mapped to new_config original_config['weekday'] = ','.join( [str(day) for day in original_config['weekday']]) original_config['begin'] = time( *(int(value) for value in original_config['begin'].split(':'))) original_config['end'] = time( *(int(value) for value in original_config['end'].split(':'))) if len(set(original_config.items()) ^ set(new_config.items())) > 0: # data has changed await self.middleware.call('datastore.update', self._config.datastore, new_config['id'], new_config) await self.middleware.call('service.restart', 'cron', {'onetime': False}) await self.middleware.call('pool.configure_resilver_priority') return await self.config()
class CloudSyncService(CRUDService): class Config: datastore = "tasks.cloudsync" datastore_extend = "cloudsync._extend" @filterable async def query(self, filters=None, options=None): tasks_or_task = await super().query(filters, options) jobs = {} for j in await self.middleware.call( "core.get_jobs", [("method", "=", "cloudsync.sync")], {"order_by": ["id"]}): try: task_id = int(j["arguments"][0]) except (IndexError, ValueError): continue if task_id in jobs and jobs[task_id]["state"] == "RUNNING": continue jobs[task_id] = j if isinstance(tasks_or_task, list): for task in tasks_or_task: task["job"] = jobs.get(task["id"]) else: tasks_or_task["job"] = jobs.get(tasks_or_task["id"]) return tasks_or_task @private async def _extend(self, cloud_sync): cloud_sync["credentials"] = cloud_sync.pop("credential") cloud_sync["encryption_password"] = await self.middleware.call( "pwenc.decrypt", cloud_sync["encryption_password"]) cloud_sync["encryption_salt"] = await self.middleware.call( "pwenc.decrypt", cloud_sync["encryption_salt"]) Cron.convert_db_format_to_schedule(cloud_sync) return cloud_sync @private async def _compress(self, cloud_sync): cloud_sync["credential"] = cloud_sync.pop("credentials") cloud_sync["encryption_password"] = await self.middleware.call( "pwenc.encrypt", cloud_sync["encryption_password"]) cloud_sync["encryption_salt"] = await self.middleware.call( "pwenc.encrypt", cloud_sync["encryption_salt"]) Cron.convert_schedule_to_db_format(cloud_sync) cloud_sync.pop('job', None) return cloud_sync @private async def _get_credentials(self, credentials_id): try: return await self.middleware.call("datastore.query", "system.cloudcredentials", [("id", "=", credentials_id)], {"get": True}) except IndexError: return None @private async def _basic_validate(self, verrors, name, data): if data["encryption"]: if not data["encryption_password"]: verrors.add( f"{name}.encryption_password", "This field is required when encryption is enabled") credentials = await self._get_credentials(data["credentials"]) if not credentials: verrors.add(f"{name}.credentials", "Invalid credentials") try: shlex.split(data["args"]) except ValueError as e: verrors.add(f"{name}.args", f"Parse error: {e.args[0]}") if verrors: raise verrors provider = REMOTES[credentials["provider"]] schema = [] if provider.buckets: schema.append(Str("bucket", required=True, empty=False)) schema.append(Str("folder", required=True)) schema.extend(provider.task_schema) schema.extend(self.common_task_schema(provider)) attributes_verrors = validate_attributes(schema, data, additional_attrs=True) if not attributes_verrors: await provider.pre_save_task(data, credentials, verrors) verrors.add_child(f"{name}.attributes", attributes_verrors) @private async def _validate(self, verrors, name, data): await self._basic_validate(verrors, name, data) for i, (limit1, limit2) in enumerate(zip(data["bwlimit"], data["bwlimit"][1:])): if limit1["time"] >= limit2["time"]: verrors.add( f"{name}.bwlimit.{i + 1}.time", f"Invalid time order: {limit1['time']}, {limit2['time']}") if data["snapshot"]: if data["direction"] != "PUSH": verrors.add(f"{name}.snapshot", "This option can only be enabled for PUSH tasks") @private async def _validate_folder(self, verrors, name, data): if data["direction"] == "PULL": folder = data["attributes"]["folder"].rstrip("/") if folder: folder_parent = os.path.normpath(os.path.join(folder, "..")) if folder_parent == ".": folder_parent = "" folder_basename = os.path.basename(folder) ls = await self.list_directory( dict( credentials=data["credentials"], encryption=data["encryption"], filename_encryption=data["filename_encryption"], encryption_password=data["encryption_password"], encryption_salt=data["encryption_salt"], attributes=dict(data["attributes"], folder=folder_parent), args=data["args"], )) for item in ls: if item["Name"] == folder_basename: if not item["IsDir"]: verrors.add(f"{name}.attributes.folder", "This is not a directory") break else: verrors.add(f"{name}.attributes.folder", "Directory does not exist") if data["direction"] == "PUSH": credentials = await self._get_credentials(data["credentials"]) provider = REMOTES[credentials["provider"]] if provider.readonly: verrors.add(f"{name}.direction", "This remote is read-only") @accepts( Dict( "cloud_sync_create", Str("description", default=""), Str("direction", enum=["PUSH", "PULL"], required=True), Str("transfer_mode", enum=["SYNC", "COPY", "MOVE"], required=True), Str("path", required=True), Int("credentials", required=True), Bool("encryption", default=False), Bool("filename_encryption", default=False), Str("encryption_password", default=""), Str("encryption_salt", default=""), Cron("schedule", required=True), Bool("follow_symlinks", default=False), Int("transfers", null=True, default=None, validators=[Range(min=1)]), List("bwlimit", default=[], items=[ Dict( "cloud_sync_bwlimit", Str("time", validators=[Time()]), Int("bandwidth", validators=[Range(min=1)], null=True)) ]), List("exclude", default=[], items=[Str("path", empty=False)]), Dict("attributes", additional_attrs=True, required=True), Bool("snapshot", default=False), Str("pre_script", default=""), Str("post_script", default=""), Str("args", default=""), Bool("enabled", default=True), register=True, )) async def do_create(self, cloud_sync): """ Creates a new cloud_sync entry. .. examples(websocket):: Create a new cloud_sync using amazon s3 attributes, which is supposed to run every hour. :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "cloudsync.create", "params": [{ "description": "s3 sync", "path": "/mnt/tank", "credentials": 1, "minute": "00", "hour": "*", "daymonth": "*", "month": "*", "attributes": { "bucket": "mybucket", "folder": "" }, "enabled": true }] } """ verrors = ValidationErrors() await self._validate(verrors, "cloud_sync", cloud_sync) if verrors: raise verrors await self._validate_folder(verrors, "cloud_sync", cloud_sync) if verrors: raise verrors cloud_sync = await self._compress(cloud_sync) cloud_sync["id"] = await self.middleware.call("datastore.insert", "tasks.cloudsync", cloud_sync) await self.middleware.call("service.restart", "cron") cloud_sync = await self._extend(cloud_sync) return cloud_sync @accepts(Int("id"), Patch("cloud_sync_create", "cloud_sync_update", ("attr", { "update": True }))) async def do_update(self, id, data): """ Updates the cloud_sync entry `id` with `data`. """ cloud_sync = await self._get_instance(id) # credentials is a foreign key for now if cloud_sync["credentials"]: cloud_sync["credentials"] = cloud_sync["credentials"]["id"] cloud_sync.update(data) verrors = ValidationErrors() await self._validate(verrors, "cloud_sync_update", cloud_sync) if verrors: raise verrors await self._validate_folder(verrors, "cloud_sync_update", cloud_sync) if verrors: raise verrors cloud_sync = await self._compress(cloud_sync) await self.middleware.call("datastore.update", "tasks.cloudsync", id, cloud_sync) await self.middleware.call("service.restart", "cron") cloud_sync = await self._extend(cloud_sync) return cloud_sync @accepts(Int("id")) async def do_delete(self, id): """ Deletes cloud_sync entry `id`. """ await self.middleware.call("datastore.delete", "tasks.cloudsync", id) await self.middleware.call("service.restart", "cron") @accepts(Int("credentials_id")) async def list_buckets(self, credentials_id): credentials = await self._get_credentials(credentials_id) if not credentials: raise CallError("Invalid credentials") provider = REMOTES[credentials["provider"]] if not provider.buckets: raise CallError("This provider does not use buckets") return await self.ls({"credentials": credentials}, "") @accepts( Dict( "cloud_sync_ls", Int("credentials", required=True), Bool("encryption", default=False), Bool("filename_encryption", default=False), Str("encryption_password", default=""), Str("encryption_salt", default=""), Dict("attributes", required=True, additional_attrs=True), Str("args", default=""), )) async def list_directory(self, cloud_sync): verrors = ValidationErrors() await self._basic_validate(verrors, "cloud_sync", dict(cloud_sync)) if verrors: raise verrors credentials = await self._get_credentials(cloud_sync["credentials"]) if REMOTES[credentials["provider"]].buckets: path = f"{cloud_sync['attributes']['bucket']}/{cloud_sync['attributes']['folder']}" else: path = cloud_sync["attributes"]["folder"] return await self.ls(dict(cloud_sync, credentials=credentials), path) @private async def ls(self, config, path): with RcloneConfig(config) as config: proc = await run([ "rclone", "--config", config.config_path, "lsjson", "remote:" + path ], check=False, encoding="utf8") if proc.returncode == 0: return json.loads(proc.stdout) else: raise CallError(proc.stderr) @item_method @accepts(Int("id")) @job(lock=lambda args: "cloud_sync:{}".format(args[-1]), lock_queue_size=1, logs=True) async def sync(self, job, id): """ Run the cloud_sync job `id`, syncing the local data to remote. """ cloud_sync = await self._get_instance(id) return await rclone(self.middleware, job, cloud_sync) @accepts() async def providers(self): return sorted([{ "name": provider.name, "title": provider.title, "credentials_schema": [{ "property": field.name, "schema": field.to_json_schema() } for field in provider.credentials_schema], "buckets": provider.buckets, "bucket_title": provider.bucket_title, "task_schema": [{ "property": field.name, "schema": field.to_json_schema() } for field in provider.task_schema + self.common_task_schema(provider)], } for provider in REMOTES.values()], key=lambda provider: provider["title"].lower()) def common_task_schema(self, provider): schema = [] if provider.fast_list: schema.append( Bool("fast_list", default=False, title="Use --fast-list", description=textwrap.dedent("""\ Use fewer transactions in exchange for more RAM. This may also speed up or slow down your transfer. See [rclone documentation](https://rclone.org/docs/#fast-list) for more details. """).rstrip())) return schema
class ReplicationService(CRUDService): class Config: datastore = 'storage.replication' datastore_prefix = 'repl_' datastore_extend = 'replication.replication_extend' @private async def replication_extend(self, data): remote_data = data.pop('remote') data['remote'] = remote_data['id'] data['remote_dedicateduser_enabled'] = remote_data[ 'ssh_remote_dedicateduser_enabled'] data['remote_port'] = remote_data['ssh_remote_port'] data['remote_cipher'] = remote_data['ssh_cipher'].upper() data['remote_dedicateduser'] = remote_data['ssh_remote_dedicateduser'] data['remote_hostkey'] = remote_data['ssh_remote_hostkey'] data['remote_hostname'] = remote_data['ssh_remote_hostname'] if not os.path.exists(REPL_RESULTFILE): data['lastresult'] = {'msg': 'Waiting'} else: with open(REPL_RESULTFILE, 'rb') as f: file_data = f.read() try: results = pickle.loads(file_data) data['lastresult'] = results[data['id']] except Exception: data['lastresult'] = {'msg': None} progressfile = f'/tmp/.repl_progress_{data["id"]}' if os.path.exists(progressfile): with open(progressfile, 'r') as f: pid = int(f.read()) title = await self.middleware.call('notifier.get_proc_title', pid) if title: reg = re.search(r'sending (\S+) \((\d+)%', title) if reg: data[ 'status'] = f'Sending {reg.groups()[0]}s {reg.groups()[1]}s' else: data['status'] = 'Sending' if 'status' not in data: data['status'] = data['lastresult'].get('msg') data['begin'] = str(data['begin']) data['end'] = str(data['end']) data['compression'] = data['compression'].upper() return data @private async def validate_data(self, data, schema_name): verrors = ValidationErrors() remote_hostname = data.pop('remote_hostname') await resolve_hostname(self.middleware, verrors, f'{schema_name}.remote_hostname', remote_hostname) remote_dedicated_user_enabled = data.pop( 'remote_dedicateduser_enabled', False) remote_dedicated_user = data.pop('remote_dedicateduser', None) if remote_dedicated_user_enabled and not remote_dedicated_user: verrors.add( f'{schema_name}.remote_dedicateduser', 'You must select a user when remote dedicated user is enabled') if not await self.middleware.call('pool.snapshottask.query', [ ('filesystem', '=', data.get('filesystem')) ]): verrors.add(f'{schema_name}.filesystem', 'Invalid Filesystem') remote_mode = data.pop('remote_mode', 'MANUAL') remote_port = data.pop('remote_port') repl_remote_dict = { 'ssh_remote_hostname': remote_hostname, 'ssh_remote_dedicateduser_enabled': remote_dedicated_user_enabled, 'ssh_remote_dedicateduser': remote_dedicated_user, 'ssh_cipher': data.pop('remote_cipher', 'STANDARD').lower() } if remote_mode == 'SEMIAUTOMATIC': token = data.pop('remote_token', None) if not token: verrors.add(f'{schema_name}.remote_token', 'This field is required') else: remote_host_key = data.pop('remote_hostkey', None) if not remote_host_key: verrors.add(f'{schema_name}.remote_hostkey', 'This field is required') else: repl_remote_dict['ssh_remote_port'] = remote_port repl_remote_dict['ssh_remote_hostkey'] = remote_host_key if verrors: raise verrors data['begin'] = time(*[int(v) for v in data.pop('begin').split(':')]) data['end'] = time(*[int(v) for v in data.pop('end').split(':')]) data['compression'] = data['compression'].lower() data.pop('remote_hostkey', None) data.pop('remote_token', None) return verrors, data, repl_remote_dict @accepts( Dict('replication_create', Bool('enabled', default=True), Bool('followdelete', default=False), Bool('remote_dedicateduser_enabled', default=False), Bool('remote_https'), Bool('userepl', default=False), Int('limit', default=0, validators=[Range(min=0)]), Int('remote_port', default=22, required=True), Str('begin', validators=[Time()]), Str('compression', enum=['OFF', 'LZ4', 'PIGZ', 'PLZIP']), Str('end', validators=[Time()]), Str('filesystem', required=True), Str('remote_cipher', enum=['STANDARD', 'FAST', 'DISABLED']), Str('remote_dedicateduser'), Str('remote_hostkey'), Str('remote_hostname', required=True), Str('remote_mode', enum=['SEMIAUTOMATIC', 'MANUAL'], required=True), Str('remote_token'), Str('zfs', required=True), register=True)) async def do_create(self, data): remote_hostname = data.get('remote_hostname') remote_dedicated_user = data.get('remote_dedicateduser') remote_port = data.get('remote_port') remote_https = data.pop('remote_https', False) remote_token = data.get('remote_token') remote_mode = data.get('remote_mode') verrors, data, repl_remote_dict = await self.validate_data( data, 'replication_create') if remote_mode == 'SEMIAUTOMATIC': remote_uri = f'ws{"s" if remote_https else ""}://{remote_hostname}:{remote_port}/websocket' try: with Client(remote_uri) as c: if not c.call('auth.token', remote_token): verrors.add('replication_create.remote_token', 'Please provide a valid token') else: try: with open(REPLICATION_KEY, 'r') as f: publickey = f.read() call_data = c.call( 'replication.pair', { 'hostname': remote_hostname, 'public-key': publickey, 'user': remote_dedicated_user, }) except Exception as e: raise CallError('Failed to set up replication ' + str(e)) else: repl_remote_dict['ssh_remote_port'] = call_data[ 'ssh_port'] repl_remote_dict['ssh_remote_hostkey'] = call_data[ 'ssh_hostkey'] except Exception as e: verrors.add( 'replication_create.remote_token', f'Failed to connect to remote host {remote_uri} with following exception {e}' ) if verrors: raise verrors remote_pk = await self.middleware.call('datastore.insert', 'storage.replremote', repl_remote_dict) await self.middleware.call('service.reload', 'ssh', {'onetime': False}) data['remote'] = remote_pk pk = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) return await self._get_instance(pk) @accepts(Int('id', required=True), Patch( 'replication_create', 'replication_update', ('attr', { 'update': True }), ('rm', { 'name': 'remote_mode' }), ('rm', { 'name': 'remote_https' }), ('rm', { 'name': 'remote_token' }), )) async def do_update(self, id, data): old = await self._get_instance(id) new = old.copy() new.update(data) verrors, new, repl_remote_dict = await self.validate_data( new, 'replication_update') new.pop('status') new.pop('lastresult') await self.middleware.call('datastore.update', 'storage.replremote', new['remote'], repl_remote_dict) await self.middleware.call('service.reload', 'ssh', {'onetime': False}) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) return await self._get_instance(id) @accepts(Int('id')) async def do_delete(self, id): response = await self.middleware.call('datastore.delete', self._config.datastore, id) return response @accepts() def public_key(self): """ Get the public SSH replication key. """ if (os.path.exists(REPLICATION_KEY) and os.path.isfile(REPLICATION_KEY)): with open(REPLICATION_KEY, 'r') as f: key = f.read() else: key = None return key @accepts( Str('host', required=True), Int('port', required=True), ) async def ssh_keyscan(self, host, port): """ Scan the SSH key on `host`:`port`. """ proc = await Popen([ "/usr/bin/ssh-keyscan", "-p", str(port), "-T", "2", str(host), ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) key, errmsg = await proc.communicate() if proc.returncode != 0 or not key: if not errmsg: errmsg = 'ssh key scan failed for unknown reason' else: errmsg = errmsg.decode() raise CallError(errmsg) return key.decode() @private @accepts( Dict( 'replication-pair-data', Str('hostname', required=True), Str('public-key', required=True), Str('user'), )) async def pair(self, data): """ Receives public key, storing it to accept SSH connection and return pertinent SSH data of this machine. """ service = await self.middleware.call('datastore.query', 'services.services', [('srv_service', '=', 'ssh')], {'get': True}) ssh = await self.middleware.call('datastore.query', 'services.ssh', None, {'get': True}) try: user = await self.middleware.call( 'datastore.query', 'account.bsdusers', [('bsdusr_username', '=', data.get('user') or 'root')], {'get': True}) except IndexError: raise ValueError('User "{}" does not exist'.format( data.get('user'))) if user['bsdusr_home'].startswith('/nonexistent'): raise CallError(f'User home directory does not exist', errno.ENOENT) # Make sure SSH is enabled if not service['srv_enable']: await self.middleware.call('datastore.update', 'services.services', service['id'], {'srv_enable': True}) await self.middleware.call('notifier.start', 'ssh') # This might be the first time of the service being enabled # which will then result in new host keys we need to grab ssh = await self.middleware.call('datastore.query', 'services.ssh', None, {'get': True}) if not os.path.exists(user['bsdusr_home']): raise ValueError('Homedir {} does not exist'.format( user['bsdusr_home'])) # If .ssh dir does not exist, create it dotsshdir = os.path.join(user['bsdusr_home'], '.ssh') if not os.path.exists(dotsshdir): os.mkdir(dotsshdir) os.chown(dotsshdir, user['bsdusr_uid'], user['bsdusr_group']['bsdgrp_gid']) # Write public key in user authorized_keys for SSH authorized_keys_file = f'{dotsshdir}/authorized_keys' with open(authorized_keys_file, 'a+') as f: f.seek(0) if data['public-key'] not in f.read(): f.write('\n' + data['public-key']) ssh_hostkey = '{0} {1}\n{0} {2}\n{0} {3}\n'.format( data['hostname'], base64.b64decode(ssh['ssh_host_rsa_key_pub'].encode()).decode(), base64.b64decode(ssh['ssh_host_ecdsa_key_pub'].encode()).decode(), base64.b64decode( ssh['ssh_host_ed25519_key_pub'].encode()).decode(), ) return { 'ssh_port': ssh['ssh_tcpport'], 'ssh_hostkey': ssh_hostkey, }