class CloudSyncService(TaskPathService):
local_fs_lock_manager = FsLockManager()
remote_fs_lock_manager = FsLockManager()
share_task_type = 'CloudSync'
class Config:
datastore = "tasks.cloudsync"
datastore_extend = "cloudsync.extend"
@filterable
async def query(self, filters=None, options=None):
"""
Query all Cloud Sync Tasks with `query-filters` and `query-options`.
"""
tasks_or_task = await super().query(filters, options)
jobs = {}
for j in await self.middleware.call(
"core.get_jobs",
[('OR', [("method", "=", "cloudsync.sync"),
("method", "=", "cloudsync.restore")])],
{"order_by": ["id"]}):
try:
task_id = int(j["arguments"][0])
except (IndexError, ValueError):
continue
if task_id in jobs and jobs[task_id]["state"] == "RUNNING":
continue
jobs[task_id] = j
if isinstance(tasks_or_task, list):
for task in tasks_or_task:
task["job"] = jobs.get(task["id"])
else:
tasks_or_task["job"] = jobs.get(tasks_or_task["id"])
return tasks_or_task
@private
async def extend(self, cloud_sync):
cloud_sync["credentials"] = cloud_sync.pop("credential")
Cron.convert_db_format_to_schedule(cloud_sync)
return cloud_sync
@private
async def _compress(self, cloud_sync):
cloud_sync["credential"] = cloud_sync.pop("credentials")
Cron.convert_schedule_to_db_format(cloud_sync)
cloud_sync.pop('job', None)
cloud_sync.pop(self.locked_field, None)
return cloud_sync
@private
async def _get_credentials(self, credentials_id):
try:
return await self.middleware.call("datastore.query",
"system.cloudcredentials",
[("id", "=", credentials_id)],
{"get": True})
except IndexError:
return None
@private
async def _basic_validate(self, verrors, name, data):
if data["encryption"]:
if not data["encryption_password"]:
verrors.add(
f"{name}.encryption_password",
"This field is required when encryption is enabled")
credentials = await self._get_credentials(data["credentials"])
if not credentials:
verrors.add(f"{name}.credentials", "Invalid credentials")
try:
shlex.split(data["args"])
except ValueError as e:
verrors.add(f"{name}.args", f"Parse error: {e.args[0]}")
if verrors:
raise verrors
provider = REMOTES[credentials["provider"]]
schema = []
if provider.buckets:
schema.append(Str("bucket", required=True, empty=False))
schema.append(Str("folder", required=True))
schema.extend(provider.task_schema)
schema.extend(self.common_task_schema(provider))
attributes_verrors = validate_attributes(schema,
data,
additional_attrs=True)
if not attributes_verrors:
await provider.pre_save_task(data, credentials, verrors)
verrors.add_child(f"{name}.attributes", attributes_verrors)
@private
async def _validate(self, verrors, name, data):
await self._basic_validate(verrors, name, data)
for i, (limit1,
limit2) in enumerate(zip(data["bwlimit"],
data["bwlimit"][1:])):
if limit1["time"] >= limit2["time"]:
verrors.add(
f"{name}.bwlimit.{i + 1}.time",
f"Invalid time order: {limit1['time']}, {limit2['time']}")
await self.validate_path_field(data, name, verrors)
if data["snapshot"]:
if data["direction"] != "PUSH":
verrors.add(f"{name}.snapshot",
"This option can only be enabled for PUSH tasks")
if data["transfer_mode"] == "MOVE":
verrors.add(
f"{name}.snapshot",
"This option can not be used for MOVE transfer mode")
if await self.middleware.call(
"pool.dataset.query",
[["name", "^",
os.path.relpath(data["path"], "/mnt") + "/"],
["type", "=", "FILESYSTEM"]]):
verrors.add(
f"{name}.snapshot",
"This option is only available for datasets that have no further "
"nesting")
@private
async def _validate_folder(self, verrors, name, data):
if data["direction"] == "PULL":
folder = data["attributes"]["folder"].rstrip("/")
if folder:
folder_parent = os.path.normpath(os.path.join(folder, ".."))
if folder_parent == ".":
folder_parent = ""
folder_basename = os.path.basename(folder)
ls = await self.list_directory(
dict(
credentials=data["credentials"],
encryption=data["encryption"],
filename_encryption=data["filename_encryption"],
encryption_password=data["encryption_password"],
encryption_salt=data["encryption_salt"],
attributes=dict(data["attributes"],
folder=folder_parent),
args=data["args"],
))
for item in ls:
if item["Name"] == folder_basename:
if not item["IsDir"]:
verrors.add(f"{name}.attributes.folder",
"This is not a directory")
break
else:
verrors.add(f"{name}.attributes.folder",
"Directory does not exist")
if data["direction"] == "PUSH":
credentials = await self._get_credentials(data["credentials"])
provider = REMOTES[credentials["provider"]]
if provider.readonly:
verrors.add(f"{name}.direction", "This remote is read-only")
@accepts(
Dict(
"cloud_sync_create",
Str("description", default=""),
Str("direction", enum=["PUSH", "PULL"], required=True),
Str("transfer_mode", enum=["SYNC", "COPY", "MOVE"], required=True),
Str("path", required=True),
Int("credentials", required=True),
Bool("encryption", default=False),
Bool("filename_encryption", default=False),
Str("encryption_password", default=""),
Str("encryption_salt", default=""),
Cron("schedule", defaults={"minute": "00"}, required=True),
Bool("follow_symlinks", default=False),
Int("transfers",
null=True,
default=None,
validators=[Range(min=1)]),
List("bwlimit",
default=[],
items=[
Dict(
"cloud_sync_bwlimit", Str("time",
validators=[Time()]),
Int("bandwidth", validators=[Range(min=1)],
null=True))
]),
List("exclude", default=[], items=[Str("path", empty=False)]),
Dict("attributes", additional_attrs=True, required=True),
Bool("snapshot", default=False),
Str("pre_script", default="", max_length=None),
Str("post_script", default="", max_length=None),
Str("args", default="", max_length=None),
Bool("enabled", default=True),
register=True,
))
async def do_create(self, cloud_sync):
"""
Creates a new cloud_sync entry.
.. examples(websocket)::
Create a new cloud_sync using amazon s3 attributes, which is supposed to run every hour.
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "cloudsync.create",
"params": [{
"description": "s3 sync",
"path": "/mnt/tank",
"credentials": 1,
"minute": "00",
"hour": "*",
"daymonth": "*",
"month": "*",
"attributes": {
"bucket": "mybucket",
"folder": ""
},
"enabled": true
}]
}
"""
verrors = ValidationErrors()
await self._validate(verrors, "cloud_sync", cloud_sync)
if verrors:
raise verrors
await self._validate_folder(verrors, "cloud_sync", cloud_sync)
if verrors:
raise verrors
cloud_sync = await self._compress(cloud_sync)
cloud_sync["id"] = await self.middleware.call("datastore.insert",
"tasks.cloudsync",
cloud_sync)
await self.middleware.call("service.restart", "cron")
cloud_sync = await self.extend(cloud_sync)
return cloud_sync
@accepts(Int("id"),
Patch("cloud_sync_create", "cloud_sync_update", ("attr", {
"update": True
})))
async def do_update(self, id, data):
"""
Updates the cloud_sync entry `id` with `data`.
"""
cloud_sync = await self.get_instance(id)
# credentials is a foreign key for now
if cloud_sync["credentials"]:
cloud_sync["credentials"] = cloud_sync["credentials"]["id"]
cloud_sync.update(data)
verrors = ValidationErrors()
await self._validate(verrors, "cloud_sync_update", cloud_sync)
if verrors:
raise verrors
await self._validate_folder(verrors, "cloud_sync_update", cloud_sync)
if verrors:
raise verrors
cloud_sync = await self._compress(cloud_sync)
await self.middleware.call("datastore.update", "tasks.cloudsync", id,
cloud_sync)
await self.middleware.call("service.restart", "cron")
return await self.get_instance(id)
@accepts(Int("id"))
async def do_delete(self, id):
"""
Deletes cloud_sync entry `id`.
"""
await self.middleware.call("cloudsync.abort", id)
await self.middleware.call("datastore.delete", "tasks.cloudsync", id)
await self.middleware.call("alert.oneshot_delete",
"CloudSyncTaskFailed", id)
await self.middleware.call("service.restart", "cron")
@accepts(Int("credentials_id"))
async def list_buckets(self, credentials_id):
credentials = await self._get_credentials(credentials_id)
if not credentials:
raise CallError("Invalid credentials")
provider = REMOTES[credentials["provider"]]
if not provider.buckets:
raise CallError("This provider does not use buckets")
return await self.ls({"credentials": credentials}, "")
@accepts(
Dict(
"cloud_sync_ls",
Int("credentials", required=True),
Bool("encryption", default=False),
Bool("filename_encryption", default=False),
Str("encryption_password", default=""),
Str("encryption_salt", default=""),
Dict("attributes", required=True, additional_attrs=True),
Str("args", default=""),
))
async def list_directory(self, cloud_sync):
"""
List contents of a remote bucket / directory.
If remote supports buckets, path is constructed by two keys "bucket"/"folder" in `attributes`.
If remote does not support buckets, path is constructed using "folder" key only in `attributes`.
"folder" is directory name and "bucket" is bucket name for remote.
Path examples:
S3 Service
`bucketname/directory/name`
Dropbox Service
`directory/name`
`credentials` is a valid id of a Cloud Sync Credential which will be used to connect to the provider.
"""
verrors = ValidationErrors()
await self._basic_validate(verrors, "cloud_sync", dict(cloud_sync))
if verrors:
raise verrors
credentials = await self._get_credentials(cloud_sync["credentials"])
path = get_remote_path(REMOTES[credentials["provider"]],
cloud_sync["attributes"])
return await self.ls(dict(cloud_sync, credentials=credentials), path)
@private
async def ls(self, config, path):
await self.middleware.call("network.general.will_perform_activity",
"cloud_sync")
decrypt_filenames = config.get("encryption") and config.get(
"filename_encryption")
async with RcloneConfig(config) as config:
proc = await run([
"rclone", "--config", config.config_path, "lsjson",
"remote:" + path
],
check=False,
encoding="utf8",
errors="ignore")
if proc.returncode == 0:
result = json.loads(proc.stdout)
if decrypt_filenames:
if result:
decrypted_names = {}
proc = await run(([
"rclone", "--config", config.config_path,
"cryptdecode", "encrypted:"
] + [item["Name"] for item in result]),
check=False,
encoding="utf8",
errors="ignore")
for line in proc.stdout.splitlines():
try:
encrypted, decrypted = line.rstrip(
"\r\n").split(" \t ", 1)
except ValueError:
continue
if decrypted != "Failed to decrypt":
decrypted_names[encrypted] = decrypted
for item in result:
if item["Name"] in decrypted_names:
item["Decrypted"] = decrypted_names[
item["Name"]]
return result
else:
raise CallError(
proc.stderr,
extra={"excerpt": lsjson_error_excerpt(proc.stderr)})
@item_method
@accepts(Int("id"),
Dict(
"cloud_sync_sync_options",
Bool("dry_run", default=False),
register=True,
))
@job(lock=lambda args: "cloud_sync:{}".format(args[-1]),
lock_queue_size=1,
logs=True)
async def sync(self, job, id, options):
"""
Run the cloud_sync job `id`, syncing the local data to remote.
"""
cloud_sync = await self.get_instance(id)
if cloud_sync['locked']:
await self.middleware.call('cloudsync.generate_locked_alert', id)
return
await self._sync(cloud_sync, options, job)
@accepts(
Patch("cloud_sync_create", "cloud_sync_sync_onetime"),
Patch("cloud_sync_sync_options", "cloud_sync_sync_onetime_options"),
)
@job(logs=True)
async def sync_onetime(self, job, cloud_sync, options):
"""
Run cloud sync task without creating it.
"""
verrors = ValidationErrors()
await self._validate(verrors, "cloud_sync_sync_onetime", cloud_sync)
if verrors:
raise verrors
await self._validate_folder(verrors, "cloud_sync_sync_onetime",
cloud_sync)
if verrors:
raise verrors
cloud_sync["credentials"] = await self._get_credentials(
cloud_sync["credentials"])
await self._sync(cloud_sync, options, job)
async def _sync(self, cloud_sync, options, job):
credentials = cloud_sync["credentials"]
local_path = cloud_sync["path"]
local_direction = FsLockDirection.READ if cloud_sync[
"direction"] == "PUSH" else FsLockDirection.WRITE
remote_path = get_remote_path(REMOTES[credentials["provider"]],
cloud_sync["attributes"])
remote_direction = FsLockDirection.READ if cloud_sync[
"direction"] == "PULL" else FsLockDirection.WRITE
directions = {
FsLockDirection.READ: "reading",
FsLockDirection.WRITE: "writing",
}
job.set_progress(
0,
f"Locking local path {local_path!r} for {directions[local_direction]}"
)
async with self.local_fs_lock_manager.lock(local_path,
local_direction):
job.set_progress(
0,
f"Locking remote path {remote_path!r} for {directions[remote_direction]}"
)
async with self.remote_fs_lock_manager.lock(
f"{credentials['id']}/{remote_path}", remote_direction):
job.set_progress(0, "Starting")
try:
await rclone(self.middleware, job, cloud_sync,
options["dry_run"])
if "id" in cloud_sync:
await self.middleware.call("alert.oneshot_delete",
"CloudSyncTaskFailed",
cloud_sync["id"])
except Exception:
if "id" in cloud_sync:
await self.middleware.call(
"alert.oneshot_create", "CloudSyncTaskFailed", {
"id": cloud_sync["id"],
"name": cloud_sync["description"],
})
raise
@item_method
@accepts(Int("id"))
async def abort(self, id):
"""
Aborts cloud sync task.
"""
cloud_sync = await self._get_instance(id)
if cloud_sync["job"] is None:
return False
if cloud_sync["job"]["state"] not in ["WAITING", "RUNNING"]:
return False
await self.middleware.call("core.job_abort", cloud_sync["job"]["id"])
return True
@accepts()
async def providers(self):
"""
Returns a list of dictionaries of supported providers for Cloud Sync Tasks.
`credentials_schema` is JSON schema for credentials attributes.
`task_schema` is JSON schema for task attributes.
`buckets` is a boolean value which is set to "true" if provider supports buckets.
Example of a single provider:
[
{
"name": "AMAZON_CLOUD_DRIVE",
"title": "Amazon Cloud Drive",
"credentials_schema": [
{
"property": "client_id",
"schema": {
"title": "Amazon Application Client ID",
"_required_": true,
"type": "string"
}
},
{
"property": "client_secret",
"schema": {
"title": "Application Key",
"_required_": true,
"type": "string"
}
}
],
"credentials_oauth": null,
"buckets": false,
"bucket_title": "Bucket",
"task_schema": []
}
]
"""
return sorted([{
"name":
provider.name,
"title":
provider.title,
"credentials_schema": [{
"property": field.name,
"schema": field.to_json_schema()
} for field in provider.credentials_schema],
"credentials_oauth":
f"{OAUTH_URL}/{provider.name.lower()}"
if provider.credentials_oauth else None,
"buckets":
provider.buckets,
"bucket_title":
provider.bucket_title,
"task_schema": [{
"property": field.name,
"schema": field.to_json_schema()
} for field in provider.task_schema +
self.common_task_schema(provider)],
} for provider in REMOTES.values()],
key=lambda provider: provider["title"].lower())
def common_task_schema(self, provider):
schema = []
if provider.fast_list:
schema.append(
Bool("fast_list",
default=False,
title="Use --fast-list",
description=textwrap.dedent("""\
Use fewer transactions in exchange for more RAM. This may also speed up or slow down your
transfer. See [rclone documentation](https://rclone.org/docs/#fast-list) for more details.
""").rstrip()))
return schema
class PeriodicSnapshotTaskService(CRUDService):
class Config:
datastore = 'storage.task'
datastore_prefix = 'task_'
datastore_extend = 'pool.snapshottask.periodic_snapshot_extend'
namespace = 'pool.snapshottask'
@private
def periodic_snapshot_extend(self, data):
data['begin'] = str(data['begin'])
data['end'] = str(data['end'])
data['ret_unit'] = data['ret_unit'].upper()
data['dow'] = [
int(day) for day in data.pop('byweekday').split(',') if day
]
data.pop('repeat_unit', None)
return data
@private
async def common_validation(self, data, schema_name):
verrors = ValidationErrors()
if not data['dow']:
verrors.add(f'{schema_name}.dow',
'At least one day must be chosen')
data['ret_unit'] = data['ret_unit'].lower()
data['begin'] = time(
*[int(value) for value in data['begin'].split(':')])
data['end'] = time(*[int(value) for value in data['end'].split(':')])
data['byweekday'] = ','.join([str(day) for day in data.pop('dow')])
return data, verrors
@accepts(
Dict('periodic_snapshot_create',
Bool('enabled', default=True),
Bool('recursive', default=False),
Int('interval',
enum=[
5, 10, 15, 30, 60, 120, 180, 240, 360, 720, 1440, 10080,
20160, 40320
],
required=True),
Int('ret_count', required=True),
List('dow',
items=[Int('day', validators=[Range(min=1, max=7)])],
required=True),
Str('begin', validators=[Time()], required=True),
Str('end', validators=[Time()], required=True),
Str('filesystem', required=True),
Str('ret_unit',
enum=['HOUR', 'DAY', 'WEEK', 'MONTH', 'YEAR'],
required=True),
register=True))
async def do_create(self, data):
data, verrors = await self.common_validation(
data, 'periodic_snapshot_create')
if data['filesystem'] not in (
await self.middleware.call('pool.filesystem_choices')):
verrors.add('periodic_snapshot_create.filesystem',
'Invalid ZFS filesystem')
if verrors:
raise verrors
data['id'] = await self.middleware.call(
'datastore.insert', self._config.datastore, data,
{'prefix': self._config.datastore_prefix})
await self.middleware.call('service.restart', 'cron')
return await self._get_instance(data['id'])
@accepts(Int('id', required=True),
Patch('periodic_snapshot_create', 'periodic_snapshot_update',
('attr', {
'update': True
})))
async def do_update(self, id, data):
old = await self._get_instance(id)
new = old.copy()
new.update(data)
new, verrors = await self.common_validation(
new, 'periodic_snapshot_update')
if old['filesystem'] != new['filesystem']:
if new['filesystem'] not in (
await self.middleware.call('pool.filesystem_choices')):
verrors.add('periodic_snapshot_update.filesystem',
'Invalid ZFS filesystem')
if verrors:
raise verrors
await self.middleware.call('datastore.update', self._config.datastore,
id, new,
{'prefix': self._config.datastore_prefix})
await self.middleware.call('service.restart', 'cron')
return await self._get_instance(id)
@accepts(Int('id'))
async def do_delete(self, id):
response = await self.middleware.call('datastore.delete',
self._config.datastore, id)
await self.middleware.call('service.restart', 'cron')
return response
class PoolResilverService(ConfigService):
class Config:
namespace = 'pool.resilver'
datastore = 'storage.resilver'
datastore_extend = 'pool.resilver.resilver_extend'
async def resilver_extend(self, data):
data['begin'] = data['begin'].strftime('%H:%M')
data['end'] = data['end'].strftime('%H:%M')
data['weekday'] = [int(v) for v in data['weekday'].split(',')]
return data
async def validate_fields_and_update(self, data, schema):
verrors = ValidationErrors()
begin = data.get('begin')
if begin:
data['begin'] = time(int(begin.split(':')[0]),
int(begin.split(':')[1]))
end = data.get('end')
if end:
data['end'] = time(int(end.split(':')[0]), int(end.split(':')[1]))
weekdays = data.get('weekday')
if weekdays:
if len([day for day in weekdays if day not in range(1, 8)]) > 0:
verrors.add(
f'{schema}.weekday',
'The week days should be in range of 1-7 inclusive')
else:
data['weekday'] = ','.join([str(day) for day in weekdays])
return verrors, data
@accepts(
Dict('pool_resilver', Str('begin', validators=[Time()]),
Str('end', validators=[Time()]), Bool('enabled'),
List('weekday', items=[Int('weekday')])))
async def do_update(self, data):
config = await self.config()
original_config = config.copy()
config.update(data)
verrors, new_config = await self.validate_fields_and_update(
config, 'pool_resilver_update')
if verrors:
raise verrors
# before checking if any changes have been made, original_config needs to be mapped to new_config
original_config['weekday'] = ','.join(
[str(day) for day in original_config['weekday']])
original_config['begin'] = time(
*(int(value) for value in original_config['begin'].split(':')))
original_config['end'] = time(
*(int(value) for value in original_config['end'].split(':')))
if len(set(original_config.items()) ^ set(new_config.items())) > 0:
# data has changed
await self.middleware.call('datastore.update',
self._config.datastore,
new_config['id'], new_config)
await self.middleware.call('service.restart', 'cron',
{'onetime': False})
await self.middleware.call('pool.configure_resilver_priority')
return await self.config()
class CloudSyncService(CRUDService):
class Config:
datastore = "tasks.cloudsync"
datastore_extend = "cloudsync._extend"
@filterable
async def query(self, filters=None, options=None):
tasks_or_task = await super().query(filters, options)
jobs = {}
for j in await self.middleware.call(
"core.get_jobs", [("method", "=", "cloudsync.sync")],
{"order_by": ["id"]}):
try:
task_id = int(j["arguments"][0])
except (IndexError, ValueError):
continue
if task_id in jobs and jobs[task_id]["state"] == "RUNNING":
continue
jobs[task_id] = j
if isinstance(tasks_or_task, list):
for task in tasks_or_task:
task["job"] = jobs.get(task["id"])
else:
tasks_or_task["job"] = jobs.get(tasks_or_task["id"])
return tasks_or_task
@private
async def _extend(self, cloud_sync):
cloud_sync["credentials"] = cloud_sync.pop("credential")
cloud_sync["encryption_password"] = await self.middleware.call(
"pwenc.decrypt", cloud_sync["encryption_password"])
cloud_sync["encryption_salt"] = await self.middleware.call(
"pwenc.decrypt", cloud_sync["encryption_salt"])
Cron.convert_db_format_to_schedule(cloud_sync)
return cloud_sync
@private
async def _compress(self, cloud_sync):
cloud_sync["credential"] = cloud_sync.pop("credentials")
cloud_sync["encryption_password"] = await self.middleware.call(
"pwenc.encrypt", cloud_sync["encryption_password"])
cloud_sync["encryption_salt"] = await self.middleware.call(
"pwenc.encrypt", cloud_sync["encryption_salt"])
Cron.convert_schedule_to_db_format(cloud_sync)
cloud_sync.pop('job', None)
return cloud_sync
@private
async def _get_credentials(self, credentials_id):
try:
return await self.middleware.call("datastore.query",
"system.cloudcredentials",
[("id", "=", credentials_id)],
{"get": True})
except IndexError:
return None
@private
async def _basic_validate(self, verrors, name, data):
if data["encryption"]:
if not data["encryption_password"]:
verrors.add(
f"{name}.encryption_password",
"This field is required when encryption is enabled")
credentials = await self._get_credentials(data["credentials"])
if not credentials:
verrors.add(f"{name}.credentials", "Invalid credentials")
try:
shlex.split(data["args"])
except ValueError as e:
verrors.add(f"{name}.args", f"Parse error: {e.args[0]}")
if verrors:
raise verrors
provider = REMOTES[credentials["provider"]]
schema = []
if provider.buckets:
schema.append(Str("bucket", required=True, empty=False))
schema.append(Str("folder", required=True))
schema.extend(provider.task_schema)
schema.extend(self.common_task_schema(provider))
attributes_verrors = validate_attributes(schema,
data,
additional_attrs=True)
if not attributes_verrors:
await provider.pre_save_task(data, credentials, verrors)
verrors.add_child(f"{name}.attributes", attributes_verrors)
@private
async def _validate(self, verrors, name, data):
await self._basic_validate(verrors, name, data)
for i, (limit1,
limit2) in enumerate(zip(data["bwlimit"],
data["bwlimit"][1:])):
if limit1["time"] >= limit2["time"]:
verrors.add(
f"{name}.bwlimit.{i + 1}.time",
f"Invalid time order: {limit1['time']}, {limit2['time']}")
if data["snapshot"]:
if data["direction"] != "PUSH":
verrors.add(f"{name}.snapshot",
"This option can only be enabled for PUSH tasks")
@private
async def _validate_folder(self, verrors, name, data):
if data["direction"] == "PULL":
folder = data["attributes"]["folder"].rstrip("/")
if folder:
folder_parent = os.path.normpath(os.path.join(folder, ".."))
if folder_parent == ".":
folder_parent = ""
folder_basename = os.path.basename(folder)
ls = await self.list_directory(
dict(
credentials=data["credentials"],
encryption=data["encryption"],
filename_encryption=data["filename_encryption"],
encryption_password=data["encryption_password"],
encryption_salt=data["encryption_salt"],
attributes=dict(data["attributes"],
folder=folder_parent),
args=data["args"],
))
for item in ls:
if item["Name"] == folder_basename:
if not item["IsDir"]:
verrors.add(f"{name}.attributes.folder",
"This is not a directory")
break
else:
verrors.add(f"{name}.attributes.folder",
"Directory does not exist")
if data["direction"] == "PUSH":
credentials = await self._get_credentials(data["credentials"])
provider = REMOTES[credentials["provider"]]
if provider.readonly:
verrors.add(f"{name}.direction", "This remote is read-only")
@accepts(
Dict(
"cloud_sync_create",
Str("description", default=""),
Str("direction", enum=["PUSH", "PULL"], required=True),
Str("transfer_mode", enum=["SYNC", "COPY", "MOVE"], required=True),
Str("path", required=True),
Int("credentials", required=True),
Bool("encryption", default=False),
Bool("filename_encryption", default=False),
Str("encryption_password", default=""),
Str("encryption_salt", default=""),
Cron("schedule", required=True),
Bool("follow_symlinks", default=False),
Int("transfers",
null=True,
default=None,
validators=[Range(min=1)]),
List("bwlimit",
default=[],
items=[
Dict(
"cloud_sync_bwlimit", Str("time",
validators=[Time()]),
Int("bandwidth", validators=[Range(min=1)],
null=True))
]),
List("exclude", default=[], items=[Str("path", empty=False)]),
Dict("attributes", additional_attrs=True, required=True),
Bool("snapshot", default=False),
Str("pre_script", default=""),
Str("post_script", default=""),
Str("args", default=""),
Bool("enabled", default=True),
register=True,
))
async def do_create(self, cloud_sync):
"""
Creates a new cloud_sync entry.
.. examples(websocket)::
Create a new cloud_sync using amazon s3 attributes, which is supposed to run every hour.
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "cloudsync.create",
"params": [{
"description": "s3 sync",
"path": "/mnt/tank",
"credentials": 1,
"minute": "00",
"hour": "*",
"daymonth": "*",
"month": "*",
"attributes": {
"bucket": "mybucket",
"folder": ""
},
"enabled": true
}]
}
"""
verrors = ValidationErrors()
await self._validate(verrors, "cloud_sync", cloud_sync)
if verrors:
raise verrors
await self._validate_folder(verrors, "cloud_sync", cloud_sync)
if verrors:
raise verrors
cloud_sync = await self._compress(cloud_sync)
cloud_sync["id"] = await self.middleware.call("datastore.insert",
"tasks.cloudsync",
cloud_sync)
await self.middleware.call("service.restart", "cron")
cloud_sync = await self._extend(cloud_sync)
return cloud_sync
@accepts(Int("id"),
Patch("cloud_sync_create", "cloud_sync_update", ("attr", {
"update": True
})))
async def do_update(self, id, data):
"""
Updates the cloud_sync entry `id` with `data`.
"""
cloud_sync = await self._get_instance(id)
# credentials is a foreign key for now
if cloud_sync["credentials"]:
cloud_sync["credentials"] = cloud_sync["credentials"]["id"]
cloud_sync.update(data)
verrors = ValidationErrors()
await self._validate(verrors, "cloud_sync_update", cloud_sync)
if verrors:
raise verrors
await self._validate_folder(verrors, "cloud_sync_update", cloud_sync)
if verrors:
raise verrors
cloud_sync = await self._compress(cloud_sync)
await self.middleware.call("datastore.update", "tasks.cloudsync", id,
cloud_sync)
await self.middleware.call("service.restart", "cron")
cloud_sync = await self._extend(cloud_sync)
return cloud_sync
@accepts(Int("id"))
async def do_delete(self, id):
"""
Deletes cloud_sync entry `id`.
"""
await self.middleware.call("datastore.delete", "tasks.cloudsync", id)
await self.middleware.call("service.restart", "cron")
@accepts(Int("credentials_id"))
async def list_buckets(self, credentials_id):
credentials = await self._get_credentials(credentials_id)
if not credentials:
raise CallError("Invalid credentials")
provider = REMOTES[credentials["provider"]]
if not provider.buckets:
raise CallError("This provider does not use buckets")
return await self.ls({"credentials": credentials}, "")
@accepts(
Dict(
"cloud_sync_ls",
Int("credentials", required=True),
Bool("encryption", default=False),
Bool("filename_encryption", default=False),
Str("encryption_password", default=""),
Str("encryption_salt", default=""),
Dict("attributes", required=True, additional_attrs=True),
Str("args", default=""),
))
async def list_directory(self, cloud_sync):
verrors = ValidationErrors()
await self._basic_validate(verrors, "cloud_sync", dict(cloud_sync))
if verrors:
raise verrors
credentials = await self._get_credentials(cloud_sync["credentials"])
if REMOTES[credentials["provider"]].buckets:
path = f"{cloud_sync['attributes']['bucket']}/{cloud_sync['attributes']['folder']}"
else:
path = cloud_sync["attributes"]["folder"]
return await self.ls(dict(cloud_sync, credentials=credentials), path)
@private
async def ls(self, config, path):
with RcloneConfig(config) as config:
proc = await run([
"rclone", "--config", config.config_path, "lsjson",
"remote:" + path
],
check=False,
encoding="utf8")
if proc.returncode == 0:
return json.loads(proc.stdout)
else:
raise CallError(proc.stderr)
@item_method
@accepts(Int("id"))
@job(lock=lambda args: "cloud_sync:{}".format(args[-1]),
lock_queue_size=1,
logs=True)
async def sync(self, job, id):
"""
Run the cloud_sync job `id`, syncing the local data to remote.
"""
cloud_sync = await self._get_instance(id)
return await rclone(self.middleware, job, cloud_sync)
@accepts()
async def providers(self):
return sorted([{
"name":
provider.name,
"title":
provider.title,
"credentials_schema": [{
"property": field.name,
"schema": field.to_json_schema()
} for field in provider.credentials_schema],
"buckets":
provider.buckets,
"bucket_title":
provider.bucket_title,
"task_schema": [{
"property": field.name,
"schema": field.to_json_schema()
} for field in provider.task_schema +
self.common_task_schema(provider)],
} for provider in REMOTES.values()],
key=lambda provider: provider["title"].lower())
def common_task_schema(self, provider):
schema = []
if provider.fast_list:
schema.append(
Bool("fast_list",
default=False,
title="Use --fast-list",
description=textwrap.dedent("""\
Use fewer transactions in exchange for more RAM. This may also speed up or slow down your
transfer. See [rclone documentation](https://rclone.org/docs/#fast-list) for more details.
""").rstrip()))
return schema
class ReplicationService(CRUDService):
class Config:
datastore = 'storage.replication'
datastore_prefix = 'repl_'
datastore_extend = 'replication.replication_extend'
@private
async def replication_extend(self, data):
remote_data = data.pop('remote')
data['remote'] = remote_data['id']
data['remote_dedicateduser_enabled'] = remote_data[
'ssh_remote_dedicateduser_enabled']
data['remote_port'] = remote_data['ssh_remote_port']
data['remote_cipher'] = remote_data['ssh_cipher'].upper()
data['remote_dedicateduser'] = remote_data['ssh_remote_dedicateduser']
data['remote_hostkey'] = remote_data['ssh_remote_hostkey']
data['remote_hostname'] = remote_data['ssh_remote_hostname']
if not os.path.exists(REPL_RESULTFILE):
data['lastresult'] = {'msg': 'Waiting'}
else:
with open(REPL_RESULTFILE, 'rb') as f:
file_data = f.read()
try:
results = pickle.loads(file_data)
data['lastresult'] = results[data['id']]
except Exception:
data['lastresult'] = {'msg': None}
progressfile = f'/tmp/.repl_progress_{data["id"]}'
if os.path.exists(progressfile):
with open(progressfile, 'r') as f:
pid = int(f.read())
title = await self.middleware.call('notifier.get_proc_title', pid)
if title:
reg = re.search(r'sending (\S+) \((\d+)%', title)
if reg:
data[
'status'] = f'Sending {reg.groups()[0]}s {reg.groups()[1]}s'
else:
data['status'] = 'Sending'
if 'status' not in data:
data['status'] = data['lastresult'].get('msg')
data['begin'] = str(data['begin'])
data['end'] = str(data['end'])
data['compression'] = data['compression'].upper()
return data
@private
async def validate_data(self, data, schema_name):
verrors = ValidationErrors()
remote_hostname = data.pop('remote_hostname')
await resolve_hostname(self.middleware, verrors,
f'{schema_name}.remote_hostname',
remote_hostname)
remote_dedicated_user_enabled = data.pop(
'remote_dedicateduser_enabled', False)
remote_dedicated_user = data.pop('remote_dedicateduser', None)
if remote_dedicated_user_enabled and not remote_dedicated_user:
verrors.add(
f'{schema_name}.remote_dedicateduser',
'You must select a user when remote dedicated user is enabled')
if not await self.middleware.call('pool.snapshottask.query', [
('filesystem', '=', data.get('filesystem'))
]):
verrors.add(f'{schema_name}.filesystem', 'Invalid Filesystem')
remote_mode = data.pop('remote_mode', 'MANUAL')
remote_port = data.pop('remote_port')
repl_remote_dict = {
'ssh_remote_hostname': remote_hostname,
'ssh_remote_dedicateduser_enabled': remote_dedicated_user_enabled,
'ssh_remote_dedicateduser': remote_dedicated_user,
'ssh_cipher': data.pop('remote_cipher', 'STANDARD').lower()
}
if remote_mode == 'SEMIAUTOMATIC':
token = data.pop('remote_token', None)
if not token:
verrors.add(f'{schema_name}.remote_token',
'This field is required')
else:
remote_host_key = data.pop('remote_hostkey', None)
if not remote_host_key:
verrors.add(f'{schema_name}.remote_hostkey',
'This field is required')
else:
repl_remote_dict['ssh_remote_port'] = remote_port
repl_remote_dict['ssh_remote_hostkey'] = remote_host_key
if verrors:
raise verrors
data['begin'] = time(*[int(v) for v in data.pop('begin').split(':')])
data['end'] = time(*[int(v) for v in data.pop('end').split(':')])
data['compression'] = data['compression'].lower()
data.pop('remote_hostkey', None)
data.pop('remote_token', None)
return verrors, data, repl_remote_dict
@accepts(
Dict('replication_create',
Bool('enabled', default=True),
Bool('followdelete', default=False),
Bool('remote_dedicateduser_enabled', default=False),
Bool('remote_https'),
Bool('userepl', default=False),
Int('limit', default=0, validators=[Range(min=0)]),
Int('remote_port', default=22, required=True),
Str('begin', validators=[Time()]),
Str('compression', enum=['OFF', 'LZ4', 'PIGZ', 'PLZIP']),
Str('end', validators=[Time()]),
Str('filesystem', required=True),
Str('remote_cipher', enum=['STANDARD', 'FAST', 'DISABLED']),
Str('remote_dedicateduser'),
Str('remote_hostkey'),
Str('remote_hostname', required=True),
Str('remote_mode',
enum=['SEMIAUTOMATIC', 'MANUAL'],
required=True),
Str('remote_token'),
Str('zfs', required=True),
register=True))
async def do_create(self, data):
remote_hostname = data.get('remote_hostname')
remote_dedicated_user = data.get('remote_dedicateduser')
remote_port = data.get('remote_port')
remote_https = data.pop('remote_https', False)
remote_token = data.get('remote_token')
remote_mode = data.get('remote_mode')
verrors, data, repl_remote_dict = await self.validate_data(
data, 'replication_create')
if remote_mode == 'SEMIAUTOMATIC':
remote_uri = f'ws{"s" if remote_https else ""}://{remote_hostname}:{remote_port}/websocket'
try:
with Client(remote_uri) as c:
if not c.call('auth.token', remote_token):
verrors.add('replication_create.remote_token',
'Please provide a valid token')
else:
try:
with open(REPLICATION_KEY, 'r') as f:
publickey = f.read()
call_data = c.call(
'replication.pair', {
'hostname': remote_hostname,
'public-key': publickey,
'user': remote_dedicated_user,
})
except Exception as e:
raise CallError('Failed to set up replication ' +
str(e))
else:
repl_remote_dict['ssh_remote_port'] = call_data[
'ssh_port']
repl_remote_dict['ssh_remote_hostkey'] = call_data[
'ssh_hostkey']
except Exception as e:
verrors.add(
'replication_create.remote_token',
f'Failed to connect to remote host {remote_uri} with following exception {e}'
)
if verrors:
raise verrors
remote_pk = await self.middleware.call('datastore.insert',
'storage.replremote',
repl_remote_dict)
await self.middleware.call('service.reload', 'ssh', {'onetime': False})
data['remote'] = remote_pk
pk = await self.middleware.call(
'datastore.insert', self._config.datastore, data,
{'prefix': self._config.datastore_prefix})
return await self._get_instance(pk)
@accepts(Int('id', required=True),
Patch(
'replication_create',
'replication_update',
('attr', {
'update': True
}),
('rm', {
'name': 'remote_mode'
}),
('rm', {
'name': 'remote_https'
}),
('rm', {
'name': 'remote_token'
}),
))
async def do_update(self, id, data):
old = await self._get_instance(id)
new = old.copy()
new.update(data)
verrors, new, repl_remote_dict = await self.validate_data(
new, 'replication_update')
new.pop('status')
new.pop('lastresult')
await self.middleware.call('datastore.update', 'storage.replremote',
new['remote'], repl_remote_dict)
await self.middleware.call('service.reload', 'ssh', {'onetime': False})
await self.middleware.call('datastore.update', self._config.datastore,
id, new,
{'prefix': self._config.datastore_prefix})
return await self._get_instance(id)
@accepts(Int('id'))
async def do_delete(self, id):
response = await self.middleware.call('datastore.delete',
self._config.datastore, id)
return response
@accepts()
def public_key(self):
"""
Get the public SSH replication key.
"""
if (os.path.exists(REPLICATION_KEY)
and os.path.isfile(REPLICATION_KEY)):
with open(REPLICATION_KEY, 'r') as f:
key = f.read()
else:
key = None
return key
@accepts(
Str('host', required=True),
Int('port', required=True),
)
async def ssh_keyscan(self, host, port):
"""
Scan the SSH key on `host`:`port`.
"""
proc = await Popen([
"/usr/bin/ssh-keyscan",
"-p",
str(port),
"-T",
"2",
str(host),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
key, errmsg = await proc.communicate()
if proc.returncode != 0 or not key:
if not errmsg:
errmsg = 'ssh key scan failed for unknown reason'
else:
errmsg = errmsg.decode()
raise CallError(errmsg)
return key.decode()
@private
@accepts(
Dict(
'replication-pair-data',
Str('hostname', required=True),
Str('public-key', required=True),
Str('user'),
))
async def pair(self, data):
"""
Receives public key, storing it to accept SSH connection and return
pertinent SSH data of this machine.
"""
service = await self.middleware.call('datastore.query',
'services.services',
[('srv_service', '=', 'ssh')],
{'get': True})
ssh = await self.middleware.call('datastore.query', 'services.ssh',
None, {'get': True})
try:
user = await self.middleware.call(
'datastore.query', 'account.bsdusers',
[('bsdusr_username', '=', data.get('user') or 'root')],
{'get': True})
except IndexError:
raise ValueError('User "{}" does not exist'.format(
data.get('user')))
if user['bsdusr_home'].startswith('/nonexistent'):
raise CallError(f'User home directory does not exist',
errno.ENOENT)
# Make sure SSH is enabled
if not service['srv_enable']:
await self.middleware.call('datastore.update', 'services.services',
service['id'], {'srv_enable': True})
await self.middleware.call('notifier.start', 'ssh')
# This might be the first time of the service being enabled
# which will then result in new host keys we need to grab
ssh = await self.middleware.call('datastore.query', 'services.ssh',
None, {'get': True})
if not os.path.exists(user['bsdusr_home']):
raise ValueError('Homedir {} does not exist'.format(
user['bsdusr_home']))
# If .ssh dir does not exist, create it
dotsshdir = os.path.join(user['bsdusr_home'], '.ssh')
if not os.path.exists(dotsshdir):
os.mkdir(dotsshdir)
os.chown(dotsshdir, user['bsdusr_uid'],
user['bsdusr_group']['bsdgrp_gid'])
# Write public key in user authorized_keys for SSH
authorized_keys_file = f'{dotsshdir}/authorized_keys'
with open(authorized_keys_file, 'a+') as f:
f.seek(0)
if data['public-key'] not in f.read():
f.write('\n' + data['public-key'])
ssh_hostkey = '{0} {1}\n{0} {2}\n{0} {3}\n'.format(
data['hostname'],
base64.b64decode(ssh['ssh_host_rsa_key_pub'].encode()).decode(),
base64.b64decode(ssh['ssh_host_ecdsa_key_pub'].encode()).decode(),
base64.b64decode(
ssh['ssh_host_ed25519_key_pub'].encode()).decode(),
)
return {
'ssh_port': ssh['ssh_tcpport'],
'ssh_hostkey': ssh_hostkey,
}
