async def load(self):
is_freenas = await self.middleware.call("system.is_freenas")
main_sources_dir = os.path.join(get_middlewared_dir(), "alert",
"source")
sources_dirs = [
os.path.join(overlay_dir, "alert", "source")
for overlay_dir in self.middleware.overlay_dirs
]
sources_dirs.insert(0, main_sources_dir)
for sources_dir in sources_dirs:
for module in load_modules(sources_dir):
for cls in load_classes(
module, AlertSource,
(FilePresenceAlertSource, ThreadedAlertSource)):
source = cls(self.middleware)
ALERT_SOURCES[source.name] = source
main_services_dir = os.path.join(
os.path.dirname(os.path.realpath(__file__)), os.path.pardir,
"alert", "service")
services_dirs = [
os.path.join(overlay_dir, "alert", "service")
for overlay_dir in self.middleware.overlay_dirs
]
services_dirs.insert(0, main_services_dir)
for services_dir in services_dirs:
for module in load_modules(services_dir):
for cls in load_classes(
module, _AlertService,
(ThreadedAlertService, ProThreadedAlertService)):
ALERT_SERVICES_FACTORIES[cls.name()] = cls
async def load(self):
main_sources_dir = os.path.join(get_middlewared_dir(), "alert",
"source")
sources_dirs = [
os.path.join(overlay_dir, "alert", "source")
for overlay_dir in self.middleware.overlay_dirs
]
sources_dirs.insert(0, main_sources_dir)
for sources_dir in sources_dirs:
for module in load_modules(sources_dir):
for cls in load_classes(
module, AlertSource,
(FilePresenceAlertSource, ThreadedAlertSource)):
source = cls(self.middleware)
if source.name in ALERT_SOURCES:
raise RuntimeError(
f"Alert source {source.name} is already registered"
)
ALERT_SOURCES[source.name] = source
main_services_dir = os.path.join(
os.path.dirname(os.path.realpath(__file__)), os.path.pardir,
"alert", "service")
services_dirs = [
os.path.join(overlay_dir, "alert", "service")
for overlay_dir in self.middleware.overlay_dirs
]
services_dirs.insert(0, main_services_dir)
for services_dir in services_dirs:
for module in load_modules(services_dir):
for cls in load_classes(
module, _AlertService,
(ThreadedAlertService, ProThreadedAlertService)):
ALERT_SERVICES_FACTORIES[cls.name()] = cls
def load_migrations(middleware):
main_sources_dir = os.path.join(get_middlewared_dir(), "migration")
sources_dirs = [os.path.join(overlay_dir, "migration") for overlay_dir in middleware.overlay_dirs]
sources_dirs.insert(0, main_sources_dir)
modules = []
for sources_dir in sources_dirs:
modules.extend(load_modules(sources_dir))
return sorted(modules, key=lambda module: module.__name__)
if provider.fast_list:
schema.append(
Bool("fast_list",
default=False,
title="Use --fast-list",
description=textwrap.dedent("""\
Use fewer transactions in exchange for more RAM. This may also speed up or slow down your
transfer. See [rclone documentation](https://rclone.org/docs/#fast-list) for more details.
""").rstrip()))
return schema
remote_classes = []
for module in load_modules(
os.path.join(get_middlewared_dir(), "rclone", "remote")):
for cls in load_classes(module, BaseRcloneRemote, []):
remote_classes.append(cls)
for method_name in cls.extra_methods:
setattr(CloudSyncService, f"{cls.name.lower()}_{method_name}",
getattr(cls, method_name))
class CloudSyncFSAttachmentDelegate(LockableFSAttachmentDelegate):
name = 'cloudsync'
title = 'CloudSync Task'
service_class = CloudSyncService
resource_name = 'path'
async def restart_reload_services(self, attachments):
await self.middleware.call('service.restart', 'cron')
def __upload(self, config_file_name):
try:
"""
First we try to open the file as a tar file.
We expect the tar file to contain at least the freenas-v1.db.
It can also contain the pwenc_secret file.
If we cannot open it as a tar, we try to proceed as it was the
raw database file.
"""
try:
with tarfile.open(config_file_name) as tar:
bundle = True
tmpdir = tempfile.mkdtemp(dir='/var/tmp/firmware')
tar.extractall(path=tmpdir)
config_file_name = os.path.join(tmpdir, 'freenas-v1.db')
except tarfile.ReadError:
bundle = False
# Currently we compare only the number of migrations for south and django
# of new and current installed database.
# This is not bullet proof as we can eventually have more migrations in a stable
# release compared to a older nightly and still be considered a downgrade, however
# this is simple enough and works in most cases.
alembic_version = None
conn = sqlite3.connect(config_file_name)
try:
cur = conn.cursor()
try:
cur.execute("SELECT version_num FROM alembic_version")
alembic_version = cur.fetchone()[0]
except sqlite3.OperationalError as e:
if e.args[0] == "no such table: alembic_version":
# FN/TN < 12
# Let's just ensure it's not a random SQLite file
cur.execute("SELECT 1 FROM django_migrations")
else:
raise
finally:
cur.close()
finally:
conn.close()
if alembic_version is not None:
for root, dirs, files in os.walk(
os.path.join(get_middlewared_dir(), "alembic",
"versions")):
found = False
for name in files:
if name.endswith(".py"):
with open(os.path.join(root, name)) as f:
if any(line.strip() ==
f"Revision ID: {alembic_version}"
for line in f.read().splitlines()):
found = True
break
if found:
break
else:
raise CallError(
'Failed to upload config, version newer than the '
'current installed.')
except Exception as e:
os.unlink(config_file_name)
if isinstance(e, CallError):
raise
else:
raise CallError(f'The uploaded file is not valid: {e}')
shutil.move(config_file_name, UPLOADED_DB_PATH)
if bundle:
for filename, destination in CONFIG_FILES.items():
file_path = os.path.join(tmpdir, filename)
if os.path.exists(file_path):
if filename == 'geli':
# Let's only copy the geli keys and not overwrite the entire directory
os.makedirs(CONFIG_FILES['geli'], exist_ok=True)
for key_path in os.listdir(file_path):
shutil.move(os.path.join(file_path, key_path),
os.path.join(destination, key_path))
elif filename == 'pwenc_secret':
shutil.move(file_path, '/data/pwenc_secret_uploaded')
else:
shutil.move(file_path, destination)
# Now we must run the migrate operation in the case the db is older
open(NEED_UPDATE_SENTINEL, 'w+').close()
if osc.IS_LINUX:
# For SCALE, we have to enable/disable services based on the uploaded database
enable_disable_units = {'enable': [], 'disable': []}
conn = sqlite3.connect(UPLOADED_DB_PATH)
try:
cursor = conn.cursor()
for service, enabled in cursor.execute(
"SELECT srv_service, srv_enable FROM services_services"
).fetchall():
try:
units = self.middleware.call_sync(
'service.systemd_units', service)
except KeyError:
# An old service which we don't have currently
continue
if enabled:
enable_disable_units['enable'].extend(units)
else:
enable_disable_units['disable'].extend(units)
finally:
conn.close()
for action in filter(lambda k: enable_disable_units[k],
enable_disable_units):
cp = subprocess.Popen(['systemctl', action] +
enable_disable_units[action],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
err = cp.communicate()[1]
if cp.returncode:
self.middleware.logger.error(
'Failed to %s %r systemctl units: %s', action,
', '.join(enable_disable_units[action]), err.decode())
sys.path.append('/usr/local/lib')
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = Model.metadata
list(load_modules(os.path.join(get_middlewared_dir(), "plugins"), depth=1))
list(load_modules("/usr/local/lib/middlewared_truenas/plugins", depth=1))
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
DATABASE_URL = f"sqlite:///{os.environ.get('FREENAS_DATABASE', FREENAS_DATABASE)}"
@Operations.register_operation("drop_references")
@BatchOperations.register_operation("drop_references", "batch_drop_references")
class DropReferencesOp(ops.MigrateOperation):
def __init__(
self,
field_name,
@private
def common_task_schema(self, provider):
schema = []
if provider.fast_list:
schema.append(Bool("fast_list", default=False, title="Use --fast-list", description=textwrap.dedent("""\
Use fewer transactions in exchange for more RAM. This may also speed up or slow down your
transfer. See [rclone documentation](https://rclone.org/docs/#fast-list) for more details.
""").rstrip()))
return schema
remote_classes = []
for module in load_modules(os.path.join(get_middlewared_dir(), "rclone", "remote")):
for cls in load_classes(module, BaseRcloneRemote, []):
remote_classes.append(cls)
for method_name in cls.extra_methods:
setattr(CloudSyncService, f"{cls.name.lower()}_{method_name}", getattr(cls, method_name))
class CloudSyncFSAttachmentDelegate(LockableFSAttachmentDelegate):
name = 'cloudsync'
title = 'CloudSync Task'
service_class = CloudSyncService
resource_name = 'path'
async def restart_reload_services(self, attachments):
await self.middleware.call('service.restart', 'cron')
def __upload(self, config_file_name):
tar_error = None
try:
"""
First we try to open the file as a tar file.
We expect the tar file to contain at least the freenas-v1.db.
It can also contain the pwenc_secret file.
If we cannot open it as a tar, we try to proceed as it was the
raw database file.
"""
try:
with tarfile.open(config_file_name) as tar:
bundle = True
tmpdir = tempfile.mkdtemp(dir='/var/tmp/firmware')
tar.extractall(path=tmpdir)
config_file_name = os.path.join(tmpdir, 'freenas-v1.db')
except tarfile.ReadError as e:
tar_error = str(e)
bundle = False
# Currently we compare only the number of migrations for south and django
# of new and current installed database.
# This is not bullet proof as we can eventually have more migrations in a stable
# release compared to a older nightly and still be considered a downgrade, however
# this is simple enough and works in most cases.
alembic_version = None
conn = sqlite3.connect(config_file_name)
try:
cur = conn.cursor()
try:
cur.execute("SELECT version_num FROM alembic_version")
alembic_version = cur.fetchone()[0]
except sqlite3.OperationalError as e:
if e.args[0] == "no such table: alembic_version":
# FN/TN < 12
# Let's just ensure it's not a random SQLite file
cur.execute("SELECT 1 FROM django_migrations")
else:
raise
finally:
cur.close()
except sqlite3.OperationalError as e:
if tar_error:
raise CallError(
f"Uploaded file is neither a valid .tar file ({tar_error}) nor valid FreeNAS/TrueNAS database "
f"file ({e}).")
else:
raise CallError(
f"Uploaded file is not a valid FreeNAS/TrueNAS database file ({e})."
)
finally:
conn.close()
if alembic_version is not None:
for root, dirs, files in os.walk(
os.path.join(get_middlewared_dir(), "alembic",
"versions")):
found = False
for name in files:
if name.endswith(".py"):
with open(os.path.join(root, name)) as f:
if any(line.strip() ==
f"Revision ID: {alembic_version}"
for line in f.read().splitlines()):
found = True
break
if found:
break
else:
raise CallError(
'Failed to upload config, version newer than the '
'current installed.')
except Exception as e:
os.unlink(config_file_name)
if isinstance(e, CallError):
raise
else:
raise CallError(f'The uploaded file is not valid: {e}')
upload = []
def move(src, dst):
shutil.move(src, dst)
upload.append(dst)
move(config_file_name, UPLOADED_DB_PATH)
if bundle:
for filename, destination in CONFIG_FILES.items():
file_path = os.path.join(tmpdir, filename)
if os.path.exists(file_path):
if filename == 'geli':
# Let's only copy the geli keys and not overwrite the entire directory
os.makedirs(CONFIG_FILES['geli'], exist_ok=True)
for key_path in os.listdir(file_path):
move(os.path.join(file_path, key_path),
os.path.join(destination, key_path))
elif filename == 'pwenc_secret':
move(file_path, '/data/pwenc_secret_uploaded')
else:
move(file_path, destination)
# Now we must run the migrate operation in the case the db is older
open(NEED_UPDATE_SENTINEL, 'w+').close()
upload.append(NEED_UPDATE_SENTINEL)
self.middleware.call_hook_sync('config.on_upload', UPLOADED_DB_PATH)
if self.middleware.call_sync('failover.licensed'):
try:
for path in upload:
self.middleware.call_sync('failover.send_small_file', path)
self.middleware.call_sync(
'failover.call_remote',
'core.call_hook',
['config.on_upload', [UPLOADED_DB_PATH]],
)
self.middleware.run_coroutine(
self.middleware.call('failover.call_remote',
'system.reboot'),
wait=False,
)
except Exception as e:
raise CallError(
f'Config uploaded successfully, but remote node responded with error: {e}. '
f'Please use Sync to Peer on the System/Failover page to perform a manual sync after reboot.',
CallError.EREMOTENODEERROR,
)
