def query(self, filters=None, options=None):
"""
Query installed plugins with `query-filters` and `query-options`.
"""
options = options or {}
self.middleware.call_sync(
'jail.check_dataset_existence') # Make sure our datasets exist.
iocage = ioc.IOCage(skip_jails=True)
resource_list = iocage.list('all', plugin=True)
pool = IOCJson().json_get_value('pool')
iocroot = IOCJson(pool).json_get_value('iocroot')
plugin_dir_path = os.path.join(iocroot, '.plugins')
plugin_jails = {
j['host_hostuuid']: j
for j in self.middleware.call_sync(
'jail.query', [['type', 'in', ['plugin', 'pluginv2']]])
}
index_jsons = {}
for repo in os.listdir(plugin_dir_path) if os.path.exists(
plugin_dir_path) else []:
index_path = os.path.join(plugin_dir_path, repo, 'INDEX')
if os.path.exists(index_path):
with contextlib.suppress(json.decoder.JSONDecodeError):
with open(index_path, 'r') as f:
index_jsons[repo] = json.loads(f.read())
for index, plugin in enumerate(resource_list):
# "plugin" is a list which we will convert to a dictionary for readability
plugin_dict = {
k: v if v != '-' else None
for k, v in zip(('jid', 'name', 'boot', 'state', 'type',
'release', 'ip4', 'ip6', 'template',
'admin_portal'), plugin)
}
plugin_output = pathlib.Path(
f'{iocroot}/jails/{plugin_dict["name"]}/root/root/PLUGIN_INFO')
plugin_info = plugin_output.read_text().strip(
) if plugin_output.is_file() else None
plugin_name = plugin_jails[plugin_dict['name']]['plugin_name']
plugin_repo = self.convert_repository_to_path(
plugin_jails[plugin_dict['name']]['plugin_repository'])
plugin_dict.update(
{
'id':
plugin_dict['name'],
'plugin_info':
plugin_info,
'plugin':
plugin_name,
'plugin_repository':
plugin_jails[plugin_dict['name']]['plugin_repository'],
**self.get_local_plugin_version(
plugin_name, index_jsons.get(plugin_repo), iocroot, plugin_dict['name'])
})
resource_list[index] = plugin_dict
return filter_list(resource_list, filters, options)
def create_job(self, job, options):
verrors = ValidationErrors()
uuid = options["uuid"]
job.set_progress(0, f'Creating: {uuid}')
try:
self.check_jail_existence(uuid, skip=False)
verrors.add('uuid', f'A jail with name {uuid} already exists')
raise verrors
except CallError:
# A jail does not exist with the provided name, we can create one
# now
verrors = self.common_validation(verrors, options)
if verrors:
raise verrors
job.set_progress(20, 'Initial validation complete')
iocage = ioc.IOCage(skip_jails=True)
release = options["release"]
template = options.get("template", False)
pkglist = options.get("pkglist", None)
basejail = options["basejail"]
empty = options["empty"]
short = options["short"]
props = options["props"]
pool = IOCJson().json_get_value("pool")
iocroot = IOCJson(pool).json_get_value("iocroot")
if template:
release = template
if (not os.path.isdir(f'{iocroot}/releases/{release}') and not template
and not empty):
job.set_progress(50, f'{release} missing, calling fetch')
self.middleware.call_sync('jail.fetch', {"release": release},
job=True)
err, msg = iocage.create(release,
props,
0,
pkglist,
template=template,
short=short,
_uuid=uuid,
basejail=basejail,
empty=empty)
if err:
raise CallError(msg)
job.set_progress(100, f'Created: {uuid}')
return True
def update_to_latest_patch(self, job, jail):
"""Updates specified jail to latest patch level."""
uuid, path, _ = self.check_jail_existence(jail)
status, jid = IOCList.list_get_jid(uuid)
conf = IOCJson(path).json_load()
# Sometimes if they don't have an existing patch level, this
# becomes 11.1 instead of 11.1-RELEASE
_release = conf["release"].rsplit("-", 1)[0]
release = _release if "-RELEASE" in _release else conf["release"]
started = False
if conf["type"] == "jail":
if not status:
self.start(jail)
started = True
else:
return False
if conf["basejail"] != "yes":
IOCFetch(release).fetch_update(True, uuid)
else:
# Basejails only need their base RELEASE updated
IOCFetch(release).fetch_update()
if started:
self.stop(jail)
return True
def upgrade(self, job, jail, release):
"""Upgrades specified jail to specified RELEASE."""
uuid, path, _ = self.check_jail_existence(jail)
status, jid = IOCList.list_get_jid(uuid)
conf = IOCJson(path).json_load()
root_path = f"{path}/root"
started = False
if conf["type"] == "jail":
if not status:
self.start(jail)
started = True
else:
return False
IOCUpgrade(conf, release, root_path).upgrade_jail()
if started:
self.stop(jail)
return True
def get_iocroot(self):
pool = IOCJson().json_get_value("pool")
return IOCJson(pool).json_get_value("iocroot")
def list_resource(self, resource, remote):
"""Returns a JSON list of the supplied resource on the host"""
self.check_dataset_existence() # Make sure our datasets exist.
iocage = ioc.IOCage(skip_jails=True)
resource = "base" if resource == "RELEASE" else resource.lower()
if resource == "plugin":
if remote:
try:
resource_list = self.middleware.call_sync(
'cache.get', 'iocage_remote_plugins')
return resource_list
except KeyError:
pass
resource_list = iocage.fetch(list=True, plugins=True,
header=False)
else:
resource_list = iocage.list("all", plugin=True)
pool = IOCJson().json_get_value("pool")
iocroot = IOCJson(pool).json_get_value("iocroot")
index_path = f'{iocroot}/.plugin_index/INDEX'
if not pathlib.Path(index_path).is_file():
index_json = None
for plugin in resource_list:
plugin += ['N/A', 'N/A']
return resource_list
else:
index_fd = open(index_path, 'r')
index_json = json.load(index_fd)
for plugin in resource_list:
for i, elem in enumerate(plugin):
# iocage returns - for None
plugin[i] = elem if elem != "-" else None
if remote:
pv = self.get_plugin_version(plugin[2])
else:
pv = self.get_local_plugin_version(
plugin[1], index_json, iocroot)
resource_list[resource_list.index(plugin)] = plugin + pv
if remote:
self.middleware.call_sync(
'cache.put', 'iocage_remote_plugins', resource_list,
86400
)
else:
index_fd.close()
elif resource == "base":
try:
if remote:
resource_list = self.middleware.call_sync(
'cache.get', 'iocage_remote_releases')
return resource_list
except KeyError:
pass
resource_list = iocage.fetch(list=True, remote=remote, http=True)
if remote:
self.middleware.call_sync(
'cache.put', 'iocage_remote_releases', resource_list,
86400
)
else:
resource_list = iocage.list(resource)
return resource_list
def fetch(self, job, options):
"""Fetches a release or plugin."""
fetch_output = {'install_notes': []}
release = options.get('release', None)
post_install = False
verrors = ValidationErrors()
self.validate_ips(verrors, options)
if verrors:
raise verrors
def progress_callback(content, exception):
msg = content['message'].strip('\r\n')
rel_up = f'* Updating {release} to the latest patch level... '
nonlocal post_install
if options['name'] is None:
if 'Downloading : base.txz' in msg and '100%' in msg:
job.set_progress(5, msg)
elif 'Downloading : lib32.txz' in msg and '100%' in msg:
job.set_progress(10, msg)
elif 'Downloading : doc.txz' in msg and '100%' in msg:
job.set_progress(15, msg)
elif 'Downloading : src.txz' in msg and '100%' in msg:
job.set_progress(20, msg)
if 'Extracting: base.txz' in msg:
job.set_progress(25, msg)
elif 'Extracting: lib32.txz' in msg:
job.set_progress(50, msg)
elif 'Extracting: doc.txz' in msg:
job.set_progress(75, msg)
elif 'Extracting: src.txz' in msg:
job.set_progress(90, msg)
elif rel_up in msg:
job.set_progress(95, msg)
else:
job.set_progress(None, msg)
else:
if ' These pkgs will be installed:' in msg:
job.set_progress(50, msg)
elif 'Installing plugin packages:' in msg:
job.set_progress(75, msg)
elif 'Command output:' in msg:
job.set_progress(90, msg)
# Sets each message going forward as important to the user
post_install = True
else:
job.set_progress(None, msg)
if post_install:
for split_msg in msg.split('\n'):
fetch_output['install_notes'].append(split_msg)
self.check_dataset_existence() # Make sure our datasets exist.
start_msg = f'{release} being fetched'
final_msg = f'{release} fetched'
iocage = ioc.IOCage(callback=progress_callback, silent=False)
if options["name"] is not None:
pool = IOCJson().json_get_value('pool')
iocroot = IOCJson(pool).json_get_value('iocroot')
plugin_index = pathlib.Path(f'{iocroot}/.plugin_index')
if not plugin_index.is_dir():
# WORKAROUND until rewritten for #39653
# We want the plugins to not prompt interactively
try:
iocage.fetch(plugin_file=True, _list=True, **options)
except Exception:
# Expected, this is to avoid it later
pass
options["plugin_file"] = True
start_msg = 'Starting plugin install'
final_msg = f"Plugin: {options['name']} installed"
options["accept"] = True
job.set_progress(0, start_msg)
iocage.fetch(**options)
if post_install and options['name'] is not None:
plugin_manifest = pathlib.Path(
f'{iocroot}/.plugin_index/{options["name"]}.json')
plugin_json = json.loads(plugin_manifest.read_text())
schema_version = plugin_json.get('plugin_schema', '1')
if schema_version.isdigit() and int(schema_version) >= 2:
plugin_output = pathlib.Path(
f'{iocroot}/jails/{options["name"]}/root/root/PLUGIN_INFO')
if plugin_output.is_file():
# Otherwise it will be the verbose output from the
# post_install script
fetch_output['install_notes'] = [
x for x in plugin_output.read_text().split('\n') if x
]
# This is to get the admin URL and such
fetch_output['install_notes'] += job.progress[
'description'].split('\n')
job.set_progress(100, final_msg)
return fetch_output
def list_resource(self, resource, remote, want_cache, branch):
"""Returns a JSON list of the supplied resource on the host"""
self.check_dataset_existence() # Make sure our datasets exist.
iocage = ioc.IOCage(skip_jails=True)
resource = "base" if resource == "RELEASE" else resource.lower()
if resource == "plugin":
if remote:
if want_cache:
try:
resource_list = self.middleware.call_sync(
'cache.get', 'iocage_remote_plugins')
return resource_list
except KeyError:
pass
resource_list = iocage.fetch(list=True,
plugins=True,
header=False,
branch=branch)
else:
resource_list = iocage.list("all", plugin=True)
pool = IOCJson().json_get_value("pool")
iocroot = IOCJson(pool).json_get_value("iocroot")
index_path = f'{iocroot}/.plugin_index/INDEX'
if not pathlib.Path(index_path).is_file():
index_json = None
for plugin in resource_list:
plugin += ['N/A', 'N/A']
return resource_list
else:
index_fd = open(index_path, 'r')
index_json = json.load(index_fd)
for plugin in resource_list:
for i, elem in enumerate(plugin):
# iocage returns - for None
plugin[i] = elem if elem != "-" else None
if remote:
pv = self.get_plugin_version(plugin[2])
else:
# plugin[1] is the UUID
plugin_output = pathlib.Path(
f'{iocroot}/jails/{plugin[1]}/root/root/PLUGIN_INFO')
if plugin_output.is_file():
plugin_info = [[
x for x in plugin_output.read_text().split('\n')
if x
]]
else:
plugin_info = [None]
pv = self.get_local_plugin_version(plugin[1], index_json,
iocroot) + plugin_info
resource_list[resource_list.index(plugin)] = plugin + pv
if remote:
self.middleware.call_sync('cache.put', 'iocage_remote_plugins',
resource_list, 86400)
else:
index_fd.close()
elif resource == "base":
try:
if remote:
resource_list = self.middleware.call_sync(
'cache.get', 'iocage_remote_releases')
return resource_list
except KeyError:
pass
resource_list = iocage.fetch(list=True, remote=remote, http=True)
if remote:
self.middleware.call_sync('cache.put',
'iocage_remote_releases',
resource_list, 86400)
elif resource == 'branches':
official_branches = requests.get(
'https://api.github.com/repos/freenas/iocage-ix-plugins/'
'branches')
official_branches.raise_for_status()
resource_list = [{
'name': b['name'],
'repo': 'official'
} for b in official_branches.json()]
else:
resource_list = iocage.list(resource)
return resource_list
def fetch(self, job, options):
"""Fetches a release or plugin."""
fetch_output = {'install_notes': []}
release = options.get('release', None)
https = options.pop('https', False)
name = options.pop('name')
jail_name = options.pop('jail_name')
post_install = False
verrors = ValidationErrors()
self.validate_ips(verrors, options)
if verrors:
raise verrors
def progress_callback(content, exception):
msg = content['message'].strip('\r\n')
rel_up = f'* Updating {release} to the latest patch level... '
nonlocal post_install
if name is None:
if 'Downloading : base.txz' in msg and '100%' in msg:
job.set_progress(5, msg)
elif 'Downloading : lib32.txz' in msg and '100%' in msg:
job.set_progress(10, msg)
elif 'Downloading : doc.txz' in msg and '100%' in msg:
job.set_progress(15, msg)
elif 'Downloading : src.txz' in msg and '100%' in msg:
job.set_progress(20, msg)
if 'Extracting: base.txz' in msg:
job.set_progress(25, msg)
elif 'Extracting: lib32.txz' in msg:
job.set_progress(50, msg)
elif 'Extracting: doc.txz' in msg:
job.set_progress(75, msg)
elif 'Extracting: src.txz' in msg:
job.set_progress(90, msg)
elif rel_up in msg:
job.set_progress(95, msg)
else:
job.set_progress(None, msg)
else:
if post_install:
for split_msg in msg.split('\n'):
fetch_output['install_notes'].append(split_msg)
if ' These pkgs will be installed:' in msg:
job.set_progress(50, msg)
elif 'Installing plugin packages:' in msg:
job.set_progress(75, msg)
elif 'Running post_install.sh' in msg:
job.set_progress(90, msg)
# Sets each message going forward as important to the user
post_install = True
else:
job.set_progress(None, msg)
self.check_dataset_existence() # Make sure our datasets exist.
start_msg = f'{release} being fetched'
final_msg = f'{release} fetched'
iocage = ioc.IOCage(callback=progress_callback, silent=False)
if name is not None:
pool = IOCJson().json_get_value('pool')
iocroot = IOCJson(pool).json_get_value('iocroot')
options["plugin_name"] = name
start_msg = 'Starting plugin install'
final_msg = f"Plugin: {name} installed"
elif name is None and https:
if 'https' not in options['server']:
options['server'] = f'https://{options["server"]}'
options["accept"] = True
options['name'] = jail_name
job.set_progress(0, start_msg)
iocage.fetch(**options)
if post_install and name is not None:
plugin_manifest = pathlib.Path(
f'{iocroot}/.plugin_index/{name}.json')
plugin_json = json.loads(plugin_manifest.read_text())
schema_version = plugin_json.get('plugin_schema', '1')
if schema_version.isdigit() and int(schema_version) >= 2:
plugin_output = pathlib.Path(
f'{iocroot}/jails/{name}/root/root/PLUGIN_INFO')
if plugin_output.is_file():
# Otherwise it will be the verbose output from the
# post_install script
fetch_output['install_notes'] = [
x for x in plugin_output.read_text().split('\n') if x
]
# This is to get the admin URL and such
fetch_output['install_notes'] += job.progress[
'description'].split('\n')
job.set_progress(100, final_msg)
return fetch_output
def create_job(self, job, options):
verrors = ValidationErrors()
uuid = options["uuid"]
job.set_progress(0, f'Creating: {uuid}')
try:
self.check_jail_existence(uuid, skip=False)
verrors.add('uuid', f'A jail with name {uuid} already exists')
raise verrors
except CallError:
# A jail does not exist with the provided name, we can create one
# now
verrors = self.common_validation(verrors, options)
if verrors:
raise verrors
job.set_progress(20, 'Initial validation complete')
if not any('resolver' in p for p in options['props']):
dc = self.middleware.call_sync(
'service.query', [('service', '=', 'domaincontroller')])[0]
dc_config = self.middleware.call_sync('domaincontroller.config')
if dc['enable'] and (dc_config['dns_forwarder'] and
dc_config['dns_backend'] == 'SAMBA_INTERNAL'):
options['props'].append(
f'resolver=nameserver {dc_config["dns_forwarder"]}')
iocage = ioc.IOCage(skip_jails=True)
release = options["release"]
template = options.get("template", False)
pkglist = options.get("pkglist", None)
basejail = options["basejail"]
empty = options["empty"]
short = options["short"]
props = options["props"]
pool = IOCJson().json_get_value("pool")
iocroot = IOCJson(pool).json_get_value("iocroot")
if template:
release = template
if (not os.path.isdir(f'{iocroot}/releases/{release}') and not template
and not empty):
job.set_progress(50, f'{release} missing, calling fetch')
self.middleware.call_sync('jail.fetch', {"release": release},
job=True)
err, msg = iocage.create(release,
props,
0,
pkglist,
template=template,
short=short,
_uuid=uuid,
basejail=basejail,
empty=empty)
if err:
raise CallError(msg)
job.set_progress(100, f'Created: {uuid}')
return True
def list_resource(self, job, resource, remote, want_cache, branch):
"""Returns a JSON list of the supplied resource on the host"""
self.check_dataset_existence() # Make sure our datasets exist.
iocage = ioc.IOCage(skip_jails=True)
resource = "base" if resource == "RELEASE" else resource.lower()
if resource == "plugin":
if remote:
if want_cache:
try:
resource_list = self.middleware.call_sync(
'cache.get', 'iocage_remote_plugins')
return resource_list
except KeyError:
pass
resource_list = iocage.fetch(list=True, plugins=True, header=False, branch=branch)
try:
plugins_versions_data = self.middleware.call_sync('cache.get', 'iocage_plugin_versions')
except KeyError:
plugins_versions_data_job = self.middleware.call_sync(
'core.get_jobs',
[['method', '=', 'jail.retrieve_plugin_versions'], ['state', '=', 'RUNNING']]
)
error = None
plugins_versions_data = {}
if plugins_versions_data_job:
try:
plugins_versions_data = self.middleware.call_sync(
'core.job_wait', plugins_versions_data_job[0]['id'], job=True
)
except CallError as e:
error = str(e)
else:
try:
plugins_versions_data = self.middleware.call_sync(
'jail.retrieve_plugin_versions', job=True
)
except Exception as e:
error = e
if error:
# Let's not make the failure fatal
self.middleware.logger.error(f'Retrieving plugins version failed: {error}')
else:
resource_list = iocage.list("all", plugin=True)
pool = IOCJson().json_get_value("pool")
iocroot = IOCJson(pool).json_get_value("iocroot")
index_path = f'{iocroot}/.plugin_index/INDEX'
plugin_jails = {
j['host_hostuuid']: j for j in self.middleware.call_sync(
'jail.query', [['type', 'in', ['plugin', 'pluginv2']]]
)
}
if not pathlib.Path(index_path).is_file():
index_json = None
else:
index_fd = open(index_path, 'r')
index_json = json.load(index_fd)
for index, plugin in enumerate(resource_list):
if remote:
# In case of remote, "plugin" is going to be a dictionary
plugin.update({
k: plugins_versions_data.get(plugin['plugin'], {}).get(k, 'N/A')
for k in ('version', 'revision', 'epoch')
})
else:
# "plugin" is a list which we will convert to a dictionary for readability
plugin_dict = {
k: v if v != '-' else None
for k, v in zip((
'jid', 'name', 'boot', 'state', 'type', 'release', 'ip4', 'ip6', 'template', 'admin_portal'
), plugin)
}
plugin_output = pathlib.Path(
f'{iocroot}/jails/{plugin[1]}/root/root/PLUGIN_INFO'
)
if plugin_output.is_file():
plugin_info = [[
x for x in plugin_output.read_text().split(
'\n') if x
]]
else:
plugin_info = None
plugin_name = plugin_jails[plugin_dict['name']]['plugin_name']
plugin_dict.update({
'plugin_info': plugin_info,
'plugin': plugin_name if plugin_name != 'none' else plugin_dict['name'],
**self.get_local_plugin_version(
plugin_name if plugin_name != 'none' else plugin_dict['name'],
index_json, iocroot, plugin_dict['name']
)
})
resource_list[index] = plugin_dict
if remote:
self.middleware.call_sync(
'cache.put', 'iocage_remote_plugins', resource_list,
86400
)
else:
index_fd.close()
elif resource == "base":
try:
if remote:
resource_list = self.middleware.call_sync(
'cache.get', 'iocage_remote_releases')
return resource_list
except KeyError:
pass
resource_list = iocage.fetch(list=True, remote=remote, http=True)
if remote:
self.middleware.call_sync(
'cache.put', 'iocage_remote_releases', resource_list,
86400
)
elif resource == 'branches':
official_branches = requests.get(
'https://api.github.com/repos/freenas/iocage-ix-plugins/'
'branches'
)
official_branches.raise_for_status()
resource_list = [
{'name': b['name'], 'repo': 'official'}
for b in official_branches.json()
]
else:
resource_list = [
{k: v if v != '-' else None for k, v in zip(('jid', 'name', 'state', 'release', 'ip4'), jail_data)}
for jail_data in iocage.list(resource)
]
return resource_list