def query(self, quick=False):
query_cache_file = '/var/cache/univention-appcenter/umc-query.json'
if quick:
try:
with open(query_cache_file) as fd:
return json.load(fd)
except (EnvironmentError, ValueError) as exc:
MODULE.error('Error returning cached query: %s' % exc)
return []
self.update_applications()
self.ucr.load()
reload_package_manager()
list_apps = get_action('list')
domain = get_action('domain')
apps = list_apps.get_apps()
if self.ucr.is_true('appcenter/docker', True):
if not self._test_for_docker_service():
raise umcm.UMC_Error(
_('The docker service is not running! The App Center will not work properly.'
) + ' ' +
_('Make sure docker.io is installed, try starting the service with "service docker start".'
))
info = domain.to_dict(apps)
with open(query_cache_file, 'w') as fd:
json.dump(info, fd)
return info
def test_string_setting_docker(installed_apache_docker_app):
content = '''[test/setting]
Type = String
Description = My Description
InitialValue = Default: @%@ldap/base@%@
Scope = inside, outside
'''
app, settings = fresh_settings(content, installed_apache_docker_app, 1)
setting, = settings
assert repr(setting) == "StringSetting(name='test/setting')"
assert setting.is_inside(app) is True
assert setting.is_outside(app) is True
assert setting.get_initial_value(
app) == 'Default: %s' % ucr_get('ldap/base')
assert setting.get_value(app) is None
with Configuring(app, revert='ucr') as config:
config.set({setting.name: 'My value'})
assert setting.get_value(app) == 'My value'
assert ucr_get(setting.name) == 'My value'
assert docker_shell(app,
'grep "test/setting: " /etc/univention/base.conf'
) == 'test/setting: My value\n'
stop = get_action('stop')
stop.call(app=app)
config.set({setting.name: 'My new value'})
start = get_action('start')
start.call(app=app)
assert ucr_get(setting.name) == 'My new value'
def main(self, args):
meta_inf_dir = os.path.join(args.path, 'meta-inf', args.ucs_version)
repo_dir = os.path.join(args.path, 'univention-repository', args.ucs_version)
if args.revert:
try:
shutil.rmtree(os.path.dirname(meta_inf_dir))
except OSError as exc:
self.warn(exc)
try:
shutil.rmtree(os.path.dirname(repo_dir))
except OSError as exc:
self.warn(exc)
use_test_appcenter = get_action('dev-use-test-appcenter')
use_test_appcenter.call_safe(revert=True)
else:
mkdir(meta_inf_dir)
mkdir(os.path.join(repo_dir, 'maintained', 'component'))
for supra_file in ['categories.ini', 'rating.ini', 'license_types.ini', 'ucs.ini']:
with open(os.path.join(meta_inf_dir, '..', supra_file), 'wb') as f:
categories = urlopen('%s/meta-inf/%s' % (default_server(), supra_file)).read()
f.write(categories)
server = 'http://%s' % args.appcenter_host
use_test_appcenter = get_action('dev-use-test-appcenter')
use_test_appcenter.call_safe(appcenter_host=server)
DevRegenerateMetaInf.call_safe(ucs_version=args.ucs_version, path=args.path, appcenter_host=server)
self.log('Local App Center server is set up at %s.' % server)
self.log('If this server should serve as an App Center server for other computers in the UCS domain, the following command has to be executed on each computer:')
self.log(' univention-app dev-use-test-appcenter --appcenter-host="%s"' % server)
def query(self, quick=False):
if not quick:
self.update_applications()
self.ucr.load()
reload_package_manager()
list_apps = get_action('list')
domain = get_action('domain')
apps = list_apps.get_apps()
if self.ucr.is_true('appcenter/docker', True):
if not self._test_for_docker_service():
raise umcm.UMC_Error(_('The docker service is not running! The App Center will not work properly.') + ' ' + _('Make sure docker.io is installed, try starting the service with "service docker start".'))
info = domain.to_dict(apps)
if quick:
ret = []
for app in info:
if app is None:
ret.append(None)
else:
short_info = {}
for attr in ['id', 'name', 'vendor', 'maintainer', 'description', 'long_description', 'app_categories', 'end_of_life', 'update_available', 'logo_name', 'is_installed_anywhere', 'is_installed', 'installations', 'rating', 'vote_for_app', 'license', 'license_description', 'candidate_needs_install_permissions']:
short_info[attr] = app[attr]
ret.append(short_info)
return ret
else:
return info
def _upgrade_docker(self, app, args):
install = get_action('install')()
action_args = install._build_namespace(
_namespace=args,
app=app,
set_vars=self._get_configure_settings(self.old_app,
filter_action=False),
send_info=False,
skip_checks=['must_not_be_installed'])
if install.call_with_namespace(action_args):
app_cache = Apps()
for _app in app_cache.get_all_apps():
if _app.plugin_of == app.id and _app.is_installed():
_app = app_cache.find(_app.id, latest=True)
if _app.docker:
_old_app = self.old_app
self._upgrade_docker(_app, args)
self.old_app = _old_app
remove = get_action('remove')()
action_args = remove._build_namespace(
_namespace=args,
app=self.old_app,
send_info=False,
skip_checks=['must_not_be_depended_on'])
remove._remove_app(self.old_app, action_args)
if remove._unregister_component(self.old_app):
update_packages()
self._call_join_script(
app, args
) # run again in case remove() called an installed unjoin script
self.old_app = app
def local_appcenter():
setup_appcenter = get_action('dev-setup-local-appcenter')
setup_appcenter.call()
yield
test_appcenter = get_action('dev-use-test-appcenter')
test_appcenter.call(revert=True)
rmtree('/var/www/meta-inf')
rmtree('/var/www/univention-repository')
def init(self):
os.umask(
0o022
) # umc umask is too restrictive for app center as it creates a lot of files in docker containers
self.ucr = ucr_instance()
self.update_applications_done = False
install_opener(self.ucr)
self._is_working = False
try:
self.package_manager = PackageManager(
info_handler=MODULE.process,
step_handler=None,
error_handler=MODULE.warn,
lock=False,
)
except SystemError as exc:
MODULE.error(str(exc))
raise umcm.UMC_Error(str(exc), status=500)
self.package_manager.set_finished(
) # currently not working. accepting new tasks
get_package_manager._package_manager = self.package_manager
# build cache
_update_modules()
get_action('list').get_apps()
# not initialize here: error prone due to network errors and also kinda slow
self._uu = None
self._cm = None
# in order to set the correct locale
locale.setlocale(locale.LC_ALL, str(self.locale))
try:
log_to_logfile()
except IOError:
pass
# connect univention.appcenter.log to the progress-method
handler = ProgressInfoHandler(self.package_manager)
handler.setLevel(logging.INFO)
get_base_logger().addHandler(handler)
percentage = ProgressPercentageHandler(self.package_manager)
percentage.setLevel(logging.DEBUG)
get_base_logger().getChild('actions.install.progress').addHandler(
percentage)
get_base_logger().getChild('actions.upgrade.progress').addHandler(
percentage)
get_base_logger().getChild('actions.remove.progress').addHandler(
percentage)
def get(self, application):
list_apps = get_action('list')
domain = get_action('domain')
apps = list_apps.get_apps()
for app in apps:
if app.id == application:
break
else:
app = None
if app is None:
raise umcm.UMC_Error(_('Could not find an application for %s') % (application,))
return domain.to_dict([app])[0]
def _revert(self, app, args):
if self._had_image_upgrade:
try:
remove = get_action('remove')
install = get_action('install')
password = self._get_password(args, ask=False)
remove.call(app=app, noninteractive=args.noninteractive, username=args.username, password=password, send_info=False, skip_checks=[], backup=False)
install.call(app=self.old_app, noninteractive=args.noninteractive, username=args.username, password=password, send_info=False, skip_checks=[])
except Exception:
pass
else:
Start.call_safe(app=self.old_app)
def _update_apps(self, args):
self.log(
'Updating all installed Apps to use the new App Center server')
logfile_logger = get_logfile_logger('dev-use-test-appcenter')
for app in Apps().get_all_locally_installed_apps():
self.log('Updating %s' % app)
register = get_action('register')
# we should only use ['component'] in container_mode()
# and None otherwise, because it is more correct. however,
# this would require credentials (for potential schema updates etc)
register.call(apps=[app], register_task=['component'])
if app.docker and not container_mode():
try:
from univention.appcenter.docker import Docker
except ImportError:
# should not happen
self.log('univention-appcenter-docker is not installed')
continue
start = get_action('start')
start.call(app=app)
docker = Docker(app, self.logger)
self.log('Updating container... (checking for appbox)')
if docker.execute('which', 'univention-app').returncode == 0:
self.log(
'... setting the new App Center inside the container')
returncode = docker.execute(
'univention-install',
'-y',
'univention-appcenter-dev',
_logger=logfile_logger).returncode
if returncode != 0:
self.warn(
'univention-install univention-appcenter-dev failed!'
)
if args.revert:
returncode = docker.execute(
'univention-app',
'dev-use-test-appcenter',
'--revert',
_logger=logfile_logger).returncode
else:
returncode = docker.execute(
'univention-app',
'dev-use-test-appcenter',
'--appcenter-host',
args.appcenter_host,
_logger=logfile_logger).returncode
if returncode != 0:
self.fatal(
'univention-app dev-use-test-appcenter failed!')
else:
self.log('... nothing to do here')
def test_password_setting_docker(installed_apache_docker_app):
content = '''[test/setting5]
Type = Password
[test/setting6]
Type = PasswordFile
Filename = /tmp/settingdir/setting6.password
'''
app, settings = fresh_settings(content, installed_apache_docker_app, 2)
password_setting, password_file_setting = settings
assert repr(password_setting) == "PasswordSetting(name='test/setting5')"
assert repr(
password_file_setting) == "PasswordFileSetting(name='test/setting6')"
assert password_setting.should_go_into_image_configuration(app) is False
assert password_file_setting.should_go_into_image_configuration(
app) is False
password_file = Docker(app).path(password_file_setting.filename)
assert password_setting.is_inside(app) is True
assert password_file_setting.is_inside(app) is True
assert not os.path.exists(password_file)
with Configuring(app, revert='ucr') as config:
config.set({
password_setting.name: 'MyPassword',
password_file_setting.name: 'FilePassword'
})
assert password_setting.get_value(app) == 'MyPassword'
assert os.path.exists(password_file)
assert open(password_file, 'rb').read() == 'FilePassword'
assert stat.S_IMODE(os.stat(password_file).st_mode) == 0600
stop = get_action('stop')
stop.call(app=app)
config.set({
password_setting.name: 'MyNewPassword2',
password_file_setting.name: 'NewFilePassword2'
})
assert password_setting.get_value(app) is None
assert password_file_setting.get_value(app) is None
start = get_action('start')
start.call(app=app)
assert password_setting.get_value(app) == 'MyPassword'
assert open(password_file, 'rb').read() == 'FilePassword'
def install_app(app):
username = re.match('uid=([^,]*),.*',
ucr_get('tests/domainadmin/account')).groups()[0]
install = get_action('install')
install.call(app=[app],
username=username,
password=ucr_get('tests/domainadmin/pwd'),
noninteractive=True)
yield app
remove = get_action('remove')
remove.call(app=[app],
username=username,
password=ucr_get('tests/domainadmin/pwd'),
noninteractive=True)
def to_dict(cls, app):
ret = super(Get, cls).to_dict(app)
configure = get_action('configure')
ret['config'] = configure.list_config(app)
ret['is_running'] = app_is_running(app)
ret['autostart'] = ucr_get('%s/autostart' % app.id, 'yes')
return ret
def install_app(app, set_vars=None):
username = re.match('uid=([^,]*),.*',
ucr_get('tests/domainadmin/account')).groups()[0]
install = get_action('install')
subprocess.run(['apt-get', 'update'], check=True)
install.call(app=[app],
username=username,
password=ucr_get('tests/domainadmin/pwd'),
noninteractive=True,
set_vars=set_vars)
yield app
remove = get_action('remove')
remove.call(app=[app],
username=username,
password=ucr_get('tests/domainadmin/pwd'),
noninteractive=True)
def verify(app, image):
from univention.appcenter.actions import get_action
update = get_action('update')()
appinfo = update.get_app_info(app)
if not appinfo:
_logger.error('Warning: Cannot check DockerImage checksum because app is not in index.json: %s' % app.id)
return # Nothing we can do here, this is mainly for ucs-test apps
try:
appfileinfo = appinfo['ini']
dockerimageinfo = appfileinfo['DockerImageManifestV2S1']
appcenter_sha256sum = dockerimageinfo['sha256']
docker_image_manifest_url = dockerimageinfo['url']
except KeyError:
_logger.error('Error looking up DockerImage checksum for %s from index.json' % app.id)
return # Nothing we can do here, this is the case of ISV Docker repos
import requests
https_request_auth = requests.auth.HTTPBasicAuth(DOCKER_READ_USER_CRED['username'], DOCKER_READ_USER_CRED['password'])
https_request_answer = requests.get(docker_image_manifest_url, auth=https_request_auth)
if not https_request_answer.ok:
raise DockerImageVerificationFailedRegistryContact(app.id, docker_image_manifest_url)
docker_image_manifest_hash = get_sha256(https_request_answer.content)
# compare with docker registry
if appcenter_sha256sum != docker_image_manifest_hash:
raise DockerImageVerificationFailedChecksum(app.id, docker_image_manifest_url)
def _remote_appcenter(self, host, function=None):
if host is None:
raise ValueError('Cannot connect to None')
if not host.endswith('.%s' % self.ucr.get('domainname')):
raise ValueError('Only connect to FQDNs within the domain')
info = get_action('info')
opts = {'version': info.get_ucs_version()}
if function is not None:
opts['function'] = function
try:
client = Client(host, self.username, self.password)
response = client.umc_command('appcenter/version2', opts)
except (HTTPError) as exc:
raise umcm.UMC_Error(
_('Problems connecting to {0} ({1}). Please update {0}!').
format(host, exc.message))
except (ConnectionError, Exception) as exc:
raise umcm.UMC_Error(
_('Problems connecting to {} ({}).').format(host, str(exc)))
err_msg = _(
'The App Center version of the this host ({}) is not compatible with the version of {} ({})'
).format(opts['version'], host, response.result.get('version'))
# i guess this is kind of bad
if response.status != 200:
raise umcm.UMC_Error(err_msg)
# remote says he is not compatible
if response.result.get('compatible', True) is False:
raise umcm.UMC_Error(err_msg)
# i'm not compatible
if not info.is_compatible(response.result.get('version')):
raise umcm.UMC_Error(err_msg)
return client
def main(self, args):
meta_inf_dir = os.path.join(args.path, 'meta-inf', args.ucs_version)
repo_dir = os.path.join(args.path, 'univention-repository', args.ucs_version, 'maintained', 'component')
self.generate_index_json(meta_inf_dir, repo_dir, args.ucs_version, args.appcenter_host)
if args.ucs_version == ucr_get('version/version'):
update = get_action('update')
update.call_safe()
def main(self, args):
from univention.appcenter.actions import get_action
if args.update:
get_action('update').call()
apps = args.app
if not apps:
apps = Apps().get_all_locally_installed_apps()
for app in apps:
self.debug('Checking %s' % app)
if not app.is_installed():
continue
upgrade_available = self._check_for_upgrades(app)
if upgrade_available is True:
ucr_save({app.ucr_upgrade_key: 'yes'})
elif upgrade_available is False:
ucr_save({app.ucr_upgrade_key: None})
return any(ucr_is_true(app.ucr_upgrade_key) for app in apps)
def get_apps(no_cache=False):
if no_cache:
AppCache().clear_cache()
get = get_action('get')
return [
get.to_dict(app) for app in Apps().get_all_apps()
if app.is_ucs_component()
]
def version(self, version=None):
info = get_action('info')
ret = info.get_compatibility()
if not info.is_compatible(version):
raise umcm.UMC_Error(
'The App Center version of the requesting host is not compatible with the version of %s (%s)'
% (get_local_fqdn(), ret))
return ret
def main(self, args):
if args.revert:
appcenter_server = 'appcenter.software-univention.de'
ucr_save({'repository/app_center/server': appcenter_server, 'update/secure_apt': 'yes', 'appcenter/index/verify': 'yes'})
else:
ucr_save({'repository/app_center/server': args.appcenter_host, 'update/secure_apt': 'no', 'appcenter/index/verify': 'no'})
update = get_action('update')
update.call()
def _configure(self, app, args, run_script=None):
if not args.configure:
return
if run_script is None:
run_script = self.get_action_name()
configure = get_action('configure')
set_vars = (args.set_vars or {}).copy()
set_vars.update(self._get_configure_settings(app))
configure.call(app=app, run_script=run_script, set_vars=set_vars)
def do_it(self, args):
app = args.app
status = 200
try:
action = self.get_action_name()
self.log('Going to %s %s (%s)' % (action, app.name, app.version))
errors, warnings = app.check(action)
can_continue = self._handle_errors(app, args, errors, True)
can_continue = self._handle_errors(
app, args, warnings, fatal=not can_continue) and can_continue
if not can_continue or not self._call_prescript(app, args):
status = 0
self.fatal('Unable to %s %s. Aborting...' % (action, app.id))
else:
try:
self._show_license(app, args)
self._show_pre_readme(app, args)
try:
self._do_it(app, args)
except (Abort, KeyboardInterrupt) as exc:
msg = str(exc)
if msg:
self.warn(msg)
self.warn('Aborting...')
status = 401
except Exception:
status = 500
raise
else:
try:
self._show_post_readme(app, args)
except Abort:
pass
except Abort:
self.warn('Cancelled...')
status = 0
except AppCenterError as exc:
status = exc.code
raise
except Exception:
raise
else:
return status == 200
finally:
if status == 0:
pass
else:
if status != 200:
self._revert(app, args)
if args.send_info:
try:
self._send_information(app, status)
except NetworkError:
self.log('Ignoring this error...')
self._register_installed_apps_in_ucr()
upgrade_search = get_action('upgrade-search')
upgrade_search.call_safe(app=[app], update=False)
def __exit__(self, exc_type, exc_val, exc_tb):
if self.revert == 'configure':
config = dict((key, None) for key in self.settings)
configure = get_action('configure')
configure.call(app=self.app, set_vars=config, run_script='no')
for setting in self.settings:
assert ucr_get(setting) is None
elif self.revert == 'ucr':
config = dict((key, None) for key in self.settings)
ucr_save(config)
def update_applications(self):
if self.ucr.is_true('appcenter/umc/update/always', True):
update = get_action('update')
try:
update.call()
except NetworkError as err:
raise umcm.UMC_Error(str(err))
except Abort:
pass
self.update_applications_done = True
def get_by_component_id(self, component_id):
domain = get_action('domain')
if isinstance(component_id, list):
requested_apps = [Apps().find_by_component_id(cid) for cid in component_id]
return domain.to_dict(requested_apps)
else:
app = Apps().find_by_component_id(component_id)
if app:
return domain.to_dict([app])[0]
else:
raise umcm.UMC_Error(_('Could not find an application for %s') % component_id)
def invoke_docker(self, function, app, force, values, progress):
for setting in app.get_settings():
if isinstance(setting, FileSetting) and not isinstance(setting, PasswordFileSetting):
if values.get(setting.name):
values[setting.name] = values[setting.name].decode('base64')
progress.title = _('%s: Running tests') % app.name
serious_problems = False
if function == 'upgrade':
_original_app = app
app = Apps().find_candidate(app)
if app is None:
# Bug #44384: Under mysterious circumstances, app may be None after the .find_candidate()
# This may happen in global App Center when the system the user is logged in has different ini files
# than the system the App shall be upgraded on. E.g., in mixed appcenter / appcenter-test environments
app = _original_app
errors, warnings = {'must_have_candidate': False}, {}
else:
errors, warnings = app.check(function)
can_continue = force # "dry_run"
if errors:
MODULE.process('Cannot %s %s: %r' % (function, app.id, errors))
serious_problems = True
can_continue = False
if warnings:
MODULE.process('Warning trying to %s %s: %r' % (function, app.id, warnings))
result = {
'serious_problems': serious_problems,
'invokation_forbidden_details': errors,
'invokation_warning_details': warnings,
'can_continue': can_continue,
'software_changes_computed': False,
}
if can_continue:
kwargs = {'noninteractive': True, 'skip_checks': ['shall_have_enough_ram', 'shall_only_be_installed_in_ad_env_with_password_service', 'must_not_have_concurrent_operation'], 'set_vars': values}
if function == 'install':
progress.title = _('Installing %s') % (app.name,)
elif function == 'uninstall':
progress.title = _('Uninstalling %s') % (app.name,)
elif function == 'upgrade':
progress.title = _('Upgrading %s') % (app.name,)
action = get_action(function)
handler = UMCProgressHandler(progress)
handler.setLevel(logging.INFO)
action.logger.addHandler(handler)
try:
result['success'] = action.call(app=app, username=self.username, password=self.password, **kwargs)
except AppCenterError as exc:
raise umcm.UMC_Error(str(exc), result=dict(
display_feedback=True,
title='%s %s' % (exc.title, exc.info)))
finally:
action.logger.removeHandler(handler)
return result
def _revert(self, app, args):
try:
password = self._get_password(args, ask=False)
remove = get_action('remove')
remove.call(app=app,
noninteractive=args.noninteractive,
username=args.username,
password=password,
send_info=False,
skip_checks=[],
backup=False)
except Exception:
pass
def configure(self, progress, app, values, autostart=None):
for setting in app.get_settings():
if isinstance(setting, FileSetting) and not isinstance(setting, PasswordFileSetting):
if values.get(setting.name):
values[setting.name] = values[setting.name].decode('base64')
configure = get_action('configure')
handler = UMCProgressHandler(progress)
handler.setLevel(logging.INFO)
configure.logger.addHandler(handler)
try:
return configure.call(app=app, set_vars=values, autostart=autostart)
finally:
configure.logger.removeHandler(handler)
def get_settings(content, app):
fname = '/tmp/app.settings'
with open(fname, 'wb') as fd:
fd.write(content)
populate = get_action('dev-populate-appcenter')
populate.call(component_id=app.component_id,
ini=app.get_ini_file(),
settings='/tmp/app.settings')
app = Apps().find(app.id)
settings = app.get_settings()
return settings
