def main(self, args):
something_changed = False
for app_cache in self._app_caches(args):
# first of all, set up local cache
mkdir(app_cache.get_cache_dir())
if self._extract_local_archive(app_cache):
something_changed = True
for appcenter_cache in self._appcenter_caches(args):
# download meta files like index.json
mkdir(appcenter_cache.get_cache_dir())
if self._download_supra_files(appcenter_cache):
appcenter_cache.clear_cache()
something_changed = True
for app_cache in self._app_caches(args):
# try it one more time (ucs.ini may have changed)
mkdir(app_cache.get_cache_dir())
if self._extract_local_archive(app_cache):
something_changed = True
# download apps based on meta files
if self._download_apps(app_cache):
app_cache.clear_cache()
something_changed = True
if something_changed:
apps_cache = Apps()
for app in apps_cache.get_all_locally_installed_apps():
newest_app = apps_cache.find_candidate(app) or app
if app < newest_app:
ucr_save({app.ucr_upgrade_key: 'yes'})
self._update_local_files()
def _unregister_app(self, app, args, lo=None, pos=None, delay=False):
if lo is None:
lo, pos = self._get_ldap_connection(args, allow_machine_connection=True)
updates = {}
for key in ucr_keys():
if key.startswith('appcenter/apps/%s/' % app.id):
updates[key] = None
if re.match('ucs/web/overview/entries/[^/]+/%s/' % app.id, key):
updates[key] = None
if re.match('appreport/%s/' % app.id, key):
updates[key] = None
if app.docker and not app.plugin_of:
try:
from univention.appcenter.actions.service import Service
except ImportError:
# univention-appcenter-docker is not installed
pass
else:
try:
init_script = Service.get_init(app)
os.unlink(init_script)
self._call_script('/usr/sbin/update-rc.d', os.path.basename(init_script), 'remove')
except OSError:
pass
ldap_object = get_app_ldap_object(app, lo, pos)
if ldap_object:
self.log('Removing localhost from LDAP object')
ldap_object.remove_localhost()
if not delay:
ucr_save(updates)
self._reload_apache()
return updates
def create(self, hostname, env):
env = {
k: yaml.scalarstring.DoubleQuotedScalarString(v)
for k, v in env.iteritems() if v is not None
}
if self.app.docker_ucr_style_env:
env.update({shell_safe(k).upper(): v for k, v in env.iteritems()})
else:
env = {shell_safe(k).upper(): v for k, v in env.iteritems()}
self._setup_env(env=env)
self._setup_yml(recreate=True, env=env)
ret, out_up = call_process2([
'docker-compose', '-p', self.app.id, 'up', '-d', '--no-build',
'--no-recreate'
],
cwd=self.app.get_compose_dir())
if ret != 0:
raise DockerCouldNotStartContainer(out_up)
self.container = self._get_main_service_container_id()
if self.container is None:
try:
out_ps = ps(only_running=True)
except Exception as e:
out_ps = str(e)
raise DockerCouldNotStartContainer(
'could not find container for service %s! docker-ps: %s docker-compose: %s)'
% (self.app.docker_main_service, out_ps, out_up))
else:
ucr_save({self.app.ucr_container_key: self.container})
return self.container
def _unregister_component(self, app):
if app.without_repository:
self.log('No repository to unregister')
return
updates = self._unregister_component_dict(app)
ucr_save(updates)
return updates
def create(self, hostname, env):
env = {
k: yaml.scalarstring.DoubleQuotedScalarString(v)
for k, v in env.iteritems()
}
env.update({shell_safe(k).upper(): v for k, v in env.iteritems()})
self._setup_yml(recreate=True, env=env)
call_process([
'docker-compose', '-p', self.app.id, 'up', '-d', '--no-build',
'--no-recreate'
],
cwd=self.app.get_compose_dir())
try:
out = ps(only_running=True)
except CalledProcessError:
return False
else:
yml_file = self.app.get_compose_file('docker-compose.yml')
content = yaml.load(open(yml_file),
yaml.RoundTripLoader,
preserve_quotes=True)
docker_image = content['services'][
self.app.docker_main_service]['image']
for line in out.splitlines():
try:
container, image = line.split()[:2]
except ValueError:
pass
else:
if image == docker_image:
ucr_save({self.app.ucr_container_key: container})
self.container = container
return container
def _do_it(self, app, args):
if not app.docker:
return super(Upgrade, self)._do_it(app, args)
mode, detail = self._docker_upgrade_mode(app)
if mode:
self.log('Upgrading %s (%r)' % (mode, detail))
if self._last_mode == (mode, detail):
self.warn('Not again!')
return
if mode == 'packages':
self._upgrade_packages(app, args)
elif mode == 'release':
self._upgrade_release(app, detail)
elif mode == 'app':
self._upgrade_app(app, args)
elif mode == 'image':
self._upgrade_image(app, args)
elif mode == 'docker':
self._upgrade_docker(app, args)
else:
self.fatal('Unable to process %r' % (mode, ))
return
self._last_mode = mode, detail
ucr_save({'appcenter/prudence/docker/%s' % app.id: None})
self._do_it(app, args)
else:
self.log('Nothing to upgrade')
def create(self, hostname, env):
ports = []
for app_id, container_port, host_port, protocol in app_ports_with_protocol():
if app_id == self.app.id:
port_definition = '%d:%d/%s' % (host_port, container_port, protocol)
ports.append(port_definition)
volumes = set(self.app.docker_volumes[:])
for app_volume in [self.app.get_data_dir(), self.app.get_conf_dir()]:
app_volume = '%s:%s' % (app_volume, app_volume)
volumes.add(app_volume)
if self.app.host_certificate_access:
cert_dir = '/etc/univention/ssl/%s.%s' % (ucr_get('hostname'), ucr_get('domainname'))
cert_volume = '%s:%s:ro' % (cert_dir, cert_dir)
volumes.add(cert_volume)
volumes.add('/sys/fs/cgroup:/sys/fs/cgroup:ro') # systemd
if ucr_is_true('appcenter/docker/container/proxy/settings', default=True):
if os.path.isfile('/etc/apt/apt.conf.d/80proxy'):
volumes.add('/etc/apt/apt.conf.d/80proxy:/etc/apt/apt.conf.d/80proxy:ro') # apt proxy
env_file = self.ucr_filter_env_file(env)
command = None
if self.app.docker_script_init:
command = shlex.split(self.app.docker_script_init)
args = shlex.split(ucr_get(self.app.ucr_docker_params_key, ''))
for tmpfs in ("/run", "/run/lock"): # systemd
args.extend(["--tmpfs", tmpfs])
seccomp_profile = "/etc/docker/seccomp-systemd.json"
args.extend(["--security-opt", "seccomp:%s" % seccomp_profile]) # systemd
args.extend(["-e", "container=docker"]) # systemd
container = create(self.image, command, hostname, ports, volumes, env_file, args)
ucr_save({self.app.ucr_container_key: container})
self.container = container
return container
def _upgrade_image(self, app, args):
docker = self._get_docker(app)
self.log('Verifying Docker registry manifest for app image %s' %
docker.image)
docker.verify()
self.log('Pulling Docker image %s' % docker.image)
docker.pull()
self.log('Saving data from old container (%s)' % self.old_app)
Start.call(app=self.old_app)
settings = self._get_configure_settings(self.old_app,
filter_action=False)
settings.update(args.set_vars or {})
args.set_vars = settings
old_docker = self._get_docker(self.old_app)
old_docker.cp_from_container('/etc/machine.secret', app.secret_on_host)
if self._backup_container(self.old_app) is False:
raise UpgradeBackupFailed()
self.log('Removing old container')
if old_docker.container:
old_docker.rm()
self._had_image_upgrade = True
self.log('Setting up new container (%s)' % app)
ucr_save({app.ucr_image_key: None})
old_configure = args.configure
args.configure = False
self._install_new_app(app, args)
self._update_converter_service(app)
args.configure = old_configure
args.set_vars = settings
self._configure(app, args)
self._register_app(app, args)
self._call_join_script(app, args)
self.old_app = app
def _register_app(self, app, args, lo=None, pos=None, delay=False):
if lo is None:
lo, pos = self._get_ldap_connection(args,
allow_machine_connection=True)
updates = {}
self.log('Registering UCR for %s' % app.id)
self.log('Marking %s as installed' % app)
if app.is_installed():
status = ucr_get(app.ucr_status_key, 'installed')
else:
status = 'installed'
ucr_save({
app.ucr_status_key: status,
app.ucr_version_key: app.version,
app.ucr_ucs_version_key: app.get_ucs_version()
})
self._register_ports(app)
updates.update(self._register_docker_variables(app))
updates.update(self._register_app_report_variables(app))
# Register app in LDAP (cn=...,cn=apps,cn=univention)
ldap_object = get_app_ldap_object(app, lo, pos, or_create=True)
self.log('Adding localhost to LDAP object')
ldap_object.add_localhost()
updates.update(self._register_overview_variables(app))
if not delay:
ucr_save(updates)
self._reload_apache()
return updates
def _register_host(self, app, args):
if not app.docker:
self.debug('App is not docker. Skip registering host')
return None, None
hostdn = ucr_get(app.ucr_hostdn_key)
lo, pos = self._get_ldap_connection(args)
if hostdn:
if lo.get(hostdn):
self.log('Already found %s as a host for %s. Better do nothing...' % (hostdn, app.id))
return hostdn, None
else:
self.warn('%s should be the host for %s. But it was not found in LDAP. Creating a new one' % (hostdn, app.id))
# quasi unique hostname; make sure it does not exceed 14 chars
# 5 chars of appid + '-' + 8 digits of Epoch
hostname = '%s-%s' % (app.id[:5], str(int((time.time() * 1000000)))[-10:-2])
password = generate_password()
self.log('Registering the container host %s for %s' % (hostname, app.id))
if app.docker_server_role == 'memberserver':
base = 'cn=memberserver,cn=computers,%s' % ucr_get('ldap/base')
else:
base = 'cn=dc,cn=computers,%s' % ucr_get('ldap/base')
while base and not lo.get(base):
base = dn2str(str2dn(base)[1:])
pos.setDn(base)
domain = ucr_get('domainname')
description = '%s (%s)' % (app.name, app.version)
policies = ['cn=app-release-update,cn=policies,%s' % ucr_get('ldap/base'), 'cn=app-update-schedule,cn=policies,%s' % ucr_get('ldap/base')]
obj = create_object_if_not_exists('computers/%s' % app.docker_server_role, lo, pos, name=hostname, description=description, domain=domain, password=password, objectFlag='docker', policies=policies)
ucr_save({app.ucr_hostdn_key: obj.dn})
return obj.dn, password
def _unregister_host(self, app, args):
hostdn = ucr_get(app.ucr_hostdn_key)
if not hostdn:
self.log('No hostdn for %s found. Nothing to remove' % app.id)
return
lo, pos = self._get_ldap_connection(args)
remove_object_if_exists('computers/%s' % app.docker_server_role, lo, pos, hostdn)
ucr_save({app.ucr_hostdn_key: None})
def main(self, args):
if args.revert:
appcenter_server = 'appcenter.software-univention.de'
ucr_save({'repository/app_center/server': appcenter_server, 'update/secure_apt': 'yes', 'appcenter/index/verify': 'yes'})
else:
ucr_save({'repository/app_center/server': args.appcenter_host, 'update/secure_apt': 'no', 'appcenter/index/verify': 'no'})
update = get_action('update')
update.call()
def enable_docker(self):
if self._test_for_docker_service():
ucr_save({'appcenter/docker': 'enabled'})
else:
raise umcm.UMC_Error(
_('Unable to start the docker service!') + ' ' +
_('Make sure docker.io is installed, try starting the service with "service docker start".'
))
def main(self, args):
prev_unmaintained = ucr_get('repository/online/unmaintained', 'no')
ucr_save({'repository/online/unmaintained': 'true'})
ret_code = self._subprocess(
['univention-install', '-y',
'ucs-test-selenium-runner']).returncode
ucr_save({'repository/online/unmaintained': prev_unmaintained})
return ret_code != 0
def _set_autostart(self, app, autostart):
if not app.docker:
return
if autostart is None:
return
if autostart not in ['yes', 'manually', 'no']:
self.warn('Autostart must be one of yes, manually, no. Not setting to %r' % autostart)
return
ucr_save({'%s/autostart' % app.id: autostart})
def _register_installed_apps_in_ucr(self):
installed_codes = []
for app in Apps().get_all_apps():
if app.is_installed():
installed_codes.append(app.code)
with catch_stdout(self.logger):
ucr_save({
'appcenter/installed': '-'.join(installed_codes),
'repository/app_center/installed': '-'.join(installed_codes), # to be deprecated
})
def __exit__(self, exc_type, exc_val, exc_tb):
if self.revert == 'configure':
config = dict((key, None) for key in self.settings)
configure = get_action('configure')
configure.call(app=self.app, set_vars=config, run_script='no')
for setting in self.settings:
assert ucr_get(setting) is None
elif self.revert == 'ucr':
config = dict((key, None) for key in self.settings)
ucr_save(config)
def _register_component_for_apps(self, apps, args):
if not self._shall_register(args, 'component'):
return
updates = {}
for app in apps:
if self._do_register(app, args):
updates.update(self._register_component(app, delay=True))
else:
updates.update(self._unregister_component_dict(app))
with catch_stdout(self.logger):
ucr_save(updates)
def _register_app_for_apps(self, apps, args):
if not self._shall_register(args, 'app'):
return
updates = {}
if apps:
lo, pos = self._get_ldap_connection(args, allow_machine_connection=True)
for app in apps:
if self._do_register(app, args):
updates.update(self._register_app(app, args, lo, pos, delay=True))
else:
updates.update(self._unregister_app(app, args, lo, pos, delay=True))
ucr_save(updates)
def main(self, args):
chromium_version = "71.0.3578.80-1~deb9u1" # Bug #48856
prev_unmaintained = ucr_get('repository/online/unmaintained', 'no')
ucr_save({'repository/online/unmaintained': 'true'})
ret_code = self._subprocess([
'univention-install', '-y', 'python-pip', 'ucs-test', 'xvfb',
'chromium=%s' % chromium_version,
'chromium-driver=%s' % chromium_version, 'python-xvfbwrapper'
]).returncode
ret_code = self._subprocess(['pip', 'install', 'selenium==3.6.0'
]).returncode or ret_code
ucr_save({'repository/online/unmaintained': prev_unmaintained})
return ret_code != 0
def test_status_and_file_setting(installed_component_app):
content = '''[test/setting3]
Type = Status
Description = My Description 3
[test/setting4]
Type = File
Filename = /tmp/settingdir/setting4.test
Description = My Description 4
[test/setting4/2]
Type = File
Filename = /tmp/%s
Description = My Description 4.2
''' % (300 * 'X')
app, settings = fresh_settings(content, installed_component_app, 3)
status_setting, file_setting, file_setting2 = settings
assert repr(status_setting) == "StatusSetting(name='test/setting3')"
assert repr(file_setting) == "FileSetting(name='test/setting4')"
assert repr(file_setting2) == "FileSetting(name='test/setting4/2')"
try:
with Configuring(app, revert='ucr') as config:
ucr_save({status_setting.name: 'My Status'})
assert status_setting.get_value(app) == 'My Status'
assert not os.path.exists(file_setting.filename)
assert file_setting.get_value(app) is None
config.set({
status_setting.name: 'My new Status',
file_setting.name: 'File content'
})
assert status_setting.get_value(app) == 'My Status'
assert os.path.exists(file_setting.filename)
assert open(file_setting.filename, 'rb').read() == 'File content'
assert file_setting.get_value(app) == 'File content'
config.set({file_setting.name: None})
assert not os.path.exists(file_setting.filename)
assert file_setting.get_value(app) is None
assert file_setting2.get_value(app) is None
config.set({file_setting2.name: 'File content 2'})
assert file_setting2.get_value(app) is None
finally:
try:
os.unlink(file_setting.filename)
except EnvironmentError:
pass
def _copy_unmaintained_packages(self, repo_dir, args):
unmaintained_ucr_var = 'repository/online/unmaintained'
old_unmaintained = ucr_get(unmaintained_ucr_var)
ucr_save({unmaintained_ucr_var: 'yes'})
try:
old_debs = glob('*.*deb')
subprocess.call(['apt-get', 'update'])
subprocess.call(['apt-get', 'download'] + args.unmaintained)
new_debs = glob('*.*deb')
for deb in new_debs:
if deb not in old_debs:
args.packages = args.packages or []
args.packages.append(deb)
finally:
ucr_save({unmaintained_ucr_var: old_unmaintained})
def _do_it(self, app, args):
self._remove_app(app, args)
self.percentage = 45
self._unregister_app(app, args)
self.percentage = 50
self._unregister_attributes(app, args)
self.percentage = 60
if self._unregister_component(app):
self._apt_get_update()
self.percentage = 70
self._unregister_files(app)
self.percentage = 80
self._call_unjoin_script(app, args)
if not app.docker:
ucr_save({'appcenter/prudence/docker/%s' % app.id: 'yes'})
def main(self, args):
from univention.appcenter.actions import get_action
if args.update:
get_action('update').call()
apps = args.app
if not apps:
apps = Apps().get_all_locally_installed_apps()
for app in apps:
self.debug('Checking %s' % app)
if not app.is_installed():
continue
upgrade_available = self._check_for_upgrades(app)
if upgrade_available is True:
ucr_save({app.ucr_upgrade_key: 'yes'})
elif upgrade_available is False:
ucr_save({app.ucr_upgrade_key: None})
return any(ucr_is_true(app.ucr_upgrade_key) for app in apps)
def _upgrade_image(self, app, args):
docker = self._get_docker(app)
if args.pull_image:
self.log('Verifying Docker registry manifest for app image %s' %
docker.image)
docker.verify()
self.log('Pulling Docker image %s' % docker.image)
docker.backup_run_file()
docker.pull()
else:
docker.setup_docker_files()
self.log('Saving data from old container (%s)' % self.old_app)
Start.call(app=self.old_app)
settings = self._get_configure_settings(self.old_app,
filter_action=False)
settings.update(args.set_vars or {})
args.set_vars = settings
old_docker = self._get_docker(self.old_app)
old_docker.cp_from_container('/etc/machine.secret', app.secret_on_host)
if self._backup_container(self.old_app) is False:
raise UpgradeBackupFailed()
self.log('Removing old container')
if old_docker.container:
old_docker.rm()
self._had_image_upgrade = True
self.log('Setting up new container (%s)' % app)
ucr_save({app.ucr_image_key: None})
old_configure = args.configure
args.configure = False
self._install_new_app(app, args)
self._update_converter_service(app)
args.configure = old_configure
args.set_vars = settings
self._configure(app, args)
self._register_app(app, args)
self._call_join_script(app, args)
if args.remove_image:
self.log('Trying to remove old image')
try:
if old_docker.rmi() != 0:
self.log(
'Failed to remove old image. Continuing anyway...')
except Exception as exc:
self.warn('Error while removing old image: %s' % exc)
self.old_app = app
def _do_it(self, app, args):
if self._install_only_master_packages(args):
self._install_master_packages(app, unregister_if_uninstalled=True)
else:
self._register_files(app)
self.percentage = 5
self._register_app(app, args)
self.percentage = 10
self._register_database(app)
self.percentage = 15
self._register_attributes(app, args)
self.percentage = 25
if self._install_app(app, args):
self.percentage = 80
self._call_join_script(app, args)
ucr_save({'appcenter/prudence/docker/%s' % app.id: 'yes'})
else:
raise Abort('Failed to install the App')
def _register_component(self, app, delay=False):
if app.docker and not container_mode():
self.log('Component needs to be registered in the container')
return {}
if app.without_repository:
self.log('No repository to register')
return {}
updates = {}
self.log('Registering component for %s' % app.id)
for _app in Apps().get_all_apps_with_id(app.id):
if _app == app:
updates.update(self._register_component_dict(_app))
else:
updates.update(self._unregister_component_dict(_app))
if not delay:
with catch_stdout(self.logger):
ucr_save(updates)
return updates
def _do_it(self, app, args):
self._unregister_listener(app)
self.percentage = 5
if not self._remove_app(app, args):
raise RemoveFailed()
self.percentage = 45
self._unregister_app(app, args)
self.percentage = 55
self._unregister_attributes(app, args)
self.percentage = 60
if self._unregister_component(app):
update_packages()
self.percentage = 70
self._unregister_files(app)
self.percentage = 80
self._call_unjoin_script(app, args)
if not app.docker:
ucr_save({'appcenter/prudence/docker/%s' % app.id: 'yes'})
def create(self, hostname, env):
env = {
k: yaml.scalarstring.DoubleQuotedScalarString(v)
for k, v in env.iteritems()
}
if self.app.docker_ucr_style_env:
env.update({shell_safe(k).upper(): v for k, v in env.iteritems()})
else:
env = {shell_safe(k).upper(): v for k, v in env.iteritems()}
self._setup_yml(recreate=True, env=env)
ret, out_up = call_process2([
'docker-compose', '-p', self.app.id, 'up', '-d', '--no-build',
'--no-recreate'
],
cwd=self.app.get_compose_dir())
if ret != 0:
raise DockerCouldNotStartContainer(out_up)
try:
out_ps = ps(only_running=True)
except CalledProcessError:
out_ps = str()
else:
yml_file = self.app.get_compose_file('docker-compose.yml')
content = yaml.load(open(yml_file),
yaml.RoundTripLoader,
preserve_quotes=True)
docker_image = content['services'][
self.app.docker_main_service]['image']
for line in out_ps.splitlines():
try:
container, image = line.split()[:2]
except ValueError:
pass
else:
if image == docker_image:
ucr_save({self.app.ucr_container_key: container})
self.container = container
return container
if self.container is None:
raise DockerCouldNotStartContainer(
'could not find container for %s (image: %s) in docker-ps %s (docker-compose: %s)'
% (self.app.docker_main_service, docker_image, out_ps, out_up))
def _do_it(self, app, args):
if self._install_only_master_packages(args):
self._install_master_packages(app, unregister_if_uninstalled=True)
else:
self._register_files(app)
self.percentage = 5
self._register_app(app, args)
self.percentage = 10
self._register_database(app)
self.percentage = 15
self._register_attributes(app, args)
self.percentage = 25
if self._install_app(app, args):
self._configure(app, args)
self._update_certificates(app, args)
self.percentage = 80
self._call_join_script(app, args)
self._register_listener(app)
ucr_save({'appcenter/prudence/docker/%s' % app.id: 'yes'})
else:
raise InstallFailed()