class Instance(Base):
def init(self):
"""Initialize the module with some values"""
super(Instance, self).init()
self.data = [
int(x)
for x in ucr.get('some/examle/ucr/variable', '1,2,3').split(',')
]
def query(self, request):
"""get all values of self.data"""
self.finished(request.id, self.data)
@sanitize(item=IntegerSanitizer(required=True))
def get(self, request):
"""get a specific item of self.data"""
try:
item = self.data[request.options['item']]
except IndexError:
MODULE.error('A invalid item was accessed.')
raise UMC_Error(_('The item %d does not exists.') %
(request.options['item'], ),
status=400)
self.finished(request.id, self.data[item])
@sanitize(IntegerSanitizer(required=True))
def put(self, request):
"""replace all data with the list provided in request.options"""
self.data = request.options
self.finished(request.id, None)
class Instance(Base):
@sanitize(
pattern=PatternSanitizer(default='.*'),
category=ChoicesSanitizer(choices=['user', 'pid', 'command', 'all'], default='all')
)
@simple_response
def query(self, pattern, category='all'):
processes = []
for process in psutil.process_iter():
try:
cpu_time = process.cpu_times()
proc = {
'timestamp': time.time(),
'cpu_time': cpu_time.user + cpu_time.system,
'user': process.username(),
'pid': process.pid,
'cpu': 0.0,
'mem': process.memory_percent(),
'command': ' '.join(process.cmdline() or []) or process.name(),
}
except psutil.NoSuchProcess:
continue
categories = [category]
if category == 'all':
categories = ['user', 'pid', 'command']
if any(pattern.match(str(proc[cat])) for cat in categories):
processes.append(proc)
# Calculate correct cpu percentage
time.sleep(1)
for process_entry in processes:
try:
process = psutil.Process(process_entry['pid'])
cpu_time = process.cpu_times()
except psutil.NoSuchProcess:
continue
elapsed_time = time.time() - process_entry.pop('timestamp')
elapsed_cpu_time = cpu_time.user + cpu_time.system - process_entry.pop('cpu_time')
cpu_percent = (elapsed_cpu_time / elapsed_time) * 100
process_entry['cpu'] = cpu_percent
return processes
@sanitize(
signal=ChoicesSanitizer(choices=['SIGTERM', 'SIGKILL']),
pid=ListSanitizer(IntegerSanitizer())
)
@simple_response
def kill(self, signal, pid):
failed = []
for pid_ in pid:
try:
process = psutil.Process(pid_)
if signal == 'SIGTERM':
process.terminate()
elif signal == 'SIGKILL':
process.kill()
except psutil.NoSuchProcess as exc:
failed.append(str(pid_))
MODULE.error('Could not %s pid %s: %s' % (signal, pid_, exc))
if failed:
failed = ', '.join(failed)
raise UMC_Error(_('No process found with PID %s') % (failed))
class Instance(Base, ProgressMixin):
def __init__(self, *args, **kwargs):
Base.__init__(self, *args, **kwargs)
ProgressMixin.__init__(self)
self._finishedLock = threading.Lock()
self._finishedResult = True
self._progressParser = util.ProgressParser()
self.__keep_alive_request = None
self._net_apply_running = 0
# reset umask to default
os.umask(0o022)
def init(self):
os.putenv('LANG', str(self.locale))
_locale.setlocale(_locale.LC_ALL, str(self.locale))
if not util.is_system_joined():
self._preload_city_data()
def _preload_city_data(self):
util.get_city_data()
util.get_country_data()
def _get_localized_label(self, label_dict):
# return the correctly loca
return label_dict.get(self.locale.language) or label_dict.get(
'en', '') or label_dict.get('', '')
def ping(self, request):
if request.options.get('keep_alive'):
self.__keep_alive_request = request
return
self.finished(request.id, None)
@simple_response
def close_browser(self):
try:
with open('/var/cache/univention-system-setup/browser.pid',
'rb') as fd:
pid = int(fd.readline().strip())
process = psutil.Process(pid)
process.kill()
return True
except IOError as exc:
MODULE.warn('cannot open browser PID file: %s' % (exc, ))
except ValueError as exc:
MODULE.error('browser PID is not a number: %s' % (exc, ))
except psutil.NoSuchProcess as exc:
MODULE.error('cannot kill process with PID: %s' % (exc, ))
return False
@simple_response
def load(self):
'''Return a dict with all necessary values for system-setup read from the current
status of the system.'''
return util.load_values(self.locale.language)
@simple_response
def save_keymap(self, layout=None):
'''Set the systems x-keymap according to
request.options[keymap]'''
# Don't set in debian installer mode
if ucr.is_true('system/setup/boot/installer'):
return True
if layout:
subprocess.call(
['/usr/bin/setxkbmap', '-display', ':0', '-layout', layout])
return True
def save(self, request):
'''Reconfigures the system according to the values specified in the dict given as
option named "values".'''
# get new values
values = request.options.get('values', {})
run_hooks = request.options.get('run_hooks', False)
script_args = []
if run_hooks:
# create a status file that indicates that save has been triggered
util.create_status_file()
# enforce particular arguments for setup scripts
script_args = [
'--appliance-mode', '--force-recreate', '--demo-mode'
]
def _thread(request, obj):
# acquire the lock until the scripts have been executed
self._finishedResult = False
obj._finishedLock.acquire()
try:
subfolders = {
'network': ['30_net'],
'certificate': ['40_ssl'],
'languages': ['15_keyboard', '20_language', '35_timezone'],
}.get(request.flavor)
self._progressParser.reset(subfolders)
if request.flavor == 'setup':
# adjust progress fractions for setup wizard with pre-configurred settings
fractions = self._progressParser.fractions
fractions['05_role/10role'] = 0
fractions['10_basis/12domainname'] = 0
fractions['10_basis/14ldap_basis'] = 0
fractions['90_postjoin/10admember'] = 0
self._progressParser.calculateFractions()
MODULE.info('saving profile values')
util.write_profile(values)
if not values:
MODULE.error('No property "values" given for save().')
return False
# in case of changes of the IP address, restart UMC server and web server
# for this we ignore changes of virtual or non-default devices
# ... no need to restart the UMC server if cleanup scripts are run anyway
restart = False
if not run_hooks:
MODULE.info('Check whether ip addresses have been changed')
for ikey, ival in values.iteritems():
if RE_IPV4.match(ikey) or RE_IPV6_DEFAULT.match(
ikey) or RE_SSL.match(ikey):
restart = True
break
MODULE.info('Restart servers: %s' % restart)
# on a joined system or on a basesystem, we can run the setup scripts
MODULE.info('runnning system setup scripts (flavor %r)' %
(request.flavor, ))
util.run_scripts(self._progressParser,
restart,
subfolders,
lang=str(self.locale),
args=script_args)
# run cleanup scripts and appliance hooks if needed
if run_hooks:
util.cleanup(with_appliance_hooks=True)
# done :)
self._finishedResult = True
return True
finally:
obj._finishedLock.release()
def _finished(thread, result):
if self.__keep_alive_request:
self.finished(self.__keep_alive_request.id, None)
self.__keep_alive_request = None
if isinstance(result, BaseException):
msg = ''.join(thread.trace + traceback.format_exception_only(
*thread.exc_info[:2]))
MODULE.warn('Exception during saving the settings: %s' %
(msg, ))
self._progressParser.current.errors.append(
_('Encountered unexpected error during setup process: %s')
% result)
self._progressParser.current.critical = True
self._finishedResult = True
thread = notifier.threads.Simple(
'save', notifier.Callback(_thread, request, self), _finished)
thread.run()
self.finished(request.id, None)
@simple_response
def join(self, values=None, dcname=None, username=None, password=None):
'''Join and reconfigure the system according to the values specified in the dict given as
option named "values".'''
# get old and new values
orgValues = util.load_values()
values = values or {}
# determine new system role
oldrole = orgValues.get('server/role', '')
newrole = values.get('server/role', oldrole)
# create a status file that indicates that save has been triggered
util.create_status_file()
def _thread(obj, username, password):
# acquire the lock until the scripts have been executed
self._finishedResult = False
obj._finishedLock.acquire()
try:
self._progressParser.reset()
# write the profile file and run setup scripts
util.auto_complete_values_for_join(values)
# on unjoined DC master the nameserver must be set to the external nameserver
if newrole == 'domaincontroller_master' and not orgValues.get(
'joined'):
for i in range(1, 4):
# overwrite these values only if they are set, because the UMC module
# will save only changed values
if values.get('dns/forwarder%d' % i):
values['nameserver%d' % i] = values.get(
'dns/forwarder%d' % i)
MODULE.info('saving profile values')
util.write_profile(values)
# unjoined DC master (that is not being converted to a basesystem) -> run the join script
MODULE.info('runnning system setup join script')
util.run_joinscript(self._progressParser,
values,
username,
password,
dcname,
lang=str(self.locale))
# done :)
self._finishedResult = True
return True
finally:
obj._finishedLock.release()
def _finished(thread, result):
if self.__keep_alive_request:
self.finished(self.__keep_alive_request.id, None)
self.__keep_alive_request = None
if isinstance(result, BaseException):
msg = ''.join(thread.trace + traceback.format_exception_only(
*thread.exc_info[:2]))
MODULE.warn('Exception during saving the settings: %s' %
(msg, ))
self._progressParser.current.errors.append(
_('Encountered unexpected error during setup process: %s')
% (result, ))
self._progressParser.current.critical = True
self._finishedResult = True
thread = notifier.threads.Simple(
'join', notifier.Callback(_thread, self, username, password),
_finished)
thread.run()
return
def check_finished(self, request):
'''Check whether the join/setup scripts are finished. This method implements a long
polling request, i.e., the request is only finished at the moment when all scripts
have been executed or due to a timeout. If it returns because of the timeout, a new
try can be started.'''
def _thread(request, obj):
def progress_info(state, **kwargs):
info = {
'component': state.fractionName,
'info': state.message,
'errors': state.errors,
'critical': state.critical,
'steps': state.percentage
}
info.update(kwargs)
MODULE.info(
'Progress state: %(steps).1f%% - %(component)s - %(info)s'
% info)
return info
# acquire the lock in order to wait for the join/setup scripts to finish
# do this for 30 sec and then return anyway
SLEEP_TIME = 0.200
WAIT_TIME = 30
ntries = WAIT_TIME / SLEEP_TIME
while not obj._finishedLock.acquire(False):
if ntries <= 0 or self._progressParser.changed and self._progressParser.current:
state = self._progressParser.current
return progress_info(state, finished=False)
time.sleep(SLEEP_TIME)
ntries -= 1
obj._finishedLock.release()
# scripts are done, return final result
# return all errors that we gathered throughout the setup
state = self._progressParser.current
return progress_info(state, finished=obj._finishedResult)
thread = notifier.threads.Simple(
'check_finished', notifier.Callback(_thread, request, self),
notifier.Callback(self.thread_finished_callback, request))
thread.run()
@simple_response(with_flavor=True)
def validate(self, values=None, flavor=None):
'''Validate the specified values given in the dict as option named "values".
Return a dict (with variable names as key) of dicts with the structure:
{ "valid": True/False, "message": "..." }'''
# init variables
messages = []
values = values or {}
orgValues = util.load_values()
is_wizard_mode = flavor == 'wizard'
# determine new system role
newrole = values.get('server/role', orgValues.get('server/role', ''))
ad_member = values.get('ad/member', orgValues.get('ad/member', ''))
# mix original and new values
allValues = copy.copy(values)
for ikey, ival in orgValues.iteritems():
if ikey not in allValues:
allValues[ikey] = ival
# helper functions
# TODO: 'valid' is not correctly evaluated in frontend
# i.e. if valid you may continue without getting message
def _check(key, check, message, critical=True):
if key not in values:
return
if not check(values[key]):
messages.append({
'message': message,
'valid': not critical,
'key': key
})
def _append(key, message):
MODULE.warn('Validation failed for key %s: %s' % (key, message))
messages.append({'key': key, 'valid': False, 'message': message})
# host and domain name
packages = set(values.get('components', []))
_check(
'hostname', util.is_hostname,
_('The hostname or the hostname part of the fully qualified domain name is invalid. Please go back to the host setting and make sure, that the hostname only contains letter (a-zA-Z) and digits (0-9).'
))
hostname_length_critical = ad_member or 'univention-samba' in packages or 'univention-samba4' in packages
appliance_str = _('the UCS system')
if ucr['umc/web/appliance/name']:
appliance_str = _('the %s appliance') % (
ucr['umc/web/appliance/name'], )
hostname_length_message = _(
'A valid NetBIOS name can not be longer than 13 characters. If Samba is installed, the hostname should be shortened.'
) if hostname_length_critical else _(
'The hostname %s is longer than 13 characters. It will not be possible to install an Active Directory compatible Domaincontroller (Samba 4) or UCS@school. The hostname cannot be changed after the installation of %s. It is recommended to shorten the hostname to maximal 13 characters.'
) % (
values.get('hostname', ''),
appliance_str,
)
_check('hostname',
lambda x: len(x) <= 13,
hostname_length_message,
critical=hostname_length_critical)
_check(
'domainname', util.is_domainname,
_("Please enter a valid fully qualified domain name (e.g. host.example.com)."
))
hostname = allValues.get('hostname', '')
domainname = allValues.get('domainname', '')
if hostname or domainname:
if len('%s%s' % (hostname, domainname)) >= 63:
_append(
'domainname',
_('The length of fully qualified domain name is greater than 63 characters.'
))
if hostname == domainname.split('.')[0]:
_append('domainname', _("Hostname is equal to domain name."))
if is_wizard_mode and not util.is_system_joined():
if newrole == 'domaincontroller_master' and not values.get(
'domainname'):
_append(
'domainname',
_("No fully qualified domain name has been specified for the system."
))
elif not values.get('hostname'):
_append('hostname',
_("No hostname has been specified for the system."))
# windows domain
_check(
'windows/domain', lambda x: x == x.upper(),
_("The windows domain name can only consist of upper case characters."
))
_check(
'windows/domain', lambda x: len(x) <= 15,
_("The windows domain name cannot be longer than 15 characters."))
_check('windows/domain', util.is_windowsdomainname,
_("The windows domain name is not valid."))
# LDAP base
_check(
'ldap/base', util.is_ldap_base,
_("The LDAP base may neither contain blanks nor any special characters. Its structure needs to consist of at least two relative distinguished names (RDN) which may only use the attribute tags 'dc', 'cn', 'c', 'o', or 'l' (e.g., dc=test,dc=net)."
))
# root password
_check(
'root_password', lambda x: len(x) >= 8,
_("The root password is too short. For security reasons, your password must contain at least 8 characters."
))
_check('root_password', util.is_ascii,
_("The root password may only contain ascii characters."))
# ssl + email
labels = {
'ssl/country': _('Country'),
'ssl/state': _('State'),
'ssl/locality': _('Location'),
'ssl/organization': _('Organization'),
'organization': _('Organization'),
'ssl/organizationalunit': _('Business unit'),
'ssl/email': _('Email address'),
'email_address': _('Email address'),
'ssl/common': _('Common name for the root SSL certificate'),
}
for maxlenth, keys in [
(2, ('ssl/country', )), (128, (
'ssl/state',
'ssl/locality',
)),
(64, ('organization', 'ssl/organization', 'ssl/organizationalunit',
'ssl/email', 'email_address', 'ssl/common'))
]:
for ikey in keys:
_check(
ikey, lambda x: len(x) <= maxlenth,
_('The following value is too long, only %(max)s characters allowed: %(name)s'
) % {
'max': maxlenth,
'name': labels[ikey]
})
for ikey in ('ssl/country', 'ssl/state', 'ssl/locality',
'ssl/organization', 'ssl/organizationalunit', 'ssl/email',
'ssl/common'):
for table in (stringprep.in_table_c21_c22, stringprep.in_table_a1,
stringprep.in_table_c8, stringprep.in_table_c3,
stringprep.in_table_c4, stringprep.in_table_c5,
lambda c: c == u'\ufffd'):
_check(
ikey, lambda x: not any(map(table, unicode(x))),
_('The value for %s contains invalid characters.') %
(labels[ikey], ))
_check('ssl/country', lambda x: len(x) == 2,
_('Country must be a country code consisting of 2 characters.'))
for ikey in ['ssl/email', 'email_address']:
_check(ikey, lambda x: x.find('@') > 0,
_("Please enter a valid email address"))
# net
try:
interfaces = network.Interfaces()
interfaces.from_dict(allValues.get('interfaces', {}))
interfaces.check_consistency()
except network.DeviceError as exc:
_append('interfaces', str(exc))
# validate the primary network interface
_check('interfaces/primary', lambda x: not x or x in interfaces,
_('The primary network device must exist.'))
# check nameservers
for ikey, iname in [('nameserver[1-3]', _('Domain name server')),
('dns/forwarder[1-3]', _('External name server'))]:
reg = re.compile('^(%s)$' % ikey)
for jkey, jval in values.iteritems():
if reg.match(jkey):
if not values.get(jkey):
# allow empty value
continue
_check(
jkey, util.is_ipaddr,
_('The specified IP address (%(name)s) is not valid: %(value)s'
) % {
'name': iname,
'value': jval
})
if is_wizard_mode and not util.is_system_joined() and (
newrole not in ['domaincontroller_master', 'basesystem']
or ad_member):
if all(nameserver in values and not values[nameserver]
for nameserver in ('nameserver1', 'nameserver2',
'nameserver3')):
# 'nameserver1'-key exists → widget is displayed → = not in UCS/debian installer mode
if not any(interface.ip4dynamic or interface.ip6dynamic
for interface in interfaces.values()):
_append('nameserver1',
_('A domain name server needs to be specified.'))
# _append('nameserver1', _('At least one domain name server needs to be given if DHCP or SLAAC is not specified.'))
# see whether the domain can be determined automatically
ucr.load()
guessed_domain = None
for obj in [values, ucr]:
for nameserver in ('nameserver1', 'nameserver2',
'nameserver3'):
nameserver = obj.get(nameserver)
if nameserver:
guessed_domain = None
if obj.get('ad/member') and obj.get('ad/address'):
try:
ad_domain_info = lookup_adds_dc(
obj.get('ad/address'),
ucr={'nameserver1': nameserver})
except failedADConnect:
pass
else:
guessed_domain = ad_domain_info['Domain']
else:
guessed_domain = util.get_ucs_domain(nameserver)
if guessed_domain:
differing_domain_name = values.get(
'domainname') and values['domainname'].lower(
) != guessed_domain.lower()
if differing_domain_name:
_append(
'domainname',
_('The specified domain name is different to the %s domain name found via the configured DNS server: %s'
) % (
_('Active Directory')
if ad_member else _('UCS'),
guessed_domain,
))
else:
# communicate guessed domainname to frontend
messages.append({
'valid': True,
'key': 'domainname',
'value': guessed_domain,
})
break
if guessed_domain:
break
if not guessed_domain:
if not values.get('domainname'):
_append(
'domainname',
_('Cannot automatically determine the domain. Please specify the server\'s fully qualified domain name.'
))
if values.get('nameserver1') and values.get('start/join'):
_append(
'nameserver1',
_('The specified nameserver %s is not part of a valid UCS domain.'
) % (values['nameserver1'], ))
# check gateways
if values.get('gateway'): # allow empty value
_check(
'gateway', util.is_ipv4addr,
_('The specified gateway IPv4 address is not valid: %s') %
values.get('gateway'))
if values.get('ipv6/gateway'): # allow empty value
_check(
'ipv6/gateway', util.is_ipv6addr,
_('The specified gateway IPv6 address is not valid: %s') %
values.get('ipv6/gateway'))
# proxy
_check(
'proxy/http', util.is_proxy,
_('The specified proxy address is not valid (e.g., http://10.201.1.1:8080): %s'
) % allValues.get('proxy/http', ''))
# software checks
if 'univention-virtual-machine-manager-node-kvm' in packages and 'univention-virtual-machine-manager-node-xen' in packages:
_append(
'components',
_('It is not possible to install KVM and XEN components on one system. Please select only one of these components.'
))
if 'univention-samba' in packages and 'univention-samba4' in packages:
_append(
'components',
_('It is not possible to install Samba 3 and Samba 4 on one system. Please select only one of these components.'
))
return messages
@sanitize(pattern=PatternSanitizer(default='.*',
required=True,
add_asterisks=True))
@simple_response
def lang_locales(self, pattern, category='language_en'):
'''Return a list of all available locales.'''
return util.get_available_locales(pattern, category)
def lang_timezones(self, request):
'''Return a list of all available time zones.'''
try:
file = open('/usr/share/univention-system-setup/locale/timezone')
except EnvironmentError:
MODULE.error(
'Cannot find locale data for timezones in /usr/share/univention-system-setup/locale'
)
self.finished(request.id, None)
return
timezones = [i.strip('\n') for i in file if not i.startswith('#')]
self.finished(request.id, timezones)
@simple_response
def lang_keyboard_model(self):
'''Return a list of all available keyboard models.'''
tree = lxml.etree.parse(open('/usr/share/X11/xkb/rules/base.xml'))
models = tree.xpath("//model")
model_result = [{
'label':
i18nXKeyboard.translate(
model.xpath('./configItem/description')[0].text),
'id':
model.xpath('./configItem/name')[0].text
} for model in models]
return model_result
@simple_response
def lang_keyboard_layout(self):
'''Return a list of all available keyboard layouts.'''
tree = lxml.etree.parse(open('/usr/share/X11/xkb/rules/base.xml'))
layouts = tree.xpath("//layout")
layout_result = [{
'label':
i18nXKeyboard.translate(
layout.xpath('./configItem/description')[0].text),
'id':
layout.xpath('./configItem/name')[0].text,
'language':
layout.xpath('./configItem/shortDescription')[0].text,
'countries':
':'.join([
icountry.text
for icountry in layout.xpath('./configItem/countryList/*')
]),
} for layout in layouts]
return layout_result
@sanitize(keyboardlayout=StringSanitizer(default='us'))
@simple_response
def lang_keyboard_variante(self, keyboardlayout):
'''Return a list of all available keyboard variantes.'''
variante_result = []
tree = lxml.etree.parse(open('/usr/share/X11/xkb/rules/base.xml'))
layouts = tree.xpath("//layout")
for layout in layouts:
layoutID = layout.xpath("./configItem/name")[0].text
if layoutID != keyboardlayout:
continue
variants = layout.xpath("./variantList/variant")
variante_result += [{
'label':
i18nXKeyboard.translate(
variant.xpath('./configItem/description')[0].text),
'id':
variant.xpath('./configItem/name')[0].text
} for variant in variants]
variante_result.insert(0, {'label': '', 'id': ''})
return variante_result
def lang_countrycodes(self, request):
'''Return a list of all countries with their two letter chcountry codes.'''
country_data = util.get_country_data()
countries = [{
'id':
icountry,
'label':
self._get_localized_label(idata.get('label', {})),
} for icountry, idata in country_data.iteritems()
if idata.get('label')]
# add the value from ucr value to the list
# this is required because invalid values will be unset in frontend
# Bug #26409
tmpUCR = univention.config_registry.ConfigRegistry()
tmpUCR.load()
ssl_country = tmpUCR.get('ssl/country')
if ssl_country not in [i['id'] for i in countries]:
countries.append({'label': ssl_country, 'id': ssl_country})
self.finished(request.id, countries)
def net_apply(self, request):
if self._net_apply_running > 0:
# do not start another process applying the network settings
return False
values = request.options.get('values', {})
demo_mode = request.options.get('demo_mode', False)
def _thread(obj):
obj._net_apply_running += 1
MODULE.process('Applying network settings')
with util.written_profile(values):
util.run_networkscrips(demo_mode)
def _finished(thread, result):
self._net_apply_running -= 1
self.finished(request.id, True)
thread = notifier.threads.Simple('net_apply',
notifier.Callback(_thread, self),
_finished)
thread.run()
@simple_response
def net_apply_check_finished(self):
if self._net_apply_running > 0:
# raise an error if net_apply command is still running...
# this allows long polling on the client side (poll until successful request)
raise RequestTimeout()
return self._net_apply_running == 0
@simple_response
def net_interfaces(self):
'''Return a list of all available network interfaces.'''
return [idev['name'] for idev in util.detect_interfaces()]
# workaround: use with_progress to make the method threaded
@simple_response(with_progress=True)
def net_dhclient(self, interface, timeout=10):
'''Request a DHCP address. Expects as options a dict containing the key
"interface" and optionally the key "timeout" (in seconds).'''
return util.dhclient(interface, timeout)
@sanitize(locale=StringSanitizer(default='en_US'))
@simple_response
def reset_locale(self, locale):
locale = Locale(locale)
locale.codeset = self.locale.codeset
MODULE.info('Switching language to: %s' % locale)
os.putenv('LANG', str(self.locale))
try:
_locale.setlocale(_locale.LC_ALL, str(locale))
except _locale.Error:
MODULE.warn(
'Locale %s is not supported, using fallback locale "C" instead.'
% locale)
_locale.setlocale(_locale.LC_ALL, 'C')
self.locale = locale
# dynamically change the translation methods
_translation.set_language(str(self.locale))
i18nXKeyboard.set_language(str(self.locale))
network._translation.set_language(str(self.locale))
AppCache().clear_cache()
@sanitize(pattern=StringSanitizer(),
max_results=IntegerSanitizer(minimum=1, default=5))
@simple_response
def find_city(self, pattern, max_results):
pattern = pattern.decode(self.locale.codeset).lower()
MODULE.info('pattern: %s' % pattern)
if not pattern:
return []
# for the given pattern, find matching cities
city_data = util.get_city_data()
matches = []
for icity in city_data:
match = None
for jlabel in icity.get('label', {}).itervalues():
label = jlabel.decode(self.locale.codeset).lower()
if pattern in label:
# matching score is the overlap if the search pattern and the matched text
# (as fraction between 0 and 1)
match_score = len(pattern) / float(len(label))
if match and match_score < match['match_score']:
# just keep the best match of a city
continue
if match_score > 0.1:
# found a match with more than 10% overlap :)
match = icity.copy()
match['match'] = jlabel
match['match_score'] = match_score
if match:
matches.append(match)
MODULE.info('Search for pattern "%s" with %s matches' %
(pattern, len(matches)))
if not matches:
return None
# add additional score w.r.t. the population size of the city
# such that the largest city gains additional 0.4 on top
max_population = max([imatch['population'] for imatch in matches])
weighted_inv_max_population = 0.6 / float(max_population)
for imatch in matches:
imatch['final_score'] = imatch[
'match_score'] + weighted_inv_max_population * imatch[
'population']
# sort matches...
matches.sort(key=lambda x: x['final_score'], reverse=True)
MODULE.info('Top 5 matches: %s' % json.dumps(matches[:5], indent=2))
matches = matches[:max_results]
# add additional information about keyboard layout, time zone etc. and
# get the correct localized labels
country_data = util.get_country_data()
for imatch in matches:
match_country = country_data.get(imatch.get('country'))
if match_country:
imatch.update(util.get_random_nameserver(match_country))
imatch.update(
dict(
default_lang=match_country.get('default_lang'),
country_label=self._get_localized_label(
match_country.get('label', {})),
label=self._get_localized_label(imatch.get('label'))
or imatch.get('match'),
))
return matches
@simple_response
def apps_query(self):
return util.get_apps(True)
@simple_response
def check_domain(self, role, nameserver):
result = {}
if role == 'ad':
try:
ad_domain_info = lookup_adds_dc(nameserver)
dc = ad_domain_info['DC DNS Name']
if dc:
result['dc_name'] = dc
domain = ad_domain_info['Domain']
result['domain'] = domain
result['ucs_master'] = util.is_ucs_domain(
nameserver, domain)
ucs_master_fqdn = util.resolve_domaincontroller_master_srv_record(
nameserver, domain)
result['ucs_master_fqdn'] = ucs_master_fqdn
result['ucs_master_reachable'] = util.is_ssh_reachable(
ucs_master_fqdn)
except (failedADConnect, connectionFailed) as exc:
MODULE.warn('ADDS DC lookup failed: %s' % (exc, ))
elif role == 'nonmaster':
domain = util.get_ucs_domain(nameserver)
if domain:
fqdn = util.resolve_domaincontroller_master_srv_record(
nameserver, domain)
else:
fqdn = util.get_fqdn(nameserver)
if fqdn:
result['dc_name'] = fqdn
domain = '.'.join(fqdn.split('.')[1:])
result['ucs_master'] = util.is_ucs_domain(nameserver, domain)
return result
@simple_response
def check_domain_join_information(self, domain_check_role, role, dns,
nameserver, address, username, password):
result = {}
if domain_check_role == 'ad':
domain = util.check_credentials_ad(nameserver, address, username,
password)
result['domain'] = domain
if dns: # "dns" means we don't want to replace the existing DC Master
ucs_master_fqdn = util.resolve_domaincontroller_master_srv_record(
nameserver, domain)
if ucs_master_fqdn:
# if we found a _domaincontroller_master._tcp SRV record the system will be a DC Backup/Slave/Member.
# We need to check the credentials of this system, too, so we ensure that the System is reachable via SSH.
# Otherwise the join will fail with strange error like "ping to ..." failed.
result.update(
receive_domaincontroller_master_information(
False, nameserver, ucs_master_fqdn, username,
password))
set_role_and_check_if_join_will_work(
role, ucs_master_fqdn, username, password)
elif domain_check_role == 'nonmaster':
result.update(
receive_domaincontroller_master_information(
dns, nameserver, address, username, password))
set_role_and_check_if_join_will_work(role, address, username,
password)
# master? basesystem? no domain check necessary
return result
@simple_response
def check_school_information(self, hostname, address, username, password):
return check_for_school_domain(hostname, address, username, password)
@simple_response
def check_repository_accessibility(self):
return get_unreachable_repository_servers()
@simple_response
def check_uid(self, uid, role, address, username, password):
return check_if_uid_is_available(uid, role, address, username,
password)
class Instance(Base):
def init(self):
MODULE.info("Initializing 'updater' module (PID = %d)" % (getpid(), ))
self._current_job = ''
self._logfile_start_line = 0
self._serial_file = Watched_File(COMPONENTS_SERIAL_FILE)
self._updates_serial = Watched_Files(UPDATE_SERIAL_FILES)
try:
self.uu = UniventionUpdater(False)
except Exception as exc: # FIXME: let it raise
MODULE.error("init() ERROR: %s" % (exc, ))
@simple_response
def query_maintenance_information(self):
ucr.load()
if ucr.is_true('license/extended_maintenance/disable_warning'):
return {'show_warning': False}
version = self.uu.get_ucs_version()
try:
url = 'http://updates.software-univention.de/download/ucs-maintenance/{}.yaml'.format(
version)
response = requests.get(url, timeout=10)
if not response.ok:
response.raise_for_status()
status = yaml.load(response.content)
if not isinstance(status, dict):
raise yaml.YAMLError(repr(status))
# the yaml file contains for maintained either false, true or extended as value.
# yaml.load converts true and false into booleans but extended into string.
_maintained_status = status.get('maintained')
maintenance_extended = _maintained_status == 'extended'
show_warning = maintenance_extended or not _maintained_status
except yaml.YAMLError as exc:
MODULE.error('The YAML format is malformed: %s' % (exc, ))
return {'show_warning': False}
except requests.exceptions.RequestException as exc:
MODULE.error("Querying maintenance information failed: %s" %
(exc, ))
return {'show_warning': False}
return {
'ucs_version': version,
'show_warning': show_warning,
'maintenance_extended': maintenance_extended,
'base_dn': ucr.get('license/base')
}
@simple_response
def poll(self):
return True
@simple_response
def query_releases(self):
"""
Returns a list of system releases suitable for the
corresponding ComboBox
"""
# be as current as possible.
self.uu.ucr_reinit()
ucr.load()
appliance_mode = ucr.is_true('server/appliance')
available_versions, blocking_components = self.uu.get_all_available_release_updates(
)
result = [{
'id': rel,
'label': 'UCS %s' % (rel, )
} for rel in available_versions]
#
# appliance_mode=no ; blocking_comp=no → add "latest version"
# appliance_mode=no ; blocking_comp=yes → no "latest version"
# appliance_mode=yes; blocking_comp=no → add "latest version"
# appliance_mode=yes; blocking_comp=yes → add "latest version"
#
if result and (appliance_mode or not blocking_components):
# UniventionUpdater returns available version in ascending order, so
# the last returned entry is the one to be flagged as 'latest' if there's
# no blocking component.
result[-1]['label'] = '%s (%s)' % (result[-1]['label'],
_('latest version'))
return result
def call_hooks(self, request):
"""
Calls the specified hooks and returns data given back by each hook
"""
def _thread(request):
result = {}
hookmanager = univention.hooks.HookManager(
HOOK_DIRECTORY) # , raise_exceptions=False
hooknames = request.options.get('hooks')
MODULE.info('requested hooks: %s' % hooknames)
for hookname in hooknames:
MODULE.info('calling hook %s' % hookname)
result[hookname] = hookmanager.call_hook(hookname)
MODULE.info('result: %r' % (result, ))
return result
thread = notifier.threads.Simple(
'call_hooks', notifier.Callback(_thread, request),
notifier.Callback(self.thread_finished_callback, request))
thread.run()
@simple_response
def updates_serial(self):
"""
Watches the three sources.list snippets for changes
"""
result = self._updates_serial.timestamp()
MODULE.info(" -> Serial for UPDATES is '%s'" % result)
return result
@simple_response
def updates_check(self):
"""
Returns the list of packages to be updated/installed
by a dist-upgrade.
"""
p0 = subprocess.Popen(['LC_ALL=C apt-get update'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(stdout, stderr) = p0.communicate()
p1 = subprocess.Popen(['LC_ALL=C apt-get -u dist-upgrade -s'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(stdout, stderr) = p1.communicate()
install = []
update = []
remove = []
for line in stdout.split('\n'):
# upgrade:
# Inst univention-updater [3.1.1-5] (3.1.1-6.408.200810311159 192.168.0.10)
# inst:
# Inst mc (1:4.6.1-6.12.200710211124 oxae-update.open-xchange.com)
#
# *** FIX *** the above example lines ignore the fact that there's
# some extra text (occasionally) after the last closing
# parenthesis. Until now, I've seen only a pair of empty
# brackets [], but who knows...
match = re.search('^Inst (\S+)\s+(.*?)\s*\((\S+)\s.*\)', line)
if match:
pkg = match.group(1)
old = match.group(2)
ver = match.group(3)
if old:
update.append([pkg, ver])
else:
install.append([pkg, ver])
elif line.startswith('Remv '):
l = line.split(' ')
pkg = l[1]
# i18n: The package version is unknown.
ver = _('unknown')
if len(l) > 2:
ver = l[2].replace('[', '').replace(']', '')
remove.append([pkg, ver])
return dict(
update=sorted(update),
install=sorted(install),
remove=sorted(remove),
)
@simple_response
def updates_available(self):
"""
Asks if there are package updates available. (don't get confused
by the name of the UniventionUpdater function that is called here.)
This is a seperate call since it can take an amount of time, thus
being invoked by a seperate button (and not in the background)
"""
ucr.load()
try:
# be as current as possible.
what = 'reinitializing UniventionUpdater'
self.uu.ucr_reinit()
what = 'checking update availability'
return self.uu.component_update_available()
except Exception as ex:
typ = str(type(ex)).strip('<>')
msg = '[while %s] [%s] %s' % (what, typ, str(ex))
MODULE.error(msg)
return False
def status(self, request): # TODO: remove unneeded things
"""One call for all single-value variables."""
result = {}
ucr.load()
try:
result['erratalevel'] = int(ucr.get('version/erratalevel', 0))
except ValueError:
result['erratalevel'] = 0
result['appliance_mode'] = ucr.is_true('server/appliance')
result['easy_mode'] = ucr.is_true('update/umc/updateprocess/easy',
False)
result['timestamp'] = int(time())
result['reboot_required'] = ucr.is_true('update/reboot/required',
False)
try:
# be as current as possible.
what = 'reinitializing UniventionUpdater'
self.uu.ucr_reinit()
what = 'getting UCS version'
result['ucs_version'] = self.uu.get_ucs_version()
# if nothing is returned -> convert to empty string.
what = 'querying available release updates'
try:
result[
'release_update_available'] = self.uu.release_update_available(
errorsto='exception')
except RequiredComponentError as exc:
result['release_update_available'] = exc.version
if result['release_update_available'] is None:
result['release_update_available'] = ''
what = 'querying update-blocking components'
blocking_components = self.uu.get_all_available_release_updates(
)[1]
result['release_update_blocking_components'] = ' '.join(
blocking_components or [])
what = "querying availability for easy mode"
if result['easy_mode']:
# updates/available should reflect the need for an update
easy_update_available = ucr.is_true('update/available', False)
# but dont rely on ucr! update/available is set during univention-upgrade --check
# but when was the last time this was run?
# release update
easy_update_available = easy_update_available or result[
'release_update_available']
# if no update seems necessary perform a real (expensive) check nonetheless
easy_update_available = easy_update_available or self.uu.component_update_available(
)
result['easy_update_available'] = bool(easy_update_available)
else:
result['easy_update_available'] = False
# Component counts are now part of the general 'status' data.
what = "counting components"
c_count = 0
e_count = 0
for comp in self.uu.get_all_components():
c_count = c_count + 1
if ucr.is_true('repository/online/component/%s' % (comp, ),
False):
e_count = e_count + 1
result['components'] = c_count
result['enabled'] = e_count
# HACK: the 'Updates' form polls on the serial file
# to refresh itself. Including the serial value
# into the form helps us to have a dependent field
# that can trigger the refresh of the "Releases"
# combobox and the 'package updates available' field.
result['serial'] = self._serial_file.timestamp()
except Exception as exc: # FIXME: don't catch everything
typ = str(type(exc)).strip('<>')
msg = '[while %s] [%s] %s' % (what, typ, exc)
result['message'] = msg
result['status'] = 1
MODULE.error(msg)
self.finished(request.id, [result])
@simple_response
def reboot(self):
"""
Reboots the computer. Simply invokes /sbin/reboot in the background
and returns success to the caller. The caller is prepared for
connection loss.
"""
subprocess.call(['/sbin/reboot'])
return True
@simple_response
def running(self):
"""
Returns the id (key into INSTALLERS) of a currently
running job, or the empty string if nothing is running.
"""
return self.__which_job_is_running()
@sanitize(
job=ChoicesSanitizer(INSTALLERS.keys() + [''], required=True),
count=IntegerSanitizer(default=0),
)
@simple_response
def updater_log_file(self, job, count):
"""
returns the content of the log file associated with
the job.
Argument 'count' has the same meaning as already known:
<0 ...... return timestamp of file (for polling)
0 ....... return whole file as a string list
>0 ...... ignore this many lines, return the rest of the file
*** NOTE *** As soon as we have looked for a running job at least once,
we know the job key and can associate it here.
TODO: honor a given 'job' argument
"""
job = self._current_job or job
if not job:
return
fname = INSTALLERS[job]['logfile']
if count < 0:
try:
return stat(fname)[9]
except (IOError, OSError):
return 0
# don't read complete file if we have an 'ignore' count
count += self._logfile_start_line
return self._logview(fname, -count)
def _logview(self, fname, count):
"""
Contains all functions needed to view or 'tail' an arbitrary text file.
Argument 'count' can have different values:
< 0 ... ignore this many lines, return the rest of the file
0 ..... return the whole file, split into lines.
> 0 ... return the last 'count' lines of the file. (a.k.a. tail -n <count>)
"""
lines = []
try:
with open(fname, 'rb') as fd:
for line in fd:
if (count < 0):
count += 1
else:
lines.append(line.rstrip().decode('utf-8', 'replace'))
if (count > 0) and (len(lines) > count):
lines.pop(0)
except (IOError, OSError):
pass
return lines
@sanitize(
job=ChoicesSanitizer(INSTALLERS.keys(), required=True), )
@simple_response
def updater_job_status(self, job): # TODO: remove this completely
"""Returns the status of the current/last update even if the job is not running anymore."""
result = {}
try:
with open(INSTALLERS[job]['statusfile'], 'rb') as fd:
for line in fd:
fields = line.strip().split('=')
if len(fields) == 2:
result['_%s_' % fields[0]] = fields[1]
except (IOError, OSError):
pass
result['running'] = '' != self.__which_job_is_running()
return result
@sanitize(
job=ChoicesSanitizer(INSTALLERS.keys(), required=True),
detail=StringSanitizer(r'^[A-Za-z0-9\.\- ]*$'),
)
@simple_response
def run_installer(self, job, detail=''):
"""
This is the function that invokes any kind of installer. Arguments accepted:
job ..... the main thing to do. can be one of:
'release' ...... perform a release update
'distupgrade' .. update all currently installed packages (distupgrade)
'check' ........ check what would be done for 'update' ... do we need this?
detail ....... an argument that specifies the subject of the installer:
for 'release' .... the target release number,
for all other subjects: detail has no meaning.
"""
MODULE.info("Starting function %r" % (job, ))
self._current_job = job
# remember initial lines of logfile before starting update to not show it in the frontend
logfile = INSTALLERS[job]['logfile']
try:
with open(logfile, 'rb') as fd:
self._logfile_start_line = sum(1 for line in fd)
except (IOError, OSError):
pass
command = INSTALLERS[job]['command']
if '%' in command:
command = command % (pipes.quote(detail).replace('\n', '').replace(
'\r', '').replace('\x00', ''), )
MODULE.info("Creating job: %r" % (command, ))
command = '''
/usr/share/univention-updater/disable-apache2-umc
%s < /dev/null
/usr/share/univention-updater/enable-apache2-umc --no-restart''' % (command, )
atjobs.add(command, comments=dict(lines=self._logfile_start_line))
return {'status': 0}
def __which_job_is_running(self):
# first check running at jobs
for atjob in atjobs.list(True):
for job, inst in INSTALLERS.iteritems():
cmd = inst['command'].split('%')[0]
if cmd in atjob.command:
self._current_job = job
try:
self._logfile_start_line = int(
atjob.comments.get('lines', 0))
except ValueError:
pass
return job
# no atjob found, parse process list (if univention-upgrade was started via CLI)
commands = [
('/usr/share/univention-updater/univention-updater-umc-dist-upgrade',
'distupgrade'),
('/usr/share/univention-updater/univention-updater', 'release'),
('/usr/sbin/univention-upgrade', 'distupgrade'
) # we don't know if it is a dist-upgrade or a release upgrade
]
for cmd, job in commands:
for process in psutil.process_iter():
try:
cmdline = process.cmdline() if callable(
process.cmdline) else process.cmdline
except psutil.NoSuchProcess:
pass
if cmd in cmdline:
self._current_job = job
self._logfile_start_line = 0
return job
return ''
class Instance(Base):
def init(self) -> None:
MODULE.info("Initializing 'updater' module (PID = %d)" % (getpid(), ))
self._current_job = ''
self._logfile_start_line = 0
self._serial_file = Watched_File(COMPONENTS_SERIAL_FILE)
try:
self.uu = UniventionUpdater(False)
except Exception as exc: # FIXME: let it raise
self.uu = None
MODULE.error("init() ERROR: %s" % (exc, ))
@simple_response
def query_maintenance_information(self) -> Dict[str, Any]:
ret = self._maintenance_information()
ret.update(self._last_update())
return ret
def _last_update(self) -> Dict[str, Any]:
status_file = '/var/lib/univention-updater/univention-updater.status'
ret = {
'last_update_failed': False,
'last_update_version': None
} # type: Dict[str, Any]
try:
fstat = stat(status_file)
mtime = datetime.fromtimestamp(fstat.st_mtime)
delta = datetime.now() - mtime
if delta.days != 0: # no fresh failure
return ret
with open(status_file) as fd:
info = dict(line.strip().split('=', 1) # type: ignore
for line in fd) # type: Dict[str, str]
ret['last_update_failed'] = info.get('status') == 'FAILED'
if ret['last_update_failed']:
ret['last_update_version'] = info.get('next_version')
except (ValueError, EnvironmentError) as exc:
MODULE.error(str(exc))
return ret
def _maintenance_information(self) -> Dict[str, Any]:
default = {'show_warning': False}
if not self.uu:
return default
ucr.load()
if ucr.is_true('license/extended_maintenance/disable_warning'):
return default
version = self.uu.current_version
for _ver, data in self.uu.get_releases(version, version):
status = data.get('status', 'unmaintained')
maintenance_extended = status == 'extended'
show_warning = maintenance_extended or status != 'maintained'
return {
'ucs_version': str(version),
'show_warning': show_warning,
'maintenance_extended': maintenance_extended,
'base_dn': ucr.get('license/base'),
}
return default
@simple_response
def query_releases(self) -> List[Dict[str, str]]:
"""
Returns a list of system releases suitable for the
corresponding ComboBox
"""
# be as current as possible.
self.uu.ucr_reinit()
ucr.load()
appliance_mode = ucr.is_true('server/appliance')
available_versions, blocking_components = self.uu.get_all_available_release_updates(
)
result = [{
'id': str(rel),
'label': 'UCS %s' % (rel, )
} for rel in available_versions]
#
# appliance_mode=no ; blocking_comp=no → add "latest version"
# appliance_mode=no ; blocking_comp=yes → no "latest version"
# appliance_mode=yes; blocking_comp=no → add "latest version"
# appliance_mode=yes; blocking_comp=yes → add "latest version"
#
if result and (appliance_mode or not blocking_components):
# UniventionUpdater returns available version in ascending order, so
# the last returned entry is the one to be flagged as 'latest' if there's
# no blocking component.
result[-1]['label'] = '%s (%s)' % (result[-1]['label'],
_('latest version'))
return result
@sanitize(hooks=ListSanitizer(StringSanitizer(minimum=1), required=True))
def call_hooks(self, request: Request) -> None:
"""
Calls the specified hooks and returns data given back by each hook
"""
def _thread(request: Request) -> Dict[str, Any]:
result = {}
hookmanager = HookManager(
HOOK_DIRECTORY) # , raise_exceptions=False
hooknames = request.options.get('hooks')
MODULE.info('requested hooks: %s' % hooknames)
for hookname in hooknames:
MODULE.info('calling hook %s' % hookname)
result[hookname] = hookmanager.call_hook(hookname)
MODULE.info('result: %r' % (result, ))
return result
thread = notifier.threads.Simple(
'call_hooks', notifier.Callback(_thread, request),
notifier.Callback(self.thread_finished_callback, request))
thread.run()
@simple_response
def updates_check(self) -> Dict[str, List[Tuple[str, str]]]:
"""
Returns the list of packages to be updated/installed
by a dist-upgrade.
"""
install = []
update = []
remove = []
apt = Cache(memonly=True)
apt.update()
apt.open()
apt.clear()
apt.upgrade(dist_upgrade=True)
for pkg in apt.get_changes():
if pkg.marked_install:
install.append((pkg.name, pkg.candidate.version))
if pkg.marked_upgrade:
update.append((pkg.name, pkg.candidate.version))
if pkg.marked_delete:
remove.append((pkg.name, pkg.installed.version))
return dict(
update=sorted(update),
install=sorted(install),
remove=sorted(remove),
)
@simple_response
def updates_available(self) -> bool:
"""
Asks if there are package updates available. (don't get confused
by the name of the UniventionUpdater function that is called here.)
This is a separate call since it can take an amount of time, thus
being invoked by a separate button (and not in the background)
"""
ucr.load()
try:
# be as current as possible.
what = 'reinitializing UniventionUpdater'
self.uu.ucr_reinit()
what = 'checking update availability'
new, upgrade, removed = self.uu.component_update_get_packages()
return any((new, upgrade, removed))
except Exception as ex:
typ = str(type(ex)).strip('<>')
msg = '[while %s] [%s] %s' % (what, typ, str(ex))
MODULE.error(msg)
return False
def status(self, request: Request) -> None: # TODO: remove unneeded things
"""One call for all single-value variables."""
result = {} # type: Dict[str, Any]
ucr.load()
try:
result['erratalevel'] = int(ucr.get('version/erratalevel', 0))
except ValueError:
result['erratalevel'] = 0
result['appliance_mode'] = ucr.is_true('server/appliance')
result['timestamp'] = int(time())
result['reboot_required'] = ucr.is_true('update/reboot/required',
False)
try:
# be as current as possible.
what = 'reinitializing UniventionUpdater'
self.uu.ucr_reinit()
what = 'getting UCS version'
result['ucs_version'] = str(self.uu.current_version)
# if nothing is returned -> convert to empty string.
what = 'querying available release updates'
try:
ver = self.uu.release_update_available(errorsto='exception')
result[
'release_update_available'] = '' if ver is None else str(
ver)
except RequiredComponentError as exc:
result['release_update_available'] = exc.version
what = 'querying update-blocking components'
blocking_components = self.uu.get_all_available_release_updates(
)[1] or set()
# check apps
if result['release_update_available']:
try:
from univention.appcenter.actions import get_action
update_check = get_action('update-check')
if update_check:
blocking_apps = update_check.get_blocking_apps(
ucs_version=result['release_update_available'])
if blocking_apps:
blocking_components.update(set(blocking_apps))
except (ImportError, ValueError):
# the new univention.appcenter package is not installed.
# Cannot be a dependency as the app center depends on updater...
raise UMC_Error(
_('Error checking if installed apps are available for next UCS version.'
))
result['release_update_blocking_components'] = ' '.join(
blocking_components)
# Component counts are now part of the general 'status' data.
what = "counting components"
components = [
bool(comp) for comp in self.uu.get_components(all=True)
]
result['components'] = len(components)
result['enabled'] = sum(components)
# HACK: the 'Updates' form polls on the serial file
# to refresh itself. Including the serial value
# into the form helps us to have a dependent field
# that can trigger the refresh of the "Releases"
# combobox and the 'package updates available' field.
result['serial'] = self._serial_file.timestamp()
except Exception as exc: # FIXME: don't catch everything
raise UMC_Error("%s %s %s" % (
_('Error contacting the update server. Please check your proxy or firewall settings, if any. Or it may be a problem with your configured DNS server.'
),
_('This is the error message:'),
exc,
),
traceback=format_exc())
self.finished(request.id, [result])
@simple_response
def running(self) -> str:
"""
Returns the id (key into INSTALLERS) of a currently
running job, or the empty string if nothing is running.
"""
return self.__which_job_is_running()
@sanitize(
job=ChoicesSanitizer(list(INSTALLERS) + [''], required=True),
count=IntegerSanitizer(default=0),
)
@simple_response
def updater_log_file(self, job: str,
count: int) -> Union[None, float, List[str]]:
"""
returns the content of the log file associated with
the job.
:param job: Job name.
:param count: has the same meaning as already known:
<0 ...... return timestamp of file (for polling)
0 ....... return whole file as a string list
>0 ...... ignore this many lines, return the rest of the file
.. note::
As soon as we have looked for a running job at least once,
we know the job key and can associate it here.
TODO: honor a given 'job' argument
"""
job = self._current_job or job
if not job:
return None
fname = INSTALLERS[job]['logfile']
if count < 0:
try:
return stat(fname).st_ctime
except EnvironmentError:
return 0
# don't read complete file if we have an 'ignore' count
count += self._logfile_start_line
return self._logview(fname, -count)
def _logview(self, fname: str, count: int) -> List[str]:
"""
Contains all functions needed to view or 'tail' an arbitrary text file.
:param count: can have different values:
< 0 ... ignore this many lines, return the rest of the file
0 ..... return the whole file, split into lines.
> 0 ... return the last 'count' lines of the file. (a.k.a. tail -n <count>)
"""
lines = []
try:
with open(fname, 'rb') as fd:
for line in fd:
if (count < 0):
count += 1
else:
lines.append(line.rstrip().decode('utf-8', 'replace'))
if (count > 0) and (len(lines) > count):
lines.pop(0)
except EnvironmentError:
pass
return lines
@sanitize(
job=ChoicesSanitizer(INSTALLERS, required=True), )
@simple_response
def updater_job_status(
self, job: str) -> Dict[str, Any]: # TODO: remove this completely
"""Returns the status of the current/last update even if the job is not running anymore."""
result = {} # type: Dict[str, Any]
try:
with open(INSTALLERS[job]['statusfile'], 'r') as fd:
for line in fd:
fields = line.strip().split('=')
if len(fields) == 2:
result['_%s_' % fields[0]] = fields[1]
except EnvironmentError:
pass
result['running'] = '' != self.__which_job_is_running()
return result
@sanitize(
job=ChoicesSanitizer(INSTALLERS, required=True),
detail=StringSanitizer(r'^[A-Za-z0-9\.\- ]*$'),
)
@simple_response
def run_installer(self, job: str, detail: str = '') -> Dict[str, int]:
"""
This is the function that invokes any kind of installer. Arguments accepted:
:param job: ..... the main thing to do. can be one of:
'release' ...... perform a release update
'distupgrade' .. update all currently installed packages (distupgrade)
:param detail: ....... an argument that specifies the subject of the installer:
for 'release' .... the target release number,
for all other subjects: detail has no meaning.
"""
MODULE.info("Starting function %r" % (job, ))
self._current_job = job
spec = INSTALLERS[job]
# remember initial lines of logfile before starting update to not show it in the frontend
logfile = spec['logfile']
try:
with open(logfile, 'rb') as fd:
self._logfile_start_line = sum(1 for line in fd)
except EnvironmentError:
pass
command = spec['command']
if '%' in command:
command = command % (pipes.quote(detail).translate({
0: None,
10: None,
13: None
}), )
MODULE.info("Creating job: %r" % (command, ))
command = '''
%s
%s < /dev/null
%s''' % (spec["prejob"], command, spec["postjob"])
atjobs.add(command, comments=dict(lines=self._logfile_start_line))
return {'status': 0}
def __which_job_is_running(self) -> str:
# first check running at jobs
for atjob in atjobs.list(True):
for job, inst in INSTALLERS.items():
cmd = inst['command'].split('%')[0]
if cmd in atjob.command:
self._current_job = job
try:
self._logfile_start_line = int(
atjob.comments.get('lines', 0))
except ValueError:
pass
return job
# no atjob found, parse process list (if univention-upgrade was started via CLI)
commands = [
('/usr/share/univention-updater/univention-updater-umc-dist-upgrade',
'distupgrade'),
('/usr/share/univention-updater/univention-updater', 'release'),
('/usr/sbin/univention-upgrade', 'distupgrade'
) # we don't know if it is a dist-upgrade or a release upgrade
]
for cmd, job in commands:
for process in psutil.process_iter():
try:
cmdline = process.cmdline() if callable(
process.cmdline) else process.cmdline
except psutil.NoSuchProcess:
pass
if cmd in cmdline:
self._current_job = job
self._logfile_start_line = 0
return job
return ''
class Instance(SchoolBaseModule):
def query(self, request):
"""Searches for internet filter rules
requests.options = {}
'pattern' -- pattern to match within the rule name or the list of domains
"""
MODULE.info('internetrules.query: options: %s' % str(request.options))
pattern = request.options.get('pattern', '').lower()
def _matchDomain(domains):
# helper function to match pattern within the list of domains
matches = [idom for idom in domains if pattern in idom.lower()]
return 0 < len(matches)
# filter out all rules that match the given pattern
result = [
dict(
name=irule.name,
type=_filterTypesInv[irule.type],
domains=len(irule.domains),
priority=irule.priority,
wlan=irule.wlan,
) for irule in rules.list()
if pattern in irule.name.lower() or _matchDomain(irule.domains)
]
MODULE.info('internetrules.query: results: %s' % str(result))
self.finished(request.id, result)
@sanitize(StringSanitizer())
def get(self, request):
"""Returns the specified rules
requests.options = [ <ruleName>, ... ]
"""
MODULE.info('internetrules.get: options: %s' % str(request.options))
result = []
# fetch all rules with the given names (we need to make sure that "name" is UTF8)
names = set(iname.encode('utf8') for iname in request.options)
result = [
dict(
name=irule.name,
type=_filterTypesInv[irule.type],
domains=irule.domains,
priority=irule.priority,
wlan=irule.wlan,
) for irule in rules.list() if irule.name in names
]
MODULE.info('internetrules.get: results: %s' % str(result))
self.finished(request.id, result)
@sanitize(DictSanitizer(dict(object=StringSanitizer()), required=True))
def remove(self, request):
"""Removes the specified rules
requests.options = [ { "object": <ruleName> }, ... ]
"""
MODULE.info('internetrules.remove: options: %s' % str(request.options))
result = []
# fetch all rules with the given names
for ientry in request.options:
iname = ientry['object']
success = False
if iname:
success = rules.remove(iname)
result.append(dict(name=iname, success=success))
MODULE.info('internetrules.remove: results: %s' % str(result))
self.finished(request.id, result)
@staticmethod
def _parseRule(iprops, forceAllProperties=False):
# validate types
for ikey, itype in (('name', basestring), ('type', basestring),
('priority', (int, basestring)), ('wlan', bool),
('domains', list)):
if ikey not in iprops:
if forceAllProperties:
# raise exception as the key is not present
raise ValueError(
_('The key "%s" has not been specified: %s') %
(ikey, iprops))
continue
if not isinstance(iprops[ikey], itype):
typeStr = ''
if isinstance(itype, tuple):
typeStr = ', '.join([i.__name__ for i in itype])
else:
typeStr = itype.__name__
raise ValueError(
_('The key "%s" needs to be of type: %s') %
(ikey, typeStr))
# validate name
if 'name' in iprops and not univention.config_registry.validate_key(
iprops['name'].encode('utf-8')):
raise ValueError(
_('Invalid rule name "%s". The name needs to be a string, the following special characters are not allowed: %s'
) %
(iprops.get('name'),
'!, ", §, $, %, &, (, ), [, ], {, }, =, ?, `, +, #, \', ",", ;, <, >, \\'
))
# validate type
if 'type' in iprops and iprops['type'] not in _filterTypes:
raise ValueError(_('Filter type is unknown: %s') % iprops['type'])
# validate domains
if 'domains' in iprops:
parsedDomains = []
for idomain in iprops['domains']:
def _validValueChar():
# helper function to check for invalid characters
for ichar in idomain:
if ichar in univention.config_registry.backend.INVALID_VALUE_CHARS:
return False
return True
if not isinstance(idomain,
basestring) or not _validValueChar():
raise ValueError(_('Invalid domain '))
# parse domain
domain = idomain
if '://' not in domain:
# make sure that we have a scheme defined for parsing
MODULE.info(
'Adding a leading scheme for parsing of domain: %s' %
idomain)
domain = 'http://%s' % domain
domain = urlparse(domain).hostname
MODULE.info('Parsed domain: %s -> %s' % (idomain, domain))
if not domain:
raise ValueError(
_('The specified domain "%s" is not valid. Please specify a valid domain name, such as "wikipedia.org", "facebook.com"'
) % idomain)
# add domain to list of parsed domains
parsedDomains.append(domain)
# save parsed domains in the dict
iprops['domains'] = parsedDomains
return iprops
@sanitize(
DictSanitizer(dict(object=DictSanitizer(dict(
name=StringSanitizer(required=True),
type=ChoicesSanitizer(list(_filterTypes.keys()), required=True),
wlan=BooleanSanitizer(required=True),
priority=IntegerSanitizer(required=True),
domains=ListSanitizer(StringSanitizer(required=True),
required=True),
),
required=True)),
required=True))
def add(self, request):
"""Add the specified new rules:
requests.options = [ {
'object': {
'name': <str>,
'type': 'whitelist' | 'blacklist',
'priority': <int> | <str>,
'wlan': <bool>,
'domains': [<str>, ...],
}
}, ... ]
"""
# try to create all specified projects
result = []
for ientry in request.options:
iprops = ientry['object']
try:
# make sure that the rule does not already exist
irule = rules.load(iprops['name'])
if irule:
raise ValueError(
_('A rule with the same name does already exist: %s') %
iprops['name'])
# parse the properties
parsedProps = self._parseRule(iprops, True)
# create a new rule from the user input
newRule = rules.Rule(
name=parsedProps['name'],
type=_filterTypes[parsedProps['type']],
priority=parsedProps['priority'],
wlan=parsedProps['wlan'],
domains=parsedProps['domains'],
)
# try to save filter rule
newRule.save()
MODULE.info('Created new rule: %s' % newRule)
# everything ok
result.append(dict(name=iprops['name'], success=True))
except (ValueError, KeyError) as e:
# data not valid... create error info
MODULE.info(
'data for internet filter rule "%s" is not valid: %s' %
(iprops.get('name'), e))
result.append(
dict(name=iprops.get('name'),
success=False,
details=str(e)))
# return the results
self.finished(request.id, result)
@sanitize(
DictSanitizer(dict(
object=DictSanitizer(dict(
name=StringSanitizer(required=True),
type=ChoicesSanitizer(list(_filterTypes.keys()),
required=True),
wlan=BooleanSanitizer(required=True),
priority=IntegerSanitizer(required=True),
domains=ListSanitizer(StringSanitizer(required=True),
required=True),
),
required=True),
options=DictSanitizer(dict(name=StringSanitizer()), required=True),
),
required=True))
def put(self, request):
"""Modify an existing rules:
requests.options = [ {
'object': {
'name': <str>, # optional
'type': 'whitelist' | 'blacklist', # optional
'priority': <int>, # optional
'wlan': <bool>, # optional
'domains': [<str>, ...], # optional
},
'options': {
'name': <str> # the original name of the object
}
}, ... ]
"""
# try to create all specified projects
result = []
for ientry in request.options:
try:
# get properties and options from entry
iprops = ientry['object']
iname = None
ioptions = ientry.get('options')
if ioptions:
iname = ioptions.get('name')
if not iname:
raise ValueError(
_('No "name" attribute has been specified in the options.'
))
# make sure that the rule already exists
irule = rules.load(iname)
if not irule:
raise ValueError(
_('The rule does not exist and cannot be modified: %s')
% iprops.get('name', ''))
# parse the properties
self._parseRule(iprops)
if iprops.get('name', iname) != iname:
# name has been changed -> remove old rule and create a new one
rules.remove(iname)
irule.name = iprops['name']
if 'type' in iprops:
# set rule type, move all domains from the previous type
oldDomains = irule.domains
irule.domains = []
irule.type = _filterTypes[iprops['type']]
irule.domains = oldDomains
if 'priority' in iprops:
# set priority
irule.priority = iprops['priority']
if 'wlan' in iprops:
# set wlan
irule.wlan = iprops['wlan']
if 'domains' in iprops:
# set domains
irule.domains = iprops['domains']
# try to save filter rule
irule.save()
MODULE.info('Saved rule: %s' % irule)
# everything ok
result.append(dict(name=iname, success=True))
except ValueError as e:
# data not valid... create error info
MODULE.info(
'data for internet filter rule "%s" is not valid: %s' %
(iprops.get('name'), e))
result.append(
dict(name=iprops.get('name'),
success=False,
details=str(e)))
# return the results
self.finished(request.id, result)
@sanitize(school=SchoolSanitizer(required=True),
pattern=StringSanitizer(default=''))
@LDAP_Connection()
def groups_query(self, request, ldap_user_read=None, ldap_position=None):
"""List all groups (classes, workgroups) and their assigned internet rule"""
pattern = LDAP_Filter.forAll(request.options.get('pattern', ''),
['name', 'description'])
school = request.options['school']
groups = [
x for x in Group.get_all(ldap_user_read, school, pattern)
if not x.self_is_computerroom()
]
internet_rules = rules.getGroupRuleName([i.name for i in groups])
name = re.compile('-%s$' % (re.escape(school)), flags=re.I)
result = [{
'name':
i.get_relative_name()
if hasattr(i, 'get_relative_name') else name.sub('', i.name),
'$dn$':
i.dn,
'rule':
internet_rules.get(i.name, 'default')
or _('-- Default (unrestricted) --')
} for i in groups]
result.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()),
key=lambda x: x['name'])
self.finished(request.id, result)
@sanitize(
DictSanitizer(
dict(
group=StringSanitizer(required=True),
rule=StringSanitizer(required=True),
)))
@LDAP_Connection()
def groups_assign(self, request, ldap_user_read=None, ldap_position=None):
"""Assigns default rules to groups:
request.options = [ { 'group': <groupDN>, 'rule': <ruleName> }, ... ]
"""
MODULE.info('internetrules.groups_assign: options: %s' %
str(request.options))
# try to load all group rules
newRules = {}
rmRules = []
for ientry in request.options:
# make sure the group exists
igrp = udm_objects.get(udm_modules.get('groups/group'), None,
ldap_user_read, ldap_position,
ientry['group'])
if not igrp:
raise UMC_Error('unknown group object')
igrp.open()
# check the rule name
irule = ientry['rule']
if irule == '$default$':
# remove the rule
rmRules.append(igrp['name'])
else:
try:
# make sure the rule name is valid
self._parseRule(dict(name=irule))
except ValueError as exc:
raise UMC_Error(str(exc))
# add new rule
newRules[igrp['name']] = irule
# assign default filter rules to groups
rules.setGroupRuleName(newRules)
rules.unsetGroupRuleName(rmRules)
MODULE.info('internetrules.groups_assign: finished')
self.finished(request.id, True)