class Instance(Base): def init(self): """Initialize the module with some values""" super(Instance, self).init() self.data = [ int(x) for x in ucr.get('some/examle/ucr/variable', '1,2,3').split(',') ] def query(self, request): """get all values of self.data""" self.finished(request.id, self.data) @sanitize(item=IntegerSanitizer(required=True)) def get(self, request): """get a specific item of self.data""" try: item = self.data[request.options['item']] except IndexError: MODULE.error('A invalid item was accessed.') raise UMC_Error(_('The item %d does not exists.') % (request.options['item'], ), status=400) self.finished(request.id, self.data[item]) @sanitize(IntegerSanitizer(required=True)) def put(self, request): """replace all data with the list provided in request.options""" self.data = request.options self.finished(request.id, None)
class Instance(Base): @sanitize( pattern=PatternSanitizer(default='.*'), category=ChoicesSanitizer(choices=['user', 'pid', 'command', 'all'], default='all') ) @simple_response def query(self, pattern, category='all'): processes = [] for process in psutil.process_iter(): try: cpu_time = process.cpu_times() proc = { 'timestamp': time.time(), 'cpu_time': cpu_time.user + cpu_time.system, 'user': process.username(), 'pid': process.pid, 'cpu': 0.0, 'mem': process.memory_percent(), 'command': ' '.join(process.cmdline() or []) or process.name(), } except psutil.NoSuchProcess: continue categories = [category] if category == 'all': categories = ['user', 'pid', 'command'] if any(pattern.match(str(proc[cat])) for cat in categories): processes.append(proc) # Calculate correct cpu percentage time.sleep(1) for process_entry in processes: try: process = psutil.Process(process_entry['pid']) cpu_time = process.cpu_times() except psutil.NoSuchProcess: continue elapsed_time = time.time() - process_entry.pop('timestamp') elapsed_cpu_time = cpu_time.user + cpu_time.system - process_entry.pop('cpu_time') cpu_percent = (elapsed_cpu_time / elapsed_time) * 100 process_entry['cpu'] = cpu_percent return processes @sanitize( signal=ChoicesSanitizer(choices=['SIGTERM', 'SIGKILL']), pid=ListSanitizer(IntegerSanitizer()) ) @simple_response def kill(self, signal, pid): failed = [] for pid_ in pid: try: process = psutil.Process(pid_) if signal == 'SIGTERM': process.terminate() elif signal == 'SIGKILL': process.kill() except psutil.NoSuchProcess as exc: failed.append(str(pid_)) MODULE.error('Could not %s pid %s: %s' % (signal, pid_, exc)) if failed: failed = ', '.join(failed) raise UMC_Error(_('No process found with PID %s') % (failed))
class Instance(Base, ProgressMixin): def __init__(self, *args, **kwargs): Base.__init__(self, *args, **kwargs) ProgressMixin.__init__(self) self._finishedLock = threading.Lock() self._finishedResult = True self._progressParser = util.ProgressParser() self.__keep_alive_request = None self._net_apply_running = 0 # reset umask to default os.umask(0o022) def init(self): os.putenv('LANG', str(self.locale)) _locale.setlocale(_locale.LC_ALL, str(self.locale)) if not util.is_system_joined(): self._preload_city_data() def _preload_city_data(self): util.get_city_data() util.get_country_data() def _get_localized_label(self, label_dict): # return the correctly loca return label_dict.get(self.locale.language) or label_dict.get( 'en', '') or label_dict.get('', '') def ping(self, request): if request.options.get('keep_alive'): self.__keep_alive_request = request return self.finished(request.id, None) @simple_response def close_browser(self): try: with open('/var/cache/univention-system-setup/browser.pid', 'rb') as fd: pid = int(fd.readline().strip()) process = psutil.Process(pid) process.kill() return True except IOError as exc: MODULE.warn('cannot open browser PID file: %s' % (exc, )) except ValueError as exc: MODULE.error('browser PID is not a number: %s' % (exc, )) except psutil.NoSuchProcess as exc: MODULE.error('cannot kill process with PID: %s' % (exc, )) return False @simple_response def load(self): '''Return a dict with all necessary values for system-setup read from the current status of the system.''' return util.load_values(self.locale.language) @simple_response def save_keymap(self, layout=None): '''Set the systems x-keymap according to request.options[keymap]''' # Don't set in debian installer mode if ucr.is_true('system/setup/boot/installer'): return True if layout: subprocess.call( ['/usr/bin/setxkbmap', '-display', ':0', '-layout', layout]) return True def save(self, request): '''Reconfigures the system according to the values specified in the dict given as option named "values".''' # get new values values = request.options.get('values', {}) run_hooks = request.options.get('run_hooks', False) script_args = [] if run_hooks: # create a status file that indicates that save has been triggered util.create_status_file() # enforce particular arguments for setup scripts script_args = [ '--appliance-mode', '--force-recreate', '--demo-mode' ] def _thread(request, obj): # acquire the lock until the scripts have been executed self._finishedResult = False obj._finishedLock.acquire() try: subfolders = { 'network': ['30_net'], 'certificate': ['40_ssl'], 'languages': ['15_keyboard', '20_language', '35_timezone'], }.get(request.flavor) self._progressParser.reset(subfolders) if request.flavor == 'setup': # adjust progress fractions for setup wizard with pre-configurred settings fractions = self._progressParser.fractions fractions['05_role/10role'] = 0 fractions['10_basis/12domainname'] = 0 fractions['10_basis/14ldap_basis'] = 0 fractions['90_postjoin/10admember'] = 0 self._progressParser.calculateFractions() MODULE.info('saving profile values') util.write_profile(values) if not values: MODULE.error('No property "values" given for save().') return False # in case of changes of the IP address, restart UMC server and web server # for this we ignore changes of virtual or non-default devices # ... no need to restart the UMC server if cleanup scripts are run anyway restart = False if not run_hooks: MODULE.info('Check whether ip addresses have been changed') for ikey, ival in values.iteritems(): if RE_IPV4.match(ikey) or RE_IPV6_DEFAULT.match( ikey) or RE_SSL.match(ikey): restart = True break MODULE.info('Restart servers: %s' % restart) # on a joined system or on a basesystem, we can run the setup scripts MODULE.info('runnning system setup scripts (flavor %r)' % (request.flavor, )) util.run_scripts(self._progressParser, restart, subfolders, lang=str(self.locale), args=script_args) # run cleanup scripts and appliance hooks if needed if run_hooks: util.cleanup(with_appliance_hooks=True) # done :) self._finishedResult = True return True finally: obj._finishedLock.release() def _finished(thread, result): if self.__keep_alive_request: self.finished(self.__keep_alive_request.id, None) self.__keep_alive_request = None if isinstance(result, BaseException): msg = ''.join(thread.trace + traceback.format_exception_only( *thread.exc_info[:2])) MODULE.warn('Exception during saving the settings: %s' % (msg, )) self._progressParser.current.errors.append( _('Encountered unexpected error during setup process: %s') % result) self._progressParser.current.critical = True self._finishedResult = True thread = notifier.threads.Simple( 'save', notifier.Callback(_thread, request, self), _finished) thread.run() self.finished(request.id, None) @simple_response def join(self, values=None, dcname=None, username=None, password=None): '''Join and reconfigure the system according to the values specified in the dict given as option named "values".''' # get old and new values orgValues = util.load_values() values = values or {} # determine new system role oldrole = orgValues.get('server/role', '') newrole = values.get('server/role', oldrole) # create a status file that indicates that save has been triggered util.create_status_file() def _thread(obj, username, password): # acquire the lock until the scripts have been executed self._finishedResult = False obj._finishedLock.acquire() try: self._progressParser.reset() # write the profile file and run setup scripts util.auto_complete_values_for_join(values) # on unjoined DC master the nameserver must be set to the external nameserver if newrole == 'domaincontroller_master' and not orgValues.get( 'joined'): for i in range(1, 4): # overwrite these values only if they are set, because the UMC module # will save only changed values if values.get('dns/forwarder%d' % i): values['nameserver%d' % i] = values.get( 'dns/forwarder%d' % i) MODULE.info('saving profile values') util.write_profile(values) # unjoined DC master (that is not being converted to a basesystem) -> run the join script MODULE.info('runnning system setup join script') util.run_joinscript(self._progressParser, values, username, password, dcname, lang=str(self.locale)) # done :) self._finishedResult = True return True finally: obj._finishedLock.release() def _finished(thread, result): if self.__keep_alive_request: self.finished(self.__keep_alive_request.id, None) self.__keep_alive_request = None if isinstance(result, BaseException): msg = ''.join(thread.trace + traceback.format_exception_only( *thread.exc_info[:2])) MODULE.warn('Exception during saving the settings: %s' % (msg, )) self._progressParser.current.errors.append( _('Encountered unexpected error during setup process: %s') % (result, )) self._progressParser.current.critical = True self._finishedResult = True thread = notifier.threads.Simple( 'join', notifier.Callback(_thread, self, username, password), _finished) thread.run() return def check_finished(self, request): '''Check whether the join/setup scripts are finished. This method implements a long polling request, i.e., the request is only finished at the moment when all scripts have been executed or due to a timeout. If it returns because of the timeout, a new try can be started.''' def _thread(request, obj): def progress_info(state, **kwargs): info = { 'component': state.fractionName, 'info': state.message, 'errors': state.errors, 'critical': state.critical, 'steps': state.percentage } info.update(kwargs) MODULE.info( 'Progress state: %(steps).1f%% - %(component)s - %(info)s' % info) return info # acquire the lock in order to wait for the join/setup scripts to finish # do this for 30 sec and then return anyway SLEEP_TIME = 0.200 WAIT_TIME = 30 ntries = WAIT_TIME / SLEEP_TIME while not obj._finishedLock.acquire(False): if ntries <= 0 or self._progressParser.changed and self._progressParser.current: state = self._progressParser.current return progress_info(state, finished=False) time.sleep(SLEEP_TIME) ntries -= 1 obj._finishedLock.release() # scripts are done, return final result # return all errors that we gathered throughout the setup state = self._progressParser.current return progress_info(state, finished=obj._finishedResult) thread = notifier.threads.Simple( 'check_finished', notifier.Callback(_thread, request, self), notifier.Callback(self.thread_finished_callback, request)) thread.run() @simple_response(with_flavor=True) def validate(self, values=None, flavor=None): '''Validate the specified values given in the dict as option named "values". Return a dict (with variable names as key) of dicts with the structure: { "valid": True/False, "message": "..." }''' # init variables messages = [] values = values or {} orgValues = util.load_values() is_wizard_mode = flavor == 'wizard' # determine new system role newrole = values.get('server/role', orgValues.get('server/role', '')) ad_member = values.get('ad/member', orgValues.get('ad/member', '')) # mix original and new values allValues = copy.copy(values) for ikey, ival in orgValues.iteritems(): if ikey not in allValues: allValues[ikey] = ival # helper functions # TODO: 'valid' is not correctly evaluated in frontend # i.e. if valid you may continue without getting message def _check(key, check, message, critical=True): if key not in values: return if not check(values[key]): messages.append({ 'message': message, 'valid': not critical, 'key': key }) def _append(key, message): MODULE.warn('Validation failed for key %s: %s' % (key, message)) messages.append({'key': key, 'valid': False, 'message': message}) # host and domain name packages = set(values.get('components', [])) _check( 'hostname', util.is_hostname, _('The hostname or the hostname part of the fully qualified domain name is invalid. Please go back to the host setting and make sure, that the hostname only contains letter (a-zA-Z) and digits (0-9).' )) hostname_length_critical = ad_member or 'univention-samba' in packages or 'univention-samba4' in packages appliance_str = _('the UCS system') if ucr['umc/web/appliance/name']: appliance_str = _('the %s appliance') % ( ucr['umc/web/appliance/name'], ) hostname_length_message = _( 'A valid NetBIOS name can not be longer than 13 characters. If Samba is installed, the hostname should be shortened.' ) if hostname_length_critical else _( 'The hostname %s is longer than 13 characters. It will not be possible to install an Active Directory compatible Domaincontroller (Samba 4) or UCS@school. The hostname cannot be changed after the installation of %s. It is recommended to shorten the hostname to maximal 13 characters.' ) % ( values.get('hostname', ''), appliance_str, ) _check('hostname', lambda x: len(x) <= 13, hostname_length_message, critical=hostname_length_critical) _check( 'domainname', util.is_domainname, _("Please enter a valid fully qualified domain name (e.g. host.example.com)." )) hostname = allValues.get('hostname', '') domainname = allValues.get('domainname', '') if hostname or domainname: if len('%s%s' % (hostname, domainname)) >= 63: _append( 'domainname', _('The length of fully qualified domain name is greater than 63 characters.' )) if hostname == domainname.split('.')[0]: _append('domainname', _("Hostname is equal to domain name.")) if is_wizard_mode and not util.is_system_joined(): if newrole == 'domaincontroller_master' and not values.get( 'domainname'): _append( 'domainname', _("No fully qualified domain name has been specified for the system." )) elif not values.get('hostname'): _append('hostname', _("No hostname has been specified for the system.")) # windows domain _check( 'windows/domain', lambda x: x == x.upper(), _("The windows domain name can only consist of upper case characters." )) _check( 'windows/domain', lambda x: len(x) <= 15, _("The windows domain name cannot be longer than 15 characters.")) _check('windows/domain', util.is_windowsdomainname, _("The windows domain name is not valid.")) # LDAP base _check( 'ldap/base', util.is_ldap_base, _("The LDAP base may neither contain blanks nor any special characters. Its structure needs to consist of at least two relative distinguished names (RDN) which may only use the attribute tags 'dc', 'cn', 'c', 'o', or 'l' (e.g., dc=test,dc=net)." )) # root password _check( 'root_password', lambda x: len(x) >= 8, _("The root password is too short. For security reasons, your password must contain at least 8 characters." )) _check('root_password', util.is_ascii, _("The root password may only contain ascii characters.")) # ssl + email labels = { 'ssl/country': _('Country'), 'ssl/state': _('State'), 'ssl/locality': _('Location'), 'ssl/organization': _('Organization'), 'organization': _('Organization'), 'ssl/organizationalunit': _('Business unit'), 'ssl/email': _('Email address'), 'email_address': _('Email address'), 'ssl/common': _('Common name for the root SSL certificate'), } for maxlenth, keys in [ (2, ('ssl/country', )), (128, ( 'ssl/state', 'ssl/locality', )), (64, ('organization', 'ssl/organization', 'ssl/organizationalunit', 'ssl/email', 'email_address', 'ssl/common')) ]: for ikey in keys: _check( ikey, lambda x: len(x) <= maxlenth, _('The following value is too long, only %(max)s characters allowed: %(name)s' ) % { 'max': maxlenth, 'name': labels[ikey] }) for ikey in ('ssl/country', 'ssl/state', 'ssl/locality', 'ssl/organization', 'ssl/organizationalunit', 'ssl/email', 'ssl/common'): for table in (stringprep.in_table_c21_c22, stringprep.in_table_a1, stringprep.in_table_c8, stringprep.in_table_c3, stringprep.in_table_c4, stringprep.in_table_c5, lambda c: c == u'\ufffd'): _check( ikey, lambda x: not any(map(table, unicode(x))), _('The value for %s contains invalid characters.') % (labels[ikey], )) _check('ssl/country', lambda x: len(x) == 2, _('Country must be a country code consisting of 2 characters.')) for ikey in ['ssl/email', 'email_address']: _check(ikey, lambda x: x.find('@') > 0, _("Please enter a valid email address")) # net try: interfaces = network.Interfaces() interfaces.from_dict(allValues.get('interfaces', {})) interfaces.check_consistency() except network.DeviceError as exc: _append('interfaces', str(exc)) # validate the primary network interface _check('interfaces/primary', lambda x: not x or x in interfaces, _('The primary network device must exist.')) # check nameservers for ikey, iname in [('nameserver[1-3]', _('Domain name server')), ('dns/forwarder[1-3]', _('External name server'))]: reg = re.compile('^(%s)$' % ikey) for jkey, jval in values.iteritems(): if reg.match(jkey): if not values.get(jkey): # allow empty value continue _check( jkey, util.is_ipaddr, _('The specified IP address (%(name)s) is not valid: %(value)s' ) % { 'name': iname, 'value': jval }) if is_wizard_mode and not util.is_system_joined() and ( newrole not in ['domaincontroller_master', 'basesystem'] or ad_member): if all(nameserver in values and not values[nameserver] for nameserver in ('nameserver1', 'nameserver2', 'nameserver3')): # 'nameserver1'-key exists → widget is displayed → = not in UCS/debian installer mode if not any(interface.ip4dynamic or interface.ip6dynamic for interface in interfaces.values()): _append('nameserver1', _('A domain name server needs to be specified.')) # _append('nameserver1', _('At least one domain name server needs to be given if DHCP or SLAAC is not specified.')) # see whether the domain can be determined automatically ucr.load() guessed_domain = None for obj in [values, ucr]: for nameserver in ('nameserver1', 'nameserver2', 'nameserver3'): nameserver = obj.get(nameserver) if nameserver: guessed_domain = None if obj.get('ad/member') and obj.get('ad/address'): try: ad_domain_info = lookup_adds_dc( obj.get('ad/address'), ucr={'nameserver1': nameserver}) except failedADConnect: pass else: guessed_domain = ad_domain_info['Domain'] else: guessed_domain = util.get_ucs_domain(nameserver) if guessed_domain: differing_domain_name = values.get( 'domainname') and values['domainname'].lower( ) != guessed_domain.lower() if differing_domain_name: _append( 'domainname', _('The specified domain name is different to the %s domain name found via the configured DNS server: %s' ) % ( _('Active Directory') if ad_member else _('UCS'), guessed_domain, )) else: # communicate guessed domainname to frontend messages.append({ 'valid': True, 'key': 'domainname', 'value': guessed_domain, }) break if guessed_domain: break if not guessed_domain: if not values.get('domainname'): _append( 'domainname', _('Cannot automatically determine the domain. Please specify the server\'s fully qualified domain name.' )) if values.get('nameserver1') and values.get('start/join'): _append( 'nameserver1', _('The specified nameserver %s is not part of a valid UCS domain.' ) % (values['nameserver1'], )) # check gateways if values.get('gateway'): # allow empty value _check( 'gateway', util.is_ipv4addr, _('The specified gateway IPv4 address is not valid: %s') % values.get('gateway')) if values.get('ipv6/gateway'): # allow empty value _check( 'ipv6/gateway', util.is_ipv6addr, _('The specified gateway IPv6 address is not valid: %s') % values.get('ipv6/gateway')) # proxy _check( 'proxy/http', util.is_proxy, _('The specified proxy address is not valid (e.g., http://10.201.1.1:8080): %s' ) % allValues.get('proxy/http', '')) # software checks if 'univention-virtual-machine-manager-node-kvm' in packages and 'univention-virtual-machine-manager-node-xen' in packages: _append( 'components', _('It is not possible to install KVM and XEN components on one system. Please select only one of these components.' )) if 'univention-samba' in packages and 'univention-samba4' in packages: _append( 'components', _('It is not possible to install Samba 3 and Samba 4 on one system. Please select only one of these components.' )) return messages @sanitize(pattern=PatternSanitizer(default='.*', required=True, add_asterisks=True)) @simple_response def lang_locales(self, pattern, category='language_en'): '''Return a list of all available locales.''' return util.get_available_locales(pattern, category) def lang_timezones(self, request): '''Return a list of all available time zones.''' try: file = open('/usr/share/univention-system-setup/locale/timezone') except EnvironmentError: MODULE.error( 'Cannot find locale data for timezones in /usr/share/univention-system-setup/locale' ) self.finished(request.id, None) return timezones = [i.strip('\n') for i in file if not i.startswith('#')] self.finished(request.id, timezones) @simple_response def lang_keyboard_model(self): '''Return a list of all available keyboard models.''' tree = lxml.etree.parse(open('/usr/share/X11/xkb/rules/base.xml')) models = tree.xpath("//model") model_result = [{ 'label': i18nXKeyboard.translate( model.xpath('./configItem/description')[0].text), 'id': model.xpath('./configItem/name')[0].text } for model in models] return model_result @simple_response def lang_keyboard_layout(self): '''Return a list of all available keyboard layouts.''' tree = lxml.etree.parse(open('/usr/share/X11/xkb/rules/base.xml')) layouts = tree.xpath("//layout") layout_result = [{ 'label': i18nXKeyboard.translate( layout.xpath('./configItem/description')[0].text), 'id': layout.xpath('./configItem/name')[0].text, 'language': layout.xpath('./configItem/shortDescription')[0].text, 'countries': ':'.join([ icountry.text for icountry in layout.xpath('./configItem/countryList/*') ]), } for layout in layouts] return layout_result @sanitize(keyboardlayout=StringSanitizer(default='us')) @simple_response def lang_keyboard_variante(self, keyboardlayout): '''Return a list of all available keyboard variantes.''' variante_result = [] tree = lxml.etree.parse(open('/usr/share/X11/xkb/rules/base.xml')) layouts = tree.xpath("//layout") for layout in layouts: layoutID = layout.xpath("./configItem/name")[0].text if layoutID != keyboardlayout: continue variants = layout.xpath("./variantList/variant") variante_result += [{ 'label': i18nXKeyboard.translate( variant.xpath('./configItem/description')[0].text), 'id': variant.xpath('./configItem/name')[0].text } for variant in variants] variante_result.insert(0, {'label': '', 'id': ''}) return variante_result def lang_countrycodes(self, request): '''Return a list of all countries with their two letter chcountry codes.''' country_data = util.get_country_data() countries = [{ 'id': icountry, 'label': self._get_localized_label(idata.get('label', {})), } for icountry, idata in country_data.iteritems() if idata.get('label')] # add the value from ucr value to the list # this is required because invalid values will be unset in frontend # Bug #26409 tmpUCR = univention.config_registry.ConfigRegistry() tmpUCR.load() ssl_country = tmpUCR.get('ssl/country') if ssl_country not in [i['id'] for i in countries]: countries.append({'label': ssl_country, 'id': ssl_country}) self.finished(request.id, countries) def net_apply(self, request): if self._net_apply_running > 0: # do not start another process applying the network settings return False values = request.options.get('values', {}) demo_mode = request.options.get('demo_mode', False) def _thread(obj): obj._net_apply_running += 1 MODULE.process('Applying network settings') with util.written_profile(values): util.run_networkscrips(demo_mode) def _finished(thread, result): self._net_apply_running -= 1 self.finished(request.id, True) thread = notifier.threads.Simple('net_apply', notifier.Callback(_thread, self), _finished) thread.run() @simple_response def net_apply_check_finished(self): if self._net_apply_running > 0: # raise an error if net_apply command is still running... # this allows long polling on the client side (poll until successful request) raise RequestTimeout() return self._net_apply_running == 0 @simple_response def net_interfaces(self): '''Return a list of all available network interfaces.''' return [idev['name'] for idev in util.detect_interfaces()] # workaround: use with_progress to make the method threaded @simple_response(with_progress=True) def net_dhclient(self, interface, timeout=10): '''Request a DHCP address. Expects as options a dict containing the key "interface" and optionally the key "timeout" (in seconds).''' return util.dhclient(interface, timeout) @sanitize(locale=StringSanitizer(default='en_US')) @simple_response def reset_locale(self, locale): locale = Locale(locale) locale.codeset = self.locale.codeset MODULE.info('Switching language to: %s' % locale) os.putenv('LANG', str(self.locale)) try: _locale.setlocale(_locale.LC_ALL, str(locale)) except _locale.Error: MODULE.warn( 'Locale %s is not supported, using fallback locale "C" instead.' % locale) _locale.setlocale(_locale.LC_ALL, 'C') self.locale = locale # dynamically change the translation methods _translation.set_language(str(self.locale)) i18nXKeyboard.set_language(str(self.locale)) network._translation.set_language(str(self.locale)) AppCache().clear_cache() @sanitize(pattern=StringSanitizer(), max_results=IntegerSanitizer(minimum=1, default=5)) @simple_response def find_city(self, pattern, max_results): pattern = pattern.decode(self.locale.codeset).lower() MODULE.info('pattern: %s' % pattern) if not pattern: return [] # for the given pattern, find matching cities city_data = util.get_city_data() matches = [] for icity in city_data: match = None for jlabel in icity.get('label', {}).itervalues(): label = jlabel.decode(self.locale.codeset).lower() if pattern in label: # matching score is the overlap if the search pattern and the matched text # (as fraction between 0 and 1) match_score = len(pattern) / float(len(label)) if match and match_score < match['match_score']: # just keep the best match of a city continue if match_score > 0.1: # found a match with more than 10% overlap :) match = icity.copy() match['match'] = jlabel match['match_score'] = match_score if match: matches.append(match) MODULE.info('Search for pattern "%s" with %s matches' % (pattern, len(matches))) if not matches: return None # add additional score w.r.t. the population size of the city # such that the largest city gains additional 0.4 on top max_population = max([imatch['population'] for imatch in matches]) weighted_inv_max_population = 0.6 / float(max_population) for imatch in matches: imatch['final_score'] = imatch[ 'match_score'] + weighted_inv_max_population * imatch[ 'population'] # sort matches... matches.sort(key=lambda x: x['final_score'], reverse=True) MODULE.info('Top 5 matches: %s' % json.dumps(matches[:5], indent=2)) matches = matches[:max_results] # add additional information about keyboard layout, time zone etc. and # get the correct localized labels country_data = util.get_country_data() for imatch in matches: match_country = country_data.get(imatch.get('country')) if match_country: imatch.update(util.get_random_nameserver(match_country)) imatch.update( dict( default_lang=match_country.get('default_lang'), country_label=self._get_localized_label( match_country.get('label', {})), label=self._get_localized_label(imatch.get('label')) or imatch.get('match'), )) return matches @simple_response def apps_query(self): return util.get_apps(True) @simple_response def check_domain(self, role, nameserver): result = {} if role == 'ad': try: ad_domain_info = lookup_adds_dc(nameserver) dc = ad_domain_info['DC DNS Name'] if dc: result['dc_name'] = dc domain = ad_domain_info['Domain'] result['domain'] = domain result['ucs_master'] = util.is_ucs_domain( nameserver, domain) ucs_master_fqdn = util.resolve_domaincontroller_master_srv_record( nameserver, domain) result['ucs_master_fqdn'] = ucs_master_fqdn result['ucs_master_reachable'] = util.is_ssh_reachable( ucs_master_fqdn) except (failedADConnect, connectionFailed) as exc: MODULE.warn('ADDS DC lookup failed: %s' % (exc, )) elif role == 'nonmaster': domain = util.get_ucs_domain(nameserver) if domain: fqdn = util.resolve_domaincontroller_master_srv_record( nameserver, domain) else: fqdn = util.get_fqdn(nameserver) if fqdn: result['dc_name'] = fqdn domain = '.'.join(fqdn.split('.')[1:]) result['ucs_master'] = util.is_ucs_domain(nameserver, domain) return result @simple_response def check_domain_join_information(self, domain_check_role, role, dns, nameserver, address, username, password): result = {} if domain_check_role == 'ad': domain = util.check_credentials_ad(nameserver, address, username, password) result['domain'] = domain if dns: # "dns" means we don't want to replace the existing DC Master ucs_master_fqdn = util.resolve_domaincontroller_master_srv_record( nameserver, domain) if ucs_master_fqdn: # if we found a _domaincontroller_master._tcp SRV record the system will be a DC Backup/Slave/Member. # We need to check the credentials of this system, too, so we ensure that the System is reachable via SSH. # Otherwise the join will fail with strange error like "ping to ..." failed. result.update( receive_domaincontroller_master_information( False, nameserver, ucs_master_fqdn, username, password)) set_role_and_check_if_join_will_work( role, ucs_master_fqdn, username, password) elif domain_check_role == 'nonmaster': result.update( receive_domaincontroller_master_information( dns, nameserver, address, username, password)) set_role_and_check_if_join_will_work(role, address, username, password) # master? basesystem? no domain check necessary return result @simple_response def check_school_information(self, hostname, address, username, password): return check_for_school_domain(hostname, address, username, password) @simple_response def check_repository_accessibility(self): return get_unreachable_repository_servers() @simple_response def check_uid(self, uid, role, address, username, password): return check_if_uid_is_available(uid, role, address, username, password)
class Instance(Base): def init(self): MODULE.info("Initializing 'updater' module (PID = %d)" % (getpid(), )) self._current_job = '' self._logfile_start_line = 0 self._serial_file = Watched_File(COMPONENTS_SERIAL_FILE) self._updates_serial = Watched_Files(UPDATE_SERIAL_FILES) try: self.uu = UniventionUpdater(False) except Exception as exc: # FIXME: let it raise MODULE.error("init() ERROR: %s" % (exc, )) @simple_response def query_maintenance_information(self): ucr.load() if ucr.is_true('license/extended_maintenance/disable_warning'): return {'show_warning': False} version = self.uu.get_ucs_version() try: url = 'http://updates.software-univention.de/download/ucs-maintenance/{}.yaml'.format( version) response = requests.get(url, timeout=10) if not response.ok: response.raise_for_status() status = yaml.load(response.content) if not isinstance(status, dict): raise yaml.YAMLError(repr(status)) # the yaml file contains for maintained either false, true or extended as value. # yaml.load converts true and false into booleans but extended into string. _maintained_status = status.get('maintained') maintenance_extended = _maintained_status == 'extended' show_warning = maintenance_extended or not _maintained_status except yaml.YAMLError as exc: MODULE.error('The YAML format is malformed: %s' % (exc, )) return {'show_warning': False} except requests.exceptions.RequestException as exc: MODULE.error("Querying maintenance information failed: %s" % (exc, )) return {'show_warning': False} return { 'ucs_version': version, 'show_warning': show_warning, 'maintenance_extended': maintenance_extended, 'base_dn': ucr.get('license/base') } @simple_response def poll(self): return True @simple_response def query_releases(self): """ Returns a list of system releases suitable for the corresponding ComboBox """ # be as current as possible. self.uu.ucr_reinit() ucr.load() appliance_mode = ucr.is_true('server/appliance') available_versions, blocking_components = self.uu.get_all_available_release_updates( ) result = [{ 'id': rel, 'label': 'UCS %s' % (rel, ) } for rel in available_versions] # # appliance_mode=no ; blocking_comp=no → add "latest version" # appliance_mode=no ; blocking_comp=yes → no "latest version" # appliance_mode=yes; blocking_comp=no → add "latest version" # appliance_mode=yes; blocking_comp=yes → add "latest version" # if result and (appliance_mode or not blocking_components): # UniventionUpdater returns available version in ascending order, so # the last returned entry is the one to be flagged as 'latest' if there's # no blocking component. result[-1]['label'] = '%s (%s)' % (result[-1]['label'], _('latest version')) return result def call_hooks(self, request): """ Calls the specified hooks and returns data given back by each hook """ def _thread(request): result = {} hookmanager = univention.hooks.HookManager( HOOK_DIRECTORY) # , raise_exceptions=False hooknames = request.options.get('hooks') MODULE.info('requested hooks: %s' % hooknames) for hookname in hooknames: MODULE.info('calling hook %s' % hookname) result[hookname] = hookmanager.call_hook(hookname) MODULE.info('result: %r' % (result, )) return result thread = notifier.threads.Simple( 'call_hooks', notifier.Callback(_thread, request), notifier.Callback(self.thread_finished_callback, request)) thread.run() @simple_response def updates_serial(self): """ Watches the three sources.list snippets for changes """ result = self._updates_serial.timestamp() MODULE.info(" -> Serial for UPDATES is '%s'" % result) return result @simple_response def updates_check(self): """ Returns the list of packages to be updated/installed by a dist-upgrade. """ p0 = subprocess.Popen(['LC_ALL=C apt-get update'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (stdout, stderr) = p0.communicate() p1 = subprocess.Popen(['LC_ALL=C apt-get -u dist-upgrade -s'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (stdout, stderr) = p1.communicate() install = [] update = [] remove = [] for line in stdout.split('\n'): # upgrade: # Inst univention-updater [3.1.1-5] (3.1.1-6.408.200810311159 192.168.0.10) # inst: # Inst mc (1:4.6.1-6.12.200710211124 oxae-update.open-xchange.com) # # *** FIX *** the above example lines ignore the fact that there's # some extra text (occasionally) after the last closing # parenthesis. Until now, I've seen only a pair of empty # brackets [], but who knows... match = re.search('^Inst (\S+)\s+(.*?)\s*\((\S+)\s.*\)', line) if match: pkg = match.group(1) old = match.group(2) ver = match.group(3) if old: update.append([pkg, ver]) else: install.append([pkg, ver]) elif line.startswith('Remv '): l = line.split(' ') pkg = l[1] # i18n: The package version is unknown. ver = _('unknown') if len(l) > 2: ver = l[2].replace('[', '').replace(']', '') remove.append([pkg, ver]) return dict( update=sorted(update), install=sorted(install), remove=sorted(remove), ) @simple_response def updates_available(self): """ Asks if there are package updates available. (don't get confused by the name of the UniventionUpdater function that is called here.) This is a seperate call since it can take an amount of time, thus being invoked by a seperate button (and not in the background) """ ucr.load() try: # be as current as possible. what = 'reinitializing UniventionUpdater' self.uu.ucr_reinit() what = 'checking update availability' return self.uu.component_update_available() except Exception as ex: typ = str(type(ex)).strip('<>') msg = '[while %s] [%s] %s' % (what, typ, str(ex)) MODULE.error(msg) return False def status(self, request): # TODO: remove unneeded things """One call for all single-value variables.""" result = {} ucr.load() try: result['erratalevel'] = int(ucr.get('version/erratalevel', 0)) except ValueError: result['erratalevel'] = 0 result['appliance_mode'] = ucr.is_true('server/appliance') result['easy_mode'] = ucr.is_true('update/umc/updateprocess/easy', False) result['timestamp'] = int(time()) result['reboot_required'] = ucr.is_true('update/reboot/required', False) try: # be as current as possible. what = 'reinitializing UniventionUpdater' self.uu.ucr_reinit() what = 'getting UCS version' result['ucs_version'] = self.uu.get_ucs_version() # if nothing is returned -> convert to empty string. what = 'querying available release updates' try: result[ 'release_update_available'] = self.uu.release_update_available( errorsto='exception') except RequiredComponentError as exc: result['release_update_available'] = exc.version if result['release_update_available'] is None: result['release_update_available'] = '' what = 'querying update-blocking components' blocking_components = self.uu.get_all_available_release_updates( )[1] result['release_update_blocking_components'] = ' '.join( blocking_components or []) what = "querying availability for easy mode" if result['easy_mode']: # updates/available should reflect the need for an update easy_update_available = ucr.is_true('update/available', False) # but dont rely on ucr! update/available is set during univention-upgrade --check # but when was the last time this was run? # release update easy_update_available = easy_update_available or result[ 'release_update_available'] # if no update seems necessary perform a real (expensive) check nonetheless easy_update_available = easy_update_available or self.uu.component_update_available( ) result['easy_update_available'] = bool(easy_update_available) else: result['easy_update_available'] = False # Component counts are now part of the general 'status' data. what = "counting components" c_count = 0 e_count = 0 for comp in self.uu.get_all_components(): c_count = c_count + 1 if ucr.is_true('repository/online/component/%s' % (comp, ), False): e_count = e_count + 1 result['components'] = c_count result['enabled'] = e_count # HACK: the 'Updates' form polls on the serial file # to refresh itself. Including the serial value # into the form helps us to have a dependent field # that can trigger the refresh of the "Releases" # combobox and the 'package updates available' field. result['serial'] = self._serial_file.timestamp() except Exception as exc: # FIXME: don't catch everything typ = str(type(exc)).strip('<>') msg = '[while %s] [%s] %s' % (what, typ, exc) result['message'] = msg result['status'] = 1 MODULE.error(msg) self.finished(request.id, [result]) @simple_response def reboot(self): """ Reboots the computer. Simply invokes /sbin/reboot in the background and returns success to the caller. The caller is prepared for connection loss. """ subprocess.call(['/sbin/reboot']) return True @simple_response def running(self): """ Returns the id (key into INSTALLERS) of a currently running job, or the empty string if nothing is running. """ return self.__which_job_is_running() @sanitize( job=ChoicesSanitizer(INSTALLERS.keys() + [''], required=True), count=IntegerSanitizer(default=0), ) @simple_response def updater_log_file(self, job, count): """ returns the content of the log file associated with the job. Argument 'count' has the same meaning as already known: <0 ...... return timestamp of file (for polling) 0 ....... return whole file as a string list >0 ...... ignore this many lines, return the rest of the file *** NOTE *** As soon as we have looked for a running job at least once, we know the job key and can associate it here. TODO: honor a given 'job' argument """ job = self._current_job or job if not job: return fname = INSTALLERS[job]['logfile'] if count < 0: try: return stat(fname)[9] except (IOError, OSError): return 0 # don't read complete file if we have an 'ignore' count count += self._logfile_start_line return self._logview(fname, -count) def _logview(self, fname, count): """ Contains all functions needed to view or 'tail' an arbitrary text file. Argument 'count' can have different values: < 0 ... ignore this many lines, return the rest of the file 0 ..... return the whole file, split into lines. > 0 ... return the last 'count' lines of the file. (a.k.a. tail -n <count>) """ lines = [] try: with open(fname, 'rb') as fd: for line in fd: if (count < 0): count += 1 else: lines.append(line.rstrip().decode('utf-8', 'replace')) if (count > 0) and (len(lines) > count): lines.pop(0) except (IOError, OSError): pass return lines @sanitize( job=ChoicesSanitizer(INSTALLERS.keys(), required=True), ) @simple_response def updater_job_status(self, job): # TODO: remove this completely """Returns the status of the current/last update even if the job is not running anymore.""" result = {} try: with open(INSTALLERS[job]['statusfile'], 'rb') as fd: for line in fd: fields = line.strip().split('=') if len(fields) == 2: result['_%s_' % fields[0]] = fields[1] except (IOError, OSError): pass result['running'] = '' != self.__which_job_is_running() return result @sanitize( job=ChoicesSanitizer(INSTALLERS.keys(), required=True), detail=StringSanitizer(r'^[A-Za-z0-9\.\- ]*$'), ) @simple_response def run_installer(self, job, detail=''): """ This is the function that invokes any kind of installer. Arguments accepted: job ..... the main thing to do. can be one of: 'release' ...... perform a release update 'distupgrade' .. update all currently installed packages (distupgrade) 'check' ........ check what would be done for 'update' ... do we need this? detail ....... an argument that specifies the subject of the installer: for 'release' .... the target release number, for all other subjects: detail has no meaning. """ MODULE.info("Starting function %r" % (job, )) self._current_job = job # remember initial lines of logfile before starting update to not show it in the frontend logfile = INSTALLERS[job]['logfile'] try: with open(logfile, 'rb') as fd: self._logfile_start_line = sum(1 for line in fd) except (IOError, OSError): pass command = INSTALLERS[job]['command'] if '%' in command: command = command % (pipes.quote(detail).replace('\n', '').replace( '\r', '').replace('\x00', ''), ) MODULE.info("Creating job: %r" % (command, )) command = ''' /usr/share/univention-updater/disable-apache2-umc %s < /dev/null /usr/share/univention-updater/enable-apache2-umc --no-restart''' % (command, ) atjobs.add(command, comments=dict(lines=self._logfile_start_line)) return {'status': 0} def __which_job_is_running(self): # first check running at jobs for atjob in atjobs.list(True): for job, inst in INSTALLERS.iteritems(): cmd = inst['command'].split('%')[0] if cmd in atjob.command: self._current_job = job try: self._logfile_start_line = int( atjob.comments.get('lines', 0)) except ValueError: pass return job # no atjob found, parse process list (if univention-upgrade was started via CLI) commands = [ ('/usr/share/univention-updater/univention-updater-umc-dist-upgrade', 'distupgrade'), ('/usr/share/univention-updater/univention-updater', 'release'), ('/usr/sbin/univention-upgrade', 'distupgrade' ) # we don't know if it is a dist-upgrade or a release upgrade ] for cmd, job in commands: for process in psutil.process_iter(): try: cmdline = process.cmdline() if callable( process.cmdline) else process.cmdline except psutil.NoSuchProcess: pass if cmd in cmdline: self._current_job = job self._logfile_start_line = 0 return job return ''
class Instance(Base): def init(self) -> None: MODULE.info("Initializing 'updater' module (PID = %d)" % (getpid(), )) self._current_job = '' self._logfile_start_line = 0 self._serial_file = Watched_File(COMPONENTS_SERIAL_FILE) try: self.uu = UniventionUpdater(False) except Exception as exc: # FIXME: let it raise self.uu = None MODULE.error("init() ERROR: %s" % (exc, )) @simple_response def query_maintenance_information(self) -> Dict[str, Any]: ret = self._maintenance_information() ret.update(self._last_update()) return ret def _last_update(self) -> Dict[str, Any]: status_file = '/var/lib/univention-updater/univention-updater.status' ret = { 'last_update_failed': False, 'last_update_version': None } # type: Dict[str, Any] try: fstat = stat(status_file) mtime = datetime.fromtimestamp(fstat.st_mtime) delta = datetime.now() - mtime if delta.days != 0: # no fresh failure return ret with open(status_file) as fd: info = dict(line.strip().split('=', 1) # type: ignore for line in fd) # type: Dict[str, str] ret['last_update_failed'] = info.get('status') == 'FAILED' if ret['last_update_failed']: ret['last_update_version'] = info.get('next_version') except (ValueError, EnvironmentError) as exc: MODULE.error(str(exc)) return ret def _maintenance_information(self) -> Dict[str, Any]: default = {'show_warning': False} if not self.uu: return default ucr.load() if ucr.is_true('license/extended_maintenance/disable_warning'): return default version = self.uu.current_version for _ver, data in self.uu.get_releases(version, version): status = data.get('status', 'unmaintained') maintenance_extended = status == 'extended' show_warning = maintenance_extended or status != 'maintained' return { 'ucs_version': str(version), 'show_warning': show_warning, 'maintenance_extended': maintenance_extended, 'base_dn': ucr.get('license/base'), } return default @simple_response def query_releases(self) -> List[Dict[str, str]]: """ Returns a list of system releases suitable for the corresponding ComboBox """ # be as current as possible. self.uu.ucr_reinit() ucr.load() appliance_mode = ucr.is_true('server/appliance') available_versions, blocking_components = self.uu.get_all_available_release_updates( ) result = [{ 'id': str(rel), 'label': 'UCS %s' % (rel, ) } for rel in available_versions] # # appliance_mode=no ; blocking_comp=no → add "latest version" # appliance_mode=no ; blocking_comp=yes → no "latest version" # appliance_mode=yes; blocking_comp=no → add "latest version" # appliance_mode=yes; blocking_comp=yes → add "latest version" # if result and (appliance_mode or not blocking_components): # UniventionUpdater returns available version in ascending order, so # the last returned entry is the one to be flagged as 'latest' if there's # no blocking component. result[-1]['label'] = '%s (%s)' % (result[-1]['label'], _('latest version')) return result @sanitize(hooks=ListSanitizer(StringSanitizer(minimum=1), required=True)) def call_hooks(self, request: Request) -> None: """ Calls the specified hooks and returns data given back by each hook """ def _thread(request: Request) -> Dict[str, Any]: result = {} hookmanager = HookManager( HOOK_DIRECTORY) # , raise_exceptions=False hooknames = request.options.get('hooks') MODULE.info('requested hooks: %s' % hooknames) for hookname in hooknames: MODULE.info('calling hook %s' % hookname) result[hookname] = hookmanager.call_hook(hookname) MODULE.info('result: %r' % (result, )) return result thread = notifier.threads.Simple( 'call_hooks', notifier.Callback(_thread, request), notifier.Callback(self.thread_finished_callback, request)) thread.run() @simple_response def updates_check(self) -> Dict[str, List[Tuple[str, str]]]: """ Returns the list of packages to be updated/installed by a dist-upgrade. """ install = [] update = [] remove = [] apt = Cache(memonly=True) apt.update() apt.open() apt.clear() apt.upgrade(dist_upgrade=True) for pkg in apt.get_changes(): if pkg.marked_install: install.append((pkg.name, pkg.candidate.version)) if pkg.marked_upgrade: update.append((pkg.name, pkg.candidate.version)) if pkg.marked_delete: remove.append((pkg.name, pkg.installed.version)) return dict( update=sorted(update), install=sorted(install), remove=sorted(remove), ) @simple_response def updates_available(self) -> bool: """ Asks if there are package updates available. (don't get confused by the name of the UniventionUpdater function that is called here.) This is a separate call since it can take an amount of time, thus being invoked by a separate button (and not in the background) """ ucr.load() try: # be as current as possible. what = 'reinitializing UniventionUpdater' self.uu.ucr_reinit() what = 'checking update availability' new, upgrade, removed = self.uu.component_update_get_packages() return any((new, upgrade, removed)) except Exception as ex: typ = str(type(ex)).strip('<>') msg = '[while %s] [%s] %s' % (what, typ, str(ex)) MODULE.error(msg) return False def status(self, request: Request) -> None: # TODO: remove unneeded things """One call for all single-value variables.""" result = {} # type: Dict[str, Any] ucr.load() try: result['erratalevel'] = int(ucr.get('version/erratalevel', 0)) except ValueError: result['erratalevel'] = 0 result['appliance_mode'] = ucr.is_true('server/appliance') result['timestamp'] = int(time()) result['reboot_required'] = ucr.is_true('update/reboot/required', False) try: # be as current as possible. what = 'reinitializing UniventionUpdater' self.uu.ucr_reinit() what = 'getting UCS version' result['ucs_version'] = str(self.uu.current_version) # if nothing is returned -> convert to empty string. what = 'querying available release updates' try: ver = self.uu.release_update_available(errorsto='exception') result[ 'release_update_available'] = '' if ver is None else str( ver) except RequiredComponentError as exc: result['release_update_available'] = exc.version what = 'querying update-blocking components' blocking_components = self.uu.get_all_available_release_updates( )[1] or set() # check apps if result['release_update_available']: try: from univention.appcenter.actions import get_action update_check = get_action('update-check') if update_check: blocking_apps = update_check.get_blocking_apps( ucs_version=result['release_update_available']) if blocking_apps: blocking_components.update(set(blocking_apps)) except (ImportError, ValueError): # the new univention.appcenter package is not installed. # Cannot be a dependency as the app center depends on updater... raise UMC_Error( _('Error checking if installed apps are available for next UCS version.' )) result['release_update_blocking_components'] = ' '.join( blocking_components) # Component counts are now part of the general 'status' data. what = "counting components" components = [ bool(comp) for comp in self.uu.get_components(all=True) ] result['components'] = len(components) result['enabled'] = sum(components) # HACK: the 'Updates' form polls on the serial file # to refresh itself. Including the serial value # into the form helps us to have a dependent field # that can trigger the refresh of the "Releases" # combobox and the 'package updates available' field. result['serial'] = self._serial_file.timestamp() except Exception as exc: # FIXME: don't catch everything raise UMC_Error("%s %s %s" % ( _('Error contacting the update server. Please check your proxy or firewall settings, if any. Or it may be a problem with your configured DNS server.' ), _('This is the error message:'), exc, ), traceback=format_exc()) self.finished(request.id, [result]) @simple_response def running(self) -> str: """ Returns the id (key into INSTALLERS) of a currently running job, or the empty string if nothing is running. """ return self.__which_job_is_running() @sanitize( job=ChoicesSanitizer(list(INSTALLERS) + [''], required=True), count=IntegerSanitizer(default=0), ) @simple_response def updater_log_file(self, job: str, count: int) -> Union[None, float, List[str]]: """ returns the content of the log file associated with the job. :param job: Job name. :param count: has the same meaning as already known: <0 ...... return timestamp of file (for polling) 0 ....... return whole file as a string list >0 ...... ignore this many lines, return the rest of the file .. note:: As soon as we have looked for a running job at least once, we know the job key and can associate it here. TODO: honor a given 'job' argument """ job = self._current_job or job if not job: return None fname = INSTALLERS[job]['logfile'] if count < 0: try: return stat(fname).st_ctime except EnvironmentError: return 0 # don't read complete file if we have an 'ignore' count count += self._logfile_start_line return self._logview(fname, -count) def _logview(self, fname: str, count: int) -> List[str]: """ Contains all functions needed to view or 'tail' an arbitrary text file. :param count: can have different values: < 0 ... ignore this many lines, return the rest of the file 0 ..... return the whole file, split into lines. > 0 ... return the last 'count' lines of the file. (a.k.a. tail -n <count>) """ lines = [] try: with open(fname, 'rb') as fd: for line in fd: if (count < 0): count += 1 else: lines.append(line.rstrip().decode('utf-8', 'replace')) if (count > 0) and (len(lines) > count): lines.pop(0) except EnvironmentError: pass return lines @sanitize( job=ChoicesSanitizer(INSTALLERS, required=True), ) @simple_response def updater_job_status( self, job: str) -> Dict[str, Any]: # TODO: remove this completely """Returns the status of the current/last update even if the job is not running anymore.""" result = {} # type: Dict[str, Any] try: with open(INSTALLERS[job]['statusfile'], 'r') as fd: for line in fd: fields = line.strip().split('=') if len(fields) == 2: result['_%s_' % fields[0]] = fields[1] except EnvironmentError: pass result['running'] = '' != self.__which_job_is_running() return result @sanitize( job=ChoicesSanitizer(INSTALLERS, required=True), detail=StringSanitizer(r'^[A-Za-z0-9\.\- ]*$'), ) @simple_response def run_installer(self, job: str, detail: str = '') -> Dict[str, int]: """ This is the function that invokes any kind of installer. Arguments accepted: :param job: ..... the main thing to do. can be one of: 'release' ...... perform a release update 'distupgrade' .. update all currently installed packages (distupgrade) :param detail: ....... an argument that specifies the subject of the installer: for 'release' .... the target release number, for all other subjects: detail has no meaning. """ MODULE.info("Starting function %r" % (job, )) self._current_job = job spec = INSTALLERS[job] # remember initial lines of logfile before starting update to not show it in the frontend logfile = spec['logfile'] try: with open(logfile, 'rb') as fd: self._logfile_start_line = sum(1 for line in fd) except EnvironmentError: pass command = spec['command'] if '%' in command: command = command % (pipes.quote(detail).translate({ 0: None, 10: None, 13: None }), ) MODULE.info("Creating job: %r" % (command, )) command = ''' %s %s < /dev/null %s''' % (spec["prejob"], command, spec["postjob"]) atjobs.add(command, comments=dict(lines=self._logfile_start_line)) return {'status': 0} def __which_job_is_running(self) -> str: # first check running at jobs for atjob in atjobs.list(True): for job, inst in INSTALLERS.items(): cmd = inst['command'].split('%')[0] if cmd in atjob.command: self._current_job = job try: self._logfile_start_line = int( atjob.comments.get('lines', 0)) except ValueError: pass return job # no atjob found, parse process list (if univention-upgrade was started via CLI) commands = [ ('/usr/share/univention-updater/univention-updater-umc-dist-upgrade', 'distupgrade'), ('/usr/share/univention-updater/univention-updater', 'release'), ('/usr/sbin/univention-upgrade', 'distupgrade' ) # we don't know if it is a dist-upgrade or a release upgrade ] for cmd, job in commands: for process in psutil.process_iter(): try: cmdline = process.cmdline() if callable( process.cmdline) else process.cmdline except psutil.NoSuchProcess: pass if cmd in cmdline: self._current_job = job self._logfile_start_line = 0 return job return ''
class Instance(SchoolBaseModule): def query(self, request): """Searches for internet filter rules requests.options = {} 'pattern' -- pattern to match within the rule name or the list of domains """ MODULE.info('internetrules.query: options: %s' % str(request.options)) pattern = request.options.get('pattern', '').lower() def _matchDomain(domains): # helper function to match pattern within the list of domains matches = [idom for idom in domains if pattern in idom.lower()] return 0 < len(matches) # filter out all rules that match the given pattern result = [ dict( name=irule.name, type=_filterTypesInv[irule.type], domains=len(irule.domains), priority=irule.priority, wlan=irule.wlan, ) for irule in rules.list() if pattern in irule.name.lower() or _matchDomain(irule.domains) ] MODULE.info('internetrules.query: results: %s' % str(result)) self.finished(request.id, result) @sanitize(StringSanitizer()) def get(self, request): """Returns the specified rules requests.options = [ <ruleName>, ... ] """ MODULE.info('internetrules.get: options: %s' % str(request.options)) result = [] # fetch all rules with the given names (we need to make sure that "name" is UTF8) names = set(iname.encode('utf8') for iname in request.options) result = [ dict( name=irule.name, type=_filterTypesInv[irule.type], domains=irule.domains, priority=irule.priority, wlan=irule.wlan, ) for irule in rules.list() if irule.name in names ] MODULE.info('internetrules.get: results: %s' % str(result)) self.finished(request.id, result) @sanitize(DictSanitizer(dict(object=StringSanitizer()), required=True)) def remove(self, request): """Removes the specified rules requests.options = [ { "object": <ruleName> }, ... ] """ MODULE.info('internetrules.remove: options: %s' % str(request.options)) result = [] # fetch all rules with the given names for ientry in request.options: iname = ientry['object'] success = False if iname: success = rules.remove(iname) result.append(dict(name=iname, success=success)) MODULE.info('internetrules.remove: results: %s' % str(result)) self.finished(request.id, result) @staticmethod def _parseRule(iprops, forceAllProperties=False): # validate types for ikey, itype in (('name', basestring), ('type', basestring), ('priority', (int, basestring)), ('wlan', bool), ('domains', list)): if ikey not in iprops: if forceAllProperties: # raise exception as the key is not present raise ValueError( _('The key "%s" has not been specified: %s') % (ikey, iprops)) continue if not isinstance(iprops[ikey], itype): typeStr = '' if isinstance(itype, tuple): typeStr = ', '.join([i.__name__ for i in itype]) else: typeStr = itype.__name__ raise ValueError( _('The key "%s" needs to be of type: %s') % (ikey, typeStr)) # validate name if 'name' in iprops and not univention.config_registry.validate_key( iprops['name'].encode('utf-8')): raise ValueError( _('Invalid rule name "%s". The name needs to be a string, the following special characters are not allowed: %s' ) % (iprops.get('name'), '!, ", §, $, %, &, (, ), [, ], {, }, =, ?, `, +, #, \', ",", ;, <, >, \\' )) # validate type if 'type' in iprops and iprops['type'] not in _filterTypes: raise ValueError(_('Filter type is unknown: %s') % iprops['type']) # validate domains if 'domains' in iprops: parsedDomains = [] for idomain in iprops['domains']: def _validValueChar(): # helper function to check for invalid characters for ichar in idomain: if ichar in univention.config_registry.backend.INVALID_VALUE_CHARS: return False return True if not isinstance(idomain, basestring) or not _validValueChar(): raise ValueError(_('Invalid domain ')) # parse domain domain = idomain if '://' not in domain: # make sure that we have a scheme defined for parsing MODULE.info( 'Adding a leading scheme for parsing of domain: %s' % idomain) domain = 'http://%s' % domain domain = urlparse(domain).hostname MODULE.info('Parsed domain: %s -> %s' % (idomain, domain)) if not domain: raise ValueError( _('The specified domain "%s" is not valid. Please specify a valid domain name, such as "wikipedia.org", "facebook.com"' ) % idomain) # add domain to list of parsed domains parsedDomains.append(domain) # save parsed domains in the dict iprops['domains'] = parsedDomains return iprops @sanitize( DictSanitizer(dict(object=DictSanitizer(dict( name=StringSanitizer(required=True), type=ChoicesSanitizer(list(_filterTypes.keys()), required=True), wlan=BooleanSanitizer(required=True), priority=IntegerSanitizer(required=True), domains=ListSanitizer(StringSanitizer(required=True), required=True), ), required=True)), required=True)) def add(self, request): """Add the specified new rules: requests.options = [ { 'object': { 'name': <str>, 'type': 'whitelist' | 'blacklist', 'priority': <int> | <str>, 'wlan': <bool>, 'domains': [<str>, ...], } }, ... ] """ # try to create all specified projects result = [] for ientry in request.options: iprops = ientry['object'] try: # make sure that the rule does not already exist irule = rules.load(iprops['name']) if irule: raise ValueError( _('A rule with the same name does already exist: %s') % iprops['name']) # parse the properties parsedProps = self._parseRule(iprops, True) # create a new rule from the user input newRule = rules.Rule( name=parsedProps['name'], type=_filterTypes[parsedProps['type']], priority=parsedProps['priority'], wlan=parsedProps['wlan'], domains=parsedProps['domains'], ) # try to save filter rule newRule.save() MODULE.info('Created new rule: %s' % newRule) # everything ok result.append(dict(name=iprops['name'], success=True)) except (ValueError, KeyError) as e: # data not valid... create error info MODULE.info( 'data for internet filter rule "%s" is not valid: %s' % (iprops.get('name'), e)) result.append( dict(name=iprops.get('name'), success=False, details=str(e))) # return the results self.finished(request.id, result) @sanitize( DictSanitizer(dict( object=DictSanitizer(dict( name=StringSanitizer(required=True), type=ChoicesSanitizer(list(_filterTypes.keys()), required=True), wlan=BooleanSanitizer(required=True), priority=IntegerSanitizer(required=True), domains=ListSanitizer(StringSanitizer(required=True), required=True), ), required=True), options=DictSanitizer(dict(name=StringSanitizer()), required=True), ), required=True)) def put(self, request): """Modify an existing rules: requests.options = [ { 'object': { 'name': <str>, # optional 'type': 'whitelist' | 'blacklist', # optional 'priority': <int>, # optional 'wlan': <bool>, # optional 'domains': [<str>, ...], # optional }, 'options': { 'name': <str> # the original name of the object } }, ... ] """ # try to create all specified projects result = [] for ientry in request.options: try: # get properties and options from entry iprops = ientry['object'] iname = None ioptions = ientry.get('options') if ioptions: iname = ioptions.get('name') if not iname: raise ValueError( _('No "name" attribute has been specified in the options.' )) # make sure that the rule already exists irule = rules.load(iname) if not irule: raise ValueError( _('The rule does not exist and cannot be modified: %s') % iprops.get('name', '')) # parse the properties self._parseRule(iprops) if iprops.get('name', iname) != iname: # name has been changed -> remove old rule and create a new one rules.remove(iname) irule.name = iprops['name'] if 'type' in iprops: # set rule type, move all domains from the previous type oldDomains = irule.domains irule.domains = [] irule.type = _filterTypes[iprops['type']] irule.domains = oldDomains if 'priority' in iprops: # set priority irule.priority = iprops['priority'] if 'wlan' in iprops: # set wlan irule.wlan = iprops['wlan'] if 'domains' in iprops: # set domains irule.domains = iprops['domains'] # try to save filter rule irule.save() MODULE.info('Saved rule: %s' % irule) # everything ok result.append(dict(name=iname, success=True)) except ValueError as e: # data not valid... create error info MODULE.info( 'data for internet filter rule "%s" is not valid: %s' % (iprops.get('name'), e)) result.append( dict(name=iprops.get('name'), success=False, details=str(e))) # return the results self.finished(request.id, result) @sanitize(school=SchoolSanitizer(required=True), pattern=StringSanitizer(default='')) @LDAP_Connection() def groups_query(self, request, ldap_user_read=None, ldap_position=None): """List all groups (classes, workgroups) and their assigned internet rule""" pattern = LDAP_Filter.forAll(request.options.get('pattern', ''), ['name', 'description']) school = request.options['school'] groups = [ x for x in Group.get_all(ldap_user_read, school, pattern) if not x.self_is_computerroom() ] internet_rules = rules.getGroupRuleName([i.name for i in groups]) name = re.compile('-%s$' % (re.escape(school)), flags=re.I) result = [{ 'name': i.get_relative_name() if hasattr(i, 'get_relative_name') else name.sub('', i.name), '$dn$': i.dn, 'rule': internet_rules.get(i.name, 'default') or _('-- Default (unrestricted) --') } for i in groups] result.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()), key=lambda x: x['name']) self.finished(request.id, result) @sanitize( DictSanitizer( dict( group=StringSanitizer(required=True), rule=StringSanitizer(required=True), ))) @LDAP_Connection() def groups_assign(self, request, ldap_user_read=None, ldap_position=None): """Assigns default rules to groups: request.options = [ { 'group': <groupDN>, 'rule': <ruleName> }, ... ] """ MODULE.info('internetrules.groups_assign: options: %s' % str(request.options)) # try to load all group rules newRules = {} rmRules = [] for ientry in request.options: # make sure the group exists igrp = udm_objects.get(udm_modules.get('groups/group'), None, ldap_user_read, ldap_position, ientry['group']) if not igrp: raise UMC_Error('unknown group object') igrp.open() # check the rule name irule = ientry['rule'] if irule == '$default$': # remove the rule rmRules.append(igrp['name']) else: try: # make sure the rule name is valid self._parseRule(dict(name=irule)) except ValueError as exc: raise UMC_Error(str(exc)) # add new rule newRules[igrp['name']] = irule # assign default filter rules to groups rules.setGroupRuleName(newRules) rules.unsetGroupRuleName(rmRules) MODULE.info('internetrules.groups_assign: finished') self.finished(request.id, True)