class Instance(Base):
def init(self):
self._hostname = ucr.get('hostname')
@sanitize(pattern=PatternSanitizer(default='.*'),
key=ChoicesSanitizer(
choices=['printer', 'description', 'location'],
required=True))
@simple_response
def list_printers(self, key, pattern):
""" Lists the printers for the overview grid. """
quota = self._quota_enabled() # we need it later
result = []
plist = self._list_printers()
for element in plist:
printer = element['printer']
data = self._printer_details(printer)
for field in data:
element[field] = data[field]
# filter according to query
if pattern.match(element[key]):
if printer in quota:
element['quota'] = quota[printer]
else:
element['quota'] = False
result.append(element)
return result
@simple_response
@log
def get_printer(self, printer=''):
""" gets detail data for one printer. """
result = self._printer_details(printer)
result['printer'] = printer
result['status'] = self._printer_status(printer)
result['quota'] = self._quota_enabled(printer)
return result
@simple_response
def list_jobs(self, printer=''):
""" returns list of jobs for one printer. """
return self._job_list(printer)
@simple_response
def list_quota(self, printer=''):
""" lists all quota entries related to this printer. """
result = []
status = None
try:
from pykota.tool import PyKotaTool
from pykota import reporter
from pykota.storages.pgstorage import PGError
except ImportError:
raise UMC_Error(
_('The print quota settings are currently disabled. Please install the package univention-printquota to enable them.'
))
reportTool = PyKotaTool()
try:
reportTool.deferredInit()
printers = reportTool.storage.getMatchingPrinters(printer)
reportingtool = reporter.openReporter(reportTool, 'html', printers,
'*', 0)
status = reportingtool.generateReport()
except PGError as exc:
MODULE.error('Cannot connect to postgres: %s' % (exc, ))
raise UMC_Error(
_('The connection to the print quota postgres database failed. Please make sure the postgres service is running and reachable.'
))
finally:
reportTool.regainPriv()
if status:
tree = lxml.html.fromstring(status)
table = tree.find_class('pykotatable')
for i in table:
for a in i.iterchildren(tag='tr'):
data = list()
for b in a.iterchildren(tag='td'):
data.append(b.text_content().strip())
if data and len(data) >= 11:
user = data[0]
# limitby = data[1]
# overcharge = data[2]
used = data[3]
soft = data[4]
hard = data[5]
# balance = data[6]
# grace = data[7]
total = data[8]
# paid = data[9]
# warn = data[10]
result.append(
dict(
user=user,
used=used,
soft=soft,
hard=hard,
total=total,
))
return result
@simple_response
def list_users(self):
""" convenience function for the username entry. Lists
all user names. We don't return this as an array of {id, label}
tuples because:
(1) id and label are always the same here
(2) at the frontend, we must do some postprocessing, and an array
is easier to handle.
(3) the ComboBox is able to handle a plain array.
"""
ucr = ConfigRegistry()
ucr.load()
identity = ucr.get('ldap/hostdn')
password = open('/etc/machine.secret').read().rstrip('\n')
server = ucr.get('ldap/server/name')
udm = UDM.credentials(identity, password, server=server).version(1)
users = udm.get('users/user').search()
return [user.props.username for user in users]
@simple_response
@log
def enable_printer(self, printer='', on=False):
""" can enable or disable a printer, depending on args.
returns empty string on success, else error message.
"""
return self._enable_printer(printer, on)
@simple_response
@log
def cancel_jobs(self, jobs, printer=''):
""" cancels one or more print jobs. Job IDs are passed
as an array that can be directly passed on to the
_shell_command() method
"""
return self._cancel_jobs(printer, jobs)
@simple_response
@log
def set_quota(self, printer='', user='', soft=0, hard=0):
""" sets quota limits for a (printer, user) combination.
optionally tries to create the corresponding user entry.
"""
if printer == '' or user == '':
return "Required parameter missing"
else:
return self._set_quota(printer, user, soft, hard)
@simple_response
@log
def reset_quota(self, printer='', users=None):
""" resets quota for a (printer, user) combination."""
users = users or []
return self._reset_quota(printer, users)
# ----------------------- Internal functions -------------------------
def _job_list(self, printer):
""" lists jobs for a given printer, directly suitable for the grid """
# *** NOTE *** we don't set language to 'neutral' since it is useful
# to get localized date/time strings.
result = []
(stdout, stderr,
status) = self._shell_command(['/usr/bin/lpstat', '-o', printer])
expr = re.compile('\s*(\S+)\s+(\S+)\s+(\d+)\s*(.*?)$')
if status == 0:
for line in stdout.split("\n"):
mobj = expr.match(line)
if mobj:
entry = {
'job': mobj.group(1),
'owner': mobj.group(2),
'size': mobj.group(3),
'date': mobj.group(4)
}
result.append(entry)
return result
def _list_printers(self):
""" returns a list of printers, along with their 'enabled' status. """
result = []
expr = re.compile('printer\s+(\S+)\s.*?(\S+abled)')
(stdout, stderr,
status) = self._shell_command(['/usr/bin/lpstat', '-p'],
{'LANG': 'C'})
if status == 0:
for line in stdout.split("\n"):
mobj = expr.match(line)
if mobj:
entry = {'printer': mobj.group(1), 'status': mobj.group(2)}
result.append(entry)
return result
def _printer_status(self, printer):
""" returns the 'enabled' status of a printer """
(stdout, stderr,
status) = self._shell_command(['/usr/bin/lpstat', '-p', printer],
{'LANG': 'C'})
if status == 0:
if ' enabled ' in stdout:
return 'enabled'
if ' disabled ' in stdout:
return 'disabled'
return 'unknown'
def _printer_details(self, printer):
""" returns as much as possible details about a printer. """
result = {}
expr = re.compile('\s+([^\s\:]+)\:\s*(.*?)$')
(stdout, stderr, status) = self._shell_command(
['/usr/bin/lpstat', '-l', '-p', printer], {'LANG': 'C'})
if status == 0:
for line in stdout.split("\n"):
mobj = expr.match(line)
if mobj:
result[mobj.group(1).lower()] = mobj.group(2)
result['server'] = self._hostname
return result
def _enable_printer(self, printer, on):
""" internal function that enables/disables a printer.
returns empty string or error message.
"""
cmd = 'univention-cups-enable' if on else 'univention-cups-disable'
(stdout, stderr, status) = self._shell_command([cmd, printer])
if status:
return stderr
return ''
def _set_quota(self, printer, user, soft, hard):
""" sets a quota entry. Can also add a user """
# Before we can set quota we have to ensure that the user is
# already known to PyKota. Fortunately these tools don't complain
# if we try to create a user that doesn't already exist.
self._shell_command(
['/usr/bin/pkusers', '--skipexisting', '--add', user],
{'LANG': 'C'})
# Caution! order of args is important!
(stdout, stderr, status) = self._shell_command([
'/usr/bin/edpykota', '--printer', printer, '--softlimit',
str(soft), '--hardlimit',
str(hard), '--add', user
], {'LANG': 'C'})
# not all errors are propagated in exit codes...
# but at least they adhere to the general rule that
# progress is printed to STDOUT and errors/warnings to STDERR
if status or len(stderr):
return stderr
return ''
def _reset_quota(self, printer, users):
""" resets the 'used' counter on a quota entry. """
cmd = ['/usr/bin/edpykota', '--printer', printer, '--reset']
# appending user names to the args array -> spaces in user names
# don't confuse edpykota (In 2.4, this was a problem)
for user in users:
if user:
cmd.append(user)
(stdout, stderr, status) = self._shell_command(cmd, {'LANG': 'C'})
if status or stderr:
return stderr
return ''
def _quota_enabled(self, printer=None):
""" returns a dictionary with printer names and their 'quota active' status.
if printer is specified, returns only quota status for this printer.
"""
result = {}
expr = re.compile('device for (\S+)\:\s*(\S+)$')
(stdout, stderr,
status) = self._shell_command(['/usr/bin/lpstat', '-v'],
{'LANG': 'C'})
if status == 0:
for line in stdout.split("\n"):
match = expr.match(line)
if match:
quota = False
if match.group(2).startswith('cupspykota'):
quota = True
result[match.group(1)] = quota
# No printer specified: return the whole list.
if printer is None:
return result
# Printer specified: return its quota value or False if not found.
return result.get(printer, False)
def _cancel_jobs(self, printer, jobs):
""" internal function that cancels a list of jobs.
returns empty string or error message.
"""
args = ['/usr/bin/cancel', '-U', '%s$' % self._hostname]
for job in jobs:
args.append(job)
args.append(printer)
(stdout, stderr, status) = self._shell_command(args)
if status:
return stderr
return ''
def _shell_command(self, args, env=None):
proc = subprocess.Popen(args=args,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
outputs = proc.communicate()
return (outputs[0], outputs[1], proc.returncode)
class Instance(umcm.Base):
def init(self):
self.ucr = univention.config_registry.ConfigRegistry()
self.ucr.load()
util.install_opener(self.ucr)
self.package_manager = PackageManager(
info_handler=MODULE.process,
step_handler=None,
error_handler=MODULE.warn,
lock=False,
always_noninteractive=True,
)
self.uu = UniventionUpdater(False)
self.component_manager = util.ComponentManager(self.ucr, self.uu)
# in order to set the correct locale for Application
locale.setlocale(locale.LC_ALL, str(self.locale))
@sanitize(email=EmailSanitizer(required=True))
@simple_response
def request_new_license(self, email):
license = LICENSE.dump_data()
if license is None:
raise umcm.UMC_CommandError(_('Cannot parse License from LDAP'))
data = {}
data['email'] = email
data['licence'] = license
data = urllib.urlencode(data)
url = 'https://license.univention.de/keyid/conversion/submit'
request = urllib2.Request(url,
data=data,
headers={'User-agent': 'UMC/AppCenter'})
try:
util.urlopen(request)
except Exception as e:
try:
# try to parse an html error
body = e.read()
detail = re.search(
'<span id="details">(?P<details>.*?)</span>',
body).group(1)
except:
detail = str(e)
raise umcm.UMC_CommandError(
_('An error occurred while sending the request: %s') % detail)
else:
return True
@sanitize(pattern=PatternSanitizer(default='.*'))
@simple_response
def query(self, pattern):
LICENSE.reload()
try:
applications = Application.all(force_reread=True)
except (urllib2.HTTPError, urllib2.URLError) as e:
raise umcm.UMC_CommandError(
_('Could not query App Center: %s') % e)
result = []
self.package_manager.reopen_cache()
for application in applications:
if pattern.search(application.name):
props = application.to_dict(self.package_manager)
# delete larger entries
for ikey in ('readmeupdate', 'licenseagreement'):
if ikey in props:
del props[ikey]
result.append(props)
return result
@sanitize(application=StringSanitizer(minimum=1, required=True))
@simple_response
def get(self, application):
LICENSE.reload()
application = Application.find(application)
self.package_manager.reopen_cache()
return application.to_dict(self.package_manager)
@sanitize(function=ChoicesSanitizer(['install', 'uninstall', 'update'],
required=True),
application=StringSanitizer(minimum=1, required=True),
force=BooleanSanitizer())
def invoke(self, request):
function = request.options.get('function')
application_id = request.options.get('application')
application = Application.find(application_id)
force = request.options.get('force')
try:
# make sure that the application cane be installed/updated
can_continue = True
result = {
'install': [],
'remove': [],
'broken': [],
}
if not application:
MODULE.info('Application not found: %s' % application_id)
can_continue = False
elif function == 'install' and not application.can_be_installed(
self.package_manager):
MODULE.info('Application cannot be installed: %s' %
application_id)
can_continue = False
elif function == 'update' and not application.can_be_updated():
MODULE.info('Application cannot be updated: %s' %
application_id)
can_continue = False
if can_continue and function in ('install', 'update'):
result = application.install_dry_run(self.package_manager,
self.component_manager,
remove_component=False)
if result['broken'] or (result['remove'] and not force):
MODULE.info('Remove component: %s' % application_id)
self.component_manager.remove_app(application)
self.package_manager.update()
can_continue = False
elif can_continue and function in ('uninstall', ) and not force:
result['remove'] = application.uninstall_dry_run(
self.package_manager)
can_continue = False
result['can_continue'] = can_continue
self.finished(request.id, result)
if can_continue:
def _thread(module, application, function):
with module.package_manager.locked(reset_status=True,
set_finished=True):
with module.package_manager.no_umc_restart():
if function in ('install', 'update'):
# dont have to add component: already added during dry_run
return application.install(
module.package_manager,
module.component_manager,
add_component=False)
else:
return application.uninstall(
module.package_manager,
module.component_manager)
def _finished(thread, result):
if isinstance(result, BaseException):
MODULE.warn('Exception during %s %s: %s' %
(function, application_id, str(result)))
thread = notifier.threads.Simple(
'invoke',
notifier.Callback(_thread, self, application, function),
_finished)
thread.run()
except LockError:
# make it thread safe: another process started a package manager
# this module instance already has a running package manager
raise umcm.UMC_CommandError(
_('Another package operation is in progress'))
@simple_response
def app_center_app_license(self, application):
application = Application.find(application)
if not application or not application.get('licensefile'):
raise umcm.UMC_CommandError(
_('No license file available for application: %s') %
(application.id))
# open the license file and replace line breaks with BR-tags
fp = util.urlopen(application.get('licensefile'))
txt = ''.join(fp.readlines()).strip()
txt = txt.replace('\n\n\n', '\n<br>\n<br>\n<br>\n')
txt = txt.replace('\n\n', '\n<br>\n<br>\n')
return txt
@simple_response
def packages_sections(self):
""" fills the 'sections' combobox in the search form """
sections = set()
for package in self.package_manager.packages():
sections.add(package.section)
return sorted(sections)
@sanitize(pattern=PatternSanitizer(required=True))
@simple_response
def packages_query(self, pattern, section='all', key='package'):
""" Query to fill the grid. Structure is fixed here. """
result = []
for package in self.package_manager.packages():
if section == 'all' or package.section == section:
toshow = False
if pattern.pattern == '^.*$':
toshow = True
elif key == 'package' and pattern.search(package.name):
toshow = True
elif key == 'description' and pattern.search(
package.candidate.raw_description):
toshow = True
if toshow:
result.append(self._package_to_dict(package, full=False))
return result
@simple_response
def packages_get(self, package):
""" retrieves full properties of one package """
package = self.package_manager.get_package(package)
if package is not None:
return self._package_to_dict(package, full=True)
else:
# TODO: 404?
return {}
@sanitize(function=MappingSanitizer(
{
'install': 'install',
'upgrade': 'install',
'uninstall': 'remove',
},
required=True),
packages=ListSanitizer(StringSanitizer(minimum=1),
required=True))
@simple_response
def packages_invoke_dry_run(self, packages, function):
packages = self.package_manager.get_packages(packages)
kwargs = {'install': [], 'remove': [], 'dry_run': True}
if function == 'install':
kwargs['install'] = packages
else:
kwargs['remove'] = packages
return dict(
zip(['install', 'remove', 'broken'],
self.package_manager.mark(**kwargs)))
@sanitize(function=MappingSanitizer(
{
'install': 'install',
'upgrade': 'install',
'uninstall': 'remove',
},
required=True),
packages=ListSanitizer(StringSanitizer(minimum=1),
required=True))
def packages_invoke(self, request):
""" executes an installer action """
packages = request.options.get('packages')
function = request.options.get('function')
try:
with self.package_manager.locked(reset_status=True):
not_found = [
pkg_name for pkg_name in packages
if self.package_manager.get_package(pkg_name) is None
]
self.finished(request.id, {'not_found': not_found})
if not not_found:
def _thread(package_manager, function, packages):
with package_manager.locked(set_finished=True):
with package_manager.no_umc_restart():
if function == 'install':
package_manager.install(*packages)
else:
package_manager.uninstall(*packages)
def _finished(thread, result):
if isinstance(result, BaseException):
MODULE.warn('Exception during %s %s: %r' %
(function, packages, str(result)))
thread = notifier.threads.Simple(
'invoke',
notifier.Callback(_thread, self.package_manager,
function, packages), _finished)
thread.run()
except LockError:
# make it thread safe: another process started a package manager
# this module instance already has a running package manager
raise umcm.UMC_CommandError(
_('Another package operation is in progress'))
@simple_response
def progress(self):
timeout = 5
return self.package_manager.poll(timeout)
def _package_to_dict(self, package, full):
""" Helper that extracts properties from a 'apt_pkg.Package' object
and stores them into a dictionary. Depending on the 'full'
switch, stores only limited (for grid display) or full
(for detail view) set of properties.
"""
installed = package.installed # may be None
candidate = package.candidate
result = {
'package': package.name,
'installed': package.is_installed,
'upgradable': package.is_upgradable,
'summary': candidate.summary,
}
# add (and translate) a combined status field
# *** NOTE *** we translate it here: if we would use the Custom Formatter
# of the grid then clicking on the sort header would not work.
if package.is_installed:
if package.is_upgradable:
result['status'] = _('upgradable')
else:
result['status'] = _('installed')
else:
result['status'] = _('not installed')
# additional fields needed for detail view
if full:
result['section'] = package.section
result['priority'] = package.priority
# Some fields differ depending on whether the package is installed or not:
if package.is_installed:
result['summary'] = installed.summary # take the current one
result['description'] = installed.description
result['installed_version'] = installed.version
result['size'] = installed.installed_size
if package.is_upgradable:
result['candidate_version'] = candidate.version
else:
del result[
'upgradable'] # not installed: don't show 'upgradable' at all
result['description'] = candidate.description
result['size'] = candidate.installed_size
result['candidate_version'] = candidate.version
# format size to handle bytes
size = result['size']
byte_mods = ['B', 'kB', 'MB']
for byte_mod in byte_mods:
if size < 10000:
break
size = float(size) / 1000 # MB, not MiB
else:
size = size * 1000 # once too often
if size == int(size):
format_string = '%d %s'
else:
format_string = '%.2f %s'
result['size'] = format_string % (size, byte_mod)
return result
@simple_response
def components_query(self):
""" Returns components list for the grid in the ComponentsPage.
"""
# be as current as possible.
self.uu.ucr_reinit()
self.ucr.load()
result = []
for comp in self.uu.get_all_components():
result.append(self.component_manager.component(comp))
return result
@sanitize_list(StringSanitizer())
@multi_response(single_values=True)
def components_get(self, iterator, component_id):
# be as current as possible.
self.uu.ucr_reinit()
self.ucr.load()
for component_id in iterator:
yield self.component_manager.component(component_id)
@sanitize_list(DictSanitizer({'object': advanced_components_sanitizer}))
@multi_response
def components_put(self, iterator, object):
"""Writes back one or more component definitions.
"""
# umc.widgets.Form wraps the real data into an array:
#
# [
# {
# 'object' : { ... a dict with the real data .. },
# 'options': None
# },
# ... more such entries ...
# ]
#
# Current approach is to return a similarly structured array,
# filled with elements, each one corresponding to one array
# element of the request:
#
# [
# {
# 'status' : a number where 0 stands for success, anything else
# is an error code
# 'message' : a result message
# 'object' : a dict of field -> error message mapping, allows
# the form to show detailed error information
# },
# ... more such entries ...
# ]
with util.set_save_commit_load(self.ucr) as super_ucr:
for object, in iterator:
yield self.component_manager.put(object, super_ucr)
self.package_manager.update()
# do the same as components_put (update)
# but dont allow adding an already existing entry
components_add = sanitize_list(
DictSanitizer({'object': add_components_sanitizer}))(components_put)
components_add.__name__ = 'components_add'
@sanitize_list(StringSanitizer())
@multi_response(single_values=True)
def components_del(self, iterator, component_id):
for component_id in iterator:
yield self.component_manager.remove(component_id)
self.package_manager.update()
@multi_response
def settings_get(self, iterator):
# *** IMPORTANT *** Our UCR copy must always be current. This is not only
# to catch up changes made via other channels (ucr command line etc),
# but also to reflect the changes we have made ourselves!
self.ucr.load()
for _ in iterator:
yield {
'maintained':
self.ucr.is_true('repository/online/maintained', False),
'unmaintained':
self.ucr.is_true('repository/online/unmaintained', False),
'server':
self.ucr.get('repository/online/server', ''),
'prefix':
self.ucr.get('repository/online/prefix', ''),
}
@sanitize_list(
DictSanitizer({'object': basic_components_sanitizer}),
min_elements=1,
max_elements=1 # moduleStore with one element...
)
@multi_response
def settings_put(self, iterator, object):
# FIXME: returns values although it should yield (multi_response)
changed = False
# Set values into our UCR copy.
try:
with util.set_save_commit_load(self.ucr) as super_ucr:
for object, in iterator:
for key, value in object.iteritems():
MODULE.info(
" ++ Setting new value for '%s' to '%s'" %
(key, value))
super_ucr.set_registry_var(
'%s/%s' % (constants.ONLINE_BASE, key), value)
changed = super_ucr.changed()
except Exception as e:
MODULE.warn(" !! Writing UCR failed: %s" % str(e))
return [{'message': str(e), 'status': constants.PUT_WRITE_ERROR}]
self.package_manager.update()
# Bug #24878: emit a warning if repository is not reachable
try:
updater = self.uu
for line in updater.print_version_repositories().split('\n'):
if line.strip():
break
else:
raise ConfigurationError()
except ConfigurationError:
msg = _(
"There is no repository at this server (or at least none for the current UCS version)"
)
MODULE.warn(" !! Updater error: %s" % msg)
response = {'message': msg, 'status': constants.PUT_UPDATER_ERROR}
# if nothing was committed, we want a different type of error code,
# just to appropriately inform the user
if changed:
response['status'] = constants.PUT_UPDATER_NOREPOS
return [response]
except:
info = sys.exc_info()
emsg = '%s: %s' % info[:2]
MODULE.warn(" !! Updater error [%s]: %s" % (emsg))
return [{
'message': str(info[1]),
'status': constants.PUT_UPDATER_ERROR
}]
return [{'status': constants.PUT_SUCCESS}]
class Instance(Base):
def init(self):
# set the language in order to return the correctly localized labels/descriptions
uit.set_language(self.locale.language)
def __create_variable_info(self, options: Dict) -> None:
"""
creating variable infos
:param options: List of options
"""
all_info = ConfigRegistryInfo(registered_only=False)
info = ConfigRegistryInfo(install_mode=True)
info.read_customized()
var = Variable()
# description
for line in options['descriptions']:
text = line['text']
if not text:
continue
if 'lang' in line:
var[f'description[{line["lang"]}]'] = text
else:
var['description'] = text
# categories
if options['categories']:
var['categories'] = ','.join(options['categories'])
# type
var['type'] = options['type']
# are there any modifications?
old_value = all_info.get_variable(options['key'])
if old_value != var:
# save
info.add_variable(options['key'], var)
info.write_customized()
def is_readonly(self, key: str) -> bool:
ucrinfo_system = ConfigRegistryInfo(registered_only=False,
load_customized=False)
var = ucrinfo_system.get_variable(key)
if var:
return var.get('readonly') in ('yes', '1', 'true')
return False
@sanitize(
DictSanitizer({
'object':
DictSanitizer({
'key': UCRKeySanitizer(required=True),
'value': StringSanitizer(default=''),
})
}))
def add(self, request: 'Request') -> None:
# does the same as put
ucr.load()
already_set = set(ucr.keys()) & set(v['object']['key']
for v in request.options)
if already_set:
raise UMC_Error(
_('The UCR variable %s is already set.') %
('", "'.join(already_set)))
self.put(request)
@sanitize(
DictSanitizer({
'object':
DictSanitizer({
'key': UCRKeySanitizer(required=True),
'value': StringSanitizer(default=''),
})
}))
def put(self, request: 'Request') -> None:
for _var in request.options:
var = _var['object']
value = var['value'] or ''
key = var['key']
if self.is_readonly(key):
raise UMC_Error(
_('The UCR variable %s is read-only and can not be changed!'
) % (key, ))
arg = [f'{key}={value}']
handler_set(arg)
# handle descriptions, type, and categories
if 'descriptions' in var or 'type' in var or 'categories' in var:
self.__create_variable_info(var)
self.finished(request.id, True)
def remove(self, request: 'Request') -> None:
variables = [
x for x in [x.get('object') for x in request.options]
if x is not None
]
for var in variables:
if self.is_readonly(var):
raise UMC_Error(
_('The UCR variable %s is read-only and can not be removed!'
) % (var, ))
handler_unset(variables)
self.finished(request.id, True)
def get(self, request: 'Request') -> None:
ucrReg = ConfigRegistry()
ucrReg.load()
ucrInfo = ConfigRegistryInfo(registered_only=False)
# iterate over all requested variables
results = []
for key in request.options:
info = ucrInfo.get_variable(str(key))
value = ucrReg.get(str(key))
if not info and (value or '' == value):
# only the value available
results.append({'key': key, 'value': value})
elif info:
# info (categories etc.) available
info['value'] = value
info['key'] = key
results.append(info.normalize())
else:
# variable not available, request failed
raise UMC_Error(
_('The UCR variable %(key)s could not be found') %
{'key': key})
self.finished(request.id, results)
def categories(self, request: 'Request') -> None:
ucrInfo = ConfigRegistryInfo(registered_only=False)
categories = []
for id, obj in ucrInfo.categories.items():
name = obj['name']
if ucrInfo.get_variables(id):
categories.append({'id': id, 'label': name})
self.finished(request.id, categories)
@sanitize(pattern=PatternSanitizer(default='.*'),
key=ChoicesSanitizer(['all', 'key', 'value', 'description'],
required=True))
@simple_response
def query(self,
pattern: str,
key: str,
category: Union[List[str], None] = None) -> Dict:
'''Returns a dictionary of configuration registry variables
found by searching for the (wildcard) expression defined by the
UMCP request. Additionally a list of configuration registry
categories can be defined.
The dictionary returned is compatible with the Dojo data store
format.'''
variables = []
if category == 'all':
# load _all_ config registry variables
base_info = ConfigRegistryInfo(registered_only=False)
else:
# load _all registered_ config registry variables
base_info = ConfigRegistryInfo()
if category in ('all', 'all-registered'):
category = None
def _match_value(name, var):
return var.value and pattern.match(var.value)
def _match_key(name, var):
return pattern.match(name)
def _match_description(name, var):
descr = var.get('description')
return descr and pattern.match(descr)
def _match_all(name, var):
return _match_value(name, var) or _match_description(
name, var) or _match_key(name, var)
func = locals().get(f'_match_{key}')
for name, var in base_info.get_variables(category).items():
if func(name, var):
variables.append({
'key': name,
'value': var.value,
'description': var.get('description', None),
})
return variables
class Instance(SchoolBaseModule):
def query(self, request):
"""Searches for internet filter rules
requests.options = {}
'pattern' -- pattern to match within the rule name or the list of domains
"""
MODULE.info('internetrules.query: options: %s' % str(request.options))
pattern = request.options.get('pattern', '').lower()
def _matchDomain(domains):
# helper function to match pattern within the list of domains
matches = [idom for idom in domains if pattern in idom.lower()]
return 0 < len(matches)
# filter out all rules that match the given pattern
result = [
dict(
name=irule.name,
type=_filterTypesInv[irule.type],
domains=len(irule.domains),
priority=irule.priority,
wlan=irule.wlan,
) for irule in rules.list()
if pattern in irule.name.lower() or _matchDomain(irule.domains)
]
MODULE.info('internetrules.query: results: %s' % str(result))
self.finished(request.id, result)
@sanitize(StringSanitizer())
def get(self, request):
"""Returns the specified rules
requests.options = [ <ruleName>, ... ]
"""
MODULE.info('internetrules.get: options: %s' % str(request.options))
result = []
# fetch all rules with the given names (we need to make sure that "name" is UTF8)
names = set(iname.encode('utf8') for iname in request.options)
result = [
dict(
name=irule.name,
type=_filterTypesInv[irule.type],
domains=irule.domains,
priority=irule.priority,
wlan=irule.wlan,
) for irule in rules.list() if irule.name in names
]
MODULE.info('internetrules.get: results: %s' % str(result))
self.finished(request.id, result)
@sanitize(DictSanitizer(dict(object=StringSanitizer()), required=True))
def remove(self, request):
"""Removes the specified rules
requests.options = [ { "object": <ruleName> }, ... ]
"""
MODULE.info('internetrules.remove: options: %s' % str(request.options))
result = []
# fetch all rules with the given names
for ientry in request.options:
iname = ientry['object']
success = False
if iname:
success = rules.remove(iname)
result.append(dict(name=iname, success=success))
MODULE.info('internetrules.remove: results: %s' % str(result))
self.finished(request.id, result)
@staticmethod
def _parseRule(iprops, forceAllProperties=False):
# validate types
for ikey, itype in (('name', basestring), ('type', basestring),
('priority', (int, basestring)), ('wlan', bool),
('domains', list)):
if ikey not in iprops:
if forceAllProperties:
# raise exception as the key is not present
raise ValueError(
_('The key "%s" has not been specified: %s') %
(ikey, iprops))
continue
if not isinstance(iprops[ikey], itype):
typeStr = ''
if isinstance(itype, tuple):
typeStr = ', '.join([i.__name__ for i in itype])
else:
typeStr = itype.__name__
raise ValueError(
_('The key "%s" needs to be of type: %s') %
(ikey, typeStr))
# validate name
if 'name' in iprops and not univention.config_registry.validate_key(
iprops['name'].encode('utf-8')):
raise ValueError(
_('Invalid rule name "%s". The name needs to be a string, the following special characters are not allowed: %s'
) %
(iprops.get('name'),
'!, ", §, $, %, &, (, ), [, ], {, }, =, ?, `, +, #, \', ",", ;, <, >, \\'
))
# validate type
if 'type' in iprops and iprops['type'] not in _filterTypes:
raise ValueError(_('Filter type is unknown: %s') % iprops['type'])
# validate domains
if 'domains' in iprops:
parsedDomains = []
for idomain in iprops['domains']:
def _validValueChar():
# helper function to check for invalid characters
for ichar in idomain:
if ichar in univention.config_registry.backend.INVALID_VALUE_CHARS:
return False
return True
if not isinstance(idomain,
basestring) or not _validValueChar():
raise ValueError(_('Invalid domain '))
# parse domain
domain = idomain
if '://' not in domain:
# make sure that we have a scheme defined for parsing
MODULE.info(
'Adding a leading scheme for parsing of domain: %s' %
idomain)
domain = 'http://%s' % domain
domain = urlparse(domain).hostname
MODULE.info('Parsed domain: %s -> %s' % (idomain, domain))
if not domain:
raise ValueError(
_('The specified domain "%s" is not valid. Please specify a valid domain name, such as "wikipedia.org", "facebook.com"'
) % idomain)
# add domain to list of parsed domains
parsedDomains.append(domain)
# save parsed domains in the dict
iprops['domains'] = parsedDomains
return iprops
@sanitize(
DictSanitizer(dict(object=DictSanitizer(dict(
name=StringSanitizer(required=True),
type=ChoicesSanitizer(list(_filterTypes.keys()), required=True),
wlan=BooleanSanitizer(required=True),
priority=IntegerSanitizer(required=True),
domains=ListSanitizer(StringSanitizer(required=True),
required=True),
),
required=True)),
required=True))
def add(self, request):
"""Add the specified new rules:
requests.options = [ {
'object': {
'name': <str>,
'type': 'whitelist' | 'blacklist',
'priority': <int> | <str>,
'wlan': <bool>,
'domains': [<str>, ...],
}
}, ... ]
"""
# try to create all specified projects
result = []
for ientry in request.options:
iprops = ientry['object']
try:
# make sure that the rule does not already exist
irule = rules.load(iprops['name'])
if irule:
raise ValueError(
_('A rule with the same name does already exist: %s') %
iprops['name'])
# parse the properties
parsedProps = self._parseRule(iprops, True)
# create a new rule from the user input
newRule = rules.Rule(
name=parsedProps['name'],
type=_filterTypes[parsedProps['type']],
priority=parsedProps['priority'],
wlan=parsedProps['wlan'],
domains=parsedProps['domains'],
)
# try to save filter rule
newRule.save()
MODULE.info('Created new rule: %s' % newRule)
# everything ok
result.append(dict(name=iprops['name'], success=True))
except (ValueError, KeyError) as e:
# data not valid... create error info
MODULE.info(
'data for internet filter rule "%s" is not valid: %s' %
(iprops.get('name'), e))
result.append(
dict(name=iprops.get('name'),
success=False,
details=str(e)))
# return the results
self.finished(request.id, result)
@sanitize(
DictSanitizer(dict(
object=DictSanitizer(dict(
name=StringSanitizer(required=True),
type=ChoicesSanitizer(list(_filterTypes.keys()),
required=True),
wlan=BooleanSanitizer(required=True),
priority=IntegerSanitizer(required=True),
domains=ListSanitizer(StringSanitizer(required=True),
required=True),
),
required=True),
options=DictSanitizer(dict(name=StringSanitizer()), required=True),
),
required=True))
def put(self, request):
"""Modify an existing rules:
requests.options = [ {
'object': {
'name': <str>, # optional
'type': 'whitelist' | 'blacklist', # optional
'priority': <int>, # optional
'wlan': <bool>, # optional
'domains': [<str>, ...], # optional
},
'options': {
'name': <str> # the original name of the object
}
}, ... ]
"""
# try to create all specified projects
result = []
for ientry in request.options:
try:
# get properties and options from entry
iprops = ientry['object']
iname = None
ioptions = ientry.get('options')
if ioptions:
iname = ioptions.get('name')
if not iname:
raise ValueError(
_('No "name" attribute has been specified in the options.'
))
# make sure that the rule already exists
irule = rules.load(iname)
if not irule:
raise ValueError(
_('The rule does not exist and cannot be modified: %s')
% iprops.get('name', ''))
# parse the properties
self._parseRule(iprops)
if iprops.get('name', iname) != iname:
# name has been changed -> remove old rule and create a new one
rules.remove(iname)
irule.name = iprops['name']
if 'type' in iprops:
# set rule type, move all domains from the previous type
oldDomains = irule.domains
irule.domains = []
irule.type = _filterTypes[iprops['type']]
irule.domains = oldDomains
if 'priority' in iprops:
# set priority
irule.priority = iprops['priority']
if 'wlan' in iprops:
# set wlan
irule.wlan = iprops['wlan']
if 'domains' in iprops:
# set domains
irule.domains = iprops['domains']
# try to save filter rule
irule.save()
MODULE.info('Saved rule: %s' % irule)
# everything ok
result.append(dict(name=iname, success=True))
except ValueError as e:
# data not valid... create error info
MODULE.info(
'data for internet filter rule "%s" is not valid: %s' %
(iprops.get('name'), e))
result.append(
dict(name=iprops.get('name'),
success=False,
details=str(e)))
# return the results
self.finished(request.id, result)
@sanitize(school=SchoolSanitizer(required=True),
pattern=StringSanitizer(default=''))
@LDAP_Connection()
def groups_query(self, request, ldap_user_read=None, ldap_position=None):
"""List all groups (classes, workgroups) and their assigned internet rule"""
pattern = LDAP_Filter.forAll(request.options.get('pattern', ''),
['name', 'description'])
school = request.options['school']
groups = [
x for x in Group.get_all(ldap_user_read, school, pattern)
if not x.self_is_computerroom()
]
internet_rules = rules.getGroupRuleName([i.name for i in groups])
name = re.compile('-%s$' % (re.escape(school)), flags=re.I)
result = [{
'name':
i.get_relative_name()
if hasattr(i, 'get_relative_name') else name.sub('', i.name),
'$dn$':
i.dn,
'rule':
internet_rules.get(i.name, 'default')
or _('-- Default (unrestricted) --')
} for i in groups]
result.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()),
key=lambda x: x['name'])
self.finished(request.id, result)
@sanitize(
DictSanitizer(
dict(
group=StringSanitizer(required=True),
rule=StringSanitizer(required=True),
)))
@LDAP_Connection()
def groups_assign(self, request, ldap_user_read=None, ldap_position=None):
"""Assigns default rules to groups:
request.options = [ { 'group': <groupDN>, 'rule': <ruleName> }, ... ]
"""
MODULE.info('internetrules.groups_assign: options: %s' %
str(request.options))
# try to load all group rules
newRules = {}
rmRules = []
for ientry in request.options:
# make sure the group exists
igrp = udm_objects.get(udm_modules.get('groups/group'), None,
ldap_user_read, ldap_position,
ientry['group'])
if not igrp:
raise UMC_Error('unknown group object')
igrp.open()
# check the rule name
irule = ientry['rule']
if irule == '$default$':
# remove the rule
rmRules.append(igrp['name'])
else:
try:
# make sure the rule name is valid
self._parseRule(dict(name=irule))
except ValueError as exc:
raise UMC_Error(str(exc))
# add new rule
newRules[igrp['name']] = irule
# assign default filter rules to groups
rules.setGroupRuleName(newRules)
rules.unsetGroupRuleName(rmRules)
MODULE.info('internetrules.groups_assign: finished')
self.finished(request.id, True)
class Instance(Base):
def init(self):
# set the language in order to return the correctly localized labels/descriptions
uit.set_language(self.locale.language)
def __create_variable_info(self, options):
all_info = ConfigRegistryInfo(registered_only=False)
info = ConfigRegistryInfo(install_mode=True)
info.read_customized()
var = Variable()
# description
for line in options['descriptions']:
text = line['text']
if not text: continue
if 'lang' in line:
var['description[%s]' % line['lang']] = text
else:
var['description'] = text
# categories
if options['categories']:
var['categories'] = ','.join(options['categories'])
# type
var['type'] = options['type']
# are there any modifications?
old_value = all_info.get_variable(options['key'])
if old_value != var:
# save
info.add_variable(options['key'], var)
info.write_customized()
def is_readonly(self, key):
ucrinfo_system = ConfigRegistryInfo(registered_only=False,
load_customized=False)
var = ucrinfo_system.get_variable(key)
if var:
return var.get('readonly') in ('yes', '1', 'true')
return False
def add(self, request):
# does the same as put
self.put(request)
def put(self, request):
message = ''
request.status = SUCCESS
success = True
if isinstance(request.options, (list, tuple)):
for _var in request.options:
try:
var = _var['object']
value = var['value'] or ''
key = var['key']
if self.is_readonly(key):
success = False
message = _(
'The UCR variable %s is read-only and can not be changed!'
) % key
break
arg = ['%s=%s' % (key.encode(), value.encode())]
ucr.handler_set(arg)
# handle descriptions, type, and categories
if 'descriptions' in var or 'type' in var or 'categories' in var:
self.__create_variable_info(var)
except KeyError:
# handle the case that neither key nor value are given for an UCR variable entry
request.status = BAD_REQUEST_INVALID_OPTS
self.finished(
request.id,
False,
message=
_('Invalid UCR variable entry, the properties "key" and "value" need to specified.'
))
return
else:
success = False
request.status = BAD_REQUEST_INVALID_OPTS
self.finished(request.id, success, message)
def remove(self, request):
variables = filter(lambda x: x is not None,
map(lambda x: x.get('object'), request.options))
for var in variables:
if self.is_readonly(var):
message = _(
'The UCR variable %s is read-only and can not be removed!'
) % var
self.finished(request.id, False, message)
return
ucr.handler_unset(variables)
self.finished(request.id, True)
def get(self, request):
ucrReg = ucr.ConfigRegistry()
ucrReg.load()
ucrInfo = ConfigRegistryInfo(registered_only=False)
# iterate over all requested variables
results = []
for key in request.options:
info = ucrInfo.get_variable(str(key))
value = ucrReg.get(str(key))
if not info and (value or '' == value):
# only the value available
results.append({'key': key, 'value': value})
elif info:
# info (categories etc.) available
info['value'] = value
info['key'] = key
results.append(info.normalize())
else:
# variable not available, request failed
request.status = BAD_REQUEST_INVALID_OPTS
self.finished(
request.id,
False,
message=_('The UCR variable %(key)s could not be found') %
{'key': key})
return
self.finished(request.id, results)
def categories(self, request):
ucrInfo = ConfigRegistryInfo(registered_only=False)
categories = []
for id, obj in ucrInfo.categories.iteritems():
name = obj['name']
categories.append({'id': id, 'label': name})
self.finished(request.id, categories)
@sanitize(pattern=PatternSanitizer(default='.*'),
key=ChoicesSanitizer(['all', 'key', 'value', 'description'],
required=True))
@simple_response
def query(self, pattern, key, category=None):
'''Returns a dictionary of configuration registry variables
found by searching for the (wildcard) expression defined by the
UMCP request. Additionally a list of configuration registry
categories can be defined.
The dictionary returned is compatible with the Dojo data store
format.'''
variables = []
if category == 'all':
# load _all_ config registry variables
base_info = ConfigRegistryInfo(registered_only=False)
else:
# load _all registered_ config registry variables
base_info = ConfigRegistryInfo()
if category in ('all', 'all-registered'):
category = None
def _match_value(name, var):
return var.value and pattern.match(var.value)
def _match_key(name, var):
return pattern.match(name)
def _match_description(name, var):
descr = var.get('description')
return descr and pattern.match(descr)
def _match_all(name, var):
return _match_value(name, var) or _match_description(
name, var) or _match_key(name, var)
func = eval('_match_%s' % key)
for name, var in base_info.get_variables(category).iteritems():
if func(name, var):
variables.append({'key': name, 'value': var.value})
return variables
def sanitize_reports_create(self, request): choices = self.reports_cfg.get_report_names(request.flavor) return dict( report=ChoicesSanitizer(choices=choices, required=True), objects=ListSanitizer(DNSanitizer(minimum=1), required=True, min_elements=1) )
class Instance(SchoolBaseModule):
def __init__(self):
SchoolBaseModule.__init__(self)
self._tmpDir = None
def init(self):
SchoolBaseModule.init(self)
# initiate paths for data distribution
util.initPaths()
def destroy(self):
self._cleanTmpDir()
def _cleanTmpDir(self):
# clean up the temporary upload directory
if self._tmpDir:
MODULE.info('Clean up temporary directory: %s' % self._tmpDir)
shutil.rmtree(self._tmpDir, ignore_errors=True)
self._tmpDir = None
@file_upload
@sanitize(DictSanitizer(dict(
filename=StringSanitizer(required=True),
tmpfile=StringSanitizer(required=True),
), required=True))
def upload(self, request):
# create a temporary upload directory, if it does not already exist
if not self._tmpDir:
self._tmpDir = tempfile.mkdtemp(prefix='ucsschool-distribution-upload-')
MODULE.info('Created temporary directory: %s' % self._tmpDir)
for file in request.options:
filename = self.__workaround_filename_bug(file)
destPath = os.path.join(self._tmpDir, filename)
MODULE.info('Received file %r, saving it to %r' % (file['tmpfile'], destPath))
shutil.move(file['tmpfile'], destPath)
self.finished(request.id, None)
def __workaround_filename_bug(self, file):
# the following code block is a heuristic to support both: fixed and unfixed Bug #37716
filename = file['filename']
try:
# The UMC-Webserver decodes filename in latin-1, need to revert
filename = filename.encode('ISO8859-1')
except UnicodeEncodeError:
# we got non-latin characters, Bug #37716 is fixed and string contains e.g. '→'
filename = file['filename'].encode('UTF-8')
else:
# the string contains at least no non-latin1 characters
try:
# try if the bytes could be UTF-8
# can't fail if Bug #37716 is fixed
filename.decode('UTF-8')
except UnicodeDecodeError:
filename = file['filename'].encode('UTF-8') # Bug #37716 was fixed
MODULE.info('Detected filename %r as %r' % (file['filename'], filename))
# the code block can be removed and replaced by filename = file['filename'].encode('UTF-8') after Bug #37716
return filename
@sanitize(
filenames=ListSanitizer(min_elements=1),
# project=StringSanitizer(allow_none=True)
)
@simple_response
def checkfiles(self, project, filenames):
'''Checks whether the given filename has already been uploaded:
request.options: { 'filenames': [ '...', ... ], project: '...' }
returns: {
'filename': '...',
'sessionDuplicate': True|False,
'projectDuplicate': True|False,
'distributed': True|False
}
'''
# load project
if project:
project = util.Project.load(project)
result = []
for ifile in filenames:
ifile = ifile.encode('UTF-8')
# check whether file has already been upload in this session
iresult = dict(sessionDuplicate=False, projectDuplicate=False, distributed=False)
iresult['filename'] = ifile
iresult['sessionDuplicate'] = self._tmpDir is not None and os.path.exists(os.path.join(self._tmpDir, ifile))
# check whether the file exists in the specified project and whether
# it has already been distributed
if project:
iresult['projectDuplicate'] = ifile in project.files
iresult['distributed'] = ifile in project.files and not os.path.exists(os.path.join(project.cachedir, ifile))
result.append(iresult)
return result
@sanitize(
pattern=PatternSanitizer(required=False, default='.*'),
filter=ChoicesSanitizer(['all', 'private'], default='private')
)
@simple_response
def query(self, pattern, filter):
result = [dict(
# only show necessary information
description=i.description,
name=i.name,
sender=i.sender.username,
recipients=len(i.recipients),
files=len(i.files),
isDistributed=i.isDistributed
) for i in util.Project.list()
if (pattern.match(i.name) or pattern.match(i.description)) and (filter == 'all' or compare_dn(i.sender.dn, self.user_dn))
]
return result
@LDAP_Connection()
def _get_sender(self, ldap_user_read=None, ldap_position=None):
'''Return a User instance of the currently logged in user.'''
try:
user = User.from_dn(self.user_dn, None, ldap_user_read)
obj = user.get_udm_object(ldap_user_read)
return util.User(obj.info, dn=obj.dn)
except udm_exceptions.base as exc:
raise UMC_Error(_('Failed to load user information: %s') % exc)
@sanitize(DictSanitizer(dict(object=DictSanitizer({}, required=True)), required=True))
def put(self, request):
"""Modify an existing project"""
result = [self._save(entry['object'], True) for entry in request.options]
self.finished(request.id, result)
@sanitize(DictSanitizer(dict(object=DictSanitizer({}, required=True)), required=True))
def add(self, request):
"""Add a new project"""
result = [self._save(entry['object'], False) for entry in request.options]
self.finished(request.id, result)
@LDAP_Connection()
def _save(self, iprops, doUpdate=True, ldap_user_read=None, ldap_position=None):
# try to open the UDM user object of the current user
sender = self._get_sender()
try:
# remove keys that may not be set from outside
for k in ('atJobNumCollect', 'atJobNumDistribute'):
iprops.pop(k, None)
# transform filenames into bytestrings
iprops['files'] = [f.encode('UTF-8') for f in iprops.get('files', [])]
# load the project or create a new one
project = None
orgProject = None
if doUpdate:
# try to load the given project
orgProject = util.Project.load(iprops.get('name', ''))
if not orgProject:
raise UMC_Error(_('The specified project does not exist: %s') % iprops['name'])
# create a new project with the updated values
project = util.Project(orgProject.dict)
project.update(iprops)
else:
# create a new project
project = util.Project(iprops)
# make sure that the project owner himself is modifying the project
if doUpdate and not compare_dn(project.sender.dn, self.user_dn):
raise UMC_Error(_('The project can only be modified by the owner himself'))
# handle time settings for distribution/collection of project files
for jsuffix, jprop, jname in (('distribute', 'starttime', _('Project distribution')), ('collect', 'deadline', _('Project collection'))):
if '%sType' % jsuffix in iprops:
# check the distribution/collection type: manual/automat
jtype = (iprops['%sType' % jsuffix]).lower()
if jtype == 'automatic':
try:
# try to parse the given time parameters
strtime = '%s %s' % (iprops['%sDate' % jsuffix], iprops['%sTime' % jsuffix])
jdate = datetime.strptime(strtime, '%Y-%m-%d %H:%M')
setattr(project, jprop, jdate)
except ValueError:
raise UMC_Error(_('Could not set date for: %s') % jname)
# make sure the execution time lies sufficiently in the future
if getattr(project, jprop) - datetime.now() < timedelta(minutes=1):
raise UMC_Error(_('The specified time needs to lie in the future for: %s') % jname)
else:
# manual distribution/collection
setattr(project, jprop, None)
if project.starttime and project.deadline:
# make sure distributing happens before collecting
if project.deadline - project.starttime < timedelta(minutes=3):
raise UMC_Error(_('Distributing the data needs to happen sufficiently long enough before collecting them'))
if 'recipients' in iprops:
# lookup the users in LDAP and save them to the project
project.recipients = [util.openRecipients(idn, ldap_user_read) for idn in iprops.get('recipients', [])]
project.recipients = [x for x in project.recipients if x]
MODULE.info('recipients: %s' % (project.recipients,))
if not doUpdate:
# set the sender (i.e., owner) of the project
project.sender = sender
# initiate project and validate its values
project.validate()
# make sure that there is no other project with the same directory name
# if we add new projects
if not doUpdate and project.isNameInUse():
MODULE.error('The project name is already in use: %s' % (project.name))
raise UMC_Error(_('The specified project directory name "%s" is already in use by a different project.') % (project.name))
# try to save project to disk
project.save()
# move new files into project directory
if self._tmpDir:
for ifile in project.files:
isrc = os.path.join(self._tmpDir, ifile)
itarget = os.path.join(project.cachedir, ifile)
if os.path.exists(isrc):
# mv file to cachedir
shutil.move(isrc, itarget)
os.chown(itarget, 0, 0)
# remove files that have been marked for removal
if doUpdate:
for ifile in set(orgProject.files) - set(project.files):
itarget = os.path.join(project.cachedir, ifile)
try:
os.remove(itarget)
except OSError:
pass
# re-distribute the project in case it has already been distributed
if doUpdate and project.isDistributed:
usersFailed = []
project.distribute(usersFailed)
if usersFailed:
# not all files could be distributed
MODULE.info('Failed processing the following users: %s' % usersFailed)
usersStr = ', '.join([Display.user(i) for i in usersFailed])
raise UMC_Error(_('The project could not distributed to the following users: %s') % usersStr)
except (IOError, OSError, UMC_Error): # TODO: catch only UMC_Error
etype, exc, etraceback = sys.exc_info()
# data not valid... create error info
MODULE.info('data for project "%s" is not valid: %s' % (iprops.get('name'), exc))
if not doUpdate:
# remove eventually created project file and cache dir
for ipath in (project.projectfile, project.cachedir):
if os.path.basename(ipath) not in os.listdir(util.DISTRIBUTION_DATA_PATH):
# no file / directory has been created yet
continue
try:
MODULE.info('cleaning up... removing: %s' % ipath)
shutil.rmtree(ipath)
except (IOError, OSError):
pass
raise UMC_Error, exc, etraceback
self._cleanTmpDir()
return {'success': True, 'name': iprops.get('name')}
@sanitize(StringSanitizer(required=True))
@LDAP_Connection()
def get(self, request, ldap_user_read=None, ldap_position=None):
"""Returns the objects for the given IDs
requests.options = [ <ID>, ... ]
return: [ { ... }, ... ]
"""
# try to load all given projects
result = []
# list of all project properties (dicts) or None if project is not valid
for iproject in [util.Project.load(iid) for iid in request.options]:
# make sure that project could be loaded
if not iproject:
result.append(None)
continue
# make sure that only the project owner himself (or an admin) is able
# to see the content of a project
if request.flavor == 'teacher' and not compare_dn(iproject.sender.dn, self.user_dn):
raise UMC_Error(_('Project details are only visible to the project owner himself or an administrator.'), status=403)
# prepare date and time properties for distribution/collection of project files
props = iproject.dict
for jjob, jsuffix in ((iproject.atJobDistribute, 'distribute'), (iproject.atJobCollect, 'collect')):
MODULE.info('check job: %s' % jsuffix)
if not jjob:
# no job is registered -> manual job distribution/collection
MODULE.info('no existing job -> manual execution')
props['%sType' % jsuffix] = 'manual'
continue
# job is registered -> prepare date and time fields
MODULE.info('job nr #%d scheduled for %s -> automatic execution' % (jjob.nr, jjob.execTime))
props['%sType' % jsuffix] = 'automatic'
props['%sDate' % jsuffix] = datetime.strftime(jjob.execTime, '%Y-%m-%d')
props['%sTime' % jsuffix] = datetime.strftime(jjob.execTime, '%H:%M')
# adjust sender / recipients properties
props['sender'] = props['sender'].username
recipients = []
for recip in props['recipients']:
recipients.append({
'id': recip.dn,
'label': recip.type == util.TYPE_USER and Display.user(recip.dict) or recip.name
})
props['recipients'] = recipients
# append final dict to result list
MODULE.info('final project dict: %s' % props)
result.append(props)
self.finished(request.id, result)
@sanitize(StringSanitizer(required=True))
def distribute(self, request):
# update the sender information of the selected projects
result = []
for iid in request.options:
MODULE.info('Distribute project: %s' % iid)
try:
# make sure that project could be loaded
iproject = util.Project.load(iid)
if not iproject:
raise IOError(_('Project "%s" could not be loaded') % iid)
# make sure that only the project owner himself (or an admin) is able
# to distribute a project
if request.flavor == 'teacher' and not compare_dn(iproject.sender.dn, self.user_dn):
raise ValueError(_('Only the owner himself or an administrator may distribute a project.'))
# project was loaded successfully... try to distribute it
usersFailed = []
iproject.distribute(usersFailed)
# raise an error in case distribution failed for some users
if usersFailed:
MODULE.info('Failed processing the following users: %s' % usersFailed)
usersStr = ', '.join([Display.user(i) for i in usersFailed])
raise IOError(_('The project could not distributed to the following users: %s') % usersStr)
# save result
result.append(dict(
name=iid,
success=True
))
except (ValueError, IOError) as exc:
result.append(dict(
name=iid,
success=False,
details=str(exc)
))
# return the results
self.finished(request.id, result)
@sanitize(StringSanitizer(required=True))
def collect(self, request):
# try to open the UDM user object of the current user
sender = self._get_sender()
# update the sender information of the selected projects
result = []
for iid in request.options:
MODULE.info('Collect project: %s' % iid)
try:
# make sure that project could be loaded
iproject = util.Project.load(iid)
if not iproject:
raise IOError(_('Project "%s" could not be loaded') % iid)
# replace the projects sender with the current logged in user
iproject.sender = sender
# project was loaded successfully... try to distribute it
dirsFailed = []
iproject.collect(dirsFailed)
# raise an error in case distribution failed for some users
if dirsFailed:
dirsStr = ', '.join(dirsFailed)
MODULE.info('Failed collecting the following dirs: %s' % dirsStr)
raise IOError(_('The following user directories could not been collected: %s') % dirsStr)
# save result
result.append(dict(
name=iid,
success=True
))
except (ValueError, IOError) as exc:
result.append(dict(
name=iid,
success=False,
details=str(exc)
))
# return the results
self.finished(request.id, result)
@sanitize(StringSanitizer(required=True))
def adopt(self, request):
# try to open the UDM user object of the current user
sender = self._get_sender()
# update the sender information of the selected projects
result = []
for iid in request.options:
try:
# make sure that project could be loaded
iproject = util.Project.load(iid)
if not iproject:
raise IOError(_('Project "%s" could not be loaded') % iid)
# project was loaded successfully
iproject.sender = sender
iproject.save()
except (ValueError, IOError) as exc:
result.append(dict(
name=iid,
success=False,
details=str(exc)
))
# return the results
self.finished(request.id, result)
@sanitize(DictSanitizer(dict(object=StringSanitizer(required=True)), required=True))
def remove(self, request):
"""Removes the specified projects"""
for iproject in [util.Project.load(ientry.get('object')) for ientry in request.options]:
if not iproject:
continue
# make sure that only the project owner himself (or an admin) is able
# to see the content of a project
if request.flavor == 'teacher' and not compare_dn(iproject.sender.dn, self.user_dn):
raise UMC_Error(_('Only the owner himself or an administrator may delete a project.'), status=403)
# purge the project
iproject.purge()
self.finished(request.id, None)
class Instance(Base):
def init(self):
MODULE.info("Initializing 'updater' module (PID = %d)" % (getpid(), ))
self._current_job = ''
self._logfile_start_line = 0
self._serial_file = Watched_File(COMPONENTS_SERIAL_FILE)
self._updates_serial = Watched_Files(UPDATE_SERIAL_FILES)
try:
self.uu = UniventionUpdater(False)
except Exception as exc: # FIXME: let it raise
MODULE.error("init() ERROR: %s" % (exc, ))
@simple_response
def query_maintenance_information(self):
ret = self._maintenance_information()
ret.update(self._last_update())
return ret
def _last_update(self):
status_file = '/var/lib/univention-updater/univention-updater.status'
ret = {'last_update_failed': False, 'last_update_version': None}
try:
mtime = stat(status_file).st_mtime
mtime = datetime.fromtimestamp(mtime)
delta = datetime.now() - mtime
if delta.days != 0: # no fresh failure
return ret
with open(status_file) as fd:
content = fd.read()
info = dict(
line.split('=', 1) for line in content.splitlines())
ret['last_update_failed'] = info.get('status') == 'FAILED'
if ret['last_update_failed']:
ret['last_update_version'] = info.get('next_version')
except (ValueError, EnvironmentError) as exc:
MODULE.error(str(exc))
pass
return ret
def _maintenance_information(self):
ucr.load()
if ucr.is_true('license/extended_maintenance/disable_warning'):
return {'show_warning': False}
version = self.uu.get_ucs_version()
try:
url = 'http://updates.software-univention.de/download/ucs-maintenance/{}.yaml'.format(
version)
response = requests.get(url, timeout=10)
if not response.ok:
response.raise_for_status()
status = yaml.load(response.content)
if not isinstance(status, dict):
raise yaml.YAMLError(repr(status))
# the yaml file contains for maintained either false, true or extended as value.
# yaml.load converts true and false into booleans but extended into string.
_maintained_status = status.get('maintained')
maintenance_extended = _maintained_status == 'extended'
show_warning = maintenance_extended or not _maintained_status
except yaml.YAMLError as exc:
MODULE.error('The YAML format is malformed: %s' % (exc, ))
return {'show_warning': False}
except requests.exceptions.RequestException as exc:
MODULE.error("Querying maintenance information failed: %s" %
(exc, ))
return {'show_warning': False}
return {
'ucs_version': version,
'show_warning': show_warning,
'maintenance_extended': maintenance_extended,
'base_dn': ucr.get('license/base')
}
@simple_response
def poll(self):
return True
@simple_response
def query_releases(self):
"""
Returns a list of system releases suitable for the
corresponding ComboBox
"""
# be as current as possible.
self.uu.ucr_reinit()
ucr.load()
appliance_mode = ucr.is_true('server/appliance')
available_versions, blocking_components = self.uu.get_all_available_release_updates(
)
result = [{
'id': rel,
'label': 'UCS %s' % (rel, )
} for rel in available_versions]
#
# appliance_mode=no ; blocking_comp=no → add "latest version"
# appliance_mode=no ; blocking_comp=yes → no "latest version"
# appliance_mode=yes; blocking_comp=no → add "latest version"
# appliance_mode=yes; blocking_comp=yes → add "latest version"
#
if result and (appliance_mode or not blocking_components):
# UniventionUpdater returns available version in ascending order, so
# the last returned entry is the one to be flagged as 'latest' if there's
# no blocking component.
result[-1]['label'] = '%s (%s)' % (result[-1]['label'],
_('latest version'))
return result
def call_hooks(self, request):
"""
Calls the specified hooks and returns data given back by each hook
"""
def _thread(request):
result = {}
hookmanager = univention.hooks.HookManager(
HOOK_DIRECTORY) # , raise_exceptions=False
hooknames = request.options.get('hooks')
MODULE.info('requested hooks: %s' % hooknames)
for hookname in hooknames:
MODULE.info('calling hook %s' % hookname)
result[hookname] = hookmanager.call_hook(hookname)
MODULE.info('result: %r' % (result, ))
return result
thread = notifier.threads.Simple(
'call_hooks', notifier.Callback(_thread, request),
notifier.Callback(self.thread_finished_callback, request))
thread.run()
@simple_response
def updates_serial(self):
"""
Watches the three sources.list snippets for changes
"""
result = self._updates_serial.timestamp()
MODULE.info(" -> Serial for UPDATES is '%s'" % result)
return result
@simple_response
def updates_check(self):
"""
Returns the list of packages to be updated/installed
by a dist-upgrade.
"""
p0 = subprocess.Popen(['LC_ALL=C apt-get update'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(stdout, stderr) = p0.communicate()
p1 = subprocess.Popen(['LC_ALL=C apt-get -u dist-upgrade -s'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(stdout, stderr) = p1.communicate()
install = []
update = []
remove = []
for line in stdout.split('\n'):
# upgrade:
# Inst univention-updater [3.1.1-5] (3.1.1-6.408.200810311159 192.168.0.10)
# inst:
# Inst mc (1:4.6.1-6.12.200710211124 oxae-update.open-xchange.com)
#
# *** FIX *** the above example lines ignore the fact that there's
# some extra text (occasionally) after the last closing
# parenthesis. Until now, I've seen only a pair of empty
# brackets [], but who knows...
match = re.search('^Inst (\S+)\s+(.*?)\s*\((\S+)\s.*\)', line)
if match:
pkg = match.group(1)
old = match.group(2)
ver = match.group(3)
if old:
update.append([pkg, ver])
else:
install.append([pkg, ver])
elif line.startswith('Remv '):
l = line.split(' ')
pkg = l[1]
# i18n: The package version is unknown.
ver = _('unknown')
if len(l) > 2:
ver = l[2].replace('[', '').replace(']', '')
remove.append([pkg, ver])
return dict(
update=sorted(update),
install=sorted(install),
remove=sorted(remove),
)
@simple_response
def updates_available(self):
"""
Asks if there are package updates available. (don't get confused
by the name of the UniventionUpdater function that is called here.)
This is a separate call since it can take an amount of time, thus
being invoked by a separate button (and not in the background)
"""
ucr.load()
try:
# be as current as possible.
what = 'reinitializing UniventionUpdater'
self.uu.ucr_reinit()
what = 'checking update availability'
return self.uu.component_update_available()
except Exception as ex:
typ = str(type(ex)).strip('<>')
msg = '[while %s] [%s] %s' % (what, typ, str(ex))
MODULE.error(msg)
return False
def status(self, request): # TODO: remove unneeded things
"""One call for all single-value variables."""
result = {}
ucr.load()
try:
result['erratalevel'] = int(ucr.get('version/erratalevel', 0))
except ValueError:
result['erratalevel'] = 0
result['appliance_mode'] = ucr.is_true('server/appliance')
result['easy_mode'] = ucr.is_true('update/umc/updateprocess/easy',
False)
result['timestamp'] = int(time())
result['reboot_required'] = ucr.is_true('update/reboot/required',
False)
try:
# be as current as possible.
what = 'reinitializing UniventionUpdater'
self.uu.ucr_reinit()
what = 'getting UCS version'
result['ucs_version'] = self.uu.get_ucs_version()
# if nothing is returned -> convert to empty string.
what = 'querying available release updates'
try:
result[
'release_update_available'] = self.uu.release_update_available(
errorsto='exception')
except RequiredComponentError as exc:
result['release_update_available'] = exc.version
if result['release_update_available'] is None:
result['release_update_available'] = ''
what = 'querying update-blocking components'
try:
blocking_components = self.uu.get_all_available_release_updates(
)[1]
except (UpdaterException, ValueError) as exc:
msg = _(
'Error contacting the update server. Please check your proxy or firewall settings, if any. Or it may be a problem with your configured DNS server.'
)
msg += ' ' + _('This is the error message:') + ' ' + str(exc)
raise UMC_Error(msg)
result['release_update_blocking_components'] = ' '.join(
blocking_components or [])
what = "querying availability for easy mode"
if result['easy_mode']:
# updates/available should reflect the need for an update
easy_update_available = ucr.is_true('update/available', False)
# but don't rely on ucr! update/available is set during univention-upgrade --check
# but when was the last time this was run?
# release update
easy_update_available = easy_update_available or result[
'release_update_available']
# if no update seems necessary perform a real (expensive) check nonetheless
easy_update_available = easy_update_available or self.uu.component_update_available(
)
result['easy_update_available'] = bool(easy_update_available)
else:
result['easy_update_available'] = False
# Component counts are now part of the general 'status' data.
what = "counting components"
c_count = 0
e_count = 0
for comp in self.uu.get_all_components():
c_count = c_count + 1
if ucr.is_true('repository/online/component/%s' % (comp, ),
False):
e_count = e_count + 1
result['components'] = c_count
result['enabled'] = e_count
# HACK: the 'Updates' form polls on the serial file
# to refresh itself. Including the serial value
# into the form helps us to have a dependent field
# that can trigger the refresh of the "Releases"
# combobox and the 'package updates available' field.
result['serial'] = self._serial_file.timestamp()
except Exception as exc: # FIXME: don't catch everything
typ = str(type(exc)).strip('<>')
msg = '[while %s] [%s] %s' % (what, typ, exc)
msg = _(
'Error contacting the update server. Please check your proxy or firewall settings, if any. Or it may be a problem with your configured DNS server.'
)
msg += ' ' + _('This is the error message:') + ' ' + str(exc)
raise UMC_Error(msg)
self.finished(request.id, [result])
@simple_response
def reboot(self):
"""
Reboots the computer. Simply invokes /sbin/reboot in the background
and returns success to the caller. The caller is prepared for
connection loss.
"""
subprocess.call(['/sbin/reboot'])
return True
@simple_response
def running(self):
"""
Returns the id (key into INSTALLERS) of a currently
running job, or the empty string if nothing is running.
"""
return self.__which_job_is_running()
@sanitize(
job=ChoicesSanitizer(INSTALLERS.keys() + [''], required=True),
count=IntegerSanitizer(default=0),
)
@simple_response
def updater_log_file(self, job, count):
"""
returns the content of the log file associated with
the job.
Argument 'count' has the same meaning as already known:
<0 ...... return timestamp of file (for polling)
0 ....... return whole file as a string list
>0 ...... ignore this many lines, return the rest of the file
*** NOTE *** As soon as we have looked for a running job at least once,
we know the job key and can associate it here.
TODO: honor a given 'job' argument
"""
job = self._current_job or job
if not job:
return
fname = INSTALLERS[job]['logfile']
if count < 0:
try:
return stat(fname)[9]
except (IOError, OSError):
return 0
# don't read complete file if we have an 'ignore' count
count += self._logfile_start_line
return self._logview(fname, -count)
def _logview(self, fname, count):
"""
Contains all functions needed to view or 'tail' an arbitrary text file.
Argument 'count' can have different values:
< 0 ... ignore this many lines, return the rest of the file
0 ..... return the whole file, split into lines.
> 0 ... return the last 'count' lines of the file. (a.k.a. tail -n <count>)
"""
lines = []
try:
with open(fname, 'rb') as fd:
for line in fd:
if (count < 0):
count += 1
else:
lines.append(line.rstrip().decode('utf-8', 'replace'))
if (count > 0) and (len(lines) > count):
lines.pop(0)
except (IOError, OSError):
pass
return lines
@sanitize(
job=ChoicesSanitizer(INSTALLERS.keys(), required=True), )
@simple_response
def updater_job_status(self, job): # TODO: remove this completely
"""Returns the status of the current/last update even if the job is not running anymore."""
result = {}
try:
with open(INSTALLERS[job]['statusfile'], 'rb') as fd:
for line in fd:
fields = line.strip().split('=')
if len(fields) == 2:
result['_%s_' % fields[0]] = fields[1]
except (IOError, OSError):
pass
result['running'] = '' != self.__which_job_is_running()
return result
@sanitize(
job=ChoicesSanitizer(INSTALLERS.keys(), required=True),
detail=StringSanitizer(r'^[A-Za-z0-9\.\- ]*$'),
)
@simple_response
def run_installer(self, job, detail=''):
"""
This is the function that invokes any kind of installer. Arguments accepted:
job ..... the main thing to do. can be one of:
'release' ...... perform a release update
'distupgrade' .. update all currently installed packages (distupgrade)
'check' ........ check what would be done for 'update' ... do we need this?
detail ....... an argument that specifies the subject of the installer:
for 'release' .... the target release number,
for all other subjects: detail has no meaning.
"""
MODULE.info("Starting function %r" % (job, ))
self._current_job = job
# remember initial lines of logfile before starting update to not show it in the frontend
logfile = INSTALLERS[job]['logfile']
try:
with open(logfile, 'rb') as fd:
self._logfile_start_line = sum(1 for line in fd)
except (IOError, OSError):
pass
command = INSTALLERS[job]['command']
if '%' in command:
command = command % (pipes.quote(detail).replace('\n', '').replace(
'\r', '').replace('\x00', ''), )
prejob = '/usr/share/univention-updater/disable-apache2-umc'
postjob = '/usr/share/univention-updater/enable-apache2-umc --no-restart'
if job == 'release':
prejob = 'ucr set updater/maintenance=true'
postjob = 'ucr set updater/maintenance=false'
MODULE.info("Creating job: %r" % (command, ))
command = '''
%s
%s < /dev/null
%s''' % (prejob, command, postjob)
atjobs.add(command, comments=dict(lines=self._logfile_start_line))
return {'status': 0}
def __which_job_is_running(self):
# first check running at jobs
for atjob in atjobs.list(True):
for job, inst in INSTALLERS.iteritems():
cmd = inst['command'].split('%')[0]
if cmd in atjob.command:
self._current_job = job
try:
self._logfile_start_line = int(
atjob.comments.get('lines', 0))
except ValueError:
pass
return job
# no atjob found, parse process list (if univention-upgrade was started via CLI)
commands = [
('/usr/share/univention-updater/univention-updater-umc-dist-upgrade',
'distupgrade'),
('/usr/share/univention-updater/univention-updater', 'release'),
('/usr/sbin/univention-upgrade', 'distupgrade'
) # we don't know if it is a dist-upgrade or a release upgrade
]
for cmd, job in commands:
for process in psutil.process_iter():
try:
cmdline = process.cmdline() if callable(
process.cmdline) else process.cmdline
except psutil.NoSuchProcess:
pass
if cmd in cmdline:
self._current_job = job
self._logfile_start_line = 0
return job
return ''
class Instance(Base):
@sanitize(pattern=PatternSanitizer(default='.*'),
category=ChoicesSanitizer(
choices=['user', 'pid', 'command', 'all'], default='all'))
@simple_response
def query(self, pattern, category='all'):
processes = []
for process in psutil.process_iter():
try:
username = process.username()
except KeyError: # fixed in psutil 2.2.0
username = str(process.uids().real)
try:
cpu_time = process.cpu_times()
proc = {
'timestamp': time.time(),
'cpu_time': cpu_time.user + cpu_time.system,
'user': username,
'pid': process.pid,
'cpu': 0.0,
'mem': process.memory_percent(),
'command': ' '.join(process.cmdline() or [])
or process.name(),
}
except psutil.NoSuchProcess:
continue
categories = [category]
if category == 'all':
categories = ['user', 'pid', 'command']
if any(pattern.match(str(proc[cat])) for cat in categories):
processes.append(proc)
# Calculate correct cpu percentage
time.sleep(1)
for process_entry in processes:
try:
process = psutil.Process(process_entry['pid'])
cpu_time = process.cpu_times()
except psutil.NoSuchProcess:
continue
elapsed_time = time.time() - process_entry.pop('timestamp')
elapsed_cpu_time = cpu_time.user + cpu_time.system - process_entry.pop(
'cpu_time')
cpu_percent = (elapsed_cpu_time / elapsed_time) * 100
process_entry['cpu'] = cpu_percent
return processes
@sanitize(signal=ChoicesSanitizer(choices=['SIGTERM', 'SIGKILL']),
pid=ListSanitizer(IntegerSanitizer()))
@simple_response
def kill(self, signal, pid):
failed = []
for pid_ in pid:
try:
process = psutil.Process(pid_)
if signal == 'SIGTERM':
process.terminate()
elif signal == 'SIGKILL':
process.kill()
except psutil.NoSuchProcess as exc:
failed.append(str(pid_))
MODULE.error('Could not %s pid %s: %s' % (signal, pid_, exc))
if failed:
failed = ', '.join(failed)
raise UMC_Error(_('No process found with PID %s') % (failed))
class Instance(Base):
def init(self):
self.ucr = univention.config_registry.ConfigRegistry()
self.ucr.load()
self.connect()
self._update_system_roles_and_versions()
def connection(func):
def _connect(self, *args, **kwargs):
if self.dbConnection is None:
self.connect()
else:
self.test_connection()
return func(self, *args, **kwargs)
return _connect
def connect(self):
# Create a connection to the pkgdb
try:
self.dbConnection = updb.open_database_connection(self.ucr,
pkgdbu=True)
except pgdb.InternalError as ex:
MODULE.error(
'Could not establish connection to the PostgreSQL server: %s' %
(ex, ))
raise UMC_Error(
_('Could not establish connection to the database.\n\n%s') %
(_server_not_running_msg(), ))
else:
self.cursor = self.dbConnection.cursor()
def test_connection(self):
# test if connection is still active
try:
self.cursor.execute('SELECT TRUE')
except pgdb.OperationalError as ex:
MODULE.error('Connection to the PostgreSQL server lost: %s' %
(ex, ))
self.dbConnection = None
try:
self.connect()
except UMC_Error:
raise UMC_Error(
_('Connection to the database lost.\n\n%s') %
(_server_not_running_msg(), ))
@simple_response
def reinit(self):
"""Method invoked when opening the module in the frontend to cache and update some values"""
self._update_system_roles_and_versions()
def _update_system_roles_and_versions(self):
""" refetchs the variable lists (system roles and system versions) """
PROPOSALS['sysrole'] = self._get_system_roles()
PROPOSALS['sysversion'] = self._get_system_versions()
PROPOSALS['sysversion_lower'] = PROPOSALS['sysversion']
PROPOSALS['sysversion_greater'] = PROPOSALS['sysversion']
@connection
def _get_system_roles(self):
return [role[0] for role in updb.sql_getall_systemroles(self.cursor)]
@connection
def _get_system_versions(self):
return [
version[0]
for version in updb.sql_getall_systemversions(self.cursor)
]
@sanitize(page=ChoicesSanitizer(choices=PAGES, required=True),
key=ChoicesSanitizer(choices=CRITERIA_OPERATOR.keys()))
@connection
@simple_response
def query(self, page, key, pattern=''):
""" Query to fill the grid. The structure of the corresponding grid
has already been fetched by the 'pkgdb/columns' command.
"""
desc = QUERIES[page]
operator = CRITERIA_OPERATOR[key]
function = desc['function']
kwargs = desc.get('args', {})
kwargs['query'] = _make_query(key, operator, pattern)
result = function(self.cursor, **kwargs)
names = desc.get('db_fields', desc['columns'])
return [_convert_to_grid(record, names) for record in result]
@sanitize(page=ChoicesSanitizer(choices=PAGES, required=True))
@connection
@simple_response
@log
def keys(self, page):
""" returns the set of search criteria suitable for the given page. """
return _combobox_data(CRITERIA[page])
@sanitize(page=ChoicesSanitizer(choices=PAGES, required=True))
@connection
@simple_response
@log
def proposals(self, page, key=''):
""" returns proposals for the query pattern that can be
presented in the frontend. This can be a single pattern
(the corresponding field will turn into a text entry)
or an array (the field will turn into a ComboBox,
with optionally translated labels)
"""
if key in PROPOSALS:
return _combobox_data(PROPOSALS[key])
# fallback for everything not explicitly listed here.
return ''
@sanitize(page=ChoicesSanitizer(choices=PAGES, required=True))
@connection
@simple_response
@log
def columns(self, page, key=''):
""" returns the structure of the results grid for a given
page+key combination. Note that design properties (width etc)
are added at the JS page (KeyTranslator.js)
"""
return QUERIES[page]['columns']
class Cloud(object):
"""
Handle cloud connections and instances.
"""
@sanitize(nodePattern=SearchSanitizer(default='*'))
def cloud_query(self, request):
"""
Searches clouds by the given pattern
options: {'nodePattern': <cloud pattern>}
return: [{
'id': <cloud name>,
'label': <cloud name>,
'group': 'cloudconnection',
'type': 'cloud',
'available': (True|False),
}, ...]
"""
self.required_options(request, 'nodePattern')
def _finished(data):
"""
Process asynchronous UVMM L_CLOUD_LIST answer.
"""
return [{
'id': d.name,
'label': d.name,
'group': _('Cloud connection'),
'type': 'cloud',
'cloudtype': d.cloudtype,
'available': d.available,
'last_error_message': d.last_error_message,
'dn': d.dn,
'search_pattern': d.search_pattern,
'ucs_images': d.ucs_images,
} for d in data]
self.uvmm.send(
'L_CLOUD_LIST',
self.process_uvmm_response(request, _finished),
pattern=request.options['nodePattern']
)
def cloud_add(self, request):
"""
Add a new cloud connection into ldap.
options: {
['cloudtype': <uvmm/cloudtype>,]
['name': <new cloud name>,]
['parameter': <key/value parameter>,]
['testconnection': true (default) / false,]
}
return: []
"""
def _finished(data):
# add cloud to ldap
ldap_cloud_connection_add(cloudtype, name, parameter, ucs_images, search_pattern, preselected_images)
return data
self.required_options(request, 'cloudtype', 'name', 'parameter', 'testconnection')
cloudtype = request.options.get('cloudtype')
name = request.options.get('name')
testconnection = request.options.get('testconnection')
parameter = request.options.get('parameter', {})
search_pattern = parameter.pop('search_pattern', '')
preselected_images = parameter.pop('preselected_images', [])
ucs_images = parameter.pop('ucs_images', True)
# add cloud to uvmm
args = parameter.copy()
args['name'] = name
args['type'] = cloudtype
args['search_pattern'] = search_pattern
args['preselected_images'] = preselected_images
args['ucs_images'] = ucs_images
self.uvmm.send(
'L_CLOUD_ADD',
self.process_uvmm_response(request, _finished),
args=args,
testconnection=testconnection
)
def cloud_list_keypair(self, request):
"""
Returns a list of keypair for the given cloud conn_name.
"""
self.required_options(request, 'conn_name')
conn_name = request.options.get('conn_name')
def _finished(data):
"""
Process asynchronous UVMM L_CLOUD_KEYPAIR_LIST answer.
"""
return [
{'id': item.name, 'label': item.name}
for conn_name, images in data.items()
for item in images
]
self.uvmm.send(
'L_CLOUD_KEYPAIR_LIST',
self.process_uvmm_response(request, _finished),
conn_name=conn_name
)
def cloud_list_size(self, request):
"""
Returns a list of hardware sizes for the given cloud conn_name.
"""
self.required_options(request, 'conn_name')
conn_name = request.options.get('conn_name')
def _finished(data):
"""
Process asynchronous UVMM L_CLOUD_SIZE_LIST answer.
"""
size_list = []
for conn_name, images in data.items():
for item in images:
size_list.append({
'id': item.id,
'label': item.u_displayname,
'disk': item.disk,
'ram': item.ram,
'vcpus': item.vcpus,
})
return size_list
self.uvmm.send(
'L_CLOUD_SIZE_LIST',
self.process_uvmm_response(request, _finished),
conn_name=conn_name
)
@sanitize(pattern=SearchSanitizer(default='*'))
def cloud_list_image(self, request):
"""
Returns a list of images by a pattern for the given cloud conn_name.
"""
self.required_options(request, 'conn_name')
conn_name = request.options.get('conn_name')
def _finished(data):
"""
Process asynchronous UVMM L_CLOUD_IMAGE_LIST answer.
"""
return [
{'id': item.id, 'label': item.name}
for conn_name, images in data.items()
for item in images
]
self.uvmm.send(
'L_CLOUD_IMAGE_LIST',
self.process_uvmm_response(request, _finished),
conn_name=conn_name
)
def cloud_list_secgroup(self, request):
"""
Returns a list of security groups for the given cloud conn_name.
"""
self.required_options(request, 'conn_name')
conn_name = request.options.get('conn_name')
network_id = request.options.get('network_id')
def _finished(data):
"""
Process asynchronous UVMM L_CLOUD_SECGROUP_LIST answer.
"""
return [
{'id': item.id, 'label': item.name}
for conn_name, images in data.items()
for item in images if network_id in ('default', item.network_id)
]
self.uvmm.send(
'L_CLOUD_SECGROUP_LIST',
self.process_uvmm_response(request, _finished),
conn_name=conn_name
)
def cloud_list_network(self, request):
"""
Returns a list of networks for the given cloud conn_name.
"""
self.required_options(request, 'conn_name')
conn_name = request.options.get('conn_name')
def _finished(data):
"""
Process asynchronous UVMM L_CLOUD_NETWORK_LIST answer.
"""
return [
{
'id': item.id,
'label': '%s %s' % (item.name, item.cidr or "")
}
for conn_name, images in data.items()
for item in images
]
self.uvmm.send(
'L_CLOUD_NETWORK_LIST',
self.process_uvmm_response(request, _finished),
conn_name=conn_name
)
def cloud_list_subnet(self, request):
"""
Returns a list of subnet for the given cloud conn_name.
"""
self.required_options(request, 'conn_name')
conn_name = request.options.get('conn_name')
network_id = request.options.get('network_id')
def _finished(data):
"""
Process asynchronous UVMM L_CLOUD_SUBNET_LIST answer.
"""
return [
{
'id': item.id,
'label': '%s %s' % (item.name, item.cidr or "")
}
for conn_name, images in data.items()
for item in images if network_id == item.network_id
]
self.uvmm.send(
'L_CLOUD_SUBNET_LIST',
self.process_uvmm_response(request, _finished),
conn_name=conn_name
)
@sanitize(domainPattern=SearchSanitizer(default='*'))
def instance_query(self, request):
"""
Returns a list of instances matching domainPattern on the clouds matching nodePattern.
options: {
['nodePattern': <cloud pattern>,]
['domainPattern': <instance pattern>,]
}
return: [{
'node_available': True,
'extra': {
'key_name': None,
'disk_config': 'MANUAL',
'flavorId': '1',
'availability_zone': 'nova',
'password': None,
'metadata': {}
},
'label': 'automagic-997898',
'type': 'instance',
'id': 'myCloud2#e2c8e274-2e17-499c-a3f9-620fb249578c',
'nodeName': 'myCloud2'
}, ... ]
"""
def _finished(data):
"""
Process asynchronous UVMM L_CLOUD_INSTANCE_LIST answer.
"""
instances = []
for conn_name, insts in data.items():
for inst in insts:
instance_uri = '%s#%s' % (conn_name, inst.id)
instances.append({
'id': instance_uri,
'label': inst.name,
'nodeName': conn_name,
'state': inst.state,
'type': 'instance',
'suspended': None, # FIXME
'description': '%s [%s]' % (inst.u_size_name, inst.state),
'node_available': inst.available,
'extra': inst.extra,
'public_ips': inst.public_ips,
'private_ips': inst.private_ips,
'u_size_name': inst.u_size_name,
'u_connection_type': inst.u_connection_type,
'keypair': inst.key_name,
'image': inst.u_image_name,
'securitygroup': inst.secgroups,
})
return instances
self.uvmm.send(
'L_CLOUD_INSTANCE_LIST',
self.process_uvmm_response(request, _finished),
conn_name=request.options.get('nodePattern', ''),
pattern=request.options['domainPattern']
)
@sanitize(state=ChoicesSanitizer(choices=('RUN', 'RESTART', 'SOFTRESTART', 'SHUTOFF', 'SHUTDOWN', 'SUSPEND', 'PAUSE', 'RESUME', 'UNPAUSE')))
def instance_state(self, request):
"""
Set the state a instance instance_id on cloud conn_name.
options: {
'uri': <conn_name#instance_id>,
'state': (RUN|RESTART|SOFTRESTART|SHUTOFF|SHUTDOWN|SUSPEND|RESUME|UNPAUSE),
}
return:
"""
self.required_options(request, 'uri', 'state')
conn_name, instance_id = urldefrag(request.options['uri'])
state = request.options['state']
self.uvmm.send(
'L_CLOUD_INSTANCE_STATE',
self.process_uvmm_response(request),
conn_name=conn_name,
instance_id=instance_id,
state=state,
)
def instance_remove(self, request):
"""
Removes a instance.
options: {
'domainURI': <domain uri>
}
return:
"""
self.required_options(request, 'domainURI')
conn_name, instance_id = urldefrag(request.options['domainURI'])
self.uvmm.send(
'L_CLOUD_INSTANCE_TERMINATE',
self.process_uvmm_response(request),
conn_name=conn_name,
instance_id=instance_id
)
def instance_add(self, request):
"""
Create a new instance on cloud conn_name.
options: {
'conn_name': <cloud connection name>,
'parameter': {...},
}
return:
"""
self.required_options(request, 'conn_name', 'name', 'parameter')
conn_name = request.options.get('conn_name')
name = request.options.get('name')
parameter = request.options.get('parameter')
args = parameter
args['name'] = name
args['security_group_ids'] = [parameter['security_group_ids']]
self.uvmm.send(
'L_CLOUD_INSTANCE_CREATE',
self.process_uvmm_response(request),
conn_name=conn_name,
args=args
)
def cloudtype_get(self, request):
"""
Returns a list of all cloudtypes from ldap.
"""
cloudtypes = []
for item in ldap_cloud_types():
cloudtypes.append({
'id': item['name'],
'label': item['name']
})
self.finished(request.id, cloudtypes)
class Instance(umcm.Base, ProgressMixin):
def init(self):
os.umask(0o022) # umc umask is too restrictive for app center as it creates a lot of files in docker containers
self.ucr = ucr_instance()
self.update_applications_done = False
util.install_opener(self.ucr)
self._remote_progress = {}
try:
self.package_manager = PackageManager(
info_handler=MODULE.process,
step_handler=None,
error_handler=MODULE.warn,
lock=False,
)
except SystemError as exc:
MODULE.error(str(exc))
raise umcm.UMC_Error(str(exc), status=500)
self.package_manager.set_finished() # currently not working. accepting new tasks
self.uu = UniventionUpdater(False)
self.component_manager = util.ComponentManager(self.ucr, self.uu)
get_package_manager._package_manager = self.package_manager
# in order to set the correct locale for Application
locale.setlocale(locale.LC_ALL, str(self.locale))
try:
log_to_logfile()
except IOError:
pass
# connect univention.appcenter.log to the progress-method
handler = ProgressInfoHandler(self.package_manager)
handler.setLevel(logging.INFO)
get_base_logger().addHandler(handler)
percentage = ProgressPercentageHandler(self.package_manager)
percentage.setLevel(logging.DEBUG)
get_base_logger().getChild('actions.install.progress').addHandler(percentage)
get_base_logger().getChild('actions.upgrade.progress').addHandler(percentage)
get_base_logger().getChild('actions.remove.progress').addHandler(percentage)
def error_handling(self, etype, exc, etraceback):
error_handling(etype, exc, etraceback)
return super(Instance, self).error_handling(exc, etype, etraceback)
@simple_response
def version(self):
info = get_action('info')
return info.get_compatibility()
@simple_response
def query(self, quick=False):
if not quick:
self.update_applications()
self.ucr.load()
reload_package_manager()
list_apps = get_action('list')
domain = get_action('domain')
apps = list_apps.get_apps()
if self.ucr.is_true('appcenter/docker', True):
if not self._test_for_docker_service():
raise umcm.UMC_Error(_('The docker service is not running! The App Center will not work properly.') + ' ' + _('Make sure docker.io is installed, try starting the service with "service docker start".'))
info = domain.to_dict(apps)
if quick:
ret = []
for app in info:
if app is None:
ret.append(None)
else:
short_info = {}
for attr in ['id', 'name', 'vendor', 'maintainer', 'description', 'long_description', 'categories', 'end_of_life', 'update_available', 'logo_name', 'is_installed_anywhere', 'is_installed', 'installations']:
short_info[attr] = app[attr]
ret.append(short_info)
return ret
else:
return info
def update_applications(self):
if self.ucr.is_true('appcenter/umc/update/always', True):
update = get_action('update')
try:
update.call()
except NetworkError as err:
raise umcm.UMC_Error(str(err))
except Abort:
pass
Application._all_applications = None
self.update_applications_done = True
def _test_for_docker_service(self):
if docker_bridge_network_conflict():
msg = _('A conflict between the system network settings and the docker bridge default network has been detected.') + '\n\n'
msg += _('Please either configure a different network for the docker bridge by setting the UCR variable docker/daemon/default/opts/bip to a different network and restart the system,') + ' '
msg += _('or disable the docker support in the AppCenter by setting appcenter/docker to false.')
raise umcm.UMC_Error(msg)
if not docker_is_running():
MODULE.warn('Docker is not running! Trying to start it now...')
call_process(['invoke-rc.d', 'docker', 'start'])
if not docker_is_running():
return False
return True
@simple_response
def enable_docker(self):
if self._test_for_docker_service():
ucr_save({'appcenter/docker': 'enabled'})
else:
raise umcm.UMC_Error(_('Unable to start the docker service!') + ' ' + _('Make sure docker.io is installed, try starting the service with "service docker start".'))
@require_apps_update
@require_password
@simple_response(with_progress=True)
def sync_ldap(self):
register = get_action('register')
register.call(username=self.username, password=self.password)
# used in updater-umc
@simple_response
def get_by_component_id(self, component_id):
domain = get_action('domain')
if isinstance(component_id, list):
requested_apps = [Apps().find_by_component_id(cid) for cid in component_id]
return domain.to_dict(requested_apps)
else:
app = Apps().find_by_component_id(component_id)
if app:
return domain.to_dict([app])[0]
else:
raise umcm.UMC_Error(_('Could not find an application for %s') % component_id)
# used in updater-umc
@simple_response
def app_updates(self):
upgrade = get_action('upgrade')
domain = get_action('domain')
return domain.to_dict(list(upgrade.iter_upgradable_apps()))
@sanitize(application=StringSanitizer(minimum=1, required=True))
@simple_response
def get(self, application):
domain = get_action('domain')
app = Apps().find(application)
if app is None:
raise umcm.UMC_Error(_('Could not find an application for %s') % (application,))
return domain.to_dict([app])[0]
@sanitize(app=AppSanitizer(required=True), values=DictSanitizer({}))
@simple_response
def configure(self, app, autostart, values):
configure = get_action('configure')
configure.call(app=app, set_vars=values, autostart=autostart)
@sanitize(app=AppSanitizer(required=True), mode=ChoicesSanitizer(['start', 'stop']))
@simple_response
def app_service(self, app, mode):
service = get_action(mode)
service.call(app=app)
@sanitize(app=AppSanitizer(required=False), action=ChoicesSanitizer(['get', 'buy', 'search']), value=StringSanitizer())
@simple_response
def track(self, app, action, value):
send_information(action, app=app, value=value)
def invoke_dry_run(self, request):
request.options['only_dry_run'] = True
self.invoke(request)
@require_password
@sanitize(
host=StringSanitizer(required=True),
function=ChoicesSanitizer(['install', 'update', 'uninstall'], required=True),
app=StringSanitizer(required=True),
force=BooleanSanitizer(),
values=DictSanitizer({})
)
@simple_response(with_progress=True)
def invoke_remote_docker(self, host, function, app, force, values, progress):
options = {'function': function, 'app': app, 'force': force, 'values': values}
client = Client(host, self.username, self.password)
result = client.umc_command('appcenter/docker/invoke', options).result
self._remote_progress[progress.id] = client, result['id']
@simple_response
def remote_progress(self, progress_id):
try:
client, remote_progress_id = self._remote_progress[progress_id]
except KeyError:
# actually happens: before invoke_remote_docker is finished, remote_progress is already called
return {}
else:
return client.umc_command('appcenter/docker/progress', {'progress_id': remote_progress_id}).result
@require_apps_update
@require_password
@sanitize(
function=MappingSanitizer({
'install': 'install',
'update': 'upgrade',
'uninstall': 'remove',
}, required=True),
app=AppSanitizer(required=True),
force=BooleanSanitizer(),
values=DictSanitizer({})
)
@simple_response(with_progress=True)
def invoke_docker(self, function, app, force, values, progress):
if function == 'upgrade':
app = Apps().find_candidate(app)
serious_problems = False
progress.title = _('%s: Running tests') % (app.name,)
errors, warnings = app.check(function)
can_continue = force # "dry_run"
if errors:
MODULE.process('Cannot %s %s: %r' % (function, app.id, errors))
serious_problems = True
can_continue = False
if warnings:
MODULE.process('Warning trying to %s %s: %r' % (function, app.id, warnings))
result = {
'serious_problems': serious_problems,
'invokation_forbidden_details': errors,
'invokation_warning_details': warnings,
'can_continue': can_continue,
'software_changes_computed': False,
}
if can_continue:
with self.locked():
kwargs = {'noninteractive': True, 'skip_checks': ['shall_have_enough_ram', 'shall_only_be_installed_in_ad_env_with_password_service', 'must_not_have_concurrent_operation']}
if function == 'install':
progress.title = _('Installing %s') % (app.name,)
kwargs['set_vars'] = values
elif function == 'uninstall':
progress.title = _('Uninstalling %s') % (app.name,)
elif function == 'upgrade':
progress.title = _('Upgrading %s') % (app.name,)
action = get_action(function)
handler = UMCProgressHandler(progress)
handler.setLevel(logging.INFO)
action.logger.addHandler(handler)
try:
result['success'] = action.call(app=app, username=self.username, password=self.password, **kwargs)
except AppCenterError as exc:
raise umcm.UMC_Error(str(exc), result=dict(
display_feedback=True,
title='%s %s' % (exc.title, exc.info)))
finally:
action.logger.removeHandler(handler)
return result
@contextmanager
def locked(self):
try:
with self.package_manager.locked(reset_status=True, set_finished=True):
yield
except LockError:
raise umcm.UMC_Error(_('Another package operation is in progress'))
@require_apps_update
@require_password
@sanitize(
function=ChoicesSanitizer(['install', 'uninstall', 'update', 'install-schema', 'update-schema'], required=True),
application=StringSanitizer(minimum=1, required=True),
force=BooleanSanitizer(),
host=StringSanitizer(),
only_dry_run=BooleanSanitizer(),
dont_remote_install=BooleanSanitizer(),
values=DictSanitizer({})
)
def invoke(self, request):
# ATTENTION!!!!!!!
# this function has to stay compatible with the very first App Center installations (Dec 2012)
# if you add new arguments that change the behaviour
# you should add a new method (see invoke_dry_run) or add a function name (e.g. install-schema)
# this is necessary because newer app center may talk remotely with older one
# that does not understand new arguments and behaves the old way (in case of
# dry_run: install application although they were asked to dry_run)
host = request.options.get('host')
function = request.options.get('function')
send_as = function
if function.startswith('install'):
function = 'install'
if function.startswith('update'):
function = 'update'
application_id = request.options.get('application')
Application.all(only_local=True) # if not yet cached, cache. but use only local inis
application = Application.find(application_id)
if application is None:
raise umcm.UMC_Error(_('Could not find an application for %s') % (application_id,))
force = request.options.get('force')
only_dry_run = request.options.get('only_dry_run')
dont_remote_install = request.options.get('dont_remote_install')
only_master_packages = send_as.endswith('schema')
MODULE.process('Try to %s (%s) %s on %s. Force? %r. Only master packages? %r. Prevent installation on other systems? %r. Only dry run? %r.' % (function, send_as, application_id, host, force, only_master_packages, dont_remote_install, only_dry_run))
# REMOTE invocation!
if host and host != self.ucr.get('hostname'):
try:
client = Client(host, self.username, self.password)
result = client.umc_command('appcenter/invoke', request.options).result
except (ConnectionError, HTTPError) as exc:
MODULE.error('Error during remote appcenter/invoke: %s' % (exc,))
result = {
'unreachable': [host],
'master_unreachable': True,
'serious_problems': True,
'software_changes_computed': True, # not really...
}
else:
if result['can_continue']:
def _thread_remote(_client, _package_manager):
with _package_manager.locked(reset_status=True, set_finished=True):
_package_manager.unlock() # not really locked locally, but busy, so "with locked()" is appropriate
Application._query_remote_progress(_client, _package_manager)
def _finished_remote(thread, result):
if isinstance(result, BaseException):
MODULE.warn('Exception during %s %s: %s' % (function, application_id, str(result)))
thread = notifier.threads.Simple('invoke', notifier.Callback(_thread_remote, client, self.package_manager), _finished_remote)
thread.run()
self.finished(request.id, result)
return
# make sure that the application can be installed/updated
can_continue = True
delayed_can_continue = True
serious_problems = False
result = {
'install': [],
'remove': [],
'broken': [],
'unreachable': [],
'master_unreachable': False,
'serious_problems': False,
'hosts_info': {},
'problems_with_hosts': False,
'serious_problems_with_hosts': False,
'invokation_forbidden_details': {},
'invokation_warning_details': {},
'software_changes_computed': False,
}
if not application:
MODULE.process('Application not found: %s' % application_id)
can_continue = False
if can_continue and not only_master_packages:
forbidden, warnings = application.check_invokation(function, self.package_manager)
if forbidden:
MODULE.process('Cannot %s %s: %r' % (function, application_id, forbidden))
result['invokation_forbidden_details'] = forbidden
can_continue = False
serious_problems = True
if warnings:
MODULE.process('Warning trying to %s %s: %r' % (function, application_id, forbidden))
result['invokation_warning_details'] = warnings
if not force:
# dont stop "immediately".
# compute the package changes!
delayed_can_continue = False
result['serious_problems'] = serious_problems
result['can_continue'] = can_continue
try:
if can_continue:
if self._working():
# make it multi-tab safe (same session many buttons to be clicked)
raise LockError()
with self.package_manager.locked(reset_status=True):
previously_registered_by_dry_run = False
if can_continue and function in ('install', 'update'):
remove_component = only_dry_run
dry_run_result, previously_registered_by_dry_run = application.install_dry_run(self.package_manager, self.component_manager, remove_component=remove_component, username=self._username, password=self.password, only_master_packages=only_master_packages, dont_remote_install=dont_remote_install, function=function, force=force)
result.update(dry_run_result)
result['software_changes_computed'] = True
serious_problems = bool(result['broken'] or result['master_unreachable'] or result['serious_problems_with_hosts'])
if serious_problems or (not force and (result['unreachable'] or result['install'] or result['remove'] or result['problems_with_hosts'])):
MODULE.process('Problems encountered or confirmation required. Removing component %s' % application.component_id)
if not remove_component:
# component was not removed automatically after dry_run
if application.candidate:
# operation on candidate failed. re-register original application
application.register(self.component_manager, self.package_manager)
else:
# operation on self failed. unregister all
application.unregister_all_and_register(None, self.component_manager, self.package_manager)
can_continue = False
elif can_continue and function in ('uninstall',) and not force:
result['remove'] = application.uninstall_dry_run(self.package_manager)
result['software_changes_computed'] = True
can_continue = False
can_continue = can_continue and delayed_can_continue and not only_dry_run
result['serious_problems'] = serious_problems
result['can_continue'] = can_continue
if can_continue and not only_dry_run:
def _thread(module, application, function):
with module.package_manager.locked(set_finished=True):
with module.package_manager.no_umc_restart(exclude_apache=True):
if function in ('install', 'update'):
# dont have to add component: already added during dry_run
return application.install(module.package_manager, module.component_manager, add_component=only_master_packages, send_as=send_as, username=self._username, password=self.password, only_master_packages=only_master_packages, dont_remote_install=dont_remote_install, previously_registered_by_dry_run=previously_registered_by_dry_run)
else:
return application.uninstall(module.package_manager, module.component_manager, self._username, self.password)
def _finished(thread, result):
if isinstance(result, BaseException):
MODULE.warn('Exception during %s %s: %s' % (function, application_id, str(result)))
thread = notifier.threads.Simple('invoke', notifier.Callback(_thread, self, application, function), _finished)
thread.run()
else:
self.package_manager.set_finished() # nothing to do, ready to take new commands
self.finished(request.id, result)
except LockError:
# make it thread safe: another process started a package manager
# this module instance already has a running package manager
raise umcm.UMC_Error(_('Another package operation is in progress'))
def keep_alive(self, request):
''' Fix for Bug #30611: UMC kills appcenter module
if no request is sent for $(ucr get umc/module/timeout).
this happens if a user logs out during a very long installation.
this function will be run by the frontend to always have one connection open
to prevent killing the module. '''
def _thread():
while not self.package_manager.progress_state._finished:
time.sleep(1)
def _finished(thread, result):
success = not isinstance(result, BaseException)
if not success:
MODULE.warn('Exception during keep_alive: %s' % result)
self.finished(request.id, success)
thread = notifier.threads.Simple('keep_alive', notifier.Callback(_thread), _finished)
thread.run()
@simple_response
def ping(self):
return True
@simple_response
def buy(self, application):
app = Apps().find(application)
if not app or not app.shop_url:
return None
ret = {}
ret['key_id'] = LICENSE.uuid
ret['ucs_version'] = self.ucr.get('version/version')
ret['app_id'] = app.id
ret['app_version'] = app.version
# ret['locale'] = locale.getlocale()[0] # done by frontend
ret['user_count'] = None # FIXME: get users and computers from license
ret['computer_count'] = None
return ret
@simple_response
def enable_disable_app(self, application, enable=True):
app = Apps().find(application)
if not app:
return
stall = get_action('stall')
stall.call(app=app, undo=enable)
@simple_response
def packages_sections(self):
""" fills the 'sections' combobox in the search form """
sections = set()
cache = apt.Cache()
for package in cache:
sections.add(package.section)
return sorted(sections)
@sanitize(pattern=PatternSanitizer(required=True))
@simple_response
def packages_query(self, pattern, section='all', key='package'):
""" Query to fill the grid. Structure is fixed here. """
result = []
for package in self.package_manager.packages(reopen=True):
if section == 'all' or package.section == section:
toshow = False
if pattern.pattern == '^.*$':
toshow = True
elif key == 'package' and pattern.search(package.name):
toshow = True
elif key == 'description' and package.candidate and pattern.search(package.candidate.raw_description):
toshow = True
if toshow:
result.append(self._package_to_dict(package, full=False))
return result
@simple_response
def packages_get(self, package):
""" retrieves full properties of one package """
package = self.package_manager.get_package(package)
if package is not None:
return self._package_to_dict(package, full=True)
else:
# TODO: 404?
return {}
@sanitize(
function=MappingSanitizer({
'install': 'install',
'upgrade': 'install',
'uninstall': 'remove',
}, required=True),
packages=ListSanitizer(StringSanitizer(minimum=1), required=True),
update=BooleanSanitizer()
)
@simple_response
def packages_invoke_dry_run(self, packages, function, update):
if update:
self.package_manager.update()
packages = self.package_manager.get_packages(packages)
kwargs = {'install': [], 'remove': [], 'dry_run': True}
if function == 'install':
kwargs['install'] = packages
else:
kwargs['remove'] = packages
return dict(zip(['install', 'remove', 'broken'], self.package_manager.mark(**kwargs)))
@sanitize(
function=MappingSanitizer({
'install': 'install',
'upgrade': 'install',
'uninstall': 'remove',
}, required=True),
packages=ListSanitizer(StringSanitizer(minimum=1), required=True)
)
def packages_invoke(self, request):
""" executes an installer action """
packages = request.options.get('packages')
function = request.options.get('function')
try:
if self._working():
# make it multi-tab safe (same session many buttons to be clicked)
raise LockError()
with self.package_manager.locked(reset_status=True):
not_found = [pkg_name for pkg_name in packages if self.package_manager.get_package(pkg_name) is None]
self.finished(request.id, {'not_found': not_found})
if not not_found:
def _thread(package_manager, function, packages):
with package_manager.locked(set_finished=True):
with package_manager.no_umc_restart(exclude_apache=True):
if function == 'install':
package_manager.install(*packages)
else:
package_manager.uninstall(*packages)
def _finished(thread, result):
if isinstance(result, BaseException):
MODULE.warn('Exception during %s %s: %r' % (function, packages, str(result)))
thread = notifier.threads.Simple('invoke', notifier.Callback(_thread, self.package_manager, function, packages), _finished)
thread.run()
else:
self.package_manager.set_finished() # nothing to do, ready to take new commands
except LockError:
# make it thread safe: another process started a package manager
# this module instance already has a running package manager
raise umcm.UMC_Error(_('Another package operation is in progress'))
def _working(self):
return not self.package_manager.progress_state._finished
@simple_response
def working(self):
# TODO: PackageManager needs is_idle() or something
# preferably the package_manager can tell what is currently executed:
# package_manager.is_working() => False or _('Installing UCC')
return self._working()
@simple_response
def custom_progress(self):
timeout = 5
return self.package_manager.poll(timeout)
def _package_to_dict(self, package, full):
""" Helper that extracts properties from a 'apt_pkg.Package' object
and stores them into a dictionary. Depending on the 'full'
switch, stores only limited (for grid display) or full
(for detail view) set of properties.
"""
installed = package.installed # may be None
found = True
candidate = package.candidate
found = candidate is not None
if not found:
candidate = NoneCandidate()
result = {
'package': package.name,
'installed': package.is_installed,
'upgradable': package.is_upgradable and found,
'summary': candidate.summary,
}
# add (and translate) a combined status field
# *** NOTE *** we translate it here: if we would use the Custom Formatter
# of the grid then clicking on the sort header would not work.
if package.is_installed:
if package.is_upgradable:
result['status'] = _('upgradable')
else:
result['status'] = _('installed')
else:
result['status'] = _('not installed')
# additional fields needed for detail view
if full:
# Some fields differ depending on whether the package is installed or not:
if package.is_installed:
result['section'] = installed.section
result['priority'] = installed.priority or ''
result['summary'] = installed.summary # take the current one
result['description'] = installed.description
result['installed_version'] = installed.version
result['size'] = installed.installed_size
if package.is_upgradable:
result['candidate_version'] = candidate.version
else:
del result['upgradable'] # not installed: don't show 'upgradable' at all
result['section'] = candidate.section
result['priority'] = candidate.priority or ''
result['description'] = candidate.description
result['size'] = candidate.installed_size
result['candidate_version'] = candidate.version
# format size to handle bytes
size = result['size']
byte_mods = ['B', 'kB', 'MB']
for byte_mod in byte_mods:
if size < 10000:
break
size = float(size) / 1000 # MB, not MiB
else:
size = size * 1000 # once too often
if size == int(size):
format_string = '%d %s'
else:
format_string = '%.2f %s'
result['size'] = format_string % (size, byte_mod)
return result
@simple_response
def components_query(self):
""" Returns components list for the grid in the ComponentsPage.
"""
# be as current as possible.
self.uu.ucr_reinit()
self.ucr.load()
result = []
for comp in self.uu.get_all_components():
result.append(self.component_manager.component(comp))
return result
@sanitize_list(StringSanitizer())
@multi_response(single_values=True)
def components_get(self, iterator, component_id):
# be as current as possible.
self.uu.ucr_reinit()
self.ucr.load()
for component_id in iterator:
yield self.component_manager.component(component_id)
@sanitize_list(DictSanitizer({'object': advanced_components_sanitizer}))
@multi_response
def components_put(self, iterator, object):
"""Writes back one or more component definitions.
"""
# umc.widgets.Form wraps the real data into an array:
#
# [
# {
# 'object' : { ... a dict with the real data .. },
# 'options': None
# },
# ... more such entries ...
# ]
#
# Current approach is to return a similarly structured array,
# filled with elements, each one corresponding to one array
# element of the request:
#
# [
# {
# 'status' : a number where 0 stands for success, anything else
# is an error code
# 'message' : a result message
# 'object' : a dict of field -> error message mapping, allows
# the form to show detailed error information
# },
# ... more such entries ...
# ]
with util.set_save_commit_load(self.ucr) as super_ucr:
for object, in iterator:
yield self.component_manager.put(object, super_ucr)
self.package_manager.update()
# do the same as components_put (update)
# but dont allow adding an already existing entry
components_add = sanitize_list(DictSanitizer({'object': add_components_sanitizer}))(components_put)
components_add.__name__ = 'components_add'
@sanitize_list(StringSanitizer())
@multi_response(single_values=True)
def components_del(self, iterator, component_id):
for component_id in iterator:
yield self.component_manager.remove(component_id)
self.package_manager.update()
@multi_response
def settings_get(self, iterator):
# *** IMPORTANT *** Our UCR copy must always be current. This is not only
# to catch up changes made via other channels (ucr command line etc),
# but also to reflect the changes we have made ourselves!
self.ucr.load()
for _ in iterator:
yield {
'unmaintained': self.ucr.is_true('repository/online/unmaintained', False),
'server': self.ucr.get('repository/online/server', ''),
'prefix': self.ucr.get('repository/online/prefix', ''),
}
@sanitize_list(
DictSanitizer({'object': basic_components_sanitizer}),
min_elements=1,
max_elements=1 # moduleStore with one element...
)
@multi_response
def settings_put(self, iterator, object):
# FIXME: returns values although it should yield (multi_response)
changed = False
# Set values into our UCR copy.
try:
with util.set_save_commit_load(self.ucr) as super_ucr:
for object, in iterator:
for key, value in object.iteritems():
MODULE.info(" ++ Setting new value for '%s' to '%s'" % (key, value))
super_ucr.set_registry_var('%s/%s' % (constants.ONLINE_BASE, key), value)
changed = super_ucr.changed()
except Exception as e:
MODULE.warn(" !! Writing UCR failed: %s" % str(e))
return [{'message': str(e), 'status': constants.PUT_WRITE_ERROR}]
self.package_manager.update()
# Bug #24878: emit a warning if repository is not reachable
try:
updater = self.uu
for line in updater.print_version_repositories().split('\n'):
if line.strip():
break
else:
raise ConfigurationError()
except ConfigurationError:
msg = _("There is no repository at this server (or at least none for the current UCS version)")
MODULE.warn(" !! Updater error: %s" % msg)
response = {'message': msg, 'status': constants.PUT_UPDATER_ERROR}
# if nothing was committed, we want a different type of error code,
# just to appropriately inform the user
if changed:
response['status'] = constants.PUT_UPDATER_NOREPOS
return [response]
except:
info = sys.exc_info()
emsg = '%s: %s' % info[:2]
MODULE.warn(" !! Updater error [%s]: %s" % (emsg))
return [{'message': str(info[1]), 'status': constants.PUT_UPDATER_ERROR}]
return [{'status': constants.PUT_SUCCESS}]