def update_installed_packages(uuid, installed_packages):
"""Stores packages json data into separate HostPackagesStatus
Args:
uuid (str) : unique ID of host
installed_packages (list): data from host
Returns:
"""
# TODO : be smarter : insert / update / delete instead of delete all / insert all ?
# is it faster ?
key = ['package','version','version','architecture','locale','maturity']
def get_key(rec):
return {k:rec[k] for k in rec}
#old_installed = HostPackagesStatus.select().where(HostPackagesStatus.host == uuid).dicts()
def encode_value(value):
if isinstance(value,unicode):
value = value.replace(u'\x00', ' ')
return value
HostPackagesStatus.delete().where(HostPackagesStatus.host == uuid).execute()
packages = []
for package in installed_packages:
package['host'] = uuid
# csv str on the client, Array on the server
package['depends'] = ensure_list(package['depends'])
package['conflicts'] = ensure_list(package['conflicts'])
# filter out all unknown fields from json data for the SQL insert
packages.append(dict([(k, encode_value(v)) for k, v in package.iteritems() if k in HostPackagesStatus._meta.fields]))
if packages:
return HostPackagesStatus.insert_many(packages).execute() # pylint: disable=no-value-for-parameter
else:
return True
def remove_from_csv_list(csv_list,new_items):
"""Remove items from csv_list"""
items = ensure_list(csv_list)
for item in new_items:
if item in items:
items.remove(item)
return ','.join(items)
def add_to_csv_list(csv_list,new_items):
"""Add items to csv_list"""
items = ensure_list(csv_list)
for item in new_items:
if not item in items:
items.append(item)
return ','.join(items)
def add_to_csv_list(csv_list, new_items):
"""Add items to csv_list"""
items = ensure_list(csv_list)
for item in new_items:
if not item in items:
items.append(item)
return ','.join(items)
def remove_from_csv_list(csv_list, new_items):
"""Remove items from csv_list"""
items = ensure_list(csv_list)
for item in new_items:
if item in items:
items.remove(item)
return ','.join(items)
def build_waptupgrade_package(waptconfigfile,
target_directory=None,
wapt_server_user=None,
wapt_server_passwd=None,
key_password=None,
sign_digests=None):
if target_directory is None:
target_directory = tempfile.gettempdir()
if not wapt_server_user:
wapt_server_user = raw_input('WAPT Server user :'******'WAPT Server password :'******'ascii')
wapt = common.Wapt(config_filename=waptconfigfile,
disable_update_server_status=True)
wapt.dbpath = r':memory:'
wapt.use_hostpackages = False
if sign_digests is None:
sign_digests = wapt.sign_digests
if not wapt.personal_certificate_path or not os.path.isfile(
wapt.personal_certificate_path):
raise Exception(
u'No personal certificate provided or not found (%s) for signing waptupgrade package'
% wapt.personal_certificate_path)
waptget = get_file_properties('wapt-get.exe')
entry = PackageEntry(waptfile=makepath(wapt.wapt_base_dir, 'waptupgrade'))
patchs_dir = makepath(entry.sourcespath, 'patchs')
mkdirs(patchs_dir)
filecopyto(makepath(wapt.wapt_base_dir, 'waptdeploy.exe'),
makepath(patchs_dir, 'waptdeploy.exe'))
entry.package = '%s-waptupgrade' % wapt.config.get(
'global', 'default_package_prefix')
rev = entry.version.split('-')[1]
entry.version = '%s-%s' % (waptget['FileVersion'], rev)
entry.inc_build()
entry.save_control_to_wapt()
entry.build_package(target_directory=target_directory)
certs = wapt.personal_certificate()
key = wapt.private_key(private_key_password=key_password)
if not certs[0].is_code_signing:
raise Exception(u'%s is not a code signing certificate' %
wapt.personal_certificate_path)
entry.sign_package(private_key=key,
certificate=certs,
private_key_password=key_password,
mds=ensure_list(sign_digests))
wapt.http_upload_package(entry.localpath,
wapt_server_user=wapt_server_user,
wapt_server_passwd=wapt_server_passwd)
return entry.as_dict()
def get_packages_filenames(packages,
with_depends=True,
waptconfigfile=None,
repo_name='wapt-templates',
remoterepo=None):
"""Returns list of package filenames (latest version) and md5 matching comma separated list of packages names and their dependencies
helps to batch download a list of selected packages using tools like curl or wget
Args:
packages (list): list of package entries as dicts or PackageEntry
with_depends (bool): get recursively the all depends filenames
waptconfigfile (str): path to wapt ini file
repo_name : section name in wapt ini file for repo parameters (repo_url, http_proxy, timeout, verify_cert)
remoterepo (WaptRemoteRepo) : remote repo to query.
Mutually exclusive with waptconfigfile and repo_name
Returns:
list: list of (wapt file basename,md5)
>>> get_packages_filenames(r"c:\users\htouvet\AppData\Local\waptconsole\waptconsole.ini","tis-firefox-esr,tis-flash,tis-wapttest")
[u'tis-firefox-esr_24.4.0-0_all.wapt', u'tis-flash_12.0.0.77-3_all.wapt', u'tis-wapttest.wapt', u'tis-wapttestsub_0.1.0-1_all.wapt', u'tis-7zip_9.2.0-15_all.wapt']
"""
result = []
defaults = {
'repo_url': 'https://store.wapt.fr/wapt',
'http_proxy': '',
'verify_cert': '0',
}
if remoterepo is None:
config = RawConfigParser(defaults=defaults)
config.read(waptconfigfile)
remoterepo = WaptRemoteRepo(name=repo_name, config=config)
remoterepo.update()
for pe in packages:
if not isinstance(pe, PackageEntry):
pe = PackageEntry(**pe)
result.append((
pe.filename,
pe.md5sum,
))
if with_depends and pe.depends:
depends_list = []
for depname in ensure_list(pe.depends):
pe_dep = remoterepo.packages_matching(depname)
if pe_dep:
depends_list.append(pe_dep[-1])
for (fn, md5) in get_packages_filenames(depends_list,
remoterepo=remoterepo):
if not fn in result:
result.append((
fn,
md5,
))
return result
def proxy_host_request(request, action):
"""Proxy a waptconsole action to wapt clients using websockets
Args:
uuid: can be a list or a single uuid
notify_user: 0/1
notify_server: 0/1
Returns:
dict:
'result':
'success' (list)
'errors' (list)
'msg' (str)
'success'(bool)
'request_time' (float)
'error_code'
"""
try:
start_time = time.time()
all_args = {k: v for k, v in request.args.iteritems()}
if request.json:
all_args.update(request.json)
uuids = ensure_list(all_args['uuid'])
del(all_args['uuid'])
timeout = float(request.args.get('timeout', app.conf.get('clients_read_timeout', 5)))
result = dict(success=[], errors=[])
tasks = []
for uuid in uuids:
try:
host_data = Hosts\
.select(Hosts.uuid, Hosts.computer_fqdn,
Hosts.server_uuid,
Hosts.listening_address,
Hosts.listening_port,
Hosts.listening_protocol,
Hosts.listening_timestamp,
)\
.where((Hosts.server_uuid == get_server_uuid()) & (Hosts.uuid == uuid) & (~Hosts.listening_address.is_null()) & (Hosts.listening_protocol == 'websockets'))\
.dicts()\
.first(1)
if host_data and host_data.get('listening_address', None):
msg = u''
logger.info(
'Launching %s with args %s for %s at address %s...' %
(action, all_args, uuid, host_data['listening_address']))
args = dict(all_args)
sid = host_data['listening_address']
computer_fqdn = host_data['computer_fqdn']
def emit_action(sid, uuid, action, action_args, computer_fqdn, timeout=5):
try:
got_result = []
def result_callback(data):
got_result.append(data)
logger.debug(u'Emit %s to %s (%s)' % (action, uuid, computer_fqdn))
socketio.emit(action, action_args, room=sid, callback=result_callback)
# wait for asynchronous answer...
wait_loop = timeout * 20
while not got_result:
wait_loop -= 1
if wait_loop < 0:
raise EWaptTimeoutWaitingForResult(u'Timeout, client did not send result within %s s' % timeout)
socketio.sleep(0.05)
# action succedded
result['success'].append(
dict(
uuid=uuid,
msg=msg,
computer_fqdn=computer_fqdn,
result=got_result[0],
))
except Exception as e:
result['errors'].append(
dict(
uuid=uuid,
msg='%s' % repr(e),
computer_fqdn='',
))
if sid:
tasks.append(socketio.start_background_task(
emit_action, sid=sid, uuid=uuid, action=action, action_args=args, computer_fqdn=computer_fqdn))
else:
result['errors'].append(
dict(
uuid=uuid,
msg=u'Host %s is not registered or not connected wia websockets' % uuid,
computer_fqdn='',
))
except Exception as e:
result['errors'].append(
dict(
uuid=uuid,
msg=u'Host %s error %s' % (uuid, repr(e)),
computer_fqdn='',
))
# wait for all background tasks to terminate or timeout...
# assume the timeout should be longer if more hosts to perform
wait_loop = timeout * len(tasks)
while True:
running = [t for t in tasks if t.is_alive()]
if not running:
break
wait_loop -= 1
if wait_loop < 0:
break
socketio.sleep(0.05)
msg = ['Success : %s, Errors: %s' % (len(result['success']), len(result['errors']))]
if result['errors']:
msg.extend(['%s: %s' % (e['computer_fqdn'], e['msg'])
for e in result['errors']])
return make_response(result,
msg='\n- '.join(msg),
success=len(result['success']) > 0,
request_time=time.time() - start_time)
except Exception as e:
return make_response_from_exception(e)
def get_packages_filenames(packages,with_depends=True,waptconfigfile=None,repo_name='wapt-templates',remoterepo=None,package_request=None):
"""Returns list of package filenames (latest version) and md5 matching comma separated list of packages names and their dependencies
helps to batch download a list of selected packages using tools like curl or wget
Args:
packages (list): list of package entries as dicts or PackageEntry
with_depends (bool): get recursively the all depends filenames
waptconfigfile (str): path to wapt ini file
repo_name : section name in wapt ini file for repo parameters (repo_url, http_proxy, timeout, verify_cert)
remoterepo (WaptRemoteRepo) : remote repo to query.
Mutually exclusive with waptconfigfile and repo_name
Returns:
list: list of (wapt file basename,md5)
>>> get_packages_filenames(r"c:\users\htouvet\AppData\Local\waptconsole\waptconsole.ini","tis-firefox-esr,tis-flash,tis-wapttest")
[u'tis-firefox-esr_24.4.0-0_all.wapt', u'tis-flash_12.0.0.77-3_all.wapt', u'tis-wapttest.wapt', u'tis-wapttestsub_0.1.0-1_all.wapt', u'tis-7zip_9.2.0-15_all.wapt']
"""
result = []
defaults = {
'repo_url':'https://store.wapt.fr/wapt',
'http_proxy':'',
'verify_cert':'0',
}
if remoterepo is None:
config = RawConfigParser(defaults=defaults)
config.read(waptconfigfile)
remoterepo = WaptRemoteRepo(name=repo_name,config=config)
remoterepo.update()
if package_request is not None and isinstance(package_request,dict):
package_request = PackageRequest(**package_request)
for pe in packages:
if not isinstance(pe,PackageEntry):
pe = PackageEntry(**pe)
# propagate capa to parent package
if package_request is None:
request_filter = PackageRequest()
if pe.locale != 'all' and pe.locale:
request_filter.locales = [pe.locale]
if pe.architecture != 'all' and pe.architecture:
request_filter.architectures = [pe.architecture]
if pe.min_os_version:
request_filter.min_os_version = pe.min_os_version
if pe.max_os_version:
request_filter.max_os_version = pe.max_os_version
else:
request_filter = package_request
result.append((pe.filename,pe.md5sum,))
if with_depends and pe.depends:
depends_list = []
for depname in ensure_list(pe.depends):
request_filter.request = depname
pe_dep = remoterepo.packages_matching(request_filter)
if pe_dep:
depends_list.append(pe_dep[-1])
for (fn,md5) in get_packages_filenames(depends_list,remoterepo = remoterepo):
if not fn in result:
result.append((fn,md5,))
return result
def build_waptupgrade_package(wapt,
mainrepo,
sources_directory=None,
target_directory=None,
wapt_server_user=None,
wapt_server_passwd=None,
key_password=None,
sign_digests=None,
priority='critical'):
if target_directory is None:
target_directory = tempfile.gettempdir()
if not wapt_server_user:
wapt_server_user = raw_input('WAPT Server user :'******'WAPT Server password :'******'ascii')
if sign_digests is None:
sign_digests = wapt.sign_digests
if not wapt.personal_certificate_path or not os.path.isfile(
wapt.personal_certificate_path):
raise Exception(
u'No personal certificate provided or not found (%s) for signing waptupgrade package'
% wapt.personal_certificate_path)
waptget = get_file_properties('wapt-get.exe')
if sources_directory is None:
sources_directory = makepath(wapt.wapt_base_dir, 'waptupgrade')
entry = PackageEntry(waptfile=sources_directory)
patchs_dir = makepath(entry.sourcespath, 'patchs')
mkdirs(patchs_dir)
filecopyto(makepath(wapt.wapt_base_dir, 'waptdeploy.exe'),
makepath(patchs_dir, 'waptdeploy.exe'))
mainrepo.update()
entry.package = '%s-waptupgrade' % wapt.config.get(
'global', 'default_package_prefix')
existing = mainrepo.packages_matching(PackageRequest(entry.package))
if existing:
rev = sorted(existing)[-1].version.split('-')[1]
else:
rev = entry.version.split('-')[1]
entry.version = '%s-%s' % (waptget['FileVersion'], rev)
entry.inc_build()
entry.priority = priority
entry.save_control_to_wapt()
entry.build_package(target_directory=target_directory)
certs = wapt.personal_certificate()
key = wapt.private_key(private_key_password=key_password)
if not certs[0].is_code_signing:
raise Exception(u'%s is not a code signing certificate' %
wapt.personal_certificate_path)
entry.sign_package(private_key=key,
certificate=certs,
private_key_password=key_password,
mds=ensure_list(sign_digests))
wapt.http_upload_package(entry.localpath,
wapt_server_user=wapt_server_user,
wapt_server_passwd=wapt_server_passwd,
progress_hook=wapt.progress_hook)
mainrepo.update()
return entry.as_dict()
def duplicate_from_file(package_filename,
new_prefix='test',
target_directory=None,
authorized_certs=None,
set_maturity=None):
r"""Duplicate a downloaded package to match prefix defined in waptconfigfile
renames all dependencies
Returns:
str: source directory
>>> from common import Wapt
>>> wapt = Wapt(config_filename = r'C:\Users\htouvet\AppData\Local\waptconsole\waptconsole.ini')
>>> sources = duplicate_from_external_repo(wapt.config_filename,r'C:\tranquilit\wapt\tests\packages\tis-wapttest.wapt')
>>> res = wapt.build_upload(sources,wapt_server_user='******',wapt_server_passwd='password')
>>> res[0]['package'].depends
u'test-wapttestsub,test-7zip'
"""
def rename_package(oldname, prefix):
sp = oldname.split('-', 1)
if len(sp) == 2:
return "%s-%s" % (prefix, sp[-1])
else:
return oldname
source_package = PackageEntry(waptfile=package_filename)
# authorized_certs is a directoyr instead a list of certificates.
if authorized_certs is not None and authorized_certs != '' and not isinstance(
authorized_certs, list):
bundle = SSLCABundle()
bundle.add_pems(makepath(authorized_certs, '*.crt'), trust_first=True)
bundle.add_pems(makepath(authorized_certs, '*.cer'), trust_first=True)
bundle.add_pems(makepath(authorized_certs, '*.pem'), trust_all=True)
else:
bundle = authorized_certs or None
source_package.unzip_package(target_dir=target_directory, cabundle=bundle)
source_package.invalidate_signature()
package = PackageEntry(waptfile=source_package.sourcespath)
oldname = source_package.package
package.package = rename_package(oldname, new_prefix)
package.inc_build()
if set_maturity is not None:
package.maturity = set_maturity
result = package['sourcespath']
# renames dependencies
if package.depends:
newdepends = []
depends = ensure_list(package.depends)
for dependname in depends:
newname = rename_package(dependname, new_prefix)
newdepends.append(newname)
package.depends = ','.join(newdepends)
# renames conflicts
if package.conflicts:
newconflicts = []
conflicts = ensure_list(package.conflicts)
for dependname in conflicts:
newname = rename_package(dependname, new_prefix)
newconflicts.append(newname)
package.conflicts = ','.join(newconflicts)
package.save_control_to_wapt()
return result
def get_packages_filenames(packages,
with_depends=True,
waptconfigfile=None,
repo_name='wapt-templates',
remoterepo=None,
package_request=None,
privaterepo=None,
local_prefix=None):
"""Returns list of package filenames (latest version) and md5 matching comma separated list of packages names and their dependencies
helps to batch download a list of selected packages using tools like curl or wget
Args:
packages (list): list of package entries as dicts or PackageEntry
with_depends (bool): get recursively the all depends filenames
waptconfigfile (str): path to wapt ini file
repo_name : section name in wapt ini file for repo parameters (repo_url, http_proxy, timeout, verify_cert)
remoterepo (WaptRemoteRepo) : remote repo to query.
Mutually exclusive with waptconfigfile and repo_name
privaterepo (WaptRemoteRepo) : local private repo where to check if dependencies already exists
Returns:
list: list of (wapt file basename,md5)
>>> get_packages_filenames(r"c:\users\htouvet\AppData\Local\waptconsole\waptconsole.ini","tis-firefox-esr,tis-flash,tis-wapttest")
[u'tis-firefox-esr_24.4.0-0_all.wapt', u'tis-flash_12.0.0.77-3_all.wapt', u'tis-wapttest.wapt', u'tis-wapttestsub_0.1.0-1_all.wapt', u'tis-7zip_9.2.0-15_all.wapt']
"""
result = []
defaults = {
'repo_url': 'https://store.wapt.fr/wapt',
'http_proxy': '',
'verify_cert': '0',
}
try:
import waptconsole
progress_hook = waptconsole.UpdateProgress
private_key_password_callback = waptconsole.GetPrivateKeyPassword
except ImportError as e:
def print_progress(show=False, n=0, max=100, msg=''):
if show:
print('%s %s/%s\r' % (msg, n, max), end='')
else:
if not msg:
msg = 'Done'
print("%s%s" % (msg, ' ' * (80 - len(msg))))
progress_hook = print_progress
private_key_password_callback = None
if remoterepo is None:
config = RawConfigParser(defaults=defaults)
config.read(waptconfigfile)
remoterepo = WaptRemoteRepo(name=repo_name, config=config)
remoterepo.private_key_password_callback = private_key_password_callback
remoterepo.update()
if privaterepo is None and waptconfigfile:
config = RawConfigParser(defaults=defaults)
config.read(waptconfigfile)
privaterepo = WaptRemoteRepo(name='wapt', config=config)
privaterepo.private_key_password_callback = private_key_password_callback
privaterepo.update()
if package_request is not None and isinstance(package_request, dict):
package_request = PackageRequest(**package_request)
for pe in packages:
if not isinstance(pe, PackageEntry):
pe = PackageEntry(**pe)
# propagate capa to parent package
if package_request is None:
request_filter = PackageRequest()
if pe.locale != 'all' and pe.locale:
request_filter.locales = [pe.locale]
if pe.architecture != 'all' and pe.architecture:
request_filter.architectures = [pe.architecture]
if pe.min_os_version:
request_filter.min_os_version = pe.min_os_version
if pe.max_os_version:
request_filter.max_os_version = pe.max_os_version
else:
request_filter = package_request
if pe.filename:
result.append((
pe.filename,
pe.md5sum,
))
if with_depends and pe.depends:
depends_list = []
for depname in ensure_list(pe.depends):
request_filter.request = depname
pe_dep = remoterepo.packages_matching(request_filter)
if privaterepo and local_prefix:
request_filter.request = change_prefix(
depname, local_prefix)
pe_local = privaterepo.packages_matching(
request_filter)
else:
pe_local = None
if pe_dep and not pe_local:
depends_list.append(pe_dep[-1])
for (fn, md5) in get_packages_filenames(depends_list,
remoterepo=remoterepo):
if not (fn, md5) in result:
result.append((
fn,
md5,
))
return result
def unzip(zipfn, target=None, filenames=None):
"""Unzip the files from zipfile with patterns in filenames to target directory
Args:
zipfn (str) : path to zipfile. (can be relative to temporary unzip location of package)
target (str) : target location. Defaults to dirname(zipfile) + basename(zipfile)
filenames (str or list of str): list of filenames / glob patterns (path sep is normally a slash)
Returns:
list : list of extracted files
>>> unzip(r'C:\tranquilit\wapt\tests\packages\tis-7zip_9.2.0-15_all.wapt')
[u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\7z920-x64.msi',
u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\7z920.msi',
u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\setup.py',
u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\WAPT/control',
u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\WAPT/wapt.psproj',
u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\WAPT/manifest.sha256',
u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\WAPT/signature']
>>> unzip(r'C:\tranquilit\wapt\tests\packages\tis-7zip_9.2.0-15_all.wapt',filenames=['*.msi','*.py'])
[u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\7z920-x64.msi',
u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\7z920.msi',
u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\setup.py']
>>> unzip(r'C:\tranquilit\wapt\tests\packages\tis-7zip_9.2.0-15_all.wapt',filenames='WAPT/*')
[u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\WAPT/control',
u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\WAPT/wapt.psproj',
u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\WAPT/manifest.sha256',
u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\WAPT/signature']
>>> unzip(r'C:\tranquilit\wapt\tests\packages\tis-7zip_9.2.0-15_all.wapt',filenames='WAPT/control')
[u'C:\\tranquilit\\wapt\\tests\\packages\\tis-7zip_9.2.0-15_all\\WAPT\\control']
.. versionadded:: 1.3.11
"""
zipf = ZipFile(zipfn, allowZip64=True)
if target is None:
target = makepath(os.path.dirname(os.path.abspath(zipfn)),
os.path.splitext(os.path.basename(zipfn))[0])
if filenames is not None:
filenames = [
pattern.replace('\\', '/') for pattern in ensure_list(filenames)
]
def match(fn, filenames):
# return True if fn matches one of the pattern in filenames
if filenames is None:
return True
else:
for pattern in filenames:
if glob.fnmatch.fnmatch(fn, pattern):
return True
return False
if filenames is not None:
files = [fn for fn in zipf.namelist() if match(fn, filenames)]
zipf.extractall(target, members=files)
else:
files = zipf.namelist()
zipf.extractall(target)
return [makepath(target, fn.replace('/', os.sep)) for fn in files]
def duplicate_from_file(package_filename,new_prefix='test',target_directory=None,authorized_certs=None,set_maturity=None):
r"""Duplicate a downloaded package to match prefix defined in waptconfigfile
renames all dependencies
Returns:
str: source directory
>>> from common import Wapt
>>> wapt = Wapt(config_filename = r'C:\Users\htouvet\AppData\Local\waptconsole\waptconsole.ini')
>>> sources = duplicate_from_external_repo(wapt.config_filename,r'C:\tranquilit\wapt\tests\packages\tis-wapttest.wapt')
>>> res = wapt.build_upload(sources,wapt_server_user='******',wapt_server_passwd='password')
>>> res[0]['package'].depends
u'test-wapttestsub,test-7zip'
"""
def rename_package(oldname,prefix):
sp = oldname.split('-',1)
if len(sp) == 2:
return "%s-%s" % (prefix,sp[-1])
else:
return oldname
source_package = PackageEntry(waptfile = package_filename)
# authorized_certs is a directoyr instead a list of certificates.
if authorized_certs is not None and authorized_certs != '' and not isinstance(authorized_certs,list):
bundle = SSLCABundle()
bundle.add_pems(makepath(authorized_certs,'*.crt'))
bundle.add_pems(makepath(authorized_certs,'*.cer'))
bundle.add_pems(makepath(authorized_certs,'*.pem'))
else:
bundle = authorized_certs or None
source_package.unzip_package(target_dir=target_directory,cabundle=bundle)
source_package.invalidate_signature()
package = PackageEntry(waptfile = source_package.sourcespath)
oldname = source_package.package
package.package = rename_package(oldname,new_prefix)
package.inc_build()
if set_maturity is not None:
package.maturity = set_maturity
result = package['sourcespath']
# renames dependencies
if package.depends:
newdepends = []
depends = ensure_list(package.depends)
for dependname in depends:
newname = rename_package(dependname,new_prefix)
newdepends.append(newname)
package.depends = ','.join(newdepends)
# renames conflicts
if package.conflicts:
newconflicts = []
conflicts = ensure_list(package.conflicts)
for dependname in conflicts:
newname = rename_package(dependname,new_prefix)
newconflicts.append(newname)
package.conflicts = ','.join(newconflicts)
package.save_control_to_wapt()
return result
def main():
parser=OptionParser(usage=__doc__,prog = 'wapt-signpackage')
parser.add_option("-c","--certificate", dest="public_key", default='', help="Path to the PEM RSA certificate to embed identitiy in control. (default: %default)")
parser.add_option("-k","--private-key", dest="private_key", default='', help="Path to the PEM RSA private key to sign packages. (default: %default)")
#parser.add_option("-w","--private-key-passwd", dest="private_key_passwd", default='', help="Path to the password of the private key. (default: %default)")
parser.add_option("-l","--loglevel", dest="loglevel", default=None, type='choice', choices=['debug','warning','info','error','critical'], metavar='LOGLEVEL',help="Loglevel (default: warning)")
parser.add_option("-m","--message-digest", dest="md", default='sha256', help="Message digest type for signatures. (default: %default)")
parser.add_option("-s","--scan-packages", dest="doscan", default=False, action='store_true', help="Rescan packages and update local Packages index after signing. (default: %default)")
parser.add_option("-r","--remove-setup", dest="removesetup", default=False, action='store_true', help="Remove setup.py. (default: %default)")
parser.add_option("-i","--inc-release", dest="increlease", default=False, action='store_true', help="Increase release number when building package (default: %default)")
parser.add_option("--maturity", dest="set_maturity", default=None, help="Set/change package maturity when signing package. (default: None)")
parser.add_option( "--keep-signature-date", dest="keep_signature_date",default=False, action='store_true', help="Keep the current package signature date, and file changetime (default: %default)")
parser.add_option( "--if-needed", dest="if_needed", default=False, action='store_true',help="Re-sign package only if needed (default: warning)")
(options,args) = parser.parse_args()
loglevel = options.loglevel
if len(logger.handlers) < 1:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(logging.Formatter(
u'%(asctime)s %(levelname)s %(message)s'))
logger.addHandler(hdlr)
if loglevel:
setloglevel(logger,loglevel)
else:
setloglevel(logger,'warning')
if len(args) < 1:
print(parser.usage)
sys.exit(1)
if not options.public_key and not options.private_key:
print('ERROR: No certificate found or specified')
sys.exit(1)
if options.private_key and os.path.isfile(options.private_key):
key = SSLPrivateKey(options.private_key)
else:
cert = SSLCertificate(options.public_key or options.private_key)
key = cert.matching_key_in_dirs()
if not key:
print('ERROR: No private key found or specified')
sys.exit(1)
args = ensure_list(args)
ca_bundle = SSLCABundle()
signers_bundle = SSLCABundle()
signers_bundle.add_certificates_from_pem(pem_filename=options.public_key)
waptpackages = []
for arg in args:
waptpackages.extend(glob.glob(arg))
errors = []
package_dirs = []
for waptpackage in waptpackages:
package_dir = os.path.abspath(os.path.dirname(waptpackage))
if not package_dir in package_dirs:
package_dirs.append(package_dir)
print('Processing %s'%waptpackage)
try:
sign_needed=False
pe = PackageEntry(waptfile = waptpackage)
if options.removesetup:
if pe.has_file('setup.py'):
with pe.as_zipfile(mode='a') as waptfile:
waptfile.remove('setup.py')
sign_needed=True
if not sign_needed and options.if_needed:
try:
pe.check_control_signature(trusted_bundle=signers_bundle,signers_bundle=signers_bundle)
for md in ensure_list(options.md):
if not pe.has_file(pe.get_signature_filename(md)):
raise Exception('Missing signature for md %s' % md)
logger.info('Skipping %s, already signed properly' % pe.asrequirement())
sign_needed = False
except Exception as e:
logger.info('Sign is needed for %s because %s' % (pe.asrequirement(),e))
sign_needed = True
if options.increlease:
pe.inc_build()
sign_needed = True
if options.set_maturity is not None and pe.maturity != options.set_maturity:
pe.maturity = options.set_maturity
sign_needed = True
if not options.if_needed or sign_needed:
pe.sign_package(private_key=key,certificate = signers_bundle.certificates(),mds = ensure_list(options.md),keep_signature_date=options.keep_signature_date)
newfn = pe.make_package_filename()
if newfn != pe.filename:
newfn_path = os.path.join(package_dir,newfn)
if not os.path.isfile(newfn_path):
print(u"Renaming file from %s to %s to match new package's properties" % (pe.filename,newfn))
shutil.move(os.path.join(package_dir,pe.filename),newfn_path)
else:
print('WARNING: unable to rename file from %s to %s because target already exists' % (pe.filename,newfn))
print('Done')
except Exception as e:
print(u'Error: %s'%ensure_unicode(e.message))
errors.append([waptpackage,repr(e)])
if options.doscan:
for package_dir in package_dirs:
if os.path.isfile(os.path.join(package_dir,'Packages')):
print(u'Launching the update of Packages index in %s ...'% ensure_unicode(package_dir))
repo = WaptLocalRepo(package_dir)
repo.update_packages_index()
print('Done')
else:
print("Don't forget to rescan your repository with wapt-scanpackages %s" % os.path.dirname(waptpackages[0]))
if errors:
print('Package not processed properly: ')
for fn,error in errors:
print(u'%s : %s' % (fn,error))
sys.exit(1)
else:
sys.exit(0)
def edit_hosts_depends(
waptconfigfile,
hosts_list,
append_depends=[],
remove_depends=[],
append_conflicts=[],
remove_conflicts=[],
sign_certs=None,
sign_key=None,
key_password=None,
wapt_server_user=None,
wapt_server_passwd=None,
cabundle=None,
):
"""Add or remove packages from host packages
Args:
Returns:
dict: { updated: of uuid of machines actually updated
unchanged : list of uuid skipped because of no change needed
discarded : list of uuid discarded due to errors}
>>> edit_hosts_depends('c:/wapt/wapt-get.ini','htlaptop.tranquilit.local','toto','tis-7zip','admin','password')
"""
if sign_certs is None:
sign_bundle_fn = inifile_readstring(waptconfigfile, u'global',
u'personal_certificate_path')
sign_bundle = SSLCABundle(sign_bundle_fn)
sign_certs = sign_bundle.certificates()
# we assume a unique signer.
if cabundle is None:
cabundle = sign_bundle
if not sign_certs:
raise Exception(u'No personal signer certificate found in %s' %
sign_bundle_fn)
try:
import waptconsole
progress_hook = waptconsole.UpdateProgress
private_key_password_callback = waptconsole.GetPrivateKeyPassword
except ImportError as e:
def print_progress(show=False, n=0, max=100, msg=''):
if show:
print('%s %s/%s\r' % (msg, n, max), end='')
else:
if not msg:
msg = 'Done'
print("%s%s" % (msg, ' ' * (80 - len(msg))))
progress_hook = print_progress
private_key_password_callback = None
if sign_key is None:
sign_key = sign_certs[0].matching_key_in_dirs(
private_key_password=key_password,
password_callback=private_key_password_callback)
progress_hook(True, 0, len(hosts_list),
'Loading %s hosts packages' % len(hosts_list))
host_repo = WaptHostRepo(name='wapt-host',
host_id=[h['uuid'] for h in hosts_list],
cabundle=cabundle)
host_repo.private_key_password_callback = private_key_password_callback
host_repo.load_config_from_file(waptconfigfile)
total_hosts = len(host_repo.packages())
discarded_uuids = [p.package for p in host_repo.discarded]
append_depends = ensure_list(append_depends)
remove_depends = ensure_list(remove_depends)
append_conflicts = ensure_list(append_conflicts)
remove_conflicts = ensure_list(remove_conflicts)
packages = []
discarded = []
unchanged = []
progress_hook(True, 0, len(hosts_list),
'Editing %s hosts' % len(hosts_list))
i = 0
try:
for hostrec in hosts_list:
host_id = hostrec['uuid']
i += 1
# don't change discarded packages.
if host_id in discarded_uuids:
discarded.append(host_id)
else:
host = host_repo.get(host_id)
if host is None:
host = PackageEntry(package=host_id, section='host')
if progress_hook(True, i, len(hosts_list),
'Editing %s' % host.package):
break
logger.debug(u'Edit host %s : +%s -%s' %
(host.package, append_depends, remove_depends))
depends = host.depends
conflicts = host.conflicts
conflicts = add_to_csv_list(conflicts, append_conflicts)
conflicts = remove_from_csv_list(conflicts, remove_conflicts)
depends = remove_from_csv_list(depends, ensure_list(conflicts))
depends = add_to_csv_list(depends, append_depends)
depends = remove_from_csv_list(depends, remove_depends)
conflicts = remove_from_csv_list(conflicts,
ensure_list(depends))
if depends != host.depends or conflicts != host.conflicts:
host.depends = depends
host.conflicts = conflicts
host.inc_build()
host_file = host.build_management_package()
host.sign_package(sign_certs, sign_key)
packages.append(host)
else:
unchanged.append(host.package)
# upload all in one step...
progress_hook(True, 3, 3, 'Upload %s host packages' % len(packages))
server = WaptServer().load_config_from_file(waptconfigfile)
server.private_key_password_callback = private_key_password_callback
server.upload_packages(packages,
auth=(wapt_server_user, wapt_server_passwd),
progress_hook=progress_hook)
return dict(updated=[p.package for p in packages],
discarded=discarded,
unchanged=unchanged)
finally:
logger.debug('Cleanup')
try:
i = 0
for s in packages:
i += 1
progress_hook(True, i, len(packages), 'Cleanup')
if os.path.isfile(s.localpath):
os.remove(s.localpath)
progress_hook(False)
except WindowsError as e:
logger.critical('Unable to remove temporary directory %s: %s' %
(s, repr(e)))
progress_hook(False)
def load(self):
"""Load waptservice parameters from global wapt-get.ini file"""
config = ConfigParser.RawConfigParser()
if os.path.exists(self.config_filename):
config.read(self.config_filename)
self.config_filedate = os.stat(self.config_filename).st_mtime
else:
raise Exception(_("FATAL. Couldn't open config file : {}").format(self.config_filename))
# lecture configuration
if config.has_section('global'):
if config.has_option('global', 'waptservice_user'):
self.waptservice_user = config.get('global', 'waptservice_user')
else:
self.waptservice_user = None
if config.has_option('global','waptservice_password'):
self.waptservice_password = config.get('global', 'waptservice_password')
else:
logger.info(u"No password set for local waptservice, using local computer security")
self.waptservice_password=None # = password
if config.has_option('global','waptservice_port'):
port = config.get('global','waptservice_port')
if port:
self.waptservice_port = int(port)
else:
self.waptservice_port = None
else:
self.waptservice_port=8088
if config.has_option('global','language'):
self.language = config.get('global','language')
if config.has_option('global','secret_key'):
self.secret_key = config.get('global','secret_key')
if config.has_option('global','waptservice_poll_timeout'):
self.waptservice_poll_timeout = int(config.get('global','waptservice_poll_timeout'))
else:
self.waptservice_poll_timeout = 10
if config.has_option('global','waptupgrade_task_period'):
self.waptupgrade_task_period = int(config.get('global','waptupgrade_task_period'))
else:
self.waptupgrade_task_period = None
if config.has_option('global','waptupdate_task_period'):
self.waptupdate_task_period = int(config.get('global','waptupdate_task_period'))
else:
self.waptupdate_task_period = 120
if config.has_option('global','waptaudit_task_period'):
self.waptaudit_task_period = config.get('global','waptaudit_task_period')
if config.has_option('global','dbpath'):
self.dbpath = config.get('global','dbpath')
else:
self.dbpath = os.path.join(wapt_root_dir,'db','waptdb.sqlite')
if self.dbpath != ':memory:':
self.dbdir = os.path.dirname(self.dbpath)
if not os.path.isdir(self.dbdir):
os.makedirs(self.dbdir)
else:
self.dbdir = None
if config.has_option('global','loglevel'):
self.loglevel = config.get('global','loglevel')
if config.has_option('global','log_to_windows_events'):
self.log_to_windows_events = config.getboolean('global','log_to_windows_events')
if config.has_option('global','allow_user_service_restart'):
self.allow_user_service_restart = config.getboolean('global','allow_user_service_restart')
if config.has_option('global','notify_user'):
self.notify_user = config.getboolean('global','notify_user')
if config.has_option('global','wapt_server'):
self.waptserver = common.WaptServer().load_config(config)
if self.waptserver.server_url:
waptserver_url = urlparse.urlparse(self.waptserver.server_url)
self.websockets_host = waptserver_url.hostname
self.websockets_proto = waptserver_url.scheme
if waptserver_url.port is None:
if waptserver_url.scheme == 'https':
self.websockets_port = 443
else:
self.websockets_port = 80
else:
self.websockets_port = waptserver_url.port
if waptserver_url.path in ('','/'):
self.websockets_root = 'socket.io'
else:
self.websockets_root = '%s/socket.io' % waptserver_url.path[1:]
else:
self.waptserver = None
self.websockets_host = None
self.websockets_proto = None
self.websockets_port = None
self.websockets_verify_cert = False
else:
self.waptserver = None
self.websockets_host = None
self.websockets_proto = None
self.websockets_port = None
self.websockets_verify_cert = False
if config.has_option('global','websockets_verify_cert'):
try:
self.websockets_verify_cert = config.getboolean('global','websockets_verify_cert')
except:
self.websockets_verify_cert = config.get('global','websockets_verify_cert')
if not os.path.isfile(self.websockets_verify_cert):
logger.warning(u'websockets_verify_cert certificate %s declared in configuration file can not be found. Waptserver websockets communication will fail' % self.websockets_verify_cert)
else:
self.websockets_verify_cert = False
if config.has_option('global','websockets_ping'):
self.websockets_ping = config.getint('global','websockets_ping')
if config.has_option('global','websockets_retry_delay'):
self.websockets_retry_delay = config.getint('global','websockets_retry_delay')
if config.has_option('global','websockets_check_config_interval'):
self.websockets_check_config_interval = config.getint('global','websockets_check_config_interval')
if config.has_option('global','websockets_hurry_interval'):
self.websockets_hurry_interval = config.getint('global','websockets_hurry_interval')
if config.has_option('global','signature_clockskew'):
self.signature_clockskew = config.getint('global','signature_clockskew')
# settings for waptexit / shutdown policy
# recommended settings :
# hiberboot_enabled = 0
# max_gpo_script_wait = 180
# pre_shutdown_timeout = 180
for param in ('hiberboot_enabled','max_gpo_script_wait','pre_shutdown_timeout'):
if config.has_option('global',param):
setattr(self,param,config.getint('global',param))
else:
setattr(self,param,None)
# waptwua settings (Enterprise)
if config.has_option('global','waptwua_enabled'):
setattr(self,'waptwua_enabled',config.getboolean('global','waptwua_enabled'))
else:
setattr(self,'waptwua_enabled',None)
for param in ['waptwua_allowed_updates','waptwua_forbidden_updates','waptwua_allowed_severities','waptwua_allowed_classifications']:
if config.has_option('global',param):
setattr(self,param,ensure_list(config.get('global',param)))
else:
setattr(self,param,None)
for param in ['waptwua_download_scheduling','waptwua_install_scheduling']:
if config.has_option('global',param):
setattr(self,param,config.get('global',param))
else:
setattr(self,param,None)
else:
raise Exception (_("FATAL, configuration file {} has no section [global]. Please check Waptserver documentation").format(self.config_filename))
def main():
parser=OptionParser(usage=__doc__,prog = 'wapt-signpackage')
parser.add_option("-c","--certificate", dest="public_key", default='', help="Path to the PEM RSA certificate to embed identitiy in control. (default: %default)")
parser.add_option("-k","--private-key", dest="private_key", default='', help="Path to the PEM RSA private key to sign packages. (default: %default)")
#parser.add_option("-w","--private-key-passwd", dest="private_key_passwd", default='', help="Path to the password of the private key. (default: %default)")
parser.add_option("-l","--loglevel", dest="loglevel", default=None, type='choice', choices=['debug','warning','info','error','critical'], metavar='LOGLEVEL',help="Loglevel (default: warning)")
parser.add_option("-m","--message-digest", dest="md", default='sha256', help="Message digest type for signatures. (default: %default)")
parser.add_option("-s","--scan-packages", dest="doscan", default=False, action='store_true', help="Rescan packages and update local Packages index after signing. (default: %default)")
parser.add_option("-r","--remove-setup", dest="removesetup", default=False, action='store_true', help="Remove setup.py. (default: %default)")
parser.add_option("-i","--inc-release", dest="increlease", default=False, action='store_true', help="Increase release number when building package (default: %default)")
parser.add_option("--maturity", dest="set_maturity", default=None, help="Set/change package maturity when signing package. (default: None)")
parser.add_option( "--keep-signature-date", dest="keep_signature_date",default=False, action='store_true', help="Keep the current package signature date, and file changetime (default: %default)")
parser.add_option( "--if-needed", dest="if_needed", default=False, action='store_true',help="Re-sign package only if needed (default: warning)")
(options,args) = parser.parse_args()
loglevel = options.loglevel
if len(logger.handlers) < 1:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(logging.Formatter(
u'%(asctime)s %(levelname)s %(message)s'))
logger.addHandler(hdlr)
if loglevel:
setloglevel(logger,loglevel)
else:
setloglevel(logger,'warning')
if len(args) < 1:
print(parser.usage)
sys.exit(1)
if not options.public_key and not options.private_key:
print('ERROR: No certificate found or specified')
sys.exit(1)
if options.private_key and os.path.isfile(options.private_key):
key = SSLPrivateKey(options.private_key)
else:
cert = SSLCertificate(options.public_key or options.private_key)
key = cert.matching_key_in_dirs()
if not key:
print('ERROR: No private key found or specified')
sys.exit(1)
args = ensure_list(args)
ca_bundle = SSLCABundle()
signers_bundle = SSLCABundle()
signers_bundle.add_certificates_from_pem(pem_filename=options.public_key)
waptpackages = []
for arg in args:
waptpackages.extend(glob.glob(arg))
errors = []
package_dirs = []
for waptpackage in waptpackages:
package_dir = os.path.abspath(os.path.dirname(waptpackage))
if not package_dir in package_dirs:
package_dirs.append(package_dir)
print('Processing %s'%waptpackage)
try:
sign_needed=False
pe = PackageEntry(waptfile = waptpackage)
if options.removesetup:
if pe.has_file('setup.py'):
with pe.as_zipfile(mode='a') as waptfile:
waptfile.remove('setup.py')
sign_needed=True
if not sign_needed and options.if_needed:
try:
pe.check_control_signature(trusted_bundle=signers_bundle,signers_bundle=signers_bundle)
for md in ensure_list(options.md):
if not pe.has_file(pe.get_signature_filename(md)):
raise Exception('Missing signature for md %s' % md)
logger.info('Skipping %s, already signed properly' % pe.asrequirement())
sign_needed = False
except Exception as e:
logger.info('Sign is needed for %s because %s' % (pe.asrequirement(),e))
sign_needed = True
if options.increlease:
pe.inc_build()
sign_needed = True
if options.set_maturity is not None and pe.maturity != options.set_maturity:
pe.maturity = options.set_maturity
sign_needed = True
if not options.if_needed or sign_needed:
pe.sign_package(private_key=key,certificate = signers_bundle.certificates(),mds = ensure_list(options.md),keep_signature_date=options.keep_signature_date)
newfn = pe.make_package_filename()
if newfn != pe.filename:
newfn_path = os.path.join(package_dir,newfn)
if not os.path.isfile(newfn_path):
print(u"Renaming file from %s to %s to match new package's properties" % (pe.filename,newfn))
os.rename(os.path.join(package_dir,pe.filename),newfn_path)
else:
print('WARNING: unable to rename file from %s to %s because target already exists' % (pe.filename,newfn))
print('Done')
except Exception as e:
print(u'Error: %s'%ensure_unicode(e.message))
errors.append([waptpackage,repr(e)])
if options.doscan:
for package_dir in package_dirs:
if os.path.isfile(os.path.join(package_dir,'Packages')):
print(u'Launching the update of Packages index in %s ...'% ensure_unicode(package_dir))
repo = WaptLocalRepo(package_dir)
repo.update_packages_index()
print('Done')
else:
print("Don't forget to rescan your repository with wapt-scanpackages %s" % os.path.dirname(waptpackages[0]))
if errors:
print('Package not processed properly: ')
for fn,error in errors:
print(u'%s : %s' % (fn,error))
sys.exit(1)
else:
sys.exit(0)
def add_ads_groups(waptconfigfile,
hosts_list,
wapt_server_user,
wapt_server_passwd,
key_password=None):
# initialise wapt api with local config file
wapt = Wapt(config_filename=waptconfigfile)
wapt.dbpath = ':memory:'
# get current packages status from repositories
wapt.update(register=False, filter_on_host_cap=False)
hosts_list = ensure_list(hosts_list)
# get the collection of hosts from waptserver inventory
all_hosts = wapt.waptserver.get(
'api/v1/hosts?columns=uuid,computer_fqdn,depends',
auth=(wapt_server_user, wapt_server_passwd))['result']
if hosts_list:
hosts = [h for h in all_hosts if h['computer_fqdn'] in hosts_list]
else:
hosts = hosts_list
result = []
for h in hosts:
try:
hostname = h['computer_fqdn']
print('Computer %s... \r' % hostname, end='')
groups = get_computer_groups(h['computer_name'])
wapt_groups = h['depends']
additional = [
group for group in groups
if not group in wapt_groups and wapt.is_available(group)
]
if additional:
# now update the host package : download and append missing packages
tmpdir = mkdtemp()
try:
package = wapt.edit_host(hostname, target_directory=tmpdir)
control = package['package']
depends = ensure_list(control.depends)
control.depends = ','.join(depends + additional)
control.save_control_to_wapt(package.sourcespath)
buid_res = wapt.build_upload(
package.sourcespath,
private_key_passwd=key_password,
wapt_server_user=wapt_server_user,
wapt_server_passwd=wapt_server_passwd,
inc_package_release=True)[0]
print(" done, new packages: %s" % (','.join(additional)))
if os.path.isfile(buid_res):
os.remove(buid_res)
result.append(hostname)
finally:
# cleanup of temporary
if os.path.isdir(tmpdir):
rmtree(tmpdir)
except Exception as e:
print(" error %s" % e)
raise
return result
def add_ads_groups(waptconfigfile,
hostdicts_list,
sign_certs=None,
sign_key=None,
key_password=None,
wapt_server_user=None,wapt_server_passwd=None,
cabundle = None):
if sign_certs is None:
sign_bundle_fn = inifile_readstring(waptconfigfile,u'global',u'personal_certificate_path')
sign_bundle = SSLCABundle(sign_bundle_fn)
sign_certs = sign_bundle.certificates()
# we assume a unique signer.
if cabundle is None:
cabundle = sign_bundle
if not sign_certs:
raise Exception(u'No personal signer certificate found in %s' % sign_bundle_fn)
if sign_key is None:
sign_key = sign_certs[0].matching_key_in_dirs(private_key_password=key_password)
main_repo = WaptRemoteRepo(name='wapt',cabundle = cabundle)
main_repo.load_config_from_file(waptconfigfile)
host_repo = WaptHostRepo(name='wapt-host',host_id=[h['uuid'] for h in hostdicts_list],cabundle = cabundle)
host_repo.load_config_from_file(waptconfigfile)
total_hosts = len(host_repo.packages())
discarded_uuids = [p.package for p in host_repo.discarded]
try:
import waptconsole
progress_hook = waptconsole.UpdateProgress
except ImportError as e:
def print_progress(show=False,n=0,max=100,msg=''):
if show:
print('%s %s/%s\r' % (msg,n,max),end='')
else:
if not msg:
msg='Done'
print("%s%s"%(msg,' '*(80-len(msg))))
progress_hook = print_progress
packages = []
discarded = []
unchanged = []
try:
progress_hook(True,0,len(hostdicts_list),'Editing %s hosts' % len(hostdicts_list))
i = 0
for h in hostdicts_list:
try:
host_package = None
host_id = h['uuid']
hostname = h['computer_name']
groups = get_computer_groups(hostname)
host_package = host_repo.get(host_id,PackageEntry(package=host_id,section='host'))
if progress_hook(True,i,len(hostdicts_list),'Checking %s' % host_package.package):
break
wapt_groups = ensure_list(host_package['depends'])
additional = [group for group in groups if not group in wapt_groups and (main_repo.get(group,None) is not None)]
if additional:
logger.info(u'Adding %s to %s' % (','.join(additional),host_package.package))
if progress_hook(True,i,len(hostdicts_list),'Editing %s' % host_package.package):
break
wapt_groups.extend(additional)
host_package.depends = ','.join(wapt_groups)
host_package.build_management_package()
host_package.inc_build()
host_file = host_package.build_management_package()
host_package.sign_package(sign_certs,sign_key)
packages.append(host_package)
else:
unchanged.append(host_package.package)
except Exception as e:
if host_package:
discarded.append(host_package.package)
logger.critical(u'Discarding because %s' % ensure_unicode(e))
# upload all in one step...
progress_hook(True,3,3,'Upload %s host packages' % len(packages))
server = WaptServer().load_config_from_file(waptconfigfile)
server.upload_packages(packages,auth=(wapt_server_user,wapt_server_passwd),progress_hook=progress_hook)
return dict(updated = packages,
discarded = discarded,
unchanged = unchanged)
finally:
logger.debug('Cleanup')
try:
i = 0
for s in packages:
i+=1
progress_hook(True,i,len(packages),'Cleanup')
if os.path.isfile(s.localpath):
os.remove(s.localpath)
progress_hook(False)
except WindowsError as e:
logger.critical('Unable to remove temporary directory %s: %s'% (s,repr(e)))
progress_hook(False)
def edit_hosts_depends(waptconfigfile,hosts_list,
append_depends=[],
remove_depends=[],
append_conflicts=[],
remove_conflicts=[],
sign_certs=None,
sign_key=None,
key_password=None,
wapt_server_user=None,wapt_server_passwd=None,
cabundle = None,
):
"""Add or remove packages from host packages
Args:
Returns:
dict: { updated: of uuid of machines actually updated
unchanged : list of uuid skipped because of no change needed
discarded : list of uuid discarded due to errors}
>>> edit_hosts_depends('c:/wapt/wapt-get.ini','htlaptop.tranquilit.local','toto','tis-7zip','admin','password')
"""
if sign_certs is None:
sign_bundle_fn = inifile_readstring(waptconfigfile,u'global',u'personal_certificate_path')
sign_bundle = SSLCABundle(sign_bundle_fn)
sign_certs = sign_bundle.certificates()
# we assume a unique signer.
if cabundle is None:
cabundle = sign_bundle
if not sign_certs:
raise Exception(u'No personal signer certificate found in %s' % sign_bundle_fn)
if sign_key is None:
sign_key = sign_certs[0].matching_key_in_dirs(private_key_password=key_password)
try:
import waptconsole
progress_hook = waptconsole.UpdateProgress
except ImportError as e:
def print_progress(show=False,n=0,max=100,msg=''):
if show:
print('%s %s/%s\r' % (msg,n,max),end='')
else:
if not msg:
msg='Done'
print("%s%s"%(msg,' '*(80-len(msg))))
progress_hook = print_progress
hosts_list = ensure_list(hosts_list)
progress_hook(True,0,len(hosts_list),'Loading %s hosts packages' % len(hosts_list))
host_repo = WaptHostRepo(name='wapt-host',host_id=hosts_list,cabundle = cabundle)
host_repo.load_config_from_file(waptconfigfile)
total_hosts = len(host_repo.packages())
discarded_uuids = [p.package for p in host_repo.discarded]
hosts_list = ensure_list(hosts_list)
append_depends = ensure_list(append_depends)
remove_depends = ensure_list(remove_depends)
append_conflicts = ensure_list(append_conflicts)
remove_conflicts = ensure_list(remove_conflicts)
packages = []
discarded = []
unchanged = []
progress_hook(True,0,len(hosts_list),'Editing %s hosts' % len(hosts_list))
i = 0
try:
for host_id in hosts_list:
i+=1
# don't change discarded packages.
if host_id in discarded_uuids:
discarded.append(host_id)
else:
host = host_repo.get(host_id)
if host is None:
host = PackageEntry(package=host_id,section='host')
if progress_hook(True,i,len(hosts_list),'Editing %s' % host.package):
break
logger.debug(u'Edit host %s : +%s -%s'%(
host.package,
append_depends,
remove_depends))
depends = host.depends
conflicts = host.conflicts
conflicts = add_to_csv_list(conflicts,append_conflicts)
conflicts = remove_from_csv_list(conflicts,remove_conflicts)
depends = remove_from_csv_list(depends,ensure_list(conflicts))
depends = add_to_csv_list(depends,append_depends)
depends = remove_from_csv_list(depends,remove_depends)
conflicts = remove_from_csv_list(conflicts,ensure_list(depends))
if depends != host.depends or conflicts != host.conflicts:
host.depends = depends
host.conflicts = conflicts
host.inc_build()
host_file = host.build_management_package()
host.sign_package(sign_certs,sign_key)
packages.append(host)
else:
unchanged.append(host.package)
# upload all in one step...
progress_hook(True,3,3,'Upload %s host packages' % len(packages))
server = WaptServer().load_config_from_file(waptconfigfile)
server.upload_packages(packages,auth=(wapt_server_user,wapt_server_passwd),progress_hook=progress_hook)
return dict(updated = [p.package for p in packages],
discarded = discarded,
unchanged = unchanged)
finally:
logger.debug('Cleanup')
try:
i = 0
for s in packages:
i+=1
progress_hook(True,i,len(packages),'Cleanup')
if os.path.isfile(s.localpath):
os.remove(s.localpath)
progress_hook(False)
except WindowsError as e:
logger.critical('Unable to remove temporary directory %s: %s'% (s,repr(e)))
progress_hook(False)
def build_waptupgrade_package(waptconfigfile,target_directory=None,wapt_server_user=None,wapt_server_passwd=None,key_password=None,sign_digests=None,priority='critical'):
if target_directory is None:
target_directory = tempfile.gettempdir()
if not wapt_server_user:
wapt_server_user = raw_input('WAPT Server user :'******'WAPT Server password :'******'ascii')
wapt = common.Wapt(config_filename=waptconfigfile,disable_update_server_status=True)
wapt.dbpath = r':memory:'
wapt.use_hostpackages = False
# try to get a progress hook inside waptconsole
try:
import waptconsole
progress_hook = waptconsole.UpdateProgress
except ImportError:
def print_progress(show=False,n=0,max=100,msg=''):
if show:
print('%s %s/%s\r' % (msg,n,max),end='')
else:
if not msg:
msg='Done'
print("%s%s"%(msg,' '*(80-len(msg))))
progress_hook = print_progress
wapt.progress_hook = progress_hook
if sign_digests is None:
sign_digests = wapt.sign_digests
if not wapt.personal_certificate_path or not os.path.isfile(wapt.personal_certificate_path):
raise Exception(u'No personal certificate provided or not found (%s) for signing waptupgrade package' % wapt.personal_certificate_path)
waptget = get_file_properties('wapt-get.exe')
entry = PackageEntry(waptfile = makepath(wapt.wapt_base_dir,'waptupgrade'))
patchs_dir = makepath(entry.sourcespath,'patchs')
mkdirs(patchs_dir)
filecopyto(makepath(wapt.wapt_base_dir,'waptdeploy.exe'),makepath(patchs_dir,'waptdeploy.exe'))
entry.package = '%s-waptupgrade' % wapt.config.get('global','default_package_prefix')
rev = entry.version.split('-')[1]
entry.version = '%s-%s' % (waptget['FileVersion'],rev)
entry.inc_build()
entry.save_control_to_wapt()
entry.build_package(target_directory=target_directory)
entry.priority = priority
certs = wapt.personal_certificate()
key = wapt.private_key(private_key_password=key_password)
if not certs[0].is_code_signing:
raise Exception(u'%s is not a code signing certificate' % wapt.personal_certificate_path)
entry.sign_package(private_key=key,certificate = certs,private_key_password=key_password,mds = ensure_list(sign_digests))
wapt.http_upload_package(entry.localpath,wapt_server_user=wapt_server_user,wapt_server_passwd=wapt_server_passwd,progress_hook=progress_hook)
return entry.as_dict()
def add_ads_groups(waptconfigfile,
hostdicts_list,
sign_certs=None,
sign_key=None,
key_password=None,
wapt_server_user=None,
wapt_server_passwd=None,
cabundle=None):
if sign_certs is None:
sign_bundle_fn = inifile_readstring(waptconfigfile, u'global',
u'personal_certificate_path')
sign_bundle = SSLCABundle(sign_bundle_fn)
sign_certs = sign_bundle.certificates()
# we assume a unique signer.
if cabundle is None:
cabundle = sign_bundle
if not sign_certs:
raise Exception(u'No personal signer certificate found in %s' %
sign_bundle_fn)
if sign_key is None:
sign_key = sign_certs[0].matching_key_in_dirs(
private_key_password=key_password)
try:
import waptconsole
progress_hook = waptconsole.UpdateProgress
private_key_password_callback = waptconsole.GetPrivateKeyPassword
except ImportError as e:
def print_progress(show=False, n=0, max=100, msg=''):
if show:
print('%s %s/%s\r' % (msg, n, max), end='')
else:
if not msg:
msg = 'Done'
print("%s%s" % (msg, ' ' * (80 - len(msg))))
progress_hook = print_progress
private_key_password_callback = None
main_repo = WaptRemoteRepo(name='wapt', cabundle=cabundle)
main_repo.load_config_from_file(waptconfigfile)
main_repo.private_key_password_callback = private_key_password_callback
host_repo = WaptHostRepo(name='wapt-host',
host_id=[h['uuid'] for h in hostdicts_list],
cabundle=cabundle)
host_repo.load_config_from_file(waptconfigfile)
host_repo.private_key_password_callback = private_key_password_callback
total_hosts = len(host_repo.packages())
discarded_uuids = [p.package for p in host_repo.discarded]
packages = []
discarded = []
unchanged = []
try:
progress_hook(True, 0, len(hostdicts_list),
'Editing %s hosts' % len(hostdicts_list))
i = 0
for h in hostdicts_list:
try:
host_package = None
host_id = h['uuid']
hostname = h['computer_name']
groups = get_computer_groups(hostname)
host_package = host_repo.get(
host_id, PackageEntry(package=host_id, section='host'))
if progress_hook(True, i, len(hostdicts_list),
'Checking %s' % host_package.package):
break
wapt_groups = ensure_list(host_package['depends'])
additional = [
group for group in groups if not group in wapt_groups and (
main_repo.get(group, None) is not None)
]
if additional:
logger.info(u'Adding %s to %s' %
(','.join(additional), host_package.package))
if progress_hook(True, i, len(hostdicts_list),
'Editing %s' % host_package.package):
break
wapt_groups.extend(additional)
host_package.depends = ','.join(wapt_groups)
host_package.build_management_package()
host_package.inc_build()
host_file = host_package.build_management_package()
host_package.sign_package(sign_certs, sign_key)
packages.append(host_package)
else:
unchanged.append(host_package.package)
except Exception as e:
if host_package:
discarded.append(host_package.package)
logger.critical(u'Discarding because %s' % ensure_unicode(e))
# upload all in one step...
progress_hook(True, 3, 3, 'Upload %s host packages' % len(packages))
server = WaptServer().load_config_from_file(waptconfigfile)
server.private_key_password_callback = private_key_password_callback
server.upload_packages(packages,
auth=(wapt_server_user, wapt_server_passwd),
progress_hook=progress_hook)
return dict(updated=packages, discarded=discarded, unchanged=unchanged)
finally:
logger.debug('Cleanup')
try:
i = 0
for s in packages:
i += 1
progress_hook(True, i, len(packages), 'Cleanup')
if os.path.isfile(s.localpath):
os.remove(s.localpath)
progress_hook(False)
except WindowsError as e:
logger.critical('Unable to remove temporary directory %s: %s' %
(s, repr(e)))
progress_hook(False)
def on_trigger_host_action(self,args,result_callback=None):
try:
start_time = time.time()
actions = args
if not isinstance(actions,list):
actions =[actions]
logger.debug('Host actions "%s" triggered by SocketIO' % ",".join([action['action'] for action in actions]))
# check signatures
if not self.wapt:
raise Exception('Wapt not available')
verified_by = None
for action in actions:
name = action['action']
verified_by = None
signer_cert_chain = SSLCABundle().add_pem(action['signer_certificate']).certificates()
chain = self.wapt.cabundle.check_certificates_chain(signer_cert_chain)
if chain:
required_attributes = ['uuid','action']
if name in ['trigger_install_packages','trigger_remove_packages','trigger_forget_packages']:
required_attributes.append('packages')
if name in ['trigger_change_description']:
required_attributes.append('computer_description')
if name in waptservice_remote_actions:
required_attributes.extend(waptservice_remote_actions[name].required_attributes)
verified_by = chain[0].verify_claim(action,max_age_secs=waptconfig.signature_clockskew,
required_attributes=required_attributes)
if not verified_by:
raise SSLVerifyException('Bad signature for action %s, aborting' % action)
if verified_by:
verified_by = verified_by.get('verified_by',None)
result = []
for action in actions:
uuid = action['uuid']
if uuid != self.wapt.host_uuid:
raise Exception('Task is not targeted to this host. task''s uuid does not match host''uuid')
name = action['action']
if name in ['trigger_cancel_all_tasks']:
data = [t.as_dict() for t in self.task_manager.cancel_all_tasks()]
result.append(data)
elif name in ['trigger_host_update','trigger_host_register']:
if name == 'trigger_host_update':
task = WaptUpdate()
elif name == 'trigger_host_register':
task = WaptRegisterComputer(computer_description = action.get('computer_description',None))
task.force = action.get('force',False)
task.notify_user = action.get('notify_user',False)
task.notify_server_on_finish = action.get('notify_server',False)
task.created_by = verified_by
data = self.task_manager.add_task(task).as_dict()
result.append(data)
elif name in ['trigger_change_description']:
desc = action.get('computer_description',None)
if desc is not None:
setuphelpers.set_computer_description(desc)
msg = u'Computer description of %s changed to %s' % (setuphelpers.get_hostname(),setuphelpers.get_computer_description())
if not setuphelpers.get_computer_description() == desc:
raise Exception(u'Computer description has not been changed')
result.append(dict(success=True,
msg = msg,
result = msg,
))
if action.get('notify_server',False):
task = WaptUpdate(created_by=verified_by)
task.notify_server_on_finish = True
self.task_manager.add_task(task)
result.append(task.as_dict())
elif name == 'trigger_host_upgrade':
notify_user = action.get('notify_user',False)
notify_server_on_finish = action.get('notify_server',False)
force = action.get('force',False)
self.wapt.update(force=False)
upgrades = self.wapt.list_upgrade()
to_install = upgrades['upgrade']+upgrades['additional']+upgrades['install']
for req in to_install:
result.append(self.task_manager.add_task(WaptPackageInstall(req,force=force,notify_user=notify_user,created_by=verified_by)).as_dict())
self.task_manager.add_task(WaptAuditPackage(packagename=req,force=False,
notify_user=notify_user,
notify_server_on_finish=False,
priority=200,
created_by=verified_by)).as_dict()
result.append(self.task_manager.add_task(WaptUpgrade(notify_user=notify_user,created_by=verified_by,priority=200)).as_dict())
result.append(self.task_manager.add_task(WaptCleanup(notify_user=False,created_by=verified_by,priority=200)).as_dict())
elif name in ['trigger_install_packages','trigger_remove_packages','trigger_forget_packages']:
packagenames = ensure_list(action['packages'])
for packagename in packagenames:
if name == 'trigger_install_packages':
task = WaptPackageInstall(packagename=packagename)
elif name == 'trigger_remove_packages':
task = WaptPackageRemove(packagename=packagename)
elif name == 'trigger_forget_packages':
task = WaptPackageForget(packagenames=packagename)
task.force = action.get('force',False)
task.notify_user = action.get('notify_user',False)
task.notify_server_on_finish = action.get('notify_server',False)
task.created_by=verified_by
result.append(self.task_manager.add_task(task).as_dict())
if name == 'trigger_install_packages':
self.task_manager.add_task(WaptAuditPackage(packagename=packagename,force=task.force,
notify_user=task.notify_user,
notify_server_on_finish=task.notify_server_on_finish,
priority=200)).as_dict()
elif name == 'trigger_waptservicerestart':
msg = setuphelpers.create_onetime_task('waptservicerestart','cmd.exe','/C net stop waptservice & net start waptservice')
result.append(dict(success=True,msg = msg,result = msg))
elif name in waptservice_remote_actions:
waptservice_remote_actions[name].trigger_action(self,action,verified_by)
else:
raise EWaptException('Unhandled remote action %s' % name)
#self.emit('trigger_update_result',{'result':data})
if result_callback:
result_callback(make_response(result,uuid=self.wapt.host_uuid,request_time=time.time()-start_time))
except BaseException as e:
logger.info('Exception for actions %s: %s' % (repr(args),repr(e)))
if result_callback:
result_callback(make_response_from_exception(e,uuid=self.wapt.host_uuid,request_time=time.time()-start_time))
def _as_attribute(cls, k, v):
if k in ['depends', 'conflicts']:
return ensure_list(v or None)
else:
return v or None
def proxy_host_request(request, action):
"""Proxy a waptconsole action to wapt clients using websockets
Args:
uuid: can be a list or a single uuid
notify_user: 0/1
notify_server: 0/1
Returns:
dict:
'result':
'success' (list)
'errors' (list)
'msg' (str)
'success'(bool)
'request_time' (float)
'error_code'
"""
try:
start_time = time.time()
all_args = {k: v for k, v in request.args.iteritems()}
if request.json:
all_args.update(request.json)
uuids = ensure_list(all_args['uuid'])
del(all_args['uuid'])
timeout = float(request.args.get('timeout', app.conf.get('clients_read_timeout', 5)))
result = dict(success=[], errors=[])
tasks = []
for uuid in uuids:
try:
host_data = Hosts\
.select(Hosts.uuid, Hosts.computer_fqdn,
Hosts.server_uuid,
Hosts.listening_address,
Hosts.listening_port,
Hosts.listening_protocol,
Hosts.listening_timestamp,
)\
.where((Hosts.server_uuid == get_server_uuid()) & (Hosts.uuid == uuid) & (~Hosts.listening_address.is_null()) & (Hosts.listening_protocol == 'websockets'))\
.dicts()\
.first(1)
if host_data and host_data.get('listening_address', None):
msg = u''
logger.info(
'Launching %s with args %s for %s at address %s...' %
(action, all_args, uuid, host_data['listening_address']))
args = dict(all_args)
sid = host_data['listening_address']
computer_fqdn = host_data['computer_fqdn']
def emit_action(sid, uuid, action, action_args, computer_fqdn, timeout=5):
try:
got_result = []
def result_callback(data):
got_result.append(data)
logger.debug(u'Emit %s to %s (%s)' % (action, uuid, computer_fqdn))
socketio.emit(action, action_args, room=sid, callback=result_callback)
# with for asynchronous answer...
wait_loop = timeout * 20
while not got_result:
wait_loop -= 1
if wait_loop < 0:
raise EWaptTimeoutWaitingForResult('Timeout, client did not send result within %s s' % timeout)
socketio.sleep(0.05)
# action succedded
result['success'].append(
dict(
uuid=uuid,
msg=msg,
computer_fqdn=computer_fqdn,
result=got_result[0],
))
except Exception as e:
result['errors'].append(
dict(
uuid=uuid,
msg='%s' % repr(e),
computer_fqdn='',
))
if sid:
tasks.append(socketio.start_background_task(
emit_action, sid=sid, uuid=uuid, action=action, action_args=args, computer_fqdn=computer_fqdn))
else:
result['errors'].append(
dict(
uuid=uuid,
msg='Host %s is not registered or not connected wia websockets' % uuid,
computer_fqdn='',
))
except Exception as e:
result['errors'].append(
dict(
uuid=uuid,
msg='Host %s error %s' % (uuid, repr(e)),
computer_fqdn='',
))
# wait for all background tasks to terminate or timeout...
# assume the timeout should be longer if more hosts to perform
wait_loop = timeout * len(tasks)
while True:
running = [t for t in tasks if t.is_alive()]
if not running:
break
wait_loop -= 1
if wait_loop < 0:
break
socketio.sleep(0.05)
msg = ['Success : %s, Errors: %s' % (len(result['success']), len(result['errors']))]
if result['errors']:
msg.extend(['%s: %s' % (e['computer_fqdn'], e['msg'])
for e in result['errors']])
return make_response(result,
msg='\n- '.join(msg),
success=len(result['success']) > 0,
request_time=time.time() - start_time)
except Exception as e:
return make_response_from_exception(e)
