def write(self, text):
'''Logs written output to listeners'''
if text and text != '\n':
if self.events:
self.events.send_multipart(
[str('PRINT'), (ensure_unicode(text)).encode('utf8')])
self.logs.append(ensure_unicode(text))
def dmi_info():
"""Hardware System information from BIOS estracted with dmidecode
Convert dmidecode -q output to python dict
Returns:
dict
>>> dmi = dmi_info()
>>> 'UUID' in dmi['System_Information']
True
>>> 'Product_Name' in dmi['System_Information']
True
"""
result = {}
# dmidecode is bugged on macOS, and prints "Bad address" repeatedly on stderr
if platform.system() != 'Darwin':
dmiout = ensure_unicode(run('dmidecode -q'))
else:
dmiout = ensure_unicode(run('dmidecode -q 2> /dev/null'))
new_section = True
for l in dmiout.splitlines():
if not l.strip() or l.startswith('#'):
new_section = True
continue
if not l.startswith('\t') or new_section:
currobject = {}
key = l.strip().replace(' ', '_')
# already here... so add as array...
if (key in result):
if not isinstance(result[key], list):
result[key] = [result[key]]
result[key].append(currobject)
else:
result[key] = currobject
if l.startswith('\t'):
print(l)
else:
if not l.startswith('\t\t'):
currarray = []
if ':' in l:
(name, value) = l.split(':', 1)
currobject[name.strip().replace(' ', '_')] = value.strip()
else:
print("Error in line : %s" % l)
else:
# first line of array
if not currarray:
currobject[name.strip().replace(' ', '_')] = currarray
currarray.append(l.strip())
new_section = False
return result
def filecopyto(filename, target):
"""Copy file from absolute or package temporary directory to target directory
If file is dll or exe, logs the original and new version.
Args:
filename (str): absolute path to file to copy,
or relative path to temporary package install content directory.
target (str) : absolute path to target directory where to copy file.
target is either a full filename or a directory name
if filename is .exe or .dll, logger prints version numbers
>>> if not os.path.isfile('c:/tmp/fc.test'):
... with open('c:/tmp/fc.test','wb') as f:
... f.write('test')
>>> if not os.path.isdir('c:/tmp/target'):
... os.mkdir('c:/tmp/target')
>>> if os.path.isfile('c:/tmp/target/fc.test'):
... os.unlink('c:/tmp/target/fc.test')
>>> filecopyto('c:/tmp/fc.test','c:/tmp/target')
>>> os.path.isfile('c:/tmp/target/fc.test')
True
"""
(dir, fn) = os.path.split(filename)
if not dir:
dir = os.getcwd()
if os.path.isdir(target):
target = os.path.join(target, os.path.basename(filename))
if os.path.isfile(target):
if os.path.splitext(target)[1] in ('.exe', '.dll'):
try:
ov = get_file_properties(target)['FileVersion']
nv = get_file_properties(filename)['FileVersion']
logger.info(u'Replacing %s (%s) -> %s' %
(ensure_unicode(target), ov, nv))
except:
logger.info(u'Replacing %s' % target)
else:
logger.info(u'Replacing %s' % target)
else:
if os.path.splitext(target)[1] in ('.exe', '.dll'):
try:
nv = get_file_properties(filename)['FileVersion']
logger.info(u'Copying %s (%s)' % (ensure_unicode(target), nv))
except:
logger.info(u'Copying %s' % (ensure_unicode(target)))
else:
logger.info(u'Copying %s' % (ensure_unicode(target)))
shutil.copy(filename, target)
def load_json(filenames=r'c:\tmp\*.json'):
"""Read a json host collection exported from wapt mongo and creates
Wapt PG Host DB instances"""
import glob
convert_map = {
'last_query_date':'last_seen_on',
'update_status':'last_update_status',
'softwares':'installed_softwares',
'packages':'installed_packages',
'wapt':'wapt_status',
'host':'host_info',
}
for fn in glob.glob(filenames):
print('Loading %s'%fn)
recs = json.load(codecs.open(fn,'rb',encoding='utf8'))
for rec in recs:
computer_fqdn = rec['host']['computer_fqdn']
uuid = rec['uuid']
try:
try:
# wapt update_status packages softwares host
newhost = Hosts()
for k in rec.keys():
if hasattr(newhost,convert_map.get(k,k)):
setattr(newhost,convert_map.get(k,k),rec[k])
else:
print '%s unknown key %s' % (computer_fqdn,k)
if 'host' in rec:
newhost.host_info = rec.get('host')
newhost.save(force_insert=True)
print('%s Inserted (%s)'%(newhost.computer_fqdn,newhost.uuid))
except IntegrityError as e:
wapt_db.rollback()
updhost = Hosts.get(uuid=uuid)
for k in rec.keys():
if hasattr(updhost,convert_map.get(k,k)):
setattr(updhost,convert_map.get(k,k),rec[k])
else:
print '%s unknown key %s' % (computer_fqdn,k)
if 'host' in rec:
newhost.host_info = rec.get('host')
updhost.save()
print('%s Updated'%computer_fqdn)
except Exception as e:
print(u'Error for %s : %s'%(ensure_unicode(computer_fqdn),ensure_unicode(e)))
wapt_db.rollback()
raise e
def migrate_pg_db(old_pgsql_root_dir,old_pgsql_data_dir,pgsql_root_dir,pgsql_data_dir):
#
try:
cwd = os.getcwd()
tmpdir = os.path.join(cwd,'temp')
mkdir_p(tmpdir)
os.chdir(tmpdir)
setuphelpers.run(r'icacls "%s" /t /grant "*S-1-1-0":(OI)(CI)(M)' % tmpdir)
old_pgsql_root_dir = old_pgsql_root_dir.replace('\\','/')
old_pgsql_data_dir = old_pgsql_data_dir.replace('\\','/')
pgsql_root_dir = pgsql_root_dir.replace('\\','/')
pgsql_data_dir = pgsql_data_dir.replace('\\','/')
cmd = r'"{pgsql_root_dir}/bin/pg_upgrade.exe" -U postgres --old-datadir "{old_pgsql_data_dir}" --new-datadir "{pgsql_data_dir}" --old-bindir "{old_pgsql_root_dir}/bin" --new-bindir "{pgsql_root_dir}/bin"'.format(**locals())
print('Running %s' % cmd)
print(run(cmd,cwd=tmpdir))
os.rename(old_pgsql_root_dir,old_pgsql_root_dir+'.old')
os.rename(old_pgsql_data_dir,old_pgsql_data_dir+'.old')
return True
except Exception as e:
print('Unable to migrate database : %s' % ensure_unicode(e))
return False
finally:
os.chdir(cwd)
def user_local_appdata():
r"""Return the local appdata profile of current user
Returns:
str: path like u'C:\\Users\\user\\AppData\\Local'
"""
return ensure_unicode(makepath(os.environ['HOME'], '.config'))
def migrate_pg_db(old_pgsql_root_dir, old_pgsql_data_dir, pgsql_root_dir,
pgsql_data_dir):
#
try:
cwd = os.getcwd()
tmpdir = os.path.join(cwd, 'temp')
mkdir_p(tmpdir)
os.chdir(tmpdir)
setuphelpers.run(r'icacls "%s" /t /grant "*S-1-1-0":(OI)(CI)(M)' %
tmpdir)
old_pgsql_root_dir = old_pgsql_root_dir.replace('\\', '/')
old_pgsql_data_dir = old_pgsql_data_dir.replace('\\', '/')
pgsql_root_dir = pgsql_root_dir.replace('\\', '/')
pgsql_data_dir = pgsql_data_dir.replace('\\', '/')
cmd = r'"{pgsql_root_dir}/bin/pg_upgrade.exe" -U postgres --old-datadir "{old_pgsql_data_dir}" --new-datadir "{pgsql_data_dir}" --old-bindir "{old_pgsql_root_dir}/bin" --new-bindir "{pgsql_root_dir}/bin"'.format(
**locals())
print('Running %s' % cmd)
print(run(cmd, cwd=tmpdir))
os.rename(old_pgsql_root_dir, old_pgsql_root_dir + '.old')
os.rename(old_pgsql_data_dir, old_pgsql_data_dir + '.old')
return True
except Exception as e:
print('Unable to migrate database : %s' % ensure_unicode(e))
return False
finally:
os.chdir(cwd)
def _run(self):
"""Launch an external 'wapt-get waptupgrade' process to upgrade local copy of wapt client"""
from setuphelpers import run
output = ensure_unicode(
run('"%s" %s' %
(os.path.join(wapt_root_dir, 'wapt-get.exe'), 'waptupgrade')))
self.result = {'result': 'OK', 'message': output}
def host_info_common_unix():
"""Read main workstation informations, returned as a dict
Returns:
dict: main properties of host, networking and windows system
.. versionchanged:: 1.4.1
returned keys changed :
dns_domain -> dnsdomain
>>> hi = host_info()
>>> 'computer_fqdn' in hi and 'connected_ips' in hi and 'computer_name' in hi and 'mac' in hi
True
"""
info = {}
try:
dmi = dmi_info()
info['system_manufacturer'] = dmi['System_Information']['Manufacturer']
info['system_productname'] = dmi['System_Information']['Product_Name']
except:
logger.warning(
'Error while running dmidecode, dmidecode needs root privileges')
pass
info['computer_name'] = socket.gethostname()
info['computer_fqdn'] = socket.getfqdn()
info['dnsdomain'] = get_domain_from_socket()
try:
if os.path.isfile('/etc/samba/smb.conf'):
config = configparser.RawConfigParser(strict=False)
config.read('/etc/samba/smb.conf')
if config.has_option('global', 'workgroup'):
info['workgroup_name'] = config.get('global', 'workgroup')
except:
info['workgroup_name'] = ''
info['connected_ips'] = get_local_IPs()
list_mac = {}
for c in networking():
if 'mac' in c and 'addr' in c:
for m in c['addr']:
if m['addr'] in info['connected_ips']:
list_mac[c['mac']] = None
info['networking'] = networking()
info['gateways'] = [get_default_gateways()]
info['dns_servers'] = get_dns_servers()
info['mac'] = list(list_mac)
info['kernel_version'] = get_kernel_version()
#Fix for vscode don't know why it doesn't work : KeyError: 'brand'
try:
info['cpu_name'] = cpuinfo.get_cpu_info()['brand']
except:
pass
info['environ'] = {k: ensure_unicode(v) for k, v in os.environ.items()}
info['main_ip'] = get_main_ip()
return info
def run(self):
while True:
func, args, kargs = self.tasks.get()
try:
func(*args, **kargs)
except Exception, e:
logger.info(u"%s" % ensure_unicode(e))
finally:
def run(self):
while True:
func, args, kargs = self.tasks.get()
try:
func(*args, **kargs)
except Exception, e:
logger.info(u"%s" % ensure_unicode(e))
finally:
def _run(self):
def cjoin(l):
return u','.join([u'%s'%p for p in l])
try:
self.result = self.wapt.cleanup(obsolete_only=not self.force)
self.summary = _(u"Packages erased : {}").format(cjoin(self.result))
except Exception as e:
self.result = {}
self.summary = _(u"Error while clearing local cache : {}").format(ensure_unicode(e))
raise Exception(self.summary)
def worker(pipe, on_write=None):
while True:
line = pipe.readline()
if not line:
break
else:
if on_write:
on_write(ensure_unicode(line))
if pipe == proc.stderr:
return_stderr.append(line)
else:
output.append(line)
def _run(self):
if self.wapt.waptserver_available():
print('Sending host status to server')
try:
self.result = self.wapt.update_server_status(force=self.force)
self.summary = _(u'WAPT Server has been notified')
print('Done.')
except Exception as e:
self.result = {}
self.summary = _(u"Error while sending to the server : {}").format(ensure_unicode(e))
else:
self.result = {}
self.summary = _(u'WAPT Server is not available')
def _run(self):
if self.wapt.waptserver_available():
try:
self.result = self.wapt.register_computer(
description=self.computer_description)
self.summary = _(u"Inventory has been sent to the WAPT server")
except Exception as e:
self.result = {}
self.summary = _(
u"Error while sending inventory to the server : {}"
).format(ensure_unicode(e))
raise
else:
self.result = {}
self.summary = _(u'WAPT Server is not available')
raise Exception(self.summary)
def as_dict(self):
return copy.deepcopy(
dict(
id=self.id,
classname=self.__class__.__name__,
priority=self.priority,
order=self.order,
create_date=self.create_date and self.create_date.isoformat(),
start_date=self.start_date and self.start_date.isoformat(),
finish_date=self.finish_date and self.finish_date.isoformat(),
logs=u'\n'.join([ensure_unicode(l) for l in self.logs]),
result=common.jsondump(self.result),
summary=self.summary,
progress=self.progress,
runstatus=self.runstatus,
description=u"{}".format(self),
pidlist=u"{0}".format(self.external_pids),
notify_user=self.notify_user,
notify_server_on_start=self.notify_server_on_start,
notify_server_on_finish=self.notify_server_on_finish,
created_by=self.created_by,
))
def get_current_user():
r"""Get the login name for the current user.
>>> get_current_user()
u'htouvet'
"""
return ensure_unicode(getpass.getuser())
def update_host_data(data):
"""Helper function to insert or update host data in db
Args :
data (dict) : data to push in DB with at least 'uuid' key
if uuid key already exists, update the data
eld insert
only keys in data are pushed to DB.
Other data (fields) are left untouched
Returns:
dict : with uuid,computer_fqdn,host_info from db after update
"""
migrate_map_13_14 = {
'packages': None,
'installed_packages': None,
'softwares': None,
'installed_softwares': None,
'update_status': 'last_update_status',
'host': 'host_info',
'wapt': 'wapt_status',
'update_status': 'last_update_status',
}
uuid = data['uuid']
try:
existing = Hosts.select(
Hosts.uuid, Hosts.computer_fqdn).where(Hosts.uuid == uuid).first()
if not existing:
logger.debug('Inserting new host %s with fields %s' %
(uuid, data.keys()))
# wapt update_status packages softwares host
newhost = Hosts()
for k in data.keys():
# manage field renaming between 1.3 and >= 1.4
target_key = migrate_map_13_14.get(k, k)
if target_key and hasattr(newhost, target_key):
set_host_field(newhost, target_key, data[k])
newhost.save(force_insert=True)
else:
logger.debug('Updating %s for fields %s' % (uuid, data.keys()))
updhost = Hosts.get(uuid=uuid)
for k in data.keys():
# manage field renaming between 1.3 and >= 1.4
target_key = migrate_map_13_14.get(k, k)
if target_key and hasattr(updhost, target_key):
set_host_field(updhost, target_key, data[k])
updhost.save()
# separate tables
# we are tolerant on errors here a we don't know exactly if client send good encoded data
# but we still want to get host in table
try:
if ('installed_softwares' in data) or ('softwares' in data):
installed_softwares = data.get('installed_softwares',
data.get('softwares', None))
if not update_installed_softwares(uuid, installed_softwares):
logger.critical(
'Unable to update installed_softwares for %s' % uuid)
except Exception as e:
logger.critical(
u'Unable to update installed_softwares for %s: %s' %
(uuid, traceback.format_exc()))
try:
if ('installed_packages' in data) or ('packages' in data):
installed_packages = data.get('installed_packages',
data.get('packages', None))
if not update_installed_packages(uuid, installed_packages):
logger.critical(
'Unable to update installed_packages for %s' % uuid)
except Exception as e:
logger.critical(u'Unable to update installed_packages for %s: %s' %
(uuid, traceback.format_exc()))
result_query = Hosts.select(Hosts.uuid, Hosts.computer_fqdn)
return result_query.where(Hosts.uuid == uuid).dicts().dicts().first(1)
except Exception as e:
logger.warning(traceback.format_exc())
logger.critical(u'Error updating data for %s : %s' %
(uuid, ensure_unicode(e)))
wapt_db.rollback()
raise e
def write(self, text):
'''Logs written output to listeners'''
if text and text != '\n':
if self.events:
self.events.post_event('PRINT',ensure_unicode(text))
self.logs.append(ensure_unicode(text))
def add_ads_groups(waptconfigfile,
hostdicts_list,
sign_certs=None,
sign_key=None,
key_password=None,
wapt_server_user=None,wapt_server_passwd=None,
cabundle = None):
if sign_certs is None:
sign_bundle_fn = inifile_readstring(waptconfigfile,u'global',u'personal_certificate_path')
sign_bundle = SSLCABundle(sign_bundle_fn)
sign_certs = sign_bundle.certificates()
# we assume a unique signer.
if cabundle is None:
cabundle = sign_bundle
if not sign_certs:
raise Exception(u'No personal signer certificate found in %s' % sign_bundle_fn)
if sign_key is None:
sign_key = sign_certs[0].matching_key_in_dirs(private_key_password=key_password)
main_repo = WaptRemoteRepo(name='wapt',cabundle = cabundle)
main_repo.load_config_from_file(waptconfigfile)
host_repo = WaptHostRepo(name='wapt-host',host_id=[h['uuid'] for h in hostdicts_list],cabundle = cabundle)
host_repo.load_config_from_file(waptconfigfile)
total_hosts = len(host_repo.packages())
discarded_uuids = [p.package for p in host_repo.discarded]
try:
import waptconsole
progress_hook = waptconsole.UpdateProgress
except ImportError as e:
def print_progress(show=False,n=0,max=100,msg=''):
if show:
print('%s %s/%s\r' % (msg,n,max),end='')
else:
if not msg:
msg='Done'
print("%s%s"%(msg,' '*(80-len(msg))))
progress_hook = print_progress
packages = []
discarded = []
unchanged = []
try:
progress_hook(True,0,len(hostdicts_list),'Editing %s hosts' % len(hostdicts_list))
i = 0
for h in hostdicts_list:
try:
host_package = None
host_id = h['uuid']
hostname = h['computer_name']
groups = get_computer_groups(hostname)
host_package = host_repo.get(host_id,PackageEntry(package=host_id,section='host'))
if progress_hook(True,i,len(hostdicts_list),'Checking %s' % host_package.package):
break
wapt_groups = ensure_list(host_package['depends'])
additional = [group for group in groups if not group in wapt_groups and (main_repo.get(group,None) is not None)]
if additional:
logger.info(u'Adding %s to %s' % (','.join(additional),host_package.package))
if progress_hook(True,i,len(hostdicts_list),'Editing %s' % host_package.package):
break
wapt_groups.extend(additional)
host_package.depends = ','.join(wapt_groups)
host_package.build_management_package()
host_package.inc_build()
host_file = host_package.build_management_package()
host_package.sign_package(sign_certs,sign_key)
packages.append(host_package)
else:
unchanged.append(host_package.package)
except Exception as e:
if host_package:
discarded.append(host_package.package)
logger.critical(u'Discarding because %s' % ensure_unicode(e))
# upload all in one step...
progress_hook(True,3,3,'Upload %s host packages' % len(packages))
server = WaptServer().load_config_from_file(waptconfigfile)
server.upload_packages(packages,auth=(wapt_server_user,wapt_server_passwd),progress_hook=progress_hook)
return dict(updated = packages,
discarded = discarded,
unchanged = unchanged)
finally:
logger.debug('Cleanup')
try:
i = 0
for s in packages:
i+=1
progress_hook(True,i,len(packages),'Cleanup')
if os.path.isfile(s.localpath):
os.remove(s.localpath)
progress_hook(False)
except WindowsError as e:
logger.critical('Unable to remove temporary directory %s: %s'% (s,repr(e)))
progress_hook(False)
def get_fqdn():
return ensure_unicode(
get_hostname()) if os.name == 'nt' else ensure_unicode(
socket.getfqdn())
def copytree2(src,
dst,
ignore=None,
onreplace=default_skip,
oncopy=default_oncopy,
enable_replace_at_reboot=True):
r"""Copy src directory to dst directory. dst is created if it doesn't exists
src can be relative to installation temporary dir
oncopy is called for each file copy. if False is returned, copy is skipped
onreplace is called when a file will be overwritten.
Args:
src (str): path to source directory (absolute path or relative to package extraction tempdir)
dst (str): path to target directory (created if not present)
ignore (func) : callback func(root_dir,filenames) which returns names to ignore
onreplace (func) : callback func(src,dst):boolean called when a file will be replaced to decide what to do.
default is to not replace if target exists. can be default_overwrite or default_overwrite_older or
custom function.
oncopy (func) : callback func(msg,src,dst) called when a file is copied.
default is to log in debug level the operation
enable_replace_at_reboot (boolean): if True, files which are locked will be scheduled for replace at next reboot
Returns:
Raises:
>>> copytree2(r'c:\tranquilit\wapt\tests',r'c:\tranquilit\wapt\tests2')
>>> isdir(r'c:\tranquilit\wapt\tests2')
True
>>> remove_tree(r'c:\tranquilit\wapt\tests2')
>>> isdir(r'c:\tranquilit\wapt\tests2')
False
"""
logger.debug('Copy tree from "%s" to "%s"' %
(ensure_unicode(src), ensure_unicode(dst)))
# path relative to temp directory...
tempdir = os.getcwd()
if not os.path.isdir(src) and os.path.isdir(os.path.join(tempdir, src)):
src = os.path.join(tempdir, src)
names = os.listdir(src)
if ignore is not None:
ignored_names = ignore(src, names)
else:
ignored_names = set()
if not os.path.isdir(dst):
if oncopy('create directory', src, dst):
os.makedirs(dst)
errors = []
skipped = []
for name in names:
if name in ignored_names:
continue
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
try:
if os.path.isdir(srcname):
if oncopy('directory', srcname, dstname):
copytree2(srcname,
dstname,
ignore=ignore,
onreplace=onreplace,
oncopy=oncopy)
else:
try:
if os.path.isfile(dstname):
if onreplace(srcname, dstname) and oncopy(
'overwrites', srcname, dstname):
os.unlink(dstname)
shutil.copy2(srcname, dstname)
else:
if oncopy('copy', srcname, dstname):
shutil.copy2(srcname, dstname)
except (IOError, os.error) as e:
# file is locked...
if enable_replace_at_reboot and e.errno in (5, 13):
filecopyto(srcname, dstname + '.pending')
replace_at_next_reboot(tmp_filename=dstname +
'.pending',
target_filename=dstname)
else:
raise
except (IOError, os.error) as why:
logger.critical(u'Error copying from "%s" to "%s" : %s' %
(ensure_unicode(src), ensure_unicode(dst),
ensure_unicode(why)))
errors.append((srcname, dstname, str(why)))
# catch the Error from the recursive copytree so that we can
# continue with other files
except shutil.Error as err:
#errors.extend(err.args[0])
errors.append(err)
try:
shutil.copystat(src, dst)
except WindowsError:
# can't copy file access times on Windows
pass
except OSError as why:
errors.extend((src, dst, str(why)))
if errors:
raise shutil.Error(errors)
def default_oncopy(msg, src, dst):
logger.debug(
u'%s : "%s" to "%s"' %
(ensure_unicode(msg), ensure_unicode(src), ensure_unicode(dst)))
return True
def main():
parser=OptionParser(usage=__doc__,prog = 'wapt-signpackage')
parser.add_option("-c","--certificate", dest="public_key", default='', help="Path to the PEM RSA certificate to embed identitiy in control. (default: %default)")
parser.add_option("-k","--private-key", dest="private_key", default='', help="Path to the PEM RSA private key to sign packages. (default: %default)")
#parser.add_option("-w","--private-key-passwd", dest="private_key_passwd", default='', help="Path to the password of the private key. (default: %default)")
parser.add_option("-l","--loglevel", dest="loglevel", default=None, type='choice', choices=['debug','warning','info','error','critical'], metavar='LOGLEVEL',help="Loglevel (default: warning)")
parser.add_option("-m","--message-digest", dest="md", default='sha256', help="Message digest type for signatures. (default: %default)")
parser.add_option("-s","--scan-packages", dest="doscan", default=False, action='store_true', help="Rescan packages and update local Packages index after signing. (default: %default)")
parser.add_option("-r","--remove-setup", dest="removesetup", default=False, action='store_true', help="Remove setup.py. (default: %default)")
parser.add_option("-i","--inc-release", dest="increlease", default=False, action='store_true', help="Increase release number when building package (default: %default)")
parser.add_option("--maturity", dest="set_maturity", default=None, help="Set/change package maturity when signing package. (default: None)")
parser.add_option( "--keep-signature-date", dest="keep_signature_date",default=False, action='store_true', help="Keep the current package signature date, and file changetime (default: %default)")
parser.add_option( "--if-needed", dest="if_needed", default=False, action='store_true',help="Re-sign package only if needed (default: warning)")
(options,args) = parser.parse_args()
loglevel = options.loglevel
if len(logger.handlers) < 1:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(logging.Formatter(
u'%(asctime)s %(levelname)s %(message)s'))
logger.addHandler(hdlr)
if loglevel:
setloglevel(logger,loglevel)
else:
setloglevel(logger,'warning')
if len(args) < 1:
print(parser.usage)
sys.exit(1)
if not options.public_key and not options.private_key:
print('ERROR: No certificate found or specified')
sys.exit(1)
if options.private_key and os.path.isfile(options.private_key):
key = SSLPrivateKey(options.private_key)
else:
cert = SSLCertificate(options.public_key or options.private_key)
key = cert.matching_key_in_dirs()
if not key:
print('ERROR: No private key found or specified')
sys.exit(1)
args = ensure_list(args)
ca_bundle = SSLCABundle()
signers_bundle = SSLCABundle()
signers_bundle.add_certificates_from_pem(pem_filename=options.public_key)
waptpackages = []
for arg in args:
waptpackages.extend(glob.glob(arg))
errors = []
package_dirs = []
for waptpackage in waptpackages:
package_dir = os.path.abspath(os.path.dirname(waptpackage))
if not package_dir in package_dirs:
package_dirs.append(package_dir)
print('Processing %s'%waptpackage)
try:
sign_needed=False
pe = PackageEntry(waptfile = waptpackage)
if options.removesetup:
if pe.has_file('setup.py'):
with pe.as_zipfile(mode='a') as waptfile:
waptfile.remove('setup.py')
sign_needed=True
if not sign_needed and options.if_needed:
try:
pe.check_control_signature(trusted_bundle=signers_bundle,signers_bundle=signers_bundle)
for md in ensure_list(options.md):
if not pe.has_file(pe.get_signature_filename(md)):
raise Exception('Missing signature for md %s' % md)
logger.info('Skipping %s, already signed properly' % pe.asrequirement())
sign_needed = False
except Exception as e:
logger.info('Sign is needed for %s because %s' % (pe.asrequirement(),e))
sign_needed = True
if options.increlease:
pe.inc_build()
sign_needed = True
if options.set_maturity is not None and pe.maturity != options.set_maturity:
pe.maturity = options.set_maturity
sign_needed = True
if not options.if_needed or sign_needed:
pe.sign_package(private_key=key,certificate = signers_bundle.certificates(),mds = ensure_list(options.md),keep_signature_date=options.keep_signature_date)
newfn = pe.make_package_filename()
if newfn != pe.filename:
newfn_path = os.path.join(package_dir,newfn)
if not os.path.isfile(newfn_path):
print(u"Renaming file from %s to %s to match new package's properties" % (pe.filename,newfn))
os.rename(os.path.join(package_dir,pe.filename),newfn_path)
else:
print('WARNING: unable to rename file from %s to %s because target already exists' % (pe.filename,newfn))
print('Done')
except Exception as e:
print(u'Error: %s'%ensure_unicode(e.message))
errors.append([waptpackage,repr(e)])
if options.doscan:
for package_dir in package_dirs:
if os.path.isfile(os.path.join(package_dir,'Packages')):
print(u'Launching the update of Packages index in %s ...'% ensure_unicode(package_dir))
repo = WaptLocalRepo(package_dir)
repo.update_packages_index()
print('Done')
else:
print("Don't forget to rescan your repository with wapt-scanpackages %s" % os.path.dirname(waptpackages[0]))
if errors:
print('Package not processed properly: ')
for fn,error in errors:
print(u'%s : %s' % (fn,error))
sys.exit(1)
else:
sys.exit(0)
def cjoin(l):
return u','.join([u'%s' % ensure_unicode(p) for p in l])
def run(cmd,
shell=True,
timeout=600,
accept_returncodes=[0, 3010],
on_write=None,
pidlist=None,
return_stderr=True,
**kwargs):
r"""Run the command cmd in a shell and return the output and error text as string
Args:
cmd : command and arguments, either as a string or as a list of arguments
shell (boolean) : True is assumed
timeout (int) : maximum time to wait for cmd completion is second (default = 600)
a TimeoutExpired exception is raised if tiemout is reached.
on_write : callback when a new line is printed on stdout or stderr by the subprocess
func(unicode_line). arg is enforced to unicode
accept_returncodes (list) : list of return code which are considered OK default = (0,1601)
pidlist (list): external list where to append the pid of the launched process.
return_stderr (bool or list) : if True, the error lines are returned to caller in result.
if a list is provided, the error lines are appended to this list
all other parameters from the psutil.Popen constructor are accepted
Returns:
RunOutput : bytes like output of stdout and optionnaly stderr streams.
returncode attribute
Raises:
CalledProcessError: if return code of cmd is not in accept_returncodes list
TimeoutExpired: if process is running for more than timeout time.
.. versionchanged:: 1.3.9
return_stderr parameters to disable stderr or get it in a separate list
return value has a returncode attribute to
.. versionchanged:: 1.4.0
output is not forced to unicode
.. versionchanged:: 1.4.1
error code 1603 is no longer accepted by default.
.. versionchanged:: 1.5.1
If cmd is unicode, encode it to default filesystem encoding before
running it.
>>> run(r'dir /B c:\windows\explorer.exe')
'explorer.exe\r\n'
>>> out = []
>>> pids = []
>>> def getlines(line):
... out.append(line)
>>> run(r'dir /B c:\windows\explorer.exe',pidlist=pids,on_write=getlines)
u'explorer.exe\r\n'
>>> print out
['explorer.exe\r\n']
>>> try:
... run(r'ping /t 127.0.0.1',timeout=3)
... except TimeoutExpired:
... print('timeout')
timeout
"""
logger.info(u'Run "%s"' % (ensure_unicode(cmd), ))
output = []
if return_stderr is None or return_stderr == False:
return_stderr = []
elif not isinstance(return_stderr, list):
return_stderr = output
if pidlist is None:
pidlist = []
# unicode cmd is not understood by shell system anyway...
if isinstance(cmd, unicode):
cmd = cmd.encode(sys.getfilesystemencoding())
try:
proc = psutil.Popen(cmd,
shell=shell,
bufsize=1,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
**kwargs)
except RuntimeError as e:
# be sure to not trigger encoding errors.
raise RuntimeError(e[0], repr(e[1]))
# keep track of launched pid if required by providing a pidlist argument to run
if not proc.pid in pidlist:
pidlist.append(proc.pid)
def worker(pipe, on_write=None):
while True:
line = pipe.readline()
if not line:
break
else:
if on_write:
on_write(ensure_unicode(line))
if pipe == proc.stderr:
return_stderr.append(line)
else:
output.append(line)
stdout_worker = RunReader(worker, proc.stdout, on_write)
stderr_worker = RunReader(worker, proc.stderr, on_write)
stdout_worker.start()
stderr_worker.start()
stdout_worker.join(timeout)
if stdout_worker.is_alive():
# kill the task and all subtasks
if proc.pid in pidlist:
pidlist.remove(proc.pid)
killtree(proc.pid)
raise TimeoutExpired(cmd, ''.join(output), timeout)
stderr_worker.join(timeout)
if stderr_worker.is_alive():
if proc.pid in pidlist:
pidlist.remove(proc.pid)
killtree(proc.pid)
raise TimeoutExpired(cmd, ''.join(output), timeout)
proc.returncode = proc.wait()
if proc.pid in pidlist:
pidlist.remove(proc.pid)
killtree(proc.pid)
if accept_returncodes is not None and not proc.returncode in accept_returncodes:
if return_stderr != output:
raise CalledProcessErrorOutput(proc.returncode, cmd,
''.join(output + return_stderr))
else:
raise CalledProcessErrorOutput(proc.returncode, cmd,
''.join(output))
else:
if proc.returncode == 0:
logger.info(u'%s command returns code %s' %
(ensure_unicode(cmd), proc.returncode))
else:
logger.warning(u'%s command returns code %s' %
(ensure_unicode(cmd), proc.returncode))
result = RunOutput(output)
result.returncode = proc.returncode
return result
def add_ads_groups(waptconfigfile,
hostdicts_list,
sign_certs=None,
sign_key=None,
key_password=None,
wapt_server_user=None,
wapt_server_passwd=None,
cabundle=None):
if sign_certs is None:
sign_bundle_fn = inifile_readstring(waptconfigfile, u'global',
u'personal_certificate_path')
sign_bundle = SSLCABundle(sign_bundle_fn)
sign_certs = sign_bundle.certificates()
# we assume a unique signer.
if cabundle is None:
cabundle = sign_bundle
if not sign_certs:
raise Exception(u'No personal signer certificate found in %s' %
sign_bundle_fn)
if sign_key is None:
sign_key = sign_certs[0].matching_key_in_dirs(
private_key_password=key_password)
try:
import waptconsole
progress_hook = waptconsole.UpdateProgress
private_key_password_callback = waptconsole.GetPrivateKeyPassword
except ImportError as e:
def print_progress(show=False, n=0, max=100, msg=''):
if show:
print('%s %s/%s\r' % (msg, n, max), end='')
else:
if not msg:
msg = 'Done'
print("%s%s" % (msg, ' ' * (80 - len(msg))))
progress_hook = print_progress
private_key_password_callback = None
main_repo = WaptRemoteRepo(name='wapt', cabundle=cabundle)
main_repo.load_config_from_file(waptconfigfile)
main_repo.private_key_password_callback = private_key_password_callback
host_repo = WaptHostRepo(name='wapt-host',
host_id=[h['uuid'] for h in hostdicts_list],
cabundle=cabundle)
host_repo.load_config_from_file(waptconfigfile)
host_repo.private_key_password_callback = private_key_password_callback
total_hosts = len(host_repo.packages())
discarded_uuids = [p.package for p in host_repo.discarded]
packages = []
discarded = []
unchanged = []
try:
progress_hook(True, 0, len(hostdicts_list),
'Editing %s hosts' % len(hostdicts_list))
i = 0
for h in hostdicts_list:
try:
host_package = None
host_id = h['uuid']
hostname = h['computer_name']
groups = get_computer_groups(hostname)
host_package = host_repo.get(
host_id, PackageEntry(package=host_id, section='host'))
if progress_hook(True, i, len(hostdicts_list),
'Checking %s' % host_package.package):
break
wapt_groups = ensure_list(host_package['depends'])
additional = [
group for group in groups if not group in wapt_groups and (
main_repo.get(group, None) is not None)
]
if additional:
logger.info(u'Adding %s to %s' %
(','.join(additional), host_package.package))
if progress_hook(True, i, len(hostdicts_list),
'Editing %s' % host_package.package):
break
wapt_groups.extend(additional)
host_package.depends = ','.join(wapt_groups)
host_package.build_management_package()
host_package.inc_build()
host_file = host_package.build_management_package()
host_package.sign_package(sign_certs, sign_key)
packages.append(host_package)
else:
unchanged.append(host_package.package)
except Exception as e:
if host_package:
discarded.append(host_package.package)
logger.critical(u'Discarding because %s' % ensure_unicode(e))
# upload all in one step...
progress_hook(True, 3, 3, 'Upload %s host packages' % len(packages))
server = WaptServer().load_config_from_file(waptconfigfile)
server.private_key_password_callback = private_key_password_callback
server.upload_packages(packages,
auth=(wapt_server_user, wapt_server_passwd),
progress_hook=progress_hook)
return dict(updated=packages, discarded=discarded, unchanged=unchanged)
finally:
logger.debug('Cleanup')
try:
i = 0
for s in packages:
i += 1
progress_hook(True, i, len(packages), 'Cleanup')
if os.path.isfile(s.localpath):
os.remove(s.localpath)
progress_hook(False)
except WindowsError as e:
logger.critical('Unable to remove temporary directory %s: %s' %
(s, repr(e)))
progress_hook(False)
def main():
parser=OptionParser(usage=__doc__,prog = 'wapt-signpackage')
parser.add_option("-c","--certificate", dest="public_key", default='', help="Path to the PEM RSA certificate to embed identitiy in control. (default: %default)")
parser.add_option("-k","--private-key", dest="private_key", default='', help="Path to the PEM RSA private key to sign packages. (default: %default)")
#parser.add_option("-w","--private-key-passwd", dest="private_key_passwd", default='', help="Path to the password of the private key. (default: %default)")
parser.add_option("-l","--loglevel", dest="loglevel", default=None, type='choice', choices=['debug','warning','info','error','critical'], metavar='LOGLEVEL',help="Loglevel (default: warning)")
parser.add_option("-m","--message-digest", dest="md", default='sha256', help="Message digest type for signatures. (default: %default)")
parser.add_option("-s","--scan-packages", dest="doscan", default=False, action='store_true', help="Rescan packages and update local Packages index after signing. (default: %default)")
parser.add_option("-r","--remove-setup", dest="removesetup", default=False, action='store_true', help="Remove setup.py. (default: %default)")
parser.add_option("-i","--inc-release", dest="increlease", default=False, action='store_true', help="Increase release number when building package (default: %default)")
parser.add_option("--maturity", dest="set_maturity", default=None, help="Set/change package maturity when signing package. (default: None)")
parser.add_option( "--keep-signature-date", dest="keep_signature_date",default=False, action='store_true', help="Keep the current package signature date, and file changetime (default: %default)")
parser.add_option( "--if-needed", dest="if_needed", default=False, action='store_true',help="Re-sign package only if needed (default: warning)")
(options,args) = parser.parse_args()
loglevel = options.loglevel
if len(logger.handlers) < 1:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(logging.Formatter(
u'%(asctime)s %(levelname)s %(message)s'))
logger.addHandler(hdlr)
if loglevel:
setloglevel(logger,loglevel)
else:
setloglevel(logger,'warning')
if len(args) < 1:
print(parser.usage)
sys.exit(1)
if not options.public_key and not options.private_key:
print('ERROR: No certificate found or specified')
sys.exit(1)
if options.private_key and os.path.isfile(options.private_key):
key = SSLPrivateKey(options.private_key)
else:
cert = SSLCertificate(options.public_key or options.private_key)
key = cert.matching_key_in_dirs()
if not key:
print('ERROR: No private key found or specified')
sys.exit(1)
args = ensure_list(args)
ca_bundle = SSLCABundle()
signers_bundle = SSLCABundle()
signers_bundle.add_certificates_from_pem(pem_filename=options.public_key)
waptpackages = []
for arg in args:
waptpackages.extend(glob.glob(arg))
errors = []
package_dirs = []
for waptpackage in waptpackages:
package_dir = os.path.abspath(os.path.dirname(waptpackage))
if not package_dir in package_dirs:
package_dirs.append(package_dir)
print('Processing %s'%waptpackage)
try:
sign_needed=False
pe = PackageEntry(waptfile = waptpackage)
if options.removesetup:
if pe.has_file('setup.py'):
with pe.as_zipfile(mode='a') as waptfile:
waptfile.remove('setup.py')
sign_needed=True
if not sign_needed and options.if_needed:
try:
pe.check_control_signature(trusted_bundle=signers_bundle,signers_bundle=signers_bundle)
for md in ensure_list(options.md):
if not pe.has_file(pe.get_signature_filename(md)):
raise Exception('Missing signature for md %s' % md)
logger.info('Skipping %s, already signed properly' % pe.asrequirement())
sign_needed = False
except Exception as e:
logger.info('Sign is needed for %s because %s' % (pe.asrequirement(),e))
sign_needed = True
if options.increlease:
pe.inc_build()
sign_needed = True
if options.set_maturity is not None and pe.maturity != options.set_maturity:
pe.maturity = options.set_maturity
sign_needed = True
if not options.if_needed or sign_needed:
pe.sign_package(private_key=key,certificate = signers_bundle.certificates(),mds = ensure_list(options.md),keep_signature_date=options.keep_signature_date)
newfn = pe.make_package_filename()
if newfn != pe.filename:
newfn_path = os.path.join(package_dir,newfn)
if not os.path.isfile(newfn_path):
print(u"Renaming file from %s to %s to match new package's properties" % (pe.filename,newfn))
shutil.move(os.path.join(package_dir,pe.filename),newfn_path)
else:
print('WARNING: unable to rename file from %s to %s because target already exists' % (pe.filename,newfn))
print('Done')
except Exception as e:
print(u'Error: %s'%ensure_unicode(e.message))
errors.append([waptpackage,repr(e)])
if options.doscan:
for package_dir in package_dirs:
if os.path.isfile(os.path.join(package_dir,'Packages')):
print(u'Launching the update of Packages index in %s ...'% ensure_unicode(package_dir))
repo = WaptLocalRepo(package_dir)
repo.update_packages_index()
print('Done')
else:
print("Don't forget to rescan your repository with wapt-scanpackages %s" % os.path.dirname(waptpackages[0]))
if errors:
print('Package not processed properly: ')
for fn,error in errors:
print(u'%s : %s' % (fn,error))
sys.exit(1)
else:
sys.exit(0)