def change_key_password(private_key_path,
old_password=None,
new_password=None):
"""Re-encrypt the PEM formated private RSA key with new_password
One must have file create rights on the directory of private_key.
Args:
private_key_path (str): FilePath to the PEM encoded RSA Key to (re)encrypt
old_password (str): Old password to decrypt the key
new_password (str): New password to encrypt the key
Returns:
str: private key path
"""
if not os.path.isfile(private_key_path):
raise Exception(u'The private key %s does not exists' %
private_key_path)
key = SSLPrivateKey(filename=private_key_path, password=old_password)
if os.path.isfile(private_key_path + '.backup'):
raise Exception(
u'Backup file %s already exists, previous password change has failed.'
% private_key_path + '.backup')
shutil.copyfile(private_key_path, private_key_path + '.backup')
try:
key.save_as_pem(filename=private_key_path, password=new_password)
os.unlink(private_key_path + '.backup')
return private_key_path
except Exception as e:
logger.critical(u'Unable to change key password: %s' % e)
shutil.copyfile(private_key_path + '.backup', private_key_path)
os.unlink(private_key_path + '.backup')
raise
def change_key_password(private_key_path,old_password=None,new_password=None):
"""Re-encrypt the PEM formated private RSA key with new_password
One must have file create rights on the directory of private_key.
Args:
private_key_path (str): FilePath to the PEM encoded RSA Key to (re)encrypt
old_password (str): Old password to decrypt the key
new_password (str): New password to encrypt the key
Returns:
str: private key path
"""
if not os.path.isfile(private_key_path):
raise Exception(u'The private key %s does not exists' % private_key_path)
key = SSLPrivateKey(filename = private_key_path,password=old_password)
if os.path.isfile(private_key_path+'.backup'):
raise Exception(u'Backup file %s already exists, previous password change has failed.' % private_key_path+'.backup')
shutil.copyfile(private_key_path,private_key_path+'.backup')
try:
key.save_as_pem(filename = private_key_path,password = new_password)
os.unlink(private_key_path+'.backup')
return private_key_path
except Exception as e:
logger.critical(u'Unable to change key password: %s' % e)
shutil.copyfile(private_key_path+'.backup',private_key_path)
os.unlink(private_key_path+'.backup')
raise
def process_tags(self, tags):
if type(tags) is str:
self.tags = {tags}
elif type(tags) is list:
self.tags = set(tags)
else:
logger.critical('Unsupport tag format provided: {0}'.format(tags))
def run(self):
# get task detail from URL
self.get_task()
if self.task is None:
logger.critical("Can't do much without task detail. exitting.")
return 1
self.mark_task_started()
# prepare sub folder
self.setup_workdir()
# start our DNS cache
self.start_dnscache()
# start scraper
self.start_scraper()
self.mark_scraper_started()
self.submit_scraper_progress()
last_check = datetime.datetime.now()
while not self.should_stop and self.container_running("scraper"):
now = datetime.datetime.now()
if (now - last_check).total_seconds() < SLEEP_INTERVAL:
self.sleep()
continue
last_check = now
self.submit_scraper_progress()
self.handle_files()
# scraper is done. check files so upload can continue
self.handle_stopped_scraper()
self.handle_files() # rescan folder
# monitor upload/check of files
while not self.should_stop and (
self.busy_zim_files or self.container_running("uploader")
or self.container_running("checker")
or self.container_running("log_uploader")):
now = datetime.datetime.now()
if (now - last_check).total_seconds() < SLEEP_INTERVAL:
self.sleep()
continue
last_check = now
self.handle_files()
self.check_scraper_log_upload()
# make sure we submit upload status for last zim and scraper log
self.handle_files()
self.check_scraper_log_upload()
# done with processing, cleaning-up and exiting
self.shutdown("succeeded" if self.scraper_succeeded else "failed")
def check_private_key(self):
logger.info(f"testing private key at {PRIVATE_KEY}…")
if (
not PRIVATE_KEY.exists()
or not PRIVATE_KEY.is_file()
or not os.access(PRIVATE_KEY, os.R_OK)
):
logger.critical(f"\tprivate key is not a readable path")
sys.exit(1)
else:
logger.info("\tprivate key is available and readable")
def __init__(self, **kwargs):
# include our class config values in the config print
kwargs.update({k: getattr(self, k) for k in self.config_keys})
kwargs.update({"OFFLINERS": SUPPORTED_OFFLINERS})
kwargs.update({"PLATFORMS_TASKS": PLATFORMS_TASKS})
self.print_config(**kwargs)
# set data holders
self.tasks = {}
self.last_poll = datetime.datetime(2020, 1, 1)
self.should_stop = False
# check workdir
self.check_workdir()
# check SSH private key
self.check_private_key()
# ensure we have valid credentials
self.check_auth()
# ensure we have access to docker API
self.check_docker()
# display resources
host_stats = query_host_stats(self.docker, self.workdir)
logger.info(
"Host hardware resources:"
"\n\tCPU : {cpu_total} (total) ; {cpu_avail} (avail)"
"\n\tRAM : {mem_total} (total) ; {mem_avail} (avail)"
"\n\tDisk: {disk_total} (configured) ; {disk_avail} (avail) ; "
"{disk_used} (reserved) ; {disk_remain} (remain)".format(
mem_total=format_size(host_stats["memory"]["total"]),
mem_avail=format_size(host_stats["memory"]["available"]),
cpu_total=host_stats["cpu"]["total"],
cpu_avail=host_stats["cpu"]["available"],
disk_avail=format_size(host_stats["disk"]["available"]),
disk_used=format_size(host_stats["disk"]["used"]),
disk_remain=format_size(host_stats["disk"]["remaining"]),
disk_total=format_size(host_stats["disk"]["total"]),
))
if host_stats["disk"]["available"] < host_stats["disk"]["remaining"]:
self.should_stop = True
logger.critical("Configured disk space is not available. Exiting.")
return
self.check_in()
# register stop/^C
self.register_signals()
self.sync_tasks_and_containers()
def check_workdir(self):
self.workdir = pathlib.Path(self.workdir).resolve()
logger.info(f"testing workdir at {self.workdir}…")
if (
not self.workdir.exists()
or not self.workdir.is_dir()
or not os.access(self.workdir, os.W_OK)
):
logger.critical(f"\tworkdir is not a writable path")
sys.exit(1)
else:
logger.info("\tworkdir is available and writable")
def check_auth(self):
self.access_token = self.refresh_token = self.token_payload = None
self.authenticated_on = datetime.datetime(2019, 1, 1)
self.authentication_expires_on = datetime.datetime(2019, 1, 1)
logger.info(f"testing authentication with {self.webapi_uri}…")
success, _, _ = self.query_api("GET", "/auth/test")
if success:
logger.info("\tauthentication successful")
else:
logger.critical("\tauthentication failed.")
sys.exit(1)
def init(self, config):
if type(config) is dict:
if {'rule', 'list'}.issubset(config):
if config['rule'] in ['ANY', 'ALL']:
self.rule = config['rule']
else:
logger.critical('Invalid tag rule provided: {0}'.format(config['rule']))
self.process_tags(config['list'])
else:
logger.critical('Invalid option set provided: {0}'.format(config))
else:
self.rule = 'ANY'
self.process_tags(config)
def poll(self, task_id=None):
self.check_cancellation() # update our tasks register
logger.debug("polling…")
self.last_poll = datetime.datetime.now()
host_stats = query_host_stats(self.docker, self.workdir)
expected_disk_avail = as_pos_int(host_stats["disk"]["total"] -
host_stats["disk"]["used"])
if host_stats["disk"]["available"] < expected_disk_avail:
self.should_stop = True
logger.critical(
f"Available disk space ({format_size(host_stats['disk']['available'])}) is lower than expected ({format_size(expected_disk_avail)}). Exiting."
)
return
success, status_code, response = self.query_api(
"GET",
"/requested-tasks/worker",
params={
"worker": self.worker_name,
"avail_cpu": host_stats["cpu"]["available"],
"avail_memory": host_stats["memory"]["available"],
"avail_disk": host_stats["disk"]["available"],
},
)
if not success:
logger.warning(f"poll failed with HTTP {status_code}: {response}")
return
if self.selfish:
response["items"] = [
t for t in response["items"] if t["worker"] == self.worker_name
]
if response["items"]:
logger.info("API is offering {nb} task(s): {ids}".format(
nb=len(response["items"]),
ids=[task["_id"] for task in response["items"]],
))
self.start_task(response["items"].pop())
# we need to allow the task to start, its container to start and
# eventually its scraper to start so docker can report to us
# the assigned resources (on the scraper) _before_ polling again
self.last_poll = datetime.datetime.now() + datetime.timedelta(
seconds=90)
def change_key_password(private_key_path,
old_password=None,
new_password=None):
if not os.path.isfile(private_key_path):
raise Exception(u'The private key %s does not exists' %
private_key_path)
key = SSLPrivateKey(filename=private_key_path, password=old_password)
if os.path.isfile(private_key_path + '.backup'):
raise Exception(
u'Backup file %s already exists, previous password change has failed.'
% private_key_path + '.backup')
shutil.copyfile(private_key_path, private_key_path + '.backup')
try:
key.save_as_pem(filename=private_key_path, password=new_password)
os.unlink(private_key_path + '.backup')
return private_key_path
except Exception as e:
logger.critical(u'Unable to change key password: %s' % e)
shutil.copyfile(private_key_path + '.backup', private_key_path)
os.unlink(private_key_path + '.backup')
raise
def check_docker(self):
logger.info(f"testing docker API on {DOCKER_SOCKET}…")
if (
not DOCKER_SOCKET.exists()
or not DOCKER_SOCKET.is_socket()
or not os.access(DOCKER_SOCKET, os.R_OK)
):
logger.critical(f"\tsocket ({DOCKER_SOCKET}) not available.")
sys.exit(1)
self.docker = docker.DockerClient(
base_url=f"unix://{DOCKER_SOCKET}", timeout=DOCKER_CLIENT_TIMEOUT
)
try:
if len(self.docker.containers.list(all=False)) < 1:
logger.warning("\tno running container, am I out-of-docker?")
except Exception as exc:
logger.critical("\tdocker API access failed: exiting.")
logger.exception(exc)
sys.exit(1)
else:
logger.info("\tdocker API access successful")
def init(self, config):
# If this is an integer, make it a length 1 list
if type(config) is int:
self.weights = [config]
# If this is a list, use it as is
elif type(config) is list:
# Make sure all the items are integers first
if not all([type(x) is int for x in config]):
logger.critical(
'Malformed weight list provided: {0}'.format(config)
)
else:
self.weights = config
# If it's a dict, make a range from the key to the value
elif type(config) is dict and len(config) == 1:
(lower, upper) = config.popitem()
# Add 1 to account for Python's range bounding
self.weights = range(lower, upper+1)
else:
logger.critical(
'Malformed weight options provided: {0}'.format(config)
)
logger.debug('Using weight list: {0}'.format(self.weights))
from monitor import *
from redis_monitor import measure
logger.debug('RedisMonitor starting...')
optlist, args = getopt.gnu_getopt(sys.argv[1:], 'd:', ['duration=']) #duration of sending measured info [min]
for opt, value in optlist:
if opt == "-d" or opt == "--duration":
constants.DURATION = int(value)
logger.info("Duration was redefined: "+str(constants.DURATION)+" min")
def handle_exception(exc_type, exc_value, exc_traceback):
if issubclass(exc_type, KeyboardInterrupt):
sys.__excepthook__(exc_type, exc_value, exc_traceback)
return
logger.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback))
sys.excepthook = handle_exception
custom = Monitor(constants.MONITOR_ID)
if not custom.prepareMonitor():
logger.critical("Cannot obtain/create monitor:(")
sys.exit(1)
logger.info("Starting loop...")
while 1 :
time.sleep(int(constants.DURATION*60))
data=measure()
custom.send(data)
logger.info('FINISHED')
def edit_hosts_depends(waptconfigfile,hosts_list,
append_depends=[],
remove_depends=[],
append_conflicts=[],
remove_conflicts=[],
sign_certs=None,
sign_key=None,
key_password=None,
wapt_server_user=None,wapt_server_passwd=None,
cabundle = None,
):
"""Add or remove packages from host packages
Args:
Returns:
dict: { updated: of uuid of machines actually updated
unchanged : list of uuid skipped because of no change needed
discarded : list of uuid discarded due to errors}
>>> edit_hosts_depends('c:/wapt/wapt-get.ini','htlaptop.tranquilit.local','toto','tis-7zip','admin','password')
"""
if sign_certs is None:
sign_bundle_fn = inifile_readstring(waptconfigfile,u'global',u'personal_certificate_path')
sign_bundle = SSLCABundle(sign_bundle_fn)
sign_certs = sign_bundle.certificates()
# we assume a unique signer.
if cabundle is None:
cabundle = sign_bundle
if not sign_certs:
raise Exception(u'No personal signer certificate found in %s' % sign_bundle_fn)
if sign_key is None:
sign_key = sign_certs[0].matching_key_in_dirs(private_key_password=key_password)
try:
import waptconsole
progress_hook = waptconsole.UpdateProgress
except ImportError as e:
def print_progress(show=False,n=0,max=100,msg=''):
if show:
print('%s %s/%s\r' % (msg,n,max),end='')
else:
if not msg:
msg='Done'
print("%s%s"%(msg,' '*(80-len(msg))))
progress_hook = print_progress
hosts_list = ensure_list(hosts_list)
progress_hook(True,0,len(hosts_list),'Loading %s hosts packages' % len(hosts_list))
host_repo = WaptHostRepo(name='wapt-host',host_id=hosts_list,cabundle = cabundle)
host_repo.load_config_from_file(waptconfigfile)
total_hosts = len(host_repo.packages())
discarded_uuids = [p.package for p in host_repo.discarded]
hosts_list = ensure_list(hosts_list)
append_depends = ensure_list(append_depends)
remove_depends = ensure_list(remove_depends)
append_conflicts = ensure_list(append_conflicts)
remove_conflicts = ensure_list(remove_conflicts)
packages = []
discarded = []
unchanged = []
progress_hook(True,0,len(hosts_list),'Editing %s hosts' % len(hosts_list))
i = 0
try:
for host_id in hosts_list:
i+=1
# don't change discarded packages.
if host_id in discarded_uuids:
discarded.append(host_id)
else:
host = host_repo.get(host_id)
if host is None:
host = PackageEntry(package=host_id,section='host')
if progress_hook(True,i,len(hosts_list),'Editing %s' % host.package):
break
logger.debug(u'Edit host %s : +%s -%s'%(
host.package,
append_depends,
remove_depends))
depends = host.depends
conflicts = host.conflicts
conflicts = add_to_csv_list(conflicts,append_conflicts)
conflicts = remove_from_csv_list(conflicts,remove_conflicts)
depends = remove_from_csv_list(depends,ensure_list(conflicts))
depends = add_to_csv_list(depends,append_depends)
depends = remove_from_csv_list(depends,remove_depends)
conflicts = remove_from_csv_list(conflicts,ensure_list(depends))
if depends != host.depends or conflicts != host.conflicts:
host.depends = depends
host.conflicts = conflicts
host.inc_build()
host_file = host.build_management_package()
host.sign_package(sign_certs,sign_key)
packages.append(host)
else:
unchanged.append(host.package)
# upload all in one step...
progress_hook(True,3,3,'Upload %s host packages' % len(packages))
server = WaptServer().load_config_from_file(waptconfigfile)
server.upload_packages(packages,auth=(wapt_server_user,wapt_server_passwd),progress_hook=progress_hook)
return dict(updated = [p.package for p in packages],
discarded = discarded,
unchanged = unchanged)
finally:
logger.debug('Cleanup')
try:
i = 0
for s in packages:
i+=1
progress_hook(True,i,len(packages),'Cleanup')
if os.path.isfile(s.localpath):
os.remove(s.localpath)
progress_hook(False)
except WindowsError as e:
logger.critical('Unable to remove temporary directory %s: %s'% (s,repr(e)))
progress_hook(False)
def add_ads_groups(waptconfigfile,
hostdicts_list,
sign_certs=None,
sign_key=None,
key_password=None,
wapt_server_user=None,wapt_server_passwd=None,
cabundle = None):
if sign_certs is None:
sign_bundle_fn = inifile_readstring(waptconfigfile,u'global',u'personal_certificate_path')
sign_bundle = SSLCABundle(sign_bundle_fn)
sign_certs = sign_bundle.certificates()
# we assume a unique signer.
if cabundle is None:
cabundle = sign_bundle
if not sign_certs:
raise Exception(u'No personal signer certificate found in %s' % sign_bundle_fn)
if sign_key is None:
sign_key = sign_certs[0].matching_key_in_dirs(private_key_password=key_password)
main_repo = WaptRemoteRepo(name='wapt',cabundle = cabundle)
main_repo.load_config_from_file(waptconfigfile)
host_repo = WaptHostRepo(name='wapt-host',host_id=[h['uuid'] for h in hostdicts_list],cabundle = cabundle)
host_repo.load_config_from_file(waptconfigfile)
total_hosts = len(host_repo.packages())
discarded_uuids = [p.package for p in host_repo.discarded]
try:
import waptconsole
progress_hook = waptconsole.UpdateProgress
except ImportError as e:
def print_progress(show=False,n=0,max=100,msg=''):
if show:
print('%s %s/%s\r' % (msg,n,max),end='')
else:
if not msg:
msg='Done'
print("%s%s"%(msg,' '*(80-len(msg))))
progress_hook = print_progress
packages = []
discarded = []
unchanged = []
try:
progress_hook(True,0,len(hostdicts_list),'Editing %s hosts' % len(hostdicts_list))
i = 0
for h in hostdicts_list:
try:
host_package = None
host_id = h['uuid']
hostname = h['computer_name']
groups = get_computer_groups(hostname)
host_package = host_repo.get(host_id,PackageEntry(package=host_id,section='host'))
if progress_hook(True,i,len(hostdicts_list),'Checking %s' % host_package.package):
break
wapt_groups = ensure_list(host_package['depends'])
additional = [group for group in groups if not group in wapt_groups and (main_repo.get(group,None) is not None)]
if additional:
logger.info(u'Adding %s to %s' % (','.join(additional),host_package.package))
if progress_hook(True,i,len(hostdicts_list),'Editing %s' % host_package.package):
break
wapt_groups.extend(additional)
host_package.depends = ','.join(wapt_groups)
host_package.build_management_package()
host_package.inc_build()
host_file = host_package.build_management_package()
host_package.sign_package(sign_certs,sign_key)
packages.append(host_package)
else:
unchanged.append(host_package.package)
except Exception as e:
if host_package:
discarded.append(host_package.package)
logger.critical(u'Discarding because %s' % ensure_unicode(e))
# upload all in one step...
progress_hook(True,3,3,'Upload %s host packages' % len(packages))
server = WaptServer().load_config_from_file(waptconfigfile)
server.upload_packages(packages,auth=(wapt_server_user,wapt_server_passwd),progress_hook=progress_hook)
return dict(updated = packages,
discarded = discarded,
unchanged = unchanged)
finally:
logger.debug('Cleanup')
try:
i = 0
for s in packages:
i+=1
progress_hook(True,i,len(packages),'Cleanup')
if os.path.isfile(s.localpath):
os.remove(s.localpath)
progress_hook(False)
except WindowsError as e:
logger.critical('Unable to remove temporary directory %s: %s'% (s,repr(e)))
progress_hook(False)
def init(self):
self.location = os.path.expanduser(self.config['location'])
if not os.path.isfile(self.location):
logger.critical('Followfile does not exist: {0}'.format(self.location))
self.handle = open(self.location)
self.handle.seek(0, 2)
from common import logger
try:
import yaml
except ImportError:
logger.critical('Failed to load PyYAML')
class main(object):
def __init__(self, config):
pass
def run(self, data):
return yaml.load(data)
except:
logger.error('failed to unpack line {} in {}: "{}"'.format(n, info_file, line.strip()))
exit(0)
if (sym.upper() == symbol.upper()):
cname = name.strip()
if len(link.strip()) > 0:
explorer = link.strip()
if args['name'] is not None:
cname = args['name'][0]
if cname is not None:
logger.debug('NAME {}'.format(cname))
else:
logger.critical('could not determine currency name for symbol {}'.format(symbol))
exit(1)
# public address
address = args['address'][0]
logger.debug('ADDRESS {}'.format(address))
# private key
privkey = args['privkey'][0]
# block explorer
qr_link = None
if explorer is not None:
def measure():
''' Prepare the metrics pattern '''
prev = None
stats = {}
for s in (constants.RESULT_PARAMS).split(";") :
key = s.split(':',1)[0]
stats[key] = []
stats['additionalResults'] = []
# logger.info("Collected statistics: %s", json.dumps(stats))
p = subprocess.Popen("ps -ef | grep -v grep | grep -o 'redis-server.*:[0-9]*' | awk -F ':' '{print $2}'", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
lines = p.stdout.readlines()
if not isinstance(lines, list) or len(lines) == 0:
e = 'There is not any running instance of Redis.'
logger.critical(e)
stats['status'].append(constants.NOK_STATE)
details={'details':str(e)}
details = urllib.quote(json.dumps(details), '{}')
stats['additionalResults'].append([details])
else:
for line in lines:
st = {}
for key in stats:
st[key] = ''
st['additionalResults'] = []
port = line.strip()
st['port'] = str(port)
redis_info = ''
try:
r = getRedis(port)
if not isinstance(r, redis.Redis):
logger.error("Cannot establish connection to Redis")
else:
redis_info = r.info()
prev = r.get('rm_prev')
if (isinstance(prev, basestring)) :
prev = json.loads(prev)
except Exception as e :
logger.critical(e)
st['status'] = constants.NOK_STATE
details={'details':str(e)}
details = urllib.quote(json.dumps(details), '{}')
st['additionalResults'].append(details)
else :
if(constants.REDIS_DB in redis_info):
db_info = redis_info[constants.REDIS_DB]
for k in db_info:
if(k in stats):
st[k] = db_info[k]
pid = str(redis_info['process_id'])
cm = commands.getoutput('ps -p'+pid+' -o %cpu,%mem | grep -v % ')
rr = cm.strip().replace(' ',' ').split(' ', 2);
cpu_pr=float(rr[0])
mem_pr=float(rr[1])
''' Collect measured values '''
for k, v in st.iteritems():
if k == 'used_memory' :
v = mem_pr
elif k == 'used_cpu' :
v = cpu_pr
elif k == 'status' :
v = constants.OK_STATE
elif(k in redis_info) :
v = redis_info[k]
st[k] = v
r.set('rm_prev', json.dumps(st)) # keep as previous value
details={'details':' Redis-'+str(redis_info['redis_version'])+'-'+st['port']+' ('+str(redis_info['process_id']) +') '+' Uptime: '+_toDate(redis_info['uptime_in_seconds'])}
details = urllib.quote(json.dumps(details), '{}')
st['additionalResults'].append(details)
''' Convert to API required form '''
for k, v in st.iteritems():
if (k == 'expired_keys' or k == 'evicted_keys' or k == 'keyspace_misses' or k == 'total_connections_received') :
if (isinstance(prev, dict) and (k in prev)) :
v = max(v - prev[k], 0)
# v = v - prev[k]
else :
v = 0
stats[k].append(v)
return_value=json.dumps(stats).replace('],', '];').lstrip('{').rstrip('}').replace('"', '').replace(' ', '').replace('}];[{', '}],[{')
logger.info('Returns: '+str(return_value))
logger.info('Stats: '+str(stats))
return return_value
def add_ads_groups(waptconfigfile,
hostdicts_list,
sign_certs=None,
sign_key=None,
key_password=None,
wapt_server_user=None,
wapt_server_passwd=None,
cabundle=None):
if sign_certs is None:
sign_bundle_fn = inifile_readstring(waptconfigfile, u'global',
u'personal_certificate_path')
sign_bundle = SSLCABundle(sign_bundle_fn)
sign_certs = sign_bundle.certificates()
# we assume a unique signer.
if cabundle is None:
cabundle = sign_bundle
if not sign_certs:
raise Exception(u'No personal signer certificate found in %s' %
sign_bundle_fn)
if sign_key is None:
sign_key = sign_certs[0].matching_key_in_dirs(
private_key_password=key_password)
try:
import waptconsole
progress_hook = waptconsole.UpdateProgress
private_key_password_callback = waptconsole.GetPrivateKeyPassword
except ImportError as e:
def print_progress(show=False, n=0, max=100, msg=''):
if show:
print('%s %s/%s\r' % (msg, n, max), end='')
else:
if not msg:
msg = 'Done'
print("%s%s" % (msg, ' ' * (80 - len(msg))))
progress_hook = print_progress
private_key_password_callback = None
main_repo = WaptRemoteRepo(name='wapt', cabundle=cabundle)
main_repo.load_config_from_file(waptconfigfile)
main_repo.private_key_password_callback = private_key_password_callback
host_repo = WaptHostRepo(name='wapt-host',
host_id=[h['uuid'] for h in hostdicts_list],
cabundle=cabundle)
host_repo.load_config_from_file(waptconfigfile)
host_repo.private_key_password_callback = private_key_password_callback
total_hosts = len(host_repo.packages())
discarded_uuids = [p.package for p in host_repo.discarded]
packages = []
discarded = []
unchanged = []
try:
progress_hook(True, 0, len(hostdicts_list),
'Editing %s hosts' % len(hostdicts_list))
i = 0
for h in hostdicts_list:
try:
host_package = None
host_id = h['uuid']
hostname = h['computer_name']
groups = get_computer_groups(hostname)
host_package = host_repo.get(
host_id, PackageEntry(package=host_id, section='host'))
if progress_hook(True, i, len(hostdicts_list),
'Checking %s' % host_package.package):
break
wapt_groups = ensure_list(host_package['depends'])
additional = [
group for group in groups if not group in wapt_groups and (
main_repo.get(group, None) is not None)
]
if additional:
logger.info(u'Adding %s to %s' %
(','.join(additional), host_package.package))
if progress_hook(True, i, len(hostdicts_list),
'Editing %s' % host_package.package):
break
wapt_groups.extend(additional)
host_package.depends = ','.join(wapt_groups)
host_package.build_management_package()
host_package.inc_build()
host_file = host_package.build_management_package()
host_package.sign_package(sign_certs, sign_key)
packages.append(host_package)
else:
unchanged.append(host_package.package)
except Exception as e:
if host_package:
discarded.append(host_package.package)
logger.critical(u'Discarding because %s' % ensure_unicode(e))
# upload all in one step...
progress_hook(True, 3, 3, 'Upload %s host packages' % len(packages))
server = WaptServer().load_config_from_file(waptconfigfile)
server.private_key_password_callback = private_key_password_callback
server.upload_packages(packages,
auth=(wapt_server_user, wapt_server_passwd),
progress_hook=progress_hook)
return dict(updated=packages, discarded=discarded, unchanged=unchanged)
finally:
logger.debug('Cleanup')
try:
i = 0
for s in packages:
i += 1
progress_hook(True, i, len(packages), 'Cleanup')
if os.path.isfile(s.localpath):
os.remove(s.localpath)
progress_hook(False)
except WindowsError as e:
logger.critical('Unable to remove temporary directory %s: %s' %
(s, repr(e)))
progress_hook(False)
def edit_hosts_depends(
waptconfigfile,
hosts_list,
append_depends=[],
remove_depends=[],
append_conflicts=[],
remove_conflicts=[],
sign_certs=None,
sign_key=None,
key_password=None,
wapt_server_user=None,
wapt_server_passwd=None,
cabundle=None,
):
"""Add or remove packages from host packages
Args:
Returns:
dict: { updated: of uuid of machines actually updated
unchanged : list of uuid skipped because of no change needed
discarded : list of uuid discarded due to errors}
>>> edit_hosts_depends('c:/wapt/wapt-get.ini','htlaptop.tranquilit.local','toto','tis-7zip','admin','password')
"""
if sign_certs is None:
sign_bundle_fn = inifile_readstring(waptconfigfile, u'global',
u'personal_certificate_path')
sign_bundle = SSLCABundle(sign_bundle_fn)
sign_certs = sign_bundle.certificates()
# we assume a unique signer.
if cabundle is None:
cabundle = sign_bundle
if not sign_certs:
raise Exception(u'No personal signer certificate found in %s' %
sign_bundle_fn)
try:
import waptconsole
progress_hook = waptconsole.UpdateProgress
private_key_password_callback = waptconsole.GetPrivateKeyPassword
except ImportError as e:
def print_progress(show=False, n=0, max=100, msg=''):
if show:
print('%s %s/%s\r' % (msg, n, max), end='')
else:
if not msg:
msg = 'Done'
print("%s%s" % (msg, ' ' * (80 - len(msg))))
progress_hook = print_progress
private_key_password_callback = None
if sign_key is None:
sign_key = sign_certs[0].matching_key_in_dirs(
private_key_password=key_password,
password_callback=private_key_password_callback)
progress_hook(True, 0, len(hosts_list),
'Loading %s hosts packages' % len(hosts_list))
host_repo = WaptHostRepo(name='wapt-host',
host_id=[h['uuid'] for h in hosts_list],
cabundle=cabundle)
host_repo.private_key_password_callback = private_key_password_callback
host_repo.load_config_from_file(waptconfigfile)
total_hosts = len(host_repo.packages())
discarded_uuids = [p.package for p in host_repo.discarded]
append_depends = ensure_list(append_depends)
remove_depends = ensure_list(remove_depends)
append_conflicts = ensure_list(append_conflicts)
remove_conflicts = ensure_list(remove_conflicts)
packages = []
discarded = []
unchanged = []
progress_hook(True, 0, len(hosts_list),
'Editing %s hosts' % len(hosts_list))
i = 0
try:
for hostrec in hosts_list:
host_id = hostrec['uuid']
i += 1
# don't change discarded packages.
if host_id in discarded_uuids:
discarded.append(host_id)
else:
host = host_repo.get(host_id)
if host is None:
host = PackageEntry(package=host_id, section='host')
if progress_hook(True, i, len(hosts_list),
'Editing %s' % host.package):
break
logger.debug(u'Edit host %s : +%s -%s' %
(host.package, append_depends, remove_depends))
depends = host.depends
conflicts = host.conflicts
conflicts = add_to_csv_list(conflicts, append_conflicts)
conflicts = remove_from_csv_list(conflicts, remove_conflicts)
depends = remove_from_csv_list(depends, ensure_list(conflicts))
depends = add_to_csv_list(depends, append_depends)
depends = remove_from_csv_list(depends, remove_depends)
conflicts = remove_from_csv_list(conflicts,
ensure_list(depends))
if depends != host.depends or conflicts != host.conflicts:
host.depends = depends
host.conflicts = conflicts
host.inc_build()
host_file = host.build_management_package()
host.sign_package(sign_certs, sign_key)
packages.append(host)
else:
unchanged.append(host.package)
# upload all in one step...
progress_hook(True, 3, 3, 'Upload %s host packages' % len(packages))
server = WaptServer().load_config_from_file(waptconfigfile)
server.private_key_password_callback = private_key_password_callback
server.upload_packages(packages,
auth=(wapt_server_user, wapt_server_passwd),
progress_hook=progress_hook)
return dict(updated=[p.package for p in packages],
discarded=discarded,
unchanged=unchanged)
finally:
logger.debug('Cleanup')
try:
i = 0
for s in packages:
i += 1
progress_hook(True, i, len(packages), 'Cleanup')
if os.path.isfile(s.localpath):
os.remove(s.localpath)
progress_hook(False)
except WindowsError as e:
logger.critical('Unable to remove temporary directory %s: %s' %
(s, repr(e)))
progress_hook(False)
def main():
parser = argparse.ArgumentParser(prog=TASK_WORKER)
parser.add_argument(
"--task-id", help="task-id to attempt to process", required=True, dest="task_id"
)
parser.add_argument(
"--webapi-uri",
help="zimfarm API URI",
required=not bool(DEFAULT_WEB_API_URL),
default=DEFAULT_WEB_API_URL,
dest="webapi_uri",
)
parser.add_argument(
"--username",
help="username to authenticate to zimfarm",
required=not bool(os.getenv("USERNAME")),
default=os.getenv("USERNAME"),
)
parser.add_argument(
"--workdir",
help="directory in which workers create task-related files",
required=not bool(os.getenv("WORKDIR")),
default=os.getenv("WORKDIR"),
dest="workdir",
)
args = parser.parse_args()
logger.info(f"starting zimfarm {TASK_WORKER} for {args.task_id}.")
try:
worker = TaskWorker(
username=args.username,
webapi_uri=args.webapi_uri,
workdir=args.workdir,
task_id=args.task_id,
)
except Exception as exc:
logger.critical(f"Unhandled exception during init: {exc}")
sys.exit(1)
try:
sys.exit(worker.run())
except Exception as exc:
logger.error(f"Unhandled exception: {exc}")
logger.exception(exc)
logger.error("exiting.")
try:
logger.info("Trying to upload exception details")
tbe = traceback.TracebackException.from_exception(exc)
worker.shutdown(
"failed",
exception="".join(tbe.format_exception_only()),
traceback="".join(tbe.format()),
)
except Exception:
logger.error("Could not submit failure details")
sys.exit(1)
import os, sys, time, getopt
import os.path as path
from common import logger
import monitor_constants as constants
from monitor import *
optlist, args = getopt.gnu_getopt(sys.argv[1:], 'd:', ['duration='])
for opt, value in optlist:
if opt == "-d" or opt == "--duration":
constants.DURATION = int(value)
logger.info("Duration was redefined: "+str(constants.DURATION)+" min")
custom = Monitor(0)
if not custom.prepareMonitor():
logger.critical("Cannot obtain/create monitor:(")
sys.exit(1)
logger.info("Start watching loop...")
def getFileName(relPathToFile, defPath):
if relPathToFile[len(relPathToFile)-1] == '/' : # path to directory
folder = relPathToFile
basename = None
else : # path to file
folder = path.dirname(relPathToFile)
basename = path.basename(relPathToFile)
filename = None
cur = path.abspath('.')
while cur != '/' : #
tmp = path.normpath(path.join(cur, folder))
