def change_key_password(private_key_path, old_password=None, new_password=None): """Re-encrypt the PEM formated private RSA key with new_password One must have file create rights on the directory of private_key. Args: private_key_path (str): FilePath to the PEM encoded RSA Key to (re)encrypt old_password (str): Old password to decrypt the key new_password (str): New password to encrypt the key Returns: str: private key path """ if not os.path.isfile(private_key_path): raise Exception(u'The private key %s does not exists' % private_key_path) key = SSLPrivateKey(filename=private_key_path, password=old_password) if os.path.isfile(private_key_path + '.backup'): raise Exception( u'Backup file %s already exists, previous password change has failed.' % private_key_path + '.backup') shutil.copyfile(private_key_path, private_key_path + '.backup') try: key.save_as_pem(filename=private_key_path, password=new_password) os.unlink(private_key_path + '.backup') return private_key_path except Exception as e: logger.critical(u'Unable to change key password: %s' % e) shutil.copyfile(private_key_path + '.backup', private_key_path) os.unlink(private_key_path + '.backup') raise
def change_key_password(private_key_path,old_password=None,new_password=None): """Re-encrypt the PEM formated private RSA key with new_password One must have file create rights on the directory of private_key. Args: private_key_path (str): FilePath to the PEM encoded RSA Key to (re)encrypt old_password (str): Old password to decrypt the key new_password (str): New password to encrypt the key Returns: str: private key path """ if not os.path.isfile(private_key_path): raise Exception(u'The private key %s does not exists' % private_key_path) key = SSLPrivateKey(filename = private_key_path,password=old_password) if os.path.isfile(private_key_path+'.backup'): raise Exception(u'Backup file %s already exists, previous password change has failed.' % private_key_path+'.backup') shutil.copyfile(private_key_path,private_key_path+'.backup') try: key.save_as_pem(filename = private_key_path,password = new_password) os.unlink(private_key_path+'.backup') return private_key_path except Exception as e: logger.critical(u'Unable to change key password: %s' % e) shutil.copyfile(private_key_path+'.backup',private_key_path) os.unlink(private_key_path+'.backup') raise
def process_tags(self, tags): if type(tags) is str: self.tags = {tags} elif type(tags) is list: self.tags = set(tags) else: logger.critical('Unsupport tag format provided: {0}'.format(tags))
def run(self): # get task detail from URL self.get_task() if self.task is None: logger.critical("Can't do much without task detail. exitting.") return 1 self.mark_task_started() # prepare sub folder self.setup_workdir() # start our DNS cache self.start_dnscache() # start scraper self.start_scraper() self.mark_scraper_started() self.submit_scraper_progress() last_check = datetime.datetime.now() while not self.should_stop and self.container_running("scraper"): now = datetime.datetime.now() if (now - last_check).total_seconds() < SLEEP_INTERVAL: self.sleep() continue last_check = now self.submit_scraper_progress() self.handle_files() # scraper is done. check files so upload can continue self.handle_stopped_scraper() self.handle_files() # rescan folder # monitor upload/check of files while not self.should_stop and ( self.busy_zim_files or self.container_running("uploader") or self.container_running("checker") or self.container_running("log_uploader")): now = datetime.datetime.now() if (now - last_check).total_seconds() < SLEEP_INTERVAL: self.sleep() continue last_check = now self.handle_files() self.check_scraper_log_upload() # make sure we submit upload status for last zim and scraper log self.handle_files() self.check_scraper_log_upload() # done with processing, cleaning-up and exiting self.shutdown("succeeded" if self.scraper_succeeded else "failed")
def check_private_key(self): logger.info(f"testing private key at {PRIVATE_KEY}…") if ( not PRIVATE_KEY.exists() or not PRIVATE_KEY.is_file() or not os.access(PRIVATE_KEY, os.R_OK) ): logger.critical(f"\tprivate key is not a readable path") sys.exit(1) else: logger.info("\tprivate key is available and readable")
def __init__(self, **kwargs): # include our class config values in the config print kwargs.update({k: getattr(self, k) for k in self.config_keys}) kwargs.update({"OFFLINERS": SUPPORTED_OFFLINERS}) kwargs.update({"PLATFORMS_TASKS": PLATFORMS_TASKS}) self.print_config(**kwargs) # set data holders self.tasks = {} self.last_poll = datetime.datetime(2020, 1, 1) self.should_stop = False # check workdir self.check_workdir() # check SSH private key self.check_private_key() # ensure we have valid credentials self.check_auth() # ensure we have access to docker API self.check_docker() # display resources host_stats = query_host_stats(self.docker, self.workdir) logger.info( "Host hardware resources:" "\n\tCPU : {cpu_total} (total) ; {cpu_avail} (avail)" "\n\tRAM : {mem_total} (total) ; {mem_avail} (avail)" "\n\tDisk: {disk_total} (configured) ; {disk_avail} (avail) ; " "{disk_used} (reserved) ; {disk_remain} (remain)".format( mem_total=format_size(host_stats["memory"]["total"]), mem_avail=format_size(host_stats["memory"]["available"]), cpu_total=host_stats["cpu"]["total"], cpu_avail=host_stats["cpu"]["available"], disk_avail=format_size(host_stats["disk"]["available"]), disk_used=format_size(host_stats["disk"]["used"]), disk_remain=format_size(host_stats["disk"]["remaining"]), disk_total=format_size(host_stats["disk"]["total"]), )) if host_stats["disk"]["available"] < host_stats["disk"]["remaining"]: self.should_stop = True logger.critical("Configured disk space is not available. Exiting.") return self.check_in() # register stop/^C self.register_signals() self.sync_tasks_and_containers()
def check_workdir(self): self.workdir = pathlib.Path(self.workdir).resolve() logger.info(f"testing workdir at {self.workdir}…") if ( not self.workdir.exists() or not self.workdir.is_dir() or not os.access(self.workdir, os.W_OK) ): logger.critical(f"\tworkdir is not a writable path") sys.exit(1) else: logger.info("\tworkdir is available and writable")
def check_auth(self): self.access_token = self.refresh_token = self.token_payload = None self.authenticated_on = datetime.datetime(2019, 1, 1) self.authentication_expires_on = datetime.datetime(2019, 1, 1) logger.info(f"testing authentication with {self.webapi_uri}…") success, _, _ = self.query_api("GET", "/auth/test") if success: logger.info("\tauthentication successful") else: logger.critical("\tauthentication failed.") sys.exit(1)
def init(self, config): if type(config) is dict: if {'rule', 'list'}.issubset(config): if config['rule'] in ['ANY', 'ALL']: self.rule = config['rule'] else: logger.critical('Invalid tag rule provided: {0}'.format(config['rule'])) self.process_tags(config['list']) else: logger.critical('Invalid option set provided: {0}'.format(config)) else: self.rule = 'ANY' self.process_tags(config)
def poll(self, task_id=None): self.check_cancellation() # update our tasks register logger.debug("polling…") self.last_poll = datetime.datetime.now() host_stats = query_host_stats(self.docker, self.workdir) expected_disk_avail = as_pos_int(host_stats["disk"]["total"] - host_stats["disk"]["used"]) if host_stats["disk"]["available"] < expected_disk_avail: self.should_stop = True logger.critical( f"Available disk space ({format_size(host_stats['disk']['available'])}) is lower than expected ({format_size(expected_disk_avail)}). Exiting." ) return success, status_code, response = self.query_api( "GET", "/requested-tasks/worker", params={ "worker": self.worker_name, "avail_cpu": host_stats["cpu"]["available"], "avail_memory": host_stats["memory"]["available"], "avail_disk": host_stats["disk"]["available"], }, ) if not success: logger.warning(f"poll failed with HTTP {status_code}: {response}") return if self.selfish: response["items"] = [ t for t in response["items"] if t["worker"] == self.worker_name ] if response["items"]: logger.info("API is offering {nb} task(s): {ids}".format( nb=len(response["items"]), ids=[task["_id"] for task in response["items"]], )) self.start_task(response["items"].pop()) # we need to allow the task to start, its container to start and # eventually its scraper to start so docker can report to us # the assigned resources (on the scraper) _before_ polling again self.last_poll = datetime.datetime.now() + datetime.timedelta( seconds=90)
def change_key_password(private_key_path, old_password=None, new_password=None): if not os.path.isfile(private_key_path): raise Exception(u'The private key %s does not exists' % private_key_path) key = SSLPrivateKey(filename=private_key_path, password=old_password) if os.path.isfile(private_key_path + '.backup'): raise Exception( u'Backup file %s already exists, previous password change has failed.' % private_key_path + '.backup') shutil.copyfile(private_key_path, private_key_path + '.backup') try: key.save_as_pem(filename=private_key_path, password=new_password) os.unlink(private_key_path + '.backup') return private_key_path except Exception as e: logger.critical(u'Unable to change key password: %s' % e) shutil.copyfile(private_key_path + '.backup', private_key_path) os.unlink(private_key_path + '.backup') raise
def check_docker(self): logger.info(f"testing docker API on {DOCKER_SOCKET}…") if ( not DOCKER_SOCKET.exists() or not DOCKER_SOCKET.is_socket() or not os.access(DOCKER_SOCKET, os.R_OK) ): logger.critical(f"\tsocket ({DOCKER_SOCKET}) not available.") sys.exit(1) self.docker = docker.DockerClient( base_url=f"unix://{DOCKER_SOCKET}", timeout=DOCKER_CLIENT_TIMEOUT ) try: if len(self.docker.containers.list(all=False)) < 1: logger.warning("\tno running container, am I out-of-docker?") except Exception as exc: logger.critical("\tdocker API access failed: exiting.") logger.exception(exc) sys.exit(1) else: logger.info("\tdocker API access successful")
def init(self, config): # If this is an integer, make it a length 1 list if type(config) is int: self.weights = [config] # If this is a list, use it as is elif type(config) is list: # Make sure all the items are integers first if not all([type(x) is int for x in config]): logger.critical( 'Malformed weight list provided: {0}'.format(config) ) else: self.weights = config # If it's a dict, make a range from the key to the value elif type(config) is dict and len(config) == 1: (lower, upper) = config.popitem() # Add 1 to account for Python's range bounding self.weights = range(lower, upper+1) else: logger.critical( 'Malformed weight options provided: {0}'.format(config) ) logger.debug('Using weight list: {0}'.format(self.weights))
from monitor import * from redis_monitor import measure logger.debug('RedisMonitor starting...') optlist, args = getopt.gnu_getopt(sys.argv[1:], 'd:', ['duration=']) #duration of sending measured info [min] for opt, value in optlist: if opt == "-d" or opt == "--duration": constants.DURATION = int(value) logger.info("Duration was redefined: "+str(constants.DURATION)+" min") def handle_exception(exc_type, exc_value, exc_traceback): if issubclass(exc_type, KeyboardInterrupt): sys.__excepthook__(exc_type, exc_value, exc_traceback) return logger.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback)) sys.excepthook = handle_exception custom = Monitor(constants.MONITOR_ID) if not custom.prepareMonitor(): logger.critical("Cannot obtain/create monitor:(") sys.exit(1) logger.info("Starting loop...") while 1 : time.sleep(int(constants.DURATION*60)) data=measure() custom.send(data) logger.info('FINISHED')
def edit_hosts_depends(waptconfigfile,hosts_list, append_depends=[], remove_depends=[], append_conflicts=[], remove_conflicts=[], sign_certs=None, sign_key=None, key_password=None, wapt_server_user=None,wapt_server_passwd=None, cabundle = None, ): """Add or remove packages from host packages Args: Returns: dict: { updated: of uuid of machines actually updated unchanged : list of uuid skipped because of no change needed discarded : list of uuid discarded due to errors} >>> edit_hosts_depends('c:/wapt/wapt-get.ini','htlaptop.tranquilit.local','toto','tis-7zip','admin','password') """ if sign_certs is None: sign_bundle_fn = inifile_readstring(waptconfigfile,u'global',u'personal_certificate_path') sign_bundle = SSLCABundle(sign_bundle_fn) sign_certs = sign_bundle.certificates() # we assume a unique signer. if cabundle is None: cabundle = sign_bundle if not sign_certs: raise Exception(u'No personal signer certificate found in %s' % sign_bundle_fn) if sign_key is None: sign_key = sign_certs[0].matching_key_in_dirs(private_key_password=key_password) try: import waptconsole progress_hook = waptconsole.UpdateProgress except ImportError as e: def print_progress(show=False,n=0,max=100,msg=''): if show: print('%s %s/%s\r' % (msg,n,max),end='') else: if not msg: msg='Done' print("%s%s"%(msg,' '*(80-len(msg)))) progress_hook = print_progress hosts_list = ensure_list(hosts_list) progress_hook(True,0,len(hosts_list),'Loading %s hosts packages' % len(hosts_list)) host_repo = WaptHostRepo(name='wapt-host',host_id=hosts_list,cabundle = cabundle) host_repo.load_config_from_file(waptconfigfile) total_hosts = len(host_repo.packages()) discarded_uuids = [p.package for p in host_repo.discarded] hosts_list = ensure_list(hosts_list) append_depends = ensure_list(append_depends) remove_depends = ensure_list(remove_depends) append_conflicts = ensure_list(append_conflicts) remove_conflicts = ensure_list(remove_conflicts) packages = [] discarded = [] unchanged = [] progress_hook(True,0,len(hosts_list),'Editing %s hosts' % len(hosts_list)) i = 0 try: for host_id in hosts_list: i+=1 # don't change discarded packages. if host_id in discarded_uuids: discarded.append(host_id) else: host = host_repo.get(host_id) if host is None: host = PackageEntry(package=host_id,section='host') if progress_hook(True,i,len(hosts_list),'Editing %s' % host.package): break logger.debug(u'Edit host %s : +%s -%s'%( host.package, append_depends, remove_depends)) depends = host.depends conflicts = host.conflicts conflicts = add_to_csv_list(conflicts,append_conflicts) conflicts = remove_from_csv_list(conflicts,remove_conflicts) depends = remove_from_csv_list(depends,ensure_list(conflicts)) depends = add_to_csv_list(depends,append_depends) depends = remove_from_csv_list(depends,remove_depends) conflicts = remove_from_csv_list(conflicts,ensure_list(depends)) if depends != host.depends or conflicts != host.conflicts: host.depends = depends host.conflicts = conflicts host.inc_build() host_file = host.build_management_package() host.sign_package(sign_certs,sign_key) packages.append(host) else: unchanged.append(host.package) # upload all in one step... progress_hook(True,3,3,'Upload %s host packages' % len(packages)) server = WaptServer().load_config_from_file(waptconfigfile) server.upload_packages(packages,auth=(wapt_server_user,wapt_server_passwd),progress_hook=progress_hook) return dict(updated = [p.package for p in packages], discarded = discarded, unchanged = unchanged) finally: logger.debug('Cleanup') try: i = 0 for s in packages: i+=1 progress_hook(True,i,len(packages),'Cleanup') if os.path.isfile(s.localpath): os.remove(s.localpath) progress_hook(False) except WindowsError as e: logger.critical('Unable to remove temporary directory %s: %s'% (s,repr(e))) progress_hook(False)
def add_ads_groups(waptconfigfile, hostdicts_list, sign_certs=None, sign_key=None, key_password=None, wapt_server_user=None,wapt_server_passwd=None, cabundle = None): if sign_certs is None: sign_bundle_fn = inifile_readstring(waptconfigfile,u'global',u'personal_certificate_path') sign_bundle = SSLCABundle(sign_bundle_fn) sign_certs = sign_bundle.certificates() # we assume a unique signer. if cabundle is None: cabundle = sign_bundle if not sign_certs: raise Exception(u'No personal signer certificate found in %s' % sign_bundle_fn) if sign_key is None: sign_key = sign_certs[0].matching_key_in_dirs(private_key_password=key_password) main_repo = WaptRemoteRepo(name='wapt',cabundle = cabundle) main_repo.load_config_from_file(waptconfigfile) host_repo = WaptHostRepo(name='wapt-host',host_id=[h['uuid'] for h in hostdicts_list],cabundle = cabundle) host_repo.load_config_from_file(waptconfigfile) total_hosts = len(host_repo.packages()) discarded_uuids = [p.package for p in host_repo.discarded] try: import waptconsole progress_hook = waptconsole.UpdateProgress except ImportError as e: def print_progress(show=False,n=0,max=100,msg=''): if show: print('%s %s/%s\r' % (msg,n,max),end='') else: if not msg: msg='Done' print("%s%s"%(msg,' '*(80-len(msg)))) progress_hook = print_progress packages = [] discarded = [] unchanged = [] try: progress_hook(True,0,len(hostdicts_list),'Editing %s hosts' % len(hostdicts_list)) i = 0 for h in hostdicts_list: try: host_package = None host_id = h['uuid'] hostname = h['computer_name'] groups = get_computer_groups(hostname) host_package = host_repo.get(host_id,PackageEntry(package=host_id,section='host')) if progress_hook(True,i,len(hostdicts_list),'Checking %s' % host_package.package): break wapt_groups = ensure_list(host_package['depends']) additional = [group for group in groups if not group in wapt_groups and (main_repo.get(group,None) is not None)] if additional: logger.info(u'Adding %s to %s' % (','.join(additional),host_package.package)) if progress_hook(True,i,len(hostdicts_list),'Editing %s' % host_package.package): break wapt_groups.extend(additional) host_package.depends = ','.join(wapt_groups) host_package.build_management_package() host_package.inc_build() host_file = host_package.build_management_package() host_package.sign_package(sign_certs,sign_key) packages.append(host_package) else: unchanged.append(host_package.package) except Exception as e: if host_package: discarded.append(host_package.package) logger.critical(u'Discarding because %s' % ensure_unicode(e)) # upload all in one step... progress_hook(True,3,3,'Upload %s host packages' % len(packages)) server = WaptServer().load_config_from_file(waptconfigfile) server.upload_packages(packages,auth=(wapt_server_user,wapt_server_passwd),progress_hook=progress_hook) return dict(updated = packages, discarded = discarded, unchanged = unchanged) finally: logger.debug('Cleanup') try: i = 0 for s in packages: i+=1 progress_hook(True,i,len(packages),'Cleanup') if os.path.isfile(s.localpath): os.remove(s.localpath) progress_hook(False) except WindowsError as e: logger.critical('Unable to remove temporary directory %s: %s'% (s,repr(e))) progress_hook(False)
def init(self): self.location = os.path.expanduser(self.config['location']) if not os.path.isfile(self.location): logger.critical('Followfile does not exist: {0}'.format(self.location)) self.handle = open(self.location) self.handle.seek(0, 2)
from common import logger try: import yaml except ImportError: logger.critical('Failed to load PyYAML') class main(object): def __init__(self, config): pass def run(self, data): return yaml.load(data)
except: logger.error('failed to unpack line {} in {}: "{}"'.format(n, info_file, line.strip())) exit(0) if (sym.upper() == symbol.upper()): cname = name.strip() if len(link.strip()) > 0: explorer = link.strip() if args['name'] is not None: cname = args['name'][0] if cname is not None: logger.debug('NAME {}'.format(cname)) else: logger.critical('could not determine currency name for symbol {}'.format(symbol)) exit(1) # public address address = args['address'][0] logger.debug('ADDRESS {}'.format(address)) # private key privkey = args['privkey'][0] # block explorer qr_link = None if explorer is not None:
def measure(): ''' Prepare the metrics pattern ''' prev = None stats = {} for s in (constants.RESULT_PARAMS).split(";") : key = s.split(':',1)[0] stats[key] = [] stats['additionalResults'] = [] # logger.info("Collected statistics: %s", json.dumps(stats)) p = subprocess.Popen("ps -ef | grep -v grep | grep -o 'redis-server.*:[0-9]*' | awk -F ':' '{print $2}'", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) lines = p.stdout.readlines() if not isinstance(lines, list) or len(lines) == 0: e = 'There is not any running instance of Redis.' logger.critical(e) stats['status'].append(constants.NOK_STATE) details={'details':str(e)} details = urllib.quote(json.dumps(details), '{}') stats['additionalResults'].append([details]) else: for line in lines: st = {} for key in stats: st[key] = '' st['additionalResults'] = [] port = line.strip() st['port'] = str(port) redis_info = '' try: r = getRedis(port) if not isinstance(r, redis.Redis): logger.error("Cannot establish connection to Redis") else: redis_info = r.info() prev = r.get('rm_prev') if (isinstance(prev, basestring)) : prev = json.loads(prev) except Exception as e : logger.critical(e) st['status'] = constants.NOK_STATE details={'details':str(e)} details = urllib.quote(json.dumps(details), '{}') st['additionalResults'].append(details) else : if(constants.REDIS_DB in redis_info): db_info = redis_info[constants.REDIS_DB] for k in db_info: if(k in stats): st[k] = db_info[k] pid = str(redis_info['process_id']) cm = commands.getoutput('ps -p'+pid+' -o %cpu,%mem | grep -v % ') rr = cm.strip().replace(' ',' ').split(' ', 2); cpu_pr=float(rr[0]) mem_pr=float(rr[1]) ''' Collect measured values ''' for k, v in st.iteritems(): if k == 'used_memory' : v = mem_pr elif k == 'used_cpu' : v = cpu_pr elif k == 'status' : v = constants.OK_STATE elif(k in redis_info) : v = redis_info[k] st[k] = v r.set('rm_prev', json.dumps(st)) # keep as previous value details={'details':' Redis-'+str(redis_info['redis_version'])+'-'+st['port']+' ('+str(redis_info['process_id']) +') '+' Uptime: '+_toDate(redis_info['uptime_in_seconds'])} details = urllib.quote(json.dumps(details), '{}') st['additionalResults'].append(details) ''' Convert to API required form ''' for k, v in st.iteritems(): if (k == 'expired_keys' or k == 'evicted_keys' or k == 'keyspace_misses' or k == 'total_connections_received') : if (isinstance(prev, dict) and (k in prev)) : v = max(v - prev[k], 0) # v = v - prev[k] else : v = 0 stats[k].append(v) return_value=json.dumps(stats).replace('],', '];').lstrip('{').rstrip('}').replace('"', '').replace(' ', '').replace('}];[{', '}],[{') logger.info('Returns: '+str(return_value)) logger.info('Stats: '+str(stats)) return return_value
def add_ads_groups(waptconfigfile, hostdicts_list, sign_certs=None, sign_key=None, key_password=None, wapt_server_user=None, wapt_server_passwd=None, cabundle=None): if sign_certs is None: sign_bundle_fn = inifile_readstring(waptconfigfile, u'global', u'personal_certificate_path') sign_bundle = SSLCABundle(sign_bundle_fn) sign_certs = sign_bundle.certificates() # we assume a unique signer. if cabundle is None: cabundle = sign_bundle if not sign_certs: raise Exception(u'No personal signer certificate found in %s' % sign_bundle_fn) if sign_key is None: sign_key = sign_certs[0].matching_key_in_dirs( private_key_password=key_password) try: import waptconsole progress_hook = waptconsole.UpdateProgress private_key_password_callback = waptconsole.GetPrivateKeyPassword except ImportError as e: def print_progress(show=False, n=0, max=100, msg=''): if show: print('%s %s/%s\r' % (msg, n, max), end='') else: if not msg: msg = 'Done' print("%s%s" % (msg, ' ' * (80 - len(msg)))) progress_hook = print_progress private_key_password_callback = None main_repo = WaptRemoteRepo(name='wapt', cabundle=cabundle) main_repo.load_config_from_file(waptconfigfile) main_repo.private_key_password_callback = private_key_password_callback host_repo = WaptHostRepo(name='wapt-host', host_id=[h['uuid'] for h in hostdicts_list], cabundle=cabundle) host_repo.load_config_from_file(waptconfigfile) host_repo.private_key_password_callback = private_key_password_callback total_hosts = len(host_repo.packages()) discarded_uuids = [p.package for p in host_repo.discarded] packages = [] discarded = [] unchanged = [] try: progress_hook(True, 0, len(hostdicts_list), 'Editing %s hosts' % len(hostdicts_list)) i = 0 for h in hostdicts_list: try: host_package = None host_id = h['uuid'] hostname = h['computer_name'] groups = get_computer_groups(hostname) host_package = host_repo.get( host_id, PackageEntry(package=host_id, section='host')) if progress_hook(True, i, len(hostdicts_list), 'Checking %s' % host_package.package): break wapt_groups = ensure_list(host_package['depends']) additional = [ group for group in groups if not group in wapt_groups and ( main_repo.get(group, None) is not None) ] if additional: logger.info(u'Adding %s to %s' % (','.join(additional), host_package.package)) if progress_hook(True, i, len(hostdicts_list), 'Editing %s' % host_package.package): break wapt_groups.extend(additional) host_package.depends = ','.join(wapt_groups) host_package.build_management_package() host_package.inc_build() host_file = host_package.build_management_package() host_package.sign_package(sign_certs, sign_key) packages.append(host_package) else: unchanged.append(host_package.package) except Exception as e: if host_package: discarded.append(host_package.package) logger.critical(u'Discarding because %s' % ensure_unicode(e)) # upload all in one step... progress_hook(True, 3, 3, 'Upload %s host packages' % len(packages)) server = WaptServer().load_config_from_file(waptconfigfile) server.private_key_password_callback = private_key_password_callback server.upload_packages(packages, auth=(wapt_server_user, wapt_server_passwd), progress_hook=progress_hook) return dict(updated=packages, discarded=discarded, unchanged=unchanged) finally: logger.debug('Cleanup') try: i = 0 for s in packages: i += 1 progress_hook(True, i, len(packages), 'Cleanup') if os.path.isfile(s.localpath): os.remove(s.localpath) progress_hook(False) except WindowsError as e: logger.critical('Unable to remove temporary directory %s: %s' % (s, repr(e))) progress_hook(False)
def edit_hosts_depends( waptconfigfile, hosts_list, append_depends=[], remove_depends=[], append_conflicts=[], remove_conflicts=[], sign_certs=None, sign_key=None, key_password=None, wapt_server_user=None, wapt_server_passwd=None, cabundle=None, ): """Add or remove packages from host packages Args: Returns: dict: { updated: of uuid of machines actually updated unchanged : list of uuid skipped because of no change needed discarded : list of uuid discarded due to errors} >>> edit_hosts_depends('c:/wapt/wapt-get.ini','htlaptop.tranquilit.local','toto','tis-7zip','admin','password') """ if sign_certs is None: sign_bundle_fn = inifile_readstring(waptconfigfile, u'global', u'personal_certificate_path') sign_bundle = SSLCABundle(sign_bundle_fn) sign_certs = sign_bundle.certificates() # we assume a unique signer. if cabundle is None: cabundle = sign_bundle if not sign_certs: raise Exception(u'No personal signer certificate found in %s' % sign_bundle_fn) try: import waptconsole progress_hook = waptconsole.UpdateProgress private_key_password_callback = waptconsole.GetPrivateKeyPassword except ImportError as e: def print_progress(show=False, n=0, max=100, msg=''): if show: print('%s %s/%s\r' % (msg, n, max), end='') else: if not msg: msg = 'Done' print("%s%s" % (msg, ' ' * (80 - len(msg)))) progress_hook = print_progress private_key_password_callback = None if sign_key is None: sign_key = sign_certs[0].matching_key_in_dirs( private_key_password=key_password, password_callback=private_key_password_callback) progress_hook(True, 0, len(hosts_list), 'Loading %s hosts packages' % len(hosts_list)) host_repo = WaptHostRepo(name='wapt-host', host_id=[h['uuid'] for h in hosts_list], cabundle=cabundle) host_repo.private_key_password_callback = private_key_password_callback host_repo.load_config_from_file(waptconfigfile) total_hosts = len(host_repo.packages()) discarded_uuids = [p.package for p in host_repo.discarded] append_depends = ensure_list(append_depends) remove_depends = ensure_list(remove_depends) append_conflicts = ensure_list(append_conflicts) remove_conflicts = ensure_list(remove_conflicts) packages = [] discarded = [] unchanged = [] progress_hook(True, 0, len(hosts_list), 'Editing %s hosts' % len(hosts_list)) i = 0 try: for hostrec in hosts_list: host_id = hostrec['uuid'] i += 1 # don't change discarded packages. if host_id in discarded_uuids: discarded.append(host_id) else: host = host_repo.get(host_id) if host is None: host = PackageEntry(package=host_id, section='host') if progress_hook(True, i, len(hosts_list), 'Editing %s' % host.package): break logger.debug(u'Edit host %s : +%s -%s' % (host.package, append_depends, remove_depends)) depends = host.depends conflicts = host.conflicts conflicts = add_to_csv_list(conflicts, append_conflicts) conflicts = remove_from_csv_list(conflicts, remove_conflicts) depends = remove_from_csv_list(depends, ensure_list(conflicts)) depends = add_to_csv_list(depends, append_depends) depends = remove_from_csv_list(depends, remove_depends) conflicts = remove_from_csv_list(conflicts, ensure_list(depends)) if depends != host.depends or conflicts != host.conflicts: host.depends = depends host.conflicts = conflicts host.inc_build() host_file = host.build_management_package() host.sign_package(sign_certs, sign_key) packages.append(host) else: unchanged.append(host.package) # upload all in one step... progress_hook(True, 3, 3, 'Upload %s host packages' % len(packages)) server = WaptServer().load_config_from_file(waptconfigfile) server.private_key_password_callback = private_key_password_callback server.upload_packages(packages, auth=(wapt_server_user, wapt_server_passwd), progress_hook=progress_hook) return dict(updated=[p.package for p in packages], discarded=discarded, unchanged=unchanged) finally: logger.debug('Cleanup') try: i = 0 for s in packages: i += 1 progress_hook(True, i, len(packages), 'Cleanup') if os.path.isfile(s.localpath): os.remove(s.localpath) progress_hook(False) except WindowsError as e: logger.critical('Unable to remove temporary directory %s: %s' % (s, repr(e))) progress_hook(False)
def main(): parser = argparse.ArgumentParser(prog=TASK_WORKER) parser.add_argument( "--task-id", help="task-id to attempt to process", required=True, dest="task_id" ) parser.add_argument( "--webapi-uri", help="zimfarm API URI", required=not bool(DEFAULT_WEB_API_URL), default=DEFAULT_WEB_API_URL, dest="webapi_uri", ) parser.add_argument( "--username", help="username to authenticate to zimfarm", required=not bool(os.getenv("USERNAME")), default=os.getenv("USERNAME"), ) parser.add_argument( "--workdir", help="directory in which workers create task-related files", required=not bool(os.getenv("WORKDIR")), default=os.getenv("WORKDIR"), dest="workdir", ) args = parser.parse_args() logger.info(f"starting zimfarm {TASK_WORKER} for {args.task_id}.") try: worker = TaskWorker( username=args.username, webapi_uri=args.webapi_uri, workdir=args.workdir, task_id=args.task_id, ) except Exception as exc: logger.critical(f"Unhandled exception during init: {exc}") sys.exit(1) try: sys.exit(worker.run()) except Exception as exc: logger.error(f"Unhandled exception: {exc}") logger.exception(exc) logger.error("exiting.") try: logger.info("Trying to upload exception details") tbe = traceback.TracebackException.from_exception(exc) worker.shutdown( "failed", exception="".join(tbe.format_exception_only()), traceback="".join(tbe.format()), ) except Exception: logger.error("Could not submit failure details") sys.exit(1)
import os, sys, time, getopt import os.path as path from common import logger import monitor_constants as constants from monitor import * optlist, args = getopt.gnu_getopt(sys.argv[1:], 'd:', ['duration=']) for opt, value in optlist: if opt == "-d" or opt == "--duration": constants.DURATION = int(value) logger.info("Duration was redefined: "+str(constants.DURATION)+" min") custom = Monitor(0) if not custom.prepareMonitor(): logger.critical("Cannot obtain/create monitor:(") sys.exit(1) logger.info("Start watching loop...") def getFileName(relPathToFile, defPath): if relPathToFile[len(relPathToFile)-1] == '/' : # path to directory folder = relPathToFile basename = None else : # path to file folder = path.dirname(relPathToFile) basename = path.basename(relPathToFile) filename = None cur = path.abspath('.') while cur != '/' : # tmp = path.normpath(path.join(cur, folder))