def onSetPrivateAction(self): utils.trace("ActionManager","onSetPublicAction","") from AppController import AppController bloopTree = AppController.getInstance().getMainViewController().getBloopTree() # Grab the currently selected items (could be multiple selections) currentItems = bloopTree.selectedItems() if currentItems is None: return # We're going to store our list of files and directories here fileList = [] dirList = [] for item in currentItems: itemData = item.getItemData() if item.isFile(): fileList.append(itemData['fileID']) elif item.isDirectory(): dirList.append(itemData['directoryID']) items = {} items['fileList'] = fileList items['dirList'] = dirList utils.trace("ActionManager","onSetPublicAction",items) # Grab the remote instance and prepare to make the call remoteSession = AppController.getInstance().getRemoteSession() remoteSession.setItemVisibility(items, 'private') # Loop through the items and set their 'filePrivate' fields to 0 for item in currentItems: item.setPrivate()
def SourceFactory(settings): """ return sourceBase derived instance for recent settings""" utils.trace( "SourceFactory called", 1 ) source = None if settings.hasTarget(): if settings.hasMultipleTargets(): url = settings.targetAt(0) else: url = settings.target() source = ArchiveSource(settings) ## \todo move settings access into info class if settings.hasSvnTarget(): url = settings.svnTarget() sourceType = utils.getVCSType( url ) if sourceType == "svn": source = SvnSource(settings) elif sourceType == "hg": source = HgSource(settings) elif sourceType == "git": source = GitSource(settings) if source == None: utils.die("none or unsupported source system set") if not source.subinfo: source.subinfo = settings source.url = url return source
def dummyUpload(self): '''Proof of concept method to upload a file to mybloop''' from base64 import b64encode '''will upload a dummy file''' fp = open('01.mp3') fp.seek(0,2) fileSize = fp.tell() fp.seek(0) chunkNum = 0 chunkSize = 131072 offset = 0 while fp.tell() < fileSize: fp.seek(offset) chunkData = fp.read(chunkSize) trace("RemoteSession","dummyUpload","Sending chunk") self.server.dummyUpload(chunkNum, chunkSize, b64encode(chunkData)) offset += chunkSize trace("RemoteSession","dummyUpload","Uploaded " + str(offset)) fp.close()
def __init__(self,uploadTokenDict,blooploadChunkFilename, remoteSession, fileObject): GetterSetter.__init__(self) if remoteSession is not None: self.setRemoteSession(remoteSession) if blooploadChunkFilename is not None: self.setBlooploadChunkFilename(blooploadChunkFilename) if fileObject is not None: self.setFileObject(fileObject) if uploadTokenDict == None and blooploadChunkFilename == None: '''They want to use an empty object''' trace("UploadToken","__init__","Using empty constructor") return if uploadTokenDict is None: trace("UploadToken","__init__","UploadToken Constructor: Missing token dict.") raise UploadTokenException("UploadToken Constructor: Missing token dict.") for x in uploadTokenDict: #trace('UploadToken','__init__', x + ':' + str(uploadTokenDict[x])) self._set(x,uploadTokenDict[x]) #End transfer. #trace('UploadToken','__init__', "The Upload Token->") #trace('UploadToken','__init__', uploadTokenDict) if uploadTokenDict['status'] == '1' : try: self._checkIncomingKeys() except UploadTokenException, e: raise e
def local_prep_worker ( worker ): virtual_env = os.path.join( self.remote_res_dir , self.tarred_testsdir ) if worker.isdir( virtual_env ): worker.execute ( "rmdir -rf %s" % virtual_env ) worker.execute ( "mkdir -p %s" % virtual_env ) worker.put ( get_virtualenv_script(), os.path.join( self.remote_res_dir, "virtualenv.py") ) trace(".") worker.execute( "python virtualenv.py %s" %\ self.tarred_testsdir, cwdir = self.remote_res_dir ) tarball = os.path.split(self.tarred_tests)[1] remote_tarball = os.path.join( self.remote_res_dir,\ tarball) # setup funkload worker.execute("./bin/easy_install %s" % self.funkload_location, cwdir = virtual_env) #unpackage tests. worker.put( self.tarred_tests, os.path.join(self.remote_res_dir , tarball)) worker.execute( "tar -xvf %s" %(tarball), cwdir = self.remote_res_dir) worker.execute("rm %s" % remote_tarball)
def processUIDS(self, view, protocol, client, account, request): if __debug__: trace("processUIDS") # request = # 0: cmd # 1: Mail Client # 2: accountUUID # 3: proocol specific info # IMAP = tuple containing # 0: imap folder UUID # 1: UID of last message # # POP = List of UIDS of seen messages if self.shuttingDown: return None if protocol == "IMAP": imapFolderUUID = request[3][0] imapFolder = view.findUUID(imapFolderUUID) # Store the last message UID for the IMAPFolder imapFolder.lastMessageUID = request[3][1] elif protocol == "POP": uids = request[3] for uid in uids: account.seenMessageUIDS[uid] = "True" view.commit()
def unpack(self): utils.trace( "MultiSource unpack", 2 ) # pylint: disable=E1101 # multiple inheritance: MultiSource is never the only # superclass, others define self.buildSystemType. self.source.buildSystemType = self.buildSystemType return self.source.unpack()
def local_prep_worker(worker): remote_res_dir = os.path.join(self.remote_res_dir, worker.name) virtual_env = os.path.join(remote_res_dir, self.tarred_testsdir) if worker.isdir(virtual_env): worker.execute("rm -rf %s" % virtual_env) worker.execute("mkdir -p %s" % virtual_env) worker.put(get_virtualenv_script(), os.path.join(remote_res_dir, "virtualenv.py")) trace(".") worker.execute("%s virtualenv.py %s" % (self.python_bin, self.tarred_testsdir), cwdir=remote_res_dir) tarball = os.path.split(self.tarred_tests)[1] remote_tarball = os.path.join(remote_res_dir, tarball) # setup funkload cmd = "./bin/easy_install setuptools ez_setup {funkload}".format(funkload=self.funkload_location) if self.distributed_packages: cmd += " %s" % self.distributed_packages worker.execute(cmd, cwdir=virtual_env) # unpackage tests. worker.put(self.tarred_tests, os.path.join(remote_res_dir, tarball)) worker.execute("tar -xvf %s" % tarball, cwdir=remote_res_dir) worker.execute("rm %s" % remote_tarball)
def processDone(self, view, protocol, client, account, request): if __debug__: trace("processDone") if self.shuttingDown: return None # request # 0: cmd # 1: client # 2: accountUUID dt = self.getDownloadTracker(account) if dt.totalDownloaded > 0: # This is a PyICU.ChoiceFormat class txt = constants.DOWNLOAD_CHANDLER_MESSAGES.format(dt.totalDownloaded) setStatusMessage(txt % \ {'accountName': account.displayName, 'numberTotal': dt.totalDownloaded, 'numberNew': dt.totalNewDownloaded, 'numberUpdates': dt.totalUpdateDownloaded, 'numberDuplicates': dt.totalIgnoreDownloaded, 'numberErrors': dt.totalErrorDownloaded}) else: setStatusMessage(constants.DOWNLOAD_NO_MESSAGES % \ {'accountName': account.displayName}) self._resetWorker(account) # Post a notification to the Mail Protocol Client # That the requested actions are finished. reactor.callFromThread(client.requestsComplete)
def main(args = None): opts = docopt.docopt( doc = __doc__, argv = args) try: if opts["--no-color"]: utils.disable_colors() if opts["--verbose"]: utils.enable_tracing() for name in opts["NAMES"]: dist = get_distribution(name) for name in required(dist): raise Error(dist.project_name, "required by", name) for req in dist.requires(): names = filter(lambda name: name != dist.project_name, required(req)) if not names: uninstall(req.project_name, dryrun = opts["--dry-run"]) else: utils.trace(req.project_name, "not uninstalled, used by", names) uninstall(dist.project_name, dryrun = opts["--dry-run"]) # else: # for dist in pip.get_installed_distributions(): # print utils.magenta(dist.project_name), dist.location except utils.Error as exc: raise SystemExit(utils.red(exc))
def _refreshCache(self, protocol): instances = None method = None if protocol in Mail.ACCOUNT_TYPES: instances = self._clientInstances.get(protocol) method = Mail.ACCOUNT_TYPES[protocol].getActiveAccounts self._view.refresh() uuidList = [] delList = [] for acc in method(self._view): uuidList.append(acc.itsUUID) for accUUID in instances.keys(): if not accUUID in uuidList: client = instances.get(accUUID) instances.pop(accUUID) del client delList.append(accUUID) if __debug__: s = len(delList) if s > 0: c = s > 1 and "Clients" or "Client" a = s > 1 and "accountUUID's" or "accountUUID" trace("removed %s%s with %s %s" % (protocol, c, a, delList))
def onDeleteAction(self, val): ''' Prepares for deletion of selected folders and files. If the user is sure, it will invoke deleteAction on a worker thread. ''' utils.trace("ActionManager","onDeleteAction","") from AppController import AppController bloopTree = AppController.getInstance().getMainViewController().getBloopTree() # Grab the currently selected items (could be multiple selections) currentItems = bloopTree.selectedItems() if currentItems is None: return if len(currentItems) > 0: from PyQt4.Qt import QMessageBox delConfirmMsg = i18n.LABEL_CONFIRM_FILE_DELETION_MESSAGE if len(currentItems) == 1 else i18n.LABEL_CONFIRM_FILES_DELETION_MESSAGE confirmDelete = QMessageBox.warning(AppController.getInstance().getMainView(), i18n.LABEL_CONFIRM_FILES_DELETION_TITLE, delConfirmMsg, QMessageBox.Yes | QMessageBox.Cancel, QMessageBox.Yes); if confirmDelete == QMessageBox.Cancel: AppController.getInstance().getMainView().statusBar().showMessage("") return AppController.getInstance().getMainView().statusBar().showMessage(i18n.LABEL_BLOOPTREE_STATUS_BAR_DELETING) self.hireWorker(self.deleteAction, currentItems)
def handleStartElement(self, name, attrs): """Called by expat parser on start element.""" if name == "funkload": self.config["version"] = attrs["version"] self.config["time"] = attrs["time"] elif name == "config": self.config[attrs["key"]] = attrs["value"] if attrs["key"] == "duration": if self.cycle_duration and attrs["value"] != self.cycle_duration: trace("Skipping file %s with different cycle duration %s" % (self.current_file, attrs["value"])) raise EndOfConfig self.cycle_duration = attrs["value"] elif attrs["key"] == "cycles": if self.cycles and attrs["value"] != self.cycles: trace( "Skipping file %s with different cycles %s != %s" % (self.current_file, attrs["value"], self.cycles) ) raise EndOfConfig self.cycles = attrs["value"] elif attrs["key"] == "node": self.nodes[self.current_file] = attrs["value"] else: self.files.append(self.current_file) raise EndOfConfig
def processEmpty(self, view, protocol, client, account, request): if __debug__: trace("processEmpty") if self.shuttingDown: return None return
def run(self): port = 8000 if self.port: port = int(self.port) server_address = ('', port) trace("Starting debug HTTP server at port %d\n" % port) httpd = BaseHTTPServer.HTTPServer(server_address, FunkLoadHTTPRequestHandler) httpd.serve_forever()
def __init__(self): object.__init__(self) utils.trace( "MultiSource __init__", 2 ) # pylint: disable=E1101 # multiple inheritance: MultiSource is never the only # superclass, others define self.source, self.subinfo etc. # TODO: This code should mostly be in the class defining self.source etc. self.source = SourceFactory(self.subinfo) self.source.localFileNames = self.localFileNames.__get__(self, MultiSource)
def final_collect(self): expr = re.compile("Log\s+xml:\s+(.*?)\n") for worker, results in self._worker_results.items(): res = expr.findall(results) if res: remote_file = res[0] file = os.path.split(remote_file)[1] local_file = os.path.join(self.distribution_output, "%s-%s" % (worker.host, file)) worker.get(remote_file, local_file) trace("* Received bench log from [%s] into %s\n" % (worker.host, local_file))
def put(self, local_path, remote_path): """ performs a copy from `local_path` to `remote_path` For performing the inverse operation, use the :meth:`get` """ try: sftp = self.connection.open_sftp() sftp.put(local_path, remote_path) except Exception, error: trace("failed to put %s->%s with error %s\n" % (local_path, remote_path, error))
def removeThreads(self, number_of_threads): """Removes threads. Used to dynamically remove threads during a debug bench run.""" self.thread_creation_lock.acquire() try: trace('* Removing threads: ') self.deleteThreads(number_of_threads) trace(' done.\n') finally: self.thread_creation_lock.release()
def extractScript(self): """Convert a tcpwatch capture into a FunkLoad script.""" files = self.searchFiles() requests = self.extractRequests(files) code = [self.convertToFunkLoad(request) for request in requests] if not code: trace("Sorry no action recorded.\n") return "" code.insert(0, "") return self.reindent("\n".join(code))
def get(self, remote_path, local_path): """ performs a copy from ``remote_path`` to ``local_path``. For performing the inverse operation, use the :meth:`put` """ try: sftp = self.connection.open_sftp() sftp.get(remote_path, local_path) except Exception, error: trace("failed to get %s->%s with error %s\n" % \ (local_path, remote_path, error))
def processError(self, view, protocol, client, account, request): if __debug__: trace("processError") if self.shuttingDown: return None #Clear the status bar message setStatusMessage(u"") self._resetWorker(account)
def prepare_workers(self, allow_errors=False): """ initialized the sandboxes in each worker node to prepare for a bench run. the additional parameter `allow_errors` essentially will make the distinction between ignoring unresponsive/inappropriate nodes - or raising an error and failing the entire bench. """ # right, lets figure out if funkload can be setup on each host def local_prep_worker(worker): virtual_env = os.path.join(self.remote_res_dir, self.tarred_testsdir) if worker.isdir(virtual_env): worker.execute("rmdir -rf %s" % virtual_env) worker.execute("mkdir -p %s" % virtual_env) worker.put(get_virtualenv_script(), os.path.join(self.remote_res_dir, "virtualenv.py")) trace(".") worker.execute("python virtualenv.py %s" % self.tarred_testsdir, cwdir=self.remote_res_dir) tarball = os.path.split(self.tarred_tests)[1] remote_tarball = os.path.join(self.remote_res_dir, tarball) # setup funkload worker.execute("./bin/easy_install %s" % self.funkload_location, cwdir=virtual_env) trace(".") worker.execute("python virtualenv.py %s" % self.remote_res_dir) # unpackage tests. worker.put(self.tarred_tests, os.path.join(self.remote_res_dir, tarball)) worker.execute("tar -xvf %s" % (tarball), cwdir=self.remote_res_dir) worker.execute("rm %s" % remote_tarball) threads = [] trace("* Preparing sandboxes for %d workers." % len(self._workers)) for worker in list(self._workers): if worker.connected: which_python = worker.execute("which python") if not which_python and not allow_errors: raise RuntimeError("%s doesn't have python" % worker.host) elif not which_python: self._workers.remove(worker) else: # python was found lets see if we need to install funkload threads.append(threading.Thread(target=local_prep_worker, args=(worker,))) elif allow_errors: self._workers.remove(worker) else: raise RuntimeError("%s is not contactable with error %s" % (worker.host, worker.error)) [k.start() for k in threads] [k.join() for k in threads] trace("\n") if not self._workers: raise RuntimeError("no workers available for distribution")
def logging(self): """Log activity during duration.""" duration = self.duration end_time = time.time() + duration trace("* Logging for %ds (until %s): " % (duration, datetime.fromtimestamp(end_time).isoformat())) set_recording_flag(True) while time.time() < end_time: # wait time.sleep(1) set_recording_flag(False) trace(" done.\n")
def startThreads(self, cycle, number_of_threads): """Starts threads.""" self.thread_creation_lock.acquire() try: trace("* Current time: %s\n" % datetime.now().isoformat()) trace("* Starting threads: ") set_recording_flag(False) threads = self.createThreads(cycle, number_of_threads) self.threads.extend(threads) finally: set_recording_flag(True) self.thread_creation_lock.release()
def _resetWorker(self, account): if __debug__: trace("_resetWorker") dt = self.getDownloadTracker(account) dt.totalDownloaded = 0 dt.totalNewDownloaded = 0 dt.totalUpdateDownloaded = 0 dt.totalIgnoreDownloaded = 0 dt.totalErrorDownloaded = 0 dt.lastTotalDownloaded = 0
def run_distributed(options, module_name, class_name, method_name, sys_args): ret = None from funkload.Distributed import DistributionMgr global _manager try: distmgr = DistributionMgr( module_name, class_name, method_name, options, sys_args) _manager = distmgr except UserWarning, error: trace(red_str("Distribution failed with:%s \n" % (error))) return 1
def log_r_X_z(self): X_m_tau = minus(self.Xz, self.tau) X_m_tau_vec = T.reshape(X_m_tau, [self.B * self.R, 1]) X_m_tau_vec.name = 'X_m_tau_vec' if self.Tau_isDiagonal: log_rX_z = -0.5 * self.R * self.B * log2pi - 0.5 * self.R * self.logDetTau \ - 0.5 * trace(dot(X_m_tau_vec.T, div(X_m_tau_vec,self.Tau))) else: log_rX_z = -0.5 * self.R * self.B * log2pi - 0.5 * self.R * self.logDetTau \ - 0.5 * trace(dot(X_m_tau_vec.T, dot(self.iTau, X_m_tau_vec))) log_rX_z.name = 'log_rX_z' return log_rX_z
def loadTestsFromTestCase(self, testCaseClass): if not issubclass(testCaseClass, FunkLoadTestCase): trace(red_str("Skipping "+ testCaseClass)) return [] testCaseNames = self.getTestCaseNames(testCaseClass) if not testCaseNames and hasattr(testCaseClass, 'runTest'): testCaseNames = ['runTest'] return [dict(module_name = testCaseClass.__module__, class_name = testCaseClass.__name__, method_name = x) for x in testCaseNames]
def KL_qp(self): if self.continuous: kappa_outer = dot(self.kappa, self.kappa.T, 'kappa_outer') AtA = dot(self.A.T, self.A) KL = 0.5*self.Q*trace(self.C) + 0.5*trace(dot(AtA,kappa_outer)) \ + 0.5*self.Q+trace(dot(AtA,self.Kappa)) - 0.5*self.Q*self.B - 0.5*self.Q*self.logDetC KL.name = 'KL_qp' else: raise RuntimeError("Case not implemented") return KL
try: pwd = String(password) x = EncryptionUtils.decryptString(cipher_text, pwd.toCharArray()) restored_text = String(x) fh = open(outputfile, "w") fh.write(str(restored_text)) fh.close() System.exit(0) except EncryptionException, e: # Catch the exact exception to get the real cause trace("SEVERE", "Error in decrypting file: %s" % e.getCause()) System.exit(-1) except: exc_type, exc_obj, exc_tb = sys.exc_info() eeString = traceback.format_exception(exc_type, exc_obj, exc_tb) trace("SEVERE", "Error in decrypting file: %s" % eeString) System.exit(-1) def encrypt_file(clear_text, password, outputfile): try: pwd = String(password) x = EncryptionUtils.encryptString(clear_text, pwd.toCharArray()) encrypted_text = String(x) fh = open(outputfile, "w") fh.write(str(encrypted_text)) fh.close() System.exit(0) except EncryptionException, e: # Catch the exact exception to get the real cause trace("SEVERE", "Error in encrypting file: %s" % e.getCause())
def calc_bias(data, metric, vanilla_result, fair_result): """ calculate bias reduction :param data: input data :param metric: similarity metric (jaccard or cosine) :param vanilla_result: vanilla mining result :param fair_result: debiased mining result :return: a dict containing bias reductions of training edges, validation edges and test edges """ biases = dict() # load graph adj_train = data['adjacency_train'] # build similarity matrix sim = utils.filter_similarity_matrix(utils.get_similarity_matrix( adj_train, metric=metric), sigma=0.75) lap = laplacian(sim) # calculate training bias biases['train'] = [ utils.trace(vanilla_result.T @ lap @ vanilla_result), utils.trace(fair_result.T @ lap @ fair_result) ] reduction = 1 - (biases['train'][1] / biases['train'][0]) biases['train'].clear() biases['train'].append(reduction) # calculating bias on validation set val_bias = [0, 0] # first one is vanilla, second one is fair for src, tgt in data['val_edge_pos']: vanilla_edge_bias = (lp_diff(vanilla_result[src], vanilla_result[tgt], ord=2)**2) * sim[src, tgt] fair_edge_bias = (lp_diff(fair_result[src], fair_result[tgt], ord=2)** 2) * sim[src, tgt] val_bias[0] += vanilla_edge_bias val_bias[1] += fair_edge_bias for src, tgt in data['val_edge_neg']: vanilla_edge_bias = (lp_diff(vanilla_result[src], vanilla_result[tgt], ord=2)**2) * sim[src, tgt] fair_edge_bias = (lp_diff(fair_result[src], fair_result[tgt], ord=2)** 2) * sim[src, tgt] val_bias[0] += vanilla_edge_bias val_bias[1] += fair_edge_bias reduction = 1 - (val_bias[1] / val_bias[0]) val_bias.clear() val_bias.append(reduction) biases['validation'] = val_bias # calculating bias on test set test_bias = [0, 0] # first one is vanilla, second one is fair for src, tgt in data['test_edge_pos']: vanilla_edge_bias = (lp_diff(vanilla_result[src], vanilla_result[tgt], ord=2)**2) * sim[src, tgt] fair_edge_bias = (lp_diff(fair_result[src], fair_result[tgt], ord=2)** 2) * sim[src, tgt] test_bias[0] += vanilla_edge_bias test_bias[1] += fair_edge_bias for src, tgt in data['test_edge_neg']: vanilla_edge_bias = (lp_diff(vanilla_result[src], vanilla_result[tgt], ord=2)**2) * sim[src, tgt] fair_edge_bias = (lp_diff(fair_result[src], fair_result[tgt], ord=2)** 2) * sim[src, tgt] test_bias[0] += vanilla_edge_bias test_bias[1] += fair_edge_bias reduction = 1 - (test_bias[1] / test_bias[0]) test_bias.clear() test_bias.append(reduction) biases['test'] = test_bias return biases
def run(self): """ """ threads = [] trace("* Starting %d workers" % len(self._workers)) self.startMonitors() for worker in self._workers: remote_res_dir = os.path.join(self.remote_res_dir, worker.name) venv = os.path.join(remote_res_dir, self.tarred_testsdir) obj = worker.threaded_execute('bin/fl-run-bench %s' % self.cmd_args, cwdir=venv) trace(".") threads.append(obj) trace("\n") [t.join() for t in threads] trace("\n") for thread, worker in zip(threads, self._workers): self._worker_results[worker] = thread.output.read() trace("* [%s] returned\n" % worker.name) err_string = thread.err.read() if err_string: trace("\n".join(" [%s]: %s" % (worker.name, k) for k \ in err_string.split("\n") if k.strip())) trace("\n") self.stopMonitors() self.correlate_statistics()
trace(red_str("Distribution failed with:%s \n" % (error))) try: distmgr.prepare_workers(allow_errors=True) ret = distmgr.run() distmgr.final_collect() except KeyboardInterrupt: trace("* ^C received *") distmgr.abort() return ret else: bench = BenchRunner(args[0], klass, method, options) # Start a HTTP server optionally if options.debugserver: http_server_thread = FunkLoadHTTPServer(bench, options.debugport) http_server_thread.start() ret = None try: ret = bench.run() except KeyboardInterrupt: trace("* ^C received *") return ret if __name__ == '__main__': ret = main() sys.exit(ret)
def __init__(self, module_name, class_name, method_name, options, cmd_args): """ mirrors the initialization of :class:`funkload.BenchRunner.BenchRunner` """ # store the args. these can be passed to BenchRunner later. self.module_name = module_name self.class_name = class_name self.method_name = method_name self.options = options self.cmd_args = cmd_args self.cmd_args += " --is-distributed" module = load_module(module_name) module_file = module.__file__ self.tarred_tests, self.tarred_testsdir = package_tests(module_file) self.remote_res_dir = "/tmp/funkload-bench-sandbox/" test = load_unittest(self.module_name, class_name, mmn_encode(method_name, 0, 0, 0), options) self.config_path = test._config_path self.result_path = test.result_path self.class_title = test.conf_get('main', 'title') self.class_description = test.conf_get('main', 'description') self.test_id = self.method_name self.test_url = test.conf_get('main', 'url') self.cycles = map(int, test.conf_getList('bench', 'cycles')) self.duration = test.conf_getInt('bench', 'duration') self.startup_delay = test.conf_getFloat('bench', 'startup_delay') self.cycle_time = test.conf_getFloat('bench', 'cycle_time') self.sleep_time = test.conf_getFloat('bench', 'sleep_time') self.sleep_time_min = test.conf_getFloat('bench', 'sleep_time_min') self.sleep_time_max = test.conf_getFloat('bench', 'sleep_time_max') if test.conf_get('distribute', 'channel_timeout', '', quiet=True): self.channel_timeout = test.conf_getFloat( 'distribute', 'channel_timeout') else: self.channel_timeout = None self.threads = [] # Contains list of ThreadData objects self.last_thread_id = -1 self.thread_creation_lock = threading.Lock() if options.python_bin: self.python_bin = options.python_bin else: self.python_bin = test.conf_get( 'distribute', 'python_bin', 'python') if options.distributed_packages: self.distributed_packages = options.distributed_packages else: self.distributed_packages = test.conf_get( 'distribute', 'packages', '') try: desc = getattr(test, self.method_name).__doc__.strip() except: desc = "" self.test_description = test.conf_get(self.method_name, 'description', desc) # make a collection output location if options.distributed_log_path: self.distribution_output = options.distributed_log_path elif test.conf_get('distribute', 'log_path', '', quiet=True): self.distribution_output = test.conf_get('distribute', 'log_path') else: raise UserWarning("log_path isn't defined in section [distribute]") # check if user has overridden the default funkload distro download # location this will be used to download funkload on the worker nodes. self.funkload_location = test.conf_get( 'distribute', 'funkload_location', 'funkload') if not os.path.isdir(self.distribution_output): os.makedirs(self.distribution_output) # check if hosts are in options workers = [] # list of (host, port, descr) if options.workerlist: for h in options.workerlist.split(","): cred_host = h.split("@") if len(cred_host) == 1: uname, pwd, host = None, None, cred_host[0] else: cred = cred_host[0] host = cred_host[1] uname_pwd = cred.split(":") if len(uname_pwd) == 1: uname, pwd = uname_pwd[0], None else: uname, pwd = uname_pwd workers.append({ "name": host, "host": host, "password": pwd, "username": uname, "channel_timeout": self.channel_timeout}) else: hosts = test.conf_get('workers', 'hosts', '', quiet=True).split() for host in hosts: host = host.strip() workers.append({ "name": host, "host": test.conf_get(host, "host",host), "password": test.conf_get(host, 'password', ''), "username": test.conf_get(host, 'username', ''), "channel_timeout": self.channel_timeout}) self._workers = [] [self._workers.append(SSHDistributor(**w)) for w in workers] self._worker_results = {} trace(str(self)) # setup monitoring monitor_hosts = [] # list of (host, port, descr) if not options.is_distributed: hosts = test.conf_get('monitor', 'hosts', '', quiet=True).split() for host in sorted(hosts): name = host host = test.conf_get(host,'host',host.strip()) monitor_hosts.append((name, host, test.conf_getInt(name, 'port'), test.conf_get(name, 'description', ''))) self.monitor_hosts = monitor_hosts # keep the test to use the result logger for monitoring # and call setUp/tearDown Cycle self.test = test
class FunkLoadTestCase(unittest.TestCase): """Unit test with browser and configuration capabilties.""" # ------------------------------------------------------------ # Initialisation # def __init__(self, methodName='runTest', options=None): """Initialise the test case. Note that methodName is encoded in bench mode to provide additional information like thread_id, concurrent virtual users...""" if mmn_is_bench(methodName): self.in_bench_mode = True else: self.in_bench_mode = False self.test_name, self.cycle, self.cvus, self.thread_id = mmn_decode( methodName) self.meta_method_name = methodName self.suite_name = self.__class__.__name__ unittest.TestCase.__init__(self, methodName=self.test_name) self._response = None self._options = options self.debug_level = getattr(options, 'debug_level', 0) self._funkload_init() self._dump_dir = getattr(options, 'dump_dir', None) self._dumping = self._dump_dir and True or False self._viewing = getattr(options, 'firefox_view', False) self._accept_invalid_links = getattr(options, 'accept_invalid_links', False) self._bench_label = getattr(options, 'label', None) self._stop_on_fail = getattr(options, 'stop_on_fail', False) self._pause = getattr(options, 'pause', False) self._keyfile_path = None self._certfile_path = None self._accept_gzip = False if self._viewing and not self._dumping: # viewing requires dumping contents self._dumping = True self._dump_dir = mkdtemp('_funkload') self._loop_mode = getattr(options, 'loop_steps', False) if self._loop_mode: if ':' in options.loop_steps: steps = options.loop_steps.split(':') self._loop_steps = range(int(steps[0]), int(steps[1])) else: self._loop_steps = [int(options.loop_steps)] self._loop_number = options.loop_number self._loop_recording = False self._loop_records = [] if sys.version_info >= (2, 5): self.__exc_info = sys.exc_info def _funkload_init(self): """Initialize a funkload test case using a configuration file.""" # look into configuration file config_path = getattr(self._options, 'config', None) if not config_path: config_directory = os.getenv('FL_CONF_PATH', '.') config_path = os.path.join(config_directory, self.__class__.__name__ + '.conf') config_path = os.path.abspath(os.path.expanduser(config_path)) if not os.path.exists(config_path): config_path = "Missing: " + config_path config = ConfigParser() config.read(config_path) self._config = config self._config_path = config_path self.conf = ConfSectionFinder(self) self.default_user_agent = self.conf_get('main', 'user_agent', 'FunkLoad/%s' % get_version(), quiet=True) if self.in_bench_mode: section = 'bench' else: section = 'ftest' self.setOkCodes( self.conf_getList(section, 'ok_codes', [200, 301, 302, 303, 307], quiet=True)) self.sleep_time_min = self.conf_getFloat(section, 'sleep_time_min', 0) self.sleep_time_max = self.conf_getFloat(section, 'sleep_time_max', 0) self._simple_fetch = self.conf_getInt(section, 'simple_fetch', 0, quiet=True) self.log_to = self.conf_get(section, 'log_to', 'console file') self.log_path = self.conf_get(section, 'log_path', 'funkload.log') self.result_path = os.path.abspath( self.conf_get(section, 'result_path', 'funkload.xml')) # init loggers if self.in_bench_mode: level = logging.INFO else: level = logging.DEBUG self.logger = get_default_logger(self.log_to, self.log_path, level=level) self.logger_result = get_default_logger(log_to="xml", log_path=self.result_path, name="FunkLoadResult") #self.logd('_funkload_init config [%s], log_to [%s],' # ' log_path [%s], result [%s].' % ( # self._config_path, self.log_to, self.log_path, self.result_path)) # init webunit browser (passing a fake methodName) self._browser = WebTestCase(methodName='log') self.clearContext() #self.logd('# FunkLoadTestCase._funkload_init done') def setOkCodes(self, ok_codes): """Set ok codes.""" self.ok_codes = map(int, ok_codes) def clearContext(self): """Reset the testcase.""" self._browser.clearContext() self._browser.css = {} self._browser.history = [] self._browser.extra_headers = [] if self.debug_level >= 3: self._browser.debug_headers = True else: self._browser.debug_headers = False self.step_success = True self.test_status = 'Successful' self.steps = 0 self.page_responses = 0 self.total_responses = 0 self.total_time = 0.0 self.total_pages = self.total_images = 0 self.total_links = self.total_redirects = 0 self.total_xmlrpc = 0 self.clearBasicAuth() self.clearHeaders() self.clearKeyAndCertificateFile() self.setUserAgent(self.default_user_agent) self.logdd('FunkLoadTestCase.clearContext done') #------------------------------------------------------------ # browser simulation # def _connect(self, url, params, ok_codes, rtype, description, redirect=False, consumer=None): """Handle fetching, logging, errors and history.""" if params is None and rtype in ('post', 'put'): # enable empty put/post params = [] t_start = time.time() try: response = self._browser.fetch(url, params, ok_codes=ok_codes, key_file=self._keyfile_path, cert_file=self._certfile_path, method=rtype, consumer=consumer) except: etype, value, tback = sys.exc_info() t_stop = time.time() t_delta = t_stop - t_start self.total_time += t_delta self.step_success = False self.test_status = 'Failure' self.logd(' Failed in %.3fs' % t_delta) if etype is HTTPError: self._log_response(value.response, rtype, description, t_start, t_stop, log_body=True) if self._dumping: self._dump_content(value.response, description) raise self.failureException, str(value.response) else: self._log_response_error(url, rtype, description, t_start, t_stop) if etype is SocketError: raise SocketError("Can't load %s." % url) raise t_stop = time.time() # Log response t_delta = t_stop - t_start self.total_time += t_delta if redirect: self.total_redirects += 1 elif rtype != 'link': self.total_pages += 1 else: self.total_links += 1 if rtype in ('put', 'post', 'get', 'delete'): # this is a valid referer for the next request self.setHeader('Referer', url) self._browser.history.append((rtype, url)) self.logd(' Done in %.3fs' % t_delta) if self._accept_gzip: if response.headers is not None and response.headers.get( 'Content-Encoding') == 'gzip': buf = StringIO(response.body) response.body = gzip.GzipFile(fileobj=buf).read() self._log_response(response, rtype, description, t_start, t_stop) if self._dumping: self._dump_content(response, description) return response def _browse(self, url_in, params_in=None, description=None, ok_codes=None, method='post', follow_redirect=True, load_auto_links=True, sleep=True): """Simulate a browser handle redirects, load/cache css and images.""" self._response = None # Loop mode if self._loop_mode: if self.steps == self._loop_steps[0]: self._loop_recording = True self.logi('Loop mode start recording') if self._loop_recording: self._loop_records.append( (url_in, params_in, description, ok_codes, method, follow_redirect, load_auto_links, False)) # ok codes if ok_codes is None: ok_codes = self.ok_codes if type(params_in) is DictType: params_in = params_in.items() params = [] if params_in: if isinstance(params_in, Data): params = params_in else: for key, value in params_in: if type(value) is DictType: for val, selected in value.items(): if selected: params.append((key, val)) elif type(value) in (ListType, TupleType): for val in value: params.append((key, val)) else: params.append((key, value)) if method == 'get' and params: url = url_in + '?' + urlencode(params) else: url = url_in if method == 'get': params = None if method == 'get': if not self.in_bench_mode: self.logd('GET: %s\n\tPage %i: %s ...' % (url, self.steps, description or '')) else: url = url_in if not self.in_bench_mode: self.logd('%s: %s %s\n\tPage %i: %s ...' % (method.upper(), url, str(params), self.steps, description or '')) # Fetching response = self._connect(url, params, ok_codes, method, description) # Check redirection if follow_redirect and response.code in (301, 302, 303, 307): max_redirect_count = 10 thread_sleep() # give a chance to other threads while response.code in (301, 302, 303, 307) and max_redirect_count: # Figure the location - which may be relative newurl = response.headers['Location'] url = urljoin(url_in, newurl) # Save the current url as the base for future redirects url_in = url self.logd(' Load redirect link: %s' % url) # Use the appropriate method for redirection if response.code in (302, 303): method = 'get' if response.code == 303: # 303 is HTTP/1.1, make sure the connection # is not in keep alive mode self.setHeader('Connection', 'close') response = self._connect(url, None, ok_codes, rtype=method, description=None, redirect=True) max_redirect_count -= 1 if not max_redirect_count: self.logd(' WARNING Too many redirects give up.') # Load auto links (css and images) response.is_html = is_html(response.body) if load_auto_links and response.is_html and not self._simple_fetch: self.logd(' Load css and images...') page = response.body t_start = time.time() c_start = self.total_time try: # pageImages is patched to call _log_response on all links self._browser.pageImages(url, page, self) except HTTPError, error: if self._accept_invalid_links: if not self.in_bench_mode: self.logd(' ' + str(error)) else: t_stop = time.time() t_delta = t_stop - t_start self.step_success = False self.test_status = 'Failure' self.logd(' Failed in ~ %.2fs' % t_delta) # XXX The duration logged for this response is wrong self._log_response(error.response, 'link', None, t_start, t_stop, log_body=True) raise self.failureException, str(error) c_stop = self.total_time self.logd(' Done in %.3fs' % (c_stop - c_start)) if sleep: self.sleep() self._response = response # Loop mode if self._loop_mode and self.steps == self._loop_steps[-1]: self._loop_recording = False self.logi('Loop mode end recording.') t_start = self.total_time count = 0 for i in range(self._loop_number): self.logi('Loop mode replay %i' % i) for record in self._loop_records: count += 1 self.steps += 1 self._browse(*record) t_delta = self.total_time - t_start text = ('End of loop: %d pages rendered in %.3fs, ' 'avg of %.3fs per page, ' '%.3f SPPS without concurrency.' % (count, t_delta, t_delta / count, count / t_delta)) self.logi(text) trace(text + '\n') return response
def run(self): """Run a test in loop.""" while (self.thread_signaller.running()): test_result = unittest.TestResult() self.test.clearContext() self.test(test_result) if test_result.wasSuccessful(): if recording(): add_cycle_result('success') if self.color: trace(green_str('.')) else: trace('.') else: if len(test_result.errors): if recording(): add_cycle_result('error') if self.color: trace(red_str('E')) else: trace('E') else: if recording(): add_cycle_result('failure') if self.color: trace(red_str('F')) else: trace('F') if self.debug: for (test, error) in test_result.errors: trace("ERROR %s: %s" % (str(test), str(error))) for (test, error) in test_result.failures: trace("FAILURE %s: %s" % (str(test), str(error))) thread_sleep(self.sleep_time)
def __init__(self, argv=None): if self.server_name is None: self.server_name = self.__class__.__name__ if argv is None: argv = sys.argv conf_path, options = self.parseArgs(argv) self.default_log_path = self.server_name + '.log' self.default_pid_path = self.server_name + '.pid' self.server = None self.quit = False # read conf conf = ConfigParser() conf.read(conf_path) self.conf_path = conf_path self.host = conf.get('server', 'host') self.port = conf.getint('server', 'port') try: self.pid_path = conf.get('server', 'pid_path') except NoOptionError: self.pid_path = self.default_pid_path try: log_path = conf.get('server', 'log_path') except NoOptionError: log_path = self.default_log_path if is_server_running(self.host, self.port): trace("Server already running on %s:%s." % (self.host, self.port)) sys.exit(0) trace('Starting %s server at http://%s:%s/' % (self.server_name, self.host, self.port)) # init logger if options.verbose: level = logging.DEBUG else: level = logging.INFO if options.debug: log_to = 'file console' else: log_to = 'file' self.logger = get_default_logger(log_to, log_path, level=level, name=self.server_name) # subclass init self._init_cb(conf, options) # daemon mode if not options.debug: trace(' as daemon.\n') close_logger(self.server_name) create_daemon() # re init the logger self.logger = get_default_logger(log_to, log_path, level=level, name=self.server_name) else: trace(' in debug mode.\n') # init rpc self.initServer()
def run(self): """Run all the cycles. return 0 on success, 1 if there were some failures and -1 on errors.""" trace(str(self)) trace("Benching\n") trace("========\n\n") cycle = total_success = total_failures = total_errors = 0 self.logr_open() trace("* setUpBench hook: ...") self.test.setUpBench() trace(' done.\n') self.getMonitorsConfig() trace('\n') for cvus in self.cycles: t_start = time.time() reset_cycle_results() text = "Cycle #%i with %s virtual users\n" % (cycle, cvus) trace(text) trace('-' * (len(text) - 1) + "\n\n") monitor_key = '%s:%s:%s' % (self.method_name, cycle, cvus) trace("* setUpCycle hook: ...") self.test.setUpCycle() trace(' done.\n') self.startMonitors(monitor_key) self.startThreads(cycle, cvus) self.logging() #self.dumpThreads() self.stopThreads() self.stopMonitors(monitor_key) cycle += 1 trace("* tearDownCycle hook: ...") self.test.tearDownCycle() trace(' done.\n') t_stop = time.time() trace("* End of cycle, %.2fs elapsed.\n" % (t_stop - t_start)) success, failures, errors = get_cycle_results() status, code = get_status(success, failures, errors, self.color) trace("* Cycle result: **%s**, " "%i success, %i failure, %i errors.\n\n" % (status, success, failures, errors)) total_success += success total_failures += failures total_errors += errors trace("* tearDownBench hook: ...") self.test.tearDownBench() trace(' done.\n\n') self.logr_close() # display bench result trace("Result\n") trace("======\n\n") trace("* Success: %s\n" % total_success) trace("* Failures: %s\n" % total_failures) trace("* Errors: %s\n\n" % total_errors) status, code = get_status(total_success, total_failures, total_errors) trace("Bench status: **%s**\n" % status) return code
def __init__(self, level): self.print_debug = trace(level)
def gen_reg(self): reg = 0 for infer in self.gnet: for param in infer.parameters(): reg += trace(param) return reg
def main(): """ReportBuilder main.""" parser = OptionParser(USAGE, formatter=TitledHelpFormatter(), version="FunkLoad %s" % get_version()) parser.add_option("-H", "--html", action="store_true", default=False, dest="html", help="Produce an html report.") parser.add_option("--org", action="store_true", default=False, dest="org", help="Org-mode report.") parser.add_option("-P", "--with-percentiles", action="store_true", default=True, dest="with_percentiles", help=("Include percentiles in tables, use 10%, 50% and" " 90% for charts, default option.")) parser.add_option("--no-percentiles", action="store_false", dest="with_percentiles", help=("No percentiles in tables display min, " "avg and max in charts.")) cur_path = os.path.abspath(os.path.curdir) parser.add_option("-d", "--diff", action="store_true", default=False, dest="diffreport", help=("Create differential report.")) parser.add_option("-t", "--trend", action="store_true", default=False, dest="trendreport", help=("Build a trend reprot.")) parser.add_option("-o", "--output-directory", type="string", dest="output_dir", help="Parent directory to store reports, the directory" "name of the report will be generated automatically.", default=cur_path) parser.add_option("-r", "--report-directory", type="string", dest="report_dir", help="Directory name to store the report.", default=None) parser.add_option( "-T", "--apdex-T", type="float", dest="apdex_t", help="Apdex T constant in second, default is set to 1.5s. " "Visit http://www.apdex.org/ for more information.", default=1.5) parser.add_option("-x", "--css", type="string", dest="css_file", help="Custom CSS file to use for the HTML reports", default=None) parser.add_option("", "--skip-definitions", action="store_true", default=False, dest="skip_definitions", help="If True, will skip the definitions") parser.add_option("-q", "--quiet", action="store_true", default=False, dest="quiet", help=("Report no system messages when generating" " html from rst.")) options, args = parser.parse_args() if options.diffreport: if len(args) != 2: parser.error("incorrect number of arguments") trace("Creating diff report ... ") output_dir = options.output_dir html_path = RenderDiff(args[0], args[1], options, css_file=options.css_file) trace("done: \n") trace("%s\n" % html_path) elif options.trendreport: if len(args) < 2: parser.error("incorrect number of arguments") trace("Creating trend report ... ") output_dir = options.output_dir html_path = RenderTrend(args, options, css_file=options.css_file) trace("done: \n") trace("%s\n" % html_path) else: if len(args) < 1: parser.error("incorrect number of arguments") if len(args) > 1: trace("Merging results files: ") f = NamedTemporaryFile(prefix='fl-mrg-', suffix='.xml') tmp_file = f.name f.close() MergeResultFiles(args, tmp_file) trace("Results merged in tmp file: %s\n" % os.path.abspath(tmp_file)) args = [tmp_file] options.xml_file = args[0] xml_parser = FunkLoadXmlParser(options.apdex_t) xml_parser.parse(options.xml_file) if options.html: trace("Creating html report: ...") html_path = RenderHtml(xml_parser.config, xml_parser.stats, xml_parser.error, xml_parser.monitor, xml_parser.monitorconfig, options, css_file=options.css_file)() trace("done: \n") trace(html_path + "\n") elif options.org: from ReportRenderOrg import RenderOrg print unicode( RenderOrg(xml_parser.config, xml_parser.stats, xml_parser.error, xml_parser.monitor, xml_parser.monitorconfig, options)).encode("utf-8") else: print unicode( RenderRst(xml_parser.config, xml_parser.stats, xml_parser.error, xml_parser.monitor, xml_parser.monitorconfig, options)).encode("utf-8")
def prepare_workers(self, allow_errors=False): """ Initialize the sandboxes in each worker node to prepare for a bench run. The additional parameter `allow_errors` will essentially make the distinction between ignoring unresponsive/inappropriate nodes - or raising an error and failing the entire bench. """ # right, lets figure out if funkload can be setup on each host def local_prep_worker(worker): remote_res_dir = os.path.join(self.remote_res_dir, worker.name) virtual_env = os.path.join( remote_res_dir, self.tarred_testsdir) if worker.isdir(virtual_env): worker.execute("rm -rf %s" % virtual_env) worker.execute("mkdir -p %s" % virtual_env) worker.put( get_virtualenv_script(), os.path.join(remote_res_dir, "virtualenv.py")) trace(".") worker.execute( "%s virtualenv.py %s" % ( self.python_bin, self.tarred_testsdir), cwdir=remote_res_dir) tarball = os.path.split(self.tarred_tests)[1] remote_tarball = os.path.join(remote_res_dir, tarball) # setup funkload cmd = "./bin/easy_install setuptools ez_setup {funkload}".format( funkload=self.funkload_location) if self.distributed_packages: cmd += " %s" % self.distributed_packages worker.execute(cmd, cwdir=virtual_env) #unpackage tests. worker.put( self.tarred_tests, os.path.join(remote_res_dir, tarball)) worker.execute( "tar -xvf %s" % tarball, cwdir=remote_res_dir) worker.execute("rm %s" % remote_tarball) # workaround for https://github.com/pypa/virtualenv/issues/330 worker.execute("rm lib64", cwdir=virtual_env) worker.execute("ln -s lib lib64", cwdir=virtual_env) threads = [] trace("* Preparing sandboxes for %d workers." % len(self._workers)) for worker in list(self._workers): if not worker.connected: if allow_errors: trace("%s is not connected, removing from pool.\n" % \ worker.name) self._workers.remove(worker) continue else: raise RuntimeError( "%s is not contactable with error %s" % ( worker.name, worker.error)) # Verify that the Python binary is available which_python = "test -x `which %s 2>&1 > /dev/null` && echo true" \ % (self.python_bin) out, err = worker.execute(which_python) if out.strip() == "true": threads.append(threading.Thread( target=local_prep_worker, args=(worker,))) elif allow_errors: trace("Cannot find Python binary at path `%s` on %s, " + \ "removing from pool" % (self.python_bin, worker.name)) self._workers.remove(worker) else: raise RuntimeError("%s is not contactable with error %s" % ( worker.name, worker.error)) [k.start() for k in threads] [k.join() for k in threads] trace("\n") if not self._workers: raise RuntimeError("no workers available for distribution")
def main(args=sys.argv[1:]): """Default main.""" # enable to load module in the current path cur_path = os.path.abspath(os.path.curdir) sys.path.insert(0, cur_path) parser = OptionParser(USAGE, formatter=TitledHelpFormatter(), version="FunkLoad %s" % get_version()) parser.add_option("-u", "--url", type="string", dest="main_url", help="Base URL to bench.") parser.add_option("-c", "--cycles", type="string", dest="bench_cycles", help="Cycles to bench, this is a list of number of " "virtual concurrent users, to run a bench with 3" "cycles with 5, 10 and 20 users use: -c 2:10:20") parser.add_option("-D", "--duration", type="string", dest="bench_duration", help="Duration of a cycle in seconds.") parser.add_option("-m", "--sleep-time-min", type="string", dest="bench_sleep_time_min", help="Minimum sleep time between requests.") parser.add_option("-M", "--sleep-time-max", type="string", dest="bench_sleep_time_max", help="Maximum sleep time between requests.") parser.add_option("-t", "--test-sleep-time", type="string", dest="bench_sleep_time", help="Sleep time between tests.") parser.add_option("-s", "--startup-delay", type="string", dest="bench_startup_delay", help="Startup delay between thread.") parser.add_option("-f", "--as-fast-as-possible", action="store_true", help="Remove sleep times between requests and between " "tests, shortcut for -m0 -M0 -t0") parser.add_option("", "--no-color", action="store_true", help="Monochrome output.") parser.add_option("", "--accept-invalid-links", action="store_true", help="Do not fail if css/image links are not reachable.") parser.add_option("", "--simple-fetch", action="store_true", dest="bench_simple_fetch", help="Don't load additional links like css or images " "when fetching an html page.") parser.add_option("-l", "--label", type="string", help="Add a label to this bench run for easier " "identification (it will be appended to the " "directory name for reports generated from it).") parser.add_option("--enable-debug-server", action="store_true", dest="debugserver", help="Instantiates a debug HTTP server which exposes an " "interface using which parameters can be modified " "at run-time. Currently supported parameters: " "/cvu?inc=<integer> to increase the number of " "CVUs, /cvu?dec=<integer> to decrease the number " "of CVUs, /getcvu returns number of CVUs ") parser.add_option("--debug-server-port", type="string", dest="debugport", help="Port at which debug server should run during the " "test") parser.add_option("--distribute", action="store_true", dest="distribute", help="Distributes the CVUs over a group of worker " "machines that are defined in the workers section") parser.add_option("--distribute-workers", type="string", dest="workerlist", help="This parameter will over-ride the list of " "workers defined in the config file. expected " "notation is uname@host,uname:pwd@host or just " "host...") parser.add_option("--distribute-python", type="string", dest="python_bin", help="When running in distributed mode, this Python " "binary will be used across all hosts.") parser.add_option("--is-distributed", action="store_true", dest="is_distributed", help="This parameter is for internal use only. it " "signals to a worker node that it is in " "distributed mode and shouldn't perform certain " "actions.") parser.add_option("--distributed-packages", type="string", dest="distributed_packages", help="Additional packages to be passed to easy_install " "on remote machines when being run in distributed " "mode.") # XXX What exactly is this checking for here?? cmd_args = " ".join([k for k in args if k.find('--distribute') < 0]) options, args = parser.parse_args(args) if len(args) != 2: parser.error("incorrect number of arguments") if not args[1].count('.'): parser.error("invalid argument; should be [class].[method]") if options.as_fast_as_possible: options.bench_sleep_time_min = '0' options.bench_sleep_time_max = '0' options.bench_sleep_time = '0' klass, method = args[1].split('.') if options.distribute: from Distributed import DistributionMgr ret = None try: distmgr = DistributionMgr(args[0], klass, method, options, cmd_args) except UserWarning, error: trace(red_str("Distribution failed with:%s \n" % (error))) try: distmgr.prepare_workers(allow_errors=True) ret = distmgr.run() distmgr.final_collect() except KeyboardInterrupt: trace("* ^C received *") distmgr.abort() return ret
def log(self, message, force=False): """Log a message.""" if force or self.verbose: trace(str(message))