Exemplo n.º 1
0
    def onSetPrivateAction(self):
        utils.trace("ActionManager","onSetPublicAction","")
        
        from AppController import AppController
        bloopTree = AppController.getInstance().getMainViewController().getBloopTree()

        # Grab the currently selected items (could be multiple selections)
        currentItems = bloopTree.selectedItems() 

        if currentItems is None:
            return
        
        # We're going to store our list of files and directories here
        fileList = []
        dirList = []
        
        for item in currentItems:
            itemData = item.getItemData()
            if item.isFile():
                fileList.append(itemData['fileID'])
            elif item.isDirectory():
                dirList.append(itemData['directoryID'])
        
        items = {}
        items['fileList'] = fileList
        items['dirList'] = dirList
        utils.trace("ActionManager","onSetPublicAction",items)
        
        # Grab the remote instance and prepare to make the call
        remoteSession = AppController.getInstance().getRemoteSession()
        remoteSession.setItemVisibility(items, 'private')
        
        # Loop through the items and set their 'filePrivate' fields to 0
        for item in currentItems:
            item.setPrivate()
Exemplo n.º 2
0
def SourceFactory(settings):
    """ return sourceBase derived instance for recent settings"""
    utils.trace( "SourceFactory called", 1 )
    source = None

    if settings.hasTarget():
        if settings.hasMultipleTargets():
            url = settings.targetAt(0)
        else:
            url = settings.target()
        source = ArchiveSource(settings)

    ## \todo move settings access into info class
    if settings.hasSvnTarget():
        url = settings.svnTarget()
        sourceType = utils.getVCSType( url )
        if sourceType == "svn":
            source = SvnSource(settings)
        elif sourceType == "hg":
            source = HgSource(settings)
        elif sourceType == "git":
            source = GitSource(settings)

    if source == None:
        utils.die("none or unsupported source system set")
    if not source.subinfo:
        source.subinfo = settings
    source.url = url
    return source
Exemplo n.º 3
0
    def dummyUpload(self):
        '''Proof of concept method to upload a file to mybloop'''
        from base64 import b64encode
        '''will upload a dummy file'''
        fp = open('01.mp3')
        fp.seek(0,2)

        fileSize = fp.tell()

        fp.seek(0)

        chunkNum = 0
        chunkSize = 131072
        offset = 0


        while fp.tell() < fileSize:

            fp.seek(offset)
            chunkData = fp.read(chunkSize)
            
            trace("RemoteSession","dummyUpload","Sending chunk")
            self.server.dummyUpload(chunkNum,
                                    chunkSize,
                                    b64encode(chunkData))

            offset += chunkSize
            trace("RemoteSession","dummyUpload","Uploaded " + str(offset))

        fp.close()
Exemplo n.º 4
0
    def __init__(self,uploadTokenDict,blooploadChunkFilename, remoteSession, fileObject):
        GetterSetter.__init__(self)

        if remoteSession is not None:
            self.setRemoteSession(remoteSession)

        if blooploadChunkFilename is not None:
            self.setBlooploadChunkFilename(blooploadChunkFilename)
        
        if fileObject is not None:
            self.setFileObject(fileObject)
        
        if uploadTokenDict == None and blooploadChunkFilename == None:
            '''They want to use an empty object'''
            trace("UploadToken","__init__","Using empty constructor")
            return

        if uploadTokenDict is None:
            trace("UploadToken","__init__","UploadToken Constructor: Missing token dict.")
            raise UploadTokenException("UploadToken Constructor: Missing token dict.")
        
        for x in uploadTokenDict:
            #trace('UploadToken','__init__', x + ':' + str(uploadTokenDict[x]))
            self._set(x,uploadTokenDict[x])

        #End transfer.
        #trace('UploadToken','__init__', "The Upload Token->")
        #trace('UploadToken','__init__', uploadTokenDict)
        
        if uploadTokenDict['status'] == '1' :
            try:
                self._checkIncomingKeys()
            except UploadTokenException, e:
                raise e
Exemplo n.º 5
0
        def local_prep_worker ( worker ):
            virtual_env = os.path.join( self.remote_res_dir , self.tarred_testsdir )
            
            if worker.isdir(  virtual_env ):
                worker.execute ( "rmdir -rf %s" % virtual_env )
            
            worker.execute ( "mkdir -p %s" % virtual_env )
            worker.put ( get_virtualenv_script(),
                         os.path.join( self.remote_res_dir,
                         "virtualenv.py") )
            trace(".")
            worker.execute( "python virtualenv.py %s" %\
                    self.tarred_testsdir,
                    cwdir = self.remote_res_dir )

            tarball = os.path.split(self.tarred_tests)[1]
            remote_tarball = os.path.join( self.remote_res_dir,\
                                           tarball)

            # setup funkload
            worker.execute("./bin/easy_install %s" % self.funkload_location,
                    cwdir = virtual_env)
            
            #unpackage tests.
            worker.put( self.tarred_tests, os.path.join(self.remote_res_dir , tarball))
            worker.execute( "tar -xvf %s" %(tarball), cwdir = self.remote_res_dir)
            worker.execute("rm %s" % remote_tarball)
Exemplo n.º 6
0
    def processUIDS(self, view, protocol, client, 
                    account, request):
        if __debug__:
            trace("processUIDS")

        # request =
        #   0: cmd
        #   1: Mail Client
        #   2: accountUUID
        #   3: proocol specific info
        #       IMAP = tuple containing
        #           0: imap folder UUID
        #           1: UID of last message
        #
        #       POP =  List of UIDS of seen messages

        if self.shuttingDown:
            return None

        if protocol == "IMAP":
            imapFolderUUID = request[3][0]
            imapFolder = view.findUUID(imapFolderUUID)

            # Store the last message UID for the IMAPFolder
            imapFolder.lastMessageUID = request[3][1]

        elif protocol == "POP":
            uids = request[3]

            for uid in uids:
                account.seenMessageUIDS[uid] = "True"

        view.commit()
Exemplo n.º 7
0
 def unpack(self):
     utils.trace( "MultiSource unpack", 2 )
     # pylint: disable=E1101
     # multiple inheritance: MultiSource is never the only
     # superclass, others define self.buildSystemType.
     self.source.buildSystemType = self.buildSystemType
     return self.source.unpack()
Exemplo n.º 8
0
        def local_prep_worker(worker):

            remote_res_dir = os.path.join(self.remote_res_dir, worker.name)
            virtual_env = os.path.join(remote_res_dir, self.tarred_testsdir)

            if worker.isdir(virtual_env):
                worker.execute("rm -rf %s" % virtual_env)

            worker.execute("mkdir -p %s" % virtual_env)
            worker.put(get_virtualenv_script(), os.path.join(remote_res_dir, "virtualenv.py"))

            trace(".")
            worker.execute("%s virtualenv.py %s" % (self.python_bin, self.tarred_testsdir), cwdir=remote_res_dir)

            tarball = os.path.split(self.tarred_tests)[1]
            remote_tarball = os.path.join(remote_res_dir, tarball)

            # setup funkload
            cmd = "./bin/easy_install setuptools ez_setup {funkload}".format(funkload=self.funkload_location)

            if self.distributed_packages:
                cmd += " %s" % self.distributed_packages

            worker.execute(cmd, cwdir=virtual_env)

            # unpackage tests.
            worker.put(self.tarred_tests, os.path.join(remote_res_dir, tarball))
            worker.execute("tar -xvf %s" % tarball, cwdir=remote_res_dir)
            worker.execute("rm %s" % remote_tarball)
Exemplo n.º 9
0
    def processDone(self, view, protocol, client,
                    account, request):
        if __debug__:
            trace("processDone")

        if self.shuttingDown:
            return None

        # request
        #     0: cmd
        #     1: client
        #     2: accountUUID
        dt = self.getDownloadTracker(account)

        if dt.totalDownloaded > 0:
            # This is a PyICU.ChoiceFormat class
            txt = constants.DOWNLOAD_CHANDLER_MESSAGES.format(dt.totalDownloaded)

            setStatusMessage(txt % \
                             {'accountName': account.displayName,
                              'numberTotal': dt.totalDownloaded,
                              'numberNew': dt.totalNewDownloaded,
                              'numberUpdates': dt.totalUpdateDownloaded,
                              'numberDuplicates': dt.totalIgnoreDownloaded,
                              'numberErrors': dt.totalErrorDownloaded})
        else:
            setStatusMessage(constants.DOWNLOAD_NO_MESSAGES % \
                            {'accountName': account.displayName})

        self._resetWorker(account)

        # Post a notification to the Mail Protocol Client
        # That the requested actions are finished.
        reactor.callFromThread(client.requestsComplete)
Exemplo n.º 10
0
def main(args = None):
	opts = docopt.docopt(
		doc = __doc__,
		argv = args)
	try:
		if opts["--no-color"]:
			utils.disable_colors()
		if opts["--verbose"]:
			utils.enable_tracing()
		for name in opts["NAMES"]:
			dist = get_distribution(name)
			for name in required(dist):
				raise Error(dist.project_name, "required by", name)
			for req in dist.requires():
				names = filter(lambda name: name != dist.project_name, required(req))
				if not names:
					uninstall(req.project_name, dryrun = opts["--dry-run"])
				else:
					utils.trace(req.project_name, "not uninstalled, used by", names)
			uninstall(dist.project_name, dryrun = opts["--dry-run"])
#		else:
#			for dist in pip.get_installed_distributions():
#				print utils.magenta(dist.project_name), dist.location
	except utils.Error as exc:
		raise SystemExit(utils.red(exc))
Exemplo n.º 11
0
    def _refreshCache(self, protocol):
        instances = None
        method = None

        if protocol in Mail.ACCOUNT_TYPES:
            instances = self._clientInstances.get(protocol)
            method = Mail.ACCOUNT_TYPES[protocol].getActiveAccounts

        self._view.refresh()

        uuidList = []
        delList  = []

        for acc in method(self._view):
            uuidList.append(acc.itsUUID)

        for accUUID in instances.keys():
            if not accUUID in uuidList:
                client = instances.get(accUUID)
                instances.pop(accUUID)
                del client
                delList.append(accUUID)

        if __debug__:
            s = len(delList)

            if s > 0:
                c = s > 1 and "Clients" or "Client"
                a = s > 1 and "accountUUID's" or "accountUUID"
                trace("removed %s%s with %s %s" % (protocol, c, a, delList))
Exemplo n.º 12
0
    def onDeleteAction(self, val):
        '''
        Prepares for deletion of selected folders and files.
        If the user is sure, it will invoke deleteAction on a worker thread.
        '''
        utils.trace("ActionManager","onDeleteAction","")

        from AppController import AppController
        bloopTree = AppController.getInstance().getMainViewController().getBloopTree()

        # Grab the currently selected items (could be multiple selections)
        currentItems = bloopTree.selectedItems() 

        if currentItems is None:
            return

        if len(currentItems) > 0:
            from PyQt4.Qt import QMessageBox
            delConfirmMsg = i18n.LABEL_CONFIRM_FILE_DELETION_MESSAGE if len(currentItems) == 1 else i18n.LABEL_CONFIRM_FILES_DELETION_MESSAGE
            confirmDelete = QMessageBox.warning(AppController.getInstance().getMainView(), 
                                                i18n.LABEL_CONFIRM_FILES_DELETION_TITLE,
                                                delConfirmMsg,
                                                QMessageBox.Yes | QMessageBox.Cancel,
                                                QMessageBox.Yes);
            if confirmDelete == QMessageBox.Cancel:
                AppController.getInstance().getMainView().statusBar().showMessage("")
                return
            
        AppController.getInstance().getMainView().statusBar().showMessage(i18n.LABEL_BLOOPTREE_STATUS_BAR_DELETING)
        self.hireWorker(self.deleteAction, currentItems)
Exemplo n.º 13
0
 def handleStartElement(self, name, attrs):
     """Called by expat parser on start element."""
     if name == "funkload":
         self.config["version"] = attrs["version"]
         self.config["time"] = attrs["time"]
     elif name == "config":
         self.config[attrs["key"]] = attrs["value"]
         if attrs["key"] == "duration":
             if self.cycle_duration and attrs["value"] != self.cycle_duration:
                 trace("Skipping file %s with different cycle duration %s" % (self.current_file, attrs["value"]))
                 raise EndOfConfig
             self.cycle_duration = attrs["value"]
         elif attrs["key"] == "cycles":
             if self.cycles and attrs["value"] != self.cycles:
                 trace(
                     "Skipping file %s with different cycles %s != %s"
                     % (self.current_file, attrs["value"], self.cycles)
                 )
                 raise EndOfConfig
             self.cycles = attrs["value"]
         elif attrs["key"] == "node":
             self.nodes[self.current_file] = attrs["value"]
     else:
         self.files.append(self.current_file)
         raise EndOfConfig
Exemplo n.º 14
0
    def processEmpty(self, view, protocol, client,
                     account, request):
        if __debug__:
            trace("processEmpty")

        if self.shuttingDown:
            return None
        return
Exemplo n.º 15
0
    def run(self):
        port = 8000
        if self.port:
            port = int(self.port)
        server_address = ('', port)
        trace("Starting debug HTTP server at port %d\n" % port)

        httpd = BaseHTTPServer.HTTPServer(server_address, FunkLoadHTTPRequestHandler)
        httpd.serve_forever()
Exemplo n.º 16
0
 def __init__(self):
     object.__init__(self)
     utils.trace( "MultiSource __init__", 2 )
     # pylint: disable=E1101
     # multiple inheritance: MultiSource is never the only
     # superclass, others define self.source, self.subinfo etc.
     # TODO: This code should mostly be in the class defining self.source etc.
     self.source = SourceFactory(self.subinfo)
     self.source.localFileNames = self.localFileNames.__get__(self, MultiSource)
Exemplo n.º 17
0
 def final_collect(self):
     expr = re.compile("Log\s+xml:\s+(.*?)\n")
     for worker, results in self._worker_results.items():
         res = expr.findall(results)
         if res:
             remote_file = res[0]
             file = os.path.split(remote_file)[1]
             local_file = os.path.join(self.distribution_output, "%s-%s" % (worker.host, file))
             worker.get(remote_file, local_file)
             trace("* Received bench log from [%s] into %s\n" % (worker.host, local_file))
Exemplo n.º 18
0
 def put(self, local_path, remote_path):
     """
     performs a copy from `local_path` to `remote_path`
     For performing the inverse operation, use the :meth:`get`
     """
     try:
         sftp = self.connection.open_sftp()
         sftp.put(local_path, remote_path)
     except Exception, error:
         trace("failed to put %s->%s with error %s\n" % (local_path, remote_path, error))
Exemplo n.º 19
0
 def removeThreads(self, number_of_threads):
     """Removes threads. Used to dynamically remove threads during a
        debug bench run."""
     self.thread_creation_lock.acquire()
     try:
         trace('* Removing threads: ')
         self.deleteThreads(number_of_threads)
         trace(' done.\n')
     finally:
         self.thread_creation_lock.release()
Exemplo n.º 20
0
 def extractScript(self):
     """Convert a tcpwatch capture into a FunkLoad script."""
     files = self.searchFiles()
     requests = self.extractRequests(files)
     code = [self.convertToFunkLoad(request) for request in requests]
     if not code:
         trace("Sorry no action recorded.\n")
         return ""
     code.insert(0, "")
     return self.reindent("\n".join(code))
Exemplo n.º 21
0
 def get(self, remote_path, local_path):
     """
     performs a copy from ``remote_path`` to ``local_path``.
     For performing the inverse operation, use the :meth:`put`
     """
     try:
         sftp = self.connection.open_sftp()
         sftp.get(remote_path, local_path)
     except Exception, error:
         trace("failed to get %s->%s with error %s\n" % \
                (local_path, remote_path, error))
Exemplo n.º 22
0
    def processError(self, view, protocol, client,
                     account, request):
        if __debug__:
            trace("processError")

        if self.shuttingDown:
            return None

        #Clear the status bar message
        setStatusMessage(u"")
        self._resetWorker(account)
Exemplo n.º 23
0
    def prepare_workers(self, allow_errors=False):
        """
        initialized the sandboxes in each worker node to prepare for a 
        bench run. the additional parameter `allow_errors` essentially
        will make the distinction between ignoring unresponsive/inappropriate
        nodes - or raising an error and failing the entire bench.
        """
        # right, lets figure out if funkload can be setup on each host

        def local_prep_worker(worker):
            virtual_env = os.path.join(self.remote_res_dir, self.tarred_testsdir)

            if worker.isdir(virtual_env):
                worker.execute("rmdir -rf %s" % virtual_env)

            worker.execute("mkdir -p %s" % virtual_env)
            worker.put(get_virtualenv_script(), os.path.join(self.remote_res_dir, "virtualenv.py"))
            trace(".")
            worker.execute("python virtualenv.py %s" % self.tarred_testsdir, cwdir=self.remote_res_dir)
            tarball = os.path.split(self.tarred_tests)[1]
            remote_tarball = os.path.join(self.remote_res_dir, tarball)

            # setup funkload
            worker.execute("./bin/easy_install %s" % self.funkload_location, cwdir=virtual_env)
            trace(".")
            worker.execute("python virtualenv.py %s" % self.remote_res_dir)
            # unpackage tests.
            worker.put(self.tarred_tests, os.path.join(self.remote_res_dir, tarball))
            worker.execute("tar -xvf %s" % (tarball), cwdir=self.remote_res_dir)
            worker.execute("rm %s" % remote_tarball)

        threads = []
        trace("* Preparing sandboxes for %d workers." % len(self._workers))
        for worker in list(self._workers):
            if worker.connected:
                which_python = worker.execute("which python")
                if not which_python and not allow_errors:
                    raise RuntimeError("%s doesn't have python" % worker.host)
                elif not which_python:
                    self._workers.remove(worker)
                else:
                    # python was found lets see if we need to install funkload

                    threads.append(threading.Thread(target=local_prep_worker, args=(worker,)))
            elif allow_errors:
                self._workers.remove(worker)
            else:
                raise RuntimeError("%s is not contactable with error %s" % (worker.host, worker.error))

        [k.start() for k in threads]
        [k.join() for k in threads]
        trace("\n")
        if not self._workers:
            raise RuntimeError("no workers available for distribution")
Exemplo n.º 24
0
 def logging(self):
     """Log activity during duration."""
     duration = self.duration
     end_time = time.time() + duration
     trace("* Logging for %ds (until %s): " % (duration, datetime.fromtimestamp(end_time).isoformat()))
     set_recording_flag(True)
     while time.time() < end_time:
         # wait
         time.sleep(1)
     set_recording_flag(False)
     trace(" done.\n")
Exemplo n.º 25
0
 def startThreads(self, cycle, number_of_threads):
     """Starts threads."""
     self.thread_creation_lock.acquire()
     try:
         trace("* Current time: %s\n" % datetime.now().isoformat())
         trace("* Starting threads: ")
         set_recording_flag(False)
         threads = self.createThreads(cycle, number_of_threads)
         self.threads.extend(threads)
     finally:
         set_recording_flag(True)
         self.thread_creation_lock.release()
Exemplo n.º 26
0
    def _resetWorker(self, account):
        if __debug__:
            trace("_resetWorker")

        dt = self.getDownloadTracker(account)

        dt.totalDownloaded = 0
        dt.totalNewDownloaded = 0
        dt.totalUpdateDownloaded = 0
        dt.totalIgnoreDownloaded = 0
        dt.totalErrorDownloaded = 0
        dt.lastTotalDownloaded = 0
Exemplo n.º 27
0
def run_distributed(options, module_name, class_name, method_name, sys_args):
    ret = None
    from funkload.Distributed import DistributionMgr
    global _manager
    
    try:
        distmgr = DistributionMgr(
            module_name, class_name, method_name, options, sys_args)
        _manager = distmgr
    except UserWarning, error:
        trace(red_str("Distribution failed with:%s \n" % (error)))
        return 1
Exemplo n.º 28
0
 def log_r_X_z(self):
     X_m_tau = minus(self.Xz, self.tau)
     X_m_tau_vec = T.reshape(X_m_tau, [self.B * self.R, 1])
     X_m_tau_vec.name = 'X_m_tau_vec'
     if self.Tau_isDiagonal:
         log_rX_z = -0.5 * self.R * self.B * log2pi - 0.5 * self.R * self.logDetTau \
         - 0.5 * trace(dot(X_m_tau_vec.T, div(X_m_tau_vec,self.Tau)))
     else:
         log_rX_z = -0.5 * self.R * self.B * log2pi - 0.5 * self.R * self.logDetTau \
         - 0.5 * trace(dot(X_m_tau_vec.T, dot(self.iTau, X_m_tau_vec)))
     log_rX_z.name = 'log_rX_z'
     return log_rX_z
Exemplo n.º 29
0
    def loadTestsFromTestCase(self, testCaseClass):
        if not issubclass(testCaseClass, FunkLoadTestCase):
            trace(red_str("Skipping "+ testCaseClass))
            return []
        testCaseNames = self.getTestCaseNames(testCaseClass)
        if not testCaseNames and hasattr(testCaseClass, 'runTest'):
            testCaseNames = ['runTest']

        return [dict(module_name = testCaseClass.__module__,
                     class_name = testCaseClass.__name__,
                     method_name = x)
                for x in testCaseNames]
    def KL_qp(self):

        if self.continuous:
            kappa_outer = dot(self.kappa, self.kappa.T, 'kappa_outer')
            AtA = dot(self.A.T, self.A)
            KL = 0.5*self.Q*trace(self.C) + 0.5*trace(dot(AtA,kappa_outer)) \
                + 0.5*self.Q+trace(dot(AtA,self.Kappa)) - 0.5*self.Q*self.B - 0.5*self.Q*self.logDetC
            KL.name = 'KL_qp'
        else:
            raise RuntimeError("Case not implemented")

        return KL
    try:
        pwd = String(password)
        x = EncryptionUtils.decryptString(cipher_text, pwd.toCharArray())
        restored_text = String(x)
        fh = open(outputfile, "w")
        fh.write(str(restored_text))
        fh.close()
        System.exit(0)
    except EncryptionException, e:
        # Catch the exact exception to get the real cause
        trace("SEVERE", "Error in decrypting file: %s" % e.getCause())
        System.exit(-1)
    except:
        exc_type, exc_obj, exc_tb = sys.exc_info()
        eeString = traceback.format_exception(exc_type, exc_obj, exc_tb)
        trace("SEVERE", "Error in decrypting file: %s" % eeString)
        System.exit(-1)


def encrypt_file(clear_text, password, outputfile):
    try:
        pwd = String(password)
        x = EncryptionUtils.encryptString(clear_text, pwd.toCharArray())
        encrypted_text = String(x)
        fh = open(outputfile, "w")
        fh.write(str(encrypted_text))
        fh.close()
        System.exit(0)
    except EncryptionException, e:
        # Catch the exact exception to get the real cause
        trace("SEVERE", "Error in encrypting file: %s" % e.getCause())
Exemplo n.º 32
0
def calc_bias(data, metric, vanilla_result, fair_result):
    """
    calculate bias reduction
    :param data: input data
    :param metric: similarity metric (jaccard or cosine)
    :param vanilla_result: vanilla mining result
    :param fair_result: debiased mining result
    :return: a dict containing bias reductions of training edges, validation edges and test edges
    """
    biases = dict()
    # load graph
    adj_train = data['adjacency_train']
    # build similarity matrix
    sim = utils.filter_similarity_matrix(utils.get_similarity_matrix(
        adj_train, metric=metric),
                                         sigma=0.75)
    lap = laplacian(sim)

    # calculate training bias
    biases['train'] = [
        utils.trace(vanilla_result.T @ lap @ vanilla_result),
        utils.trace(fair_result.T @ lap @ fair_result)
    ]
    reduction = 1 - (biases['train'][1] / biases['train'][0])
    biases['train'].clear()
    biases['train'].append(reduction)

    # calculating bias on validation set
    val_bias = [0, 0]  # first one is vanilla, second one is fair
    for src, tgt in data['val_edge_pos']:
        vanilla_edge_bias = (lp_diff(vanilla_result[src],
                                     vanilla_result[tgt],
                                     ord=2)**2) * sim[src, tgt]
        fair_edge_bias = (lp_diff(fair_result[src], fair_result[tgt], ord=2)**
                          2) * sim[src, tgt]
        val_bias[0] += vanilla_edge_bias
        val_bias[1] += fair_edge_bias
    for src, tgt in data['val_edge_neg']:
        vanilla_edge_bias = (lp_diff(vanilla_result[src],
                                     vanilla_result[tgt],
                                     ord=2)**2) * sim[src, tgt]
        fair_edge_bias = (lp_diff(fair_result[src], fair_result[tgt], ord=2)**
                          2) * sim[src, tgt]
        val_bias[0] += vanilla_edge_bias
        val_bias[1] += fair_edge_bias
    reduction = 1 - (val_bias[1] / val_bias[0])
    val_bias.clear()
    val_bias.append(reduction)
    biases['validation'] = val_bias

    # calculating bias on test set
    test_bias = [0, 0]  # first one is vanilla, second one is fair
    for src, tgt in data['test_edge_pos']:
        vanilla_edge_bias = (lp_diff(vanilla_result[src],
                                     vanilla_result[tgt],
                                     ord=2)**2) * sim[src, tgt]
        fair_edge_bias = (lp_diff(fair_result[src], fair_result[tgt], ord=2)**
                          2) * sim[src, tgt]
        test_bias[0] += vanilla_edge_bias
        test_bias[1] += fair_edge_bias
    for src, tgt in data['test_edge_neg']:
        vanilla_edge_bias = (lp_diff(vanilla_result[src],
                                     vanilla_result[tgt],
                                     ord=2)**2) * sim[src, tgt]
        fair_edge_bias = (lp_diff(fair_result[src], fair_result[tgt], ord=2)**
                          2) * sim[src, tgt]
        test_bias[0] += vanilla_edge_bias
        test_bias[1] += fair_edge_bias
    reduction = 1 - (test_bias[1] / test_bias[0])
    test_bias.clear()
    test_bias.append(reduction)
    biases['test'] = test_bias

    return biases
Exemplo n.º 33
0
    def run(self):
        """
        """
        threads = []
        trace("* Starting %d workers" % len(self._workers))

        self.startMonitors()
        for worker in self._workers:
            remote_res_dir = os.path.join(self.remote_res_dir, worker.name)
            venv = os.path.join(remote_res_dir, self.tarred_testsdir)
            obj = worker.threaded_execute('bin/fl-run-bench %s' %
                                          self.cmd_args,
                                          cwdir=venv)
            trace(".")
            threads.append(obj)
        trace("\n")
        [t.join() for t in threads]
        trace("\n")

        for thread, worker in zip(threads, self._workers):
            self._worker_results[worker] = thread.output.read()
            trace("* [%s] returned\n" % worker.name)
            err_string = thread.err.read()
            if err_string:
                trace("\n".join("  [%s]: %s" % (worker.name, k) for k \
                        in err_string.split("\n") if k.strip()))
            trace("\n")

        self.stopMonitors()
        self.correlate_statistics()
Exemplo n.º 34
0
            trace(red_str("Distribution failed with:%s \n" % (error)))

        try:
            distmgr.prepare_workers(allow_errors=True)
            ret = distmgr.run()
            distmgr.final_collect()
        except KeyboardInterrupt:
            trace("* ^C received *")
            distmgr.abort()

        return ret
    else:
        bench = BenchRunner(args[0], klass, method, options)

        # Start a HTTP server optionally
        if options.debugserver:
            http_server_thread = FunkLoadHTTPServer(bench, options.debugport)
            http_server_thread.start()

        ret = None
        try:
            ret = bench.run()
        except KeyboardInterrupt:
            trace("* ^C received *")
        return ret


if __name__ == '__main__':
    ret = main()
    sys.exit(ret)
Exemplo n.º 35
0
    def __init__(self, module_name, class_name, method_name, options,
                                                                cmd_args):
        """
        mirrors the initialization of :class:`funkload.BenchRunner.BenchRunner`
        """
        # store the args. these can be passed to BenchRunner later.
        self.module_name = module_name
        self.class_name = class_name
        self.method_name = method_name
        self.options = options
        self.cmd_args = cmd_args

        self.cmd_args += " --is-distributed"

        module = load_module(module_name)
        module_file = module.__file__
        self.tarred_tests, self.tarred_testsdir = package_tests(module_file)

        self.remote_res_dir = "/tmp/funkload-bench-sandbox/"

        test = load_unittest(self.module_name, class_name,
                             mmn_encode(method_name, 0, 0, 0), options)

        self.config_path = test._config_path
        self.result_path = test.result_path
        self.class_title = test.conf_get('main', 'title')
        self.class_description = test.conf_get('main', 'description')
        self.test_id = self.method_name
        self.test_url = test.conf_get('main', 'url')
        self.cycles = map(int, test.conf_getList('bench', 'cycles'))
        self.duration = test.conf_getInt('bench', 'duration')
        self.startup_delay = test.conf_getFloat('bench', 'startup_delay')
        self.cycle_time = test.conf_getFloat('bench', 'cycle_time')
        self.sleep_time = test.conf_getFloat('bench', 'sleep_time')
        self.sleep_time_min = test.conf_getFloat('bench', 'sleep_time_min')
        self.sleep_time_max = test.conf_getFloat('bench', 'sleep_time_max')
        if test.conf_get('distribute', 'channel_timeout', '', quiet=True):
            self.channel_timeout = test.conf_getFloat(
                                        'distribute', 'channel_timeout')
        else:
            self.channel_timeout = None
        self.threads = []  # Contains list of ThreadData objects
        self.last_thread_id = -1
        self.thread_creation_lock = threading.Lock()

        if options.python_bin:
            self.python_bin = options.python_bin
        else:
            self.python_bin = test.conf_get(
                'distribute', 'python_bin', 'python')

        if options.distributed_packages:
            self.distributed_packages = options.distributed_packages
        else:
            self.distributed_packages = test.conf_get(
                                                'distribute', 'packages', '')

        try:
            desc = getattr(test, self.method_name).__doc__.strip()
        except:
            desc = ""
        self.test_description = test.conf_get(self.method_name, 'description',
                                              desc)
        # make a collection output location
        if options.distributed_log_path:
            self.distribution_output = options.distributed_log_path
        elif test.conf_get('distribute', 'log_path', '', quiet=True):
            self.distribution_output = test.conf_get('distribute', 'log_path')
        else:
            raise UserWarning("log_path isn't defined in section [distribute]")

        # check if user has overridden the default funkload distro download
        # location this will be used to download funkload on the worker nodes.
        self.funkload_location = test.conf_get(
            'distribute', 'funkload_location', 'funkload')

        if not os.path.isdir(self.distribution_output):
            os.makedirs(self.distribution_output)

        # check if hosts are in options
        workers = []                  # list of (host, port, descr)
        if options.workerlist:
            for h in  options.workerlist.split(","):
                cred_host = h.split("@")
                if len(cred_host) == 1:
                    uname, pwd, host = None, None, cred_host[0]
                else:
                    cred = cred_host[0]
                    host = cred_host[1]
                    uname_pwd = cred.split(":")
                    if len(uname_pwd) == 1:
                        uname, pwd = uname_pwd[0], None
                    else:
                        uname, pwd = uname_pwd

                workers.append({
                    "name": host,
                    "host": host,
                    "password": pwd,
                    "username": uname,
                    "channel_timeout": self.channel_timeout})
        else:
            hosts = test.conf_get('workers', 'hosts', '', quiet=True).split()
            for host in hosts:
                host = host.strip()
                workers.append({
                    "name": host,
                    "host": test.conf_get(host, "host",host),
                    "password": test.conf_get(host, 'password', ''),
                    "username": test.conf_get(host, 'username', ''),
                    "channel_timeout": self.channel_timeout})

        self._workers = []
        [self._workers.append(SSHDistributor(**w)) for w in workers]
        self._worker_results = {}
        trace(str(self))

        # setup monitoring
        monitor_hosts = []                  # list of (host, port, descr)
        if not options.is_distributed:
            hosts = test.conf_get('monitor', 'hosts', '', quiet=True).split()
            for host in sorted(hosts):
                name = host
                host = test.conf_get(host,'host',host.strip())
                monitor_hosts.append((name, host, test.conf_getInt(name, 'port'),
                                      test.conf_get(name, 'description', '')))
        self.monitor_hosts = monitor_hosts
        # keep the test to use the result logger for monitoring
        # and call setUp/tearDown Cycle
        self.test = test
Exemplo n.º 36
0
class FunkLoadTestCase(unittest.TestCase):
    """Unit test with browser and configuration capabilties."""

    # ------------------------------------------------------------
    # Initialisation
    #
    def __init__(self, methodName='runTest', options=None):
        """Initialise the test case.

        Note that methodName is encoded in bench mode to provide additional
        information like thread_id, concurrent virtual users..."""
        if mmn_is_bench(methodName):
            self.in_bench_mode = True
        else:
            self.in_bench_mode = False
        self.test_name, self.cycle, self.cvus, self.thread_id = mmn_decode(
            methodName)
        self.meta_method_name = methodName
        self.suite_name = self.__class__.__name__
        unittest.TestCase.__init__(self, methodName=self.test_name)
        self._response = None
        self._options = options
        self.debug_level = getattr(options, 'debug_level', 0)
        self._funkload_init()
        self._dump_dir = getattr(options, 'dump_dir', None)
        self._dumping = self._dump_dir and True or False
        self._viewing = getattr(options, 'firefox_view', False)
        self._accept_invalid_links = getattr(options, 'accept_invalid_links',
                                             False)
        self._bench_label = getattr(options, 'label', None)
        self._stop_on_fail = getattr(options, 'stop_on_fail', False)
        self._pause = getattr(options, 'pause', False)
        self._keyfile_path = None
        self._certfile_path = None
        self._accept_gzip = False
        if self._viewing and not self._dumping:
            # viewing requires dumping contents
            self._dumping = True
            self._dump_dir = mkdtemp('_funkload')
        self._loop_mode = getattr(options, 'loop_steps', False)
        if self._loop_mode:
            if ':' in options.loop_steps:
                steps = options.loop_steps.split(':')
                self._loop_steps = range(int(steps[0]), int(steps[1]))
            else:
                self._loop_steps = [int(options.loop_steps)]
            self._loop_number = options.loop_number
            self._loop_recording = False
            self._loop_records = []
        if sys.version_info >= (2, 5):
            self.__exc_info = sys.exc_info

    def _funkload_init(self):
        """Initialize a funkload test case using a configuration file."""
        # look into configuration file
        config_path = getattr(self._options, 'config', None)
        if not config_path:
            config_directory = os.getenv('FL_CONF_PATH', '.')
            config_path = os.path.join(config_directory,
                                       self.__class__.__name__ + '.conf')
        config_path = os.path.abspath(os.path.expanduser(config_path))
        if not os.path.exists(config_path):
            config_path = "Missing: " + config_path
        config = ConfigParser()
        config.read(config_path)
        self._config = config
        self._config_path = config_path
        self.conf = ConfSectionFinder(self)
        self.default_user_agent = self.conf_get('main',
                                                'user_agent',
                                                'FunkLoad/%s' % get_version(),
                                                quiet=True)
        if self.in_bench_mode:
            section = 'bench'
        else:
            section = 'ftest'
        self.setOkCodes(
            self.conf_getList(section,
                              'ok_codes', [200, 301, 302, 303, 307],
                              quiet=True))
        self.sleep_time_min = self.conf_getFloat(section, 'sleep_time_min', 0)
        self.sleep_time_max = self.conf_getFloat(section, 'sleep_time_max', 0)
        self._simple_fetch = self.conf_getInt(section,
                                              'simple_fetch',
                                              0,
                                              quiet=True)
        self.log_to = self.conf_get(section, 'log_to', 'console file')
        self.log_path = self.conf_get(section, 'log_path', 'funkload.log')
        self.result_path = os.path.abspath(
            self.conf_get(section, 'result_path', 'funkload.xml'))

        # init loggers
        if self.in_bench_mode:
            level = logging.INFO
        else:
            level = logging.DEBUG
        self.logger = get_default_logger(self.log_to,
                                         self.log_path,
                                         level=level)
        self.logger_result = get_default_logger(log_to="xml",
                                                log_path=self.result_path,
                                                name="FunkLoadResult")
        #self.logd('_funkload_init config [%s], log_to [%s],'
        #          ' log_path [%s], result [%s].' % (
        #    self._config_path, self.log_to, self.log_path, self.result_path))

        # init webunit browser (passing a fake methodName)
        self._browser = WebTestCase(methodName='log')
        self.clearContext()

        #self.logd('# FunkLoadTestCase._funkload_init done')

    def setOkCodes(self, ok_codes):
        """Set ok codes."""
        self.ok_codes = map(int, ok_codes)

    def clearContext(self):
        """Reset the testcase."""
        self._browser.clearContext()
        self._browser.css = {}
        self._browser.history = []
        self._browser.extra_headers = []
        if self.debug_level >= 3:
            self._browser.debug_headers = True
        else:
            self._browser.debug_headers = False
        self.step_success = True
        self.test_status = 'Successful'
        self.steps = 0
        self.page_responses = 0
        self.total_responses = 0
        self.total_time = 0.0
        self.total_pages = self.total_images = 0
        self.total_links = self.total_redirects = 0
        self.total_xmlrpc = 0
        self.clearBasicAuth()
        self.clearHeaders()
        self.clearKeyAndCertificateFile()
        self.setUserAgent(self.default_user_agent)

        self.logdd('FunkLoadTestCase.clearContext done')

    #------------------------------------------------------------
    # browser simulation
    #
    def _connect(self,
                 url,
                 params,
                 ok_codes,
                 rtype,
                 description,
                 redirect=False,
                 consumer=None):
        """Handle fetching, logging, errors and history."""
        if params is None and rtype in ('post', 'put'):
            # enable empty put/post
            params = []
        t_start = time.time()
        try:
            response = self._browser.fetch(url,
                                           params,
                                           ok_codes=ok_codes,
                                           key_file=self._keyfile_path,
                                           cert_file=self._certfile_path,
                                           method=rtype,
                                           consumer=consumer)
        except:
            etype, value, tback = sys.exc_info()
            t_stop = time.time()
            t_delta = t_stop - t_start
            self.total_time += t_delta
            self.step_success = False
            self.test_status = 'Failure'
            self.logd(' Failed in %.3fs' % t_delta)
            if etype is HTTPError:
                self._log_response(value.response,
                                   rtype,
                                   description,
                                   t_start,
                                   t_stop,
                                   log_body=True)
                if self._dumping:
                    self._dump_content(value.response, description)
                raise self.failureException, str(value.response)
            else:
                self._log_response_error(url, rtype, description, t_start,
                                         t_stop)
                if etype is SocketError:
                    raise SocketError("Can't load %s." % url)
                raise
        t_stop = time.time()
        # Log response
        t_delta = t_stop - t_start
        self.total_time += t_delta
        if redirect:
            self.total_redirects += 1
        elif rtype != 'link':
            self.total_pages += 1
        else:
            self.total_links += 1

        if rtype in ('put', 'post', 'get', 'delete'):
            # this is a valid referer for the next request
            self.setHeader('Referer', url)
        self._browser.history.append((rtype, url))
        self.logd(' Done in %.3fs' % t_delta)
        if self._accept_gzip:
            if response.headers is not None and response.headers.get(
                    'Content-Encoding') == 'gzip':
                buf = StringIO(response.body)
                response.body = gzip.GzipFile(fileobj=buf).read()
        self._log_response(response, rtype, description, t_start, t_stop)
        if self._dumping:
            self._dump_content(response, description)
        return response

    def _browse(self,
                url_in,
                params_in=None,
                description=None,
                ok_codes=None,
                method='post',
                follow_redirect=True,
                load_auto_links=True,
                sleep=True):
        """Simulate a browser handle redirects, load/cache css and images."""
        self._response = None
        # Loop mode
        if self._loop_mode:
            if self.steps == self._loop_steps[0]:
                self._loop_recording = True
                self.logi('Loop mode start recording')
            if self._loop_recording:
                self._loop_records.append(
                    (url_in, params_in, description, ok_codes, method,
                     follow_redirect, load_auto_links, False))
        # ok codes
        if ok_codes is None:
            ok_codes = self.ok_codes
        if type(params_in) is DictType:
            params_in = params_in.items()
        params = []
        if params_in:
            if isinstance(params_in, Data):
                params = params_in
            else:
                for key, value in params_in:
                    if type(value) is DictType:
                        for val, selected in value.items():
                            if selected:
                                params.append((key, val))
                    elif type(value) in (ListType, TupleType):
                        for val in value:
                            params.append((key, val))
                    else:
                        params.append((key, value))

        if method == 'get' and params:
            url = url_in + '?' + urlencode(params)
        else:
            url = url_in
        if method == 'get':
            params = None

        if method == 'get':
            if not self.in_bench_mode:
                self.logd('GET: %s\n\tPage %i: %s ...' %
                          (url, self.steps, description or ''))
        else:
            url = url_in
            if not self.in_bench_mode:
                self.logd('%s: %s %s\n\tPage %i: %s ...' %
                          (method.upper(), url, str(params), self.steps,
                           description or ''))
            # Fetching
        response = self._connect(url, params, ok_codes, method, description)

        # Check redirection
        if follow_redirect and response.code in (301, 302, 303, 307):
            max_redirect_count = 10
            thread_sleep()  # give a chance to other threads
            while response.code in (301, 302, 303, 307) and max_redirect_count:
                # Figure the location - which may be relative
                newurl = response.headers['Location']
                url = urljoin(url_in, newurl)
                # Save the current url as the base for future redirects
                url_in = url
                self.logd(' Load redirect link: %s' % url)
                # Use the appropriate method for redirection
                if response.code in (302, 303):
                    method = 'get'
                if response.code == 303:
                    # 303 is HTTP/1.1, make sure the connection
                    # is not in keep alive mode
                    self.setHeader('Connection', 'close')
                response = self._connect(url,
                                         None,
                                         ok_codes,
                                         rtype=method,
                                         description=None,
                                         redirect=True)
                max_redirect_count -= 1
            if not max_redirect_count:
                self.logd(' WARNING Too many redirects give up.')

        # Load auto links (css and images)
        response.is_html = is_html(response.body)
        if load_auto_links and response.is_html and not self._simple_fetch:
            self.logd(' Load css and images...')
            page = response.body
            t_start = time.time()
            c_start = self.total_time
            try:
                # pageImages is patched to call _log_response on all links
                self._browser.pageImages(url, page, self)
            except HTTPError, error:
                if self._accept_invalid_links:
                    if not self.in_bench_mode:
                        self.logd('  ' + str(error))
                else:
                    t_stop = time.time()
                    t_delta = t_stop - t_start
                    self.step_success = False
                    self.test_status = 'Failure'
                    self.logd('  Failed in ~ %.2fs' % t_delta)
                    # XXX The duration logged for this response is wrong
                    self._log_response(error.response,
                                       'link',
                                       None,
                                       t_start,
                                       t_stop,
                                       log_body=True)
                    raise self.failureException, str(error)
            c_stop = self.total_time
            self.logd('  Done in %.3fs' % (c_stop - c_start))
        if sleep:
            self.sleep()
        self._response = response

        # Loop mode
        if self._loop_mode and self.steps == self._loop_steps[-1]:
            self._loop_recording = False
            self.logi('Loop mode end recording.')
            t_start = self.total_time
            count = 0
            for i in range(self._loop_number):
                self.logi('Loop mode replay %i' % i)
                for record in self._loop_records:
                    count += 1
                    self.steps += 1
                    self._browse(*record)
            t_delta = self.total_time - t_start
            text = ('End of loop: %d pages rendered in %.3fs, '
                    'avg of %.3fs per page, '
                    '%.3f SPPS without concurrency.' %
                    (count, t_delta, t_delta / count, count / t_delta))
            self.logi(text)
            trace(text + '\n')

        return response
Exemplo n.º 37
0
 def run(self):
     """Run a test in loop."""
     while (self.thread_signaller.running()):
         test_result = unittest.TestResult()
         self.test.clearContext()
         self.test(test_result)
         if test_result.wasSuccessful():
             if recording():
                 add_cycle_result('success')
                 if self.color:
                     trace(green_str('.'))
                 else:
                     trace('.')
         else:
             if len(test_result.errors):
                 if recording():
                     add_cycle_result('error')
                 if self.color:
                     trace(red_str('E'))
                 else:
                     trace('E')
             else:
                 if recording():
                     add_cycle_result('failure')
                 if self.color:
                     trace(red_str('F'))
                 else:
                     trace('F')
             if self.debug:
                 for (test, error) in test_result.errors:
                     trace("ERROR %s: %s" % (str(test), str(error)))
                 for (test, error) in test_result.failures:
                     trace("FAILURE %s: %s" % (str(test), str(error)))
         thread_sleep(self.sleep_time)
Exemplo n.º 38
0
    def __init__(self, argv=None):
        if self.server_name is None:
            self.server_name = self.__class__.__name__
        if argv is None:
            argv = sys.argv
        conf_path, options = self.parseArgs(argv)
        self.default_log_path = self.server_name + '.log'
        self.default_pid_path = self.server_name + '.pid'
        self.server = None
        self.quit = False

        # read conf
        conf = ConfigParser()
        conf.read(conf_path)
        self.conf_path = conf_path
        self.host = conf.get('server', 'host')
        self.port = conf.getint('server', 'port')
        try:
            self.pid_path = conf.get('server', 'pid_path')
        except NoOptionError:
            self.pid_path = self.default_pid_path
        try:
            log_path = conf.get('server', 'log_path')
        except NoOptionError:
            log_path = self.default_log_path

        if is_server_running(self.host, self.port):
            trace("Server already running on %s:%s." % (self.host, self.port))
            sys.exit(0)

        trace('Starting %s server at http://%s:%s/' %
              (self.server_name, self.host, self.port))
        # init logger
        if options.verbose:
            level = logging.DEBUG
        else:
            level = logging.INFO
        if options.debug:
            log_to = 'file console'
        else:
            log_to = 'file'
        self.logger = get_default_logger(log_to,
                                         log_path,
                                         level=level,
                                         name=self.server_name)
        # subclass init
        self._init_cb(conf, options)

        # daemon mode
        if not options.debug:
            trace(' as daemon.\n')
            close_logger(self.server_name)
            create_daemon()
            # re init the logger
            self.logger = get_default_logger(log_to,
                                             log_path,
                                             level=level,
                                             name=self.server_name)
        else:
            trace(' in debug mode.\n')

        # init rpc
        self.initServer()
Exemplo n.º 39
0
    def run(self):
        """Run all the cycles.

        return 0 on success, 1 if there were some failures and -1 on errors."""
        trace(str(self))
        trace("Benching\n")
        trace("========\n\n")
        cycle = total_success = total_failures = total_errors = 0

        self.logr_open()
        trace("* setUpBench hook: ...")
        self.test.setUpBench()
        trace(' done.\n')
        self.getMonitorsConfig()
        trace('\n')
        for cvus in self.cycles:
            t_start = time.time()
            reset_cycle_results()
            text = "Cycle #%i with %s virtual users\n" % (cycle, cvus)
            trace(text)
            trace('-' * (len(text) - 1) + "\n\n")
            monitor_key = '%s:%s:%s' % (self.method_name, cycle, cvus)
            trace("* setUpCycle hook: ...")
            self.test.setUpCycle()
            trace(' done.\n')
            self.startMonitors(monitor_key)
            self.startThreads(cycle, cvus)
            self.logging()
            #self.dumpThreads()
            self.stopThreads()
            self.stopMonitors(monitor_key)
            cycle += 1
            trace("* tearDownCycle hook: ...")
            self.test.tearDownCycle()
            trace(' done.\n')
            t_stop = time.time()
            trace("* End of cycle, %.2fs elapsed.\n" % (t_stop - t_start))
            success, failures, errors = get_cycle_results()
            status, code = get_status(success, failures, errors, self.color)
            trace("* Cycle result: **%s**, "
                  "%i success, %i failure, %i errors.\n\n" %
                  (status, success, failures, errors))
            total_success += success
            total_failures += failures
            total_errors += errors
        trace("* tearDownBench hook: ...")
        self.test.tearDownBench()
        trace(' done.\n\n')
        self.logr_close()

        # display bench result
        trace("Result\n")
        trace("======\n\n")
        trace("* Success: %s\n" % total_success)
        trace("* Failures: %s\n" % total_failures)
        trace("* Errors: %s\n\n" % total_errors)
        status, code = get_status(total_success, total_failures, total_errors)
        trace("Bench status: **%s**\n" % status)
        return code
Exemplo n.º 40
0
 def __init__(self, level):
     self.print_debug = trace(level)
Exemplo n.º 41
0
 def gen_reg(self):
     reg = 0
     for infer in self.gnet:
         for param in infer.parameters():
             reg += trace(param)
     return reg
Exemplo n.º 42
0
def main():
    """ReportBuilder main."""
    parser = OptionParser(USAGE,
                          formatter=TitledHelpFormatter(),
                          version="FunkLoad %s" % get_version())
    parser.add_option("-H",
                      "--html",
                      action="store_true",
                      default=False,
                      dest="html",
                      help="Produce an html report.")
    parser.add_option("--org",
                      action="store_true",
                      default=False,
                      dest="org",
                      help="Org-mode report.")
    parser.add_option("-P",
                      "--with-percentiles",
                      action="store_true",
                      default=True,
                      dest="with_percentiles",
                      help=("Include percentiles in tables, use 10%, 50% and"
                            " 90% for charts, default option."))
    parser.add_option("--no-percentiles",
                      action="store_false",
                      dest="with_percentiles",
                      help=("No percentiles in tables display min, "
                            "avg and max in charts."))
    cur_path = os.path.abspath(os.path.curdir)
    parser.add_option("-d",
                      "--diff",
                      action="store_true",
                      default=False,
                      dest="diffreport",
                      help=("Create differential report."))
    parser.add_option("-t",
                      "--trend",
                      action="store_true",
                      default=False,
                      dest="trendreport",
                      help=("Build a trend reprot."))
    parser.add_option("-o",
                      "--output-directory",
                      type="string",
                      dest="output_dir",
                      help="Parent directory to store reports, the directory"
                      "name of the report will be generated automatically.",
                      default=cur_path)
    parser.add_option("-r",
                      "--report-directory",
                      type="string",
                      dest="report_dir",
                      help="Directory name to store the report.",
                      default=None)
    parser.add_option(
        "-T",
        "--apdex-T",
        type="float",
        dest="apdex_t",
        help="Apdex T constant in second, default is set to 1.5s. "
        "Visit http://www.apdex.org/ for more information.",
        default=1.5)
    parser.add_option("-x",
                      "--css",
                      type="string",
                      dest="css_file",
                      help="Custom CSS file to use for the HTML reports",
                      default=None)
    parser.add_option("",
                      "--skip-definitions",
                      action="store_true",
                      default=False,
                      dest="skip_definitions",
                      help="If True, will skip the definitions")
    parser.add_option("-q",
                      "--quiet",
                      action="store_true",
                      default=False,
                      dest="quiet",
                      help=("Report no system messages when generating"
                            " html from rst."))

    options, args = parser.parse_args()
    if options.diffreport:
        if len(args) != 2:
            parser.error("incorrect number of arguments")
        trace("Creating diff report ... ")
        output_dir = options.output_dir
        html_path = RenderDiff(args[0],
                               args[1],
                               options,
                               css_file=options.css_file)
        trace("done: \n")
        trace("%s\n" % html_path)
    elif options.trendreport:
        if len(args) < 2:
            parser.error("incorrect number of arguments")
        trace("Creating trend report ... ")
        output_dir = options.output_dir
        html_path = RenderTrend(args, options, css_file=options.css_file)
        trace("done: \n")
        trace("%s\n" % html_path)
    else:
        if len(args) < 1:
            parser.error("incorrect number of arguments")
        if len(args) > 1:
            trace("Merging results files: ")
            f = NamedTemporaryFile(prefix='fl-mrg-', suffix='.xml')
            tmp_file = f.name
            f.close()
            MergeResultFiles(args, tmp_file)
            trace("Results merged in tmp file: %s\n" %
                  os.path.abspath(tmp_file))
            args = [tmp_file]
        options.xml_file = args[0]
        xml_parser = FunkLoadXmlParser(options.apdex_t)
        xml_parser.parse(options.xml_file)
        if options.html:
            trace("Creating html report: ...")
            html_path = RenderHtml(xml_parser.config,
                                   xml_parser.stats,
                                   xml_parser.error,
                                   xml_parser.monitor,
                                   xml_parser.monitorconfig,
                                   options,
                                   css_file=options.css_file)()
            trace("done: \n")
            trace(html_path + "\n")
        elif options.org:
            from ReportRenderOrg import RenderOrg
            print unicode(
                RenderOrg(xml_parser.config, xml_parser.stats,
                          xml_parser.error, xml_parser.monitor,
                          xml_parser.monitorconfig, options)).encode("utf-8")
        else:
            print unicode(
                RenderRst(xml_parser.config, xml_parser.stats,
                          xml_parser.error, xml_parser.monitor,
                          xml_parser.monitorconfig, options)).encode("utf-8")
Exemplo n.º 43
0
    def prepare_workers(self, allow_errors=False):
        """
        Initialize the sandboxes in each worker node to prepare for a
        bench run. The additional parameter `allow_errors` will essentially
        make the distinction between ignoring unresponsive/inappropriate
        nodes - or raising an error and failing the entire bench.
        """
        # right, lets figure out if funkload can be setup on each host

        def local_prep_worker(worker):

            remote_res_dir = os.path.join(self.remote_res_dir, worker.name)
            virtual_env = os.path.join(
                remote_res_dir, self.tarred_testsdir)

            if worker.isdir(virtual_env):
                worker.execute("rm -rf %s" % virtual_env)

            worker.execute("mkdir -p %s" % virtual_env)
            worker.put(
                get_virtualenv_script(),
                os.path.join(remote_res_dir, "virtualenv.py"))

            trace(".")
            worker.execute(
                "%s virtualenv.py %s" % (
                    self.python_bin, self.tarred_testsdir),
                cwdir=remote_res_dir)

            tarball = os.path.split(self.tarred_tests)[1]
            remote_tarball = os.path.join(remote_res_dir, tarball)

            # setup funkload
            cmd = "./bin/easy_install setuptools ez_setup {funkload}".format(
                funkload=self.funkload_location)

            if self.distributed_packages:
                cmd += " %s" % self.distributed_packages

            worker.execute(cmd, cwdir=virtual_env)

            #unpackage tests.
            worker.put(
                self.tarred_tests, os.path.join(remote_res_dir, tarball))
            worker.execute(
                "tar -xvf %s" % tarball,
                cwdir=remote_res_dir)
            worker.execute("rm %s" % remote_tarball)

            # workaround for https://github.com/pypa/virtualenv/issues/330
            worker.execute("rm lib64", cwdir=virtual_env)
            worker.execute("ln -s lib lib64", cwdir=virtual_env)

        threads = []
        trace("* Preparing sandboxes for %d workers." % len(self._workers))
        for worker in list(self._workers):
            if not worker.connected:
                if allow_errors:
                    trace("%s is not connected, removing from pool.\n" % \
                                                                 worker.name)
                    self._workers.remove(worker)
                    continue
                else:
                    raise RuntimeError(
                        "%s is not contactable with error %s" % (
                            worker.name, worker.error))

            # Verify that the Python binary is available
            which_python = "test -x `which %s 2>&1 > /dev/null` && echo true" \
                    % (self.python_bin)
            out, err = worker.execute(which_python)

            if out.strip() == "true":
                threads.append(threading.Thread(
                    target=local_prep_worker,
                    args=(worker,)))
            elif allow_errors:
                trace("Cannot find Python binary at path `%s` on %s, " + \
                      "removing from pool" % (self.python_bin, worker.name))
                self._workers.remove(worker)
            else:
                raise RuntimeError("%s is not contactable with error %s" % (
                    worker.name, worker.error))

        [k.start() for k in threads]
        [k.join() for k in threads]
        trace("\n")
        if not self._workers:
            raise RuntimeError("no workers available for distribution")
Exemplo n.º 44
0
def main(args=sys.argv[1:]):
    """Default main."""
    # enable to load module in the current path
    cur_path = os.path.abspath(os.path.curdir)
    sys.path.insert(0, cur_path)

    parser = OptionParser(USAGE,
                          formatter=TitledHelpFormatter(),
                          version="FunkLoad %s" % get_version())
    parser.add_option("-u",
                      "--url",
                      type="string",
                      dest="main_url",
                      help="Base URL to bench.")
    parser.add_option("-c",
                      "--cycles",
                      type="string",
                      dest="bench_cycles",
                      help="Cycles to bench, this is a list of number of "
                      "virtual concurrent users, to run a bench with 3"
                      "cycles with 5, 10 and 20 users use: -c 2:10:20")
    parser.add_option("-D",
                      "--duration",
                      type="string",
                      dest="bench_duration",
                      help="Duration of a cycle in seconds.")
    parser.add_option("-m",
                      "--sleep-time-min",
                      type="string",
                      dest="bench_sleep_time_min",
                      help="Minimum sleep time between requests.")
    parser.add_option("-M",
                      "--sleep-time-max",
                      type="string",
                      dest="bench_sleep_time_max",
                      help="Maximum sleep time between requests.")
    parser.add_option("-t",
                      "--test-sleep-time",
                      type="string",
                      dest="bench_sleep_time",
                      help="Sleep time between tests.")
    parser.add_option("-s",
                      "--startup-delay",
                      type="string",
                      dest="bench_startup_delay",
                      help="Startup delay between thread.")
    parser.add_option("-f",
                      "--as-fast-as-possible",
                      action="store_true",
                      help="Remove sleep times between requests and between "
                      "tests, shortcut for -m0 -M0 -t0")
    parser.add_option("",
                      "--no-color",
                      action="store_true",
                      help="Monochrome output.")
    parser.add_option("",
                      "--accept-invalid-links",
                      action="store_true",
                      help="Do not fail if css/image links are not reachable.")
    parser.add_option("",
                      "--simple-fetch",
                      action="store_true",
                      dest="bench_simple_fetch",
                      help="Don't load additional links like css or images "
                      "when fetching an html page.")
    parser.add_option("-l",
                      "--label",
                      type="string",
                      help="Add a label to this bench run for easier "
                      "identification (it will be appended to the "
                      "directory name for reports generated from it).")
    parser.add_option("--enable-debug-server",
                      action="store_true",
                      dest="debugserver",
                      help="Instantiates a debug HTTP server which exposes an "
                      "interface using which parameters can be modified "
                      "at run-time. Currently supported parameters: "
                      "/cvu?inc=<integer> to increase the number of "
                      "CVUs, /cvu?dec=<integer> to decrease the number "
                      "of CVUs, /getcvu returns number of CVUs ")
    parser.add_option("--debug-server-port",
                      type="string",
                      dest="debugport",
                      help="Port at which debug server should run during the "
                      "test")
    parser.add_option("--distribute",
                      action="store_true",
                      dest="distribute",
                      help="Distributes the CVUs over a group of worker "
                      "machines that are defined in the workers section")
    parser.add_option("--distribute-workers",
                      type="string",
                      dest="workerlist",
                      help="This parameter will  over-ride the list of "
                      "workers defined in the config file. expected "
                      "notation is uname@host,uname:pwd@host or just "
                      "host...")
    parser.add_option("--distribute-python",
                      type="string",
                      dest="python_bin",
                      help="When running in distributed mode, this Python "
                      "binary will be used across all hosts.")
    parser.add_option("--is-distributed",
                      action="store_true",
                      dest="is_distributed",
                      help="This parameter is for internal use only. it "
                      "signals to a worker node that it is in "
                      "distributed mode and shouldn't perform certain "
                      "actions.")
    parser.add_option("--distributed-packages",
                      type="string",
                      dest="distributed_packages",
                      help="Additional packages to be passed to easy_install "
                      "on remote machines when being run in distributed "
                      "mode.")

    # XXX What exactly is this checking for here??
    cmd_args = " ".join([k for k in args if k.find('--distribute') < 0])

    options, args = parser.parse_args(args)

    if len(args) != 2:
        parser.error("incorrect number of arguments")

    if not args[1].count('.'):
        parser.error("invalid argument; should be [class].[method]")

    if options.as_fast_as_possible:
        options.bench_sleep_time_min = '0'
        options.bench_sleep_time_max = '0'
        options.bench_sleep_time = '0'

    klass, method = args[1].split('.')
    if options.distribute:
        from Distributed import DistributionMgr
        ret = None
        try:
            distmgr = DistributionMgr(args[0], klass, method, options,
                                      cmd_args)
        except UserWarning, error:
            trace(red_str("Distribution failed with:%s \n" % (error)))

        try:
            distmgr.prepare_workers(allow_errors=True)
            ret = distmgr.run()
            distmgr.final_collect()
        except KeyboardInterrupt:
            trace("* ^C received *")
            distmgr.abort()

        return ret
Exemplo n.º 45
0
 def log(self, message, force=False):
     """Log a message."""
     if force or self.verbose:
         trace(str(message))