示例#1
0
 def _run_ipbenchd(self, user, host):
     ssh_dest = '%s@%s' % (user, host)
     remotecmd = siteconfig.get('IPBENCHD_PATH')
     cmd = ['ssh'
            ] + siteconfig.get('SSH_ARGS').split() + [ssh_dest, remotecmd]
     debug.verbose('spawning ipbenchd on %s' % host)
     return subprocess.Popen(cmd)
示例#2
0
 def get_modules(self, build, machine):
     cardName = "e1000"
     modules = super(WebCommon, self).get_modules(build, machine)
     modules.add_module("e1000n",
                        ["core=%d" % machine.get_coreids()[3]])  # 1
     modules.add_module(
         "NGD_mng",
         [
             "core=%d" % machine.get_coreids()[1],  #2
             "cardname=%s" % cardName
         ])
     modules.add_module(
         "netd",
         [
             "core=%d" % machine.get_coreids()[1],  #2
             "cardname=%s" % cardName
         ])
     nfsip = socket.gethostbyname(siteconfig.get('WEBSERVER_NFS_HOST'))
     modules.add_module(
         "webserver",
         [
             "core=%d" % machine.get_coreids()[2],  #2
             cardName,
             nfsip,
             siteconfig.get('WEBSERVER_NFS_PATH')
         ])
     #                                         siteconfig.get('WEBSERVER_NFS_TEST_PATH')])
     return modules
示例#3
0
 def get_modules(self, build, machine):
     cardName = "e1000"
     modules = super(WebCommon, self).get_modules(build, machine)
     modules.add_module("e1000n", ["core=%d" % machine.get_coreids()[1]])
     modules.add_module("NGD_mng", ["core=%d" % machine.get_coreids()[2], "cardname=%s" % cardName])
     modules.add_module("netd", ["core=%d" % machine.get_coreids()[2], "cardname=%s" % cardName])
     nfsip = socket.gethostbyname(siteconfig.get("WEBSERVER_NFS_HOST"))
     modules.add_module(
         "webserver", ["core=%d" % machine.get_coreids()[3], cardName, nfsip, siteconfig.get("WEBSERVER_NFS_PATH")]
     )
     return modules
示例#4
0
    def __init__(self, name='', rootlogger=False):
        self.name = name

        self.systemConfig = siteconfig.inifile('all')

        url = self.systemConfig.get('servers', self.name, '')
        port = urlExtract('port', url)
        addr = ('', port)
        SimpleXMLRPCServer.__init__(self, addr, logRequests=False)

        logurl = self.systemConfig.get('servers', 'log.tcp', 'dtc0:9020')
        logaddr = urlExtract('address', logurl)
        logport = urlExtract('port', logurl)

        locallevel = siteconfig.get('logging', 'locallevel', 'info').lower()
        globallevel = siteconfig.get('logging', 'globallevel', 'info').lower()
        try:
            llevel = loglevels[locallevel]
        except:
            llevel = logging.INFO
        try:
            glevel = loglevels[globallevel]
        except:
            glevel = logging.INFO

        if rootlogger:
            fmt = logging.Formatter(
                '%(asctime)s %(server)-6s %(application)-12s %(name)-12s %(levelname)-6s %(message)s',
                datefmt='%H:%M:%S')
        else:
            fmt = logging.Formatter(
                '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                datefmt='%H:%M:%S')
            socketloghandler = logging.handlers.SocketHandler(logaddr, logport)
            socketloghandler.setLevel(glevel)
            logging.getLogger('').addHandler(socketloghandler)

        console = logging.StreamHandler(sys.stdout)
        console.setLevel(llevel)
        console.setFormatter(fmt)
        logging.getLogger('').addHandler(console)
        logging.root.setLevel(llevel)

        ## Internal system functions
        self.register_introspection_functions()

        ## Common functions
        self.register_function(self.ident)
        self.register_function(self.status)

        self.log = logging.getLogger('main')
        self.log.info('Running on port: %d', port)
示例#5
0
 def get_modules(self, build, machine):
     modules = super(WebCommon, self).get_modules(build, machine)
     modules.add_module("e10k", ["auto"])
     modules.add_module("net_sockets_server", ["nospawn"])
     nfsip = socket.gethostbyname(siteconfig.get('WEBSERVER_NFS_HOST'))
     modules.add_module(
         "webserver",
         [
             "core=%d" % machine.get_coreids()[0],  #2
             nfsip,
             siteconfig.get('WEBSERVER_NFS_PATH')
         ])
     return modules
示例#6
0
    def get_modules(self, build, machine):
        modules = super(NFSTest, self).get_modules(build, machine)
        modules.add_module("sfn5122f", ["auto"])
        modules.add_module("net_sockets_server", ["nospawn"])
        nfsip = socket.gethostbyname(siteconfig.get('WEBSERVER_NFS_HOST'))
        nfspath = siteconfig.get('WEBSERVER_1G_PATH')
        nfsfile = siteconfig.get('WEBSERVER_1G_FILE')

        modules.add_module("netthroughput", [
            "core=%d" % machine.get_coreids()[2], "nfs://" + nfsip + nfspath,
            "/nfs/" + nfsfile
        ])
        return modules
示例#7
0
    def get_modules(self, build, machine):
        cardName = "e1000"
        modules = super(VMKitTest, self).get_modules(build, machine)
        modules.add_module("serial_pc16550d", ["auto"])
        modules.add_module("lpc_timer", ["auto"])
        modules.add_module("e1000n", ["auto"])
        modules.add_module("net_sockets_server", ["nospawn"])

        nfsip = socket.gethostbyname(siteconfig.get('WEBSERVER_NFS_HOST'))
        modules.add_module(
            "vmkitmon",
            [cardName, "nfs://" + nfsip + siteconfig.get('WEBSERVER_VM_PATH')])
        return modules
    def get_modules(self, build, machine):
        cardName = "e1000"
        modules = super(NFSTest, self).get_modules(build, machine)
        modules.add_module("e1000n", ["auto"])
        modules.add_module("NGD_mng", ["auto"])
        modules.add_module("netd", ["auto"])
        nfsip = socket.gethostbyname(siteconfig.get('WEBSERVER_NFS_HOST'))
        nfspath = siteconfig.get('WEBSERVER_1G_PATH')
        nfsfile = siteconfig.get('WEBSERVER_1G_FILE')

        modules.add_module("netthroughput", [
            "core=%d" % machine.get_coreids()[2], "nfs://" + nfsip + nfspath,
            "/nfs/" + nfsfile
        ])
        return modules
示例#9
0
    def get_modules(self, build, machine):
        cardName = "e1000"
        modules = super(NetCommon, self).get_modules(build, machine)
        modules.add_module("net_sockets_server", ["auto"])
        nfsip = socket.gethostbyname(siteconfig.get('WEBSERVER_NFS_HOST'))
        modules.add_module(
            "webserver",
            [
                "core=%d" % machine.get_coreids()[0],  #2
                cardName,
                nfsip,
                siteconfig.get('WEBSERVER_NFS_TEST_PATH')
            ])

        return modules
示例#10
0
    def getpage(self, server, page):
        debug.verbose("requesting http://%s/%s" % (server, page))
        c = httplib.HTTPConnection(server, timeout=WEBSERVER_TIMEOUT)
        c.request("GET", "/" + page)
        r = c.getresponse()

        debug.verbose("server replied %s %s" % (r.status, r.reason))
        assert (r.status / 100) == 2  # check for success response

        try:
            local_path = siteconfig.get("WEBSERVER_LOCAL_PATH")
        except AttributeError:
            local_path = None
        local = os.path.join(local_path, page) if local_path else None
        if local and os.path.isfile(local) and os.access(local, os.R_OK):
            debug.verbose("comparing content to %s" % local)
            l = open(local, "r")
            # read from both files and compare
            CHUNKSIZE = 4096
            while True:
                remote_data = r.read(CHUNKSIZE)
                local_data = l.read(CHUNKSIZE)
                assert remote_data == local_data
                if len(local_data) < CHUNKSIZE:
                    break

        c.close()
示例#11
0
    def getpage(self, server, page):
        debug.verbose('requesting http://%s/%s' % (server, page))
        c = httplib.HTTPConnection(server, timeout=WEBSERVER_TIMEOUT)
        c.request('GET', '/' + page)
        r = c.getresponse()

        debug.verbose('server replied %s %s' % (r.status, r.reason))
        assert((r.status / 100) == 2) # check for success response

        try:
            local_path = siteconfig.get('WEBSERVER_LOCAL_PATH')
        except AttributeError:
            local_path = None
        local = os.path.join(local_path, page) if local_path else None
        if local and os.path.isfile(local) and os.access(local, os.R_OK):
            debug.verbose('comparing content to %s' % local)
            l = open(local, 'r')
            # read from both files and compare
            CHUNKSIZE = 4096
            while True:
                remote_data = r.read(CHUNKSIZE)
                local_data = l.read(CHUNKSIZE)
                if remote_data != local_data:
                    print "Remote and local data did not match:"
                    print "Remote data\n"
                    print remote_data
                    print "Local data\n"
                    print local_data
                assert(remote_data == local_data)
                if len(local_data) < CHUNKSIZE:
                    break

            debug.verbose('contents matched for %s' % local)
        c.close()
示例#12
0
    def _run_ipbench(self, args, logfile):
        cmd = [siteconfig.get('IPBENCH_PATH')] + args
        firstrun = True
        for _ in range(IPBENCH_ITERATIONS):
            if firstrun:
                firstrun = False
            else:
                # sleep a moment to let things settle down between runs
                debug.verbose('sleeping between ipbench runs')
                time.sleep(IPBENCH_SLEEPTIME)

            debug.verbose('running ipbench: %s' % ' '.join(cmd))
            child = subprocess.Popen(cmd, stdout=subprocess.PIPE)
            timeout = datetime.datetime.now() + IPBENCH_TIMEOUT
            while True:
                # wait for some output
                (rlist, _, _) = select_timeout(timeout, [child.stdout])
                if not rlist:
                    debug.warning('ipbench run timed out')
                    child.terminate()
                    child.wait()
                    raise TimeoutError('waiting for ipbench')
                # read one char at a time to avoid blocking
                c = child.stdout.read(1)
                if c == '':
                    break # EOF
                logfile.write(c)
            child.wait()
            assert(child.returncode == 0) # check for successful exit
示例#13
0
    def runtests(self, target):
        nclients = siteconfig.get('HTTPERF_MAXCLIENTS')
        firstrun = True
        totalrate = HTTPERF_STARTRATE
        while True:
            if firstrun:
                firstrun = False
            else:
                # sleep a moment to let things settle down between runs
                debug.verbose('sleeping between httperf runs')
                time.sleep(HTTPERF_SLEEPTIME)

            # compute rate and total number of connections for each client
            rate = totalrate / nclients
            nconns = HTTPERF_DURATION * rate

            debug.log('starting httperf: %d clients, %d conns, rate %d (%d per client)' %
                      (nclients, nconns, totalrate, rate))
            self._runtest(target, nclients, nconns, rate)

            # decide whether to keep going...
            results = self._process_run(self.nruns)
            if not results.passed():
                debug.log('previous test failed, stopping')
                break
            elif results.request_rate < (0.9 * results.connect_rate):
                debug.log('request rate below 90% of connect rate, stopping')
                break
            elif results.reply_rate < (0.9 * results.request_rate):
                debug.log('reply rate below 90% of request rate, stopping')
                break
            else:
                totalrate += HTTPERF_RATEINCREMENT
                continue
示例#14
0
    def getpage(self, server, page):
        debug.verbose('requesting http://%s/%s' % (server, page))
        c = httplib.HTTPConnection(server, timeout=WEBSERVER_TIMEOUT)
        c.request('GET', '/' + page)
        r = c.getresponse()

        debug.verbose('server replied %s %s' % (r.status, r.reason))
        assert ((r.status / 100) == 2)  # check for success response

        try:
            local_path = siteconfig.get('WEBSERVER_LOCAL_PATH')
        except AttributeError:
            local_path = None
        local = os.path.join(local_path, page) if local_path else None
        if local and os.path.isfile(local) and os.access(local, os.R_OK):
            debug.verbose('comparing content to %s' % local)
            l = open(local, 'r')
            # read from both files and compare
            CHUNKSIZE = 4096
            while True:
                remote_data = r.read(CHUNKSIZE)
                local_data = l.read(CHUNKSIZE)
                if remote_data != local_data:
                    print "Remote and local data did not match:"
                    print "Remote data\n"
                    print remote_data
                    print "Local data\n"
                    print local_data
                assert (remote_data == local_data)
                if len(local_data) < CHUNKSIZE:
                    break

            debug.verbose('contents matched for %s' % local)
        c.close()
示例#15
0
    def get_modules(self, build, machine):
        cardName = "e1000"
        modules = super(VMKitTest, self).get_modules(build, machine)
        modules.add_module("serial")
        modules.add_module("lpc_timer")
        modules.add_module("e1000n", ["core=%d" % machine.get_coreids()[3]])
        modules.add_module("NGD_mng", ["core=%d" % machine.get_coreids()[2],
                                    "cardname=%s"%cardName])
        modules.add_module("netd", ["core=%d" % machine.get_coreids()[2],
                                    "cardname=%s"%cardName])

        nfsip = socket.gethostbyname(siteconfig.get('WEBSERVER_NFS_HOST'))
        modules.add_module("vmkitmon", [cardName,
                                       "nfs://" + nfsip +
                                       siteconfig.get('WEBSERVER_VM_PATH')])
        return modules
示例#16
0
    def runtests(self, target):
        nclients = siteconfig.get('HTTPERF_MAXCLIENTS')
        firstrun = True
        totalrate = HTTPERF_STARTRATE
        while True:
            if firstrun:
                firstrun = False
            else:
                # sleep a moment to let things settle down between runs
                debug.verbose('sleeping between httperf runs')
                time.sleep(HTTPERF_SLEEPTIME)

            # compute rate and total number of connections for each client
            rate = totalrate / nclients
            nconns = HTTPERF_DURATION * rate

            debug.log(
                'starting httperf: %d clients, %d conns, rate %d (%d per client)'
                % (nclients, nconns, totalrate, rate))
            self._runtest(target, nclients, nconns, rate)

            # decide whether to keep going...
            results = self._process_run(self.nruns)
            if not results.passed():
                debug.log('previous test failed, stopping')
                break
            elif results.request_rate < (0.9 * results.connect_rate):
                debug.log('request rate below 90% of connect rate, stopping')
                break
            elif results.reply_rate < (0.9 * results.request_rate):
                debug.log('reply rate below 90% of request rate, stopping')
                break
            else:
                totalrate += HTTPERF_RATEINCREMENT
                continue
示例#17
0
    def _run_ipbench(self, args, logfile):
        cmd = [siteconfig.get('IPBENCH_PATH')] + args
        firstrun = True
        for _ in range(IPBENCH_ITERATIONS):
            if firstrun:
                firstrun = False
            else:
                # sleep a moment to let things settle down between runs
                debug.verbose('sleeping between ipbench runs')
                time.sleep(IPBENCH_SLEEPTIME)

            debug.verbose('running ipbench: %s' % ' '.join(cmd))
            child = subprocess.Popen(cmd, stdout=subprocess.PIPE)
            timeout = datetime.datetime.now() + IPBENCH_TIMEOUT
            while True:
                # wait for some output
                (rlist, _, _) = select_timeout(timeout, [child.stdout])
                if not rlist:
                    debug.warning('ipbench run timed out')
                    child.terminate()
                    child.wait()
                    raise TimeoutError('waiting for ipbench')
                # read one char at a time to avoid blocking
                c = child.stdout.read(1)
                if c == '':
                    break  # EOF
                logfile.write(c)
            child.wait()
            assert (child.returncode == 0)  # check for successful exit
示例#18
0
    def getTxChannelInfo(self, dtc, setup):
        info = {}
        section = '%s mode:%d' % (dtc, setup)

        txf = self.experimentConfig.get(section, 'txfrequency', 0)
        txch = self.experimentConfig.get(section, 'txchannel', 'normal')
        txchannel = siteconfig.get('tx channels', txch, '')

        # amisr and new sondrestrom way. The freq is named directly in the exp file
        mo = re.match('tx([c0-9]*)frequency([0-9]*)', txf, re.I)
        if not mo is None:
            txid = mo.group(1)
            txlo = mo.group(2)
            txfreqname = txf
            txloname = 'tx%slo%s' % (txid, txlo)
            txbandname = 'tx%sband' % (txid)
        else:
            raise Exception('Can not interpret txfrequency')

        # substitute config "dso" with real txlo name
        if txchannel:
            txchannel = txchannel.replace('dso', txloname)
            txchannel = txchannel.replace('txband', txbandname)
        else:
            raise Exception('Tx channel not specified')

        info['channelname'] = txch
        info['loname'] = txloname
        info['bandname'] = txbandname
        info['frequencyname'] = txfreqname
        info['frequencyalgorithm'] = txchannel

        return info
示例#19
0
 def __init__(self, logfile, user, host, target, nconns, rate):
     self.user = user
     self.host = host
     self.httperf_path = siteconfig.get("HTTPERF_PATH")
     cmd = "%s %s" % (self.httperf_path, HTTPERF_BASE_ARGS)
     cmd += " --num-conns %d --rate %d --server %s --uri %s" % (nconns, rate, target, HTTPERF_URI)
     self.proc = self._launchssh(cmd, stdout=subprocess.PIPE, bufsize=0)
     self.logfile = logfile
示例#20
0
 def __init__(self, logfile, user, host, target, nconns, rate):
     self.user = user
     self.host = host
     self.httperf_path = siteconfig.get('HTTPERF_PATH')
     cmd = '%s %s' % (self.httperf_path, HTTPERF_BASE_ARGS)
     cmd += ' --num-conns %d --rate %d --server %s --uri %s' % (
         nconns, rate, target, HTTPERF_URI)
     self.proc = self._launchssh(cmd, stdout=subprocess.PIPE, bufsize=0)
     self.logfile = logfile
示例#21
0
 def _cleanup_ipbenchd(self, user, host):
     # run a remote killall to get rid of ipbenchd
     ssh_dest = '%s@%s' % (user, host)
     remotecmd = 'killall -q python'
     cmd = ['ssh'] + siteconfig.get('SSH_ARGS').split() + [ssh_dest, remotecmd]
     debug.verbose('killing ipbenchd on %s' % host)
     retcode = subprocess.call(cmd)
     if retcode != 0:
         debug.warning('failed to killall python on %s!' % host)
示例#22
0
    def get_modules(self, build, machine):
        cardName = "e1000"
        modules = super(NFSTest, self).get_modules(build, machine)
        modules.add_module("e1000n", ["core=%d" % machine.get_coreids()[1]])
        modules.add_module("NGD_mng", ["core=%d" % machine.get_coreids()[2],
                                    "cardname=%s"%cardName])
        modules.add_module("netd", ["core=%d" % machine.get_coreids()[2],
                                    "cardname=%s"%cardName])
        if use_emmentaler :
            nfsip = socket.gethostbyname(siteconfig.get('WEBSERVER_NFS_HOST'))
            nfspath = "/local/nfs/harness_nfs/"
        else :
            nfsip = socket.gethostbyname(siteconfig.get('NFS_SERVER_HOST'))
            nfspath = "/shared/harness_nfs/"

        modules.add_module("netthroughput",
                ["core=%d" % machine.get_coreids()[2], "nfs://" + nfsip +
                          nfspath , "/nfs/G1.file"])
        return modules
示例#23
0
 def _cleanup_ipbenchd(self, user, host):
     # run a remote killall to get rid of ipbenchd
     ssh_dest = '%s@%s' % (user, host)
     remotecmd = 'killall -q python'
     cmd = ['ssh'
            ] + siteconfig.get('SSH_ARGS').split() + [ssh_dest, remotecmd]
     debug.verbose('killing ipbenchd on %s' % host)
     retcode = subprocess.call(cmd)
     if retcode != 0:
         debug.warning('failed to killall python on %s!' % host)
示例#24
0
 def _get_hake_conf(self, srcdir, archs):
     default_config = {
         "source_dir": "\"%s\"" % srcdir,
         "architectures": "[" + ", ".join("\"%s\"" % a for a in archs) + "]",
         "install_dir": "\".\"",
         "toolroot": "Nothing",
         "arm_toolspec": "Nothing",
         "aarch64_toolspec": "Nothing",
         "thumb_toolspec": "Nothing",
         "armeb_toolspec": "Nothing",
         "x86_toolspec": "Nothing",
         "k1om_toolspec": "Nothing",
         "cache_dir": "\"%s\"" % os.path.expanduser("~/.cache/barrelfish/"),
         "hagfish_location" : "\"%s\"" % siteconfig.get('HAGFISH_LOCATION')
     }
     return default_config
示例#25
0
    def getRxChannelInfo(self, dtc, setup):
        info = {}
        id = int(dtc[3:])
        section = '%s mode:%d' % (dtc, setup)

        rxloname = 'rx%dlo0' % (id)
        rxbandname = 'rx%dband' % (id)
        rxch = self.experimentConfig.get(section, 'rxchannel', 'ionline')
        rxchannel = siteconfig.get('rx channels', rxch, '')
        if rxchannel:
            rxchannel = rxchannel.replace('nco', rxloname)
            rxchannel = rxchannel.replace('rxband', rxbandname)
        else:
            raise Exception('Rx channel not specified')

        tuningmethod = self.experimentConfig.getint(section, 'tuningmethod', 0)
        if tuningmethod:
            tuningsection = 'tuning method %d' % (tuningmethod)
            method = siteconfig.vars(tuningsection)
            if self.experimentConfig.has_section(tuningsection):
                method = self.experimentConfig.vars(tuningsection)
            if method:
                tuningalgorithm = method
            else:
                raise Exception('Not tuning method specified for method: %d' %
                                (tuningmethod))
        else:
            tuningalgorithm = {}

        info['channelname'] = rxch
        info['loname'] = rxloname
        info['bandname'] = rxbandname
        info['frequencyalgorithm'] = rxchannel
        info['tuningmethod'] = tuningmethod
        info['tuningalgorithm'] = tuningalgorithm

        return info
示例#26
0
 def _launchssh(self, remotecmd, **kwargs):
     ssh_dest = '%s@%s' % (self.user, self.host)
     cmd = ['ssh'] + siteconfig.get('SSH_ARGS').split() + [ssh_dest, remotecmd]
     return subprocess.Popen(cmd, **kwargs)
示例#27
0
 def _run_ipbenchd(self, user, host):
     ssh_dest = '%s@%s' % (user, host)
     remotecmd = siteconfig.get('IPBENCHD_PATH')
     cmd = ['ssh'] + siteconfig.get('SSH_ARGS').split() + [ssh_dest, remotecmd]
     debug.verbose('spawning ipbenchd on %s' % host)
     return subprocess.Popen(cmd)
示例#28
0
 def _launchssh(self, remotecmd, **kwargs):
     ssh_dest = "%s@%s" % (self.user, self.host)
     cmd = ["ssh"] + siteconfig.get("SSH_ARGS").split() + [ssh_dest, remotecmd]
     return subprocess.Popen(cmd, **kwargs)
示例#29
0
 def _launchssh(self, remotecmd, **kwargs):
     ssh_dest = '%s@%s' % (self.user, self.host)
     cmd = ['ssh'
            ] + siteconfig.get('SSH_ARGS').split() + [ssh_dest, remotecmd]
     return subprocess.Popen(cmd, **kwargs)
示例#30
0
class experiment(baseExperiment):
    def __init__(self, exppath='', expid='', identity='', options={}):
        baseExperiment.__init__(self, exppath)
        self.expId = expid
        self.identity = identity
        self.options = options
        self.log.info('Loading experiment: %s' % (os.path.basename(exppath)))

        bconf = self.binConfig

        self.state = LockableSet()

        self.name = os.environ['computername']

        self._filenumber = 0

        self.intQueue = Queue()
        self.threadShutdown = Event()
        thread.start_new_thread(self.intFinishThread, ())

        self.syncSettings = Queue()

        self.radac = radacInterface()
        # set radac event recipient
        self.radac.register(self.radacEvent)

        url = self.systemConfig.get('servers', self.name, '')
        syncurl = self.systemConfig.get('servers', 'sync', '')
        syncaddr = urlExtract('address', syncurl)
        if url <> '':
            proxyport = urlExtract('proxyport', url)
            proxyurl = 'http://%s:%d' % (syncaddr, proxyport)
            self.sync = ServerProxy(proxyurl)
            self.log.info('Proxy url: %s' % (self.sync._ServerProxy__host))
        else:
            self.log.error('Could not find url for %s' % (self.name))

        # Dtc name and id
        self.dtc = self.name
        try:
            self.id = int(self.dtc.strip(ALPHA))
            self.log.debug('Id: %d' % (self.id))
        except:
            self.id = 0
            self.log.critical('Could not extract id number from dtc name: %s' %
                              (self.dtc))

        # Create buffer to hold data going to disk
        self.log.info('Starting storage thread')
        self.h5Buffer = h5Buffer()

        # Data storage thread
        self.dataStorage = h5Storage('DataStorage', self.h5Buffer)

        # Display storage thread
        # self.displayStorage = h5Storage('DisplayStorage',self.h5Buffer,options=['lock'])

        # Initialize integrator
        self.integrator = wrapIntegrator(self)
        self.integrator.registerEvent('integrating', self.integrating)

        # low level system limits
        defaultlim = self.systemConfig.get('limits', 'default', '')
        if defaultlim <> '':
            try:
                limits = systemCheck.LIMITS[defaultlim]
            except:
                self.log.error(
                    'No default limits for: %s\nGoing to use AMISR limits' %
                    (defaultlim))
                limits = systemCheck.LIMITS['amisr']
        else:
            limits = self.systemConfig.vars('limits')
        self.log.info('Low level limits: %s' % (str(limits)))
        # radac.setLimits(limits)
        self.limits = limits

        self.datapath = '/tmp'  # if datapath configuration fails completely it will try to store in /tmp

        # Get RecordsPerFile from system ini and possibly overloaded in the experiment file
        self.recordsPerFile = self.systemConfig.getint('Data',
                                                       'RecordsPerFile', 250)
        self.recordsPerFile = self.experimentConfig.getint(
            'Common Parameters', 'RecordsPerFile', self.recordsPerFile)
        self.log.info('RecordsPerFile: %d' % (self.recordsPerFile))

        #Display record filename
        try:
            defaultDisplayfilename = 'c:/tmp/%s.h5' % (self.dtc)
            systemDisplayfilename = self.systemConfig.get(
                self.dtc, 'displayfilename', defaultDisplayfilename)
            expDisplayfilename = self.experimentConfig.get(
                'common parameters', 'displayfilename', systemDisplayfilename)
            displayfilename = self.experimentConfig.get(
                self.dtc, 'displayfilename', expDisplayfilename)
            self.dataStorage.setDisplayFilename(displayfilename)
            self.log.info('Display filename: %s' % (displayfilename))
        except Exception, inst:
            self.log.exception(inst)

        # Load setups

        # gather external setup stuff that is needed
        extconf = {
            'nradacheaderwords': self.radac.readRegister('nheaderwords')
        }
        self.log.info('Loading setups')
        self.setups = []
        modes = self.experimentConfig.options('Modes')
        for mode in modes:
            self.setups.append(setup(self, mode, extconf))

        self._setup = 0
        # self.configureSetup(self._setup) ## Always start with setup 0

        # Write experiment relevant info to h5buffer

        # Handle [include data] section of experiment file
        if self.experimentConfig.has_section('include data'):
            include = self.experimentConfig.vars('include data')
            forme = [
                v for v in include.items()
                if v[0].lower().find(self.dtc.lower()) <> -1
            ]
            forall = [
                v for v in include.items() if v[0].lower().find('all') <> -1
            ]
            forme.extend(forall)
            for dst, fn in forme:
                sdst = dst.split(':')
                h5path = sdst[1]
                if not h5path.startswith('/'):
                    h5path = '/' + h5path
                fp = '\\'.join([self.experimentConfigPath, fn])
                self.h5Buffer.includeFile(h5path, fp)

        # Datafile version tracking
        major = 1
        minor = 0
        self.h5Buffer.setAttribute('/Major', major)
        self.h5Buffer.setAttribute('/Minor', minor)
        self.h5Buffer.setAttribute(
            '/Description', 'RADAC data file version: %d.%d' % (major, minor))

        # Site information from system.ini
        self.log.info('Reading Site information')
        self.h5Buffer.setStatic('/Site/Name',
                                siteconfig.get('site', 'name', 'unknown'))
        self.h5Buffer.setStatic('/Site/Code',
                                siteconfig.getint('site', 'code', -1))
        self.h5Buffer.setStatic('/Site/Latitude',
                                siteconfig.getfloat('site', 'latitude', 0.0))
        self.h5Buffer.setAttribute('/Site/Latitude/Unit', u'º')
        self.h5Buffer.setStatic('/Site/Longitude',
                                siteconfig.getfloat('site', 'longitude', 0.0))
        self.h5Buffer.setAttribute('/Site/Longitude/Unit', u'º')
        self.h5Buffer.setStatic('/Site/Altitude',
                                siteconfig.getfloat('site', 'altitude', 0.0))
        self.h5Buffer.setAttribute('/Site/Altitude/Unit', 'm')

        # Program version and radac info
        self.log.info('Reading Program and Radac version info')
        self.h5Buffer.setStatic('/Setup/Program', sys.argv[0])
        self.h5Buffer.setAttribute('/Setup/Program/Version', self.getVersion())
        self.h5Buffer.setStatic(
            '/Setup/RadacInfo',
            [self.radac.info['versionnumber'], self.radac.info['versiondate']])
        self.h5Buffer.setAttribute('/Setup/RadacInfo/VersionString',
                                   '%x' % (self.radac.info['versionnumber']))
        self.h5Buffer.setAttribute('/Setup/RadacInfo/VersionDateString',
                                   '%i' % (self.radac.info['versiondate']))

        # System.ini and experiment file
        self.log.info('Reading System.ini and experiment ini files')
        self.h5Buffer.setStatic('/Setup/Systemfile', bconf.text('system'))
        self.h5Buffer.setStatic('/Setup/Experimentfile',
                                bconf.text(self.experimentConfigFilename))

        # Load the constants section from siteconfig
        self.log.info('Reading Constants and attributes')
        if siteconfig.hassection('constants'):
            consts = siteconfig.vars('constants')
            for p, v in consts.items():
                self.h5Buffer.setStatic(p, float(v))
        if siteconfig.hassection('constant attributes'):
            attrs = siteconfig.vars('constant attributes')
            for p, v in attrs.items():
                self.h5Buffer.setAttribute(p, v)

        # Dynamic boi and eoi placeholders and attributes
        self.log.info('Initial setup of time arrays and attributes')
        self.h5Buffer.setDynamic('/Time/RadacTime', np.array([0.0, 0.0]))
        self.h5Buffer.setAttribute('/Time/RadacTime/Unit', u'µs')
        self.h5Buffer.setAttribute(
            '/Time/RadacTime/Descriptions',
            'µSeconds since 00:00:00 UTC on January 1, 1970')
        self.h5Buffer.setDynamic('/Time/RadacTimeString',
                                 np.array(['x' * 30, 'x' * 30]))
        self.h5Buffer.setAttribute('/Time/RadacTimeString/Format',
                                   u'YYYY-MM-DD HH:MM:SS.µµµµµµ')
        self.h5Buffer.setDynamic('/Time/MatlabTime', np.array([0.0, 0.0]))
        self.h5Buffer.setAttribute('/Time/MatlabTime/Unit', 'Days')
        self.h5Buffer.setAttribute('/Time/MatlabTime/Description',
                                   'Days since 00:00:00 on January 1, 0000')
        self.h5Buffer.setDynamic('/Time/UnixTime', np.array([0, 0]))
        self.h5Buffer.setAttribute('/Time/UnixTime/Unit', 's')
        self.h5Buffer.setAttribute(
            '/Time/UnixTime/Description',
            'Seconds since 00:00:00 UTC on January 1, 1970')
        self.h5Buffer.setDynamic('/Time/Synchronized', np.array([0, 0]))
        self.h5Buffer.setAttribute('/Time/Synchronized/Description',
                                   '0=Not Synchronized, 1=Synchronized')

        self.h5Buffer.setDynamic('/Integration/MissedPulses', np.array([0]))
        self.h5Buffer.setAttribute('/Integration/MissedPulses/Desciption',
                                   'Number of missed pulses in integration')
        self.h5Buffer.setAttribute('/Integration/MissedPulses/Unit', 'Count')

        # Load proxy interfaces for external data sources
        self.proxies = proxies(self)

        self.h5Buffer.synchronize()

        self.log.info('Experiment initialized')
示例#31
0
def get_ticket_for_exception(exc_type=None,
                             exc_value=None,
                             exc_traceback=None,
                             title=None):
    """Get (or create) a ticket for the given exception.
    
    Uses :func:`exception_uuid` to identify existing tickets by looking for the
    tag in square brackets in the title. E.g. ``Something happened. [deadbeef]``.
    
    :param str title: Title to use if creating a new ticket. Defaults to the
        exception type and value.
    :returns: The ID of the ticket.
    
    """
    if exc_type and exc_value and exc_traceback:
        mini_uuid = exception_uuid(exc_type, exc_value, exc_traceback)
    else:
        mini_uuid = None

    # Get the Shotgun.
    shotgun = get_shotgun()

    # Look for an existing ticket, or create a new one.
    if mini_uuid:
        ticket = shotgun.find_one('Ticket',
                                  [('title', 'contains', '[%s]' % mini_uuid)])
        if ticket:
            return ticket['id']

    if title is None:
        if exc_type and exc_value:
            title = '%s: %s' % (exc_type.__name__, exc_value)
        else:
            title = 'New Ticket'

    uuid_tag = ' [%s]' % mini_uuid if mini_uuid else ''

    # Automatically truncate to 255 escaped chars. Remember that the uuid
    # tag will consume 11, and another 3 for ellipsis, and a guess as to
    # how many due to the string escaping.
    if len(title.encode('string-escape')) + len(uuid_tag) > 255:
        title = title[:255 -
                      (len(title.encode('string-escape')) - len(title)) -
                      len(uuid_tag) - 3] + '...'
    title = '%s%s' % (title, uuid_tag)

    # Lookup where we are storing this ticket.
    project_id = siteconfig.get('SGACTIONS_TICKET_PROJECT')
    if not project_id:
        raise ValueError('SGACTIONS_TICKET_PROJECT must be set via siteconfig')
    tool_id = siteconfig.get('SGACTIONS_TICKET_TOOL')

    ticket = shotgun.create(
        'Ticket',
        dict(
            title=title,
            sg_status_list='rev',  # Pending Review.
            project={
                'type': 'Project',
                'id': project_id
            },
            sg_tool={
                'type': 'Tool',
                'id': tool_id
            } if tool_id else None,
        ))

    return ticket['id']