Пример #1
0
    def _fetchLogicals(self, logicals, srcs):
        fetched = []
        if len(logicals) == 0:
            return []
        log.info("need fetch for %s from %s" %(str(logicals),str(srcs)))
        d = dict(srcs)
        for lf in logicals:
            self._fetchLock.acquire()
            if self.filemap.existsForRead(lf):
                self._fetchLock.release()
                log.debug("satisfied by other thread")
                continue
            start = time.time()
        
            self.fetchFile = lf
            phy = self.filemap.mapWriteFile(lf)
            if lf not in d:
                log.error("Missing source for %s" %lf)
                continue
            log.debug("fetching %s from %s" % (lf, d[lf]))
            self._fetchPhysical(phy, d[lf])
            fetched.append((lf, phy))
            self.fetchFile = None
            end = time.time()
            diff = end-start
            statistics.tracker().logTransfer(d[lf], diff)
            self._fetchLock.release()

        return fetched
Пример #2
0
    def __init__(self, remote, config, script, outMap,
                 customizer=lambda p,s,cf: True):
        """
    remote -- a list of executors where jobs can be sent.
    config -- a configuration object, e.g. the global config object
    script -- the script to be executed
    outMap -- a filemap object to be used to map between logical and
    physical filenames

    customizer -- a function accepting a parser instance, scheduler
    instance, and a command factory instance.  This function will be
    called after construction (but before parsing) to apply any
    customization desired or necessary for an environment.
    """
        self.config = config
        self.parser = Parser()
        self.scheduler = Scheduler(config, self._publishIfOutput)
        self.parser.commandHandler(self.scheduler.schedule)
        self._commandFactory = CommandFactory(self.config)
        self.buildTime = time.time()
        self.fail = None
        if not customizer(self.parser,
                          self.scheduler,
                          self._commandFactory):
            self.fail = "Error applying frontend customization."
            return

        self.remoteExec = remote
        self.outMap = LinkedMap(outMap, self.taskId())
        self._publishedFiles = []
        self.stat = statistics.tracker().scriptStart((self.taskId(), script, self))
        if True: # set to False to profile the parse step
            self._parseScript(script)
            self.stat.markParseFinish()
            return
        try:
            pobject = profile.Profile()
            presult = pobject.runctx('self._parseScript(script)', globals(),locals())
            presult.dump_stats("/home/wangd/parse.pyprofile")
        except StandardError, e:
            self.fail = str((e,e.__doc__, str(e)))
            raise e