def _fetchLogicals(self, logicals, srcs):
fetched = []
if len(logicals) == 0:
return []
log.info("need fetch for %s from %s" %(str(logicals),str(srcs)))
d = dict(srcs)
for lf in logicals:
self._fetchLock.acquire()
if self.filemap.existsForRead(lf):
self._fetchLock.release()
log.debug("satisfied by other thread")
continue
start = time.time()
self.fetchFile = lf
phy = self.filemap.mapWriteFile(lf)
if lf not in d:
log.error("Missing source for %s" %lf)
continue
log.debug("fetching %s from %s" % (lf, d[lf]))
self._fetchPhysical(phy, d[lf])
fetched.append((lf, phy))
self.fetchFile = None
end = time.time()
diff = end-start
statistics.tracker().logTransfer(d[lf], diff)
self._fetchLock.release()
return fetched
def __init__(self, remote, config, script, outMap,
customizer=lambda p,s,cf: True):
"""
remote -- a list of executors where jobs can be sent.
config -- a configuration object, e.g. the global config object
script -- the script to be executed
outMap -- a filemap object to be used to map between logical and
physical filenames
customizer -- a function accepting a parser instance, scheduler
instance, and a command factory instance. This function will be
called after construction (but before parsing) to apply any
customization desired or necessary for an environment.
"""
self.config = config
self.parser = Parser()
self.scheduler = Scheduler(config, self._publishIfOutput)
self.parser.commandHandler(self.scheduler.schedule)
self._commandFactory = CommandFactory(self.config)
self.buildTime = time.time()
self.fail = None
if not customizer(self.parser,
self.scheduler,
self._commandFactory):
self.fail = "Error applying frontend customization."
return
self.remoteExec = remote
self.outMap = LinkedMap(outMap, self.taskId())
self._publishedFiles = []
self.stat = statistics.tracker().scriptStart((self.taskId(), script, self))
if True: # set to False to profile the parse step
self._parseScript(script)
self.stat.markParseFinish()
return
try:
pobject = profile.Profile()
presult = pobject.runctx('self._parseScript(script)', globals(),locals())
presult.dump_stats("/home/wangd/parse.pyprofile")
except StandardError, e:
self.fail = str((e,e.__doc__, str(e)))
raise e