def clientTest(): import SOAPpy serverConf = Config("swampsoap.conf") serverConf.read() server = SOAPpy.SOAPProxy("http://localhost:%d/%s" %(serverConf.servicePort, serverConf.serviceSoapPath)) if len(sys.argv) > 2: import readline while True: print server.pyInterface(raw_input()) else: server.reset() tok = server.newScriptedFlow(""" ncwa -a time -dtime,0,3 camsom1pdf/camsom1pdf_10_clm.nc timeavg.nc ncwa -a lon timeavg.nc timelonavg.nc ncwa -a time -dtime,0,2 camsom1pdf/camsom1pdf_10_clm.nc timeavg.nc """) print "submitted, got token: ", tok while True: ret = server.pollState(tok) if ret is not None: print "finish, code ", ret break time.sleep(1) outUrls = server.pollOutputs(tok) print "actual outs are at", outUrls for u in outUrls: # simple fetch, since we are single-threaded. urllib.urlretrieve(u[1], u[0])
def testParser3(): logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s', datefmt='%d%b%Y %H:%M:%S') p = Parser() portionlist = ["ncwa in.nc temp.nc", "ncwa temp.nc temp.nc", "ncwa temp.nc out.nc"] portion = "\n".join(portionlist) cf = CommandFactory(Config.dummyConfig()) p.parseScript(portion, cf)
def testParser2(): logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s', datefmt='%d%b%Y %H:%M:%S') wholelist = open("full_resamp.swamp").readlines() test = [ "".join(wholelist[:10]), "".join(wholelist[:400]), "".join(wholelist), testScript4] p = Parser() cf = CommandFactory(Config.dummyConfig()) p.parseScript(test[1], cf)
def testSwampInterface(): from swamp.execution import LocalExecutor from swamp.execution import FakeExecutor #logging.basicConfig(level=logging.DEBUG) wholelist = open("full_resamp.swamp").readlines() test = [ "".join(wholelist[:10]), "".join(wholelist[:6000]), "".join(wholelist), testScript4] c = Config("swamp.conf") c.read() fe = FakeExecutor() le = LocalExecutor.newInstance(c) #si = SwampInterface(fe) si = SwampInterface(c, le) log.info("after configread at " + time.ctime()) #evilly force the interface to use a remote executor assert len(si.remote) > 0 si.executor = si.remote[0] taskid = si.submit(test[1]) log.info("finish at " + time.ctime()) print "submitted with taskid=", taskid
def __init__(self, cfgName=None): if cfgName: config = Config(cfgName) else: config = Config() config.read() self._setupLogging(config) config.writePid() le = LocalExecutor.newInstance(config) #always make one self.filemap = FileMapper("f"+str(os.getpid()), config.execSourcePath, config.execResultPath, config.execResultPath) self.publishedPaths = [(config.servicePubPath, config.execResultPath), (config.servicePubPath + "s", config.execScratchPath), (config.servicePubPath + "b", config.execBulkPath)] self.exportTemplate = lambda s: "http://%s:%d/%s/" % (config.serviceHostname, config.servicePort, s) self.exportPrefix = map( self.exportTemplate, map(itemgetter(0), self.publishedPaths)) self.publishedFuncs = [self.reset, self.newScriptedFlow, self.discardFlow, self.pollState, self.pollOutputs, #self.pollJob, self.pyInterface, self.registerWorker, self.unregisterWorker] self.swampInterface = SwampInterface(config, le) self.config = config self._setupVariablePreload(self.swampInterface) self.swampInterface.startLoop() self.token = 0 self.tokenLock = threading.Lock() self.jobs = {} self.discardedJobs = {} self._setupMaster() pass