def submitJobs(runner, zipDir, chunkIds, splitDir, idFname, outDir):
#chunkIds = os.listdir(splitDir)
for chunkId in chunkIds:
chunkFname = join(splitDir, chunkId)
outFname = os.path.join(outDir, chunkId+".articles.gz")
maxCommon.mustNotExist(outFname)
thisFilePath = __file__
command = "%s %s %s {check in line %s} {check in line %s} {check out exists %s}" % (sys.executable, thisFilePath, abspath(zipDir), abspath(idFname), abspath(chunkFname), abspath(outFname))
runner.submit(command)
runner.finish(wait=True)
def submitJobs(runner, zipDir, splitDir, idFname, outDir):
chunkIds = os.listdir(splitDir)
for chunkId in chunkIds:
chunkFname = join(splitDir, chunkId)
outFname = os.path.join(outDir, chunkId+".articles.gz")
maxCommon.mustNotExist(outFname)
thisFilePath = __file__
command = "%s %s %s {check in exists %s} %s {check out exists+ %s}" % \
(sys.executable, thisFilePath, zipDir, chunkFname, idFname, outFname)
runner.submit(command)
runner.finish(wait=True)
def submitConvertJobs(runner, zipDir, updateId, chunkIds, splitDir, idFname, outDir):
for chunkId in chunkIds:
chunkFname = join(splitDir, str(chunkId))
outFname = join(outDir, str(updateId)+"_"+str(chunkId)+".articles.gz")
maxCommon.mustNotExist(outFname)
thisFilePath = __file__
params = [zipDir, idFname, chunkFname, "{check out exists %s}" % outFname]
#command = "%s %s %s {check in line %s} {check in line %s} {check out exists+ %s}" % (sys.executable, thisFilePath, zipDir, idFname, chunkFname, outFname)
#runner.submit(command)
runner.submitPythonFunc("pubConvBing.py", "convertOneChunk", params)
runner.finish(wait=True)
def submitConvertJobs(runner, zipDir, updateId, chunkIds, splitDir, idFname, outDir):
for chunkId in chunkIds:
chunkFname = join(splitDir, str(chunkId))
outFname = join(outDir, str(updateId)+"_"+str(chunkId)+".articles.gz")
maxCommon.mustNotExist(outFname)
thisFilePath = __file__
params = [zipDir, idFname, chunkFname, "{check out exists %s}" % outFname]
#command = "%s %s %s {check in line %s} {check in line %s} {check out exists+ %s}" % (sys.executable, thisFilePath, zipDir, idFname, chunkFname, outFname)
#runner.submit(command)
runner.submitPythonFunc("pubConvBing.py", "convertOneChunk", params)
runner.finish(wait=True)
def submitJobs(runner, splitDir, outDir):
#runner = maxRun.Runner(delayTime=3, maxJob=maxJob)
#runner = maxRun.Runner(delayTime=3, maxJob=maxJob)
chunkIds = os.listdir(splitDir)
for chunkId in chunkIds:
chunkFname = join(splitDir, chunkId)
outFname = os.path.join(outDir, chunkId+".articles.gz")
maxCommon.mustNotExist(outFname)
thisFilePath = __file__
command = "%s %s {check in line %s} {check out exists+ %s}" % (sys.executable, thisFilePath, chunkFname, outFname)
runner.submit(command)
runner.finish(wait=True)