Exemple #1
0
def submitJobs(runner, zipDir, chunkIds, splitDir, idFname, outDir):
    #chunkIds = os.listdir(splitDir)
    for chunkId in chunkIds:
        chunkFname = join(splitDir, chunkId)
        outFname = os.path.join(outDir, chunkId+".articles.gz")
        maxCommon.mustNotExist(outFname)
        thisFilePath = __file__
        command = "%s %s %s {check in line %s} {check in line %s} {check out exists %s}" % (sys.executable, thisFilePath, abspath(zipDir), abspath(idFname), abspath(chunkFname), abspath(outFname))
        runner.submit(command)
    runner.finish(wait=True)
Exemple #2
0
def submitJobs(runner, zipDir, splitDir, idFname, outDir):
    chunkIds = os.listdir(splitDir)
    for chunkId in chunkIds:
        chunkFname = join(splitDir, chunkId)
        outFname = os.path.join(outDir, chunkId+".articles.gz")
        maxCommon.mustNotExist(outFname)
        thisFilePath = __file__
        command = "%s %s %s {check in exists %s} %s {check out exists+ %s}" % \
            (sys.executable, thisFilePath, zipDir, chunkFname, idFname, outFname)
        runner.submit(command)
    runner.finish(wait=True)
Exemple #3
0
def submitConvertJobs(runner, zipDir, updateId, chunkIds, splitDir, idFname, outDir):
    for chunkId in chunkIds:
        chunkFname = join(splitDir, str(chunkId))
        outFname = join(outDir, str(updateId)+"_"+str(chunkId)+".articles.gz")
        maxCommon.mustNotExist(outFname)
        thisFilePath = __file__
        params = [zipDir, idFname, chunkFname, "{check out exists %s}" % outFname]
        #command = "%s %s %s {check in line %s} {check in line %s} {check out exists+ %s}" % (sys.executable, thisFilePath, zipDir, idFname, chunkFname, outFname)
        #runner.submit(command)
        runner.submitPythonFunc("pubConvBing.py", "convertOneChunk", params)
    runner.finish(wait=True)
Exemple #4
0
def submitConvertJobs(runner, zipDir, updateId, chunkIds, splitDir, idFname, outDir):
    for chunkId in chunkIds:
        chunkFname = join(splitDir, str(chunkId))
        outFname = join(outDir, str(updateId)+"_"+str(chunkId)+".articles.gz")
        maxCommon.mustNotExist(outFname)
        thisFilePath = __file__
        params = [zipDir, idFname, chunkFname, "{check out exists %s}" % outFname]
        #command = "%s %s %s {check in line %s} {check in line %s} {check out exists+ %s}" % (sys.executable, thisFilePath, zipDir, idFname, chunkFname, outFname)
        #runner.submit(command)
        runner.submitPythonFunc("pubConvBing.py", "convertOneChunk", params)
    runner.finish(wait=True)
def submitJobs(runner, splitDir, outDir):
    #runner = maxRun.Runner(delayTime=3, maxJob=maxJob)
    #runner = maxRun.Runner(delayTime=3, maxJob=maxJob)
    chunkIds = os.listdir(splitDir)
    for chunkId in chunkIds:
        chunkFname = join(splitDir, chunkId)
        outFname = os.path.join(outDir, chunkId+".articles.gz")
        maxCommon.mustNotExist(outFname)
        thisFilePath = __file__
        command = "%s %s {check in line %s} {check out exists+ %s}" % (sys.executable, thisFilePath, chunkFname, outFname)
        runner.submit(command)
    runner.finish(wait=True)