def up(job, inputFileID1, inputFileID2, outputFileStoreID):
"""Merges the two files and places them in the output.
"""
if random.random() > success_ratio:
raise RuntimeError() #This error is a test error, it does not mean the tests have failed.
with job.fileStore.updateGlobalFileStream(outputFileStoreID) as fileHandle:
with job.fileStore.readGlobalFileStream( inputFileID1 ) as inputFileHandle1:
with job.fileStore.readGlobalFileStream( inputFileID2 ) as inputFileHandle2:
merge(inputFileHandle1, inputFileHandle2, fileHandle)
job.fileStore.logToMaster( "Merging %s and %s to %s"
% (inputFileID1, inputFileID2, outputFileStoreID) )
def up(job, inputFileID1, inputFileID2, outputFileStoreID):
"""Merges the two files and places them in the output.
"""
if random.random() > success_ratio:
raise RuntimeError() #This error is a test error, it does not mean the tests have failed.
with job.fileStore.updateGlobalFileStream(outputFileStoreID) as fileHandle:
with job.fileStore.readGlobalFileStream( inputFileID1 ) as inputFileHandle1:
with job.fileStore.readGlobalFileStream( inputFileID2 ) as inputFileHandle2:
merge(inputFileHandle1, inputFileHandle2, fileHandle)
job.fileStore.logToMaster( "Merging %s and %s to %s"
% (inputFileID1, inputFileID2, outputFileStoreID) )
def up(job, inputFileID1, inputFileID2, memory=sortMemory):
"""Merges the two files and places them in the output.
"""
with job.fileStore.writeGlobalFileStream() as (fileHandle, outputFileStoreID):
with job.fileStore.readGlobalFileStream( inputFileID1 ) as inputFileHandle1:
with job.fileStore.readGlobalFileStream( inputFileID2 ) as inputFileHandle2:
merge(inputFileHandle1, inputFileHandle2, fileHandle)
job.fileStore.logToMaster( "Merging %s and %s to %s"
% (inputFileID1, inputFileID2, outputFileStoreID) )
#Cleanup up the input files - these deletes will occur after the completion is successful.
job.fileStore.deleteGlobalFile(inputFileID1)
job.fileStore.deleteGlobalFile(inputFileID2)
return outputFileStoreID
def up(job, inputFileID1, inputFileID2, memory=sortMemory):
"""Merges the two files and places them in the output.
"""
with job.fileStore.writeGlobalFileStream() as (fileHandle, outputFileStoreID):
with job.fileStore.readGlobalFileStream( inputFileID1 ) as inputFileHandle1:
with job.fileStore.readGlobalFileStream( inputFileID2 ) as inputFileHandle2:
merge(inputFileHandle1, inputFileHandle2, fileHandle)
job.fileStore.logToMaster( "Merging %s and %s to %s"
% (inputFileID1, inputFileID2, outputFileStoreID) )
#Cleanup up the input files - these deletes will occur after the completion is successful.
job.fileStore.deleteGlobalFile(inputFileID1)
job.fileStore.deleteGlobalFile(inputFileID2)
return outputFileStoreID
def up(job, inputFileID1, inputFileID2):
"""Merges the two files and places them in the output.
"""
if random.random() > success_ratio:
raise RuntimeError() #This error is a test error, it does not mean the tests have failed.
with job.fileStore.writeGlobalFileStream() as (fileHandle, outputFileStoreID):
with job.fileStore.readGlobalFileStream( inputFileID1 ) as inputFileHandle1:
with job.fileStore.readGlobalFileStream( inputFileID2 ) as inputFileHandle2:
merge(inputFileHandle1, inputFileHandle2, fileHandle)
job.fileStore.logToMaster( "Merging %s and %s to %s"
% (inputFileID1, inputFileID2, outputFileStoreID) )
#Cleanup up the input files - these deletes will occur after the completion is successful.
job.fileStore.deleteGlobalFile(inputFileID1)
job.fileStore.deleteGlobalFile(inputFileID2)
return outputFileStoreID
def testMerge(self):
for test in xrange(self.testNo):
tempFile1 = os.path.join(self.tempDir, "fileToSort1.txt")
tempFile2 = os.path.join(self.tempDir, "fileToSort2.txt")
tempFile3 = os.path.join(self.tempDir, "mergedFile.txt")
makeFileToSort(tempFile1)
makeFileToSort(tempFile2)
sort(tempFile1)
sort(tempFile2)
with open(tempFile3, 'w') as fileHandle:
with open(tempFile1) as tempFileHandle1:
with open(tempFile2) as tempFileHandle2:
merge(tempFileHandle1, tempFileHandle2, fileHandle)
lines1 = loadFile(tempFile1) + loadFile(tempFile2)
lines1.sort()
with open(tempFile3, 'r') as f:
lines2 = f.readlines()
checkEqual(lines1, lines2)
def testMerge(self):
for test in xrange(self.testNo):
tempFile1 = os.path.join(self.tempDir, "fileToSort1.txt")
tempFile2 = os.path.join(self.tempDir, "fileToSort2.txt")
tempFile3 = os.path.join(self.tempDir, "mergedFile.txt")
makeFileToSort(tempFile1)
makeFileToSort(tempFile2)
sort(tempFile1)
sort(tempFile2)
with open(tempFile3, 'w') as fileHandle:
with open(tempFile1) as tempFileHandle1:
with open(tempFile2) as tempFileHandle2:
merge(tempFileHandle1, tempFileHandle2, fileHandle)
lines1 = self._loadFile(tempFile1) + self._loadFile(tempFile2)
lines1.sort()
with open(tempFile3, 'r') as f:
lines2 = f.readlines()
self.assertEquals(lines1, lines2)