def merge(self, filenames = None):
"""merge runs from parallel computations.
return false if segmented file is not complete.
"""
if filenames == None: filenames = self.mFilenames
for f in filenames:
if SegmentedFile.isComplete( f ): return True
self.info( "merging file %s from %i chunks" % (f, self.mNumChunks) )
# check if all parts have finished and are present
if self.mNumChunks > 1:
for chunk in range( self.mNumChunks ):
fn = SegmentedFile.mangle(f, self.getSlice( chunk ) )
if not SegmentedFile.isComplete( fn ):
self.info( "file %s is incomplete - merging aborted" % fn )
return False
self.info( "all files complete" )
SegmentedFile.merge( f )
return True
def checkContents(self):
self.create()
self.assertEqual(SegmentedFile.merge(self.mFilename), True)
self.checkToken(self.mFilename)
infile = SegmentedFile.openfile(self.mFilename, "r")
data = [x for x in infile]
self.assertEqual(data[1], "header1\n")
self.assertEqual(data[0], "#comment1\n")
self.assertEqual(data[12], "#comment2\n")
self.assertEqual([int(x) for x in data[2:12] + data[13:]], range(20))
def checkContents(self):
self.create()
self.assertEqual( SegmentedFile.merge( self.mFilename ), True )
self.checkToken( self.mFilename )
infile = SegmentedFile.openfile( self.mFilename, "r" )
data = [ x for x in infile ]
self.assertEqual( data[1], "header1\n" )
self.assertEqual( data[0], "#comment1\n" )
self.assertEqual( data[12], "#comment2\n" )
self.assertEqual( [int(x) for x in data[2:12] + data[13:]], range( 20 ) )
def testMerge(self):
self.create()
SegmentedFile.merge(self.mFilename, has_header=self.mHasHeader)
self.checkContents()
def merge(self):
SegmentedFile.merge( self.mFilenameGraph )
def testMerge(self):
self.create()
SegmentedFile.merge( self.mFilename, has_header=self.mHasHeader )
self.checkContents()