def merge(self, filenames = None):
"""merge runs from parallel computations.
return false if segmented file is not complete.
"""
if filenames == None: filenames = self.mFilenames
for f in filenames:
if SegmentedFile.isComplete( f ): return True
self.info( "merging file %s from %i chunks" % (f, self.mNumChunks) )
# check if all parts have finished and are present
if self.mNumChunks > 1:
for chunk in range( self.mNumChunks ):
fn = SegmentedFile.mangle(f, self.getSlice( chunk ) )
if not SegmentedFile.isComplete( fn ):
self.info( "file %s is incomplete - merging aborted" % fn )
return False
self.info( "all files complete" )
SegmentedFile.merge( f )
return True
def isComplete( self ):
'''check if files are complete'''
if AddaModuleRecord.isComplete( self ):
return True
# If all the data files are complete, re-compute fit, transfer and overhang
# only and then return as complete
if SegmentedFile.isComplete( SegmentedFile.mangle( self.mFilenameData, self.getSlice()) ):
return True
if SegmentedFile.isComplete( self.mFilenameData ):
return self.merge()
return False
def merge(self, filenames=None):
"""merge runs from parallel computations.
"""
if SegmentedFile.isComplete(self.mFilenameFit):
return True
# remove unwanted results
for x in (self.mFilenameTransfer, self.mFilenameOverhang,
self.mFilenameFit):
for fn in glob.glob("%s.0*" % x):
os.remove(fn)
# merge the details file if all is complete
if glob.glob("%s.0*" % self.mFilenameDetails):
if not AddaModuleRecord.merge(self, (self.mFilenameDetails, )):
return False
if not AddaModuleRecord.merge(self, (self.mFilenameData, )):
return False
self.mNumChunks = 1
self.readPreviousData(self.mFilenameData)
self.finish()
return True
def isComplete(self):
'''check if files are complete'''
if AddaModuleRecord.isComplete(self):
return True
# If all the data files are complete, re-compute fit, transfer and overhang
# only and then return as complete
if SegmentedFile.isComplete(
SegmentedFile.mangle(self.mFilenameData, self.getSlice())):
return True
if SegmentedFile.isComplete(self.mFilenameData):
return self.merge()
return False
def merge(self, filenames = None ):
"""merge runs from parallel computations.
"""
if SegmentedFile.isComplete( self.mFilenameFit ):
return True
# remove unwanted results
for x in (self.mFilenameTransfer, self.mFilenameOverhang, self.mFilenameFit):
for fn in glob.glob( "%s.0*" % x ):
os.remove(fn)
# merge the details file if all is complete
if glob.glob( "%s.0*" % self.mFilenameDetails):
if not AddaModuleRecord.merge( self, (self.mFilenameDetails, ) ): return False
if not AddaModuleRecord.merge( self, (self.mFilenameData, ) ): return False
self.mNumChunks = 1
self.readPreviousData( self.mFilenameData )
self.finish()
return True
def isComplete(self):
fn, fi = ProfileLibrary.getFileNames(self.mFilenameProfile +
self.getSlice())
return SegmentedFile.isComplete(fi)
def isComplete( self ):
fn, fi = ProfileLibrary.getFileNames( self.mFilenameProfile + self.getSlice() )
return SegmentedFile.isComplete( fi )
def isComplete( self ):
"""return if this step is complete."""
for f in self.mFilenames:
if not SegmentedFile.isComplete( SegmentedFile.mangle( f, self.getSlice()) ):
return False
return True