def mergeAndMove(remotePath): lines = xrd.ls(remotePath, lopt = True) filesToMerge = [] for line in lines: words = line.split() newPath = words[-1] print 'Merging files in', newPath if words[0][0] == 'd': if newPath.endswith('failed'): xrd.cleanup(newPath, force = True) else: mergeAndMove(newPath) elif newPath.endswith('.root'): filesToMerge.append(newPath) if len(filesToMerge): pathParts = remotePath.replace(sourcedir, '')[1:].split('/') # <timestamp> <pd> crab_<pd>_<reco>-v* ... pd = pathParts[1] recov = pathParts[2] reco = recov.replace('crab_' + pd + '_', '') reco = reco[:reco.rfind('-v')] fileName = 'metfilters_%s.root' % time.strftime('%y%m%d%H%M%S') outFile = '/'.join((config.scratchdir, 'merged', reco, pd, fileName)) proc = subprocess.Popen(['hadd', outFile + '.tmp'] + ['root://eoscms.cern.ch/' + f for f in filesToMerge]) proc.wait() if proc.returncode == 0: os.rename(outFile + '.tmp', outFile) for path in filesToMerge: xrd.rm(path) else: os.remove(outFile + '.tmp')
if proc.returncode == 0: os.rename(outFile + '.tmp', outFile) for path in filesToMerge: xrd.rm(path) else: os.remove(outFile + '.tmp') if __name__ == '__main__': dbcursor.execute('SELECT `name` FROM `primarydatasets`') for reco in config.reconstructions: for name in [row[0] for row in dbcursor]: if not os.path.isdir('/'.join((config.scratchdir, 'merged', reco, name))): os.mkdir('/'.join((config.scratchdir, 'merged', reco, name))) for tsdir in xrd.ls(sourcedir): #temporary if int(os.path.basename(tsdir)) < 151108000000: continue for pddir in xrd.ls(tsdir): for recovdir in xrd.ls(pddir): mergeAndMove(recovdir) xrd.cleanup(sourcedir) print 'Done.'