def merge(job, files, outputdir="", args=""): rm = RootMerger() rm.files = files rm.args = "" jobsToMerge = [] job, jobList = getJobList(job) for j in jobList: for f in files: src = os.path.join(j.outputdir, f) if os.path.isfile(src): jobsToMerge.append(j) print jobsToMerge #if outputdir!="": # rm.merge(jobsToMerge,outputdir) #else: # Merge first in jobs own outputdirectory print job.outputdir rm.merge(jobsToMerge, job.outputdir) if outputdir != "": for f in files: src = os.path.join(job.outputdir, f) name = j.name + "_" + f dest = os.path.join(outputdir, name) print src print dest shutil.move(src, dest) print "Merging of {0} done".format(job.name)
def merge(job,files,outputdir="",args=""): rm = RootMerger() rm.files = files rm.args = "" jobsToMerge = [] job, jobList = getJobList(job) for j in jobList: for f in files: src = os.path.join(j.outputdir,f) if os.path.isfile(src): jobsToMerge.append(j) print jobsToMerge #if outputdir!="": # rm.merge(jobsToMerge,outputdir) #else: # Merge first in jobs own outputdirectory print job.outputdir rm.merge(jobsToMerge,job.outputdir) if outputdir!="": for f in files: src = os.path.join(job.outputdir,f) name = j.name +"_"+ f dest = os.path.join(outputdir,name) print src print dest shutil.move(src,dest) print "Merging of {0} done".format(job.name)
def download(job,targetDir=None,force_redownload=False,sub_list=None): # check if job id or job object has been given job, jobList = getJobList(job,sub_list) # Ok sub_list overwrites the rest # if no targetdir specified, don't do magic with it if isinstance(targetDir, str): commands.getstatusoutput('mkdir -p ' + targetDir) if not targetDir.endswith("/"): targetDir = targetDir + "/" downloadedIDs = [] failedIDs = [] # download files for sj in jobList: for f in sj.outputfiles.get("*.root"): if ""==f.lfn: print "no lfn for file ", f.namePattern, " from job", str(job.id), ".", str(sj.id) continue if isinstance(targetDir,str): f.localDir = targetDir ending = re.sub(".*\.","",f.namePattern) # the file ending is everything after the last . mainPart = re.sub("\."+ending,"",f.namePattern) # the main part of the filename is everything before the ending (removing the .) newMainPart = mainPart + "_" + str(sj.id) targetFileName = targetDir + newMainPart + "." + ending automaticName = f.localDir + f.namePattern else: f.localDir = sj.outputdir automaticName = sj.outputdir + f.namePattern targetFileName = sj.outputdir + f.namePattern successful = False if (os.path.isfile(targetFileName)) : print "file ", f.namePattern, " from job", str(job.id), ".", str(sj.id), " already loaded" if force_redownload: print "load it anyway" f.get() if automaticName!=targetFileName: stat, out = commands.getstatusoutput("mv " + automaticName + " " + targetFileName) if stat!=0: print "error in renaming ", output if (os.path.isfile(targetFileName)) : downloadedIDs.append(sj.id) else: print "Download of {0} failed".format(sj.id) failedIDs.append(sj.id) else: f.get() if automaticName!=targetFileName: #shutil.move(automaticName,targetFileName) stat, out = commands.getstatusoutput("mv " + automaticName + " " + targetFileName) if stat!=0: print "error in renaming ", output if (os.path.isfile(targetFileName)) : downloadedIDs.append(sj.id) else: failedIDs.append(sj.id) print "Download of {0} failed".format(sj.id) if ( (len(job.subjobs)>0 ) and ( len(downloadedIDs)==len(job.subjobs) ) ): print "All subjobs downloaded!!! Have fun!" return downloadedIDs,failedIDs
def queueDownload(joblist,targetDirBase=None,force_redownload=False): '''joblist may be jobs.select, or array of jobs or array of jobids''' for job in joblist: job,joblist = getJobList(job) targetdir=None if isinstance(targetDirBase,str): targetdir = targetDirBase+"/"+job.name queues.add(download,kwargs={"job":job,"targetDir":targetdir,"force_redownload":force_redownload})
def remove(job): '''Removes locally stored files.''' job, jobList = getJobList(job) for j in jobList: dir = j.outputdir files = os.listdir(dir) for file in files: if file.endswith(".root"): #print os.path.join(dir,file) os.remove(os.path.join(dir,file))
def remove(job): '''Removes locally stored files.''' job, jobList = getJobList(job) for j in jobList: dir = j.outputdir files = os.listdir(dir) for file in files: if file.endswith(".root"): #print os.path.join(dir,file) os.remove(os.path.join(dir, file))
def queueDownload(joblist, targetDirBase=None, force_redownload=False): '''joblist may be jobs.select, or array of jobs or array of jobids''' for job in joblist: job, joblist = getJobList(job) targetdir = None if isinstance(targetDirBase, str): targetdir = targetDirBase + "/" + job.name queues.add(download, kwargs={ "job": job, "targetDir": targetdir, "force_redownload": force_redownload })
def removeLFNs(job): ''' remove all outputfiles (ending with .root) of all subjobs of job ''' job, jobList = getJobList(job) success = True for sj in jobList: for f in sj.outputfiles.get("*.root"): if ""!=f.lfn: print "Removing ", f.namePattern, " of job ",sj.id try: f.remove() except: # TODO: what if removal failed because it has been removed already? success = False return success
def removeLFNs(job): ''' remove all outputfiles (ending with .root) of all subjobs of job ''' job, jobList = getJobList(job) success = True for sj in jobList: for f in sj.outputfiles.get("*.root"): if "" != f.lfn: print "Removing ", f.namePattern, " of job ", sj.id try: f.remove() except: # TODO: what if removal failed because it has been removed already? success = False return success
def move(job,outputdir,remove = False, overwrite = False): job, jobList = getJobList(job) for j in jobList: dir = j.outputdir files = os.listdir(dir) # explicitly create directory and delete files with same name pattern #try: # os.mkdir(targetDir) #except: # print "Directory {0} already exists.".format(outputdir) # # print "Deleting files with same name pattern as output." # try: # files = [] # for f in job.outputfiles: # files += [f for f in glob.glob(f.namePattern+"*")] # for f in files: # os.remove(f) # print "Delete directory." # except: # print "Something went wrong." # if targetdir doesn't end with / then add it try: os.mkdir(outputdir) except: print "Directory {0} already exists.".format(outputdir) if (overwrite): print "Deleting old directory." try: shutil.rmtree(outputdir) print "Delete directory." except: print "Something went wrong." for file in files: if file.endswith(".root"): dest = file dest = dest.replace(".root","_"+str(j.id)+".root") dest = os.path.join(outputdir,dest) src = os.path.join(dir,file) print dest print src shutil.move(src,dest) if (remove): os.remove(src)
def move(job, outputdir, remove=False, overwrite=False): job, jobList = getJobList(job) for j in jobList: dir = j.outputdir files = os.listdir(dir) # explicitly create directory and delete files with same name pattern #try: # os.mkdir(targetDir) #except: # print "Directory {0} already exists.".format(outputdir) # # print "Deleting files with same name pattern as output." # try: # files = [] # for f in job.outputfiles: # files += [f for f in glob.glob(f.namePattern+"*")] # for f in files: # os.remove(f) # print "Delete directory." # except: # print "Something went wrong." # if targetdir doesn't end with / then add it try: os.mkdir(outputdir) except: print "Directory {0} already exists.".format(outputdir) if (overwrite): print "Deleting old directory." try: shutil.rmtree(outputdir) print "Delete directory." except: print "Something went wrong." for file in files: if file.endswith(".root"): dest = file dest = dest.replace(".root", "_" + str(j.id) + ".root") dest = os.path.join(outputdir, dest) src = os.path.join(dir, file) print dest print src shutil.move(src, dest) if (remove): os.remove(src)
def download(job, targetDir=None, force_redownload=False, sub_list=None): # check if job id or job object has been given job, jobList = getJobList(job, sub_list) # Ok sub_list overwrites the rest # if no targetdir specified, don't do magic with it if isinstance(targetDir, str): commands.getstatusoutput('mkdir -p ' + targetDir) if not targetDir.endswith("/"): targetDir = targetDir + "/" downloadedIDs = [] failedIDs = [] # download files for sj in jobList: for f in sj.outputfiles.get("*.root"): if "" == f.lfn: print "no lfn for file ", f.namePattern, " from job", str( job.id), ".", str(sj.id) continue if isinstance(targetDir, str): f.localDir = targetDir ending = re.sub( ".*\.", "", f.namePattern ) # the file ending is everything after the last . mainPart = re.sub( "\." + ending, "", f.namePattern ) # the main part of the filename is everything before the ending (removing the .) newMainPart = mainPart + "_" + str(sj.id) targetFileName = targetDir + newMainPart + "." + ending automaticName = f.localDir + f.namePattern else: f.localDir = sj.outputdir automaticName = sj.outputdir + f.namePattern targetFileName = sj.outputdir + f.namePattern successful = False if (os.path.isfile(targetFileName)): print "file ", f.namePattern, " from job", str( job.id), ".", str(sj.id), " already loaded" if force_redownload: print "load it anyway" f.get() if automaticName != targetFileName: stat, out = commands.getstatusoutput("mv " + automaticName + " " + targetFileName) if stat != 0: print "error in renaming ", output if (os.path.isfile(targetFileName)): downloadedIDs.append(sj.id) else: print "Download of {0} failed".format(sj.id) failedIDs.append(sj.id) else: f.get() if automaticName != targetFileName: #shutil.move(automaticName,targetFileName) stat, out = commands.getstatusoutput("mv " + automaticName + " " + targetFileName) if stat != 0: print "error in renaming ", output if (os.path.isfile(targetFileName)): downloadedIDs.append(sj.id) else: failedIDs.append(sj.id) print "Download of {0} failed".format(sj.id) if ((len(job.subjobs) > 0) and (len(downloadedIDs) == len(job.subjobs))): print "All subjobs downloaded!!! Have fun!" return downloadedIDs, failedIDs