def __init__(self): # get the sensitive information from config file login="" password="" dbaseName="" dbaseHost="" for line in open(".config","r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] self.sql = SQLHandler(dbaseName,login,password,dbaseHost)
login="" password="" dbaseName="" dbaseHost="" for line in open(".config","r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] sql = SQLHandler(dbaseName,login,password,dbaseHost) if not options.stdout == True: log = logHandler("logs/ProductionWorkflow.txt") else: log = logHandler("") sleepTime = int(300) ID=int(0) DataSet = "" DataTier = "" useLocalDBS = "" RunSelection = "" skipPAT = "" CMSSW_VER = ""
login="" password="" dbaseName="" dbaseHost="" for line in open("../.config","r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] sql = SQLHandler(dbaseName,login,password,dbaseHost) ############### ### METHODS ### ############### def insertSimRequest(campaign,CMSSWversion_sim,GlobalTag_sim,Template_sim,PublishName_sim,LHEList,nEvents,prio,email): global options values = [] values.append([]) values[len(values)-1].append("Date") values[len(values)-1].append(datetime.date.today().strftime("%Y-%m-%d")) values.append([])
login = "" password = "" dbaseName = "" dbaseHost = "" for line in open("../.config", "r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] sql = SQLHandler(dbaseName, login, password, dbaseHost) ################# ### MAIN LOOP ### ################# if not len(sys.argv) >= 3: print "Usage: python updateJSONFile.py <search_string> <new_json_url>" ## get the datasets from TopDB containing <search_string> print "\n-> Searching datasets in TopDB containing: " + sys.argv[1] sql.createQuery("SELECT", "datasets", "id,name", "name REGEXP '" + sys.argv[1] + "'")
stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True).stdout.read().strip() + "/AutoMaticTopTreeProducer" for line in open("../.config", "r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] sql = SQLHandler(dbaseName, login, password, dbaseHost) # checking the CMSSW versions installed -> insert into topDB where needed for entry in os.listdir(baseDir): if os.path.isdir(baseDir + "/" + entry) == True and not entry.rfind("CMSSW") == -1: print "Local version found: " + entry isInDB = bool(False) # check if this version exists in topDB otherwise add it options = "version REGEXP '" + entry + "'"
class RequestHandler: sleepTime = int(300) requests = [] def __init__(self): # get the sensitive information from config file login="" password="" dbaseName="" dbaseHost="" for line in open(".config","r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] self.sql = SQLHandler(dbaseName,login,password,dbaseHost) self.getDataSetProperties() def patParentID(self,name): self.sql.createQuery("SELECT","patuples","dataset_id","name REGEXP '"+name+"' LIMIT 0,1") f = open("ttprod_sql2.out","w") f.write(self.sql.execQuery()) f.close() lines = open("ttprod_sql2.out","r").readlines() if len(lines) == 2: return int(lines[1]) else: return -1 def getDataSetProperties(self): self.sql.createQuery("SELECT","requests","*","Status = 'Pending' OR Status = 'Queued' ORDER BY `Priority` DESC") sqlFile="ttprod_sql_"+str(strftime("%d%m%Y_%H%M%S"))+".out" sqlFile2="ttprod_sql2_"+str(strftime("%d%m%Y_%H%M%S"))+".out" f = open(sqlFile,"w") f.write(self.sql.execQuery()) f.close() for res in open(sqlFile,"r"): #print res line = res.split("\n")[0] sqlRes = line.split(" ") if len(sqlRes) < 11: continue if not sqlRes[1].rfind("/") == -1: request = Request() request.ID = sqlRes[0] request.DataSet = sqlRes[1] request.DataTier = sqlRes[2] request.RunSelection = sqlRes[3] request.FLFilterPath = sqlRes[4] request.useLocalDBS = sqlRes[5] request.skipPAT = sqlRes[6] request.DontStorePat = sqlRes[7] request.CMSSW_VER = sqlRes[8] request.GlobalTag = sqlRes[9] request.Priority = int(sqlRes[10]) if not request.Priority == 0: # in case it is reco->toptree (+- using pat) self.sql.createQuery("SELECT","datasets","CMSSWversion,process","name REGEXP '"+request.DataSet+"' LIMIT 0,1") f = open(sqlFile2,"w") f.write(self.sql.execQuery()) f.close() for res in open(sqlFile2,"r"): #print res line = res.split("\n")[0] sqlRes = line.split(" ") if len(sqlRes) < 2: continue if not sqlRes[0] == "CMSSWversion": #print "Ver: "+sqlRes[0] request.CMSSW_VER_SAMPLE = sqlRes[0]; if sqlRes[1] == "TTJets" or sqlRes[1] == "TTbar": request.wantGenEvent = True; #print sqlRes[0] #print request.CMSSW_VER os.remove(sqlFile2) # in case we start from pat dataset_id = self.patParentID(request.DataSet) #print dataset_id if not dataset_id == -1: #print "going from PAT" self.sql.createQuery("SELECT","datasets","CMSSWversion,process","id = '"+str(dataset_id)+"' LIMIT 0,1") f = open(sqlFile2,"w") f.write(self.sql.execQuery()) f.close() for res in open(sqlFile2,"r"): #print res line = res.split("\n")[0] sqlRes = line.split(" ") if len(sqlRes) < 2: continue if not sqlRes[0] == "CMSSWversion": request.CMSSW_VER_SAMPLE = sqlRes[0]; #print sqlRes[0] #print request.CMSSW_VER if sqlRes[1] == "TTJets" or sqlRes[1] == "TTbar": request.wantGenEvent = True; if not sqlRes[1] == "Data": request.DataTier = "PAT-MC" os.remove(sqlFile2) self.requests.append(request) os.remove(sqlFile)
login = "" password = "" dbaseName = "" dbaseHost = "" for line in open("../.config", "r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] sql = SQLHandler(dbaseName, login, password, dbaseHost) ############### ### METHODS ### ############### def rmSRMdir(dir): global srmHost if os.path.exists(dir): log.output(" ---> Going into directory " + dir) tmp = os.listdir(dir)
class Request: ID=int(0) Email = "" Campaign="" CMSSW_sim="" GT_sim="" PublishName_sim="" CMSSW_top="" GT_top="" nEvents="" LHEDir="" Template="" Priority=int(0) doDry=bool(False) def __init__(self): # get the sensitive information from config file login="" password="" dbaseName="" dbaseHost="" for line in open(".config","r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] self.sql = SQLHandler(dbaseName,login,password,dbaseHost) def setQueued(self): self.sql.createQuery("UPDATE","simrequests","","SET `Status` = 'Queued' WHERE `id` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def setRunning(self): self.sql.createQuery("UPDATE","simrequests","","SET `Status` = 'Running' WHERE `id` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def setDone(self): self.sql.createQuery("UPDATE","simrequests","","SET `Status` = 'Done', `Priority` = '0' WHERE `id` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def invalidate(self): self.sql.createQuery("UPDATE","simrequests","","SET `Status` = 'Pending', `Priority` = '0' WHERE `id` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def process(self): cmssw="" gt="" publish="" template="" LHEDir="" template=" -f "+self.Template+" " LHEDir="-d "+self.LHEDir+" " cmssw=self.CMSSW_sim gt=self.GT_sim publish=self.PublishName_sim cmd ="python AutoMaticSIMProducer.py -c "+cmssw+" -g "+gt+" -p "+publish+" -a "+self.Campaign+" --email "+self.Email if not template == "": cmd+=template if not LHEDir == "": cmd+=LHEDir if not self.nEvents == "": cmd+=" -n "+self.nEvents if self.doDry: cmd+=" --dry-run " cmd += " --setLogFile logs/log-TopDB-SIMRequest-"+str(self.ID)+".txt" cmd += " >& logs/stdout" log.output(cmd) #print cmd #sys.exit(1) #return 0 #cmd = "sleep 1" pExe = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) # wait until the process is terminated while pExe.poll() == None: pExe.poll() time.sleep(10) if options.stdout: log.output(pExe.stdout.read()) exitCode = int(pExe.poll()) # possible exit codes: 0 (everything went fine) 1 (python error, need to send email) 2 (script error, mail sent by AutoMaticTopTreeProducer.py) return exitCode #return 0 def resetQueue(self): self.sql.createQuery("UPDATE","simrequests","","SET `Status` = 'Pending' WHERE `Status` = 'Queued'") self.sql.execQuery()
login = "" password = "" dbaseName = "" dbaseHost = "" for line in open(".config", "r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] sql = SQLHandler(dbaseName, login, password, dbaseHost) if not options.stdout == True: log = logHandler("logs/ProductionWorkflow.txt") else: log = logHandler("") sleepTime = int(300) ID = int(0) DataSet = "" DataTier = "" useLocalDBS = "" RunSelection = "" skipPAT = "" CMSSW_VER = ""
class RequestHandler: sleepTime = int(30) requests = [] def __init__(self,host): # get the sensitive information from config file login="" password="" dbaseName="" dbaseHost="" for line in open(".config","r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] self.sql = SQLHandler(dbaseName,login,password,dbaseHost) self.getDataSetProperties(host) def getDataSetProperties(self,host): self.sql.createQuery("SELECT","skimrequests","*","(Status = 'Pending' OR Status = 'Queued') AND host = '"+host+"'") sqlFile = "sql_"+strftime("%d%m%Y_%H%M%S")+".out" f = open(sqlFile,"w") f.write(self.sql.execQuery()) f.close() for res in open(sqlFile,"r"): #print res line = res.split("\n")[0] sqlRes = line.split(" ") if len(sqlRes) < 10: continue if sqlRes[0].rfind("ID") == -1: request = Request(); request.ID = sqlRes[0] request.Username = sqlRes[1] request.Email = sqlRes[2] request.SamplePath = sqlRes[4] request.JSON = sqlRes[6] request.WorkingDir = sqlRes[7] request.nThreads = int(sqlRes[8]) request.nGroupFiles = int(sqlRes[9]) request.walltime = sqlRes[10] request.publishName = sqlRes[11] request.announce = int(sqlRes[12]) request.host = sqlRes[13] #print request.ID+" "+request.SamplePath+" "+request.Username tmp = sqlRes[3].split('\r') for line in tmp: line = line.strip("\n").lstrip('\n') split = line.split("\\n") if len(split) == 1: request.GridProxy += split[0]+'\n' else: request.GridProxy += split[1]+'\n' tmp = sqlRes[5].split('\r') #print tmp[0].split("\\n") for line in tmp[0].split("\\n"): #line = line.strip("\n").lstrip('\n') #print line split = line.split("\\n") if len(split) == 1: request.SkimXML += split[0]+'\n' else: request.SkimXML += split[1]+'\n' #print request.SkimXML self.requests.append(request) #sys.exit(1); os.remove(sqlFile)
class Request: ID=int(0) Username = "" Email = "" GridProxy = "" SamplePath = "" SkimXML = "" JSON = "" WorkingDir = "" nThreads = int(0) nGroupFiles = int(0) publishName = "" announce = int(0) walltime = "" host = "" wantGenEvent = bool(False) def __init__(self): # get the sensitive information from config file login="" password="" dbaseName="" dbaseHost="" for line in open(".config","r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] self.sql = SQLHandler(dbaseName,login,password,dbaseHost) def setQueued(self): if not self.SamplePath == "": self.sql.createQuery("UPDATE","skimrequests","","SET `Status` = 'Queued' WHERE `ID` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def setRunning(self): if not self.SamplePath == "": self.sql.createQuery("UPDATE","skimrequests","","SET `Status` = 'Running' WHERE `ID` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def setDone(self): if not self.SamplePath == "": self.sql.createQuery("UPDATE","skimrequests","","SET `Status` = 'Done' WHERE `ID` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def invalidate(self): if not self.SamplePath == "": self.sql.createQuery("UPDATE","skimrequests","","SET `Status` = 'Pending' WHERE `ID` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def process(self): stamp = strftime("%d%m%Y_%H%M%S") TopSkimDir = Popen("echo $HOME", shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True).stdout.read().strip()+"/TopSkim/" #TopSkimDir = "/user/mmaes/AutoMaticTopTreeProducer-DEV/UserCode/mmaes/TopSkim/" #export VO_CMS_SW_DIR="/swmgrs/cmss/" #export SCRAM_ARCH="slc5_ia32_gcc434" #source $VO_CMS_SW_DIR/cmsset_default.sh #cd ../../TopSkim #python SkimTopTree.py --mtop-runmode --toptree_location=/pnfs/iihe/cms/store/user/dhondt/TToBLNu_TuneZ2_tW-channel_7TeV-madgraph/Fall10-START38_V12-v2/18112010_093940/TOPTREE -t CMSSW_38X_v1/TopBrussels/TopTreeProducer/ --skim_xml=skim_1291979767.xml -n 50 -j 1 [email protected] >& logs/topskim-error.txt # get the JSON File if needed if not self.JSON == "": #print self.JSON.split(";")[0] #JSONDest = "/localgrid/dhondt/JSON_"+stamp+".txt" #JSONDest = "/localgrid/mmaes/OFFICIAL_JSON_"+stamp+".json" JSONDest = "/localgrid/dhondt/OFFICIAL_JSON_"+stamp+".json" #print JSONDest Popen("wget -O "+JSONDest+" "+self.JSON.split(";")[0], shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True).stdout.read() # store the skim XML SkimFile = "skim_"+stamp+".xml" xmlFile = open(TopSkimDir+SkimFile,"w") xmlFile.write(self.SkimXML) xmlFile.close() # store the CRAB report JSON XML if len(self.JSON.split(";")) == 2: JSONFile = "/localgrid/dhondt/MERGED_JSON"+stamp+".json" cJSONFile = open(JSONFile,"w") cJSONFile.write(mergeJSON(JSONDest,self.JSON.split(";")[1])) cJSONFile.close() # if we use a JSON we need to change useJSON and JSONFile in the skim XML if not self.JSON == "": self.SkimXML = "" for line in open(TopSkimDir+SkimFile): if line.rfind("useJSON") == -1: self.SkimXML += line else: splitline = line.split("useJSON") self.SkimXML += splitline[0]+"useJSON=\"1\" JSONFile=\""+JSONFile+"\"/>\n" xmlFile = open(TopSkimDir+SkimFile,"w") xmlFile.write(self.SkimXML) xmlFile.close() #print self.SkimXML # store the grid proxy ProxyFile = "grid_proxy_"+strftime("%d%m%Y_%H%M%S")+".proxy" pFile = open(TopSkimDir+ProxyFile,"w") pFile.write(self.GridProxy) pFile.close() cmd = "" #cmd += "export VO_CMS_SW_DIR=\"/swmgrs/cmss/\";" #cmd += "export SCRAM_ARCH=\"slc5_ia32_gcc434\";" #cmd += "source $VO_CMS_SW_DIR/cmsset_default.sh;" cmd += "export X509_USER_PROXY=\""+TopSkimDir+ProxyFile+"\";" cmd += "cd "+TopSkimDir+";" cmd += "python SkimTopTree.py --srmcp --mtop-runmode --mtop-setuser="******" --toptree_location="+self.SamplePath+" -t "+self.WorkingDir+" --skim_xml="+SkimFile+" -j "+str(self.nThreads)+" -n "+str(self.nGroupFiles)+" -w "+str(self.walltime)+" --email "+self.Email #cmd += "python SkimTopTree.py --mtop-runmode --mtop-setuser="******" --toptree_location="+self.SamplePath+" --skim_xml="+SkimFile+" -n "+str(self.nThreads)+" -j "+str(self.nGroupFiles)+" --email "+self.Email if self.host == "PBS": cmd += " --use-pbs" if not self.publishName == "": cmd += " -p "+self.publishName if self.announce: cmd += " -a" #cmd += "voms-proxy-info" log.output(cmd) #sys.exit(1) #cmd = "" pExe = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) while pExe.poll() == None: pExe.poll() time.sleep(10) if options.stdout: log.output(pExe.stdout.read()) exitCode = int(pExe.poll()) # clean up try: #os.remove(TopSkimDir+ProxyFile) #os.remove(TopSkimDir+SkimFile) os.remove(JSONDest) os.remove(JSONFile) except: log.output("Error cleaning up") return exitCode def resetQueue(self): self.sql.createQuery("UPDATE","skimrequests","","SET `Status` = 'Pending' WHERE `Status` = 'Queued'") self.sql.execQuery()
class RequestHandler: sleepTime = int(30) requests = [] def __init__(self, host): # get the sensitive information from config file login = "" password = "" dbaseName = "" dbaseHost = "" for line in open(".config", "r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] self.sql = SQLHandler(dbaseName, login, password, dbaseHost) self.getDataSetProperties(host) def getDataSetProperties(self, host): self.sql.createQuery( "SELECT", "skimrequests", "*", "(Status = 'Pending' OR Status = 'Queued') AND host = '" + host + "'") sqlFile = "sql_" + strftime("%d%m%Y_%H%M%S") + ".out" f = open(sqlFile, "w") f.write(self.sql.execQuery()) f.close() for res in open(sqlFile, "r"): #print res line = res.split("\n")[0] sqlRes = line.split(" ") if len(sqlRes) < 10: continue if sqlRes[0].rfind("ID") == -1: request = Request() request.ID = sqlRes[0] request.Username = sqlRes[1] request.Email = sqlRes[2] request.SamplePath = sqlRes[4] request.JSON = sqlRes[6] request.WorkingDir = sqlRes[7] request.nThreads = int(sqlRes[8]) request.nGroupFiles = int(sqlRes[9]) request.walltime = sqlRes[10] request.publishName = sqlRes[11] request.announce = int(sqlRes[12]) request.host = sqlRes[13] #print request.ID+" "+request.SamplePath+" "+request.Username tmp = sqlRes[3].split('\r') for line in tmp: line = line.strip("\n").lstrip('\n') split = line.split("\\n") if len(split) == 1: request.GridProxy += split[0] + '\n' else: request.GridProxy += split[1] + '\n' tmp = sqlRes[5].split('\r') #print tmp[0].split("\\n") for line in tmp[0].split("\\n"): #line = line.strip("\n").lstrip('\n') #print line split = line.split("\\n") if len(split) == 1: request.SkimXML += split[0] + '\n' else: request.SkimXML += split[1] + '\n' #print request.SkimXML self.requests.append(request) #sys.exit(1); os.remove(sqlFile)
class Request: ID = int(0) Username = "" Email = "" GridProxy = "" SamplePath = "" SkimXML = "" JSON = "" WorkingDir = "" nThreads = int(0) nGroupFiles = int(0) publishName = "" announce = int(0) walltime = "" host = "" wantGenEvent = bool(False) def __init__(self): # get the sensitive information from config file login = "" password = "" dbaseName = "" dbaseHost = "" for line in open(".config", "r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] self.sql = SQLHandler(dbaseName, login, password, dbaseHost) def setQueued(self): if not self.SamplePath == "": self.sql.createQuery( "UPDATE", "skimrequests", "", "SET `Status` = 'Queued' WHERE `ID` = " + str(self.ID) + " LIMIT 1") self.sql.execQuery() def setRunning(self): if not self.SamplePath == "": self.sql.createQuery( "UPDATE", "skimrequests", "", "SET `Status` = 'Running' WHERE `ID` = " + str(self.ID) + " LIMIT 1") self.sql.execQuery() def setDone(self): if not self.SamplePath == "": self.sql.createQuery( "UPDATE", "skimrequests", "", "SET `Status` = 'Done' WHERE `ID` = " + str(self.ID) + " LIMIT 1") self.sql.execQuery() def invalidate(self): if not self.SamplePath == "": self.sql.createQuery( "UPDATE", "skimrequests", "", "SET `Status` = 'Pending' WHERE `ID` = " + str(self.ID) + " LIMIT 1") self.sql.execQuery() def process(self): stamp = strftime("%d%m%Y_%H%M%S") TopSkimDir = Popen("echo $HOME", shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True).stdout.read().strip() + "/TopSkim/" #TopSkimDir = "/user/mmaes/AutoMaticTopTreeProducer-DEV/UserCode/mmaes/TopSkim/" #export VO_CMS_SW_DIR="/swmgrs/cmss/" #export SCRAM_ARCH="slc5_ia32_gcc434" #source $VO_CMS_SW_DIR/cmsset_default.sh #cd ../../TopSkim #python SkimTopTree.py --mtop-runmode --toptree_location=/pnfs/iihe/cms/store/user/dhondt/TToBLNu_TuneZ2_tW-channel_7TeV-madgraph/Fall10-START38_V12-v2/18112010_093940/TOPTREE -t CMSSW_38X_v1/TopBrussels/TopTreeProducer/ --skim_xml=skim_1291979767.xml -n 50 -j 1 [email protected] >& logs/topskim-error.txt # get the JSON File if needed if not self.JSON == "": #print self.JSON.split(";")[0] #JSONDest = "/localgrid/dhondt/JSON_"+stamp+".txt" #JSONDest = "/localgrid/mmaes/OFFICIAL_JSON_"+stamp+".json" JSONDest = "/localgrid/dhondt/OFFICIAL_JSON_" + stamp + ".json" #print JSONDest Popen("wget -O " + JSONDest + " " + self.JSON.split(";")[0], shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True).stdout.read() # store the skim XML SkimFile = "skim_" + stamp + ".xml" xmlFile = open(TopSkimDir + SkimFile, "w") xmlFile.write(self.SkimXML) xmlFile.close() # store the CRAB report JSON XML if len(self.JSON.split(";")) == 2: JSONFile = "/localgrid/dhondt/MERGED_JSON" + stamp + ".json" cJSONFile = open(JSONFile, "w") cJSONFile.write(mergeJSON(JSONDest, self.JSON.split(";")[1])) cJSONFile.close() # if we use a JSON we need to change useJSON and JSONFile in the skim XML if not self.JSON == "": self.SkimXML = "" for line in open(TopSkimDir + SkimFile): if line.rfind("useJSON") == -1: self.SkimXML += line else: splitline = line.split("useJSON") self.SkimXML += splitline[ 0] + "useJSON=\"1\" JSONFile=\"" + JSONFile + "\"/>\n" xmlFile = open(TopSkimDir + SkimFile, "w") xmlFile.write(self.SkimXML) xmlFile.close() #print self.SkimXML # store the grid proxy ProxyFile = "grid_proxy_" + strftime("%d%m%Y_%H%M%S") + ".proxy" pFile = open(TopSkimDir + ProxyFile, "w") pFile.write(self.GridProxy) pFile.close() cmd = "" #cmd += "export VO_CMS_SW_DIR=\"/swmgrs/cmss/\";" #cmd += "export SCRAM_ARCH=\"slc5_ia32_gcc434\";" #cmd += "source $VO_CMS_SW_DIR/cmsset_default.sh;" cmd += "export X509_USER_PROXY=\"" + TopSkimDir + ProxyFile + "\";" cmd += "cd " + TopSkimDir + ";" cmd += "python SkimTopTree.py --srmcp --mtop-runmode --mtop-setuser="******" --toptree_location=" + self.SamplePath + " -t " + self.WorkingDir + " --skim_xml=" + SkimFile + " -j " + str( self.nThreads) + " -n " + str(self.nGroupFiles) + " -w " + str( self.walltime) + " --email " + self.Email #cmd += "python SkimTopTree.py --mtop-runmode --mtop-setuser="******" --toptree_location="+self.SamplePath+" --skim_xml="+SkimFile+" -n "+str(self.nThreads)+" -j "+str(self.nGroupFiles)+" --email "+self.Email if self.host == "PBS": cmd += " --use-pbs" if not self.publishName == "": cmd += " -p " + self.publishName if self.announce: cmd += " -a" #cmd += "voms-proxy-info" log.output(cmd) #sys.exit(1) #cmd = "" pExe = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) while pExe.poll() == None: pExe.poll() time.sleep(10) if options.stdout: log.output(pExe.stdout.read()) exitCode = int(pExe.poll()) # clean up try: #os.remove(TopSkimDir+ProxyFile) #os.remove(TopSkimDir+SkimFile) os.remove(JSONDest) os.remove(JSONFile) except: log.output("Error cleaning up") return exitCode def resetQueue(self): self.sql.createQuery( "UPDATE", "skimrequests", "", "SET `Status` = 'Pending' WHERE `Status` = 'Queued'") self.sql.execQuery()
class Request: ID=int(0) DataSet = "" DataTier = "" useLocalDBS = "" RunSelection = "" skipPAT = "" DontStorePat = "" CMSSW_VER = "" CMSSW_VER_SAMPLE = "" GlobalTag = "" Priority = int(0) FLFilterPath = int(-1) wantGenEvent = bool(False) def __init__(self): # get the sensitive information from config file login="" password="" dbaseName="" dbaseHost="" for line in open(".config","r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] self.sql = SQLHandler(dbaseName,login,password,dbaseHost) def setQueued(self): if not self.DataSet == "": self.sql.createQuery("UPDATE","requests","","SET `Status` = 'Queued' WHERE `ID` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def setRunning(self): if not self.DataSet == "": self.sql.createQuery("UPDATE","requests","","SET `Status` = 'Running' WHERE `ID` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def setDone(self): if not self.DataSet == "": self.sql.createQuery("UPDATE","requests","","SET `Status` = 'Done', `Priority` = '0' WHERE `ID` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def invalidate(self): if not self.DataSet == "": self.sql.createQuery("UPDATE","requests","","SET `Status` = 'Pending', `Priority` = '0' WHERE `ID` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def process(self): #print self.RunSelection cmd ='python AutoMaticTopTreeProducer.py -c '+self.CMSSW_VER+' --cmssw_sample '+self.CMSSW_VER_SAMPLE+' -g '+self.GlobalTag+' -d '+self.DataSet if self.skipPAT == "1": cmd += " --skip-pat" if self.DontStorePat == "1": cmd += " --dont-store-pat" if self.DataTier == "PAT": cmd += " --start-from-pat" if self.DataTier == "PAT-MC": cmd += " --start-from-pat-mc" if options.doPBS: cmd += " --pbs-submit" if options.dryRun: cmd += " --dry-run" if self.wantGenEvent: cmd += " --addGenEvent" if int(self.FLFilterPath) > 0: cmd += " --flavourHistoryFilterPath="+str(self.FLFilterPath) if not self.useLocalDBS == "": cmd += " --dbsInst "+self.useLocalDBS if not self.DataSet.rfind("USER") == -1 and self.DataTier.rfind("PAT") == -1: # then we need a datatier cmd += " -t "+self.DataTier cmd += " --setLogFile logs/log-TopDB-Request-"+str(self.ID)+".txt" if not self.RunSelection == "NULL" and not self.RunSelection == "": cmd += " -r "+str(self.RunSelection) #if options.stdout: # cmd += " --log-stdout" #else: cmd += " >& logs/stdout" log.output(cmd) #return 0 #cmd = "sleep 10" pExe = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) # wait until the process is terminated while pExe.poll() == None: pExe.poll() time.sleep(10) if options.stdout: log.output(pExe.stdout.read()) exitCode = int(pExe.poll()) # possible exit codes: 0 (everything went fine) 1 (python error, need to send email) 2 (script error, mail sent by AutoMaticTopTreeProducer.py) return exitCode #return 0 def resetQueue(self): self.sql.createQuery("UPDATE","requests","","SET `Status` = 'Pending' WHERE `Status` = 'Queued'") self.sql.execQuery()
from aiogram.dispatcher import FSMContext from aiogram import Bot, Dispatcher, executor, types from aiogram.contrib.fsm_storage.memory import MemoryStorage from stateHandler import STATES from sqlHandler import SQLHandler from configHandler import ConfigHandler from sqlHandler.consts import TYPES, CONSTS, STRINGS from errorHandling import errorHandler logging.basicConfig(level=logging.INFO) bot = Bot(token=ConfigHandler.token) dispatcher = Dispatcher(bot, storage=MemoryStorage()) database = SQLHandler() def api_count_check() -> list: country_codes = TYPES.params_list result = [] for param in country_codes: response = requests.get( ConfigHandler.count_link, params=param ).json() result.append(response.get('count')) return result @dispatcher.message_handler(commands=['start']) async def start(message: types.Message):
class RequestHandler: sleepTime = int(300) requests = [] def __init__(self): # get the sensitive information from config file login="" password="" dbaseName="" dbaseHost="" for line in open(".config","r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] self.sql = SQLHandler(dbaseName,login,password,dbaseHost) self.getDataSetProperties() def getDataSetProperties(self): self.sql.createQuery("SELECT","simrequests","*","Status = 'Pending' OR Status = 'Queued' ORDER BY `Priority` DESC") sqlFile="sql_"+str(strftime("%d%m%Y_%H%M%S"))+".out" f = open(sqlFile,"w") f.write(self.sql.execQuery()) f.close() for res in open(sqlFile,"r"): #print res line = res.split("\n")[0] sqlRes = line.split(" ") if len(sqlRes) < 14: continue for a in xrange(len(sqlRes)): sqlRes[a]=sqlRes[a].split("\n")[0] sqlRes[a]=sqlRes[a].split("\r")[0] if sqlRes[0].rfind("id") == -1: request = Request() request.ID = sqlRes[0] request.Campaign=sqlRes[3] request.CMSSW_sim=sqlRes[4] request.GT_sim=sqlRes[5] request.nEvents=sqlRes[11] request.LHEDir=sqlRes[10] request.Template=sqlRes[6] request.PublishName_sim=sqlRes[9] request.CMSSW_top="" request.GT_top="" request.Priority = int(sqlRes[12]) if not request.Priority == 0: self.requests.append(request) request.Email = sqlRes[13] os.remove(sqlFile)
login="" password="" dbaseName="" dbaseHost="" for line in open("../.config","r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] sql = SQLHandler(dbaseName,login,password,dbaseHost) ################# ### MAIN LOOP ### ################# if not len(sys.argv) >= 3: print "Usage: python updateJSONFile.py <search_string> <new_json_url>" ## get the datasets from TopDB containing <search_string> print "\n-> Searching datasets in TopDB containing: "+sys.argv[1] sql.createQuery("SELECT","datasets","id,name","name REGEXP '"+sys.argv[1]+"'")
class Request: ID=int(0) removeId=int(0) removeType="" user="" comment="" log=logHandler("") def __init__(self): # get the sensitive information from config file login="" password="" dbaseName="" dbaseHost="" for line in open(".config","r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] self.sql = SQLHandler(dbaseName,login,password,dbaseHost) def setQueued(self): self.sql.createQuery("UPDATE","removals","","SET `Status` = 'Queued' WHERE `id` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def setRunning(self): self.comment = self.comment+" <br>"+strftime("%Y-%m-%d %H:%M:%S", localtime())+": Starting removal" self.sql.createQuery("UPDATE","removals","","SET `Status` = 'Running' , `AdminAction` = '"+self.comment+"' WHERE `id` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def setDone(self): self.comment = self.comment+" <br>"+strftime("%Y-%m-%d %H:%M:%S", localtime())+": Finished removal" self.sql.createQuery("UPDATE","removals","","SET `Status` = 'Done' , `AdminAction` = '"+self.comment+"' WHERE `id` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def invalidate(self): self.comment = self.comment+" <br>"+strftime("%Y-%m-%d %H:%M:%S", localtime())+": <b>Error encountered during removal</b>" self.sql.createQuery("UPDATE","removals","","SET `Status` = 'Frozen', `AdminAction` = '"+self.comment+"' WHERE `id` = "+str(self.ID)+" LIMIT 1") self.sql.execQuery() def process(self): # setup proper log location self.log = logHandler("logs/log-TopDB-CleaningAgent-"+str(self.ID)+".txt") self.log.output("****** Removing "+self.removeType+" with ID "+str(self.removeId)+" as requested by "+self.user+" ******") # pat to remove id = [] storagePath = [] dbsPublish = [] CffFilePath = [] # toptree to remove idTop = [] storagePathTop = [] mergedTopLocation = [] storagePathTopMail=[] ## REMOVE ONLY TOPTREE if self.removeType == "toptree": self.sql.createQuery("SELECT","toptrees","id,StoragePath,TopTreeLocation","id = '"+str(self.removeId)+"'") result = self.sql.execQuery().split('\n') if len(result) == 1: self.log.output(" ---> ERROR: TopTree was not found in TopDB") return 1 else: idTop.append(result[1].split("\t")[0]) storagePathTop.append(result[1].split("\t")[1]) mergedTopLocation.append(result[1].split("\t")[2]) ## REMOVE PAT + ALL DOWNSTREAM TOPTREES elif self.removeType == "patuple": self.sql.createQuery("SELECT","patuples","id,StoragePath,name,CffFilePath","id = '"+str(self.removeId)+"'") result = self.sql.execQuery().split('\n') if len(result) == 1: self.log.output(" ---> RRROR: PatTuple was not found in TopDB") return 1 else: id.append(result[1].split("\t")[0]) storagePath.append(result[1].split("\t")[1]) dbsPublish.append(result[1].split("\t")[2]) CffFilePath.append(result[1].split("\t")[3]) self.sql.createQuery("SELECT","toptrees","id,StoragePath,TopTreeLocation","patuple_id = '"+id[len(id)-1].split("\\n")[0]+"'") result2 = self.sql.execQuery().split('\n') if len(result2) > 1: for j in range(1,len(result2)-1): idTop.append(result2[j].split("\t")[0]) storagePathTop.append(result2[j].split("\t")[1]) mergedTopLocation.append(result2[j].split("\t")[2]) ## REMOVE DATASET + ALL DOWNSTREAM PAT + ALL DOWNSTREAM TOPTREES elif self.removeType == "dataset": self.sql.createQuery("SELECT","patuples","id,StoragePath,name,CffFilePath","dataset_id = '"+str(self.removeId)+"'") result = self.sql.execQuery().split('\n') if len(result) > 1: for i in range(1,len(result)-1): id.append(result[i].split("\t")[0]) storagePath.append(result[i].split("\t")[1]) dbsPublish.append(result[i].split("\t")[2]) CffFilePath.append(result[i].split("\t")[3]) self.sql.createQuery("SELECT","toptrees","id,StoragePath,TopTreeLocation","patuple_id = '"+id[len(id)-1].split("\\n")[0]+"'") result2 = self.sql.execQuery().split('\n') if len(result2) > 1: for j in range(1,len(result2)-1): idTop.append(result2[j].split("\t")[0]) storagePathTop.append(result2[j].split("\t")[1]) mergedTopLocation.append(result2[j].split("\t")[2]) ## CLEAN LEFTOVER FILES FROM FAILED PRODUCTION, CLEAN PATUPLES WITHOUT ANY TOPTREES elif self.removeType == "cleanpnfs": self.log.output("--> Cleaning up PNFS area for dhondt") self.log.output(" ---> Searching for PNFS directories from broken production") dirs = [] for dir in os.listdir("/pnfs/iihe/cms/store/user/dhondt/"): if not dir.rfind("Skimmed-TopTrees") == -1: continue; #if dir.rfind("7TeV_T2") == -1: continue # this is just to make testing go fast pExe = Popen("find /pnfs/iihe/cms/store/user/dhondt/"+dir+" -name TOPTREE", shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) out = pExe.stdout.read() for file in out.split("\n"): split = file.split("/") dirName = "" for i in xrange(0,len(split)-1): dirName += split[i]+"/" dirName = dirName.rstrip("/") if dirs.count(dirName) == 0 and len(dirName) > 0: dirs.append(dirName+"/TOPTREE") self.log.output(" ----> "+str(len(dirs))+" directory(s) found in total, cross-referencing TopDB...") for i in xrange(0,len(dirs)): self.sql.createQuery("SELECT","toptrees","id","StoragePath REGEXP '"+dirs[i]+"'") result = self.sql.execQuery().split('\n') self.sql.createQuery("SELECT","patuples","id","StoragePath REGEXP '"+dirs[i]+"'") result2 = self.sql.execQuery().split('\n') self.sql.createQuery("SELECT","gensims","id","PNFSPath REGEXP '"+dirs[i]+"'") result3 = self.sql.execQuery().split('\n') self.sql.createQuery("SELECT","recos","id","PNFSPath REGEXP '"+dirs[i]+"'") result4 = self.sql.execQuery().split('\n') if len(result) < 2 and len(result2) < 2 and len(result3) < 2 and len(result4) < 2 and storagePathTopMail.count(dirs[i]) == 0: filestat = os.stat(dirs[i]) filedate = filestat.st_mtime now = int(time.time()) last_mod=int(filedate) time_diff=now-last_mod if time_diff/(60*60) > 720: # just want the dir to be old enough to not remove ongoing prod self.log.output(" ----> Directory "+dirs[i]+" is not in TopDB, it should be removed! (Age: "+str(time_diff/(60*60*24))+" days)") #idTop.append(-9999) storagePathTopMail.append(dirs[i]) self.log.output(" ----> "+str(len(storagePathTopMail))+" directory(s) need removal!") self.log.output(" ---> Searching for PATuples that don't have a TopTree assigned") self.sql.createQuery("SELECT","patuples","id,StoragePath,name,CffFilePath","") result2 = self.sql.execQuery().split('\n') self.sql.createQuery("SELECT","toptrees","patuple_id","") result3 = self.sql.execQuery().split('\n') for i in result2: if i == "" or not i.rfind("id") == -1: continue tmpid = i.split("\t")[0] found=bool(False) for j in result3: if j == "": continue if tmpid == j: found=bool(True) #if not found: #id.append(i.split("\t")[0]) #storagePath.append(i.split("\t")[1]) #dbsPublish.append(i.split("\t")[2]) #CffFilePath.append(i.split("\t")[3]) msg = "Dear admins," if len(storagePathTopMail) > 0: msg += "\n\n The automatic TopDB PNFS cleaning tool has found "+str(len(storagePathTopMail))+" directories on PNFS not corresponding to any entry in the TopDB database." msg += "\n\n Please have a look at the following list:" for s in storagePathTopMail: msg += "\n\n \t rm -rfv "+s else: msg += "\n\n The automatic TopDB PNFS cleaning tool has found NO directories on PNFS not corresponding to any entry in the TopDB database." msg += "\n\nCheers,\nHector the cleaning agent" mail = MailHandler() mail.sendMail("error","Report from TopDB PNFS cleaning",msg) ## CLEAN LEFTOVER FILES FROM FAILED PRODUCTION, CLEAN PATUPLES WITHOUT ANY TOPTREES elif self.removeType == "cleancrablogs": days=50 self.log.output(" ---> Listing Configuration directories") self.log.output(" ---> Checking every Configuration directory (older than "+str(days)+" days) for large amounts of *.stdout from CRAB") ldirs = [] cleanup_ldirsToRemove = [] basedir=(Popen("echo $HOME", shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True).stdout.read()).strip()+"/AutoMaticTopTreeProducer/" for dir in os.listdir(basedir): if dir.rfind("CMSSW_") == -1: continue; pExe = Popen("find "+basedir+dir.strip()+"/ -name crab_*.cfg", shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) out = pExe.stdout.read() for file in out.split("\n"): split = file.split("/") dirName = "" for i in xrange(0,len(split)-1): dirName += split[i]+"/" dirName = dirName.rstrip("/") if ldirs.count(dirName) == 0 and len(dirName) > 0: ldirs.append(dirName.split("/AutoMaticTopTreeProducer/")[1]) # becase we don't want it to crash on changes /home /user # time to clean out some big chunks of stdout files if not dirName == "": if not dirName.find("find: ") == -1: dirName = dirName.split("find: ")[1] #print dirName filestat = os.stat(dirName) filedate = filestat.st_mtime now = int(time.time()) last_mod=int(filedate) time_diff=now-last_mod if time_diff/(60*60*24) > days: #self.log.output(" ---> Cleaning CRAB stdout files in "+dirName+" (Age: "+str(time_diff/(3600*24))+" days)") crabdir="" for dir in os.listdir(dirName): if not dir.rfind("TOPTREE_") == -1 and dir.rfind(".py") == -1 and os.path.isdir(dirName+"/"+dir): crabdir=dirName+"/"+dir if not crabdir == "": numfiles=int(0) keepstdout="" keepstderr="" keepxml="" if os.path.exists(crabdir+"/log/crab.log"): self.log.output(" ---> Cleaning crab.log in "+crabdir+"/log/ (Age: "+str(time_diff/(3600*24))+" days)") os.unlink(crabdir+"/log/crab.log") #sys.exit(1) for file in os.listdir(crabdir+"/res"): if not file.rfind(".stdout") == -1: if os.path.getsize(crabdir+"/res/"+file) > 0 and keepstdout == "": keepstdout=file numfiles=numfiles+1 #print keepstdout if not os.path.isdir(crabdir+"/res"): numfiles=0 #print numfiles #print str(numfiles)+" "+dirName if numfiles > 2 and dirName.rfind("Run201") == -1: print numfiles self.log.output(" ---> Cleaning CRAB stdout files in "+crabdir+" (Age: "+str(time_diff/(3600*24))+" days)") keepstderr=keepstdout.split(".stdout")[0]+".stderr" keepxml="crab_fjr_"+(keepstdout.split(".stdout")[0]).split("CMSSW_")[1]+".xml" for file in os.listdir(crabdir+"/res"): if not os.path.isdir(crabdir+"/res/"+file) and file.rfind("Submission") == -1 and file.rfind(".json") == -1 and not file == keepxml and not file == keepstdout and not file == keepstderr: self.log.output(" ---> Removing crab output "+file) os.unlink(crabdir+"/res/"+file) elif not file.rfind("Submission") == -1: self.log.output(" ---> Removing old Submission_X dir: "+file) shutil.rmtree(crabdir+"/res/"+file) elif not dirName.rfind("Run201") == -1: if os.path.exists(crabdir+"/res/.shrunk"): continue self.log.output(" ---> (DATA PRODUCTION) Removing unuseful lines from stdout files in "+crabdir+" (Age: "+str(time_diff/(3600*24))+" days)") for file in os.listdir(crabdir+"/res"): if not file.rfind("Submission") == -1: self.log.output(" ---> Removing old Submission_X dir: "+file) shutil.rmtree(crabdir+"/res/"+file) elif not os.path.isdir(crabdir+"/res/"+file) and file.rfind("Submission") == -1 and file.rfind(".json") == -1 and not file.rfind(".stdout") == -1: self.log.output(" ---> Shrinking crab output "+file) tmpfile = open(crabdir+"/res/"+file+"_tmp","w") for line in open(crabdir+"/res/"+file): if line.rfind("Begin processing") == -1 and line.rfind("Vertex") == -1 and line.rfind("%MSG") == -1: tmpfile.write(line) os.unlink(crabdir+"/res/"+file) os.rename(crabdir+"/res/"+file+"_tmp",crabdir+"/res/"+file) f = open(crabdir+"/res/.shrunk","w") # leave a stamp that this dir is fixed f.close() self.log.output(" ----> "+str(len(ldirs))+" Configuration directory(s) found in total, cross-referencing TopDB...") self.log.output("") ldirs = [] # disable this for now for i in xrange(0,len(ldirs)): self.sql.createQuery("SELECT","toptrees","id","CffFilePath REGEXP '"+ldirs[i]+"'") result = self.sql.execQuery().split('\n') self.sql.createQuery("SELECT","patuples","id","CffFilePath REGEXP '"+ldirs[i]+"'") result2 = self.sql.execQuery().split('\n') self.sql.createQuery("SELECT","gensims","id","CffPath REGEXP '"+ldirs[i]+"'") result3 = self.sql.execQuery().split('\n') self.sql.createQuery("SELECT","recos","id","CffPath REGEXP '"+ldirs[i]+"'") result4 = self.sql.execQuery().split('\n') if len(result) < 2 and len(result2) < 2 and len(result3) < 2 and len(result4) < 2 and cleanup_ldirsToRemove.count(ldirs[i]) == 0: filestat = os.stat(basedir+"/"+ldirs[i]) filedate = filestat.st_mtime now = int(time.time()) last_mod=int(filedate) time_diff=now-last_mod if time_diff/(60*60*24) > days: # just want the dir to be old enough to not remove ongoing prod self.log.output(" ----> Directory "+ldirs[i]+" is not in TopDB, it should be removed! (Age: "+str(time_diff/(60*60*24))+" days)") cleanup_ldirsToRemove.append(ldirs[i]) self.log.output(" ----> "+str(len(cleanup_ldirsToRemove))+" directory(s) need removal!") ## SUMMARY OF THE REMOVAL self.log.output(" --> Summary of the removal") for i in range(0,len(id)): self.log.output(" * Removing PATtuple with ID "+str(id[i])+" at "+storagePath[i]) for i in range(0,len(idTop)): self.log.output(" * Removing TopTree with ID "+str(idTop[i])+" at "+storagePathTop[i]) #if self.removeType == "cleanpnfs": # return 0; # START REMOVAL time.sleep(20) log.output(" --> Starting the removal procedure") rm = RemoveHelper(self.sql,self.log) for i in range(0,len(id)): rm.rmSRMdir(storagePath[i]) rm.rmFromTopDB("patuples",id[i]) rm.invalDBS(dbsPublish[i],CffFilePath[i]) for i in range(0,len(idTop)): rm.rmSRMdir(storagePathTop[i]) if idTop[i] > 0: rm.rmFromTopDB("toptrees",idTop[i]) if self.removeType == "dataset": rm.rmFromTopDB("datasets",self.removeId) self.log.output(" --> Ended removal procedure") return 0 def resetQueue(self): self.sql.createQuery("UPDATE","removals","","SET `Status` = 'Pending' WHERE `Status` = 'Queued'") self.sql.execQuery()
class RequestHandler: sleepTime = int(30) requests = [] def __init__(self): # get the sensitive information from config file login="" password="" dbaseName="" dbaseHost="" for line in open(".config","r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] self.sql = SQLHandler(dbaseName,login,password,dbaseHost) self.getProperties() def getProperties(self): self.sql.createQuery("SELECT","removals","*","Status = 'Pending' OR Status = 'Queued' ORDER BY `ID` DESC") timestamp = strftime("%d%m%Y_%H%M%S") f = open("sql"+str(timestamp)+".out","w") f.write(self.sql.execQuery()) f.close() for res in open("sql"+str(timestamp)+".out","r"): #print res line = res.split("\n")[0] sqlRes = line.split(" ") for a in xrange(len(sqlRes)): sqlRes[a]=sqlRes[a].split("\n")[0] sqlRes[a]=sqlRes[a].split("\r")[0] if sqlRes[0].rfind("ID") == -1 and len(sqlRes) > 6: request = Request() request.ID = sqlRes[0] request.removeId = sqlRes[4] request.removeType = sqlRes[3] request.user = sqlRes[1] request.comment = sqlRes[6] self.requests.append(request) os.remove("sql"+str(timestamp)+".out")
login="" password="" dbaseName="" dbaseHost="" for line in open("../.config","r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] sql = SQLHandler(dbaseName,login,password,dbaseHost) ############### ### METHODS ### ############### def rmSRMdir (dir): global srmHost if os.path.exists(dir): log.output(" ---> Going into directory "+dir) tmp = os.listdir(dir)
baseDir = ( Popen("echo $HOME", shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True).stdout.read().strip() + "/AutoMaticTopTreeProducer" ) for line in open("../.config", "r"): if not line.rfind("DBUser") == -1: login = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBPass") == -1: password = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBHost") == -1: dbaseHost = line.split(" ")[1].split("\n")[0] elif not line.rfind("DBName") == -1: dbaseName = line.split(" ")[1].split("\n")[0] sql = SQLHandler(dbaseName, login, password, dbaseHost) # checking the CMSSW versions installed -> insert into topDB where needed for entry in os.listdir(baseDir): if os.path.isdir(baseDir + "/" + entry) == True and not entry.rfind("CMSSW") == -1: print "Local version found: " + entry isInDB = bool(False) # check if this version exists in topDB otherwise add it options = "version REGEXP '" + entry + "'"