def __init__(self):
# get the sensitive information from config file
login=""
password=""
dbaseName=""
dbaseHost=""
for line in open(".config","r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
self.sql = SQLHandler(dbaseName,login,password,dbaseHost)
login=""
password=""
dbaseName=""
dbaseHost=""
for line in open(".config","r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
sql = SQLHandler(dbaseName,login,password,dbaseHost)
if not options.stdout == True:
log = logHandler("logs/ProductionWorkflow.txt")
else:
log = logHandler("")
sleepTime = int(300)
ID=int(0)
DataSet = ""
DataTier = ""
useLocalDBS = ""
RunSelection = ""
skipPAT = ""
CMSSW_VER = ""
login=""
password=""
dbaseName=""
dbaseHost=""
for line in open("../.config","r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
sql = SQLHandler(dbaseName,login,password,dbaseHost)
###############
### METHODS ###
###############
def insertSimRequest(campaign,CMSSWversion_sim,GlobalTag_sim,Template_sim,PublishName_sim,LHEList,nEvents,prio,email):
global options
values = []
values.append([])
values[len(values)-1].append("Date")
values[len(values)-1].append(datetime.date.today().strftime("%Y-%m-%d"))
values.append([])
login = ""
password = ""
dbaseName = ""
dbaseHost = ""
for line in open("../.config", "r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
sql = SQLHandler(dbaseName, login, password, dbaseHost)
#################
### MAIN LOOP ###
#################
if not len(sys.argv) >= 3:
print "Usage: python updateJSONFile.py <search_string> <new_json_url>"
## get the datasets from TopDB containing <search_string>
print "\n-> Searching datasets in TopDB containing: " + sys.argv[1]
sql.createQuery("SELECT", "datasets", "id,name",
"name REGEXP '" + sys.argv[1] + "'")
stdin=PIPE,
stdout=PIPE,
stderr=STDOUT,
close_fds=True).stdout.read().strip() + "/AutoMaticTopTreeProducer"
for line in open("../.config", "r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
sql = SQLHandler(dbaseName, login, password, dbaseHost)
# checking the CMSSW versions installed -> insert into topDB where needed
for entry in os.listdir(baseDir):
if os.path.isdir(baseDir + "/" +
entry) == True and not entry.rfind("CMSSW") == -1:
print "Local version found: " + entry
isInDB = bool(False)
# check if this version exists in topDB otherwise add it
options = "version REGEXP '" + entry + "'"
class RequestHandler:
sleepTime = int(300)
requests = []
def __init__(self):
# get the sensitive information from config file
login=""
password=""
dbaseName=""
dbaseHost=""
for line in open(".config","r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
self.sql = SQLHandler(dbaseName,login,password,dbaseHost)
self.getDataSetProperties()
def patParentID(self,name):
self.sql.createQuery("SELECT","patuples","dataset_id","name REGEXP '"+name+"' LIMIT 0,1")
f = open("ttprod_sql2.out","w")
f.write(self.sql.execQuery())
f.close()
lines = open("ttprod_sql2.out","r").readlines()
if len(lines) == 2:
return int(lines[1])
else:
return -1
def getDataSetProperties(self):
self.sql.createQuery("SELECT","requests","*","Status = 'Pending' OR Status = 'Queued' ORDER BY `Priority` DESC")
sqlFile="ttprod_sql_"+str(strftime("%d%m%Y_%H%M%S"))+".out"
sqlFile2="ttprod_sql2_"+str(strftime("%d%m%Y_%H%M%S"))+".out"
f = open(sqlFile,"w")
f.write(self.sql.execQuery())
f.close()
for res in open(sqlFile,"r"):
#print res
line = res.split("\n")[0]
sqlRes = line.split(" ")
if len(sqlRes) < 11:
continue
if not sqlRes[1].rfind("/") == -1:
request = Request()
request.ID = sqlRes[0]
request.DataSet = sqlRes[1]
request.DataTier = sqlRes[2]
request.RunSelection = sqlRes[3]
request.FLFilterPath = sqlRes[4]
request.useLocalDBS = sqlRes[5]
request.skipPAT = sqlRes[6]
request.DontStorePat = sqlRes[7]
request.CMSSW_VER = sqlRes[8]
request.GlobalTag = sqlRes[9]
request.Priority = int(sqlRes[10])
if not request.Priority == 0:
# in case it is reco->toptree (+- using pat)
self.sql.createQuery("SELECT","datasets","CMSSWversion,process","name REGEXP '"+request.DataSet+"' LIMIT 0,1")
f = open(sqlFile2,"w")
f.write(self.sql.execQuery())
f.close()
for res in open(sqlFile2,"r"):
#print res
line = res.split("\n")[0]
sqlRes = line.split(" ")
if len(sqlRes) < 2:
continue
if not sqlRes[0] == "CMSSWversion":
#print "Ver: "+sqlRes[0]
request.CMSSW_VER_SAMPLE = sqlRes[0];
if sqlRes[1] == "TTJets" or sqlRes[1] == "TTbar":
request.wantGenEvent = True;
#print sqlRes[0]
#print request.CMSSW_VER
os.remove(sqlFile2)
# in case we start from pat
dataset_id = self.patParentID(request.DataSet)
#print dataset_id
if not dataset_id == -1:
#print "going from PAT"
self.sql.createQuery("SELECT","datasets","CMSSWversion,process","id = '"+str(dataset_id)+"' LIMIT 0,1")
f = open(sqlFile2,"w")
f.write(self.sql.execQuery())
f.close()
for res in open(sqlFile2,"r"):
#print res
line = res.split("\n")[0]
sqlRes = line.split(" ")
if len(sqlRes) < 2:
continue
if not sqlRes[0] == "CMSSWversion":
request.CMSSW_VER_SAMPLE = sqlRes[0];
#print sqlRes[0]
#print request.CMSSW_VER
if sqlRes[1] == "TTJets" or sqlRes[1] == "TTbar":
request.wantGenEvent = True;
if not sqlRes[1] == "Data":
request.DataTier = "PAT-MC"
os.remove(sqlFile2)
self.requests.append(request)
os.remove(sqlFile)
login = ""
password = ""
dbaseName = ""
dbaseHost = ""
for line in open("../.config", "r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
sql = SQLHandler(dbaseName, login, password, dbaseHost)
###############
### METHODS ###
###############
def rmSRMdir(dir):
global srmHost
if os.path.exists(dir):
log.output(" ---> Going into directory " + dir)
tmp = os.listdir(dir)
class Request:
ID=int(0)
Email = ""
Campaign=""
CMSSW_sim=""
GT_sim=""
PublishName_sim=""
CMSSW_top=""
GT_top=""
nEvents=""
LHEDir=""
Template=""
Priority=int(0)
doDry=bool(False)
def __init__(self):
# get the sensitive information from config file
login=""
password=""
dbaseName=""
dbaseHost=""
for line in open(".config","r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
self.sql = SQLHandler(dbaseName,login,password,dbaseHost)
def setQueued(self):
self.sql.createQuery("UPDATE","simrequests","","SET `Status` = 'Queued' WHERE `id` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def setRunning(self):
self.sql.createQuery("UPDATE","simrequests","","SET `Status` = 'Running' WHERE `id` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def setDone(self):
self.sql.createQuery("UPDATE","simrequests","","SET `Status` = 'Done', `Priority` = '0' WHERE `id` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def invalidate(self):
self.sql.createQuery("UPDATE","simrequests","","SET `Status` = 'Pending', `Priority` = '0' WHERE `id` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def process(self):
cmssw=""
gt=""
publish=""
template=""
LHEDir=""
template=" -f "+self.Template+" "
LHEDir="-d "+self.LHEDir+" "
cmssw=self.CMSSW_sim
gt=self.GT_sim
publish=self.PublishName_sim
cmd ="python AutoMaticSIMProducer.py -c "+cmssw+" -g "+gt+" -p "+publish+" -a "+self.Campaign+" --email "+self.Email
if not template == "":
cmd+=template
if not LHEDir == "":
cmd+=LHEDir
if not self.nEvents == "":
cmd+=" -n "+self.nEvents
if self.doDry:
cmd+=" --dry-run "
cmd += " --setLogFile logs/log-TopDB-SIMRequest-"+str(self.ID)+".txt"
cmd += " >& logs/stdout"
log.output(cmd)
#print cmd
#sys.exit(1)
#return 0
#cmd = "sleep 1"
pExe = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
# wait until the process is terminated
while pExe.poll() == None:
pExe.poll()
time.sleep(10)
if options.stdout:
log.output(pExe.stdout.read())
exitCode = int(pExe.poll())
# possible exit codes: 0 (everything went fine) 1 (python error, need to send email) 2 (script error, mail sent by AutoMaticTopTreeProducer.py)
return exitCode
#return 0
def resetQueue(self):
self.sql.createQuery("UPDATE","simrequests","","SET `Status` = 'Pending' WHERE `Status` = 'Queued'")
self.sql.execQuery()
login = ""
password = ""
dbaseName = ""
dbaseHost = ""
for line in open(".config", "r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
sql = SQLHandler(dbaseName, login, password, dbaseHost)
if not options.stdout == True:
log = logHandler("logs/ProductionWorkflow.txt")
else:
log = logHandler("")
sleepTime = int(300)
ID = int(0)
DataSet = ""
DataTier = ""
useLocalDBS = ""
RunSelection = ""
skipPAT = ""
CMSSW_VER = ""
class RequestHandler:
sleepTime = int(30)
requests = []
def __init__(self,host):
# get the sensitive information from config file
login=""
password=""
dbaseName=""
dbaseHost=""
for line in open(".config","r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
self.sql = SQLHandler(dbaseName,login,password,dbaseHost)
self.getDataSetProperties(host)
def getDataSetProperties(self,host):
self.sql.createQuery("SELECT","skimrequests","*","(Status = 'Pending' OR Status = 'Queued') AND host = '"+host+"'")
sqlFile = "sql_"+strftime("%d%m%Y_%H%M%S")+".out"
f = open(sqlFile,"w")
f.write(self.sql.execQuery())
f.close()
for res in open(sqlFile,"r"):
#print res
line = res.split("\n")[0]
sqlRes = line.split(" ")
if len(sqlRes) < 10:
continue
if sqlRes[0].rfind("ID") == -1:
request = Request();
request.ID = sqlRes[0]
request.Username = sqlRes[1]
request.Email = sqlRes[2]
request.SamplePath = sqlRes[4]
request.JSON = sqlRes[6]
request.WorkingDir = sqlRes[7]
request.nThreads = int(sqlRes[8])
request.nGroupFiles = int(sqlRes[9])
request.walltime = sqlRes[10]
request.publishName = sqlRes[11]
request.announce = int(sqlRes[12])
request.host = sqlRes[13]
#print request.ID+" "+request.SamplePath+" "+request.Username
tmp = sqlRes[3].split('\r')
for line in tmp:
line = line.strip("\n").lstrip('\n')
split = line.split("\\n")
if len(split) == 1:
request.GridProxy += split[0]+'\n'
else:
request.GridProxy += split[1]+'\n'
tmp = sqlRes[5].split('\r')
#print tmp[0].split("\\n")
for line in tmp[0].split("\\n"):
#line = line.strip("\n").lstrip('\n')
#print line
split = line.split("\\n")
if len(split) == 1:
request.SkimXML += split[0]+'\n'
else:
request.SkimXML += split[1]+'\n'
#print request.SkimXML
self.requests.append(request)
#sys.exit(1);
os.remove(sqlFile)
class Request:
ID=int(0)
Username = ""
Email = ""
GridProxy = ""
SamplePath = ""
SkimXML = ""
JSON = ""
WorkingDir = ""
nThreads = int(0)
nGroupFiles = int(0)
publishName = ""
announce = int(0)
walltime = ""
host = ""
wantGenEvent = bool(False)
def __init__(self):
# get the sensitive information from config file
login=""
password=""
dbaseName=""
dbaseHost=""
for line in open(".config","r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
self.sql = SQLHandler(dbaseName,login,password,dbaseHost)
def setQueued(self):
if not self.SamplePath == "":
self.sql.createQuery("UPDATE","skimrequests","","SET `Status` = 'Queued' WHERE `ID` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def setRunning(self):
if not self.SamplePath == "":
self.sql.createQuery("UPDATE","skimrequests","","SET `Status` = 'Running' WHERE `ID` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def setDone(self):
if not self.SamplePath == "":
self.sql.createQuery("UPDATE","skimrequests","","SET `Status` = 'Done' WHERE `ID` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def invalidate(self):
if not self.SamplePath == "":
self.sql.createQuery("UPDATE","skimrequests","","SET `Status` = 'Pending' WHERE `ID` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def process(self):
stamp = strftime("%d%m%Y_%H%M%S")
TopSkimDir = Popen("echo $HOME", shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True).stdout.read().strip()+"/TopSkim/"
#TopSkimDir = "/user/mmaes/AutoMaticTopTreeProducer-DEV/UserCode/mmaes/TopSkim/"
#export VO_CMS_SW_DIR="/swmgrs/cmss/"
#export SCRAM_ARCH="slc5_ia32_gcc434"
#source $VO_CMS_SW_DIR/cmsset_default.sh
#cd ../../TopSkim
#python SkimTopTree.py --mtop-runmode --toptree_location=/pnfs/iihe/cms/store/user/dhondt/TToBLNu_TuneZ2_tW-channel_7TeV-madgraph/Fall10-START38_V12-v2/18112010_093940/TOPTREE -t CMSSW_38X_v1/TopBrussels/TopTreeProducer/ --skim_xml=skim_1291979767.xml -n 50 -j 1 [email protected] >& logs/topskim-error.txt
# get the JSON File if needed
if not self.JSON == "":
#print self.JSON.split(";")[0]
#JSONDest = "/localgrid/dhondt/JSON_"+stamp+".txt"
#JSONDest = "/localgrid/mmaes/OFFICIAL_JSON_"+stamp+".json"
JSONDest = "/localgrid/dhondt/OFFICIAL_JSON_"+stamp+".json"
#print JSONDest
Popen("wget -O "+JSONDest+" "+self.JSON.split(";")[0], shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True).stdout.read()
# store the skim XML
SkimFile = "skim_"+stamp+".xml"
xmlFile = open(TopSkimDir+SkimFile,"w")
xmlFile.write(self.SkimXML)
xmlFile.close()
# store the CRAB report JSON XML
if len(self.JSON.split(";")) == 2:
JSONFile = "/localgrid/dhondt/MERGED_JSON"+stamp+".json"
cJSONFile = open(JSONFile,"w")
cJSONFile.write(mergeJSON(JSONDest,self.JSON.split(";")[1]))
cJSONFile.close()
# if we use a JSON we need to change useJSON and JSONFile in the skim XML
if not self.JSON == "":
self.SkimXML = ""
for line in open(TopSkimDir+SkimFile):
if line.rfind("useJSON") == -1:
self.SkimXML += line
else:
splitline = line.split("useJSON")
self.SkimXML += splitline[0]+"useJSON=\"1\" JSONFile=\""+JSONFile+"\"/>\n"
xmlFile = open(TopSkimDir+SkimFile,"w")
xmlFile.write(self.SkimXML)
xmlFile.close()
#print self.SkimXML
# store the grid proxy
ProxyFile = "grid_proxy_"+strftime("%d%m%Y_%H%M%S")+".proxy"
pFile = open(TopSkimDir+ProxyFile,"w")
pFile.write(self.GridProxy)
pFile.close()
cmd = ""
#cmd += "export VO_CMS_SW_DIR=\"/swmgrs/cmss/\";"
#cmd += "export SCRAM_ARCH=\"slc5_ia32_gcc434\";"
#cmd += "source $VO_CMS_SW_DIR/cmsset_default.sh;"
cmd += "export X509_USER_PROXY=\""+TopSkimDir+ProxyFile+"\";"
cmd += "cd "+TopSkimDir+";"
cmd += "python SkimTopTree.py --srmcp --mtop-runmode --mtop-setuser="******" --toptree_location="+self.SamplePath+" -t "+self.WorkingDir+" --skim_xml="+SkimFile+" -j "+str(self.nThreads)+" -n "+str(self.nGroupFiles)+" -w "+str(self.walltime)+" --email "+self.Email
#cmd += "python SkimTopTree.py --mtop-runmode --mtop-setuser="******" --toptree_location="+self.SamplePath+" --skim_xml="+SkimFile+" -n "+str(self.nThreads)+" -j "+str(self.nGroupFiles)+" --email "+self.Email
if self.host == "PBS":
cmd += " --use-pbs"
if not self.publishName == "":
cmd += " -p "+self.publishName
if self.announce:
cmd += " -a"
#cmd += "voms-proxy-info"
log.output(cmd)
#sys.exit(1)
#cmd = ""
pExe = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
while pExe.poll() == None:
pExe.poll()
time.sleep(10)
if options.stdout:
log.output(pExe.stdout.read())
exitCode = int(pExe.poll())
# clean up
try:
#os.remove(TopSkimDir+ProxyFile)
#os.remove(TopSkimDir+SkimFile)
os.remove(JSONDest)
os.remove(JSONFile)
except:
log.output("Error cleaning up")
return exitCode
def resetQueue(self):
self.sql.createQuery("UPDATE","skimrequests","","SET `Status` = 'Pending' WHERE `Status` = 'Queued'")
self.sql.execQuery()
class RequestHandler:
sleepTime = int(30)
requests = []
def __init__(self, host):
# get the sensitive information from config file
login = ""
password = ""
dbaseName = ""
dbaseHost = ""
for line in open(".config", "r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
self.sql = SQLHandler(dbaseName, login, password, dbaseHost)
self.getDataSetProperties(host)
def getDataSetProperties(self, host):
self.sql.createQuery(
"SELECT", "skimrequests", "*",
"(Status = 'Pending' OR Status = 'Queued') AND host = '" + host +
"'")
sqlFile = "sql_" + strftime("%d%m%Y_%H%M%S") + ".out"
f = open(sqlFile, "w")
f.write(self.sql.execQuery())
f.close()
for res in open(sqlFile, "r"):
#print res
line = res.split("\n")[0]
sqlRes = line.split(" ")
if len(sqlRes) < 10:
continue
if sqlRes[0].rfind("ID") == -1:
request = Request()
request.ID = sqlRes[0]
request.Username = sqlRes[1]
request.Email = sqlRes[2]
request.SamplePath = sqlRes[4]
request.JSON = sqlRes[6]
request.WorkingDir = sqlRes[7]
request.nThreads = int(sqlRes[8])
request.nGroupFiles = int(sqlRes[9])
request.walltime = sqlRes[10]
request.publishName = sqlRes[11]
request.announce = int(sqlRes[12])
request.host = sqlRes[13]
#print request.ID+" "+request.SamplePath+" "+request.Username
tmp = sqlRes[3].split('\r')
for line in tmp:
line = line.strip("\n").lstrip('\n')
split = line.split("\\n")
if len(split) == 1:
request.GridProxy += split[0] + '\n'
else:
request.GridProxy += split[1] + '\n'
tmp = sqlRes[5].split('\r')
#print tmp[0].split("\\n")
for line in tmp[0].split("\\n"):
#line = line.strip("\n").lstrip('\n')
#print line
split = line.split("\\n")
if len(split) == 1:
request.SkimXML += split[0] + '\n'
else:
request.SkimXML += split[1] + '\n'
#print request.SkimXML
self.requests.append(request)
#sys.exit(1);
os.remove(sqlFile)
class Request:
ID = int(0)
Username = ""
Email = ""
GridProxy = ""
SamplePath = ""
SkimXML = ""
JSON = ""
WorkingDir = ""
nThreads = int(0)
nGroupFiles = int(0)
publishName = ""
announce = int(0)
walltime = ""
host = ""
wantGenEvent = bool(False)
def __init__(self):
# get the sensitive information from config file
login = ""
password = ""
dbaseName = ""
dbaseHost = ""
for line in open(".config", "r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
self.sql = SQLHandler(dbaseName, login, password, dbaseHost)
def setQueued(self):
if not self.SamplePath == "":
self.sql.createQuery(
"UPDATE", "skimrequests", "",
"SET `Status` = 'Queued' WHERE `ID` = " + str(self.ID) +
" LIMIT 1")
self.sql.execQuery()
def setRunning(self):
if not self.SamplePath == "":
self.sql.createQuery(
"UPDATE", "skimrequests", "",
"SET `Status` = 'Running' WHERE `ID` = " + str(self.ID) +
" LIMIT 1")
self.sql.execQuery()
def setDone(self):
if not self.SamplePath == "":
self.sql.createQuery(
"UPDATE", "skimrequests", "",
"SET `Status` = 'Done' WHERE `ID` = " + str(self.ID) +
" LIMIT 1")
self.sql.execQuery()
def invalidate(self):
if not self.SamplePath == "":
self.sql.createQuery(
"UPDATE", "skimrequests", "",
"SET `Status` = 'Pending' WHERE `ID` = " + str(self.ID) +
" LIMIT 1")
self.sql.execQuery()
def process(self):
stamp = strftime("%d%m%Y_%H%M%S")
TopSkimDir = Popen("echo $HOME",
shell=True,
stdin=PIPE,
stdout=PIPE,
stderr=STDOUT,
close_fds=True).stdout.read().strip() + "/TopSkim/"
#TopSkimDir = "/user/mmaes/AutoMaticTopTreeProducer-DEV/UserCode/mmaes/TopSkim/"
#export VO_CMS_SW_DIR="/swmgrs/cmss/"
#export SCRAM_ARCH="slc5_ia32_gcc434"
#source $VO_CMS_SW_DIR/cmsset_default.sh
#cd ../../TopSkim
#python SkimTopTree.py --mtop-runmode --toptree_location=/pnfs/iihe/cms/store/user/dhondt/TToBLNu_TuneZ2_tW-channel_7TeV-madgraph/Fall10-START38_V12-v2/18112010_093940/TOPTREE -t CMSSW_38X_v1/TopBrussels/TopTreeProducer/ --skim_xml=skim_1291979767.xml -n 50 -j 1 [email protected] >& logs/topskim-error.txt
# get the JSON File if needed
if not self.JSON == "":
#print self.JSON.split(";")[0]
#JSONDest = "/localgrid/dhondt/JSON_"+stamp+".txt"
#JSONDest = "/localgrid/mmaes/OFFICIAL_JSON_"+stamp+".json"
JSONDest = "/localgrid/dhondt/OFFICIAL_JSON_" + stamp + ".json"
#print JSONDest
Popen("wget -O " + JSONDest + " " + self.JSON.split(";")[0],
shell=True,
stdin=PIPE,
stdout=PIPE,
stderr=STDOUT,
close_fds=True).stdout.read()
# store the skim XML
SkimFile = "skim_" + stamp + ".xml"
xmlFile = open(TopSkimDir + SkimFile, "w")
xmlFile.write(self.SkimXML)
xmlFile.close()
# store the CRAB report JSON XML
if len(self.JSON.split(";")) == 2:
JSONFile = "/localgrid/dhondt/MERGED_JSON" + stamp + ".json"
cJSONFile = open(JSONFile, "w")
cJSONFile.write(mergeJSON(JSONDest, self.JSON.split(";")[1]))
cJSONFile.close()
# if we use a JSON we need to change useJSON and JSONFile in the skim XML
if not self.JSON == "":
self.SkimXML = ""
for line in open(TopSkimDir + SkimFile):
if line.rfind("useJSON") == -1:
self.SkimXML += line
else:
splitline = line.split("useJSON")
self.SkimXML += splitline[
0] + "useJSON=\"1\" JSONFile=\"" + JSONFile + "\"/>\n"
xmlFile = open(TopSkimDir + SkimFile, "w")
xmlFile.write(self.SkimXML)
xmlFile.close()
#print self.SkimXML
# store the grid proxy
ProxyFile = "grid_proxy_" + strftime("%d%m%Y_%H%M%S") + ".proxy"
pFile = open(TopSkimDir + ProxyFile, "w")
pFile.write(self.GridProxy)
pFile.close()
cmd = ""
#cmd += "export VO_CMS_SW_DIR=\"/swmgrs/cmss/\";"
#cmd += "export SCRAM_ARCH=\"slc5_ia32_gcc434\";"
#cmd += "source $VO_CMS_SW_DIR/cmsset_default.sh;"
cmd += "export X509_USER_PROXY=\"" + TopSkimDir + ProxyFile + "\";"
cmd += "cd " + TopSkimDir + ";"
cmd += "python SkimTopTree.py --srmcp --mtop-runmode --mtop-setuser="******" --toptree_location=" + self.SamplePath + " -t " + self.WorkingDir + " --skim_xml=" + SkimFile + " -j " + str(
self.nThreads) + " -n " + str(self.nGroupFiles) + " -w " + str(
self.walltime) + " --email " + self.Email
#cmd += "python SkimTopTree.py --mtop-runmode --mtop-setuser="******" --toptree_location="+self.SamplePath+" --skim_xml="+SkimFile+" -n "+str(self.nThreads)+" -j "+str(self.nGroupFiles)+" --email "+self.Email
if self.host == "PBS":
cmd += " --use-pbs"
if not self.publishName == "":
cmd += " -p " + self.publishName
if self.announce:
cmd += " -a"
#cmd += "voms-proxy-info"
log.output(cmd)
#sys.exit(1)
#cmd = ""
pExe = Popen(cmd,
shell=True,
stdin=PIPE,
stdout=PIPE,
stderr=STDOUT,
close_fds=True)
while pExe.poll() == None:
pExe.poll()
time.sleep(10)
if options.stdout:
log.output(pExe.stdout.read())
exitCode = int(pExe.poll())
# clean up
try:
#os.remove(TopSkimDir+ProxyFile)
#os.remove(TopSkimDir+SkimFile)
os.remove(JSONDest)
os.remove(JSONFile)
except:
log.output("Error cleaning up")
return exitCode
def resetQueue(self):
self.sql.createQuery(
"UPDATE", "skimrequests", "",
"SET `Status` = 'Pending' WHERE `Status` = 'Queued'")
self.sql.execQuery()
class Request:
ID=int(0)
DataSet = ""
DataTier = ""
useLocalDBS = ""
RunSelection = ""
skipPAT = ""
DontStorePat = ""
CMSSW_VER = ""
CMSSW_VER_SAMPLE = ""
GlobalTag = ""
Priority = int(0)
FLFilterPath = int(-1)
wantGenEvent = bool(False)
def __init__(self):
# get the sensitive information from config file
login=""
password=""
dbaseName=""
dbaseHost=""
for line in open(".config","r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
self.sql = SQLHandler(dbaseName,login,password,dbaseHost)
def setQueued(self):
if not self.DataSet == "":
self.sql.createQuery("UPDATE","requests","","SET `Status` = 'Queued' WHERE `ID` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def setRunning(self):
if not self.DataSet == "":
self.sql.createQuery("UPDATE","requests","","SET `Status` = 'Running' WHERE `ID` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def setDone(self):
if not self.DataSet == "":
self.sql.createQuery("UPDATE","requests","","SET `Status` = 'Done', `Priority` = '0' WHERE `ID` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def invalidate(self):
if not self.DataSet == "":
self.sql.createQuery("UPDATE","requests","","SET `Status` = 'Pending', `Priority` = '0' WHERE `ID` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def process(self):
#print self.RunSelection
cmd ='python AutoMaticTopTreeProducer.py -c '+self.CMSSW_VER+' --cmssw_sample '+self.CMSSW_VER_SAMPLE+' -g '+self.GlobalTag+' -d '+self.DataSet
if self.skipPAT == "1":
cmd += " --skip-pat"
if self.DontStorePat == "1":
cmd += " --dont-store-pat"
if self.DataTier == "PAT":
cmd += " --start-from-pat"
if self.DataTier == "PAT-MC":
cmd += " --start-from-pat-mc"
if options.doPBS:
cmd += " --pbs-submit"
if options.dryRun:
cmd += " --dry-run"
if self.wantGenEvent:
cmd += " --addGenEvent"
if int(self.FLFilterPath) > 0:
cmd += " --flavourHistoryFilterPath="+str(self.FLFilterPath)
if not self.useLocalDBS == "":
cmd += " --dbsInst "+self.useLocalDBS
if not self.DataSet.rfind("USER") == -1 and self.DataTier.rfind("PAT") == -1: # then we need a datatier
cmd += " -t "+self.DataTier
cmd += " --setLogFile logs/log-TopDB-Request-"+str(self.ID)+".txt"
if not self.RunSelection == "NULL" and not self.RunSelection == "":
cmd += " -r "+str(self.RunSelection)
#if options.stdout:
# cmd += " --log-stdout"
#else:
cmd += " >& logs/stdout"
log.output(cmd)
#return 0
#cmd = "sleep 10"
pExe = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
# wait until the process is terminated
while pExe.poll() == None:
pExe.poll()
time.sleep(10)
if options.stdout:
log.output(pExe.stdout.read())
exitCode = int(pExe.poll())
# possible exit codes: 0 (everything went fine) 1 (python error, need to send email) 2 (script error, mail sent by AutoMaticTopTreeProducer.py)
return exitCode
#return 0
def resetQueue(self):
self.sql.createQuery("UPDATE","requests","","SET `Status` = 'Pending' WHERE `Status` = 'Queued'")
self.sql.execQuery()
from aiogram.dispatcher import FSMContext
from aiogram import Bot, Dispatcher, executor, types
from aiogram.contrib.fsm_storage.memory import MemoryStorage
from stateHandler import STATES
from sqlHandler import SQLHandler
from configHandler import ConfigHandler
from sqlHandler.consts import TYPES, CONSTS, STRINGS
from errorHandling import errorHandler
logging.basicConfig(level=logging.INFO)
bot = Bot(token=ConfigHandler.token)
dispatcher = Dispatcher(bot, storage=MemoryStorage())
database = SQLHandler()
def api_count_check() -> list:
country_codes = TYPES.params_list
result = []
for param in country_codes:
response = requests.get(
ConfigHandler.count_link, params=param
).json()
result.append(response.get('count'))
return result
@dispatcher.message_handler(commands=['start'])
async def start(message: types.Message):
class RequestHandler:
sleepTime = int(300)
requests = []
def __init__(self):
# get the sensitive information from config file
login=""
password=""
dbaseName=""
dbaseHost=""
for line in open(".config","r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
self.sql = SQLHandler(dbaseName,login,password,dbaseHost)
self.getDataSetProperties()
def getDataSetProperties(self):
self.sql.createQuery("SELECT","simrequests","*","Status = 'Pending' OR Status = 'Queued' ORDER BY `Priority` DESC")
sqlFile="sql_"+str(strftime("%d%m%Y_%H%M%S"))+".out"
f = open(sqlFile,"w")
f.write(self.sql.execQuery())
f.close()
for res in open(sqlFile,"r"):
#print res
line = res.split("\n")[0]
sqlRes = line.split(" ")
if len(sqlRes) < 14:
continue
for a in xrange(len(sqlRes)):
sqlRes[a]=sqlRes[a].split("\n")[0]
sqlRes[a]=sqlRes[a].split("\r")[0]
if sqlRes[0].rfind("id") == -1:
request = Request()
request.ID = sqlRes[0]
request.Campaign=sqlRes[3]
request.CMSSW_sim=sqlRes[4]
request.GT_sim=sqlRes[5]
request.nEvents=sqlRes[11]
request.LHEDir=sqlRes[10]
request.Template=sqlRes[6]
request.PublishName_sim=sqlRes[9]
request.CMSSW_top=""
request.GT_top=""
request.Priority = int(sqlRes[12])
if not request.Priority == 0:
self.requests.append(request)
request.Email = sqlRes[13]
os.remove(sqlFile)
login=""
password=""
dbaseName=""
dbaseHost=""
for line in open("../.config","r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
sql = SQLHandler(dbaseName,login,password,dbaseHost)
#################
### MAIN LOOP ###
#################
if not len(sys.argv) >= 3:
print "Usage: python updateJSONFile.py <search_string> <new_json_url>"
## get the datasets from TopDB containing <search_string>
print "\n-> Searching datasets in TopDB containing: "+sys.argv[1]
sql.createQuery("SELECT","datasets","id,name","name REGEXP '"+sys.argv[1]+"'")
class Request:
ID=int(0)
removeId=int(0)
removeType=""
user=""
comment=""
log=logHandler("")
def __init__(self):
# get the sensitive information from config file
login=""
password=""
dbaseName=""
dbaseHost=""
for line in open(".config","r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
self.sql = SQLHandler(dbaseName,login,password,dbaseHost)
def setQueued(self):
self.sql.createQuery("UPDATE","removals","","SET `Status` = 'Queued' WHERE `id` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def setRunning(self):
self.comment = self.comment+" <br>"+strftime("%Y-%m-%d %H:%M:%S", localtime())+": Starting removal"
self.sql.createQuery("UPDATE","removals","","SET `Status` = 'Running' , `AdminAction` = '"+self.comment+"' WHERE `id` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def setDone(self):
self.comment = self.comment+" <br>"+strftime("%Y-%m-%d %H:%M:%S", localtime())+": Finished removal"
self.sql.createQuery("UPDATE","removals","","SET `Status` = 'Done' , `AdminAction` = '"+self.comment+"' WHERE `id` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def invalidate(self):
self.comment = self.comment+" <br>"+strftime("%Y-%m-%d %H:%M:%S", localtime())+": <b>Error encountered during removal</b>"
self.sql.createQuery("UPDATE","removals","","SET `Status` = 'Frozen', `AdminAction` = '"+self.comment+"' WHERE `id` = "+str(self.ID)+" LIMIT 1")
self.sql.execQuery()
def process(self):
# setup proper log location
self.log = logHandler("logs/log-TopDB-CleaningAgent-"+str(self.ID)+".txt")
self.log.output("****** Removing "+self.removeType+" with ID "+str(self.removeId)+" as requested by "+self.user+" ******")
# pat to remove
id = []
storagePath = []
dbsPublish = []
CffFilePath = []
# toptree to remove
idTop = []
storagePathTop = []
mergedTopLocation = []
storagePathTopMail=[]
## REMOVE ONLY TOPTREE
if self.removeType == "toptree":
self.sql.createQuery("SELECT","toptrees","id,StoragePath,TopTreeLocation","id = '"+str(self.removeId)+"'")
result = self.sql.execQuery().split('\n')
if len(result) == 1:
self.log.output(" ---> ERROR: TopTree was not found in TopDB")
return 1
else:
idTop.append(result[1].split("\t")[0])
storagePathTop.append(result[1].split("\t")[1])
mergedTopLocation.append(result[1].split("\t")[2])
## REMOVE PAT + ALL DOWNSTREAM TOPTREES
elif self.removeType == "patuple":
self.sql.createQuery("SELECT","patuples","id,StoragePath,name,CffFilePath","id = '"+str(self.removeId)+"'")
result = self.sql.execQuery().split('\n')
if len(result) == 1:
self.log.output(" ---> RRROR: PatTuple was not found in TopDB")
return 1
else:
id.append(result[1].split("\t")[0])
storagePath.append(result[1].split("\t")[1])
dbsPublish.append(result[1].split("\t")[2])
CffFilePath.append(result[1].split("\t")[3])
self.sql.createQuery("SELECT","toptrees","id,StoragePath,TopTreeLocation","patuple_id = '"+id[len(id)-1].split("\\n")[0]+"'")
result2 = self.sql.execQuery().split('\n')
if len(result2) > 1:
for j in range(1,len(result2)-1):
idTop.append(result2[j].split("\t")[0])
storagePathTop.append(result2[j].split("\t")[1])
mergedTopLocation.append(result2[j].split("\t")[2])
## REMOVE DATASET + ALL DOWNSTREAM PAT + ALL DOWNSTREAM TOPTREES
elif self.removeType == "dataset":
self.sql.createQuery("SELECT","patuples","id,StoragePath,name,CffFilePath","dataset_id = '"+str(self.removeId)+"'")
result = self.sql.execQuery().split('\n')
if len(result) > 1:
for i in range(1,len(result)-1):
id.append(result[i].split("\t")[0])
storagePath.append(result[i].split("\t")[1])
dbsPublish.append(result[i].split("\t")[2])
CffFilePath.append(result[i].split("\t")[3])
self.sql.createQuery("SELECT","toptrees","id,StoragePath,TopTreeLocation","patuple_id = '"+id[len(id)-1].split("\\n")[0]+"'")
result2 = self.sql.execQuery().split('\n')
if len(result2) > 1:
for j in range(1,len(result2)-1):
idTop.append(result2[j].split("\t")[0])
storagePathTop.append(result2[j].split("\t")[1])
mergedTopLocation.append(result2[j].split("\t")[2])
## CLEAN LEFTOVER FILES FROM FAILED PRODUCTION, CLEAN PATUPLES WITHOUT ANY TOPTREES
elif self.removeType == "cleanpnfs":
self.log.output("--> Cleaning up PNFS area for dhondt")
self.log.output(" ---> Searching for PNFS directories from broken production")
dirs = []
for dir in os.listdir("/pnfs/iihe/cms/store/user/dhondt/"):
if not dir.rfind("Skimmed-TopTrees") == -1:
continue;
#if dir.rfind("7TeV_T2") == -1: continue # this is just to make testing go fast
pExe = Popen("find /pnfs/iihe/cms/store/user/dhondt/"+dir+" -name TOPTREE", shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
out = pExe.stdout.read()
for file in out.split("\n"):
split = file.split("/")
dirName = ""
for i in xrange(0,len(split)-1):
dirName += split[i]+"/"
dirName = dirName.rstrip("/")
if dirs.count(dirName) == 0 and len(dirName) > 0:
dirs.append(dirName+"/TOPTREE")
self.log.output(" ----> "+str(len(dirs))+" directory(s) found in total, cross-referencing TopDB...")
for i in xrange(0,len(dirs)):
self.sql.createQuery("SELECT","toptrees","id","StoragePath REGEXP '"+dirs[i]+"'")
result = self.sql.execQuery().split('\n')
self.sql.createQuery("SELECT","patuples","id","StoragePath REGEXP '"+dirs[i]+"'")
result2 = self.sql.execQuery().split('\n')
self.sql.createQuery("SELECT","gensims","id","PNFSPath REGEXP '"+dirs[i]+"'")
result3 = self.sql.execQuery().split('\n')
self.sql.createQuery("SELECT","recos","id","PNFSPath REGEXP '"+dirs[i]+"'")
result4 = self.sql.execQuery().split('\n')
if len(result) < 2 and len(result2) < 2 and len(result3) < 2 and len(result4) < 2 and storagePathTopMail.count(dirs[i]) == 0:
filestat = os.stat(dirs[i])
filedate = filestat.st_mtime
now = int(time.time())
last_mod=int(filedate)
time_diff=now-last_mod
if time_diff/(60*60) > 720: # just want the dir to be old enough to not remove ongoing prod
self.log.output(" ----> Directory "+dirs[i]+" is not in TopDB, it should be removed! (Age: "+str(time_diff/(60*60*24))+" days)")
#idTop.append(-9999)
storagePathTopMail.append(dirs[i])
self.log.output(" ----> "+str(len(storagePathTopMail))+" directory(s) need removal!")
self.log.output(" ---> Searching for PATuples that don't have a TopTree assigned")
self.sql.createQuery("SELECT","patuples","id,StoragePath,name,CffFilePath","")
result2 = self.sql.execQuery().split('\n')
self.sql.createQuery("SELECT","toptrees","patuple_id","")
result3 = self.sql.execQuery().split('\n')
for i in result2:
if i == "" or not i.rfind("id") == -1: continue
tmpid = i.split("\t")[0]
found=bool(False)
for j in result3:
if j == "": continue
if tmpid == j:
found=bool(True)
#if not found:
#id.append(i.split("\t")[0])
#storagePath.append(i.split("\t")[1])
#dbsPublish.append(i.split("\t")[2])
#CffFilePath.append(i.split("\t")[3])
msg = "Dear admins,"
if len(storagePathTopMail) > 0:
msg += "\n\n The automatic TopDB PNFS cleaning tool has found "+str(len(storagePathTopMail))+" directories on PNFS not corresponding to any entry in the TopDB database."
msg += "\n\n Please have a look at the following list:"
for s in storagePathTopMail:
msg += "\n\n \t rm -rfv "+s
else:
msg += "\n\n The automatic TopDB PNFS cleaning tool has found NO directories on PNFS not corresponding to any entry in the TopDB database."
msg += "\n\nCheers,\nHector the cleaning agent"
mail = MailHandler()
mail.sendMail("error","Report from TopDB PNFS cleaning",msg)
## CLEAN LEFTOVER FILES FROM FAILED PRODUCTION, CLEAN PATUPLES WITHOUT ANY TOPTREES
elif self.removeType == "cleancrablogs":
days=50
self.log.output(" ---> Listing Configuration directories")
self.log.output(" ---> Checking every Configuration directory (older than "+str(days)+" days) for large amounts of *.stdout from CRAB")
ldirs = []
cleanup_ldirsToRemove = []
basedir=(Popen("echo $HOME", shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True).stdout.read()).strip()+"/AutoMaticTopTreeProducer/"
for dir in os.listdir(basedir):
if dir.rfind("CMSSW_") == -1:
continue;
pExe = Popen("find "+basedir+dir.strip()+"/ -name crab_*.cfg", shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
out = pExe.stdout.read()
for file in out.split("\n"):
split = file.split("/")
dirName = ""
for i in xrange(0,len(split)-1):
dirName += split[i]+"/"
dirName = dirName.rstrip("/")
if ldirs.count(dirName) == 0 and len(dirName) > 0:
ldirs.append(dirName.split("/AutoMaticTopTreeProducer/")[1]) # becase we don't want it to crash on changes /home /user
# time to clean out some big chunks of stdout files
if not dirName == "":
if not dirName.find("find: ") == -1:
dirName = dirName.split("find: ")[1]
#print dirName
filestat = os.stat(dirName)
filedate = filestat.st_mtime
now = int(time.time())
last_mod=int(filedate)
time_diff=now-last_mod
if time_diff/(60*60*24) > days:
#self.log.output(" ---> Cleaning CRAB stdout files in "+dirName+" (Age: "+str(time_diff/(3600*24))+" days)")
crabdir=""
for dir in os.listdir(dirName):
if not dir.rfind("TOPTREE_") == -1 and dir.rfind(".py") == -1 and os.path.isdir(dirName+"/"+dir):
crabdir=dirName+"/"+dir
if not crabdir == "":
numfiles=int(0)
keepstdout=""
keepstderr=""
keepxml=""
if os.path.exists(crabdir+"/log/crab.log"):
self.log.output(" ---> Cleaning crab.log in "+crabdir+"/log/ (Age: "+str(time_diff/(3600*24))+" days)")
os.unlink(crabdir+"/log/crab.log")
#sys.exit(1)
for file in os.listdir(crabdir+"/res"):
if not file.rfind(".stdout") == -1:
if os.path.getsize(crabdir+"/res/"+file) > 0 and keepstdout == "":
keepstdout=file
numfiles=numfiles+1
#print keepstdout
if not os.path.isdir(crabdir+"/res"):
numfiles=0
#print numfiles
#print str(numfiles)+" "+dirName
if numfiles > 2 and dirName.rfind("Run201") == -1:
print numfiles
self.log.output(" ---> Cleaning CRAB stdout files in "+crabdir+" (Age: "+str(time_diff/(3600*24))+" days)")
keepstderr=keepstdout.split(".stdout")[0]+".stderr"
keepxml="crab_fjr_"+(keepstdout.split(".stdout")[0]).split("CMSSW_")[1]+".xml"
for file in os.listdir(crabdir+"/res"):
if not os.path.isdir(crabdir+"/res/"+file) and file.rfind("Submission") == -1 and file.rfind(".json") == -1 and not file == keepxml and not file == keepstdout and not file == keepstderr:
self.log.output(" ---> Removing crab output "+file)
os.unlink(crabdir+"/res/"+file)
elif not file.rfind("Submission") == -1:
self.log.output(" ---> Removing old Submission_X dir: "+file)
shutil.rmtree(crabdir+"/res/"+file)
elif not dirName.rfind("Run201") == -1:
if os.path.exists(crabdir+"/res/.shrunk"):
continue
self.log.output(" ---> (DATA PRODUCTION) Removing unuseful lines from stdout files in "+crabdir+" (Age: "+str(time_diff/(3600*24))+" days)")
for file in os.listdir(crabdir+"/res"):
if not file.rfind("Submission") == -1:
self.log.output(" ---> Removing old Submission_X dir: "+file)
shutil.rmtree(crabdir+"/res/"+file)
elif not os.path.isdir(crabdir+"/res/"+file) and file.rfind("Submission") == -1 and file.rfind(".json") == -1 and not file.rfind(".stdout") == -1:
self.log.output(" ---> Shrinking crab output "+file)
tmpfile = open(crabdir+"/res/"+file+"_tmp","w")
for line in open(crabdir+"/res/"+file):
if line.rfind("Begin processing") == -1 and line.rfind("Vertex") == -1 and line.rfind("%MSG") == -1:
tmpfile.write(line)
os.unlink(crabdir+"/res/"+file)
os.rename(crabdir+"/res/"+file+"_tmp",crabdir+"/res/"+file)
f = open(crabdir+"/res/.shrunk","w") # leave a stamp that this dir is fixed
f.close()
self.log.output(" ----> "+str(len(ldirs))+" Configuration directory(s) found in total, cross-referencing TopDB...")
self.log.output("")
ldirs = [] # disable this for now
for i in xrange(0,len(ldirs)):
self.sql.createQuery("SELECT","toptrees","id","CffFilePath REGEXP '"+ldirs[i]+"'")
result = self.sql.execQuery().split('\n')
self.sql.createQuery("SELECT","patuples","id","CffFilePath REGEXP '"+ldirs[i]+"'")
result2 = self.sql.execQuery().split('\n')
self.sql.createQuery("SELECT","gensims","id","CffPath REGEXP '"+ldirs[i]+"'")
result3 = self.sql.execQuery().split('\n')
self.sql.createQuery("SELECT","recos","id","CffPath REGEXP '"+ldirs[i]+"'")
result4 = self.sql.execQuery().split('\n')
if len(result) < 2 and len(result2) < 2 and len(result3) < 2 and len(result4) < 2 and cleanup_ldirsToRemove.count(ldirs[i]) == 0:
filestat = os.stat(basedir+"/"+ldirs[i])
filedate = filestat.st_mtime
now = int(time.time())
last_mod=int(filedate)
time_diff=now-last_mod
if time_diff/(60*60*24) > days: # just want the dir to be old enough to not remove ongoing prod
self.log.output(" ----> Directory "+ldirs[i]+" is not in TopDB, it should be removed! (Age: "+str(time_diff/(60*60*24))+" days)")
cleanup_ldirsToRemove.append(ldirs[i])
self.log.output(" ----> "+str(len(cleanup_ldirsToRemove))+" directory(s) need removal!")
## SUMMARY OF THE REMOVAL
self.log.output(" --> Summary of the removal")
for i in range(0,len(id)):
self.log.output(" * Removing PATtuple with ID "+str(id[i])+" at "+storagePath[i])
for i in range(0,len(idTop)):
self.log.output(" * Removing TopTree with ID "+str(idTop[i])+" at "+storagePathTop[i])
#if self.removeType == "cleanpnfs":
# return 0;
# START REMOVAL
time.sleep(20)
log.output(" --> Starting the removal procedure")
rm = RemoveHelper(self.sql,self.log)
for i in range(0,len(id)):
rm.rmSRMdir(storagePath[i])
rm.rmFromTopDB("patuples",id[i])
rm.invalDBS(dbsPublish[i],CffFilePath[i])
for i in range(0,len(idTop)):
rm.rmSRMdir(storagePathTop[i])
if idTop[i] > 0:
rm.rmFromTopDB("toptrees",idTop[i])
if self.removeType == "dataset":
rm.rmFromTopDB("datasets",self.removeId)
self.log.output(" --> Ended removal procedure")
return 0
def resetQueue(self):
self.sql.createQuery("UPDATE","removals","","SET `Status` = 'Pending' WHERE `Status` = 'Queued'")
self.sql.execQuery()
class RequestHandler:
sleepTime = int(30)
requests = []
def __init__(self):
# get the sensitive information from config file
login=""
password=""
dbaseName=""
dbaseHost=""
for line in open(".config","r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
self.sql = SQLHandler(dbaseName,login,password,dbaseHost)
self.getProperties()
def getProperties(self):
self.sql.createQuery("SELECT","removals","*","Status = 'Pending' OR Status = 'Queued' ORDER BY `ID` DESC")
timestamp = strftime("%d%m%Y_%H%M%S")
f = open("sql"+str(timestamp)+".out","w")
f.write(self.sql.execQuery())
f.close()
for res in open("sql"+str(timestamp)+".out","r"):
#print res
line = res.split("\n")[0]
sqlRes = line.split(" ")
for a in xrange(len(sqlRes)):
sqlRes[a]=sqlRes[a].split("\n")[0]
sqlRes[a]=sqlRes[a].split("\r")[0]
if sqlRes[0].rfind("ID") == -1 and len(sqlRes) > 6:
request = Request()
request.ID = sqlRes[0]
request.removeId = sqlRes[4]
request.removeType = sqlRes[3]
request.user = sqlRes[1]
request.comment = sqlRes[6]
self.requests.append(request)
os.remove("sql"+str(timestamp)+".out")
login=""
password=""
dbaseName=""
dbaseHost=""
for line in open("../.config","r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
sql = SQLHandler(dbaseName,login,password,dbaseHost)
###############
### METHODS ###
###############
def rmSRMdir (dir):
global srmHost
if os.path.exists(dir):
log.output(" ---> Going into directory "+dir)
tmp = os.listdir(dir)
baseDir = (
Popen("echo $HOME", shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True).stdout.read().strip()
+ "/AutoMaticTopTreeProducer"
)
for line in open("../.config", "r"):
if not line.rfind("DBUser") == -1:
login = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBPass") == -1:
password = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBHost") == -1:
dbaseHost = line.split(" ")[1].split("\n")[0]
elif not line.rfind("DBName") == -1:
dbaseName = line.split(" ")[1].split("\n")[0]
sql = SQLHandler(dbaseName, login, password, dbaseHost)
# checking the CMSSW versions installed -> insert into topDB where needed
for entry in os.listdir(baseDir):
if os.path.isdir(baseDir + "/" + entry) == True and not entry.rfind("CMSSW") == -1:
print "Local version found: " + entry
isInDB = bool(False)
# check if this version exists in topDB otherwise add it
options = "version REGEXP '" + entry + "'"
