def requestDeletions(site,dsetNames):
phedex = phedexApi.phedexApi(logPath='./')
# compose data for deletion request
check,data = phedex.xmlData(datasets=dsetNames,instance='prod',level='block')
if check:
print " ERROR - phedexApi.xmlData failed"
sys.exit(1)
# here the request is really sent
message = 'IntelROCCS -- Cache Release Request -- Deprecated Datasets'
check,response = phedex.delete(node=site,data=data,comments=message,instance='prod')
if check:
print " ERROR - phedexApi.delete failed"
print site
print response
sys.exit(1)
respo = response.read()
matchObj = re.search(r'"id":"(\d+)"',respo)
reqid = int(matchObj.group(1))
del phedex
# here we brute force deletion to be approved
phedex = phedexApi.phedexApi(logPath='./')
check,response = phedex.updateRequest(decision='approve',request=reqid,node=site,instance='prod')
if check:
print " ERROR - phedexApi.updateRequest failed - reqid="+ str(reqid)
print response
del phedex
def submitTransferRequest(self,site,datasets2trans):
if len(datasets2trans) < 1:
return
phedex = phedexApi.phedexApi(logPath='./')
# compose data for deletion request
check,data = phedex.xmlData(datasets=datasets2trans,instance='prod')
if check:
print " ERROR - phedexApi.xmlData failed"
print data
sys.exit(1)
# here the request is really sent
message = 'IntelROCCS -- Automatic Transfer Request'
check,response = phedex.subscribe(node=site,data=data,comments=message)
if check:
print " ERROR - phedexApi.subscribe failed"
print response
sys.exit(1)
respo = response.read()
matchObj = re.search(r'"id":"(\d+)"',respo)
reqid = int(matchObj.group(1))
rdate = (re.search(r'"request_date":"(.*?)"',respo)).group(1)
rdate = rdate[:-3]
self.dbInfoHandler.logRequest(site,datasets2trans,reqid,rdate,0)
def submitDeletionRequest(self,site,datasets2del):
if len(datasets2del) < 1:
return
phedex = phedexApi.phedexApi(logPath='./')
# compose data for deletion request
check,data = phedex.xmlData(datasets=datasets2del,instance='prod')
if check:
print " ERROR - phedexApi.xmlData failed"
sys.exit(1)
# here the request is really sent
message = 'IntelROCCS -- Automatic Cache Release Request (next check ' + \
'in about %s hours).'%(os.environ['DETOX_CYCLE_HOURS']) + \
' Summary at: http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/result/'
check,response = phedex.delete(node=site,data=data,comments=message,instance='prod')
if check:
print " ERROR - phedexApi.delete failed"
print response
sys.exit(1)
respo = response.read()
matchObj = re.search(r'"id":"(\d+)"',respo)
reqid = int(matchObj.group(1))
rdate = (re.search(r'"request_date":"(.*?)"',respo)).group(1)
rdate = rdate[:-3]
self.dbInforhandler.logRequest(site,datasets2del,reqid,rdate,1)
def submitTransferRequest(self, site, datasets2trans):
if len(datasets2trans) < 1:
return
phedex = phedexApi.phedexApi(logPath='./')
# compose data for deletion request
check, data = phedex.xmlData(datasets=datasets2trans, instance='prod')
if check:
print " ERROR - phedexApi.xmlData failed"
print data
sys.exit(1)
# here the request is really sent
message = 'IntelROCCS -- Automatic Transfer Request'
check, response = phedex.subscribe(node=site,
data=data,
comments=message)
if check:
print " ERROR - phedexApi.subscribe failed"
print response
sys.exit(1)
respo = response.read()
matchObj = re.search(r'"id":"(\d+)"', respo)
reqid = int(matchObj.group(1))
rdate = (re.search(r'"request_date":"(.*?)"', respo)).group(1)
rdate = rdate[:-3]
self.dbInfoHandler.logRequest(site, datasets2trans, reqid, rdate, 0)
def submitDeletionRequest(self, site, datasets2del):
if len(datasets2del) < 1:
return
phedex = phedexApi.phedexApi(logPath='./')
# compose data for deletion request
check, data = phedex.xmlData(datasets=datasets2del, instance='prod')
if check:
print " ERROR - phedexApi.xmlData failed"
sys.exit(1)
# here the request is really sent
message = 'IntelROCCS -- Automatic Cache Release Request (next check ' + \
'in about %s hours).'%(os.environ['DETOX_CYCLE_HOURS']) + \
' Summary at: http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/result/'
check, response = phedex.delete(node=site,
data=data,
comments=message,
instance='prod')
if check:
print " ERROR - phedexApi.delete failed"
print response
sys.exit(1)
respo = response.read()
matchObj = re.search(r'"id":"(\d+)"', respo)
reqid = int(matchObj.group(1))
rdate = (re.search(r'"request_date":"(.*?)"', respo)).group(1)
rdate = rdate[:-3]
self.dbInforhandler.logRequest(site, datasets2del, reqid, rdate, 1)
def submitUpdateRequest(self,site,reqid):
# here we brute force deletion to be approved
phedex = phedexApi.phedexApi(logPath='./')
check,response = phedex.updateRequest(decision='approve',request=reqid,node=site,instance='prod')
if check:
print " ERROR - phedexApi.updateRequest failed - reqid="+ str(reqid)
print response
del phedex
def submitUpdateRequest(self,site,reqid):
# here we brute force deletion to be approved
phedex = phedexApi.phedexApi(logPath='./')
check,response = phedex.updateRequest(decision='approve',request=reqid,node=site,instance='prod')
if check:
print " ERROR - phedexApi.updateRequest failed - reqid="+ str(reqid)
print response
del phedex
def submitRequest(site, datasets=[]):
if len(datasets) < 1:
print " ERROR - Trying to submit empty request for " + site
return
phedex = phedexApi.phedexApi(logPath='./')
# compose data for deletion request
check,data = phedex.xmlData(datasets=datasets,instance='prod')
if check:
print " ERROR - phedexApi.xmlData failed"
sys.exit(1)
# here the request is really sent
message = 'IntelROCCS -- Automatic Cache Release Request (if not acted upon will repeat ' + \
'in about %s hours).'%(os.environ['DETOX_CYCLE_HOURS']) + \
' Summary at: http://t3serv001.mit.edu/~cmsprod/IntelROCCS/Detox/result/'
check,response = phedex.delete(node=site,data=data,comments=message,instance='prod')
if check:
print " ERROR - phedexApi.delete failed"
print response
sys.exit(1)
respo = response.read()
matchObj = re.search(r'"id":"(\d+)"',respo)
id = int(matchObj.group(1))
matchObj = re.search(r'"request_date":"(.*?)"',respo)
date = matchObj.group(1)
date = date[:-3]
myCnf = os.environ['DETOX_MYSQL_CONFIG']
for dataset in datasets:
rank = float(datasetInfo[dataset][0])
size = float(datasetInfo[dataset][1])
group = datasetGroup[dataset]
db = MySQLdb.connect(read_default_file=myCnf,read_default_group="mysql")
cursor = db.cursor()
sql = "insert into Requests(RequestId,RequestType,SiteName,Dataset,Size,Rank,GroupName," + \
"TimeStamp) values ('%d', '%d', '%s', '%s', '%d', '%d', '%s', '%s' )" % \
(id, 1, site, dataset,size,rank,group,date)
# ! this could be done in one line but it is just nice to see what is deleted !
try:
cursor.execute(sql)
db.commit()
except:
print "caught an exception"
db.rollback()
db.close()
def requestDeletions(site, dsetNames):
phedex = phedexApi.phedexApi(logPath='./')
# compose data for deletion request
check, data = phedex.xmlData(datasets=dsetNames,
instance='prod',
level='block')
if check:
print " ERROR - phedexApi.xmlData failed"
sys.exit(1)
# here the request is really sent
message = 'IntelROCCS -- Cache Release Request -- Deprecated Datasets'
check, response = phedex.delete(node=site,
data=data,
comments=message,
instance='prod')
if check:
print " ERROR - phedexApi.delete failed"
print site
print response
sys.exit(1)
respo = response.read()
matchObj = re.search(r'"id":"(\d+)"', respo)
reqid = int(matchObj.group(1))
del phedex
# here we brute force deletion to be approved
phedex = phedexApi.phedexApi(logPath='./')
check, response = phedex.updateRequest(decision='approve',
request=reqid,
node=site,
instance='prod')
if check:
print " ERROR - phedexApi.updateRequest failed - reqid=" + str(reqid)
print response
del phedex
def printRunawaySets(self):
siteSizes = {}
siteSets = {}
for dset in self.phedexDatasets:
if dset not in self.otherDatasets:
continue
dataset = self.otherDatasets[dset]
for site in dataset.siteNames:
group = dataset.group(site)
size = dataset.size(site)
if site not in siteSizes:
siteSizes[site] = 0
siteSets[site] = 0
siteSizes[site] = siteSizes[site] + size / 1000
siteSets[site] = siteSets[site] + 1
if site not in self.runAwayGroups:
dsetName = dataset.dataset
if 'AOD' in dsetName:
# if group != 'RelVal' and group != 'DataOps':
if not site.startswith('T1_'):
if group == 'local':
continue
print site
print dataset.dataset
print group + '--> AnalysisOps'
phedex = phedexApi.phedexApi(logPath='./')
check, response = phedex.changeGroup(
site, dsetName, 'AnalysisOps')
if check:
print " ERROR - phedexApi.updateRequest failed"
print response
del phedex
else:
del phedex
continue
self.runAwayGroups[site] = [group]
else:
if group not in self.runAwayGroups[site]:
self.runAwayGroups[site].append(group)
if (len(siteSizes) < 1):
return
print " !! WARNING !! - those sites have datasets in wrong groups"
for site in sorted(siteSizes):
print ' %3d %6.2f TB' % (siteSets[site],
siteSizes[site]) + ": " + site
def printRunawaySets(self):
siteSizes = {}
siteSets = {}
for dset in self.phedexDatasets:
if dset not in self.otherDatasets:
continue
dataset = self.otherDatasets[dset]
for site in dataset.siteNames:
group = dataset.group(site)
size = dataset.size(site)
if site not in siteSizes:
siteSizes[site] = 0
siteSets[site] = 0
siteSizes[site] = siteSizes[site] + size/1000
siteSets[site] = siteSets[site] + 1
if site not in self.runAwayGroups:
dsetName = dataset.dataset
if 'AOD' in dsetName:
# if group != 'RelVal' and group != 'DataOps':
if not site.startswith('T1_'):
if group == 'local':
continue
print site
print dataset.dataset
print group + '--> AnalysisOps'
phedex = phedexApi.phedexApi(logPath='./')
check,response = phedex.changeGroup(site,dsetName,'AnalysisOps')
if check:
print " ERROR - phedexApi.updateRequest failed"
print response
del phedex
else:
del phedex
continue
self.runAwayGroups[site] = [group]
else:
if group not in self.runAwayGroups[site]:
self.runAwayGroups[site].append(group)
if (len(siteSizes) < 1):
return
print " !! WARNING !! - those sites have datasets in wrong groups"
for site in sorted(siteSizes):
print ' %3d %6.2f TB'%(siteSets[site],siteSizes[site]) + ": " + site
def __init__(self):
config = ConfigParser.RawConfigParser()
config.read(os.path.join(os.path.dirname(__file__), 'api.cfg'))
self.phedexCache = config.get('phedex', 'cache')
self.cacheDeadline = config.getint('phedex', 'expiration_timer')
self.phedexApi = phedexApi.phedexApi()
def __init__(self):
config = ConfigParser.RawConfigParser()
config.read(os.path.join(os.path.dirname(__file__), 'api.cfg'))
self.phedexCache = config.get('phedex', 'cache')
self.cacheDeadline = config.getint('phedex', 'expiration_timer')
self.phedexApi = phedexApi.phedexApi()
