def main():
"""
_main_
"""
# Start services
if 'WMAGENT_CONFIG' not in os.environ:
os.environ[
'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
connectToDB()
myPhEDEx = PhEDEx()
myThread = threading.currentThread()
print "Please remember to shutdown the PhEDExInjector first, you have 10 seconds before the script starts."
time.sleep(10)
# Get the files that the PhEDExInjector would look for
formatter = DBFormatter(logging, myThread.dbi)
formatter.sql = query
results = formatter.execute()
sortedBlocks = defaultdict(set)
for lfn, block in results:
sortedBlocks[block].add(lfn)
# Check with block-level calls
foundFiles = set()
for block in sortedBlocks:
result = myPhEDEx._getResult('data', args={'block': block}, verb='GET')
for dbs in result['phedex']['dbs']:
for dataset in dbs['dataset']:
blockChunk = dataset['block']
for blockInfo in blockChunk:
for fileInfo in blockInfo['file']:
if fileInfo['lfn'] in sortedBlocks[block]:
foundFiles.add(fileInfo['lfn'])
if not foundFiles:
print "I didn't find an abnormal file, feel free to panic!. Please contact a developer."
return 0
print "Found %d files that are already registered in PhEDEx but the buffer doesn't know" % len(
foundFiles)
print "Fixing them now..."
# Fix it!
binds = []
for lfn in foundFiles:
binds.append({'lfn': lfn})
formatter.dbi.processData(modification,
binds,
conn=None,
transaction=False,
returnCursor=False)
print "Fixed them! :)"
print "You can restart the PhEDExInjector now, have a nice day!"
return 0
def main():
"""
_main_
"""
# Start services
if 'WMAGENT_CONFIG' not in os.environ:
os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
connectToDB()
myPhEDEx = PhEDEx()
myThread = threading.currentThread()
print "Please remember to shutdown the PhEDExInjector first, you have 10 seconds before the script starts."
time.sleep(10)
# Get the files that the PhEDExInjector would look for
formatter = DBFormatter(logging, myThread.dbi)
formatter.sql = query
results = formatter.execute()
sortedBlocks = defaultdict(set)
for lfn, block in results:
sortedBlocks[block].add(lfn)
# Check with block-level calls
foundFiles = set()
for block in sortedBlocks:
result = myPhEDEx._getResult('data', args = {'block' : block}, verb = 'GET')
for dbs in result['phedex']['dbs']:
for dataset in dbs['dataset']:
blockChunk = dataset['block']
for blockInfo in blockChunk:
for fileInfo in blockInfo['file']:
if fileInfo['lfn'] in sortedBlocks[block]:
foundFiles.add(fileInfo['lfn'])
if not foundFiles:
print "I didn't find an abnormal file, feel free to panic!. Please contact a developer."
return 0
print "Found %d files that are already registered in PhEDEx but the buffer doesn't know" % len(foundFiles)
print "Fixing them now..."
# Fix it!
binds = []
for lfn in foundFiles:
binds.append({'lfn' :lfn})
formatter.dbi.processData(modification, binds,
conn = None,
transaction = False,
returnCursor = False)
print "Fixed them! :)"
print "You can restart the PhEDExInjector now, have a nice day!"
return 0
def main():
"""
_main_
"""
if 'WMAGENT_CONFIG' not in os.environ:
os.environ[
'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
if 'manage' not in os.environ:
os.environ[
'manage'] = '/data/srv/wmagent/current/config/wmagent/manage'
connectToDB()
myPhEDEx = PhEDEx()
myDBS = DBS3Reader('https://cmsweb.cern.ch/dbs/prod/global/DBSReader/')
myThread = threading.currentThread()
print "Shutting down PhEDExInjector..."
subprocess.call([
os.environ['manage'], "execute-agent", "wmcoreD", "--shutdown",
"--component=PhEDExInjector"
],
stdout=open(os.devnull, 'wb'))
time.sleep(5)
## TASK1: query DB for files not injected in phedex yet
# Get the files that the PhEDExInjector would look for
formatter = DBFormatter(logging, myThread.dbi)
formatter.sql = getQuery
results = formatter.execute()
fileList = []
fileList = [lfn[0] for lfn in results]
## TASK2: makes lfns a bit shorter to sort and uniq them
reducedLfns = [lfn.rsplit('/', 2)[0] for lfn in fileList]
reducedLfns = list(set(reducedLfns))
## TASK3: build uniq dataset names and check whether PhEDEx and DBS contain
## the same number of files. If so, then those lfns are healthy
print "Checking %d dataset in both PhEDEx and DBS ..." % len(reducedLfns)
crippleLfns, healthyLfns = [], []
i = 0
n = len(reducedLfns)
for lfn in reducedLfns:
try:
lfnAux = lfn.split('/')
dset = '/' + lfnAux[4] + '/' + lfnAux[3] + '-' + lfnAux[
6] + '/' + lfnAux[5]
result = myPhEDEx._getResult('blockreplicas',
args={'dataset': dset},
verb='GET')
phedexFiles = 0
for item in result["phedex"]["block"]:
phedexFiles += item['files']
## TODO: ValidFile is only available for > 0.9.95pre5. Once all agents are
## upgraded, then we can start using this new query.
#result = myDBS.listDatasetFileDetails(dset)
#dbsFiles = 0
#for item in result.itervalues():
# dbsFiles += 1 if item['ValidFile'] else 0
# This call returns valid+invalid number of filesfiles
result = myDBS.listDatasetFiles(dset)
dbsFiles = len(result)
if phedexFiles == dbsFiles:
healthyLfns.append(lfn)
else:
crippleLfns.append(lfn)
except:
print "Error with:", lfn
i += 1
if i % 100 == 0:
print '%d/%d files processed' % (i, n)
## TASK4: map the short cripple and healthy lists to the full original lfns
## TODO: this code looks terrible... IMPROVE IT!
if crippleLfns:
filesToCheck = []
for lfn in crippleLfns:
#filesToCheck = [file for file in fileList if lfn in file]
for file in fileList:
if lfn in file:
filesToCheck.append(file)
else:
filesToCheck = []
if healthyLfns:
filesInPhedex = []
for lfn in healthyLfns:
#filesInPhedex = [file for file in fileList if lfn in file]
for file in fileList:
if lfn in file:
filesInPhedex.append(file)
else:
filesInPhedex = []
## TASK5: query PhEDEx for each cripple file (filesToCheck)
## and build the final file lists
missingFiles = []
i = 0
n = len(filesToCheck)
for file in filesToCheck:
try:
result = myPhEDEx._getResult('data',
args={'file': file},
verb='GET')
if len(result['phedex']['dbs']):
filesInPhedex.append(file)
else:
missingFiles.append(file)
except:
print "Error contacting Phedex", file
i += 1
if i % 100 == 0:
print '%d/%d files processed' % (i, n)
if not filesInPhedex:
print "There are no files to be updated in the buffer. Contact a developer."
print "Starting PhEDExInjector now ..."
subprocess.call([
os.environ['manage'], "execute-agent", "wmcoreD", "--start",
"--component=PhEDExInjector"
],
stdout=open(os.devnull, 'wb'))
return 0
print "Found %d out of %d files that are already registered in PhEDEx \
but buffer doesn't know" % (len(filesInPhedex), len(fileList))
print "Fixing them now, it may take several minutes ..."
## TASK6: time to actually fix these files
binds = []
for file in filesInPhedex:
binds.append({'lfn': file})
formatter.dbi.processData(setQuery,
binds,
conn=None,
transaction=False,
returnCursor=False)
print "Rows were successfully updated! Good job!"
print "Starting PhEDExInjector now ..."
subprocess.call([
os.environ['manage'], "execute-agent", "wmcoreD", "--start",
"--component=PhEDExInjector"
],
stdout=open(os.devnull, 'wb'))
print "Done!"
return 0
def main():
"""
_main_
"""
if 'WMAGENT_CONFIG' not in os.environ:
os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
if 'manage' not in os.environ:
os.environ['manage'] = '/data/srv/wmagent/current/config/wmagent/manage'
connectToDB()
myPhEDEx = PhEDEx()
myDBS = DBS3Reader('https://cmsweb.cern.ch/dbs/prod/global/DBSReader/')
myThread = threading.currentThread()
print "Shutting down PhEDExInjector..."
subprocess.call([os.environ['manage'], "execute-agent", "wmcoreD", "--shutdown",
"--component=PhEDExInjector"], stdout=open(os.devnull, 'wb'))
time.sleep(5)
## TASK1: query DB for files not injected in phedex yet
# Get the files that the PhEDExInjector would look for
formatter = DBFormatter(logging, myThread.dbi)
formatter.sql = getQuery
results = formatter.execute()
fileList = []
fileList = [lfn[0] for lfn in results]
## TASK2: makes lfns a bit shorter to sort and uniq them
reducedLfns = [lfn.rsplit('/',2)[0] for lfn in fileList]
reducedLfns = list(set(reducedLfns))
## TASK3: build uniq dataset names and check whether PhEDEx and DBS contain
## the same number of files. If so, then those lfns are healthy
print "Checking %d dataset in both PhEDEx and DBS ..." % len(reducedLfns)
crippleLfns, healthyLfns = [], []
i = 0
n = len(reducedLfns)
for lfn in reducedLfns:
try:
lfnAux = lfn.split ('/')
dset = '/'+lfnAux[4]+'/'+lfnAux[3]+'-'+lfnAux[6]+'/'+lfnAux[5]
result = myPhEDEx._getResult('blockreplicas', args = {'dataset' : dset}, verb = 'GET')
phedexFiles = 0
for item in result["phedex"]["block"]:
phedexFiles += item['files']
## TODO: ValidFile is only available for > 0.9.95pre5. Once all agents are
## upgraded, then we can start using this new query.
#result = myDBS.listDatasetFileDetails(dset)
#dbsFiles = 0
#for item in result.itervalues():
# dbsFiles += 1 if item['ValidFile'] else 0
# This call returns valid+invalid number of filesfiles
result = myDBS.listDatasetFiles(dset)
dbsFiles = len(result)
if phedexFiles == dbsFiles:
healthyLfns.append(lfn)
else:
crippleLfns.append(lfn)
except:
print "Error with:",lfn
i += 1
if i % 100 == 0:
print '%d/%d files processed'%(i,n)
## TASK4: map the short cripple and healthy lists to the full original lfns
## TODO: this code looks terrible... IMPROVE IT!
if crippleLfns:
filesToCheck = []
for lfn in crippleLfns:
#filesToCheck = [file for file in fileList if lfn in file]
for file in fileList:
if lfn in file:
filesToCheck.append(file)
else:
filesToCheck = []
if healthyLfns:
filesInPhedex = []
for lfn in healthyLfns:
#filesInPhedex = [file for file in fileList if lfn in file]
for file in fileList:
if lfn in file:
filesInPhedex.append(file)
else:
filesInPhedex = []
## TASK5: query PhEDEx for each cripple file (filesToCheck)
## and build the final file lists
missingFiles = []
i = 0
n = len(filesToCheck)
for file in filesToCheck:
try:
result = myPhEDEx._getResult('data', args = {'file' : file}, verb = 'GET')
if len(result['phedex']['dbs']):
filesInPhedex.append(file)
else:
missingFiles.append(file)
except:
print "Error contacting Phedex", file
i += 1
if i % 100 == 0:
print '%d/%d files processed'%(i,n)
if not filesInPhedex:
print "There are no files to be updated in the buffer. Contact a developer."
print "Starting PhEDExInjector now ..."
subprocess.call([os.environ['manage'], "execute-agent", "wmcoreD", "--start",
"--component=PhEDExInjector"], stdout=open(os.devnull, 'wb'))
return 0
print "Found %d out of %d files that are already registered in PhEDEx \
but buffer doesn't know" % (len(filesInPhedex), len(fileList))
print "Fixing them now, it may take several minutes ..."
## TASK6: time to actually fix these files
binds = []
for file in filesInPhedex:
binds.append({'lfn': file})
formatter.dbi.processData(setQuery, binds,
conn = None,
transaction = False,
returnCursor = False)
print "Rows were successfully updated! Good job!"
print "Starting PhEDExInjector now ..."
subprocess.call([os.environ['manage'], "execute-agent", "wmcoreD", "--start",
"--component=PhEDExInjector"], stdout=open(os.devnull, 'wb'))
print "Done!"
return 0
