def main(): """ _main_ """ # Start services if 'WMAGENT_CONFIG' not in os.environ: os.environ[ 'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py' connectToDB() myPhEDEx = PhEDEx() myThread = threading.currentThread() print "Please remember to shutdown the PhEDExInjector first, you have 10 seconds before the script starts." time.sleep(10) # Get the files that the PhEDExInjector would look for formatter = DBFormatter(logging, myThread.dbi) formatter.sql = query results = formatter.execute() sortedBlocks = defaultdict(set) for lfn, block in results: sortedBlocks[block].add(lfn) # Check with block-level calls foundFiles = set() for block in sortedBlocks: result = myPhEDEx._getResult('data', args={'block': block}, verb='GET') for dbs in result['phedex']['dbs']: for dataset in dbs['dataset']: blockChunk = dataset['block'] for blockInfo in blockChunk: for fileInfo in blockInfo['file']: if fileInfo['lfn'] in sortedBlocks[block]: foundFiles.add(fileInfo['lfn']) if not foundFiles: print "I didn't find an abnormal file, feel free to panic!. Please contact a developer." return 0 print "Found %d files that are already registered in PhEDEx but the buffer doesn't know" % len( foundFiles) print "Fixing them now..." # Fix it! binds = [] for lfn in foundFiles: binds.append({'lfn': lfn}) formatter.dbi.processData(modification, binds, conn=None, transaction=False, returnCursor=False) print "Fixed them! :)" print "You can restart the PhEDExInjector now, have a nice day!" return 0
def main(): """ _main_ """ # Start services if 'WMAGENT_CONFIG' not in os.environ: os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py' connectToDB() myPhEDEx = PhEDEx() myThread = threading.currentThread() print "Please remember to shutdown the PhEDExInjector first, you have 10 seconds before the script starts." time.sleep(10) # Get the files that the PhEDExInjector would look for formatter = DBFormatter(logging, myThread.dbi) formatter.sql = query results = formatter.execute() sortedBlocks = defaultdict(set) for lfn, block in results: sortedBlocks[block].add(lfn) # Check with block-level calls foundFiles = set() for block in sortedBlocks: result = myPhEDEx._getResult('data', args = {'block' : block}, verb = 'GET') for dbs in result['phedex']['dbs']: for dataset in dbs['dataset']: blockChunk = dataset['block'] for blockInfo in blockChunk: for fileInfo in blockInfo['file']: if fileInfo['lfn'] in sortedBlocks[block]: foundFiles.add(fileInfo['lfn']) if not foundFiles: print "I didn't find an abnormal file, feel free to panic!. Please contact a developer." return 0 print "Found %d files that are already registered in PhEDEx but the buffer doesn't know" % len(foundFiles) print "Fixing them now..." # Fix it! binds = [] for lfn in foundFiles: binds.append({'lfn' :lfn}) formatter.dbi.processData(modification, binds, conn = None, transaction = False, returnCursor = False) print "Fixed them! :)" print "You can restart the PhEDExInjector now, have a nice day!" return 0
def main(): """ _main_ """ if 'WMAGENT_CONFIG' not in os.environ: os.environ[ 'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py' if 'manage' not in os.environ: os.environ[ 'manage'] = '/data/srv/wmagent/current/config/wmagent/manage' connectToDB() myPhEDEx = PhEDEx() myDBS = DBS3Reader('https://cmsweb.cern.ch/dbs/prod/global/DBSReader/') myThread = threading.currentThread() print "Shutting down PhEDExInjector..." subprocess.call([ os.environ['manage'], "execute-agent", "wmcoreD", "--shutdown", "--component=PhEDExInjector" ], stdout=open(os.devnull, 'wb')) time.sleep(5) ## TASK1: query DB for files not injected in phedex yet # Get the files that the PhEDExInjector would look for formatter = DBFormatter(logging, myThread.dbi) formatter.sql = getQuery results = formatter.execute() fileList = [] fileList = [lfn[0] for lfn in results] ## TASK2: makes lfns a bit shorter to sort and uniq them reducedLfns = [lfn.rsplit('/', 2)[0] for lfn in fileList] reducedLfns = list(set(reducedLfns)) ## TASK3: build uniq dataset names and check whether PhEDEx and DBS contain ## the same number of files. If so, then those lfns are healthy print "Checking %d dataset in both PhEDEx and DBS ..." % len(reducedLfns) crippleLfns, healthyLfns = [], [] i = 0 n = len(reducedLfns) for lfn in reducedLfns: try: lfnAux = lfn.split('/') dset = '/' + lfnAux[4] + '/' + lfnAux[3] + '-' + lfnAux[ 6] + '/' + lfnAux[5] result = myPhEDEx._getResult('blockreplicas', args={'dataset': dset}, verb='GET') phedexFiles = 0 for item in result["phedex"]["block"]: phedexFiles += item['files'] ## TODO: ValidFile is only available for > 0.9.95pre5. Once all agents are ## upgraded, then we can start using this new query. #result = myDBS.listDatasetFileDetails(dset) #dbsFiles = 0 #for item in result.itervalues(): # dbsFiles += 1 if item['ValidFile'] else 0 # This call returns valid+invalid number of filesfiles result = myDBS.listDatasetFiles(dset) dbsFiles = len(result) if phedexFiles == dbsFiles: healthyLfns.append(lfn) else: crippleLfns.append(lfn) except: print "Error with:", lfn i += 1 if i % 100 == 0: print '%d/%d files processed' % (i, n) ## TASK4: map the short cripple and healthy lists to the full original lfns ## TODO: this code looks terrible... IMPROVE IT! if crippleLfns: filesToCheck = [] for lfn in crippleLfns: #filesToCheck = [file for file in fileList if lfn in file] for file in fileList: if lfn in file: filesToCheck.append(file) else: filesToCheck = [] if healthyLfns: filesInPhedex = [] for lfn in healthyLfns: #filesInPhedex = [file for file in fileList if lfn in file] for file in fileList: if lfn in file: filesInPhedex.append(file) else: filesInPhedex = [] ## TASK5: query PhEDEx for each cripple file (filesToCheck) ## and build the final file lists missingFiles = [] i = 0 n = len(filesToCheck) for file in filesToCheck: try: result = myPhEDEx._getResult('data', args={'file': file}, verb='GET') if len(result['phedex']['dbs']): filesInPhedex.append(file) else: missingFiles.append(file) except: print "Error contacting Phedex", file i += 1 if i % 100 == 0: print '%d/%d files processed' % (i, n) if not filesInPhedex: print "There are no files to be updated in the buffer. Contact a developer." print "Starting PhEDExInjector now ..." subprocess.call([ os.environ['manage'], "execute-agent", "wmcoreD", "--start", "--component=PhEDExInjector" ], stdout=open(os.devnull, 'wb')) return 0 print "Found %d out of %d files that are already registered in PhEDEx \ but buffer doesn't know" % (len(filesInPhedex), len(fileList)) print "Fixing them now, it may take several minutes ..." ## TASK6: time to actually fix these files binds = [] for file in filesInPhedex: binds.append({'lfn': file}) formatter.dbi.processData(setQuery, binds, conn=None, transaction=False, returnCursor=False) print "Rows were successfully updated! Good job!" print "Starting PhEDExInjector now ..." subprocess.call([ os.environ['manage'], "execute-agent", "wmcoreD", "--start", "--component=PhEDExInjector" ], stdout=open(os.devnull, 'wb')) print "Done!" return 0
def main(): """ _main_ """ if 'WMAGENT_CONFIG' not in os.environ: os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py' if 'manage' not in os.environ: os.environ['manage'] = '/data/srv/wmagent/current/config/wmagent/manage' connectToDB() myPhEDEx = PhEDEx() myDBS = DBS3Reader('https://cmsweb.cern.ch/dbs/prod/global/DBSReader/') myThread = threading.currentThread() print "Shutting down PhEDExInjector..." subprocess.call([os.environ['manage'], "execute-agent", "wmcoreD", "--shutdown", "--component=PhEDExInjector"], stdout=open(os.devnull, 'wb')) time.sleep(5) ## TASK1: query DB for files not injected in phedex yet # Get the files that the PhEDExInjector would look for formatter = DBFormatter(logging, myThread.dbi) formatter.sql = getQuery results = formatter.execute() fileList = [] fileList = [lfn[0] for lfn in results] ## TASK2: makes lfns a bit shorter to sort and uniq them reducedLfns = [lfn.rsplit('/',2)[0] for lfn in fileList] reducedLfns = list(set(reducedLfns)) ## TASK3: build uniq dataset names and check whether PhEDEx and DBS contain ## the same number of files. If so, then those lfns are healthy print "Checking %d dataset in both PhEDEx and DBS ..." % len(reducedLfns) crippleLfns, healthyLfns = [], [] i = 0 n = len(reducedLfns) for lfn in reducedLfns: try: lfnAux = lfn.split ('/') dset = '/'+lfnAux[4]+'/'+lfnAux[3]+'-'+lfnAux[6]+'/'+lfnAux[5] result = myPhEDEx._getResult('blockreplicas', args = {'dataset' : dset}, verb = 'GET') phedexFiles = 0 for item in result["phedex"]["block"]: phedexFiles += item['files'] ## TODO: ValidFile is only available for > 0.9.95pre5. Once all agents are ## upgraded, then we can start using this new query. #result = myDBS.listDatasetFileDetails(dset) #dbsFiles = 0 #for item in result.itervalues(): # dbsFiles += 1 if item['ValidFile'] else 0 # This call returns valid+invalid number of filesfiles result = myDBS.listDatasetFiles(dset) dbsFiles = len(result) if phedexFiles == dbsFiles: healthyLfns.append(lfn) else: crippleLfns.append(lfn) except: print "Error with:",lfn i += 1 if i % 100 == 0: print '%d/%d files processed'%(i,n) ## TASK4: map the short cripple and healthy lists to the full original lfns ## TODO: this code looks terrible... IMPROVE IT! if crippleLfns: filesToCheck = [] for lfn in crippleLfns: #filesToCheck = [file for file in fileList if lfn in file] for file in fileList: if lfn in file: filesToCheck.append(file) else: filesToCheck = [] if healthyLfns: filesInPhedex = [] for lfn in healthyLfns: #filesInPhedex = [file for file in fileList if lfn in file] for file in fileList: if lfn in file: filesInPhedex.append(file) else: filesInPhedex = [] ## TASK5: query PhEDEx for each cripple file (filesToCheck) ## and build the final file lists missingFiles = [] i = 0 n = len(filesToCheck) for file in filesToCheck: try: result = myPhEDEx._getResult('data', args = {'file' : file}, verb = 'GET') if len(result['phedex']['dbs']): filesInPhedex.append(file) else: missingFiles.append(file) except: print "Error contacting Phedex", file i += 1 if i % 100 == 0: print '%d/%d files processed'%(i,n) if not filesInPhedex: print "There are no files to be updated in the buffer. Contact a developer." print "Starting PhEDExInjector now ..." subprocess.call([os.environ['manage'], "execute-agent", "wmcoreD", "--start", "--component=PhEDExInjector"], stdout=open(os.devnull, 'wb')) return 0 print "Found %d out of %d files that are already registered in PhEDEx \ but buffer doesn't know" % (len(filesInPhedex), len(fileList)) print "Fixing them now, it may take several minutes ..." ## TASK6: time to actually fix these files binds = [] for file in filesInPhedex: binds.append({'lfn': file}) formatter.dbi.processData(setQuery, binds, conn = None, transaction = False, returnCursor = False) print "Rows were successfully updated! Good job!" print "Starting PhEDExInjector now ..." subprocess.call([os.environ['manage'], "execute-agent", "wmcoreD", "--start", "--component=PhEDExInjector"], stdout=open(os.devnull, 'wb')) print "Done!" return 0