def updateFileLog(fileHash, log, runLogCreationTimestamp, runLogBackend): '''Uploads the log of a file and the creationTimestamp of the run where the file has been processed. Called from online, after processing a file. ''' logging.debug('dropBox::updateFileLog(%s, %s [len], %s, %s)', fileHash, len(log), runLogCreationTimestamp, runLogBackend) dataAccess.updateFileLogLog(fileHash, log, runLogCreationTimestamp, runLogBackend) # Send email to user (fileName, statusCode, uploadTimestamp, finishTimestamp, username, userText, metadata) = dataAccess.getFileInformation(fileHash) userText = loadJson(userText) if userText is None: userText = '(userText too big to be displayed)' metadata = loadJson(metadata) if metadata is None: metadata = '(metadata too big to be displayed)' else: metadata = service.getPrettifiedJSON(metadata) fileInformation = { 'fileName': fileName, 'fileHash': fileHash, 'statusCode': statusCode, 'statusString': Constants.inverseMapping[statusCode], 'uploadTimestamp': uploadTimestamp, 'finishTimestamp': finishTimestamp, 'username': username, 'userText': userText, 'metadata': metadata, 'log': logPack.unpack(log), } toAddresses = [username] if service.settings['productionLevel'] in set(['int', 'pro']): toAddresses.append(config.notificationsEgroup) dataAccess.insertEmail(config.subjectTemplate.render(fileInformation), config.bodyTemplate.render(fileInformation), username, toAddresses) # If it failed for any reason, send an SMS to the shifter phone via email if service.settings['productionLevel'] in set(['int', 'pro']) and int(statusCode) != Constants.PROCESSING_OK: dataAccess.insertEmail(config.smsTemplate.render(fileInformation), '.', username, [config.shifterPhoneSMSAddress])
def main(): dropBoxRuns = replay.calculateOldDropBoxRuns() with open( '/afs/cern.ch/cms/DB/conddb/test/dropbox/replay/runInfoFromLogForReplay.json', 'rb') as f: filesPairs = json.load(f) dropBoxRunsPairs = {} emptyPairs = 0 # Replay all the runs i = 0 for runTimestamp in sorted(dropBoxRuns): i += 1 logging.info('[%s/%s] %s: Replaying run...', i, len(dropBoxRuns), runTimestamp) mergedPair = [None, None] j = 0 for fileName in dropBoxRuns[runTimestamp]: j += 1 fileName = '%s.db' % fileName[:-len('.tar.bz2')] if fileName in filesPairs: pair = filesPairs[fileName] else: pair = (None, None) logging.info(' [%s/%s] %s: %s...', j, len(dropBoxRuns[runTimestamp]), fileName, pair) mergedPair[0] = max(mergedPair[0], pair[0]) mergedPair[1] = max(mergedPair[1], pair[1]) if mergedPair[0] is None and mergedPair[1] is None: emptyPairs += 1 dropBoxRunsPairs[runTimestamp.strftime('%Y-%m-%d %H:%M:%S,%f') [:-3]] = mergedPair logging.info('Empty pairs: %s', emptyPairs) with open('runInfo.json', 'wb') as f: f.write(service.getPrettifiedJSON(dropBoxRunsPairs))
def getStatus(): '''Returns a the list of failed processed files in the latest hour. For check_mk agent dropBox.check_mk.py. Cached to prevent abuse/bugs loading the database. ''' return service.getPrettifiedJSON(connection.fetch(''' select fileHash, statusCode, creationTimestamp from fileLog where mod(statusCode, 100) in (10, 20) and creationTimestamp > sysdate - (1/24) and not exists ( select fileHash from fileAcks where fileAcks.fileHash = fileLog.fileHash ) order by creationTimestamp '''))
def main(): dropBoxRuns = replay.calculateOldDropBoxRuns() with open('/afs/cern.ch/cms/DB/conddb/test/dropbox/replay/runInfoFromLogForReplay.json', 'rb') as f: filesPairs = json.load(f) dropBoxRunsPairs = {} emptyPairs = 0 # Replay all the runs i = 0 for runTimestamp in sorted(dropBoxRuns): i += 1 logging.info('[%s/%s] %s: Replaying run...', i, len(dropBoxRuns), runTimestamp) mergedPair = [None, None] j = 0 for fileName in dropBoxRuns[runTimestamp]: j += 1 fileName = '%s.db' % fileName[:-len('.tar.bz2')] if fileName in filesPairs: pair = filesPairs[fileName] else: pair = (None, None) logging.info(' [%s/%s] %s: %s...', j, len(dropBoxRuns[runTimestamp]), fileName, pair) mergedPair[0] = max(mergedPair[0], pair[0]) mergedPair[1] = max(mergedPair[1], pair[1]) if mergedPair[0] is None and mergedPair[1] is None: emptyPairs += 1 dropBoxRunsPairs[runTimestamp.strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]] = mergedPair logging.info('Empty pairs: %s', emptyPairs) with open('runInfo.json', 'wb') as f: f.write(service.getPrettifiedJSON(dropBoxRunsPairs))
def generateServiceApi(cherrypyClass): '''CherryPy class decorator for generating the api() and apih() methods for CMS DB Web services. ''' # Get the methods' details methods = {} for (name, method) in inspect.getmembers(cherrypyClass, predicate=inspect.ismethod): if hasattr(method, 'exposed') and method.exposed == True: argspec = inspect.getargspec(method) methods[name] = { 'short_spec': str(argspec), 'doc': inspect.getdoc(method), 'args': argspec[0][1:], 'varargs': argspec[1], 'keywords': argspec[2], 'defaults': argspec[3], } template = ''' <!DOCTYPE html> <html> <head> <style type="text/css"> table { border-collapse: collapse; } td, th { border: 1px solid #7A7A7A; padding: 3px; vertical-align: middle; } th { background-color: #CACACA; text-align: center; } pre { margin: 0; } span { color: blue; font-weight: bold; } </style> <title>Exposed methods</title> </head> <body> <h1>Exposed methods</h1> <table> <tr> <th>Method</th> <th>Documentation</th> </tr> {% for method in methods %} <tr> <td> <span>{{method}}</span> {{methods[method]['short_spec']}} </td> <td> <pre>{{methods[method]['doc']}}</pre> </td> </tr> {% endfor %} </table> </body> </html> ''' # Pregenerate the JSON and HTML, so that it is done only once jsonApi = service.getPrettifiedJSON(methods) humanApi = jinja2.Template(template).render(methods=methods) # Generate the api() and apih() methods @cherrypy.expose def api(self): '''Returns the API of the class in JSON. ''' return service.setResponseJSON(jsonApi, encode=False) @cherrypy.expose def apih(self): '''Returns the API of the class in HTML (i.e. for humans). ''' return humanApi # Add the methods to the class cherrypyClass.api = api cherrypyClass.apih = apih return cherrypyClass
def generateServiceApi(cherrypyClass): '''CherryPy class decorator for generating the api() and apih() methods for CMS DB Web services. ''' # Get the methods' details methods = {} for (name, method) in inspect.getmembers(cherrypyClass, predicate = inspect.ismethod): if hasattr(method, 'exposed') and method.exposed == True: argspec = inspect.getargspec(method) methods[name] = { 'short_spec': str(argspec), 'doc': inspect.getdoc(method), 'args': argspec[0][1:], 'varargs': argspec[1], 'keywords': argspec[2], 'defaults': argspec[3], } template = ''' <!DOCTYPE html> <html> <head> <style type="text/css"> table { border-collapse: collapse; } td, th { border: 1px solid #7A7A7A; padding: 3px; vertical-align: middle; } th { background-color: #CACACA; text-align: center; } pre { margin: 0; } span { color: blue; font-weight: bold; } </style> <title>Exposed methods</title> </head> <body> <h1>Exposed methods</h1> <table> <tr> <th>Method</th> <th>Documentation</th> </tr> {% for method in methods %} <tr> <td> <span>{{method}}</span> {{methods[method]['short_spec']}} </td> <td> <pre>{{methods[method]['doc']}}</pre> </td> </tr> {% endfor %} </table> </body> </html> ''' # Pregenerate the JSON and HTML, so that it is done only once jsonApi = service.getPrettifiedJSON(methods) humanApi = jinja2.Template(template).render(methods = methods) # Generate the api() and apih() methods @cherrypy.expose def api(self): '''Returns the API of the class in JSON. ''' return service.setResponseJSON(jsonApi, encode = False) @cherrypy.expose def apih(self): '''Returns the API of the class in HTML (i.e. for humans). ''' return humanApi # Add the methods to the class cherrypyClass.api = api cherrypyClass.apih = apih return cherrypyClass