def updateFileLog(fileHash, log, runLogCreationTimestamp, runLogBackend):
'''Uploads the log of a file and the creationTimestamp of the run
where the file has been processed.
Called from online, after processing a file.
'''
logging.debug('dropBox::updateFileLog(%s, %s [len], %s, %s)', fileHash, len(log), runLogCreationTimestamp, runLogBackend)
dataAccess.updateFileLogLog(fileHash, log, runLogCreationTimestamp, runLogBackend)
# Send email to user
(fileName, statusCode, uploadTimestamp, finishTimestamp, username, userText, metadata) = dataAccess.getFileInformation(fileHash)
userText = loadJson(userText)
if userText is None:
userText = '(userText too big to be displayed)'
metadata = loadJson(metadata)
if metadata is None:
metadata = '(metadata too big to be displayed)'
else:
metadata = service.getPrettifiedJSON(metadata)
fileInformation = {
'fileName': fileName,
'fileHash': fileHash,
'statusCode': statusCode,
'statusString': Constants.inverseMapping[statusCode],
'uploadTimestamp': uploadTimestamp,
'finishTimestamp': finishTimestamp,
'username': username,
'userText': userText,
'metadata': metadata,
'log': logPack.unpack(log),
}
toAddresses = [username]
if service.settings['productionLevel'] in set(['int', 'pro']):
toAddresses.append(config.notificationsEgroup)
dataAccess.insertEmail(config.subjectTemplate.render(fileInformation), config.bodyTemplate.render(fileInformation), username, toAddresses)
# If it failed for any reason, send an SMS to the shifter phone via email
if service.settings['productionLevel'] in set(['int', 'pro']) and int(statusCode) != Constants.PROCESSING_OK:
dataAccess.insertEmail(config.smsTemplate.render(fileInformation), '.', username, [config.shifterPhoneSMSAddress])
def main():
dropBoxRuns = replay.calculateOldDropBoxRuns()
with open(
'/afs/cern.ch/cms/DB/conddb/test/dropbox/replay/runInfoFromLogForReplay.json',
'rb') as f:
filesPairs = json.load(f)
dropBoxRunsPairs = {}
emptyPairs = 0
# Replay all the runs
i = 0
for runTimestamp in sorted(dropBoxRuns):
i += 1
logging.info('[%s/%s] %s: Replaying run...', i, len(dropBoxRuns),
runTimestamp)
mergedPair = [None, None]
j = 0
for fileName in dropBoxRuns[runTimestamp]:
j += 1
fileName = '%s.db' % fileName[:-len('.tar.bz2')]
if fileName in filesPairs:
pair = filesPairs[fileName]
else:
pair = (None, None)
logging.info(' [%s/%s] %s: %s...', j,
len(dropBoxRuns[runTimestamp]), fileName, pair)
mergedPair[0] = max(mergedPair[0], pair[0])
mergedPair[1] = max(mergedPair[1], pair[1])
if mergedPair[0] is None and mergedPair[1] is None:
emptyPairs += 1
dropBoxRunsPairs[runTimestamp.strftime('%Y-%m-%d %H:%M:%S,%f')
[:-3]] = mergedPair
logging.info('Empty pairs: %s', emptyPairs)
with open('runInfo.json', 'wb') as f:
f.write(service.getPrettifiedJSON(dropBoxRunsPairs))
def getStatus():
'''Returns a the list of failed processed files in the latest hour.
For check_mk agent dropBox.check_mk.py.
Cached to prevent abuse/bugs loading the database.
'''
return service.getPrettifiedJSON(connection.fetch('''
select fileHash, statusCode, creationTimestamp
from fileLog
where mod(statusCode, 100) in (10, 20)
and creationTimestamp > sysdate - (1/24)
and not exists (
select fileHash
from fileAcks
where fileAcks.fileHash = fileLog.fileHash
)
order by creationTimestamp
'''))
def main():
dropBoxRuns = replay.calculateOldDropBoxRuns()
with open('/afs/cern.ch/cms/DB/conddb/test/dropbox/replay/runInfoFromLogForReplay.json', 'rb') as f:
filesPairs = json.load(f)
dropBoxRunsPairs = {}
emptyPairs = 0
# Replay all the runs
i = 0
for runTimestamp in sorted(dropBoxRuns):
i += 1
logging.info('[%s/%s] %s: Replaying run...', i, len(dropBoxRuns), runTimestamp)
mergedPair = [None, None]
j = 0
for fileName in dropBoxRuns[runTimestamp]:
j += 1
fileName = '%s.db' % fileName[:-len('.tar.bz2')]
if fileName in filesPairs:
pair = filesPairs[fileName]
else:
pair = (None, None)
logging.info(' [%s/%s] %s: %s...', j, len(dropBoxRuns[runTimestamp]), fileName, pair)
mergedPair[0] = max(mergedPair[0], pair[0])
mergedPair[1] = max(mergedPair[1], pair[1])
if mergedPair[0] is None and mergedPair[1] is None:
emptyPairs += 1
dropBoxRunsPairs[runTimestamp.strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]] = mergedPair
logging.info('Empty pairs: %s', emptyPairs)
with open('runInfo.json', 'wb') as f:
f.write(service.getPrettifiedJSON(dropBoxRunsPairs))
def generateServiceApi(cherrypyClass):
'''CherryPy class decorator for generating the api() and apih() methods for CMS DB Web services.
'''
# Get the methods' details
methods = {}
for (name, method) in inspect.getmembers(cherrypyClass,
predicate=inspect.ismethod):
if hasattr(method, 'exposed') and method.exposed == True:
argspec = inspect.getargspec(method)
methods[name] = {
'short_spec': str(argspec),
'doc': inspect.getdoc(method),
'args': argspec[0][1:],
'varargs': argspec[1],
'keywords': argspec[2],
'defaults': argspec[3],
}
template = '''
<!DOCTYPE html>
<html>
<head>
<style type="text/css">
table {
border-collapse: collapse;
}
td, th {
border: 1px solid #7A7A7A;
padding: 3px;
vertical-align: middle;
}
th {
background-color: #CACACA;
text-align: center;
}
pre {
margin: 0;
}
span {
color: blue;
font-weight: bold;
}
</style>
<title>Exposed methods</title>
</head>
<body>
<h1>Exposed methods</h1>
<table>
<tr>
<th>Method</th>
<th>Documentation</th>
</tr>
{% for method in methods %}
<tr>
<td>
<span>{{method}}</span> {{methods[method]['short_spec']}}
</td>
<td>
<pre>{{methods[method]['doc']}}</pre>
</td>
</tr>
{% endfor %}
</table>
</body>
</html>
'''
# Pregenerate the JSON and HTML, so that it is done only once
jsonApi = service.getPrettifiedJSON(methods)
humanApi = jinja2.Template(template).render(methods=methods)
# Generate the api() and apih() methods
@cherrypy.expose
def api(self):
'''Returns the API of the class in JSON.
'''
return service.setResponseJSON(jsonApi, encode=False)
@cherrypy.expose
def apih(self):
'''Returns the API of the class in HTML (i.e. for humans).
'''
return humanApi
# Add the methods to the class
cherrypyClass.api = api
cherrypyClass.apih = apih
return cherrypyClass
def generateServiceApi(cherrypyClass):
'''CherryPy class decorator for generating the api() and apih() methods for CMS DB Web services.
'''
# Get the methods' details
methods = {}
for (name, method) in inspect.getmembers(cherrypyClass, predicate = inspect.ismethod):
if hasattr(method, 'exposed') and method.exposed == True:
argspec = inspect.getargspec(method)
methods[name] = {
'short_spec': str(argspec),
'doc': inspect.getdoc(method),
'args': argspec[0][1:],
'varargs': argspec[1],
'keywords': argspec[2],
'defaults': argspec[3],
}
template = '''
<!DOCTYPE html>
<html>
<head>
<style type="text/css">
table {
border-collapse: collapse;
}
td, th {
border: 1px solid #7A7A7A;
padding: 3px;
vertical-align: middle;
}
th {
background-color: #CACACA;
text-align: center;
}
pre {
margin: 0;
}
span {
color: blue;
font-weight: bold;
}
</style>
<title>Exposed methods</title>
</head>
<body>
<h1>Exposed methods</h1>
<table>
<tr>
<th>Method</th>
<th>Documentation</th>
</tr>
{% for method in methods %}
<tr>
<td>
<span>{{method}}</span> {{methods[method]['short_spec']}}
</td>
<td>
<pre>{{methods[method]['doc']}}</pre>
</td>
</tr>
{% endfor %}
</table>
</body>
</html>
'''
# Pregenerate the JSON and HTML, so that it is done only once
jsonApi = service.getPrettifiedJSON(methods)
humanApi = jinja2.Template(template).render(methods = methods)
# Generate the api() and apih() methods
@cherrypy.expose
def api(self):
'''Returns the API of the class in JSON.
'''
return service.setResponseJSON(jsonApi, encode = False)
@cherrypy.expose
def apih(self):
'''Returns the API of the class in HTML (i.e. for humans).
'''
return humanApi
# Add the methods to the class
cherrypyClass.api = api
cherrypyClass.apih = apih
return cherrypyClass
