def deleteCollection(self, collection):
"""Delete all reports and logs for a particular collection."""
self.logreplock.acquire()
try:
for reportType in [liblog.RAW_REPORT, liblog.SUMMARY_REPORT]:
reports = self.getLogReports(collection, reportType)
for report in reports:
# stop running job if report is being (re)generated.
if report.completeState != COMPLETE:
self.stopRunningJob(self.jobName(report))
# delete data files if any.
(html_file, valid_file) = liblog.get_report_filenames(self.entConfig,
reportType, report.reportName, collection)
self.RemoveReportFiles(html_file, valid_file)
self.reportCount[reportType] -= len(reports)
logging.info('Delete total %d reports of type %s for collection %s.' % (
len(reports), reportType, collection))
listfile = liblog.get_report_list_filename(self.entConfig,
reportType, collection)
(err, out) = E.run_fileutil_command(self.entConfig,
'rm -f %s' % listfile)
if err:
logging.error('Cannot remove list file %s.' % listfile)
report_collection_dir = liblog.get_report_collection_dir(self.entConfig,
collection)
(err, out) = E.run_fileutil_command(self.entConfig,
'rmdir %s' % report_collection_dir)
if err:
logging.error('Cannot delete unused directory %s' % \
report_collection_dir)
finally:
self.logreplock.release()
def doLogReport(self, jobName, jobToken, collection, reportName, reportDate,
withResults, topCount, diagnosticTerms, update):
"""The actual work done in a worker thread to generate a summary report."""
(html_file, valid_file) = liblog.get_report_filenames(self.entConfig,
liblog.SUMMARY_REPORT, reportName, collection)
liblog.MakeGoogleDir(self.entConfig, os.path.dirname(html_file))
new_html_file = tempfile.mktemp('.report')
new_valid_file = tempfile.mktemp('.report_valid')
args = []
args.append(commands.mkarg(self.entConfig.GetEntHome()))
args.append(commands.mkarg(collection))
args.append(commands.mkarg(reportDate))
args.append(withResults)
args.append(topCount)
args.append(commands.mkarg(diagnosticTerms))
args.append(commands.mkarg(html_file))
args.append(commands.mkarg(valid_file))
args.append(commands.mkarg(new_html_file))
args.append(commands.mkarg(new_valid_file))
cmd = ('. %s && cd %s/enterprise/legacy/logs && '
'alarm %s nice -n 15 ./log_report.py %s' %
(self.cfg.getGlobalParam('ENTERPRISE_BASHRC'),
self.cfg.getGlobalParam('MAIN_GOOGLE3_DIR'),
COMMAND_TIMEOUT_PERIOD, string.join(args, ' ')))
logging.info('doLogReport(): CMD = %s' % cmd)
returnCode = E.system(cmd)
self.handleResult(jobName, jobToken, returnCode, liblog.SUMMARY_REPORT,
collection, reportName, update,
html_file, valid_file, new_html_file, new_valid_file)
def getReport(self, collection, reportName):
"""Return body of a summary report."""
self.logreplock.acquire()
try:
reports = self.getLogReports(collection, liblog.SUMMARY_REPORT)
found = false
incomplete = false
for report in reports:
if report.reportName == reportName:
found = true
if (report.completeState != COMPLETE and
report.completeState != COMPLETE_REGENERATE):
incomplete = true
break
if not found:
logging.error('Report %s not found' % reportName)
return (C.REPORT_NAME_NOT_FOUND, None, None)
elif incomplete:
logging.error('Report %s is incomplete' % reportName)
return (C.REPORT_INCOMPLETE, report.toString(), None)
(html_file, _) = liblog.get_report_filenames(self.entConfig,
liblog.SUMMARY_REPORT, reportName, collection)
try:
reportContents = gfile.GFile(html_file, 'r').read()
except IOError:
return (C.REPORT_INTERNAL_ERROR, report.toString(), None)
finally:
self.logreplock.release()
return (C.REPORT_OK, report.toString(), reportContents)
def CopyRawReportFromGfsToLocal(self, reportName, collection):
"""Make a local copy of a raw report so file_handler can use."""
(remoteName, _) = liblog.get_report_filenames(self.entConfig,
liblog.RAW_REPORT, reportName, collection)
localName = liblog.get_local_raw_report_filename(self.entConfig,
reportName, collection)
liblog.MakeDir(os.path.dirname(localName))
(err, out) = E.run_fileutil_command(self.entConfig, 'copy -f %s %s' % \
(remoteName, localName),
COMMAND_TIMEOUT_PERIOD)
if err:
logging.error('Failed to make copy from gfs for %s. Error: %s' % \
(localName, out))
return false
return true
def deleteReport(self, collection, reportType,
reportName, doCancel=false):
"""Delete a complete report or cancel a pending report."""
self.logreplock.acquire()
(html_file, valid_file) = liblog.get_report_filenames(self.entConfig,
reportType, reportName, collection)
try:
found = false
reportList = self.getLogReports(collection, reportType)
for report in reportList:
if reportName == report.reportName:
found = true
break
if not found:
logging.error('Report not found')
return C.REPORT_NAME_NOT_FOUND
if (doCancel and not (report.completeState == PENDING or
report.completeState == COMPLETE_REGENERATE)):
# doCancel should apply for running report only.
return C.REPORT_ALREADY_COMPLETE
if (not doCancel and not (report.completeState == COMPLETE or
report.completeState == FAILURE)):
# delete (doCancel==false) should apply for complete/failed report only.
return C.REPORT_INCOMPLETE
if doCancel:
self.stopRunningJob(self.jobName(report))
if report.completeState == COMPLETE_REGENERATE:
# Just recover original state, don't delete the old report.
report.completeState = COMPLETE
report.isFinal = string.split(report.isFinal, '|')[0]
report.creationDate = string.split(report.creationDate, '|')[0]
else:
self.reportCount[report.reportType] -= 1
reportList.remove(report)
self.RemoveReportFiles(html_file, valid_file)
if not self.setLogReports(reportType, collection, reportList):
logging.error('Failed to update report list.')
return C.REPORT_INTERNAL_ERROR
finally:
self.logreplock.release()
return C.REPORT_OK
def doLogDump(self, jobName, jobToken, collection, reportName,
reportDate, update):
"""The actual work done in a worker thread to generate a raw log report."""
(html_file, valid_file) = liblog.get_report_filenames(self.entConfig,
liblog.RAW_REPORT, reportName, collection)
liblog.MakeGoogleDir(self.entConfig, os.path.dirname(html_file))
# (TODO): Change this once the we move to python2.4 to use a safer call to
# mkstemp which accepts the target directory name as an argument
new_html_temp = tempfile.mktemp('.log')
# create file in /export/hda3 instead of /tmp. The / partition is very
# small compared to the /export/hda3
new_html_file = '/export/hda3' + new_html_temp
# need to perform a check about file existance
while os.path.exists(new_html_file):
new_html_temp = tempfile.mktemp('.log')
new_html_file = '/export/hda3' + new_html_temp
new_valid_file = tempfile.mktemp('.log_valid')
args = []
args.append(commands.mkarg(collection))
args.append(commands.mkarg(reportDate))
args.append(commands.mkarg(html_file))
args.append(commands.mkarg(valid_file))
args.append(commands.mkarg(new_valid_file))
cmd = ('(. %s && cd %s/enterprise/legacy/logs && '
'alarm %s nice -n 15 ./apache_log.py %s %s) > %s' %
(self.cfg.getGlobalParam('ENTERPRISE_BASHRC'),
self.cfg.getGlobalParam('MAIN_GOOGLE3_DIR'),
COMMAND_TIMEOUT_PERIOD,
commands.mkarg(self.cfg.globalParams.GetEntHome()),
string.join(args, ' '),
commands.mkarg(new_html_file)))
logging.info('doLogDump(): CMD = %s' % cmd)
returnCode = E.system(cmd)
self.handleResult(jobName, jobToken, returnCode, liblog.RAW_REPORT,
collection, reportName, update,
html_file, valid_file, new_html_file, new_valid_file)
