def deleteCollection(self, collection): """Delete all reports and logs for a particular collection.""" self.logreplock.acquire() try: for reportType in [liblog.RAW_REPORT, liblog.SUMMARY_REPORT]: reports = self.getLogReports(collection, reportType) for report in reports: # stop running job if report is being (re)generated. if report.completeState != COMPLETE: self.stopRunningJob(self.jobName(report)) # delete data files if any. (html_file, valid_file) = liblog.get_report_filenames(self.entConfig, reportType, report.reportName, collection) self.RemoveReportFiles(html_file, valid_file) self.reportCount[reportType] -= len(reports) logging.info('Delete total %d reports of type %s for collection %s.' % ( len(reports), reportType, collection)) listfile = liblog.get_report_list_filename(self.entConfig, reportType, collection) (err, out) = E.run_fileutil_command(self.entConfig, 'rm -f %s' % listfile) if err: logging.error('Cannot remove list file %s.' % listfile) report_collection_dir = liblog.get_report_collection_dir(self.entConfig, collection) (err, out) = E.run_fileutil_command(self.entConfig, 'rmdir %s' % report_collection_dir) if err: logging.error('Cannot delete unused directory %s' % \ report_collection_dir) finally: self.logreplock.release()
def doLogReport(self, jobName, jobToken, collection, reportName, reportDate, withResults, topCount, diagnosticTerms, update): """The actual work done in a worker thread to generate a summary report.""" (html_file, valid_file) = liblog.get_report_filenames(self.entConfig, liblog.SUMMARY_REPORT, reportName, collection) liblog.MakeGoogleDir(self.entConfig, os.path.dirname(html_file)) new_html_file = tempfile.mktemp('.report') new_valid_file = tempfile.mktemp('.report_valid') args = [] args.append(commands.mkarg(self.entConfig.GetEntHome())) args.append(commands.mkarg(collection)) args.append(commands.mkarg(reportDate)) args.append(withResults) args.append(topCount) args.append(commands.mkarg(diagnosticTerms)) args.append(commands.mkarg(html_file)) args.append(commands.mkarg(valid_file)) args.append(commands.mkarg(new_html_file)) args.append(commands.mkarg(new_valid_file)) cmd = ('. %s && cd %s/enterprise/legacy/logs && ' 'alarm %s nice -n 15 ./log_report.py %s' % (self.cfg.getGlobalParam('ENTERPRISE_BASHRC'), self.cfg.getGlobalParam('MAIN_GOOGLE3_DIR'), COMMAND_TIMEOUT_PERIOD, string.join(args, ' '))) logging.info('doLogReport(): CMD = %s' % cmd) returnCode = E.system(cmd) self.handleResult(jobName, jobToken, returnCode, liblog.SUMMARY_REPORT, collection, reportName, update, html_file, valid_file, new_html_file, new_valid_file)
def getReport(self, collection, reportName): """Return body of a summary report.""" self.logreplock.acquire() try: reports = self.getLogReports(collection, liblog.SUMMARY_REPORT) found = false incomplete = false for report in reports: if report.reportName == reportName: found = true if (report.completeState != COMPLETE and report.completeState != COMPLETE_REGENERATE): incomplete = true break if not found: logging.error('Report %s not found' % reportName) return (C.REPORT_NAME_NOT_FOUND, None, None) elif incomplete: logging.error('Report %s is incomplete' % reportName) return (C.REPORT_INCOMPLETE, report.toString(), None) (html_file, _) = liblog.get_report_filenames(self.entConfig, liblog.SUMMARY_REPORT, reportName, collection) try: reportContents = gfile.GFile(html_file, 'r').read() except IOError: return (C.REPORT_INTERNAL_ERROR, report.toString(), None) finally: self.logreplock.release() return (C.REPORT_OK, report.toString(), reportContents)
def CopyRawReportFromGfsToLocal(self, reportName, collection): """Make a local copy of a raw report so file_handler can use.""" (remoteName, _) = liblog.get_report_filenames(self.entConfig, liblog.RAW_REPORT, reportName, collection) localName = liblog.get_local_raw_report_filename(self.entConfig, reportName, collection) liblog.MakeDir(os.path.dirname(localName)) (err, out) = E.run_fileutil_command(self.entConfig, 'copy -f %s %s' % \ (remoteName, localName), COMMAND_TIMEOUT_PERIOD) if err: logging.error('Failed to make copy from gfs for %s. Error: %s' % \ (localName, out)) return false return true
def deleteReport(self, collection, reportType, reportName, doCancel=false): """Delete a complete report or cancel a pending report.""" self.logreplock.acquire() (html_file, valid_file) = liblog.get_report_filenames(self.entConfig, reportType, reportName, collection) try: found = false reportList = self.getLogReports(collection, reportType) for report in reportList: if reportName == report.reportName: found = true break if not found: logging.error('Report not found') return C.REPORT_NAME_NOT_FOUND if (doCancel and not (report.completeState == PENDING or report.completeState == COMPLETE_REGENERATE)): # doCancel should apply for running report only. return C.REPORT_ALREADY_COMPLETE if (not doCancel and not (report.completeState == COMPLETE or report.completeState == FAILURE)): # delete (doCancel==false) should apply for complete/failed report only. return C.REPORT_INCOMPLETE if doCancel: self.stopRunningJob(self.jobName(report)) if report.completeState == COMPLETE_REGENERATE: # Just recover original state, don't delete the old report. report.completeState = COMPLETE report.isFinal = string.split(report.isFinal, '|')[0] report.creationDate = string.split(report.creationDate, '|')[0] else: self.reportCount[report.reportType] -= 1 reportList.remove(report) self.RemoveReportFiles(html_file, valid_file) if not self.setLogReports(reportType, collection, reportList): logging.error('Failed to update report list.') return C.REPORT_INTERNAL_ERROR finally: self.logreplock.release() return C.REPORT_OK
def doLogDump(self, jobName, jobToken, collection, reportName, reportDate, update): """The actual work done in a worker thread to generate a raw log report.""" (html_file, valid_file) = liblog.get_report_filenames(self.entConfig, liblog.RAW_REPORT, reportName, collection) liblog.MakeGoogleDir(self.entConfig, os.path.dirname(html_file)) # (TODO): Change this once the we move to python2.4 to use a safer call to # mkstemp which accepts the target directory name as an argument new_html_temp = tempfile.mktemp('.log') # create file in /export/hda3 instead of /tmp. The / partition is very # small compared to the /export/hda3 new_html_file = '/export/hda3' + new_html_temp # need to perform a check about file existance while os.path.exists(new_html_file): new_html_temp = tempfile.mktemp('.log') new_html_file = '/export/hda3' + new_html_temp new_valid_file = tempfile.mktemp('.log_valid') args = [] args.append(commands.mkarg(collection)) args.append(commands.mkarg(reportDate)) args.append(commands.mkarg(html_file)) args.append(commands.mkarg(valid_file)) args.append(commands.mkarg(new_valid_file)) cmd = ('(. %s && cd %s/enterprise/legacy/logs && ' 'alarm %s nice -n 15 ./apache_log.py %s %s) > %s' % (self.cfg.getGlobalParam('ENTERPRISE_BASHRC'), self.cfg.getGlobalParam('MAIN_GOOGLE3_DIR'), COMMAND_TIMEOUT_PERIOD, commands.mkarg(self.cfg.globalParams.GetEntHome()), string.join(args, ' '), commands.mkarg(new_html_file))) logging.info('doLogDump(): CMD = %s' % cmd) returnCode = E.system(cmd) self.handleResult(jobName, jobToken, returnCode, liblog.RAW_REPORT, collection, reportName, update, html_file, valid_file, new_html_file, new_valid_file)