def _doUploadFile(self, oSrcFile, sSrcFilename, sDescription, sKind, sMime): """ Uploads the given file to the test manager. """ # Prepare header and url. dHeader = dict(self._dHttpHeader) dHeader['Content-Type'] = 'application/octet-stream' self._writeOutput('%s: _doUploadFile: sHeader=%s' % ( utils.getTimePrefix(), dHeader, )) oSrcFile.seek(0, 2) self._writeOutput('%s: _doUploadFile: size=%d' % ( utils.getTimePrefix(), oSrcFile.tell(), )) oSrcFile.seek(0) from common import constants sUrl = self._sTmServerPath + '&' \ + self._fnUrlEncode({ constants.tbreq.UPLOAD_PARAM_NAME: os.path.basename(sSrcFilename), constants.tbreq.UPLOAD_PARAM_DESC: sDescription, constants.tbreq.UPLOAD_PARAM_KIND: sKind, constants.tbreq.UPLOAD_PARAM_MIME: sMime, constants.tbreq.ALL_PARAM_ACTION: constants.tbreq.UPLOAD, }) # Retry loop. secStart = utils.timestampSecond() while True: try: oConn = self._fnTmConnect() oConn.request('POST', sUrl, oSrcFile.read(), dHeader) fRc = self._processTmStatusResponse(oConn, '_doUploadFile', fClose=True) oConn.close() if fRc is not None: return fRc except: logXcpt('warning: exception during UPLOAD request') if utils.timestampSecond( ) - secStart >= self.kcSecTestManagerRetryTimeout: self._writeOutput('%s: _doUploadFile: Timed out.' % (utils.getTimePrefix(), )) break try: oSrcFile.seek(0) except: logXcpt() break self._writeOutput('%s: _doUploadFile: Retrying...' % (utils.getTimePrefix(), )) time.sleep(2) return False
def logAllStacks(cFrames=None): """ Logs the stacks of all python threads. """ sTsPrf = utils.getTimePrefix() sCaller = utils.getCallerName() g_oLock.acquire() cThread = 0 for idThread, oStack in sys._current_frames().items(): # >=2.5, a bit ugly - pylint: disable=W0212 try: if cThread > 0: g_oReporter.log(1, '', sCaller, sTsPrf) g_oReporter.log(1, 'Thread %s (%#x)' % (idThread, idThread), sCaller, sTsPrf) try: asInfo = traceback.format_stack(oStack, cFrames) except: g_oReporter.log(1, ' Stack formatting failed w/ exception', sCaller, sTsPrf) else: for sInfo in asInfo: asLines = sInfo.splitlines() for sLine in asLines: g_oReporter.log(1, sLine, sCaller, sTsPrf) except: pass cThread += 1 g_oLock.release() return None
def logAllStacks(cFrames = None): """ Logs the stacks of all python threads. """ sTsPrf = utils.getTimePrefix(); sCaller = utils.getCallerName(); g_oLock.acquire(); cThread = 0; for idThread, oStack in sys._current_frames().items(): # >=2.5, a bit ugly - pylint: disable=W0212 try: if cThread > 0: g_oReporter.log(1, '', sCaller, sTsPrf); g_oReporter.log(1, 'Thread %s (%#x)' % (idThread, idThread), sCaller, sTsPrf); try: asInfo = traceback.format_stack(oStack, cFrames); except: g_oReporter.log(1, ' Stack formatting failed w/ exception', sCaller, sTsPrf); else: for sInfo in asInfo: asLines = sInfo.splitlines(); for sLine in asLines: g_oReporter.log(1, sLine, sCaller, sTsPrf); except: pass; cThread += 1; g_oLock.release(); return None;
def addLogFile(sFilename, sKind, sDescription='', sAltName=None): """ Adds the specified log file to the report if the file exists. The sDescription is a free form description of the log file. The sKind parameter is for adding some machine parsable hint what kind of log file this really is. Returns True on success, False on failure (no ENOENT errors are logged). """ sTsPrf = utils.getTimePrefix() sCaller = utils.getCallerName() fRc = False if sAltName is None: sAltName = sFilename try: oSrcFile = utils.openNoInherit(sFilename, 'rb') except IOError, oXcpt: if oXcpt.errno != errno.ENOENT: logXcpt('addLogFile(%s,%s,%s)' % (sFilename, sDescription, sKind)) else: logXcpt('addLogFile(%s,%s,%s) IOError' % (sFilename, sDescription, sKind))
def log(sMessage, sCaller = None, sTsPrf = None): """ Print out a message and flush stdout """ if sTsPrf is None: sTsPrf = utils.getTimePrefix(); print('[%s] %s' % (sTsPrf, sMessage,)); sys.stdout.flush(); _ = sCaller;
def logXcptWorker(iLevel, fIncErrors, sPrefix="", sText=None, cFrames=1): """ Log an exception, optionally with a preceeding message and more than one call frame. """ g_oLock.acquire(); if fIncErrors: g_oReporter.testIncErrors(); ## @todo skip all this if iLevel is too high! # Try get exception info. sTsPrf = utils.getTimePrefix(); try: oType, oValue, oTraceback = sys.exc_info(); except: oType = oValue = oTraceback = None; if oType is not None: # Try format the info try: rc = 0; sCaller = utils.getCallerName(oTraceback.tb_frame); if sText is not None: rc = g_oReporter.log(iLevel, "%s%s" % (sPrefix, sText), sCaller, sTsPrf); asInfo = []; try: asInfo = asInfo + traceback.format_exception_only(oType, oValue); if cFrames is not None and cFrames <= 1: asInfo = asInfo + traceback.format_tb(oTraceback, 1); else: asInfo.append('Traceback:') asInfo = asInfo + traceback.format_tb(oTraceback, cFrames); asInfo.append('Stack:') asInfo = asInfo + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames); except: g_oReporter.log(0, 'internal-error: Hit exception #2! %s' % (traceback.format_exc()), sCaller, sTsPrf); if len(asInfo) > 0: # Do the logging. for sItem in asInfo: asLines = sItem.splitlines(); for sLine in asLines: rc = g_oReporter.log(iLevel, '%s%s' % (sPrefix, sLine), sCaller, sTsPrf); else: g_oReporter.log(iLevel, 'No exception info...', sCaller, sTsPrf); rc = -3; except: g_oReporter.log(0, 'internal-error: Hit exception! %s' % (traceback.format_exc()), None, sTsPrf); rc = -2; else: g_oReporter.log(0, 'internal-error: No exception! %s' % (utils.getCallerName(iFrame=3)), utils.getCallerName(iFrame=3), sTsPrf); rc = -1; g_oLock.release(); return rc;
def log2(sMessage, sCaller = None, sTsPrf = None): """ Debug logging, will later be disabled by default. """ if True: if sTsPrf is None: sTsPrf = utils.getTimePrefix(); print('[%s] %s' % (sTsPrf, sMessage,)); sys.stdout.flush() _ = sCaller;
def log2(sText): """Log level 2: Writes the specfied text to the log.""" g_oLock.acquire(); try: rc = g_oReporter.log(2, sText, utils.getCallerName(), utils.getTimePrefix()); except: rc = -1; g_oLock.release(); return rc;
def _doUploadFile(self, oSrcFile, sSrcFilename, sDescription, sKind, sMime): """ Uploads the given file to the test manager. """ # Prepare header and url. dHeader = dict(self._dHttpHeader); dHeader['Content-Type'] = 'application/octet-stream'; self._writeOutput('%s: _doUploadFile: sHeader=%s' % (utils.getTimePrefix(), dHeader,)); oSrcFile.seek(0, 2); self._writeOutput('%s: _doUploadFile: size=%d' % (utils.getTimePrefix(), oSrcFile.tell(),)); oSrcFile.seek(0); from common import constants; sUrl = self._sTmServerPath + '&' \ + self._fnUrlEncode({ constants.tbreq.UPLOAD_PARAM_NAME: os.path.basename(sSrcFilename), constants.tbreq.UPLOAD_PARAM_DESC: sDescription, constants.tbreq.UPLOAD_PARAM_KIND: sKind, constants.tbreq.UPLOAD_PARAM_MIME: sMime, constants.tbreq.ALL_PARAM_ACTION: constants.tbreq.UPLOAD, }); # Retry loop. secStart = utils.timestampSecond(); while True: try: oConn = self._fnTmConnect(); oConn.request('POST', sUrl, oSrcFile.read(), dHeader); fRc = self._processTmStatusResponse(oConn, '_doUploadFile', fClose = True); oConn.close(); if fRc is not None: return fRc; except: logXcpt('warning: exception during UPLOAD request'); if utils.timestampSecond() - secStart >= self.kcSecTestManagerRetryTimeout: self._writeOutput('%s: _doUploadFile: Timed out.' % (utils.getTimePrefix(),)); break; try: oSrcFile.seek(0); except: logXcpt(); break; self._writeOutput('%s: _doUploadFile: Retrying...' % (utils.getTimePrefix(), )); time.sleep(2); return False;
def log2(sText): """Log level 2: Writes the specfied text to the log.""" g_oLock.acquire() try: rc = g_oReporter.log(2, sText, utils.getCallerName(), utils.getTimePrefix()) except: rc = -1 g_oLock.release() return rc
def log2(sMessage, sCaller=None, sTsPrf=None): """ Debug logging, will later be disabled by default. """ if True is True: # pylint: disable=comparison-with-itself if sTsPrf is None: sTsPrf = utils.getTimePrefix() print('[%s] %s' % ( sTsPrf, sMessage, )) sys.stdout.flush() _ = sCaller
def _logXcptWorker(fnLogger, sPrefix = '', sText = None, cFrames = 1, fnLogger1 = log): """ Log an exception, optionally with a preceeding message and more than one call frame. """ ## @todo skip all this if iLevel is too high! # Try get exception info. sTsPrf = utils.getTimePrefix(); try: oType, oValue, oTraceback = sys.exc_info(); except: oType = oValue = oTraceback = None; if oType is not None: # Try format the info try: rc = 0; sCaller = utils.getCallerName(oTraceback.tb_frame); if sText is not None: rc = fnLogger('%s%s' % (sPrefix, sText), sCaller, sTsPrf); asInfo = []; try: asInfo = asInfo + traceback.format_exception_only(oType, oValue); if cFrames is not None and cFrames <= 1: asInfo = asInfo + traceback.format_tb(oTraceback, 1); else: asInfo.append('Traceback:') asInfo = asInfo + traceback.format_tb(oTraceback, cFrames); asInfo.append('Stack:') asInfo = asInfo + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames); except: fnLogger1('internal-error: Hit exception #2! %s' % (traceback.format_exc()), sCaller, sTsPrf); if len(asInfo) > 0: # Do the logging. for sItem in asInfo: asLines = sItem.splitlines(); for sLine in asLines: rc = fnLogger('%s%s' % (sPrefix, sLine), sCaller, sTsPrf); else: fnLogger('No exception info...', sCaller, sTsPrf); rc = -3; except: fnLogger1('internal-error: Hit exception! %s' % (traceback.format_exc()), None, sTsPrf); rc = -2; else: fnLogger1('internal-error: No exception! %s' % (utils.getCallerName(iFrame=3)), utils.getCallerName(iFrame=3), sTsPrf); rc = -1; return rc;
def fatal(sText): """ Writes a fatal error to the log. This will add an error to the current test. Always returns False for the convenience of methods returning boolean success indicators. """ g_oLock.acquire(); g_oReporter.testIncErrors(); try: g_oReporter.log(0, 'fatal error: %s' % (sText), utils.getCallerName(), utils.getTimePrefix()); except: pass g_oLock.release(); return False;
def _processTmStatusResponse(self, oConn, sOperation, fClose = True): """ Processes HTTP reponse from the test manager. Returns True, False or None. None should be retried, the others not. May raise exception on HTTP issue (retry ok). """ import httplib; from common import constants; # Read the response and (optionally) close the connection. oResponse = oConn.getresponse(); try: sRspBody = oResponse.read(); except httplib.IncompleteRead, oXcpt: self._writeOutput('%s: %s: Warning: httplib.IncompleteRead: %s [expected %s, got %s]' % (utils.getTimePrefix(), sOperation, oXcpt, oXcpt.expected, len(oXcpt.partial),)); sRspBody = oXcpt.partial;
def errorTimeout(sText): """ Flags the current test as having timed out and writes the specified message to the log. This will add an error to the current test. Always returns False for the convenience of methods returning boolean success indicators. """ g_oLock.acquire(); g_oReporter.testSetTimedOut(); try: g_oReporter.log(0, 'timeout-error: %s' % (sText), utils.getCallerName(), utils.getTimePrefix()); except: pass; g_oLock.release(); return False;
def write(self, sText): """file.write""" if isinstance(sText, array.array): try: sText = sText.tostring(); except: pass; g_oLock.acquire(); try: sTsPrf = utils.getTimePrefix(); sCaller = utils.getCallerName(); asLines = sText.splitlines(); for sLine in asLines: g_oReporter.log(0, '%s: %s' % (self.sPrefix, sLine), sCaller, sTsPrf); except: traceback.print_exc(); g_oLock.release(); return None;
def errorTimeout(sText): """ Flags the current test as having timed out and writes the specified message to the log. This will add an error to the current test. Always returns False for the convenience of methods returning boolean success indicators. """ g_oLock.acquire() g_oReporter.testSetTimedOut() try: g_oReporter.log(0, 'timeout-error: %s' % (sText), utils.getCallerName(), utils.getTimePrefix()) except: pass g_oLock.release() return False
def fatal(sText): """ Writes a fatal error to the log. This will add an error to the current test. Always returns False for the convenience of methods returning boolean success indicators. """ g_oLock.acquire() g_oReporter.testIncErrors() try: g_oReporter.log(0, 'fatal error: %s' % (sText), utils.getCallerName(), utils.getTimePrefix()) except: pass g_oLock.release() return False
def write(self, sText): """file.write""" if isinstance(sText, array.array): try: sText = sText.tostring() except: pass g_oLock.acquire() try: sTsPrf = utils.getTimePrefix() sCaller = utils.getCallerName() asLines = sText.splitlines() for sLine in asLines: g_oReporter.log(0, '%s: %s' % (self.sPrefix, sLine), sCaller, sTsPrf) except: traceback.print_exc() g_oLock.release() return None
def addLogFile(sFilename, sKind, sDescription = '', sAltName = None): """ Adds the specified log file to the report if the file exists. The sDescription is a free form description of the log file. The sKind parameter is for adding some machine parsable hint what kind of log file this really is. Returns True on success, False on failure (no ENOENT errors are logged). """ sTsPrf = utils.getTimePrefix(); sCaller = utils.getCallerName(); fRc = False; if sAltName is None: sAltName = sFilename; try: oSrcFile = utils.openNoInherit(sFilename, 'rb'); except IOError, oXcpt: if oXcpt.errno != errno.ENOENT: logXcpt('addLogFile(%s,%s,%s)' % (sFilename, sDescription, sKind)); else: logXcpt('addLogFile(%s,%s,%s) IOError' % (sFilename, sDescription, sKind));
def _processTmStatusResponse(self, oConn, sOperation, fClose=True): """ Processes HTTP reponse from the test manager. Returns True, False or None. None should be retried, the others not. May raise exception on HTTP issue (retry ok). """ import httplib from common import constants # Read the response and (optionally) close the connection. oResponse = oConn.getresponse() try: sRspBody = oResponse.read() except httplib.IncompleteRead, oXcpt: self._writeOutput( '%s: %s: Warning: httplib.IncompleteRead: %s [expected %s, got %s]' % ( utils.getTimePrefix(), sOperation, oXcpt, oXcpt.expected, len(oXcpt.partial), )) sRspBody = oXcpt.partial
def subXmlWrite(self, oFileWrapper, sRawXml, sCaller): """Called by the file wrapper write method for test pipes.""" return self.log(0, 'raw xml%s: %s' % (oFileWrapper.sPrefix, sRawXml), sCaller, utils.getTimePrefix());
def subXmlWrite(self, oFileWrapper, sRawXml, sCaller): """Called by the file wrapper write method for test pipes.""" return self.log(0, 'raw xml%s: %s' % (oFileWrapper.sPrefix, sRawXml), sCaller, utils.getTimePrefix())
class RemoteReporter(ReporterBase): """ Reporter that talks to the test manager server. """ ## The XML sync min time (seconds). kcSecXmlFlushMin = 30 ## The XML sync max time (seconds). kcSecXmlFlushMax = 120 ## The XML sync idle time before flushing (seconds). kcSecXmlFlushIdle = 5 ## The XML sync line count threshold. kcLinesXmlFlush = 512 ## The retry timeout. kcSecTestManagerRetryTimeout = 120 ## The request timeout. kcSecTestManagerRequestTimeout = 30 def __init__(self): ReporterBase.__init__(self) self.sTestManagerUrl = os.environ.get('TESTBOX_MANAGER_URL') self.sTestBoxUuid = os.environ.get('TESTBOX_UUID') self.idTestBox = int(os.environ.get('TESTBOX_ID')) self.idTestSet = int(os.environ.get('TESTBOX_TEST_SET_ID')) self._asXml = [] self._secTsXmlFlush = utils.timestampSecond() self._secTsXmlLast = self._secTsXmlFlush self._fXmlFlushing = False self.oOutput = sys.stdout # Hack for __del__ output. self.fFlushEachLine = True self.fDebugXml = 'TESTDRIVER_REPORTER_DEBUG_XML' in os.environ # Prepare the TM connecting. import urlparse import httplib import urllib from common import constants self._fnUrlEncode = urllib.urlencode self._fnUrlParseQs = urlparse.parse_qs self._oParsedTmUrl = urlparse.urlparse(self.sTestManagerUrl) if sys.version_info[0] >= 3 \ or (sys.version_info[0] == 2 and sys.version_info[1] >= 6): if self._oParsedTmUrl.scheme == 'https': # pylint: disable=E1101 self._fnTmConnect = lambda: httplib.HTTPSConnection( self._oParsedTmUrl.hostname, timeout=self.kcSecTestManagerRequestTimeout) else: self._fnTmConnect = lambda: httplib.HTTPConnection( self._oParsedTmUrl.hostname, timeout=self.kcSecTestManagerRequestTimeout) else: if self._oParsedTmUrl.scheme == 'https': # pylint: disable=E1101 self._fnTmConnect = lambda: httplib.HTTPSConnection( self._oParsedTmUrl.hostname) else: self._fnTmConnect = lambda: httplib.HTTPConnection( self._oParsedTmUrl.hostname) self._dHttpHeader = \ { 'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8', 'User-Agent': 'TestDriverReporter/%s.0 (%s, %s)' % (__version__, utils.getHostOs(), utils.getHostArch(),), 'Accept': 'text/plain,application/x-www-form-urlencoded', 'Accept-Encoding': 'identity', 'Cache-Control': 'max-age=0', #'Connection': 'keep-alive', } dParams = { constants.tbreq.ALL_PARAM_TESTBOX_UUID: self.sTestBoxUuid, constants.tbreq.ALL_PARAM_TESTBOX_ID: self.idTestBox, constants.tbreq.RESULT_PARAM_TEST_SET_ID: self.idTestSet, } self._sTmServerPath = '/%s/testboxdisp.py?%s' \ % ( self._oParsedTmUrl.path.strip('/'), # pylint: disable=E1101 urllib.urlencode(dParams), ) def __del__(self): """Flush pending log messages?""" if len(self._asXml) > 0: self._xmlDoFlush(self._asXml, fRetry=True, fDtor=True) def _writeOutput(self, sText): """ Does the actual writing and flushing. """ print >> self.oOutput, sText.encode('ascii', 'replace') if self.fFlushEachLine: self.oOutput.flush() return None # # Talking to TM. # def _processTmStatusResponse(self, oConn, sOperation, fClose=True): """ Processes HTTP reponse from the test manager. Returns True, False or None. None should be retried, the others not. May raise exception on HTTP issue (retry ok). """ import httplib from common import constants # Read the response and (optionally) close the connection. oResponse = oConn.getresponse() try: sRspBody = oResponse.read() except httplib.IncompleteRead, oXcpt: self._writeOutput( '%s: %s: Warning: httplib.IncompleteRead: %s [expected %s, got %s]' % ( utils.getTimePrefix(), sOperation, oXcpt, oXcpt.expected, len(oXcpt.partial), )) sRspBody = oXcpt.partial if fClose is True: try: oConn.close() except: pass # Check the content type. sContentType = oResponse.getheader('Content-Type') if sContentType is not None and sContentType == 'application/x-www-form-urlencoded; charset=utf-8': # Parse the body and check the RESULT parameter. dResponse = self._fnUrlParseQs(sRspBody, strict_parsing=True) sResult = dResponse.get(constants.tbresp.ALL_PARAM_RESULT, None) if isinstance(sResult, list): sResult = sResult[0] if len( sResult) == 1 else '%d results' % (len(sResult), ) if sResult is not None: if sResult == constants.tbresp.STATUS_ACK: return True if sResult == constants.tbresp.STATUS_NACK: self._writeOutput('%s: %s: Failed (%s). (dResponse=%s)' % ( utils.getTimePrefix(), sOperation, sResult, dResponse, )) return False self._writeOutput('%s: %s: Failed - dResponse=%s' % ( utils.getTimePrefix(), sOperation, dResponse, )) else: self._writeOutput('%s: %s: Unexpected Content-Type: %s' % ( utils.getTimePrefix(), sOperation, sContentType, )) self._writeOutput('%s: %s: Body: %s' % ( utils.getTimePrefix(), sOperation, sRspBody, )) return None
def logXcptWorker(iLevel, fIncErrors, sPrefix="", sText=None, cFrames=1): """ Log an exception, optionally with a preceeding message and more than one call frame. """ g_oLock.acquire() if fIncErrors: g_oReporter.testIncErrors() ## @todo skip all this if iLevel is too high! # Try get exception info. sTsPrf = utils.getTimePrefix() try: oType, oValue, oTraceback = sys.exc_info() except: oType = oValue = oTraceback = None if oType is not None: # Try format the info try: rc = 0 sCaller = utils.getCallerName(oTraceback.tb_frame) if sText is not None: rc = g_oReporter.log(iLevel, "%s%s" % (sPrefix, sText), sCaller, sTsPrf) asInfo = [] try: asInfo = asInfo + traceback.format_exception_only( oType, oValue) if cFrames is not None and cFrames <= 1: asInfo = asInfo + traceback.format_tb(oTraceback, 1) else: asInfo.append('Traceback:') asInfo = asInfo + traceback.format_tb(oTraceback, cFrames) asInfo.append('Stack:') asInfo = asInfo + traceback.format_stack( oTraceback.tb_frame.f_back, cFrames) except: g_oReporter.log( 0, 'internal-error: Hit exception #2! %s' % (traceback.format_exc()), sCaller, sTsPrf) if len(asInfo) > 0: # Do the logging. for sItem in asInfo: asLines = sItem.splitlines() for sLine in asLines: rc = g_oReporter.log(iLevel, '%s%s' % (sPrefix, sLine), sCaller, sTsPrf) else: g_oReporter.log(iLevel, 'No exception info...', sCaller, sTsPrf) rc = -3 except: g_oReporter.log( 0, 'internal-error: Hit exception! %s' % (traceback.format_exc()), None, sTsPrf) rc = -2 else: g_oReporter.log( 0, 'internal-error: No exception! %s' % (utils.getCallerName(iFrame=3)), utils.getCallerName(iFrame=3), sTsPrf) rc = -1 g_oLock.release() return rc