def setUp(self): """ Setup for unit tests """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() testname = self.id().split('.')[-1] logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', datefmt='%m-%d %H:%M', filename='service_unittests.log', filemode='w') logger_name = 'Service%s' % testname.replace('test', '', 1) self.logger = logging.getLogger(logger_name) # self.cache_path = tempfile.mkdtemp() test_dict = { 'logger': self.logger, 'endpoint': 'https://github.com/dmwm' } self.myService = Service(test_dict) test_dict[ 'endpoint'] = 'http://cmssw-test.cvs.cern.ch/cgi-bin/cmssw.cgi' self.myService2 = Service(test_dict) self.testUrl = 'http://cern.ch' self.port = 8888 cherrypy.config.update({'server.socket_port': self.port})
def testStaleCache(self): dict = { 'logger': self.logger, 'endpoint': 'http://cmssw.cvs.cern.ch', 'cacheduration': 0.0002, 'maxcachereuse': 0.001, 'timeout': 10, 'usestalecache': True, #'cachepath' : self.cache_path, #'req_cache_path': '%s/requests' % self.cache_path } service = Service(dict) cache = 'stalecachetest' # Start test from a clear cache service.clearCache(cache) cachefile = service.cacheFileName(cache) # first check that the exception raises when the file doesn't exist self.logger.info('first call to refreshCache - should fail') self.assertRaises(HTTPException, service.refreshCache, cache, '/lies') cacheddata = 'this data is mouldy' f = open(cachefile, 'w') f.write(cacheddata) f.close() self.logger.info('second call to refreshCache - should pass') data = service.refreshCache(cache, '/lies').read() self.assertEquals(cacheddata, data) # sleep a while so the file expires in the cache # FIXME: RACY time.sleep(2) self.logger.info( 'third call to refreshCache - should return stale cache') data = service.refreshCache(cache, '/lies').read() self.assertEquals(cacheddata, data) # sleep a while longer so the cache is dead # FIXME: RACY time.sleep(5) self.logger.info('fourth call to refreshCache - cache should be dead') self.assertRaises(HTTPException, service.refreshCache, cache, '/lies') # touch the file and expire it f = open(cachefile, 'w') f.write('foo') f.close() time.sleep(2) self.logger.info('fifth call to refreshCache - do not use stale cache') # now our service cache is less permissive, the following should fail service['usestalecache'] = False self.assertRaises(HTTPException, service.refreshCache, cache, '/lies') service.cacheFileName(cache)
def testCacheDuration(self): myConfig = { 'logger': self.logger, 'endpoint': 'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi', 'cacheduration': 100 } service = Service(myConfig) self.assertEqual(service['cacheduration'], myConfig['cacheduration'])
def testCacheLifetime(self): """Cache deleted if created by Service - else left alone""" dict = {'logger': self.logger, 'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi', 'cacheduration': 100} os.environ.pop('TMPDIR', None) # Mac sets this by default service = Service(dict) cache_path = service['cachepath'] self.assertTrue(os.path.isdir(cache_path)) del service self.assertFalse(os.path.exists(cache_path)) cache_path = tempfile.mkdtemp() dict['cachepath'] = cache_path service = Service(dict) del service self.assertTrue(os.path.isdir(cache_path)) Permissions.owner_readwriteexec(cache_path)
def testNoCacheDuration(self): dict = {'logger': self.logger, 'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi', 'cacheduration': None, #'cachepath' : self.cache_path, #'req_cache_path': '%s/requests' % self.cache_path } service = Service(dict) self.assertEqual( service['cacheduration'] , dict['cacheduration'] )
def testStaleCache(self): myConfig = { 'logger': self.logger, 'endpoint': 'https://github.com/dmwm', 'usestalecache': True, } service = Service(myConfig) service.getData('%s/socketresettest' % self.testDir, '/WMCore/blob/master/setup.py#L11') self.assertEqual(service['usestalecache'], myConfig['usestalecache'])
def testCacheLifetime(self): """Cache deleted if created by Service - else left alone""" myConfig = { 'logger': self.logger, 'endpoint': 'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi', 'cacheduration': 24 } os.environ.pop('TMPDIR', None) # Mac sets this by default service = Service(myConfig) cache_path = service['cachepath'] self.assertTrue(os.path.isdir(cache_path)) del service self.assertFalse(os.path.exists(cache_path)) cache_path = tempfile.mkdtemp() myConfig['cachepath'] = cache_path service = Service(myConfig) del service self.assertTrue(os.path.isdir(cache_path))
def testSocketTimeout(self): dict = {'logger': self.logger, 'endpoint': 'https://github.com/dmwm', 'cacheduration': None, 'timeout': 10, } service = Service(dict) deftimeout = socket.getdefaulttimeout() service.getData('%s/socketresettest' % self.testDir, '/WMCore/blob/master/setup.py#L11') assert deftimeout == socket.getdefaulttimeout()
def testSocketTimeout(self): myConfig = { 'logger': self.logger, 'endpoint': 'https://github.com/dmwm', 'cacheduration': None, 'timeout': 10, } service = Service(myConfig) service.getData('%s/socketresettest' % self.testDir, '/WMCore/blob/master/setup.py#L11') self.assertEqual(service['timeout'], myConfig['timeout'])
def testCachePath(self): cache_path = tempfile.mkdtemp() dict = {'logger': self.logger, 'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi', 'cachepath' : cache_path, 'req_cache_path': '%s/requests' % cache_path } service = Service(dict) # We append hostname to the cachepath, so that we can talk to two # services on different hosts self.assertEqual(service['cachepath'], '%s/cmssw.cvs.cern.ch' % dict['cachepath'] ) shutil.rmtree(cache_path, ignore_errors = True)
def testSocketTimeout(self): dict = { 'logger': self.logger, 'endpoint': 'http://cmssw.cvs.cern.ch/', 'cacheduration': None, 'timeout': 10, #'cachepath' : self.cache_path, #'req_cache_path': '%s/requests' % self.cache_path } service = Service(dict) deftimeout = socket.getdefaulttimeout() service.getData('%s/socketresettest' % self.testDir, '/cgi-bin/cmssw.cgi') assert deftimeout == socket.getdefaulttimeout()
def testNoCache(self): """Cache disabled""" dict = {'logger': self.logger, 'endpoint': 'https://github.com/dmwm', 'cachepath': None, } service = Service(dict) self.assertEqual(service['cachepath'], dict['cachepath']) self.assertEqual(service['requests']['cachepath'], dict['cachepath']) self.assertEqual(service['requests']['req_cache_path'], dict['cachepath']) out = service.refreshCache('shouldntbeused', '/').read() self.assertTrue('html' in out)
def testBadStatusLine(self): """ _BadStatusLine_ """ FORMAT = '%(message)s' logging.basicConfig(format=FORMAT) dummyLogger = logging.getLogger('john') test_dict = {'logger': self.logger,'endpoint':'http://127.0.0.1:%i/badstatus' % self.port, 'usestalecache': True} myService = Service(test_dict) # Have to fudge the status line in the Request object as cherrypy won't # Allow bad statuses to be raised myService['requests'] = CrappyRequest('http://bad.com', {}) self.assertRaises(BadStatusLine, myService.getData, 'foo', '')
def __init__(self, config): TrustStore.__init__(self, config) defaultdict = {'endpoint': self.store, 'cacheduration': config.duration, 'cachepath': config.path, 'method': 'GET'} logging.basicConfig(level = logging.DEBUG, format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s', datefmt = '%m-%d %H:%M', filename = defaultdict['cachepath'] + '/regsvc.log', filemode = 'w') defaultdict['logger'] = logging.getLogger('OIDRegSvcTrustStore') self.svc = Service(defaultdict)
def testTruncatedResponse(self): """ _TruncatedResponse_ """ cherrypy.tree.mount(CrappyServer()) cherrypy.engine.start() FORMAT = '%(message)s' logging.basicConfig(format=FORMAT) dummyLogger = logging.getLogger('john') test_dict = {'logger': self.logger,'endpoint':'http://127.0.0.1:%i/truncated' % self.port, 'usestalecache': True} myService = Service(test_dict) self.assertRaises(IncompleteRead, myService.getData, 'foo', '') cherrypy.engine.exit() cherrypy.engine.stop()
def testSlowResponse(self): """ _SlowResponse_ """ cherrypy.tree.mount(SlowServer()) cherrypy.engine.start() FORMAT = '%(message)s' logging.basicConfig(format=FORMAT) dummyLogger = logging.getLogger('john') test_dict = {'logger': self.logger,'endpoint':'http://127.0.0.1:%i/slow' % self.port, 'usestalecache': True} myService = Service(test_dict) startTime = int(time.time()) self.assertRaises(socket.timeout, myService.getData, 'foo', '') self.assertTrue(int(time.time()) - startTime < 130, "Error: Timeout took too long") cherrypy.engine.exit() cherrypy.engine.stop()
def notestZ_InterruptedConnection(self): """ _InterruptedConnection_ What happens if we shut down the server while the connection is still active? Confirm that the cache works as expected """ cherrypy.tree.mount(RegularServer(), "/reg1") cherrypy.engine.start() FORMAT = '%(message)s' logging.basicConfig(format=FORMAT) dummyLogger = logging.getLogger('john') test_dict = { 'logger': self.logger, 'endpoint': 'http://127.0.0.1:%i/reg1/regular' % self.port, 'usestalecache': True, "cacheduration": 0.005 } myService = Service(test_dict) self.assertRaises(HTTPException, myService.getData, 'foo', 'THISISABADURL') data = myService.refreshCache('foo', '') dataString = data.read() self.assertEqual(dataString, "This is silly.") data.close() # Now stop the server and confirm that it is down cherrypy.server.stop() self.assertRaises(socket.error, myService.forceRefresh, 'foo', '') # Make sure we can still read from the cache data = myService.refreshCache('foo', '') dataString = data.read() self.assertEqual(dataString, "This is silly.") data.close() # Mount a backup server del cherrypy.tree.apps['/reg1'] cherrypy.tree.mount(BackupServer(), "/reg1") # Expire cache time.sleep(30) self.assertRaises(socket.error, myService.forceRefresh, 'foo', '') # get expired cache results while the server is down data = myService.refreshCache('foo', '') dataString = data.read() self.assertEqual(dataString, "This is silly.") data.close() # Restart server cherrypy.server.start() # Confirm new server is in place data = myService.refreshCache('foo', '') dataString = data.read() self.assertEqual(dataString, "This is nuts.") data.close() cherrypy.engine.exit() cherrypy.engine.stop() return
def testUsingStaleCache(self): myConfig = { 'logger': self.logger, 'endpoint': 'https://cmssdt.cern.ch/SDT/', 'cacheduration': 0.0005, # cache file lasts 1.8 secs 'timeout': 10, 'usestalecache': True, # 'cachepath' : self.cache_path, # 'req_cache_path': '%s/requests' % self.cache_path } service = Service(myConfig) cache = 'stalecachetest' # Start test from a clear cache service.clearCache(cache) cachefile = service.cacheFileName(cache) self.logger.info( '1st call to refreshCache - should fail, there is no cache file') self.assertRaises(HTTPException, service.refreshCache, cache, '/lies') cacheddata = 'this data is mouldy' with open(cachefile, 'w') as f: f.write(cacheddata) self.logger.info( '2nd call to refreshCache - should pass, data comes from the valid cache' ) data = service.refreshCache(cache, '/lies').read() self.assertEqual(cacheddata, data) # power nap to avoid letting the cache expire time.sleep(1) self.logger.info( '3rd call to refreshCache - should pass, cache is still valid') data = service.refreshCache(cache, '/lies').read() self.assertEqual(cacheddata, data) # sleep a while longer so the cache dies out time.sleep(2) self.logger.info( '4th call to refreshCache - should fail, cache is dead now') self.assertRaises(HTTPException, service.refreshCache, cache, '/lies') # touch/renew the file again cacheddata = 'foo' with open(cachefile, 'w') as f: f.write(cacheddata) # disable usage of stale cache, so doesn't call the endpoint if cache is valid service['usestalecache'] = False self.logger.info( '5th call to refreshCache - should pass, cache is still valid') data = service.refreshCache(cache, '/lies').read() self.assertEqual(cacheddata, data) # consider the cache dead service['cacheduration'] = 0 time.sleep(1) self.logger.info( '6th call to refreshCache - should fail, cache is dead now') self.assertRaises(HTTPException, service.refreshCache, cache, '/lies')