def __init__(self, dict = {}):
"""
"""
dict.setdefault('secure', False)
if not dict.has_key('endpoint'):
dict['endpoint'] = "%cmsweb.cern.ch/workqueue/" % \
((dict['secure'] and "https://" or "http://"))
if dict.has_key('cachepath'):
pass
elif os.getenv('WORKQUEUE_CACHE_DIR'):
dict['cachepath'] = os.getenv('WORKQUEUE_CACHE_DIR') + '/.workqueue_cache'
elif os.getenv('HOME'):
dict['cachepath'] = os.getenv('HOME') + '/.workqueue_cache'
else:
dict['cachepath'] = '/tmp/.workqueue_' + pwd.getpwuid(os.getuid())[0]
if not os.path.isdir(dict['cachepath']):
os.makedirs(dict['cachepath'])
if 'logger' not in dict.keys():
logging.basicConfig(level = logging.DEBUG,
format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt = '%m-%d %H:%M',
filename = dict['cachepath'] + '/jsonparser.log',
filemode = 'w')
dict['logger'] = logging.getLogger('WorkQueueParser')
dict.setdefault("accept_type", "application/json+thunker")
dict.setdefault("content_type", "application/json")
self.encoder = JsonWrapper.dumps
self.decoder = self.jsonThunkerDecoder
Service.__init__(self, dict)
def __init__(self, dict = {}, secure = False):
"""
responseType will be either xml or json
"""
if not dict.has_key('endpoint'):
#TODO needs to change proper default location
dict['endpoint'] = "%scmssrv49.fnal.gov:8585/reqMgr/" % \
((secure and "https://" or "http://"))
if dict.has_key('cachepath'):
pass
elif os.getenv('REQUESTMGR_CACHE_DIR'):
dict['cachepath'] = os.getenv('REQUESTMGR_CACHE_DIR') + '/.requestmgr_cache'
elif os.getenv('HOME'):
dict['cachepath'] = os.getenv('HOME') + '/.requestmgr_cache'
else:
dict['cachepath'] = '/tmp/.requestmgr_' + pwd.getpwuid(os.getuid())[0]
if not os.path.isdir(dict['cachepath']):
os.makedirs(dict['cachepath'])
if 'logger' not in dict.keys():
logging.basicConfig(level = logging.DEBUG,
format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt = '%m-%d %H:%M',
filename = dict['cachepath'] + '/jsonparser.log',
filemode = 'w')
dict['logger'] = logging.getLogger('RequestMgrParser')
dict['accept_type'] = 'text/json'
Service.__init__(self, dict)
def __init__(self, dict={}, responseType="xml"):
"""
responseType will be either xml or json
"""
self.responseType = responseType.lower()
#if self.responseType == 'json':
#self.parser = JSONParser()
#elif self.responseType == 'xml':
#self.parser = XMLParser()
if os.getenv('WMBS_SERV_CACHE_DIR'):
dict['cachepath'] = os.getenv('WMBS_SERV_CACHE_DIR') + '/.wmbs_service_cache'
elif os.getenv('HOME'):
dict['cachepath'] = os.getenv('HOME') + '/.wmbs_service_cache'
else:
dict['cachepath'] = '/tmp/wmbs_service_cache_' + pwd.getpwuid(os.getuid())[0]
if not os.path.isdir(dict['cachepath']):
os.mkdir(dict['cachepath'])
if 'logger' not in dict.keys():
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=dict['cachepath'] + '/wmbs_service.log',
filemode='w')
dict['logger'] = logging.getLogger('WMBSParser')
#TODO if service doesn't need to be authorized, have switch to use Service
Service.__init__(self, dict)
def setUp(self):
"""
Setup for unit tests
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
testname = self.id().split('.')[-1]
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='service_unittests.log',
filemode='w')
logger_name = 'Service%s' % testname.replace('test', '', 1)
self.logger = logging.getLogger(logger_name)
#self.cache_path = tempfile.mkdtemp()
test_dict = {'logger': self.logger,
#'cachepath' : self.cache_path,
#'req_cache_path': '%s/requests' % self.cache_path,
'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi'}
self.myService = Service(test_dict)
test_dict['endpoint'] = 'http://cmssw-test.cvs.cern.ch/cgi-bin/cmssw.cgi'
self.myService2 = Service(test_dict)
self.testUrl = 'http://cern.ch'
self.port = 8888
cherrypy.config.update({'server.socket_port': self.port})
def __init__(self, dict={}, responseType="json", secure=False):
"""
responseType will be either xml or json
"""
self.responseType = responseType.lower()
dict["timeout"] = 300
if not dict.has_key('endpoint'):
dict['endpoint'] = "%scmsweb.cern.ch/phedex/datasvc/%s/prod/" % \
((secure and "https://" or "http://"),
self.responseType)
if dict.has_key('cachepath'):
pass
elif os.getenv('CMS_PHEDEX_CACHE_DIR'):
dict['cachepath'] = os.getenv(
'CMS_PHEDEX_CACHE_DIR') + '/.cms_phedexcache'
elif os.getenv('HOME'):
dict['cachepath'] = os.getenv('HOME') + '/.cms_phedexcache'
else:
dict['cachepath'] = '/tmp/phedex_' + pwd.getpwuid(os.getuid())[0]
if not os.path.isdir(dict['cachepath']):
os.makedirs(dict['cachepath'])
if 'logger' not in dict.keys():
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=dict['cachepath'] + '/phedexdbjsonparser.log',
filemode='w')
dict['logger'] = logging.getLogger('PhEDExParser')
Service.__init__(self, dict)
def __init__(self, config={}):
config = dict(config) ### copy dict since mutables are shared between instances
config['endpoint'] = "https://cmsweb.cern.ch/sitedb/data/prod/"
config['accept_type'] = "application/json"
config['content_type'] = "application/json"
if os.getenv('CMS_SITEDB_CACHE_DIR'):
config['cachepath'] = os.getenv('CMS_SITEDB_CACHE_DIR') + '/.cms_sitedbcache'
elif os.getenv('HOME'):
config['cachepath'] = os.getenv('HOME') + '/.cms_sitedbcache'
else:
import pwd
config['cachepath'] = '/tmp/sitedbjson_' + pwd.getpwuid(os.getuid())[0]
if not os.path.isdir(config['cachepath']):
os.mkdir(config['cachepath'])
if 'logger' not in config.keys():
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=config['cachepath'] + '/sitedbjsonparser.log',
filemode='w')
config['logger'] = logging.getLogger('SiteDBParser')
Service.__init__(self, config)
def __init__(self, dict = {}, responseType = "json", secure = False):
"""
responseType will be either xml or json
"""
self.responseType = responseType.lower()
dict["timeout"] = 300
if not dict.has_key('endpoint'):
dict['endpoint'] = "%scmsweb.cern.ch/phedex/datasvc/%s/prod/" % \
((secure and "https://" or "http://"),
self.responseType)
if dict.has_key('cachepath'):
pass
elif os.getenv('CMS_PHEDEX_CACHE_DIR'):
dict['cachepath'] = os.getenv('CMS_PHEDEX_CACHE_DIR') + '/.cms_phedexcache'
elif os.getenv('HOME'):
dict['cachepath'] = os.getenv('HOME') + '/.cms_phedexcache'
else:
dict['cachepath'] = '/tmp/phedex_' + pwd.getpwuid(os.getuid())[0]
if not os.path.isdir(dict['cachepath']):
os.makedirs(dict['cachepath'])
if 'logger' not in dict.keys():
logging.basicConfig(level = logging.DEBUG,
format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt = '%m-%d %H:%M',
filename = dict['cachepath'] + '/phedexdbjsonparser.log',
filemode = 'w')
dict['logger'] = logging.getLogger('PhEDExParser')
Service.__init__(self, dict)
def __init__(self, config={}):
config = dict(
config) ### copy dict since mutables are shared between instances
config['endpoint'] = "https://cmsweb.cern.ch/sitedb/data/prod/"
config['accept_type'] = "application/json"
config['content_type'] = "application/json"
if os.getenv('CMS_SITEDB_CACHE_DIR'):
config['cachepath'] = os.getenv(
'CMS_SITEDB_CACHE_DIR') + '/.cms_sitedbcache'
elif os.getenv('HOME'):
config['cachepath'] = os.getenv('HOME') + '/.cms_sitedbcache'
else:
import pwd
config['cachepath'] = '/tmp/sitedbjson_' + pwd.getpwuid(
os.getuid())[0]
if not os.path.isdir(config['cachepath']):
os.mkdir(config['cachepath'])
if 'logger' not in config.keys():
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=config['cachepath'] + '/sitedbjsonparser.log',
filemode='w')
config['logger'] = logging.getLogger('SiteDBParser')
Service.__init__(self, config)
def __init__(self, dict={}):
"""
"""
dict.setdefault('secure', False)
if not dict.has_key('endpoint'):
dict['endpoint'] = "%cmsweb.cern.ch/workqueue/" % \
((dict['secure'] and "https://" or "http://"))
if dict.has_key('cachepath'):
pass
elif os.getenv('WORKQUEUE_CACHE_DIR'):
dict['cachepath'] = os.getenv(
'WORKQUEUE_CACHE_DIR') + '/.workqueue_cache'
elif os.getenv('HOME'):
dict['cachepath'] = os.getenv('HOME') + '/.workqueue_cache'
else:
dict['cachepath'] = '/tmp/.workqueue_' + pwd.getpwuid(
os.getuid())[0]
if not os.path.isdir(dict['cachepath']):
os.makedirs(dict['cachepath'])
if 'logger' not in dict.keys():
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=dict['cachepath'] + '/jsonparser.log',
filemode='w')
dict['logger'] = logging.getLogger('WorkQueueParser')
dict.setdefault("accept_type", "application/json+thunker")
dict.setdefault("content_type", "application/json")
self.encoder = JsonWrapper.dumps
self.decoder = self.jsonThunkerDecoder
Service.__init__(self, dict)
def setUp(self):
"""
Setup for unit tests
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
testname = self.id().split('.')[-1]
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='service_unittests.log',
filemode='w')
logger_name = 'Service%s' % testname.replace('test', '', 1)
self.logger = logging.getLogger(logger_name)
# self.cache_path = tempfile.mkdtemp()
test_dict = {
'logger': self.logger,
'endpoint': 'https://github.com/dmwm'
}
self.myService = Service(test_dict)
test_dict[
'endpoint'] = 'http://cmssw-test.cvs.cern.ch/cgi-bin/cmssw.cgi'
self.myService2 = Service(test_dict)
self.testUrl = 'http://cern.ch'
self.port = 8888
cherrypy.config.update({'server.socket_port': self.port})
def __init__(self, dict={}, secure=False):
"""
responseType will be either xml or json
"""
if not dict.has_key('endpoint'):
#TODO needs to change proper default location
dict['endpoint'] = "%scmssrv49.fnal.gov:8585/reqMgr/" % \
((secure and "https://" or "http://"))
if dict.has_key('cachepath'):
pass
elif os.getenv('REQUESTMGR_CACHE_DIR'):
dict['cachepath'] = os.getenv(
'REQUESTMGR_CACHE_DIR') + '/.requestmgr_cache'
elif os.getenv('HOME'):
dict['cachepath'] = os.getenv('HOME') + '/.requestmgr_cache'
else:
dict['cachepath'] = '/tmp/.requestmgr_' + pwd.getpwuid(
os.getuid())[0]
if not os.path.isdir(dict['cachepath']):
os.makedirs(dict['cachepath'])
if 'logger' not in dict.keys():
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename=dict['cachepath'] + '/jsonparser.log',
filemode='w')
dict['logger'] = logging.getLogger('RequestMgrParser')
dict['accept_type'] = 'text/json'
Service.__init__(self, dict)
def testStaleCache(self):
myConfig = {'logger': self.logger,
'endpoint': 'https://github.com/dmwm',
'usestalecache': True,
}
service = Service(myConfig)
service.getData('%s/socketresettest' % self.testDir, '/WMCore/blob/master/setup.py#L11')
self.assertEqual(service['usestalecache'], myConfig['usestalecache'])
def __init__(self, dict={}):
dict['endpoint'] = dict.get('endpoint', 'https://cmsweb.cern.ch/crabcache/')
Service.__init__(self, dict)
if dict.has_key('proxyfilename'):
#in case there is some code I have not updated in ticket #3780. Should not be required... but...
self['logger'].warning('The UserFileCache proxyfilename parameter has been replace with the more'
' general (ckey/cert) pair.')
def __init__(self, dict={}):
try:
Service.__init__(self, dict)
self["requests"] = SSLRequests(self["requests"]["host"])
except WMException, ex:
msg = str(ex)
self["logger"].exception(msg)
raise WMException(msg)
def testSocketTimeout(self):
myConfig = {'logger': self.logger,
'endpoint': 'https://github.com/dmwm',
'cacheduration': None,
'timeout': 10,
}
service = Service(myConfig)
service.getData('%s/socketresettest' % self.testDir, '/WMCore/blob/master/setup.py#L11')
self.assertEqual(service['timeout'], myConfig['timeout'])
def __init__(self, dict={}):
try:
Service.__init__(self, dict)
self["requests"] = SSLRequests(self["requests"]["host"])
except WMException, ex:
msg = str(ex)
self["logger"].exception(msg)
raise WMException(msg)
def testZ_InterruptedConnection(self):
"""
_InterruptedConnection_
What happens if we shut down the server while
the connection is still active?
Confirm that the cache works as expected
"""
cherrypy.tree.mount(RegularServer(), "/reg1")
cherrypy.engine.start()
FORMAT = '%(message)s'
logging.basicConfig(format=FORMAT)
logger = logging.getLogger('john')
test_dict = {'logger': self.logger,'endpoint':'http://localhost:%i/reg1/regular' % self.port,
'usestalecache': False, "cacheduration": 0.005}
myService = Service(test_dict)
self.assertRaises(HTTPException, myService.getData, 'foo', 'THISISABADURL')
data = myService.refreshCache('foo', '')
dataString = data.read()
self.assertEqual(dataString, "This is silly.")
data.close()
# Now stop the server and confirm that it is down
cherrypy.server.stop()
self.assertRaises(socket.error, myService.forceRefresh, 'foo', '')
# Make sure we can still read from the cache
data = myService.refreshCache('foo', '')
dataString = data.read()
self.assertEqual(dataString, "This is silly.")
data.close()
# Mount a backup server
del cherrypy.tree.apps['/reg1']
cherrypy.tree.mount(BackupServer(), "/reg1")
# Expire cache
time.sleep(30)
self.assertRaises(socket.error, myService.forceRefresh, 'foo', '')
# Restart server
cherrypy.server.start()
# Confirm new server is in place
data = myService.refreshCache('foo', '')
dataString = data.read()
self.assertEqual(dataString, "This is nuts.")
data.close()
cherrypy.engine.exit()
cherrypy.engine.stop()
return
def __init__(self, url=None):
"""
responseType will be either xml or json
"""
defaultURL = "https://cmssdt.cern.ch/SDT/cgi-bin/ReleasesXML?anytype=1"
params = {}
params["timeout"] = 300
params['endpoint'] = url or defaultURL
params.setdefault('cacheduration', 3600)
Service.__init__(self, params)
def __init__(self, dict={}):
dict["endpoint"] = dict.get("endpoint", "https://cmsweb.cern.ch/crabcache/")
Service.__init__(self, dict)
if dict.has_key("proxyfilename"):
# in case there is some code I have not updated in ticket #3780. Should not be required... but...
self["logger"].warning(
"The UserFileCache proxyfilename parameter has been replace with the more"
+ " general (ckey/cert) pair."
)
def __init__(self, dict={}):
dict["endpoint"] = dict.get("endpoint", "https://cmsweb.cern.ch/crabcache/")
Service.__init__(self, dict)
self["requests"]["accept_type"] = "application/json"
if "proxyfilename" in dict:
# in case there is some code I have not updated in ticket #3780. Should not be required... but...
self["logger"].warning(
"The UserFileCache proxyfilename parameter has been replace with the more" " general (ckey/cert) pair."
)
def testSocketTimeout(self):
dict = {'logger': self.logger,
'endpoint': 'https://github.com/dmwm',
'cacheduration': None,
'timeout': 10,
}
service = Service(dict)
deftimeout = socket.getdefaulttimeout()
service.getData('%s/socketresettest' % self.testDir, '/WMCore/blob/master/setup.py#L11')
assert deftimeout == socket.getdefaulttimeout()
def __init__(self, url, logger=None):
params = {}
params['endpoint'] = url
params['cacheduration'] = 0
params['accept_type'] = 'application/json'
params['content_type'] = 'application/json'
params['method'] = 'GET'
params['logger'] = logger if logger else logging.getLogger()
Service.__init__(self, params)
def testSocketTimeout(self):
dict = {'logger': self.logger,
'endpoint': 'https://github.com/dmwm',
'cacheduration': None,
'timeout': 10,
}
service = Service(dict)
deftimeout = socket.getdefaulttimeout()
service.getData('%s/socketresettest' % self.testDir, '/WMCore/blob/master/setup.py#L11')
assert deftimeout == socket.getdefaulttimeout()
def testStaleCache(self):
myConfig = {
'logger': self.logger,
'endpoint': 'https://github.com/dmwm',
'usestalecache': True,
}
service = Service(myConfig)
service.getData('%s/socketresettest' % self.testDir,
'/WMCore/blob/master/setup.py#L11')
self.assertEqual(service['usestalecache'], myConfig['usestalecache'])
def __init__(self, url, logger=None):
params = {}
params['endpoint'] = url
params['cacheduration'] = 0
params['accept_type'] = 'application/json'
params['content_type'] = 'application/json'
params['method'] = 'GET'
params['logger'] = logger if logger else logging.getLogger()
Service.__init__(self, params)
def testStaleCache(self):
dict = {
'logger': self.logger,
'endpoint': 'http://cmssw.cvs.cern.ch',
'cacheduration': 0.0002,
'maxcachereuse': 0.001,
'timeout': 10,
'usestalecache': True,
#'cachepath' : self.cache_path,
#'req_cache_path': '%s/requests' % self.cache_path
}
service = Service(dict)
cache = 'stalecachetest'
# Start test from a clear cache
service.clearCache(cache)
cachefile = service.cacheFileName(cache)
# first check that the exception raises when the file doesn't exist
self.logger.info('first call to refreshCache - should fail')
self.assertRaises(HTTPException, service.refreshCache, cache, '/lies')
cacheddata = 'this data is mouldy'
f = open(cachefile, 'w')
f.write(cacheddata)
f.close()
self.logger.info('second call to refreshCache - should pass')
data = service.refreshCache(cache, '/lies').read()
self.assertEquals(cacheddata, data)
# sleep a while so the file expires in the cache
# FIXME: RACY
time.sleep(2)
self.logger.info(
'third call to refreshCache - should return stale cache')
data = service.refreshCache(cache, '/lies').read()
self.assertEquals(cacheddata, data)
# sleep a while longer so the cache is dead
# FIXME: RACY
time.sleep(5)
self.logger.info('fourth call to refreshCache - cache should be dead')
self.assertRaises(HTTPException, service.refreshCache, cache, '/lies')
# touch the file and expire it
f = open(cachefile, 'w')
f.write('foo')
f.close()
time.sleep(2)
self.logger.info('fifth call to refreshCache - do not use stale cache')
# now our service cache is less permissive, the following should fail
service['usestalecache'] = False
self.assertRaises(HTTPException, service.refreshCache, cache, '/lies')
service.cacheFileName(cache)
def __init__(self, mydict=None):
if mydict==None: #dangerous {} default value
mydict = {}
mydict['endpoint'] = mydict.get('endpoint', 'https://cmsweb.cern.ch/crabcache/')
Service.__init__(self, mydict)
self['requests']['accept_type'] = 'application/json'
if 'proxyfilename' in mydict:
#in case there is some code I have not updated in ticket #3780. Should not be required... but...
self['logger'].warning('The UserFileCache proxyfilename parameter has been replace with the more'
' general (ckey/cert) pair.')
def __init__(self, config={}, logger=None):
config = dict(config)
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv(
"WMCORE_USE_CRIC", False):
# just to make sure we don't use SiteDB anywhere when CRIC flag is true
raise RuntimeError("Stop using SiteDB, use CRIC instead!")
else:
config.setdefault('endpoint',
"https://cmsweb.cern.ch/sitedb/data/prod/")
config.setdefault('logger', logging.getLogger())
Service.__init__(self, config)
def __init__(self, dict={}):
"""
responseType will be JSON
"""
dict.setdefault("accept_type", "application/json")
dict.setdefault("content_type", "application/json")
self.encoder = JsonWrapper.dumps
self.decoder = JsonWrapper.loads
Service.__init__(self, dict)
def __init__(self, dict={}):
"""
responseType will be either xml or json
"""
dict.setdefault("accept_type", "application/json")
dict.setdefault("content_type", "application/json")
self.encoder = JsonWrapper.dumps
self.decoder = JsonWrapper.loads
Service.__init__(self, dict)
def __init__(self, dict={}):
"""
responseType will be JSON
"""
dict.setdefault("accept_type", "application/json")
dict.setdefault("content_type", "application/json")
self.encoder = json.dumps
self.decoder = json.loads
Service.__init__(self, dict)
def __init__(self, url=None, logger=None):
params = {}
defaultURL = "https://cms-cric.cern.ch/"
params['endpoint'] = url or defaultURL
params.setdefault('cacheduration', 3600)
params.setdefault('accept_type', 'application/json')
params.setdefault('content_type', 'application/json')
params['logger'] = logger if logger else logging.getLogger()
Service.__init__(self, params)
self['logger'].info("DEBUG: Initializing CRIC with url: %s",
self['endpoint'])
def testSocketTimeout(self):
myConfig = {
'logger': self.logger,
'endpoint': 'https://github.com/dmwm',
'cacheduration': None,
'timeout': 10,
}
service = Service(myConfig)
service.getData('%s/socketresettest' % self.testDir,
'/WMCore/blob/master/setup.py#L11')
self.assertEqual(service['timeout'], myConfig['timeout'])
def __init__(self, dict={}):
"""
responseType will be either xml or json
"""
dict.setdefault("accept_type", "application/json")
dict.setdefault("content_type", "application/json")
self.encoder = json.dumps
self.decoder = json.loads
Service.__init__(self, dict)
def testSocketTimeout(self):
dict = {'logger': self.logger,
'endpoint':'http://cmssw.cvs.cern.ch/',
'cacheduration': None,
'timeout': 10,
#'cachepath' : self.cache_path,
#'req_cache_path': '%s/requests' % self.cache_path
}
service = Service(dict)
deftimeout = socket.getdefaulttimeout()
service.getData('%s/socketresettest' % self.testDir, '/cgi-bin/cmssw.cgi')
assert deftimeout == socket.getdefaulttimeout()
def __init__(self, mydict=None):
if mydict == None: #dangerous {} default value
mydict = {}
mydict['endpoint'] = mydict.get('endpoint',
'https://cmsweb.cern.ch/crabcache/')
Service.__init__(self, mydict)
self['requests']['accept_type'] = 'application/json'
if 'proxyfilename' in mydict:
#in case there is some code I have not updated in ticket #3780. Should not be required... but...
self['logger'].warning(
'The UserFileCache proxyfilename parameter has been replace with the more'
' general (ckey/cert) pair.')
def testNoCache(self):
"""Cache disabled"""
dict = {'logger': self.logger,
'endpoint': 'https://github.com/dmwm',
'cachepath': None,
}
service = Service(dict)
self.assertEqual(service['cachepath'], dict['cachepath'])
self.assertEqual(service['requests']['cachepath'], dict['cachepath'])
self.assertEqual(service['requests']['req_cache_path'], dict['cachepath'])
out = service.refreshCache('shouldntbeused', '/').read()
self.assertTrue('html' in out)
def testNoCache(self):
"""Cache disabled"""
dict = {'logger': self.logger,
'endpoint': 'https://github.com/dmwm',
'cachepath': None,
}
service = Service(dict)
self.assertEqual(service['cachepath'], dict['cachepath'])
self.assertEqual(service['requests']['cachepath'], dict['cachepath'])
self.assertEqual(service['requests']['req_cache_path'], dict['cachepath'])
out = service.refreshCache('shouldntbeused', '/').read()
self.assertTrue('html' in out)
def testSocketTimeout(self):
dict = {
'logger': self.logger,
'endpoint': 'http://cmssw.cvs.cern.ch/',
'cacheduration': None,
'timeout': 10,
#'cachepath' : self.cache_path,
#'req_cache_path': '%s/requests' % self.cache_path
}
service = Service(dict)
deftimeout = socket.getdefaulttimeout()
service.getData('%s/socketresettest' % self.testDir,
'/cgi-bin/cmssw.cgi')
assert deftimeout == socket.getdefaulttimeout()
def __init__(self, httpDict=None, responseType="json", logger=None):
"""
responseType will be either xml or json
"""
httpDict = httpDict or {}
self.responseType = responseType.lower()
httpDict['logger'] = logger if logger else logging.getLogger()
httpDict["timeout"] = 300
if 'endpoint' not in httpDict:
httpDict['endpoint'] = "https://cmsweb.cern.ch/phedex/datasvc/%s/prod/" % self.responseType
httpDict.setdefault('cacheduration', 0)
Service.__init__(self, httpDict)
def __init__(self, dict = {}, secure = False):
"""
responseType will be either xml or json
"""
if not dict.has_key('endpoint'):
#TODO needs to change proper default location
dict['endpoint'] = "%scmssrv49.fnal.gov:8585/reqMgr/" % \
((secure and "https://" or "http://"))
dict['accept_type'] = 'text/json'
dict.setdefault('cacheduration', 0)
Service.__init__(self, dict)
def __init__(self, dict = None, responseType = "json", secure = True):
"""
responseType will be either xml or json
"""
if not dict:
dict = {}
self.responseType = responseType.lower()
dict["timeout"] = 300
if 'endpoint' not in dict:
dict['endpoint'] = "https://cmsweb.cern.ch/phedex/datasvc/%s/prod/" % self.responseType
dict.setdefault('cacheduration', 0)
Service.__init__(self, dict)
def __init__(self, dict=None):
if not dict:
dict = {}
if not dict.has_key('endpoint'):
dict['endpoint'] = "http://cms-xen38.fnal.gov:7725/userfilecache/"
if not dict.has_key('proxyfilename'):
dict['proxyfilename'] = None
if not dict.has_key('capath'):
dict['capath'] = None
#TODO: Temporary flag used to indicate that the UserFileCache is talking with the new REST
#Remove when the branch 3.0.x of the CRABClient is deprecated
if not dict.has_key('newrest'):
dict['newrest'] = False
Service.__init__(self, dict)
def testStaleCache(self):
dict = {'logger': self.logger,
'endpoint':'http://cmssw.cvs.cern.ch',
'cacheduration': 0.0002,
'maxcachereuse': 0.001,
'timeout': 10,
'usestalecache': True,
#'cachepath' : self.cache_path,
#'req_cache_path': '%s/requests' % self.cache_path
}
service = Service(dict)
cache = 'stalecachetest'
# Start test from a clear cache
service.clearCache(cache)
cachefile = service.cacheFileName(cache)
# first check that the exception raises when the file doesn't exist
self.logger.info('first call to refreshCache - should fail')
self.assertRaises(HTTPException, service.refreshCache, cache, '/lies')
cacheddata = 'this data is mouldy'
f = open(cachefile, 'w')
f.write(cacheddata)
f.close()
self.logger.info('second call to refreshCache - should pass')
data = service.refreshCache(cache, '/lies').read()
self.assertEquals(cacheddata, data)
# sleep a while so the file expires in the cache
# FIXME: RACY
time.sleep(2)
self.logger.info('third call to refreshCache - should return stale cache')
data = service.refreshCache(cache, '/lies').read()
self.assertEquals(cacheddata, data)
# sleep a while longer so the cache is dead
# FIXME: RACY
time.sleep(5)
self.logger.info('fourth call to refreshCache - cache should be dead')
self.assertRaises(HTTPException, service.refreshCache, cache, '/lies')
# touch the file and expire it
f = open(cachefile, 'w')
f.write('foo')
f.close()
time.sleep(2)
self.logger.info('fifth call to refreshCache - do not use stale cache')
# now our service cache is less permissive, the following should fail
service['usestalecache'] = False
self.assertRaises(HTTPException, service.refreshCache, cache, '/lies')
service.cacheFileName(cache)
def __init__(self, httpDict=None, responseType="json", logger=None,
dbsUrl='https://cmsweb.cern.ch/dbs/prod/global/DBSReader'):
"""
responseType will be either xml or json
"""
self.dbsUrl = dbsUrl
httpDict = httpDict or {}
self.responseType = responseType.lower()
httpDict['logger'] = logger if logger else logging.getLogger()
httpDict["timeout"] = 300
if 'endpoint' not in httpDict:
httpDict['endpoint'] = "https://cmsweb.cern.ch/phedex/datasvc/%s/prod/" % self.responseType
httpDict.setdefault('cacheduration', 0)
Service.__init__(self, httpDict)
def __init__(self, config):
TrustStore.__init__(self, config)
defaultdict = {'endpoint': self.store,
'cacheduration': config.duration,
'cachepath': config.path,
'method': 'GET'}
logging.basicConfig(level = logging.DEBUG,
format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt = '%m-%d %H:%M',
filename = defaultdict['cachepath'] + '/regsvc.log',
filemode = 'w')
defaultdict['logger'] = logging.getLogger('OIDRegSvcTrustStore')
self.svc = Service(defaultdict)
def __init__(self, dict={}):
"""
responseType will be either xml or json
"""
#if self.responseType == 'json':
#self.parser = JSONParser()
#elif self.responseType == 'xml':
#self.parser = XMLParser()
dict.setdefault("accept_type", "application/json")
dict.setdefault("content_type", "application/json")
self.encoder = JsonWrapper.dumps
self.decoder = JsonWrapper.loads
Service.__init__(self, dict)
def __init__(self, dict):
defaultdict = {'endpoint': "https://cmsweb.cern.ch/registration/",
'cacheduration': 1,
'cachepath': '/tmp'}
defaultdict.update(dict)
defaultdict["method"] = 'PUT'
if 'logger' not in defaultdict.keys():
logging.basicConfig(level = logging.DEBUG,
format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt = '%m-%d %H:%M',
filename = defaultdict['cachepath'] + '/regsvc.log',
filemode = 'w')
defaultdict['logger'] = logging.getLogger('RegService')
Service.__init__(self, defaultdict)
JSONRequests.__init__(self, defaultdict['endpoint'])
def __init__(self, url, header={}):
"""
responseType will be either xml or json
"""
httpDict = {}
# url is end point
httpDict['endpoint'] = "%s/data" % url
# cherrypy converts request.body to params when content type is set
# application/x-www-form-urlencodeds
httpDict.setdefault("content_type", 'application/json')
httpDict.setdefault('cacheduration', 0)
httpDict.setdefault("accept_type", "application/json")
httpDict.update(header)
self.encoder = json.dumps
Service.__init__(self, httpDict)
def __init__(self, url=None, logger=None, **kwargs):
"""
responseType will be either xml or json
"""
defaultURL = "https://cmssdt.cern.ch/SDT/cgi-bin/ReleasesXML"
# all releases types and all their archs
self.tcArgs = kwargs
self.tcArgs.setdefault("anytype", 1)
self.tcArgs.setdefault("anyarch", 1)
params = {}
params["timeout"] = 300
params['endpoint'] = url or defaultURL
params.setdefault('cacheduration', 3600)
params['logger'] = logger if logger else logging.getLogger()
Service.__init__(self, params)
def __init__(self, dict = {}, secure = False):
"""
responseType will be either xml or json
"""
if not dict.has_key('endpoint'):
#TODO needs to change proper default location
dict['endpoint'] = "%scmssrv49.fnal.gov:8585/reqMgr/" % \
((secure and "https://" or "http://"))
dict.setdefault("accept_type", "application/json")
# cherrypy converts request.body to params when content type is set
# application/x-www-form-urlencodeds
dict.setdefault("content_type", 'application/x-www-form-urlencoded')
dict.setdefault('cacheduration', 0)
Service.__init__(self, dict)
def testCacheDuration(self):
myConfig = {
'logger': self.logger,
'endpoint': 'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi',
'cacheduration': 100
}
service = Service(myConfig)
self.assertEqual(service['cacheduration'], myConfig['cacheduration'])
def __init__(self, url, header = {}):
"""
responseType will be either xml or json
"""
httpDict = {}
# url is end point
httpDict['endpoint'] = "%s/data" % url
# cherrypy converts request.body to params when content type is set
# application/x-www-form-urlencodeds
httpDict.setdefault("content_type", 'application/json')
httpDict.setdefault('cacheduration', 0)
httpDict.setdefault("accept_type", "application/json")
httpDict.update(header)
self.encoder = JsonWrapper.dumps
Service.__init__(self, httpDict)
def testCacheLifetime(self):
"""Cache deleted if created by Service - else left alone"""
dict = {'logger': self.logger,
'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi',
'cacheduration': 100}
os.environ.pop('TMPDIR', None) # Mac sets this by default
service = Service(dict)
cache_path = service['cachepath']
self.assertTrue(os.path.isdir(cache_path))
del service
self.assertFalse(os.path.exists(cache_path))
cache_path = tempfile.mkdtemp()
dict['cachepath'] = cache_path
service = Service(dict)
del service
self.assertTrue(os.path.isdir(cache_path))
Permissions.owner_readwriteexec(cache_path)
def testNoCacheDuration(self):
dict = {'logger': self.logger,
'endpoint':'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi',
'cacheduration': None,
#'cachepath' : self.cache_path,
#'req_cache_path': '%s/requests' % self.cache_path
}
service = Service(dict)
self.assertEqual( service['cacheduration'] , dict['cacheduration'] )
def testCacheLifetime(self):
"""Cache deleted if created by Service - else left alone"""
myConfig = {
'logger': self.logger,
'endpoint': 'http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi',
'cacheduration': 24
}
os.environ.pop('TMPDIR', None) # Mac sets this by default
service = Service(myConfig)
cache_path = service['cachepath']
self.assertTrue(os.path.isdir(cache_path))
del service
self.assertFalse(os.path.exists(cache_path))
cache_path = tempfile.mkdtemp()
myConfig['cachepath'] = cache_path
service = Service(myConfig)
del service
self.assertTrue(os.path.isdir(cache_path))