def __init__(self, config=None):
self.base = os.environ.get('WEB_BASE', '') # defines base path for HREF in templates
if config and not isinstance(config, dict):
config = config.dictionary_()
if not config:
config = {'base': self.base}
print "\n### Configuration:"
pprint.pprint(config)
TemplatedPage.__init__(self, config)
imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd()+'/images')
self.imgdir = config.get('imgdir', imgdir)
cssdir = os.environ.get('RM_CSSPATH', os.getcwd()+'/css')
self.cssdir = config.get('cssdir', cssdir)
jsdir = os.environ.get('RM_JSPATH', os.getcwd()+'/js')
self.jsdir = config.get('jsdir', jsdir)
yuidir = os.environ.get('YUI_ROOT', os.getcwd()+'/yui')
self.yuidir = config.get('yuidir', yuidir)
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
})
self._cache = {}
def css(self, *args, **kwargs):
"""
Cat together the specified css files and return a single css include.
Get css by calling: /controllers/css/file1/file2/file3
"""
mime_types = ['text/css']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
})
args = list(args)
scripts = self.check_scripts(args, self.cssmap)
idx = "-".join(scripts)
if idx not in self.cache.keys():
data = '@CHARSET "UTF-8";'
for script in args:
if self.cssmap.has_key(script):
path = os.path.join(sys.path[0], self.cssmap[script])
path = os.path.normpath(path)
ifile = open(path)
data = "\n".join ([data, ifile.read().\
replace('@CHARSET "UTF-8";', '')])
ifile.close()
set_headers ("text/css")
self.cache[idx] = minify(data)
return self.cache[idx]
def __init__(self, config):
TemplatedPage.__init__(self, config)
self.base = '' # defines base path for HREF in templates
self.imgdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'images')
if not os.path.isdir(self.imgdir):
self.imgdir = os.environ['DAS_IMAGESPATH']
self.cssdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'css')
if not os.path.isdir(self.cssdir):
self.cssdir = os.environ['DAS_CSSPATH']
self.jsdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'js')
if not os.path.isdir(self.jsdir):
self.jsdir = os.environ['DAS_JSPATH']
if not os.environ.has_key('YUI_ROOT'):
msg = 'YUI_ROOT is not set in environment'
raise Exception(msg)
self.yuidir = os.environ['YUI_ROOT']
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
})
self._cache = {}
def __init__(self, config):
TemplatedPage.__init__(self, config)
self.base = '' # defines base path for HREF in templates
self.imgdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'images')
if not os.path.isdir(self.imgdir):
self.imgdir = os.environ['DAS_IMAGESPATH']
self.cssdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'css')
if not os.path.isdir(self.cssdir):
self.cssdir = os.environ['DAS_CSSPATH']
self.jsdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'js')
if not os.path.isdir(self.jsdir):
self.jsdir = os.environ['DAS_JSPATH']
if 'YUI_ROOT' not in os.environ:
msg = 'YUI_ROOT is not set in environment'
raise Exception(msg)
self.yuidir = os.environ['YUI_ROOT']
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
})
self._cache = {}
def main():
conf = {
'/': {
'tools.sessions.on': True,
'tools.staticdir.root': path.abspath(getcwd())
},
'/static': {
'tools.staticdir.on': True,
'tools.staticdir.dir': './public'
}
}
config.update({
'tools.response_headers.on':
True,
'tools.response_headers.headers':
[('Access-Control-Allow-Origin', '*')],
'server.socket_host':
ip,
'server.socket_port':
port
})
quickstart(ResumeServer(), '/', conf)
def configure(self):
"""Configure server, CherryPy and the rest."""
config = self.config['web_server']
cpconfig["engine.autoreload_on"] = False
cpconfig["server.environment"] = config.get("environment", "production")
cpconfig["server.thread_pool"] = int(config.get("thread_pool", 30))
cpconfig["server.socket_port"] = int(config.get("port", 8080))
self.pid = config.get('pid', '/tmp/das_web.pid')
cpconfig["server.socket_host"] = config.get("host", "0.0.0.0")
cpconfig["server.socket_queue_size"] = \
int(config.get("socket_queue_size", 100))
cpconfig["tools.expires.secs"] = int(config.get("expires", 300))
cpconfig["log.screen"] = bool(config.get("log_screen", True))
cpconfig["log.access_file"] = config.get("access_log_file", None)
cpconfig["log.error_file"] = config.get("error_log_file", None)
cpconfig['request.show_tracebacks'] = False
log.error_log.setLevel(config.get("error_log_level", logging.DEBUG))
log.access_log.setLevel(config.get("access_log_level", logging.DEBUG))
cpconfig.update ({
'tools.expires.on': True,
'tools.response_headers.on':True,
'tools.etags.on':True,
'tools.etags.autotags':True,
'tools.encode.on': True,
'tools.proxy.on': True,
'tools.gzip.on': True,
})
log("loading config: %s" % cpconfig,
context='web',
severity=logging.DEBUG,
traceback=False)
def js(self, *args, **kwargs):
"""
Cat together the specified js files and return a single js include.
Get js by calling: /controllers/js/file1/file2/file3
"""
mime_types = ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
'tools.encode.on': True,
})
args = list(args)
scripts = self.check_scripts(args, self.jsmap)
idx = "-".join(scripts)
if idx not in self.cache.keys():
data = ''
for script in args:
path = os.path.join(sys.path[0], self.jsmap[script])
path = os.path.normpath(path)
ifile = open(path)
data = "\n".join ([data, ifile.read()])
ifile.close()
self.cache[idx] = data
return self.cache[idx]
def configure(self):
"""Configure server, CherryPy and the rest."""
config = self.config['web_server']
cpconfig["engine.autoreload_on"] = False
cpconfig["engine.timeout_monitor.on"] = config.get("timeout_monitor", False)
cpconfig["server.environment"] = config.get("environment", "production")
cpconfig["server.thread_pool"] = int(config.get("thread_pool", 30))
cpconfig["server.socket_port"] = int(config.get("port", 8080))
self.pid = config.get('pid', '/tmp/das_web.pid')
cpconfig["server.socket_host"] = config.get("host", "0.0.0.0")
cpconfig["server.socket_queue_size"] = \
int(config.get("socket_queue_size", 100))
cpconfig["tools.expires.secs"] = int(config.get("expires", 300))
cpconfig["tools.sessions.timeout"] = int(config.get("session_timeout", 60))
cpconfig["log.screen"] = bool(config.get("log_screen", True))
cpconfig["log.access_file"] = config.get("access_log_file", None)
cpconfig["log.error_file"] = config.get("error_log_file", None)
cpconfig['request.show_tracebacks'] = False
log.error_log.setLevel(config.get("error_log_level", logging.DEBUG))
log.access_log.setLevel(config.get("access_log_level", logging.DEBUG))
# SecurityModule config
# Registers secmodv2 into cherrypy.tools so it can be used through
# decorators
class SecConfig(object):
pass
security = self.config['security']
secconfig = SecConfig()
secsection = ConfigSection('security')
for key, val in security.items():
setattr(secconfig, key, val)
if security.get('module', '') == '':
print("### DAS behind NullAuth, should NOT be used in production")
cherrypy.tools.secmodv2 = NullAuth(secconfig)
else:
print("### DAS behind FrontEndAuth")
cherrypy.tools.secmodv2 = FrontEndAuth(secconfig)
cherrypy.config.update({'tools.secmodv2.on': True,
'tools.secmodv2.role': security.get('role', ''),
'tools.secmodv2.group': security.get('group' ''),
'tools.secmodv2.site': security.get('site', '')})
cpconfig.update ({\
'tools.expires.on': True,\
'tools.response_headers.on':True,\
'tools.etags.on':True,\
'tools.etags.autotags':True,\
'tools.encode.on': True,\
'tools.proxy.on': True,\
'tools.gzip.on': True,\
})
log("loading config: %s" % cpconfig,\
context='web',\
severity=logging.DEBUG,\
traceback=False)
def __init__(self, config):
TemplatedPage.__init__(self, config)
dbs = config.section_('dbs')
phedex = config.section_('phedex')
dbsconfig = {'dbs':dbs.url, 'dbsinst':dbs.instance,
'dbsparams':dbs.params, 'phedex':phedex.url}
self.dbs = DBS(dbsconfig)
self.securityApi = ""
self.fmConfig = config.section_('fmws')
self.verbose = self.fmConfig.verbose
self.day_transfer = self.fmConfig.day_transfer
self.max_transfer = self.fmConfig.max_transfer
self.file_manager = config.section_('file_manager')
self.transfer_dir = self.file_manager.base_directory
self.download_dir = self.fmConfig.download_area
self.fmgr = FileManager()
self.fmgr.configure(fm_config(config))
self.voms_timer = 0
self.userDict = {}
self.userDictPerDay = {}
self.url = "/filemover"
# prevent users from partial retrieval requests
cherrypy.response.headers['Accept-Ranges'] = 'none'
# internal settings
self.base = '' # defines base path for HREF in templates
self.imgdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'images')
if not os.path.isdir(self.imgdir):
self.imgdir = os.environ['FM_IMAGESPATH']
self.cssdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'css')
if not os.path.isdir(self.cssdir):
self.cssdir = os.environ['FM_CSSPATH']
self.jsdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'js')
if not os.path.isdir(self.jsdir):
self.jsdir = os.environ['FM_JSPATH']
if not os.environ.has_key('YUI_ROOT'):
msg = 'YUI_ROOT is not set'
raise Exception(msg)
self.yuidir = os.environ['YUI_ROOT']
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
self.cache = {}
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
'tools.etags.on' : False,
})
def update_config(configfile=None, modulename=None):
"""Updates the system configuration either from a ConfigObj
(INI-style) config file, a module name specified in dotted notation
or both (the module name is assumed to have a ".cfg" extension).
If both are specified, the module is called first,
followed by the config file. This means that the config file's options
override the options in the module file."""
configdict = config_obj(configfile, modulename).dict()
configure_loggers(configdict)
config.update(configdict)
def update_config(configfile=None, modulename=None):
"""Updates the system configuration either from a ConfigObj
(INI-style) config file, a module name specified in dotted notation
or both (the module name is assumed to have a ".cfg" extension).
If both are specified, the module is called first,
followed by the config file. This means that the config file's options
override the options in the module file."""
configdict = config_obj(configfile, modulename).dict()
configure_loggers(configdict)
config.update(configdict)
def main():
"""docstring for main"""
config.update("config.ini")
time_interval = config.get('processor.timeout')
db = create_engine("mysql://%s:%s@localhost/%s" % (authwebdb.AUTHWEB_USER, authwebdb.AUTHWEB_PASSWD, authwebdb.AUTHWEB_DB))
db.echo = False
metadata = MetaData(db)
auth_app_table = Table('auth_app', metadata, autoload=True)
mapper(authwebdb.AuthApp, auth_app_table)
auth_query_table = Table('auth_query', metadata, autoload=True)
mapper(authwebdb.AuthQuery, auth_query_table, order_by=auth_query_table.created_time)
session = create_session()
query_auth_app = session.query(authwebdb.AuthApp)
authapp = query_auth_app.selectfirst()
cur_id = authapp.current_query_id
if cur_id != None:
query_auth_query = session.query(authwebdb.AuthQuery)
current_query = query_auth_query.get_by(query_id = cur_id)
if current_query.state == authwebdb.QUERY_STATE_PROCESSING:
conn = MySQLdb.connect(user=authwebdb.AUTHWEB_USER, passwd=authwebdb.AUTHWEB_PASSWD, db=cur_id)
cursor = conn.cursor()
query_string = "select count(*) from article;"
cursor.execute(query_string)
first_num = int(cursor.fetchone()[0])
cursor.close()
time.sleep(time_interval)
conn = MySQLdb.connect(user=authwebdb.AUTHWEB_USER, passwd=authwebdb.AUTHWEB_PASSWD, db=cur_id)
cursor = conn.cursor()
query_string = "select count(*) from article;"
cursor.execute(query_string)
second_num = int(cursor.fetchone()[0])
cursor.close()
conn.close()
if first_num == second_num:
print "Restarting process."
os.system("cd %s ; %s > /dev/null 2>&1 &" % (os.getcwd(), COMMAND))
else:
time.sleep(time_interval)
else:
time.sleep(time_interval)
def run_server(app):
app_logged = TransLogger(application=app)
tree.graft(app_logged, '/')
config.update({
'engine.autoreload.on': True,
'log.screen': True,
'server.socket_port': 5432,
'server.socket_host': '0.0.0.0'
})
engine.start()
engine.block()
def css(self, *args, **kwargs):
"""
cat together the specified css files and return a single css include
get css by calling: /controllers/css/file1/file2/file3
"""
mimetypes = ['text/css']
cherryconf.update({'tools.encode.on': True, 'tools.gzip.on': True})
data = self.return_cache_response(args, kwargs, 'css')
setHeaders(mimetypes, len(data))
return data
def css(self, *args, **kwargs):
"""
cat together the specified css files and return a single css include
get css by calling: /controllers/css/file1/file2/file3
"""
mimetypes = ['text/css']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True
})
data = self.return_cache_response(args, kwargs, 'css')
setHeaders(mimetypes, len(data))
return data
def js(self, *args, **kwargs):
"""
cat together the specified js files and return a single js include
get js by calling: /controllers/js/file1/file2/file3
"""
mimetypes = ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.gzip.on': True,
'tools.encode.on': True,
})
data = self.return_cache_response(args, kwargs, 'js')
setHeaders(mimetypes, len(data))
return data
def js(self, *args, **kwargs):
"""
cat together the specified js files and return a single js include
get js by calling: /controllers/js/file1/file2/file3
"""
mimetypes = [
'application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript'
]
cherryconf.update({
'tools.gzip.on': True,
'tools.encode.on': True,
})
data = self.return_cache_response(args, kwargs, 'js')
setHeaders(mimetypes, len(data))
return data
def update_config(configfile=None, modulename=None):
"""Update the system configuration from given config file and/or module.
'configfile' is a ConfigObj (INI-style) config file, 'modulename' a module
path in dotted notation. The function looks for files with a ".cfg"
extension if the given module name refers to a package directory or a file
with the base name of the right-most part of the module path and a ".cfg"
extension added.
If both 'configfile' and 'modulname' are specified, the module is read
first, followed by the config file. This means that the config file's
options override the options in the module file.
"""
configdict = config_obj(configfile, modulename).dict()
configure_loggers(configdict)
config.update(configdict)
def __init__(self, dburi, dbname='mongodb', dbcoll='stats',
size=10*1024*1024, interval=5):
self.dburi = dburi
self.conn = Connection(dburi)
if dbname not in self.conn.database_names():
dbptr = self.conn[dbname]
dbptr.create_collection(dbcoll, {'capped':True, 'size':size})
self.coll = self.conn[dbname][dbcoll]
self.attr = []
for key, val in self.conn[dbname].command( { "serverStatus" : 1 } ).items():
if isinstance(val, dict):
for akey, aval in val.items():
if isinstance(aval, dict):
for kkk, vvv in aval.items():
if not isinstance(vvv, dict):
self.attr.append('%s.%s.%s' % (key, akey, kkk))
else:
self.attr.append('%s.%s' % (key, akey))
# start thread with db_updater
thread.start_new_thread(db_updater, (dburi, dbname, dbcoll, interval))
# setup js/css dirs
self.jsdir = '%s/%s' % (__file__.rsplit('/', 1)[0], 'js')
if os.environ.has_key('JSPATH'):
self.jsdir = os.environ['JSPATH']
if not os.path.isdir(self.jsdir):
raise Exception('JS path is not set')
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
'request.show_tracebacks': False,
'server.environment':'production',
'server.socket_host':'0.0.0.0',
})
self._cache = {}
def __init__(self, app, config, mount):
self.base = config.base
self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1])
if config and not isinstance(config, dict):
web_config = config.dictionary_()
if not config:
web_config = {'base': self.base}
TemplatedPage.__init__(self, web_config)
imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images')
self.imgdir = web_config.get('imgdir', imgdir)
cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css')
self.cssdir = web_config.get('cssdir', cssdir)
jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js')
self.jsdir = web_config.get('jsdir', jsdir)
spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs')
self.spdir = web_config.get('spdir', spdir)
# read scripts area and initialize data-ops scripts
self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts')
self.sdir = web_config.get('sdir', self.sdir)
self.sdict_thr = web_config.get('sdict_thr', 600) # put reasonable 10 min interval
self.sdict = {'ts': time.time()} # placeholder for data-ops scripts
self.update_scripts(force=True)
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs')
self.std_specs = spec_list(std_specs_dir)
self.std_specs.sort()
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
})
self._cache = {}
# initialize access to reqmgr2 APIs
self.reqmgr_url = config.reqmgr.reqmgr2_url
self.reqmgr = ReqMgr(self.reqmgr_url)
# only gets current view (This might cause to reponse time much longer,
# If upto date view is not needed overwrite Fale)
self.reqmgr._noStale = True
# get fields which we'll use in templates
cdict = config.reqmgr.dictionary_()
self.couch_url = cdict.get('couch_host', '')
self.couch_dbname = cdict.get('couch_reqmgr_db', '')
self.couch_wdbname = cdict.get('couch_workload_summary_db', '')
self.acdc_url = cdict.get('acdc_host', '')
self.acdc_dbname = cdict.get('acdc_db', '')
self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url)
self.dbs_url = cdict.get('dbs_url', '')
self.dqm_url = cdict.get('dqm_url', '')
self.sw_ver = cdict.get('default_sw_version', 'CMSSW_7_6_1')
self.sw_arch = cdict.get('default_sw_scramarch', 'slc6_amd64_gcc493')
# LogDB holder
centralurl = cdict.get("central_logdb_url", "")
identifier = cdict.get("log_reporter", "reqmgr2")
self.logdb = LogDB(centralurl, identifier)
# local team cache which will request data from wmstats
base, uri = self.reqmgr_url.split('://')
base_url = '%s://%s' % (base, uri.split('/')[0])
self.wmstatsurl = cdict.get('wmstats_url', '%s/wmstatsserver' % base_url)
if not self.wmstatsurl:
raise Exception('ReqMgr2 configuration file does not provide wmstats url')
self.team_cache = []
# fetch assignment arguments specification from StdBase
self.assignArgs = StdBase().getWorkloadAssignArgs()
self.assignArgs = {key: val['default'] for key, val in self.assignArgs.items()}
def anonymous():
urls = database.scope.get_anonymous_urls()
if urls:
urls.add('/oauth/token')
if request.path_info in urls:
return 'anonymous'
config.update({
'tools.sessions.on': True,
'tools.session_auth.on': True,
'tools.session_auth.check_username_and_password':check,
'tools.session_auth.anonymous':anonymous,
})
@expose
@tools.response_headers(headers = [('Content-Type', 'image/svg')])
def configure(self):
"""Configure server, CherryPy and the rest."""
try:
cpconfig.update ({"server.environment": self.config['environment']})
except:
cpconfig.update ({"server.environment": 'production'})
try:
cpconfig.update ({"server.thread_pool": self.config['thread_pool']})
except:
cpconfig.update ({"server.thread_pool": 30})
try:
cpconfig.update ({"server.socket_queue_size": \
self.config['socket_queue_size']})
except:
cpconfig.update ({"server.socket_queue_size": 15})
try:
cpconfig.update ({"server.socket_port": int(self.config['port'])})
except:
cpconfig.update ({"server.socket_port": 8080})
try:
cpconfig.update ({"server.socket_host": self.config['host']})
except:
cpconfig.update ({"server.socket_host": '0.0.0.0'})
try:
cpconfig.update ({'tools.expires.secs':
int(self.config['expires'])})
except:
cpconfig.update ({'tools.expires.secs': 300})
try:
cpconfig.update ({'tools.staticdir.on': True})
cpconfig.update ({'tools.staticdir.dir': self.config['doc_dir']})
except:
cpconfig.update ({'tools.staticdir.on': False})
try:
cpconfig.update ({'log.screen':
bool(self.config['log_screen'])})
except:
cpconfig.update ({'log.screen': True})
try:
cpconfig.update ({'log.access_file':
self.config['access_log_file']})
except:
cpconfig.update ({'log.access_file': None})
try:
cpconfig.update ({'log.error_file':
self.config['error_log_file']})
except:
cpconfig.update ({'log.error_file': None})
try:
log.error_log.setLevel(self.config['error_log_level'])
except:
log.error_log.setLevel(logging.DEBUG)
try:
log.access_log.setLevel(self.config['access_log_level'])
except:
log.access_log.setLevel(logging.DEBUG)
cpconfig.update ({
'tools.expires.on': True,
'tools.response_headers.on':True,
'tools.etags.on':True,
'tools.etags.autotags':True,
'tools.encode.on': True,
'tools.gzip.on': True
})
#cpconfig.update ({'request.show_tracebacks': False})
#cpconfig.update ({'request.error_response': self.handle_error})
#cpconfig.update ({'tools.proxy.on': True})
#cpconfig.update ({'proxy.tool.base': '%s:%s'
# % (socket.gethostname(), opts.port)})
log("loading config: %s" % cpconfig,
context=self.app,
severity=logging.DEBUG,
traceback=False)
from cherrypy import wsgiserver, config
from main import app
import sys
dispatcher = wsgiserver.WSGIPathInfoDispatcher({'/': app})
server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 5000), dispatcher)
config.update({'log.screen': True})
if __name__ == '__main__':
try:
if sys.argv[1] == 'debug':
app.debug = True
except IndexError:
pass
try:
server.start()
except KeyboardInterrupt:
server.stop()
def __init__(self, app, config, mount):
self.base = config.base
self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1])
if config and not isinstance(config, dict):
web_config = config.dictionary_()
if not config:
web_config = {'base': self.base}
TemplatedPage.__init__(self, web_config)
imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images')
self.imgdir = web_config.get('imgdir', imgdir)
cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css')
self.cssdir = web_config.get('cssdir', cssdir)
jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js')
self.jsdir = web_config.get('jsdir', jsdir)
spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs')
self.spdir = web_config.get('spdir', spdir)
# read scripts area and initialize data-ops scripts
self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts')
self.sdir = web_config.get('sdir', self.sdir)
self.sdict_thr = web_config.get('sdict_thr',
600) # put reasonable 10 min interval
self.sdict = {'ts': time.time()} # placeholder for data-ops scripts
self.update_scripts(force=True)
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs')
self.std_specs = spec_list(std_specs_dir)
self.std_specs.sort()
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += [
'application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript'
]
cherryconf.update({
'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
})
self._cache = {}
# initialize access to reqmgr2 APIs
self.reqmgr_url = config.reqmgr.reqmgr2_url
self.reqmgr = ReqMgr(self.reqmgr_url)
# only gets current view (This might cause to reponse time much longer,
# If upto date view is not needed overwrite Fale)
self.reqmgr._noStale = True
# get fields which we'll use in templates
cdict = config.reqmgr.dictionary_()
self.couch_url = cdict.get('couch_host', '')
self.couch_dbname = cdict.get('couch_reqmgr_db', '')
self.couch_wdbname = cdict.get('couch_workload_summary_db', '')
self.acdc_url = cdict.get('acdc_host', '')
self.acdc_dbname = cdict.get('acdc_db', '')
self.configcache_url = cdict.get('couch_config_cache_url',
self.couch_url)
self.dbs_url = cdict.get('dbs_url', '')
self.dqm_url = cdict.get('dqm_url', '')
self.sw_ver = cdict.get('default_sw_version', 'CMSSW_7_6_1')
self.sw_arch = cdict.get('default_sw_scramarch', 'slc6_amd64_gcc493')
# LogDB holder
centralurl = cdict.get("central_logdb_url", "")
identifier = cdict.get("log_reporter", "reqmgr2")
self.logdb = LogDB(centralurl, identifier)
# local team cache which will request data from wmstats
base, uri = self.reqmgr_url.split('://')
base_url = '%s://%s' % (base, uri.split('/')[0])
self.wmstatsurl = cdict.get('wmstats_url',
'%s/wmstatsserver' % base_url)
if not self.wmstatsurl:
raise Exception(
'ReqMgr2 configuration file does not provide wmstats url')
self.team_cache = []
# fetch assignment arguments specification from StdBase
self.assignArgs = StdBase().getWorkloadAssignArgs()
self.assignArgs = {
key: val['default']
for key, val in self.assignArgs.items()
}
import sys, os
from cherrypy import (config as server_conf,
quickstart)
from cide.index import Welcome
from pdb import set_trace as dbg
#static_base_path = os.path.dirname(os.path.abspath(os.getcwd()))
#conf = dict()
#conf = {
# '/': {
# 'tools.sessions.on': True,
# 'tools.staticdir.root': static_base_path,
# 'tools.sessions.locking': 'explicit'
# },
# '/static': {
# 'tools.staticdir.on': True,
# 'tools.staticdir.dir': '..:/public'
# }
#}
conf = sys.argv[1]
server_conf.update(conf)
quickstart(Welcome(), "/", conf)
def start(self) -> None:
config.update({
'server.socket_host': environ['WEB_HOST'],
'server.socket_port': int(environ['WEB_PORT'])
})
quickstart(self)
def update(configvalues):
"""Updates the configuration with the values from the dictionary."""
return config.update(configvalues)
def configure(self):
"""Configure server, CherryPy and the rest."""
config = self.config['web_server']
cpconfig["engine.autoreload_on"] = False
cpconfig["engine.timeout_monitor.on"] = config.get(
"timeout_monitor", False)
cpconfig["server.environment"] = config.get("environment",
"production")
cpconfig["server.thread_pool"] = int(config.get("thread_pool", 30))
cpconfig["server.socket_port"] = int(config.get("port", 8080))
self.pid = config.get('pid', '/tmp/das_web.pid')
cpconfig["server.socket_host"] = config.get("host", "0.0.0.0")
cpconfig["server.socket_queue_size"] = \
int(config.get("socket_queue_size", 100))
cpconfig["tools.expires.secs"] = int(config.get("expires", 300))
cpconfig["tools.sessions.timeout"] = int(
config.get("session_timeout", 60))
cpconfig["log.screen"] = bool(config.get("log_screen", True))
cpconfig["log.access_file"] = config.get("access_log_file", None)
cpconfig["log.error_file"] = config.get("error_log_file", None)
cpconfig['request.show_tracebacks'] = False
log.error_log.setLevel(config.get("error_log_level", logging.DEBUG))
log.access_log.setLevel(config.get("access_log_level", logging.DEBUG))
# SecurityModule config
# Registers secmodv2 into cherrypy.tools so it can be used through
# decorators
class SecConfig(object):
pass
security = self.config['security']
secconfig = SecConfig()
try:
secsection = ConfigSection('security')
except NameError:
pass
for key, val in security.items():
setattr(secconfig, key, val)
if security.get('module', '') == '':
cherrypy.tools.secmodv2 = NullAuth(secconfig)
print("### DAS behind NullAuth, should NOT be used in production")
else:
try:
cherrypy.tools.secmodv2 = FrontEndAuth(secconfig)
print("### DAS behind FrontEndAuth")
except ImportError:
cherrypy.tools.secmodv2 = NullAuth(secconfig)
print(
"### DAS behind NullAuth, should NOT be used in production"
)
cherrypy.config.update({
'tools.secmodv2.on':
True,
'tools.secmodv2.role':
security.get('role', ''),
'tools.secmodv2.group':
security.get('group'
''),
'tools.secmodv2.site':
security.get('site', '')
})
cpconfig.update ({\
'tools.expires.on': True,\
'tools.response_headers.on':True,\
'tools.etags.on':True,\
'tools.etags.autotags':True,\
'tools.encode.on': True,\
'tools.proxy.on': True,\
'tools.gzip.on': True,\
})
log("loading config: %s" % cpconfig,\
context='web',\
severity=logging.DEBUG,\
traceback=False)
def __init__(self, app, config, mount):
self.base = config.base
self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1])
if config and not isinstance(config, dict):
web_config = config.dictionary_()
if not config:
web_config = {'base': self.base}
TemplatedPage.__init__(self, web_config)
imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd()+'/images')
self.imgdir = web_config.get('imgdir', imgdir)
cssdir = os.environ.get('RM_CSSPATH', os.getcwd()+'/css')
self.cssdir = web_config.get('cssdir', cssdir)
jsdir = os.environ.get('RM_JSPATH', os.getcwd()+'/js')
self.jsdir = web_config.get('jsdir', jsdir)
spdir = os.environ.get('RM_SPECPATH', os.getcwd()+'/specs')
self.spdir = web_config.get('spdir', spdir)
# read scripts area and initialize data-ops scripts
self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd()+'/scripts')
self.sdir = web_config.get('sdir', self.sdir)
self.sdict_thr = web_config.get('sdict_thr', 600) # put reasonable 10 min interval
self.sdict = {'ts':time.time()} # placeholder for data-ops scripts
self.update_scripts(force=True)
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs')
self.std_specs = spec_list(std_specs_dir, 'WMSpec.StdSpecs')
self.std_specs.sort()
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += ['application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript']
cherryconf.update({'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
})
self._cache = {}
# initialize rest API
statedir = '/tmp'
app = RESTMain(config, statedir) # REST application
mount = '/rest' # mount point for cherrypy service
api = RestApiHub(app, config.reqmgr, mount)
# initialize access to reqmgr2 APIs
self.reqmgr = ReqMgr(config.reqmgr.reqmgr2_url)
# only gets current view (This might cause to reponse time much longer,
# If upto date view is not needed overwrite Fale)
self.reqmgr._noStale = True
# admin helpers
self.admin_info = Info(app, api, config.reqmgr, mount=mount+'/info')
self.admin_group = Group(app, api, config.reqmgr, mount=mount+'/group')
self.admin_team = Team(app, api, config.reqmgr, mount=mount+'/team')
# get fields which we'll use in templates
cdict = config.reqmgr.dictionary_()
self.couch_url = cdict.get('couch_host', '')
self.couch_dbname = cdict.get('couch_reqmgr_db', '')
self.couch_wdbname = cdict.get('couch_workload_summary_db', '')
self.acdc_url = cdict.get('acdc_host', '')
self.acdc_dbname = cdict.get('acdc_db', '')
self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url)
self.dbs_url = cdict.get('dbs_url', '')
self.dqm_url = cdict.get('dqm_url', '')
self.sw_ver = cdict.get('default_sw_version', 'CMSSW_5_2_5')
self.sw_arch = cdict.get('default_sw_scramarch', 'slc5_amd64_gcc434')
def bind(self):
CherryPyConfig.update({'server.socket_host': self._config.Interface,
'server.socket_port': self._config.Port,
})
def update(configvalues):
"""Updates the configuration with the values from the dictionary."""
return config.update(configvalues)
#group_name = ddf.group.unique()[0]
#ddf_js = ddf.head().to_json(orient='values')
#whole = json.loads(json.dumps(ddf.to_dict('records')[:300]).encode('utf-8').decode('utf-8', errors='ignore'))
whole = ddf[:300].to_json(orient='records')
obj = {
'rubric_id': rubric_id,
'rubric_name': rubric_name,
'group_name': group_name,
'done_count': len(done),
'total_count': total_raw,
'likes_max': max_likes,
'likes_mean': mean_likes,
#'xars_list': xars,
'whole_data': whole
}
return json.dumps(obj).encode('utf-8')
if __name__ == '__main__':
df = pd.read_pickle('df_indexed.pickle')
#print(len(df))
config.update({
'tools.staticdir.index': "index_polarity.html",
'tools.staticdir.dir': os.getcwd(),
'tools.staticdir.on': True,
'server.socket_host': '0.0.0.0'
})
quickstart(Characterize())
def __init__(self, app, config, mount):
self.base = config.base
self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1])
if config and not isinstance(config, dict):
web_config = config.dictionary_()
if not config:
web_config = {'base': self.base}
TemplatedPage.__init__(self, web_config)
imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images')
self.imgdir = web_config.get('imgdir', imgdir)
cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css')
self.cssdir = web_config.get('cssdir', cssdir)
jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js')
self.jsdir = web_config.get('jsdir', jsdir)
spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs')
self.spdir = web_config.get('spdir', spdir)
# read scripts area and initialize data-ops scripts
self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts')
self.sdir = web_config.get('sdir', self.sdir)
self.sdict_thr = web_config.get('sdict_thr',
600) # put reasonable 10 min interval
self.sdict = {'ts': time.time()} # placeholder for data-ops scripts
self.update_scripts(force=True)
# To be filled at run time
self.cssmap = {}
self.jsmap = {}
self.imgmap = {}
self.yuimap = {}
std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs')
self.std_specs = spec_list(std_specs_dir, 'WMSpec.StdSpecs')
self.std_specs.sort()
# Update CherryPy configuration
mime_types = ['text/css']
mime_types += [
'application/javascript', 'text/javascript',
'application/x-javascript', 'text/x-javascript'
]
cherryconf.update({
'tools.encode.on': True,
'tools.gzip.on': True,
'tools.gzip.mime_types': mime_types,
})
self._cache = {}
# initialize rest API
statedir = '/tmp'
app = RESTMain(config, statedir) # REST application
mount = '/rest' # mount point for cherrypy service
api = RestApiHub(app, config.reqmgr, mount)
# initialize access to reqmgr2 APIs
self.reqmgr = ReqMgr(config.reqmgr.reqmgr2_url)
# only gets current view (This might cause to reponse time much longer,
# If upto date view is not needed overwrite Fale)
self.reqmgr._noStale = True
# admin helpers
self.admin_info = Info(app, api, config.reqmgr, mount=mount + '/info')
self.admin_group = Group(app,
api,
config.reqmgr,
mount=mount + '/group')
self.admin_team = Team(app, api, config.reqmgr, mount=mount + '/team')
# get fields which we'll use in templates
cdict = config.reqmgr.dictionary_()
self.couch_url = cdict.get('couch_host', '')
self.couch_dbname = cdict.get('couch_reqmgr_db', '')
self.couch_wdbname = cdict.get('couch_workload_summary_db', '')
self.acdc_url = cdict.get('acdc_host', '')
self.acdc_dbname = cdict.get('acdc_db', '')
self.configcache_url = cdict.get('couch_config_cache_url',
self.couch_url)
self.dbs_url = cdict.get('dbs_url', '')
self.dqm_url = cdict.get('dqm_url', '')
self.sw_ver = cdict.get('default_sw_version', 'CMSSW_5_2_5')
self.sw_arch = cdict.get('default_sw_scramarch', 'slc5_amd64_gcc434')
def bind(self):
CherryPyConfig.update({
'server.socket_host': self._config.Interface,
'server.socket_port': self._config.Port,
})