def main():
logging.basicConfig(level=logging.INFO, format='%(message)s')
bind_address = (LOCALHOST, MINIJACK_FCGI_PORT)
cpu_count = multiprocessing.cpu_count()
fork_args = {
'minSpare': 1,
'maxSpare': min(cpu_count * 2, 8),
'maxChildren': max(cpu_count * 4, 16),
'maxRequests': 64}
server = WSGIServer(wsgi.application, bindAddress=bind_address, **fork_args)
server.run()
def main():
logging.basicConfig(level=logging.INFO, format='%(message)s')
bind_address = (LOCALHOST, MINIJACK_FCGI_PORT)
cpu_count = multiprocessing.cpu_count()
fork_args = {
'minSpare': 1,
'maxSpare': min(cpu_count * 2, 8),
'maxChildren': max(cpu_count * 4, 16),
'maxRequests': 64
}
server = WSGIServer(wsgi.application,
bindAddress=bind_address,
**fork_args)
server.run()
def __init__(self, config, dynamo_server):
self.socket = config.socket
self.modules_config = config.modules_config.clone()
self.dynamo_server = dynamo_server
# Preforked WSGI server
# Preforking = have at minimum min_idle and at maximum max_idle child processes listening to the out-facing port.
# There can be at most max_procs children. Each child process is single-use to ensure changes to shared resources (e.g. inventory)
# made in a child process does not affect the other processes.
prefork_config = {'minSpare': config.get('min_idle', 1), 'maxSpare': config.get('max_idle', 5), 'maxChildren': config.get('max_procs', 10), 'maxRequests': 1}
self.wsgi_server = WSGIServer(self.main, bindAddress = config.socket, umask = 0, **prefork_config)
self.server_proc = None
self.active_count = multiprocessing.Value('I', 0, lock = True)
HTMLMixin.contents_path = config.contents_path
# common mixin class used by all page-generating modules
with open(HTMLMixin.contents_path + '/html/header_common.html') as source:
HTMLMixin.header_html = source.read()
with open(HTMLMixin.contents_path + '/html/footer_common.html') as source:
HTMLMixin.footer_html = source.read()
# cookie string -> (user name, user id)
self.known_users = {}
# Log file path (start a rotating log if specified)
self.log_path = None
self.debug = config.get('debug', False)
def main(wsgi=True):
if wsgi:
from flup.server.fcgi_fork import WSGIServer
application = tornado.wsgi.WSGIApplication(routes)
WSGIServer(application=application, bindAddress=('', 18611)).run()
else:
application = tornado.web.Application(routes)
application.listen(8888)
tornado.ioloop.IOLoop.instance().start()
def RunAsFastCGI(address, port, instance):
"""Starts an XML-RPC server in given address:port.
Args:
address: IP address to bind
port: Port for server to listen
instance: Server instance for incoming XML RPC requests.
Return:
Never returns
"""
application = MyXMLRPCApp(instance=instance)
bind_address = (address, port)
cpu_count = multiprocessing.cpu_count()
fork_args = dict()
fork_args['minSpare'] = 4
fork_args['maxSpare'] = cpu_count * 2
fork_args['maxChildren'] = cpu_count * 100
fork_args['maxRequests'] = 16
server = WSGIServer(application, bindAddress=bind_address, **fork_args)
server.run()
def __init__(self, config, dynamo_server):
self.socket = config.socket
self.modules_config = config.modules_config.clone()
self.dynamo_server = dynamo_server
# Preforked WSGI server
# Preforking = have at minimum min_idle and at maximum max_idle child processes listening to the out-facing port.
# There can be at most max_procs children. Each child process is single-use to ensure changes to shared resources (e.g. inventory)
# made in a child process does not affect the other processes.
prefork_config = {
'minSpare': config.get('min_idle', 1),
'maxSpare': config.get('max_idle', 5),
'maxChildren': config.get('max_procs', 10),
'maxRequests': 1
}
self.wsgi_server = WSGIServer(self.main,
bindAddress=config.socket,
umask=0,
**prefork_config)
self.server_proc = None
self.active_count = multiprocessing.Value('I', 0, lock=True)
HTMLMixin.contents_path = config.contents_path
# common mixin class used by all page-generating modules
with open(HTMLMixin.contents_path +
'/html/header_common.html') as source:
HTMLMixin.header_html = source.read()
with open(HTMLMixin.contents_path +
'/html/footer_common.html') as source:
HTMLMixin.footer_html = source.read()
# cookie string -> (user name, user id)
self.known_users = {}
# Log file path (start a rotating log if specified)
self.log_path = None
self.debug = config.get('debug', False)
grouping=request.args.get('grouping'),
positive_query=positive_query,
negative_query=negative_query)
except ProcessingException as e:
return render_template('queries_error.html',
page_title=page_title,
page_url=page_url,
error_message=e)
except Exception as e:
return render_template('queries_unknown_error.html',
page_title=page_title,
page_url=page_url,
error_type=type(e),
error_message=e)
@app.errorhandler(404)
def page_not_found(error):
return render_template('page_not_found.html', title=u'Page not found'), 404
if __name__ == '__main__':
if os.uname()[1].startswith('tools-webgrid'):
from flup.server.fcgi_fork import WSGIServer
WSGIServer(app).run()
else:
if os.environ.get('LOCAL_ENVIRONMENT', False):
app.run(host='0.0.0.0')
else:
app.run()
#!/usr/bin/python
from flup.server.fcgi_fork import WSGIServer
from pdf_to_djvu_cgi import myapp
import os
import sys
if __name__ == '__main__':
sys.stderr = open(os.path.expanduser('~/log/pdf_to_djvu_fcgi.err'), 'a', 0)
prefork_args = {
'maxRequests': 100,
'maxSpare': 1,
'minSpare': 1,
'maxChildren': 1
}
WSGIServer(myapp, **prefork_args).run()
#!/home2/ngtherap/python27/bin/python2.7
import cgit
cgitb.enable()
import cgi
import threading
import time
import requests
import cgitb
from bs4 import BeautifulSoup
import html5lib
import sys
from flup.server.fcgi_fork import WSGIServer
def test_app(environ, start_response):
start_response('200 OK', [('Content-Type', 'text/plain')])
yield 'Hello, world!\n'
WSGIServer(test_app).run()
#! /usr/bin/env python
import sys
from paste.deploy import loadapp
from flup.server.fcgi_fork import WSGIServer
config = sys.argv[1]
app = loadapp(":".join(("config", config)))
server = WSGIServer(app)
server.run()
wsgi_opts['bindAddress'] = None
else:
return fastcgi_help("Invalid combination of host, port, socket.")
if options["daemonize"] is None:
# Default to daemonizing if we're running on a socket/named pipe.
daemonize = (wsgi_opts['bindAddress'] is not None)
else:
if options["daemonize"].lower() in ('true', 'yes', 't'):
daemonize = True
elif options["daemonize"].lower() in ('false', 'no', 'f'):
daemonize = False
else:
return fastcgi_help(
"ERROR: Invalid option for daemonize parameter.")
if daemonize:
from django.utils.daemonize import become_daemon
become_daemon(our_home_dir=options["workdir"])
if options["pidfile"]:
fp = open(options["pidfile"], "w")
fp.write("%d\n" % os.getpid())
fp.close()
WSGIServer(WSGIHandler(), **wsgi_opts).run()
if __name__ == '__main__':
runfastcgi(sys.argv[1:])
return show_login(env, start_response)
elif env['REQUEST_URI'] == '/cgi-bin/sign-up':
return show_register(env, start_response)
elif env['REQUEST_URI'] == '/cgi-bin/do-register':
return do_register(env, start_response)
elif env['REQUEST_URI'] == '/cgi-bin/' or env['REQUEST_URI'].startswith(
'/cgi-bin/do-login?'):
start_response(b'302 Found', [
(b'Location', '/~' + urllib.parse.quote(env['REMOTE_USER']) + '/')
])
return []
elif env['REQUEST_URI'].startswith('/~'):
return dav_responder(env, start_response)
else:
start_response(b'404 Not Found', [])
return []
def app(env, start_response):
if env.get('FCGI_ROLE') == 'AUTHORIZER':
return fcgi_authorizer(env, start_response)
else:
return fcgi_responder(env, start_response)
serv = WSGIServer(app,
bindAddress=('::1', 808, 0),
roles=(FCGI_AUTHORIZER, FCGI_RESPONDER))
serv.debug = False
serv.run()
try:
if config.getboolean(form_key, 'spam.check') \
and looks_like_spam(" ".join(to_check), config, form_key):
start_response('403 Forbidden', [('Content-Type', 'text/plain')])
return "I don't like SPAM!"
except NoOptionError:
pass
useful_fields = ["{0}: {1}".format(*f)
for f in useful_fields
if f[0] not in ignored_fields]
message_buffer.write(context.get("message", my_config["message"]))
message_buffer.write("\n\n")
message_buffer.write("\n".join(useful_fields))
send_message(message_buffer,
context.get("subject", my_config["subject"]),
my_config["to"],
my_config["from"])
redirect_location = context.get("redirect", my_config["redirect"])
start_response('302 Found', [('Location', redirect_location)])
return ""
if __name__ == "__main__":
from flup.server.fcgi_fork import WSGIServer
WSGIServer(email_app, maxSpare=1).run()
class WebServer(object):
User = collections.namedtuple('User', ['name', 'dn', 'id', 'authlist'])
@staticmethod
def format_dn(dn_string):
"""Read the DN string in the environ and return (user name, user id)."""
dn_parts = []
start = 0
end = 0
while True:
end = dn_string.find(',', end)
if end == -1:
dn_parts.append(dn_string[start:])
break
if end == 0:
raise exceptions.AuthorizationError()
if dn_string[end - 1] == '\\':
# if this was an escaped comma, move ahead
end += 1
continue
dn_parts.append(dn_string[start:end])
end += 2 # skip ', '
start = end
dn = ''
for part in dn_parts:
key, _, value = part.partition(' = ')
dn += '/' + key + '=' + value
return dn
def __init__(self, config, dynamo_server):
self.socket = config.socket
self.modules_config = config.modules_config.clone()
self.dynamo_server = dynamo_server
# Preforked WSGI server
# Preforking = have at minimum min_idle and at maximum max_idle child processes listening to the out-facing port.
# There can be at most max_procs children. Each child process is single-use to ensure changes to shared resources (e.g. inventory)
# made in a child process does not affect the other processes.
prefork_config = {'minSpare': config.get('min_idle', 1), 'maxSpare': config.get('max_idle', 5), 'maxChildren': config.get('max_procs', 10), 'maxRequests': 1}
self.wsgi_server = WSGIServer(self.main, bindAddress = config.socket, umask = 0, **prefork_config)
self.server_proc = None
self.active_count = multiprocessing.Value('I', 0, lock = True)
HTMLMixin.contents_path = config.contents_path
# common mixin class used by all page-generating modules
with open(HTMLMixin.contents_path + '/html/header_common.html') as source:
HTMLMixin.header_html = source.read()
with open(HTMLMixin.contents_path + '/html/footer_common.html') as source:
HTMLMixin.footer_html = source.read()
# cookie string -> (user name, user id)
self.known_users = {}
# Log file path (start a rotating log if specified)
self.log_path = None
self.debug = config.get('debug', False)
def start(self):
if self.server_proc and self.server_proc.is_alive():
raise RuntimeError('Web server is already running')
self.server_proc = multiprocessing.Process(target = self._serve)
self.server_proc.daemon = True
self.server_proc.start()
LOG.info('Started web server (PID %d).', self.server_proc.pid)
def stop(self):
LOG.info('Stopping web server (PID %d).', self.server_proc.pid)
self.server_proc.terminate()
LOG.debug('Waiting for web server to join.')
self.server_proc.join(5)
if self.server_proc.is_alive():
# SIGTERM got ignored
LOG.info('Web server failed to stop. Sending KILL signal..')
try:
os.kill(self.server_proc.pid, signal.SIGKILL)
except:
pass
self.server_proc.join(5)
if self.server_proc.is_alive():
LOG.warning('Web server (PID %d) is stuck.', self.server_proc.pid)
self.server_proc = None
return
LOG.debug('Web server joined.')
self.server_proc = None
def restart(self):
LOG.info('Restarting web server (PID %d).', self.server_proc.pid)
# Replace the active_count by a temporary object (won't be visible from subprocs)
old_active_count = self.active_count
self.active_count = multiprocessing.Value('I', 0, lock = True)
# A new WSGI server will overtake the socket. New requests will be handled by new_server_proc
LOG.debug('Starting new web server.')
new_server_proc = multiprocessing.Process(target = self._serve)
new_server_proc.daemon = True
new_server_proc.start()
# Drain and stop the main server
LOG.debug('Waiting for web server to drain.')
elapsed = 0.
while old_active_count.value != 0:
time.sleep(0.2)
elapsed += 0.2
if elapsed >= 10.:
break
self.stop()
self.server_proc = new_server_proc
LOG.info('Started web server (PID %d).', self.server_proc.pid)
def _serve(self):
if self.log_path:
reset_logger()
root_logger = logging.getLogger()
log_handler = logging.handlers.RotatingFileHandler(self.log_path, maxBytes = 10000000, backupCount = 100)
log_handler.setFormatter(logging.Formatter(fmt = '%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
root_logger.addHandler(log_handler)
# If the inventory loads super-fast in the main server, we get reset while loading the defaults
# Block the terminate signal to avoid KeyboardInterrupt error messages
try:
# Ignore one specific warning issued by accident when a web page crashes and dumps a stack trace
# cgitb scans all exception attributes with dir(exc) + getattr(exc, attr) which results in accessing
# exception.message, a deprecated attribute.
warnings.filterwarnings('ignore', 'BaseException.message.*', DeprecationWarning, '.*cgitb.*', 173)
load_modules()
# Set up module defaults
# Using the same piece of code as serverutils, but only picking up fullauth or all configurations
for key, config in self.dynamo_server.defaults_config.items():
try:
myconf = config['fullauth']
except KeyError:
try:
myconf = config['all']
except KeyError:
continue
modname, clsname = key.split(':')
module = __import__('dynamo.' + modname, globals(), locals(), [clsname])
cls = getattr(module, clsname)
cls.set_default(myconf)
except KeyboardInterrupt:
os._exit(0)
try:
self.wsgi_server.run()
except SystemExit as exc:
LOG.debug('Web server subprocess %d exiting', os.getpid())
# Server subprocesses terminate with sys.exit and land here
# Because sys.exit performs garbage collection, which can disrupt shared resources (e.g. close connection to DB)
# we need to translate it to os._exit
os._exit(exc.code)
def main(self, environ, start_response):
# Increment the active count so that the parent process won't be killed before this function returns
with self.active_count.get_lock():
self.active_count.value += 1
try:
agent = environ['HTTP_USER_AGENT']
except KeyError:
agent = 'Unknown'
# Log file is a shared resource - write within the lock
LOG.info('%s-%s %s (%s:%s %s)', environ['REQUEST_SCHEME'], environ['REQUEST_METHOD'], environ['REQUEST_URI'], environ['REMOTE_ADDR'], environ['REMOTE_PORT'], agent)
# Then immediately switch to logging to a buffer
root_logger = logging.getLogger()
stream = cStringIO.StringIO()
original_handler = root_logger.handlers.pop()
handler = logging.StreamHandler(stream)
handler.setFormatter(logging.Formatter(fmt = '%(levelname)s:%(name)s: %(message)s'))
root_logger.addHandler(handler)
stdout = sys.stdout
stderr = sys.stderr
sys.stdout = stream
sys.stderr = stream
try:
self.code = 200 # HTTP response code
self.content_type = 'application/json' # content type string
self.headers = [] # list of header tuples
self.callback = None # set to callback function name if this is a JSONP request
self.message = '' # string
self.phedex_request = '' # backward compatibility
content = self._main(environ)
# Maybe we can use some standard library?
if self.code == 200:
status = 'OK'
elif self.code == 400:
status = 'Bad Request'
elif self.code == 403:
status = 'Forbidden'
elif self.code == 404:
status = 'Not Found'
elif self.code == 500:
status = 'Internal Server Error'
elif self.code == 503:
status = 'Service Unavailable'
if self.content_type == 'application/json':
if self.phedex_request != '':
if type(content) is not dict:
self.code == 500
status = 'Internal Server Error'
else:
url = '%s://%s' % (environ['REQUEST_SCHEME'], environ['HTTP_HOST'])
if (environ['REQUEST_SCHEME'] == 'http' and environ['SERVER_PORT'] != '80') or \
(environ['REQUEST_SCHEME'] == 'https' and environ['SERVER_PORT'] != '443'):
url += '%s' % environ['SERVER_PORT']
url += environ['REQUEST_URI']
json_data = {'phedex': {'call_time': 0, 'instance': 'prod', 'request_call': self.phedex_request,
'request_date': time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime()), 'request_timestamp': time.time(),
'request_url': url, 'request_version': '2.2.1'}}
json_data['phedex'].update(content)
content = json.dumps(json_data)
else:
json_data = {'result': status, 'message': self.message}
if content is not None:
json_data['data'] = content
# replace content with the json string
if self.callback is not None:
content = '%s(%s)' % (self.callback, json.dumps(json_data))
else:
content = json.dumps(json_data)
headers = [('Content-Type', self.content_type)] + self.headers
start_response('%d %s' % (self.code, status), headers)
return content + '\n'
finally:
sys.stdout = stdout
sys.stderr = stderr
root_logger.handlers.pop()
root_logger.addHandler(original_handler)
delim = '--------------'
log_tmp = stream.getvalue().strip()
if len(log_tmp) == 0:
log = 'empty log'
else:
log = 'return:\n%s\n%s%s' % (delim, ''.join(' %s\n' % line for line in log_tmp.split('\n')), delim)
with self.active_count.get_lock():
LOG.info('%s-%s %s (%s:%s) %s', environ['REQUEST_SCHEME'], environ['REQUEST_METHOD'], environ['REQUEST_URI'], environ['REMOTE_ADDR'], environ['REMOTE_PORT'], log)
self.active_count.value -= 1
def _main(self, environ):
"""
Body of the WSGI callable. Steps:
1. Determine protocol. If HTTPS, identify the user.
2. If js or css is requested, respond.
3. Find the module class and instantiate it.
4. Parse the query string into a dictionary.
5. Call the run() function of the module class.
6. Respond.
"""
authorizer = None
## Step 1
if environ['REQUEST_SCHEME'] == 'http':
# No auth
user, dn, user_id = None, None, 0
authlist = []
elif environ['REQUEST_SCHEME'] == 'https':
authorizer = self.dynamo_server.manager.master.create_authorizer()
# Client DN must match a known user
try:
dn = WebServer.format_dn(environ['SSL_CLIENT_S_DN'])
userinfo = authorizer.identify_user(dn = dn, check_trunc = True)
if userinfo is None:
raise exceptions.AuthorizationError()
user, user_id, dn = userinfo
except exceptions.AuthorizationError:
self.code = 403
self.message = 'Unknown user. Client name: %s' % environ['SSL_CLIENT_S_DN']
return
except:
return self._internal_server_error()
authlist = authorizer.list_user_auth(user)
else:
self.code = 400
self.message = 'Only HTTP or HTTPS requests are allowed.'
return
## Step 2
mode = environ['SCRIPT_NAME'].strip('/')
if mode == 'js' or mode == 'css':
try:
source = open(HTMLMixin.contents_path + '/' + mode + environ['PATH_INFO'])
except IOError:
self.code = 404
self.content_type = 'text/plain'
return 'Invalid request %s%s.\n' % (mode, environ['PATH_INFO'])
else:
if mode == 'js':
self.content_type = 'text/javascript'
else:
self.content_type = 'text/css'
content = source.read() + '\n'
source.close()
return content
## Step 3
if mode != 'data' and mode != 'web' and mode != 'registry' and mode != 'phedexdata': # registry and phedexdata for backward compatibility
self.code = 404
self.message = 'Invalid request %s.' % mode
return
if mode == 'phedexdata':
mode = 'data'
self.phedex_request = environ['PATH_INFO'][1:]
module, _, command = environ['PATH_INFO'][1:].partition('/')
try:
cls = modules[mode][module][command]
except KeyError:
# Was a new module added perhaps?
load_modules()
try: # again
cls = modules[mode][module][command]
except KeyError:
self.code = 404
self.message = 'Invalid request %s/%s.' % (module, command)
return
try:
provider = cls(self.modules_config)
except:
return self._internal_server_error()
if provider.must_authenticate and user is None:
self.code = 400
self.message = 'Resource only available with HTTPS.'
return
if provider.write_enabled:
self.dynamo_server.manager.master.lock()
try:
if self.dynamo_server.manager.master.inhibit_write():
# We need to give up here instead of waiting, because the web server processes will be flushed out as soon as
# inventory is updated after the current writing process is done
self.code = 503
self.message = 'Server cannot execute %s/%s at the moment because the inventory is being updated.' % (module, command)
return
else:
self.dynamo_server.manager.master.start_write_web(socket.gethostname(), os.getpid())
# stop is called from the DynamoServer upon successful inventory update
except:
self.dynamo_server.manager.master.stop_write_web()
raise
finally:
self.dynamo_server.manager.master.unlock()
if provider.require_authorizer:
if authorizer is None:
authorizer = self.dynamo_server.manager.master.create_authorizer()
provider.authorizer = authorizer
if provider.require_appmanager:
provider.appmanager = self.dynamo_server.manager.master.create_appmanager()
try:
## Step 4
post_request = None
if environ['REQUEST_METHOD'] == 'POST':
try:
content_type = environ['CONTENT_TYPE']
except KeyError:
content_type = 'application/x-www-form-urlencoded'
# In principle we should grab CONTENT_LENGTH from environ and only read as many bytes as given, but wsgi.input seems to know where the EOF is
try:
content_length = environ['CONTENT_LENGTH']
except KeyError:
# length -1: rely on wsgi.input having an EOF at the end
content_length = -1
post_data = environ['wsgi.input'].read(content_length)
# Even though our default content type is URL form, we check if this is a JSON
try:
json_data = json.loads(post_data)
except:
if content_type == 'application/json':
self.code = 400
self.message = 'Could not parse input.'
return
else:
content_type = 'application/json'
provider.input_data = json_data
unicode2str(provider.input_data)
if content_type == 'application/x-www-form-urlencoded':
try:
post_request = parse_qs(post_data)
except:
self.code = 400
self.message = 'Could not parse input.'
elif content_type != 'application/json':
self.code = 400
self.message = 'Unknown Content-Type %s.' % content_type
get_request = parse_qs(environ['QUERY_STRING'])
if post_request is not None:
for key, value in post_request.iteritems():
if key in get_request:
# return dict of parse_qs is {key: list}
get_request[key].extend(post_request[key])
else:
get_request[key] = post_request[key]
unicode2str(get_request)
request = {}
for key, value in get_request.iteritems():
if key.endswith('[]'):
key = key[:-2]
request[key] = map(escape, value)
else:
if len(value) == 1:
request[key] = escape(value[0])
else:
request[key] = map(escape, value)
## Step 5
caller = WebServer.User(user, dn, user_id, authlist)
if self.dynamo_server.inventory.loaded:
inventory = self.dynamo_server.inventory.create_proxy()
if provider.write_enabled:
inventory._update_commands = []
else:
inventory = DummyInventory()
content = provider.run(caller, request, inventory)
if provider.write_enabled:
self.dynamo_server._send_updates(inventory)
except (exceptions.AuthorizationError, exceptions.ResponseDenied, exceptions.MissingParameter,
exceptions.ExtraParameter, exceptions.IllFormedRequest, exceptions.InvalidRequest) as ex:
self.code = 400
self.message = str(ex)
return
except exceptions.TryAgain as ex:
self.code = 503
self.message = str(ex)
return
except:
return self._internal_server_error()
## Step 6
self.message = provider.message
self.content_type = provider.content_type
self.headers = provider.additional_headers
if 'callback' in request:
self.callback = request['callback']
return content
def _internal_server_error(self):
self.code = 500
self.content_type = 'text/plain'
exc_type, exc, tb = sys.exc_info()
if self.debug:
response = 'Caught exception %s while waiting for task to complete.\n' % exc_type.__name__
response += 'Traceback (most recent call last):\n'
response += ''.join(traceback.format_tb(tb)) + '\n'
response += '%s: %s\n' % (exc_type.__name__, str(exc))
return response
else:
return 'Internal server error! (' + exc_type.__name__ + ': ' + str(exc) + ')\n'
print >> sys.stderr, sys.argv[0].split("/")[-1] + ": " + str(err.msg)
print >> sys.stderr, "\t for help use --help"
return -2
try:
f = open(pidfile, 'w')
f.write(str(os.getpid()) + '\n')
f.close()
except IOError, e:
print "!! Error writing to pid file, check pid file path: %s" % pidfile
return -1
try:
WSGIServer(gserve,
bindAddress=(host, port),
minSpare=minspare,
maxSpare=maxspare,
maxChildren=maxchildren,
maxRequests=maxrequests).run()
except Exception, e:
print "!! WSGIServer raised exception"
print e
if __name__ == "__main__":
sys.exit(main())
########NEW FILE########
__FILENAME__ = messaging
#messaging.py
#this is a module used for messaging. It allows multiple classes
#to handle various types of messages. It should work on all python
#versions >= 1.5.2
def __init__(self, *args, **kw):
WSGIServer.__init__(self, *args, **kw)
regerror.regerror.__init__(self, logpath, 'admin')
k.make_public()
return open(cachefilename)
standalone = False
if standalone:
PORT = 8080
httpd = make_server('', PORT, imagserver)
print 'Starting up HTTP server on port %i...' % PORT
# Respond to requests until process is killed
httpd.serve_forever()
# FastCGI
WSGIServer(save_imagserver).run()
########NEW FILE########
__FILENAME__ = imageserver_import
#!/usr/bin/env python
# encoding: utf-8
"""
imageserver_import.py
Created by Maximillian Dornseif on 2009-01-29.
Copyright (c) 2009 HUDORA. All rights reserved.
"""
import base64
import datetime
import hashlib
from esp.datatree.models import get_lowest_parent
from esp.users.models import UserBit
user = User.objects.get(username=username, is_active=True)
except User.DoesNotExist:
return unauthorized()
# Check the password and any permission given
if not ( user.check_password(password) and user.is_authenticated() ):
return unauthorized()
qsc = get_lowest_parent('Q/Static/' + environ['REQUEST_URI'].strip('/'))
verb = get_lowest_parent('V/Flags/Public')
if not UserBit.UserHasPerms(user, qsc, verb):
return unauthorized()
# By now we've verified the user's login and permissions
environ['REMOTE_USER'] = username
# Eventually pass all requests to Trac
from trac.web.main import dispatch_request
return dispatch_request(environ, start_response)
if __name__ == '__main__':
if len( sys.argv ) == 3:
# Assume the arguments are a socket and a pidfile.
from flup.server.fcgi_fork import WSGIServer
from django.utils.daemonize import become_daemon
become_daemon()
with open( sys.argv[2], 'w' ) as pidfile:
pidfile.write( '%d\n' % os.getpid() )
WSGIServer(handler, bindAddress=sys.argv[1]).run()
else:
from flup.server.fcgi import WSGIServer
WSGIServer(handler).run()
class WebServer(object):
User = collections.namedtuple('User', ['name', 'dn', 'id', 'authlist'])
@staticmethod
def format_dn(dn_string):
"""Read the DN string in the environ and return (user name, user id)."""
dn_parts = []
start = 0
end = 0
while True:
end = dn_string.find(',', end)
if end == -1:
dn_parts.append(dn_string[start:])
break
if end == 0:
raise exceptions.AuthorizationError()
if dn_string[end - 1] == '\\':
# if this was an escaped comma, move ahead
end += 1
continue
dn_parts.append(dn_string[start:end])
end += 2 # skip ', '
start = end
dn = ''
for part in dn_parts:
key, _, value = part.partition(' = ')
dn += '/' + key + '=' + value
return dn
def __init__(self, config, dynamo_server):
self.socket = config.socket
self.modules_config = config.modules_config.clone()
self.dynamo_server = dynamo_server
# Preforked WSGI server
# Preforking = have at minimum min_idle and at maximum max_idle child processes listening to the out-facing port.
# There can be at most max_procs children. Each child process is single-use to ensure changes to shared resources (e.g. inventory)
# made in a child process does not affect the other processes.
prefork_config = {
'minSpare': config.get('min_idle', 1),
'maxSpare': config.get('max_idle', 5),
'maxChildren': config.get('max_procs', 10),
'maxRequests': 1
}
self.wsgi_server = WSGIServer(self.main,
bindAddress=config.socket,
umask=0,
**prefork_config)
self.server_proc = None
self.active_count = multiprocessing.Value('I', 0, lock=True)
HTMLMixin.contents_path = config.contents_path
# common mixin class used by all page-generating modules
with open(HTMLMixin.contents_path +
'/html/header_common.html') as source:
HTMLMixin.header_html = source.read()
with open(HTMLMixin.contents_path +
'/html/footer_common.html') as source:
HTMLMixin.footer_html = source.read()
# cookie string -> (user name, user id)
self.known_users = {}
# Log file path (start a rotating log if specified)
self.log_path = None
self.debug = config.get('debug', False)
def start(self):
if self.server_proc and self.server_proc.is_alive():
raise RuntimeError('Web server is already running')
self.server_proc = multiprocessing.Process(target=self._serve)
self.server_proc.daemon = True
self.server_proc.start()
LOG.info('Started web server (PID %d).', self.server_proc.pid)
def stop(self):
LOG.info('Stopping web server (PID %d).', self.server_proc.pid)
self.server_proc.terminate()
LOG.debug('Waiting for web server to join.')
self.server_proc.join(5)
if self.server_proc.is_alive():
# SIGTERM got ignored
LOG.info('Web server failed to stop. Sending KILL signal..')
try:
os.kill(self.server_proc.pid, signal.SIGKILL)
except:
pass
self.server_proc.join(5)
if self.server_proc.is_alive():
LOG.warning('Web server (PID %d) is stuck.',
self.server_proc.pid)
self.server_proc = None
return
LOG.debug('Web server joined.')
self.server_proc = None
def restart(self):
LOG.info('Restarting web server (PID %d).', self.server_proc.pid)
# Replace the active_count by a temporary object (won't be visible from subprocs)
old_active_count = self.active_count
self.active_count = multiprocessing.Value('I', 0, lock=True)
# A new WSGI server will overtake the socket. New requests will be handled by new_server_proc
LOG.debug('Starting new web server.')
new_server_proc = multiprocessing.Process(target=self._serve)
new_server_proc.daemon = True
new_server_proc.start()
# Drain and stop the main server
LOG.debug('Waiting for web server to drain.')
elapsed = 0.
while old_active_count.value != 0:
time.sleep(0.2)
elapsed += 0.2
if elapsed >= 10.:
break
self.stop()
self.server_proc = new_server_proc
LOG.info('Started web server (PID %d).', self.server_proc.pid)
def _serve(self):
if self.log_path:
reset_logger()
root_logger = logging.getLogger()
log_handler = logging.handlers.RotatingFileHandler(
self.log_path, maxBytes=10000000, backupCount=100)
log_handler.setFormatter(
logging.Formatter(
fmt='%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
root_logger.addHandler(log_handler)
# If the inventory loads super-fast in the main server, we get reset while loading the defaults
# Block the terminate signal to avoid KeyboardInterrupt error messages
try:
# Ignore one specific warning issued by accident when a web page crashes and dumps a stack trace
# cgitb scans all exception attributes with dir(exc) + getattr(exc, attr) which results in accessing
# exception.message, a deprecated attribute.
warnings.filterwarnings('ignore', 'BaseException.message.*',
DeprecationWarning, '.*cgitb.*', 173)
load_modules()
# Set up module defaults
# Using the same piece of code as serverutils, but only picking up fullauth or all configurations
for key, config in self.dynamo_server.defaults_config.items():
try:
myconf = config['fullauth']
except KeyError:
try:
myconf = config['all']
except KeyError:
continue
modname, clsname = key.split(':')
module = __import__('dynamo.' + modname, globals(), locals(),
[clsname])
cls = getattr(module, clsname)
cls.set_default(myconf)
except KeyboardInterrupt:
os._exit(0)
try:
self.wsgi_server.run()
except SystemExit as exc:
LOG.debug('Web server subprocess %d exiting', os.getpid())
# Server subprocesses terminate with sys.exit and land here
# Because sys.exit performs garbage collection, which can disrupt shared resources (e.g. close connection to DB)
# we need to translate it to os._exit
os._exit(exc.code)
def main(self, environ, start_response):
# Increment the active count so that the parent process won't be killed before this function returns
with self.active_count.get_lock():
self.active_count.value += 1
try:
agent = environ['HTTP_USER_AGENT']
except KeyError:
agent = 'Unknown'
# Log file is a shared resource - write within the lock
LOG.info('%s-%s %s (%s:%s %s)', environ['REQUEST_SCHEME'],
environ['REQUEST_METHOD'], environ['REQUEST_URI'],
environ['REMOTE_ADDR'], environ['REMOTE_PORT'], agent)
# Then immediately switch to logging to a buffer
root_logger = logging.getLogger()
stream = cStringIO.StringIO()
original_handler = root_logger.handlers.pop()
handler = logging.StreamHandler(stream)
handler.setFormatter(
logging.Formatter(fmt='%(levelname)s:%(name)s: %(message)s'))
root_logger.addHandler(handler)
stdout = sys.stdout
stderr = sys.stderr
sys.stdout = stream
sys.stderr = stream
try:
self.code = 200 # HTTP response code
self.content_type = 'application/json' # content type string
self.headers = [] # list of header tuples
self.callback = None # set to callback function name if this is a JSONP request
self.message = '' # string
self.phedex_request = '' # backward compatibility
content = self._main(environ)
# Maybe we can use some standard library?
if self.code == 200:
status = 'OK'
elif self.code == 400:
status = 'Bad Request'
elif self.code == 403:
status = 'Forbidden'
elif self.code == 404:
status = 'Not Found'
elif self.code == 500:
status = 'Internal Server Error'
elif self.code == 503:
status = 'Service Unavailable'
if self.content_type == 'application/json':
if self.phedex_request != '':
if type(content) is not dict:
self.code == 500
status = 'Internal Server Error'
else:
url = '%s://%s' % (environ['REQUEST_SCHEME'],
environ['HTTP_HOST'])
if (environ['REQUEST_SCHEME'] == 'http' and environ['SERVER_PORT'] != '80') or \
(environ['REQUEST_SCHEME'] == 'https' and environ['SERVER_PORT'] != '443'):
url += '%s' % environ['SERVER_PORT']
url += environ['REQUEST_URI']
json_data = {
'phedex': {
'call_time':
0,
'instance':
'prod',
'request_call':
self.phedex_request,
'request_date':
time.strftime('%Y-%m-%d %H:%M:%S UTC',
time.gmtime()),
'request_timestamp':
time.time(),
'request_url':
url,
'request_version':
'2.2.1'
}
}
json_data['phedex'].update(content)
content = json.dumps(json_data)
else:
json_data = {'result': status, 'message': self.message}
if content is not None:
json_data['data'] = content
# replace content with the json string
start = time.time()
if self.callback is not None:
content = '%s(%s)' % (self.callback,
json.dumps(json_data))
else:
content = json.dumps(json_data)
root_logger.info('Make JSON: %s seconds',
time.time() - start)
headers = [('Content-Type', self.content_type)] + self.headers
start_response('%d %s' % (self.code, status), headers)
return content + '\n'
finally:
sys.stdout = stdout
sys.stderr = stderr
root_logger.handlers.pop()
root_logger.addHandler(original_handler)
delim = '--------------'
log_tmp = stream.getvalue().strip()
if len(log_tmp) == 0:
log = 'empty log'
else:
log = 'return:\n%s\n%s%s' % (delim, ''.join(
' %s\n' % line for line in log_tmp.split('\n')), delim)
with self.active_count.get_lock():
LOG.info('%s-%s %s (%s:%s) %s', environ['REQUEST_SCHEME'],
environ['REQUEST_METHOD'], environ['REQUEST_URI'],
environ['REMOTE_ADDR'], environ['REMOTE_PORT'], log)
self.active_count.value -= 1
def _main(self, environ):
"""
Body of the WSGI callable. Steps:
1. Determine protocol. If HTTPS, identify the user.
2. If js or css is requested, respond.
3. Find the module class and instantiate it.
4. Parse the query string into a dictionary.
5. Call the run() function of the module class.
6. Respond.
"""
authorizer = None
## Step 1
if environ['REQUEST_SCHEME'] == 'http':
# No auth
user, dn, user_id = None, None, 0
authlist = []
elif environ['REQUEST_SCHEME'] == 'https':
authorizer = self.dynamo_server.manager.master.create_authorizer()
# Client DN must match a known user
try:
dn = WebServer.format_dn(environ['SSL_CLIENT_S_DN'])
userinfo = authorizer.identify_user(dn=dn, check_trunc=True)
if userinfo is None:
raise exceptions.AuthorizationError()
user, user_id, dn = userinfo
except exceptions.AuthorizationError:
self.code = 403
self.message = 'Unknown user. Client name: %s' % environ[
'SSL_CLIENT_S_DN']
return
except:
return self._internal_server_error()
authlist = authorizer.list_user_auth(user)
else:
self.code = 400
self.message = 'Only HTTP or HTTPS requests are allowed.'
return
## Step 2
mode = environ['SCRIPT_NAME'].strip('/')
if mode == 'js' or mode == 'css':
try:
source = open(HTMLMixin.contents_path + '/' + mode +
environ['PATH_INFO'])
except IOError:
self.code = 404
self.content_type = 'text/plain'
return 'Invalid request %s%s.\n' % (mode, environ['PATH_INFO'])
else:
if mode == 'js':
self.content_type = 'text/javascript'
else:
self.content_type = 'text/css'
content = source.read() + '\n'
source.close()
return content
## Step 3
if mode != 'data' and mode != 'web' and mode != 'registry' and mode != 'phedexdata': # registry and phedexdata for backward compatibility
self.code = 404
self.message = 'Invalid request %s.' % mode
return
if mode == 'phedexdata':
mode = 'data'
self.phedex_request = environ['PATH_INFO'][1:]
module, _, command = environ['PATH_INFO'][1:].partition('/')
try:
cls = modules[mode][module][command]
except KeyError:
# Was a new module added perhaps?
load_modules()
try: # again
cls = modules[mode][module][command]
except KeyError:
self.code = 404
self.message = 'Invalid request %s/%s.' % (module, command)
return
try:
provider = cls(self.modules_config)
except:
return self._internal_server_error()
if provider.must_authenticate and user is None:
self.code = 400
self.message = 'Resource only available with HTTPS.'
return
if provider.write_enabled:
self.dynamo_server.manager.master.lock()
try:
if self.dynamo_server.manager.master.inhibit_write():
# We need to give up here instead of waiting, because the web server processes will be flushed out as soon as
# inventory is updated after the current writing process is done
self.code = 503
self.message = 'Server cannot execute %s/%s at the moment because the inventory is being updated.' % (
module, command)
return
else:
self.dynamo_server.manager.master.start_write_web(
socket.gethostname(), os.getpid())
# stop is called from the DynamoServer upon successful inventory update
except:
self.dynamo_server.manager.master.stop_write_web()
raise
finally:
self.dynamo_server.manager.master.unlock()
if provider.require_authorizer:
if authorizer is None:
authorizer = self.dynamo_server.manager.master.create_authorizer(
)
provider.authorizer = authorizer
if provider.require_appmanager:
provider.appmanager = self.dynamo_server.manager.master.create_appmanager(
)
try:
## Step 4
post_request = None
if environ['REQUEST_METHOD'] == 'POST':
try:
content_type = environ['CONTENT_TYPE']
except KeyError:
content_type = 'application/x-www-form-urlencoded'
# In principle we should grab CONTENT_LENGTH from environ and only read as many bytes as given, but wsgi.input seems to know where the EOF is
try:
content_length = environ['CONTENT_LENGTH']
except KeyError:
# length -1: rely on wsgi.input having an EOF at the end
content_length = -1
post_data = environ['wsgi.input'].read(content_length)
# Even though our default content type is URL form, we check if this is a JSON
try:
json_data = json.loads(post_data)
except:
if content_type == 'application/json':
self.code = 400
self.message = 'Could not parse input.'
return
else:
content_type = 'application/json'
provider.input_data = json_data
unicode2str(provider.input_data)
if content_type == 'application/x-www-form-urlencoded':
try:
post_request = parse_qs(post_data)
except:
self.code = 400
self.message = 'Could not parse input.'
elif content_type != 'application/json':
self.code = 400
self.message = 'Unknown Content-Type %s.' % content_type
get_request = parse_qs(environ['QUERY_STRING'])
if post_request is not None:
for key, value in post_request.iteritems():
if key in get_request:
# return dict of parse_qs is {key: list}
get_request[key].extend(post_request[key])
else:
get_request[key] = post_request[key]
unicode2str(get_request)
request = {}
for key, value in get_request.iteritems():
if key.endswith('[]'):
key = key[:-2]
request[key] = map(escape, value)
else:
if len(value) == 1:
request[key] = escape(value[0])
else:
request[key] = map(escape, value)
## Step 5
caller = WebServer.User(user, dn, user_id, authlist)
if self.dynamo_server.inventory.loaded:
inventory = self.dynamo_server.inventory.create_proxy()
if provider.write_enabled:
inventory._update_commands = []
else:
inventory = DummyInventory()
content = provider.run(caller, request, inventory)
if provider.write_enabled:
self.dynamo_server._send_updates(inventory)
except (exceptions.AuthorizationError, exceptions.ResponseDenied,
exceptions.MissingParameter, exceptions.ExtraParameter,
exceptions.IllFormedRequest, exceptions.InvalidRequest) as ex:
self.code = 400
self.message = str(ex)
return
except exceptions.TryAgain as ex:
self.code = 503
self.message = str(ex)
return
except:
return self._internal_server_error()
## Step 6
self.message = provider.message
self.content_type = provider.content_type
self.headers = provider.additional_headers
if 'callback' in request:
self.callback = request['callback']
return content
def _internal_server_error(self):
self.code = 500
self.content_type = 'text/plain'
exc_type, exc, tb = sys.exc_info()
if self.debug:
response = 'Caught exception %s while waiting for task to complete.\n' % exc_type.__name__
response += 'Traceback (most recent call last):\n'
response += ''.join(traceback.format_tb(tb)) + '\n'
response += '%s: %s\n' % (exc_type.__name__, str(exc))
return response
else:
return 'Internal server error! (' + exc_type.__name__ + ': ' + str(
exc) + ')\n'
#!/usr/bin/python
from TileCache.Service import wsgiApp
if __name__ == '__main__':
from flup.server.fcgi_fork import WSGIServer
WSGIServer(wsgiApp).run()
