class ChildFactory:
def preExec(self):
os.setpgrp()
def __init__(self, configuration, name):
self.log = Logger('worker ' + str(name), configuration.log.worker)
def createProcess(self, program, universal=False):
try:
process = subprocess.Popen(
program.split(' '),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=universal,
preexec_fn=self.preExec,
)
self.log.debug('spawn process %s' % program)
except KeyboardInterrupt:
process = None
except (subprocess.CalledProcessError, OSError, ValueError):
self.log.error('could not spawn process %s' % program)
process = None
if process:
try:
fcntl.fcntl(process.stderr, fcntl.F_SETFL, os.O_NONBLOCK)
except IOError:
self.destroyProcess(process)
process = None
return process
def destroyProcess(self, process):
try:
process.terminate()
process.wait()
self.log.info('terminated process PID %s' % process.pid)
except OSError, e:
# No such processs
if e[0] != errno.ESRCH:
self.log.error('PID %s died' % process.pid)
class ChildFactory:
def preExec (self):
os.setpgrp()
def __init__ (self, configuration, name):
self.log = Logger('worker ' + str(name), configuration.log.worker)
def createProcess (self, program, universal=False):
try:
process = subprocess.Popen([program],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=universal,
preexec_fn=self.preExec,
)
self.log.debug('spawn process %s' % program)
except KeyboardInterrupt:
process = None
except (subprocess.CalledProcessError,OSError,ValueError):
self.log.error('could not spawn process %s' % program)
process = None
if process:
try:
fcntl.fcntl(process.stderr, fcntl.F_SETFL, os.O_NONBLOCK)
except IOError:
self.destroyProcess(process)
process = None
return process
def destroyProcess (self, process):
try:
process.terminate()
process.wait()
self.log.info('terminated process PID %s' % process.pid)
except OSError, e:
# No such processs
if e[0] != errno.ESRCH:
self.log.error('PID %s died' % process.pid)
class Supervisor (object):
alarm_time = 0.1 # regular backend work
second_frequency = int(1/alarm_time) # when we record history
minute_frequency = int(60/alarm_time) # when we want to average history
increase_frequency = int(5/alarm_time) # when we add workers
decrease_frequency = int(60/alarm_time) # when we remove workers
saturation_frequency = int(20/alarm_time) # when we report connection saturation
interface_frequency = int(300/alarm_time) # when we check for new interfaces
# import os
# clear = [hex(ord(c)) for c in os.popen('clear').read()]
# clear = ''.join([chr(int(c,16)) for c in ['0x1b', '0x5b', '0x48', '0x1b', '0x5b', '0x32', '0x4a']])
def __init__ (self,configuration):
configuration = load()
self.configuration = configuration
# Only here so the introspection code can find them
self.log = Logger('supervisor', configuration.log.supervisor)
self.log.error('Starting exaproxy version %s' % configuration.proxy.version)
self.signal_log = Logger('signal', configuration.log.signal)
self.log_writer = SysLogWriter('log', configuration.log.destination, configuration.log.enable, level=configuration.log.level)
self.usage_writer = UsageWriter('usage', configuration.usage.destination, configuration.usage.enable)
self.log_writer.setIdentifier(configuration.daemon.identifier)
#self.usage_writer.setIdentifier(configuration.daemon.identifier)
if configuration.debug.log:
self.log_writer.toggleDebug()
self.usage_writer.toggleDebug()
self.log.error('python version %s' % sys.version.replace(os.linesep,' '))
self.log.debug('starting %s' % sys.argv[0])
self.pid = PID(self.configuration)
self.daemon = Daemon(self.configuration)
self.poller = Poller(self.configuration.daemon)
self.poller.setupRead('read_proxy') # Listening proxy sockets
self.poller.setupRead('read_web') # Listening webserver sockets
self.poller.setupRead('read_icap') # Listening icap sockets
self.poller.setupRead('read_workers') # Pipes carrying responses from the child processes
self.poller.setupRead('read_resolver') # Sockets currently listening for DNS responses
self.poller.setupRead('read_client') # Active clients
self.poller.setupRead('opening_client') # Clients we have not yet read a request from
self.poller.setupWrite('write_client') # Active clients with buffered data to send
self.poller.setupWrite('write_resolver') # Active DNS requests with buffered data to send
self.poller.setupRead('read_download') # Established connections
self.poller.setupWrite('write_download') # Established connections we have buffered data to send to
self.poller.setupWrite('opening_download') # Opening connections
self.monitor = Monitor(self)
self.page = Page(self)
self.manager = RedirectorManager(
self.configuration,
self.poller,
)
self.content = ContentManager(self,configuration)
self.client = ClientManager(self.poller, configuration)
self.resolver = ResolverManager(self.poller, self.configuration, configuration.dns.retries*10)
self.proxy = Server('http proxy',self.poller,'read_proxy', configuration.http.connections)
self.web = Server('web server',self.poller,'read_web', configuration.web.connections)
self.icap = Server('icap server',self.poller,'read_icap', configuration.icap.connections)
self.reactor = Reactor(self.configuration, self.web, self.proxy, self.icap, self.manager, self.content, self.client, self.resolver, self.log_writer, self.usage_writer, self.poller)
self._shutdown = True if self.daemon.filemax == 0 else False # stop the program
self._softstop = False # stop once all current connection have been dealt with
self._reload = False # unimplemented
self._toggle_debug = False # start logging a lot
self._decrease_spawn_limit = 0
self._increase_spawn_limit = 0
self._refork = False # unimplemented
self._pdb = False # turn on pdb debugging
self._listen = None # listening change ? None: no, True: listen, False: stop listeing
self.wait_time = 5.0 # how long do we wait at maximum once we have been soft-killed
self.local = set() # what addresses are on our local interfaces
self.interfaces()
signal.signal(signal.SIGQUIT, self.sigquit)
signal.signal(signal.SIGINT, self.sigterm)
signal.signal(signal.SIGTERM, self.sigterm)
# signal.signal(signal.SIGABRT, self.sigabrt)
# signal.signal(signal.SIGHUP, self.sighup)
signal.signal(signal.SIGTRAP, self.sigtrap)
signal.signal(signal.SIGUSR1, self.sigusr1)
signal.signal(signal.SIGUSR2, self.sigusr2)
signal.signal(signal.SIGTTOU, self.sigttou)
signal.signal(signal.SIGTTIN, self.sigttin)
signal.signal(signal.SIGALRM, self.sigalrm)
# make sure we always have data in history
# (done in zero for dependencies reasons)
self.monitor.zero()
def sigquit (self,signum, frame):
if self._softstop:
self.signal_log.critical('multiple SIG INT received, shutdown')
self._shutdown = True
else:
self.signal_log.critical('SIG INT received, soft-stop')
self._softstop = True
self._listen = False
def sigterm (self,signum, frame):
self.signal_log.critical('SIG TERM received, shutdown request')
if os.environ.get('PDB',False):
self._pdb = True
else:
self._shutdown = True
# def sigabrt (self,signum, frame):
# self.signal_log.info('SIG INFO received, refork request')
# self._refork = True
# def sighup (self,signum, frame):
# self.signal_log.info('SIG HUP received, reload request')
# self._reload = True
def sigtrap (self,signum, frame):
self.signal_log.critical('SIG TRAP received, toggle debug')
self._toggle_debug = True
def sigusr1 (self,signum, frame):
self.signal_log.critical('SIG USR1 received, decrease worker number')
self._decrease_spawn_limit += 1
def sigusr2 (self,signum, frame):
self.signal_log.critical('SIG USR2 received, increase worker number')
self._increase_spawn_limit += 1
def sigttou (self,signum, frame):
self.signal_log.critical('SIG TTOU received, stop listening')
self._listen = False
def sigttin (self,signum, frame):
self.signal_log.critical('SIG IN received, star listening')
self._listen = True
def sigalrm (self,signum, frame):
self.signal_log.debug('SIG ALRM received, timed actions')
self.reactor.running = False
signal.setitimer(signal.ITIMER_REAL,self.alarm_time,self.alarm_time)
def interfaces (self):
local = set(['127.0.0.1','::1'])
for interface in getifaddrs():
if interface.family not in (AF_INET,AF_INET6):
continue
if interface.address not in self.local:
self.log.info('found new local ip %s (%s)' % (interface.address,interface.name))
local.add(interface.address)
for ip in self.local:
if ip not in local:
self.log.info('removed local ip %s' % ip)
if local == self.local:
self.log.info('no ip change')
else:
self.local = local
def run (self):
if self.daemon.drop_privileges():
self.log.critical('Could not drop privileges to \'%s\'. Refusing to run as root' % self.daemon.user)
self.log.critical('Set the environment value USER to change the unprivileged user')
self._shutdown = True
elif not self.initialise():
self._shutdown = True
signal.setitimer(signal.ITIMER_REAL,self.alarm_time,self.alarm_time)
count_second = 0
count_minute = 0
count_increase = 0
count_decrease = 0
count_saturation = 0
count_interface = 0
while True:
count_second = (count_second + 1) % self.second_frequency
count_minute = (count_minute + 1) % self.minute_frequency
count_increase = (count_increase + 1) % self.increase_frequency
count_decrease = (count_decrease + 1) % self.decrease_frequency
count_saturation = (count_saturation + 1) % self.saturation_frequency
count_interface = (count_interface + 1) % self.interface_frequency
try:
if self._pdb:
self._pdb = False
import pdb
pdb.set_trace()
# check for IO change with select
self.reactor.run()
# must follow the reactor so we are sure to go through the reactor at least once
# and flush any logs
if self._shutdown:
self._shutdown = False
self.shutdown()
break
elif self._reload:
self._reload = False
self.reload()
elif self._refork:
self._refork = False
self.signal_log.warning('refork not implemented')
# stop listening to new connections
# refork the program (as we have been updated)
# just handle current open connection
if self._softstop:
if self._listen == False:
self.proxy.rejecting()
self._listen = None
if self.client.softstop():
self._shutdown = True
# only change listening if we are not shutting down
elif self._listen is not None:
if self._listen:
self._shutdown = not self.proxy.accepting()
self._listen = None
else:
self.proxy.rejecting()
self._listen = None
if self._toggle_debug:
self._toggle_debug = False
self.log_writer.toggleDebug()
if self._increase_spawn_limit:
number = self._increase_spawn_limit
self._increase_spawn_limit = 0
self.manager.low += number
self.manager.high = max(self.manager.low,self.manager.high)
for _ in range(number):
self.manager.increase()
if self._decrease_spawn_limit:
number = self._decrease_spawn_limit
self._decrease_spawn_limit = 0
self.manager.high = max(1,self.manager.high-number)
self.manager.low = min(self.manager.high,self.manager.low)
for _ in range(number):
self.manager.decrease()
# save our monitoring stats
if count_second == 0:
self.monitor.second()
expired = self.reactor.client.expire()
self.reactor.log.debug('events : ' + ', '.join('%s:%d' % (k,len(v)) for (k,v) in self.reactor.events.items()))
else:
expired = 0
if expired:
self.proxy.notifyClose(None, count=expired)
if count_minute == 0:
self.monitor.minute()
# make sure we have enough workers
if count_increase == 0:
self.manager.provision()
# and every so often remove useless workers
if count_decrease == 0:
self.manager.deprovision()
# report if we saw too many connections
if count_saturation == 0:
self.proxy.saturation()
self.web.saturation()
if self.configuration.daemon.poll_interfaces and count_interface == 0:
self.interfaces()
except KeyboardInterrupt:
self.log.critical('^C received')
self._shutdown = True
except OSError,e:
# This shoould never happen as we are limiting how many connections we accept
if e.errno == 24: # Too many open files
self.log.critical('Too many opened files, shutting down')
for line in traceback.format_exc().split('\n'):
self.log.critical(line)
self._shutdown = True
else:
self.log.critical('unrecoverable io error')
for line in traceback.format_exc().split('\n'):
self.log.critical(line)
self._shutdown = True
finally:
class ClientManager (object):
unproxy = ProxyProtocol().parseRequest
def __init__(self, poller, configuration):
self.total_sent4 = 0L
self.total_sent6 = 0L
self.total_requested = 0L
self.norequest = TimeCache(configuration.http.idle_connect)
self.bysock = {}
self.byname = {}
self.buffered = []
self._nextid = 0
self.poller = poller
self.log = Logger('client', configuration.log.client)
self.proxied = configuration.http.proxied
self.max_buffer = configuration.http.header_size
def __contains__(self, item):
return item in self.byname
def getnextid(self):
self._nextid += 1
return str(self._nextid)
def expire (self,number=100):
count = 0
for sock in self.norequest.expired(number):
client = self.norequest.get(sock,[None,])[0]
if client:
self.cleanup(sock,client.name)
count += 1
return count
def newConnection(self, sock, peer, source):
name = self.getnextid()
client = Client(name, sock, peer, self.log, self.max_buffer)
self.norequest[sock] = client, source
self.byname[name] = client, source
# watch for the opening request
self.poller.addReadSocket('opening_client', client.sock)
#self.log.info('new id %s (socket %s) in clients : %s' % (name, sock, sock in self.bysock))
return peer
def readRequest(self, sock):
"""Read only the initial HTTP headers sent by the client"""
client, source = self.norequest.get(sock, (None, None))
if client:
name, peer, request, content = client.readData()
if request:
self.total_requested += 1
# headers can be read only once
self.norequest.pop(sock, (None, None))
# we have now read the client's opening request
self.poller.removeReadSocket('opening_client', client.sock)
elif request is None:
self.cleanup(sock, client.name)
else:
self.log.error('trying to read headers from a client that does not exist %s' % sock)
name, peer, request, content, source = None, None, None, None, None
if request and self.proxied is True and source == 'proxy':
client_ip, client_request = self.unproxy(request)
if client_ip and client_request:
peer = client_ip
request = client_request
client.setPeer(client_ip)
return name, peer, request, content, source
def readDataBySocket(self, sock):
client, source = self.bysock.get(sock, (None, None))
if client:
name, peer, request, content = client.readData()
if request:
self.total_requested += 1
# Parsing of the new request will be handled asynchronously. Ensure that
# we do not read anything from the client until a request has been sent
# to the remote webserver.
# Since we just read a request, we know that the cork is not currently
# set and so there's no risk of it being erroneously removed.
self.poller.corkReadSocket('read_client', sock)
elif request is None:
self.cleanup(sock, client.name)
else:
self.log.error('trying to read from a client that does not exist %s' % sock)
name, peer, request, content = None, None, None, None
return name, peer, request, content, source
def readDataByName(self, name):
client, source = self.byname.get(name, (None, None))
if client:
name, peer, request, content = client.readData()
if request:
self.total_requested += 1
# Parsing of the new request will be handled asynchronously. Ensure that
# we do not read anything from the client until a request has been sent
# to the remote webserver.
# Since we just read a request, we know that the cork is not currently
# set and so there's no risk of it being erroneously removed.
self.poller.corkReadSocket('read_client', client.sock)
elif request is None:
self.cleanup(client.sock, name)
else:
self.log.error('trying to read from a client that does not exist %s' % name)
name, peer, request, content = None, None, None, None
return name, peer, request, content
def sendDataBySocket(self, sock, data):
client, source = self.bysock.get(sock, (None, None))
if client:
name = client.name
res = client.writeData(data)
if res is None:
# close the client connection
self.cleanup(sock, client.name)
buffered, had_buffer, sent4, sent6 = None, None, 0, 0
result = None
buffer_change = None
else:
buffered, had_buffer, sent4, sent6 = res
self.total_sent4 += sent4
self.total_sent6 += sent6
result = buffered
if buffered:
if sock not in self.buffered:
self.buffered.append(sock)
buffer_change = True
# watch for the socket's send buffer becoming less than full
self.poller.addWriteSocket('write_client', client.sock)
else:
buffer_change = False
elif had_buffer and sock in self.buffered:
self.buffered.remove(sock)
buffer_change = True
# we no longer care about writing to the client
self.poller.removeWriteSocket('write_client', client.sock)
else:
buffer_change = False
else:
result = None
buffer_change = None
name = None
return result, buffer_change, name, source
def sendDataByName(self, name, data):
client, source = self.byname.get(name, (None, None))
if client:
res = client.writeData(data)
if res is None:
# we cannot write to the client so clean it up
self.cleanup(client.sock, name)
buffered, had_buffer, sent4, sent6 = None, None, 0, 0
result = None
buffer_change = None
else:
buffered, had_buffer, sent4, sent6 = res
self.total_sent4 += sent4
self.total_sent6 += sent6
result = buffered
if buffered:
if client.sock not in self.buffered:
self.buffered.append(client.sock)
buffer_change = True
# watch for the socket's send buffer becoming less than full
self.poller.addWriteSocket('write_client', client.sock)
else:
buffer_change = False
elif had_buffer and client.sock in self.buffered:
self.buffered.remove(client.sock)
buffer_change = True
# we no longer care about writing to the client
self.poller.removeWriteSocket('write_client', client.sock)
else:
buffer_change = False
else:
result = None
buffer_change = None
return result, buffer_change, client
def startData(self, name, data, remaining):
# NOTE: soo ugly but fast to code
nb_to_read = 0
if type(remaining) == type(''):
if 'chunked' in remaining:
mode = 'chunked'
else:
mode = 'passthrough'
elif remaining > 0:
mode = 'transfer'
nb_to_read = remaining
elif remaining == 0:
mode = 'request'
else:
mode = 'passthrough'
client, source = self.byname.get(name, (None, None))
if client:
try:
command, d = data
except (ValueError, TypeError):
self.log.error('invalid command sent to client %s' % name)
self.cleanup(client.sock, name)
res = None
else:
if client.sock not in self.bysock:
# Start checking for content sent by the client
self.bysock[client.sock] = client, source
# watch for the client sending new data
self.poller.addReadSocket('read_client', client.sock)
# make sure we don't somehow end up with this still here
self.norequest.pop(client.sock, (None,None))
# NOTE: always done already in readRequest
self.poller.removeReadSocket('opening_client', client.sock)
res = client.startData(command, d)
else:
res = client.restartData(command, d)
# If we are here then we must have prohibited reading from the client
# and it must otherwise have been in a readable state
self.poller.uncorkReadSocket('read_client', client.sock)
if res is not None:
buffered, had_buffer, sent4, sent6 = res
# buffered data we read with the HTTP headers
name, peer, request, content = client.readRelated(mode,nb_to_read)
if request:
self.total_requested += 1
self.log.info('reading multiple requests')
self.cleanup(client.sock, name)
buffered, had_buffer = None, None
content = None
elif request is None:
self.cleanup(client.sock, name)
buffered, had_buffer = None, None
content = None
else:
# we cannot write to the client so clean it up
self.cleanup(client.sock, name)
buffered, had_buffer = None, None
content = None
if buffered:
if client.sock not in self.buffered:
self.buffered.append(client.sock)
# watch for the socket's send buffer becoming less than full
self.poller.addWriteSocket('write_client', client.sock)
elif had_buffer and client.sock in self.buffered:
self.buffered.remove(client.sock)
# we no longer care about writing to the client
self.poller.removeWriteSocket('write_client', client.sock)
else:
content = None
return client, content, source
def corkUploadByName(self, name):
client, source = self.byname.get(name, (None, None))
if client:
self.poller.corkReadSocket('read_client', client.sock)
def uncorkUploadByName(self, name):
client, source = self.byname.get(name, (None, None))
if client:
if client.sock in self.bysock:
self.poller.uncorkReadSocket('read_client', client.sock)
def cleanup(self, sock, name):
self.log.debug('cleanup for socket %s' % sock)
client, source = self.bysock.get(sock, (None,None))
client, source = (client,None) if client else self.norequest.get(sock, (None,None))
client, source = (client,None) or self.byname.get(name, (None,None))
self.bysock.pop(sock, None)
self.norequest.pop(sock, (None,None))
self.byname.pop(name, None)
if client:
self.poller.removeWriteSocket('write_client', client.sock)
self.poller.removeReadSocket('read_client', client.sock)
self.poller.removeReadSocket('opening_client', client.sock)
client.shutdown()
else:
self.log.error('COULD NOT CLEAN UP SOCKET %s' % sock)
if sock in self.buffered:
self.buffered.remove(sock)
def softstop (self):
if len(self.byname) > 0 or len(self.norequest) > 0:
return False
self.log.critical('no more client connection, exiting.')
return True
def stop(self):
for client, source in self.bysock.itervalues():
client.shutdown()
for client, source in self.norequest.itervalues():
client.shutdown()
self.poller.clearRead('read_client')
self.poller.clearRead('opening_client')
self.poller.clearWrite('write_client')
self.bysock = {}
self.norequest = {}
self.byname = {}
self.buffered = []
class Supervisor (object):
alarm_time = 0.1 # regular backend work
second_frequency = int(1/alarm_time) # when we record history
minute_frequency = int(60/alarm_time) # when we want to average history
increase_frequency = int(5/alarm_time) # when we add workers
decrease_frequency = int(60/alarm_time) # when we remove workers
saturation_frequency = int(20/alarm_time) # when we report connection saturation
interface_frequency = int(300/alarm_time) # when we check for new interfaces
# import os
# clear = [hex(ord(c)) for c in os.popen('clear').read()]
# clear = ''.join([chr(int(c,16)) for c in ['0x1b', '0x5b', '0x48', '0x1b', '0x5b', '0x32', '0x4a']])
def __init__ (self,configuration):
configuration = load()
self.configuration = configuration
# Only here so the introspection code can find them
self.log = Logger('supervisor', configuration.log.supervisor)
self.log.error('Starting exaproxy version %s' % configuration.proxy.version)
self.signal_log = Logger('signal', configuration.log.signal)
self.log_writer = SysLogWriter('log', configuration.log.destination, configuration.log.enable, level=configuration.log.level)
self.usage_writer = UsageWriter('usage', configuration.usage.destination, configuration.usage.enable)
sys.exitfunc = self.log_writer.writeMessages
self.log_writer.setIdentifier(configuration.daemon.identifier)
#self.usage_writer.setIdentifier(configuration.daemon.identifier)
if configuration.debug.log:
self.log_writer.toggleDebug()
self.usage_writer.toggleDebug()
self.log.error('python version %s' % sys.version.replace(os.linesep,' '))
self.log.debug('starting %s' % sys.argv[0])
self.pid = PID(self.configuration)
self.daemon = Daemon(self.configuration)
self.poller = Poller(self.configuration.daemon)
self.poller.setupRead('read_proxy') # Listening proxy sockets
self.poller.setupRead('read_web') # Listening webserver sockets
self.poller.setupRead('read_icap') # Listening icap sockets
self.poller.setupRead('read_redirector') # Pipes carrying responses from the redirector process
self.poller.setupRead('read_resolver') # Sockets currently listening for DNS responses
self.poller.setupRead('read_client') # Active clients
self.poller.setupRead('opening_client') # Clients we have not yet read a request from
self.poller.setupWrite('write_client') # Active clients with buffered data to send
self.poller.setupWrite('write_resolver') # Active DNS requests with buffered data to send
self.poller.setupRead('read_download') # Established connections
self.poller.setupWrite('write_download') # Established connections we have buffered data to send to
self.poller.setupWrite('opening_download') # Opening connections
self.monitor = Monitor(self)
self.page = Page(self)
self.content = ContentManager(self,configuration)
self.client = ClientManager(self.poller, configuration)
self.resolver = ResolverManager(self.poller, self.configuration, configuration.dns.retries*10)
self.proxy = Server('http proxy',self.poller,'read_proxy', configuration.http.connections)
self.web = Server('web server',self.poller,'read_web', configuration.web.connections)
self.icap = Server('icap server',self.poller,'read_icap', configuration.icap.connections)
self._shutdown = True if self.daemon.filemax == 0 else False # stop the program
self._softstop = False # stop once all current connection have been dealt with
self._reload = False # unimplemented
self._toggle_debug = False # start logging a lot
self._decrease_spawn_limit = 0
self._increase_spawn_limit = 0
self._refork = False # unimplemented
self._pdb = False # turn on pdb debugging
self._listen = None # listening change ? None: no, True: listen, False: stop listeing
self.wait_time = 5.0 # how long do we wait at maximum once we have been soft-killed
self.local = set() # what addresses are on our local interfaces
if not self.initialise():
self._shutdown = True
elif self.daemon.drop_privileges():
self.log.critical('Could not drop privileges to \'%s\'. Refusing to run as root' % self.daemon.user)
self.log.critical('Set the environment value USER to change the unprivileged user')
self._shutdown = True
# fork the redirector process before performing any further setup
redirector = fork_redirector(self.poller, self.configuration)
# create threads _after_ all forking is done
self.redirector = redirector_message_thread(redirector)
self.reactor = Reactor(self.configuration, self.web, self.proxy, self.icap, self.redirector, self.content, self.client, self.resolver, self.log_writer, self.usage_writer, self.poller)
self.interfaces()
signal.signal(signal.SIGQUIT, self.sigquit)
signal.signal(signal.SIGINT, self.sigterm)
signal.signal(signal.SIGTERM, self.sigterm)
# signal.signal(signal.SIGABRT, self.sigabrt)
# signal.signal(signal.SIGHUP, self.sighup)
signal.signal(signal.SIGTRAP, self.sigtrap)
signal.signal(signal.SIGUSR1, self.sigusr1)
signal.signal(signal.SIGUSR2, self.sigusr2)
signal.signal(signal.SIGTTOU, self.sigttou)
signal.signal(signal.SIGTTIN, self.sigttin)
signal.signal(signal.SIGALRM, self.sigalrm)
# make sure we always have data in history
# (done in zero for dependencies reasons)
self.monitor.zero()
def exit (self):
sys.exit()
def sigquit (self,signum, frame):
if self._softstop:
self.signal_log.critical('multiple SIG INT received, shutdown')
self._shutdown = True
else:
self.signal_log.critical('SIG INT received, soft-stop')
self._softstop = True
self._listen = False
def sigterm (self,signum, frame):
self.signal_log.critical('SIG TERM received, shutdown request')
if os.environ.get('PDB',False):
self._pdb = True
else:
self._shutdown = True
# def sigabrt (self,signum, frame):
# self.signal_log.info('SIG INFO received, refork request')
# self._refork = True
# def sighup (self,signum, frame):
# self.signal_log.info('SIG HUP received, reload request')
# self._reload = True
def sigtrap (self,signum, frame):
self.signal_log.critical('SIG TRAP received, toggle debug')
self._toggle_debug = True
def sigusr1 (self,signum, frame):
self.signal_log.critical('SIG USR1 received, decrease worker number')
self._decrease_spawn_limit += 1
def sigusr2 (self,signum, frame):
self.signal_log.critical('SIG USR2 received, increase worker number')
self._increase_spawn_limit += 1
def sigttou (self,signum, frame):
self.signal_log.critical('SIG TTOU received, stop listening')
self._listen = False
def sigttin (self,signum, frame):
self.signal_log.critical('SIG IN received, star listening')
self._listen = True
def sigalrm (self,signum, frame):
self.reactor.running = False
signal.setitimer(signal.ITIMER_REAL,self.alarm_time,self.alarm_time)
def interfaces (self):
local = set(['127.0.0.1','::1'])
for interface in getifaddrs():
if interface.family not in (AF_INET,AF_INET6):
continue
if interface.address not in self.local:
self.log.info('found new local ip %s (%s)' % (interface.address,interface.name))
local.add(interface.address)
for ip in self.local:
if ip not in local:
self.log.info('removed local ip %s' % ip)
if local == self.local:
self.log.info('no ip change')
else:
self.local = local
def run (self):
signal.setitimer(signal.ITIMER_REAL,self.alarm_time,self.alarm_time)
count_second = 0
count_minute = 0
count_saturation = 0
count_interface = 0
while True:
count_second = (count_second + 1) % self.second_frequency
count_minute = (count_minute + 1) % self.minute_frequency
count_saturation = (count_saturation + 1) % self.saturation_frequency
count_interface = (count_interface + 1) % self.interface_frequency
try:
if self._pdb:
self._pdb = False
import pdb
pdb.set_trace()
# check for IO change with select
status = self.reactor.run()
if status is False:
self._shutdown = True
# must follow the reactor so we are sure to go through the reactor at least once
# and flush any logs
if self._shutdown:
self._shutdown = False
self.shutdown()
break
elif self._reload:
self._reload = False
self.reload()
elif self._refork:
self._refork = False
self.signal_log.warning('refork not implemented')
# stop listening to new connections
# refork the program (as we have been updated)
# just handle current open connection
if self._softstop:
if self._listen == False:
self.proxy.rejecting()
self._listen = None
if self.client.softstop():
self._shutdown = True
# only change listening if we are not shutting down
elif self._listen is not None:
if self._listen:
self._shutdown = not self.proxy.accepting()
self._listen = None
else:
self.proxy.rejecting()
self._listen = None
if self._toggle_debug:
self._toggle_debug = False
self.log_writer.toggleDebug()
if self._decrease_spawn_limit:
count = self._decrease_spawn_limit
self.redirector.decreaseSpawnLimit(count)
self._decrease_spawn_limit = 0
if self._increase_spawn_limit:
count = self._increase_spawn_limit
self.redirector.increaseSpawnLimit(count)
self._increase_spawn_limit = 0
# save our monitoring stats
if count_second == 0:
self.monitor.second()
expired = self.reactor.client.expire()
else:
expired = 0
if expired:
self.proxy.notifyClose(None, count=expired)
if count_minute == 0:
self.monitor.minute()
# report if we saw too many connections
if count_saturation == 0:
self.proxy.saturation()
self.web.saturation()
if self.configuration.daemon.poll_interfaces and count_interface == 0:
self.interfaces()
except KeyboardInterrupt:
self.log.critical('^C received')
self._shutdown = True
except OSError,e:
# This shoould never happen as we are limiting how many connections we accept
if e.errno == 24: # Too many open files
self.log.critical('Too many opened files, shutting down')
for line in traceback.format_exc().split('\n'):
self.log.critical(line)
self._shutdown = True
else:
self.log.critical('unrecoverable io error')
for line in traceback.format_exc().split('\n'):
self.log.critical(line)
self._shutdown = True
finally:
try:
import cProfile as profile
except:
import profile
if not configuration.profile.destination or configuration.profile.destination == 'stdout':
profile.run('Supervisor().run()')
__exit(configuration.debug.memory, 0)
notice = ''
profiled = configuration.profile.destination
if os.path.isdir(profiled):
notice = 'profile can not use this filename as outpout, it is not a directory (%s)' % profiled
if os.path.exists(configuration.profile.destination):
notice = 'profile can not use this filename as outpout, it already exists (%s)' % profiled
if not notice:
log.debug('profiling ....')
profile.run('main()', filename=configuration.profile.destination)
else:
log.debug("-" * len(notice))
log.debug(notice)
log.debug("-" * len(notice))
main()
__exit(configuration.debug.memory, 0)
if __name__ == '__main__':
main()
class ClientManager(object):
unproxy = ProxyProtocol().parseRequest
def __init__(self, poller, configuration):
self.total_sent4 = 0L
self.total_sent6 = 0L
self.total_requested = 0L
self.norequest = TimeCache(configuration.http.idle_connect)
self.bysock = {}
self.byname = {}
self.buffered = []
self._nextid = 0
self.poller = poller
self.log = Logger('client', configuration.log.client)
self.proxied = configuration.http.proxied
self.max_buffer = configuration.http.header_size
def __contains__(self, item):
return item in self.byname
def getnextid(self):
self._nextid += 1
return str(self._nextid)
def expire(self, number=100):
count = 0
for sock in self.norequest.expired(number):
client = self.norequest.get(sock, [
None,
])[0]
if client:
self.cleanup(sock, client.name)
count += 1
return count
def newConnection(self, sock, peer, source):
name = self.getnextid()
client = Client(name, sock, peer, self.log, self.max_buffer)
self.norequest[sock] = client, source
self.byname[name] = client, source
# watch for the opening request
self.poller.addReadSocket('opening_client', client.sock)
#self.log.info('new id %s (socket %s) in clients : %s' % (name, sock, sock in self.bysock))
return peer
def readRequest(self, sock):
"""Read only the initial HTTP headers sent by the client"""
client, source = self.norequest.get(sock, (None, None))
if client:
name, peer, request, content = client.readData()
if request:
self.total_requested += 1
# headers can be read only once
self.norequest.pop(sock, (None, None))
# we have now read the client's opening request
self.poller.removeReadSocket('opening_client', client.sock)
elif request is None:
self.cleanup(sock, client.name)
else:
self.log.error(
'trying to read headers from a client that does not exist %s' %
sock)
name, peer, request, content, source = None, None, None, None, None
if request and self.proxied is True and source == 'proxy':
client_ip, client_request = self.unproxy(request)
if client_ip and client_request:
peer = client_ip
request = client_request
client.setPeer(client_ip)
return name, peer, request, content, source
def readDataBySocket(self, sock):
client, source = self.bysock.get(sock, (None, None))
if client:
name, peer, request, content = client.readData()
if request:
self.total_requested += 1
# Parsing of the new request will be handled asynchronously. Ensure that
# we do not read anything from the client until a request has been sent
# to the remote webserver.
# Since we just read a request, we know that the cork is not currently
# set and so there's no risk of it being erroneously removed.
self.poller.corkReadSocket('read_client', sock)
elif request is None:
self.cleanup(sock, client.name)
else:
self.log.error(
'trying to read from a client that does not exist %s' % sock)
name, peer, request, content = None, None, None, None
return name, peer, request, content, source
def readDataByName(self, name):
client, source = self.byname.get(name, (None, None))
if client:
name, peer, request, content = client.readData()
if request:
self.total_requested += 1
# Parsing of the new request will be handled asynchronously. Ensure that
# we do not read anything from the client until a request has been sent
# to the remote webserver.
# Since we just read a request, we know that the cork is not currently
# set and so there's no risk of it being erroneously removed.
self.poller.corkReadSocket('read_client', client.sock)
elif request is None:
self.cleanup(client.sock, name)
else:
self.log.error(
'trying to read from a client that does not exist %s' % name)
name, peer, request, content = None, None, None, None
return name, peer, request, content
def sendDataBySocket(self, sock, data):
client, source = self.bysock.get(sock, (None, None))
if client:
name = client.name
res = client.writeData(data)
if res is None:
# close the client connection
self.cleanup(sock, client.name)
buffered, had_buffer, sent4, sent6 = None, None, 0, 0
result = None
buffer_change = None
else:
buffered, had_buffer, sent4, sent6 = res
self.total_sent4 += sent4
self.total_sent6 += sent6
result = buffered
if buffered:
if sock not in self.buffered:
self.buffered.append(sock)
buffer_change = True
# watch for the socket's send buffer becoming less than full
self.poller.addWriteSocket('write_client', client.sock)
else:
buffer_change = False
elif had_buffer and sock in self.buffered:
self.buffered.remove(sock)
buffer_change = True
# we no longer care about writing to the client
self.poller.removeWriteSocket('write_client', client.sock)
else:
buffer_change = False
else:
result = None
buffer_change = None
name = None
return result, buffer_change, name, source
def sendDataByName(self, name, data):
client, source = self.byname.get(name, (None, None))
if client:
res = client.writeData(data)
if res is None:
# we cannot write to the client so clean it up
self.cleanup(client.sock, name)
buffered, had_buffer, sent4, sent6 = None, None, 0, 0
result = None
buffer_change = None
else:
buffered, had_buffer, sent4, sent6 = res
self.total_sent4 += sent4
self.total_sent6 += sent6
result = buffered
if buffered:
if client.sock not in self.buffered:
self.buffered.append(client.sock)
buffer_change = True
# watch for the socket's send buffer becoming less than full
self.poller.addWriteSocket('write_client', client.sock)
else:
buffer_change = False
elif had_buffer and client.sock in self.buffered:
self.buffered.remove(client.sock)
buffer_change = True
# we no longer care about writing to the client
self.poller.removeWriteSocket('write_client', client.sock)
else:
buffer_change = False
else:
result = None
buffer_change = None
return result, buffer_change, client
def startData(self, name, data, remaining):
# NOTE: soo ugly but fast to code
nb_to_read = 0
if type(remaining) == type(''):
if 'chunked' in remaining:
mode = 'chunked'
else:
mode = 'passthrough'
elif remaining > 0:
mode = 'transfer'
nb_to_read = remaining
elif remaining == 0:
mode = 'request'
else:
mode = 'passthrough'
client, source = self.byname.get(name, (None, None))
if client:
try:
command, d = data
except (ValueError, TypeError):
self.log.error('invalid command sent to client %s' % name)
self.cleanup(client.sock, name)
res = None
else:
if client.sock not in self.bysock:
# Start checking for content sent by the client
self.bysock[client.sock] = client, source
# watch for the client sending new data
self.poller.addReadSocket('read_client', client.sock)
# make sure we don't somehow end up with this still here
self.norequest.pop(client.sock, (None, None))
# NOTE: always done already in readRequest
self.poller.removeReadSocket('opening_client', client.sock)
res = client.startData(command, d)
else:
res = client.restartData(command, d)
# If we are here then we must have prohibited reading from the client
# and it must otherwise have been in a readable state
self.poller.uncorkReadSocket('read_client', client.sock)
if res is not None:
buffered, had_buffer, sent4, sent6 = res
# buffered data we read with the HTTP headers
name, peer, request, content = client.readRelated(
mode, nb_to_read)
if request:
self.total_requested += 1
self.log.info('reading multiple requests')
self.cleanup(client.sock, name)
buffered, had_buffer = None, None
content = None
elif request is None:
self.cleanup(client.sock, name)
buffered, had_buffer = None, None
content = None
else:
# we cannot write to the client so clean it up
self.cleanup(client.sock, name)
buffered, had_buffer = None, None
content = None
if buffered:
if client.sock not in self.buffered:
self.buffered.append(client.sock)
# watch for the socket's send buffer becoming less than full
self.poller.addWriteSocket('write_client', client.sock)
elif had_buffer and client.sock in self.buffered:
self.buffered.remove(client.sock)
# we no longer care about writing to the client
self.poller.removeWriteSocket('write_client', client.sock)
else:
content = None
return client, content, source
def corkUploadByName(self, name):
client, source = self.byname.get(name, (None, None))
if client:
self.poller.corkReadSocket('read_client', client.sock)
def uncorkUploadByName(self, name):
client, source = self.byname.get(name, (None, None))
if client:
if client.sock in self.bysock:
self.poller.uncorkReadSocket('read_client', client.sock)
def cleanup(self, sock, name):
self.log.debug('cleanup for socket %s' % sock)
client, source = self.bysock.get(sock, (None, None))
client, source = (client, None) if client else self.norequest.get(
sock, (None, None))
client, source = (client, None) or self.byname.get(name, (None, None))
self.bysock.pop(sock, None)
self.norequest.pop(sock, (None, None))
self.byname.pop(name, None)
if client:
self.poller.removeWriteSocket('write_client', client.sock)
self.poller.removeReadSocket('read_client', client.sock)
self.poller.removeReadSocket('opening_client', client.sock)
client.shutdown()
else:
self.log.error('COULD NOT CLEAN UP SOCKET %s' % sock)
if sock in self.buffered:
self.buffered.remove(sock)
def softstop(self):
if len(self.byname) > 0 or len(self.norequest) > 0:
return False
self.log.critical('no more client connection, exiting.')
return True
def stop(self):
for client, source in self.bysock.itervalues():
client.shutdown()
for client, source in self.norequest.itervalues():
client.shutdown()
self.poller.clearRead('read_client')
self.poller.clearRead('opening_client')
self.poller.clearWrite('write_client')
self.bysock = {}
self.norequest = {}
self.byname = {}
self.buffered = []
class ContentManager(object):
downloader_factory = Content
def __init__(self, supervisor, configuration):
self.total_sent4 = 0L
self.total_sent6 = 0L
self.opening = {}
self.established = {}
self.byclientid = {}
self.buffered = []
self.retry = []
self.configuration = configuration
self.supervisor = supervisor
self.poller = supervisor.poller
self.log = Logger('download', configuration.log.download)
self.location = os.path.realpath(os.path.normpath(configuration.web.html))
self.page = supervisor.page
self._header = {}
def hasClient(self, client_id):
return client_id in self.byclientid
def getLocalContent(self, code, name):
filename = os.path.normpath(os.path.join(self.location, name))
if not filename.startswith(self.location + os.path.sep):
filename = ''
if os.path.isfile(filename):
try:
stat = os.stat(filename)
except IOError:
# NOTE: we are always returning an HTTP/1.1 response
content = 'close', http(501, 'local file is inaccessible %s' % str(filename))
else:
if filename in self._header :
cache_time, header = self._header[filename]
else:
cache_time, header = None, None
if cache_time is None or cache_time < stat.st_mtime:
header = file_header(code, stat.st_size, filename)
self._header[filename] = stat.st_size, header
content = 'file', (header, filename)
else:
self.log.debug('local file is missing for %s: %s' % (str(name), str(filename)))
# NOTE: we are always returning an HTTP/1.1 response
content = 'close', http(501, 'could not serve missing file %s' % str(filename))
return content
def readLocalContent(self, code, reason, data={}):
filename = os.path.normpath(os.path.join(self.location, reason))
if not filename.startswith(self.location + os.path.sep):
filename = ''
if os.path.isfile(filename):
try:
with open(filename) as fd:
body = fd.read() % data
# NOTE: we are always returning an HTTP/1.1 response
content = 'close', http(code, body)
except IOError:
self.log.debug('local file is missing for %s: %s' % (str(reason), str(filename)))
# NOTE: we are always returning an HTTP/1.1 response
content = 'close', http(501, 'could not serve missing file %s' % str(reason))
else:
self.log.debug('local file is missing for %s: %s' % (str(reason), str(filename)))
# NOTE: we are always returning an HTTP/1.1 response
content = 'close', http(501, 'could not serve missing file %s' % str(reason))
return content
def getDownloader(self, client_id, host, port, command, request):
downloader = self.byclientid.get(client_id, None)
if downloader:
# NOTE: with pipeline, consequent request could go to other sites if the browser knows we are a proxy
# NOTE: therefore the second request could reach the first site
# NOTE: and we could kill the connection before the data is fully back to the client
# NOTE: in practice modern browser are too clever and test for it !
if host != downloader.host or port != downloader.port:
self.endClientDownload(client_id)
downloader = None
else:
newdownloader = False
if isipv4(host):
bind = self.configuration.tcp4.bind
elif isipv6(host):
bind = self.configuration.tcp6.bind
else:
# should really never happen
self.log.critical('the host IP address is neither IPv4 or IPv6 .. what year is it ?')
return None, False
if downloader is None:
# supervisor.local is replaced when interface are changed, so do not cache or reference it in this class
if host in self.supervisor.local:
for h,p in self.configuration.security.local:
if (h == '*' or h == host) and (p == '*' or p == port):
break
else:
# we did not break
return None, False
downloader = self.downloader_factory(client_id, host, port, bind, command, request, self.log)
newdownloader = True
if downloader.sock is None:
return None, False
return downloader, newdownloader
def getContent(self, client_id, command, args):
try:
if command == 'download':
try:
host, port, upgrade, length, request = args.split('\0', 4)
except (ValueError, TypeError), e:
raise ParsingError()
downloader, newdownloader = self.getDownloader(client_id, host, int(port), command, request)
if downloader is not None:
content = ('stream', '')
if upgrade in ('', 'http/1.0', 'http/1.1'):
length = int(length) if length.isdigit() else length
else:
length = -1
else:
content = self.getLocalContent('400', 'noconnect.html')
length = 0
elif command == 'connect':
try:
host, port, request = args.split('\0', 2)
except (ValueError, TypeError), e:
raise ParsingError()
downloader, newdownloader = self.getDownloader(client_id, host, int(port), command, '')
if downloader is not None:
content = ('stream', '')
length = -1 # the client can send as much data as it wants
else:
content = self.getLocalContent('400', 'noconnect.html')
length = 0
__exit(configuration.debug.memory,0)
try:
import cProfile as profile
except:
import profile
if not configuration.profile.destination or configuration.profile.destination == 'stdout':
profile.run('Supervisor().run()')
__exit(configuration.debug.memory,0)
notice = ''
profiled = configuration.profile.destination
if os.path.isdir(profiled):
notice = 'profile can not use this filename as outpout, it is not a directory (%s)' % profiled
if os.path.exists(configuration.profile.destination):
notice = 'profile can not use this filename as outpout, it already exists (%s)' % profiled
if not notice:
log.debug('profiling ....')
profile.run('main()',filename=configuration.profile.destination)
else:
log.debug("-"*len(notice))
log.debug(notice)
log.debug("-"*len(notice))
main()
__exit(configuration.debug.memory,0)
if __name__ == '__main__':
main()
class Redirector(Thread):
# TODO : if the program is a function, fork and run :)
ICAPParser = ICAPParser
def __init__(self, configuration, name, request_box, program):
self.configuration = configuration
self.icap_parser = self.ICAPParser(configuration)
self.enabled = configuration.redirector.enable
self.protocol = configuration.redirector.protocol
self._transparent = configuration.http.transparent
self.log = Logger('worker ' + str(name), configuration.log.worker)
self.usage = UsageLogger('usage', configuration.log.worker)
self.universal = True if self.protocol == 'url' else False
self.icap = self.protocol[len('icap://'):].split(
'/')[0] if self.protocol.startswith('icap://') else ''
r, w = os.pipe() # pipe for communication with the main thread
self.response_box_write = os.fdopen(w, 'w',
0) # results are written here
self.response_box_read = os.fdopen(r, 'r',
0) # read from the main thread
self.wid = name # a unique name
self.creation = time.time() # when the thread was created
# self.last_worked = self.creation # when the thread last picked a task
self.request_box = request_box # queue with HTTP headers to process
self.program = program # the squid redirector program to fork
self.running = True # the thread is active
self.stats_timestamp = None # time of the most recent outstanding request to generate stats
self._proxy = 'ExaProxy-%s-id-%d' % (configuration.proxy.version,
os.getpid())
if self.protocol == 'url':
self.classify = self._classify_url
if self.protocol.startswith('icap://'):
self.classify = self._classify_icap
# Do not move, we need the forking AFTER the setup
self.process = self._createProcess(
) # the forked program to handle classification
Thread.__init__(self)
def _createProcess(self):
if not self.enabled:
return
def preexec(): # Don't forward signals.
os.setpgrp()
try:
process = subprocess.Popen(
[
self.program,
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=self.universal,
preexec_fn=preexec,
)
self.log.debug('spawn process %s' % self.program)
except KeyboardInterrupt:
process = None
except (subprocess.CalledProcessError, OSError, ValueError):
self.log.error('could not spawn process %s' % self.program)
process = None
if process:
try:
fcntl.fcntl(process.stderr, fcntl.F_SETFL, os.O_NONBLOCK)
except IOError:
self.destroyProcess()
process = None
return process
def destroyProcess(self):
if not self.enabled:
return
self.log.debug('destroying process %s' % self.program)
if not self.process:
return
try:
if self.process:
self.process.terminate()
self.process.wait()
self.log.info('terminated process PID %s' % self.process.pid)
except OSError, e:
# No such processs
if e[0] != errno.ESRCH:
self.log.error('PID %s died' % self.process.pid)
class ContentManager(object):
downloader_factory = Content
def __init__(self, supervisor, configuration):
self.total_sent4 = 0L
self.total_sent6 = 0L
self.opening = {}
self.established = {}
self.byclientid = {}
self.buffered = []
self.retry = []
self.configuration = configuration
self.supervisor = supervisor
self.poller = supervisor.poller
self.log = Logger('download', configuration.log.download)
self.location = os.path.realpath(
os.path.normpath(configuration.web.html))
self.page = supervisor.page
self._header = {}
def hasClient(self, client_id):
return client_id in self.byclientid
def getLocalContent(self, code, name):
filename = os.path.normpath(os.path.join(self.location, name))
if not filename.startswith(self.location + os.path.sep):
filename = ''
if os.path.isfile(filename):
try:
stat = os.stat(filename)
except IOError:
# NOTE: we are always returning an HTTP/1.1 response
content = 'close', http(
501, 'local file is inaccessible %s' % str(filename))
else:
if filename in self._header:
cache_time, header = self._header[filename]
else:
cache_time, header = None, None
if cache_time is None or cache_time < stat.st_mtime:
header = file_header(code, stat.st_size, filename)
self._header[filename] = stat.st_size, header
content = 'file', (header, filename)
else:
self.log.debug('local file is missing for %s: %s' %
(str(name), str(filename)))
# NOTE: we are always returning an HTTP/1.1 response
content = 'close', http(
501, 'could not serve missing file %s' % str(filename))
return content
def readLocalContent(self, code, reason, data={}):
filename = os.path.normpath(os.path.join(self.location, reason))
if not filename.startswith(self.location + os.path.sep):
filename = ''
if os.path.isfile(filename):
try:
with open(filename) as fd:
body = fd.read() % data
# NOTE: we are always returning an HTTP/1.1 response
content = 'close', http(code, body)
except IOError:
self.log.debug('local file is missing for %s: %s' %
(str(reason), str(filename)))
# NOTE: we are always returning an HTTP/1.1 response
content = 'close', http(
501, 'could not serve missing file %s' % str(reason))
else:
self.log.debug('local file is missing for %s: %s' %
(str(reason), str(filename)))
# NOTE: we are always returning an HTTP/1.1 response
content = 'close', http(
501, 'could not serve missing file %s' % str(reason))
return content
def getDownloader(self, client_id, host, port, command, request):
downloader = self.byclientid.get(client_id, None)
if downloader:
# NOTE: with pipeline, consequent request could go to other sites if the browser knows we are a proxy
# NOTE: therefore the second request could reach the first site
# NOTE: and we could kill the connection before the data is fully back to the client
# NOTE: in practice modern browser are too clever and test for it !
if host != downloader.host or port != downloader.port:
self.endClientDownload(client_id)
downloader = None
else:
newdownloader = False
if isipv4(host):
bind = self.configuration.tcp4.bind
elif isipv6(host):
bind = self.configuration.tcp6.bind
else:
# should really never happen
self.log.critical(
'the host IP address is neither IPv4 or IPv6 .. what year is it ?'
)
return None, False
if downloader is None:
# supervisor.local is replaced when interface are changed, so do not cache or reference it in this class
if host in self.supervisor.local:
for h, p in self.configuration.security.local:
if (h == '*' or h == host) and (p == '*' or p == port):
break
else:
# we did not break
return None, False
downloader = self.downloader_factory(client_id, host, port, bind,
command, request, self.log)
newdownloader = True
if downloader.sock is None:
return None, False
return downloader, newdownloader
def getContent(self, client_id, command, args):
try:
if command == 'download':
try:
host, port, upgrade, length, request = args
except (ValueError, TypeError), e:
raise ParsingError()
downloader, newdownloader = self.getDownloader(
client_id, host, int(port), command, request)
if downloader is not None:
content = ('stream', '')
if upgrade in ('', 'http/1.0', 'http/1.1'):
length = int(length) if length.isdigit() else length
else:
length = -1
else:
content = self.getLocalContent('400', 'noconnect.html')
length = 0
elif command == 'connect':
try:
host, port, request = args
except (ValueError, TypeError), e:
raise ParsingError()
downloader, newdownloader = self.getDownloader(
client_id, host, int(port), command, '')
if downloader is not None:
content = ('stream', '')
length = -1 # the client can send as much data as it wants
else:
content = self.getLocalContent('400', 'noconnect.html')
length = 0
class Redirector (Thread):
# TODO : if the program is a function, fork and run :)
def __init__ (self, configuration, name, request_box, program):
self.configuration = configuration
self.enabled = configuration.redirector.enable
self.protocol = configuration.redirector.protocol
self._transparent = configuration.http.transparent
self.log = Logger('worker ' + str(name), configuration.log.worker)
self.usage = UsageLogger('usage', configuration.log.worker)
self.universal = True if self.protocol == 'url' else False
self.icap = self.protocol[len('icap://'):].split('/')[0] if self.protocol.startswith('icap://') else ''
r, w = os.pipe() # pipe for communication with the main thread
self.response_box_write = os.fdopen(w,'w',0) # results are written here
self.response_box_read = os.fdopen(r,'r',0) # read from the main thread
self.wid = name # a unique name
self.creation = time.time() # when the thread was created
# self.last_worked = self.creation # when the thread last picked a task
self.request_box = request_box # queue with HTTP headers to process
self.program = program # the squid redirector program to fork
self.running = True # the thread is active
self.stats_timestamp = None # time of the most recent outstanding request to generate stats
self._proxy = 'ExaProxy-%s-id-%d' % (configuration.proxy.version,os.getpid())
if self.protocol == 'url':
self.classify = self._classify_url
if self.protocol.startswith('icap://'):
self.classify = self._classify_icap
# Do not move, we need the forking AFTER the setup
self.process = self._createProcess() # the forked program to handle classification
Thread.__init__(self)
def _createProcess (self):
if not self.enabled:
return
def preexec(): # Don't forward signals.
os.setpgrp()
try:
process = subprocess.Popen([self.program,],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=self.universal,
preexec_fn=preexec,
)
self.log.debug('spawn process %s' % self.program)
except KeyboardInterrupt:
process = None
except (subprocess.CalledProcessError,OSError,ValueError):
self.log.error('could not spawn process %s' % self.program)
process = None
if process:
try:
fcntl.fcntl(process.stderr, fcntl.F_SETFL, os.O_NONBLOCK)
except IOError:
self.destroyProcess()
process = None
return process
def destroyProcess (self):
if not self.enabled:
return
self.log.debug('destroying process %s' % self.program)
if not self.process:
return
try:
if self.process:
self.process.terminate()
self.process.wait()
self.log.info('terminated process PID %s' % self.process.pid)
except OSError, e:
# No such processs
if e[0] != errno.ESRCH:
self.log.error('PID %s died' % self.process.pid)
class Supervisor(object):
alarm_time = 0.1 # regular backend work
second_frequency = int(1 / alarm_time) # when we record history
minute_frequency = int(60 / alarm_time) # when we want to average history
increase_frequency = int(5 / alarm_time) # when we add workers
decrease_frequency = int(60 / alarm_time) # when we remove workers
saturation_frequency = int(
20 / alarm_time) # when we report connection saturation
interface_frequency = int(300 /
alarm_time) # when we check for new interfaces
# import os
# clear = [hex(ord(c)) for c in os.popen('clear').read()]
# clear = ''.join([chr(int(c,16)) for c in ['0x1b', '0x5b', '0x48', '0x1b', '0x5b', '0x32', '0x4a']])
def __init__(self, configuration):
self.configuration = configuration
# Only here so the introspection code can find them
self.log = Logger('supervisor', configuration.log.supervisor)
self.log.error('Starting exaproxy version %s' %
configuration.proxy.version)
self.signal_log = Logger('signal', configuration.log.signal)
self.log_writer = SysLogWriter('log',
configuration.log.destination,
configuration.log.enable,
level=configuration.log.level)
self.usage_writer = UsageWriter('usage',
configuration.usage.destination,
configuration.usage.enable)
sys.exitfunc = self.log_writer.writeMessages
self.log_writer.setIdentifier(configuration.daemon.identifier)
#self.usage_writer.setIdentifier(configuration.daemon.identifier)
if configuration.debug.log:
self.log_writer.toggleDebug()
self.usage_writer.toggleDebug()
self.log.error('python version %s' %
sys.version.replace(os.linesep, ' '))
self.log.debug('starting %s' % sys.argv[0])
self.pid = PID(self.configuration)
self.daemon = Daemon(self.configuration)
self.poller = Poller(self.configuration.daemon)
self.poller.setupRead('read_proxy') # Listening proxy sockets
self.poller.setupRead('read_web') # Listening webserver sockets
self.poller.setupRead('read_icap') # Listening icap sockets
self.poller.setupRead('read_tls') # Listening tls sockets
self.poller.setupRead('read_passthrough') # Listening raw data sockets
self.poller.setupRead(
'read_redirector'
) # Pipes carrying responses from the redirector process
self.poller.setupRead(
'read_resolver') # Sockets currently listening for DNS responses
self.poller.setupRead('read_client') # Active clients
self.poller.setupRead(
'opening_client') # Clients we have not yet read a request from
self.poller.setupWrite(
'write_client') # Active clients with buffered data to send
self.poller.setupWrite(
'write_resolver') # Active DNS requests with buffered data to send
self.poller.setupRead('read_download') # Established connections
self.poller.setupWrite(
'write_download'
) # Established connections we have buffered data to send to
self.poller.setupWrite('opening_download') # Opening connections
self.poller.setupRead('read_interrupt') # Scheduled events
self.poller.setupRead(
'read_control'
) # Responses from commands sent to the redirector process
self.monitor = Monitor(self)
self.page = Page(self)
self.content = ContentManager(self, configuration)
self.client = ClientManager(self.poller, configuration)
self.resolver = ResolverManager(self.poller, self.configuration,
configuration.dns.retries * 10)
self.proxy = Server('http proxy', self.poller, 'read_proxy',
configuration.http)
self.web = Server('web server', self.poller, 'read_web',
configuration.web)
self.icap = Server('icap server', self.poller, 'read_icap',
configuration.icap)
self.tls = Server('tls server', self.poller, 'read_tls',
configuration.tls)
self.passthrough = InterceptServer('passthrough server', self.poller,
'read_passthrough',
configuration.passthrough)
self._shutdown = True if self.daemon.filemax == 0 else False # stop the program
self._softstop = False # stop once all current connection have been dealt with
self._reload = False # unimplemented
self._toggle_debug = False # start logging a lot
self._decrease_spawn_limit = 0
self._increase_spawn_limit = 0
self._refork = False # unimplemented
self._pdb = False # turn on pdb debugging
self._listen = None # listening change ? None: no, True: listen, False: stop listeing
self.wait_time = 5.0 # how long do we wait at maximum once we have been soft-killed
self.local = set() # what addresses are on our local interfaces
if not self.initialise():
self._shutdown = True
elif self.daemon.drop_privileges():
self.log.critical(
'Could not drop privileges to \'%s\'. Refusing to run as root'
% self.daemon.user)
self.log.critical(
'Set the environment value USER to change the unprivileged user'
)
self._shutdown = True
# fork the redirector process before performing any further setup
redirector = fork_redirector(self.poller, self.configuration)
# use simple blocking IO for communication with the redirector process
self.redirector = redirector_message_thread(redirector)
# NOTE: create threads _after_ all forking is done
# regularly interrupt the reactor for maintenance
self.interrupt_scheduler = alarm_thread(self.poller, self.alarm_time)
self.reactor = Reactor(self.configuration, self.web, self.proxy,
self.passthrough, self.icap, self.tls,
self.redirector, self.content, self.client,
self.resolver, self.log_writer,
self.usage_writer, self.poller)
self.interfaces()
signal.signal(signal.SIGQUIT, self.sigquit)
signal.signal(signal.SIGINT, self.sigterm)
signal.signal(signal.SIGTERM, self.sigterm)
# signal.signal(signal.SIGABRT, self.sigabrt)
# signal.signal(signal.SIGHUP, self.sighup)
signal.signal(signal.SIGTRAP, self.sigtrap)
signal.signal(signal.SIGUSR1, self.sigusr1)
signal.signal(signal.SIGUSR2, self.sigusr2)
signal.signal(signal.SIGTTOU, self.sigttou)
signal.signal(signal.SIGTTIN, self.sigttin)
# make sure we always have data in history
# (done in zero for dependencies reasons)
if self._shutdown is False:
self.redirector.requestStats()
command, control_data = self.redirector.readResponse()
stats_data = control_data if command == 'STATS' else None
stats = self.monitor.statistics(stats_data)
ok = self.monitor.zero(stats)
if ok:
self.redirector.requestStats()
else:
self._shutdown = True
def exit(self):
sys.exit()
def sigquit(self, signum, frame):
if self._softstop:
self.signal_log.critical('multiple SIG INT received, shutdown')
self._shutdown = True
else:
self.signal_log.critical('SIG INT received, soft-stop')
self._softstop = True
self._listen = False
def sigterm(self, signum, frame):
self.signal_log.critical('SIG TERM received, shutdown request')
if os.environ.get('PDB', False):
self._pdb = True
else:
self._shutdown = True
# def sigabrt (self,signum, frame):
# self.signal_log.info('SIG INFO received, refork request')
# self._refork = True
# def sighup (self,signum, frame):
# self.signal_log.info('SIG HUP received, reload request')
# self._reload = True
def sigtrap(self, signum, frame):
self.signal_log.critical('SIG TRAP received, toggle debug')
self._toggle_debug = True
def sigusr1(self, signum, frame):
self.signal_log.critical('SIG USR1 received, decrease worker number')
self._decrease_spawn_limit += 1
def sigusr2(self, signum, frame):
self.signal_log.critical('SIG USR2 received, increase worker number')
self._increase_spawn_limit += 1
def sigttou(self, signum, frame):
self.signal_log.critical('SIG TTOU received, stop listening')
self._listen = False
def sigttin(self, signum, frame):
self.signal_log.critical('SIG IN received, star listening')
self._listen = True
def interfaces(self):
local = {'127.0.0.1', '::1'}
for interface in getifaddrs():
if interface.family not in (AF_INET, AF_INET6):
continue
if interface.address not in self.local:
self.log.info('found new local ip %s (%s)' %
(interface.address, interface.name))
local.add(interface.address)
for ip in self.local:
if ip not in local:
self.log.info('removed local ip %s' % ip)
if local == self.local:
self.log.info('no ip change')
else:
self.local = local
def run(self):
count_second = 0
count_minute = 0
count_saturation = 0
count_interface = 0
events = {'read_interrupt'}
while True:
count_second = (count_second + 1) % self.second_frequency
count_minute = (count_minute + 1) % self.minute_frequency
count_saturation = (count_saturation +
1) % self.saturation_frequency
count_interface = (count_interface + 1) % self.interface_frequency
try:
if self._pdb:
self._pdb = False
import pdb
pdb.set_trace()
# prime the alarm
if 'read_interrupt' in events:
self.interrupt_scheduler.setAlarm()
# check for IO change with select
status, events = self.reactor.run()
# shut down the server if a child process disappears
if status is False:
self._shutdown = True
# respond to control responses immediately
if 'read_control' in events:
command, control_data = self.redirector.readResponse()
if command == 'STATS':
ok = self.doStats(count_second, count_minute,
control_data)
if ok is False:
self._shutdown = True
# jump straight back into the reactor if we haven't yet received an
# interrupt event
if 'read_interrupt' not in events:
continue
# clear the alarm condition
self.interrupt_scheduler.acknowledgeAlarm()
# must follow the reactor so we are sure to go through the reactor at least once
# and flush any logs
if self._shutdown:
self._shutdown = False
self.shutdown()
break
elif self._reload:
self._reload = False
self.reload()
elif self._refork:
self._refork = False
self.signal_log.warning('refork not implemented')
# stop listening to new connections
# refork the program (as we have been updated)
# just handle current open connection
# ask the redirector process for stats
self.redirector.requestStats()
if self._softstop:
if self._listen == False:
self.proxy.rejecting()
self._listen = None
if self.client.softstop():
self._shutdown = True
# only change listening if we are not shutting down
elif self._listen is not None:
if self._listen:
self._shutdown = not self.proxy.accepting()
self._listen = None
else:
self.proxy.rejecting()
self._listen = None
if self._toggle_debug:
self._toggle_debug = False
self.log_writer.toggleDebug()
if self._decrease_spawn_limit:
count = self._decrease_spawn_limit
self.redirector.decreaseSpawnLimit(count)
self._decrease_spawn_limit = 0
if self._increase_spawn_limit:
count = self._increase_spawn_limit
self.redirector.increaseSpawnLimit(count)
self._increase_spawn_limit = 0
# cleanup idle connections
# TODO: track all idle connections, not just the ones that have never sent data
expired = self.reactor.client.expire()
for expire_source, expire_count in expired.items():
if expire_source == 'proxy':
self.proxy.notifyClose(None, count=expire_count)
elif expire_source == 'icap':
self.icap.notifyClose(None, count=expire_count)
elif expire_source == 'passthrough':
self.passthrough.notifyClose(None, count=expire_count)
elif expire_source == 'tls':
self.tls.notifyClose(None, count=expire_count)
elif expire_source == 'web':
self.web.notifyClose(None, count=expire_count)
# report if we saw too many connections
if count_saturation == 0:
self.proxy.saturation()
self.web.saturation()
if self.configuration.daemon.poll_interfaces and count_interface == 0:
self.interfaces()
except KeyboardInterrupt:
self.log.critical('^C received')
self._shutdown = True
except OSError, e:
# This shoould never happen as we are limiting how many connections we accept
if e.errno == 24: # Too many open files
self.log.critical('Too many opened files, shutting down')
for line in traceback.format_exc().split('\n'):
self.log.critical(line)
self._shutdown = True
else:
self.log.critical('unrecoverable io error')
for line in traceback.format_exc().split('\n'):
self.log.critical(line)
self._shutdown = True
finally:
