def __init__(self, configuration, poller):
self.low = configuration.redirector.minimum # minimum concurrent redirector workers
self.high = configuration.redirector.maximum # maximum concurrent redirector workers
self.poller = poller
self.configuration = configuration
self.queue = Queue(
) # store requests we do not immediately have the resources to process
self.nextid = 1 # unique id to give to the next spawned worker
self.worker = {} # worker tasks for each spawned child
self.processes = {
} # worker tasks indexed by file descriptors we can poll
self.available = set(
) # workers that are currently available to handle new requests
self.active = {
} # workers that are currently busy waiting for a response from the spawned process
self.stopping = set(
) # workers we want to stop as soon as they stop being active
program = configuration.redirector.program
protocol = configuration.redirector.protocol
self.redirector_factory = RedirectorFactory(configuration, program,
protocol)
self.log = Logger('manager', configuration.log.manager)
def __init__ (self, configuration, name, program, protocol):
self.configuration = configuration
self.http_parser = self.HTTPParser(configuration)
self.tls_parser = self.TLSParser(configuration)
self.enabled = bool(program is not None) and configuration.redirector.enable
self._transparent = configuration.http.transparent
self.log = Logger('worker ' + str(name), configuration.log.worker)
self.usage = UsageLogger('usage', configuration.log.worker)
self.response_factory = self.ResponseFactory()
self.child_factory = self.ChildFactory(configuration, name)
self.wid = name # a unique name
self.creation = time.time() # when the thread was created
# self.last_worked = self.creation # when the thread last picked a task
self.program = program # the squid redirector program to fork
self.running = True # the thread is active
self.stats_timestamp = None # time of the most recent outstanding request to generate stats
self._proxy = 'ExaProxy-%s-id-%d' % (configuration.proxy.version,os.getpid())
universal = configuration.redirector.protocol == 'url'
# Do not move, we need the forking AFTER the setup
if program:
self.process = self.child_factory.createProcess(self.program, universal=universal)
else:
self.process = None
def __init__ (self, configuration, querier, decider, logger, poller):
self.querier = querier # Incoming requests from the proxy
self.decider = decider # Decides how each request should be handled
self.logger = logger # Log writing interfaces
self.poller = poller
# NOT the same logger the rest of the proxy uses since we're running in a different process
self.log = Logger('redirector', configuration.log.supervisor)
self.running = True
def __init__(self, name, poller, read_name, max_clients):
self.socks = {}
self.name = name
self.poller = poller
self.read_name = read_name
self.max_clients = max_clients
self.client_count = 0
self.saturated = False # we are receiving more connections than we can handle
self.binding = set()
self.serving = True # We are currenrly listening
self.log = Logger('server', configuration.log.server)
self.log.info('server [%s] accepting up to %d clients' % (name, max_clients))
def __init__(self, poller, configuration):
self.total_sent4 = 0L
self.total_sent6 = 0L
self.total_requested = 0L
self.norequest = TimeCache(configuration.http.idle_connect)
self.bysock = {}
self.byname = {}
self.buffered = []
self._nextid = 0
self.poller = poller
self.log = Logger('client', configuration.log.client)
self.proxied = configuration.http.proxied
self.max_buffer = configuration.http.header_size
def __init__ (self,configuration,poller):
self.configuration = configuration
self.low = configuration.redirector.minimum # minimum number of workers at all time
self.high = configuration.redirector.maximum # maximum numbe of workers at all time
self.program = configuration.redirector.program # what program speaks the squid redirector API
self.nextid = 1 # incremental number to make the name of the next worker
self.queue = Queue() # queue with HTTP headers to process
self.poller = poller # poller interface that checks for events on sockets
self.worker = {} # our workers threads
self.closing = set() # workers that are currently closing
self.running = True # we are running
self.log = Logger('manager', configuration.log.manager)
def __init__(self, supervisor, configuration):
self.total_sent4 = 0L
self.total_sent6 = 0L
self.opening = {}
self.established = {}
self.byclientid = {}
self.buffered = []
self.retry = []
self.configuration = configuration
self.supervisor = supervisor
self.poller = supervisor.poller
self.log = Logger('download', configuration.log.download)
self.location = os.path.realpath(os.path.normpath(configuration.web.html))
self.page = supervisor.page
self._header = {}
def __init__(self, configuration, web, proxy, decider, content, client,
resolver, logger, usage, poller):
self.web = web # Manage listening web sockets
self.proxy = proxy # Manage listening proxy sockets
self.decider = decider # Task manager for handling child decider processes
self.content = content # The Content Download manager
self.client = client # Currently open client connections
self.resolver = resolver # The DNS query manager
self.poller = poller # Interface to the poller
self.logger = logger # Log writing interfaces
self.usage = usage # Request logging
self.running = True # Until we stop we run :)
self.nb_events = 0L # Number of events received
self.nb_loops = 0L # Number of loop iteration
self.events = [] # events so we can report them once in a while
self.log = Logger('supervisor', configuration.log.supervisor)
def __init__(self, poller, configuration):
self.total_sent4 = 0L
self.total_sent6 = 0L
self.total_requested = 0L
self.norequest = TimeCache(configuration.http.idle_connect)
self.bysock = {}
self.byname = {}
self.buffered = []
self._nextid = 0
self.poller = poller
self.log = Logger('client', configuration.log.client)
self.http_max_buffer = configuration.http.header_size
self.icap_max_buffer = configuration.icap.header_size
self.tls_max_buffer = configuration.tls.header_size
self.passthrough_max_buffer = 0
self.proxied = {
'proxy' : configuration.http.proxied,
'icap' : configuration.icap.proxied,
'tls' : configuration.tls.proxied,
}
def __init__(self, poller, configuration, max_workers):
self.poller = poller
self.configuration = configuration
self.resolver_factory = self.resolverFactory(configuration)
# The actual work is done in the worker
self.worker = self.resolver_factory.createUDPClient()
# All currently active clients (one UDP and many TCP)
self.workers = {}
self.workers[self.worker.socket] = self.worker
self.poller.addReadSocket('read_resolver', self.worker.socket)
# Track the clients currently expecting results
self.clients = {} # client_id : identifier
# Key should be the hostname rather than the request ID?
self.resolving = {} # identifier, worker_id :
# TCP workers that have not yet sent a complete request
self.sending = {} # sock :
# Maximum number of entry we will cache (1024 DNS lookup per second !)
# assuming 1k per entry, which is a lot, it mean 20Mb of memory
# which at the default of 900 seconds of cache is 22 new host per seonds
self.max_entries = 1024 * 20
# track the current queries and when they were started
self.active = []
self.cache = {}
self.cached = deque()
self.max_workers = max_workers
self.worker_count = len(self.workers) # the UDP client
self.waiting = []
self.log = Logger('resolver', configuration.log.resolver)
self.chained = {}
def __init__(self, configuration, name, request_box, program):
self.configuration = configuration
self.icap_parser = self.ICAPParser(configuration)
self.enabled = configuration.redirector.enable
self.protocol = configuration.redirector.protocol
self._transparent = configuration.http.transparent
self.log = Logger('worker ' + str(name), configuration.log.worker)
self.usage = UsageLogger('usage', configuration.log.worker)
self.universal = True if self.protocol == 'url' else False
self.icap = self.protocol[len('icap://'):].split(
'/')[0] if self.protocol.startswith('icap://') else ''
r, w = os.pipe() # pipe for communication with the main thread
self.response_box_write = os.fdopen(w, 'w',
0) # results are written here
self.response_box_read = os.fdopen(r, 'r',
0) # read from the main thread
self.wid = name # a unique name
self.creation = time.time() # when the thread was created
# self.last_worked = self.creation # when the thread last picked a task
self.request_box = request_box # queue with HTTP headers to process
self.program = program # the squid redirector program to fork
self.running = True # the thread is active
self.stats_timestamp = None # time of the most recent outstanding request to generate stats
self._proxy = 'ExaProxy-%s-id-%d' % (configuration.proxy.version,
os.getpid())
if self.protocol == 'url':
self.classify = self._classify_url
if self.protocol.startswith('icap://'):
self.classify = self._classify_icap
# Do not move, we need the forking AFTER the setup
self.process = self._createProcess(
) # the forked program to handle classification
Thread.__init__(self)
def __init__(self, supervisor):
self.supervisor = supervisor
self.monitor = supervisor.monitor
self.email_sent = False
self.log = Logger('web', supervisor.configuration.log.web)
def __init__ (self,configuration):
configuration = load()
self.configuration = configuration
# Only here so the introspection code can find them
self.log = Logger('supervisor', configuration.log.supervisor)
self.log.error('Starting exaproxy version %s' % configuration.proxy.version)
self.signal_log = Logger('signal', configuration.log.signal)
self.log_writer = SysLogWriter('log', configuration.log.destination, configuration.log.enable, level=configuration.log.level)
self.usage_writer = UsageWriter('usage', configuration.usage.destination, configuration.usage.enable)
self.log_writer.setIdentifier(configuration.daemon.identifier)
#self.usage_writer.setIdentifier(configuration.daemon.identifier)
if configuration.debug.log:
self.log_writer.toggleDebug()
self.usage_writer.toggleDebug()
self.log.error('python version %s' % sys.version.replace(os.linesep,' '))
self.log.debug('starting %s' % sys.argv[0])
self.pid = PID(self.configuration)
self.daemon = Daemon(self.configuration)
self.poller = Poller(self.configuration.daemon)
self.poller.setupRead('read_proxy') # Listening proxy sockets
self.poller.setupRead('read_web') # Listening webserver sockets
self.poller.setupRead('read_icap') # Listening icap sockets
self.poller.setupRead('read_workers') # Pipes carrying responses from the child processes
self.poller.setupRead('read_resolver') # Sockets currently listening for DNS responses
self.poller.setupRead('read_client') # Active clients
self.poller.setupRead('opening_client') # Clients we have not yet read a request from
self.poller.setupWrite('write_client') # Active clients with buffered data to send
self.poller.setupWrite('write_resolver') # Active DNS requests with buffered data to send
self.poller.setupRead('read_download') # Established connections
self.poller.setupWrite('write_download') # Established connections we have buffered data to send to
self.poller.setupWrite('opening_download') # Opening connections
self.monitor = Monitor(self)
self.page = Page(self)
self.manager = RedirectorManager(
self.configuration,
self.poller,
)
self.content = ContentManager(self,configuration)
self.client = ClientManager(self.poller, configuration)
self.resolver = ResolverManager(self.poller, self.configuration, configuration.dns.retries*10)
self.proxy = Server('http proxy',self.poller,'read_proxy', configuration.http.connections)
self.web = Server('web server',self.poller,'read_web', configuration.web.connections)
self.icap = Server('icap server',self.poller,'read_icap', configuration.icap.connections)
self.reactor = Reactor(self.configuration, self.web, self.proxy, self.icap, self.manager, self.content, self.client, self.resolver, self.log_writer, self.usage_writer, self.poller)
self._shutdown = True if self.daemon.filemax == 0 else False # stop the program
self._softstop = False # stop once all current connection have been dealt with
self._reload = False # unimplemented
self._toggle_debug = False # start logging a lot
self._decrease_spawn_limit = 0
self._increase_spawn_limit = 0
self._refork = False # unimplemented
self._pdb = False # turn on pdb debugging
self._listen = None # listening change ? None: no, True: listen, False: stop listeing
self.wait_time = 5.0 # how long do we wait at maximum once we have been soft-killed
self.local = set() # what addresses are on our local interfaces
self.interfaces()
signal.signal(signal.SIGQUIT, self.sigquit)
signal.signal(signal.SIGINT, self.sigterm)
signal.signal(signal.SIGTERM, self.sigterm)
# signal.signal(signal.SIGABRT, self.sigabrt)
# signal.signal(signal.SIGHUP, self.sighup)
signal.signal(signal.SIGTRAP, self.sigtrap)
signal.signal(signal.SIGUSR1, self.sigusr1)
signal.signal(signal.SIGUSR2, self.sigusr2)
signal.signal(signal.SIGTTOU, self.sigttou)
signal.signal(signal.SIGTTIN, self.sigttin)
signal.signal(signal.SIGALRM, self.sigalrm)
# make sure we always have data in history
# (done in zero for dependencies reasons)
self.monitor.zero()
# encoding: utf-8
"""
async/__init__.py
Created by David Farrar on 2012-01-31.
Copyright (c) 2011-2013 Exa Networks. All rights reserved.
"""
import sys
import select
from exaproxy.util.log.logger import Logger
from exaproxy.configuration import load
configuration = load()
log = Logger('supervisor', configuration.log.supervisor)
def Poller(configuration, speed=None):
reactor = configuration.reactor
if reactor == 'best':
if sys.platform.startswith('linux'):
configuration.reactor = 'epoll'
elif sys.platform.startswith('freebsd'):
configuration.reactor = 'kqueue'
elif sys.platform.startswith('darwin'):
configuration.reactor = 'kqueue'
else:
log.error(
'we could not autodetect an high performance reactor for your OS'
Created by Thomas Mangin on 2011-11-30.
Copyright (c) 2011-2013 Exa Networks. All rights reserved.
"""
import socket
import errno
from exaproxy.util.log.logger import Logger
from exaproxy.network.errno_list import errno_block
from exaproxy.configuration import load
IP_TRANSPARENT = 19
configuration = load()
log = Logger('server', configuration.log.server)
def isipv4(address):
try:
socket.inet_pton(socket.AF_INET, address)
return True
except socket.error:
return False
def isipv6(address):
try:
socket.inet_pton(socket.AF_INET6, address)
return True
except socket.error:
return False
def __init__(self, configuration, name):
self.log = Logger('worker ' + str(name), configuration.log.worker)
def __init__(self, configuration):
self.configuration = configuration
# Only here so the introspection code can find them
self.log = Logger('supervisor', configuration.log.supervisor)
self.log.error('Starting exaproxy version %s' %
configuration.proxy.version)
self.signal_log = Logger('signal', configuration.log.signal)
self.log_writer = SysLogWriter('log',
configuration.log.destination,
configuration.log.enable,
level=configuration.log.level)
self.usage_writer = UsageWriter('usage',
configuration.usage.destination,
configuration.usage.enable)
sys.exitfunc = self.log_writer.writeMessages
self.log_writer.setIdentifier(configuration.daemon.identifier)
#self.usage_writer.setIdentifier(configuration.daemon.identifier)
if configuration.debug.log:
self.log_writer.toggleDebug()
self.usage_writer.toggleDebug()
self.log.error('python version %s' %
sys.version.replace(os.linesep, ' '))
self.log.debug('starting %s' % sys.argv[0])
self.pid = PID(self.configuration)
self.daemon = Daemon(self.configuration)
self.poller = Poller(self.configuration.daemon)
self.poller.setupRead('read_proxy') # Listening proxy sockets
self.poller.setupRead('read_web') # Listening webserver sockets
self.poller.setupRead('read_icap') # Listening icap sockets
self.poller.setupRead('read_tls') # Listening tls sockets
self.poller.setupRead('read_passthrough') # Listening raw data sockets
self.poller.setupRead(
'read_redirector'
) # Pipes carrying responses from the redirector process
self.poller.setupRead(
'read_resolver') # Sockets currently listening for DNS responses
self.poller.setupRead('read_client') # Active clients
self.poller.setupRead(
'opening_client') # Clients we have not yet read a request from
self.poller.setupWrite(
'write_client') # Active clients with buffered data to send
self.poller.setupWrite(
'write_resolver') # Active DNS requests with buffered data to send
self.poller.setupRead('read_download') # Established connections
self.poller.setupWrite(
'write_download'
) # Established connections we have buffered data to send to
self.poller.setupWrite('opening_download') # Opening connections
self.poller.setupRead('read_interrupt') # Scheduled events
self.poller.setupRead(
'read_control'
) # Responses from commands sent to the redirector process
self.monitor = Monitor(self)
self.page = Page(self)
self.content = ContentManager(self, configuration)
self.client = ClientManager(self.poller, configuration)
self.resolver = ResolverManager(self.poller, self.configuration,
configuration.dns.retries * 10)
self.proxy = Server('http proxy', self.poller, 'read_proxy',
configuration.http)
self.web = Server('web server', self.poller, 'read_web',
configuration.web)
self.icap = Server('icap server', self.poller, 'read_icap',
configuration.icap)
self.tls = Server('tls server', self.poller, 'read_tls',
configuration.tls)
self.passthrough = InterceptServer('passthrough server', self.poller,
'read_passthrough',
configuration.passthrough)
self._shutdown = True if self.daemon.filemax == 0 else False # stop the program
self._softstop = False # stop once all current connection have been dealt with
self._reload = False # unimplemented
self._toggle_debug = False # start logging a lot
self._decrease_spawn_limit = 0
self._increase_spawn_limit = 0
self._refork = False # unimplemented
self._pdb = False # turn on pdb debugging
self._listen = None # listening change ? None: no, True: listen, False: stop listeing
self.wait_time = 5.0 # how long do we wait at maximum once we have been soft-killed
self.local = set() # what addresses are on our local interfaces
if not self.initialise():
self._shutdown = True
elif self.daemon.drop_privileges():
self.log.critical(
'Could not drop privileges to \'%s\'. Refusing to run as root'
% self.daemon.user)
self.log.critical(
'Set the environment value USER to change the unprivileged user'
)
self._shutdown = True
# fork the redirector process before performing any further setup
redirector = fork_redirector(self.poller, self.configuration)
# use simple blocking IO for communication with the redirector process
self.redirector = redirector_message_thread(redirector)
# NOTE: create threads _after_ all forking is done
# regularly interrupt the reactor for maintenance
self.interrupt_scheduler = alarm_thread(self.poller, self.alarm_time)
self.reactor = Reactor(self.configuration, self.web, self.proxy,
self.passthrough, self.icap, self.tls,
self.redirector, self.content, self.client,
self.resolver, self.log_writer,
self.usage_writer, self.poller)
self.interfaces()
signal.signal(signal.SIGQUIT, self.sigquit)
signal.signal(signal.SIGINT, self.sigterm)
signal.signal(signal.SIGTERM, self.sigterm)
# signal.signal(signal.SIGABRT, self.sigabrt)
# signal.signal(signal.SIGHUP, self.sighup)
signal.signal(signal.SIGTRAP, self.sigtrap)
signal.signal(signal.SIGUSR1, self.sigusr1)
signal.signal(signal.SIGUSR2, self.sigusr2)
signal.signal(signal.SIGTTOU, self.sigttou)
signal.signal(signal.SIGTTIN, self.sigttin)
# make sure we always have data in history
# (done in zero for dependencies reasons)
if self._shutdown is False:
self.redirector.requestStats()
command, control_data = self.redirector.readResponse()
stats_data = control_data if command == 'STATS' else None
stats = self.monitor.statistics(stats_data)
ok = self.monitor.zero(stats)
if ok:
self.redirector.requestStats()
else:
self._shutdown = True
