def connection_ready(self, stream):
''' Invoked when the connection is ready '''
method = self.conf["http.client.method"]
stdout = self.conf["http.client.stdout"]
uri = self.conf["http.client.uri"]
request = Message()
if method == "PUT":
fpath = uri.split("/")[-1]
if not os.path.exists(fpath):
LOG.error("* Local file does not exist: %s" % fpath)
sys.exit(1)
request.compose(method=method, uri=uri, keepalive=False,
mimetype="text/plain", body=open(fpath, "rb"))
else:
request.compose(method=method, uri=uri, keepalive=False)
response = Message()
if method == "GET" and not stdout:
fpath = uri.split("/")[-1]
if os.path.exists(fpath):
LOG.error("* Local file already exists: %s" % fpath)
sys.exit(1)
response.body = open(fpath, "wb")
else:
response.body = sys.stdout
stream.send_request(request, response)
def closed(self, exception=None):
if self.close_complete:
return
self.close_complete = True
if exception:
LOG.error("* Connection %s: %s" % (self.logname, exception))
elif self.eof:
LOG.debug("* Connection %s: EOF" % (self.logname))
else:
LOG.debug("* Closed connection %s" % (self.logname))
self.connection_lost(exception)
self.parent.connection_lost(self)
atclosev, self.atclosev = self.atclosev, set()
for func in atclosev:
try:
func(self, exception)
except (KeyboardInterrupt, SystemExit):
raise
except:
LOG.oops("Error in atclosev")
if self.measurer:
self.measurer.unregister_stream(self)
self.send_octets = None
self.send_ticks = 0
self.recv_ticks = 0
self.sock.soclose()
def run_queue(self):
''' If possible run the first test in queue '''
# Adapted from neubot/rendezvous/client.py
if not self.queue:
return
if self.running:
return
#
# Subscribe BEFORE starting the test, otherwise we
# may miss the 'testdone' event if the connection
# to the negotiator service fails, and we will stay
# stuck forever.
#
NOTIFIER.subscribe('testdone', self.test_done)
# Prevent concurrent tests
self.running = True
# Safely run first element in queue
try:
self._do_run_queue()
except (SystemExit, KeyboardInterrupt):
raise
except:
exc = asyncore.compact_traceback()
error = str(exc)
LOG.error('runner_core: catched exception: %s' % error)
NOTIFIER.publish('testdone')
def lookup_country(self, address):
''' Lookup for country entry '''
country = self.countries.country_code_by_addr(address)
if not country:
LOG.error("Geolocator: %s: not found" % address)
return ""
return utils.stringify(country)
def open_or_die(self):
''' Open the database or die '''
if not GEOIP:
LOG.error("Missing dependency: GeoIP")
LOG.info("Please install GeoIP python wrappers, e.g.")
LOG.info(" sudo apt-get install python-geoip")
sys.exit(1)
path = CONFIG.get("rendezvous.geoip_wrapper.country_database",
COUNTRY_DATABASE)
#
# Detect the common error case, i.e. that the user has
# not downloaded the database. If something fancy is
# going on, let the GeoIP library stacktrace for us.
#
if not os.path.exists(path):
LOG.error("Missing GeoLiteCountry database: %s" % path)
LOG.info("Please download it from "
"<http://www.maxmind.com/app/geolitecountry>.")
sys.exit(1)
self.countries = GEOIP.open(path, GEOIP.GEOIP_STANDARD)
def cleanup(self, message=""):
if not self.finished:
self.finished = True
if message:
LOG.error("* speedtest: %s" % message)
while self.streams:
self.streams.popleft().close()
self.child = None
NOTIFIER.publish(TESTDONE)
def connect(self, endpoint, conf):
self.endpoint = endpoint
self.family = socket.AF_INET
if conf["net.stream.ipv6"]:
self.family = socket.AF_INET6
rcvbuf = conf["net.stream.rcvbuf"]
sndbuf = conf["net.stream.sndbuf"]
try:
addrinfo = getaddrinfo(endpoint[0], endpoint[1], self.family,
socket.SOCK_STREAM)
except socket.error, exception:
LOG.error("* Connection to %s failed: %s" % (endpoint, exception))
self.parent._connection_failed(self, exception)
return
def listen(self, endpoint, conf):
self.endpoint = endpoint
self.family = socket.AF_INET
if conf["net.stream.ipv6"]:
self.family = socket.AF_INET6
rcvbuf = conf["net.stream.rcvbuf"]
sndbuf = conf["net.stream.sndbuf"]
try:
addrinfo = getaddrinfo(endpoint[0], endpoint[1], self.family,
socket.SOCK_STREAM, 0, socket.AI_PASSIVE)
except socket.error, exception:
LOG.error("* Bind %s failed: %s" % (self.endpoint, exception))
self.parent.bind_failed(self, exception)
return
def connect(self, endpoint, conf, measurer=None):
self.endpoint = endpoint
self.family = socket.AF_INET
if conf.get("net.stream.ipv6", False):
self.family = socket.AF_INET6
self.measurer = measurer
rcvbuf = conf.get("net.stream.rcvbuf", 0)
sndbuf = conf.get("net.stream.sndbuf", 0)
try:
addrinfo = getaddrinfo(endpoint[0], endpoint[1], self.family,
socket.SOCK_STREAM)
except socket.error, exception:
LOG.error("* Connection to %s failed: %s" % (endpoint, exception))
self.parent._connection_failed(self, exception)
return
def run_queue(self):
''' If possible run the first test in queue '''
# Adapted from neubot/rendezvous/client.py
if not self.queue:
return
if self.running:
return
#
# Subscribe BEFORE starting the test, otherwise we
# may miss the 'testdone' event if the connection
# to the negotiator service fails, and we will stay
# stuck forever.
#
NOTIFIER.subscribe('testdone', self.test_done)
# Prevent concurrent tests
self.running = True
# Make a copy of current settings
conf = CONFIG.copy()
# Make sure we abide to M-Lab policy
if privacy.count_valid(conf, 'privacy.') != 3:
privacy.complain()
NOTIFIER.publish('testdone')
# Run speedtest
elif self.queue[0][0] == 'speedtest':
conf['speedtest.client.uri'] = self.queue[0][1]
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
# Run bittorrent
elif self.queue[0][0] == 'bittorrent':
conf['bittorrent._uri'] = self.queue[0][1]
bittorrent.run(POLLER, conf)
# Safety net
else:
LOG.error('Asked to run an unknown test')
NOTIFIER.publish('testdone')
def test_done(self, *baton):
''' Invoked when the test is done '''
#
# Stop streaming test events to interested parties
# via the log streaming API.
# Do not stop processing immediately and give HTTP
# code time to stream logs to the client in case
# connections fails immediately.
# This must not be done when we're processing the
# somewhat internal 'rendezvous' test.
#
if self.queue[0][0] != 'rendezvous':
POLLER.sched(2, LOG.stop_streaming)
# Paranoid
if baton[0] != 'testdone':
raise RuntimeError('Invoked for the wrong event')
# Notify the caller that the test is done
callback, ctx = self.queue.popleft()[1:]
try:
if ctx:
callback(ctx)
else:
callback()
except (KeyboardInterrupt, SystemExit):
raise
except:
exc = asyncore.compact_traceback()
error = str(exc)
LOG.error('runner_core: catched exception: %s' % error)
#
# Allow for more tests
# If callback() adds one more test, that would
# be run by the run_queue() invocation below.
#
self.running = False
# Eventually run next queued test
self.run_queue()
def process_request(self, stream, request):
''' Process a request and generate the response '''
response = Message()
if not request.uri.startswith("/"):
response.compose(code="403", reason="Forbidden",
body="403 Forbidden")
stream.send_response(request, response)
return
for prefix, child in self.childs.items():
if request.uri.startswith(prefix):
child.process_request(stream, request)
return
rootdir = self.conf.get("http.server.rootdir", "")
if not rootdir:
response.compose(code="403", reason="Forbidden",
body="403 Forbidden")
stream.send_response(request, response)
return
if request.uri == "/":
response.compose_redirect(stream, "/api/index")
stream.send_response(request, response)
return
# Paranoid mode: ON
rootdir = utils.asciiify(rootdir)
uripath = utils.asciiify(request.uri)
fullpath = os.path.normpath(rootdir + uripath)
fullpath = utils.asciiify(fullpath)
if not fullpath.startswith(rootdir):
response.compose(code="403", reason="Forbidden",
body="403 Forbidden")
stream.send_response(request, response)
return
try:
filep = open(fullpath, "rb")
except (IOError, OSError):
LOG.error("HTTP: Not Found: %s (WWW: %s)" % (fullpath, rootdir))
response.compose(code="404", reason="Not Found",
body="404 Not Found")
stream.send_response(request, response)
return
if self.conf.get("http.server.mime", True):
mimetype, encoding = mimetypes.guess_type(fullpath)
# Do not attempt SSI if the resource is, say, gzipped
if not encoding:
if mimetype == "text/html":
ssi = self.conf.get("http.server.ssi", False)
if ssi:
body = ssi_replace(rootdir, filep)
filep = StringIO.StringIO(body)
#XXX Do we need to enforce the charset?
if mimetype in ("text/html", "application/x-javascript"):
mimetype += "; charset=UTF-8"
else:
response["content-encoding"] = encoding
else:
mimetype = "text/plain"
response.compose(code="200", reason="Ok", body=filep,
mimetype=mimetype)
if request.method == "HEAD":
utils.safe_seek(filep, 0, os.SEEK_END)
stream.send_response(request, response)
import os.path
import sys
if __name__ == "__main__":
sys.path.insert(0, ".")
from neubot.config import CONFIG
from neubot.log import LOG
from neubot import utils
try:
import GeoIP
except ImportError:
LOG.error("Missing dependency: GeoIP")
LOG.info("Please install GeoIP python wrappers, e.g.")
LOG.info(" sudo apt-get install python-geoip")
sys.exit(1)
COUNTRY_DATABASE = "/usr/local/share/GeoIP/GeoIP.dat"
class Geolocator(object):
def __init__(self):
self.countries = None
def open_or_die(self):
path = CONFIG.get("rendezvous.geoip_wrapper.country_database",
COUNTRY_DATABASE)
#
LOG.complete("success!")
logging.info("INFO w/ logging.info")
# The following should work because it should not interpolate
logging.debug("DEBUG w/ logging.debug", "ciao")
logging.warning("WARNING w/ logging.warning")
logging.error("ERROR w/ logging.error")
LOG.verbose()
logging.info("INFO w/ logging.info")
logging.debug("DEBUG w/ logging.debug")
logging.warning("WARNING w/ logging.warning")
logging.error("ERROR w/ logging.error")
LOG.error("testing neubot logger -- This is an error message")
LOG.warning("testing neubot logger -- This is an warning message")
LOG.info("testing neubot logger -- This is an info message")
LOG.debug("testing neubot logger -- This is a debug message")
print compat.json.dumps(LOG.listify())
try:
raise Exception("Testing LOG.exception")
except (KeyboardInterrupt, SystemExit):
raise
except:
LOG.exception()
LOG.exception(func=LOG.warning)
LOG.start("Testing the in-progress feature")
LOG.progress("...")
def closed(self, exception=None):
LOG.error("* Connection to %s failed: %s" % (self.endpoint, exception))
self.parent._connection_failed(self, exception)
def closed(self, exception=None):
LOG.error("* Bind %s failed: %s" % (self.endpoint, exception))
self.parent.bind_failed(self, exception) # XXX
result = sock.connect_ex(ainfo[4])
if result not in INPROGRESS:
raise socket.error(result, os.strerror(result))
self.sock = sock
self.timestamp = utils.ticks()
self.poller.set_writable(self)
if result != 0:
LOG.debug("* Connecting to %s ..." % str(endpoint))
self.parent.started_connecting(self)
return
except socket.error, exception:
last_exception = exception
LOG.error("* Connection to %s failed: %s" % (endpoint, last_exception))
self.parent._connection_failed(self, last_exception)
def fileno(self):
return self.sock.fileno()
def handle_write(self):
self.poller.unset_writable(self)
# See http://cr.yp.to/docs/connect.html
try:
self.sock.getpeername()
except socket.error, exception:
# MacOSX getpeername() fails with EINVAL
if exception[0] in (errno.ENOTCONN, errno.EINVAL):
try:
