def connection_made(self, sock, rtt=0):
if rtt:
LOG.debug("ClientHTTP: latency: %s" % utils.time_formatter(rtt))
self.rtt = rtt
stream = ClientStream(self.poller)
stream.attach(self, sock, self.conf, self.measurer)
self.connection_ready(stream)
def check_response(self, response):
if response.code != "200":
raise ValueError("Bad HTTP response code")
if response["content-type"] != "application/json":
raise ValueError("Unexpected contenty type")
octets = response.body.read()
dictionary = json.loads(octets)
LOG.debug("APIStateTracker: received JSON: " +
json.dumps(dictionary, ensure_ascii=True))
if not "events" in dictionary:
return
if not "current" in dictionary:
raise ValueError("Incomplete dictionary")
t = dictionary["t"]
if not type(t) == types.IntType and not type(t) == types.LongType:
raise ValueError("Invalid type for current event time")
if t < 0:
raise ValueError("Invalid value for current event time")
self.timestamp = t
self.process_dictionary(dictionary)
def _send_handshake(self):
''' Convenience function to send handshake '''
LOG.debug("> HANDSHAKE infohash=%s id=%s" %
(self.parent.infohash.encode("hex"),
self.parent.my_id.encode("hex")))
self.start_send("".join((chr(len(PROTOCOL_NAME)), PROTOCOL_NAME,
FLAGS, self.parent.infohash, self.parent.my_id)))
def attach(self, parent, sock, conf):
self.parent = parent
self.conf = conf
self.filenum = sock.fileno()
self.myname = sock.getsockname()
self.peername = sock.getpeername()
self.logname = str((self.myname, self.peername))
LOG.debug("* Connection made %s" % str(self.logname))
if conf["net.stream.secure"]:
if not ssl:
raise RuntimeError("SSL support not available")
server_side = conf["net.stream.server_side"]
certfile = conf["net.stream.certfile"]
# wrap_socket distinguishes between None and ''
if not certfile:
certfile = None
ssl_sock = ssl.wrap_socket(sock, do_handshake_on_connect=False,
certfile=certfile, server_side=server_side)
self.sock = SSLWrapper(ssl_sock)
self.recv_ssl_needs_kickoff = not server_side
else:
self.sock = SocketWrapper(sock)
self.connection_made()
def closed(self, exception=None):
if self.close_complete:
return
self.close_complete = True
if exception:
LOG.error("* Connection %s: %s" % (self.logname, exception))
elif self.eof:
LOG.debug("* Connection %s: EOF" % (self.logname))
else:
LOG.debug("* Closed connection %s" % (self.logname))
self.connection_lost(exception)
self.parent.connection_lost(self)
atclosev, self.atclosev = self.atclosev, set()
for func in atclosev:
try:
func(self, exception)
except (KeyboardInterrupt, SystemExit):
raise
except:
LOG.oops("Error in atclosev")
if self.measurer:
self.measurer.unregister_stream(self)
self.send_octets = None
self.send_ticks = 0
self.recv_ticks = 0
self.sock.soclose()
def got_response_negotiating(self, stream, request, response):
m = json.loads(response.body.read())
PROPERTIES = ("authorization", "queue_pos", "real_address", "unchoked")
for k in PROPERTIES:
self.conf["_%s" % k] = m[k]
if not self.conf["_unchoked"]:
LOG.complete("done (queue_pos %d)" % m["queue_pos"])
STATE.update("negotiate", {"queue_pos": m["queue_pos"]})
self.connection_ready(stream)
else:
LOG.complete("done (unchoked)")
sha1 = hashlib.sha1()
sha1.update(m["authorization"])
self.conf["bittorrent.my_id"] = sha1.digest()
LOG.debug("* My ID: %s" % sha1.hexdigest())
self.http_stream = stream
self.negotiating = False
peer = PeerNeubot(self.poller)
peer.complete = self.peer_test_complete
peer.connection_lost = self.peer_connection_lost
peer.connection_failed = self.peer_connection_failed
peer.configure(self.conf)
peer.connect((self.http_stream.peername[0],
self.conf["bittorrent.port"]))
def prettyprintbody(self, prefix):
''' Pretty print body '''
if self["content-type"] not in ("application/json", "text/xml",
"application/xml"):
return
# Grab the whole body
if not isinstance(self.body, basestring):
body = self.body.read()
else:
body = self.body
# Decode the body
if self["content-type"] == "application/json":
string = compat.json.dumps(compat.json.loads(body),
indent=4, sort_keys=True)
elif self["content-type"] in ("text/xml", "application/xml"):
string = body
# Prettyprint
for line in string.split("\n"):
LOG.debug("%s %s" % (prefix, line.rstrip()))
# Seek to the beginning if needed
if not isinstance(self.body, basestring):
utils.safe_seek(self.body, 0)
def merge_api(self, dictlike, database=None):
# enforce all-or-nothing
LOG.debug("config: reading properties from /api/config")
map(lambda t: self.merge_kv(t, dry=True), dictlike.iteritems())
map(self.merge_kv, dictlike.iteritems())
if database:
table_config.update(database, dictlike.iteritems())
def connection_lost(self, stream):
if runner_core.test_is_running():
LOG.debug("RendezVous: don't _schedule(): test in progress")
return
if self._task:
LOG.debug("RendezVous: don't _schedule(): we already have a task")
return
self._schedule()
def connection_lost(self, stream):
if NOTIFIER.is_subscribed("testdone"):
LOG.debug("RendezVous: don't _schedule(): test in progress")
return
if self._task:
LOG.debug("RendezVous: don't _schedule(): we already have a task")
return
self._schedule()
def connection_made(self, sock, rtt=0):
''' Invoked when the connection is created '''
if rtt:
LOG.debug("ClientHTTP: latency: %s" % utils.time_formatter(rtt))
self.rtt = rtt
stream = ClientStream(self.poller)
stream.attach(self, sock, self.conf)
self.connection_ready(stream)
def __setitem__(self, key, value):
if key in self:
ovalue = self[key]
cast = utils.smart_cast(ovalue)
else:
ovalue = "(none)"
cast = utils.smart_cast(value)
value = cast(value)
LOG.debug("config: %s: %s -> %s" % (key, ovalue, value))
dict.__setitem__(self, key, value)
def recv_complete(self, s):
while s and not (self.close_pending or self.close_complete):
# If we don't know the length then read it
if self.left == 0:
amt = min(len(s), 4 - self.count)
self.buff.append(s[:amt])
s = buffer(s, amt)
self.count += amt
if self.count == 4:
self.left = toint("".join(self.buff))
if self.left == 0:
LOG.debug("< KEEPALIVE")
del self.buff[:]
self.count = 0
elif self.count > 4:
raise RuntimeError("Invalid self.count")
# Bufferize and pass upstream messages
elif self.left > 0:
amt = min(len(s), self.left)
if self.count <= self.smallmessage:
self.buff.append(s[:amt])
else:
if self.buff:
self._got_message_start("".join(self.buff))
# This means "big message"
del self.buff[:]
mp = buffer(s, 0, amt)
self._got_message_part(mp)
s = buffer(s, amt)
self.left -= amt
self.count += amt
if self.left == 0:
# Small or big message?
if self.buff:
self._got_message("".join(self.buff))
else:
self._got_message_end()
del self.buff[:]
self.count = 0
elif self.left < 0:
raise RuntimeError("Invalid self.left")
# Something's wrong
else:
raise RuntimeError("Invalid self.left")
if not (self.close_pending or self.close_complete):
self.start_recv()
def update(self, name, event=None, publish=True):
if not event:
event = {}
if name in STATES:
self._current = name
self._t = self._T()
self._events[name] = event
LOG.debug("state: %s %s" % (name, event))
if publish:
self._publish(STATECHANGE, self._t)
def send_piece(self, index, begin, block):
if not isinstance(block, basestring):
length = utils.file_length(block)
LOG.debug("> PIECE %d %d len=%d" % (index, begin, length))
preamble = struct.pack("!cII", PIECE, index, begin)
l = len(preamble) + length
d = [tobinary(l), ]
d.extend(preamble)
s = "".join(d)
self.start_send(s)
self.start_send(block)
return
LOG.debug("> PIECE %d %d len=%d" % (index, begin, len(block)))
self._send_message(struct.pack("!cII%ss" % len(block), PIECE,
index, begin, block))
def recv_complete(self, s):
''' Invoked when recv() completes '''
while s and not (self.close_pending or self.close_complete):
# If we don't know the length then read it
if self.left == 0:
amt = min(len(s), 4 - self.count)
self.buff.append(s[:amt])
s = buffer(s, amt)
self.count += amt
if self.count == 4:
self.left = toint("".join(self.buff))
if self.left == 0:
LOG.debug("< KEEPALIVE")
elif self.left > MAXMESSAGE:
raise RuntimeError('Message too big')
del self.buff[:]
self.count = 0
elif self.count > 4:
raise RuntimeError("Invalid self.count")
# Bufferize and pass upstream messages
elif self.left > 0:
amt = min(len(s), self.left)
self.buff.append(s[:amt])
s = buffer(s, amt)
self.left -= amt
self.count += amt
if self.left == 0:
self._got_message("".join(self.buff))
del self.buff[:]
self.count = 0
elif self.left < 0:
raise RuntimeError("Invalid self.left")
# Something's wrong
else:
raise RuntimeError("Invalid self.left")
if not (self.close_pending or self.close_complete):
self.start_recv()
def connect(self, endpoint, conf):
# Connect first address in a list
if ' ' in endpoint[0]:
LOG.debug('* Connecting to %s' % str(endpoint))
for address in endpoint[0].split():
epnt = (address.strip(), endpoint[1])
self.epnts.append(epnt)
endpoint = self.epnts.popleft()
self.endpoint = endpoint
sock = utils_net.connect(endpoint)
if not sock:
self._connection_failed()
return
self.sock = sock
self.timestamp = utils.ticks()
self.poller.set_writable(self)
def main(args):
common.main("agent", "Run in background, periodically run tests", args)
conf = CONFIG.copy()
privacy.complain_if_needed()
if conf["agent.api"]:
server = HTTP_SERVER
LOG.debug("* API server root directory: %s" % WWW)
conf["http.server.rootdir"] = WWW
conf["http.server.ssi"] = True
conf["http.server.bind_or_die"] = True
server.configure(conf)
server.register_child(ServerAPI(POLLER), "/api")
server.listen((conf["agent.api.address"], conf["agent.api.port"]))
if conf["agent.daemonize"]:
system.change_dir()
system.go_background()
system.write_pidfile()
LOG.redirect()
if conf["agent.use_syslog"]:
LOG.redirect()
system.drop_privileges(LOG.error)
#
# When we run as an agent we also save logs into
# the database, to easily access and show them via
# the web user interface.
#
LOG.use_database()
if conf["agent.rendezvous"]:
client = ClientRendezvous(POLLER)
client.configure(conf)
client.connect_uri()
POLLER.loop()
def attach(self, parent, sock, conf, measurer=None):
self.parent = parent
self.conf = conf
self.filenum = sock.fileno()
self.myname = sock.getsockname()
self.peername = sock.getpeername()
self.logname = str((self.myname, self.peername))
self.measurer = measurer
if self.measurer:
self.measurer.register_stream(self)
LOG.debug("* Connection made %s" % str(self.logname))
if conf.get("net.stream.secure", False):
if not ssl:
raise RuntimeError("SSL support not available")
server_side = conf.get("net.stream.server_side", False)
certfile = conf.get("net.stream.certfile", None)
so = ssl.wrap_socket(sock, do_handshake_on_connect=False,
certfile=certfile, server_side=server_side)
self.sock = SSLWrapper(so)
self.recv_ssl_needs_kickoff = not server_side
else:
self.sock = SocketWrapper(sock)
if conf.get("net.stream.obfuscate", False):
key = conf.get("net.stream.key", None)
self.encrypt = arcfour_new(key).encrypt
self.decrypt = arcfour_new(key).decrypt
self.connection_made()
def serialize_headers(self):
''' Serialize message headers '''
vector = []
if self.method:
vector.append(self.method)
vector.append(" ")
if self.pathquery:
vector.append(self.pathquery)
elif self.uri:
vector.append(self.uri)
else:
vector.append("/")
vector.append(" ")
vector.append(self.protocol)
else:
vector.append(self.protocol)
vector.append(" ")
vector.append(self.code)
vector.append(" ")
vector.append(self.reason)
LOG.debug("> %s" % ("".join(vector)))
vector.append("\r\n")
for key, value in self.headers.items():
key = "-".join(map(lambda s: s.capitalize(), key.split("-")))
vector.append(key)
vector.append(": ")
vector.append(value)
LOG.debug("> %s: %s" % (key, value))
vector.append("\r\n")
LOG.debug(">")
vector.append("\r\n")
string = "".join(vector)
string = utils.stringify(string)
return StringIO.StringIO(string)
def _got_line(self, line):
""" We've got a line... what do we do? """
if self.state == FIRSTLINE:
line = line.strip()
LOG.debug("< %s" % line)
vector = line.split(None, 2)
if len(vector) == 3:
if line.startswith("HTTP"):
protocol, code, reason = vector
if protocol in PROTOCOLS:
self.got_response_line(protocol, code, reason)
else:
method, uri, protocol = vector
if protocol in PROTOCOLS:
self.got_request_line(method, uri, protocol)
if protocol not in PROTOCOLS:
raise RuntimeError("Invalid protocol")
else:
self.state = HEADER
else:
raise RuntimeError("Invalid first line")
elif self.state == HEADER:
if line.strip():
LOG.debug("< %s" % line)
# not handling mime folding
index = line.find(":")
if index >= 0:
key, value = line.split(":", 1)
self.got_header(key.strip(), value.strip())
else:
raise RuntimeError("Invalid header line")
else:
LOG.debug("<")
self.state, self.left = self.got_end_of_headers()
if self.state == ERROR:
# allow upstream to filter out unwanted requests
self.close()
elif self.state == FIRSTLINE:
# this is the case of an empty body
self.got_end_of_body()
elif self.state == CHUNK_LENGTH:
vector = line.split()
if vector:
length = int(vector[0], 16)
if length < 0:
raise RuntimeError("Negative chunk-length")
elif length == 0:
self.state = TRAILER
else:
self.left = length
self.state = CHUNK
else:
raise RuntimeError("Invalid chunk-length line")
elif self.state == CHUNK_END:
if line.strip():
raise RuntimeError("Invalid chunk-end line")
else:
self.state = CHUNK_LENGTH
elif self.state == TRAILER:
if not line.strip():
self.state = FIRSTLINE
self.got_end_of_body()
else:
# Ignoring trailers
pass
else:
raise RuntimeError("Not expecting a line")
def recv_complete(self, data):
""" We've received successfully some data """
if self.close_complete or self.close_pending:
return
# This one should be debug2 as well
# LOG.debug("HTTP receiver: got %d bytes" % len(data))
# merge with previous fragments (if any)
if self.incoming:
self.incoming.append(data)
data = "".join(self.incoming)
del self.incoming[:]
# consume the current fragment
offset = 0
length = len(data)
while length > 0:
# ostate = self.state # needed by commented-out code below
# when we know the length we're looking for a piece
if self.left > 0:
count = min(self.left, length)
piece = buffer(data, offset, count)
self.left -= count
offset += count
length -= count
self._got_piece(piece)
# otherwise we're looking for the next line
elif self.left == 0:
index = data.find("\n", offset)
if index == -1:
if length > MAXLINE:
raise RuntimeError("Line too long")
break
index = index + 1
line = data[offset:index]
length -= index - offset
offset = index
self._got_line(line)
# robustness
else:
raise RuntimeError("Left become negative")
# robustness
if self.close_complete or self.close_pending:
return
# Should be debug2() not debug()
# LOG.debug("HTTP receiver: %s -> %s" %
# (STATES[ostate], STATES[self.state]))
# keep the eventual remainder for later
if length > 0:
remainder = data[offset:]
self.incoming.append(remainder)
LOG.debug("HTTP receiver: remainder %d" % len(remainder))
# get the next fragment
self.start_recv()
def send_unchoke(self):
''' Send the UNCHOKE message '''
LOG.debug("> UNCHOKE")
self._send_message(UNCHOKE)
def send_request(self, index, begin, length):
''' Send the REQUEST message '''
LOG.debug("> REQUEST %d %d %d" % (index, begin, length))
self._send_message(struct.pack("!cIII", REQUEST, index, begin, length))
def _got_message(self, message):
''' Invoked when we receive a complete message '''
if not self.complete:
if (len(message) != 68 or message[0] != chr(19) or
message[1:20] != PROTOCOL_NAME):
raise RuntimeError("Invalid handshake")
self.id = message[-20:]
infohash = message[-40:-20]
LOG.debug("< HANDSHAKE infohash=%s id=%s" %
(infohash.encode("hex"), self.id.encode("hex")))
#
# In Neubot the listener does not have an infohash
# and handshakes, including connector infohash, after
# it receives the connector handshake.
#
if not self.parent.infohash:
self.parent.infohash = infohash
self._send_handshake()
elif infohash != self.parent.infohash:
raise RuntimeError("Invalid infohash")
self.complete = True
self.parent.connection_ready(self)
return
t = message[0]
if t not in MESSAGESET:
raise RuntimeError("Invalid message type")
if INVALID_LENGTH[t](len(message)):
raise RuntimeError("Invalid message length")
if t in [BITFIELD] and self.got_anything:
raise RuntimeError("Bitfield after we got something")
self.got_anything = True
if t == CHOKE:
LOG.debug("< CHOKE")
self.parent.got_choke(self)
elif t == UNCHOKE:
LOG.debug("< UNCHOKE")
self.parent.got_unchoke(self)
elif t == INTERESTED:
LOG.debug("< INTERESTED")
self.parent.got_interested(self)
elif t == NOT_INTERESTED:
LOG.debug("< NOT_INTERESTED")
self.parent.got_not_interested(self)
elif t == HAVE:
i = struct.unpack("!xI", message)[0]
if i >= self.parent.numpieces:
raise RuntimeError("HAVE: index out of bounds")
LOG.debug("< HAVE %d" % i)
self.parent.got_have(i)
elif t == BITFIELD:
LOG.debug("< BITFIELD {bitfield}")
self.parent.got_bitfield(message[1:])
elif t == REQUEST:
i, a, b = struct.unpack("!xIII", message)
LOG.debug("< REQUEST %d %d %d" % (i, a, b))
if i >= self.parent.numpieces:
raise RuntimeError("REQUEST: index out of bounds")
self.parent.got_request(self, i, a, b)
elif t == CANCEL:
i, a, b = struct.unpack("!xIII", message)
LOG.debug("< CANCEL %d %d %d" % (i, a, b))
if i >= self.parent.numpieces:
raise RuntimeError("CANCEL: index out of bounds")
# NOTE Ignore CANCEL message
elif t == PIECE:
n = len(message) - 9
i, a, b = struct.unpack("!xII%ss" % n, message)
LOG.debug("< PIECE %d %d len=%d" % (i, a, n))
if i >= self.parent.numpieces:
raise RuntimeError("PIECE: index out of bounds")
self.parent.got_piece(self, i, a, b)
def send_piece(self, index, begin, block):
''' Send the PIECE message '''
LOG.debug("> PIECE %d %d len=%d" % (index, begin, len(block)))
self._send_message(struct.pack("!cII%ss" % len(block), PIECE,
index, begin, block))
def send_keepalive(self):
''' Send the KEEPALIVE message '''
LOG.debug("> KEEPALIVE")
self._send_message('')
def send_have(self, index):
''' Send the HAVE message '''
LOG.debug("> HAVE %d" % index)
self._send_message(struct.pack("!cI", HAVE, index))
def send_bitfield(self, bitfield):
''' Send the BITFIELD message '''
LOG.debug("> BITFIELD {bitfield}")
self._send_message(BITFIELD, bitfield)
def send_cancel(self, index, begin, length):
''' Send the CANCEL message '''
LOG.debug("> CANCEL %d %d %d" % (index, begin, length))
self._send_message(struct.pack("!cIII", CANCEL, index, begin, length))
