def _request_stream_from_others(cls, stream, destination):
unconfirmed_tickets = cls._request_stream(stream, destination)
if not unconfirmed_tickets:
raise HTTPError(412)
unconfirmed_tickets = set(unconfirmed_tickets)
bad_tickets = set()
for ticket in unconfirmed_tickets:
ticket.source.update_rtt()
if Node.me().supernode and not ticket.source.rtt:
log.info("Informing web server that %s is unresponsive "
"and should be deleted", ticket.source)
NodesAPI(settings.ASTRAL_WEBSERVER).unregister(
ticket.source.absolute_url())
bad_tickets.append(ticket)
unconfirmed_tickets = unconfirmed_tickets - bad_tickets
log.debug("Received %d unconfirmed tickets: %s",
len(unconfirmed_tickets), unconfirmed_tickets)
closest = min(unconfirmed_tickets, key=lambda t: t.source.rtt)
log.debug("Closest ticket of the unconfirmed ones is %s", closest)
TicketsAPI(closest.source.uri()).confirm(closest.absolute_url())
closest.confirmed = True
session.commit()
for ticket in set(unconfirmed_tickets) - set([closest]):
TicketsAPI(ticket.source.uri()).cancel(ticket.absolute_url())
ticket.delete()
session.commit()
return closest
def register_with_supernode(self):
Node.update_supernode_rtt()
# TODO hacky hacky hacky. moving query inside of the node causes
# SQLAlchemy session errors.
session.commit()
session.close_all()
if not self.node().supernode:
if not Node.supernodes():
self.node().supernode = True
log.info("Registering %s as a supernode, none others found",
self.node())
self.register_with_origin()
else:
for supernode in Node.supernodes().order_by('rtt'):
self.node().primary_supernode = supernode
try:
NodesAPI(self.node().primary_supernode.uri()).register(
self.node().to_dict())
except RequestError, e:
log.warning("Can't connect to supernode %s to register"
": %s", self.node().primary_supernode, e)
log.info("Informing web server that supernode %s is "
"unresponsive and should be deleted", supernode)
NodesAPI(settings.ASTRAL_WEBSERVER).unregister(
supernode.absolute_url())
self.node().primary_supernode = None
except RequestFailed, e:
log.warning("%s rejected us as a child node: %s",
supernode, e)
self.node().primary_supernode = None
else:
self.load_dynamic_bootstrap_nodes(
self.node().primary_supernode.uri())
def run(self):
UpstreamCheckThread(self.node, self.upstream_limit).run()
Node.me(refresh=True)
self.load_static_bootstrap_nodes()
self.load_dynamic_bootstrap_nodes()
self.register_with_supernode()
self.load_streams()
session.commit()
def delete(self, stream_slug, destination_uuid=None):
"""Stop forwarding the stream to the requesting node."""
ticket = self._load_ticket(stream_slug, destination_uuid)
if not ticket:
raise HTTPError(404)
ticket.delete()
session.commit()
TicketDeletionPropagationThread(ticket, self.request).start()
def put(self, stream_slug, destination_uuid=None):
"""Edit tickets, most likely just confirming them."""
ticket = self._load_ticket(stream_slug, destination_uuid)
if ticket:
ticket.confirmed = self.get_json_argument('confirmed')
if ticket.confirmed:
log.info("Confirmed %s", ticket)
session.commit()
def run(self):
UpstreamCheckThread(self.node, self.upstream_limit).run()
Node.me(refresh=True)
self.load_static_bootstrap_nodes()
self.load_dynamic_bootstrap_nodes()
self.register_with_supernode()
self.load_streams()
session.commit()
def test_resume_stream(self):
stream = StreamFactory()
session.commit()
data = {'streaming': True}
eq_(stream.streaming, False)
self.http_client.fetch(
HTTPRequest(self.get_url(stream.absolute_url()),
'PUT',
body=json.dumps(data)), self.stop)
response = self.wait()
eq_(response.code, 200)
eq_(stream.streaming, True)
def from_dict(cls, data):
stream = Stream.get_by(name=data['name'])
if not stream:
source = Node.get_by(uuid=data.get('source') or
data.get('source_uuid'))
if not source:
log.debug("%s had a source not in our database -- not creating "
"Stream", data)
return
else:
stream = cls(source=source, name=data['name'],
description=data.get('description', ''))
session.commit()
return stream
def post(self):
"""Add the node specified in POSTed JSON to the list of known nodes."""
uuid = self.get_json_argument('uuid')
if (Node.me().supernode and self.get_json_argument(
'primary_supernode_uuid', '') == Node.me().uuid):
children_count = Node.query.filter_by(primary_supernode=Node.me()
).count()
if children_count > settings.SUPERNODE_MAX_CHILDREN:
raise HTTPError(503)
if not Node.get_by(uuid=uuid):
# TODO what if supernode changes? need to update
self.request.arguments.setdefault('ip_address',
self.request.remote_ip)
Node.from_dict(self.request.arguments)
session.commit()
def handle_ticket_request(cls, stream, destination):
log.debug("Trying to create a ticket to serve %s to %s",
stream, destination)
new_ticket = cls._already_streaming(stream, destination)
if new_ticket:
log.info("%s already has a ticket for %s (%s)", destination,
stream, new_ticket)
if not new_ticket.source == Node.me():
log.info("Refreshing with source %s to be sure",
new_ticket.source)
existing_ticket = cls._request_stream_from_node(stream,
new_ticket.source, destination)
else:
existing_ticket = new_ticket
if existing_ticket:
existing_ticket.refreshed = datetime.datetime.now()
# In case we lost the tunnel, just make sure it exists
existing_ticket.queue_tunnel_creation()
session.commit()
return existing_ticket
log.info("%s didn't confirm our old ticket %s, must get a new "
"one", new_ticket.source, new_ticket)
if stream.source != Node.me():
new_ticket = cls._offer_ourselves(stream, destination)
if new_ticket:
log.info("We can stream %s to %s, created %s",
stream, destination, new_ticket)
# In case we lost the tunnel, just make sure it exists
new_ticket.queue_tunnel_creation()
elif Node.me().supernode or destination == Node.me():
log.info("Propagating the request for streaming %s to %s to "
"our other known nodes", stream, destination)
new_ticket = cls._request_stream_from_others(stream,
destination)
else:
raise HTTPError(412)
else:
new_ticket = Ticket(stream=stream, source_port=stream.source_port,
destination=destination)
log.info("%s is the source of %s, created %s", Node.me(),
stream, new_ticket)
session.commit()
return new_ticket
def get(self, stream_slug, destination_uuid=None):
ticket = self._load_ticket(stream_slug, destination_uuid)
if ticket:
# TODO this block is somewhat duplicated from TicketsHandler.post,
# where we refresh an existing ticket.
if not ticket.source == Node.me():
log.info("Refreshing %s with the source", ticket)
ticket = TicketsHandler._request_stream_from_node(ticket.stream,
ticket.source, ticket.destination,
existing_ticket=ticket)
if ticket:
ticket.refreshed = datetime.datetime.now()
# In case we lost the tunnel, just make sure it exists
ticket.queue_tunnel_creation()
session.commit()
# TODO this is unideal, but we need to get the new port if it
# changed. combination of sleep and db flush seems to do it
# somewhat reliably, but it's still a race condition.
import time
time.sleep(0.5)
ticket = self._load_ticket(stream_slug, destination_uuid)
self.write({'ticket': ticket.to_dict()})
def register_with_supernode(self):
Node.update_supernode_rtt()
# TODO hacky hacky hacky. moving query inside of the node causes
# SQLAlchemy session errors.
session.commit()
session.close_all()
if not self.node().supernode:
if not Node.supernodes():
self.node().supernode = True
log.info("Registering %s as a supernode, none others found",
self.node())
self.register_with_origin()
else:
for supernode in Node.supernodes().order_by('rtt'):
self.node().primary_supernode = supernode
try:
NodesAPI(self.node().primary_supernode.uri()).register(
self.node().to_dict())
except RequestError, e:
log.warning(
"Can't connect to supernode %s to register"
": %s",
self.node().primary_supernode, e)
log.info(
"Informing web server that supernode %s is "
"unresponsive and should be deleted", supernode)
NodesAPI(settings.ASTRAL_WEBSERVER).unregister(
supernode.absolute_url())
self.node().primary_supernode = None
except RequestFailed, e:
log.warning("%s rejected us as a child node: %s",
supernode, e)
self.node().primary_supernode = None
else:
self.load_dynamic_bootstrap_nodes(
self.node().primary_supernode.uri())
# reload to get a new database session
ticket = Ticket.get_by(id=ticket.id)
if ticket.source == Node.me():
source_ip = "127.0.0.1"
else:
source_ip = ticket.source.ip_address
try:
port = self.create_tunnel(ticket.id, source_ip,
ticket.source_port, self.tunnels)
except Exception, e:
log.warning("Couldn't create a tunnel for %s: %s",
ticket, e)
else:
if not ticket.destination_port or ticket.destination_port != port:
ticket.destination_port = port
session.commit()
self.close_expired_tunnels()
finally:
TUNNEL_QUEUE.task_done()
def _handle_stream(self, stream):
"""Update the enabled flag on all tunnels that are for this stream."""
# reload to get a new database session
stream = Stream.get_by(slug=stream.slug)
try:
port = self.create_tunnel(stream.slug, "127.0.0.1",
settings.RTMP_PORT, self.stream_tunnels)
except Exception, e:
log.warning("Couldn't create a tunnel for %s: %s",
stream, e)
else:
for supernode in Node.supernodes():
if supernode != Node.me():
try:
NodesAPI(supernode.uri()).register(
self.node().to_dict())
except RequestError, e:
log.warning("Can't connect to supernode %s: %s",
supernode, e)
supernode.delete()
log.info(
"Informing web server that %s is unresponsive "
"and should be deleted", supernode)
NodesAPI(settings.ASTRAL_WEBSERVER).unregister(
supernode.absolute_url())
except RequestFailed:
log.warning(
"%s threw an error - sure it's not "
"running on another computer in your LAN with "
"the same remote IP?", supernode)
supernode.delete()
session.commit()
def run(self):
UpstreamCheckThread(self.node, self.upstream_limit).run()
Node.me(refresh=True)
self.load_static_bootstrap_nodes()
self.load_dynamic_bootstrap_nodes()
self.register_with_supernode()
self.load_streams()
session.commit()
def setUp(self):
super(TicketsHandlerTest, self).setUp()
session.commit()
mockito.when(NodeAPI).ping().thenReturn(42)
mockito.when(TicketsAPI).confirm(mockito.any()).thenReturn(True)
mockito.when(TicketsAPI).cancel(mockito.any()).thenReturn(True)
def load_static_bootstrap_nodes(self):
log.info("Loading static bootstrap nodes %s", settings.BOOTSTRAP_NODES)
nodes = [Node.from_dict(node) for node in settings.BOOTSTRAP_NODES]
session.commit()
log.debug("Loaded static bootstrap nodes %s", nodes)
def shutdown(self):
self._unregister_from_origin()
self._unregister_from_supernode()
self._cancel_tickets()
self._unregister_from_all()
session.commit()
def load_static_bootstrap_nodes(self):
log.info("Loading static bootstrap nodes %s",
settings.BOOTSTRAP_NODES)
nodes = [Node.from_dict(node) for node in settings.BOOTSTRAP_NODES]
session.commit()
log.debug("Loaded static bootstrap nodes %s", nodes)
def shutdown(self):
self._unregister_from_origin()
self._unregister_from_supernode()
self._cancel_tickets()
self._unregister_from_all()
session.commit()