class Game(Protocol): def __init__(self, users): self.users = users self.queue = DeferredQueue() def connectionMade(self): global state connections['GAME'] = self state = "GO" self.flag = True def dataReceived(self, data): self.queue.put(data) self.queue.get().addCallback(self.ForwardData) def ForwardData(self, data): positions = pickle.loads(data) if is_client == "host": gs.player2.rect = positions["p2_rect"] gs.player2.image = pygame.transform.rotate(gs.player2.orig_image, positions["p2_angle"]) if positions["firing"] == True: mx = positions["mouse_x"] my = positions["mouse_y"] self.laser = P2_Laser(gs.player2.rect.centerx, gs.player2.rect.centery,mx,my,gs) self.laser.rect.x = gs.player2.rect.centerx self.laser.rect.y = gs.player2.rect.centery # Add the laser to the lists gs.laser_list2.add(self.laser) gs.all_sprites_list.add(self.laser) else: gs.player1.rect = positions["p1_rect"] gs.player1.image = pygame.transform.rotate(gs.player1.orig_image, positions["p1_angle"]) if positions["firing"] == True: mx = positions["mouse_x"] my = positions["mouse_y"] self.laser = P1_Laser(gs.player1.rect.centerx, gs.player1.rect.centery,mx,my,gs) self.laser.rect.x = gs.player1.rect.centerx self.laser.rect.y = gs.player1.rect.centery # Add the laser to the lists gs.laser_list1.add(self.laser) gs.all_sprites_list.add(self.laser) if positions["change"] == True: gs.player2.rect.x = positions["newx"] gs.player2.rect.y = positions["newy"] if self.queue.waiting > 0: self.queue.get().addCallback(self.ForwardData) def connectionLost(self,reason): print "Connection lost - goodbye!" reactor.stop() def clientConnectionLost(self, reason): print "Connection lost - goodbye!" reactor.stop()
class TestStreamingClient(VumiTestCase):
def setUp(self):
self.fake_http = FakeHttpServer(self.handle_request)
self.request_queue = DeferredQueue()
self.client = StreamingClient(self.fake_http.get_agent)
self.messages_received = DeferredQueue()
self.errors_received = DeferredQueue()
self.disconnects_received = DeferredQueue()
def reason_trapper(reason):
if reason.trap(ResponseDone):
self.disconnects_received.put(reason.getErrorMessage())
self.receiver = self.client.stream(
Message,
self.messages_received.put, self.errors_received.put,
"http://vumi-go-api.example.com/", on_disconnect=reason_trapper)
def handle_request(self, request):
self.request_queue.put(request)
return NOT_DONE_YET
def test_default_agent_factory(self):
"""
If `None` is passed as the `agent_factory`, `Agent` is used instead.
"""
self.assertNotIsInstance(self.client.agent, Agent)
self.assertIsInstance(StreamingClient(None).agent, Agent)
self.assertIsInstance(StreamingClient().agent, Agent)
@inlineCallbacks
def test_callback_on_disconnect(self):
req = yield self.request_queue.get()
req.write(
'%s\n' % (Message(foo='bar').to_json().encode('utf-8'),))
req.finish()
message = yield self.messages_received.get()
self.assertEqual(message['foo'], 'bar')
reason = yield self.disconnects_received.get()
# this is the error message we get when a ResponseDone is raised
# which happens when the remote server closes the connection.
self.assertEqual(reason, 'Response body fully received')
@inlineCallbacks
def test_invalid_json(self):
req = yield self.request_queue.get()
req.write("Hello\n")
req.finish()
err = yield self.assertFailure(
self.errors_received.get(), VumiBridgeInvalidJsonError)
self.assertEqual(err.args, ("Hello",))
class TcpProxyProtocol(Protocol, object):
"""A simple TCP proxy"""
def __init__(self):
"""Create a new TCP proxy.
`self.server_queue` contains messages from end server to client.
`self.client_queue` contains messages from client to end server.
"""
self.server_queue = DeferredQueue()
self.client_queue = DeferredQueue()
self.server_queue.get().addCallback(self.serverQueueCallback)
def connectServer(self, hostname, port):
"""Tell the proxy what the end server is and start the connection.
The messages in `self.client_queue` will automatically be consumed.
This method should only be called once.
:param str hostname:
:param int port:
"""
endpoint = TCP4ClientEndpoint(reactor, hostname, port)
protocol = ServerProtocol(
self.server_queue, self.client_queue)
connectProtocol(endpoint, protocol)
def serverQueueCallback(self, data):
"""A callback for `self.server_queue`
:param str data: data from server queue
"""
if data is False:
self.transport.loseConnection()
return
self.transport.write(data)
self.server_queue.get().addCallback(self.serverQueueCallback)
def dataReceived(self, data):
"""Received data from client, put into client queue
"""
self.client_queue.put(data)
def connectionLost(self, why):
"""Client closed connection, or some other issue. close connection
to server
"""
self.client_queue.put(False)
class TestStreamingClient(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.mock_server = MockHttpServer(self.handle_request)
self.add_cleanup(self.mock_server.stop)
yield self.mock_server.start()
self.url = self.mock_server.url
self.client = StreamingClient()
self.messages_received = DeferredQueue()
self.errors_received = DeferredQueue()
self.disconnects_received = DeferredQueue()
def reason_trapper(reason):
if reason.trap(ResponseDone):
self.disconnects_received.put(reason.getErrorMessage())
self.receiver = self.client.stream(
Message,
self.messages_received.put, self.errors_received.put,
self.url, on_disconnect=reason_trapper)
def handle_request(self, request):
self.mock_server.queue.put(request)
return NOT_DONE_YET
@inlineCallbacks
def test_callback_on_disconnect(self):
req = yield self.mock_server.queue.get()
req.write(
'%s\n' % (Message(foo='bar').to_json().encode('utf-8'),))
req.finish()
message = yield self.messages_received.get()
self.assertEqual(message['foo'], 'bar')
reason = yield self.disconnects_received.get()
# this is the error message we get when a ResponseDone is raised
# which happens when the remote server closes the connection.
self.assertEqual(reason, 'Response body fully received')
@inlineCallbacks
def test_invalid_json(self):
req = yield self.mock_server.queue.get()
req.write("Hello\n")
req.finish()
try:
yield self.errors_received.get()
except VumiBridgeInvalidJsonError, e:
self.assertEqual(e.args, ("Hello",))
else:
class home_DataConn(protocol.Protocol): def __init__(self): print 'home listening for data connection from work' self.queue = DeferredQueue() def connectionMade(self): home_connections['data_conn'] = self self.queue.get().addCallback(self.forwardData) def dataReceived(self, data): self.queue.put(data) def forwardData(self, data): home_connections['client_conn'].transport.write(data) self.queue.get().addCallback(self.forwardData)
class work_ServiceConn(protocol.Protocol): def __init__(self): print 'service connection established' self.queue = DeferredQueue() def connectionMade(self): work_connections['service_conn'] = self self.queue.get().addCallback(self.forwardData) def dataReceived(self, data): self.queue.put(data) def forwardData(self, data): work_connections['data_conn'].transport.write(data) self.queue.get().addCallback(self.forwardData)
class QueuePoller(object):
implements(IPoller)
def __init__(self, settings):
self.q = RedisSpiderQueue(settings)
self.dq = DeferredQueue(size=1)
@inlineCallbacks
def poll(self):
if self.dq.pending:
return
c = yield maybeDeferred(self.q.count)
if c:
msg = yield maybeDeferred(self.q.pop)
returnValue(self.dq.put(self._message(msg)))
def next(self):
return self.dq.get()
def _message(self, queue_msg):
d = queue_msg.copy()
d['_project'] = SCRAPY_PROJECT
d['_spider'] = d.pop('name')
d['_job'] = d.pop('jobid', uuid.uuid1().hex)
return d
class QueuePoller(object):
def __init__(self, config):
self.config = config
self.update_projects()
self.dq = DeferredQueue()
@inlineCallbacks
def poll(self):
if not self.dq.waiting:
return
for p, q in iteritems(self.queues):
c = yield maybeDeferred(q.count)
if c:
msg = yield maybeDeferred(q.pop)
if msg is not None: # In case of a concurrently accessed queue
returnValue(self.dq.put(self._message(msg, p)))
def next(self):
return self.dq.get()
def update_projects(self):
self.queues = get_spider_queues(self.config)
def _message(self, queue_msg, project):
d = queue_msg.copy()
d['_project'] = project
d['_spider'] = d.pop('name')
return d
def test_messages_stream(self):
url = '%s/%s/messages.json' % (self.url, self.conversation.key)
messages = DeferredQueue()
errors = DeferredQueue()
receiver = self.client.stream(
TransportUserMessage, messages.put, errors.put, url,
Headers(self.auth_headers))
msg1 = yield self.app_helper.make_dispatch_inbound(
'in 1', message_id='1', conv=self.conversation)
msg2 = yield self.app_helper.make_dispatch_inbound(
'in 2', message_id='2', conv=self.conversation)
rm1 = yield messages.get()
rm2 = yield messages.get()
receiver.disconnect()
# Sometimes messages arrive out of order if we're hitting real redis.
rm1, rm2 = sorted([rm1, rm2], key=lambda m: m['message_id'])
self.assertEqual(msg1['message_id'], rm1['message_id'])
self.assertEqual(msg2['message_id'], rm2['message_id'])
self.assertEqual(errors.size, None)
def test_health_response(self):
health_url = 'http://%s:%s%s' % (
self.addr.host, self.addr.port, self.config['health_path'])
response = yield http_request_full(health_url, method='GET')
self.assertEqual(response.delivered_body, '0')
yield self.app_helper.make_dispatch_inbound(
'in 1', message_id='1', conv=self.conversation)
queue = DeferredQueue()
stream_url = '%s/%s/messages.json' % (self.url, self.conversation.key)
stream_receiver = self.client.stream(
TransportUserMessage, queue.put, queue.put, stream_url,
Headers(self.auth_headers))
yield queue.get()
response = yield http_request_full(health_url, method='GET')
self.assertEqual(response.delivered_body, '1')
stream_receiver.disconnect()
response = yield http_request_full(health_url, method='GET')
self.assertEqual(response.delivered_body, '0')
self.assertEqual(self.app.client_manager.clients, {
'sphex.stream.message.%s' % (self.conversation.key,): []
})
class QueuePoller(object):
implements(IPoller)
def __init__(self, config):
self.config = config
self.update_projects()
self.dq = DeferredQueue(size=1)
@inlineCallbacks
def poll(self):
if self.dq.pending:
return
for p, q in self.queues.iteritems():
c = yield maybeDeferred(q.count)
if c:
msg = yield maybeDeferred(q.pop)
returnValue(self.dq.put(self._message(msg, p)))
def next(self):
return self.dq.get()
def update_projects(self):
self.queues = get_spider_queues(self.config)
def _message(self, queue_msg, project):
d = queue_msg.copy()
d['_project'] = project
d['_spider'] = d.pop('name')
return d
class ProcessPool(object): def __init__(self, count=10): self.limiter = DeferredSemaphore(count) self.processes = [spawnProcess() for _ in xrange(count)] self.workQueue = DeferredQueue() for process in self.processes: process.onconnect.addCallback(self._prepareForWork) @inlineCallbacks def _prepareForWork(self, proto): deferred, func, args = yield self.workQueue.get() proto.queueWork(deferred, func, *args) def requeue(result): self._prepareForWork(proto) return result deferred.addCallback(requeue) returnValue(proto) def queueWork(self, function, *args): resultDeferred = Deferred() innerDeferred = Deferred() self.workQueue.put((innerDeferred, function, args)) def callResult(obj): resultDeferred.callback(obj) return obj innerDeferred.addCallback(callResult) return resultDeferred def stop(self): for process in self.processes: process.protocol.kill() process.transport.loseConnection()
class SSMIServerProtocol(Protocol):
delimiter = TruteqTransportProtocol.delimiter
def __init__(self):
self.receive_queue = DeferredQueue()
self._buf = b""
def dataReceived(self, data):
self._buf += data
self.parse_commands()
def parse_commands(self):
while self.delimiter in self._buf:
line, _, self._buf = self._buf.partition(self.delimiter)
if line:
self.receive_queue.put(SSMIRequest.parse(line))
def send(self, command):
self.transport.write(str(command))
self.transport.write(self.delimiter)
return wait0()
def receive(self):
return self.receive_queue.get()
def disconnect(self):
self.transport.loseConnection()
def test_events_stream(self):
url = '%s/%s/events.json' % (self.url, self.conversation.key)
events = DeferredQueue()
errors = DeferredQueue()
receiver = yield self.client.stream(TransportEvent, events.put,
events.put, url,
Headers(self.auth_headers))
msg1 = yield self.app_helper.make_stored_outbound(
self.conversation, 'out 1', message_id='1')
ack1 = yield self.app_helper.make_dispatch_ack(
msg1, conv=self.conversation)
msg2 = yield self.app_helper.make_stored_outbound(
self.conversation, 'out 2', message_id='2')
ack2 = yield self.app_helper.make_dispatch_ack(
msg2, conv=self.conversation)
ra1 = yield events.get()
ra2 = yield events.get()
receiver.disconnect()
self.assertEqual(ack1['event_id'], ra1['event_id'])
self.assertEqual(ack2['event_id'], ra2['event_id'])
self.assertEqual(errors.size, None)
def async_receive_stream(self, func, *args, **kw):
queue = DeferredQueue()
def _execute():
for result in func(*args, **kw):
reactor.callFromThread(queue.put, result)
_ = threads.deferToThread(_execute)
while 1:
yield queue.get()
class home_ClientConn(protocol.Protocol):
def __init__(self):
print 'client connection established'
self.queue = DeferredQueue()
def connectionMade(self):
home_connections['client_conn'] = self
home_connections['command_conn'].transport.write('begin data connect')
reactor.listenTCP(80061, DataConnFactory())
self.queue.get().addCallback(self.forwardData)
def dataReceived(self, data):
self.queue.put(data)
def forwardData(self, data):
home_connections['data_conn'].transport.write(data)
self.queue.get().addCallback(self.forwardData)
class GameConn(protocol.Protocol): #simply reiterate what they said.
def connectionMade(self):
self.transport.write("Beginning Player Connection...")
self.queue = DeferredQueue()
def dataReceived(self, data):
#"As soon as any data is received, print it."
print data
self.queue.put(data)
self.queue.get().addCallback(self.send_to_server)
def send_to_server(self, data):
self.transport.write(data)
def connectionLost(self, reason):
pass
def get(self, timeout=None):
deferred = DeferredQueue.get(self)
call_id = None
if timeout:
call_id = self.clock.callLater(timeout, self._timeout, deferred)
deferred.addCallback(self._raiseIfClosed, call_id)
return deferred
class DataConn(Protocol):
def __init__(self):
self.queue = DeferredQueue()
def connectionMade(self):
print "connection made on DataConn"
pass
def dataReceived(self, data):
self.queue.put(data)
def sendData(self, data):
connectionsDict["Client"].transport.write(data)
self.queue.get().addCallback(self.sendData)
def startForwarding(self):
self.queue.get().addCallback(self.sendData)
class GameConn(protocol.Protocol): #respond to data received
def connectionMade(self):
global num_players
num_players+=1
playerConnections[str(num_players)] = self
self.queue = DeferredQueue()
self.transport.write("Connected to server as player " + str(num_players))
print self.queue
def connectionLost(self, reason):
pass
def dataReceived(self, data):
print data
self.queue.put(data)
self.queue.get().addCallback(self.send_to_client)
def send_to_client(self, data):
pass
class QueuePoller(object):
implements(IPoller)
def __init__(self, config, app):
self.config = config
self.update_projects()
self.dq = DeferredQueue(size=1)
self.max_jobs_per_project = self.config.getint('max_jobs_per_project', 4)
@inlineCallbacks
def poll(self, launcher):
if self.dq.pending:
return
for p, q in self.queues.iteritems():
c = yield maybeDeferred(q.count)
if c and self._has_slot_for_project(p, launcher):
msg = yield maybeDeferred(q.pop)
returnValue(self.dq.put(self._message(msg, p)))
def _has_slot_for_project(self, project_name, launcher):
running_jobs = 0
spiders = launcher.processes.values()
for s in spiders:
if s.project == project_name:
running_jobs += 1
return running_jobs < self.max_jobs_per_project
def next(self):
return self.dq.get()
def update_projects(self):
self.queues = get_spider_queues(self.config)
def _message(self, queue_msg, project):
d = queue_msg.copy()
d['_project'] = project
d['_spider'] = d.pop('name')
return d
@property
def launcher(self):
"""
Copied from website.Root
Should do some refactory to avoid this duplicated code
"""
app = IServiceCollection(self.app, self.app)
return app.getServiceNamed('launcher')
def test__handles_missing_system_handler_on_notification(self):
# Captured notifications from the database will go here.
notices = DeferredQueue()
class PostgresListenerServiceSpy(PostgresListenerService):
"""Send notices off to `notices` right after processing."""
def doRead(self):
try:
self.connection.connection.poll()
except Exception:
self.loseConnection(Failure(error.ConnectionLost()))
else:
# Copy the pending notices now but don't put them in the
# queue until after the real doRead has processed them.
notifies = list(self.connection.connection.notifies)
try:
return super().doRead()
finally:
for notice in notifies:
notices.put(notice)
listener = PostgresListenerServiceSpy()
# Change notifications to a frozenset. This makes sure that
# the system message does not go into the queue. Instead if should
# call the handler directly in `doRead`.
listener.notifications = frozenset()
yield listener.startService()
# Use a randomised channel name even though LISTEN/NOTIFY is
# per-database and _not_ per-cluster.
channel = factory.make_name("sys_test", sep="_").lower()
self.assertTrue(listener.isSystemChannel(channel))
payload = factory.make_name("payload")
yield deferToDatabase(listener.registerChannel, channel)
listener.listeners[channel] = []
try:
# Notify our channel with a payload and wait for it to come back.
yield deferToDatabase(self.send_notification, channel, payload)
while True:
notice = yield notices.get()
if notice.channel == channel:
self.assertThat(notice.payload, Equals(payload))
# Our channel has been deleted from the listeners map.
self.assertFalse(channel in listener.listeners)
break
finally:
yield listener.stopService()
class MockXmlOverTcpServer(MockServer):
def __init__(self):
self.responses = {}
self.received_queue = DeferredQueue()
def wait_for_data(self):
return self.received_queue.get()
def send_data(self, data):
self.transport.write(data)
def dataReceived(self, data):
response = self.responses.get(data)
if response is not None:
self.transport.write(response)
self.received_queue.put(data)
def test_deferred_queue_receiver(self):
ebc = EventBus()
queue = DeferredQueue()
ebc.subscribe('', lambda _, msg: queue.put(msg))
for i in range(10):
ebc.publish('', i)
self.assertEqual(len(queue.pending), 10)
for i in range(10):
msg = yield queue.get()
self.assertEqual(msg, i)
self.assertEqual(len(queue.pending), 0)
class MockXmlOverTcpServer(MockServer):
def __init__(self):
self.responses = {}
self.received_queue = DeferredQueue()
def wait_for_data(self):
return self.received_queue.get()
def send_data(self, data):
self.transport.write(data)
def dataReceived(self, data):
response = self.responses.get(data)
if response is not None:
self.transport.write(response)
self.received_queue.put(data)
def test_deferred_queue_receiver(self):
ebc = EventBus()
queue = DeferredQueue()
ebc.subscribe('', lambda _, msg: queue.put(msg))
for i in xrange(10):
ebc.publish('', i)
self.assertEqual(len(queue.pending), 10)
for i in xrange(10):
msg = yield queue.get()
self.assertEqual(msg, i)
self.assertEqual(len(queue.pending), 0)
def test_backlog_on_connect(self):
for i in range(10):
yield self.app_helper.make_dispatch_inbound(
'in %s' % (i,), message_id=str(i), conv=self.conversation)
queue = DeferredQueue()
url = '%s/%s/messages.json' % (self.url, self.conversation.key)
receiver = self.client.stream(
TransportUserMessage, queue.put, queue.put, url,
Headers(self.auth_headers))
for i in range(10):
received = yield queue.get()
self.assertEqual(received['message_id'], str(i))
receiver.disconnect()
class ConsumerQueue(object):
def __init__(self, stop_on_error=False, empty=None):
self.stop_on_error = stop_on_error
self.empty = empty
self.queue = DeferredQueue()
self.size = 0
self.running = True
self._deferred = Deferred()
def _consume_next(self, *args):
if not self.running:
return
self._deferred = self.queue.get()
self._deferred.addCallbacks(self._consumer, self._error)
def _consumer(self, item):
self.size -= 1
r = self.consume(item)
if self.size == 0 and self.empty is not None:
self.empty()
if isinstance(r, Deferred):
r.addCallbacks(self._consume_next, self._consume_next)
else:
self._consume_next()
def _error(self, fail):
self.error(fail)
if not self.stop_on_error:
self._consume_next()
def add(self, item):
self.size += 1
self.queue.put(item)
def consume(self, item):
raise NotImplementedError
def error(self, fail):
raise NotImplementedError
def start(self):
self.running = True
self._consume_next()
def stop(self):
self.running = False
self._deferred.cancel()
def test_broker_restart(self):
"""
restart the kafka broker and verify that the group rejoins
"""
record_stream = DeferredQueue(backlog=1)
def processor(consumer, records):
log.debug('processor(%r, %r)', consumer, records)
record_stream.put(records)
coord = ConsumerGroup(
self.client,
self.id(),
topics=[self.topic],
processor=processor,
retry_backoff_ms=100,
heartbeat_interval_ms=1000,
fatal_backoff_ms=2000,
)
join_de = self.when_called(coord, 'on_join_complete')
coord.start()
self.addCleanup(coord.stop)
yield join_de
self.assertIn(self.topic, coord.consumers)
self.assertEqual(len(coord.consumers[self.topic]), self.num_partitions)
self.assertEqual(coord.consumers[self.topic][0].topic, self.topic)
self.assertEqual(coord.consumers[self.topic][0].partition, 0)
# restart the broker and see that we re-join and still work
leave_de = self.when_called(coord, 'on_group_leave')
prepare_de = self.when_called(coord, 'on_join_prepare')
join_de = self.when_called(coord, 'on_join_complete')
self.harness.brokers[0].stop()
yield leave_de
self.assertEqual(len(coord.consumers), 0)
self.harness.brokers[0].restart()
yield prepare_de
yield join_de
self.assertIn(self.topic, coord.consumers)
self.assertEqual(len(coord.consumers[self.topic]), self.num_partitions)
for part in range(self.num_partitions):
values = yield self.send_messages(part, [part])
msgs = yield record_stream.get()
self.assertEqual(msgs[0].partition, part)
self.assertEqual(msgs[0].message.value, values[0])
def test_backlog_on_connect(self):
for i in range(10):
yield self.app_helper.make_dispatch_inbound('in %s' % (i, ),
message_id=str(i),
conv=self.conversation)
queue = DeferredQueue()
url = '%s/%s/messages.json' % (self.url, self.conversation.key)
receiver = self.client.stream(TransportUserMessage,
queue.put, queue.put, url,
Headers(self.auth_headers))
for i in range(10):
received = yield queue.get()
self.assertEqual(received['message_id'], str(i))
receiver.disconnect()
class ToyXmlOverTcpClient(XmlOverTcpClient):
_PACKET_RECEIVED_HANDLERS = {'DummyPacket': 'dummy_packet_received'}
def __init__(self):
XmlOverTcpClient.__init__(self, 'root', 'toor', '1029384756')
self.PACKET_RECEIVED_HANDLERS.update(self._PACKET_RECEIVED_HANDLERS)
self.received_dummy_packets = []
self.received_data_request_packets = []
self.disconnected = False
self.session_id_counter = count()
self.generated_session_ids = []
self.request_id_counter = count()
self.generated_request_ids = []
self.received_queue = DeferredQueue()
def wait_for_data(self):
return self.received_queue.get()
def connectionMade(self):
pass
def dataReceived(self, data):
XmlOverTcpClient.dataReceived(self, data)
self.received_queue.put(data)
def dummy_packet_received(self, session_id, params):
self.received_dummy_packets.append((session_id, params))
def data_request_received(self, session_id, params):
self.received_data_request_packets.append((session_id, params))
def disconnect(self):
self.disconnected = True
@classmethod
def session_id_from_nr(cls, nr):
return cls.serialize_header_field(nr, cls.SESSION_ID_HEADER_SIZE)
def gen_session_id(self):
return self.session_id_from_nr(next(self.session_id_counter))
def gen_request_id(self):
return str(next(self.request_id_counter))
class Pipeline(Deferred):
"""Run a generator pipelined over a set of inputs."""
def __init__(self, pipe, width=5):
Deferred.__init__(self)
self.pipe = pipe
self.width = width
self.waiting = None
self.running = 0
self.results = DeferredQueue()
def run(self, inputs, *a, **kw):
self._produce(inputs, *a, **kw)
self._consume().chainDeferred(self)
@inlineCallbacks
def _produce(self, inputs, *a, **kw):
prev = None
for i, e in enumerate(inputs):
if self.running >= self.width:
# wait to add more pipes
self.waiting = Deferred()
yield self.waiting
self.running += 1
p = self.pipe(e, *a, **kw)
pr = PipeRunner(p, prev=prev)
pr.addBoth(self.results.put)
if i == 0:
pr.run()
prev = pr
pr.addBoth(lambda _: self.results.put(DONE))
@inlineCallbacks
def _consume(self):
results = []
while 1:
r = yield self.results.get()
if r == DONE:
break
self.running -= 1
results.append(r)
if self.waiting:
d = self.waiting
self.waiting = None
d.callback(None)
returnValue(results)
class Pipeline(Deferred):
"""Run a generator pipelined over a set of inputs."""
def __init__(self, pipe, width=5):
Deferred.__init__(self)
self.pipe = pipe
self.width = width
self.waiting = None
self.running = 0
self.results = DeferredQueue()
def run(self, inputs, *a, **kw):
self._produce(inputs, *a, **kw)
self._consume().chainDeferred(self)
@inlineCallbacks
def _produce(self, inputs, *a, **kw):
prev = None
for i, e in enumerate(inputs):
if self.running >= self.width:
# wait to add more pipes
self.waiting = Deferred()
yield self.waiting
self.running += 1
p = self.pipe(e, *a, **kw)
pr = PipeRunner(p, prev=prev)
pr.addBoth(self.results.put)
if i == 0:
pr.run()
prev = pr
pr.addBoth(lambda _: self.results.put(DONE))
@inlineCallbacks
def _consume(self):
results = []
while 1:
r = yield self.results.get()
if r == DONE:
break
self.running -= 1
results.append(r)
if self.waiting:
d = self.waiting
self.waiting = None
d.callback(None)
returnValue(results)
class TestGoConversationTransportBase(VumiTestCase):
transport_class = None
def setUp(self):
self.tx_helper = self.add_helper(TransportHelper(self.transport_class))
self.fake_http = FakeHttpServer(self.handle_inbound_request)
self.clock = Clock()
self._request_queue = DeferredQueue()
self._pending_reqs = []
self.add_cleanup(self.finish_requests)
@inlineCallbacks
def get_transport(self, **config):
defaults = {
'account_key': 'account-key',
'conversation_key': 'conversation-key',
'access_token': 'access-token',
}
defaults.update(config)
transport = yield self.tx_helper.get_transport(defaults, start=False)
transport.agent_factory = self.fake_http.get_agent
yield transport.startWorker()
yield self.setup_transport(transport)
returnValue(transport)
def setup_transport(self, transport):
pass
@inlineCallbacks
def finish_requests(self):
for req in self._pending_reqs:
if not req.finished:
yield req.finish()
def handle_inbound_request(self, request):
self._request_queue.put(request)
return NOT_DONE_YET
@inlineCallbacks
def get_next_request(self):
req = yield self._request_queue.get()
self._pending_reqs.append(req)
returnValue(req)
def pull_message(self, count=1):
url = '%s/%s/messages.json' % (self.url, self.conversation.key)
messages = DeferredQueue()
errors = DeferredQueue()
receiver = self.client.stream(TransportUserMessage, messages.put,
errors.put, url,
Headers(self.auth_headers))
received_messages = []
for msg_id in range(count):
yield self.app_helper.make_dispatch_inbound('in %s' % (msg_id, ),
message_id=str(msg_id),
conv=self.conversation)
recv_msg = yield messages.get()
received_messages.append(recv_msg)
receiver.disconnect()
returnValue((receiver, received_messages))
class StubbyIrcServer(ServerFactory):
protocol = StubbyIrcProtocol
def __init__(self, *args, **kw):
# ServerFactory.__init__(self, *args, **kw)
self.client = None
self.events = DeferredQueue()
def buildProtocol(self, addr):
self.client = self.protocol(self.events)
return self.client
@inlineCallbacks
def filter_events(self, command_type):
while True:
ev = yield self.events.get()
if ev[1] == command_type:
returnValue(ev)
class StubbyIrcServer(ServerFactory):
protocol = StubbyIrcServerProtocol
def startFactory(self):
self.server = None
self.events = DeferredQueue()
self.finished_d = Deferred()
def buildProtocol(self, addr):
self.server = ServerFactory.buildProtocol(self, addr)
self.server.factory = self
return self.server
@inlineCallbacks
def filter_events(self, command_type):
while True:
ev = yield self.events.get()
if ev[1] == command_type:
returnValue(ev)
def pull_message(self, count=1):
url = '%s/%s/messages.json' % (self.url, self.conversation.key)
messages = DeferredQueue()
errors = DeferredQueue()
receiver = self.client.stream(
TransportUserMessage, messages.put, errors.put, url,
Headers(self.auth_headers))
received_messages = []
for msg_id in range(count):
yield self.app_helper.make_dispatch_inbound(
'in %s' % (msg_id,), message_id=str(msg_id),
conv=self.conversation)
recv_msg = yield messages.get()
received_messages.append(recv_msg)
receiver.disconnect()
returnValue((receiver, received_messages))
class TestGoConversationTransportBase(VumiTestCase):
transport_class = None
def setUp(self):
self.tx_helper = self.add_helper(TransportHelper(self.transport_class))
self.fake_http = FakeHttpServer(self.handle_inbound_request)
self.clock = Clock()
self._request_queue = DeferredQueue()
self._pending_reqs = []
self.add_cleanup(self.finish_requests)
@inlineCallbacks
def get_transport(self, start=True, **config):
defaults = {
'account_key': 'account-key',
'conversation_key': 'conversation-key',
'access_token': 'access-token',
'publish_status': True,
}
defaults.update(config)
transport = yield self.tx_helper.get_transport(defaults, start=False)
transport.agent_factory = self.fake_http.get_agent
if start:
yield transport.startWorker()
returnValue(transport)
@inlineCallbacks
def finish_requests(self):
for req in self._pending_reqs:
if not req.finished:
yield req.finish()
def handle_inbound_request(self, request):
self._request_queue.put(request)
return NOT_DONE_YET
@inlineCallbacks
def get_next_request(self):
req = yield self._request_queue.get()
self._pending_reqs.append(req)
returnValue(req)
class FakeServer(object):
"""
Fake server container for testing client/server interactions.
"""
def __init__(self, server_factory, auto_accept=True, on_connect=None):
self.server_factory = server_factory
self.auto_accept = auto_accept
self.connection_queue = DeferredQueue()
self.on_connect = on_connect
# Public API.
@classmethod
def for_protocol(cls, protocol, *args, **kw):
factory = ServerFactory.forProtocol(protocol)
return cls(factory, *args, **kw)
@property
def endpoint(self):
"""
Get an endpoint that connects clients to this server.
"""
return FakeServerEndpoint(self)
def await_connection(self):
"""
Wait for a client to start connecting, and then return a
:class:`FakeConnection` object.
"""
return self.connection_queue.get()
# Internal stuff.
def _handle_connection(self):
conn = FakeConnection(self)
if self.on_connect is not None:
conn._connected_d.addCallback(lambda _: self.on_connect(conn))
self.connection_queue.put(conn)
if self.auto_accept:
conn.accept_connection()
return conn._accept_d
class BattleshipsProcessProtocol(ProcessProtocol):
def __init__(self, name):
self.name = name
self.buf = ''
self.queue = DeferredQueue()
self.on_crash = Deferred()
self.err = ''
def errReceived(self, data):
self.err += data
def outReceived(self, data):
self.buf += data
lines = self.buf.split('\n')
self.buf = lines[-1]
for l in lines[:-1]:
self.lineReceived(l)
def lineReceived(self, line):
mo = re.match(r'^([A-Z])(\d+)$', line, flags=re.I)
if mo:
col = ord(mo.group(1).upper()) - 64
row = int(mo.group(2))
self.queue.put((col, row))
def processExited(self, status):
if self.err:
print self.name, "crashed with error:"
print self.err
self.on_crash.errback(status)
def getMove(self):
return self.queue.get()
def sendResult(self, result):
self.transport.write(result + '\n')
def close(self):
try:
self.transport.signalProcess('TERM')
except ProcessExitedAlready:
pass
class PostgresListenerServiceSpy(PostgresListenerService):
"""Save received notifies `captured_notifies` before processing them.."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Captured notifications from the database will go here.
self._captured_notifies = DeferredQueue()
# Change notifications to a frozenset. This makes sure that the system
# message does not go into the queue. Instead it should call the
# handler directly in `doRead`.
self.notifications = frozenset()
def _process_notifies(self):
for notify in self.connection.connection.notifies:
self._captured_notifies.put(notify)
super()._process_notifies()
@inlineCallbacks
def wait_notification(self, channel):
"""Wait for a notification to be received."""
while True:
notice = yield self._captured_notifies.get()
if notice.channel == channel:
returnValue(notice)
class BroadcomOnuHandler(object):
def __init__(self, adapter, device_id):
self.adapter = adapter
self.adapter_agent = adapter.adapter_agent
self.device_id = device_id
self.log = structlog.get_logger(device_id=device_id)
self.incoming_messages = DeferredQueue()
self.proxy_address = None
self.tx_id = 0
def receive_message(self, msg):
self.incoming_messages.put(msg)
def activate(self, device):
self.log.info('activating')
# first we verify that we got parent reference and proxy info
assert device.parent_id
assert device.proxy_address.device_id
assert device.proxy_address.channel_id
# register for proxied messages right away
self.proxy_address = device.proxy_address
self.adapter_agent.register_for_proxied_messages(device.proxy_address)
# populate device info
device.root = True
device.vendor = 'Broadcom'
device.model = 'n/a'
device.hardware_version = 'to be filled'
device.firmware_version = 'to be filled'
device.software_version = 'to be filled'
device.serial_number = uuid4().hex
device.connect_status = ConnectStatus.REACHABLE
self.adapter_agent.update_device(device)
# register physical ports
uni_port = Port(port_no=2,
label='UNI facing Ethernet port',
type=Port.ETHERNET_UNI,
admin_state=AdminState.ENABLED,
oper_status=OperStatus.ACTIVE)
self.adapter_agent.add_port(device.id, uni_port)
self.adapter_agent.add_port(
device.id,
Port(port_no=1,
label='PON port',
type=Port.PON_ONU,
admin_state=AdminState.ENABLED,
oper_status=OperStatus.ACTIVE,
peers=[
Port.PeerPort(device_id=device.parent_id,
port_no=device.parent_port_no)
]))
# add uni port to logical device
parent_device = self.adapter_agent.get_device(device.parent_id)
logical_device_id = parent_device.parent_id
assert logical_device_id
port_no = device.proxy_address.channel_id
cap = OFPPF_1GB_FD | OFPPF_FIBER
self.adapter_agent.add_logical_port(
logical_device_id,
LogicalPort(id='uni-{}'.format(port_no),
ofp_port=ofp_port(
port_no=port_no,
hw_addr=mac_str_to_tuple(
'00:00:00:00:%02x:%02x' %
((port_no >> 8) & 0xff, port_no & 0xff)),
name='uni-{}'.format(port_no),
config=0,
state=OFPPS_LIVE,
curr=cap,
advertised=cap,
peer=cap,
curr_speed=OFPPF_1GB_FD,
max_speed=OFPPF_1GB_FD),
device_id=device.id,
device_port_no=uni_port.port_no))
reactor.callLater(10, self.message_exchange)
device = self.adapter_agent.get_device(device.id)
device.oper_status = OperStatus.ACTIVE
self.adapter_agent.update_device(device)
@inlineCallbacks
def update_flow_table(self, device, flows):
#
# We need to proxy through the OLT to get to the ONU
# Configuration from here should be using OMCI
#
self.log.info('bulk-flow-update', device_id=device.id, flows=flows)
def is_downstream(port):
return port == 2 # Need a better way
def is_upstream(port):
return not is_downstream(port)
for flow in flows:
try:
_in_port = fd.get_in_port(flow)
assert _in_port is not None
if is_downstream(_in_port):
self.log.info('downstream-flow')
elif is_upstream(_in_port):
self.log.info('upstream-flow')
else:
raise Exception('port should be 1 or 2 by our convention')
_out_port = fd.get_out_port(flow) # may be None
self.log.info('out-port', out_port=_out_port)
for field in fd.get_ofb_fields(flow):
if field.type == fd.ETH_TYPE:
_type = field.eth_type
self.log.info('field-type-eth-type', eth_type=_type)
elif field.type == fd.IP_PROTO:
_proto = field.ip_proto
self.log.info('field-type-ip-proto', ip_proto=_proto)
elif field.type == fd.IN_PORT:
_port = field.port
self.log.info('field-type-in-port', in_port=_port)
elif field.type == fd.VLAN_VID:
_vlan_vid = field.vlan_vid & 0xfff
self.log.info('field-type-vlan-vid', vlan=_vlan_vid)
elif field.type == fd.VLAN_PCP:
_vlan_pcp = field.vlan_pcp
self.log.info('field-type-vlan-pcp', pcp=_vlan_pcp)
elif field.type == fd.UDP_DST:
_udp_dst = field.udp_dst
self.log.info('field-type-udp-dst', udp_dst=_udp_dst)
elif field.type == fd.UDP_SRC:
_udp_src = field.udp_src
self.log.info('field-type-udp-src', udp_src=_udp_src)
elif field.type == fd.IPV4_DST:
_ipv4_dst = field.ipv4_dst
self.log.info('field-type-ipv4-dst',
ipv4_dst=_ipv4_dst)
elif field.type == fd.IPV4_SRC:
_ipv4_src = field.ipv4_src
self.log.info('field-type-ipv4-src',
ipv4_dst=_ipv4_src)
elif field.type == fd.METADATA:
_metadata = field.metadata
self.log.info('field-type-metadata',
metadata=_metadata)
else:
raise NotImplementedError('field.type={}'.format(
field.type))
for action in fd.get_actions(flow):
if action.type == fd.OUTPUT:
_output = action.output.port
self.log.info('action-type-output',
output=_output,
in_port=_in_port)
elif action.type == fd.POP_VLAN:
self.log.info('action-type-pop-vlan', in_port=_in_port)
elif action.type == fd.PUSH_VLAN:
_push_tpid = action.push.ethertype
log.info('action-type-push-vlan',
push_tpid=_push_tpid,
in_port=_in_port)
if action.push.ethertype != 0x8100:
self.log.error('unhandled-tpid',
ethertype=action.push.ethertype)
elif action.type == fd.SET_FIELD:
_field = action.set_field.field.ofb_field
assert (action.set_field.field.oxm_class ==
OFPXMC_OPENFLOW_BASIC)
self.log.info('action-type-set-field',
field=_field,
in_port=_in_port)
if _field.type == fd.VLAN_VID:
self.log.info('set-field-type-valn-vid',
vlan_vid=_field.vlan_vid & 0xfff)
else:
self.log.error('unsupported-action-set-field-type',
field_type=_field.type)
else:
log.error('unsupported-action-type',
action_type=action.type,
in_port=_in_port)
#
# All flows created from ONU adapter should be OMCI based
#
except Exception as e:
log.exception('failed-to-install-flow', e=e, flow=flow)
def get_tx_id(self):
self.tx_id += 1
return self.tx_id
def send_omci_message(self, frame):
_frame = hexify(str(frame))
self.log.info('send-omci-message-%s' % _frame)
device = self.adapter_agent.get_device(self.device_id)
try:
self.adapter_agent.send_proxied_message(device.proxy_address,
_frame)
except Exception as e:
self.log.info('send-omci-message-exception', exc=str(e))
def send_get_circuit_pack(self, entity_id=0):
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciGet.message_id,
omci_message=OmciGet(
entity_class=CircuitPack.class_id,
entity_id=entity_id,
attributes_mask=CircuitPack.mask_for('vendor_id')))
self.send_omci_message(frame)
def send_mib_reset(self, entity_id=0):
frame = OmciFrame(transaction_id=self.get_tx_id(),
message_type=OmciMibReset.message_id,
omci_message=OmciMibReset(
entity_class=OntData.class_id,
entity_id=entity_id))
self.send_omci_message(frame)
def send_create_gal_ethernet_profile(self, entity_id,
max_gem_payload_size):
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciCreate.message_id,
omci_message=OmciCreate(
entity_class=GalEthernetProfile.class_id,
entity_id=entity_id,
data=dict(max_gem_payload_size=max_gem_payload_size)))
self.send_omci_message(frame)
def send_set_tcont(self, entity_id, alloc_id):
data = dict(alloc_id=alloc_id)
frame = OmciFrame(transaction_id=self.get_tx_id(),
message_type=OmciSet.message_id,
omci_message=OmciSet(
entity_class=Tcont.class_id,
entity_id=entity_id,
attributes_mask=Tcont.mask_for(*data.keys()),
data=data))
self.send_omci_message(frame)
def send_create_8021p_mapper_service_profile(self, entity_id):
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciCreate.message_id,
omci_message=OmciCreate(
entity_class=Ieee8021pMapperServiceProfile.class_id,
entity_id=entity_id,
data=dict(
tp_pointer=OmciNullPointer,
interwork_tp_pointer_for_p_bit_priority_0=OmciNullPointer,
)))
self.send_omci_message(frame)
def send_create_mac_bridge_service_profile(self, entity_id):
frame = OmciFrame(transaction_id=self.get_tx_id(),
message_type=OmciCreate.message_id,
omci_message=OmciCreate(
entity_class=MacBridgeServiceProfile.class_id,
entity_id=entity_id,
data=dict(spanning_tree_ind=False,
learning_ind=True,
priority=0x8000,
max_age=20 * 256,
hello_time=2 * 256,
forward_delay=15 * 256,
unknown_mac_address_discard=True)))
self.send_omci_message(frame)
def send_create_gem_port_network_ctp(self, entity_id, port_id, tcont_id,
direction, tm):
_directions = {"upstream": 1, "downstream": 2, "bi-directional": 3}
if _directions.has_key(direction):
_direction = _directions[direction]
else:
self.log.error('invalid-gem-port-direction', direction=direction)
raise ValueError(
'Invalid GEM port direction: {_dir}'.format(_dir=direction))
frame = OmciFrame(transaction_id=self.get_tx_id(),
message_type=OmciCreate.message_id,
omci_message=OmciCreate(
entity_class=GemPortNetworkCtp.class_id,
entity_id=entity_id,
data=dict(
port_id=port_id,
tcont_pointer=tcont_id,
direction=_direction,
traffic_management_pointer_upstream=tm)))
self.send_omci_message(frame)
def send_create_multicast_gem_interworking_tp(self, entity_id,
gem_port_net_ctp_id):
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciCreate.message_id,
omci_message=OmciCreate(
entity_class=MulticastGemInterworkingTp.class_id,
entity_id=entity_id,
data=dict(gem_port_network_ctp_pointer=gem_port_net_ctp_id,
interworking_option=0,
service_profile_pointer=0x1)))
self.send_omci_message(frame)
def send_create_gem_inteworking_tp(self, entity_id, gem_port_net_ctp_id,
service_profile_id):
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciCreate.message_id,
omci_message=OmciCreate(
entity_class=GemInterworkingTp.class_id,
entity_id=entity_id,
data=dict(gem_port_network_ctp_pointer=gem_port_net_ctp_id,
interworking_option=5,
service_profile_pointer=service_profile_id,
interworking_tp_pointer=0x0,
gal_profile_pointer=0x1)))
self.send_omci_message(frame)
def send_set_8021p_mapper_service_profile(self, entity_id,
interwork_tp_id):
data = dict(interwork_tp_pointer_for_p_bit_priority_0=interwork_tp_id)
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciSet.message_id,
omci_message=OmciSet(
entity_class=Ieee8021pMapperServiceProfile.class_id,
entity_id=entity_id,
attributes_mask=Ieee8021pMapperServiceProfile.mask_for(
*data.keys()),
data=data))
self.send_omci_message(frame)
def send_create_mac_bridge_port_configuration_data(self, entity_id,
bridge_id, port_id,
tp_type, tp_id):
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciCreate.message_id,
omci_message=OmciCreate(
entity_class=MacBridgePortConfigurationData.class_id,
entity_id=entity_id,
data=dict(bridge_id_pointer=bridge_id,
port_num=port_id,
tp_type=tp_type,
tp_pointer=tp_id)))
self.send_omci_message(frame)
def send_create_vlan_tagging_filter_data(self, entity_id, vlan_id):
frame = OmciFrame(transaction_id=self.get_tx_id(),
message_type=OmciCreate.message_id,
omci_message=OmciCreate(
entity_class=VlanTaggingFilterData.class_id,
entity_id=entity_id,
data=dict(vlan_filter_0=vlan_id,
forward_operation=0x10,
number_of_entries=1)))
self.send_omci_message(frame)
def send_create_extended_vlan_tagging_operation_configuration_data(
self, entity_id, assoc_type, assoc_me):
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciCreate.message_id,
omci_message=OmciCreate(
entity_class=ExtendedVlanTaggingOperationConfigurationData.
class_id,
entity_id=entity_id,
data=dict(association_type=assoc_type,
associated_me_pointer=assoc_me)))
self.send_omci_message(frame)
def send_set_extended_vlan_tagging_operation_tpid_configuration_data(
self, entity_id, input_tpid, output_tpid):
data = dict(
input_tpid=input_tpid,
output_tpid=output_tpid,
downstream_mode=0, # inverse of upstream
)
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciSet.message_id,
omci_message=OmciSet(
entity_class=ExtendedVlanTaggingOperationConfigurationData.
class_id,
entity_id=entity_id,
attributes_mask=ExtendedVlanTaggingOperationConfigurationData.
mask_for(*data.keys()),
data=data))
self.send_omci_message(frame)
def send_set_extended_vlan_tagging_operation_vlan_configuration_data_untagged(
self, entity_id, filter_inner_vid, treatment_inner_vid):
data = dict(
received_frame_vlan_tagging_operation_table=VlanTaggingOperation(
filter_outer_priority=15,
filter_outer_vid=4096,
filter_outer_tpid_de=0,
filter_inner_priority=15,
filter_inner_vid=filter_inner_vid,
filter_inner_tpid_de=0,
filter_ether_type=0,
treatment_tags_to_remove=0,
treatment_outer_priority=15,
treatment_outer_vid=0,
treatment_outer_tpid_de=0,
treatment_inner_priority=0,
treatment_inner_vid=treatment_inner_vid,
treatment_inner_tpid_de=4))
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciSet.message_id,
omci_message=OmciSet(
entity_class=ExtendedVlanTaggingOperationConfigurationData.
class_id,
entity_id=entity_id,
attributes_mask=ExtendedVlanTaggingOperationConfigurationData.
mask_for(*data.keys()),
data=data))
self.send_omci_message(frame)
def send_set_extended_vlan_tagging_operation_vlan_configuration_data_single_tag(
self, entity_id, filter_inner_priority, filter_inner_vid,
filter_inner_tpid_de, treatment_tags_to_remove,
treatment_inner_priority, treatment_inner_vid):
data = dict(
received_frame_vlan_tagging_operation_table=VlanTaggingOperation(
filter_outer_priority=15,
filter_outer_vid=4096,
filter_outer_tpid_de=0,
filter_inner_priority=filter_inner_priority,
filter_inner_vid=filter_inner_vid,
filter_inner_tpid_de=filter_inner_tpid_de,
filter_ether_type=0,
treatment_tags_to_remove=treatment_tags_to_remove,
treatment_outer_priority=15,
treatment_outer_vid=0,
treatment_outer_tpid_de=0,
treatment_inner_priority=treatment_inner_priority,
treatment_inner_vid=treatment_inner_vid,
treatment_inner_tpid_de=4))
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciSet.message_id,
omci_message=OmciSet(
entity_class=ExtendedVlanTaggingOperationConfigurationData.
class_id,
entity_id=entity_id,
attributes_mask=ExtendedVlanTaggingOperationConfigurationData.
mask_for(*data.keys()),
data=data))
self.send_omci_message(frame)
def send_create_multicast_operations_profile(self, entity_id, igmp_ver):
frame = OmciFrame(transaction_id=self.get_tx_id(),
message_type=OmciCreate.message_id,
omci_message=OmciCreate(
entity_class=MulticastOperationsProfile.class_id,
entity_id=entity_id,
data=dict(igmp_version=igmp_ver,
igmp_function=0,
immediate_leave=0)))
self.send_omci_message(frame)
def send_set_multicast_operations_profile_acl_row0(self, entity_id,
acl_table, row_key,
gem_port, vlan, src_ip,
dst_ip_start,
dst_ip_end):
row0 = AccessControlRow0(set_ctrl=1,
row_part_id=0,
test=0,
row_key=row_key,
gem_port_id=gem_port,
vlan_id=vlan,
src_ip=src_ip,
dst_ip_start=dst_ip_start,
dst_ip_end=dst_ip_end,
ipm_group_bw=0)
if acl_table == 'dynamic':
data = dict(dynamic_access_control_list_table=row0)
else:
data = dict(static_access_control_list_table=row0)
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciSet.message_id,
omci_message=OmciSet(
entity_class=MulticastOperationsProfile.class_id,
entity_id=entity_id,
attributes_mask=MulticastOperationsProfile.mask_for(
*data.keys()),
data=data))
self.send_omci_message(frame)
def send_set_multicast_operations_profile_ds_igmp_mcast_tci(
self, entity_id, ctrl_type, tci):
data = dict(ds_igmp_mcast_tci=DownstreamIgmpMulticastTci(
ctrl_type=ctrl_type, tci=tci))
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciSet.message_id,
omci_message=OmciSet(
entity_class=MulticastOperationsProfile.class_id,
entity_id=entity_id,
attributes_mask=MulticastOperationsProfile.mask_for(
*data.keys()),
data=data))
self.send_omci_message(frame)
def send_create_multicast_subscriber_config_info(self, entity_id, me_type,
mcast_oper_profile):
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciCreate.message_id,
omci_message=OmciCreate(
entity_class=MulticastSubscriberConfigInfo.class_id,
entity_id=entity_id,
data=dict(
me_type=me_type,
mcast_operations_profile_pointer=mcast_oper_profile)))
self.send_omci_message(frame)
def send_set_multicast_subscriber_config_info(self,
entity_id,
max_groups=0,
max_mcast_bw=0,
bw_enforcement=0):
data = dict(max_simultaneous_groups=max_groups,
max_multicast_bandwidth=max_mcast_bw,
bandwidth_enforcement=bw_enforcement)
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciSet.message_id,
omci_message=OmciSet(
entity_class=MulticastSubscriberConfigInfo.class_id,
entity_id=entity_id,
attributes_mask=MulticastSubscriberConfigInfo.mask_for(
*data.keys()),
data=data))
self.send_omci_message(frame)
def send_set_multicast_service_package(self, entity_id, row_key, vid_uni,
max_groups, max_mcast_bw,
mcast_oper_profile):
data = dict(multicast_service_package_table=MulticastServicePackage(
set_ctrl=1,
row_key=row_key,
vid_uni=vid_uni,
max_simultaneous_groups=max_groups,
max_multicast_bw=max_mcast_bw,
mcast_operations_profile_pointer=mcast_oper_profile))
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciSet.message_id,
omci_message=OmciSet(
entity_class=MulticastSubscriberConfigInfo.class_id,
entity_id=entity_id,
attributes_mask=MulticastSubscriberConfigInfo.mask_for(
*data.keys()),
data=data))
self.send_omci_message(frame)
def send_set_multicast_allowed_preview_groups_row0(self, entity_id,
row_key, src_ip,
vlan_id_ani,
vlan_id_uni):
data = dict(allowed_preview_groups_table=AllowedPreviewGroupsRow0(
set_ctrl=1,
row_part_id=0,
row_key=row_key,
src_ip=src_ip,
vlan_id_ani=vlan_id_ani,
vlan_id_uni=vlan_id_uni))
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciSet.message_id,
omci_message=OmciSet(
entity_class=MulticastSubscriberConfigInfo.class_id,
entity_id=entity_id,
attributes_mask=MulticastSubscriberConfigInfo.mask_for(
*data.keys()),
data=data))
self.send_omci_message(frame)
def send_set_multicast_allowed_preview_groups_row1(self, entity_id,
row_key, dst_ip,
duration, time_left):
data = dict(allowed_preview_groups_table=AllowedPreviewGroupsRow1(
set_ctrl=1,
row_part_id=1,
row_key=row_key,
dst_ip=dst_ip,
duration=duration,
time_left=time_left))
frame = OmciFrame(
transaction_id=self.get_tx_id(),
message_type=OmciSet.message_id,
omci_message=OmciSet(
entity_class=MulticastSubscriberConfigInfo.class_id,
entity_id=entity_id,
attributes_mask=MulticastSubscriberConfigInfo.mask_for(
*data.keys()),
data=data))
self.send_omci_message(frame)
@inlineCallbacks
def wait_for_response(self):
log.info('wait-for-response')
try:
response = yield self.incoming_messages.get()
log.info('got-response')
# resp = OmciFrame(response)
# resp.show()
except Exception as e:
self.log.info('wait-for-response-exception', exc=str(e))
@inlineCallbacks
def message_exchange(self):
log.info('message_exchange')
# reset incoming message queue
while self.incoming_messages.pending:
_ = yield self.incoming_messages.get()
# construct message
# MIB Reset - OntData - 0
self.send_mib_reset()
yield self.wait_for_response()
# Create AR - GalEthernetProfile - 1
self.send_create_gal_ethernet_profile(1, 48)
yield self.wait_for_response()
# Set AR - TCont - 32768 - 1024
self.send_set_tcont(0x8000, 0x400)
yield self.wait_for_response()
# Set AR - TCont - 32769 - 1025
self.send_set_tcont(0x8001, 0x401)
yield self.wait_for_response()
# Set AR - TCont - 32770 - 1026
self.send_set_tcont(0x8002, 0x402)
yield self.wait_for_response()
# Set AR - TCont - 32771 - 1027
self.send_set_tcont(0x8003, 0x403)
yield self.wait_for_response()
# Create AR - 802.1pMapperServiceProfile - 32768
self.send_create_8021p_mapper_service_profile(0x8000)
yield self.wait_for_response()
# Create AR - 802.1pMapperServiceProfile - 32769
self.send_create_8021p_mapper_service_profile(0x8001)
yield self.wait_for_response()
# Create AR - 802.1pMapperServiceProfile - 32770
self.send_create_8021p_mapper_service_profile(0x8002)
yield self.wait_for_response()
# Create AR - 802.1pMapperServiceProfile - 32771
self.send_create_8021p_mapper_service_profile(0x8003)
yield self.wait_for_response()
# Create AR - MacBridgeServiceProfile - 513
self.send_create_mac_bridge_service_profile(0x201)
yield self.wait_for_response()
# Create AR - GemPortNetworkCtp - 256 - 1024 - 32768
self.send_create_gem_port_network_ctp(0x100, 0x400, 0x8000,
"bi-directional", 0x100)
yield self.wait_for_response()
# Create AR - GemPortNetworkCtp - 257 - 1025 - 32769
self.send_create_gem_port_network_ctp(0x101, 0x401, 0x8001,
"bi-directional", 0x100)
yield self.wait_for_response()
# Create AR - GemPortNetworkCtp - 258 - 1026 - 32770
self.send_create_gem_port_network_ctp(0x102, 0x402, 0x8002,
"bi-directional", 0x100)
yield self.wait_for_response()
# Create AR - GemPortNetworkCtp - 259 - 1027 - 32771
self.send_create_gem_port_network_ctp(0x103, 0x403, 0x8003,
"bi-directional", 0x100)
yield self.wait_for_response()
# Create AR - GemPortNetworkCtp - 260 - 4000 - 0
self.send_create_gem_port_network_ctp(0x104, 0x0FA0, 0, "downstream",
0)
yield self.wait_for_response()
# Create AR - MulticastGemInterworkingTp - 6 - 260
self.send_create_multicast_gem_interworking_tp(0x6, 0x104)
yield self.wait_for_response()
# Create AR - GemInterworkingTp - 32769 - 256 -32768 - 1
self.send_create_gem_inteworking_tp(0x8001, 0x100, 0x8000)
yield self.wait_for_response()
# Create AR - GemInterworkingTp - 32770 - 257 -32769 - 1
self.send_create_gem_inteworking_tp(0x8002, 0x101, 0x8001)
yield self.wait_for_response()
# Create AR - GemInterworkingTp - 32771 - 258 -32770 - 1
self.send_create_gem_inteworking_tp(0x8003, 0x102, 0x8002)
yield self.wait_for_response()
# Create AR - GemInterworkingTp - 32772 - 259 -32771 - 1
self.send_create_gem_inteworking_tp(0x8004, 0x103, 0x8003)
yield self.wait_for_response()
# Set AR - 802.1pMapperServiceProfile - 32768 - 32769
self.send_set_8021p_mapper_service_profile(0x8000, 0x8001)
yield self.wait_for_response()
# Set AR - 802.1pMapperServiceProfile - 32769 - 32770
self.send_set_8021p_mapper_service_profile(0x8001, 0x8002)
yield self.wait_for_response()
# Set AR - 802.1pMapperServiceProfile - 32770 - 32771
self.send_set_8021p_mapper_service_profile(0x8002, 0x8003)
yield self.wait_for_response()
# Set AR - 802.1pMapperServiceProfile - 32771 - 32772
self.send_set_8021p_mapper_service_profile(0x8003, 0x8004)
yield self.wait_for_response()
# Create AR - MacBridgePortConfigData - 8449 - 513 - 2 - 3 - 32768
self.send_create_mac_bridge_port_configuration_data(
0x2101, 0x201, 2, 3, 0x8000)
yield self.wait_for_response()
# Create AR - MacBridgePortConfigData - 8450 - 513 - 3 - 3 - 32769
self.send_create_mac_bridge_port_configuration_data(
0x2102, 0x201, 3, 3, 0x8001)
yield self.wait_for_response()
# Create AR - MacBridgePortConfigData - 8451 - 513 - 4 - 3 - 32770
self.send_create_mac_bridge_port_configuration_data(
0x2103, 0x201, 4, 3, 0x8002)
yield self.wait_for_response()
# Create AR - MacBridgePortConfigData - 8452 - 513 - 5 - 3 - 32771
self.send_create_mac_bridge_port_configuration_data(
0x2104, 0x201, 5, 3, 0x8003)
yield self.wait_for_response()
# Create AR - MacBridgePortConfigData - 9000 - 513 - 6 - 6 - 6
self.send_create_mac_bridge_port_configuration_data(
0x2328, 0x201, 6, 6, 6)
yield self.wait_for_response()
# Create AR - VlanTaggingFilterData - 8449 - 040000000000000000000000000000000000000000000000
self.send_create_vlan_tagging_filter_data(0x2101, 0x0400)
yield self.wait_for_response()
# Create AR - VlanTaggingFilterData - 8450 - 040100000000000000000000000000000000000000000000
self.send_create_vlan_tagging_filter_data(0x2102, 0x0401)
yield self.wait_for_response()
# Create AR - VlanTaggingFilterData - 8451 - 040200000000000000000000000000000000000000000000
self.send_create_vlan_tagging_filter_data(0x2103, 0x0402)
yield self.wait_for_response()
# Create AR - VlanTaggingFilterData - 8452 - 040300000000000000000000000000000000000000000000
self.send_create_vlan_tagging_filter_data(0x2104, 0x0403)
yield self.wait_for_response()
# Create AR - MulticastOperationsProfile
self.send_create_multicast_operations_profile(0x201, 3)
yield self.wait_for_response()
# Set AR - MulticastOperationsProfile - Dynamic Access Control List table
self.send_set_multicast_operations_profile_acl_row0(
0x201, 'dynamic', 0, 0x0fa0, 0x0fa0, '0.0.0.0', '224.0.0.0',
'239.255.255.255')
yield self.wait_for_response()
# Create AR - MulticastSubscriberConfigInfo
self.send_create_multicast_subscriber_config_info(0x201, 0, 0x201)
yield self.wait_for_response()
# Set AR - MulticastOperationsProfile - Downstream IGMP Multicast TCI
self.send_set_multicast_operations_profile_ds_igmp_mcast_tci(
0x201, 3, 0x401)
yield self.wait_for_response()
# Create AR - ExtendedVlanTaggingOperationConfigData - 514 - 2 - 0x105
self.send_create_extended_vlan_tagging_operation_configuration_data(
0x202, 2, 0x102)
yield self.wait_for_response()
# Set AR - ExtendedVlanTaggingOperationConfigData - 514 - 8100 - 8100
self.send_set_extended_vlan_tagging_operation_tpid_configuration_data(
0x202, 0x8100, 0x8100)
yield self.wait_for_response()
# Set AR - ExtendedVlanTaggingOperationConfigData
# 514 - RxVlanTaggingOperationTable - add VLAN 1025 to untagged pkts
#self.send_set_extended_vlan_tagging_operation_vlan_configuration_data_untagged(0x202, 0x1000, 0x401)
#yield self.wait_for_response()
# Set AR - ExtendedVlanTaggingOperationConfigData
# 514 - RxVlanTaggingOperationTable - add VLAN 1025 to priority tagged pkts
self.send_set_extended_vlan_tagging_operation_vlan_configuration_data_single_tag(
0x202, 0, 0, 0, 1, 8, 0x401)
yield self.wait_for_response()
# Create AR - MacBridgePortConfigData - 513 - 513 - 1 - 1 - 0x105
self.send_create_mac_bridge_port_configuration_data(
0x201, 0x201, 1, 1, 0x102)
yield self.wait_for_response()
class MQTTService(ClientService):
# Default publish QoS
QoS = 0
def __init__(self, options, **kargs):
self.options = options
setLogLevel(namespace=NAMESPACE, levelStr=options['log_level'])
setLogLevel(namespace=PROTOCOL_NAMESPACE,
levelStr=options['log_messages'])
self.factory = MQTTFactory(profile=MQTTFactory.PUBLISHER)
self.endpoint = clientFromString(reactor, self.options['broker'])
self.task = None
if self.options['username'] == "":
self.options['username'] = None
self.options['password'] = None
ClientService.__init__(self,
self.endpoint,
self.factory,
retryPolicy=backoffPolicy(
initialDelay=INITIAL_DELAY,
factor=FACTOR,
maxDelay=MAX_DELAY))
self.queue = DeferredQueue()
# -----------
# Service API
# -----------
def startService(self):
log.info("starting MQTT Client Service")
self.whenConnected().addCallback(self.connectToBroker)
super().startService()
@inlineCallbacks
def stopService(self):
try:
yield ClientService.stopService(self)
except Exception as e:
log.failure("Exception {excp!s}", excp=e)
reactor.stop()
@inlineCallbacks
def reloadService(self, options):
setLogLevel(namespace=NAMESPACE, levelStr=options['log_level'])
setLogLevel(namespace=PROTOCOL_NAMESPACE,
levelStr=options['log_messages'])
log.info("new log level is {lvl}", lvl=options['log_level'])
self.options = options
# -------------
# log stats API
# -------------
# --------------
# Helper methods
# ---------------
@inlineCallbacks
def connectToBroker(self, protocol):
'''
Connect to MQTT broker
'''
self.protocol = protocol
self.protocol.onPublish = self.publish
self.protocol.onDisconnection = self.onDisconnection
self.protocol.setWindowSize(3)
try:
yield self.protocol.connect("tessw-publisher" + '@' + HOSTNAME,
username=self.options['username'],
password=self.options['password'],
keepalive=self.options['keepalive'])
except Exception as e:
log.failure("Connecting to {broker} raised {excp!s}",
broker=self.options['broker'],
excp=e)
else:
log.info("Connected to {broker}", broker=self.options['broker'])
reactor.callLater(0, self.publish)
def onDisconnection(self, reason):
'''
Disconenction handler.
Tells ClientService what to do when the connection is lost
'''
log.warn("tessw-publisher lost connection with its MQTT broker")
if self.task:
self.task.cancel()
self.task = None
self.whenConnected().addCallback(self.connectToBroker)
def addRegisterRequest(self, photometer_info):
topic = "{0}/{1}".format(self.options['topic'], "register")
self.queue.put((topic, photometer_info))
def addReading(self, reading):
topic = "{0}/{1}/{2}".format(self.options['topic'], reading['name'],
"reading")
self.queue.put((topic, reading))
@inlineCallbacks
def publish(self):
log.info("Entering Registry & Data Publishing Phase")
while True:
try:
topic, reading = yield self.queue.get()
msg = json.dumps(reading)
yield self.protocol.publish(topic=topic,
qos=self.QoS,
message=msg)
except MQTTStateError as e:
log.error("{excp}", excp=e)
except Exception as e:
log.failure("Error when publishing: {excp!s}", excp=e)
else:
pass
class OMCI_CC(object):
""" Handle OMCI Communication Channel specifics for Adtran ONUs"""
def __init__(self,
adapter_agent,
device_id,
custom_me_entries=None,
alarm_queue_limit=_MAX_INCOMING_ALARM_MESSAGES,
avc_queue_limit=_MAX_INCOMING_ALARM_MESSAGES,
test_results_queue_limit=_MAX_INCOMING_TEST_RESULT_MESSAGES):
self.log = structlog.get_logger(device_id=device_id)
self._adapter_agent = adapter_agent
self._device_id = device_id
self._proxy_address = None
self._tx_tid = 1
self._enabled = False
self._requests = dict(
) # Tx ID -> (timestamp, deferred, tx_frame, timeout)
self._alarm_queue = DeferredQueue(size=alarm_queue_limit)
self._avc_queue = DeferredQueue(size=avc_queue_limit)
self._test_results_queue = DeferredQueue(size=test_results_queue_limit)
# Statistics
self._tx_frames = 0
self._rx_frames = 0
self._rx_unknown_tid = 0 # Rx OMCI with no Tx TID match
self._rx_onu_frames = 0 # Autonomously generated ONU frames
self._rx_alarm_overflow = 0 # Autonomously generated ONU alarms rx overflow
self._rx_avc_overflow = 0 # Autonomously generated ONU AVC rx overflow
self._rx_onu_discards = 0 # Autonomously generated ONU unknown message types
self._rx_timeouts = 0
self._tx_errors = 0 # Exceptions during tx request
self._consecutive_errors = 0 # Rx & Tx errors in a row, a good RX resets this to 0
self._reply_min = sys.maxint # Fastest successful tx -> rx
self._reply_max = 0 # Longest successful tx -> rx
self._reply_sum = 0.0 # Total seconds for successful tx->rx (float for average)
# If a list of custom ME Entities classes were provided, insert them into
# main class_id to entity map.
# TODO: If this class becomes hidden from the ONU DA, move this to the OMCI State Machine runner
if custom_me_entries is not None:
add_onu_me_entities(custom_me_entries)
def __str__(self):
return "OMCISupport: {}".format(self._device_id)
@property
def enabled(self):
return self._enabled
@enabled.setter
def enabled(self, value):
"""
Enable/disable the OMCI Communications Channel
:param value: (boolean) True to enable, False to disable
"""
assert isinstance(value, bool), 'enabled is a boolean'
if self._enabled != value:
self._enabled = value
if self._enabled:
self._start()
else:
self._stop()
@property
def tx_frames(self):
return self._tx_frames
@property
def rx_frames(self):
return self._rx_frames
@property
def rx_unknown_tid(self):
return self._rx_unknown_tid # Tx TID not found
@property
def rx_onu_frames(self):
return self._rx_onu_frames
@property
def rx_alarm_overflow(self):
return self._rx_alarm_overflow # Alarm ONU autonomous overflows
@property
def rx_avc_overflow(self):
return self._rx_avc_overflow # Attribute Value change autonomous overflows
@property
def rx_onu_discards(self):
return self._rx_onu_discards # Attribute Value change autonomous overflows
@property
def rx_timeouts(self):
return self._rx_timeouts
@property
def tx_errors(self):
return self._tx_errors
@property
def consecutive_errors(self):
return self._consecutive_errors
@property
def reply_min(self):
return int(round(self._reply_min * 1000.0)) # Milliseconds
@property
def reply_max(self):
return int(round(self._reply_max * 1000.0)) # Milliseconds
@property
def reply_average(self):
avg = self._reply_sum / self._rx_frames if self._rx_frames > 0 else 0.0
return int(round(avg * 1000.0)) # Milliseconds
@property
def get_alarm_message(self):
"""
Attempt to retrieve and remove an ONU Alarm Message from the ONU
autonomous message queue.
TODO: We may want to deprecate this, see TODO comment around line 399 in
the _request_success() method below
:return: a Deferred which fires with the next Alarm Frame available in
the queue.
"""
return self._alarm_queue.get()
@property
def get_avc_message(self):
"""
Attempt to retrieve and remove an ONU Attribute Value Change (AVC)
Message from the ONU autonomous message queue.
TODO: We may want to deprecate this, see TODO comment around line 399 in
the _request_success() method below
:return: a Deferred which fires with the next AVC Frame available in
the queue.
"""
return self._avc_queue.get()
@property
def get_test_results(self):
"""
Attempt to retrieve and remove an ONU Test Results Message from the
ONU autonomous message queue.
TODO: We may want to deprecate this, see TODO comment around line 399 in
the _request_success() method below
:return: a Deferred which fires with the next Test Results Frame is
available in the queue.
"""
return self._test_results_queue.get()
def _start(self):
"""
Start the OMCI Communications Channel
"""
assert self._enabled, 'Start should only be called if enabled'
#
# TODO: Perform any other common startup tasks here
#
self.flush()
device = self._adapter_agent.get_device(self._device_id)
self._proxy_address = device.proxy_address
def _stop(self):
"""
Stop the OMCI Communications Channel
"""
assert not self._enabled, 'Stop should only be called if disabled'
#
# TODO: Perform common shutdown tasks here
#
self.flush()
self._proxy_address = None
# TODO: What is best way to clean up any outstanding futures for these queues
self._alarm_queue = None
self._avc_queue = None
self._test_results_queue = None
def _receive_onu_message(self, rx_frame):
""" Autonomously generated ONU frame Rx handler"""
from twisted.internet.defer import QueueOverflow
self.log.debug('rx-onu-frame',
frame_type=type(rx_frame),
frame=hexify(str(rx_frame)))
# TODO: Signal, via defer if Alarm Overflow or just an event?
msg_type = rx_frame.fields['message_type']
self._rx_onu_frames += 1
if msg_type == EntityOperations.AlarmNotification:
try:
self._alarm_queue.put(
(rx_frame, arrow.utcnow().float_timestamp))
except QueueOverflow:
self._rx_alarm_overflow += 1
self.log.warn('onu-rx-alarm-overflow',
cnt=self._rx_alarm_overflow)
elif msg_type == EntityOperations.AttributeValueChange:
try:
self._alarm_queue.put(
(rx_frame, arrow.utcnow().float_timestamp))
except QueueOverflow:
self._rx_avc_overflow += 1
self.log.warn('onu-rx-avc-overflow', cnt=self._rx_avc_overflow)
else:
# TODO: Need to add test results message support
self.log.warn('onu-unsupported-autonomous-message', type=msg_type)
self._rx_onu_discards += 1
def receive_message(self, msg):
"""
Receive and OMCI message from the proxy channel to the OLT.
Call this from your ONU Adapter on a new OMCI Rx on the proxy channel
"""
if self.enabled:
try:
now = arrow.utcnow()
d = None
try:
rx_frame = OmciFrame(msg)
rx_tid = rx_frame.fields['transaction_id']
if rx_tid == 0:
return self._receive_onu_message(rx_frame)
self._rx_frames += 1
self._consecutive_errors = 0
except KeyError as e:
# Unknown, Unsupported, or vendor-specific ME. Key is the unknown classID
# TODO: Can we create a temporary one to hold it so upload does not always fail on new ME's?
self.log.exception('frame-decode-key-error',
msg=hexlify(msg),
e=e)
return
except Exception as e:
self.log.exception('frame-decode', msg=hexlify(msg), e=e)
return
try:
(ts, d, _, _) = self._requests.pop(rx_tid)
ts_diff = now - arrow.Arrow.utcfromtimestamp(ts)
secs = ts_diff.total_seconds()
self._reply_sum += secs
if secs < self._reply_min:
self._reply_min = secs
if secs > self._reply_max:
self._reply_max = secs
# TODO: Could also validate response type based on request action
except KeyError:
# Possible late Rx on a message that timed-out
self._rx_unknown_tid += 1
self.log.warn('tx-message-missing',
rx_id=rx_tid,
msg=hexlify(msg))
return
except Exception as e:
self.log.exception('frame-match', msg=hexlify(msg), e=e)
if d is not None:
return d.errback(failure.Failure(e))
return
d.callback(rx_frame)
except Exception as e:
self.log.exception('rx-msg', e=e)
def flush(self, max_age=0):
limit = arrow.utcnow().float_timestamp - max_age
old = [
tid for tid, (ts, _, _, _) in self._requests.iteritems()
if ts <= limit
]
for tid in old:
(_, d, _, _) = self._requests.pop(tid)
if d is not None and not d.called:
d.cancel()
self._requests = dict()
if max_age == 0:
# Flush autonomous messages (Alarms & AVCs)
while self._alarm_queue.pending:
_ = yield self._alarm_queue.get()
while self._avc_queue.pending:
_ = yield self._avc_queue.get()
def _get_tx_tid(self):
"""
Get the next Transaction ID for a tx. Note TID=0 is reserved
for autonomously generated messages from an ONU
:return: (int) TID
"""
tx_tid, self._tx_tid = self._tx_tid, self._tx_tid + 1
if self._tx_tid > MAX_OMCI_TX_ID:
self._tx_tid = 1
return tx_tid
def _request_failure(self, value, tx_tid):
"""
Handle a transmit failure and/or Rx timeout
:param value: (Failure) Twisted failure
:param tx_tid: (int) Associated Tx TID
"""
if tx_tid in self._requests:
(_, _, _, timeout) = self._requests.pop(tx_tid)
else:
timeout = 0
if isinstance(value, failure.Failure):
value.trap(CancelledError)
self._rx_timeouts += 1
self._consecutive_errors += 1
self.log.info('timeout', tx_id=tx_tid, timeout=timeout)
value = failure.Failure(TimeoutError(timeout, "Deferred"))
return value
def _request_success(self, rx_frame):
"""
Handle transmit success (a matching Rx was received)
:param rx_frame: (OmciFrame) OMCI response frame with matching TID
:return: (OmciFrame) OMCI response frame with matching TID
"""
#
# TODO: Here we could update the MIB database if we did a set/create/delete
# or perhaps a verify if a GET. Also could increment mib counter
#
# TODO: A better way to perform this in VOLTHA v1.3 would be to provide
# a pub/sub capability for external users/tasks to monitor responses
# that could optionally take a filter. This would allow a MIB-Sync
# task to easily watch all AVC notifications as well as Set/Create/Delete
# operations and keep them serialized. It may also be a better/easier
# way to handle things if we containerize OpenOMCI.
#
try:
if isinstance(rx_frame.omci_message, OmciGetResponse):
pass # TODO: Implement MIB check or remove
elif isinstance(rx_frame.omci_message, OmciSetResponse):
pass # TODO: Implement MIB update
elif isinstance(rx_frame.omci_message, OmciCreateResponse):
pass # TODO: Implement MIB update
elif isinstance(rx_frame.omci_message, OmciDeleteResponse):
pass # TODO: Implement MIB update
except Exception as e:
self.log.exception('omci-message', e=e)
return rx_frame
def send(self, frame, timeout=DEFAULT_OMCI_TIMEOUT):
"""
Send the OMCI Frame to the ONU via the proxy_channel
:param frame: (OMCIFrame) Message to send
:param timeout: (int) Rx Timeout. 0=Forever
:return: (deferred) A deferred that fires when the response frame is received
or if an error/timeout occurs
"""
self.flush(max_age=MAX_OMCI_REQUEST_AGE)
assert timeout <= MAX_OMCI_REQUEST_AGE, \
'Maximum timeout is {} seconds'.format(MAX_OMCI_REQUEST_AGE)
assert isinstance(frame, OmciFrame), \
"Invalid frame class '{}'".format(type(frame))
if not self.enabled or self._proxy_address is None:
# TODO custom exceptions throughout this code would be helpful
return fail(
result=failure.Failure(Exception('OMCI is not enabled')))
try:
tx_tid = frame.fields['transaction_id']
if tx_tid is None:
tx_tid = self._get_tx_tid()
frame.fields['transaction_id'] = tx_tid
assert tx_tid not in self._requests, 'TX TID {} is already exists'.format(
tx_tid)
assert tx_tid >= 0, 'Invalid Tx TID: {}'.format(tx_tid)
ts = arrow.utcnow().float_timestamp
d = defer.Deferred()
self._adapter_agent.send_proxied_message(self._proxy_address,
hexify(str(frame)))
self._tx_frames += 1
self._requests[tx_tid] = (ts, d, frame, timeout)
d.addCallbacks(self._request_success,
self._request_failure,
errbackArgs=(tx_tid, ))
if timeout > 0:
d.addTimeout(timeout, reactor)
except Exception as e:
self._tx_errors += 1
self._consecutive_errors += 1
self.log.exception('send-omci', e=e)
return fail(result=failure.Failure(e))
return d
###################################################################################
# MIB Action shortcuts
def send_mib_reset(self, timeout=DEFAULT_OMCI_TIMEOUT):
"""
Perform a MIB Reset
"""
self.log.debug('send-mib-reset')
frame = OntDataFrame().mib_reset()
return self.send(frame, timeout)
def send_mib_upload(self, timeout=DEFAULT_OMCI_TIMEOUT):
self.log.debug('send-mib-upload')
frame = OntDataFrame().mib_upload()
return self.send(frame, timeout)
def send_mib_upload_next(self, seq_no, timeout=DEFAULT_OMCI_TIMEOUT):
self.log.debug('send-mib-upload-next')
frame = OntDataFrame(seq_no).mib_upload_next()
return self.send(frame, timeout)
def send_reboot(self, timeout=DEFAULT_OMCI_TIMEOUT):
"""
Send an ONU Device reboot request (ONU-G ME).
"""
self.log.debug('send-mib-reboot')
frame = OntGFrame().reboot()
return self.send(frame, timeout)
def test_shared_interface(self):
queue1 = DeferredQueue()
queue2 = DeferredQueue()
# two senders hooked up to the same interface (sharing it)
# here we test if they can both send
pin1 = self.mgr.open_port('veth0', none).up()
pin2 = self.mgr.open_port('veth0', none).up()
pout1 = self.mgr.open_port('veth1', lambda p, f: queue1.put(
(p, f))).up()
filter = BpfProgramFilter('ip dst host 123.123.123.123')
pout2 = self.mgr.open_port('veth1',
lambda p, f: queue2.put((p, f)),
filter=filter).up()
# sending from pin1, should be received by pout1
bogus_frame = 'bogus packet'
bogus_frame_padded = bogus_frame + '\x00' * (FrameIOPort.MIN_PKT_SIZE -
len(bogus_frame))
pin1.send(bogus_frame_padded)
port, frame = yield queue1.get()
self.assertEqual(port, pout1)
self.assertEqual(frame, bogus_frame_padded)
self.assertEqual(len(queue1.pending), 0)
self.assertEqual(len(queue2.pending), 0)
# sending from pin2, should be received by pout1
pin2.send(bogus_frame_padded)
port, frame = yield queue1.get()
self.assertEqual(port, pout1)
self.assertEqual(frame, bogus_frame_padded)
self.assertEqual(len(queue1.pending), 0)
self.assertEqual(len(queue2.pending), 0)
# sending from pin1, should be received by both pouts
ip_packet = str(Ether() / IP(dst='123.123.123.123'))
pin1.send(ip_packet)
port, frame = yield queue1.get()
self.assertEqual(port, pout1)
self.assertEqual(frame, ip_packet)
self.assertEqual(len(queue1.pending), 0)
port, frame = yield queue2.get()
self.assertEqual(port, pout2)
self.assertEqual(frame, ip_packet)
self.assertEqual(len(queue2.pending), 0)
# sending from pin2, should be received by pout1
ip_packet = str(Ether() / IP(dst='123.123.123.123'))
pin2.send(ip_packet)
port, frame = yield queue1.get()
self.assertEqual(port, pout1)
self.assertEqual(frame, ip_packet)
self.assertEqual(len(queue1.pending), 0)
port, frame = yield queue2.get()
self.assertEqual(port, pout2)
self.assertEqual(frame, ip_packet)
self.assertEqual(len(queue2.pending), 0)
self.mgr.close_port(pin1)
self.mgr.close_port(pin2)
self.mgr.close_port(pout1)
self.mgr.close_port(pout2)
class TibitOnuAdapter(object):
name = 'tibit_onu'
supported_device_types = [
DeviceType(id='tibit_onu', adapter=name, accepts_bulk_flow_update=True)
]
def __init__(self, adapter_agent, config):
self.adapter_agent = adapter_agent
self.config = config
self.descriptor = Adapter(
id=self.name,
vendor='Tibit Communications Inc.',
version='0.1',
config=AdapterConfig(log_level=LogLevel.INFO))
self.incoming_messages = DeferredQueue()
def start(self):
log.debug('starting')
log.info('started')
def stop(self):
log.debug('stopping')
log.info('stopped')
def adapter_descriptor(self):
return self.descriptor
def device_types(self):
return DeviceTypes(items=self.supported_device_types)
def health(self):
return HealthStatus(state=HealthStatus.HealthState.HEALTHY)
def change_master_state(self, master):
raise NotImplementedError()
def adopt_device(self, device):
log.info('adopt-device', device=device)
reactor.callLater(0.1, self._onu_device_activation, device)
return device
@inlineCallbacks
def _onu_device_activation(self, device):
# first we verify that we got parent reference and proxy info
assert device.parent_id
assert device.proxy_address.device_id
assert device.proxy_address.channel_id
# TODO: For now, pretend that we were able to contact the device and obtain
# additional information about it. Should add real message.
device.vendor = 'Tibit Communications, Inc.'
device.model = '10G GPON ONU'
device.hardware_version = 'fa161020'
device.firmware_version = '16.12.02'
device.software_version = '1.0'
device.serial_number = uuid4().hex
device.connect_status = ConnectStatus.REACHABLE
self.adapter_agent.update_device(device)
# then shortly after we create some ports for the device
uni_port = Port(port_no=2,
label='UNI facing Ethernet port',
type=Port.ETHERNET_UNI,
admin_state=AdminState.ENABLED,
oper_status=OperStatus.ACTIVE)
self.adapter_agent.add_port(device.id, uni_port)
self.adapter_agent.add_port(
device.id,
Port(port_no=1,
label='PON port',
type=Port.PON_ONU,
admin_state=AdminState.ENABLED,
oper_status=OperStatus.ACTIVE,
peers=[
Port.PeerPort(device_id=device.parent_id,
port_no=device.parent_port_no)
]))
# TODO adding vports to the logical device shall be done by agent?
# then we create the logical device port that corresponds to the UNI
# port of the device
# obtain logical device id
parent_device = self.adapter_agent.get_device(device.parent_id)
logical_device_id = parent_device.parent_id
assert logical_device_id
# we are going to use the proxy_address.channel_id as unique number
# and name for the virtual ports, as this is guaranteed to be unique
# in the context of the OLT port, so it is also unique in the context
# of the logical device
port_no = device.proxy_address.channel_id
cap = OFPPF_10GB_FD | OFPPF_FIBER
self.adapter_agent.add_logical_port(
logical_device_id,
LogicalPort(id=str(port_no),
ofp_port=ofp_port(port_no=port_no,
hw_addr=mac_str_to_tuple(
device.mac_address),
name='uni-{}'.format(port_no),
config=0,
state=OFPPS_LIVE,
curr=cap,
advertised=cap,
peer=cap,
curr_speed=OFPPF_10GB_FD,
max_speed=OFPPF_10GB_FD),
device_id=device.id,
device_port_no=uni_port.port_no))
# simulate a proxied message sending and receving a reply
reply = yield self._message_exchange(device)
# and finally update to "ACTIVE"
device = self.adapter_agent.get_device(device.id)
device.oper_status = OperStatus.ACTIVE
self.adapter_agent.update_device(device)
self.start_kpi_collection(device.id)
def abandon_device(self, device):
raise NotImplementedError(0)
def deactivate_device(self, device):
raise NotImplementedError()
def update_flows_bulk(self, device, flows, groups):
log.info('########################################')
log.info('bulk-flow-update',
device_id=device.id,
flows=flows,
groups=groups)
assert len(groups.items) == 0, "Cannot yet deal with groups"
Clause = {v: k for k, v in ClauseSubtypeEnum.iteritems()}
Operator = {v: k for k, v in RuleOperatorEnum.iteritems()}
for flow in flows.items:
in_port = get_in_port(flow)
assert in_port is not None
precedence = 255 - min(flow.priority / 256, 255)
if in_port == 2:
log.info('#### Upstream Rule ####')
dn_req = EOAMPayload(body=CablelabsOUI() /
DPoEOpcode_SetRequest())
for field in get_ofb_fields(flow):
if field.type == ETH_TYPE:
_type = field.eth_type
log.info('#### field.type == ETH_TYPE ####',
field_type=_type)
elif field.type == IP_PROTO:
_proto = field.ip_proto
log.info('#### field.type == IP_PROTO ####')
elif field.type == IN_PORT:
_port = field.port
log.info('#### field.type == IN_PORT ####', port=_port)
elif field.type == VLAN_VID:
_vlan_vid = field.vlan_vid & 0xfff
log.info('#### field.type == VLAN_VID ####',
vlan=_vlan_vid)
elif field.type == VLAN_PCP:
_vlan_pcp = field.vlan_pcp
log.info('#### field.type == VLAN_PCP ####',
pcp=_vlan_pcp)
elif field.type == UDP_DST:
_udp_dst = field.udp_dst
log.info('#### field.type == UDP_DST ####')
elif field.type == IPV4_DST:
_ipv4_dst = field.ipv4_dst
log.info('#### field.type == IPV4_DST ####')
else:
log.info('#### field.type == NOT IMPLEMENTED!! ####')
raise NotImplementedError('field.type={}'.format(
field.type))
for action in get_actions(flow):
if action.type == OUTPUT:
log.info('#### action.type == OUTPUT ####')
elif action.type == POP_VLAN:
log.info('#### action.type == POP_VLAN ####')
elif action.type == PUSH_VLAN:
log.info('#### action.type == PUSH_VLAN ####')
if action.push.ethertype != 0x8100:
log.error('unhandled-tpid',
ethertype=action.push.ethertype)
elif action.type == SET_FIELD:
log.info('#### action.type == SET_FIELD ####')
assert (action.set_field.field.oxm_class ==
ofp.OFPXMC_OPENFLOW_BASIC)
field = action.set_field.field.ofb_field
if field.type == VLAN_VID:
pass
else:
log.error('unsupported-action-set-field-type',
field_type=field.type)
else:
log.error('UNSUPPORTED-ACTION-TYPE',
action_type=action.type)
elif in_port == 1:
log.info('#### Downstream Rule ####')
#### Loop through fields again...
for field in get_ofb_fields(flow):
if field.type == ETH_TYPE:
_type = field.eth_type
log.info('#### field.type == ETH_TYPE ####',
in_port=in_port,
match=_type)
elif field.type == IP_PROTO:
_proto = field.ip_proto
log.info('#### field.type == IP_PROTO ####',
in_port=in_port,
ip_proto=ip_proto)
elif field.type == IN_PORT:
_port = field.port
log.info('#### field.type == IN_PORT ####')
elif field.type == VLAN_VID:
_vlan_vid = field.vlan_vid & 0xfff
log.info('#### field.type == VLAN_VID ####')
elif field.type == VLAN_PCP:
_vlan_pcp = field.vlan_pcp
log.info('#### field.type == VLAN_PCP ####')
elif field.type == UDP_DST:
_udp_dst = field.udp_dst
log.info('#### field.type == UDP_DST ####')
elif field.type == IPV4_DST:
_ipv4_dst = field.ipv4_dst
log.info('#### field.type == IPV4_DST ####')
a = int(hex(_ipv4_dst)[2:4], 16)
b = int(hex(_ipv4_dst)[4:6], 16)
c = int(hex(_ipv4_dst)[6:8], 16)
d = int(hex(_ipv4_dst)[8:], 16)
dn_req = EOAMPayload(
body=CablelabsOUI() / DPoEOpcode_SetRequest() /
AddStaticMacAddress(
mac=mcastIp2McastMac('%d.%d.%d.%d' %
(a, b, c, d))))
# send message
log.info('ONU-send-proxied-message')
self.adapter_agent.send_proxied_message(
device.proxy_address, dn_req)
else:
raise NotImplementedError('field.type={}'.format(
field.type))
for action in get_actions(flow):
if action.type == OUTPUT:
log.info('#### action.type == OUTPUT ####')
elif action.type == POP_VLAN:
log.info('#### action.type == POP_VLAN ####')
elif action.type == PUSH_VLAN:
log.info('#### action.type == PUSH_VLAN ####')
if action.push.ethertype != 0x8100:
log.error('unhandled-ether-type',
ethertype=action.push.ethertype)
elif action.type == SET_FIELD:
log.info('#### action.type == SET_FIELD ####')
assert (action.set_field.field.oxm_class ==
ofp.OFPXMC_OPENFLOW_BASIC)
field = action.set_field.field.ofb_field
if field.type == VLAN_VID:
pass
else:
log.error('unsupported-action-set-field-type',
field_type=field.type)
else:
log.error('UNSUPPORTED-ACTION-TYPE',
action_type=action.type)
else:
raise Exception('Port should be 1 or 2 by our convention')
def update_flows_incrementally(self, device, flow_changes, group_changes):
raise NotImplementedError()
def send_proxied_message(self, proxy_address, msg):
raise NotImplementedError()
def receive_proxied_message(self, proxy_address, msg):
log.info('receive-proxied-message',
proxy_address=proxy_address,
msg=msg.show(dump=True))
self.incoming_messages.put(msg)
@inlineCallbacks
def _message_exchange(self, device):
# register for receiving async messages
self.adapter_agent.register_for_proxied_messages(device.proxy_address)
# reset incoming message queue
while self.incoming_messages.pending:
_ = yield self.incoming_messages.get()
# construct message
msg = EOAMPayload(body=CablelabsOUI() / DPoEOpcode_GetRequest() /
DeviceId())
# send message
log.info('ONU-send-proxied-message')
self.adapter_agent.send_proxied_message(device.proxy_address, msg)
# wait till we detect incoming message
yield self.incoming_messages.get()
# construct install of igmp query address
msg = EOAMPayload(body=CablelabsOUI() / DPoEOpcode_SetRequest() /
AddStaticMacAddress(mac='01:00:5e:00:00:01'))
# send message
log.info('ONU-send-proxied-message')
self.adapter_agent.send_proxied_message(device.proxy_address, msg)
# wait till we detect incoming message
yield self.incoming_messages.get()
# by returning we allow the device to be shown as active, which
# indirectly verified that message passing works
def receive_packet_out(self, logical_device_id, egress_port_no, msg):
log.info('packet-out',
logical_device_id=logical_device_id,
egress_port_no=egress_port_no,
msg_len=len(msg))
def start_kpi_collection(self, device_id):
"""TMP Simulate periodic KPI metric collection from the device"""
import random
@inlineCallbacks # pretend that we need to do async calls
def _collect(device_id, prefix):
try:
# Step 1: gather metrics from device (pretend it here) - examples
uni_port_metrics = yield dict(
tx_pkts=random.randint(0, 100),
rx_pkts=random.randint(0, 100),
tx_bytes=random.randint(0, 100000),
rx_bytes=random.randint(0, 100000),
)
pon_port_metrics = yield dict(
tx_pkts=uni_port_metrics['rx_pkts'],
rx_pkts=uni_port_metrics['tx_pkts'],
tx_bytes=uni_port_metrics['rx_bytes'],
rx_bytes=uni_port_metrics['tx_bytes'],
)
onu_metrics = yield dict(cpu_util=20 + 5 * random.random(),
buffer_util=10 + 10 * random.random())
# Step 2: prepare the KpiEvent for submission
# we can time-stamp them here (or could use time derived from OLT
ts = arrow.utcnow().timestamp
kpi_event = KpiEvent(
type=KpiEventType.slice,
ts=ts,
prefixes={
# OLT-level
prefix:
MetricValuePairs(metrics=onu_metrics),
# OLT NNI port
prefix + '.nni':
MetricValuePairs(metrics=uni_port_metrics),
# OLT PON port
prefix + '.pon':
MetricValuePairs(metrics=pon_port_metrics)
})
# Step 3: submit
self.adapter_agent.submit_kpis(kpi_event)
except Exception as e:
log.exception('failed-to-submit-kpis', e=e)
prefix = 'voltha.{}.{}'.format(self.name, device_id)
lc = LoopingCall(_collect, device_id, prefix)
lc.start(interval=15) # TODO make this configurable
class TibitOnuAdapter(object):
name = 'tibit_onu'
supported_device_types = [
DeviceType(
id='tibit_onu',
adapter=name,
accepts_bulk_flow_update=True
)
]
def __init__(self, adapter_agent, config):
self.adapter_agent = adapter_agent
self.config = config
self.descriptor = Adapter(
id=self.name,
vendor='Tibit Communications Inc.',
version='0.1',
config=AdapterConfig(log_level=LogLevel.INFO)
)
self.incoming_messages = DeferredQueue()
self.mode = "GPON"
def start(self):
log.debug('starting')
log.info('started')
def stop(self):
log.debug('stopping')
log.info('stopped')
def adapter_descriptor(self):
return self.descriptor
def device_types(self):
return DeviceTypes(items=self.supported_device_types)
def health(self):
return HealthStatus(state=HealthStatus.HealthState.HEALTHY)
def change_master_state(self, master):
raise NotImplementedError()
def update_pm_config(self, device, pm_configs):
raise NotImplementedError()
def adopt_device(self, device):
log.info('adopt-device', device=device)
reactor.callLater(0.1, self._onu_device_activation, device)
return device
def reconcile_device(self, device):
raise NotImplementedError()
@inlineCallbacks
def _onu_device_activation(self, device):
# first we verify that we got parent reference and proxy info
assert device.parent_id
assert device.proxy_address.device_id
assert device.proxy_address.channel_id
# Device information will be updated later on
device.vendor = 'Tibit Communications, Inc.'
device.model = '10G GPON ONU'
device.connect_status = ConnectStatus.REACHABLE
self.adapter_agent.update_device(device)
# then shortly after we create some ports for the device
uni_port = Port(
port_no=2,
label='UNI facing Ethernet port',
type=Port.ETHERNET_UNI,
admin_state=AdminState.ENABLED,
oper_status=OperStatus.ACTIVE
)
self.adapter_agent.add_port(device.id, uni_port)
self.adapter_agent.add_port(device.id, Port(
port_no=1,
label='PON port',
type=Port.PON_ONU,
admin_state=AdminState.ENABLED,
oper_status=OperStatus.ACTIVE,
peers=[
Port.PeerPort(
device_id=device.parent_id,
port_no=device.parent_port_no
)
]
))
# TODO adding vports to the logical device shall be done by agent?
# then we create the logical device port that corresponds to the UNI
# port of the device
# obtain logical device id
parent_device = self.adapter_agent.get_device(device.parent_id)
logical_device_id = parent_device.parent_id
assert logical_device_id
# we are going to use the proxy_address.channel_id as unique number
# and name for the virtual ports, as this is guaranteed to be unique
# in the context of the OLT port, so it is also unique in the context
# of the logical device
port_no = device.proxy_address.channel_id
cap = OFPPF_10GB_FD | OFPPF_FIBER
self.adapter_agent.add_logical_port(logical_device_id, LogicalPort(
id=str(port_no),
ofp_port=ofp_port(
port_no=port_no,
hw_addr=mac_str_to_tuple(device.mac_address),
name='uni-{}'.format(port_no),
config=0,
state=OFPPS_LIVE,
curr=cap,
advertised=cap,
peer=cap,
curr_speed=OFPPF_10GB_FD,
max_speed=OFPPF_10GB_FD
),
device_id=device.id,
device_port_no=uni_port.port_no
))
# simulate a proxied message sending and receving a reply
reply = yield self._message_exchange(device)
# TODO - Need to add validation of reply and decide what to do upon failure
# and finally update to "ACTIVE"
device = self.adapter_agent.get_device(device.id)
device.oper_status = OperStatus.ACTIVE
self.adapter_agent.update_device(device)
# TODO - Disable Stats Reporting for the moment
#self.start_kpi_collection(device.id)
def abandon_device(self, device):
raise NotImplementedError(0
)
def disable_device(self, device):
log.info('disabling', device_id=device.id)
# Disable all ports on that device
self.adapter_agent.disable_all_ports(device.id)
# Update the device operational status to UNKNOWN
device.oper_status = OperStatus.UNKNOWN
device.connect_status = ConnectStatus.UNREACHABLE
self.adapter_agent.update_device(device)
# Remove the uni logical port from the OLT, if still present
parent_device = self.adapter_agent.get_device(device.parent_id)
assert parent_device
logical_device_id = parent_device.parent_id
assert logical_device_id
port_no = device.proxy_address.channel_id
# port_id = 'uni-{}'.format(port_no)
port_id = '{}'.format(port_no)
try:
port = self.adapter_agent.get_logical_port(logical_device_id,
port_id)
self.adapter_agent.delete_logical_port(logical_device_id, port)
except KeyError:
log.info('logical-port-not-found', device_id=device.id,
portid=port_id)
# Remove pon port from parent
#self.adapter_agent.delete_port_reference_from_parent(device.id,
# self.pon_port)
# Just updating the port status may be an option as well
# port.ofp_port.config = OFPPC_NO_RECV
# yield self.adapter_agent.update_logical_port(logical_device_id,
# port)
# Unregister for proxied message
self.adapter_agent.unregister_for_proxied_messages(device.proxy_address)
# TODO:
# 1) Remove all flows from the device
# 2) Remove the device from ponsim
log.info('disabled', device_id=device.id)
return device
def reenable_device(self, device):
log.info('re-enabling', device_id=device.id)
# First we verify that we got parent reference and proxy info
assert device.parent_id
assert device.proxy_address.device_id
assert device.proxy_address.channel_id
# Re-register for proxied messages right away
#self.proxy_address = device.proxy_address
self.adapter_agent.register_for_proxied_messages(device.proxy_address)
# Re-enable the ports on that device
self.adapter_agent.enable_all_ports(device.id)
# Add the pon port reference to the parent
#self.adapter_agent.add_port_reference_to_parent(device.id,
# self.pon_port)
# Update the connect status to REACHABLE
device.connect_status = ConnectStatus.REACHABLE
self.adapter_agent.update_device(device)
# re-add uni port to logical device
parent_device = self.adapter_agent.get_device(device.parent_id)
logical_device_id = parent_device.parent_id
assert logical_device_id
port_no = device.proxy_address.channel_id
cap = OFPPF_10GB_FD | OFPPF_FIBER
self.adapter_agent.add_logical_port(logical_device_id, LogicalPort(
# id='uni-{}'.format(port_no),
id= str(port_no),
ofp_port=ofp_port(
port_no=port_no,
hw_addr=mac_str_to_tuple(device.mac_address),
name='uni-{}'.format(port_no),
config=0,
state=OFPPS_LIVE,
curr=cap,
advertised=cap,
peer=cap,
curr_speed=OFPPF_10GB_FD,
max_speed=OFPPF_10GB_FD
),
device_id=device.id,
device_port_no=2
))
device = self.adapter_agent.get_device(device.id)
device.oper_status = OperStatus.ACTIVE
self.adapter_agent.update_device(device)
log.info('re-enabled', device_id=device.id)
@inlineCallbacks
def reboot_device(self, device):
log.info('Rebooting ONU: {}'.format(device.mac_address))
# Update the operational status to ACTIVATING and connect status to
# UNREACHABLE
previous_oper_status = device.oper_status
previous_conn_status = device.connect_status
device.oper_status = OperStatus.ACTIVATING
device.connect_status = ConnectStatus.UNREACHABLE
self.adapter_agent.update_device(device)
msg = (
EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) /
EOAM_DpoeMsg(dpoe_opcode = Dpoe_Opcodes["Set Request"],
body=DeviceReset())/
EndOfPDU()
)
action = "Device Reset"
# send message
log.info('ONU-send-proxied-message to {} for ONU: {}'.format(action, device.mac_address))
self.adapter_agent.send_proxied_message(device.proxy_address, msg)
rc = []
yield self._handle_set_resp(device, action, rc)
# Change the operational status back to its previous state.
device.oper_status = previous_oper_status
device.connect_status = previous_conn_status
self.adapter_agent.update_device(device)
log.info('ONU Rebooted: {}'.format(device.mac_address))
def download_image(self, device, request):
raise NotImplementedError()
def get_image_download_status(self, device, request):
raise NotImplementedError()
def cancel_image_download(self, device, request):
raise NotImplementedError()
def activate_image_update(self, device, request):
raise NotImplementedError()
def revert_image_update(self, device, request):
raise NotImplementedError()
def self_test_device(self, device):
"""
This is called to Self a device based on a NBI call.
:param device: A Voltha.Device object.
:return: Will return result of self test
"""
log.info('self-test-device', device=device.id)
raise NotImplementedError()
def delete_device(self, device):
log.info('deleting', device_id=device.id)
# A delete request may be received when an OLT is disabled
# TODO:
# 1) Remove all flows from the device
# 2) Remove the device from ponsim
log.info('deleted', device_id=device.id)
def get_device_details(self, device):
raise NotImplementedError()
@inlineCallbacks
def update_flows_bulk(self, device, flows, groups):
log.info('########################################')
log.info('bulk-flow-update', device_id=device.id,
flows=flows, groups=groups)
assert len(groups.items) == 0, "Cannot yet deal with groups"
# Only do something if there are flows to program
if (len(flows.items) > 0):
# Clear the existing entries in the Static MAC Address Table
yield self._send_clear_static_mac_table(device)
# Re-add the IGMP Multicast Address
yield self._send_igmp_mcast_addr(device)
Clause = {v: k for k, v in ClauseSubtypeEnum.iteritems()}
Operator = {v: k for k, v in RuleOperatorEnum.iteritems()}
for flow in flows.items:
in_port = get_in_port(flow)
assert in_port is not None
precedence = 255 - min(flow.priority / 256, 255)
if in_port == 2:
log.info('#### Upstream Rule ####')
up_req = UserPortObject()
up_req /= PortIngressRuleHeader(precedence=precedence)
for field in get_ofb_fields(flow):
if field.type == ETH_TYPE:
_type = field.eth_type
log.info('#### field.type == ETH_TYPE ####',field_type=_type)
elif field.type == IP_PROTO:
_proto = field.ip_proto
log.info('#### field.type == IP_PROTO ####')
elif field.type == IN_PORT:
_port = field.port
log.info('#### field.type == IN_PORT ####', port=_port)
elif field.type == VLAN_VID:
_vlan_vid = field.vlan_vid & 0xfff
log.info('#### field.type == VLAN_VID ####', vlan=_vlan_vid)
up_req /= PortIngressRuleClauseMatchLength02(fieldcode=Clause['C-VLAN Tag'], fieldinstance=0,
operator=Operator['=='], match=_vlan_vid)
elif field.type == VLAN_PCP:
_vlan_pcp = field.vlan_pcp
log.info('#### field.type == VLAN_PCP ####', pcp=_vlan_pcp)
elif field.type == UDP_DST:
_udp_dst = field.udp_dst
log.info('#### field.type == UDP_DST ####', udp_dst=_udp_dst)
elif field.type == IPV4_DST:
_ipv4_dst = field.ipv4_dst
log.info('#### field.type == IPV4_DST ####', ipv4_dst=_ipv4_dst)
elif field.type == METADATA:
_metadata = field.table_metadata
log.info('#### field.type == METADATA ####', metadata=_metadata)
else:
log.info('#### field.type == NOT IMPLEMENTED!! ####')
raise NotImplementedError('field.type={}'.format(
field.type))
for action in get_actions(flow):
if action.type == OUTPUT:
log.info('#### action.type == OUTPUT ####')
up_req /= PortIngressRuleResultInsert(fieldcode=Clause['C-VLAN Tag'])
elif action.type == POP_VLAN:
log.info('#### action.type == POP_VLAN ####')
elif action.type == PUSH_VLAN:
log.info('#### action.type == PUSH_VLAN ####')
up_req /= PortIngressRuleResultInsert(fieldcode=Clause['C-VLAN Tag'])
# if action.push.ethertype != 0x8100:
# log.error('unhandled-tpid',
# ethertype=action.push.ethertype)
elif action.type == SET_FIELD:
log.info('#### action.type == SET_FIELD ####')
assert (action.set_field.field.oxm_class ==
ofp.OFPXMC_OPENFLOW_BASIC)
field = action.set_field.field.ofb_field
if field.type == VLAN_VID:
log.info("#### action.field.vlan {} ####".format(field.vlan_vid & 0xfff))
up_req /= PortIngressRuleResultSet(
fieldcode=Clause['C-VLAN Tag'], value=field.vlan_vid & 0xfff)
else:
log.error('unsupported-action-set-field-type',
field_type=field.type)
else:
log.error('UNSUPPORTED-ACTION-TYPE',
action_type=action.type)
up_req /= PortIngressRuleTerminator()
up_req /= AddPortIngressRule()
msg = (
EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) /
EOAM_DpoeMsg(dpoe_opcode = Dpoe_Opcodes["Set Request"], body=up_req)/
EndOfPDU()
)
# send message
action = "Set ONU US Rule"
log.info('ONU-send-proxied-message to {} for ONU: {}'.format(action, device.mac_address))
self.adapter_agent.send_proxied_message(device.proxy_address, msg)
# Get and process the Set Response
rc = []
yield self._handle_set_resp(device, action, rc)
elif in_port == 1:
log.info('#### Downstream Rule ####')
Is_MCast = False
dn_req = PonPortObject()
dn_req /= PortIngressRuleHeader(precedence=precedence)
#### Loop through fields again...
for field in get_ofb_fields(flow):
if field.type == ETH_TYPE:
_type = field.eth_type
log.info('#### field.type == ETH_TYPE ####', in_port=in_port,
match=_type)
elif field.type == IP_PROTO:
_proto = field.ip_proto
log.info('#### field.type == IP_PROTO ####', in_port=in_port,
ip_proto=_proto)
elif field.type == IN_PORT:
_port = field.port
log.info('#### field.type == IN_PORT ####')
elif field.type == VLAN_VID:
_vlan_vid = field.vlan_vid & 0xfff
log.info('#### field.type == VLAN_VID ####')
elif field.type == VLAN_PCP:
_vlan_pcp = field.vlan_pcp
log.info('#### field.type == VLAN_PCP ####')
elif field.type == UDP_DST:
_udp_dst = field.udp_dst
log.info('#### field.type == UDP_DST ####')
elif field.type == IPV4_DST:
_ipv4_dst = field.ipv4_dst
log.info('#### field.type == IPV4_DST ####')
a = int(hex(_ipv4_dst)[2:4], 16)
b = int(hex(_ipv4_dst)[4:6], 16)
c = int(hex(_ipv4_dst)[6:8], 16)
d = int(hex(_ipv4_dst)[8:], 16)
dn_req = AddStaticMacAddress(mac=mcastIp2McastMac('%d.%d.%d.%d' % (a,b,c,d)))
Is_MCast = True
else:
raise NotImplementedError('field.type={}'.format(
field.type))
for action in get_actions(flow):
if action.type == OUTPUT:
log.info('#### action.type == OUTPUT ####')
elif action.type == POP_VLAN:
log.info('#### action.type == POP_VLAN ####')
dn_req /= PortIngressRuleClauseMatchLength02(fieldcode=Clause['C-VLAN Tag'], fieldinstance=0,
operator=Operator['=='], match=_vlan_vid)
dn_req /= PortIngressRuleResultReplace(fieldcode=Clause['C-VLAN Tag'])
dn_req /= PortIngressRuleResultSet(
fieldcode=Clause['C-VLAN Tag'], value=field.vlan_vid & 0xfff)
elif action.type == PUSH_VLAN:
log.info('#### action.type == PUSH_VLAN ####')
if action.push.ethertype != 0x8100:
log.error('unhandled-ether-type',
ethertype=action.push.ethertype)
elif action.type == SET_FIELD:
log.info('#### action.type == SET_FIELD ####')
assert (action.set_field.field.oxm_class ==
ofp.OFPXMC_OPENFLOW_BASIC)
field = action.set_field.field.ofb_field
if field.type == VLAN_VID:
dn_req /= PortIngressRuleClauseMatchLength02(fieldcode=Clause['C-VLAN Tag'], fieldinstance=0,
operator=Operator['=='], match=_vlan_vid)
dn_req /= PortIngressRuleResultReplace(fieldcode=Clause['C-VLAN Tag'])
dn_req /= PortIngressRuleResultSet(
fieldcode=Clause['C-VLAN Tag'], value=field.vlan_vid & 0xfff)
else:
log.error('unsupported-action-set-field-type',
field_type=field.type)
else:
log.error('UNSUPPORTED-ACTION-TYPE',
action_type=action.type)
if Is_MCast is True:
action = "Set Static IP MCAST address"
else:
dn_req /= PortIngressRuleTerminator()
dn_req /= AddPortIngressRule()
action = "Set ONU DS Rule"
msg = (
EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) /
EOAM_DpoeMsg(dpoe_opcode = Dpoe_Opcodes["Set Request"], body=dn_req)/
EndOfPDU()
)
# send message
log.info('ONU-send-proxied-message to {} for ONU: {}'.format(action, device.mac_address))
self.adapter_agent.send_proxied_message(device.proxy_address, msg)
# Get and process the Set Response
rc = []
yield self._handle_set_resp(device, action, rc)
else:
raise Exception('Port should be 1 or 2 by our convention')
log.info('bulk-flow-update finished', device_id=device.id,
flows=flows, groups=groups)
log.info('########################################')
def update_flows_incrementally(self, device, flow_changes, group_changes):
raise NotImplementedError()
def send_proxied_message(self, proxy_address, msg):
raise NotImplementedError()
def receive_proxied_message(self, proxy_address, msg):
log.info('receive-proxied-message',
proxy_address=proxy_address, msg=msg.show(dump=True))
self.incoming_messages.put(msg)
def create_interface(self, device, data):
raise NotImplementedError()
def update_interface(self, device, data):
raise NotImplementedError()
def remove_interface(self, device, data):
raise NotImplementedError()
def receive_onu_detect_state(self, device_id, state):
raise NotImplementedError()
def create_tcont(self, device, tcont_data, traffic_descriptor_data):
raise NotImplementedError()
def update_tcont(self, device, tcont_data, traffic_descriptor_data):
raise NotImplementedError()
def remove_tcont(self, device, tcont_data, traffic_descriptor_data):
raise NotImplementedError()
def create_gemport(self, device, data):
raise NotImplementedError()
def update_gemport(self, device, data):
raise NotImplementedError()
def remove_gemport(self, device, data):
raise NotImplementedError()
def create_multicast_gemport(self, device, data):
raise NotImplementedError()
def update_multicast_gemport(self, device, data):
raise NotImplementedError()
def remove_multicast_gemport(self, device, data):
raise NotImplementedError()
def create_multicast_distribution_set(self, device, data):
raise NotImplementedError()
def update_multicast_distribution_set(self, device, data):
raise NotImplementedError()
def remove_multicast_distribution_set(self, device, data):
raise NotImplementedError()
@inlineCallbacks
def _message_exchange(self, device):
# register for receiving async messages
self.adapter_agent.register_for_proxied_messages(device.proxy_address)
# reset incoming message queue
while self.incoming_messages.pending:
_ = yield self.incoming_messages.get()
# send out ping frame to ONU device get device information
ping_frame = (
EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) /
EOAM_DpoeMsg(dpoe_opcode=Dpoe_Opcodes["Get Request"],
body=VendorName() /
OnuMode() /
HardwareVersion() /
ManufacturerInfo()
) /
EndOfPDU()
)
log.info('ONU-send-proxied-message to Get Version Info for ONU: {}'.format(device.mac_address))
self.adapter_agent.send_proxied_message(device.proxy_address, ping_frame)
# Loop until we have a Get Response
ack = False
while not ack:
frame = yield self.incoming_messages.get()
respType = get_oam_msg_type(log, frame)
if (respType == RxedOamMsgTypeEnum["DPoE Get Response"]):
ack = True
else:
# Handle unexpected events/OMCI messages
check_resp(log, frame)
if ack:
log.info('ONU-response received for Get Version Info for ONU: {}'.format(device.mac_address))
self._process_ping_frame_response(device, frame)
if self.mode.upper()[0] == "G": # GPON
hw_vers = int(device.hardware_version, 16)
if hw_vers >= 0x170618:
mcastLidx = 0x04bc
elif hw_vers >= 0x170517:
mcastLidx = 0x14bc
else:
mcastLidx = 0x10bc
log.info("Using Multicast LIDX {:04X}".format(mcastLidx))
# construct multicast LLID set
msg = (
EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) /
EOAM_DpoeMsg(dpoe_opcode=Dpoe_Opcodes["Multicast Register"],body=MulticastRegisterSet(MulticastLink=mcastLidx, UnicastLink=0)
))
# send message
log.info('ONU-send-proxied-message to Multicast Register Set for ONU: {}'.format(device.mac_address))
self.adapter_agent.send_proxied_message(device.proxy_address, msg)
# The MulticastRegisterSet does not currently return a response. Just hope it worked.
# by returning we allow the device to be shown as active, which
# indirectly verified that message passing works
def receive_packet_out(self, logical_device_id, egress_port_no, msg):
log.info('packet-out', logical_device_id=logical_device_id,
egress_port_no=egress_port_no, msg_len=len(msg))
def receive_inter_adapter_message(self, msg):
raise NotImplementedError()
def suppress_alarm(self, filter):
raise NotImplementedError()
def unsuppress_alarm(self, filter):
raise NotImplementedError()
def start_kpi_collection(self, device_id):
"""TMP Simulate periodic KPI metric collection from the device"""
import random
@inlineCallbacks # pretend that we need to do async calls
def _collect(device_id, prefix):
try:
# Step 1: gather metrics from device (pretend it here) - examples
uni_port_metrics = yield dict(
tx_pkts=random.randint(0, 100),
rx_pkts=random.randint(0, 100),
tx_bytes=random.randint(0, 100000),
rx_bytes=random.randint(0, 100000),
)
pon_port_metrics = yield dict(
tx_pkts=uni_port_metrics['rx_pkts'],
rx_pkts=uni_port_metrics['tx_pkts'],
tx_bytes=uni_port_metrics['rx_bytes'],
rx_bytes=uni_port_metrics['tx_bytes'],
)
onu_metrics = yield dict(
cpu_util=20 + 5 * random.random(),
buffer_util=10 + 10 * random.random()
)
# Step 2: prepare the KpiEvent for submission
# we can time-stamp them here (or could use time derived from OLT
ts = arrow.utcnow().timestamp
kpi_event = KpiEvent(
type=KpiEventType.slice,
ts=ts,
prefixes={
# OLT-level
prefix: MetricValuePairs(metrics=onu_metrics),
# OLT NNI port
prefix + '.nni': MetricValuePairs(metrics=uni_port_metrics),
# OLT PON port
prefix + '.pon': MetricValuePairs(metrics=pon_port_metrics)
}
)
# Step 3: submit
self.adapter_agent.submit_kpis(kpi_event)
except Exception as e:
log.exception('failed-to-submit-kpis', e=e)
prefix = 'voltha.{}.{}'.format(self.name, device_id)
lc = LoopingCall(_collect, device_id, prefix)
lc.start(interval=15) # TODO make this configurable
# Methods for Get / Set Response Processing from eoam_messages
@inlineCallbacks
def _send_igmp_mcast_addr(self, device):
# construct install of igmp query address
msg = (
EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) /
EOAM_DpoeMsg(dpoe_opcode=Dpoe_Opcodes["Set Request"],body=AddStaticMacAddress(mac='01:00:5e:00:00:01')
))
action = "Set Static IGMP MAC address"
# send message
log.info('ONU-send-proxied-message to {} for ONU: {}'.format(action, device.mac_address))
self.adapter_agent.send_proxied_message(device.proxy_address, msg)
rc = []
yield self._handle_set_resp(device, action, rc)
@inlineCallbacks
def _send_clear_static_mac_table(self, device):
# construct install of igmp query address
msg = (
EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) /
EOAM_DpoeMsg(dpoe_opcode=Dpoe_Opcodes["Set Request"],body=ClearStaticMacTable()
))
action = "Clear Static MAC Table"
# send message
log.info('ONU-send-proxied-message to {} for ONU: {}'.format(action, device.mac_address))
self.adapter_agent.send_proxied_message(device.proxy_address, msg)
rc = []
yield self._handle_set_resp(device, action, rc)
@inlineCallbacks
def _handle_set_resp(self, device, action, retcode):
# Get and process the Set Response
ack = False
start_time = time.time()
# Loop until we have a set response or timeout
while not ack:
frame = yield self.incoming_messages.get()
#TODO - Need to add propoer timeout functionality
#if (time.time() - start_time) > TIBIT_MSG_WAIT_TIME or (frame is None):
# break # don't wait forever
respType = get_oam_msg_type(log, frame)
#Check that the message received is a Set Response
if (respType == RxedOamMsgTypeEnum["DPoE Set Response"]):
ack = True
else:
log.info('Received Unexpected OAM Message 0x{:X} while waiting for Set Resp for {}'.format(respType,action))
# Handle unexpected events/OMCI messages
check_resp(log, frame)
# Verify Set Response
rc = False
if ack:
(rc,branch,leaf,status) = check_set_resp(log, frame)
if (rc is False):
log.info('Set Response had errors - Branch 0x{:X} Leaf 0x{:0>4X} {}'.format(branch, leaf, DPoEVariableResponseCodes[status]))
if (rc is True):
log.info('ONU-response received for {} for ONU: {}'.format(action, device.mac_address))
else:
log.info('BAD ONU-response received for {} for ONU: {}'.format(action, device.mac_address))
retcode.append(rc)
def _process_ping_frame_response(self, device, frame):
vendor = [0xD7, 0x0011]
ponMode = [0xB7, 0x0105]
hw_version = [0xD7, 0x0013]
manufacturer = [0xD7, 0x0006]
branch_leaf_pairs = [vendor, ponMode, hw_version, manufacturer]
for pair in branch_leaf_pairs:
temp_pair = pair
(rc, value) = (get_value_from_msg(log, frame, pair[0], pair[1]))
temp_pair.append(rc)
temp_pair.append(value)
if rc:
overall_rc = True
else:
log.info('Failed to get valid response for Branch 0x{:X} Leaf 0x{:0>4X} '.format(temp_pair[0], temp_pair[1]))
ack = True
if vendor[rc]:
device.vendor = vendor.pop()
if device.vendor.endswith(''):
device.vendor = device.vendor[:-1]
else:
device.vendor = "UNKNOWN"
# mode: 3 = EPON OLT, 7 = GPON OLT
# mode: 2 = EPON ONU, 6 = GPON ONU
if ponMode[rc]:
value = ponMode.pop()
mode = "UNKNOWN"
self.mode = "UNKNOWN"
if value == 6:
mode = "10G GPON ONU"
self.mode = "GPON"
if value == 2:
mode = "10G EPON ONU"
self.mode = "EPON"
if value == 1:
mode = "10G Point to Point"
self.mode = "Unsupported"
device.model = mode
else:
device.model = "UNKNOWN"
self.mode = "UNKNOWN"
log.info("PON Mode is {}".format(self.mode))
if hw_version[rc]:
device.hardware_version = hw_version.pop()
device.hardware_version = device.hardware_version.replace("FA","")
if device.hardware_version.endswith(''):
device.hardware_version = device.hardware_version[:-1]
else:
device.hardware_version = "UNKNOWN"
if manufacturer[rc]:
manu_value = manufacturer.pop()
device.firmware_version = re.search('\Firmware: (.+?) ', manu_value).group(1)
image_1 = Image(version = \
re.search('\Build: (.+?) ', manu_value).group(1))
device.images.image.extend([ image_1 ])
device.serial_number = re.search('\Serial #: (.+?) ', manu_value).group(1)
else:
device.firmware_version = "UNKNOWN"
image_1 = Image(version="UNKNOWN")
device.images.image.extend([ image_1 ])
device.serial_number = "UNKNOWN"
device.connect_status = ConnectStatus.REACHABLE
class EsmeTransceiver(Protocol):
bind_pdu = BindTransceiver
clock = reactor
noisy = True
unbind_timeout = 2
OPEN_STATE = 'OPEN'
CLOSED_STATE = 'CLOSED'
BOUND_STATE_TRX = 'BOUND_TRX'
BOUND_STATE_TX = 'BOUND_TX'
BOUND_STATE_RX = 'BOUND_RX'
BOUND_STATES = set([
BOUND_STATE_RX,
BOUND_STATE_TX,
BOUND_STATE_TRX,
])
def __init__(self, vumi_transport):
"""
An SMPP 3.4 client suitable for use by a Vumi Transport.
:param SmppTransceiverProtocol vumi_transport:
The transport that is using this protocol to communicate
with an SMSC.
"""
self.vumi_transport = vumi_transport
self.config = self.vumi_transport.get_static_config()
self.buffer = b''
self.state = self.CLOSED_STATE
self.deliver_sm_processor = self.vumi_transport.deliver_sm_processor
self.dr_processor = self.vumi_transport.dr_processor
self.sequence_generator = self.vumi_transport.sequence_generator
self.enquire_link_call = LoopingCall(self.enquire_link)
self.drop_link_call = None
self.idle_timeout = self.config.smpp_enquire_link_interval * 2
self.disconnect_call = self.clock.callLater(
self.idle_timeout, self.disconnect,
'Disconnecting, no response from SMSC for longer '
'than %s seconds' % (self.idle_timeout, ))
self.unbind_resp_queue = DeferredQueue()
def emit(self, msg):
if self.noisy:
log.debug(msg)
def connectionMade(self):
self.state = self.OPEN_STATE
log.msg('Connection made, current state: %s' % (self.state, ))
@inlineCallbacks
def bind(self,
system_id,
password,
system_type,
interface_version='34',
addr_ton='',
addr_npi='',
address_range=''):
"""
Send the `bind_transmitter`, `bind_transceiver` or `bind_receiver`
PDU to the SMSC in order to establish the connection.
:param str system_id:
Identifies the ESME system requesting to bind.
:param str password:
The password may be used by the SMSC to authenticate the
ESME requesting to bind.
:param str system_type:
Identifies the type of ESME system requesting to bind
with the SMSC.
:param str interface_version:
Indicates the version of the SMPP protocol supported by the
ESME.
:param str addr_ton:
Indicates Type of Number of the ESME address.
:param str addr_npi:
Numbering Plan Indicator for ESME address.
:param str address_range:
The ESME address.
"""
sequence_number = yield self.sequence_generator.next()
pdu = self.bind_pdu(sequence_number,
system_id=system_id,
password=password,
system_type=system_type,
interface_version=interface_version,
addr_ton=addr_ton,
addr_npi=addr_npi,
address_range=address_range)
self.send_pdu(pdu)
self.drop_link_call = self.clock.callLater(
self.config.smpp_bind_timeout, self.drop_link)
def drop_link(self):
"""
Called if the SMPP connection is not bound within
``smpp_bind_timeout`` amount of seconds
"""
if self.is_bound():
return
return self.disconnect(
'Dropping link due to binding delay. Current state: %s' %
(self.state))
def disconnect(self, log_msg=None):
"""
Forcibly close the connection, logging ``log_msg`` if provided.
:param str log_msg:
The entry to write to the log file.
"""
if log_msg is not None:
log.warning(log_msg)
if not self.connected:
return succeed(self.transport.loseConnection())
d = self.unbind()
d.addCallback(lambda _: self.unbind_resp_queue.get())
d.addBoth(lambda *a: self.transport.loseConnection())
# Give the SMSC a few seconds to respond with an unbind_resp
self.clock.callLater(self.unbind_timeout, d.cancel)
return d
def connectionLost(self, reason):
"""
:param Exception reason:
The reason for the connection closed, generally a
``ConnectionDone``
"""
self.state = self.CLOSED_STATE
if self.enquire_link_call.running:
self.enquire_link_call.stop()
if self.drop_link_call is not None and self.drop_link_call.active():
self.drop_link_call.cancel()
if self.disconnect_call.active():
self.disconnect_call.cancel()
def is_bound(self):
"""
Returns ``True`` if the connection is in one of the known
values of ``self.BOUND_STATES``
"""
return self.state in self.BOUND_STATES
@require_bind
@inlineCallbacks
def enquire_link(self):
"""
Ping the SMSC to see if they're still around.
"""
sequence_number = yield self.sequence_generator.next()
self.send_pdu(EnquireLink(sequence_number))
returnValue(sequence_number)
def send_pdu(self, pdu):
"""
Send a PDU to the SMSC
:param smpp.pdu_builder.PDU pdu:
The PDU object to send.
"""
self.emit('OUTGOING >> %r' % (pdu.get_obj(), ))
return self.transport.write(pdu.get_bin())
def dataReceived(self, data):
self.buffer += data
data = self.handle_buffer()
while data is not None:
self.on_pdu(unpack_pdu(data))
data = self.handle_buffer()
def handle_buffer(self):
pdu_found = chop_pdu_stream(self.buffer)
if pdu_found is None:
return
data, self.buffer = pdu_found
return data
def on_pdu(self, pdu):
"""
Handle a PDU that was received & decoded.
:param dict pdu:
The dict result one gets when calling ``smpp.pdu.unpack_pdu()``
on the received PDU
"""
self.emit('INCOMING << %r' % (pdu, ))
handler = getattr(self, 'handle_%s' % (command_id(pdu), ),
self.on_unsupported_command_id)
return maybeDeferred(handler, pdu)
def on_unsupported_command_id(self, pdu):
"""
Called when an SMPP PDU is received for which no handler function has
been defined.
:param dict pdu:
The dict result one gets when calling ``smpp.pdu.unpack_pdu()``
on the received PDU
"""
log.warning('Received unsupported SMPP command_id: %r' %
(command_id(pdu), ))
def handle_bind_transceiver_resp(self, pdu):
if not pdu_ok(pdu):
log.warning('Unable to bind: %r' % (command_status(pdu), ))
self.transport.loseConnection()
return
self.state = self.BOUND_STATE_TRX
return self.on_smpp_bind(seq_no(pdu))
def handle_bind_transmitter_resp(self, pdu):
if not pdu_ok(pdu):
log.warning('Unable to bind: %r' % (command_status(pdu), ))
self.transport.loseConnection()
return
self.state = self.BOUND_STATE_TX
return self.on_smpp_bind(seq_no(pdu))
def handle_bind_receiver_resp(self, pdu):
if not pdu_ok(pdu):
log.warning('Unable to bind: %r' % (command_status(pdu), ))
self.transport.loseConnection()
return
self.state = self.BOUND_STATE_RX
return self.on_smpp_bind(seq_no(pdu))
def on_smpp_bind(self, sequence_number):
"""Called when the bind has been setup"""
self.drop_link_call.cancel()
self.enquire_link_call.start(self.config.smpp_enquire_link_interval)
def handle_unbind(self, pdu):
return self.send_pdu(UnbindResp(seq_no(pdu)))
def handle_submit_sm_resp(self, pdu):
return self.on_submit_sm_resp(seq_no(pdu), message_id(pdu),
command_status(pdu))
def on_submit_sm_resp(self, sequence_number, smpp_message_id,
command_status):
"""
Called when a ``submit_sm_resp`` command was received.
:param int sequence_number:
The sequence_number of the command, should correlate with the
sequence_number of the ``submit_sm`` command that this is a
response to.
:param str smpp_message_id:
The message id that the SMSC is using for this message.
This will be referred to in the delivery reports (if any).
:param str command_status:
The SMPP command_status for this command. Will determine if
the ``submit_sm`` command was successful or not. Refer to the
SMPP specification for full list of options.
"""
log.warning('onSubmitSMResp called but not implemented by ESME class.')
@inlineCallbacks
def handle_deliver_sm(self, pdu):
# These operate before the PDUs ``short_message`` or
# ``message_payload`` fields have been string decoded.
# NOTE: order is important!
pdu_handler_chain = [
self.dr_processor.handle_delivery_report_pdu,
self.deliver_sm_processor.handle_multipart_pdu,
self.deliver_sm_processor.handle_ussd_pdu,
]
for handler in pdu_handler_chain:
handled = yield handler(pdu)
if handled:
self.send_pdu(
DeliverSMResp(seq_no(pdu), command_status='ESME_ROK'))
return
# At this point we either have a DR in the message payload
# or have a normal SMS that needs to be decoded and handled.
content_parts = self.deliver_sm_processor.decode_pdus([pdu])
if not all([isinstance(part, unicode) for part in content_parts]):
command_status = self.config.deliver_sm_decoding_error
log.msg('Not all parts of the PDU were able to be decoded. '
'Responding with %s.' % (command_status, ),
parts=content_parts)
self.send_pdu(
DeliverSMResp(seq_no(pdu), command_status=command_status))
return
content = u''.join(content_parts)
was_cdr = yield self.dr_processor.handle_delivery_report_content(
content)
if was_cdr:
self.send_pdu(DeliverSMResp(seq_no(pdu),
command_status='ESME_ROK'))
return
handled = yield self.deliver_sm_processor.handle_short_message_pdu(pdu)
if handled:
self.send_pdu(DeliverSMResp(seq_no(pdu),
command_status="ESME_ROK"))
return
command_status = self.config.deliver_sm_decoding_error
log.warning('Unable to process message. '
'Responding with %s.' % (command_status, ),
content=content,
pdu=pdu.get_obj())
self.send_pdu(DeliverSMResp(seq_no(pdu),
command_status=command_status))
def handle_enquire_link(self, pdu):
return self.send_pdu(EnquireLinkResp(seq_no(pdu)))
def handle_enquire_link_resp(self, pdu):
self.disconnect_call.reset(self.idle_timeout)
@require_bind
@inlineCallbacks
def submit_sm(self,
vumi_message_id,
destination_addr,
source_addr='',
esm_class=0,
protocol_id=0,
priority_flag=0,
schedule_delivery_time='',
validity_period='',
replace_if_present=0,
data_coding=0,
sm_default_msg_id=0,
sm_length=0,
short_message='',
optional_parameters=None,
**configured_parameters):
"""
Put a `submit_sm` command on the wire.
:param str source_addr:
Address of SME which originated this message.
If unknown leave blank.
:param str destination_addr:
Destination address of this short message.
For mobile terminated messages, this is the directory number
of the recipient MS.
:param str service_type:
The service_type parameter can be used to indicate the SMS
Application service associated with the message.
If unknown leave blank.
:param int source_addr_ton:
Type of Number for source address.
:param int source_addr_npi:
Numbering Plan Indicator for source address.
:param int dest_addr_ton:
Type of Number for destination.
:param int dest_addr_npi:
Numbering Plan Indicator for destination.
:param int esm_class:
Indicates Message Mode & Message Type.
:param int protocol_id:
Protocol Identifier. Network specific field.
:param int priority_flag:
Designates the priority level of the message.
:param str schedule_delivery_time:
The short message is to be scheduled by the SMSC for delivery.
Leave blank for immediate delivery.
:param str validity_period:
The validity period of this message.
Leave blank for SMSC default.
:param int registered_delivery:
Indicator to signify if an SMSC delivery receipt or an SME
acknowledgement is required.
:param int replace_if_present:
Flag indicating if submitted message should replace an
existing message.
:param int data_coding:
Defines the encoding scheme of the short message user data.
:param int sm_default_msg_id:
Indicates the short message to send from a list of pre- defined
('canned') short messages stored on the SMSC.
Leave blank if not using an SMSC canned message.
:param int sm_length:
Length in octets of the short_message user data.
This is automatically calculated and set during PDU encoding,
no need to specify.
:param int short_message:
Up to 254 octets of short message user data.
The exact physical limit for short_message size may vary
according to the underlying network.
Applications which need to send messages longer than 254
octets should use the message_payload parameter. In this
case the sm_length field should be set to zero.
:param dict optional_parameters:
keys and values to be embedded in the PDU as tag-length-values.
Refer to the SMPP specification and your SMSCs instructions
on what valid and suitable keys and values are.
:returns: list of 1 sequence number (int) for consistency with other
submit_sm calls.
:rtype: list
"""
configured_param_values = {
'service_type': self.config.service_type,
'source_addr_ton': self.config.source_addr_ton,
'source_addr_npi': self.config.source_addr_npi,
'dest_addr_ton': self.config.dest_addr_ton,
'dest_addr_npi': self.config.dest_addr_npi,
'registered_delivery': self.config.registered_delivery,
}
configured_param_values.update(configured_parameters)
sequence_number = yield self.sequence_generator.next()
pdu = SubmitSM(sequence_number=sequence_number,
source_addr=source_addr,
destination_addr=destination_addr,
esm_class=esm_class,
protocol_id=protocol_id,
priority_flag=priority_flag,
schedule_delivery_time=schedule_delivery_time,
validity_period=validity_period,
replace_if_present=replace_if_present,
data_coding=data_coding,
sm_default_msg_id=sm_default_msg_id,
sm_length=sm_length,
short_message=short_message,
**configured_param_values)
if optional_parameters:
for key, value in optional_parameters.items():
pdu.add_optional_parameter(key, value)
yield self.vumi_transport.message_stash.set_sequence_number_message_id(
sequence_number, vumi_message_id)
self.send_pdu(pdu)
returnValue([sequence_number])
def submit_sm_long(self, vumi_message_id, destination_addr, long_message,
**pdu_params):
"""
Send a `submit_sm` command with the message encoded in the
``message_payload`` optional parameter.
Same parameters apply as for ``submit_sm`` with the exception
that the ``short_message`` keyword argument is disallowed
because it conflicts with the ``long_message`` field.
:returns: list of 1 sequence number, int.
:rtype: list
"""
if 'short_message' in pdu_params:
raise EsmeProtocolError(
'short_message not allowed when sending a long message'
'in the message_payload')
optional_parameters = pdu_params.pop('optional_parameters', {}).copy()
optional_parameters.update({
'message_payload': (''.join('%02x' % ord(c) for c in long_message))
})
return self.submit_sm(vumi_message_id,
destination_addr,
short_message='',
sm_length=0,
optional_parameters=optional_parameters,
**pdu_params)
def _fits_in_one_message(self, message):
if len(message) <= GSM_MAX_SMS_BYTES:
return True
# NOTE: We already have byte strings here, so we assume that printable
# ASCII characters are all the same as single-width GSM 03.38
# characters.
if len(message) <= GSM_MAX_SMS_7BIT_CHARS:
# TODO: We need better character handling and counting stuff.
return all(0x20 <= ord(ch) <= 0x7f for ch in message)
return False
def csm_split_message(self, message):
"""
Chop the message into 130 byte chunks to leave 10 bytes for the
user data header the SMSC is presumably going to add for us. This is
a guess based mostly on optimism and the hope that we'll never have
to deal with this stuff in production.
NOTE: If we have utf-8 encoded data, we might break in the
middle of a multibyte character. This should be ok since
the message is only decoded after re-assembly of all
individual segments.
:param str message:
The message to split
:returns: list of strings
:rtype: list
"""
if self._fits_in_one_message(message):
return [message]
payload_length = GSM_MAX_SMS_BYTES - 10
split_msg = []
while message:
split_msg.append(message[:payload_length])
message = message[payload_length:]
return split_msg
@inlineCallbacks
def submit_csm_sar(self, vumi_message_id, destination_addr, **pdu_params):
"""
Submit a concatenated SMS to the SMSC using the optional
SAR parameter names in the various PDUS.
:returns: List of sequence numbers (int) for each of the segments.
:rtype: list
"""
split_msg = self.csm_split_message(pdu_params.pop('short_message'))
if len(split_msg) == 1:
# There is only one part, so send it without SAR stuff.
sequence_numbers = yield self.submit_sm(vumi_message_id,
destination_addr,
short_message=split_msg[0],
**pdu_params)
returnValue(sequence_numbers)
optional_parameters = pdu_params.pop('optional_parameters', {}).copy()
ref_num = yield self.sequence_generator.next()
sequence_numbers = []
yield self.vumi_transport.message_stash.init_multipart_info(
vumi_message_id, len(split_msg))
for i, msg in enumerate(split_msg):
pdu_params = pdu_params.copy()
optional_parameters.update({
# Reference number must be between 00 & FFFF
'sar_msg_ref_num': (ref_num % 0xFFFF),
'sar_total_segments': len(split_msg),
'sar_segment_seqnum': i + 1,
})
sequence_number = yield self.submit_sm(
vumi_message_id,
destination_addr,
short_message=msg,
optional_parameters=optional_parameters,
**pdu_params)
sequence_numbers.extend(sequence_number)
returnValue(sequence_numbers)
@inlineCallbacks
def submit_csm_udh(self, vumi_message_id, destination_addr, **pdu_params):
"""
Submit a concatenated SMS to the SMSC using user data headers (UDH)
in the message content.
Same parameters apply as for ``submit_sm`` with the exception
that the ``esm_class`` keyword argument is disallowed
because the SMPP spec mandates a value that is to be set for UDH.
:returns: List of sequence numbers (int) for each of the segments.
:rtype: list
"""
if 'esm_class' in pdu_params:
raise EsmeProtocolError(
'Cannot specify esm_class, GSM spec sets this at 0x40 '
'for concatenated messages using UDH.')
pdu_params = pdu_params.copy()
split_msg = self.csm_split_message(pdu_params.pop('short_message'))
if len(split_msg) == 1:
# There is only one part, so send it without UDH stuff.
sequence_numbers = yield self.submit_sm(vumi_message_id,
destination_addr,
short_message=split_msg[0],
**pdu_params)
returnValue(sequence_numbers)
ref_num = yield self.sequence_generator.next()
sequence_numbers = []
yield self.vumi_transport.message_stash.init_multipart_info(
vumi_message_id, len(split_msg))
for i, msg in enumerate(split_msg):
# 0x40 is the UDHI flag indicating that this payload contains a
# user data header.
# NOTE: Looking at the SMPP specs I can find no requirement
# for this anywhere.
pdu_params['esm_class'] = 0x40
# See http://en.wikipedia.org/wiki/User_Data_Header and
# http://en.wikipedia.org/wiki/Concatenated_SMS for an
# explanation of the magic numbers below. We should probably
# abstract this out into a class that makes it less magic and
# opaque.
udh = ''.join([
'\05', # Full UDH header length
'\00', # Information Element Identifier for Concatenated SMS
'\03', # header length
# Reference number must be between 00 & FF
chr(ref_num % 0xFF),
chr(len(split_msg)),
chr(i + 1),
])
short_message = udh + msg
sequence_number = yield self.submit_sm(vumi_message_id,
destination_addr,
short_message=short_message,
**pdu_params)
sequence_numbers.extend(sequence_number)
returnValue(sequence_numbers)
@require_bind
@inlineCallbacks
def query_sm(self,
message_id,
source_addr_ton=0,
source_addr_npi=0,
source_addr=''):
"""
Query the SMSC for the status of an earlier sent message.
:param str message_id:
Message ID of the message whose state is to be queried.
This must be the SMSC assigned Message ID allocated to the
original short message when submitted to the SMSC by the
submit_sm, data_sm or submit_multi command, and returned
in the response PDU by the SMSC.
:param int source_addr_ton:
Type of Number of message originator. This is used for
verification purposes, and must match that supplied in the
original request PDU (e.g. submit_sm).
:param int source_addr_npi:
Numbering Plan Identity of message originator. This is used
for verification purposes, and must match that supplied in
the original request PDU (e.g. submit_sm).
:param str source_addr:
Address of message originator.
This is used for verification purposes, and must match that
supplied in the original request PDU (e.g. submit_sm).
"""
sequence_number = yield self.sequence_generator.next()
pdu = QuerySM(sequence_number=sequence_number,
message_id=message_id,
source_addr=source_addr,
source_addr_npi=source_addr_npi,
source_addr_ton=source_addr_ton)
self.send_pdu(pdu)
returnValue([sequence_number])
@inlineCallbacks
def unbind(self):
sequence_number = yield self.sequence_generator.next()
self.send_pdu(Unbind(sequence_number))
returnValue([sequence_number])
def handle_unbind_resp(self, pdu):
self.unbind_resp_queue.put(pdu)
class AirtelBfTransportTestCase(TransportTestCase):
transport_name = 'airtel'
transport_type = 'sms'
transport_class = AirtelBfHttpTransport
send_path = '/sendsms/index'
send_port = 9999
@inlineCallbacks
def setUp(self):
yield super(AirtelBfTransportTestCase, self).setUp()
self.airtel_sms_calls = DeferredQueue()
self.mock_airtel_sms = MockHttpServer(self.handle_request)
yield self.mock_airtel_sms.start()
self.config = {
'transport_name': self.transport_name,
'receive_path': 'sendsms',
'receive_port': 9998,
'outbound_url': self.mock_airtel_sms.url,
'default_shortcode': '3411',
'login': '******',
'password': '******'
}
self.transport = yield self.get_transport(self.config)
self.transport_url = self.transport.get_transport_url()
@inlineCallbacks
def tearDown(self):
yield self.mock_airtel_sms.stop()
yield super(AirtelBfTransportTestCase, self).tearDown()
def handle_request(self, request):
self.airtel_sms_calls.put(request)
return ''
@inlineCallbacks
def test_sending_sms(self):
yield self.dispatch(self.mkmsg_out())
req = yield self.airtel_sms_calls.get()
self.assertEqual(req.path, '/')
self.assertEqual(req.method, 'GET')
self.assertEqual(
{
'DA': ['9292'],
'SOA': ['+41791234567'],
'content': ['hello world'],
'u': ['texttochange'],
'p': ['password']
}, req.args)
[smsg] = self.get_dispatched_events()
self.assertEqual(
self.mkmsg_ack(user_message_id='1', sent_message_id='1'), smsg)
@inlineCallbacks
def test_sending_sms_complex(self):
yield self.dispatch(
self.mkmsg_out(
from_addr="+2261",
content=
u'setoffre #A#BELG=10/tete+300000/pu# envoy\xe9 depuis SIMAgriMobile'
))
req = yield self.airtel_sms_calls.get()
self.assertEqual(req.path, '/')
self.assertEqual(req.method, 'GET')
self.assertEqual(
'setoffre #A#BELG=10/tete+300000/pu# envoyé depuis SIMAgriMobile',
req.args['content'][0])
[smsg] = self.get_dispatched_events()
self.assertEqual(
self.mkmsg_ack(user_message_id='1', sent_message_id='1'), smsg)
def mkurl_raw(self, **params):
return '%s%s?%s' % (self.transport_url, self.config['receive_path'],
urlencode(params))
def mkurl(self, content, from_addr, **kw):
params = {'message': content, 'msisdn': from_addr}
params.update(kw)
return self.mkurl_raw(**params)
@inlineCallbacks
def test_receiving_sms(self):
url = self.mkurl('Hello envoy\xc3\xa9', '+2261')
response = yield http_request_full(url, method='GET')
[smsg] = self.get_dispatched_messages()
self.assertEqual(response.code, http.OK)
self.assertEqual(u'Hello envoy\xe9', smsg['content'])
self.assertEqual('3411', smsg['to_addr'])
self.assertEqual('+2261', smsg['from_addr'])
@inlineCallbacks
def test_receiving_sms_accent(self):
url = self.mkurl('Hello envoyé', '+2261')
response = yield http_request_full(url, method='GET')
[smsg] = self.get_dispatched_messages()
self.assertEqual(response.code, http.OK)
self.assertEqual(u'Hello envoy\xe9', smsg['content'])
self.assertEqual('3411', smsg['to_addr'])
self.assertEqual('+2261', smsg['from_addr'])
@inlineCallbacks
def test_receiving_sms_fail(self):
params = {
'message': 'Hello',
'to_addr': '+2261',
}
url = self.mkurl_raw(**params)
response = yield http_request_full(url, method='GET')
self.assertEqual(0, len(self.get_dispatched_messages()))
self.assertEqual(response.code, http.INTERNAL_SERVER_ERROR)
class MarathonSyncTest(TestCase):
def setUp(self):
self.marathon_sync = MarathonSync('http://1.2.3.4:8080', [])
# We use this to mock requests going to Marathon
self.requests = DeferredQueue()
def mock_requests(method, url, headers, data):
d = Deferred()
self.requests.put({
'method': method,
'url': url,
'data': data,
'deferred': d,
})
return d
self.patch(self.marathon_sync, 'requester', mock_requests)
def tearDown(self):
pass
@inlineCallbacks
def test_delete_unknown_apps(self):
self.marathon_sync.groups = [{
'id': 'test1',
'apps': [{
'id': 'app1'
}, {
'id': 'app2'
}]
}, {
'id': 'test2',
'apps': [{
'id': 'app1'
}]
}]
d = self.marathon_sync.delete_unknown_apps()
d.addErrback(log.err)
get_request = yield self.requests.get()
self.assertEqual(get_request['url'], 'http://1.2.3.4:8080/v2/apps')
self.assertEqual(get_request['method'], 'GET')
# Return 2 out of 3 of the known apps plus 1 unknown app
get_request['deferred'].callback(
FakeResponse(
200, [],
json.dumps({
'apps': [{
'id': '/test1/app1'
}, {
'id': '/test2/app1'
}, {
'id': '/app2'
}]
})))
delete_request = yield self.requests.get()
self.assertEqual(delete_request['url'],
'http://1.2.3.4:8080/v2/apps/app2')
self.assertEqual(delete_request['method'], 'DELETE')
delete_request['deferred'].callback(
FakeResponse(
200, [],
json.dumps({
"version":
"2015-08-25T10:06:19.918Z",
"deploymentId":
"7828f718-ef25-426b-9d80-82ad3b39c8ae"
})))
yield d
@inlineCallbacks
def test_sync_apps(self):
self.marathon_sync.groups = [{
'id': 'test1',
'apps': [{
'id': 'app1'
}, {
'id': 'app2'
}]
}, {
'id': 'test2',
'apps': [{
'id': 'app1'
}]
}]
d = self.marathon_sync.sync_apps()
d.addErrback(log.err)
request1 = yield self.requests.get()
self.assertEqual(request1['url'], 'http://1.2.3.4:8080/v2/groups')
self.assertEqual(request1['method'], 'PUT')
self.assertEqual(
request1['data'],
json.dumps({
'id': 'test1',
'apps': [{
'id': 'app1'
}, {
'id': 'app2'
}]
}))
request1['deferred'].callback(
FakeResponse(
200, [],
json.dumps({
"version":
"2015-08-25T08:36:47.314Z",
"deploymentId":
"54ba4426-433d-4d38-b97e-5b9e3ac9d027"
})))
request2 = yield self.requests.get()
self.assertEqual(request2['url'], 'http://1.2.3.4:8080/v2/groups')
self.assertEqual(request2['method'], 'PUT')
self.assertEqual(request2['data'],
json.dumps({
'id': 'test2',
'apps': [{
'id': 'app1'
}]
}))
request2['deferred'].callback(
FakeResponse(
200, [],
json.dumps({
"version":
"2015-08-25T09:50:54.340Z",
"deploymentId":
"54393ec8-e93e-4132-abe6-e78e4545a965"
})))
yield d
class TestMxitTransport(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.mock_http = MockHttpServer(self.handle_request)
self.mock_request_queue = DeferredQueue()
yield self.mock_http.start()
self.addCleanup(self.mock_http.stop)
config = {
'web_port': 0,
'web_path': '/api/v1/mxit/mobiportal/',
'client_id': 'client_id',
'client_secret': 'client_secret',
'api_send_url': self.mock_http.url,
'api_auth_url': self.mock_http.url,
}
self.sample_loc_str = 'cc,cn,sc,sn,cc,c,noi,cfb,ci'
self.sample_profile_str = 'lc,cc,dob,gender,tariff'
self.sample_html_str = '<&>'
self.sample_req_headers = {
'X-Device-User-Agent': 'ua',
'X-Mxit-Contact': 'contact',
'X-Mxit-USERID-R': 'user-id',
'X-Mxit-Nick': 'nick',
'X-Mxit-Location': self.sample_loc_str,
'X-Mxit-Profile': self.sample_profile_str,
'X-Mxit-User-Input': self.sample_html_str,
}
self.sample_menu_resp = "\n".join([
"Hello!",
"1. option 1",
"2. option 2",
"3. option 3",
])
# same as above but the o's are replaced with
# http://www.fileformat.info/info/unicode/char/f8/index.htm
slashed_o = '\xc3\xb8'
self.sample_unicode_menu_resp = unicode(
self.sample_menu_resp.replace('o', slashed_o), 'utf-8')
self.tx_helper = self.add_helper(TransportHelper(MxitTransport))
self.transport = yield self.tx_helper.get_transport(config)
# NOTE: priming redis with an access token
self.transport.redis.set(self.transport.access_token_key, 'foo')
self.url = self.transport.get_transport_url(config['web_path'])
def handle_request(self, request):
self.mock_request_queue.put(request)
return NOT_DONE_YET
def test_is_mxit_request(self):
req = Request(None, True)
self.assertFalse(self.transport.is_mxit_request(req))
req.requestHeaders.addRawHeader('X-Mxit-Contact', 'foo')
self.assertTrue(self.transport.is_mxit_request(req))
def test_noop(self):
self.assertEqual(self.transport.noop('foo'), 'foo')
def test_parse_location(self):
self.assertEqual(self.transport.parse_location(self.sample_loc_str), {
'country_code': 'cc',
'country_name': 'cn',
'subdivision_code': 'sc',
'subdivision_name': 'sn',
'city_code': 'cc',
'city': 'c',
'network_operator_id': 'noi',
'client_features_bitset': 'cfb',
'cell_id': 'ci',
})
def test_parse_profile(self):
self.assertEqual(
self.transport.parse_profile(self.sample_profile_str),
{
'country_code': 'cc',
'date_of_birth': 'dob',
'gender': 'gender',
'language_code': 'lc',
'tariff_plan': 'tariff',
})
def test_html_decode(self):
self.assertEqual(
self.transport.html_decode(self.sample_html_str), '<&>')
def test_get_request_data(self):
req = Request(None, True)
headers = req.requestHeaders
for key, value in self.sample_req_headers.items():
headers.addRawHeader(key, value)
data = self.transport.get_request_data(req)
self.assertEqual(data, {
'X-Device-User-Agent': 'ua',
'X-Mxit-Contact': 'contact',
'X-Mxit-Location': {
'cell_id': 'ci',
'city': 'c',
'city_code': 'cc',
'client_features_bitset': 'cfb',
'country_code': 'cc',
'country_name': 'cn',
'network_operator_id': 'noi',
'subdivision_code': 'sc',
'subdivision_name': 'sn',
},
'X-Mxit-Nick': 'nick',
'X-Mxit-Profile': {
'country_code': 'cc',
'date_of_birth': 'dob',
'gender': 'gender',
'language_code': 'lc',
'tariff_plan': 'tariff',
},
'X-Mxit-USERID-R': 'user-id',
'X-Mxit-User-Input': u'<&>',
})
def test_get_request_content_from_header(self):
req = Request(None, True)
req.requestHeaders.addRawHeader('X-Mxit-User-Input', 'foo')
self.assertEqual(self.transport.get_request_content(req), 'foo')
def test_get_quote_plus_request_content_from_header(self):
req = Request(None, True)
req.requestHeaders.addRawHeader('X-Mxit-User-Input', 'foo+bar')
self.assertEqual(
self.transport.get_request_content(req), 'foo bar')
def test_get_quoted_request_content_from_header(self):
req = Request(None, True)
req.requestHeaders.addRawHeader('X-Mxit-User-Input', 'foo%20bar')
self.assertEqual(
self.transport.get_request_content(req), 'foo bar')
def test_get_request_content_from_args(self):
req = Request(None, True)
req.args = {'input': ['bar']}
self.assertEqual(self.transport.get_request_content(req), 'bar')
def test_get_request_content_when_missing(self):
req = Request(None, True)
self.assertEqual(self.transport.get_request_content(req), None)
@inlineCallbacks
def test_invalid_request(self):
resp = yield http_request_full(self.url)
self.assertEqual(resp.code, BAD_REQUEST)
@inlineCallbacks
def test_request(self):
resp_d = http_request_full(
self.url, headers=self.sample_req_headers)
[msg] = yield self.tx_helper.wait_for_dispatched_inbound(1)
self.tx_helper.make_dispatch_reply(msg, self.sample_menu_resp)
resp = yield resp_d
self.assertTrue('1. option 1' in resp.delivered_body)
self.assertTrue('2. option 2' in resp.delivered_body)
self.assertTrue('3. option 3' in resp.delivered_body)
self.assertTrue('?input=1' in resp.delivered_body)
self.assertTrue('?input=2' in resp.delivered_body)
self.assertTrue('?input=3' in resp.delivered_body)
def test_response_parser(self):
header, items = ResponseParser.parse(self.sample_menu_resp)
self.assertEqual(header, 'Hello!')
self.assertEqual(items, [
('1', 'option 1'),
('2', 'option 2'),
('3', 'option 3'),
])
header, items = ResponseParser.parse('foo!')
self.assertEqual(header, 'foo!')
self.assertEqual(items, [])
@inlineCallbacks
def test_unicode_rendering(self):
resp_d = http_request_full(
self.url, headers=self.sample_req_headers)
[msg] = yield self.tx_helper.wait_for_dispatched_inbound(1)
self.tx_helper.make_dispatch_reply(msg, self.sample_unicode_menu_resp)
resp = yield resp_d
self.assertTrue(
'Hell\xc3\xb8' in resp.delivered_body)
self.assertTrue(
'\xc3\xb8pti\xc3\xb8n 1' in resp.delivered_body)
@inlineCallbacks
def test_outbound_that_is_not_a_reply(self):
d = self.tx_helper.make_dispatch_outbound(
content="Send!", to_addr="mxit-1", from_addr="mxit-2")
req = yield self.mock_request_queue.get()
body = json.load(req.content)
self.assertEqual(body, {
'Body': 'Send!',
'To': 'mxit-1',
'From': 'mxit-2',
'ContainsMarkup': 'true',
'Spool': 'true',
})
[auth] = req.requestHeaders.getRawHeaders('Authorization')
# primed access token
self.assertEqual(auth, 'Bearer foo')
req.finish()
yield d
@inlineCallbacks
def test_getting_access_token(self):
transport = self.transport
redis = transport.redis
# clear primed value
yield redis.delete(transport.access_token_key)
d = transport.get_access_token()
req = yield self.mock_request_queue.get()
[auth] = req.requestHeaders.getRawHeaders('Authorization')
self.assertEqual(
auth, 'Basic %s' % (
base64.b64encode('client_id:client_secret')))
self.assertEqual(
'scope=message%2Fsend&grant_type=client_credentials',
req.content.read())
req.write(json.dumps({
'access_token': 'access_token',
'expires_in': '10'
}))
req.finish()
access_token = yield d
self.assertEqual(access_token, 'access_token')
self.assertFalse(isinstance(access_token, unicode))
ttl = yield redis.ttl(transport.access_token_key)
self.assertTrue(
0 < ttl <= (transport.access_token_auto_decay * 10))
class SpoofTcpProxyProtocol(Protocol, object):
"""This is a special kind of TCP proxy where the connection
is initially routed to a spoof server, and then when
identifying information is sent from the client,
the connection is made to the right server and
messages are relayed.
This assumes that the spoof server and the real server
gives the exact same response up to the time the real
server is connected to.
"""
def __init__(self, spoof_hostname, spoof_port):
"""Create a new spoof TCP proxy.
:param str spoof_hostname: the hostname of the spoof server
:param int spoof_port: the port of the spoof server
"""
# how many bytes have been sent by the spoof server?
self.spoof_messages_length = 0
# for the actual server connection
self.server_queue = DeferredQueue()
self.client_queue = DeferredQueue()
# for the spoofed connection
self.spoof_client_queue = DeferredQueue()
self.spoof_server_queue = DeferredQueue()
self._connectServer(spoof_hostname, spoof_port,
self.spoof_server_queue, self.spoof_client_queue)
# add callbacks
self.server_queue.get().addCallback(self.serverQueueCallback)
self.spoof_server_queue.get().addCallback(
self.spoofServerQueueCallback)
def serverQueueCallback(self, data):
"""A callback for `self.server_queue`
This only starts sending data after
:code:`spoof_messages_length` has gone to 0. If the
incoming data is longer than
:code:`spoof_messages_length`, then that many bytes is
truncated from the beginning and the rest is sent.
:param str data: data from server queue
"""
if data is False:
self.transport.loseConnection()
return
assert self.spoof_messages_length >= 0
if self.spoof_messages_length == 0:
self.transport.write(data)
else:
if self.spoof_messages_length < len(data):
data = data[self.spoof_messages_length:]
self.spoof_messages_length = 0
self.transport.write(data)
else:
self.spoof_messages_length -= len(data)
self.server_queue.get().addCallback(self.serverQueueCallback)
def spoofServerQueueCallback(self, data):
"""A callback for `self.spoof_server_queue`
:param str data: data from server queue
"""
if self.spoof_server_queue is not None:
if data is False:
self.transport.loseConnection()
return
self.spoof_messages_length += len(data)
self.transport.write(data)
self.spoof_server_queue.get().addCallback(
self.spoofServerQueueCallback)
def _connectServer(self, hostname, port, server_queue, client_queue):
"""A helper function for connecting to (hostname, port)
with the given server and client queues.
:param str hostname:
:param int port:
:param DeferredQueue server_queue:
:param DeferredQueue client_queue:
"""
endpoint = TCP4ClientEndpoint(reactor, hostname, port)
protocol = ServerProtocol(server_queue, client_queue)
connectProtocol(endpoint, protocol)
def connectServer(self, hostname, port):
"""Tell the proxy what the end server is and start the connection. This closes the connection to the spoofed
server.
:param str hostname:
:param int port:
:param DeferredQueue server_queue:
:param DeferredQueue client_queue:
"""
# close connection
spoof_client_queue = self.spoof_client_queue
# setting to None first prevents a race condition
self.spoof_client_queue = None
self.spoof_server_queue = None
spoof_client_queue.put(False)
self._connectServer(hostname, port, self.server_queue,
self.client_queue)
def dataReceived(self, data):
"""Received data from client, put into client queue
"""
self.client_queue.put(data)
if self.spoof_client_queue is not None:
self.spoof_client_queue.put(data)
def connectionLost(self, why):
"""Client closed connection, or some other issue. close connection
to server
"""
# TODO pretty sure this only allows client to close connection, not the
# other way around
self.client_queue.put(False)
if self.spoof_client_queue is not None:
self.spoof_client_queue.put(False)
class Server(Protocol,QObject):
# Initialize PyQt Signals
# For some godforsaken reason, these are necessarily declared within
# the class scope but outside the __init__() function. It doesn't
# work otherwise.
chatSignal = pyqtSignal(QString)
userUpdateSignal = pyqtSignal(QString)
challengeSignal = pyqtSignal(QString)
reidentifySignal = pyqtSignal()
def __init__(self):
# Make sure Protocol.__init__() and QObject.__init__() get called (double inheritance)
super(Server, self).__init__()
# initialize vars
self.username = ''
self.opponent = ''
self.inGame = False
self.challenger = ''
# initialize queue
self.queue = DeferredQueue()
# initialize the incoming data queue
self.startQueuing()
self.quitting = False # to prevent notifying the user of disconnection when they exit
# Called by reactor upon connection
def connectionMade(self):
self.startGUI()
# Starts the PyQt GUI which will handle user input and ouptut
def startGUI(self):
thread = threading.Thread(target = run_gui, args = (self,))
thread.daemon = True # Thread will die when calling process dies
thread.start()
# Called by reactor when TCP packet received over this connection
def dataReceived(self, data):
self.queue.put(data)
# Continuously grabs data from the
def queueData(self,data):
data = data.rstrip()
self.handleData(data)
self.queue.get().addCallback(self.queueData)
# Kicks off the queueData loop
def startQueuing(self):
self.queue.get().addCallback(self.queueData)
# Handle data sent by the server
def handleData(self, msg):
# Server messages come colon-delineated:
# <element>:<element>:<element>:<element>
data = msg.split(':')
cmmd = data[0]
if (cmmd == 'opponent'):
# Server is telling you to start a game with another user.
# This will come in the form of:
# "opponent:<username>:<1|2|...|n>"
# where the last argument indicates the order of turn
self.inGame = True
self.opponent = data[1]
self.challenger = data[1]
# Update "Available" list to all users
self.transport.write("refresh:null")
# Start the game
self.initializeGame()
# tell the game who's turn it is
turn = data[2]
if turn == "1":
reactor.gs.Turn.My_Turn()
else:
reactor.gs.Turn.Opponents_Turn()
elif (cmmd == 'reject'):
# Someone has rejected your game challenge:
# reject:null
self.inGame = False
# Notify user in GUI chat box
self.chatSignal.emit("Game rejected")
elif (cmmd == 'reidentify'):
self.reidentifySignal.emit() # This signal picked up by GUI
elif (cmmd == 'challenge'):
# Challenge receieved from another user
# challenge:<fromUserID>
if self.inGame == False:
self.challenger = data[1]
# Notify user in GUI chat box
self.chatSignal.emit("SERVER MESSAGE: You have received a challenge from " + data[1])
# Prompt response from user via GUI
self.challengeSignal.emit(data[1])
elif (cmmd == 'opponentMove'):
# Opponent has made a move; pass to game:
# opponentMove:<moveID>
moveID = data[1]
reactor.gs.opponentMove(moveID)
elif (cmmd == 'forfeit'):
# Opponent has forfeited the game
# forfeit:null
# Notify the user via GUI chat box
self.chatSignal.emit("SERVER MESSAGE: " + self.challenger + " forfeited.")
self.inGame = False
# Quietly exit the game (we already know the result - game shouldn't announce it)
reactor.gs.quietQuit()
# Since the 'forfeit' command from the other user triggered the server to set
# both users (self and opponent) to available again, we need to send a global
# refresh to reflect these changes
self.transport.write("refresh:null")
elif (cmmd == 'winner'):
# Opponent has notified you that you have won your game
self.chatSignal.emit("You won against " + self.challenger + "!")
self.inGame = False
# Tell the server that both you and your opponent are now available
self.transport.write("available:" + self.username + ":" + self.challenger)
# Quietly exit the game (we already know the result - game shouldn't announce it)
reactor.gs.quietQuit()
elif (cmmd == 'loser'):
# Opponent has notified you that you have lost your game
self.chatSignal.emit("You lost against " + self.challenger + ".")
self.inGame = False
# Tell the server that both you and your opponent are now available
self.transport.write("available:" + self.username + ":" + self.challenger)
# Quietly exit the game (we already know the result - game shouldn't announce it)
reactor.gs.quietQuit()
elif (cmmd == 'tied'):
# Opponent has notified you that you have tied your game
self.chatSignal.emit("You tied " + self.challenger + ".")
self.inGame = False
# Tell the server that both you and your opponent are now available
self.transport.write("available:" + self.username + ":" + self.challenger)
# Quietly exit the game (we already know the result - game shouldn't announce it)
reactor.gs.quietQuit()
elif (cmmd == 'users' and data[2] == 'available'):
# The server is sending a list of Online and Available users:
# users:<u1+u2+u3+u4>:available:<u1+u2>
# Note: user lists are '+' delineated
userList = data[1]
availableList = data[3]
# Simply pass these lists (no parsing yet) to the GUI,
# delineated with a colon between the two
self.userUpdateSignal.emit(userList + ":" + availableList)
elif (cmmd == 'msg'):
# This is a general server message to the user:
# msg:<msg_string>
# Simply pass it along to the GUI for display in the chat box,
# after we prepend it with a special server stamp.
msg = "SERVER MESSAGE: " + data[1]
self.chatSignal.emit(msg)
elif (cmmd == 'chat'):
# Incoming chat message from another user (via the server)
# Simply pass it to the GUI via the chatSignal
msg = data[1]
self.chatSignal.emit(msg)
else:
# data was receieved but was unable to be interpreted
print "Received unrecognized message from server: " + msg
# This function will be called by the PyGame instance,
# and will notify the server of a game move like so:
# move:<opponent_username>:<move_id>
def sendMove(self,moveID):
self.transport.write("move:" + self.opponent + ":" + moveID)
# Start the pygame instance
def initializeGame(self):
# Initialize the GameSpace.
# Reactor needs to be passed to it so it can issue server calls.
reactor.gs = GameSpace(reactor)
# GameSpace also needs this protocol to issue server calls.
reactor.gs.protocol = self
# Start game loop
self.lc = LoopingCall(reactor.gs.loop)
self.lc.start(1/60)
# Called by a PyGame instance (into which we had passed a
# reference to this protocol.)
def gameEnded(self,msg):
self.inGame = False
msg = msg.rstrip() # remove any pesky trailing whitespace
# Notify server (and, by extension, the opponent) that the game has ended, and how it has
# Note: 'msg' will be either "forfeit", "won", "lost", or "tied"
self.transport.write(msg + ":" + self.challenger)
# Write end game condition to the GUI chatbox
if msg == "forfeit":
self.chatSignal.emit("You forfeited the game against " + self.challenger + ".")
elif msg == "won":
self.chatSignal.emit("You won against " + self.challenger + "!")
elif msg == "lost":
self.chatSignal.emit("You lost against " + self.challenger + ".")
elif msg == "tied":
self.chatSignal.emit("You tied " + self.challenger + ".")
# After the GUI exits, we want the program to exit as well
def guiExit(self):
self.quitting = True
self.transport.loseConnection()
reactor.stop()
return
class BrcmOpenomciOnuHandler(object):
def __init__(self, adapter, device_id):
self.log = structlog.get_logger(device_id=device_id)
self.log.debug('function-entry')
self.adapter = adapter
self.adapter_agent = adapter.adapter_agent
self.parent_adapter = None
self.parent_id = None
self.device_id = device_id
self.incoming_messages = DeferredQueue()
self.event_messages = DeferredQueue()
self.proxy_address = None
self.tx_id = 0
self._enabled = False
self.alarms = None
self.pm_metrics = None
self._omcc_version = OMCCVersion.Unknown
self._total_tcont_count = 0 # From ANI-G ME
self._qos_flexibility = 0 # From ONT2_G ME
self._onu_indication = None
self._unis = dict() # Port # -> UniPort
self._pon = None
# TODO: probably shouldnt be hardcoded, determine from olt maybe?
self._pon_port_number = 100
self.logical_device_id = None
self._heartbeat = HeartBeat.create(self, device_id)
# Set up OpenOMCI environment
self._onu_omci_device = None
self._dev_info_loaded = False
self._deferred = None
self._in_sync_subscription = None
self._connectivity_subscription = None
self._capabilities_subscription = None
self.mac_bridge_service_profile_entity_id = 0x201
self.gal_enet_profile_entity_id = 0x1
self._tp_service_specific_task = dict()
self._tech_profile_download_done = dict()
# Initialize KV store client
self.args = registry('main').get_args()
if self.args.backend == 'etcd':
host, port = self.args.etcd.split(':', 1)
self.kv_client = EtcdStore(
host, port, TechProfile.KV_STORE_TECH_PROFILE_PATH_PREFIX)
elif self.args.backend == 'consul':
host, port = self.args.consul.split(':', 1)
self.kv_client = ConsulStore(
host, port, TechProfile.KV_STORE_TECH_PROFILE_PATH_PREFIX)
else:
self.log.error('Invalid-backend')
raise Exception("Invalid-backend-for-kv-store")
# Handle received ONU event messages
reactor.callLater(0, self.handle_onu_events)
@property
def enabled(self):
return self._enabled
@enabled.setter
def enabled(self, value):
if self._enabled != value:
self._enabled = value
@property
def omci_agent(self):
return self.adapter.omci_agent
@property
def omci_cc(self):
return self._onu_omci_device.omci_cc if self._onu_omci_device is not None else None
@property
def heartbeat(self):
return self._heartbeat
@property
def uni_ports(self):
return self._unis.values()
def uni_port(self, port_no_or_name):
if isinstance(port_no_or_name, (str, unicode)):
return next(
(uni for uni in self.uni_ports if uni.name == port_no_or_name),
None)
assert isinstance(port_no_or_name, int), 'Invalid parameter type'
return next((uni for uni in self.uni_ports
if uni.logical_port_number == port_no_or_name), None)
@property
def pon_port(self):
return self._pon
def receive_message(self, msg):
if self.omci_cc is not None:
self.omci_cc.receive_message(msg)
# Called once when the adapter creates the device/onu instance
def activate(self, device):
self.log.debug('function-entry', device=device)
# first we verify that we got parent reference and proxy info
assert device.parent_id
assert device.proxy_address.device_id
# register for proxied messages right away
self.proxy_address = device.proxy_address
self.adapter_agent.register_for_proxied_messages(device.proxy_address)
self.parent_id = device.parent_id
parent_device = self.adapter_agent.get_device(self.parent_id)
if parent_device.type == 'openolt':
self.parent_adapter = registry('adapter_loader'). \
get_agent(parent_device.adapter).adapter
if self.enabled is not True:
self.log.info('activating-new-onu')
# populate what we know. rest comes later after mib sync
device.root = True
device.vendor = 'Broadcom'
device.connect_status = ConnectStatus.REACHABLE
device.oper_status = OperStatus.DISCOVERED
device.reason = 'activating-onu'
# pm_metrics requires a logical device id
parent_device = self.adapter_agent.get_device(device.parent_id)
self.logical_device_id = parent_device.parent_id
assert self.logical_device_id, 'Invalid logical device ID'
self.adapter_agent.update_device(device)
self.log.debug('set-device-discovered')
self._init_pon_state(device)
############################################################################
# Setup PM configuration for this device
# Pass in ONU specific options
kwargs = {
OnuPmMetrics.DEFAULT_FREQUENCY_KEY:
OnuPmMetrics.DEFAULT_ONU_COLLECTION_FREQUENCY,
'heartbeat': self.heartbeat,
OnuOmciPmMetrics.OMCI_DEV_KEY: self._onu_omci_device
}
self.pm_metrics = OnuPmMetrics(self.adapter_agent,
self.device_id,
self.logical_device_id,
grouped=True,
freq_override=False,
**kwargs)
pm_config = self.pm_metrics.make_proto()
self._onu_omci_device.set_pm_config(
self.pm_metrics.omci_pm.openomci_interval_pm)
self.log.info("initial-pm-config", pm_config=pm_config)
self.adapter_agent.update_device_pm_config(pm_config, init=True)
############################################################################
# Setup Alarm handler
self.alarms = AdapterAlarms(self.adapter_agent, device.id,
self.logical_device_id)
# Note, ONU ID and UNI intf set in add_uni_port method
self._onu_omci_device.alarm_synchronizer.set_alarm_params(
mgr=self.alarms, ani_ports=[self._pon])
self.enabled = True
else:
self.log.info('onu-already-activated')
# Called once when the adapter needs to re-create device. usually on vcore restart
def reconcile(self, device):
self.log.debug('function-entry', device=device)
# first we verify that we got parent reference and proxy info
assert device.parent_id
assert device.proxy_address.device_id
# register for proxied messages right away
self.proxy_address = device.proxy_address
self.adapter_agent.register_for_proxied_messages(device.proxy_address)
if self.enabled is not True:
self.log.info('reconciling-broadcom-onu-device')
self._init_pon_state(device)
# need to restart state machines on vcore restart. there is no indication to do it for us.
self._onu_omci_device.start()
device.reason = "restarting-openomci"
self.adapter_agent.update_device(device)
# TODO: this is probably a bit heavy handed
# Force a reboot for now. We need indications to reflow to reassign tconts and gems given vcore went away
# This may not be necessary when mib resync actually works
reactor.callLater(1, self.reboot)
self.enabled = True
else:
self.log.info('onu-already-activated')
@inlineCallbacks
def handle_onu_events(self):
event_msg = yield self.event_messages.get()
try:
if event_msg['event'] == 'download_tech_profile':
tp_path = event_msg['event_data']
uni_id = event_msg['uni_id']
self.load_and_configure_tech_profile(uni_id, tp_path)
except Exception as e:
self.log.error("exception-handling-onu-event", e=e)
# Handle next event
reactor.callLater(0, self.handle_onu_events)
def _init_pon_state(self, device):
self.log.debug('function-entry', device=device)
self._pon = PonPort.create(self, self._pon_port_number)
self.adapter_agent.add_port(device.id, self._pon.get_port())
self.log.debug('added-pon-port-to-agent', pon=self._pon)
parent_device = self.adapter_agent.get_device(device.parent_id)
self.logical_device_id = parent_device.parent_id
self.adapter_agent.update_device(device)
# Create and start the OpenOMCI ONU Device Entry for this ONU
self._onu_omci_device = self.omci_agent.add_device(
self.device_id,
self.adapter_agent,
support_classes=self.adapter.broadcom_omci,
custom_me_map=self.adapter.custom_me_entities())
# Port startup
if self._pon is not None:
self._pon.enabled = True
# TODO: move to UniPort
def update_logical_port(self, logical_device_id, port_id, state):
try:
self.log.info('updating-logical-port',
logical_port_id=port_id,
logical_device_id=logical_device_id,
state=state)
logical_port = self.adapter_agent.get_logical_port(
logical_device_id, port_id)
logical_port.ofp_port.state = state
self.adapter_agent.update_logical_port(logical_device_id,
logical_port)
except Exception as e:
self.log.exception("exception-updating-port", e=e)
def delete(self, device):
self.log.info('delete-onu', device=device)
if self.parent_adapter:
try:
self.parent_adapter.delete_child_device(self.parent_id, device)
except AttributeError:
self.log.debug('parent-device-delete-child-not-implemented')
else:
self.log.debug("parent-adapter-not-available")
def _create_tconts(self, uni_id, us_scheduler):
alloc_id = us_scheduler['alloc_id']
q_sched_policy = us_scheduler['q_sched_policy']
self.log.debug('create-tcont', us_scheduler=us_scheduler)
tcontdict = dict()
tcontdict['alloc-id'] = alloc_id
tcontdict['q_sched_policy'] = q_sched_policy
tcontdict['uni_id'] = uni_id
# TODO: Not sure what to do with any of this...
tddata = dict()
tddata['name'] = 'not-sure-td-profile'
tddata['fixed-bandwidth'] = "not-sure-fixed"
tddata['assured-bandwidth'] = "not-sure-assured"
tddata['maximum-bandwidth'] = "not-sure-max"
tddata['additional-bw-eligibility-indicator'] = "not-sure-additional"
td = OnuTrafficDescriptor.create(tddata)
tcont = OnuTCont.create(self, tcont=tcontdict, td=td)
self._pon.add_tcont(tcont)
self.log.debug('pon-add-tcont', tcont=tcont)
# Called when there is an olt up indication, providing the gem port id chosen by the olt handler
def _create_gemports(self, uni_id, gem_ports, alloc_id_ref, direction):
self.log.debug('create-gemport',
gem_ports=gem_ports,
direction=direction)
for gem_port in gem_ports:
gemdict = dict()
gemdict['gemport_id'] = gem_port['gemport_id']
gemdict['direction'] = direction
gemdict['alloc_id_ref'] = alloc_id_ref
gemdict['encryption'] = gem_port['aes_encryption']
gemdict['discard_config'] = dict()
gemdict['discard_config']['max_probability'] = \
gem_port['discard_config']['max_probability']
gemdict['discard_config']['max_threshold'] = \
gem_port['discard_config']['max_threshold']
gemdict['discard_config']['min_threshold'] = \
gem_port['discard_config']['min_threshold']
gemdict['discard_policy'] = gem_port['discard_policy']
gemdict['max_q_size'] = gem_port['max_q_size']
gemdict['pbit_map'] = gem_port['pbit_map']
gemdict['priority_q'] = gem_port['priority_q']
gemdict['scheduling_policy'] = gem_port['scheduling_policy']
gemdict['weight'] = gem_port['weight']
gemdict['uni_id'] = uni_id
gem_port = OnuGemPort.create(self, gem_port=gemdict)
self._pon.add_gem_port(gem_port)
self.log.debug('pon-add-gemport', gem_port=gem_port)
def _do_tech_profile_configuration(self, uni_id, tp):
num_of_tconts = tp['num_of_tconts']
us_scheduler = tp['us_scheduler']
alloc_id = us_scheduler['alloc_id']
self._create_tconts(uni_id, us_scheduler)
upstream_gem_port_attribute_list = tp[
'upstream_gem_port_attribute_list']
self._create_gemports(uni_id, upstream_gem_port_attribute_list,
alloc_id, "UPSTREAM")
downstream_gem_port_attribute_list = tp[
'downstream_gem_port_attribute_list']
self._create_gemports(uni_id, downstream_gem_port_attribute_list,
alloc_id, "DOWNSTREAM")
def load_and_configure_tech_profile(self, uni_id, tp_path):
self.log.debug("loading-tech-profile-configuration",
uni_id=uni_id,
tp_path=tp_path)
if uni_id not in self._tp_service_specific_task:
self._tp_service_specific_task[uni_id] = dict()
if uni_id not in self._tech_profile_download_done:
self._tech_profile_download_done[uni_id] = dict()
if tp_path not in self._tech_profile_download_done[uni_id]:
self._tech_profile_download_done[uni_id][tp_path] = False
if not self._tech_profile_download_done[uni_id][tp_path]:
try:
if tp_path in self._tp_service_specific_task[uni_id]:
self.log.info("tech-profile-config-already-in-progress",
tp_path=tp_path)
return
tp = self.kv_client[tp_path]
tp = ast.literal_eval(tp)
self.log.debug("tp-instance", tp=tp)
self._do_tech_profile_configuration(uni_id, tp)
def success(_results):
self.log.info("tech-profile-config-done-successfully")
device = self.adapter_agent.get_device(self.device_id)
device.reason = 'tech-profile-config-download-success'
self.adapter_agent.update_device(device)
if tp_path in self._tp_service_specific_task[uni_id]:
del self._tp_service_specific_task[uni_id][tp_path]
self._tech_profile_download_done[uni_id][tp_path] = True
def failure(_reason):
self.log.warn('tech-profile-config-failure-retrying',
_reason=_reason)
device = self.adapter_agent.get_device(self.device_id)
device.reason = 'tech-profile-config-download-failure-retrying'
self.adapter_agent.update_device(device)
if tp_path in self._tp_service_specific_task[uni_id]:
del self._tp_service_specific_task[uni_id][tp_path]
self._deferred = reactor.callLater(
_STARTUP_RETRY_WAIT,
self.load_and_configure_tech_profile, uni_id, tp_path)
self.log.info('downloading-tech-profile-configuration')
self._tp_service_specific_task[uni_id][tp_path] = \
BrcmTpServiceSpecificTask(self.omci_agent, self, uni_id)
self._deferred = \
self._onu_omci_device.task_runner.queue_task(self._tp_service_specific_task[uni_id][tp_path])
self._deferred.addCallbacks(success, failure)
except Exception as e:
self.log.exception("error-loading-tech-profile", e=e)
else:
self.log.info("tech-profile-config-already-done")
def update_pm_config(self, device, pm_config):
# TODO: This has not been tested
self.log.info('update_pm_config', pm_config=pm_config)
self.pm_metrics.update(pm_config)
# Calling this assumes the onu is active/ready and had at least an initial mib downloaded. This gets called from
# flow decomposition that ultimately comes from onos
def update_flow_table(self, device, flows):
self.log.debug('function-entry', device=device, flows=flows)
#
# We need to proxy through the OLT to get to the ONU
# Configuration from here should be using OMCI
#
# self.log.info('bulk-flow-update', device_id=device.id, flows=flows)
# no point in pushing omci flows if the device isnt reachable
if device.connect_status != ConnectStatus.REACHABLE or \
device.admin_state != AdminState.ENABLED:
self.log.warn("device-disabled-or-offline-skipping-flow-update",
admin=device.admin_state,
connect=device.connect_status)
return
def is_downstream(port):
return port == self._pon_port_number
def is_upstream(port):
return not is_downstream(port)
for flow in flows:
_type = None
_port = None
_vlan_vid = None
_udp_dst = None
_udp_src = None
_ipv4_dst = None
_ipv4_src = None
_metadata = None
_output = None
_push_tpid = None
_field = None
_set_vlan_vid = None
self.log.debug('bulk-flow-update', device_id=device.id, flow=flow)
try:
_in_port = fd.get_in_port(flow)
assert _in_port is not None
_out_port = fd.get_out_port(flow) # may be None
if is_downstream(_in_port):
self.log.debug('downstream-flow',
in_port=_in_port,
out_port=_out_port)
uni_port = self.uni_port(_out_port)
elif is_upstream(_in_port):
self.log.debug('upstream-flow',
in_port=_in_port,
out_port=_out_port)
uni_port = self.uni_port(_in_port)
else:
raise Exception('port should be 1 or 2 by our convention')
self.log.debug('flow-ports',
in_port=_in_port,
out_port=_out_port,
uni_port=str(uni_port))
for field in fd.get_ofb_fields(flow):
if field.type == fd.ETH_TYPE:
_type = field.eth_type
self.log.debug('field-type-eth-type', eth_type=_type)
elif field.type == fd.IP_PROTO:
_proto = field.ip_proto
self.log.debug('field-type-ip-proto', ip_proto=_proto)
elif field.type == fd.IN_PORT:
_port = field.port
self.log.debug('field-type-in-port', in_port=_port)
elif field.type == fd.VLAN_VID:
_vlan_vid = field.vlan_vid & 0xfff
self.log.debug('field-type-vlan-vid', vlan=_vlan_vid)
elif field.type == fd.VLAN_PCP:
_vlan_pcp = field.vlan_pcp
self.log.debug('field-type-vlan-pcp', pcp=_vlan_pcp)
elif field.type == fd.UDP_DST:
_udp_dst = field.udp_dst
self.log.debug('field-type-udp-dst', udp_dst=_udp_dst)
elif field.type == fd.UDP_SRC:
_udp_src = field.udp_src
self.log.debug('field-type-udp-src', udp_src=_udp_src)
elif field.type == fd.IPV4_DST:
_ipv4_dst = field.ipv4_dst
self.log.debug('field-type-ipv4-dst',
ipv4_dst=_ipv4_dst)
elif field.type == fd.IPV4_SRC:
_ipv4_src = field.ipv4_src
self.log.debug('field-type-ipv4-src',
ipv4_dst=_ipv4_src)
elif field.type == fd.METADATA:
_metadata = field.table_metadata
self.log.debug('field-type-metadata',
metadata=_metadata)
else:
raise NotImplementedError('field.type={}'.format(
field.type))
for action in fd.get_actions(flow):
if action.type == fd.OUTPUT:
_output = action.output.port
self.log.debug('action-type-output',
output=_output,
in_port=_in_port)
elif action.type == fd.POP_VLAN:
self.log.debug('action-type-pop-vlan',
in_port=_in_port)
elif action.type == fd.PUSH_VLAN:
_push_tpid = action.push.ethertype
self.log.debug('action-type-push-vlan',
push_tpid=_push_tpid,
in_port=_in_port)
if action.push.ethertype != 0x8100:
self.log.error('unhandled-tpid',
ethertype=action.push.ethertype)
elif action.type == fd.SET_FIELD:
_field = action.set_field.field.ofb_field
assert (action.set_field.field.oxm_class ==
OFPXMC_OPENFLOW_BASIC)
self.log.debug('action-type-set-field',
field=_field,
in_port=_in_port)
if _field.type == fd.VLAN_VID:
_set_vlan_vid = _field.vlan_vid & 0xfff
self.log.debug('set-field-type-vlan-vid',
vlan_vid=_set_vlan_vid)
else:
self.log.error('unsupported-action-set-field-type',
field_type=_field.type)
else:
self.log.error('unsupported-action-type',
action_type=action.type,
in_port=_in_port)
# TODO: We only set vlan omci flows. Handle omci matching ethertypes at some point in another task
if _type is not None:
self.log.warn('ignoring-flow-with-ethType', ethType=_type)
elif _set_vlan_vid is None or _set_vlan_vid == 0:
self.log.warn('ignorning-flow-that-does-not-set-vlanid')
else:
self.log.warn('set-vlanid',
uni_id=uni_port.port_number,
set_vlan_vid=_set_vlan_vid)
self._add_vlan_filter_task(device, uni_port, _set_vlan_vid)
except Exception as e:
self.log.exception('failed-to-install-flow', e=e, flow=flow)
def _add_vlan_filter_task(self, device, uni_port, _set_vlan_vid):
assert uni_port is not None
def success(_results):
self.log.info('vlan-tagging-success',
uni_port=uni_port,
vlan=_set_vlan_vid)
device.reason = 'omci-flows-pushed'
self._vlan_filter_task = None
def failure(_reason):
self.log.warn('vlan-tagging-failure',
uni_port=uni_port,
vlan=_set_vlan_vid)
device.reason = 'omci-flows-failed-retrying'
self._vlan_filter_task = reactor.callLater(
_STARTUP_RETRY_WAIT, self._add_vlan_filter_task, device,
uni_port, _set_vlan_vid)
self.log.info('setting-vlan-tag')
self._vlan_filter_task = BrcmVlanFilterTask(self.omci_agent,
self.device_id, uni_port,
_set_vlan_vid)
self._deferred = self._onu_omci_device.task_runner.queue_task(
self._vlan_filter_task)
self._deferred.addCallbacks(success, failure)
def get_tx_id(self):
self.log.debug('function-entry')
self.tx_id += 1
return self.tx_id
# TODO: Actually conform to or create a proper interface.
# this and the other functions called from the olt arent very clear.
# Called each time there is an onu "up" indication from the olt handler
def create_interface(self, data):
self.log.debug('function-entry', data=data)
self._onu_indication = data
onu_device = self.adapter_agent.get_device(self.device_id)
self.log.debug('starting-openomci-statemachine')
self._subscribe_to_events()
reactor.callLater(1, self._onu_omci_device.start)
onu_device.reason = "starting-openomci"
self.adapter_agent.update_device(onu_device)
self._heartbeat.enabled = True
# Currently called each time there is an onu "down" indication from the olt handler
# TODO: possibly other reasons to "update" from the olt?
def update_interface(self, data):
self.log.debug('function-entry', data=data)
oper_state = data.get('oper_state', None)
onu_device = self.adapter_agent.get_device(self.device_id)
if oper_state == 'down':
self.log.debug('stopping-openomci-statemachine')
reactor.callLater(0, self._onu_omci_device.stop)
# Let TP download happen again
for uni_id in self._tp_service_specific_task:
self._tp_service_specific_task[uni_id].clear()
for uni_id in self._tech_profile_download_done:
self._tech_profile_download_done[uni_id].clear()
self.disable_ports(onu_device)
onu_device.reason = "stopping-openomci"
onu_device.connect_status = ConnectStatus.UNREACHABLE
onu_device.oper_status = OperStatus.DISCOVERED
self.adapter_agent.update_device(onu_device)
else:
self.log.debug('not-changing-openomci-statemachine')
# Not currently called by olt or anything else
def remove_interface(self, data):
self.log.debug('function-entry', data=data)
onu_device = self.adapter_agent.get_device(self.device_id)
self.log.debug('stopping-openomci-statemachine')
reactor.callLater(0, self._onu_omci_device.stop)
# Let TP download happen again
for uni_id in self._tp_service_specific_task:
self._tp_service_specific_task[uni_id].clear()
for uni_id in self._tech_profile_download_done:
self._tech_profile_download_done[uni_id].clear()
self.disable_ports(onu_device)
onu_device.reason = "stopping-openomci"
self.adapter_agent.update_device(onu_device)
# TODO: im sure there is more to do here
# Not currently called. Would be called presumably from the olt handler
def remove_gemport(self, data):
self.log.debug('remove-gemport', data=data)
gem_port = GemportsConfigData()
gem_port.CopyFrom(data)
device = self.adapter_agent.get_device(self.device_id)
if device.connect_status != ConnectStatus.REACHABLE:
self.log.error('device-unreachable')
return
# Not currently called. Would be called presumably from the olt handler
def remove_tcont(self, tcont_data, traffic_descriptor_data):
self.log.debug('remove-tcont',
tcont_data=tcont_data,
traffic_descriptor_data=traffic_descriptor_data)
device = self.adapter_agent.get_device(self.device_id)
if device.connect_status != ConnectStatus.REACHABLE:
self.log.error('device-unreachable')
return
# TODO: Create some omci task that encompases this what intended
# Not currently called. Would be called presumably from the olt handler
def create_multicast_gemport(self, data):
self.log.debug('function-entry', data=data)
# TODO: create objects and populate for later omci calls
def disable(self, device):
self.log.debug('function-entry', device=device)
try:
self.log.info('sending-uni-lock-towards-device', device=device)
def stop_anyway(reason):
# proceed with disable regardless if we could reach the onu. for example onu is unplugged
self.log.debug('stopping-openomci-statemachine')
reactor.callLater(0, self._onu_omci_device.stop)
# Let TP download happen again
for uni_id in self._tp_service_specific_task:
self._tp_service_specific_task[uni_id].clear()
for uni_id in self._tech_profile_download_done:
self._tech_profile_download_done[uni_id].clear()
self.disable_ports(device)
device.oper_status = OperStatus.UNKNOWN
device.reason = "omci-admin-lock"
self.adapter_agent.update_device(device)
# lock all the unis
task = BrcmUniLockTask(self.omci_agent, self.device_id, lock=True)
self._deferred = self._onu_omci_device.task_runner.queue_task(task)
self._deferred.addCallbacks(stop_anyway, stop_anyway)
except Exception as e:
log.exception('exception-in-onu-disable', exception=e)
def reenable(self, device):
self.log.debug('function-entry', device=device)
try:
# Start up OpenOMCI state machines for this device
# this will ultimately resync mib and unlock unis on successful redownloading the mib
self.log.debug('restarting-openomci-statemachine')
self._subscribe_to_events()
device.reason = "restarting-openomci"
self.adapter_agent.update_device(device)
reactor.callLater(1, self._onu_omci_device.start)
self._heartbeat.enabled = True
except Exception as e:
log.exception('exception-in-onu-reenable', exception=e)
def reboot(self):
self.log.info('reboot-device')
device = self.adapter_agent.get_device(self.device_id)
if device.connect_status != ConnectStatus.REACHABLE:
self.log.error("device-unreachable")
return
def success(_results):
self.log.info('reboot-success', _results=_results)
self.disable_ports(device)
device.connect_status = ConnectStatus.UNREACHABLE
device.oper_status = OperStatus.DISCOVERED
device.reason = "rebooting"
self.adapter_agent.update_device(device)
def failure(_reason):
self.log.info('reboot-failure', _reason=_reason)
self._deferred = self._onu_omci_device.reboot()
self._deferred.addCallbacks(success, failure)
def disable_ports(self, onu_device):
self.log.info('disable-ports',
device_id=self.device_id,
onu_device=onu_device)
# Disable all ports on that device
self.adapter_agent.disable_all_ports(self.device_id)
parent_device = self.adapter_agent.get_device(onu_device.parent_id)
assert parent_device
logical_device_id = parent_device.parent_id
assert logical_device_id
ports = self.adapter_agent.get_ports(onu_device.id, Port.ETHERNET_UNI)
for port in ports:
port_id = 'uni-{}'.format(port.port_no)
# TODO: move to UniPort
self.update_logical_port(logical_device_id, port_id,
OFPPS_LINK_DOWN)
def enable_ports(self, onu_device):
self.log.info('enable-ports',
device_id=self.device_id,
onu_device=onu_device)
# Disable all ports on that device
self.adapter_agent.enable_all_ports(self.device_id)
parent_device = self.adapter_agent.get_device(onu_device.parent_id)
assert parent_device
logical_device_id = parent_device.parent_id
assert logical_device_id
ports = self.adapter_agent.get_ports(onu_device.id, Port.ETHERNET_UNI)
for port in ports:
port_id = 'uni-{}'.format(port.port_no)
# TODO: move to UniPort
self.update_logical_port(logical_device_id, port_id, OFPPS_LIVE)
# Called just before openomci state machine is started. These listen for events from selected state machines,
# most importantly, mib in sync. Which ultimately leads to downloading the mib
def _subscribe_to_events(self):
self.log.debug('function-entry')
# OMCI MIB Database sync status
bus = self._onu_omci_device.event_bus
topic = OnuDeviceEntry.event_bus_topic(
self.device_id, OnuDeviceEvents.MibDatabaseSyncEvent)
self._in_sync_subscription = bus.subscribe(topic, self.in_sync_handler)
# OMCI Capabilities
bus = self._onu_omci_device.event_bus
topic = OnuDeviceEntry.event_bus_topic(
self.device_id, OnuDeviceEvents.OmciCapabilitiesEvent)
self._capabilities_subscription = bus.subscribe(
topic, self.capabilties_handler)
# Called when the mib is in sync
def in_sync_handler(self, _topic, msg):
self.log.debug('function-entry', _topic=_topic, msg=msg)
if self._in_sync_subscription is not None:
try:
in_sync = msg[IN_SYNC_KEY]
if in_sync:
# Only call this once
bus = self._onu_omci_device.event_bus
bus.unsubscribe(self._in_sync_subscription)
self._in_sync_subscription = None
# Start up device_info load
self.log.debug('running-mib-sync')
reactor.callLater(0, self._mib_in_sync)
except Exception as e:
self.log.exception('in-sync', e=e)
def capabilties_handler(self, _topic, _msg):
self.log.debug('function-entry', _topic=_topic, msg=_msg)
if self._capabilities_subscription is not None:
self.log.debug('capabilities-handler-done')
# Mib is in sync, we can now query what we learned and actually start pushing ME (download) to the ONU.
# Currently uses a basic mib download task that create a bridge with a single gem port and uni, only allowing EAP
# Implement your own MibDownloadTask if you wish to setup something different by default
def _mib_in_sync(self):
self.log.debug('function-entry')
omci = self._onu_omci_device
in_sync = omci.mib_db_in_sync
device = self.adapter_agent.get_device(self.device_id)
device.reason = 'discovery-mibsync-complete'
self.adapter_agent.update_device(device)
if not self._dev_info_loaded:
self.log.info('loading-device-data-from-mib',
in_sync=in_sync,
already_loaded=self._dev_info_loaded)
omci_dev = self._onu_omci_device
config = omci_dev.configuration
# TODO: run this sooner somehow. shouldnt have to wait for mib sync to push an initial download
# In Sync, we can register logical ports now. Ideally this could occur on
# the first time we received a successful (no timeout) OMCI Rx response.
try:
# sort the lists so we get consistent port ordering.
ani_list = sorted(
config.ani_g_entities) if config.ani_g_entities else []
uni_list = sorted(
config.uni_g_entities) if config.uni_g_entities else []
pptp_list = sorted(
config.pptp_entities) if config.pptp_entities else []
veip_list = sorted(
config.veip_entities) if config.veip_entities else []
if ani_list is None or (pptp_list is None
and veip_list is None):
device.reason = 'onu-missing-required-elements'
self.log.warn("no-ani-or-unis")
self.adapter_agent.update_device(device)
raise Exception("onu-missing-required-elements")
# Currently logging the ani, pptp, veip, and uni for information purposes.
# Actually act on the veip/pptp as its ME is the most correct one to use in later tasks.
# And in some ONU the UNI-G list is incomplete or incorrect...
for entity_id in ani_list:
ani_value = config.ani_g_entities[entity_id]
self.log.debug("discovered-ani",
entity_id=entity_id,
value=ani_value)
# TODO: currently only one OLT PON port/ANI, so this works out. With NGPON there will be 2..?
self._total_tcont_count = ani_value.get(
'total-tcont-count')
self.log.debug("set-total-tcont-count",
tcont_count=self._total_tcont_count)
for entity_id in uni_list:
uni_value = config.uni_g_entities[entity_id]
self.log.debug("discovered-uni",
entity_id=entity_id,
value=uni_value)
uni_entities = OrderedDict()
for entity_id in pptp_list:
pptp_value = config.pptp_entities[entity_id]
self.log.debug("discovered-pptp",
entity_id=entity_id,
value=pptp_value)
uni_entities[entity_id] = UniType.PPTP
for entity_id in veip_list:
veip_value = config.veip_entities[entity_id]
self.log.debug("discovered-veip",
entity_id=entity_id,
value=veip_value)
uni_entities[entity_id] = UniType.VEIP
uni_id = 0
for entity_id, uni_type in uni_entities.iteritems():
try:
self._add_uni_port(entity_id, uni_id, uni_type)
uni_id += 1
except AssertionError as e:
self.log.warn("could not add UNI",
entity_id=entity_id,
uni_type=uni_type,
e=e)
multi_uni = len(self._unis) > 1
for uni_port in self._unis.itervalues():
uni_port.add_logical_port(uni_port.port_number, multi_uni)
self.adapter_agent.update_device(device)
self._qos_flexibility = config.qos_configuration_flexibility or 0
self._omcc_version = config.omcc_version or OMCCVersion.Unknown
if self._unis:
self._dev_info_loaded = True
else:
device.reason = 'no-usable-unis'
self.adapter_agent.update_device(device)
self.log.warn("no-usable-unis")
raise Exception("no-usable-unis")
except Exception as e:
self.log.exception('device-info-load', e=e)
self._deferred = reactor.callLater(_STARTUP_RETRY_WAIT,
self._mib_in_sync)
else:
self.log.info('device-info-already-loaded',
in_sync=in_sync,
already_loaded=self._dev_info_loaded)
if self._dev_info_loaded:
if device.admin_state == AdminState.ENABLED:
def success(_results):
self.log.info('mib-download-success', _results=_results)
device = self.adapter_agent.get_device(self.device_id)
device.reason = 'initial-mib-downloaded'
device.oper_status = OperStatus.ACTIVE
device.connect_status = ConnectStatus.REACHABLE
self.enable_ports(device)
self.adapter_agent.update_device(device)
self._mib_download_task = None
def failure(_reason):
self.log.warn('mib-download-failure-retrying',
_reason=_reason)
device.reason = 'initial-mib-download-failure-retrying'
self.adapter_agent.update_device(device)
self._deferred = reactor.callLater(_STARTUP_RETRY_WAIT,
self._mib_in_sync)
# Download an initial mib that creates simple bridge that can pass EAP. On success (above) finally set
# the device to active/reachable. This then opens up the handler to openflow pushes from outside
self.log.info('downloading-initial-mib-configuration')
self._mib_download_task = BrcmMibDownloadTask(
self.omci_agent, self)
self._deferred = self._onu_omci_device.task_runner.queue_task(
self._mib_download_task)
self._deferred.addCallbacks(success, failure)
else:
self.log.info('admin-down-disabling')
self.disable(device)
else:
self.log.info('device-info-not-loaded-skipping-mib-download')
def _add_uni_port(self, entity_id, uni_id, uni_type=UniType.PPTP):
self.log.debug('function-entry')
device = self.adapter_agent.get_device(self.device_id)
parent_device = self.adapter_agent.get_device(device.parent_id)
parent_adapter_agent = registry('adapter_loader').get_agent(
parent_device.adapter)
if parent_adapter_agent is None:
self.log.error('parent-adapter-could-not-be-retrieved')
# TODO: This knowledge is locked away in openolt. and it assumes one onu equals one uni...
parent_device = self.adapter_agent.get_device(device.parent_id)
parent_adapter = parent_adapter_agent.adapter.devices[parent_device.id]
uni_no = parent_adapter.platform.mk_uni_port_num(
self._onu_indication.intf_id, self._onu_indication.onu_id, uni_id)
# TODO: Some or parts of this likely need to move to UniPort. especially the format stuff
uni_name = "uni-{}".format(uni_no)
mac_bridge_port_num = uni_id + 1 # TODO +1 is only to test non-zero index
self.log.debug('uni-port-inputs',
uni_no=uni_no,
uni_id=uni_id,
uni_name=uni_name,
uni_type=uni_type,
entity_id=entity_id,
mac_bridge_port_num=mac_bridge_port_num)
uni_port = UniPort.create(self, uni_name, uni_id, uni_no, uni_name,
uni_type)
uni_port.entity_id = entity_id
uni_port.enabled = True
uni_port.mac_bridge_port_num = mac_bridge_port_num
self.log.debug("created-uni-port", uni=uni_port)
self.adapter_agent.add_port(device.id, uni_port.get_port())
parent_adapter_agent.add_port(device.parent_id, uni_port.get_port())
self._unis[uni_port.port_number] = uni_port
self._onu_omci_device.alarm_synchronizer.set_alarm_params(
onu_id=self._onu_indication.onu_id, uni_ports=self._unis.values())
# TODO: this should be in the PonPortclass
pon_port = self._pon.get_port()
# Delete reference to my own UNI as peer from parent.
# TODO why is this here, add_port_reference_to_parent already prunes duplicates
me_as_peer = Port.PeerPort(device_id=device.parent_id,
port_no=uni_port.port_number)
partial_pon_port = Port(
port_no=pon_port.port_no,
label=pon_port.label,
type=pon_port.type,
admin_state=pon_port.admin_state,
oper_status=pon_port.oper_status,
peers=[me_as_peer]
) # only list myself as a peer to avoid deleting all other UNIs from parent
self.adapter_agent.delete_port_reference_from_parent(
self.device_id, partial_pon_port)
pon_port.peers.extend([me_as_peer])
self._pon._port = pon_port
self.adapter_agent.add_port_reference_to_parent(
self.device_id, pon_port)
class ServerProtocol(Protocol, object):
"""The client protocol that talks to the end server in a TCP proxy.
"""
def __init__(self, server_queue, client_queue):
"""Create a new protocol.
:code:`server_queue` and :code:`client_queue` corresponds to the variables
in the TCP proxy.
:code:`self.wait_queue` is used to handle the race condition where
:code:`self.client_queue` is ready to be consumed, but the connection has
not been established.
:param DeferredQueue server_queue:
:param DeferredQueue client_queue:
"""
self.server_queue = server_queue
self.client_queue = client_queue
self.wait_queue = DeferredQueue()
self.client_queue.get().addCallback(self.clientQueueCallback)
def clientQueueCallback(self, data):
"""A callback for the client queue.
If the data is the literal False, then close the connection.
Otherwise, add this data to our wait queue.
:param data: the data from the client queue
"""
if data is False:
self.transport.loseConnection()
else:
self.wait_queue.put(data)
self.client_queue.get().addCallback(self.clientQueueCallback)
def emptyWaitQueue(self):
"""Starts emptying the wait queue. Note that a connection
must already be made (self.transport must be ready for use)
"""
def _emptyWaitQueueHelper(data):
self.transport.write(data)
self.wait_queue.get().addCallback(_emptyWaitQueueHelper)
assert self.transport is not None
self.wait_queue.get().addCallback(_emptyWaitQueueHelper)
def connectionMade(self):
"""Connection to target server is established. Empty the wait queue.
"""
self.emptyWaitQueue()
def dataReceived(self, data):
"""Received data from target server, put into server queue
:param str data:
"""
self.server_queue.put(data)
def connectionLost(self, why):
"""Server closed connection, or some other issue. close connection
to server
"""
self.server_queue.put(False)
class OMCI_CC(object):
""" Handle OMCI Communication Channel specifics for Adtran ONUs"""
_frame_to_event_type = {
OmciMibResetResponse.message_id: RxEvent.MIB_Reset,
OmciMibUploadResponse.message_id: RxEvent.MIB_Upload,
OmciMibUploadNextResponse.message_id: RxEvent.MIB_Upload_Next,
OmciCreateResponse.message_id: RxEvent.Create,
OmciDeleteResponse.message_id: RxEvent.Delete,
OmciSetResponse.message_id: RxEvent.Set,
OmciGetAllAlarmsResponse.message_id: RxEvent.Get_ALARM_Get,
OmciGetAllAlarmsNextResponse.message_id: RxEvent.Get_ALARM_Get_Next
}
def __init__(self, adapter_agent, device_id, me_map=None,
alarm_queue_limit=_MAX_INCOMING_ALARM_MESSAGES,
avc_queue_limit=_MAX_INCOMING_ALARM_MESSAGES,
test_results_queue_limit=_MAX_INCOMING_TEST_RESULT_MESSAGES):
self.log = structlog.get_logger(device_id=device_id)
self._adapter_agent = adapter_agent
self._device_id = device_id
self._proxy_address = None
self._tx_tid = 1
self._enabled = False
self._requests = dict() # Tx ID -> (timestamp, deferred, tx_frame, timeout)
self._alarm_queue = DeferredQueue(size=alarm_queue_limit)
self._avc_queue = DeferredQueue(size=avc_queue_limit)
self._test_results_queue = DeferredQueue(size=test_results_queue_limit)
self._me_map = me_map
# Statistics
self._tx_frames = 0
self._rx_frames = 0
self._rx_unknown_tid = 0 # Rx OMCI with no Tx TID match
self._rx_onu_frames = 0 # Autonomously generated ONU frames
self._rx_alarm_overflow = 0 # Autonomously generated ONU alarms rx overflow
self._rx_avc_overflow = 0 # Autonomously generated ONU AVC rx overflow
self._rx_onu_discards = 0 # Autonomously generated ONU unknown message types
self._rx_timeouts = 0
self._rx_unknown_me = 0 # Number of managed entities Rx without a decode definition
self._tx_errors = 0 # Exceptions during tx request
self._consecutive_errors = 0 # Rx & Tx errors in a row, a good RX resets this to 0
self._reply_min = sys.maxint # Fastest successful tx -> rx
self._reply_max = 0 # Longest successful tx -> rx
self._reply_sum = 0.0 # Total seconds for successful tx->rx (float for average)
self.event_bus = EventBusClient()
# If a list of custom ME Entities classes were provided, insert them into
# main class_id to entity map.
# TODO: If this class becomes hidden from the ONU DA, move this to the OMCI State Machine runner
def __str__(self):
return "OMCISupport: {}".format(self._device_id)
@staticmethod
def event_bus_topic(device_id, event):
"""
Get the topic name for a given event Frame Type
:param device_id: (str) ONU Device ID
:param event: (OmciCCRxEvents) Type of event
:return: (str) Topic string
"""
assert event in OmciCCRxEvents, \
'Event {} is not an OMCI-CC Rx Event'.format(event.name)
return 'omci-rx:{}:{}'.format(device_id, event.name)
@property
def enabled(self):
return self._enabled
@enabled.setter
def enabled(self, value):
"""
Enable/disable the OMCI Communications Channel
:param value: (boolean) True to enable, False to disable
"""
assert isinstance(value, bool), 'enabled is a boolean'
if self._enabled != value:
self._enabled = value
if self._enabled:
self._start()
else:
self._stop()
@property
def tx_frames(self):
return self._tx_frames
@property
def rx_frames(self):
return self._rx_frames
@property
def rx_unknown_tid(self):
return self._rx_unknown_tid # Tx TID not found
@property
def rx_unknown_me(self):
return self._rx_unknown_me
@property
def rx_onu_frames(self):
return self._rx_onu_frames
@property
def rx_alarm_overflow(self):
return self._rx_alarm_overflow # Alarm ONU autonomous overflows
@property
def rx_avc_overflow(self):
return self._rx_avc_overflow # Attribute Value change autonomous overflows
@property
def rx_onu_discards(self):
return self._rx_onu_discards # Attribute Value change autonomous overflows
@property
def rx_timeouts(self):
return self._rx_timeouts
@property
def tx_errors(self):
return self._tx_errors
@property
def consecutive_errors(self):
return self._consecutive_errors
@property
def reply_min(self):
return int(round(self._reply_min * 1000.0)) # Milliseconds
@property
def reply_max(self):
return int(round(self._reply_max * 1000.0)) # Milliseconds
@property
def reply_average(self):
avg = self._reply_sum / self._rx_frames if self._rx_frames > 0 else 0.0
return int(round(avg * 1000.0)) # Milliseconds
@property
def get_alarm_message(self):
"""
Attempt to retrieve and remove an ONU Alarm Message from the ONU
autonomous message queue.
TODO: We may want to deprecate this, see TODO comment around line 399 in
the _request_success() method below
:return: a Deferred which fires with the next Alarm Frame available in
the queue.
"""
return self._alarm_queue.get()
@property
def get_avc_message(self):
"""
Attempt to retrieve and remove an ONU Attribute Value Change (AVC)
Message from the ONU autonomous message queue.
TODO: We may want to deprecate this, see TODO comment around line 399 in
the _request_success() method below
:return: a Deferred which fires with the next AVC Frame available in
the queue.
"""
return self._avc_queue.get()
@property
def get_test_results(self):
"""
Attempt to retrieve and remove an ONU Test Results Message from the
ONU autonomous message queue.
TODO: We may want to deprecate this, see TODO comment around line 399 in
the _request_success() method below
:return: a Deferred which fires with the next Test Results Frame is
available in the queue.
"""
return self._test_results_queue.get()
def _start(self):
"""
Start the OMCI Communications Channel
"""
assert self._enabled, 'Start should only be called if enabled'
self.flush()
device = self._adapter_agent.get_device(self._device_id)
self._proxy_address = device.proxy_address
def _stop(self):
"""
Stop the OMCI Communications Channel
"""
assert not self._enabled, 'Stop should only be called if disabled'
self.flush()
self._proxy_address = None
# TODO: What is best way to clean up any outstanding futures for these queues
self._alarm_queue = None
self._avc_queue = None
self._test_results_queue = None
def _receive_onu_message(self, rx_frame):
""" Autonomously generated ONU frame Rx handler"""
from twisted.internet.defer import QueueOverflow
self.log.debug('rx-onu-frame', frame_type=type(rx_frame),
frame=hexify(str(rx_frame)))
# TODO: Signal, via defer if Alarm Overflow or just an event?
msg_type = rx_frame.fields['message_type']
self._rx_onu_frames += 1
msg = {TX_REQUEST_KEY: None,
RX_RESPONSE_KEY: rx_frame}
if msg_type == EntityOperations.AlarmNotification.value:
topic = OMCI_CC.event_bus_topic(self._device_id, RxEvent.Alarm_Notification)
reactor.callLater(0, self.event_bus.publish, topic, msg)
try:
self._alarm_queue.put((rx_frame, arrow.utcnow().float_timestamp))
except QueueOverflow:
self._rx_alarm_overflow += 1
self.log.warn('onu-rx-alarm-overflow', cnt=self._rx_alarm_overflow)
elif msg_type == EntityOperations.AttributeValueChange.value:
topic = OMCI_CC.event_bus_topic(self._device_id, RxEvent.AVC_Notification)
reactor.callLater(0, self.event_bus.publish, topic, msg)
try:
self._alarm_queue.put((rx_frame, arrow.utcnow().float_timestamp))
except QueueOverflow:
self._rx_avc_overflow += 1
self.log.warn('onu-rx-avc-overflow', cnt=self._rx_avc_overflow)
elif msg_type == EntityOperations.TestResult.value:
topic = OMCI_CC.event_bus_topic(self._device_id, RxEvent.Test_Result)
reactor.callLater(0, self.event_bus.publish, topic, msg)
try:
self._test_results_queue.put((rx_frame, arrow.utcnow().float_timestamp))
except QueueOverflow:
self.log.warn('onu-rx-test-results-overflow')
else:
# TODO: Need to add test results message support
self.log.warn('onu-unsupported-autonomous-message', type=msg_type)
self._rx_onu_discards += 1
def receive_message(self, msg):
"""
Receive and OMCI message from the proxy channel to the OLT.
Call this from your ONU Adapter on a new OMCI Rx on the proxy channel
"""
if self.enabled:
try:
now = arrow.utcnow()
d = None
# NOTE: Since we may need to do an independent ME map on a per-ONU basis
# save the current value of the entity_id_to_class_map, then
# replace it with our custom one before decode, and then finally
# restore it later. Tried other ways but really made the code messy.
saved_me_map = omci_entities.entity_id_to_class_map
omci_entities.entity_id_to_class_map = self._me_map
try:
rx_frame = OmciFrame(msg)
rx_tid = rx_frame.fields['transaction_id']
if rx_tid == 0:
return self._receive_onu_message(rx_frame)
# Previously unreachable if this is the very first Rx or we
# have been running consecutive errors
if self._rx_frames == 0 or self._consecutive_errors != 0:
reactor.callLater(0, self._publish_connectivity_event, True)
self._rx_frames += 1
self._consecutive_errors = 0
except KeyError as e:
# Unknown, Unsupported, or vendor-specific ME. Key is the unknown classID
self.log.debug('frame-decode-key-error', msg=hexlify(msg), e=e)
rx_frame = self._decode_unknown_me(msg)
self._rx_unknown_me += 1
rx_tid = rx_frame.fields.get('transaction_id')
except Exception as e:
self.log.exception('frame-decode', msg=hexlify(msg), e=e)
return
finally:
omci_entities.entity_id_to_class_map = saved_me_map # Always restore it.
try:
(ts, d, tx_frame, _) = self._requests.pop(rx_tid)
ts_diff = now - arrow.Arrow.utcfromtimestamp(ts)
secs = ts_diff.total_seconds()
self._reply_sum += secs
if secs < self._reply_min:
self._reply_min = secs
if secs > self._reply_max:
self._reply_max = secs
except KeyError as e:
# Possible late Rx on a message that timed-out
self._rx_unknown_tid += 1
self.log.warn('tx-message-missing', rx_id=rx_tid, msg=hexlify(msg))
return
except Exception as e:
self.log.exception('frame-match', msg=hexlify(msg), e=e)
if d is not None:
return d.errback(failure.Failure(e))
return
# Notify sender of completed request
reactor.callLater(0, d.callback, rx_frame)
# Publish Rx event to listeners in a different task
reactor.callLater(0, self._publish_rx_frame, tx_frame, rx_frame)
except Exception as e:
self.log.exception('rx-msg', e=e)
def _decode_unknown_me(self, msg):
"""
Decode an ME for an unsupported class ID. This should only occur for a subset
of message types (Get, Set, MIB Upload Next, ...) and they should only be
responses as well.
There are some times below that are commented out. For VOLTHA 2.0, it is
expected that any get, set, create, delete for unique (often vendor) MEs
will be coded by the ONU utilizing it and supplied to OpenOMCI as a
vendor-specific ME during device initialization.
:param msg: (str) Binary data
:return: (OmciFrame) resulting frame
"""
from struct import unpack
(tid, msg_type, framing) = unpack('!HBB', msg[0:4])
assert framing == 0xa, 'Only basic OMCI framing supported at this time'
msg = msg[4:]
# TODO: Commented out items below are future work (not expected for VOLTHA v2.0)
(msg_class, kwargs) = {
# OmciCreateResponse.message_id: (OmciCreateResponse, None),
# OmciDeleteResponse.message_id: (OmciDeleteResponse, None),
# OmciSetResponse.message_id: (OmciSetResponse, None),
# OmciGetResponse.message_id: (OmciGetResponse, None),
# OmciGetAllAlarmsNextResponse.message_id: (OmciGetAllAlarmsNextResponse, None),
OmciMibUploadNextResponse.message_id: (OmciMibUploadNextResponse,
{
'entity_class': unpack('!H', msg[0:2])[0],
'entity_id': unpack('!H', msg[2:4])[0],
'object_entity_class': unpack('!H', msg[4:6])[0],
'object_entity_id': unpack('!H', msg[6:8])[0],
'object_attributes_mask': unpack('!H', msg[8:10])[0],
'object_data': {
UNKNOWN_CLASS_ATTRIBUTE_KEY: hexlify(msg[10:-4])
},
}),
# OmciAlarmNotification.message_id: (OmciAlarmNotification, None),
# OmciAttributeValueChange.message_id: (OmciAttributeValueChange,
# {
# 'entity_class': unpack('!H', msg[0:2])[0],
# 'entity_id': unpack('!H', msg[2:4])[0],
# 'data': {
# UNKNOWN_CLASS_ATTRIBUTE_KEY: hexlify(msg[4:-8])
# },
# }),
# OmciTestResult.message_id: (OmciTestResult, None),
}.get(msg_type, None)
if msg_class is None:
raise TypeError('Unsupport Message Type for Unknown Decode: {}',
msg_type)
return OmciFrame(transaction_id=tid, message_type=msg_type,
omci_message=msg_class(**kwargs))
def _publish_rx_frame(self, tx_frame, rx_frame):
"""
Notify listeners of successful response frame
:param tx_frame: (OmciFrame) Original request frame
:param rx_frame: (OmciFrame) Response frame
"""
if self._enabled and isinstance(rx_frame, OmciFrame):
frame_type = rx_frame.fields['omci_message'].message_id
event_type = OMCI_CC._frame_to_event_type.get(frame_type)
if event_type is not None:
topic = OMCI_CC.event_bus_topic(self._device_id, event_type)
msg = {TX_REQUEST_KEY: tx_frame,
RX_RESPONSE_KEY: rx_frame}
self.event_bus.publish(topic=topic, msg=msg)
def _publish_connectivity_event(self, connected):
"""
Notify listeners of Rx/Tx connectivity over OMCI
:param connected: (bool) True if connectivity transitioned from unreachable
to reachable
"""
if self._enabled:
topic = OMCI_CC.event_bus_topic(self._device_id,
RxEvent.Connectivity)
msg = {CONNECTED_KEY: connected}
self.event_bus.publish(topic=topic, msg=msg)
def flush(self, max_age=0):
limit = arrow.utcnow().float_timestamp - max_age
old = [tid for tid, (ts, _, _, _) in self._requests.iteritems()
if ts <= limit]
for tid in old:
(_, d, _, _) = self._requests.pop(tid)
if d is not None and not d.called:
d.cancel()
self._requests = dict()
if max_age == 0:
# Flush autonomous messages (Alarms & AVCs)
while self._alarm_queue.pending:
_ = yield self._alarm_queue.get()
while self._avc_queue.pending:
_ = yield self._avc_queue.get()
def _get_tx_tid(self):
"""
Get the next Transaction ID for a tx. Note TID=0 is reserved
for autonomously generated messages from an ONU
:return: (int) TID
"""
tx_tid, self._tx_tid = self._tx_tid, self._tx_tid + 1
if self._tx_tid > MAX_OMCI_TX_ID:
self._tx_tid = 1
return tx_tid
def _request_failure(self, value, tx_tid):
"""
Handle a transmit failure and/or Rx timeout
:param value: (Failure) Twisted failure
:param tx_tid: (int) Associated Tx TID
"""
if tx_tid in self._requests:
(_, _, _, timeout) = self._requests.pop(tx_tid)
else:
timeout = 0
if isinstance(value, failure.Failure):
value.trap(CancelledError)
self._rx_timeouts += 1
self._consecutive_errors += 1
if self._consecutive_errors == 1:
reactor.callLater(0, self._publish_connectivity_event, False)
self.log.info('timeout', tx_id=tx_tid, timeout=timeout)
value = failure.Failure(TimeoutError(timeout, "Deferred"))
return value
def _request_success(self, rx_frame):
"""
Handle transmit success (a matching Rx was received)
:param rx_frame: (OmciFrame) OMCI response frame with matching TID
:return: (OmciFrame) OMCI response frame with matching TID
"""
# At this point, no additional processing is required
# Continue with Rx Success callbacks.
return rx_frame
def send(self, frame, timeout=DEFAULT_OMCI_TIMEOUT):
"""
Send the OMCI Frame to the ONU via the proxy_channel
:param frame: (OMCIFrame) Message to send
:param timeout: (int) Rx Timeout. 0=Forever
:return: (deferred) A deferred that fires when the response frame is received
or if an error/timeout occurs
"""
self.flush(max_age=MAX_OMCI_REQUEST_AGE)
assert timeout <= MAX_OMCI_REQUEST_AGE, \
'Maximum timeout is {} seconds'.format(MAX_OMCI_REQUEST_AGE)
assert isinstance(frame, OmciFrame), \
"Invalid frame class '{}'".format(type(frame))
if not self.enabled or self._proxy_address is None:
# TODO custom exceptions throughout this code would be helpful
return fail(result=failure.Failure(Exception('OMCI is not enabled')))
try:
tx_tid = frame.fields['transaction_id']
if tx_tid is None:
tx_tid = self._get_tx_tid()
frame.fields['transaction_id'] = tx_tid
assert tx_tid not in self._requests, 'TX TID {} is already exists'.format(tx_tid)
assert tx_tid >= 0, 'Invalid Tx TID: {}'.format(tx_tid)
ts = arrow.utcnow().float_timestamp
d = defer.Deferred()
# NOTE: Since we may need to do an independent ME map on a per-ONU basis
# save the current value of the entity_id_to_class_map, then
# replace it with our custom one before decode, and then finally
# restore it later. Tried other ways but really made the code messy.
saved_me_map = omci_entities.entity_id_to_class_map
omci_entities.entity_id_to_class_map = self._me_map
try:
self._adapter_agent.send_proxied_message(self._proxy_address,
hexify(str(frame)))
finally:
omci_entities.entity_id_to_class_map = saved_me_map
self._tx_frames += 1
self._requests[tx_tid] = (ts, d, frame, timeout)
d.addCallbacks(self._request_success, self._request_failure,
errbackArgs=(tx_tid,))
if timeout > 0:
d.addTimeout(timeout, reactor)
except Exception as e:
self._tx_errors += 1
self._consecutive_errors += 1
if self._consecutive_errors == 1:
reactor.callLater(0, self._publish_connectivity_event, False)
self.log.exception('send-omci', e=e)
return fail(result=failure.Failure(e))
return d
###################################################################################
# MIB Action shortcuts
def send_mib_reset(self, timeout=DEFAULT_OMCI_TIMEOUT):
"""
Perform a MIB Reset
"""
self.log.debug('send-mib-reset')
frame = OntDataFrame().mib_reset()
return self.send(frame, timeout)
def send_mib_upload(self, timeout=DEFAULT_OMCI_TIMEOUT):
self.log.debug('send-mib-upload')
frame = OntDataFrame().mib_upload()
return self.send(frame, timeout)
def send_mib_upload_next(self, seq_no, timeout=DEFAULT_OMCI_TIMEOUT):
self.log.debug('send-mib-upload-next')
frame = OntDataFrame(sequence_number=seq_no).mib_upload_next()
return self.send(frame, timeout)
def send_reboot(self, timeout=DEFAULT_OMCI_TIMEOUT):
"""
Send an ONU Device reboot request (ONU-G ME).
NOTICE: This method is being deprecated and replaced with a tasks to preform this function
"""
self.log.debug('send-mib-reboot')
frame = OntGFrame().reboot()
return self.send(frame, timeout)
def send_get_all_alarm(self, alarm_retrieval_mode=0, timeout=DEFAULT_OMCI_TIMEOUT):
self.log.debug('send_get_alarm')
frame = OntDataFrame().get_all_alarm(alarm_retrieval_mode)
return self.send(frame, timeout)
def send_get_all_alarm_next(self, seq_no, timeout=DEFAULT_OMCI_TIMEOUT):
self.log.debug('send_get_alarm_next')
frame = OntDataFrame().get_all_alarm_next(seq_no)
return self.send(frame, timeout)
class PostgresProtocol(StatefulProtocol):
def getInitialState(self):
return (self.getHeader, 5)
def getHandler(self, tag):
handlers = {
'R': self.handle_Authentication,
'S': self.handle_ParameterStatus,
'K': self.handle_BackendKeyData,
'Z': self.handle_ReadyForQuery,
'E': self.handle_ErrorResponse,
'T': self.handle_RowDescription,
'D': self.handle_DataRow,
'C': self.handle_CommandComplete
}
if tag in handlers:
return handlers[tag]
else:
print 'UNKNOWN TAG %r' % (tag, )
return self.handle_Unknown
def getHeader(self, header):
tag, length = struct.unpack('!cI', header)
handler = self.getHandler(tag)
return handler, length - 4
def connectionMade(self):
self.parameters = {}
self.cancellationKey = None
self.backendPID = None
self.transactionStatus = None
self._dfds = deque()
self._queries = DeferredQueue()
self._currentResults = []
self._currentRowDescription = []
self.send_StartupMessage()
def changeStatus(self, newStatus):
self.transactionStatus = newStatus
if newStatus == 'I':
query = self._queries.get()
query.addCallback(self.send_Query)
def runQuery(self, query):
d = Deferred()
self._dfds.append(d)
self._queries.put(query)
return d
def send(self, tag, payload):
if tag is None:
tag = ''
length = len(payload) + 4
length_packed = struct.pack('!I', length)
data = ''.join([tag, length_packed, payload])
self.transport.write(data)
def send_StartupMessage(self):
payload = {'user': '******', 'database': 'asdf'}
data = struct.pack('!hh', 3, 0)
for k, v in payload.items():
data += '%s\x00%s\x00' % (k, v)
data += '\x00'
self.send(None, data)
def send_Query(self, query):
if not query.endswith('\x00'):
query = query + '\x00'
self.send('Q', query)
@handler
def handle_Authentication(self, data):
if len(data) == 8:
print 'md5 auth required'
elif len(data) == 4:
print 'r4'
@handler
def handle_Unknown(self, data):
print 'unknown handler, data: %r' % (data, )
@handler
def handle_ParameterStatus(self, data):
param, value, empty = data.split('\x00')
self.parameters[param] = value
@handler
def handle_BackendKeyData(self, data):
self.backendPID, self.cancellationKey = struct.unpack('!II', data)
@handler
def handle_ReadyForQuery(self, data):
self.changeStatus(data)
@handler
def handle_ErrorResponse(self, data):
fields = data.split('\x00')
for field in fields:
if not field:
continue
fieldType = field[0]
fieldValue = field[1:]
print 'SERVER ERROR! %s: %s' % (fieldType, fieldValue)
@handler
def handle_RowDescription(self, data):
self._currentRowDescription = lib.parse_rowDescription(data, len(data))
@handler
def handle_DataRow(self, data):
numCols = struct.unpack('!h', data[:2])[0]
data = data[2:]
row = []
lib.x(numCols, ffi.new('char[]', data))
while data:
fieldLen = struct.unpack('!I', data[:4])[0]
fieldData = data[4:4 + fieldLen]
data = data[4 + fieldLen:]
row.append(fieldData)
self._currentResults.append(row)
@handler
def handle_CommandComplete(self, data):
dfd = self._dfds.popleft()
dfd.callback(self._currentResults)
self._currentRowDescription = []
self._currentResults = []
