class Game(Protocol): def __init__(self, users): self.users = users self.queue = DeferredQueue() def connectionMade(self): global state connections['GAME'] = self state = "GO" self.flag = True def dataReceived(self, data): self.queue.put(data) self.queue.get().addCallback(self.ForwardData) def ForwardData(self, data): positions = pickle.loads(data) if is_client == "host": gs.player2.rect = positions["p2_rect"] gs.player2.image = pygame.transform.rotate(gs.player2.orig_image, positions["p2_angle"]) if positions["firing"] == True: mx = positions["mouse_x"] my = positions["mouse_y"] self.laser = P2_Laser(gs.player2.rect.centerx, gs.player2.rect.centery,mx,my,gs) self.laser.rect.x = gs.player2.rect.centerx self.laser.rect.y = gs.player2.rect.centery # Add the laser to the lists gs.laser_list2.add(self.laser) gs.all_sprites_list.add(self.laser) else: gs.player1.rect = positions["p1_rect"] gs.player1.image = pygame.transform.rotate(gs.player1.orig_image, positions["p1_angle"]) if positions["firing"] == True: mx = positions["mouse_x"] my = positions["mouse_y"] self.laser = P1_Laser(gs.player1.rect.centerx, gs.player1.rect.centery,mx,my,gs) self.laser.rect.x = gs.player1.rect.centerx self.laser.rect.y = gs.player1.rect.centery # Add the laser to the lists gs.laser_list1.add(self.laser) gs.all_sprites_list.add(self.laser) if positions["change"] == True: gs.player2.rect.x = positions["newx"] gs.player2.rect.y = positions["newy"] if self.queue.waiting > 0: self.queue.get().addCallback(self.ForwardData) def connectionLost(self,reason): print "Connection lost - goodbye!" reactor.stop() def clientConnectionLost(self, reason): print "Connection lost - goodbye!" reactor.stop()
class TestStreamingClient(VumiTestCase): def setUp(self): self.fake_http = FakeHttpServer(self.handle_request) self.request_queue = DeferredQueue() self.client = StreamingClient(self.fake_http.get_agent) self.messages_received = DeferredQueue() self.errors_received = DeferredQueue() self.disconnects_received = DeferredQueue() def reason_trapper(reason): if reason.trap(ResponseDone): self.disconnects_received.put(reason.getErrorMessage()) self.receiver = self.client.stream( Message, self.messages_received.put, self.errors_received.put, "http://vumi-go-api.example.com/", on_disconnect=reason_trapper) def handle_request(self, request): self.request_queue.put(request) return NOT_DONE_YET def test_default_agent_factory(self): """ If `None` is passed as the `agent_factory`, `Agent` is used instead. """ self.assertNotIsInstance(self.client.agent, Agent) self.assertIsInstance(StreamingClient(None).agent, Agent) self.assertIsInstance(StreamingClient().agent, Agent) @inlineCallbacks def test_callback_on_disconnect(self): req = yield self.request_queue.get() req.write( '%s\n' % (Message(foo='bar').to_json().encode('utf-8'),)) req.finish() message = yield self.messages_received.get() self.assertEqual(message['foo'], 'bar') reason = yield self.disconnects_received.get() # this is the error message we get when a ResponseDone is raised # which happens when the remote server closes the connection. self.assertEqual(reason, 'Response body fully received') @inlineCallbacks def test_invalid_json(self): req = yield self.request_queue.get() req.write("Hello\n") req.finish() err = yield self.assertFailure( self.errors_received.get(), VumiBridgeInvalidJsonError) self.assertEqual(err.args, ("Hello",))
class TcpProxyProtocol(Protocol, object): """A simple TCP proxy""" def __init__(self): """Create a new TCP proxy. `self.server_queue` contains messages from end server to client. `self.client_queue` contains messages from client to end server. """ self.server_queue = DeferredQueue() self.client_queue = DeferredQueue() self.server_queue.get().addCallback(self.serverQueueCallback) def connectServer(self, hostname, port): """Tell the proxy what the end server is and start the connection. The messages in `self.client_queue` will automatically be consumed. This method should only be called once. :param str hostname: :param int port: """ endpoint = TCP4ClientEndpoint(reactor, hostname, port) protocol = ServerProtocol( self.server_queue, self.client_queue) connectProtocol(endpoint, protocol) def serverQueueCallback(self, data): """A callback for `self.server_queue` :param str data: data from server queue """ if data is False: self.transport.loseConnection() return self.transport.write(data) self.server_queue.get().addCallback(self.serverQueueCallback) def dataReceived(self, data): """Received data from client, put into client queue """ self.client_queue.put(data) def connectionLost(self, why): """Client closed connection, or some other issue. close connection to server """ self.client_queue.put(False)
class TestStreamingClient(VumiTestCase): @inlineCallbacks def setUp(self): self.mock_server = MockHttpServer(self.handle_request) self.add_cleanup(self.mock_server.stop) yield self.mock_server.start() self.url = self.mock_server.url self.client = StreamingClient() self.messages_received = DeferredQueue() self.errors_received = DeferredQueue() self.disconnects_received = DeferredQueue() def reason_trapper(reason): if reason.trap(ResponseDone): self.disconnects_received.put(reason.getErrorMessage()) self.receiver = self.client.stream( Message, self.messages_received.put, self.errors_received.put, self.url, on_disconnect=reason_trapper) def handle_request(self, request): self.mock_server.queue.put(request) return NOT_DONE_YET @inlineCallbacks def test_callback_on_disconnect(self): req = yield self.mock_server.queue.get() req.write( '%s\n' % (Message(foo='bar').to_json().encode('utf-8'),)) req.finish() message = yield self.messages_received.get() self.assertEqual(message['foo'], 'bar') reason = yield self.disconnects_received.get() # this is the error message we get when a ResponseDone is raised # which happens when the remote server closes the connection. self.assertEqual(reason, 'Response body fully received') @inlineCallbacks def test_invalid_json(self): req = yield self.mock_server.queue.get() req.write("Hello\n") req.finish() try: yield self.errors_received.get() except VumiBridgeInvalidJsonError, e: self.assertEqual(e.args, ("Hello",)) else:
class home_DataConn(protocol.Protocol): def __init__(self): print 'home listening for data connection from work' self.queue = DeferredQueue() def connectionMade(self): home_connections['data_conn'] = self self.queue.get().addCallback(self.forwardData) def dataReceived(self, data): self.queue.put(data) def forwardData(self, data): home_connections['client_conn'].transport.write(data) self.queue.get().addCallback(self.forwardData)
class work_ServiceConn(protocol.Protocol): def __init__(self): print 'service connection established' self.queue = DeferredQueue() def connectionMade(self): work_connections['service_conn'] = self self.queue.get().addCallback(self.forwardData) def dataReceived(self, data): self.queue.put(data) def forwardData(self, data): work_connections['data_conn'].transport.write(data) self.queue.get().addCallback(self.forwardData)
class QueuePoller(object): implements(IPoller) def __init__(self, settings): self.q = RedisSpiderQueue(settings) self.dq = DeferredQueue(size=1) @inlineCallbacks def poll(self): if self.dq.pending: return c = yield maybeDeferred(self.q.count) if c: msg = yield maybeDeferred(self.q.pop) returnValue(self.dq.put(self._message(msg))) def next(self): return self.dq.get() def _message(self, queue_msg): d = queue_msg.copy() d['_project'] = SCRAPY_PROJECT d['_spider'] = d.pop('name') d['_job'] = d.pop('jobid', uuid.uuid1().hex) return d
class QueuePoller(object): def __init__(self, config): self.config = config self.update_projects() self.dq = DeferredQueue() @inlineCallbacks def poll(self): if not self.dq.waiting: return for p, q in iteritems(self.queues): c = yield maybeDeferred(q.count) if c: msg = yield maybeDeferred(q.pop) if msg is not None: # In case of a concurrently accessed queue returnValue(self.dq.put(self._message(msg, p))) def next(self): return self.dq.get() def update_projects(self): self.queues = get_spider_queues(self.config) def _message(self, queue_msg, project): d = queue_msg.copy() d['_project'] = project d['_spider'] = d.pop('name') return d
def test_messages_stream(self): url = '%s/%s/messages.json' % (self.url, self.conversation.key) messages = DeferredQueue() errors = DeferredQueue() receiver = self.client.stream( TransportUserMessage, messages.put, errors.put, url, Headers(self.auth_headers)) msg1 = yield self.app_helper.make_dispatch_inbound( 'in 1', message_id='1', conv=self.conversation) msg2 = yield self.app_helper.make_dispatch_inbound( 'in 2', message_id='2', conv=self.conversation) rm1 = yield messages.get() rm2 = yield messages.get() receiver.disconnect() # Sometimes messages arrive out of order if we're hitting real redis. rm1, rm2 = sorted([rm1, rm2], key=lambda m: m['message_id']) self.assertEqual(msg1['message_id'], rm1['message_id']) self.assertEqual(msg2['message_id'], rm2['message_id']) self.assertEqual(errors.size, None)
def test_health_response(self): health_url = 'http://%s:%s%s' % ( self.addr.host, self.addr.port, self.config['health_path']) response = yield http_request_full(health_url, method='GET') self.assertEqual(response.delivered_body, '0') yield self.app_helper.make_dispatch_inbound( 'in 1', message_id='1', conv=self.conversation) queue = DeferredQueue() stream_url = '%s/%s/messages.json' % (self.url, self.conversation.key) stream_receiver = self.client.stream( TransportUserMessage, queue.put, queue.put, stream_url, Headers(self.auth_headers)) yield queue.get() response = yield http_request_full(health_url, method='GET') self.assertEqual(response.delivered_body, '1') stream_receiver.disconnect() response = yield http_request_full(health_url, method='GET') self.assertEqual(response.delivered_body, '0') self.assertEqual(self.app.client_manager.clients, { 'sphex.stream.message.%s' % (self.conversation.key,): [] })
class QueuePoller(object): implements(IPoller) def __init__(self, config): self.config = config self.update_projects() self.dq = DeferredQueue(size=1) @inlineCallbacks def poll(self): if self.dq.pending: return for p, q in self.queues.iteritems(): c = yield maybeDeferred(q.count) if c: msg = yield maybeDeferred(q.pop) returnValue(self.dq.put(self._message(msg, p))) def next(self): return self.dq.get() def update_projects(self): self.queues = get_spider_queues(self.config) def _message(self, queue_msg, project): d = queue_msg.copy() d['_project'] = project d['_spider'] = d.pop('name') return d
class ProcessPool(object): def __init__(self, count=10): self.limiter = DeferredSemaphore(count) self.processes = [spawnProcess() for _ in xrange(count)] self.workQueue = DeferredQueue() for process in self.processes: process.onconnect.addCallback(self._prepareForWork) @inlineCallbacks def _prepareForWork(self, proto): deferred, func, args = yield self.workQueue.get() proto.queueWork(deferred, func, *args) def requeue(result): self._prepareForWork(proto) return result deferred.addCallback(requeue) returnValue(proto) def queueWork(self, function, *args): resultDeferred = Deferred() innerDeferred = Deferred() self.workQueue.put((innerDeferred, function, args)) def callResult(obj): resultDeferred.callback(obj) return obj innerDeferred.addCallback(callResult) return resultDeferred def stop(self): for process in self.processes: process.protocol.kill() process.transport.loseConnection()
class SSMIServerProtocol(Protocol): delimiter = TruteqTransportProtocol.delimiter def __init__(self): self.receive_queue = DeferredQueue() self._buf = b"" def dataReceived(self, data): self._buf += data self.parse_commands() def parse_commands(self): while self.delimiter in self._buf: line, _, self._buf = self._buf.partition(self.delimiter) if line: self.receive_queue.put(SSMIRequest.parse(line)) def send(self, command): self.transport.write(str(command)) self.transport.write(self.delimiter) return wait0() def receive(self): return self.receive_queue.get() def disconnect(self): self.transport.loseConnection()
def test_events_stream(self): url = '%s/%s/events.json' % (self.url, self.conversation.key) events = DeferredQueue() errors = DeferredQueue() receiver = yield self.client.stream(TransportEvent, events.put, events.put, url, Headers(self.auth_headers)) msg1 = yield self.app_helper.make_stored_outbound( self.conversation, 'out 1', message_id='1') ack1 = yield self.app_helper.make_dispatch_ack( msg1, conv=self.conversation) msg2 = yield self.app_helper.make_stored_outbound( self.conversation, 'out 2', message_id='2') ack2 = yield self.app_helper.make_dispatch_ack( msg2, conv=self.conversation) ra1 = yield events.get() ra2 = yield events.get() receiver.disconnect() self.assertEqual(ack1['event_id'], ra1['event_id']) self.assertEqual(ack2['event_id'], ra2['event_id']) self.assertEqual(errors.size, None)
def async_receive_stream(self, func, *args, **kw): queue = DeferredQueue() def _execute(): for result in func(*args, **kw): reactor.callFromThread(queue.put, result) _ = threads.deferToThread(_execute) while 1: yield queue.get()
class home_ClientConn(protocol.Protocol): def __init__(self): print 'client connection established' self.queue = DeferredQueue() def connectionMade(self): home_connections['client_conn'] = self home_connections['command_conn'].transport.write('begin data connect') reactor.listenTCP(80061, DataConnFactory()) self.queue.get().addCallback(self.forwardData) def dataReceived(self, data): self.queue.put(data) def forwardData(self, data): home_connections['data_conn'].transport.write(data) self.queue.get().addCallback(self.forwardData)
class GameConn(protocol.Protocol): #simply reiterate what they said. def connectionMade(self): self.transport.write("Beginning Player Connection...") self.queue = DeferredQueue() def dataReceived(self, data): #"As soon as any data is received, print it." print data self.queue.put(data) self.queue.get().addCallback(self.send_to_server) def send_to_server(self, data): self.transport.write(data) def connectionLost(self, reason): pass
def get(self, timeout=None): deferred = DeferredQueue.get(self) call_id = None if timeout: call_id = self.clock.callLater(timeout, self._timeout, deferred) deferred.addCallback(self._raiseIfClosed, call_id) return deferred
class DataConn(Protocol): def __init__(self): self.queue = DeferredQueue() def connectionMade(self): print "connection made on DataConn" pass def dataReceived(self, data): self.queue.put(data) def sendData(self, data): connectionsDict["Client"].transport.write(data) self.queue.get().addCallback(self.sendData) def startForwarding(self): self.queue.get().addCallback(self.sendData)
class GameConn(protocol.Protocol): #respond to data received def connectionMade(self): global num_players num_players+=1 playerConnections[str(num_players)] = self self.queue = DeferredQueue() self.transport.write("Connected to server as player " + str(num_players)) print self.queue def connectionLost(self, reason): pass def dataReceived(self, data): print data self.queue.put(data) self.queue.get().addCallback(self.send_to_client) def send_to_client(self, data): pass
class QueuePoller(object): implements(IPoller) def __init__(self, config, app): self.config = config self.update_projects() self.dq = DeferredQueue(size=1) self.max_jobs_per_project = self.config.getint('max_jobs_per_project', 4) @inlineCallbacks def poll(self, launcher): if self.dq.pending: return for p, q in self.queues.iteritems(): c = yield maybeDeferred(q.count) if c and self._has_slot_for_project(p, launcher): msg = yield maybeDeferred(q.pop) returnValue(self.dq.put(self._message(msg, p))) def _has_slot_for_project(self, project_name, launcher): running_jobs = 0 spiders = launcher.processes.values() for s in spiders: if s.project == project_name: running_jobs += 1 return running_jobs < self.max_jobs_per_project def next(self): return self.dq.get() def update_projects(self): self.queues = get_spider_queues(self.config) def _message(self, queue_msg, project): d = queue_msg.copy() d['_project'] = project d['_spider'] = d.pop('name') return d @property def launcher(self): """ Copied from website.Root Should do some refactory to avoid this duplicated code """ app = IServiceCollection(self.app, self.app) return app.getServiceNamed('launcher')
def test__handles_missing_system_handler_on_notification(self): # Captured notifications from the database will go here. notices = DeferredQueue() class PostgresListenerServiceSpy(PostgresListenerService): """Send notices off to `notices` right after processing.""" def doRead(self): try: self.connection.connection.poll() except Exception: self.loseConnection(Failure(error.ConnectionLost())) else: # Copy the pending notices now but don't put them in the # queue until after the real doRead has processed them. notifies = list(self.connection.connection.notifies) try: return super().doRead() finally: for notice in notifies: notices.put(notice) listener = PostgresListenerServiceSpy() # Change notifications to a frozenset. This makes sure that # the system message does not go into the queue. Instead if should # call the handler directly in `doRead`. listener.notifications = frozenset() yield listener.startService() # Use a randomised channel name even though LISTEN/NOTIFY is # per-database and _not_ per-cluster. channel = factory.make_name("sys_test", sep="_").lower() self.assertTrue(listener.isSystemChannel(channel)) payload = factory.make_name("payload") yield deferToDatabase(listener.registerChannel, channel) listener.listeners[channel] = [] try: # Notify our channel with a payload and wait for it to come back. yield deferToDatabase(self.send_notification, channel, payload) while True: notice = yield notices.get() if notice.channel == channel: self.assertThat(notice.payload, Equals(payload)) # Our channel has been deleted from the listeners map. self.assertFalse(channel in listener.listeners) break finally: yield listener.stopService()
class MockXmlOverTcpServer(MockServer): def __init__(self): self.responses = {} self.received_queue = DeferredQueue() def wait_for_data(self): return self.received_queue.get() def send_data(self, data): self.transport.write(data) def dataReceived(self, data): response = self.responses.get(data) if response is not None: self.transport.write(response) self.received_queue.put(data)
def test_deferred_queue_receiver(self): ebc = EventBus() queue = DeferredQueue() ebc.subscribe('', lambda _, msg: queue.put(msg)) for i in range(10): ebc.publish('', i) self.assertEqual(len(queue.pending), 10) for i in range(10): msg = yield queue.get() self.assertEqual(msg, i) self.assertEqual(len(queue.pending), 0)
class MockXmlOverTcpServer(MockServer): def __init__(self): self.responses = {} self.received_queue = DeferredQueue() def wait_for_data(self): return self.received_queue.get() def send_data(self, data): self.transport.write(data) def dataReceived(self, data): response = self.responses.get(data) if response is not None: self.transport.write(response) self.received_queue.put(data)
def test_deferred_queue_receiver(self): ebc = EventBus() queue = DeferredQueue() ebc.subscribe('', lambda _, msg: queue.put(msg)) for i in xrange(10): ebc.publish('', i) self.assertEqual(len(queue.pending), 10) for i in xrange(10): msg = yield queue.get() self.assertEqual(msg, i) self.assertEqual(len(queue.pending), 0)
def test_backlog_on_connect(self): for i in range(10): yield self.app_helper.make_dispatch_inbound( 'in %s' % (i,), message_id=str(i), conv=self.conversation) queue = DeferredQueue() url = '%s/%s/messages.json' % (self.url, self.conversation.key) receiver = self.client.stream( TransportUserMessage, queue.put, queue.put, url, Headers(self.auth_headers)) for i in range(10): received = yield queue.get() self.assertEqual(received['message_id'], str(i)) receiver.disconnect()
class ConsumerQueue(object): def __init__(self, stop_on_error=False, empty=None): self.stop_on_error = stop_on_error self.empty = empty self.queue = DeferredQueue() self.size = 0 self.running = True self._deferred = Deferred() def _consume_next(self, *args): if not self.running: return self._deferred = self.queue.get() self._deferred.addCallbacks(self._consumer, self._error) def _consumer(self, item): self.size -= 1 r = self.consume(item) if self.size == 0 and self.empty is not None: self.empty() if isinstance(r, Deferred): r.addCallbacks(self._consume_next, self._consume_next) else: self._consume_next() def _error(self, fail): self.error(fail) if not self.stop_on_error: self._consume_next() def add(self, item): self.size += 1 self.queue.put(item) def consume(self, item): raise NotImplementedError def error(self, fail): raise NotImplementedError def start(self): self.running = True self._consume_next() def stop(self): self.running = False self._deferred.cancel()
def test_broker_restart(self): """ restart the kafka broker and verify that the group rejoins """ record_stream = DeferredQueue(backlog=1) def processor(consumer, records): log.debug('processor(%r, %r)', consumer, records) record_stream.put(records) coord = ConsumerGroup( self.client, self.id(), topics=[self.topic], processor=processor, retry_backoff_ms=100, heartbeat_interval_ms=1000, fatal_backoff_ms=2000, ) join_de = self.when_called(coord, 'on_join_complete') coord.start() self.addCleanup(coord.stop) yield join_de self.assertIn(self.topic, coord.consumers) self.assertEqual(len(coord.consumers[self.topic]), self.num_partitions) self.assertEqual(coord.consumers[self.topic][0].topic, self.topic) self.assertEqual(coord.consumers[self.topic][0].partition, 0) # restart the broker and see that we re-join and still work leave_de = self.when_called(coord, 'on_group_leave') prepare_de = self.when_called(coord, 'on_join_prepare') join_de = self.when_called(coord, 'on_join_complete') self.harness.brokers[0].stop() yield leave_de self.assertEqual(len(coord.consumers), 0) self.harness.brokers[0].restart() yield prepare_de yield join_de self.assertIn(self.topic, coord.consumers) self.assertEqual(len(coord.consumers[self.topic]), self.num_partitions) for part in range(self.num_partitions): values = yield self.send_messages(part, [part]) msgs = yield record_stream.get() self.assertEqual(msgs[0].partition, part) self.assertEqual(msgs[0].message.value, values[0])
def test_backlog_on_connect(self): for i in range(10): yield self.app_helper.make_dispatch_inbound('in %s' % (i, ), message_id=str(i), conv=self.conversation) queue = DeferredQueue() url = '%s/%s/messages.json' % (self.url, self.conversation.key) receiver = self.client.stream(TransportUserMessage, queue.put, queue.put, url, Headers(self.auth_headers)) for i in range(10): received = yield queue.get() self.assertEqual(received['message_id'], str(i)) receiver.disconnect()
class ToyXmlOverTcpClient(XmlOverTcpClient): _PACKET_RECEIVED_HANDLERS = {'DummyPacket': 'dummy_packet_received'} def __init__(self): XmlOverTcpClient.__init__(self, 'root', 'toor', '1029384756') self.PACKET_RECEIVED_HANDLERS.update(self._PACKET_RECEIVED_HANDLERS) self.received_dummy_packets = [] self.received_data_request_packets = [] self.disconnected = False self.session_id_counter = count() self.generated_session_ids = [] self.request_id_counter = count() self.generated_request_ids = [] self.received_queue = DeferredQueue() def wait_for_data(self): return self.received_queue.get() def connectionMade(self): pass def dataReceived(self, data): XmlOverTcpClient.dataReceived(self, data) self.received_queue.put(data) def dummy_packet_received(self, session_id, params): self.received_dummy_packets.append((session_id, params)) def data_request_received(self, session_id, params): self.received_data_request_packets.append((session_id, params)) def disconnect(self): self.disconnected = True @classmethod def session_id_from_nr(cls, nr): return cls.serialize_header_field(nr, cls.SESSION_ID_HEADER_SIZE) def gen_session_id(self): return self.session_id_from_nr(next(self.session_id_counter)) def gen_request_id(self): return str(next(self.request_id_counter))
class Pipeline(Deferred): """Run a generator pipelined over a set of inputs.""" def __init__(self, pipe, width=5): Deferred.__init__(self) self.pipe = pipe self.width = width self.waiting = None self.running = 0 self.results = DeferredQueue() def run(self, inputs, *a, **kw): self._produce(inputs, *a, **kw) self._consume().chainDeferred(self) @inlineCallbacks def _produce(self, inputs, *a, **kw): prev = None for i, e in enumerate(inputs): if self.running >= self.width: # wait to add more pipes self.waiting = Deferred() yield self.waiting self.running += 1 p = self.pipe(e, *a, **kw) pr = PipeRunner(p, prev=prev) pr.addBoth(self.results.put) if i == 0: pr.run() prev = pr pr.addBoth(lambda _: self.results.put(DONE)) @inlineCallbacks def _consume(self): results = [] while 1: r = yield self.results.get() if r == DONE: break self.running -= 1 results.append(r) if self.waiting: d = self.waiting self.waiting = None d.callback(None) returnValue(results)
class Pipeline(Deferred): """Run a generator pipelined over a set of inputs.""" def __init__(self, pipe, width=5): Deferred.__init__(self) self.pipe = pipe self.width = width self.waiting = None self.running = 0 self.results = DeferredQueue() def run(self, inputs, *a, **kw): self._produce(inputs, *a, **kw) self._consume().chainDeferred(self) @inlineCallbacks def _produce(self, inputs, *a, **kw): prev = None for i, e in enumerate(inputs): if self.running >= self.width: # wait to add more pipes self.waiting = Deferred() yield self.waiting self.running += 1 p = self.pipe(e, *a, **kw) pr = PipeRunner(p, prev=prev) pr.addBoth(self.results.put) if i == 0: pr.run() prev = pr pr.addBoth(lambda _: self.results.put(DONE)) @inlineCallbacks def _consume(self): results = [] while 1: r = yield self.results.get() if r == DONE: break self.running -= 1 results.append(r) if self.waiting: d = self.waiting self.waiting = None d.callback(None) returnValue(results)
class TestGoConversationTransportBase(VumiTestCase): transport_class = None def setUp(self): self.tx_helper = self.add_helper(TransportHelper(self.transport_class)) self.fake_http = FakeHttpServer(self.handle_inbound_request) self.clock = Clock() self._request_queue = DeferredQueue() self._pending_reqs = [] self.add_cleanup(self.finish_requests) @inlineCallbacks def get_transport(self, **config): defaults = { 'account_key': 'account-key', 'conversation_key': 'conversation-key', 'access_token': 'access-token', } defaults.update(config) transport = yield self.tx_helper.get_transport(defaults, start=False) transport.agent_factory = self.fake_http.get_agent yield transport.startWorker() yield self.setup_transport(transport) returnValue(transport) def setup_transport(self, transport): pass @inlineCallbacks def finish_requests(self): for req in self._pending_reqs: if not req.finished: yield req.finish() def handle_inbound_request(self, request): self._request_queue.put(request) return NOT_DONE_YET @inlineCallbacks def get_next_request(self): req = yield self._request_queue.get() self._pending_reqs.append(req) returnValue(req)
def pull_message(self, count=1): url = '%s/%s/messages.json' % (self.url, self.conversation.key) messages = DeferredQueue() errors = DeferredQueue() receiver = self.client.stream(TransportUserMessage, messages.put, errors.put, url, Headers(self.auth_headers)) received_messages = [] for msg_id in range(count): yield self.app_helper.make_dispatch_inbound('in %s' % (msg_id, ), message_id=str(msg_id), conv=self.conversation) recv_msg = yield messages.get() received_messages.append(recv_msg) receiver.disconnect() returnValue((receiver, received_messages))
class StubbyIrcServer(ServerFactory): protocol = StubbyIrcProtocol def __init__(self, *args, **kw): # ServerFactory.__init__(self, *args, **kw) self.client = None self.events = DeferredQueue() def buildProtocol(self, addr): self.client = self.protocol(self.events) return self.client @inlineCallbacks def filter_events(self, command_type): while True: ev = yield self.events.get() if ev[1] == command_type: returnValue(ev)
class StubbyIrcServer(ServerFactory): protocol = StubbyIrcServerProtocol def startFactory(self): self.server = None self.events = DeferredQueue() self.finished_d = Deferred() def buildProtocol(self, addr): self.server = ServerFactory.buildProtocol(self, addr) self.server.factory = self return self.server @inlineCallbacks def filter_events(self, command_type): while True: ev = yield self.events.get() if ev[1] == command_type: returnValue(ev)
def pull_message(self, count=1): url = '%s/%s/messages.json' % (self.url, self.conversation.key) messages = DeferredQueue() errors = DeferredQueue() receiver = self.client.stream( TransportUserMessage, messages.put, errors.put, url, Headers(self.auth_headers)) received_messages = [] for msg_id in range(count): yield self.app_helper.make_dispatch_inbound( 'in %s' % (msg_id,), message_id=str(msg_id), conv=self.conversation) recv_msg = yield messages.get() received_messages.append(recv_msg) receiver.disconnect() returnValue((receiver, received_messages))
class TestGoConversationTransportBase(VumiTestCase): transport_class = None def setUp(self): self.tx_helper = self.add_helper(TransportHelper(self.transport_class)) self.fake_http = FakeHttpServer(self.handle_inbound_request) self.clock = Clock() self._request_queue = DeferredQueue() self._pending_reqs = [] self.add_cleanup(self.finish_requests) @inlineCallbacks def get_transport(self, start=True, **config): defaults = { 'account_key': 'account-key', 'conversation_key': 'conversation-key', 'access_token': 'access-token', 'publish_status': True, } defaults.update(config) transport = yield self.tx_helper.get_transport(defaults, start=False) transport.agent_factory = self.fake_http.get_agent if start: yield transport.startWorker() returnValue(transport) @inlineCallbacks def finish_requests(self): for req in self._pending_reqs: if not req.finished: yield req.finish() def handle_inbound_request(self, request): self._request_queue.put(request) return NOT_DONE_YET @inlineCallbacks def get_next_request(self): req = yield self._request_queue.get() self._pending_reqs.append(req) returnValue(req)
class FakeServer(object): """ Fake server container for testing client/server interactions. """ def __init__(self, server_factory, auto_accept=True, on_connect=None): self.server_factory = server_factory self.auto_accept = auto_accept self.connection_queue = DeferredQueue() self.on_connect = on_connect # Public API. @classmethod def for_protocol(cls, protocol, *args, **kw): factory = ServerFactory.forProtocol(protocol) return cls(factory, *args, **kw) @property def endpoint(self): """ Get an endpoint that connects clients to this server. """ return FakeServerEndpoint(self) def await_connection(self): """ Wait for a client to start connecting, and then return a :class:`FakeConnection` object. """ return self.connection_queue.get() # Internal stuff. def _handle_connection(self): conn = FakeConnection(self) if self.on_connect is not None: conn._connected_d.addCallback(lambda _: self.on_connect(conn)) self.connection_queue.put(conn) if self.auto_accept: conn.accept_connection() return conn._accept_d
class BattleshipsProcessProtocol(ProcessProtocol): def __init__(self, name): self.name = name self.buf = '' self.queue = DeferredQueue() self.on_crash = Deferred() self.err = '' def errReceived(self, data): self.err += data def outReceived(self, data): self.buf += data lines = self.buf.split('\n') self.buf = lines[-1] for l in lines[:-1]: self.lineReceived(l) def lineReceived(self, line): mo = re.match(r'^([A-Z])(\d+)$', line, flags=re.I) if mo: col = ord(mo.group(1).upper()) - 64 row = int(mo.group(2)) self.queue.put((col, row)) def processExited(self, status): if self.err: print self.name, "crashed with error:" print self.err self.on_crash.errback(status) def getMove(self): return self.queue.get() def sendResult(self, result): self.transport.write(result + '\n') def close(self): try: self.transport.signalProcess('TERM') except ProcessExitedAlready: pass
class PostgresListenerServiceSpy(PostgresListenerService): """Save received notifies `captured_notifies` before processing them..""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Captured notifications from the database will go here. self._captured_notifies = DeferredQueue() # Change notifications to a frozenset. This makes sure that the system # message does not go into the queue. Instead it should call the # handler directly in `doRead`. self.notifications = frozenset() def _process_notifies(self): for notify in self.connection.connection.notifies: self._captured_notifies.put(notify) super()._process_notifies() @inlineCallbacks def wait_notification(self, channel): """Wait for a notification to be received.""" while True: notice = yield self._captured_notifies.get() if notice.channel == channel: returnValue(notice)
class BroadcomOnuHandler(object): def __init__(self, adapter, device_id): self.adapter = adapter self.adapter_agent = adapter.adapter_agent self.device_id = device_id self.log = structlog.get_logger(device_id=device_id) self.incoming_messages = DeferredQueue() self.proxy_address = None self.tx_id = 0 def receive_message(self, msg): self.incoming_messages.put(msg) def activate(self, device): self.log.info('activating') # first we verify that we got parent reference and proxy info assert device.parent_id assert device.proxy_address.device_id assert device.proxy_address.channel_id # register for proxied messages right away self.proxy_address = device.proxy_address self.adapter_agent.register_for_proxied_messages(device.proxy_address) # populate device info device.root = True device.vendor = 'Broadcom' device.model = 'n/a' device.hardware_version = 'to be filled' device.firmware_version = 'to be filled' device.software_version = 'to be filled' device.serial_number = uuid4().hex device.connect_status = ConnectStatus.REACHABLE self.adapter_agent.update_device(device) # register physical ports uni_port = Port(port_no=2, label='UNI facing Ethernet port', type=Port.ETHERNET_UNI, admin_state=AdminState.ENABLED, oper_status=OperStatus.ACTIVE) self.adapter_agent.add_port(device.id, uni_port) self.adapter_agent.add_port( device.id, Port(port_no=1, label='PON port', type=Port.PON_ONU, admin_state=AdminState.ENABLED, oper_status=OperStatus.ACTIVE, peers=[ Port.PeerPort(device_id=device.parent_id, port_no=device.parent_port_no) ])) # add uni port to logical device parent_device = self.adapter_agent.get_device(device.parent_id) logical_device_id = parent_device.parent_id assert logical_device_id port_no = device.proxy_address.channel_id cap = OFPPF_1GB_FD | OFPPF_FIBER self.adapter_agent.add_logical_port( logical_device_id, LogicalPort(id='uni-{}'.format(port_no), ofp_port=ofp_port( port_no=port_no, hw_addr=mac_str_to_tuple( '00:00:00:00:%02x:%02x' % ((port_no >> 8) & 0xff, port_no & 0xff)), name='uni-{}'.format(port_no), config=0, state=OFPPS_LIVE, curr=cap, advertised=cap, peer=cap, curr_speed=OFPPF_1GB_FD, max_speed=OFPPF_1GB_FD), device_id=device.id, device_port_no=uni_port.port_no)) reactor.callLater(10, self.message_exchange) device = self.adapter_agent.get_device(device.id) device.oper_status = OperStatus.ACTIVE self.adapter_agent.update_device(device) @inlineCallbacks def update_flow_table(self, device, flows): # # We need to proxy through the OLT to get to the ONU # Configuration from here should be using OMCI # self.log.info('bulk-flow-update', device_id=device.id, flows=flows) def is_downstream(port): return port == 2 # Need a better way def is_upstream(port): return not is_downstream(port) for flow in flows: try: _in_port = fd.get_in_port(flow) assert _in_port is not None if is_downstream(_in_port): self.log.info('downstream-flow') elif is_upstream(_in_port): self.log.info('upstream-flow') else: raise Exception('port should be 1 or 2 by our convention') _out_port = fd.get_out_port(flow) # may be None self.log.info('out-port', out_port=_out_port) for field in fd.get_ofb_fields(flow): if field.type == fd.ETH_TYPE: _type = field.eth_type self.log.info('field-type-eth-type', eth_type=_type) elif field.type == fd.IP_PROTO: _proto = field.ip_proto self.log.info('field-type-ip-proto', ip_proto=_proto) elif field.type == fd.IN_PORT: _port = field.port self.log.info('field-type-in-port', in_port=_port) elif field.type == fd.VLAN_VID: _vlan_vid = field.vlan_vid & 0xfff self.log.info('field-type-vlan-vid', vlan=_vlan_vid) elif field.type == fd.VLAN_PCP: _vlan_pcp = field.vlan_pcp self.log.info('field-type-vlan-pcp', pcp=_vlan_pcp) elif field.type == fd.UDP_DST: _udp_dst = field.udp_dst self.log.info('field-type-udp-dst', udp_dst=_udp_dst) elif field.type == fd.UDP_SRC: _udp_src = field.udp_src self.log.info('field-type-udp-src', udp_src=_udp_src) elif field.type == fd.IPV4_DST: _ipv4_dst = field.ipv4_dst self.log.info('field-type-ipv4-dst', ipv4_dst=_ipv4_dst) elif field.type == fd.IPV4_SRC: _ipv4_src = field.ipv4_src self.log.info('field-type-ipv4-src', ipv4_dst=_ipv4_src) elif field.type == fd.METADATA: _metadata = field.metadata self.log.info('field-type-metadata', metadata=_metadata) else: raise NotImplementedError('field.type={}'.format( field.type)) for action in fd.get_actions(flow): if action.type == fd.OUTPUT: _output = action.output.port self.log.info('action-type-output', output=_output, in_port=_in_port) elif action.type == fd.POP_VLAN: self.log.info('action-type-pop-vlan', in_port=_in_port) elif action.type == fd.PUSH_VLAN: _push_tpid = action.push.ethertype log.info('action-type-push-vlan', push_tpid=_push_tpid, in_port=_in_port) if action.push.ethertype != 0x8100: self.log.error('unhandled-tpid', ethertype=action.push.ethertype) elif action.type == fd.SET_FIELD: _field = action.set_field.field.ofb_field assert (action.set_field.field.oxm_class == OFPXMC_OPENFLOW_BASIC) self.log.info('action-type-set-field', field=_field, in_port=_in_port) if _field.type == fd.VLAN_VID: self.log.info('set-field-type-valn-vid', vlan_vid=_field.vlan_vid & 0xfff) else: self.log.error('unsupported-action-set-field-type', field_type=_field.type) else: log.error('unsupported-action-type', action_type=action.type, in_port=_in_port) # # All flows created from ONU adapter should be OMCI based # except Exception as e: log.exception('failed-to-install-flow', e=e, flow=flow) def get_tx_id(self): self.tx_id += 1 return self.tx_id def send_omci_message(self, frame): _frame = hexify(str(frame)) self.log.info('send-omci-message-%s' % _frame) device = self.adapter_agent.get_device(self.device_id) try: self.adapter_agent.send_proxied_message(device.proxy_address, _frame) except Exception as e: self.log.info('send-omci-message-exception', exc=str(e)) def send_get_circuit_pack(self, entity_id=0): frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciGet.message_id, omci_message=OmciGet( entity_class=CircuitPack.class_id, entity_id=entity_id, attributes_mask=CircuitPack.mask_for('vendor_id'))) self.send_omci_message(frame) def send_mib_reset(self, entity_id=0): frame = OmciFrame(transaction_id=self.get_tx_id(), message_type=OmciMibReset.message_id, omci_message=OmciMibReset( entity_class=OntData.class_id, entity_id=entity_id)) self.send_omci_message(frame) def send_create_gal_ethernet_profile(self, entity_id, max_gem_payload_size): frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciCreate.message_id, omci_message=OmciCreate( entity_class=GalEthernetProfile.class_id, entity_id=entity_id, data=dict(max_gem_payload_size=max_gem_payload_size))) self.send_omci_message(frame) def send_set_tcont(self, entity_id, alloc_id): data = dict(alloc_id=alloc_id) frame = OmciFrame(transaction_id=self.get_tx_id(), message_type=OmciSet.message_id, omci_message=OmciSet( entity_class=Tcont.class_id, entity_id=entity_id, attributes_mask=Tcont.mask_for(*data.keys()), data=data)) self.send_omci_message(frame) def send_create_8021p_mapper_service_profile(self, entity_id): frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciCreate.message_id, omci_message=OmciCreate( entity_class=Ieee8021pMapperServiceProfile.class_id, entity_id=entity_id, data=dict( tp_pointer=OmciNullPointer, interwork_tp_pointer_for_p_bit_priority_0=OmciNullPointer, ))) self.send_omci_message(frame) def send_create_mac_bridge_service_profile(self, entity_id): frame = OmciFrame(transaction_id=self.get_tx_id(), message_type=OmciCreate.message_id, omci_message=OmciCreate( entity_class=MacBridgeServiceProfile.class_id, entity_id=entity_id, data=dict(spanning_tree_ind=False, learning_ind=True, priority=0x8000, max_age=20 * 256, hello_time=2 * 256, forward_delay=15 * 256, unknown_mac_address_discard=True))) self.send_omci_message(frame) def send_create_gem_port_network_ctp(self, entity_id, port_id, tcont_id, direction, tm): _directions = {"upstream": 1, "downstream": 2, "bi-directional": 3} if _directions.has_key(direction): _direction = _directions[direction] else: self.log.error('invalid-gem-port-direction', direction=direction) raise ValueError( 'Invalid GEM port direction: {_dir}'.format(_dir=direction)) frame = OmciFrame(transaction_id=self.get_tx_id(), message_type=OmciCreate.message_id, omci_message=OmciCreate( entity_class=GemPortNetworkCtp.class_id, entity_id=entity_id, data=dict( port_id=port_id, tcont_pointer=tcont_id, direction=_direction, traffic_management_pointer_upstream=tm))) self.send_omci_message(frame) def send_create_multicast_gem_interworking_tp(self, entity_id, gem_port_net_ctp_id): frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciCreate.message_id, omci_message=OmciCreate( entity_class=MulticastGemInterworkingTp.class_id, entity_id=entity_id, data=dict(gem_port_network_ctp_pointer=gem_port_net_ctp_id, interworking_option=0, service_profile_pointer=0x1))) self.send_omci_message(frame) def send_create_gem_inteworking_tp(self, entity_id, gem_port_net_ctp_id, service_profile_id): frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciCreate.message_id, omci_message=OmciCreate( entity_class=GemInterworkingTp.class_id, entity_id=entity_id, data=dict(gem_port_network_ctp_pointer=gem_port_net_ctp_id, interworking_option=5, service_profile_pointer=service_profile_id, interworking_tp_pointer=0x0, gal_profile_pointer=0x1))) self.send_omci_message(frame) def send_set_8021p_mapper_service_profile(self, entity_id, interwork_tp_id): data = dict(interwork_tp_pointer_for_p_bit_priority_0=interwork_tp_id) frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciSet.message_id, omci_message=OmciSet( entity_class=Ieee8021pMapperServiceProfile.class_id, entity_id=entity_id, attributes_mask=Ieee8021pMapperServiceProfile.mask_for( *data.keys()), data=data)) self.send_omci_message(frame) def send_create_mac_bridge_port_configuration_data(self, entity_id, bridge_id, port_id, tp_type, tp_id): frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciCreate.message_id, omci_message=OmciCreate( entity_class=MacBridgePortConfigurationData.class_id, entity_id=entity_id, data=dict(bridge_id_pointer=bridge_id, port_num=port_id, tp_type=tp_type, tp_pointer=tp_id))) self.send_omci_message(frame) def send_create_vlan_tagging_filter_data(self, entity_id, vlan_id): frame = OmciFrame(transaction_id=self.get_tx_id(), message_type=OmciCreate.message_id, omci_message=OmciCreate( entity_class=VlanTaggingFilterData.class_id, entity_id=entity_id, data=dict(vlan_filter_0=vlan_id, forward_operation=0x10, number_of_entries=1))) self.send_omci_message(frame) def send_create_extended_vlan_tagging_operation_configuration_data( self, entity_id, assoc_type, assoc_me): frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciCreate.message_id, omci_message=OmciCreate( entity_class=ExtendedVlanTaggingOperationConfigurationData. class_id, entity_id=entity_id, data=dict(association_type=assoc_type, associated_me_pointer=assoc_me))) self.send_omci_message(frame) def send_set_extended_vlan_tagging_operation_tpid_configuration_data( self, entity_id, input_tpid, output_tpid): data = dict( input_tpid=input_tpid, output_tpid=output_tpid, downstream_mode=0, # inverse of upstream ) frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciSet.message_id, omci_message=OmciSet( entity_class=ExtendedVlanTaggingOperationConfigurationData. class_id, entity_id=entity_id, attributes_mask=ExtendedVlanTaggingOperationConfigurationData. mask_for(*data.keys()), data=data)) self.send_omci_message(frame) def send_set_extended_vlan_tagging_operation_vlan_configuration_data_untagged( self, entity_id, filter_inner_vid, treatment_inner_vid): data = dict( received_frame_vlan_tagging_operation_table=VlanTaggingOperation( filter_outer_priority=15, filter_outer_vid=4096, filter_outer_tpid_de=0, filter_inner_priority=15, filter_inner_vid=filter_inner_vid, filter_inner_tpid_de=0, filter_ether_type=0, treatment_tags_to_remove=0, treatment_outer_priority=15, treatment_outer_vid=0, treatment_outer_tpid_de=0, treatment_inner_priority=0, treatment_inner_vid=treatment_inner_vid, treatment_inner_tpid_de=4)) frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciSet.message_id, omci_message=OmciSet( entity_class=ExtendedVlanTaggingOperationConfigurationData. class_id, entity_id=entity_id, attributes_mask=ExtendedVlanTaggingOperationConfigurationData. mask_for(*data.keys()), data=data)) self.send_omci_message(frame) def send_set_extended_vlan_tagging_operation_vlan_configuration_data_single_tag( self, entity_id, filter_inner_priority, filter_inner_vid, filter_inner_tpid_de, treatment_tags_to_remove, treatment_inner_priority, treatment_inner_vid): data = dict( received_frame_vlan_tagging_operation_table=VlanTaggingOperation( filter_outer_priority=15, filter_outer_vid=4096, filter_outer_tpid_de=0, filter_inner_priority=filter_inner_priority, filter_inner_vid=filter_inner_vid, filter_inner_tpid_de=filter_inner_tpid_de, filter_ether_type=0, treatment_tags_to_remove=treatment_tags_to_remove, treatment_outer_priority=15, treatment_outer_vid=0, treatment_outer_tpid_de=0, treatment_inner_priority=treatment_inner_priority, treatment_inner_vid=treatment_inner_vid, treatment_inner_tpid_de=4)) frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciSet.message_id, omci_message=OmciSet( entity_class=ExtendedVlanTaggingOperationConfigurationData. class_id, entity_id=entity_id, attributes_mask=ExtendedVlanTaggingOperationConfigurationData. mask_for(*data.keys()), data=data)) self.send_omci_message(frame) def send_create_multicast_operations_profile(self, entity_id, igmp_ver): frame = OmciFrame(transaction_id=self.get_tx_id(), message_type=OmciCreate.message_id, omci_message=OmciCreate( entity_class=MulticastOperationsProfile.class_id, entity_id=entity_id, data=dict(igmp_version=igmp_ver, igmp_function=0, immediate_leave=0))) self.send_omci_message(frame) def send_set_multicast_operations_profile_acl_row0(self, entity_id, acl_table, row_key, gem_port, vlan, src_ip, dst_ip_start, dst_ip_end): row0 = AccessControlRow0(set_ctrl=1, row_part_id=0, test=0, row_key=row_key, gem_port_id=gem_port, vlan_id=vlan, src_ip=src_ip, dst_ip_start=dst_ip_start, dst_ip_end=dst_ip_end, ipm_group_bw=0) if acl_table == 'dynamic': data = dict(dynamic_access_control_list_table=row0) else: data = dict(static_access_control_list_table=row0) frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciSet.message_id, omci_message=OmciSet( entity_class=MulticastOperationsProfile.class_id, entity_id=entity_id, attributes_mask=MulticastOperationsProfile.mask_for( *data.keys()), data=data)) self.send_omci_message(frame) def send_set_multicast_operations_profile_ds_igmp_mcast_tci( self, entity_id, ctrl_type, tci): data = dict(ds_igmp_mcast_tci=DownstreamIgmpMulticastTci( ctrl_type=ctrl_type, tci=tci)) frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciSet.message_id, omci_message=OmciSet( entity_class=MulticastOperationsProfile.class_id, entity_id=entity_id, attributes_mask=MulticastOperationsProfile.mask_for( *data.keys()), data=data)) self.send_omci_message(frame) def send_create_multicast_subscriber_config_info(self, entity_id, me_type, mcast_oper_profile): frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciCreate.message_id, omci_message=OmciCreate( entity_class=MulticastSubscriberConfigInfo.class_id, entity_id=entity_id, data=dict( me_type=me_type, mcast_operations_profile_pointer=mcast_oper_profile))) self.send_omci_message(frame) def send_set_multicast_subscriber_config_info(self, entity_id, max_groups=0, max_mcast_bw=0, bw_enforcement=0): data = dict(max_simultaneous_groups=max_groups, max_multicast_bandwidth=max_mcast_bw, bandwidth_enforcement=bw_enforcement) frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciSet.message_id, omci_message=OmciSet( entity_class=MulticastSubscriberConfigInfo.class_id, entity_id=entity_id, attributes_mask=MulticastSubscriberConfigInfo.mask_for( *data.keys()), data=data)) self.send_omci_message(frame) def send_set_multicast_service_package(self, entity_id, row_key, vid_uni, max_groups, max_mcast_bw, mcast_oper_profile): data = dict(multicast_service_package_table=MulticastServicePackage( set_ctrl=1, row_key=row_key, vid_uni=vid_uni, max_simultaneous_groups=max_groups, max_multicast_bw=max_mcast_bw, mcast_operations_profile_pointer=mcast_oper_profile)) frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciSet.message_id, omci_message=OmciSet( entity_class=MulticastSubscriberConfigInfo.class_id, entity_id=entity_id, attributes_mask=MulticastSubscriberConfigInfo.mask_for( *data.keys()), data=data)) self.send_omci_message(frame) def send_set_multicast_allowed_preview_groups_row0(self, entity_id, row_key, src_ip, vlan_id_ani, vlan_id_uni): data = dict(allowed_preview_groups_table=AllowedPreviewGroupsRow0( set_ctrl=1, row_part_id=0, row_key=row_key, src_ip=src_ip, vlan_id_ani=vlan_id_ani, vlan_id_uni=vlan_id_uni)) frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciSet.message_id, omci_message=OmciSet( entity_class=MulticastSubscriberConfigInfo.class_id, entity_id=entity_id, attributes_mask=MulticastSubscriberConfigInfo.mask_for( *data.keys()), data=data)) self.send_omci_message(frame) def send_set_multicast_allowed_preview_groups_row1(self, entity_id, row_key, dst_ip, duration, time_left): data = dict(allowed_preview_groups_table=AllowedPreviewGroupsRow1( set_ctrl=1, row_part_id=1, row_key=row_key, dst_ip=dst_ip, duration=duration, time_left=time_left)) frame = OmciFrame( transaction_id=self.get_tx_id(), message_type=OmciSet.message_id, omci_message=OmciSet( entity_class=MulticastSubscriberConfigInfo.class_id, entity_id=entity_id, attributes_mask=MulticastSubscriberConfigInfo.mask_for( *data.keys()), data=data)) self.send_omci_message(frame) @inlineCallbacks def wait_for_response(self): log.info('wait-for-response') try: response = yield self.incoming_messages.get() log.info('got-response') # resp = OmciFrame(response) # resp.show() except Exception as e: self.log.info('wait-for-response-exception', exc=str(e)) @inlineCallbacks def message_exchange(self): log.info('message_exchange') # reset incoming message queue while self.incoming_messages.pending: _ = yield self.incoming_messages.get() # construct message # MIB Reset - OntData - 0 self.send_mib_reset() yield self.wait_for_response() # Create AR - GalEthernetProfile - 1 self.send_create_gal_ethernet_profile(1, 48) yield self.wait_for_response() # Set AR - TCont - 32768 - 1024 self.send_set_tcont(0x8000, 0x400) yield self.wait_for_response() # Set AR - TCont - 32769 - 1025 self.send_set_tcont(0x8001, 0x401) yield self.wait_for_response() # Set AR - TCont - 32770 - 1026 self.send_set_tcont(0x8002, 0x402) yield self.wait_for_response() # Set AR - TCont - 32771 - 1027 self.send_set_tcont(0x8003, 0x403) yield self.wait_for_response() # Create AR - 802.1pMapperServiceProfile - 32768 self.send_create_8021p_mapper_service_profile(0x8000) yield self.wait_for_response() # Create AR - 802.1pMapperServiceProfile - 32769 self.send_create_8021p_mapper_service_profile(0x8001) yield self.wait_for_response() # Create AR - 802.1pMapperServiceProfile - 32770 self.send_create_8021p_mapper_service_profile(0x8002) yield self.wait_for_response() # Create AR - 802.1pMapperServiceProfile - 32771 self.send_create_8021p_mapper_service_profile(0x8003) yield self.wait_for_response() # Create AR - MacBridgeServiceProfile - 513 self.send_create_mac_bridge_service_profile(0x201) yield self.wait_for_response() # Create AR - GemPortNetworkCtp - 256 - 1024 - 32768 self.send_create_gem_port_network_ctp(0x100, 0x400, 0x8000, "bi-directional", 0x100) yield self.wait_for_response() # Create AR - GemPortNetworkCtp - 257 - 1025 - 32769 self.send_create_gem_port_network_ctp(0x101, 0x401, 0x8001, "bi-directional", 0x100) yield self.wait_for_response() # Create AR - GemPortNetworkCtp - 258 - 1026 - 32770 self.send_create_gem_port_network_ctp(0x102, 0x402, 0x8002, "bi-directional", 0x100) yield self.wait_for_response() # Create AR - GemPortNetworkCtp - 259 - 1027 - 32771 self.send_create_gem_port_network_ctp(0x103, 0x403, 0x8003, "bi-directional", 0x100) yield self.wait_for_response() # Create AR - GemPortNetworkCtp - 260 - 4000 - 0 self.send_create_gem_port_network_ctp(0x104, 0x0FA0, 0, "downstream", 0) yield self.wait_for_response() # Create AR - MulticastGemInterworkingTp - 6 - 260 self.send_create_multicast_gem_interworking_tp(0x6, 0x104) yield self.wait_for_response() # Create AR - GemInterworkingTp - 32769 - 256 -32768 - 1 self.send_create_gem_inteworking_tp(0x8001, 0x100, 0x8000) yield self.wait_for_response() # Create AR - GemInterworkingTp - 32770 - 257 -32769 - 1 self.send_create_gem_inteworking_tp(0x8002, 0x101, 0x8001) yield self.wait_for_response() # Create AR - GemInterworkingTp - 32771 - 258 -32770 - 1 self.send_create_gem_inteworking_tp(0x8003, 0x102, 0x8002) yield self.wait_for_response() # Create AR - GemInterworkingTp - 32772 - 259 -32771 - 1 self.send_create_gem_inteworking_tp(0x8004, 0x103, 0x8003) yield self.wait_for_response() # Set AR - 802.1pMapperServiceProfile - 32768 - 32769 self.send_set_8021p_mapper_service_profile(0x8000, 0x8001) yield self.wait_for_response() # Set AR - 802.1pMapperServiceProfile - 32769 - 32770 self.send_set_8021p_mapper_service_profile(0x8001, 0x8002) yield self.wait_for_response() # Set AR - 802.1pMapperServiceProfile - 32770 - 32771 self.send_set_8021p_mapper_service_profile(0x8002, 0x8003) yield self.wait_for_response() # Set AR - 802.1pMapperServiceProfile - 32771 - 32772 self.send_set_8021p_mapper_service_profile(0x8003, 0x8004) yield self.wait_for_response() # Create AR - MacBridgePortConfigData - 8449 - 513 - 2 - 3 - 32768 self.send_create_mac_bridge_port_configuration_data( 0x2101, 0x201, 2, 3, 0x8000) yield self.wait_for_response() # Create AR - MacBridgePortConfigData - 8450 - 513 - 3 - 3 - 32769 self.send_create_mac_bridge_port_configuration_data( 0x2102, 0x201, 3, 3, 0x8001) yield self.wait_for_response() # Create AR - MacBridgePortConfigData - 8451 - 513 - 4 - 3 - 32770 self.send_create_mac_bridge_port_configuration_data( 0x2103, 0x201, 4, 3, 0x8002) yield self.wait_for_response() # Create AR - MacBridgePortConfigData - 8452 - 513 - 5 - 3 - 32771 self.send_create_mac_bridge_port_configuration_data( 0x2104, 0x201, 5, 3, 0x8003) yield self.wait_for_response() # Create AR - MacBridgePortConfigData - 9000 - 513 - 6 - 6 - 6 self.send_create_mac_bridge_port_configuration_data( 0x2328, 0x201, 6, 6, 6) yield self.wait_for_response() # Create AR - VlanTaggingFilterData - 8449 - 040000000000000000000000000000000000000000000000 self.send_create_vlan_tagging_filter_data(0x2101, 0x0400) yield self.wait_for_response() # Create AR - VlanTaggingFilterData - 8450 - 040100000000000000000000000000000000000000000000 self.send_create_vlan_tagging_filter_data(0x2102, 0x0401) yield self.wait_for_response() # Create AR - VlanTaggingFilterData - 8451 - 040200000000000000000000000000000000000000000000 self.send_create_vlan_tagging_filter_data(0x2103, 0x0402) yield self.wait_for_response() # Create AR - VlanTaggingFilterData - 8452 - 040300000000000000000000000000000000000000000000 self.send_create_vlan_tagging_filter_data(0x2104, 0x0403) yield self.wait_for_response() # Create AR - MulticastOperationsProfile self.send_create_multicast_operations_profile(0x201, 3) yield self.wait_for_response() # Set AR - MulticastOperationsProfile - Dynamic Access Control List table self.send_set_multicast_operations_profile_acl_row0( 0x201, 'dynamic', 0, 0x0fa0, 0x0fa0, '0.0.0.0', '224.0.0.0', '239.255.255.255') yield self.wait_for_response() # Create AR - MulticastSubscriberConfigInfo self.send_create_multicast_subscriber_config_info(0x201, 0, 0x201) yield self.wait_for_response() # Set AR - MulticastOperationsProfile - Downstream IGMP Multicast TCI self.send_set_multicast_operations_profile_ds_igmp_mcast_tci( 0x201, 3, 0x401) yield self.wait_for_response() # Create AR - ExtendedVlanTaggingOperationConfigData - 514 - 2 - 0x105 self.send_create_extended_vlan_tagging_operation_configuration_data( 0x202, 2, 0x102) yield self.wait_for_response() # Set AR - ExtendedVlanTaggingOperationConfigData - 514 - 8100 - 8100 self.send_set_extended_vlan_tagging_operation_tpid_configuration_data( 0x202, 0x8100, 0x8100) yield self.wait_for_response() # Set AR - ExtendedVlanTaggingOperationConfigData # 514 - RxVlanTaggingOperationTable - add VLAN 1025 to untagged pkts #self.send_set_extended_vlan_tagging_operation_vlan_configuration_data_untagged(0x202, 0x1000, 0x401) #yield self.wait_for_response() # Set AR - ExtendedVlanTaggingOperationConfigData # 514 - RxVlanTaggingOperationTable - add VLAN 1025 to priority tagged pkts self.send_set_extended_vlan_tagging_operation_vlan_configuration_data_single_tag( 0x202, 0, 0, 0, 1, 8, 0x401) yield self.wait_for_response() # Create AR - MacBridgePortConfigData - 513 - 513 - 1 - 1 - 0x105 self.send_create_mac_bridge_port_configuration_data( 0x201, 0x201, 1, 1, 0x102) yield self.wait_for_response()
class MQTTService(ClientService): # Default publish QoS QoS = 0 def __init__(self, options, **kargs): self.options = options setLogLevel(namespace=NAMESPACE, levelStr=options['log_level']) setLogLevel(namespace=PROTOCOL_NAMESPACE, levelStr=options['log_messages']) self.factory = MQTTFactory(profile=MQTTFactory.PUBLISHER) self.endpoint = clientFromString(reactor, self.options['broker']) self.task = None if self.options['username'] == "": self.options['username'] = None self.options['password'] = None ClientService.__init__(self, self.endpoint, self.factory, retryPolicy=backoffPolicy( initialDelay=INITIAL_DELAY, factor=FACTOR, maxDelay=MAX_DELAY)) self.queue = DeferredQueue() # ----------- # Service API # ----------- def startService(self): log.info("starting MQTT Client Service") self.whenConnected().addCallback(self.connectToBroker) super().startService() @inlineCallbacks def stopService(self): try: yield ClientService.stopService(self) except Exception as e: log.failure("Exception {excp!s}", excp=e) reactor.stop() @inlineCallbacks def reloadService(self, options): setLogLevel(namespace=NAMESPACE, levelStr=options['log_level']) setLogLevel(namespace=PROTOCOL_NAMESPACE, levelStr=options['log_messages']) log.info("new log level is {lvl}", lvl=options['log_level']) self.options = options # ------------- # log stats API # ------------- # -------------- # Helper methods # --------------- @inlineCallbacks def connectToBroker(self, protocol): ''' Connect to MQTT broker ''' self.protocol = protocol self.protocol.onPublish = self.publish self.protocol.onDisconnection = self.onDisconnection self.protocol.setWindowSize(3) try: yield self.protocol.connect("tessw-publisher" + '@' + HOSTNAME, username=self.options['username'], password=self.options['password'], keepalive=self.options['keepalive']) except Exception as e: log.failure("Connecting to {broker} raised {excp!s}", broker=self.options['broker'], excp=e) else: log.info("Connected to {broker}", broker=self.options['broker']) reactor.callLater(0, self.publish) def onDisconnection(self, reason): ''' Disconenction handler. Tells ClientService what to do when the connection is lost ''' log.warn("tessw-publisher lost connection with its MQTT broker") if self.task: self.task.cancel() self.task = None self.whenConnected().addCallback(self.connectToBroker) def addRegisterRequest(self, photometer_info): topic = "{0}/{1}".format(self.options['topic'], "register") self.queue.put((topic, photometer_info)) def addReading(self, reading): topic = "{0}/{1}/{2}".format(self.options['topic'], reading['name'], "reading") self.queue.put((topic, reading)) @inlineCallbacks def publish(self): log.info("Entering Registry & Data Publishing Phase") while True: try: topic, reading = yield self.queue.get() msg = json.dumps(reading) yield self.protocol.publish(topic=topic, qos=self.QoS, message=msg) except MQTTStateError as e: log.error("{excp}", excp=e) except Exception as e: log.failure("Error when publishing: {excp!s}", excp=e) else: pass
class OMCI_CC(object): """ Handle OMCI Communication Channel specifics for Adtran ONUs""" def __init__(self, adapter_agent, device_id, custom_me_entries=None, alarm_queue_limit=_MAX_INCOMING_ALARM_MESSAGES, avc_queue_limit=_MAX_INCOMING_ALARM_MESSAGES, test_results_queue_limit=_MAX_INCOMING_TEST_RESULT_MESSAGES): self.log = structlog.get_logger(device_id=device_id) self._adapter_agent = adapter_agent self._device_id = device_id self._proxy_address = None self._tx_tid = 1 self._enabled = False self._requests = dict( ) # Tx ID -> (timestamp, deferred, tx_frame, timeout) self._alarm_queue = DeferredQueue(size=alarm_queue_limit) self._avc_queue = DeferredQueue(size=avc_queue_limit) self._test_results_queue = DeferredQueue(size=test_results_queue_limit) # Statistics self._tx_frames = 0 self._rx_frames = 0 self._rx_unknown_tid = 0 # Rx OMCI with no Tx TID match self._rx_onu_frames = 0 # Autonomously generated ONU frames self._rx_alarm_overflow = 0 # Autonomously generated ONU alarms rx overflow self._rx_avc_overflow = 0 # Autonomously generated ONU AVC rx overflow self._rx_onu_discards = 0 # Autonomously generated ONU unknown message types self._rx_timeouts = 0 self._tx_errors = 0 # Exceptions during tx request self._consecutive_errors = 0 # Rx & Tx errors in a row, a good RX resets this to 0 self._reply_min = sys.maxint # Fastest successful tx -> rx self._reply_max = 0 # Longest successful tx -> rx self._reply_sum = 0.0 # Total seconds for successful tx->rx (float for average) # If a list of custom ME Entities classes were provided, insert them into # main class_id to entity map. # TODO: If this class becomes hidden from the ONU DA, move this to the OMCI State Machine runner if custom_me_entries is not None: add_onu_me_entities(custom_me_entries) def __str__(self): return "OMCISupport: {}".format(self._device_id) @property def enabled(self): return self._enabled @enabled.setter def enabled(self, value): """ Enable/disable the OMCI Communications Channel :param value: (boolean) True to enable, False to disable """ assert isinstance(value, bool), 'enabled is a boolean' if self._enabled != value: self._enabled = value if self._enabled: self._start() else: self._stop() @property def tx_frames(self): return self._tx_frames @property def rx_frames(self): return self._rx_frames @property def rx_unknown_tid(self): return self._rx_unknown_tid # Tx TID not found @property def rx_onu_frames(self): return self._rx_onu_frames @property def rx_alarm_overflow(self): return self._rx_alarm_overflow # Alarm ONU autonomous overflows @property def rx_avc_overflow(self): return self._rx_avc_overflow # Attribute Value change autonomous overflows @property def rx_onu_discards(self): return self._rx_onu_discards # Attribute Value change autonomous overflows @property def rx_timeouts(self): return self._rx_timeouts @property def tx_errors(self): return self._tx_errors @property def consecutive_errors(self): return self._consecutive_errors @property def reply_min(self): return int(round(self._reply_min * 1000.0)) # Milliseconds @property def reply_max(self): return int(round(self._reply_max * 1000.0)) # Milliseconds @property def reply_average(self): avg = self._reply_sum / self._rx_frames if self._rx_frames > 0 else 0.0 return int(round(avg * 1000.0)) # Milliseconds @property def get_alarm_message(self): """ Attempt to retrieve and remove an ONU Alarm Message from the ONU autonomous message queue. TODO: We may want to deprecate this, see TODO comment around line 399 in the _request_success() method below :return: a Deferred which fires with the next Alarm Frame available in the queue. """ return self._alarm_queue.get() @property def get_avc_message(self): """ Attempt to retrieve and remove an ONU Attribute Value Change (AVC) Message from the ONU autonomous message queue. TODO: We may want to deprecate this, see TODO comment around line 399 in the _request_success() method below :return: a Deferred which fires with the next AVC Frame available in the queue. """ return self._avc_queue.get() @property def get_test_results(self): """ Attempt to retrieve and remove an ONU Test Results Message from the ONU autonomous message queue. TODO: We may want to deprecate this, see TODO comment around line 399 in the _request_success() method below :return: a Deferred which fires with the next Test Results Frame is available in the queue. """ return self._test_results_queue.get() def _start(self): """ Start the OMCI Communications Channel """ assert self._enabled, 'Start should only be called if enabled' # # TODO: Perform any other common startup tasks here # self.flush() device = self._adapter_agent.get_device(self._device_id) self._proxy_address = device.proxy_address def _stop(self): """ Stop the OMCI Communications Channel """ assert not self._enabled, 'Stop should only be called if disabled' # # TODO: Perform common shutdown tasks here # self.flush() self._proxy_address = None # TODO: What is best way to clean up any outstanding futures for these queues self._alarm_queue = None self._avc_queue = None self._test_results_queue = None def _receive_onu_message(self, rx_frame): """ Autonomously generated ONU frame Rx handler""" from twisted.internet.defer import QueueOverflow self.log.debug('rx-onu-frame', frame_type=type(rx_frame), frame=hexify(str(rx_frame))) # TODO: Signal, via defer if Alarm Overflow or just an event? msg_type = rx_frame.fields['message_type'] self._rx_onu_frames += 1 if msg_type == EntityOperations.AlarmNotification: try: self._alarm_queue.put( (rx_frame, arrow.utcnow().float_timestamp)) except QueueOverflow: self._rx_alarm_overflow += 1 self.log.warn('onu-rx-alarm-overflow', cnt=self._rx_alarm_overflow) elif msg_type == EntityOperations.AttributeValueChange: try: self._alarm_queue.put( (rx_frame, arrow.utcnow().float_timestamp)) except QueueOverflow: self._rx_avc_overflow += 1 self.log.warn('onu-rx-avc-overflow', cnt=self._rx_avc_overflow) else: # TODO: Need to add test results message support self.log.warn('onu-unsupported-autonomous-message', type=msg_type) self._rx_onu_discards += 1 def receive_message(self, msg): """ Receive and OMCI message from the proxy channel to the OLT. Call this from your ONU Adapter on a new OMCI Rx on the proxy channel """ if self.enabled: try: now = arrow.utcnow() d = None try: rx_frame = OmciFrame(msg) rx_tid = rx_frame.fields['transaction_id'] if rx_tid == 0: return self._receive_onu_message(rx_frame) self._rx_frames += 1 self._consecutive_errors = 0 except KeyError as e: # Unknown, Unsupported, or vendor-specific ME. Key is the unknown classID # TODO: Can we create a temporary one to hold it so upload does not always fail on new ME's? self.log.exception('frame-decode-key-error', msg=hexlify(msg), e=e) return except Exception as e: self.log.exception('frame-decode', msg=hexlify(msg), e=e) return try: (ts, d, _, _) = self._requests.pop(rx_tid) ts_diff = now - arrow.Arrow.utcfromtimestamp(ts) secs = ts_diff.total_seconds() self._reply_sum += secs if secs < self._reply_min: self._reply_min = secs if secs > self._reply_max: self._reply_max = secs # TODO: Could also validate response type based on request action except KeyError: # Possible late Rx on a message that timed-out self._rx_unknown_tid += 1 self.log.warn('tx-message-missing', rx_id=rx_tid, msg=hexlify(msg)) return except Exception as e: self.log.exception('frame-match', msg=hexlify(msg), e=e) if d is not None: return d.errback(failure.Failure(e)) return d.callback(rx_frame) except Exception as e: self.log.exception('rx-msg', e=e) def flush(self, max_age=0): limit = arrow.utcnow().float_timestamp - max_age old = [ tid for tid, (ts, _, _, _) in self._requests.iteritems() if ts <= limit ] for tid in old: (_, d, _, _) = self._requests.pop(tid) if d is not None and not d.called: d.cancel() self._requests = dict() if max_age == 0: # Flush autonomous messages (Alarms & AVCs) while self._alarm_queue.pending: _ = yield self._alarm_queue.get() while self._avc_queue.pending: _ = yield self._avc_queue.get() def _get_tx_tid(self): """ Get the next Transaction ID for a tx. Note TID=0 is reserved for autonomously generated messages from an ONU :return: (int) TID """ tx_tid, self._tx_tid = self._tx_tid, self._tx_tid + 1 if self._tx_tid > MAX_OMCI_TX_ID: self._tx_tid = 1 return tx_tid def _request_failure(self, value, tx_tid): """ Handle a transmit failure and/or Rx timeout :param value: (Failure) Twisted failure :param tx_tid: (int) Associated Tx TID """ if tx_tid in self._requests: (_, _, _, timeout) = self._requests.pop(tx_tid) else: timeout = 0 if isinstance(value, failure.Failure): value.trap(CancelledError) self._rx_timeouts += 1 self._consecutive_errors += 1 self.log.info('timeout', tx_id=tx_tid, timeout=timeout) value = failure.Failure(TimeoutError(timeout, "Deferred")) return value def _request_success(self, rx_frame): """ Handle transmit success (a matching Rx was received) :param rx_frame: (OmciFrame) OMCI response frame with matching TID :return: (OmciFrame) OMCI response frame with matching TID """ # # TODO: Here we could update the MIB database if we did a set/create/delete # or perhaps a verify if a GET. Also could increment mib counter # # TODO: A better way to perform this in VOLTHA v1.3 would be to provide # a pub/sub capability for external users/tasks to monitor responses # that could optionally take a filter. This would allow a MIB-Sync # task to easily watch all AVC notifications as well as Set/Create/Delete # operations and keep them serialized. It may also be a better/easier # way to handle things if we containerize OpenOMCI. # try: if isinstance(rx_frame.omci_message, OmciGetResponse): pass # TODO: Implement MIB check or remove elif isinstance(rx_frame.omci_message, OmciSetResponse): pass # TODO: Implement MIB update elif isinstance(rx_frame.omci_message, OmciCreateResponse): pass # TODO: Implement MIB update elif isinstance(rx_frame.omci_message, OmciDeleteResponse): pass # TODO: Implement MIB update except Exception as e: self.log.exception('omci-message', e=e) return rx_frame def send(self, frame, timeout=DEFAULT_OMCI_TIMEOUT): """ Send the OMCI Frame to the ONU via the proxy_channel :param frame: (OMCIFrame) Message to send :param timeout: (int) Rx Timeout. 0=Forever :return: (deferred) A deferred that fires when the response frame is received or if an error/timeout occurs """ self.flush(max_age=MAX_OMCI_REQUEST_AGE) assert timeout <= MAX_OMCI_REQUEST_AGE, \ 'Maximum timeout is {} seconds'.format(MAX_OMCI_REQUEST_AGE) assert isinstance(frame, OmciFrame), \ "Invalid frame class '{}'".format(type(frame)) if not self.enabled or self._proxy_address is None: # TODO custom exceptions throughout this code would be helpful return fail( result=failure.Failure(Exception('OMCI is not enabled'))) try: tx_tid = frame.fields['transaction_id'] if tx_tid is None: tx_tid = self._get_tx_tid() frame.fields['transaction_id'] = tx_tid assert tx_tid not in self._requests, 'TX TID {} is already exists'.format( tx_tid) assert tx_tid >= 0, 'Invalid Tx TID: {}'.format(tx_tid) ts = arrow.utcnow().float_timestamp d = defer.Deferred() self._adapter_agent.send_proxied_message(self._proxy_address, hexify(str(frame))) self._tx_frames += 1 self._requests[tx_tid] = (ts, d, frame, timeout) d.addCallbacks(self._request_success, self._request_failure, errbackArgs=(tx_tid, )) if timeout > 0: d.addTimeout(timeout, reactor) except Exception as e: self._tx_errors += 1 self._consecutive_errors += 1 self.log.exception('send-omci', e=e) return fail(result=failure.Failure(e)) return d ################################################################################### # MIB Action shortcuts def send_mib_reset(self, timeout=DEFAULT_OMCI_TIMEOUT): """ Perform a MIB Reset """ self.log.debug('send-mib-reset') frame = OntDataFrame().mib_reset() return self.send(frame, timeout) def send_mib_upload(self, timeout=DEFAULT_OMCI_TIMEOUT): self.log.debug('send-mib-upload') frame = OntDataFrame().mib_upload() return self.send(frame, timeout) def send_mib_upload_next(self, seq_no, timeout=DEFAULT_OMCI_TIMEOUT): self.log.debug('send-mib-upload-next') frame = OntDataFrame(seq_no).mib_upload_next() return self.send(frame, timeout) def send_reboot(self, timeout=DEFAULT_OMCI_TIMEOUT): """ Send an ONU Device reboot request (ONU-G ME). """ self.log.debug('send-mib-reboot') frame = OntGFrame().reboot() return self.send(frame, timeout)
def test_shared_interface(self): queue1 = DeferredQueue() queue2 = DeferredQueue() # two senders hooked up to the same interface (sharing it) # here we test if they can both send pin1 = self.mgr.open_port('veth0', none).up() pin2 = self.mgr.open_port('veth0', none).up() pout1 = self.mgr.open_port('veth1', lambda p, f: queue1.put( (p, f))).up() filter = BpfProgramFilter('ip dst host 123.123.123.123') pout2 = self.mgr.open_port('veth1', lambda p, f: queue2.put((p, f)), filter=filter).up() # sending from pin1, should be received by pout1 bogus_frame = 'bogus packet' bogus_frame_padded = bogus_frame + '\x00' * (FrameIOPort.MIN_PKT_SIZE - len(bogus_frame)) pin1.send(bogus_frame_padded) port, frame = yield queue1.get() self.assertEqual(port, pout1) self.assertEqual(frame, bogus_frame_padded) self.assertEqual(len(queue1.pending), 0) self.assertEqual(len(queue2.pending), 0) # sending from pin2, should be received by pout1 pin2.send(bogus_frame_padded) port, frame = yield queue1.get() self.assertEqual(port, pout1) self.assertEqual(frame, bogus_frame_padded) self.assertEqual(len(queue1.pending), 0) self.assertEqual(len(queue2.pending), 0) # sending from pin1, should be received by both pouts ip_packet = str(Ether() / IP(dst='123.123.123.123')) pin1.send(ip_packet) port, frame = yield queue1.get() self.assertEqual(port, pout1) self.assertEqual(frame, ip_packet) self.assertEqual(len(queue1.pending), 0) port, frame = yield queue2.get() self.assertEqual(port, pout2) self.assertEqual(frame, ip_packet) self.assertEqual(len(queue2.pending), 0) # sending from pin2, should be received by pout1 ip_packet = str(Ether() / IP(dst='123.123.123.123')) pin2.send(ip_packet) port, frame = yield queue1.get() self.assertEqual(port, pout1) self.assertEqual(frame, ip_packet) self.assertEqual(len(queue1.pending), 0) port, frame = yield queue2.get() self.assertEqual(port, pout2) self.assertEqual(frame, ip_packet) self.assertEqual(len(queue2.pending), 0) self.mgr.close_port(pin1) self.mgr.close_port(pin2) self.mgr.close_port(pout1) self.mgr.close_port(pout2)
class TibitOnuAdapter(object): name = 'tibit_onu' supported_device_types = [ DeviceType(id='tibit_onu', adapter=name, accepts_bulk_flow_update=True) ] def __init__(self, adapter_agent, config): self.adapter_agent = adapter_agent self.config = config self.descriptor = Adapter( id=self.name, vendor='Tibit Communications Inc.', version='0.1', config=AdapterConfig(log_level=LogLevel.INFO)) self.incoming_messages = DeferredQueue() def start(self): log.debug('starting') log.info('started') def stop(self): log.debug('stopping') log.info('stopped') def adapter_descriptor(self): return self.descriptor def device_types(self): return DeviceTypes(items=self.supported_device_types) def health(self): return HealthStatus(state=HealthStatus.HealthState.HEALTHY) def change_master_state(self, master): raise NotImplementedError() def adopt_device(self, device): log.info('adopt-device', device=device) reactor.callLater(0.1, self._onu_device_activation, device) return device @inlineCallbacks def _onu_device_activation(self, device): # first we verify that we got parent reference and proxy info assert device.parent_id assert device.proxy_address.device_id assert device.proxy_address.channel_id # TODO: For now, pretend that we were able to contact the device and obtain # additional information about it. Should add real message. device.vendor = 'Tibit Communications, Inc.' device.model = '10G GPON ONU' device.hardware_version = 'fa161020' device.firmware_version = '16.12.02' device.software_version = '1.0' device.serial_number = uuid4().hex device.connect_status = ConnectStatus.REACHABLE self.adapter_agent.update_device(device) # then shortly after we create some ports for the device uni_port = Port(port_no=2, label='UNI facing Ethernet port', type=Port.ETHERNET_UNI, admin_state=AdminState.ENABLED, oper_status=OperStatus.ACTIVE) self.adapter_agent.add_port(device.id, uni_port) self.adapter_agent.add_port( device.id, Port(port_no=1, label='PON port', type=Port.PON_ONU, admin_state=AdminState.ENABLED, oper_status=OperStatus.ACTIVE, peers=[ Port.PeerPort(device_id=device.parent_id, port_no=device.parent_port_no) ])) # TODO adding vports to the logical device shall be done by agent? # then we create the logical device port that corresponds to the UNI # port of the device # obtain logical device id parent_device = self.adapter_agent.get_device(device.parent_id) logical_device_id = parent_device.parent_id assert logical_device_id # we are going to use the proxy_address.channel_id as unique number # and name for the virtual ports, as this is guaranteed to be unique # in the context of the OLT port, so it is also unique in the context # of the logical device port_no = device.proxy_address.channel_id cap = OFPPF_10GB_FD | OFPPF_FIBER self.adapter_agent.add_logical_port( logical_device_id, LogicalPort(id=str(port_no), ofp_port=ofp_port(port_no=port_no, hw_addr=mac_str_to_tuple( device.mac_address), name='uni-{}'.format(port_no), config=0, state=OFPPS_LIVE, curr=cap, advertised=cap, peer=cap, curr_speed=OFPPF_10GB_FD, max_speed=OFPPF_10GB_FD), device_id=device.id, device_port_no=uni_port.port_no)) # simulate a proxied message sending and receving a reply reply = yield self._message_exchange(device) # and finally update to "ACTIVE" device = self.adapter_agent.get_device(device.id) device.oper_status = OperStatus.ACTIVE self.adapter_agent.update_device(device) self.start_kpi_collection(device.id) def abandon_device(self, device): raise NotImplementedError(0) def deactivate_device(self, device): raise NotImplementedError() def update_flows_bulk(self, device, flows, groups): log.info('########################################') log.info('bulk-flow-update', device_id=device.id, flows=flows, groups=groups) assert len(groups.items) == 0, "Cannot yet deal with groups" Clause = {v: k for k, v in ClauseSubtypeEnum.iteritems()} Operator = {v: k for k, v in RuleOperatorEnum.iteritems()} for flow in flows.items: in_port = get_in_port(flow) assert in_port is not None precedence = 255 - min(flow.priority / 256, 255) if in_port == 2: log.info('#### Upstream Rule ####') dn_req = EOAMPayload(body=CablelabsOUI() / DPoEOpcode_SetRequest()) for field in get_ofb_fields(flow): if field.type == ETH_TYPE: _type = field.eth_type log.info('#### field.type == ETH_TYPE ####', field_type=_type) elif field.type == IP_PROTO: _proto = field.ip_proto log.info('#### field.type == IP_PROTO ####') elif field.type == IN_PORT: _port = field.port log.info('#### field.type == IN_PORT ####', port=_port) elif field.type == VLAN_VID: _vlan_vid = field.vlan_vid & 0xfff log.info('#### field.type == VLAN_VID ####', vlan=_vlan_vid) elif field.type == VLAN_PCP: _vlan_pcp = field.vlan_pcp log.info('#### field.type == VLAN_PCP ####', pcp=_vlan_pcp) elif field.type == UDP_DST: _udp_dst = field.udp_dst log.info('#### field.type == UDP_DST ####') elif field.type == IPV4_DST: _ipv4_dst = field.ipv4_dst log.info('#### field.type == IPV4_DST ####') else: log.info('#### field.type == NOT IMPLEMENTED!! ####') raise NotImplementedError('field.type={}'.format( field.type)) for action in get_actions(flow): if action.type == OUTPUT: log.info('#### action.type == OUTPUT ####') elif action.type == POP_VLAN: log.info('#### action.type == POP_VLAN ####') elif action.type == PUSH_VLAN: log.info('#### action.type == PUSH_VLAN ####') if action.push.ethertype != 0x8100: log.error('unhandled-tpid', ethertype=action.push.ethertype) elif action.type == SET_FIELD: log.info('#### action.type == SET_FIELD ####') assert (action.set_field.field.oxm_class == ofp.OFPXMC_OPENFLOW_BASIC) field = action.set_field.field.ofb_field if field.type == VLAN_VID: pass else: log.error('unsupported-action-set-field-type', field_type=field.type) else: log.error('UNSUPPORTED-ACTION-TYPE', action_type=action.type) elif in_port == 1: log.info('#### Downstream Rule ####') #### Loop through fields again... for field in get_ofb_fields(flow): if field.type == ETH_TYPE: _type = field.eth_type log.info('#### field.type == ETH_TYPE ####', in_port=in_port, match=_type) elif field.type == IP_PROTO: _proto = field.ip_proto log.info('#### field.type == IP_PROTO ####', in_port=in_port, ip_proto=ip_proto) elif field.type == IN_PORT: _port = field.port log.info('#### field.type == IN_PORT ####') elif field.type == VLAN_VID: _vlan_vid = field.vlan_vid & 0xfff log.info('#### field.type == VLAN_VID ####') elif field.type == VLAN_PCP: _vlan_pcp = field.vlan_pcp log.info('#### field.type == VLAN_PCP ####') elif field.type == UDP_DST: _udp_dst = field.udp_dst log.info('#### field.type == UDP_DST ####') elif field.type == IPV4_DST: _ipv4_dst = field.ipv4_dst log.info('#### field.type == IPV4_DST ####') a = int(hex(_ipv4_dst)[2:4], 16) b = int(hex(_ipv4_dst)[4:6], 16) c = int(hex(_ipv4_dst)[6:8], 16) d = int(hex(_ipv4_dst)[8:], 16) dn_req = EOAMPayload( body=CablelabsOUI() / DPoEOpcode_SetRequest() / AddStaticMacAddress( mac=mcastIp2McastMac('%d.%d.%d.%d' % (a, b, c, d)))) # send message log.info('ONU-send-proxied-message') self.adapter_agent.send_proxied_message( device.proxy_address, dn_req) else: raise NotImplementedError('field.type={}'.format( field.type)) for action in get_actions(flow): if action.type == OUTPUT: log.info('#### action.type == OUTPUT ####') elif action.type == POP_VLAN: log.info('#### action.type == POP_VLAN ####') elif action.type == PUSH_VLAN: log.info('#### action.type == PUSH_VLAN ####') if action.push.ethertype != 0x8100: log.error('unhandled-ether-type', ethertype=action.push.ethertype) elif action.type == SET_FIELD: log.info('#### action.type == SET_FIELD ####') assert (action.set_field.field.oxm_class == ofp.OFPXMC_OPENFLOW_BASIC) field = action.set_field.field.ofb_field if field.type == VLAN_VID: pass else: log.error('unsupported-action-set-field-type', field_type=field.type) else: log.error('UNSUPPORTED-ACTION-TYPE', action_type=action.type) else: raise Exception('Port should be 1 or 2 by our convention') def update_flows_incrementally(self, device, flow_changes, group_changes): raise NotImplementedError() def send_proxied_message(self, proxy_address, msg): raise NotImplementedError() def receive_proxied_message(self, proxy_address, msg): log.info('receive-proxied-message', proxy_address=proxy_address, msg=msg.show(dump=True)) self.incoming_messages.put(msg) @inlineCallbacks def _message_exchange(self, device): # register for receiving async messages self.adapter_agent.register_for_proxied_messages(device.proxy_address) # reset incoming message queue while self.incoming_messages.pending: _ = yield self.incoming_messages.get() # construct message msg = EOAMPayload(body=CablelabsOUI() / DPoEOpcode_GetRequest() / DeviceId()) # send message log.info('ONU-send-proxied-message') self.adapter_agent.send_proxied_message(device.proxy_address, msg) # wait till we detect incoming message yield self.incoming_messages.get() # construct install of igmp query address msg = EOAMPayload(body=CablelabsOUI() / DPoEOpcode_SetRequest() / AddStaticMacAddress(mac='01:00:5e:00:00:01')) # send message log.info('ONU-send-proxied-message') self.adapter_agent.send_proxied_message(device.proxy_address, msg) # wait till we detect incoming message yield self.incoming_messages.get() # by returning we allow the device to be shown as active, which # indirectly verified that message passing works def receive_packet_out(self, logical_device_id, egress_port_no, msg): log.info('packet-out', logical_device_id=logical_device_id, egress_port_no=egress_port_no, msg_len=len(msg)) def start_kpi_collection(self, device_id): """TMP Simulate periodic KPI metric collection from the device""" import random @inlineCallbacks # pretend that we need to do async calls def _collect(device_id, prefix): try: # Step 1: gather metrics from device (pretend it here) - examples uni_port_metrics = yield dict( tx_pkts=random.randint(0, 100), rx_pkts=random.randint(0, 100), tx_bytes=random.randint(0, 100000), rx_bytes=random.randint(0, 100000), ) pon_port_metrics = yield dict( tx_pkts=uni_port_metrics['rx_pkts'], rx_pkts=uni_port_metrics['tx_pkts'], tx_bytes=uni_port_metrics['rx_bytes'], rx_bytes=uni_port_metrics['tx_bytes'], ) onu_metrics = yield dict(cpu_util=20 + 5 * random.random(), buffer_util=10 + 10 * random.random()) # Step 2: prepare the KpiEvent for submission # we can time-stamp them here (or could use time derived from OLT ts = arrow.utcnow().timestamp kpi_event = KpiEvent( type=KpiEventType.slice, ts=ts, prefixes={ # OLT-level prefix: MetricValuePairs(metrics=onu_metrics), # OLT NNI port prefix + '.nni': MetricValuePairs(metrics=uni_port_metrics), # OLT PON port prefix + '.pon': MetricValuePairs(metrics=pon_port_metrics) }) # Step 3: submit self.adapter_agent.submit_kpis(kpi_event) except Exception as e: log.exception('failed-to-submit-kpis', e=e) prefix = 'voltha.{}.{}'.format(self.name, device_id) lc = LoopingCall(_collect, device_id, prefix) lc.start(interval=15) # TODO make this configurable
class TibitOnuAdapter(object): name = 'tibit_onu' supported_device_types = [ DeviceType( id='tibit_onu', adapter=name, accepts_bulk_flow_update=True ) ] def __init__(self, adapter_agent, config): self.adapter_agent = adapter_agent self.config = config self.descriptor = Adapter( id=self.name, vendor='Tibit Communications Inc.', version='0.1', config=AdapterConfig(log_level=LogLevel.INFO) ) self.incoming_messages = DeferredQueue() self.mode = "GPON" def start(self): log.debug('starting') log.info('started') def stop(self): log.debug('stopping') log.info('stopped') def adapter_descriptor(self): return self.descriptor def device_types(self): return DeviceTypes(items=self.supported_device_types) def health(self): return HealthStatus(state=HealthStatus.HealthState.HEALTHY) def change_master_state(self, master): raise NotImplementedError() def update_pm_config(self, device, pm_configs): raise NotImplementedError() def adopt_device(self, device): log.info('adopt-device', device=device) reactor.callLater(0.1, self._onu_device_activation, device) return device def reconcile_device(self, device): raise NotImplementedError() @inlineCallbacks def _onu_device_activation(self, device): # first we verify that we got parent reference and proxy info assert device.parent_id assert device.proxy_address.device_id assert device.proxy_address.channel_id # Device information will be updated later on device.vendor = 'Tibit Communications, Inc.' device.model = '10G GPON ONU' device.connect_status = ConnectStatus.REACHABLE self.adapter_agent.update_device(device) # then shortly after we create some ports for the device uni_port = Port( port_no=2, label='UNI facing Ethernet port', type=Port.ETHERNET_UNI, admin_state=AdminState.ENABLED, oper_status=OperStatus.ACTIVE ) self.adapter_agent.add_port(device.id, uni_port) self.adapter_agent.add_port(device.id, Port( port_no=1, label='PON port', type=Port.PON_ONU, admin_state=AdminState.ENABLED, oper_status=OperStatus.ACTIVE, peers=[ Port.PeerPort( device_id=device.parent_id, port_no=device.parent_port_no ) ] )) # TODO adding vports to the logical device shall be done by agent? # then we create the logical device port that corresponds to the UNI # port of the device # obtain logical device id parent_device = self.adapter_agent.get_device(device.parent_id) logical_device_id = parent_device.parent_id assert logical_device_id # we are going to use the proxy_address.channel_id as unique number # and name for the virtual ports, as this is guaranteed to be unique # in the context of the OLT port, so it is also unique in the context # of the logical device port_no = device.proxy_address.channel_id cap = OFPPF_10GB_FD | OFPPF_FIBER self.adapter_agent.add_logical_port(logical_device_id, LogicalPort( id=str(port_no), ofp_port=ofp_port( port_no=port_no, hw_addr=mac_str_to_tuple(device.mac_address), name='uni-{}'.format(port_no), config=0, state=OFPPS_LIVE, curr=cap, advertised=cap, peer=cap, curr_speed=OFPPF_10GB_FD, max_speed=OFPPF_10GB_FD ), device_id=device.id, device_port_no=uni_port.port_no )) # simulate a proxied message sending and receving a reply reply = yield self._message_exchange(device) # TODO - Need to add validation of reply and decide what to do upon failure # and finally update to "ACTIVE" device = self.adapter_agent.get_device(device.id) device.oper_status = OperStatus.ACTIVE self.adapter_agent.update_device(device) # TODO - Disable Stats Reporting for the moment #self.start_kpi_collection(device.id) def abandon_device(self, device): raise NotImplementedError(0 ) def disable_device(self, device): log.info('disabling', device_id=device.id) # Disable all ports on that device self.adapter_agent.disable_all_ports(device.id) # Update the device operational status to UNKNOWN device.oper_status = OperStatus.UNKNOWN device.connect_status = ConnectStatus.UNREACHABLE self.adapter_agent.update_device(device) # Remove the uni logical port from the OLT, if still present parent_device = self.adapter_agent.get_device(device.parent_id) assert parent_device logical_device_id = parent_device.parent_id assert logical_device_id port_no = device.proxy_address.channel_id # port_id = 'uni-{}'.format(port_no) port_id = '{}'.format(port_no) try: port = self.adapter_agent.get_logical_port(logical_device_id, port_id) self.adapter_agent.delete_logical_port(logical_device_id, port) except KeyError: log.info('logical-port-not-found', device_id=device.id, portid=port_id) # Remove pon port from parent #self.adapter_agent.delete_port_reference_from_parent(device.id, # self.pon_port) # Just updating the port status may be an option as well # port.ofp_port.config = OFPPC_NO_RECV # yield self.adapter_agent.update_logical_port(logical_device_id, # port) # Unregister for proxied message self.adapter_agent.unregister_for_proxied_messages(device.proxy_address) # TODO: # 1) Remove all flows from the device # 2) Remove the device from ponsim log.info('disabled', device_id=device.id) return device def reenable_device(self, device): log.info('re-enabling', device_id=device.id) # First we verify that we got parent reference and proxy info assert device.parent_id assert device.proxy_address.device_id assert device.proxy_address.channel_id # Re-register for proxied messages right away #self.proxy_address = device.proxy_address self.adapter_agent.register_for_proxied_messages(device.proxy_address) # Re-enable the ports on that device self.adapter_agent.enable_all_ports(device.id) # Add the pon port reference to the parent #self.adapter_agent.add_port_reference_to_parent(device.id, # self.pon_port) # Update the connect status to REACHABLE device.connect_status = ConnectStatus.REACHABLE self.adapter_agent.update_device(device) # re-add uni port to logical device parent_device = self.adapter_agent.get_device(device.parent_id) logical_device_id = parent_device.parent_id assert logical_device_id port_no = device.proxy_address.channel_id cap = OFPPF_10GB_FD | OFPPF_FIBER self.adapter_agent.add_logical_port(logical_device_id, LogicalPort( # id='uni-{}'.format(port_no), id= str(port_no), ofp_port=ofp_port( port_no=port_no, hw_addr=mac_str_to_tuple(device.mac_address), name='uni-{}'.format(port_no), config=0, state=OFPPS_LIVE, curr=cap, advertised=cap, peer=cap, curr_speed=OFPPF_10GB_FD, max_speed=OFPPF_10GB_FD ), device_id=device.id, device_port_no=2 )) device = self.adapter_agent.get_device(device.id) device.oper_status = OperStatus.ACTIVE self.adapter_agent.update_device(device) log.info('re-enabled', device_id=device.id) @inlineCallbacks def reboot_device(self, device): log.info('Rebooting ONU: {}'.format(device.mac_address)) # Update the operational status to ACTIVATING and connect status to # UNREACHABLE previous_oper_status = device.oper_status previous_conn_status = device.connect_status device.oper_status = OperStatus.ACTIVATING device.connect_status = ConnectStatus.UNREACHABLE self.adapter_agent.update_device(device) msg = ( EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) / EOAM_DpoeMsg(dpoe_opcode = Dpoe_Opcodes["Set Request"], body=DeviceReset())/ EndOfPDU() ) action = "Device Reset" # send message log.info('ONU-send-proxied-message to {} for ONU: {}'.format(action, device.mac_address)) self.adapter_agent.send_proxied_message(device.proxy_address, msg) rc = [] yield self._handle_set_resp(device, action, rc) # Change the operational status back to its previous state. device.oper_status = previous_oper_status device.connect_status = previous_conn_status self.adapter_agent.update_device(device) log.info('ONU Rebooted: {}'.format(device.mac_address)) def download_image(self, device, request): raise NotImplementedError() def get_image_download_status(self, device, request): raise NotImplementedError() def cancel_image_download(self, device, request): raise NotImplementedError() def activate_image_update(self, device, request): raise NotImplementedError() def revert_image_update(self, device, request): raise NotImplementedError() def self_test_device(self, device): """ This is called to Self a device based on a NBI call. :param device: A Voltha.Device object. :return: Will return result of self test """ log.info('self-test-device', device=device.id) raise NotImplementedError() def delete_device(self, device): log.info('deleting', device_id=device.id) # A delete request may be received when an OLT is disabled # TODO: # 1) Remove all flows from the device # 2) Remove the device from ponsim log.info('deleted', device_id=device.id) def get_device_details(self, device): raise NotImplementedError() @inlineCallbacks def update_flows_bulk(self, device, flows, groups): log.info('########################################') log.info('bulk-flow-update', device_id=device.id, flows=flows, groups=groups) assert len(groups.items) == 0, "Cannot yet deal with groups" # Only do something if there are flows to program if (len(flows.items) > 0): # Clear the existing entries in the Static MAC Address Table yield self._send_clear_static_mac_table(device) # Re-add the IGMP Multicast Address yield self._send_igmp_mcast_addr(device) Clause = {v: k for k, v in ClauseSubtypeEnum.iteritems()} Operator = {v: k for k, v in RuleOperatorEnum.iteritems()} for flow in flows.items: in_port = get_in_port(flow) assert in_port is not None precedence = 255 - min(flow.priority / 256, 255) if in_port == 2: log.info('#### Upstream Rule ####') up_req = UserPortObject() up_req /= PortIngressRuleHeader(precedence=precedence) for field in get_ofb_fields(flow): if field.type == ETH_TYPE: _type = field.eth_type log.info('#### field.type == ETH_TYPE ####',field_type=_type) elif field.type == IP_PROTO: _proto = field.ip_proto log.info('#### field.type == IP_PROTO ####') elif field.type == IN_PORT: _port = field.port log.info('#### field.type == IN_PORT ####', port=_port) elif field.type == VLAN_VID: _vlan_vid = field.vlan_vid & 0xfff log.info('#### field.type == VLAN_VID ####', vlan=_vlan_vid) up_req /= PortIngressRuleClauseMatchLength02(fieldcode=Clause['C-VLAN Tag'], fieldinstance=0, operator=Operator['=='], match=_vlan_vid) elif field.type == VLAN_PCP: _vlan_pcp = field.vlan_pcp log.info('#### field.type == VLAN_PCP ####', pcp=_vlan_pcp) elif field.type == UDP_DST: _udp_dst = field.udp_dst log.info('#### field.type == UDP_DST ####', udp_dst=_udp_dst) elif field.type == IPV4_DST: _ipv4_dst = field.ipv4_dst log.info('#### field.type == IPV4_DST ####', ipv4_dst=_ipv4_dst) elif field.type == METADATA: _metadata = field.table_metadata log.info('#### field.type == METADATA ####', metadata=_metadata) else: log.info('#### field.type == NOT IMPLEMENTED!! ####') raise NotImplementedError('field.type={}'.format( field.type)) for action in get_actions(flow): if action.type == OUTPUT: log.info('#### action.type == OUTPUT ####') up_req /= PortIngressRuleResultInsert(fieldcode=Clause['C-VLAN Tag']) elif action.type == POP_VLAN: log.info('#### action.type == POP_VLAN ####') elif action.type == PUSH_VLAN: log.info('#### action.type == PUSH_VLAN ####') up_req /= PortIngressRuleResultInsert(fieldcode=Clause['C-VLAN Tag']) # if action.push.ethertype != 0x8100: # log.error('unhandled-tpid', # ethertype=action.push.ethertype) elif action.type == SET_FIELD: log.info('#### action.type == SET_FIELD ####') assert (action.set_field.field.oxm_class == ofp.OFPXMC_OPENFLOW_BASIC) field = action.set_field.field.ofb_field if field.type == VLAN_VID: log.info("#### action.field.vlan {} ####".format(field.vlan_vid & 0xfff)) up_req /= PortIngressRuleResultSet( fieldcode=Clause['C-VLAN Tag'], value=field.vlan_vid & 0xfff) else: log.error('unsupported-action-set-field-type', field_type=field.type) else: log.error('UNSUPPORTED-ACTION-TYPE', action_type=action.type) up_req /= PortIngressRuleTerminator() up_req /= AddPortIngressRule() msg = ( EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) / EOAM_DpoeMsg(dpoe_opcode = Dpoe_Opcodes["Set Request"], body=up_req)/ EndOfPDU() ) # send message action = "Set ONU US Rule" log.info('ONU-send-proxied-message to {} for ONU: {}'.format(action, device.mac_address)) self.adapter_agent.send_proxied_message(device.proxy_address, msg) # Get and process the Set Response rc = [] yield self._handle_set_resp(device, action, rc) elif in_port == 1: log.info('#### Downstream Rule ####') Is_MCast = False dn_req = PonPortObject() dn_req /= PortIngressRuleHeader(precedence=precedence) #### Loop through fields again... for field in get_ofb_fields(flow): if field.type == ETH_TYPE: _type = field.eth_type log.info('#### field.type == ETH_TYPE ####', in_port=in_port, match=_type) elif field.type == IP_PROTO: _proto = field.ip_proto log.info('#### field.type == IP_PROTO ####', in_port=in_port, ip_proto=_proto) elif field.type == IN_PORT: _port = field.port log.info('#### field.type == IN_PORT ####') elif field.type == VLAN_VID: _vlan_vid = field.vlan_vid & 0xfff log.info('#### field.type == VLAN_VID ####') elif field.type == VLAN_PCP: _vlan_pcp = field.vlan_pcp log.info('#### field.type == VLAN_PCP ####') elif field.type == UDP_DST: _udp_dst = field.udp_dst log.info('#### field.type == UDP_DST ####') elif field.type == IPV4_DST: _ipv4_dst = field.ipv4_dst log.info('#### field.type == IPV4_DST ####') a = int(hex(_ipv4_dst)[2:4], 16) b = int(hex(_ipv4_dst)[4:6], 16) c = int(hex(_ipv4_dst)[6:8], 16) d = int(hex(_ipv4_dst)[8:], 16) dn_req = AddStaticMacAddress(mac=mcastIp2McastMac('%d.%d.%d.%d' % (a,b,c,d))) Is_MCast = True else: raise NotImplementedError('field.type={}'.format( field.type)) for action in get_actions(flow): if action.type == OUTPUT: log.info('#### action.type == OUTPUT ####') elif action.type == POP_VLAN: log.info('#### action.type == POP_VLAN ####') dn_req /= PortIngressRuleClauseMatchLength02(fieldcode=Clause['C-VLAN Tag'], fieldinstance=0, operator=Operator['=='], match=_vlan_vid) dn_req /= PortIngressRuleResultReplace(fieldcode=Clause['C-VLAN Tag']) dn_req /= PortIngressRuleResultSet( fieldcode=Clause['C-VLAN Tag'], value=field.vlan_vid & 0xfff) elif action.type == PUSH_VLAN: log.info('#### action.type == PUSH_VLAN ####') if action.push.ethertype != 0x8100: log.error('unhandled-ether-type', ethertype=action.push.ethertype) elif action.type == SET_FIELD: log.info('#### action.type == SET_FIELD ####') assert (action.set_field.field.oxm_class == ofp.OFPXMC_OPENFLOW_BASIC) field = action.set_field.field.ofb_field if field.type == VLAN_VID: dn_req /= PortIngressRuleClauseMatchLength02(fieldcode=Clause['C-VLAN Tag'], fieldinstance=0, operator=Operator['=='], match=_vlan_vid) dn_req /= PortIngressRuleResultReplace(fieldcode=Clause['C-VLAN Tag']) dn_req /= PortIngressRuleResultSet( fieldcode=Clause['C-VLAN Tag'], value=field.vlan_vid & 0xfff) else: log.error('unsupported-action-set-field-type', field_type=field.type) else: log.error('UNSUPPORTED-ACTION-TYPE', action_type=action.type) if Is_MCast is True: action = "Set Static IP MCAST address" else: dn_req /= PortIngressRuleTerminator() dn_req /= AddPortIngressRule() action = "Set ONU DS Rule" msg = ( EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) / EOAM_DpoeMsg(dpoe_opcode = Dpoe_Opcodes["Set Request"], body=dn_req)/ EndOfPDU() ) # send message log.info('ONU-send-proxied-message to {} for ONU: {}'.format(action, device.mac_address)) self.adapter_agent.send_proxied_message(device.proxy_address, msg) # Get and process the Set Response rc = [] yield self._handle_set_resp(device, action, rc) else: raise Exception('Port should be 1 or 2 by our convention') log.info('bulk-flow-update finished', device_id=device.id, flows=flows, groups=groups) log.info('########################################') def update_flows_incrementally(self, device, flow_changes, group_changes): raise NotImplementedError() def send_proxied_message(self, proxy_address, msg): raise NotImplementedError() def receive_proxied_message(self, proxy_address, msg): log.info('receive-proxied-message', proxy_address=proxy_address, msg=msg.show(dump=True)) self.incoming_messages.put(msg) def create_interface(self, device, data): raise NotImplementedError() def update_interface(self, device, data): raise NotImplementedError() def remove_interface(self, device, data): raise NotImplementedError() def receive_onu_detect_state(self, device_id, state): raise NotImplementedError() def create_tcont(self, device, tcont_data, traffic_descriptor_data): raise NotImplementedError() def update_tcont(self, device, tcont_data, traffic_descriptor_data): raise NotImplementedError() def remove_tcont(self, device, tcont_data, traffic_descriptor_data): raise NotImplementedError() def create_gemport(self, device, data): raise NotImplementedError() def update_gemport(self, device, data): raise NotImplementedError() def remove_gemport(self, device, data): raise NotImplementedError() def create_multicast_gemport(self, device, data): raise NotImplementedError() def update_multicast_gemport(self, device, data): raise NotImplementedError() def remove_multicast_gemport(self, device, data): raise NotImplementedError() def create_multicast_distribution_set(self, device, data): raise NotImplementedError() def update_multicast_distribution_set(self, device, data): raise NotImplementedError() def remove_multicast_distribution_set(self, device, data): raise NotImplementedError() @inlineCallbacks def _message_exchange(self, device): # register for receiving async messages self.adapter_agent.register_for_proxied_messages(device.proxy_address) # reset incoming message queue while self.incoming_messages.pending: _ = yield self.incoming_messages.get() # send out ping frame to ONU device get device information ping_frame = ( EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) / EOAM_DpoeMsg(dpoe_opcode=Dpoe_Opcodes["Get Request"], body=VendorName() / OnuMode() / HardwareVersion() / ManufacturerInfo() ) / EndOfPDU() ) log.info('ONU-send-proxied-message to Get Version Info for ONU: {}'.format(device.mac_address)) self.adapter_agent.send_proxied_message(device.proxy_address, ping_frame) # Loop until we have a Get Response ack = False while not ack: frame = yield self.incoming_messages.get() respType = get_oam_msg_type(log, frame) if (respType == RxedOamMsgTypeEnum["DPoE Get Response"]): ack = True else: # Handle unexpected events/OMCI messages check_resp(log, frame) if ack: log.info('ONU-response received for Get Version Info for ONU: {}'.format(device.mac_address)) self._process_ping_frame_response(device, frame) if self.mode.upper()[0] == "G": # GPON hw_vers = int(device.hardware_version, 16) if hw_vers >= 0x170618: mcastLidx = 0x04bc elif hw_vers >= 0x170517: mcastLidx = 0x14bc else: mcastLidx = 0x10bc log.info("Using Multicast LIDX {:04X}".format(mcastLidx)) # construct multicast LLID set msg = ( EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) / EOAM_DpoeMsg(dpoe_opcode=Dpoe_Opcodes["Multicast Register"],body=MulticastRegisterSet(MulticastLink=mcastLidx, UnicastLink=0) )) # send message log.info('ONU-send-proxied-message to Multicast Register Set for ONU: {}'.format(device.mac_address)) self.adapter_agent.send_proxied_message(device.proxy_address, msg) # The MulticastRegisterSet does not currently return a response. Just hope it worked. # by returning we allow the device to be shown as active, which # indirectly verified that message passing works def receive_packet_out(self, logical_device_id, egress_port_no, msg): log.info('packet-out', logical_device_id=logical_device_id, egress_port_no=egress_port_no, msg_len=len(msg)) def receive_inter_adapter_message(self, msg): raise NotImplementedError() def suppress_alarm(self, filter): raise NotImplementedError() def unsuppress_alarm(self, filter): raise NotImplementedError() def start_kpi_collection(self, device_id): """TMP Simulate periodic KPI metric collection from the device""" import random @inlineCallbacks # pretend that we need to do async calls def _collect(device_id, prefix): try: # Step 1: gather metrics from device (pretend it here) - examples uni_port_metrics = yield dict( tx_pkts=random.randint(0, 100), rx_pkts=random.randint(0, 100), tx_bytes=random.randint(0, 100000), rx_bytes=random.randint(0, 100000), ) pon_port_metrics = yield dict( tx_pkts=uni_port_metrics['rx_pkts'], rx_pkts=uni_port_metrics['tx_pkts'], tx_bytes=uni_port_metrics['rx_bytes'], rx_bytes=uni_port_metrics['tx_bytes'], ) onu_metrics = yield dict( cpu_util=20 + 5 * random.random(), buffer_util=10 + 10 * random.random() ) # Step 2: prepare the KpiEvent for submission # we can time-stamp them here (or could use time derived from OLT ts = arrow.utcnow().timestamp kpi_event = KpiEvent( type=KpiEventType.slice, ts=ts, prefixes={ # OLT-level prefix: MetricValuePairs(metrics=onu_metrics), # OLT NNI port prefix + '.nni': MetricValuePairs(metrics=uni_port_metrics), # OLT PON port prefix + '.pon': MetricValuePairs(metrics=pon_port_metrics) } ) # Step 3: submit self.adapter_agent.submit_kpis(kpi_event) except Exception as e: log.exception('failed-to-submit-kpis', e=e) prefix = 'voltha.{}.{}'.format(self.name, device_id) lc = LoopingCall(_collect, device_id, prefix) lc.start(interval=15) # TODO make this configurable # Methods for Get / Set Response Processing from eoam_messages @inlineCallbacks def _send_igmp_mcast_addr(self, device): # construct install of igmp query address msg = ( EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) / EOAM_DpoeMsg(dpoe_opcode=Dpoe_Opcodes["Set Request"],body=AddStaticMacAddress(mac='01:00:5e:00:00:01') )) action = "Set Static IGMP MAC address" # send message log.info('ONU-send-proxied-message to {} for ONU: {}'.format(action, device.mac_address)) self.adapter_agent.send_proxied_message(device.proxy_address, msg) rc = [] yield self._handle_set_resp(device, action, rc) @inlineCallbacks def _send_clear_static_mac_table(self, device): # construct install of igmp query address msg = ( EOAMPayload() / EOAM_VendSpecificMsg(oui=CableLabs_OUI) / EOAM_DpoeMsg(dpoe_opcode=Dpoe_Opcodes["Set Request"],body=ClearStaticMacTable() )) action = "Clear Static MAC Table" # send message log.info('ONU-send-proxied-message to {} for ONU: {}'.format(action, device.mac_address)) self.adapter_agent.send_proxied_message(device.proxy_address, msg) rc = [] yield self._handle_set_resp(device, action, rc) @inlineCallbacks def _handle_set_resp(self, device, action, retcode): # Get and process the Set Response ack = False start_time = time.time() # Loop until we have a set response or timeout while not ack: frame = yield self.incoming_messages.get() #TODO - Need to add propoer timeout functionality #if (time.time() - start_time) > TIBIT_MSG_WAIT_TIME or (frame is None): # break # don't wait forever respType = get_oam_msg_type(log, frame) #Check that the message received is a Set Response if (respType == RxedOamMsgTypeEnum["DPoE Set Response"]): ack = True else: log.info('Received Unexpected OAM Message 0x{:X} while waiting for Set Resp for {}'.format(respType,action)) # Handle unexpected events/OMCI messages check_resp(log, frame) # Verify Set Response rc = False if ack: (rc,branch,leaf,status) = check_set_resp(log, frame) if (rc is False): log.info('Set Response had errors - Branch 0x{:X} Leaf 0x{:0>4X} {}'.format(branch, leaf, DPoEVariableResponseCodes[status])) if (rc is True): log.info('ONU-response received for {} for ONU: {}'.format(action, device.mac_address)) else: log.info('BAD ONU-response received for {} for ONU: {}'.format(action, device.mac_address)) retcode.append(rc) def _process_ping_frame_response(self, device, frame): vendor = [0xD7, 0x0011] ponMode = [0xB7, 0x0105] hw_version = [0xD7, 0x0013] manufacturer = [0xD7, 0x0006] branch_leaf_pairs = [vendor, ponMode, hw_version, manufacturer] for pair in branch_leaf_pairs: temp_pair = pair (rc, value) = (get_value_from_msg(log, frame, pair[0], pair[1])) temp_pair.append(rc) temp_pair.append(value) if rc: overall_rc = True else: log.info('Failed to get valid response for Branch 0x{:X} Leaf 0x{:0>4X} '.format(temp_pair[0], temp_pair[1])) ack = True if vendor[rc]: device.vendor = vendor.pop() if device.vendor.endswith(''): device.vendor = device.vendor[:-1] else: device.vendor = "UNKNOWN" # mode: 3 = EPON OLT, 7 = GPON OLT # mode: 2 = EPON ONU, 6 = GPON ONU if ponMode[rc]: value = ponMode.pop() mode = "UNKNOWN" self.mode = "UNKNOWN" if value == 6: mode = "10G GPON ONU" self.mode = "GPON" if value == 2: mode = "10G EPON ONU" self.mode = "EPON" if value == 1: mode = "10G Point to Point" self.mode = "Unsupported" device.model = mode else: device.model = "UNKNOWN" self.mode = "UNKNOWN" log.info("PON Mode is {}".format(self.mode)) if hw_version[rc]: device.hardware_version = hw_version.pop() device.hardware_version = device.hardware_version.replace("FA","") if device.hardware_version.endswith(''): device.hardware_version = device.hardware_version[:-1] else: device.hardware_version = "UNKNOWN" if manufacturer[rc]: manu_value = manufacturer.pop() device.firmware_version = re.search('\Firmware: (.+?) ', manu_value).group(1) image_1 = Image(version = \ re.search('\Build: (.+?) ', manu_value).group(1)) device.images.image.extend([ image_1 ]) device.serial_number = re.search('\Serial #: (.+?) ', manu_value).group(1) else: device.firmware_version = "UNKNOWN" image_1 = Image(version="UNKNOWN") device.images.image.extend([ image_1 ]) device.serial_number = "UNKNOWN" device.connect_status = ConnectStatus.REACHABLE
class EsmeTransceiver(Protocol): bind_pdu = BindTransceiver clock = reactor noisy = True unbind_timeout = 2 OPEN_STATE = 'OPEN' CLOSED_STATE = 'CLOSED' BOUND_STATE_TRX = 'BOUND_TRX' BOUND_STATE_TX = 'BOUND_TX' BOUND_STATE_RX = 'BOUND_RX' BOUND_STATES = set([ BOUND_STATE_RX, BOUND_STATE_TX, BOUND_STATE_TRX, ]) def __init__(self, vumi_transport): """ An SMPP 3.4 client suitable for use by a Vumi Transport. :param SmppTransceiverProtocol vumi_transport: The transport that is using this protocol to communicate with an SMSC. """ self.vumi_transport = vumi_transport self.config = self.vumi_transport.get_static_config() self.buffer = b'' self.state = self.CLOSED_STATE self.deliver_sm_processor = self.vumi_transport.deliver_sm_processor self.dr_processor = self.vumi_transport.dr_processor self.sequence_generator = self.vumi_transport.sequence_generator self.enquire_link_call = LoopingCall(self.enquire_link) self.drop_link_call = None self.idle_timeout = self.config.smpp_enquire_link_interval * 2 self.disconnect_call = self.clock.callLater( self.idle_timeout, self.disconnect, 'Disconnecting, no response from SMSC for longer ' 'than %s seconds' % (self.idle_timeout, )) self.unbind_resp_queue = DeferredQueue() def emit(self, msg): if self.noisy: log.debug(msg) def connectionMade(self): self.state = self.OPEN_STATE log.msg('Connection made, current state: %s' % (self.state, )) @inlineCallbacks def bind(self, system_id, password, system_type, interface_version='34', addr_ton='', addr_npi='', address_range=''): """ Send the `bind_transmitter`, `bind_transceiver` or `bind_receiver` PDU to the SMSC in order to establish the connection. :param str system_id: Identifies the ESME system requesting to bind. :param str password: The password may be used by the SMSC to authenticate the ESME requesting to bind. :param str system_type: Identifies the type of ESME system requesting to bind with the SMSC. :param str interface_version: Indicates the version of the SMPP protocol supported by the ESME. :param str addr_ton: Indicates Type of Number of the ESME address. :param str addr_npi: Numbering Plan Indicator for ESME address. :param str address_range: The ESME address. """ sequence_number = yield self.sequence_generator.next() pdu = self.bind_pdu(sequence_number, system_id=system_id, password=password, system_type=system_type, interface_version=interface_version, addr_ton=addr_ton, addr_npi=addr_npi, address_range=address_range) self.send_pdu(pdu) self.drop_link_call = self.clock.callLater( self.config.smpp_bind_timeout, self.drop_link) def drop_link(self): """ Called if the SMPP connection is not bound within ``smpp_bind_timeout`` amount of seconds """ if self.is_bound(): return return self.disconnect( 'Dropping link due to binding delay. Current state: %s' % (self.state)) def disconnect(self, log_msg=None): """ Forcibly close the connection, logging ``log_msg`` if provided. :param str log_msg: The entry to write to the log file. """ if log_msg is not None: log.warning(log_msg) if not self.connected: return succeed(self.transport.loseConnection()) d = self.unbind() d.addCallback(lambda _: self.unbind_resp_queue.get()) d.addBoth(lambda *a: self.transport.loseConnection()) # Give the SMSC a few seconds to respond with an unbind_resp self.clock.callLater(self.unbind_timeout, d.cancel) return d def connectionLost(self, reason): """ :param Exception reason: The reason for the connection closed, generally a ``ConnectionDone`` """ self.state = self.CLOSED_STATE if self.enquire_link_call.running: self.enquire_link_call.stop() if self.drop_link_call is not None and self.drop_link_call.active(): self.drop_link_call.cancel() if self.disconnect_call.active(): self.disconnect_call.cancel() def is_bound(self): """ Returns ``True`` if the connection is in one of the known values of ``self.BOUND_STATES`` """ return self.state in self.BOUND_STATES @require_bind @inlineCallbacks def enquire_link(self): """ Ping the SMSC to see if they're still around. """ sequence_number = yield self.sequence_generator.next() self.send_pdu(EnquireLink(sequence_number)) returnValue(sequence_number) def send_pdu(self, pdu): """ Send a PDU to the SMSC :param smpp.pdu_builder.PDU pdu: The PDU object to send. """ self.emit('OUTGOING >> %r' % (pdu.get_obj(), )) return self.transport.write(pdu.get_bin()) def dataReceived(self, data): self.buffer += data data = self.handle_buffer() while data is not None: self.on_pdu(unpack_pdu(data)) data = self.handle_buffer() def handle_buffer(self): pdu_found = chop_pdu_stream(self.buffer) if pdu_found is None: return data, self.buffer = pdu_found return data def on_pdu(self, pdu): """ Handle a PDU that was received & decoded. :param dict pdu: The dict result one gets when calling ``smpp.pdu.unpack_pdu()`` on the received PDU """ self.emit('INCOMING << %r' % (pdu, )) handler = getattr(self, 'handle_%s' % (command_id(pdu), ), self.on_unsupported_command_id) return maybeDeferred(handler, pdu) def on_unsupported_command_id(self, pdu): """ Called when an SMPP PDU is received for which no handler function has been defined. :param dict pdu: The dict result one gets when calling ``smpp.pdu.unpack_pdu()`` on the received PDU """ log.warning('Received unsupported SMPP command_id: %r' % (command_id(pdu), )) def handle_bind_transceiver_resp(self, pdu): if not pdu_ok(pdu): log.warning('Unable to bind: %r' % (command_status(pdu), )) self.transport.loseConnection() return self.state = self.BOUND_STATE_TRX return self.on_smpp_bind(seq_no(pdu)) def handle_bind_transmitter_resp(self, pdu): if not pdu_ok(pdu): log.warning('Unable to bind: %r' % (command_status(pdu), )) self.transport.loseConnection() return self.state = self.BOUND_STATE_TX return self.on_smpp_bind(seq_no(pdu)) def handle_bind_receiver_resp(self, pdu): if not pdu_ok(pdu): log.warning('Unable to bind: %r' % (command_status(pdu), )) self.transport.loseConnection() return self.state = self.BOUND_STATE_RX return self.on_smpp_bind(seq_no(pdu)) def on_smpp_bind(self, sequence_number): """Called when the bind has been setup""" self.drop_link_call.cancel() self.enquire_link_call.start(self.config.smpp_enquire_link_interval) def handle_unbind(self, pdu): return self.send_pdu(UnbindResp(seq_no(pdu))) def handle_submit_sm_resp(self, pdu): return self.on_submit_sm_resp(seq_no(pdu), message_id(pdu), command_status(pdu)) def on_submit_sm_resp(self, sequence_number, smpp_message_id, command_status): """ Called when a ``submit_sm_resp`` command was received. :param int sequence_number: The sequence_number of the command, should correlate with the sequence_number of the ``submit_sm`` command that this is a response to. :param str smpp_message_id: The message id that the SMSC is using for this message. This will be referred to in the delivery reports (if any). :param str command_status: The SMPP command_status for this command. Will determine if the ``submit_sm`` command was successful or not. Refer to the SMPP specification for full list of options. """ log.warning('onSubmitSMResp called but not implemented by ESME class.') @inlineCallbacks def handle_deliver_sm(self, pdu): # These operate before the PDUs ``short_message`` or # ``message_payload`` fields have been string decoded. # NOTE: order is important! pdu_handler_chain = [ self.dr_processor.handle_delivery_report_pdu, self.deliver_sm_processor.handle_multipart_pdu, self.deliver_sm_processor.handle_ussd_pdu, ] for handler in pdu_handler_chain: handled = yield handler(pdu) if handled: self.send_pdu( DeliverSMResp(seq_no(pdu), command_status='ESME_ROK')) return # At this point we either have a DR in the message payload # or have a normal SMS that needs to be decoded and handled. content_parts = self.deliver_sm_processor.decode_pdus([pdu]) if not all([isinstance(part, unicode) for part in content_parts]): command_status = self.config.deliver_sm_decoding_error log.msg('Not all parts of the PDU were able to be decoded. ' 'Responding with %s.' % (command_status, ), parts=content_parts) self.send_pdu( DeliverSMResp(seq_no(pdu), command_status=command_status)) return content = u''.join(content_parts) was_cdr = yield self.dr_processor.handle_delivery_report_content( content) if was_cdr: self.send_pdu(DeliverSMResp(seq_no(pdu), command_status='ESME_ROK')) return handled = yield self.deliver_sm_processor.handle_short_message_pdu(pdu) if handled: self.send_pdu(DeliverSMResp(seq_no(pdu), command_status="ESME_ROK")) return command_status = self.config.deliver_sm_decoding_error log.warning('Unable to process message. ' 'Responding with %s.' % (command_status, ), content=content, pdu=pdu.get_obj()) self.send_pdu(DeliverSMResp(seq_no(pdu), command_status=command_status)) def handle_enquire_link(self, pdu): return self.send_pdu(EnquireLinkResp(seq_no(pdu))) def handle_enquire_link_resp(self, pdu): self.disconnect_call.reset(self.idle_timeout) @require_bind @inlineCallbacks def submit_sm(self, vumi_message_id, destination_addr, source_addr='', esm_class=0, protocol_id=0, priority_flag=0, schedule_delivery_time='', validity_period='', replace_if_present=0, data_coding=0, sm_default_msg_id=0, sm_length=0, short_message='', optional_parameters=None, **configured_parameters): """ Put a `submit_sm` command on the wire. :param str source_addr: Address of SME which originated this message. If unknown leave blank. :param str destination_addr: Destination address of this short message. For mobile terminated messages, this is the directory number of the recipient MS. :param str service_type: The service_type parameter can be used to indicate the SMS Application service associated with the message. If unknown leave blank. :param int source_addr_ton: Type of Number for source address. :param int source_addr_npi: Numbering Plan Indicator for source address. :param int dest_addr_ton: Type of Number for destination. :param int dest_addr_npi: Numbering Plan Indicator for destination. :param int esm_class: Indicates Message Mode & Message Type. :param int protocol_id: Protocol Identifier. Network specific field. :param int priority_flag: Designates the priority level of the message. :param str schedule_delivery_time: The short message is to be scheduled by the SMSC for delivery. Leave blank for immediate delivery. :param str validity_period: The validity period of this message. Leave blank for SMSC default. :param int registered_delivery: Indicator to signify if an SMSC delivery receipt or an SME acknowledgement is required. :param int replace_if_present: Flag indicating if submitted message should replace an existing message. :param int data_coding: Defines the encoding scheme of the short message user data. :param int sm_default_msg_id: Indicates the short message to send from a list of pre- defined ('canned') short messages stored on the SMSC. Leave blank if not using an SMSC canned message. :param int sm_length: Length in octets of the short_message user data. This is automatically calculated and set during PDU encoding, no need to specify. :param int short_message: Up to 254 octets of short message user data. The exact physical limit for short_message size may vary according to the underlying network. Applications which need to send messages longer than 254 octets should use the message_payload parameter. In this case the sm_length field should be set to zero. :param dict optional_parameters: keys and values to be embedded in the PDU as tag-length-values. Refer to the SMPP specification and your SMSCs instructions on what valid and suitable keys and values are. :returns: list of 1 sequence number (int) for consistency with other submit_sm calls. :rtype: list """ configured_param_values = { 'service_type': self.config.service_type, 'source_addr_ton': self.config.source_addr_ton, 'source_addr_npi': self.config.source_addr_npi, 'dest_addr_ton': self.config.dest_addr_ton, 'dest_addr_npi': self.config.dest_addr_npi, 'registered_delivery': self.config.registered_delivery, } configured_param_values.update(configured_parameters) sequence_number = yield self.sequence_generator.next() pdu = SubmitSM(sequence_number=sequence_number, source_addr=source_addr, destination_addr=destination_addr, esm_class=esm_class, protocol_id=protocol_id, priority_flag=priority_flag, schedule_delivery_time=schedule_delivery_time, validity_period=validity_period, replace_if_present=replace_if_present, data_coding=data_coding, sm_default_msg_id=sm_default_msg_id, sm_length=sm_length, short_message=short_message, **configured_param_values) if optional_parameters: for key, value in optional_parameters.items(): pdu.add_optional_parameter(key, value) yield self.vumi_transport.message_stash.set_sequence_number_message_id( sequence_number, vumi_message_id) self.send_pdu(pdu) returnValue([sequence_number]) def submit_sm_long(self, vumi_message_id, destination_addr, long_message, **pdu_params): """ Send a `submit_sm` command with the message encoded in the ``message_payload`` optional parameter. Same parameters apply as for ``submit_sm`` with the exception that the ``short_message`` keyword argument is disallowed because it conflicts with the ``long_message`` field. :returns: list of 1 sequence number, int. :rtype: list """ if 'short_message' in pdu_params: raise EsmeProtocolError( 'short_message not allowed when sending a long message' 'in the message_payload') optional_parameters = pdu_params.pop('optional_parameters', {}).copy() optional_parameters.update({ 'message_payload': (''.join('%02x' % ord(c) for c in long_message)) }) return self.submit_sm(vumi_message_id, destination_addr, short_message='', sm_length=0, optional_parameters=optional_parameters, **pdu_params) def _fits_in_one_message(self, message): if len(message) <= GSM_MAX_SMS_BYTES: return True # NOTE: We already have byte strings here, so we assume that printable # ASCII characters are all the same as single-width GSM 03.38 # characters. if len(message) <= GSM_MAX_SMS_7BIT_CHARS: # TODO: We need better character handling and counting stuff. return all(0x20 <= ord(ch) <= 0x7f for ch in message) return False def csm_split_message(self, message): """ Chop the message into 130 byte chunks to leave 10 bytes for the user data header the SMSC is presumably going to add for us. This is a guess based mostly on optimism and the hope that we'll never have to deal with this stuff in production. NOTE: If we have utf-8 encoded data, we might break in the middle of a multibyte character. This should be ok since the message is only decoded after re-assembly of all individual segments. :param str message: The message to split :returns: list of strings :rtype: list """ if self._fits_in_one_message(message): return [message] payload_length = GSM_MAX_SMS_BYTES - 10 split_msg = [] while message: split_msg.append(message[:payload_length]) message = message[payload_length:] return split_msg @inlineCallbacks def submit_csm_sar(self, vumi_message_id, destination_addr, **pdu_params): """ Submit a concatenated SMS to the SMSC using the optional SAR parameter names in the various PDUS. :returns: List of sequence numbers (int) for each of the segments. :rtype: list """ split_msg = self.csm_split_message(pdu_params.pop('short_message')) if len(split_msg) == 1: # There is only one part, so send it without SAR stuff. sequence_numbers = yield self.submit_sm(vumi_message_id, destination_addr, short_message=split_msg[0], **pdu_params) returnValue(sequence_numbers) optional_parameters = pdu_params.pop('optional_parameters', {}).copy() ref_num = yield self.sequence_generator.next() sequence_numbers = [] yield self.vumi_transport.message_stash.init_multipart_info( vumi_message_id, len(split_msg)) for i, msg in enumerate(split_msg): pdu_params = pdu_params.copy() optional_parameters.update({ # Reference number must be between 00 & FFFF 'sar_msg_ref_num': (ref_num % 0xFFFF), 'sar_total_segments': len(split_msg), 'sar_segment_seqnum': i + 1, }) sequence_number = yield self.submit_sm( vumi_message_id, destination_addr, short_message=msg, optional_parameters=optional_parameters, **pdu_params) sequence_numbers.extend(sequence_number) returnValue(sequence_numbers) @inlineCallbacks def submit_csm_udh(self, vumi_message_id, destination_addr, **pdu_params): """ Submit a concatenated SMS to the SMSC using user data headers (UDH) in the message content. Same parameters apply as for ``submit_sm`` with the exception that the ``esm_class`` keyword argument is disallowed because the SMPP spec mandates a value that is to be set for UDH. :returns: List of sequence numbers (int) for each of the segments. :rtype: list """ if 'esm_class' in pdu_params: raise EsmeProtocolError( 'Cannot specify esm_class, GSM spec sets this at 0x40 ' 'for concatenated messages using UDH.') pdu_params = pdu_params.copy() split_msg = self.csm_split_message(pdu_params.pop('short_message')) if len(split_msg) == 1: # There is only one part, so send it without UDH stuff. sequence_numbers = yield self.submit_sm(vumi_message_id, destination_addr, short_message=split_msg[0], **pdu_params) returnValue(sequence_numbers) ref_num = yield self.sequence_generator.next() sequence_numbers = [] yield self.vumi_transport.message_stash.init_multipart_info( vumi_message_id, len(split_msg)) for i, msg in enumerate(split_msg): # 0x40 is the UDHI flag indicating that this payload contains a # user data header. # NOTE: Looking at the SMPP specs I can find no requirement # for this anywhere. pdu_params['esm_class'] = 0x40 # See http://en.wikipedia.org/wiki/User_Data_Header and # http://en.wikipedia.org/wiki/Concatenated_SMS for an # explanation of the magic numbers below. We should probably # abstract this out into a class that makes it less magic and # opaque. udh = ''.join([ '\05', # Full UDH header length '\00', # Information Element Identifier for Concatenated SMS '\03', # header length # Reference number must be between 00 & FF chr(ref_num % 0xFF), chr(len(split_msg)), chr(i + 1), ]) short_message = udh + msg sequence_number = yield self.submit_sm(vumi_message_id, destination_addr, short_message=short_message, **pdu_params) sequence_numbers.extend(sequence_number) returnValue(sequence_numbers) @require_bind @inlineCallbacks def query_sm(self, message_id, source_addr_ton=0, source_addr_npi=0, source_addr=''): """ Query the SMSC for the status of an earlier sent message. :param str message_id: Message ID of the message whose state is to be queried. This must be the SMSC assigned Message ID allocated to the original short message when submitted to the SMSC by the submit_sm, data_sm or submit_multi command, and returned in the response PDU by the SMSC. :param int source_addr_ton: Type of Number of message originator. This is used for verification purposes, and must match that supplied in the original request PDU (e.g. submit_sm). :param int source_addr_npi: Numbering Plan Identity of message originator. This is used for verification purposes, and must match that supplied in the original request PDU (e.g. submit_sm). :param str source_addr: Address of message originator. This is used for verification purposes, and must match that supplied in the original request PDU (e.g. submit_sm). """ sequence_number = yield self.sequence_generator.next() pdu = QuerySM(sequence_number=sequence_number, message_id=message_id, source_addr=source_addr, source_addr_npi=source_addr_npi, source_addr_ton=source_addr_ton) self.send_pdu(pdu) returnValue([sequence_number]) @inlineCallbacks def unbind(self): sequence_number = yield self.sequence_generator.next() self.send_pdu(Unbind(sequence_number)) returnValue([sequence_number]) def handle_unbind_resp(self, pdu): self.unbind_resp_queue.put(pdu)
class AirtelBfTransportTestCase(TransportTestCase): transport_name = 'airtel' transport_type = 'sms' transport_class = AirtelBfHttpTransport send_path = '/sendsms/index' send_port = 9999 @inlineCallbacks def setUp(self): yield super(AirtelBfTransportTestCase, self).setUp() self.airtel_sms_calls = DeferredQueue() self.mock_airtel_sms = MockHttpServer(self.handle_request) yield self.mock_airtel_sms.start() self.config = { 'transport_name': self.transport_name, 'receive_path': 'sendsms', 'receive_port': 9998, 'outbound_url': self.mock_airtel_sms.url, 'default_shortcode': '3411', 'login': '******', 'password': '******' } self.transport = yield self.get_transport(self.config) self.transport_url = self.transport.get_transport_url() @inlineCallbacks def tearDown(self): yield self.mock_airtel_sms.stop() yield super(AirtelBfTransportTestCase, self).tearDown() def handle_request(self, request): self.airtel_sms_calls.put(request) return '' @inlineCallbacks def test_sending_sms(self): yield self.dispatch(self.mkmsg_out()) req = yield self.airtel_sms_calls.get() self.assertEqual(req.path, '/') self.assertEqual(req.method, 'GET') self.assertEqual( { 'DA': ['9292'], 'SOA': ['+41791234567'], 'content': ['hello world'], 'u': ['texttochange'], 'p': ['password'] }, req.args) [smsg] = self.get_dispatched_events() self.assertEqual( self.mkmsg_ack(user_message_id='1', sent_message_id='1'), smsg) @inlineCallbacks def test_sending_sms_complex(self): yield self.dispatch( self.mkmsg_out( from_addr="+2261", content= u'setoffre #A#BELG=10/tete+300000/pu# envoy\xe9 depuis SIMAgriMobile' )) req = yield self.airtel_sms_calls.get() self.assertEqual(req.path, '/') self.assertEqual(req.method, 'GET') self.assertEqual( 'setoffre #A#BELG=10/tete+300000/pu# envoyé depuis SIMAgriMobile', req.args['content'][0]) [smsg] = self.get_dispatched_events() self.assertEqual( self.mkmsg_ack(user_message_id='1', sent_message_id='1'), smsg) def mkurl_raw(self, **params): return '%s%s?%s' % (self.transport_url, self.config['receive_path'], urlencode(params)) def mkurl(self, content, from_addr, **kw): params = {'message': content, 'msisdn': from_addr} params.update(kw) return self.mkurl_raw(**params) @inlineCallbacks def test_receiving_sms(self): url = self.mkurl('Hello envoy\xc3\xa9', '+2261') response = yield http_request_full(url, method='GET') [smsg] = self.get_dispatched_messages() self.assertEqual(response.code, http.OK) self.assertEqual(u'Hello envoy\xe9', smsg['content']) self.assertEqual('3411', smsg['to_addr']) self.assertEqual('+2261', smsg['from_addr']) @inlineCallbacks def test_receiving_sms_accent(self): url = self.mkurl('Hello envoyé', '+2261') response = yield http_request_full(url, method='GET') [smsg] = self.get_dispatched_messages() self.assertEqual(response.code, http.OK) self.assertEqual(u'Hello envoy\xe9', smsg['content']) self.assertEqual('3411', smsg['to_addr']) self.assertEqual('+2261', smsg['from_addr']) @inlineCallbacks def test_receiving_sms_fail(self): params = { 'message': 'Hello', 'to_addr': '+2261', } url = self.mkurl_raw(**params) response = yield http_request_full(url, method='GET') self.assertEqual(0, len(self.get_dispatched_messages())) self.assertEqual(response.code, http.INTERNAL_SERVER_ERROR)
class MarathonSyncTest(TestCase): def setUp(self): self.marathon_sync = MarathonSync('http://1.2.3.4:8080', []) # We use this to mock requests going to Marathon self.requests = DeferredQueue() def mock_requests(method, url, headers, data): d = Deferred() self.requests.put({ 'method': method, 'url': url, 'data': data, 'deferred': d, }) return d self.patch(self.marathon_sync, 'requester', mock_requests) def tearDown(self): pass @inlineCallbacks def test_delete_unknown_apps(self): self.marathon_sync.groups = [{ 'id': 'test1', 'apps': [{ 'id': 'app1' }, { 'id': 'app2' }] }, { 'id': 'test2', 'apps': [{ 'id': 'app1' }] }] d = self.marathon_sync.delete_unknown_apps() d.addErrback(log.err) get_request = yield self.requests.get() self.assertEqual(get_request['url'], 'http://1.2.3.4:8080/v2/apps') self.assertEqual(get_request['method'], 'GET') # Return 2 out of 3 of the known apps plus 1 unknown app get_request['deferred'].callback( FakeResponse( 200, [], json.dumps({ 'apps': [{ 'id': '/test1/app1' }, { 'id': '/test2/app1' }, { 'id': '/app2' }] }))) delete_request = yield self.requests.get() self.assertEqual(delete_request['url'], 'http://1.2.3.4:8080/v2/apps/app2') self.assertEqual(delete_request['method'], 'DELETE') delete_request['deferred'].callback( FakeResponse( 200, [], json.dumps({ "version": "2015-08-25T10:06:19.918Z", "deploymentId": "7828f718-ef25-426b-9d80-82ad3b39c8ae" }))) yield d @inlineCallbacks def test_sync_apps(self): self.marathon_sync.groups = [{ 'id': 'test1', 'apps': [{ 'id': 'app1' }, { 'id': 'app2' }] }, { 'id': 'test2', 'apps': [{ 'id': 'app1' }] }] d = self.marathon_sync.sync_apps() d.addErrback(log.err) request1 = yield self.requests.get() self.assertEqual(request1['url'], 'http://1.2.3.4:8080/v2/groups') self.assertEqual(request1['method'], 'PUT') self.assertEqual( request1['data'], json.dumps({ 'id': 'test1', 'apps': [{ 'id': 'app1' }, { 'id': 'app2' }] })) request1['deferred'].callback( FakeResponse( 200, [], json.dumps({ "version": "2015-08-25T08:36:47.314Z", "deploymentId": "54ba4426-433d-4d38-b97e-5b9e3ac9d027" }))) request2 = yield self.requests.get() self.assertEqual(request2['url'], 'http://1.2.3.4:8080/v2/groups') self.assertEqual(request2['method'], 'PUT') self.assertEqual(request2['data'], json.dumps({ 'id': 'test2', 'apps': [{ 'id': 'app1' }] })) request2['deferred'].callback( FakeResponse( 200, [], json.dumps({ "version": "2015-08-25T09:50:54.340Z", "deploymentId": "54393ec8-e93e-4132-abe6-e78e4545a965" }))) yield d
class TestMxitTransport(VumiTestCase): @inlineCallbacks def setUp(self): self.mock_http = MockHttpServer(self.handle_request) self.mock_request_queue = DeferredQueue() yield self.mock_http.start() self.addCleanup(self.mock_http.stop) config = { 'web_port': 0, 'web_path': '/api/v1/mxit/mobiportal/', 'client_id': 'client_id', 'client_secret': 'client_secret', 'api_send_url': self.mock_http.url, 'api_auth_url': self.mock_http.url, } self.sample_loc_str = 'cc,cn,sc,sn,cc,c,noi,cfb,ci' self.sample_profile_str = 'lc,cc,dob,gender,tariff' self.sample_html_str = '<&>' self.sample_req_headers = { 'X-Device-User-Agent': 'ua', 'X-Mxit-Contact': 'contact', 'X-Mxit-USERID-R': 'user-id', 'X-Mxit-Nick': 'nick', 'X-Mxit-Location': self.sample_loc_str, 'X-Mxit-Profile': self.sample_profile_str, 'X-Mxit-User-Input': self.sample_html_str, } self.sample_menu_resp = "\n".join([ "Hello!", "1. option 1", "2. option 2", "3. option 3", ]) # same as above but the o's are replaced with # http://www.fileformat.info/info/unicode/char/f8/index.htm slashed_o = '\xc3\xb8' self.sample_unicode_menu_resp = unicode( self.sample_menu_resp.replace('o', slashed_o), 'utf-8') self.tx_helper = self.add_helper(TransportHelper(MxitTransport)) self.transport = yield self.tx_helper.get_transport(config) # NOTE: priming redis with an access token self.transport.redis.set(self.transport.access_token_key, 'foo') self.url = self.transport.get_transport_url(config['web_path']) def handle_request(self, request): self.mock_request_queue.put(request) return NOT_DONE_YET def test_is_mxit_request(self): req = Request(None, True) self.assertFalse(self.transport.is_mxit_request(req)) req.requestHeaders.addRawHeader('X-Mxit-Contact', 'foo') self.assertTrue(self.transport.is_mxit_request(req)) def test_noop(self): self.assertEqual(self.transport.noop('foo'), 'foo') def test_parse_location(self): self.assertEqual(self.transport.parse_location(self.sample_loc_str), { 'country_code': 'cc', 'country_name': 'cn', 'subdivision_code': 'sc', 'subdivision_name': 'sn', 'city_code': 'cc', 'city': 'c', 'network_operator_id': 'noi', 'client_features_bitset': 'cfb', 'cell_id': 'ci', }) def test_parse_profile(self): self.assertEqual( self.transport.parse_profile(self.sample_profile_str), { 'country_code': 'cc', 'date_of_birth': 'dob', 'gender': 'gender', 'language_code': 'lc', 'tariff_plan': 'tariff', }) def test_html_decode(self): self.assertEqual( self.transport.html_decode(self.sample_html_str), '<&>') def test_get_request_data(self): req = Request(None, True) headers = req.requestHeaders for key, value in self.sample_req_headers.items(): headers.addRawHeader(key, value) data = self.transport.get_request_data(req) self.assertEqual(data, { 'X-Device-User-Agent': 'ua', 'X-Mxit-Contact': 'contact', 'X-Mxit-Location': { 'cell_id': 'ci', 'city': 'c', 'city_code': 'cc', 'client_features_bitset': 'cfb', 'country_code': 'cc', 'country_name': 'cn', 'network_operator_id': 'noi', 'subdivision_code': 'sc', 'subdivision_name': 'sn', }, 'X-Mxit-Nick': 'nick', 'X-Mxit-Profile': { 'country_code': 'cc', 'date_of_birth': 'dob', 'gender': 'gender', 'language_code': 'lc', 'tariff_plan': 'tariff', }, 'X-Mxit-USERID-R': 'user-id', 'X-Mxit-User-Input': u'<&>', }) def test_get_request_content_from_header(self): req = Request(None, True) req.requestHeaders.addRawHeader('X-Mxit-User-Input', 'foo') self.assertEqual(self.transport.get_request_content(req), 'foo') def test_get_quote_plus_request_content_from_header(self): req = Request(None, True) req.requestHeaders.addRawHeader('X-Mxit-User-Input', 'foo+bar') self.assertEqual( self.transport.get_request_content(req), 'foo bar') def test_get_quoted_request_content_from_header(self): req = Request(None, True) req.requestHeaders.addRawHeader('X-Mxit-User-Input', 'foo%20bar') self.assertEqual( self.transport.get_request_content(req), 'foo bar') def test_get_request_content_from_args(self): req = Request(None, True) req.args = {'input': ['bar']} self.assertEqual(self.transport.get_request_content(req), 'bar') def test_get_request_content_when_missing(self): req = Request(None, True) self.assertEqual(self.transport.get_request_content(req), None) @inlineCallbacks def test_invalid_request(self): resp = yield http_request_full(self.url) self.assertEqual(resp.code, BAD_REQUEST) @inlineCallbacks def test_request(self): resp_d = http_request_full( self.url, headers=self.sample_req_headers) [msg] = yield self.tx_helper.wait_for_dispatched_inbound(1) self.tx_helper.make_dispatch_reply(msg, self.sample_menu_resp) resp = yield resp_d self.assertTrue('1. option 1' in resp.delivered_body) self.assertTrue('2. option 2' in resp.delivered_body) self.assertTrue('3. option 3' in resp.delivered_body) self.assertTrue('?input=1' in resp.delivered_body) self.assertTrue('?input=2' in resp.delivered_body) self.assertTrue('?input=3' in resp.delivered_body) def test_response_parser(self): header, items = ResponseParser.parse(self.sample_menu_resp) self.assertEqual(header, 'Hello!') self.assertEqual(items, [ ('1', 'option 1'), ('2', 'option 2'), ('3', 'option 3'), ]) header, items = ResponseParser.parse('foo!') self.assertEqual(header, 'foo!') self.assertEqual(items, []) @inlineCallbacks def test_unicode_rendering(self): resp_d = http_request_full( self.url, headers=self.sample_req_headers) [msg] = yield self.tx_helper.wait_for_dispatched_inbound(1) self.tx_helper.make_dispatch_reply(msg, self.sample_unicode_menu_resp) resp = yield resp_d self.assertTrue( 'Hell\xc3\xb8' in resp.delivered_body) self.assertTrue( '\xc3\xb8pti\xc3\xb8n 1' in resp.delivered_body) @inlineCallbacks def test_outbound_that_is_not_a_reply(self): d = self.tx_helper.make_dispatch_outbound( content="Send!", to_addr="mxit-1", from_addr="mxit-2") req = yield self.mock_request_queue.get() body = json.load(req.content) self.assertEqual(body, { 'Body': 'Send!', 'To': 'mxit-1', 'From': 'mxit-2', 'ContainsMarkup': 'true', 'Spool': 'true', }) [auth] = req.requestHeaders.getRawHeaders('Authorization') # primed access token self.assertEqual(auth, 'Bearer foo') req.finish() yield d @inlineCallbacks def test_getting_access_token(self): transport = self.transport redis = transport.redis # clear primed value yield redis.delete(transport.access_token_key) d = transport.get_access_token() req = yield self.mock_request_queue.get() [auth] = req.requestHeaders.getRawHeaders('Authorization') self.assertEqual( auth, 'Basic %s' % ( base64.b64encode('client_id:client_secret'))) self.assertEqual( 'scope=message%2Fsend&grant_type=client_credentials', req.content.read()) req.write(json.dumps({ 'access_token': 'access_token', 'expires_in': '10' })) req.finish() access_token = yield d self.assertEqual(access_token, 'access_token') self.assertFalse(isinstance(access_token, unicode)) ttl = yield redis.ttl(transport.access_token_key) self.assertTrue( 0 < ttl <= (transport.access_token_auto_decay * 10))
class SpoofTcpProxyProtocol(Protocol, object): """This is a special kind of TCP proxy where the connection is initially routed to a spoof server, and then when identifying information is sent from the client, the connection is made to the right server and messages are relayed. This assumes that the spoof server and the real server gives the exact same response up to the time the real server is connected to. """ def __init__(self, spoof_hostname, spoof_port): """Create a new spoof TCP proxy. :param str spoof_hostname: the hostname of the spoof server :param int spoof_port: the port of the spoof server """ # how many bytes have been sent by the spoof server? self.spoof_messages_length = 0 # for the actual server connection self.server_queue = DeferredQueue() self.client_queue = DeferredQueue() # for the spoofed connection self.spoof_client_queue = DeferredQueue() self.spoof_server_queue = DeferredQueue() self._connectServer(spoof_hostname, spoof_port, self.spoof_server_queue, self.spoof_client_queue) # add callbacks self.server_queue.get().addCallback(self.serverQueueCallback) self.spoof_server_queue.get().addCallback( self.spoofServerQueueCallback) def serverQueueCallback(self, data): """A callback for `self.server_queue` This only starts sending data after :code:`spoof_messages_length` has gone to 0. If the incoming data is longer than :code:`spoof_messages_length`, then that many bytes is truncated from the beginning and the rest is sent. :param str data: data from server queue """ if data is False: self.transport.loseConnection() return assert self.spoof_messages_length >= 0 if self.spoof_messages_length == 0: self.transport.write(data) else: if self.spoof_messages_length < len(data): data = data[self.spoof_messages_length:] self.spoof_messages_length = 0 self.transport.write(data) else: self.spoof_messages_length -= len(data) self.server_queue.get().addCallback(self.serverQueueCallback) def spoofServerQueueCallback(self, data): """A callback for `self.spoof_server_queue` :param str data: data from server queue """ if self.spoof_server_queue is not None: if data is False: self.transport.loseConnection() return self.spoof_messages_length += len(data) self.transport.write(data) self.spoof_server_queue.get().addCallback( self.spoofServerQueueCallback) def _connectServer(self, hostname, port, server_queue, client_queue): """A helper function for connecting to (hostname, port) with the given server and client queues. :param str hostname: :param int port: :param DeferredQueue server_queue: :param DeferredQueue client_queue: """ endpoint = TCP4ClientEndpoint(reactor, hostname, port) protocol = ServerProtocol(server_queue, client_queue) connectProtocol(endpoint, protocol) def connectServer(self, hostname, port): """Tell the proxy what the end server is and start the connection. This closes the connection to the spoofed server. :param str hostname: :param int port: :param DeferredQueue server_queue: :param DeferredQueue client_queue: """ # close connection spoof_client_queue = self.spoof_client_queue # setting to None first prevents a race condition self.spoof_client_queue = None self.spoof_server_queue = None spoof_client_queue.put(False) self._connectServer(hostname, port, self.server_queue, self.client_queue) def dataReceived(self, data): """Received data from client, put into client queue """ self.client_queue.put(data) if self.spoof_client_queue is not None: self.spoof_client_queue.put(data) def connectionLost(self, why): """Client closed connection, or some other issue. close connection to server """ # TODO pretty sure this only allows client to close connection, not the # other way around self.client_queue.put(False) if self.spoof_client_queue is not None: self.spoof_client_queue.put(False)
class Server(Protocol,QObject): # Initialize PyQt Signals # For some godforsaken reason, these are necessarily declared within # the class scope but outside the __init__() function. It doesn't # work otherwise. chatSignal = pyqtSignal(QString) userUpdateSignal = pyqtSignal(QString) challengeSignal = pyqtSignal(QString) reidentifySignal = pyqtSignal() def __init__(self): # Make sure Protocol.__init__() and QObject.__init__() get called (double inheritance) super(Server, self).__init__() # initialize vars self.username = '' self.opponent = '' self.inGame = False self.challenger = '' # initialize queue self.queue = DeferredQueue() # initialize the incoming data queue self.startQueuing() self.quitting = False # to prevent notifying the user of disconnection when they exit # Called by reactor upon connection def connectionMade(self): self.startGUI() # Starts the PyQt GUI which will handle user input and ouptut def startGUI(self): thread = threading.Thread(target = run_gui, args = (self,)) thread.daemon = True # Thread will die when calling process dies thread.start() # Called by reactor when TCP packet received over this connection def dataReceived(self, data): self.queue.put(data) # Continuously grabs data from the def queueData(self,data): data = data.rstrip() self.handleData(data) self.queue.get().addCallback(self.queueData) # Kicks off the queueData loop def startQueuing(self): self.queue.get().addCallback(self.queueData) # Handle data sent by the server def handleData(self, msg): # Server messages come colon-delineated: # <element>:<element>:<element>:<element> data = msg.split(':') cmmd = data[0] if (cmmd == 'opponent'): # Server is telling you to start a game with another user. # This will come in the form of: # "opponent:<username>:<1|2|...|n>" # where the last argument indicates the order of turn self.inGame = True self.opponent = data[1] self.challenger = data[1] # Update "Available" list to all users self.transport.write("refresh:null") # Start the game self.initializeGame() # tell the game who's turn it is turn = data[2] if turn == "1": reactor.gs.Turn.My_Turn() else: reactor.gs.Turn.Opponents_Turn() elif (cmmd == 'reject'): # Someone has rejected your game challenge: # reject:null self.inGame = False # Notify user in GUI chat box self.chatSignal.emit("Game rejected") elif (cmmd == 'reidentify'): self.reidentifySignal.emit() # This signal picked up by GUI elif (cmmd == 'challenge'): # Challenge receieved from another user # challenge:<fromUserID> if self.inGame == False: self.challenger = data[1] # Notify user in GUI chat box self.chatSignal.emit("SERVER MESSAGE: You have received a challenge from " + data[1]) # Prompt response from user via GUI self.challengeSignal.emit(data[1]) elif (cmmd == 'opponentMove'): # Opponent has made a move; pass to game: # opponentMove:<moveID> moveID = data[1] reactor.gs.opponentMove(moveID) elif (cmmd == 'forfeit'): # Opponent has forfeited the game # forfeit:null # Notify the user via GUI chat box self.chatSignal.emit("SERVER MESSAGE: " + self.challenger + " forfeited.") self.inGame = False # Quietly exit the game (we already know the result - game shouldn't announce it) reactor.gs.quietQuit() # Since the 'forfeit' command from the other user triggered the server to set # both users (self and opponent) to available again, we need to send a global # refresh to reflect these changes self.transport.write("refresh:null") elif (cmmd == 'winner'): # Opponent has notified you that you have won your game self.chatSignal.emit("You won against " + self.challenger + "!") self.inGame = False # Tell the server that both you and your opponent are now available self.transport.write("available:" + self.username + ":" + self.challenger) # Quietly exit the game (we already know the result - game shouldn't announce it) reactor.gs.quietQuit() elif (cmmd == 'loser'): # Opponent has notified you that you have lost your game self.chatSignal.emit("You lost against " + self.challenger + ".") self.inGame = False # Tell the server that both you and your opponent are now available self.transport.write("available:" + self.username + ":" + self.challenger) # Quietly exit the game (we already know the result - game shouldn't announce it) reactor.gs.quietQuit() elif (cmmd == 'tied'): # Opponent has notified you that you have tied your game self.chatSignal.emit("You tied " + self.challenger + ".") self.inGame = False # Tell the server that both you and your opponent are now available self.transport.write("available:" + self.username + ":" + self.challenger) # Quietly exit the game (we already know the result - game shouldn't announce it) reactor.gs.quietQuit() elif (cmmd == 'users' and data[2] == 'available'): # The server is sending a list of Online and Available users: # users:<u1+u2+u3+u4>:available:<u1+u2> # Note: user lists are '+' delineated userList = data[1] availableList = data[3] # Simply pass these lists (no parsing yet) to the GUI, # delineated with a colon between the two self.userUpdateSignal.emit(userList + ":" + availableList) elif (cmmd == 'msg'): # This is a general server message to the user: # msg:<msg_string> # Simply pass it along to the GUI for display in the chat box, # after we prepend it with a special server stamp. msg = "SERVER MESSAGE: " + data[1] self.chatSignal.emit(msg) elif (cmmd == 'chat'): # Incoming chat message from another user (via the server) # Simply pass it to the GUI via the chatSignal msg = data[1] self.chatSignal.emit(msg) else: # data was receieved but was unable to be interpreted print "Received unrecognized message from server: " + msg # This function will be called by the PyGame instance, # and will notify the server of a game move like so: # move:<opponent_username>:<move_id> def sendMove(self,moveID): self.transport.write("move:" + self.opponent + ":" + moveID) # Start the pygame instance def initializeGame(self): # Initialize the GameSpace. # Reactor needs to be passed to it so it can issue server calls. reactor.gs = GameSpace(reactor) # GameSpace also needs this protocol to issue server calls. reactor.gs.protocol = self # Start game loop self.lc = LoopingCall(reactor.gs.loop) self.lc.start(1/60) # Called by a PyGame instance (into which we had passed a # reference to this protocol.) def gameEnded(self,msg): self.inGame = False msg = msg.rstrip() # remove any pesky trailing whitespace # Notify server (and, by extension, the opponent) that the game has ended, and how it has # Note: 'msg' will be either "forfeit", "won", "lost", or "tied" self.transport.write(msg + ":" + self.challenger) # Write end game condition to the GUI chatbox if msg == "forfeit": self.chatSignal.emit("You forfeited the game against " + self.challenger + ".") elif msg == "won": self.chatSignal.emit("You won against " + self.challenger + "!") elif msg == "lost": self.chatSignal.emit("You lost against " + self.challenger + ".") elif msg == "tied": self.chatSignal.emit("You tied " + self.challenger + ".") # After the GUI exits, we want the program to exit as well def guiExit(self): self.quitting = True self.transport.loseConnection() reactor.stop() return
class BrcmOpenomciOnuHandler(object): def __init__(self, adapter, device_id): self.log = structlog.get_logger(device_id=device_id) self.log.debug('function-entry') self.adapter = adapter self.adapter_agent = adapter.adapter_agent self.parent_adapter = None self.parent_id = None self.device_id = device_id self.incoming_messages = DeferredQueue() self.event_messages = DeferredQueue() self.proxy_address = None self.tx_id = 0 self._enabled = False self.alarms = None self.pm_metrics = None self._omcc_version = OMCCVersion.Unknown self._total_tcont_count = 0 # From ANI-G ME self._qos_flexibility = 0 # From ONT2_G ME self._onu_indication = None self._unis = dict() # Port # -> UniPort self._pon = None # TODO: probably shouldnt be hardcoded, determine from olt maybe? self._pon_port_number = 100 self.logical_device_id = None self._heartbeat = HeartBeat.create(self, device_id) # Set up OpenOMCI environment self._onu_omci_device = None self._dev_info_loaded = False self._deferred = None self._in_sync_subscription = None self._connectivity_subscription = None self._capabilities_subscription = None self.mac_bridge_service_profile_entity_id = 0x201 self.gal_enet_profile_entity_id = 0x1 self._tp_service_specific_task = dict() self._tech_profile_download_done = dict() # Initialize KV store client self.args = registry('main').get_args() if self.args.backend == 'etcd': host, port = self.args.etcd.split(':', 1) self.kv_client = EtcdStore( host, port, TechProfile.KV_STORE_TECH_PROFILE_PATH_PREFIX) elif self.args.backend == 'consul': host, port = self.args.consul.split(':', 1) self.kv_client = ConsulStore( host, port, TechProfile.KV_STORE_TECH_PROFILE_PATH_PREFIX) else: self.log.error('Invalid-backend') raise Exception("Invalid-backend-for-kv-store") # Handle received ONU event messages reactor.callLater(0, self.handle_onu_events) @property def enabled(self): return self._enabled @enabled.setter def enabled(self, value): if self._enabled != value: self._enabled = value @property def omci_agent(self): return self.adapter.omci_agent @property def omci_cc(self): return self._onu_omci_device.omci_cc if self._onu_omci_device is not None else None @property def heartbeat(self): return self._heartbeat @property def uni_ports(self): return self._unis.values() def uni_port(self, port_no_or_name): if isinstance(port_no_or_name, (str, unicode)): return next( (uni for uni in self.uni_ports if uni.name == port_no_or_name), None) assert isinstance(port_no_or_name, int), 'Invalid parameter type' return next((uni for uni in self.uni_ports if uni.logical_port_number == port_no_or_name), None) @property def pon_port(self): return self._pon def receive_message(self, msg): if self.omci_cc is not None: self.omci_cc.receive_message(msg) # Called once when the adapter creates the device/onu instance def activate(self, device): self.log.debug('function-entry', device=device) # first we verify that we got parent reference and proxy info assert device.parent_id assert device.proxy_address.device_id # register for proxied messages right away self.proxy_address = device.proxy_address self.adapter_agent.register_for_proxied_messages(device.proxy_address) self.parent_id = device.parent_id parent_device = self.adapter_agent.get_device(self.parent_id) if parent_device.type == 'openolt': self.parent_adapter = registry('adapter_loader'). \ get_agent(parent_device.adapter).adapter if self.enabled is not True: self.log.info('activating-new-onu') # populate what we know. rest comes later after mib sync device.root = True device.vendor = 'Broadcom' device.connect_status = ConnectStatus.REACHABLE device.oper_status = OperStatus.DISCOVERED device.reason = 'activating-onu' # pm_metrics requires a logical device id parent_device = self.adapter_agent.get_device(device.parent_id) self.logical_device_id = parent_device.parent_id assert self.logical_device_id, 'Invalid logical device ID' self.adapter_agent.update_device(device) self.log.debug('set-device-discovered') self._init_pon_state(device) ############################################################################ # Setup PM configuration for this device # Pass in ONU specific options kwargs = { OnuPmMetrics.DEFAULT_FREQUENCY_KEY: OnuPmMetrics.DEFAULT_ONU_COLLECTION_FREQUENCY, 'heartbeat': self.heartbeat, OnuOmciPmMetrics.OMCI_DEV_KEY: self._onu_omci_device } self.pm_metrics = OnuPmMetrics(self.adapter_agent, self.device_id, self.logical_device_id, grouped=True, freq_override=False, **kwargs) pm_config = self.pm_metrics.make_proto() self._onu_omci_device.set_pm_config( self.pm_metrics.omci_pm.openomci_interval_pm) self.log.info("initial-pm-config", pm_config=pm_config) self.adapter_agent.update_device_pm_config(pm_config, init=True) ############################################################################ # Setup Alarm handler self.alarms = AdapterAlarms(self.adapter_agent, device.id, self.logical_device_id) # Note, ONU ID and UNI intf set in add_uni_port method self._onu_omci_device.alarm_synchronizer.set_alarm_params( mgr=self.alarms, ani_ports=[self._pon]) self.enabled = True else: self.log.info('onu-already-activated') # Called once when the adapter needs to re-create device. usually on vcore restart def reconcile(self, device): self.log.debug('function-entry', device=device) # first we verify that we got parent reference and proxy info assert device.parent_id assert device.proxy_address.device_id # register for proxied messages right away self.proxy_address = device.proxy_address self.adapter_agent.register_for_proxied_messages(device.proxy_address) if self.enabled is not True: self.log.info('reconciling-broadcom-onu-device') self._init_pon_state(device) # need to restart state machines on vcore restart. there is no indication to do it for us. self._onu_omci_device.start() device.reason = "restarting-openomci" self.adapter_agent.update_device(device) # TODO: this is probably a bit heavy handed # Force a reboot for now. We need indications to reflow to reassign tconts and gems given vcore went away # This may not be necessary when mib resync actually works reactor.callLater(1, self.reboot) self.enabled = True else: self.log.info('onu-already-activated') @inlineCallbacks def handle_onu_events(self): event_msg = yield self.event_messages.get() try: if event_msg['event'] == 'download_tech_profile': tp_path = event_msg['event_data'] uni_id = event_msg['uni_id'] self.load_and_configure_tech_profile(uni_id, tp_path) except Exception as e: self.log.error("exception-handling-onu-event", e=e) # Handle next event reactor.callLater(0, self.handle_onu_events) def _init_pon_state(self, device): self.log.debug('function-entry', device=device) self._pon = PonPort.create(self, self._pon_port_number) self.adapter_agent.add_port(device.id, self._pon.get_port()) self.log.debug('added-pon-port-to-agent', pon=self._pon) parent_device = self.adapter_agent.get_device(device.parent_id) self.logical_device_id = parent_device.parent_id self.adapter_agent.update_device(device) # Create and start the OpenOMCI ONU Device Entry for this ONU self._onu_omci_device = self.omci_agent.add_device( self.device_id, self.adapter_agent, support_classes=self.adapter.broadcom_omci, custom_me_map=self.adapter.custom_me_entities()) # Port startup if self._pon is not None: self._pon.enabled = True # TODO: move to UniPort def update_logical_port(self, logical_device_id, port_id, state): try: self.log.info('updating-logical-port', logical_port_id=port_id, logical_device_id=logical_device_id, state=state) logical_port = self.adapter_agent.get_logical_port( logical_device_id, port_id) logical_port.ofp_port.state = state self.adapter_agent.update_logical_port(logical_device_id, logical_port) except Exception as e: self.log.exception("exception-updating-port", e=e) def delete(self, device): self.log.info('delete-onu', device=device) if self.parent_adapter: try: self.parent_adapter.delete_child_device(self.parent_id, device) except AttributeError: self.log.debug('parent-device-delete-child-not-implemented') else: self.log.debug("parent-adapter-not-available") def _create_tconts(self, uni_id, us_scheduler): alloc_id = us_scheduler['alloc_id'] q_sched_policy = us_scheduler['q_sched_policy'] self.log.debug('create-tcont', us_scheduler=us_scheduler) tcontdict = dict() tcontdict['alloc-id'] = alloc_id tcontdict['q_sched_policy'] = q_sched_policy tcontdict['uni_id'] = uni_id # TODO: Not sure what to do with any of this... tddata = dict() tddata['name'] = 'not-sure-td-profile' tddata['fixed-bandwidth'] = "not-sure-fixed" tddata['assured-bandwidth'] = "not-sure-assured" tddata['maximum-bandwidth'] = "not-sure-max" tddata['additional-bw-eligibility-indicator'] = "not-sure-additional" td = OnuTrafficDescriptor.create(tddata) tcont = OnuTCont.create(self, tcont=tcontdict, td=td) self._pon.add_tcont(tcont) self.log.debug('pon-add-tcont', tcont=tcont) # Called when there is an olt up indication, providing the gem port id chosen by the olt handler def _create_gemports(self, uni_id, gem_ports, alloc_id_ref, direction): self.log.debug('create-gemport', gem_ports=gem_ports, direction=direction) for gem_port in gem_ports: gemdict = dict() gemdict['gemport_id'] = gem_port['gemport_id'] gemdict['direction'] = direction gemdict['alloc_id_ref'] = alloc_id_ref gemdict['encryption'] = gem_port['aes_encryption'] gemdict['discard_config'] = dict() gemdict['discard_config']['max_probability'] = \ gem_port['discard_config']['max_probability'] gemdict['discard_config']['max_threshold'] = \ gem_port['discard_config']['max_threshold'] gemdict['discard_config']['min_threshold'] = \ gem_port['discard_config']['min_threshold'] gemdict['discard_policy'] = gem_port['discard_policy'] gemdict['max_q_size'] = gem_port['max_q_size'] gemdict['pbit_map'] = gem_port['pbit_map'] gemdict['priority_q'] = gem_port['priority_q'] gemdict['scheduling_policy'] = gem_port['scheduling_policy'] gemdict['weight'] = gem_port['weight'] gemdict['uni_id'] = uni_id gem_port = OnuGemPort.create(self, gem_port=gemdict) self._pon.add_gem_port(gem_port) self.log.debug('pon-add-gemport', gem_port=gem_port) def _do_tech_profile_configuration(self, uni_id, tp): num_of_tconts = tp['num_of_tconts'] us_scheduler = tp['us_scheduler'] alloc_id = us_scheduler['alloc_id'] self._create_tconts(uni_id, us_scheduler) upstream_gem_port_attribute_list = tp[ 'upstream_gem_port_attribute_list'] self._create_gemports(uni_id, upstream_gem_port_attribute_list, alloc_id, "UPSTREAM") downstream_gem_port_attribute_list = tp[ 'downstream_gem_port_attribute_list'] self._create_gemports(uni_id, downstream_gem_port_attribute_list, alloc_id, "DOWNSTREAM") def load_and_configure_tech_profile(self, uni_id, tp_path): self.log.debug("loading-tech-profile-configuration", uni_id=uni_id, tp_path=tp_path) if uni_id not in self._tp_service_specific_task: self._tp_service_specific_task[uni_id] = dict() if uni_id not in self._tech_profile_download_done: self._tech_profile_download_done[uni_id] = dict() if tp_path not in self._tech_profile_download_done[uni_id]: self._tech_profile_download_done[uni_id][tp_path] = False if not self._tech_profile_download_done[uni_id][tp_path]: try: if tp_path in self._tp_service_specific_task[uni_id]: self.log.info("tech-profile-config-already-in-progress", tp_path=tp_path) return tp = self.kv_client[tp_path] tp = ast.literal_eval(tp) self.log.debug("tp-instance", tp=tp) self._do_tech_profile_configuration(uni_id, tp) def success(_results): self.log.info("tech-profile-config-done-successfully") device = self.adapter_agent.get_device(self.device_id) device.reason = 'tech-profile-config-download-success' self.adapter_agent.update_device(device) if tp_path in self._tp_service_specific_task[uni_id]: del self._tp_service_specific_task[uni_id][tp_path] self._tech_profile_download_done[uni_id][tp_path] = True def failure(_reason): self.log.warn('tech-profile-config-failure-retrying', _reason=_reason) device = self.adapter_agent.get_device(self.device_id) device.reason = 'tech-profile-config-download-failure-retrying' self.adapter_agent.update_device(device) if tp_path in self._tp_service_specific_task[uni_id]: del self._tp_service_specific_task[uni_id][tp_path] self._deferred = reactor.callLater( _STARTUP_RETRY_WAIT, self.load_and_configure_tech_profile, uni_id, tp_path) self.log.info('downloading-tech-profile-configuration') self._tp_service_specific_task[uni_id][tp_path] = \ BrcmTpServiceSpecificTask(self.omci_agent, self, uni_id) self._deferred = \ self._onu_omci_device.task_runner.queue_task(self._tp_service_specific_task[uni_id][tp_path]) self._deferred.addCallbacks(success, failure) except Exception as e: self.log.exception("error-loading-tech-profile", e=e) else: self.log.info("tech-profile-config-already-done") def update_pm_config(self, device, pm_config): # TODO: This has not been tested self.log.info('update_pm_config', pm_config=pm_config) self.pm_metrics.update(pm_config) # Calling this assumes the onu is active/ready and had at least an initial mib downloaded. This gets called from # flow decomposition that ultimately comes from onos def update_flow_table(self, device, flows): self.log.debug('function-entry', device=device, flows=flows) # # We need to proxy through the OLT to get to the ONU # Configuration from here should be using OMCI # # self.log.info('bulk-flow-update', device_id=device.id, flows=flows) # no point in pushing omci flows if the device isnt reachable if device.connect_status != ConnectStatus.REACHABLE or \ device.admin_state != AdminState.ENABLED: self.log.warn("device-disabled-or-offline-skipping-flow-update", admin=device.admin_state, connect=device.connect_status) return def is_downstream(port): return port == self._pon_port_number def is_upstream(port): return not is_downstream(port) for flow in flows: _type = None _port = None _vlan_vid = None _udp_dst = None _udp_src = None _ipv4_dst = None _ipv4_src = None _metadata = None _output = None _push_tpid = None _field = None _set_vlan_vid = None self.log.debug('bulk-flow-update', device_id=device.id, flow=flow) try: _in_port = fd.get_in_port(flow) assert _in_port is not None _out_port = fd.get_out_port(flow) # may be None if is_downstream(_in_port): self.log.debug('downstream-flow', in_port=_in_port, out_port=_out_port) uni_port = self.uni_port(_out_port) elif is_upstream(_in_port): self.log.debug('upstream-flow', in_port=_in_port, out_port=_out_port) uni_port = self.uni_port(_in_port) else: raise Exception('port should be 1 or 2 by our convention') self.log.debug('flow-ports', in_port=_in_port, out_port=_out_port, uni_port=str(uni_port)) for field in fd.get_ofb_fields(flow): if field.type == fd.ETH_TYPE: _type = field.eth_type self.log.debug('field-type-eth-type', eth_type=_type) elif field.type == fd.IP_PROTO: _proto = field.ip_proto self.log.debug('field-type-ip-proto', ip_proto=_proto) elif field.type == fd.IN_PORT: _port = field.port self.log.debug('field-type-in-port', in_port=_port) elif field.type == fd.VLAN_VID: _vlan_vid = field.vlan_vid & 0xfff self.log.debug('field-type-vlan-vid', vlan=_vlan_vid) elif field.type == fd.VLAN_PCP: _vlan_pcp = field.vlan_pcp self.log.debug('field-type-vlan-pcp', pcp=_vlan_pcp) elif field.type == fd.UDP_DST: _udp_dst = field.udp_dst self.log.debug('field-type-udp-dst', udp_dst=_udp_dst) elif field.type == fd.UDP_SRC: _udp_src = field.udp_src self.log.debug('field-type-udp-src', udp_src=_udp_src) elif field.type == fd.IPV4_DST: _ipv4_dst = field.ipv4_dst self.log.debug('field-type-ipv4-dst', ipv4_dst=_ipv4_dst) elif field.type == fd.IPV4_SRC: _ipv4_src = field.ipv4_src self.log.debug('field-type-ipv4-src', ipv4_dst=_ipv4_src) elif field.type == fd.METADATA: _metadata = field.table_metadata self.log.debug('field-type-metadata', metadata=_metadata) else: raise NotImplementedError('field.type={}'.format( field.type)) for action in fd.get_actions(flow): if action.type == fd.OUTPUT: _output = action.output.port self.log.debug('action-type-output', output=_output, in_port=_in_port) elif action.type == fd.POP_VLAN: self.log.debug('action-type-pop-vlan', in_port=_in_port) elif action.type == fd.PUSH_VLAN: _push_tpid = action.push.ethertype self.log.debug('action-type-push-vlan', push_tpid=_push_tpid, in_port=_in_port) if action.push.ethertype != 0x8100: self.log.error('unhandled-tpid', ethertype=action.push.ethertype) elif action.type == fd.SET_FIELD: _field = action.set_field.field.ofb_field assert (action.set_field.field.oxm_class == OFPXMC_OPENFLOW_BASIC) self.log.debug('action-type-set-field', field=_field, in_port=_in_port) if _field.type == fd.VLAN_VID: _set_vlan_vid = _field.vlan_vid & 0xfff self.log.debug('set-field-type-vlan-vid', vlan_vid=_set_vlan_vid) else: self.log.error('unsupported-action-set-field-type', field_type=_field.type) else: self.log.error('unsupported-action-type', action_type=action.type, in_port=_in_port) # TODO: We only set vlan omci flows. Handle omci matching ethertypes at some point in another task if _type is not None: self.log.warn('ignoring-flow-with-ethType', ethType=_type) elif _set_vlan_vid is None or _set_vlan_vid == 0: self.log.warn('ignorning-flow-that-does-not-set-vlanid') else: self.log.warn('set-vlanid', uni_id=uni_port.port_number, set_vlan_vid=_set_vlan_vid) self._add_vlan_filter_task(device, uni_port, _set_vlan_vid) except Exception as e: self.log.exception('failed-to-install-flow', e=e, flow=flow) def _add_vlan_filter_task(self, device, uni_port, _set_vlan_vid): assert uni_port is not None def success(_results): self.log.info('vlan-tagging-success', uni_port=uni_port, vlan=_set_vlan_vid) device.reason = 'omci-flows-pushed' self._vlan_filter_task = None def failure(_reason): self.log.warn('vlan-tagging-failure', uni_port=uni_port, vlan=_set_vlan_vid) device.reason = 'omci-flows-failed-retrying' self._vlan_filter_task = reactor.callLater( _STARTUP_RETRY_WAIT, self._add_vlan_filter_task, device, uni_port, _set_vlan_vid) self.log.info('setting-vlan-tag') self._vlan_filter_task = BrcmVlanFilterTask(self.omci_agent, self.device_id, uni_port, _set_vlan_vid) self._deferred = self._onu_omci_device.task_runner.queue_task( self._vlan_filter_task) self._deferred.addCallbacks(success, failure) def get_tx_id(self): self.log.debug('function-entry') self.tx_id += 1 return self.tx_id # TODO: Actually conform to or create a proper interface. # this and the other functions called from the olt arent very clear. # Called each time there is an onu "up" indication from the olt handler def create_interface(self, data): self.log.debug('function-entry', data=data) self._onu_indication = data onu_device = self.adapter_agent.get_device(self.device_id) self.log.debug('starting-openomci-statemachine') self._subscribe_to_events() reactor.callLater(1, self._onu_omci_device.start) onu_device.reason = "starting-openomci" self.adapter_agent.update_device(onu_device) self._heartbeat.enabled = True # Currently called each time there is an onu "down" indication from the olt handler # TODO: possibly other reasons to "update" from the olt? def update_interface(self, data): self.log.debug('function-entry', data=data) oper_state = data.get('oper_state', None) onu_device = self.adapter_agent.get_device(self.device_id) if oper_state == 'down': self.log.debug('stopping-openomci-statemachine') reactor.callLater(0, self._onu_omci_device.stop) # Let TP download happen again for uni_id in self._tp_service_specific_task: self._tp_service_specific_task[uni_id].clear() for uni_id in self._tech_profile_download_done: self._tech_profile_download_done[uni_id].clear() self.disable_ports(onu_device) onu_device.reason = "stopping-openomci" onu_device.connect_status = ConnectStatus.UNREACHABLE onu_device.oper_status = OperStatus.DISCOVERED self.adapter_agent.update_device(onu_device) else: self.log.debug('not-changing-openomci-statemachine') # Not currently called by olt or anything else def remove_interface(self, data): self.log.debug('function-entry', data=data) onu_device = self.adapter_agent.get_device(self.device_id) self.log.debug('stopping-openomci-statemachine') reactor.callLater(0, self._onu_omci_device.stop) # Let TP download happen again for uni_id in self._tp_service_specific_task: self._tp_service_specific_task[uni_id].clear() for uni_id in self._tech_profile_download_done: self._tech_profile_download_done[uni_id].clear() self.disable_ports(onu_device) onu_device.reason = "stopping-openomci" self.adapter_agent.update_device(onu_device) # TODO: im sure there is more to do here # Not currently called. Would be called presumably from the olt handler def remove_gemport(self, data): self.log.debug('remove-gemport', data=data) gem_port = GemportsConfigData() gem_port.CopyFrom(data) device = self.adapter_agent.get_device(self.device_id) if device.connect_status != ConnectStatus.REACHABLE: self.log.error('device-unreachable') return # Not currently called. Would be called presumably from the olt handler def remove_tcont(self, tcont_data, traffic_descriptor_data): self.log.debug('remove-tcont', tcont_data=tcont_data, traffic_descriptor_data=traffic_descriptor_data) device = self.adapter_agent.get_device(self.device_id) if device.connect_status != ConnectStatus.REACHABLE: self.log.error('device-unreachable') return # TODO: Create some omci task that encompases this what intended # Not currently called. Would be called presumably from the olt handler def create_multicast_gemport(self, data): self.log.debug('function-entry', data=data) # TODO: create objects and populate for later omci calls def disable(self, device): self.log.debug('function-entry', device=device) try: self.log.info('sending-uni-lock-towards-device', device=device) def stop_anyway(reason): # proceed with disable regardless if we could reach the onu. for example onu is unplugged self.log.debug('stopping-openomci-statemachine') reactor.callLater(0, self._onu_omci_device.stop) # Let TP download happen again for uni_id in self._tp_service_specific_task: self._tp_service_specific_task[uni_id].clear() for uni_id in self._tech_profile_download_done: self._tech_profile_download_done[uni_id].clear() self.disable_ports(device) device.oper_status = OperStatus.UNKNOWN device.reason = "omci-admin-lock" self.adapter_agent.update_device(device) # lock all the unis task = BrcmUniLockTask(self.omci_agent, self.device_id, lock=True) self._deferred = self._onu_omci_device.task_runner.queue_task(task) self._deferred.addCallbacks(stop_anyway, stop_anyway) except Exception as e: log.exception('exception-in-onu-disable', exception=e) def reenable(self, device): self.log.debug('function-entry', device=device) try: # Start up OpenOMCI state machines for this device # this will ultimately resync mib and unlock unis on successful redownloading the mib self.log.debug('restarting-openomci-statemachine') self._subscribe_to_events() device.reason = "restarting-openomci" self.adapter_agent.update_device(device) reactor.callLater(1, self._onu_omci_device.start) self._heartbeat.enabled = True except Exception as e: log.exception('exception-in-onu-reenable', exception=e) def reboot(self): self.log.info('reboot-device') device = self.adapter_agent.get_device(self.device_id) if device.connect_status != ConnectStatus.REACHABLE: self.log.error("device-unreachable") return def success(_results): self.log.info('reboot-success', _results=_results) self.disable_ports(device) device.connect_status = ConnectStatus.UNREACHABLE device.oper_status = OperStatus.DISCOVERED device.reason = "rebooting" self.adapter_agent.update_device(device) def failure(_reason): self.log.info('reboot-failure', _reason=_reason) self._deferred = self._onu_omci_device.reboot() self._deferred.addCallbacks(success, failure) def disable_ports(self, onu_device): self.log.info('disable-ports', device_id=self.device_id, onu_device=onu_device) # Disable all ports on that device self.adapter_agent.disable_all_ports(self.device_id) parent_device = self.adapter_agent.get_device(onu_device.parent_id) assert parent_device logical_device_id = parent_device.parent_id assert logical_device_id ports = self.adapter_agent.get_ports(onu_device.id, Port.ETHERNET_UNI) for port in ports: port_id = 'uni-{}'.format(port.port_no) # TODO: move to UniPort self.update_logical_port(logical_device_id, port_id, OFPPS_LINK_DOWN) def enable_ports(self, onu_device): self.log.info('enable-ports', device_id=self.device_id, onu_device=onu_device) # Disable all ports on that device self.adapter_agent.enable_all_ports(self.device_id) parent_device = self.adapter_agent.get_device(onu_device.parent_id) assert parent_device logical_device_id = parent_device.parent_id assert logical_device_id ports = self.adapter_agent.get_ports(onu_device.id, Port.ETHERNET_UNI) for port in ports: port_id = 'uni-{}'.format(port.port_no) # TODO: move to UniPort self.update_logical_port(logical_device_id, port_id, OFPPS_LIVE) # Called just before openomci state machine is started. These listen for events from selected state machines, # most importantly, mib in sync. Which ultimately leads to downloading the mib def _subscribe_to_events(self): self.log.debug('function-entry') # OMCI MIB Database sync status bus = self._onu_omci_device.event_bus topic = OnuDeviceEntry.event_bus_topic( self.device_id, OnuDeviceEvents.MibDatabaseSyncEvent) self._in_sync_subscription = bus.subscribe(topic, self.in_sync_handler) # OMCI Capabilities bus = self._onu_omci_device.event_bus topic = OnuDeviceEntry.event_bus_topic( self.device_id, OnuDeviceEvents.OmciCapabilitiesEvent) self._capabilities_subscription = bus.subscribe( topic, self.capabilties_handler) # Called when the mib is in sync def in_sync_handler(self, _topic, msg): self.log.debug('function-entry', _topic=_topic, msg=msg) if self._in_sync_subscription is not None: try: in_sync = msg[IN_SYNC_KEY] if in_sync: # Only call this once bus = self._onu_omci_device.event_bus bus.unsubscribe(self._in_sync_subscription) self._in_sync_subscription = None # Start up device_info load self.log.debug('running-mib-sync') reactor.callLater(0, self._mib_in_sync) except Exception as e: self.log.exception('in-sync', e=e) def capabilties_handler(self, _topic, _msg): self.log.debug('function-entry', _topic=_topic, msg=_msg) if self._capabilities_subscription is not None: self.log.debug('capabilities-handler-done') # Mib is in sync, we can now query what we learned and actually start pushing ME (download) to the ONU. # Currently uses a basic mib download task that create a bridge with a single gem port and uni, only allowing EAP # Implement your own MibDownloadTask if you wish to setup something different by default def _mib_in_sync(self): self.log.debug('function-entry') omci = self._onu_omci_device in_sync = omci.mib_db_in_sync device = self.adapter_agent.get_device(self.device_id) device.reason = 'discovery-mibsync-complete' self.adapter_agent.update_device(device) if not self._dev_info_loaded: self.log.info('loading-device-data-from-mib', in_sync=in_sync, already_loaded=self._dev_info_loaded) omci_dev = self._onu_omci_device config = omci_dev.configuration # TODO: run this sooner somehow. shouldnt have to wait for mib sync to push an initial download # In Sync, we can register logical ports now. Ideally this could occur on # the first time we received a successful (no timeout) OMCI Rx response. try: # sort the lists so we get consistent port ordering. ani_list = sorted( config.ani_g_entities) if config.ani_g_entities else [] uni_list = sorted( config.uni_g_entities) if config.uni_g_entities else [] pptp_list = sorted( config.pptp_entities) if config.pptp_entities else [] veip_list = sorted( config.veip_entities) if config.veip_entities else [] if ani_list is None or (pptp_list is None and veip_list is None): device.reason = 'onu-missing-required-elements' self.log.warn("no-ani-or-unis") self.adapter_agent.update_device(device) raise Exception("onu-missing-required-elements") # Currently logging the ani, pptp, veip, and uni for information purposes. # Actually act on the veip/pptp as its ME is the most correct one to use in later tasks. # And in some ONU the UNI-G list is incomplete or incorrect... for entity_id in ani_list: ani_value = config.ani_g_entities[entity_id] self.log.debug("discovered-ani", entity_id=entity_id, value=ani_value) # TODO: currently only one OLT PON port/ANI, so this works out. With NGPON there will be 2..? self._total_tcont_count = ani_value.get( 'total-tcont-count') self.log.debug("set-total-tcont-count", tcont_count=self._total_tcont_count) for entity_id in uni_list: uni_value = config.uni_g_entities[entity_id] self.log.debug("discovered-uni", entity_id=entity_id, value=uni_value) uni_entities = OrderedDict() for entity_id in pptp_list: pptp_value = config.pptp_entities[entity_id] self.log.debug("discovered-pptp", entity_id=entity_id, value=pptp_value) uni_entities[entity_id] = UniType.PPTP for entity_id in veip_list: veip_value = config.veip_entities[entity_id] self.log.debug("discovered-veip", entity_id=entity_id, value=veip_value) uni_entities[entity_id] = UniType.VEIP uni_id = 0 for entity_id, uni_type in uni_entities.iteritems(): try: self._add_uni_port(entity_id, uni_id, uni_type) uni_id += 1 except AssertionError as e: self.log.warn("could not add UNI", entity_id=entity_id, uni_type=uni_type, e=e) multi_uni = len(self._unis) > 1 for uni_port in self._unis.itervalues(): uni_port.add_logical_port(uni_port.port_number, multi_uni) self.adapter_agent.update_device(device) self._qos_flexibility = config.qos_configuration_flexibility or 0 self._omcc_version = config.omcc_version or OMCCVersion.Unknown if self._unis: self._dev_info_loaded = True else: device.reason = 'no-usable-unis' self.adapter_agent.update_device(device) self.log.warn("no-usable-unis") raise Exception("no-usable-unis") except Exception as e: self.log.exception('device-info-load', e=e) self._deferred = reactor.callLater(_STARTUP_RETRY_WAIT, self._mib_in_sync) else: self.log.info('device-info-already-loaded', in_sync=in_sync, already_loaded=self._dev_info_loaded) if self._dev_info_loaded: if device.admin_state == AdminState.ENABLED: def success(_results): self.log.info('mib-download-success', _results=_results) device = self.adapter_agent.get_device(self.device_id) device.reason = 'initial-mib-downloaded' device.oper_status = OperStatus.ACTIVE device.connect_status = ConnectStatus.REACHABLE self.enable_ports(device) self.adapter_agent.update_device(device) self._mib_download_task = None def failure(_reason): self.log.warn('mib-download-failure-retrying', _reason=_reason) device.reason = 'initial-mib-download-failure-retrying' self.adapter_agent.update_device(device) self._deferred = reactor.callLater(_STARTUP_RETRY_WAIT, self._mib_in_sync) # Download an initial mib that creates simple bridge that can pass EAP. On success (above) finally set # the device to active/reachable. This then opens up the handler to openflow pushes from outside self.log.info('downloading-initial-mib-configuration') self._mib_download_task = BrcmMibDownloadTask( self.omci_agent, self) self._deferred = self._onu_omci_device.task_runner.queue_task( self._mib_download_task) self._deferred.addCallbacks(success, failure) else: self.log.info('admin-down-disabling') self.disable(device) else: self.log.info('device-info-not-loaded-skipping-mib-download') def _add_uni_port(self, entity_id, uni_id, uni_type=UniType.PPTP): self.log.debug('function-entry') device = self.adapter_agent.get_device(self.device_id) parent_device = self.adapter_agent.get_device(device.parent_id) parent_adapter_agent = registry('adapter_loader').get_agent( parent_device.adapter) if parent_adapter_agent is None: self.log.error('parent-adapter-could-not-be-retrieved') # TODO: This knowledge is locked away in openolt. and it assumes one onu equals one uni... parent_device = self.adapter_agent.get_device(device.parent_id) parent_adapter = parent_adapter_agent.adapter.devices[parent_device.id] uni_no = parent_adapter.platform.mk_uni_port_num( self._onu_indication.intf_id, self._onu_indication.onu_id, uni_id) # TODO: Some or parts of this likely need to move to UniPort. especially the format stuff uni_name = "uni-{}".format(uni_no) mac_bridge_port_num = uni_id + 1 # TODO +1 is only to test non-zero index self.log.debug('uni-port-inputs', uni_no=uni_no, uni_id=uni_id, uni_name=uni_name, uni_type=uni_type, entity_id=entity_id, mac_bridge_port_num=mac_bridge_port_num) uni_port = UniPort.create(self, uni_name, uni_id, uni_no, uni_name, uni_type) uni_port.entity_id = entity_id uni_port.enabled = True uni_port.mac_bridge_port_num = mac_bridge_port_num self.log.debug("created-uni-port", uni=uni_port) self.adapter_agent.add_port(device.id, uni_port.get_port()) parent_adapter_agent.add_port(device.parent_id, uni_port.get_port()) self._unis[uni_port.port_number] = uni_port self._onu_omci_device.alarm_synchronizer.set_alarm_params( onu_id=self._onu_indication.onu_id, uni_ports=self._unis.values()) # TODO: this should be in the PonPortclass pon_port = self._pon.get_port() # Delete reference to my own UNI as peer from parent. # TODO why is this here, add_port_reference_to_parent already prunes duplicates me_as_peer = Port.PeerPort(device_id=device.parent_id, port_no=uni_port.port_number) partial_pon_port = Port( port_no=pon_port.port_no, label=pon_port.label, type=pon_port.type, admin_state=pon_port.admin_state, oper_status=pon_port.oper_status, peers=[me_as_peer] ) # only list myself as a peer to avoid deleting all other UNIs from parent self.adapter_agent.delete_port_reference_from_parent( self.device_id, partial_pon_port) pon_port.peers.extend([me_as_peer]) self._pon._port = pon_port self.adapter_agent.add_port_reference_to_parent( self.device_id, pon_port)
class ServerProtocol(Protocol, object): """The client protocol that talks to the end server in a TCP proxy. """ def __init__(self, server_queue, client_queue): """Create a new protocol. :code:`server_queue` and :code:`client_queue` corresponds to the variables in the TCP proxy. :code:`self.wait_queue` is used to handle the race condition where :code:`self.client_queue` is ready to be consumed, but the connection has not been established. :param DeferredQueue server_queue: :param DeferredQueue client_queue: """ self.server_queue = server_queue self.client_queue = client_queue self.wait_queue = DeferredQueue() self.client_queue.get().addCallback(self.clientQueueCallback) def clientQueueCallback(self, data): """A callback for the client queue. If the data is the literal False, then close the connection. Otherwise, add this data to our wait queue. :param data: the data from the client queue """ if data is False: self.transport.loseConnection() else: self.wait_queue.put(data) self.client_queue.get().addCallback(self.clientQueueCallback) def emptyWaitQueue(self): """Starts emptying the wait queue. Note that a connection must already be made (self.transport must be ready for use) """ def _emptyWaitQueueHelper(data): self.transport.write(data) self.wait_queue.get().addCallback(_emptyWaitQueueHelper) assert self.transport is not None self.wait_queue.get().addCallback(_emptyWaitQueueHelper) def connectionMade(self): """Connection to target server is established. Empty the wait queue. """ self.emptyWaitQueue() def dataReceived(self, data): """Received data from target server, put into server queue :param str data: """ self.server_queue.put(data) def connectionLost(self, why): """Server closed connection, or some other issue. close connection to server """ self.server_queue.put(False)
class OMCI_CC(object): """ Handle OMCI Communication Channel specifics for Adtran ONUs""" _frame_to_event_type = { OmciMibResetResponse.message_id: RxEvent.MIB_Reset, OmciMibUploadResponse.message_id: RxEvent.MIB_Upload, OmciMibUploadNextResponse.message_id: RxEvent.MIB_Upload_Next, OmciCreateResponse.message_id: RxEvent.Create, OmciDeleteResponse.message_id: RxEvent.Delete, OmciSetResponse.message_id: RxEvent.Set, OmciGetAllAlarmsResponse.message_id: RxEvent.Get_ALARM_Get, OmciGetAllAlarmsNextResponse.message_id: RxEvent.Get_ALARM_Get_Next } def __init__(self, adapter_agent, device_id, me_map=None, alarm_queue_limit=_MAX_INCOMING_ALARM_MESSAGES, avc_queue_limit=_MAX_INCOMING_ALARM_MESSAGES, test_results_queue_limit=_MAX_INCOMING_TEST_RESULT_MESSAGES): self.log = structlog.get_logger(device_id=device_id) self._adapter_agent = adapter_agent self._device_id = device_id self._proxy_address = None self._tx_tid = 1 self._enabled = False self._requests = dict() # Tx ID -> (timestamp, deferred, tx_frame, timeout) self._alarm_queue = DeferredQueue(size=alarm_queue_limit) self._avc_queue = DeferredQueue(size=avc_queue_limit) self._test_results_queue = DeferredQueue(size=test_results_queue_limit) self._me_map = me_map # Statistics self._tx_frames = 0 self._rx_frames = 0 self._rx_unknown_tid = 0 # Rx OMCI with no Tx TID match self._rx_onu_frames = 0 # Autonomously generated ONU frames self._rx_alarm_overflow = 0 # Autonomously generated ONU alarms rx overflow self._rx_avc_overflow = 0 # Autonomously generated ONU AVC rx overflow self._rx_onu_discards = 0 # Autonomously generated ONU unknown message types self._rx_timeouts = 0 self._rx_unknown_me = 0 # Number of managed entities Rx without a decode definition self._tx_errors = 0 # Exceptions during tx request self._consecutive_errors = 0 # Rx & Tx errors in a row, a good RX resets this to 0 self._reply_min = sys.maxint # Fastest successful tx -> rx self._reply_max = 0 # Longest successful tx -> rx self._reply_sum = 0.0 # Total seconds for successful tx->rx (float for average) self.event_bus = EventBusClient() # If a list of custom ME Entities classes were provided, insert them into # main class_id to entity map. # TODO: If this class becomes hidden from the ONU DA, move this to the OMCI State Machine runner def __str__(self): return "OMCISupport: {}".format(self._device_id) @staticmethod def event_bus_topic(device_id, event): """ Get the topic name for a given event Frame Type :param device_id: (str) ONU Device ID :param event: (OmciCCRxEvents) Type of event :return: (str) Topic string """ assert event in OmciCCRxEvents, \ 'Event {} is not an OMCI-CC Rx Event'.format(event.name) return 'omci-rx:{}:{}'.format(device_id, event.name) @property def enabled(self): return self._enabled @enabled.setter def enabled(self, value): """ Enable/disable the OMCI Communications Channel :param value: (boolean) True to enable, False to disable """ assert isinstance(value, bool), 'enabled is a boolean' if self._enabled != value: self._enabled = value if self._enabled: self._start() else: self._stop() @property def tx_frames(self): return self._tx_frames @property def rx_frames(self): return self._rx_frames @property def rx_unknown_tid(self): return self._rx_unknown_tid # Tx TID not found @property def rx_unknown_me(self): return self._rx_unknown_me @property def rx_onu_frames(self): return self._rx_onu_frames @property def rx_alarm_overflow(self): return self._rx_alarm_overflow # Alarm ONU autonomous overflows @property def rx_avc_overflow(self): return self._rx_avc_overflow # Attribute Value change autonomous overflows @property def rx_onu_discards(self): return self._rx_onu_discards # Attribute Value change autonomous overflows @property def rx_timeouts(self): return self._rx_timeouts @property def tx_errors(self): return self._tx_errors @property def consecutive_errors(self): return self._consecutive_errors @property def reply_min(self): return int(round(self._reply_min * 1000.0)) # Milliseconds @property def reply_max(self): return int(round(self._reply_max * 1000.0)) # Milliseconds @property def reply_average(self): avg = self._reply_sum / self._rx_frames if self._rx_frames > 0 else 0.0 return int(round(avg * 1000.0)) # Milliseconds @property def get_alarm_message(self): """ Attempt to retrieve and remove an ONU Alarm Message from the ONU autonomous message queue. TODO: We may want to deprecate this, see TODO comment around line 399 in the _request_success() method below :return: a Deferred which fires with the next Alarm Frame available in the queue. """ return self._alarm_queue.get() @property def get_avc_message(self): """ Attempt to retrieve and remove an ONU Attribute Value Change (AVC) Message from the ONU autonomous message queue. TODO: We may want to deprecate this, see TODO comment around line 399 in the _request_success() method below :return: a Deferred which fires with the next AVC Frame available in the queue. """ return self._avc_queue.get() @property def get_test_results(self): """ Attempt to retrieve and remove an ONU Test Results Message from the ONU autonomous message queue. TODO: We may want to deprecate this, see TODO comment around line 399 in the _request_success() method below :return: a Deferred which fires with the next Test Results Frame is available in the queue. """ return self._test_results_queue.get() def _start(self): """ Start the OMCI Communications Channel """ assert self._enabled, 'Start should only be called if enabled' self.flush() device = self._adapter_agent.get_device(self._device_id) self._proxy_address = device.proxy_address def _stop(self): """ Stop the OMCI Communications Channel """ assert not self._enabled, 'Stop should only be called if disabled' self.flush() self._proxy_address = None # TODO: What is best way to clean up any outstanding futures for these queues self._alarm_queue = None self._avc_queue = None self._test_results_queue = None def _receive_onu_message(self, rx_frame): """ Autonomously generated ONU frame Rx handler""" from twisted.internet.defer import QueueOverflow self.log.debug('rx-onu-frame', frame_type=type(rx_frame), frame=hexify(str(rx_frame))) # TODO: Signal, via defer if Alarm Overflow or just an event? msg_type = rx_frame.fields['message_type'] self._rx_onu_frames += 1 msg = {TX_REQUEST_KEY: None, RX_RESPONSE_KEY: rx_frame} if msg_type == EntityOperations.AlarmNotification.value: topic = OMCI_CC.event_bus_topic(self._device_id, RxEvent.Alarm_Notification) reactor.callLater(0, self.event_bus.publish, topic, msg) try: self._alarm_queue.put((rx_frame, arrow.utcnow().float_timestamp)) except QueueOverflow: self._rx_alarm_overflow += 1 self.log.warn('onu-rx-alarm-overflow', cnt=self._rx_alarm_overflow) elif msg_type == EntityOperations.AttributeValueChange.value: topic = OMCI_CC.event_bus_topic(self._device_id, RxEvent.AVC_Notification) reactor.callLater(0, self.event_bus.publish, topic, msg) try: self._alarm_queue.put((rx_frame, arrow.utcnow().float_timestamp)) except QueueOverflow: self._rx_avc_overflow += 1 self.log.warn('onu-rx-avc-overflow', cnt=self._rx_avc_overflow) elif msg_type == EntityOperations.TestResult.value: topic = OMCI_CC.event_bus_topic(self._device_id, RxEvent.Test_Result) reactor.callLater(0, self.event_bus.publish, topic, msg) try: self._test_results_queue.put((rx_frame, arrow.utcnow().float_timestamp)) except QueueOverflow: self.log.warn('onu-rx-test-results-overflow') else: # TODO: Need to add test results message support self.log.warn('onu-unsupported-autonomous-message', type=msg_type) self._rx_onu_discards += 1 def receive_message(self, msg): """ Receive and OMCI message from the proxy channel to the OLT. Call this from your ONU Adapter on a new OMCI Rx on the proxy channel """ if self.enabled: try: now = arrow.utcnow() d = None # NOTE: Since we may need to do an independent ME map on a per-ONU basis # save the current value of the entity_id_to_class_map, then # replace it with our custom one before decode, and then finally # restore it later. Tried other ways but really made the code messy. saved_me_map = omci_entities.entity_id_to_class_map omci_entities.entity_id_to_class_map = self._me_map try: rx_frame = OmciFrame(msg) rx_tid = rx_frame.fields['transaction_id'] if rx_tid == 0: return self._receive_onu_message(rx_frame) # Previously unreachable if this is the very first Rx or we # have been running consecutive errors if self._rx_frames == 0 or self._consecutive_errors != 0: reactor.callLater(0, self._publish_connectivity_event, True) self._rx_frames += 1 self._consecutive_errors = 0 except KeyError as e: # Unknown, Unsupported, or vendor-specific ME. Key is the unknown classID self.log.debug('frame-decode-key-error', msg=hexlify(msg), e=e) rx_frame = self._decode_unknown_me(msg) self._rx_unknown_me += 1 rx_tid = rx_frame.fields.get('transaction_id') except Exception as e: self.log.exception('frame-decode', msg=hexlify(msg), e=e) return finally: omci_entities.entity_id_to_class_map = saved_me_map # Always restore it. try: (ts, d, tx_frame, _) = self._requests.pop(rx_tid) ts_diff = now - arrow.Arrow.utcfromtimestamp(ts) secs = ts_diff.total_seconds() self._reply_sum += secs if secs < self._reply_min: self._reply_min = secs if secs > self._reply_max: self._reply_max = secs except KeyError as e: # Possible late Rx on a message that timed-out self._rx_unknown_tid += 1 self.log.warn('tx-message-missing', rx_id=rx_tid, msg=hexlify(msg)) return except Exception as e: self.log.exception('frame-match', msg=hexlify(msg), e=e) if d is not None: return d.errback(failure.Failure(e)) return # Notify sender of completed request reactor.callLater(0, d.callback, rx_frame) # Publish Rx event to listeners in a different task reactor.callLater(0, self._publish_rx_frame, tx_frame, rx_frame) except Exception as e: self.log.exception('rx-msg', e=e) def _decode_unknown_me(self, msg): """ Decode an ME for an unsupported class ID. This should only occur for a subset of message types (Get, Set, MIB Upload Next, ...) and they should only be responses as well. There are some times below that are commented out. For VOLTHA 2.0, it is expected that any get, set, create, delete for unique (often vendor) MEs will be coded by the ONU utilizing it and supplied to OpenOMCI as a vendor-specific ME during device initialization. :param msg: (str) Binary data :return: (OmciFrame) resulting frame """ from struct import unpack (tid, msg_type, framing) = unpack('!HBB', msg[0:4]) assert framing == 0xa, 'Only basic OMCI framing supported at this time' msg = msg[4:] # TODO: Commented out items below are future work (not expected for VOLTHA v2.0) (msg_class, kwargs) = { # OmciCreateResponse.message_id: (OmciCreateResponse, None), # OmciDeleteResponse.message_id: (OmciDeleteResponse, None), # OmciSetResponse.message_id: (OmciSetResponse, None), # OmciGetResponse.message_id: (OmciGetResponse, None), # OmciGetAllAlarmsNextResponse.message_id: (OmciGetAllAlarmsNextResponse, None), OmciMibUploadNextResponse.message_id: (OmciMibUploadNextResponse, { 'entity_class': unpack('!H', msg[0:2])[0], 'entity_id': unpack('!H', msg[2:4])[0], 'object_entity_class': unpack('!H', msg[4:6])[0], 'object_entity_id': unpack('!H', msg[6:8])[0], 'object_attributes_mask': unpack('!H', msg[8:10])[0], 'object_data': { UNKNOWN_CLASS_ATTRIBUTE_KEY: hexlify(msg[10:-4]) }, }), # OmciAlarmNotification.message_id: (OmciAlarmNotification, None), # OmciAttributeValueChange.message_id: (OmciAttributeValueChange, # { # 'entity_class': unpack('!H', msg[0:2])[0], # 'entity_id': unpack('!H', msg[2:4])[0], # 'data': { # UNKNOWN_CLASS_ATTRIBUTE_KEY: hexlify(msg[4:-8]) # }, # }), # OmciTestResult.message_id: (OmciTestResult, None), }.get(msg_type, None) if msg_class is None: raise TypeError('Unsupport Message Type for Unknown Decode: {}', msg_type) return OmciFrame(transaction_id=tid, message_type=msg_type, omci_message=msg_class(**kwargs)) def _publish_rx_frame(self, tx_frame, rx_frame): """ Notify listeners of successful response frame :param tx_frame: (OmciFrame) Original request frame :param rx_frame: (OmciFrame) Response frame """ if self._enabled and isinstance(rx_frame, OmciFrame): frame_type = rx_frame.fields['omci_message'].message_id event_type = OMCI_CC._frame_to_event_type.get(frame_type) if event_type is not None: topic = OMCI_CC.event_bus_topic(self._device_id, event_type) msg = {TX_REQUEST_KEY: tx_frame, RX_RESPONSE_KEY: rx_frame} self.event_bus.publish(topic=topic, msg=msg) def _publish_connectivity_event(self, connected): """ Notify listeners of Rx/Tx connectivity over OMCI :param connected: (bool) True if connectivity transitioned from unreachable to reachable """ if self._enabled: topic = OMCI_CC.event_bus_topic(self._device_id, RxEvent.Connectivity) msg = {CONNECTED_KEY: connected} self.event_bus.publish(topic=topic, msg=msg) def flush(self, max_age=0): limit = arrow.utcnow().float_timestamp - max_age old = [tid for tid, (ts, _, _, _) in self._requests.iteritems() if ts <= limit] for tid in old: (_, d, _, _) = self._requests.pop(tid) if d is not None and not d.called: d.cancel() self._requests = dict() if max_age == 0: # Flush autonomous messages (Alarms & AVCs) while self._alarm_queue.pending: _ = yield self._alarm_queue.get() while self._avc_queue.pending: _ = yield self._avc_queue.get() def _get_tx_tid(self): """ Get the next Transaction ID for a tx. Note TID=0 is reserved for autonomously generated messages from an ONU :return: (int) TID """ tx_tid, self._tx_tid = self._tx_tid, self._tx_tid + 1 if self._tx_tid > MAX_OMCI_TX_ID: self._tx_tid = 1 return tx_tid def _request_failure(self, value, tx_tid): """ Handle a transmit failure and/or Rx timeout :param value: (Failure) Twisted failure :param tx_tid: (int) Associated Tx TID """ if tx_tid in self._requests: (_, _, _, timeout) = self._requests.pop(tx_tid) else: timeout = 0 if isinstance(value, failure.Failure): value.trap(CancelledError) self._rx_timeouts += 1 self._consecutive_errors += 1 if self._consecutive_errors == 1: reactor.callLater(0, self._publish_connectivity_event, False) self.log.info('timeout', tx_id=tx_tid, timeout=timeout) value = failure.Failure(TimeoutError(timeout, "Deferred")) return value def _request_success(self, rx_frame): """ Handle transmit success (a matching Rx was received) :param rx_frame: (OmciFrame) OMCI response frame with matching TID :return: (OmciFrame) OMCI response frame with matching TID """ # At this point, no additional processing is required # Continue with Rx Success callbacks. return rx_frame def send(self, frame, timeout=DEFAULT_OMCI_TIMEOUT): """ Send the OMCI Frame to the ONU via the proxy_channel :param frame: (OMCIFrame) Message to send :param timeout: (int) Rx Timeout. 0=Forever :return: (deferred) A deferred that fires when the response frame is received or if an error/timeout occurs """ self.flush(max_age=MAX_OMCI_REQUEST_AGE) assert timeout <= MAX_OMCI_REQUEST_AGE, \ 'Maximum timeout is {} seconds'.format(MAX_OMCI_REQUEST_AGE) assert isinstance(frame, OmciFrame), \ "Invalid frame class '{}'".format(type(frame)) if not self.enabled or self._proxy_address is None: # TODO custom exceptions throughout this code would be helpful return fail(result=failure.Failure(Exception('OMCI is not enabled'))) try: tx_tid = frame.fields['transaction_id'] if tx_tid is None: tx_tid = self._get_tx_tid() frame.fields['transaction_id'] = tx_tid assert tx_tid not in self._requests, 'TX TID {} is already exists'.format(tx_tid) assert tx_tid >= 0, 'Invalid Tx TID: {}'.format(tx_tid) ts = arrow.utcnow().float_timestamp d = defer.Deferred() # NOTE: Since we may need to do an independent ME map on a per-ONU basis # save the current value of the entity_id_to_class_map, then # replace it with our custom one before decode, and then finally # restore it later. Tried other ways but really made the code messy. saved_me_map = omci_entities.entity_id_to_class_map omci_entities.entity_id_to_class_map = self._me_map try: self._adapter_agent.send_proxied_message(self._proxy_address, hexify(str(frame))) finally: omci_entities.entity_id_to_class_map = saved_me_map self._tx_frames += 1 self._requests[tx_tid] = (ts, d, frame, timeout) d.addCallbacks(self._request_success, self._request_failure, errbackArgs=(tx_tid,)) if timeout > 0: d.addTimeout(timeout, reactor) except Exception as e: self._tx_errors += 1 self._consecutive_errors += 1 if self._consecutive_errors == 1: reactor.callLater(0, self._publish_connectivity_event, False) self.log.exception('send-omci', e=e) return fail(result=failure.Failure(e)) return d ################################################################################### # MIB Action shortcuts def send_mib_reset(self, timeout=DEFAULT_OMCI_TIMEOUT): """ Perform a MIB Reset """ self.log.debug('send-mib-reset') frame = OntDataFrame().mib_reset() return self.send(frame, timeout) def send_mib_upload(self, timeout=DEFAULT_OMCI_TIMEOUT): self.log.debug('send-mib-upload') frame = OntDataFrame().mib_upload() return self.send(frame, timeout) def send_mib_upload_next(self, seq_no, timeout=DEFAULT_OMCI_TIMEOUT): self.log.debug('send-mib-upload-next') frame = OntDataFrame(sequence_number=seq_no).mib_upload_next() return self.send(frame, timeout) def send_reboot(self, timeout=DEFAULT_OMCI_TIMEOUT): """ Send an ONU Device reboot request (ONU-G ME). NOTICE: This method is being deprecated and replaced with a tasks to preform this function """ self.log.debug('send-mib-reboot') frame = OntGFrame().reboot() return self.send(frame, timeout) def send_get_all_alarm(self, alarm_retrieval_mode=0, timeout=DEFAULT_OMCI_TIMEOUT): self.log.debug('send_get_alarm') frame = OntDataFrame().get_all_alarm(alarm_retrieval_mode) return self.send(frame, timeout) def send_get_all_alarm_next(self, seq_no, timeout=DEFAULT_OMCI_TIMEOUT): self.log.debug('send_get_alarm_next') frame = OntDataFrame().get_all_alarm_next(seq_no) return self.send(frame, timeout)
class PostgresProtocol(StatefulProtocol): def getInitialState(self): return (self.getHeader, 5) def getHandler(self, tag): handlers = { 'R': self.handle_Authentication, 'S': self.handle_ParameterStatus, 'K': self.handle_BackendKeyData, 'Z': self.handle_ReadyForQuery, 'E': self.handle_ErrorResponse, 'T': self.handle_RowDescription, 'D': self.handle_DataRow, 'C': self.handle_CommandComplete } if tag in handlers: return handlers[tag] else: print 'UNKNOWN TAG %r' % (tag, ) return self.handle_Unknown def getHeader(self, header): tag, length = struct.unpack('!cI', header) handler = self.getHandler(tag) return handler, length - 4 def connectionMade(self): self.parameters = {} self.cancellationKey = None self.backendPID = None self.transactionStatus = None self._dfds = deque() self._queries = DeferredQueue() self._currentResults = [] self._currentRowDescription = [] self.send_StartupMessage() def changeStatus(self, newStatus): self.transactionStatus = newStatus if newStatus == 'I': query = self._queries.get() query.addCallback(self.send_Query) def runQuery(self, query): d = Deferred() self._dfds.append(d) self._queries.put(query) return d def send(self, tag, payload): if tag is None: tag = '' length = len(payload) + 4 length_packed = struct.pack('!I', length) data = ''.join([tag, length_packed, payload]) self.transport.write(data) def send_StartupMessage(self): payload = {'user': '******', 'database': 'asdf'} data = struct.pack('!hh', 3, 0) for k, v in payload.items(): data += '%s\x00%s\x00' % (k, v) data += '\x00' self.send(None, data) def send_Query(self, query): if not query.endswith('\x00'): query = query + '\x00' self.send('Q', query) @handler def handle_Authentication(self, data): if len(data) == 8: print 'md5 auth required' elif len(data) == 4: print 'r4' @handler def handle_Unknown(self, data): print 'unknown handler, data: %r' % (data, ) @handler def handle_ParameterStatus(self, data): param, value, empty = data.split('\x00') self.parameters[param] = value @handler def handle_BackendKeyData(self, data): self.backendPID, self.cancellationKey = struct.unpack('!II', data) @handler def handle_ReadyForQuery(self, data): self.changeStatus(data) @handler def handle_ErrorResponse(self, data): fields = data.split('\x00') for field in fields: if not field: continue fieldType = field[0] fieldValue = field[1:] print 'SERVER ERROR! %s: %s' % (fieldType, fieldValue) @handler def handle_RowDescription(self, data): self._currentRowDescription = lib.parse_rowDescription(data, len(data)) @handler def handle_DataRow(self, data): numCols = struct.unpack('!h', data[:2])[0] data = data[2:] row = [] lib.x(numCols, ffi.new('char[]', data)) while data: fieldLen = struct.unpack('!I', data[:4])[0] fieldData = data[4:4 + fieldLen] data = data[4 + fieldLen:] row.append(fieldData) self._currentResults.append(row) @handler def handle_CommandComplete(self, data): dfd = self._dfds.popleft() dfd.callback(self._currentResults) self._currentRowDescription = [] self._currentResults = []