def __init__(self, topology_name, topology_id, instance, stream_port, metrics_port, topo_pex_file_path): # Basic information about this heron instance self.topology_name = topology_name self.topology_id = topology_id self.instance = instance self.stream_port = stream_port self.metrics_port = metrics_port self.topo_pex_file_abs_path = os.path.abspath(topo_pex_file_path) self.sys_config = system_config.get_sys_config() self.in_stream = HeronCommunicator(producer_cb=None, consumer_cb=None) self.out_stream = HeronCommunicator(producer_cb=None, consumer_cb=None) self.socket_map = dict() self.looper = GatewayLooper(self.socket_map) # Initialize metrics related self.out_metrics = HeronCommunicator() self.out_metrics.\ register_capacity(self.sys_config[constants.INSTANCE_INTERNAL_METRICS_WRITE_QUEUE_CAPACITY]) self.metrics_collector = MetricsCollector(self.looper, self.out_metrics) self.gateway_metrics = GatewayMetrics(self.metrics_collector) self.py_metrics = PyMetrics(self.metrics_collector) # Create socket options and socket clients socket_options = create_socket_options() self._stmgr_client = \ SingleThreadStmgrClient(self.looper, self, self.STREAM_MGR_HOST, stream_port, topology_name, topology_id, instance, self.socket_map, self.gateway_metrics, socket_options) self._metrics_client = \ MetricsManagerClient(self.looper, self.METRICS_MGR_HOST, metrics_port, instance, self.out_metrics, self.in_stream, self.out_stream, self.socket_map, socket_options, self.gateway_metrics, self.py_metrics) self.my_pplan_helper = None self.serializer = None # my_instance is a AssignedInstance tuple self.my_instance = None self.is_instance_started = False self.is_stateful_started = False self.stateful_state = None # Debugging purposes def go_trace(_, stack): with open("/tmp/trace.log", "w") as f: traceback.print_stack(stack, file=f) self.looper.register_timer_task_in_sec(self.looper.exit_loop, 0.0) signal.signal(signal.SIGUSR1, go_trace)
def __init__(self): socket_options = SocketOptions(32768, 16, 32768, 16, 1024000, 1024000) sys_config = { constants.INSTANCE_RECONNECT_METRICSMGR_INTERVAL_SEC: 10, constants.INSTANCE_METRICS_SYSTEM_SAMPLE_INTERVAL_SEC: 10 } stream = HeronCommunicator(producer_cb=None, consumer_cb=None) MetricsManagerClient.__init__(self, EventLooper(), self.HOST, self.PORT, mock_protobuf.get_mock_instance(), HeronCommunicator(), stream, stream, {}, socket_options, Mock(), sys_config) self.register_req_called = False
def __init__(self): socket_options = SocketOptions(32768, 16, 32768, 16, 1024000, 1024000) with patch( "heron.common.src.python.config.system_config.get_sys_config", side_effect=lambda: { constants.INSTANCE_RECONNECT_METRICSMGR_INTERVAL_SEC: 10, constants.INSTANCE_METRICS_SYSTEM_SAMPLE_INTERVAL_SEC: 10 }): stream = HeronCommunicator(producer_cb=None, consumer_cb=None) MetricsManagerClient.__init__(self, EventLooper(), self.HOST, self.PORT, mock_protobuf.get_mock_instance(), HeronCommunicator(), stream, stream, {}, socket_options, Mock(), Mock()) self.register_req_called = False
def test_generic(self): """Unittest with no producer, consumer specified""" communicator = HeronCommunicator(producer_cb=None, consumer_cb=None) for obj in mock_generator.prim_list: communicator.offer(obj) for obj in mock_generator.prim_list: self.assertEqual(obj, communicator.poll())
def test_consumer_callback(self): def callback(): self.global_value = 10 # test consumer cb communicator = HeronCommunicator(producer_cb=None, consumer_cb=callback) self.assertEqual(self.global_value, 6) communicator.offer("object") self.assertEqual(self.global_value, 10)
def test_producer_callback(self): def callback(): self.global_value = 10 # test producer cb communicator = HeronCommunicator(producer_cb=callback, consumer_cb=None) communicator.offer("object") self.assertEqual(self.global_value, 6) ret = communicator.poll() self.assertEqual(ret, "object") self.assertEqual(self.global_value, 10)
def __init__(self): self.registered_timers = [] super(MockMetricsCollector, self).__init__(None, HeronCommunicator())
def _prepare_sample_success(): pplan, instances = get_a_sample_pplan() pplan_helper = PhysicalPlanHelper(pplan, instances[0]["instance_id"], "topology.pex.path") out_stream = HeronCommunicator(producer_cb=None, consumer_cb=None) return pplan_helper, out_stream
def test_empty(self): communicator = HeronCommunicator(producer_cb=None, consumer_cb=None) with self.assertRaises(Queue.Empty): communicator.poll()