def register_metrics(self, context): """Registers metrics to context :param context: Topology Context """ sys_config = system_config.get_sys_config() interval = float(sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC]) collector = context.get_metrics_collector() super(ComponentMetrics, self).register_metrics(collector, interval)
def register_metrics(self, context): """Registers metrics to context :param context: Topology Context """ sys_config = system_config.get_sys_config() interval = float(sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC]) collector = context.get_metrics_collector() super(ComponentMetrics, self).register_metrics(collector, interval)
def __init__(self, metrics_collector): self.process = psutil.Process() # total sys cpu time self.sys_cpu_time = AssignableMetrics(0) # total user cpu time self.user_cpu_time = AssignableMetrics(0) # threads cpu usage self.threads = MultiAssignableMetrics() # number of open file descriptors self.fd_nums = AssignableMetrics(0) # number of threads self.num_threads = AssignableMetrics([]) # rss: aka "Resident Set Size" # this is the non-swapped physical memory a process has used. self.physical_memory = AssignableMetrics(0) # vms: "Virtual Memory Size", this is the total # amount of virtual memory used by the process. self.virtual_memory = AssignableMetrics(0) # stats about three generations of GC # count is the number of objects in one generation # threshold is the collect frequency of one generation self.g1_count, self.g1_threshold = AssignableMetrics( 0), AssignableMetrics(0) self.g2_count, self.g2_threshold = AssignableMetrics( 0), AssignableMetrics(0) self.g3_count, self.g3_threshold = AssignableMetrics( 0), AssignableMetrics(0) PY_SYS_CPU_TIME = '__py-sys-cpu-time-secs' PY_USER_CPU_TIME = '__py-user-cpu-time-secs' PY_FD_NUMS = '__py-file-descriptors-number' PY_PHYSICAL_MEMORY = '__py-physical-memory-byte' PY_VIRTUAL_MEMORY = '__py-virtual-memory-byte' PY_GC_GENERATION_1_COUNT = '__py-generation-1-count' PY_GC_GENERATION_2_COUNT = '__py-generation-2-count' PY_GC_GENERATION_3_COUNT = '__py-generation-3-count' PY_GC_GENERATION_1_THRESHOLD = '__py-generation-1-collection-threshold' PY_GC_GENERATION_2_THRESHOLD = '__py-generation-2-collection-threshold' PY_GC_GENERATION_3_THRESHOLD = '__py-generation-3-collection-threshold' self.metrics = { PY_SYS_CPU_TIME: self.sys_cpu_time, PY_USER_CPU_TIME: self.user_cpu_time, PY_FD_NUMS: self.fd_nums, PY_PHYSICAL_MEMORY: self.physical_memory, PY_VIRTUAL_MEMORY: self.virtual_memory, PY_GC_GENERATION_1_COUNT: self.g1_count, PY_GC_GENERATION_2_COUNT: self.g2_count, PY_GC_GENERATION_3_COUNT: self.g3_count, PY_GC_GENERATION_1_THRESHOLD: self.g1_threshold, PY_GC_GENERATION_2_THRESHOLD: self.g2_threshold, PY_GC_GENERATION_3_THRESHOLD: self.g3_threshold } super(PyMetrics, self).__init__(self.metrics) sys_config = system_config.get_sys_config() interval = float( sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC]) self.register_metrics(metrics_collector, interval)
def main(): if len(sys.argv) != 13: print_usage(sys.argv[0]) sys.exit(1) topology_name = sys.argv[1] topology_id = sys.argv[2] instance_id = sys.argv[3] component_name = sys.argv[4] task_id = sys.argv[5] component_index = sys.argv[6] stmgr_id = sys.argv[7] stmgr_port = sys.argv[8] metrics_port = sys.argv[9] sys_config = yaml_config_reader(sys.argv[10]) override_config = yaml_config_reader(sys.argv[11]) topology_pex_file_path = sys.argv[12] Log.debug("System config: " + str(sys_config)) Log.debug("Override config: " + str(override_config)) system_config.set_sys_config(sys_config, override_config) # get combined configuration sys_config = system_config.get_sys_config() # create the protobuf instance instance_info = physical_plan_pb2.InstanceInfo() instance_info.task_id = int(task_id) instance_info.component_index = int(component_index) instance_info.component_name = component_name instance = physical_plan_pb2.Instance() instance.instance_id = instance_id instance.stmgr_id = stmgr_id instance.info.MergeFrom(instance_info) # Logging init log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY]) max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES] max_log_bytes = sys_config[constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB log_file = os.path.join(log_dir, instance_id + ".log.0") log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=max_log_files, max_bytes=max_log_bytes) Log.info("\nStarting instance: " + instance_id + " for topology: " + topology_name + " and topologyId: " + topology_id + " for component: " + component_name + " with taskId: " + task_id + " and componentIndex: " + component_index + " and stmgrId: " + stmgr_id + " and stmgrPort: " + stmgr_port + " and metricsManagerPort: " + metrics_port + "\n **Topology Pex file located at: " + topology_pex_file_path) heron_instance = SingleThreadHeronInstance(topology_name, topology_id, instance, stmgr_port, metrics_port, topology_pex_file_path) heron_instance.start()
def __init__(self, looper, heron_instance_cls, strmgr_host, port, topology_name, topology_id, instance, sock_map, gateway_metrics, socket_options): HeronClient.__init__(self, looper, strmgr_host, port, sock_map, socket_options) self.heron_instance_cls = heron_instance_cls self.topology_name = topology_name self.topology_id = topology_id # physical_plan_pb2.Instance message self.instance = instance self.gateway_metrics = gateway_metrics self.sys_config = system_config.get_sys_config() self.is_registered = False
def __init__(self, looper, heron_instance_cls, strmgr_host, port, topology_name, topology_id, instance, sock_map, gateway_metrics, socket_options): HeronClient.__init__(self, looper, strmgr_host, port, sock_map, socket_options) self.heron_instance_cls = heron_instance_cls self.topology_name = topology_name self.topology_id = topology_id # physical_plan_pb2.Instance message self.instance = instance self.gateway_metrics = gateway_metrics self.sys_config = system_config.get_sys_config() self.is_registered = False
def __init__(self, topology_name, topology_id, instance, stream_port, metrics_port, topo_pex_file_path): # Basic information about this heron instance self.topology_name = topology_name self.topology_id = topology_id self.instance = instance self.stream_port = stream_port self.metrics_port = metrics_port self.topo_pex_file_abs_path = os.path.abspath(topo_pex_file_path) self.sys_config = system_config.get_sys_config() self.in_stream = HeronCommunicator(producer_cb=None, consumer_cb=None) self.out_stream = HeronCommunicator(producer_cb=None, consumer_cb=None) self.socket_map = dict() self.looper = GatewayLooper(self.socket_map) # Initialize metrics related self.out_metrics = HeronCommunicator() self.out_metrics.\ register_capacity(self.sys_config[constants.INSTANCE_INTERNAL_METRICS_WRITE_QUEUE_CAPACITY]) self.metrics_collector = MetricsCollector(self.looper, self.out_metrics) self.gateway_metrics = GatewayMetrics(self.metrics_collector) self.py_metrics = PyMetrics(self.metrics_collector) # Create socket options and socket clients socket_options = create_socket_options() self._stmgr_client = \ SingleThreadStmgrClient(self.looper, self, self.STREAM_MGR_HOST, stream_port, topology_name, topology_id, instance, self.socket_map, self.gateway_metrics, socket_options) self._metrics_client = \ MetricsManagerClient(self.looper, self.METRICS_MGR_HOST, metrics_port, instance, self.out_metrics, self.in_stream, self.out_stream, self.socket_map, socket_options, self.gateway_metrics, self.py_metrics) self.my_pplan_helper = None self.serializer = None # my_instance is a AssignedInstance tuple self.my_instance = None self.is_instance_started = False self.is_stateful_started = False self.stateful_state = None # Debugging purposes def go_trace(_, stack): with open("/tmp/trace.log", "w") as f: traceback.print_stack(stack, file=f) self.looper.register_timer_task_in_sec(self.looper.exit_loop, 0.0) signal.signal(signal.SIGUSR1, go_trace)
def __init__(self, metrics_collector): # total sys cpu time self.sys_cpu_time = AssignableMetrics(0) # total user cpu time self.user_cpu_time = AssignableMetrics(0) # threads cpu usage. Not supported # Add it back when we find an alternative to psutil # self.threads = MultiAssignableMetrics() # number of open file descriptors. Not supported # Add it back when we find an alternative to psutil # self.fd_nums = AssignableMetrics(0) # rss: aka "Resident Set Size" # this is the non-swapped physical memory a process has used. self.physical_memory = AssignableMetrics(0) # vms: "Virtual Memory Size", this is the total # amount of virtual memory used by the process. Not supported # Add it back when we find an alternative to psutil # self.virtual_memory = AssignableMetrics(0) # stats about three generations of GC # count is the number of objects in one generation # threshold is the collect frequency of one generation self.g1_count, self.g1_threshold = AssignableMetrics(0), AssignableMetrics(0) self.g2_count, self.g2_threshold = AssignableMetrics(0), AssignableMetrics(0) self.g3_count, self.g3_threshold = AssignableMetrics(0), AssignableMetrics(0) PY_SYS_CPU_TIME = '__py-sys-cpu-time-secs' PY_USER_CPU_TIME = '__py-user-cpu-time-secs' # PY_FD_NUMS = '__py-file-descriptors-number' PY_PHYSICAL_MEMORY = '__py-physical-memory-byte' # PY_VIRTUAL_MEMORY = '__py-virtual-memory-byte' PY_GC_GENERATION_1_COUNT = '__py-generation-1-count' PY_GC_GENERATION_2_COUNT = '__py-generation-2-count' PY_GC_GENERATION_3_COUNT = '__py-generation-3-count' PY_GC_GENERATION_1_THRESHOLD = '__py-generation-1-collection-threshold' PY_GC_GENERATION_2_THRESHOLD = '__py-generation-2-collection-threshold' PY_GC_GENERATION_3_THRESHOLD = '__py-generation-3-collection-threshold' self.metrics = {PY_SYS_CPU_TIME: self.sys_cpu_time, PY_USER_CPU_TIME: self.user_cpu_time, # PY_FD_NUMS: self.fd_nums, PY_PHYSICAL_MEMORY: self.physical_memory, # PY_VIRTUAL_MEMORY: self.virtual_memory, PY_GC_GENERATION_1_COUNT: self.g1_count, PY_GC_GENERATION_2_COUNT: self.g2_count, PY_GC_GENERATION_3_COUNT: self.g3_count, PY_GC_GENERATION_1_THRESHOLD: self.g1_threshold, PY_GC_GENERATION_2_THRESHOLD: self.g2_threshold, PY_GC_GENERATION_3_THRESHOLD: self.g3_threshold} super(PyMetrics, self).__init__(self.metrics) sys_config = system_config.get_sys_config() interval = float(sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC]) self.register_metrics(metrics_collector, interval)
def __init__(self, topology_name, topology_id, instance, stream_port, metrics_port, topo_pex_file_path): # Basic information about this heron instance self.topology_name = topology_name self.topology_id = topology_id self.instance = instance self.stream_port = stream_port self.metrics_port = metrics_port self.topo_pex_file_abs_path = os.path.abspath(topo_pex_file_path) self.sys_config = system_config.get_sys_config() self.in_stream = HeronCommunicator(producer_cb=None, consumer_cb=None) self.out_stream = HeronCommunicator(producer_cb=None, consumer_cb=None) self.socket_map = dict() self.looper = GatewayLooper(self.socket_map) # Initialize metrics related self.out_metrics = HeronCommunicator() self.out_metrics.\ register_capacity(self.sys_config[constants.INSTANCE_INTERNAL_METRICS_WRITE_QUEUE_CAPACITY]) self.metrics_collector = MetricsCollector(self.looper, self.out_metrics) self.gateway_metrics = GatewayMetrics(self.metrics_collector) self.py_metrics = PyMetrics(self.metrics_collector) # Create socket options and socket clients socket_options = create_socket_options() self._stmgr_client = \ SingleThreadStmgrClient(self.looper, self, self.STREAM_MGR_HOST, stream_port, topology_name, topology_id, instance, self.socket_map, self.gateway_metrics, socket_options) self._metrics_client = \ MetricsManagerClient(self.looper, self.METRICS_MGR_HOST, metrics_port, instance, self.out_metrics, self.in_stream, self.out_stream, self.socket_map, socket_options, self.gateway_metrics, self.py_metrics) self.my_pplan_helper = None self.serializer = None # my_instance is a AssignedInstance tuple self.my_instance = None self.is_instance_started = False self.is_stateful_started = False self.stateful_state = None # Debugging purposes def go_trace(_, stack): with open("/tmp/trace.log", "w") as f: traceback.print_stack(stack, file=f) self.looper.register_timer_task_in_sec(self.looper.exit_loop, 0.0) signal.signal(signal.SIGUSR1, go_trace)
def __init__(self, pplan_helper, in_stream, out_stream, looper): self.pplan_helper = pplan_helper self.in_stream = in_stream self.output_helper = OutgoingTupleHelper(self.pplan_helper, out_stream) self.looper = looper self.sys_config = system_config.get_sys_config() # will set a root logger here self.logger = logging.getLogger() context = pplan_helper.context mode = context.get_cluster_config().get(api_constants.TOPOLOGY_RELIABILITY_MODE, api_constants.TopologyReliabilityMode.ATMOST_ONCE) self.is_stateful = bool(mode == api_constants.TopologyReliabilityMode.EFFECTIVELY_ONCE) self._stateful_state = None self.serializer = SerializerHelper.get_serializer(pplan_helper.context) self._initialized_global_metrics = False
def __init__(self, pplan_helper, in_stream, out_stream, looper): self.pplan_helper = pplan_helper self.in_stream = in_stream self.output_helper = OutgoingTupleHelper(self.pplan_helper, out_stream) self.looper = looper self.sys_config = system_config.get_sys_config() # will set a root logger here self.logger = logging.getLogger() context = pplan_helper.context mode = context.get_cluster_config().get(api_constants.TOPOLOGY_RELIABILITY_MODE, api_constants.TopologyReliabilityMode.ATMOST_ONCE) self.is_stateful = bool(mode == api_constants.TopologyReliabilityMode.EFFECTIVELY_ONCE) self._stateful_state = None self.serializer = SerializerHelper.get_serializer(pplan_helper.context) self._initialized_global_metrics = False
def __init__(self, looper, metrics_host, port, instance, out_metrics, in_stream, out_stream, sock_map, socket_options, gateway_metrics, py_metrics): HeronClient.__init__(self, looper, metrics_host, port, sock_map, socket_options) self.instance = instance self.out_queue = out_metrics self.in_stream = in_stream self.out_stream = out_stream self.gateway_metrics = gateway_metrics self.py_metrics = py_metrics self.sys_config = system_config.get_sys_config() self._add_metrics_client_tasks() Log.debug('start updating in and out stream metrics') self._update_in_out_stream_metrics_tasks() self._update_py_metrics()
def __init__(self, pplan_helper, out_stream): self.out_stream = out_stream self.pplan_helper = pplan_helper self.current_data_tuple_set = None self.current_control_tuple_set = None self.current_data_tuple_size_in_bytes = 0 self.total_data_emitted_in_bytes = 0 self.sys_config = system_config.get_sys_config() # read the config values self.data_tuple_set_capacity = self.sys_config[constants.INSTANCE_SET_DATA_TUPLE_CAPACITY] self.max_data_tuple_size_in_bytes =\ self.sys_config.get(constants.INSTANCE_SET_DATA_TUPLE_SIZE_BYTES, sys.maxsize) self.control_tuple_set_capacity = self.sys_config[constants.INSTANCE_SET_CONTROL_TUPLE_CAPACITY]
def __init__(self, pplan_helper, out_stream): self.out_stream = out_stream self.pplan_helper = pplan_helper self.current_data_tuple_set = None self.current_control_tuple_set = None self.current_data_tuple_size_in_bytes = 0 self.total_data_emitted_in_bytes = 0 self.sys_config = system_config.get_sys_config() # read the config values self.data_tuple_set_capacity = self.sys_config[constants.INSTANCE_SET_DATA_TUPLE_CAPACITY] self.max_data_tuple_size_in_bytes =\ self.sys_config.get(constants.INSTANCE_SET_DATA_TUPLE_SIZE_BYTES, sys.maxsize) self.control_tuple_set_capacity = self.sys_config[constants.INSTANCE_SET_CONTROL_TUPLE_CAPACITY]
def __init__(self, looper, metrics_host, port, instance, out_metrics, in_stream, out_stream, sock_map, socket_options, gateway_metrics, py_metrics): HeronClient.__init__(self, looper, metrics_host, port, sock_map, socket_options) self.instance = instance self.out_queue = out_metrics self.in_stream = in_stream self.out_stream = out_stream self.gateway_metrics = gateway_metrics self.py_metrics = py_metrics self.sys_config = system_config.get_sys_config() self._add_metrics_client_tasks() Log.debug('start updating in and out stream metrics') self._update_in_out_stream_metrics_tasks() self._update_py_metrics()
def create_socket_options(): """Creates SocketOptions object from a given sys_config dict""" sys_config = system_config.get_sys_config() opt_list = [const.INSTANCE_NETWORK_WRITE_BATCH_SIZE_BYTES, const.INSTANCE_NETWORK_WRITE_BATCH_TIME_MS, const.INSTANCE_NETWORK_READ_BATCH_SIZE_BYTES, const.INSTANCE_NETWORK_READ_BATCH_TIME_MS, const.INSTANCE_NETWORK_OPTIONS_SOCKET_RECEIVED_BUFFER_SIZE_BYTES, const.INSTANCE_NETWORK_OPTIONS_SOCKET_SEND_BUFFER_SIZE_BYTES] Log.debug("In create_socket_options()") try: value_lst = [int(sys_config[opt]) for opt in opt_list] sock_opt = SocketOptions(*value_lst) return sock_opt except ValueError as e: # couldn't convert to int raise ValueError("Invalid value in sys_config: %s" % str(e)) except KeyError as e: # option key was not found raise KeyError("Incomplete sys_config: %s" % str(e))
def create_socket_options(): """Creates SocketOptions object from a given sys_config dict""" sys_config = system_config.get_sys_config() opt_list = [const.INSTANCE_NETWORK_WRITE_BATCH_SIZE_BYTES, const.INSTANCE_NETWORK_WRITE_BATCH_TIME_MS, const.INSTANCE_NETWORK_READ_BATCH_SIZE_BYTES, const.INSTANCE_NETWORK_READ_BATCH_TIME_MS, const.INSTANCE_NETWORK_OPTIONS_SOCKET_RECEIVED_BUFFER_SIZE_BYTES, const.INSTANCE_NETWORK_OPTIONS_SOCKET_SEND_BUFFER_SIZE_BYTES] Log.debug("In create_socket_options()") try: value_lst = [int(sys_config[opt]) for opt in opt_list] sock_opt = SocketOptions(*value_lst) return sock_opt except ValueError as e: # couldn't convert to int raise ValueError("Invalid value in sys_config: %s" % str(e)) except KeyError as e: # option key was not found raise KeyError("Incomplete sys_config: %s" % str(e))
def __init__(self, metrics_collector): sys_config = system_config.get_sys_config() super(GatewayMetrics, self).__init__(self.metrics) interval = float(sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC]) self.register_metrics(metrics_collector, interval)
def __init__(self, metrics_collector): sys_config = system_config.get_sys_config() super().__init__(self.metrics) interval = float( sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC]) self.register_metrics(metrics_collector, interval)
def cli( topology_name: str, topology_id: str, instance_id: str, component_name: str, task_id: int, component_index: int, stmgr_id: str, stmgr_port: int, metricsmgr_port: int, config_file: str, override_config_file: str, topology_pex: str, max_ram: int, ) -> None: """Heron Python Instance.""" sys_config = yaml_config_reader(config_file) override_config = yaml_config_reader(override_config_file) system_config.set_sys_config(sys_config, override_config) # get combined configuration sys_config = system_config.get_sys_config() # set resource limits set_resource_limit(max_ram) # create the protobuf instance instance_info = physical_plan_pb2.InstanceInfo() instance_info.task_id = task_id instance_info.component_index = component_index instance_info.component_name = component_name instance = physical_plan_pb2.Instance() instance.instance_id = instance_id instance.stmgr_id = stmgr_id instance.info.MergeFrom(instance_info) # Logging init log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY]) max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES] max_log_bytes = sys_config[ constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB log_file = os.path.join(log_dir, instance_id + ".log.0") log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=max_log_files, max_bytes=max_log_bytes) Log.info( f"\nStarting instance: {instance_id} for topology: {topology_name}" f" and topologyId: {topology_id} for component: {component_name}" f" with taskId: {task_id} and componentIndex: {component_index}" f" and stmgrId: {stmgr_id} and stmgrPort: {stmgr_port}" f" and metricsManagerPort: {metricsmgr_port}" f"\n **Topology Pex file located at: {topology_pex}") Log.debug(f"System config: {sys_config}") Log.debug(f"Override config: {override_config}") Log.debug(f"Maximum RAM: {max_ram}") heron_instance = SingleThreadHeronInstance(topology_name, topology_id, instance, stmgr_port, metricsmgr_port, topology_pex) heron_instance.start()
def main(): parser = argparse.ArgumentParser(description='Heron Python Instance') parser.add_argument('--topology_name', required=True, help='Topology Name') parser.add_argument('--topology_id', required=True, help='Topology Id') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--component_name', required=True, help='Component Name') parser.add_argument('--task_id', required=True, help='Task Id', type=int) parser.add_argument('--component_index', required=True, help='Component Index', type=int) parser.add_argument('--stmgr_id', required=True, help='StMgr Id') parser.add_argument('--stmgr_port', required=True, help='StMgr Port', type=int) parser.add_argument('--metricsmgr_port', required=True, help='MetricsMgr Port', type=int) parser.add_argument('--sys_config', required=True, help='System Config File') parser.add_argument('--override_config', required=True, help='Override Config File') parser.add_argument('--topology_pex', required=True, help='Topology Pex File') parser.add_argument('--max_ram', required=True, help='Maximum RAM to limit', type=int) args = parser.parse_args() sys_config = yaml_config_reader(args.sys_config) override_config = yaml_config_reader(args.override_config) system_config.set_sys_config(sys_config, override_config) # get combined configuration sys_config = system_config.get_sys_config() # set resource limits set_resource_limit(args.max_ram) # create the protobuf instance instance_info = physical_plan_pb2.InstanceInfo() instance_info.task_id = args.task_id instance_info.component_index = args.component_index instance_info.component_name = args.component_name instance = physical_plan_pb2.Instance() instance.instance_id = args.instance_id instance.stmgr_id = args.stmgr_id instance.info.MergeFrom(instance_info) # Logging init log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY]) max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES] max_log_bytes = sys_config[constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB log_file = os.path.join(log_dir, args.instance_id + ".log.0") log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=max_log_files, max_bytes=max_log_bytes) Log.info("\nStarting instance: " + args.instance_id + " for topology: " + args.topology_name + " and topologyId: " + args.topology_id + " for component: " + args.component_name + " with taskId: " + str(args.task_id) + " and componentIndex: " + str(args.component_index) + " and stmgrId: " + args.stmgr_id + " and stmgrPort: " + str(args.stmgr_port) + " and metricsManagerPort: " + str(args.metricsmgr_port) + "\n **Topology Pex file located at: " + args.topology_pex) Log.debug("System config: " + str(sys_config)) Log.debug("Override config: " + str(override_config)) Log.debug("Maximum RAM: " + str(args.max_ram)) heron_instance = SingleThreadHeronInstance(args.topology_name, args.topology_id, instance, args.stmgr_port, args.metricsmgr_port, args.topology_pex) heron_instance.start()
def main(): parser = argparse.ArgumentParser(description='Heron Python Instance') parser.add_argument('--topology_name', required=True, help='Topology Name') parser.add_argument('--topology_id', required=True, help='Topology Id') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--component_name', required=True, help='Component Name') parser.add_argument('--task_id', required=True, help='Task Id', type=int) parser.add_argument('--component_index', required=True, help='Component Index', type=int) parser.add_argument('--stmgr_id', required=True, help='StMgr Id') parser.add_argument('--stmgr_port', required=True, help='StMgr Port', type=int) parser.add_argument('--metricsmgr_port', required=True, help='MetricsMgr Port', type=int) parser.add_argument('--sys_config', required=True, help='System Config File') parser.add_argument('--override_config', required=True, help='Override Config File') parser.add_argument('--topology_pex', required=True, help='Topology Pex File') parser.add_argument('--max_ram', required=True, help='Maximum Ram to limit', type=int) args = parser.parse_args() sys_config = yaml_config_reader(args.sys_config) override_config = yaml_config_reader(args.override_config) system_config.set_sys_config(sys_config, override_config) # get combined configuration sys_config = system_config.get_sys_config() # set resource limits set_resource_limit(args.max_ram) # create the protobuf instance instance_info = physical_plan_pb2.InstanceInfo() instance_info.task_id = args.task_id instance_info.component_index = args.component_index instance_info.component_name = args.component_name instance = physical_plan_pb2.Instance() instance.instance_id = args.instance_id instance.stmgr_id = args.stmgr_id instance.info.MergeFrom(instance_info) # Logging init log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY]) max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES] max_log_bytes = sys_config[ constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB log_file = os.path.join(log_dir, args.instance_id + ".log.0") log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=max_log_files, max_bytes=max_log_bytes) Log.info("\nStarting instance: " + args.instance_id + " for topology: " + args.topology_name + " and topologyId: " + args.topology_id + " for component: " + args.component_name + " with taskId: " + str(args.task_id) + " and componentIndex: " + str(args.component_index) + " and stmgrId: " + args.stmgr_id + " and stmgrPort: " + str(args.stmgr_port) + " and metricsManagerPort: " + str(args.metricsmgr_port) + "\n **Topology Pex file located at: " + args.topology_pex) Log.debug("System config: " + str(sys_config)) Log.debug("Override config: " + str(override_config)) Log.debug("Maximum Ram: " + str(args.max_ram)) heron_instance = SingleThreadHeronInstance(args.topology_name, args.topology_id, instance, args.stmgr_port, args.metricsmgr_port, args.topology_pex) heron_instance.start()