def register_metrics(self, context):
"""Registers metrics to context
:param context: Topology Context
"""
sys_config = system_config.get_sys_config()
interval = float(sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC])
collector = context.get_metrics_collector()
super(ComponentMetrics, self).register_metrics(collector, interval)
def register_metrics(self, context):
"""Registers metrics to context
:param context: Topology Context
"""
sys_config = system_config.get_sys_config()
interval = float(sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC])
collector = context.get_metrics_collector()
super(ComponentMetrics, self).register_metrics(collector, interval)
def __init__(self, metrics_collector):
self.process = psutil.Process()
# total sys cpu time
self.sys_cpu_time = AssignableMetrics(0)
# total user cpu time
self.user_cpu_time = AssignableMetrics(0)
# threads cpu usage
self.threads = MultiAssignableMetrics()
# number of open file descriptors
self.fd_nums = AssignableMetrics(0)
# number of threads
self.num_threads = AssignableMetrics([])
# rss: aka "Resident Set Size"
# this is the non-swapped physical memory a process has used.
self.physical_memory = AssignableMetrics(0)
# vms: "Virtual Memory Size", this is the total
# amount of virtual memory used by the process.
self.virtual_memory = AssignableMetrics(0)
# stats about three generations of GC
# count is the number of objects in one generation
# threshold is the collect frequency of one generation
self.g1_count, self.g1_threshold = AssignableMetrics(
0), AssignableMetrics(0)
self.g2_count, self.g2_threshold = AssignableMetrics(
0), AssignableMetrics(0)
self.g3_count, self.g3_threshold = AssignableMetrics(
0), AssignableMetrics(0)
PY_SYS_CPU_TIME = '__py-sys-cpu-time-secs'
PY_USER_CPU_TIME = '__py-user-cpu-time-secs'
PY_FD_NUMS = '__py-file-descriptors-number'
PY_PHYSICAL_MEMORY = '__py-physical-memory-byte'
PY_VIRTUAL_MEMORY = '__py-virtual-memory-byte'
PY_GC_GENERATION_1_COUNT = '__py-generation-1-count'
PY_GC_GENERATION_2_COUNT = '__py-generation-2-count'
PY_GC_GENERATION_3_COUNT = '__py-generation-3-count'
PY_GC_GENERATION_1_THRESHOLD = '__py-generation-1-collection-threshold'
PY_GC_GENERATION_2_THRESHOLD = '__py-generation-2-collection-threshold'
PY_GC_GENERATION_3_THRESHOLD = '__py-generation-3-collection-threshold'
self.metrics = {
PY_SYS_CPU_TIME: self.sys_cpu_time,
PY_USER_CPU_TIME: self.user_cpu_time,
PY_FD_NUMS: self.fd_nums,
PY_PHYSICAL_MEMORY: self.physical_memory,
PY_VIRTUAL_MEMORY: self.virtual_memory,
PY_GC_GENERATION_1_COUNT: self.g1_count,
PY_GC_GENERATION_2_COUNT: self.g2_count,
PY_GC_GENERATION_3_COUNT: self.g3_count,
PY_GC_GENERATION_1_THRESHOLD: self.g1_threshold,
PY_GC_GENERATION_2_THRESHOLD: self.g2_threshold,
PY_GC_GENERATION_3_THRESHOLD: self.g3_threshold
}
super(PyMetrics, self).__init__(self.metrics)
sys_config = system_config.get_sys_config()
interval = float(
sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC])
self.register_metrics(metrics_collector, interval)
def main():
if len(sys.argv) != 13:
print_usage(sys.argv[0])
sys.exit(1)
topology_name = sys.argv[1]
topology_id = sys.argv[2]
instance_id = sys.argv[3]
component_name = sys.argv[4]
task_id = sys.argv[5]
component_index = sys.argv[6]
stmgr_id = sys.argv[7]
stmgr_port = sys.argv[8]
metrics_port = sys.argv[9]
sys_config = yaml_config_reader(sys.argv[10])
override_config = yaml_config_reader(sys.argv[11])
topology_pex_file_path = sys.argv[12]
Log.debug("System config: " + str(sys_config))
Log.debug("Override config: " + str(override_config))
system_config.set_sys_config(sys_config, override_config)
# get combined configuration
sys_config = system_config.get_sys_config()
# create the protobuf instance
instance_info = physical_plan_pb2.InstanceInfo()
instance_info.task_id = int(task_id)
instance_info.component_index = int(component_index)
instance_info.component_name = component_name
instance = physical_plan_pb2.Instance()
instance.instance_id = instance_id
instance.stmgr_id = stmgr_id
instance.info.MergeFrom(instance_info)
# Logging init
log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY])
max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES]
max_log_bytes = sys_config[constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB
log_file = os.path.join(log_dir, instance_id + ".log.0")
log.init_rotating_logger(level=logging.INFO, logfile=log_file,
max_files=max_log_files, max_bytes=max_log_bytes)
Log.info("\nStarting instance: " + instance_id + " for topology: " + topology_name +
" and topologyId: " + topology_id + " for component: " + component_name +
" with taskId: " + task_id + " and componentIndex: " + component_index +
" and stmgrId: " + stmgr_id + " and stmgrPort: " + stmgr_port +
" and metricsManagerPort: " + metrics_port +
"\n **Topology Pex file located at: " + topology_pex_file_path)
heron_instance = SingleThreadHeronInstance(topology_name, topology_id, instance, stmgr_port,
metrics_port, topology_pex_file_path)
heron_instance.start()
def __init__(self, looper, heron_instance_cls, strmgr_host, port, topology_name, topology_id,
instance, sock_map, gateway_metrics, socket_options):
HeronClient.__init__(self, looper, strmgr_host, port, sock_map, socket_options)
self.heron_instance_cls = heron_instance_cls
self.topology_name = topology_name
self.topology_id = topology_id
# physical_plan_pb2.Instance message
self.instance = instance
self.gateway_metrics = gateway_metrics
self.sys_config = system_config.get_sys_config()
self.is_registered = False
def __init__(self, looper, heron_instance_cls, strmgr_host, port, topology_name, topology_id,
instance, sock_map, gateway_metrics, socket_options):
HeronClient.__init__(self, looper, strmgr_host, port, sock_map, socket_options)
self.heron_instance_cls = heron_instance_cls
self.topology_name = topology_name
self.topology_id = topology_id
# physical_plan_pb2.Instance message
self.instance = instance
self.gateway_metrics = gateway_metrics
self.sys_config = system_config.get_sys_config()
self.is_registered = False
def __init__(self, topology_name, topology_id, instance, stream_port,
metrics_port, topo_pex_file_path):
# Basic information about this heron instance
self.topology_name = topology_name
self.topology_id = topology_id
self.instance = instance
self.stream_port = stream_port
self.metrics_port = metrics_port
self.topo_pex_file_abs_path = os.path.abspath(topo_pex_file_path)
self.sys_config = system_config.get_sys_config()
self.in_stream = HeronCommunicator(producer_cb=None, consumer_cb=None)
self.out_stream = HeronCommunicator(producer_cb=None, consumer_cb=None)
self.socket_map = dict()
self.looper = GatewayLooper(self.socket_map)
# Initialize metrics related
self.out_metrics = HeronCommunicator()
self.out_metrics.\
register_capacity(self.sys_config[constants.INSTANCE_INTERNAL_METRICS_WRITE_QUEUE_CAPACITY])
self.metrics_collector = MetricsCollector(self.looper,
self.out_metrics)
self.gateway_metrics = GatewayMetrics(self.metrics_collector)
self.py_metrics = PyMetrics(self.metrics_collector)
# Create socket options and socket clients
socket_options = create_socket_options()
self._stmgr_client = \
SingleThreadStmgrClient(self.looper, self, self.STREAM_MGR_HOST, stream_port,
topology_name, topology_id, instance, self.socket_map,
self.gateway_metrics, socket_options)
self._metrics_client = \
MetricsManagerClient(self.looper, self.METRICS_MGR_HOST, metrics_port, instance,
self.out_metrics, self.in_stream, self.out_stream,
self.socket_map, socket_options, self.gateway_metrics, self.py_metrics)
self.my_pplan_helper = None
self.serializer = None
# my_instance is a AssignedInstance tuple
self.my_instance = None
self.is_instance_started = False
self.is_stateful_started = False
self.stateful_state = None
# Debugging purposes
def go_trace(_, stack):
with open("/tmp/trace.log", "w") as f:
traceback.print_stack(stack, file=f)
self.looper.register_timer_task_in_sec(self.looper.exit_loop, 0.0)
signal.signal(signal.SIGUSR1, go_trace)
def __init__(self, metrics_collector):
# total sys cpu time
self.sys_cpu_time = AssignableMetrics(0)
# total user cpu time
self.user_cpu_time = AssignableMetrics(0)
# threads cpu usage. Not supported
# Add it back when we find an alternative to psutil
# self.threads = MultiAssignableMetrics()
# number of open file descriptors. Not supported
# Add it back when we find an alternative to psutil
# self.fd_nums = AssignableMetrics(0)
# rss: aka "Resident Set Size"
# this is the non-swapped physical memory a process has used.
self.physical_memory = AssignableMetrics(0)
# vms: "Virtual Memory Size", this is the total
# amount of virtual memory used by the process. Not supported
# Add it back when we find an alternative to psutil
# self.virtual_memory = AssignableMetrics(0)
# stats about three generations of GC
# count is the number of objects in one generation
# threshold is the collect frequency of one generation
self.g1_count, self.g1_threshold = AssignableMetrics(0), AssignableMetrics(0)
self.g2_count, self.g2_threshold = AssignableMetrics(0), AssignableMetrics(0)
self.g3_count, self.g3_threshold = AssignableMetrics(0), AssignableMetrics(0)
PY_SYS_CPU_TIME = '__py-sys-cpu-time-secs'
PY_USER_CPU_TIME = '__py-user-cpu-time-secs'
# PY_FD_NUMS = '__py-file-descriptors-number'
PY_PHYSICAL_MEMORY = '__py-physical-memory-byte'
# PY_VIRTUAL_MEMORY = '__py-virtual-memory-byte'
PY_GC_GENERATION_1_COUNT = '__py-generation-1-count'
PY_GC_GENERATION_2_COUNT = '__py-generation-2-count'
PY_GC_GENERATION_3_COUNT = '__py-generation-3-count'
PY_GC_GENERATION_1_THRESHOLD = '__py-generation-1-collection-threshold'
PY_GC_GENERATION_2_THRESHOLD = '__py-generation-2-collection-threshold'
PY_GC_GENERATION_3_THRESHOLD = '__py-generation-3-collection-threshold'
self.metrics = {PY_SYS_CPU_TIME: self.sys_cpu_time,
PY_USER_CPU_TIME: self.user_cpu_time,
# PY_FD_NUMS: self.fd_nums,
PY_PHYSICAL_MEMORY: self.physical_memory,
# PY_VIRTUAL_MEMORY: self.virtual_memory,
PY_GC_GENERATION_1_COUNT: self.g1_count,
PY_GC_GENERATION_2_COUNT: self.g2_count,
PY_GC_GENERATION_3_COUNT: self.g3_count,
PY_GC_GENERATION_1_THRESHOLD: self.g1_threshold,
PY_GC_GENERATION_2_THRESHOLD: self.g2_threshold,
PY_GC_GENERATION_3_THRESHOLD: self.g3_threshold}
super(PyMetrics, self).__init__(self.metrics)
sys_config = system_config.get_sys_config()
interval = float(sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC])
self.register_metrics(metrics_collector, interval)
def __init__(self, topology_name, topology_id, instance,
stream_port, metrics_port, topo_pex_file_path):
# Basic information about this heron instance
self.topology_name = topology_name
self.topology_id = topology_id
self.instance = instance
self.stream_port = stream_port
self.metrics_port = metrics_port
self.topo_pex_file_abs_path = os.path.abspath(topo_pex_file_path)
self.sys_config = system_config.get_sys_config()
self.in_stream = HeronCommunicator(producer_cb=None, consumer_cb=None)
self.out_stream = HeronCommunicator(producer_cb=None, consumer_cb=None)
self.socket_map = dict()
self.looper = GatewayLooper(self.socket_map)
# Initialize metrics related
self.out_metrics = HeronCommunicator()
self.out_metrics.\
register_capacity(self.sys_config[constants.INSTANCE_INTERNAL_METRICS_WRITE_QUEUE_CAPACITY])
self.metrics_collector = MetricsCollector(self.looper, self.out_metrics)
self.gateway_metrics = GatewayMetrics(self.metrics_collector)
self.py_metrics = PyMetrics(self.metrics_collector)
# Create socket options and socket clients
socket_options = create_socket_options()
self._stmgr_client = \
SingleThreadStmgrClient(self.looper, self, self.STREAM_MGR_HOST, stream_port,
topology_name, topology_id, instance, self.socket_map,
self.gateway_metrics, socket_options)
self._metrics_client = \
MetricsManagerClient(self.looper, self.METRICS_MGR_HOST, metrics_port, instance,
self.out_metrics, self.in_stream, self.out_stream,
self.socket_map, socket_options, self.gateway_metrics, self.py_metrics)
self.my_pplan_helper = None
self.serializer = None
# my_instance is a AssignedInstance tuple
self.my_instance = None
self.is_instance_started = False
self.is_stateful_started = False
self.stateful_state = None
# Debugging purposes
def go_trace(_, stack):
with open("/tmp/trace.log", "w") as f:
traceback.print_stack(stack, file=f)
self.looper.register_timer_task_in_sec(self.looper.exit_loop, 0.0)
signal.signal(signal.SIGUSR1, go_trace)
def __init__(self, pplan_helper, in_stream, out_stream, looper):
self.pplan_helper = pplan_helper
self.in_stream = in_stream
self.output_helper = OutgoingTupleHelper(self.pplan_helper, out_stream)
self.looper = looper
self.sys_config = system_config.get_sys_config()
# will set a root logger here
self.logger = logging.getLogger()
context = pplan_helper.context
mode = context.get_cluster_config().get(api_constants.TOPOLOGY_RELIABILITY_MODE,
api_constants.TopologyReliabilityMode.ATMOST_ONCE)
self.is_stateful = bool(mode == api_constants.TopologyReliabilityMode.EFFECTIVELY_ONCE)
self._stateful_state = None
self.serializer = SerializerHelper.get_serializer(pplan_helper.context)
self._initialized_global_metrics = False
def __init__(self, pplan_helper, in_stream, out_stream, looper):
self.pplan_helper = pplan_helper
self.in_stream = in_stream
self.output_helper = OutgoingTupleHelper(self.pplan_helper, out_stream)
self.looper = looper
self.sys_config = system_config.get_sys_config()
# will set a root logger here
self.logger = logging.getLogger()
context = pplan_helper.context
mode = context.get_cluster_config().get(api_constants.TOPOLOGY_RELIABILITY_MODE,
api_constants.TopologyReliabilityMode.ATMOST_ONCE)
self.is_stateful = bool(mode == api_constants.TopologyReliabilityMode.EFFECTIVELY_ONCE)
self._stateful_state = None
self.serializer = SerializerHelper.get_serializer(pplan_helper.context)
self._initialized_global_metrics = False
def __init__(self, looper, metrics_host, port, instance,
out_metrics, in_stream, out_stream, sock_map, socket_options,
gateway_metrics, py_metrics):
HeronClient.__init__(self, looper, metrics_host, port, sock_map, socket_options)
self.instance = instance
self.out_queue = out_metrics
self.in_stream = in_stream
self.out_stream = out_stream
self.gateway_metrics = gateway_metrics
self.py_metrics = py_metrics
self.sys_config = system_config.get_sys_config()
self._add_metrics_client_tasks()
Log.debug('start updating in and out stream metrics')
self._update_in_out_stream_metrics_tasks()
self._update_py_metrics()
def __init__(self, pplan_helper, out_stream):
self.out_stream = out_stream
self.pplan_helper = pplan_helper
self.current_data_tuple_set = None
self.current_control_tuple_set = None
self.current_data_tuple_size_in_bytes = 0
self.total_data_emitted_in_bytes = 0
self.sys_config = system_config.get_sys_config()
# read the config values
self.data_tuple_set_capacity = self.sys_config[constants.INSTANCE_SET_DATA_TUPLE_CAPACITY]
self.max_data_tuple_size_in_bytes =\
self.sys_config.get(constants.INSTANCE_SET_DATA_TUPLE_SIZE_BYTES, sys.maxsize)
self.control_tuple_set_capacity = self.sys_config[constants.INSTANCE_SET_CONTROL_TUPLE_CAPACITY]
def __init__(self, pplan_helper, out_stream):
self.out_stream = out_stream
self.pplan_helper = pplan_helper
self.current_data_tuple_set = None
self.current_control_tuple_set = None
self.current_data_tuple_size_in_bytes = 0
self.total_data_emitted_in_bytes = 0
self.sys_config = system_config.get_sys_config()
# read the config values
self.data_tuple_set_capacity = self.sys_config[constants.INSTANCE_SET_DATA_TUPLE_CAPACITY]
self.max_data_tuple_size_in_bytes =\
self.sys_config.get(constants.INSTANCE_SET_DATA_TUPLE_SIZE_BYTES, sys.maxsize)
self.control_tuple_set_capacity = self.sys_config[constants.INSTANCE_SET_CONTROL_TUPLE_CAPACITY]
def __init__(self, looper, metrics_host, port, instance, out_metrics,
in_stream, out_stream, sock_map, socket_options,
gateway_metrics, py_metrics):
HeronClient.__init__(self, looper, metrics_host, port, sock_map,
socket_options)
self.instance = instance
self.out_queue = out_metrics
self.in_stream = in_stream
self.out_stream = out_stream
self.gateway_metrics = gateway_metrics
self.py_metrics = py_metrics
self.sys_config = system_config.get_sys_config()
self._add_metrics_client_tasks()
Log.debug('start updating in and out stream metrics')
self._update_in_out_stream_metrics_tasks()
self._update_py_metrics()
def create_socket_options():
"""Creates SocketOptions object from a given sys_config dict"""
sys_config = system_config.get_sys_config()
opt_list = [const.INSTANCE_NETWORK_WRITE_BATCH_SIZE_BYTES,
const.INSTANCE_NETWORK_WRITE_BATCH_TIME_MS,
const.INSTANCE_NETWORK_READ_BATCH_SIZE_BYTES,
const.INSTANCE_NETWORK_READ_BATCH_TIME_MS,
const.INSTANCE_NETWORK_OPTIONS_SOCKET_RECEIVED_BUFFER_SIZE_BYTES,
const.INSTANCE_NETWORK_OPTIONS_SOCKET_SEND_BUFFER_SIZE_BYTES]
Log.debug("In create_socket_options()")
try:
value_lst = [int(sys_config[opt]) for opt in opt_list]
sock_opt = SocketOptions(*value_lst)
return sock_opt
except ValueError as e:
# couldn't convert to int
raise ValueError("Invalid value in sys_config: %s" % str(e))
except KeyError as e:
# option key was not found
raise KeyError("Incomplete sys_config: %s" % str(e))
def create_socket_options():
"""Creates SocketOptions object from a given sys_config dict"""
sys_config = system_config.get_sys_config()
opt_list = [const.INSTANCE_NETWORK_WRITE_BATCH_SIZE_BYTES,
const.INSTANCE_NETWORK_WRITE_BATCH_TIME_MS,
const.INSTANCE_NETWORK_READ_BATCH_SIZE_BYTES,
const.INSTANCE_NETWORK_READ_BATCH_TIME_MS,
const.INSTANCE_NETWORK_OPTIONS_SOCKET_RECEIVED_BUFFER_SIZE_BYTES,
const.INSTANCE_NETWORK_OPTIONS_SOCKET_SEND_BUFFER_SIZE_BYTES]
Log.debug("In create_socket_options()")
try:
value_lst = [int(sys_config[opt]) for opt in opt_list]
sock_opt = SocketOptions(*value_lst)
return sock_opt
except ValueError as e:
# couldn't convert to int
raise ValueError("Invalid value in sys_config: %s" % str(e))
except KeyError as e:
# option key was not found
raise KeyError("Incomplete sys_config: %s" % str(e))
def __init__(self, metrics_collector): sys_config = system_config.get_sys_config() super(GatewayMetrics, self).__init__(self.metrics) interval = float(sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC]) self.register_metrics(metrics_collector, interval)
def __init__(self, metrics_collector):
sys_config = system_config.get_sys_config()
super().__init__(self.metrics)
interval = float(
sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC])
self.register_metrics(metrics_collector, interval)
def cli(
topology_name: str,
topology_id: str,
instance_id: str,
component_name: str,
task_id: int,
component_index: int,
stmgr_id: str,
stmgr_port: int,
metricsmgr_port: int,
config_file: str,
override_config_file: str,
topology_pex: str,
max_ram: int,
) -> None:
"""Heron Python Instance."""
sys_config = yaml_config_reader(config_file)
override_config = yaml_config_reader(override_config_file)
system_config.set_sys_config(sys_config, override_config)
# get combined configuration
sys_config = system_config.get_sys_config()
# set resource limits
set_resource_limit(max_ram)
# create the protobuf instance
instance_info = physical_plan_pb2.InstanceInfo()
instance_info.task_id = task_id
instance_info.component_index = component_index
instance_info.component_name = component_name
instance = physical_plan_pb2.Instance()
instance.instance_id = instance_id
instance.stmgr_id = stmgr_id
instance.info.MergeFrom(instance_info)
# Logging init
log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY])
max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES]
max_log_bytes = sys_config[
constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB
log_file = os.path.join(log_dir, instance_id + ".log.0")
log.init_rotating_logger(level=logging.INFO,
logfile=log_file,
max_files=max_log_files,
max_bytes=max_log_bytes)
Log.info(
f"\nStarting instance: {instance_id} for topology: {topology_name}"
f" and topologyId: {topology_id} for component: {component_name}"
f" with taskId: {task_id} and componentIndex: {component_index}"
f" and stmgrId: {stmgr_id} and stmgrPort: {stmgr_port}"
f" and metricsManagerPort: {metricsmgr_port}"
f"\n **Topology Pex file located at: {topology_pex}")
Log.debug(f"System config: {sys_config}")
Log.debug(f"Override config: {override_config}")
Log.debug(f"Maximum RAM: {max_ram}")
heron_instance = SingleThreadHeronInstance(topology_name, topology_id,
instance, stmgr_port,
metricsmgr_port, topology_pex)
heron_instance.start()
def main():
parser = argparse.ArgumentParser(description='Heron Python Instance')
parser.add_argument('--topology_name', required=True, help='Topology Name')
parser.add_argument('--topology_id', required=True, help='Topology Id')
parser.add_argument('--instance_id', required=True, help='Instance Id')
parser.add_argument('--component_name', required=True, help='Component Name')
parser.add_argument('--task_id', required=True, help='Task Id', type=int)
parser.add_argument('--component_index', required=True, help='Component Index', type=int)
parser.add_argument('--stmgr_id', required=True, help='StMgr Id')
parser.add_argument('--stmgr_port', required=True, help='StMgr Port', type=int)
parser.add_argument('--metricsmgr_port', required=True, help='MetricsMgr Port', type=int)
parser.add_argument('--sys_config', required=True, help='System Config File')
parser.add_argument('--override_config', required=True, help='Override Config File')
parser.add_argument('--topology_pex', required=True, help='Topology Pex File')
parser.add_argument('--max_ram', required=True, help='Maximum RAM to limit', type=int)
args = parser.parse_args()
sys_config = yaml_config_reader(args.sys_config)
override_config = yaml_config_reader(args.override_config)
system_config.set_sys_config(sys_config, override_config)
# get combined configuration
sys_config = system_config.get_sys_config()
# set resource limits
set_resource_limit(args.max_ram)
# create the protobuf instance
instance_info = physical_plan_pb2.InstanceInfo()
instance_info.task_id = args.task_id
instance_info.component_index = args.component_index
instance_info.component_name = args.component_name
instance = physical_plan_pb2.Instance()
instance.instance_id = args.instance_id
instance.stmgr_id = args.stmgr_id
instance.info.MergeFrom(instance_info)
# Logging init
log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY])
max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES]
max_log_bytes = sys_config[constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB
log_file = os.path.join(log_dir, args.instance_id + ".log.0")
log.init_rotating_logger(level=logging.INFO, logfile=log_file,
max_files=max_log_files, max_bytes=max_log_bytes)
Log.info("\nStarting instance: " + args.instance_id + " for topology: " + args.topology_name +
" and topologyId: " + args.topology_id + " for component: " + args.component_name +
" with taskId: " + str(args.task_id) + " and componentIndex: " +
str(args.component_index) +
" and stmgrId: " + args.stmgr_id + " and stmgrPort: " + str(args.stmgr_port) +
" and metricsManagerPort: " + str(args.metricsmgr_port) +
"\n **Topology Pex file located at: " + args.topology_pex)
Log.debug("System config: " + str(sys_config))
Log.debug("Override config: " + str(override_config))
Log.debug("Maximum RAM: " + str(args.max_ram))
heron_instance = SingleThreadHeronInstance(args.topology_name, args.topology_id, instance,
args.stmgr_port, args.metricsmgr_port,
args.topology_pex)
heron_instance.start()
def main():
parser = argparse.ArgumentParser(description='Heron Python Instance')
parser.add_argument('--topology_name', required=True, help='Topology Name')
parser.add_argument('--topology_id', required=True, help='Topology Id')
parser.add_argument('--instance_id', required=True, help='Instance Id')
parser.add_argument('--component_name',
required=True,
help='Component Name')
parser.add_argument('--task_id', required=True, help='Task Id', type=int)
parser.add_argument('--component_index',
required=True,
help='Component Index',
type=int)
parser.add_argument('--stmgr_id', required=True, help='StMgr Id')
parser.add_argument('--stmgr_port',
required=True,
help='StMgr Port',
type=int)
parser.add_argument('--metricsmgr_port',
required=True,
help='MetricsMgr Port',
type=int)
parser.add_argument('--sys_config',
required=True,
help='System Config File')
parser.add_argument('--override_config',
required=True,
help='Override Config File')
parser.add_argument('--topology_pex',
required=True,
help='Topology Pex File')
parser.add_argument('--max_ram',
required=True,
help='Maximum Ram to limit',
type=int)
args = parser.parse_args()
sys_config = yaml_config_reader(args.sys_config)
override_config = yaml_config_reader(args.override_config)
system_config.set_sys_config(sys_config, override_config)
# get combined configuration
sys_config = system_config.get_sys_config()
# set resource limits
set_resource_limit(args.max_ram)
# create the protobuf instance
instance_info = physical_plan_pb2.InstanceInfo()
instance_info.task_id = args.task_id
instance_info.component_index = args.component_index
instance_info.component_name = args.component_name
instance = physical_plan_pb2.Instance()
instance.instance_id = args.instance_id
instance.stmgr_id = args.stmgr_id
instance.info.MergeFrom(instance_info)
# Logging init
log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY])
max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES]
max_log_bytes = sys_config[
constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB
log_file = os.path.join(log_dir, args.instance_id + ".log.0")
log.init_rotating_logger(level=logging.INFO,
logfile=log_file,
max_files=max_log_files,
max_bytes=max_log_bytes)
Log.info("\nStarting instance: " + args.instance_id + " for topology: " +
args.topology_name + " and topologyId: " + args.topology_id +
" for component: " + args.component_name + " with taskId: " +
str(args.task_id) + " and componentIndex: " +
str(args.component_index) + " and stmgrId: " + args.stmgr_id +
" and stmgrPort: " + str(args.stmgr_port) +
" and metricsManagerPort: " + str(args.metricsmgr_port) +
"\n **Topology Pex file located at: " + args.topology_pex)
Log.debug("System config: " + str(sys_config))
Log.debug("Override config: " + str(override_config))
Log.debug("Maximum Ram: " + str(args.max_ram))
heron_instance = SingleThreadHeronInstance(args.topology_name,
args.topology_id, instance,
args.stmgr_port,
args.metricsmgr_port,
args.topology_pex)
heron_instance.start()
