def get_mock_instance(instance_id="instance_id",
stmgr_id="Stmgr_id",
info=get_mock_instance_info()):
"""Returns a mock protobuf Instance object from physical_plan_pb2"""
instance = physical_plan_pb2.Instance()
instance.instance_id = instance_id
instance.stmgr_id = stmgr_id
instance.info.CopyFrom(info)
return instance
def main():
if len(sys.argv) != 12:
print_usage(sys.argv[0])
sys.exit(1)
topology_name = sys.argv[1]
topology_id = sys.argv[2]
instance_id = sys.argv[3]
component_name = sys.argv[4]
task_id = sys.argv[5]
component_index = sys.argv[6]
stmgr_id = sys.argv[7]
stmgr_port = sys.argv[8]
metrics_port = sys.argv[9]
sys_config = yaml_config_reader(sys.argv[10])
topology_pex_file_path = sys.argv[11]
system_config.set_sys_config(sys_config)
# create the protobuf instance
instance_info = physical_plan_pb2.InstanceInfo()
instance_info.task_id = int(task_id)
instance_info.component_index = int(component_index)
instance_info.component_name = component_name
instance = physical_plan_pb2.Instance()
instance.instance_id = instance_id
instance.stmgr_id = stmgr_id
instance.info.MergeFrom(instance_info)
# Logging init
log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY])
max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES]
max_log_bytes = sys_config[
constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB
log_file = os.path.join(log_dir, instance_id + ".log.0")
log.init_rotating_logger(level=logging.INFO,
logfile=log_file,
max_files=max_log_files,
max_bytes=max_log_bytes)
Log.info("\nStarting instance: " + instance_id + " for topology: " +
topology_name + " and topologyId: " + topology_id +
" for component: " + component_name + " with taskId: " + task_id +
" and componentIndex: " + component_index + " and stmgrId: " +
stmgr_id + " and stmgrPort: " + stmgr_port +
" and metricsManagerPort: " + metrics_port +
"\n **Topology Pex file located at: " + topology_pex_file_path)
Log.debug("System config: " + str(sys_config))
heron_instance = SingleThreadHeronInstance(topology_name, topology_id,
instance, stmgr_port,
metrics_port,
topology_pex_file_path)
heron_instance.start()
def create_mock_instance(self, i):
instance = protoPPlan.Instance()
instance.instance_id = f"mock_instance{i}"
instance.stmgr_id = f"mock_stream{i}"
instance_info = protoPPlan.InstanceInfo()
instance_info.task_id = i
instance_info.component_index = i
instance_info.component_name = f"mock_spout"
instance.info.CopyFrom(instance_info)
return instance
def cli(
topology_name: str,
topology_id: str,
instance_id: str,
component_name: str,
task_id: int,
component_index: int,
stmgr_id: str,
stmgr_port: int,
metricsmgr_port: int,
config_file: str,
override_config_file: str,
topology_pex: str,
max_ram: int,
) -> None:
"""Heron Python Instance."""
sys_config = yaml_config_reader(config_file)
override_config = yaml_config_reader(override_config_file)
system_config.set_sys_config(sys_config, override_config)
# get combined configuration
sys_config = system_config.get_sys_config()
# set resource limits
set_resource_limit(max_ram)
# create the protobuf instance
instance_info = physical_plan_pb2.InstanceInfo()
instance_info.task_id = task_id
instance_info.component_index = component_index
instance_info.component_name = component_name
instance = physical_plan_pb2.Instance()
instance.instance_id = instance_id
instance.stmgr_id = stmgr_id
instance.info.MergeFrom(instance_info)
# Logging init
log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY])
max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES]
max_log_bytes = sys_config[
constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB
log_file = os.path.join(log_dir, instance_id + ".log.0")
log.init_rotating_logger(level=logging.INFO,
logfile=log_file,
max_files=max_log_files,
max_bytes=max_log_bytes)
Log.info(
f"\nStarting instance: {instance_id} for topology: {topology_name}"
f" and topologyId: {topology_id} for component: {component_name}"
f" with taskId: {task_id} and componentIndex: {component_index}"
f" and stmgrId: {stmgr_id} and stmgrPort: {stmgr_port}"
f" and metricsManagerPort: {metricsmgr_port}"
f"\n **Topology Pex file located at: {topology_pex}")
Log.debug(f"System config: {sys_config}")
Log.debug(f"Override config: {override_config}")
Log.debug(f"Maximum RAM: {max_ram}")
heron_instance = SingleThreadHeronInstance(topology_name, topology_id,
instance, stmgr_port,
metricsmgr_port, topology_pex)
heron_instance.start()
def main():
parser = argparse.ArgumentParser(description='Heron Python Instance')
parser.add_argument('--topology_name', required=True, help='Topology Name')
parser.add_argument('--topology_id', required=True, help='Topology Id')
parser.add_argument('--instance_id', required=True, help='Instance Id')
parser.add_argument('--component_name',
required=True,
help='Component Name')
parser.add_argument('--task_id', required=True, help='Task Id', type=int)
parser.add_argument('--component_index',
required=True,
help='Component Index',
type=int)
parser.add_argument('--stmgr_id', required=True, help='StMgr Id')
parser.add_argument('--stmgr_port',
required=True,
help='StMgr Port',
type=int)
parser.add_argument('--metricsmgr_port',
required=True,
help='MetricsMgr Port',
type=int)
parser.add_argument('--sys_config',
required=True,
help='System Config File')
parser.add_argument('--override_config',
required=True,
help='Override Config File')
parser.add_argument('--topology_pex',
required=True,
help='Topology Pex File')
parser.add_argument('--max_ram',
required=True,
help='Maximum Ram to limit',
type=int)
args = parser.parse_args()
sys_config = yaml_config_reader(args.sys_config)
override_config = yaml_config_reader(args.override_config)
system_config.set_sys_config(sys_config, override_config)
# get combined configuration
sys_config = system_config.get_sys_config()
# set resource limits
set_resource_limit(args.max_ram)
# create the protobuf instance
instance_info = physical_plan_pb2.InstanceInfo()
instance_info.task_id = args.task_id
instance_info.component_index = args.component_index
instance_info.component_name = args.component_name
instance = physical_plan_pb2.Instance()
instance.instance_id = args.instance_id
instance.stmgr_id = args.stmgr_id
instance.info.MergeFrom(instance_info)
# Logging init
log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY])
max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES]
max_log_bytes = sys_config[
constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB
log_file = os.path.join(log_dir, args.instance_id + ".log.0")
log.init_rotating_logger(level=logging.INFO,
logfile=log_file,
max_files=max_log_files,
max_bytes=max_log_bytes)
Log.info("\nStarting instance: " + args.instance_id + " for topology: " +
args.topology_name + " and topologyId: " + args.topology_id +
" for component: " + args.component_name + " with taskId: " +
str(args.task_id) + " and componentIndex: " +
str(args.component_index) + " and stmgrId: " + args.stmgr_id +
" and stmgrPort: " + str(args.stmgr_port) +
" and metricsManagerPort: " + str(args.metricsmgr_port) +
"\n **Topology Pex file located at: " + args.topology_pex)
Log.debug("System config: " + str(sys_config))
Log.debug("Override config: " + str(override_config))
Log.debug("Maximum Ram: " + str(args.max_ram))
heron_instance = SingleThreadHeronInstance(args.topology_name,
args.topology_id, instance,
args.stmgr_port,
args.metricsmgr_port,
args.topology_pex)
heron_instance.start()
