def get_mock_instance(instance_id="instance_id", stmgr_id="Stmgr_id", info=get_mock_instance_info()): """Returns a mock protobuf Instance object from physical_plan_pb2""" instance = physical_plan_pb2.Instance() instance.instance_id = instance_id instance.stmgr_id = stmgr_id instance.info.CopyFrom(info) return instance
def main(): if len(sys.argv) != 12: print_usage(sys.argv[0]) sys.exit(1) topology_name = sys.argv[1] topology_id = sys.argv[2] instance_id = sys.argv[3] component_name = sys.argv[4] task_id = sys.argv[5] component_index = sys.argv[6] stmgr_id = sys.argv[7] stmgr_port = sys.argv[8] metrics_port = sys.argv[9] sys_config = yaml_config_reader(sys.argv[10]) topology_pex_file_path = sys.argv[11] system_config.set_sys_config(sys_config) # create the protobuf instance instance_info = physical_plan_pb2.InstanceInfo() instance_info.task_id = int(task_id) instance_info.component_index = int(component_index) instance_info.component_name = component_name instance = physical_plan_pb2.Instance() instance.instance_id = instance_id instance.stmgr_id = stmgr_id instance.info.MergeFrom(instance_info) # Logging init log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY]) max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES] max_log_bytes = sys_config[ constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB log_file = os.path.join(log_dir, instance_id + ".log.0") log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=max_log_files, max_bytes=max_log_bytes) Log.info("\nStarting instance: " + instance_id + " for topology: " + topology_name + " and topologyId: " + topology_id + " for component: " + component_name + " with taskId: " + task_id + " and componentIndex: " + component_index + " and stmgrId: " + stmgr_id + " and stmgrPort: " + stmgr_port + " and metricsManagerPort: " + metrics_port + "\n **Topology Pex file located at: " + topology_pex_file_path) Log.debug("System config: " + str(sys_config)) heron_instance = SingleThreadHeronInstance(topology_name, topology_id, instance, stmgr_port, metrics_port, topology_pex_file_path) heron_instance.start()
def create_mock_instance(self, i): instance = protoPPlan.Instance() instance.instance_id = f"mock_instance{i}" instance.stmgr_id = f"mock_stream{i}" instance_info = protoPPlan.InstanceInfo() instance_info.task_id = i instance_info.component_index = i instance_info.component_name = f"mock_spout" instance.info.CopyFrom(instance_info) return instance
def cli( topology_name: str, topology_id: str, instance_id: str, component_name: str, task_id: int, component_index: int, stmgr_id: str, stmgr_port: int, metricsmgr_port: int, config_file: str, override_config_file: str, topology_pex: str, max_ram: int, ) -> None: """Heron Python Instance.""" sys_config = yaml_config_reader(config_file) override_config = yaml_config_reader(override_config_file) system_config.set_sys_config(sys_config, override_config) # get combined configuration sys_config = system_config.get_sys_config() # set resource limits set_resource_limit(max_ram) # create the protobuf instance instance_info = physical_plan_pb2.InstanceInfo() instance_info.task_id = task_id instance_info.component_index = component_index instance_info.component_name = component_name instance = physical_plan_pb2.Instance() instance.instance_id = instance_id instance.stmgr_id = stmgr_id instance.info.MergeFrom(instance_info) # Logging init log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY]) max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES] max_log_bytes = sys_config[ constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB log_file = os.path.join(log_dir, instance_id + ".log.0") log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=max_log_files, max_bytes=max_log_bytes) Log.info( f"\nStarting instance: {instance_id} for topology: {topology_name}" f" and topologyId: {topology_id} for component: {component_name}" f" with taskId: {task_id} and componentIndex: {component_index}" f" and stmgrId: {stmgr_id} and stmgrPort: {stmgr_port}" f" and metricsManagerPort: {metricsmgr_port}" f"\n **Topology Pex file located at: {topology_pex}") Log.debug(f"System config: {sys_config}") Log.debug(f"Override config: {override_config}") Log.debug(f"Maximum RAM: {max_ram}") heron_instance = SingleThreadHeronInstance(topology_name, topology_id, instance, stmgr_port, metricsmgr_port, topology_pex) heron_instance.start()
def main(): parser = argparse.ArgumentParser(description='Heron Python Instance') parser.add_argument('--topology_name', required=True, help='Topology Name') parser.add_argument('--topology_id', required=True, help='Topology Id') parser.add_argument('--instance_id', required=True, help='Instance Id') parser.add_argument('--component_name', required=True, help='Component Name') parser.add_argument('--task_id', required=True, help='Task Id', type=int) parser.add_argument('--component_index', required=True, help='Component Index', type=int) parser.add_argument('--stmgr_id', required=True, help='StMgr Id') parser.add_argument('--stmgr_port', required=True, help='StMgr Port', type=int) parser.add_argument('--metricsmgr_port', required=True, help='MetricsMgr Port', type=int) parser.add_argument('--sys_config', required=True, help='System Config File') parser.add_argument('--override_config', required=True, help='Override Config File') parser.add_argument('--topology_pex', required=True, help='Topology Pex File') parser.add_argument('--max_ram', required=True, help='Maximum Ram to limit', type=int) args = parser.parse_args() sys_config = yaml_config_reader(args.sys_config) override_config = yaml_config_reader(args.override_config) system_config.set_sys_config(sys_config, override_config) # get combined configuration sys_config = system_config.get_sys_config() # set resource limits set_resource_limit(args.max_ram) # create the protobuf instance instance_info = physical_plan_pb2.InstanceInfo() instance_info.task_id = args.task_id instance_info.component_index = args.component_index instance_info.component_name = args.component_name instance = physical_plan_pb2.Instance() instance.instance_id = args.instance_id instance.stmgr_id = args.stmgr_id instance.info.MergeFrom(instance_info) # Logging init log_dir = os.path.abspath(sys_config[constants.HERON_LOGGING_DIRECTORY]) max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES] max_log_bytes = sys_config[ constants.HERON_LOGGING_MAXIMUM_SIZE_MB] * constants.MB log_file = os.path.join(log_dir, args.instance_id + ".log.0") log.init_rotating_logger(level=logging.INFO, logfile=log_file, max_files=max_log_files, max_bytes=max_log_bytes) Log.info("\nStarting instance: " + args.instance_id + " for topology: " + args.topology_name + " and topologyId: " + args.topology_id + " for component: " + args.component_name + " with taskId: " + str(args.task_id) + " and componentIndex: " + str(args.component_index) + " and stmgrId: " + args.stmgr_id + " and stmgrPort: " + str(args.stmgr_port) + " and metricsManagerPort: " + str(args.metricsmgr_port) + "\n **Topology Pex file located at: " + args.topology_pex) Log.debug("System config: " + str(sys_config)) Log.debug("Override config: " + str(override_config)) Log.debug("Maximum Ram: " + str(args.max_ram)) heron_instance = SingleThreadHeronInstance(args.topology_name, args.topology_id, instance, args.stmgr_port, args.metricsmgr_port, args.topology_pex) heron_instance.start()