Example #1
0
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file, topology_name):
  '''
  Launch a topology given topology jar, its definition file and configurations
  :param cl_args:
  :param tmp_dir:
  :param topology_file:
  :param topology_defn_file:
  :return:
  '''
  # get the normalized path for topology.tar.gz
  topology_pkg_path = config.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz'))

  # get the release yaml file
  release_yaml_file = config.get_heron_release_file()

  # create a tar package with the cluster configuration and generated config files
  config_path = cl_args['config_path']
  tar_pkg_files = [topology_file, topology_defn_file]
  generated_config_files = [release_yaml_file, cl_args['override_config_file']]

  config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files)

  # pass the args to submitter main
  args = [
      "--cluster", cl_args['cluster'],
      "--role", cl_args['role'],
      "--environment", cl_args['environ'],
      "--heron_home", config.get_heron_dir(),
      "--config_path", config_path,
      "--override_config_file", cl_args['override_config_file'],
      "--release_file", release_yaml_file,
      "--topology_package", topology_pkg_path,
      "--topology_defn", topology_defn_file,
      "--topology_bin", topology_file   # pex file if pex specified
  ]

  if Log.getEffectiveLevel() == logging.DEBUG:
    args.append("--verbose")

  lib_jars = config.get_heron_libs(
      jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()
  )
  extra_jars = cl_args['extra_launch_classpath'].split(':')

  # invoke the submitter to submit and launch the topology
  main_class = 'com.twitter.heron.scheduler.SubmitterMain'
  resp = execute.heron_class(
      class_name=main_class,
      lib_jars=lib_jars,
      extra_jars=extra_jars,
      args=args,
      java_defines=[])
  err_context = "Failed to launch topology '%s'" % topology_name
  succ_context = "Successfully launched topology '%s'" % topology_name
  resp.add_context(err_context, succ_context)
  return resp
Example #2
0
def run(command, parser, cl_args, unknown_args, action):
    '''
  helper function to take action on topologies
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :param action:        description of action taken
  :return:
  '''
    try:
        topology_name = cl_args['topology-name']

        new_args = [
            "--cluster",
            cl_args['cluster'],
            "--role",
            cl_args['role'],
            "--environment",
            cl_args['environ'],
            "--heron_home",
            config.get_heron_dir(),
            "--config_path",
            cl_args['config_path'],
            "--override_config_file",
            cl_args['override_config_file'],
            "--release_file",
            config.get_heron_release_file(),
            "--topology_name",
            topology_name,
            "--command",
            command,
        ]

        if Log.getEffectiveLevel() == logging.DEBUG:
            new_args.append("--verbose")

        lib_jars = config.get_heron_libs(jars.scheduler_jars() +
                                         jars.statemgr_jars())

        # invoke the runtime manager to kill the topology
        execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
                            lib_jars,
                            extra_jars=[],
                            args=new_args)

    except Exception:
        Log.error('Failed to %s \'%s\'' % (action, topology_name))
        return False

    Log.info('Successfully %s \'%s\'' % (action, topology_name))
    return True
Example #3
0
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
    '''
  helper function to take action on topologies
  :param command:
  :param cl_args:
  :param action:        description of action taken
  :return:
  '''
    topology_name = cl_args['topology-name']

    new_args = [
        "--cluster",
        cl_args['cluster'],
        "--role",
        cl_args['role'],
        "--environment",
        cl_args['environ'],
        "--submit_user",
        cl_args['submit_user'],
        "--heron_home",
        config.get_heron_dir(),
        "--config_path",
        cl_args['config_path'],
        "--override_config_file",
        cl_args['override_config_file'],
        "--release_file",
        config.get_heron_release_file(),
        "--topology_name",
        topology_name,
        "--command",
        command,
    ]
    new_args += extra_args

    lib_jars = config.get_heron_libs(jars.scheduler_jars() +
                                     jars.statemgr_jars())
    lib_jars += extra_lib_jars

    if Log.getEffectiveLevel() == logging.DEBUG:
        new_args.append("--verbose")

    # invoke the runtime manager to kill the topology
    result = execute.heron_class(
        'com.twitter.heron.scheduler.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args=new_args)

    err_msg = "Failed to %s: %s" % (action, topology_name)
    succ_msg = "Successfully %s: %s" % (action, topology_name)
    result.add_context(err_msg, succ_msg)
    return result
Example #4
0
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file):
  '''
  Launch a topology given topology jar, its definition file and configurations
  :param cl_args:
  :param tmp_dir:
  :param topology_file:
  :param topology_defn_file:
  :return:
  '''
  # get the normalized path for topology.tar.gz
  topology_pkg_path = config.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz'))

  # get the release yaml file
  release_yaml_file = config.get_heron_release_file()

  # create a tar package with the cluster configuration and generated config files
  config_path = cl_args['config_path']
  tar_pkg_files = [topology_file, topology_defn_file]
  generated_config_files = [release_yaml_file, cl_args['override_config_file']]

  config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files)

  # pass the args to submitter main
  args = [
      "--cluster", cl_args['cluster'],
      "--role", cl_args['role'],
      "--environment", cl_args['environ'],
      "--heron_home", config.get_heron_dir(),
      "--config_path", config_path,
      "--override_config_file", cl_args['override_config_file'],
      "--release_file", release_yaml_file,
      "--topology_package", topology_pkg_path,
      "--topology_defn", topology_defn_file,
      "--topology_bin", topology_file   # pex file if pex specified
  ]

  if Log.getEffectiveLevel() == logging.DEBUG:
    args.append("--verbose")

  lib_jars = config.get_heron_libs(
      jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()
  )

  # invoke the submitter to submit and launch the topology
  execute.heron_class(
      'com.twitter.heron.scheduler.SubmitterMain',
      lib_jars,
      extra_jars=[],
      args=args,
      java_defines=[]
  )
Example #5
0
def run(command, parser, cl_args, unknown_args):
    """ run the update command """
    topology_name = cl_args['topology-name']
    try:
        new_args = [
            "--cluster",
            cl_args['cluster'],
            "--role",
            cl_args['role'],
            "--environment",
            cl_args['environ'],
            "--heron_home",
            config.get_heron_dir(),
            "--config_path",
            cl_args['config_path'],
            "--override_config_file",
            cl_args['override_config_file'],
            "--release_file",
            config.get_heron_release_file(),
            "--topology_name",
            topology_name,
            "--command",
            command,
            "--component_parallelism",
            ','.join(cl_args['component_parallelism']),
        ]

        if Log.getEffectiveLevel() == logging.DEBUG:
            new_args.append("--verbose")

        lib_jars = config.get_heron_libs(jars.scheduler_jars() +
                                         jars.statemgr_jars() +
                                         jars.packing_jars())

        # invoke the runtime manager to kill the topology
        execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
                            lib_jars,
                            extra_jars=[],
                            args=new_args)

    except Exception as ex:
        Log.error('Failed to update topology \'%s\': %s', topology_name,
                  traceback.format_exc(ex))
        return False

    Log.info('Successfully updated topology \'%s\'' % topology_name)
    return True
Example #6
0
def run(command, parser, cl_args, unknown_args, action):
  '''
  helper function to take action on topologies
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :param action:        description of action taken
  :return:
  '''
  try:
    topology_name = cl_args['topology-name']

    new_args = [
        "--cluster", cl_args['cluster'],
        "--role", cl_args['role'],
        "--environment", cl_args['environ'],
        "--heron_home", config.get_heron_dir(),
        "--config_path", cl_args['config_path'],
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", config.get_heron_release_file(),
        "--topology_name", topology_name,
        "--command", command,
    ]

    if Log.getEffectiveLevel() == logging.DEBUG:
      new_args.append("--verbose")

    lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())

    # invoke the runtime manager to kill the topology
    execute.heron_class(
        'com.twitter.heron.scheduler.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args=new_args
    )

  except Exception:
    Log.error('Failed to %s \'%s\'' % (action, topology_name))
    return False

  Log.info('Successfully executed %s \'%s\'' % (action, topology_name))
  return True
Example #7
0
def run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
  '''
  helper function to take action on topologies
  :param command:
  :param cl_args:
  :param action:        description of action taken
  :return:
  '''
  topology_name = cl_args['topology-name']

  new_args = [
      "--cluster", cl_args['cluster'],
      "--role", cl_args['role'],
      "--environment", cl_args['environ'],
      "--submit_user", cl_args['submit_user'],
      "--heron_home", config.get_heron_dir(),
      "--config_path", cl_args['config_path'],
      "--override_config_file", cl_args['override_config_file'],
      "--release_file", config.get_heron_release_file(),
      "--topology_name", topology_name,
      "--command", command,
  ]
  new_args += extra_args

  lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())
  lib_jars += extra_lib_jars

  if Log.getEffectiveLevel() == logging.DEBUG:
    new_args.append("--verbose")

  # invoke the runtime manager to kill the topology
  result = execute.heron_class(
      'com.twitter.heron.scheduler.RuntimeManagerMain',
      lib_jars,
      extra_jars=[],
      args=new_args
  )

  err_msg = "Failed to %s: %s" % (action, topology_name)
  succ_msg = "Successfully %s: %s" % (action, topology_name)
  result.add_context(err_msg, succ_msg)
  return result
Example #8
0
def run(command, parser, cl_args, unknown_args):
  """ run the update command """
  topology_name = cl_args['topology-name']
  try:
    new_args = [
        "--cluster", cl_args['cluster'],
        "--role", cl_args['role'],
        "--environment", cl_args['environ'],
        "--heron_home", config.get_heron_dir(),
        "--config_path", cl_args['config_path'],
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", config.get_heron_release_file(),
        "--topology_name", topology_name,
        "--command", command,
        "--component_parallelism", ','.join(cl_args['component_parallelism']),
    ]

    if Log.getEffectiveLevel() == logging.DEBUG:
      new_args.append("--verbose")

    lib_jars = config.get_heron_libs(
        jars.scheduler_jars() + jars.statemgr_jars() + jars.packing_jars()
    )

    # invoke the runtime manager to kill the topology
    execute.heron_class(
        'com.twitter.heron.scheduler.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args=new_args
    )

  except Exception as ex:
    Log.error('Failed to update topology \'%s\': %s', topology_name, traceback.format_exc(ex))
    return False

  Log.info('Successfully updated topology \'%s\'' % topology_name)
  return True
Example #9
0
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file,
                      topology_name):
    '''
  Launch a topology given topology jar, its definition file and configurations
  :param cl_args:
  :param tmp_dir:
  :param topology_file:
  :param topology_defn_file:
  :return:
  '''
    # get the normalized path for topology.tar.gz
    topology_pkg_path = config.normalized_class_path(
        os.path.join(tmp_dir, 'topology.tar.gz'))

    # get the release yaml file
    release_yaml_file = config.get_heron_release_file()

    # create a tar package with the cluster configuration and generated config files
    config_path = cl_args['config_path']
    tar_pkg_files = [topology_file, topology_defn_file]
    generated_config_files = [
        release_yaml_file, cl_args['override_config_file']
    ]

    config.create_tar(topology_pkg_path, tar_pkg_files, config_path,
                      generated_config_files)

    # pass the args to submitter main
    args = [
        "--cluster",
        cl_args['cluster'],
        "--role",
        cl_args['role'],
        "--environment",
        cl_args['environ'],
        "--submit_user",
        cl_args['submit_user'],
        "--heron_home",
        config.get_heron_dir(),
        "--config_path",
        config_path,
        "--override_config_file",
        cl_args['override_config_file'],
        "--release_file",
        release_yaml_file,
        "--topology_package",
        topology_pkg_path,
        "--topology_defn",
        topology_defn_file,
        "--topology_bin",
        os.path.basename(topology_file)  # pex file if pex specified
    ]

    if Log.getEffectiveLevel() == logging.DEBUG:
        args.append("--verbose")

    if cl_args["dry_run"]:
        args.append("--dry_run")
        if "dry_run_format" in cl_args:
            args += ["--dry_run_format", cl_args["dry_run_format"]]

    lib_jars = config.get_heron_libs(jars.scheduler_jars() +
                                     jars.uploader_jars() +
                                     jars.statemgr_jars() +
                                     jars.packing_jars())
    extra_jars = cl_args['extra_launch_classpath'].split(':')

    # invoke the submitter to submit and launch the topology
    main_class = 'com.twitter.heron.scheduler.SubmitterMain'
    res = execute.heron_class(class_name=main_class,
                              lib_jars=lib_jars,
                              extra_jars=extra_jars,
                              args=args,
                              java_defines=[])
    err_context = "Failed to launch topology '%s'" % topology_name
    if cl_args["dry_run"]:
        err_context += " in dry-run mode"
    succ_context = "Successfully launched topology '%s'" % topology_name
    if cl_args["dry_run"]:
        succ_context += " in dry-run mode"
    res.add_context(err_context, succ_context)
    return res