Beispiel #1
0
def run(command, parser, cl_args, unknown_args, action):
    '''
  helper function to take action on topologies
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :param action:        description of action taken
  :return:
  '''
    try:
        topology_name = cl_args['topology-name']

        new_args = [
            "--cluster",
            cl_args['cluster'],
            "--role",
            cl_args['role'],
            "--environment",
            cl_args['environ'],
            "--heron_home",
            utils.get_heron_dir(),
            "--config_path",
            cl_args['config_path'],
            "--override_config_file",
            cl_args['override_config_file'],
            "--release_file",
            utils.get_heron_release_file(),
            "--topology_name",
            topology_name,
            "--command",
            command,
        ]

        if opts.verbose():
            new_args.append("--verbose")

        lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
                                        jars.statemgr_jars())

        # invoke the runtime manager to kill the topology
        execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
                            lib_jars,
                            extra_jars=[],
                            args=new_args)

    except Exception:
        Log.error('Failed to %s \'%s\'' % (action, topology_name))
        return False

    Log.info('Successfully %s \'%s\'' % (action, topology_name))
    return True
Beispiel #2
0
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file):
  '''
  Launch a topology given topology jar, its definition file and configurations
  :param cl_args:
  :param tmp_dir:
  :param topology_file:
  :param topology_defn_file:
  :return:
  '''
  # get the normalized path for topology.tar.gz
  topology_pkg_path = utils.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz'))

  # get the release yaml file
  release_yaml_file = utils.get_heron_release_file()

  # create a tar package with the cluster configuration and generated config files
  config_path = cl_args['config_path']
  tar_pkg_files = [topology_file, topology_defn_file]
  generated_config_files = [release_yaml_file, cl_args['override_config_file']]

  utils.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files)

  # pass the args to submitter main
  args = [
      "--cluster", cl_args['cluster'],
      "--role", cl_args['role'],
      "--environment", cl_args['environ'],
      "--heron_home", utils.get_heron_dir(),
      "--config_path", config_path,
      "--override_config_file", cl_args['override_config_file'],
      "--release_file", release_yaml_file,
      "--topology_package", topology_pkg_path,
      "--topology_defn", topology_defn_file,
      "--topology_jar", topology_file
  ]

  if opts.verbose():
    args.append("--verbose")

  lib_jars = utils.get_heron_libs(
      jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()
  )

  # invoke the submitter to submit and launch the topology
  execute.heron_class(
      'com.twitter.heron.scheduler.SubmitterMain',
      lib_jars,
      extra_jars=[],
      args=args,
      java_defines=[]
  )
Beispiel #3
0
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file):
    '''
  Launch a topology given topology jar, its definition file and configurations
  :param cl_args:
  :param tmp_dir:
  :param topology_file:
  :param topology_defn_file:
  :return:
  '''
    # get the normalized path for topology.tar.gz
    topology_pkg_path = utils.normalized_class_path(
        os.path.join(tmp_dir, 'topology.tar.gz'))

    # get the release yaml file
    release_yaml_file = utils.get_heron_release_file()

    # create a tar package with the cluster configuration and generated config files
    config_path = cl_args['config_path']
    tar_pkg_files = [topology_file, topology_defn_file]
    generated_config_files = [
        release_yaml_file, cl_args['override_config_file']
    ]

    utils.create_tar(topology_pkg_path, tar_pkg_files, config_path,
                     generated_config_files)

    # pass the args to submitter main
    args = [
        "--cluster", cl_args['cluster'], "--role", cl_args['role'],
        "--environment", cl_args['environ'], "--heron_home",
        utils.get_heron_dir(), "--config_path", config_path,
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", release_yaml_file, "--topology_package",
        topology_pkg_path, "--topology_defn", topology_defn_file,
        "--topology_jar", topology_file
    ]

    if opts.verbose():
        args.append("--verbose")

    lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
                                    jars.uploader_jars() +
                                    jars.statemgr_jars() + jars.packing_jars())

    # invoke the submitter to submit and launch the topology
    execute.heron_class('com.twitter.heron.scheduler.SubmitterMain',
                        lib_jars,
                        extra_jars=[],
                        args=args,
                        java_defines=[])
Beispiel #4
0
def run(command, parser, cl_args, unknown_args, action):
  '''
  helper function to take action on topologies
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :param action:        description of action taken
  :return:
  '''
  try:
    topology_name = cl_args['topology-name']

    new_args = [
        "--cluster", cl_args['cluster'],
        "--role", cl_args['role'],
        "--environment", cl_args['environ'],
        "--heron_home", utils.get_heron_dir(),
        "--config_path", cl_args['config_path'],
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", utils.get_heron_release_file(),
        "--topology_name", topology_name,
        "--command", command,
    ]

    if opts.verbose():
      new_args.append("--verbose")

    lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())

    # invoke the runtime manager to kill the topology
    execute.heron_class(
        'com.twitter.heron.scheduler.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args=new_args
    )

  except Exception:
    Log.error('Failed to %s \'%s\'' % (action, topology_name))
    return False

  Log.info('Successfully %s \'%s\'' % (action, topology_name))
  return True
Beispiel #5
0
def add_config(parser):
  '''
  :param parser:
  :return:
  '''
  # the default config path
  default_config_path = utils.get_heron_conf_dir()

  parser.add_argument(
      '--config-path',
      metavar='(a string; path to cluster config; default: "' + default_config_path + '")',
      default=os.path.join(utils.get_heron_dir(), default_config_path))

  parser.add_argument(
      '--config-property',
      metavar='(key=value; a config key and its value; default: [])',
      action='append',
      default=[])
  return parser
Beispiel #6
0
def run(command, parser, cl_args, unknown_args):
  '''
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :return:
  '''
  try:
    topology_name = cl_args['topology-name']
    container_id = cl_args['container-id']

    new_args = [
        "--cluster", cl_args['cluster'],
        "--role", cl_args['role'],
        "--environment", cl_args['environ'],
        "--heron_home", utils.get_heron_dir(),
        "--config_path", cl_args['config_path'],
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", utils.get_heron_release_file(),
        "--topology_name", topology_name,
        "--command", command,
        "--container_id", str(container_id)
    ]

    lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())

    # invoke the runtime manager to kill the topology
    execute.heron_class(
        'com.twitter.heron.scheduler.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args=new_args
    )

  except Exception as ex:
    print 'Error: %s' % str(ex)
    Log.error('Failed to restart topology \'%s\'' % topology_name)
    return False

  Log.info('Successfully restarted topology \'%s\'' % topology_name)
  return True
Beispiel #7
0
def add_config(parser):
    '''
  :param parser:
  :return:
  '''
    # the default config path
    default_config_path = utils.get_heron_conf_dir()

    parser.add_argument(
        '--config-path',
        metavar='(a string; path to cluster config; default: "' +
        default_config_path + '")',
        default=os.path.join(utils.get_heron_dir(), default_config_path))

    parser.add_argument(
        '--config-property',
        metavar='(key=value; a config key and its value; default: [])',
        action='append',
        default=[])
    return parser
Beispiel #8
0
def run(command, parser, cl_args, unknown_args):
    '''
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :return:
  '''
    try:
        topology_name = cl_args['topology-name']
        container_id = cl_args['container-id']

        new_args = [
            "--cluster", cl_args['cluster'], "--role", cl_args['role'],
            "--environment", cl_args['environ'], "--heron_home",
            utils.get_heron_dir(), "--config_path", cl_args['config_path'],
            "--override_config_file", cl_args['override_config_file'],
            "--release_file",
            utils.get_heron_release_file(), "--topology_name", topology_name,
            "--command", command, "--container_id",
            str(container_id)
        ]

        lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
                                        jars.statemgr_jars())

        # invoke the runtime manager to kill the topology
        execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
                            lib_jars,
                            extra_jars=[],
                            args=new_args)

    except Exception as ex:
        print 'Error: %s' % str(ex)
        Log.error('Failed to restart topology \'%s\'' % topology_name)
        return False

    Log.info('Successfully restarted topology \'%s\'' % topology_name)
    return True