Ejemplo n.º 1
0
def submit_fatjar(cl_args, unknown_args, tmp_dir):

  # execute main of the topology to create the topology definition
  topology_file = cl_args['topology-file-name']
  try:
    execute.heron_class(
      cl_args['topology-class-name'],
      utils.get_heron_libs(jars.topology_jars()),
      extra_jars = [topology_file],
      args = tuple(unknown_args),
      javaDefines = cl_args['javaDefines'])

  except Exception as ex:
    Log.error("Unable to execute topology main class")
    return False

  try:
    launch_topologies(cl_args, topology_file, tmp_dir)

  except Exception as ex:
    return False

  finally:
    shutil.rmtree(tmp_dir)

  return True
Ejemplo n.º 2
0
def heron_tar(class_name, topology_tar, arguments, tmpdir_root, javaDefines):
    # Extract tar to a tmp folder.
    tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp')

    with contextlib.closing(tarfile.open(topology_tar)) as tar:
        tar.extractall(path=tmpdir)

    # A tar generated by pants has all dependency jars under libs/
    # in addition to the topology jar at top level. Pants keeps
    # filename for jar and tar the same except for extension.

    # TODO: We need to extract heron-core to separate folder.
    topology_jar = os.path.basename(topology_tar).replace(
        ".tar.gz", "").replace(".tar", "") + ".jar"

    extra_jars = [
        os.path.join(tmpdir, "heron-instance.jar"),
        os.path.join(tmpdir, topology_jar),
        os.path.join(tmpdir, "*"),
        os.path.join(tmpdir, "libs/*")
    ]

    lib_jars = utils.get_heron_libs(jars.topology_jars())

    # Now execute the class
    heron_class(class_name, lib_jars, extra_jars, arguments, javaDefines)
Ejemplo n.º 3
0
def heron_tar(class_name, topology_tar, arguments, tmpdir_root, javaDefines):
  # Extract tar to a tmp folder.
  tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp')

  with contextlib.closing(tarfile.open(topology_tar)) as tar:
    tar.extractall(path=tmpdir)

  # A tar generated by pants has all dependency jars under libs/
  # in addition to the topology jar at top level. Pants keeps
  # filename for jar and tar the same except for extension.

  # TODO: We need to extract heron-core to separate folder.
  topology_jar = os.path.basename(topology_tar).replace(".tar.gz", "").replace(".tar", "") + ".jar"

  extra_jars = [
      os.path.join(tmpdir, "heron-instance.jar"),
      os.path.join(tmpdir, topology_jar),
      os.path.join(tmpdir, "*"),
      os.path.join(tmpdir, "libs/*")
  ]

  lib_jars = utils.get_heron_libs(jars.topology_jars())

  # Now execute the class
  heron_class(class_name, lib_jars, extra_jars, arguments, javaDefines)
Ejemplo n.º 4
0
def run(command, parser, cl_args, unknown_args):

    try:
        topology_name = cl_args['topology-name']

        new_args = [
            "--cluster",
            cl_args['cluster'],
            "--role",
            cl_args['role'],
            "--environment",
            cl_args['environ'],
            "--heron_home",
            utils.get_heron_dir(),
            "--config_path",
            cl_args['config_path'],
            "--override_config_file",
            cl_args['override_config_file'],
            "--release_file",
            utils.get_heron_release_file(),
            "--topology_name",
            topology_name,
            "--command",
            command,
        ]

        if opts.verbose():
            new_args.append("--verbose")

        lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
                                        jars.statemgr_jars())

        # invoke the runtime manager to kill the topology
        execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
                            lib_jars,
                            extra_jars=[],
                            args=new_args)

    except Exception as ex:
        print 'Error: %s' % str(ex)
        Log.error('Failed to activate topology \'%s\'' % topology_name)
        return False

    Log.info('Successfully activated topology \'%s\'' % topology_name)
    return True
Ejemplo n.º 5
0
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file):

  # get the normalized path for topology.tar.gz
  topology_pkg_path = utils.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz'))

  # get the release yaml file
  release_yaml_file = utils.get_heron_release_file()

  # create a tar package with the cluster configuration and generated config files
  config_path = cl_args['config_path']
  tar_pkg_files = [topology_file, topology_defn_file]
  generated_config_files = [release_yaml_file, cl_args['override_config_file']]

  utils.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files)

  # pass the args to submitter main
  args = [
      "--cluster", cl_args['cluster'],
      "--role", cl_args['role'],
      "--environment", cl_args['environ'],
      "--heron_home", utils.get_heron_dir(),
      "--config_path", config_path,
      "--override_config_file", cl_args['override_config_file'],
      "--release_file", release_yaml_file,
      "--topology_package", topology_pkg_path,
      "--topology_defn", topology_defn_file,
      "--topology_jar", topology_file
  ]

  if opts.verbose():
    args.append("--verbose")

  lib_jars = utils.get_heron_libs(
      jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()
  )

  # invoke the submitter to submit and launch the topology
  execute.heron_class(
      'com.twitter.heron.scheduler.SubmitterMain',
      lib_jars,
      extra_jars=[],
      args = args,
      javaDefines = cl_args['javaDefines']
  )
Ejemplo n.º 6
0
def run(command, parser, cl_args, unknown_args):
    try:
        topology_name = cl_args["topology-name"]

        new_args = [
            "--cluster",
            cl_args["cluster"],
            "--role",
            cl_args["role"],
            "--environment",
            cl_args["environ"],
            "--heron_home",
            utils.get_heron_dir(),
            "--config_path",
            cl_args["config_path"],
            "--override_config_file",
            cl_args["override_config_file"],
            "--release_file",
            utils.get_heron_release_file(),
            "--topology_name",
            topology_name,
            "--command",
            command,
        ]

        if opts.verbose():
            new_args.append("--verbose")

        lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())

        # invoke the runtime manager to kill the topology
        execute.heron_class("com.twitter.heron.scheduler.RuntimeManagerMain", lib_jars, extra_jars=[], args=new_args)

    except Exception as ex:
        Log.error("Failed to kill topology '%s'" % topology_name)
        return False

    Log.info("Successfully killed topology '%s'" % topology_name)
    return True
Ejemplo n.º 7
0
def run(command, parser, cl_args, unknown_args):

  try:
    topology_name = cl_args['topology-name']

    new_args = [
        "--cluster", cl_args['cluster'],
        "--role", cl_args['role'],
        "--environment", cl_args['environ'],
        "--heron_home", utils.get_heron_dir(),
        "--config_path", cl_args['config_path'],
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", utils.get_heron_release_file(),
        "--topology_name", topology_name,
        "--command", command,
    ]

    if opts.verbose():
      new_args.append("--verbose")

    lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())

    # invoke the runtime manager to kill the topology
    execute.heron_class(
        'com.twitter.heron.scheduler.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args= new_args
    )

  except Exception as ex:
    print 'Error: %s' % str(ex)
    Log.error('Failed to deactivate topology \'%s\'' % topology_name)
    return False

  Log.info('Successfully deactivated topology \'%s\'' % topology_name)
  return True