Exemple #1
0
def task_descriptor_for_package_and_initial_task(package_dict, start_handler,
                                                 start_args, package_path,
                                                 master_uri, args):
    def resolve_arg(value):
        try:
            return args[value["__args__"]]
        except IndexError:
            if "default" in value:
                print >> sys.stderr, "Positional argument", value[
                    "__args__"], "not specified; using default", value[
                        "default"]
                return value["default"]
            else:
                print >> sys.stderr, "Mandatory argument", value[
                    "__args__"], "not specified."
                sys.exit(-1)

    def resolve_env(value):
        try:
            return os.environ[value["__env__"]]
        except KeyError:
            if "default" in value:
                print >> sys.stderr, "Environment variable", value[
                    "__env__"], "not specified; using default", value[
                        "default"]
                return value["default"]
            else:
                print >> sys.stderr, "Mandatory environment variable", value[
                    "__env__"], "not specified."
                sys.exit(-1)

    env_and_args_callbacks = {"__args__": resolve_arg, "__env__": resolve_env}
    package_dict = resolve_vars(package_dict, env_and_args_callbacks)
    start_args = resolve_vars(start_args, env_and_args_callbacks)

    submit_package_dict = dict([(k,
                                 ref_of_object(k, v, package_path, master_uri))
                                for (k, v) in package_dict.items()])
    #for key, ref in submit_package_dict.items():
    #    print >>sys.stderr, key, '-->', simplejson.dumps(ref, cls=SWReferenceJSONEncoder)
    package_ref = ref_of_string(pickle.dumps(submit_package_dict), master_uri)

    resolved_args = resolve_vars(
        start_args,
        {"__package__": lambda x: submit_package_dict[x["__package__"]]})

    return build_init_descriptor(start_handler, resolved_args, package_ref,
                                 master_uri, ref_of_string)
Exemple #2
0
def task_descriptor_for_package_and_initial_task(package_dict, start_handler, start_args, package_path, master_uri, args):

    def resolve_arg(value):
        try:
            return args[value["__args__"]]
        except IndexError:
            if "default" in value:
                print >>sys.stderr, "Positional argument", value["__args__"], "not specified; using default", value["default"]
                return value["default"]
            else:
                print >>sys.stderr, "Mandatory argument", value["__args__"], "not specified."
                sys.exit(-1)

    def resolve_env(value):
        try:
            return os.environ[value["__env__"]]
        except KeyError:
            if "default" in value:
                print >>sys.stderr, "Environment variable", value["__env__"], "not specified; using default", value["default"]
                return value["default"]
            else:
                print >>sys.stderr, "Mandatory environment variable", value["__env__"], "not specified."
                sys.exit(-1)

    env_and_args_callbacks = {"__args__": resolve_arg,
                              "__env__": resolve_env}
    package_dict = resolve_vars(package_dict, env_and_args_callbacks)
    start_args = resolve_vars(start_args, env_and_args_callbacks)

    submit_package_dict = dict([(k, ref_of_object(k, v, package_path, master_uri)) for (k, v) in package_dict.items()])
    #for key, ref in submit_package_dict.items():
    #    print >>sys.stderr, key, '-->', simplejson.dumps(ref, cls=SWReferenceJSONEncoder)
    package_ref = ref_of_string(pickle.dumps(submit_package_dict), master_uri)

    resolved_args = resolve_vars(start_args, {"__package__": lambda x: submit_package_dict[x["__package__"]]})

    return build_init_descriptor(start_handler, resolved_args, package_ref, master_uri, ref_of_string)
Exemple #3
0
def jar(my_args=sys.argv):

    parser = OptionParser(usage='Usage: ciel jar [options] JAR_FILE CLASS_NAME [args...]')
    parser.add_option("-m", "--master", action="store", dest="master", help="URI of the cluster master", metavar="MASTER", default=ciel.config.get('cluster', 'master', 'http://localhost:8000'))
    parser.add_option("-j", "--extra-jar", action="append", dest="extra_jars", help="Filename of additional JAR to load", metavar="JAR_FILE", default=[])
    parser.add_option("-P", "--package", action="append", dest="package", help="Additional file to upload", metavar="ID=FILENAME", default=[])
    parser.add_option("-n", "--num-outputs", action="store", dest="num_outputs", help="Number of outputs for root task", type="int", metavar="N", default=1)
    parser.add_option("-L", "--jar-lib", action="store", dest="jar_lib", help="Directory containing CIEL bindings JARs", type="str", metavar="PATH", default=ciel.config.get('java', 'jar_lib'))
 
    (options, args) = parser.parse_args(args=my_args)
    master_uri = options.master

    if master_uri is None or master_uri == "":
        print >>sys.stderr, ("Must specify a master with -m or `ciel config --set cluster.master URL`")
        sys.exit(-1)
    elif len(args) < 2:
        print >>sys.stderr, "Must specify a fully-qualified class to run"
        parser.print_help()
        sys.exit(-1)

    jars = options.extra_jars + [args[1]]

    # Consult the config to see where the standard JARs are installed.
    jar_path = options.jar_lib
    if jar_path is None:
        print >>sys.stderr, "Could not find CIEL bindings. Set the JAR libary path using one of:"
        print >>sys.stderr, "\tciel jar (--jar-lib|-L) PATH ..."
        print >>sys.stderr, "\tciel config --set java.jar_lib PATH"
        sys.exit(-1)

    for jar_file in glob.glob(os.path.join(jar_path, '*.jar')):
        jars = jars + [jar_file]

    class_name = args[2]
    args = args[3:]

    def upload_jar(filename):
        with open(filename, 'r') as infile:
            return ref_of_string(infile.read(), master_uri)
        
    jar_refs = [upload_jar(j) for j in jars]

    package_dict = {}
    for binding in options.package:
        id, filename = binding.split("=", 2)
        with open(filename, 'r') as infile:
            package_dict[id] = ref_of_string(infile.read(), master_uri)

    package_ref = ref_of_string(pickle.dumps(package_dict), master_uri)

    args = {'jar_lib' : jar_refs,
            'class_name' : class_name,
            'args' : args,
            'n_outputs' : options.num_outputs}

    init_descriptor = build_init_descriptor("java2", args, package_ref, master_uri, ref_of_string)

    job_descriptor = submit_job_for_task(init_descriptor, master_uri)

    job_url = urlparse.urljoin(master_uri, "control/browse/job/%s" % job_descriptor['job_id'])

    result = await_job(job_descriptor['job_id'], master_uri)


    try:
        reflist = simple_retrieve_object_for_ref(result, "json", job_descriptor['job_id'], master_uri)
    except:
        print >>sys.stderr, "Error getting list of references as a result."

    try:
        j_return = retrieve_object_for_ref(reflist[0], "json", None)
    except:
        try:
            j_return = retrieve_object_for_ref(reflist[0], "noop", None)
        except:
            print >>sys.stderr, "Error parsing job result."
            sys.exit(-1)

    print j_return
Exemple #4
0
def jar(my_args=sys.argv):

    parser = OptionParser(
        usage='Usage: ciel jar [options] JAR_FILE CLASS_NAME [args...]')
    parser.add_option("-m",
                      "--master",
                      action="store",
                      dest="master",
                      help="URI of the cluster master",
                      metavar="MASTER",
                      default=ciel.config.get('cluster', 'master',
                                              'http://localhost:8000'))
    parser.add_option("-j",
                      "--extra-jar",
                      action="append",
                      dest="extra_jars",
                      help="Filename of additional JAR to load",
                      metavar="JAR_FILE",
                      default=[])
    parser.add_option("-P",
                      "--package",
                      action="append",
                      dest="package",
                      help="Additional file to upload",
                      metavar="ID=FILENAME",
                      default=[])
    parser.add_option("-n",
                      "--num-outputs",
                      action="store",
                      dest="num_outputs",
                      help="Number of outputs for root task",
                      type="int",
                      metavar="N",
                      default=1)
    parser.add_option("-L",
                      "--jar-lib",
                      action="store",
                      dest="jar_lib",
                      help="Directory containing CIEL bindings JARs",
                      type="str",
                      metavar="PATH",
                      default=ciel.config.get('java', 'jar_lib'))

    (options, args) = parser.parse_args(args=my_args)
    master_uri = options.master

    if master_uri is None or master_uri == "":
        print >> sys.stderr, (
            "Must specify a master with -m or `ciel config --set cluster.master URL`"
        )
        sys.exit(-1)
    elif len(args) < 2:
        print >> sys.stderr, "Must specify a fully-qualified class to run"
        parser.print_help()
        sys.exit(-1)

    jars = options.extra_jars + [args[1]]

    # Consult the config to see where the standard JARs are installed.
    jar_path = options.jar_lib
    if jar_path is None:
        print >> sys.stderr, "Could not find CIEL bindings. Set the JAR libary path using one of:"
        print >> sys.stderr, "\tciel jar (--jar-lib|-L) PATH ..."
        print >> sys.stderr, "\tciel config --set java.jar_lib PATH"
        sys.exit(-1)

    for jar_file in glob.glob(os.path.join(jar_path, '*.jar')):
        jars = jars + [jar_file]

    class_name = args[2]
    args = args[3:]

    def upload_jar(filename):
        with open(filename, 'r') as infile:
            return ref_of_string(infile.read(), master_uri)

    jar_refs = [upload_jar(j) for j in jars]

    package_dict = {}
    for binding in options.package:
        id, filename = binding.split("=", 2)
        with open(filename, 'r') as infile:
            package_dict[id] = ref_of_string(infile.read(), master_uri)

    package_ref = ref_of_string(pickle.dumps(package_dict), master_uri)

    args = {
        'jar_lib': jar_refs,
        'class_name': class_name,
        'args': args,
        'n_outputs': options.num_outputs
    }

    init_descriptor = build_init_descriptor("java2", args, package_ref,
                                            master_uri, ref_of_string)

    job_descriptor = submit_job_for_task(init_descriptor, master_uri)

    job_url = urlparse.urljoin(
        master_uri, "control/browse/job/%s" % job_descriptor['job_id'])

    result = await_job(job_descriptor['job_id'], master_uri)

    try:
        reflist = simple_retrieve_object_for_ref(result, "json",
                                                 job_descriptor['job_id'],
                                                 master_uri)
    except:
        print >> sys.stderr, "Error getting list of references as a result."

    try:
        j_return = retrieve_object_for_ref(reflist[0], "json", None)
    except:
        try:
            j_return = retrieve_object_for_ref(reflist[0], "noop", None)
        except:
            print >> sys.stderr, "Error parsing job result."
            sys.exit(-1)

    print j_return