def task_descriptor_for_package_and_initial_task(package_dict, start_handler, start_args, package_path, master_uri, args): def resolve_arg(value): try: return args[value["__args__"]] except IndexError: if "default" in value: print >> sys.stderr, "Positional argument", value[ "__args__"], "not specified; using default", value[ "default"] return value["default"] else: print >> sys.stderr, "Mandatory argument", value[ "__args__"], "not specified." sys.exit(-1) def resolve_env(value): try: return os.environ[value["__env__"]] except KeyError: if "default" in value: print >> sys.stderr, "Environment variable", value[ "__env__"], "not specified; using default", value[ "default"] return value["default"] else: print >> sys.stderr, "Mandatory environment variable", value[ "__env__"], "not specified." sys.exit(-1) env_and_args_callbacks = {"__args__": resolve_arg, "__env__": resolve_env} package_dict = resolve_vars(package_dict, env_and_args_callbacks) start_args = resolve_vars(start_args, env_and_args_callbacks) submit_package_dict = dict([(k, ref_of_object(k, v, package_path, master_uri)) for (k, v) in package_dict.items()]) #for key, ref in submit_package_dict.items(): # print >>sys.stderr, key, '-->', simplejson.dumps(ref, cls=SWReferenceJSONEncoder) package_ref = ref_of_string(pickle.dumps(submit_package_dict), master_uri) resolved_args = resolve_vars( start_args, {"__package__": lambda x: submit_package_dict[x["__package__"]]}) return build_init_descriptor(start_handler, resolved_args, package_ref, master_uri, ref_of_string)
def task_descriptor_for_package_and_initial_task(package_dict, start_handler, start_args, package_path, master_uri, args): def resolve_arg(value): try: return args[value["__args__"]] except IndexError: if "default" in value: print >>sys.stderr, "Positional argument", value["__args__"], "not specified; using default", value["default"] return value["default"] else: print >>sys.stderr, "Mandatory argument", value["__args__"], "not specified." sys.exit(-1) def resolve_env(value): try: return os.environ[value["__env__"]] except KeyError: if "default" in value: print >>sys.stderr, "Environment variable", value["__env__"], "not specified; using default", value["default"] return value["default"] else: print >>sys.stderr, "Mandatory environment variable", value["__env__"], "not specified." sys.exit(-1) env_and_args_callbacks = {"__args__": resolve_arg, "__env__": resolve_env} package_dict = resolve_vars(package_dict, env_and_args_callbacks) start_args = resolve_vars(start_args, env_and_args_callbacks) submit_package_dict = dict([(k, ref_of_object(k, v, package_path, master_uri)) for (k, v) in package_dict.items()]) #for key, ref in submit_package_dict.items(): # print >>sys.stderr, key, '-->', simplejson.dumps(ref, cls=SWReferenceJSONEncoder) package_ref = ref_of_string(pickle.dumps(submit_package_dict), master_uri) resolved_args = resolve_vars(start_args, {"__package__": lambda x: submit_package_dict[x["__package__"]]}) return build_init_descriptor(start_handler, resolved_args, package_ref, master_uri, ref_of_string)
def jar(my_args=sys.argv): parser = OptionParser(usage='Usage: ciel jar [options] JAR_FILE CLASS_NAME [args...]') parser.add_option("-m", "--master", action="store", dest="master", help="URI of the cluster master", metavar="MASTER", default=ciel.config.get('cluster', 'master', 'http://localhost:8000')) parser.add_option("-j", "--extra-jar", action="append", dest="extra_jars", help="Filename of additional JAR to load", metavar="JAR_FILE", default=[]) parser.add_option("-P", "--package", action="append", dest="package", help="Additional file to upload", metavar="ID=FILENAME", default=[]) parser.add_option("-n", "--num-outputs", action="store", dest="num_outputs", help="Number of outputs for root task", type="int", metavar="N", default=1) parser.add_option("-L", "--jar-lib", action="store", dest="jar_lib", help="Directory containing CIEL bindings JARs", type="str", metavar="PATH", default=ciel.config.get('java', 'jar_lib')) (options, args) = parser.parse_args(args=my_args) master_uri = options.master if master_uri is None or master_uri == "": print >>sys.stderr, ("Must specify a master with -m or `ciel config --set cluster.master URL`") sys.exit(-1) elif len(args) < 2: print >>sys.stderr, "Must specify a fully-qualified class to run" parser.print_help() sys.exit(-1) jars = options.extra_jars + [args[1]] # Consult the config to see where the standard JARs are installed. jar_path = options.jar_lib if jar_path is None: print >>sys.stderr, "Could not find CIEL bindings. Set the JAR libary path using one of:" print >>sys.stderr, "\tciel jar (--jar-lib|-L) PATH ..." print >>sys.stderr, "\tciel config --set java.jar_lib PATH" sys.exit(-1) for jar_file in glob.glob(os.path.join(jar_path, '*.jar')): jars = jars + [jar_file] class_name = args[2] args = args[3:] def upload_jar(filename): with open(filename, 'r') as infile: return ref_of_string(infile.read(), master_uri) jar_refs = [upload_jar(j) for j in jars] package_dict = {} for binding in options.package: id, filename = binding.split("=", 2) with open(filename, 'r') as infile: package_dict[id] = ref_of_string(infile.read(), master_uri) package_ref = ref_of_string(pickle.dumps(package_dict), master_uri) args = {'jar_lib' : jar_refs, 'class_name' : class_name, 'args' : args, 'n_outputs' : options.num_outputs} init_descriptor = build_init_descriptor("java2", args, package_ref, master_uri, ref_of_string) job_descriptor = submit_job_for_task(init_descriptor, master_uri) job_url = urlparse.urljoin(master_uri, "control/browse/job/%s" % job_descriptor['job_id']) result = await_job(job_descriptor['job_id'], master_uri) try: reflist = simple_retrieve_object_for_ref(result, "json", job_descriptor['job_id'], master_uri) except: print >>sys.stderr, "Error getting list of references as a result." try: j_return = retrieve_object_for_ref(reflist[0], "json", None) except: try: j_return = retrieve_object_for_ref(reflist[0], "noop", None) except: print >>sys.stderr, "Error parsing job result." sys.exit(-1) print j_return
def jar(my_args=sys.argv): parser = OptionParser( usage='Usage: ciel jar [options] JAR_FILE CLASS_NAME [args...]') parser.add_option("-m", "--master", action="store", dest="master", help="URI of the cluster master", metavar="MASTER", default=ciel.config.get('cluster', 'master', 'http://localhost:8000')) parser.add_option("-j", "--extra-jar", action="append", dest="extra_jars", help="Filename of additional JAR to load", metavar="JAR_FILE", default=[]) parser.add_option("-P", "--package", action="append", dest="package", help="Additional file to upload", metavar="ID=FILENAME", default=[]) parser.add_option("-n", "--num-outputs", action="store", dest="num_outputs", help="Number of outputs for root task", type="int", metavar="N", default=1) parser.add_option("-L", "--jar-lib", action="store", dest="jar_lib", help="Directory containing CIEL bindings JARs", type="str", metavar="PATH", default=ciel.config.get('java', 'jar_lib')) (options, args) = parser.parse_args(args=my_args) master_uri = options.master if master_uri is None or master_uri == "": print >> sys.stderr, ( "Must specify a master with -m or `ciel config --set cluster.master URL`" ) sys.exit(-1) elif len(args) < 2: print >> sys.stderr, "Must specify a fully-qualified class to run" parser.print_help() sys.exit(-1) jars = options.extra_jars + [args[1]] # Consult the config to see where the standard JARs are installed. jar_path = options.jar_lib if jar_path is None: print >> sys.stderr, "Could not find CIEL bindings. Set the JAR libary path using one of:" print >> sys.stderr, "\tciel jar (--jar-lib|-L) PATH ..." print >> sys.stderr, "\tciel config --set java.jar_lib PATH" sys.exit(-1) for jar_file in glob.glob(os.path.join(jar_path, '*.jar')): jars = jars + [jar_file] class_name = args[2] args = args[3:] def upload_jar(filename): with open(filename, 'r') as infile: return ref_of_string(infile.read(), master_uri) jar_refs = [upload_jar(j) for j in jars] package_dict = {} for binding in options.package: id, filename = binding.split("=", 2) with open(filename, 'r') as infile: package_dict[id] = ref_of_string(infile.read(), master_uri) package_ref = ref_of_string(pickle.dumps(package_dict), master_uri) args = { 'jar_lib': jar_refs, 'class_name': class_name, 'args': args, 'n_outputs': options.num_outputs } init_descriptor = build_init_descriptor("java2", args, package_ref, master_uri, ref_of_string) job_descriptor = submit_job_for_task(init_descriptor, master_uri) job_url = urlparse.urljoin( master_uri, "control/browse/job/%s" % job_descriptor['job_id']) result = await_job(job_descriptor['job_id'], master_uri) try: reflist = simple_retrieve_object_for_ref(result, "json", job_descriptor['job_id'], master_uri) except: print >> sys.stderr, "Error getting list of references as a result." try: j_return = retrieve_object_for_ref(reflist[0], "json", None) except: try: j_return = retrieve_object_for_ref(reflist[0], "noop", None) except: print >> sys.stderr, "Error parsing job result." sys.exit(-1) print j_return