def task_descriptor_for_package_and_initial_task(package_dict, start_handler,
start_args, package_path,
master_uri, args):
def resolve_arg(value):
try:
return args[value["__args__"]]
except IndexError:
if "default" in value:
print >> sys.stderr, "Positional argument", value[
"__args__"], "not specified; using default", value[
"default"]
return value["default"]
else:
print >> sys.stderr, "Mandatory argument", value[
"__args__"], "not specified."
sys.exit(-1)
def resolve_env(value):
try:
return os.environ[value["__env__"]]
except KeyError:
if "default" in value:
print >> sys.stderr, "Environment variable", value[
"__env__"], "not specified; using default", value[
"default"]
return value["default"]
else:
print >> sys.stderr, "Mandatory environment variable", value[
"__env__"], "not specified."
sys.exit(-1)
env_and_args_callbacks = {"__args__": resolve_arg, "__env__": resolve_env}
package_dict = resolve_vars(package_dict, env_and_args_callbacks)
start_args = resolve_vars(start_args, env_and_args_callbacks)
submit_package_dict = dict([(k,
ref_of_object(k, v, package_path, master_uri))
for (k, v) in package_dict.items()])
#for key, ref in submit_package_dict.items():
# print >>sys.stderr, key, '-->', simplejson.dumps(ref, cls=SWReferenceJSONEncoder)
package_ref = ref_of_string(pickle.dumps(submit_package_dict), master_uri)
resolved_args = resolve_vars(
start_args,
{"__package__": lambda x: submit_package_dict[x["__package__"]]})
return build_init_descriptor(start_handler, resolved_args, package_ref,
master_uri, ref_of_string)
def task_descriptor_for_package_and_initial_task(package_dict, start_handler, start_args, package_path, master_uri, args):
def resolve_arg(value):
try:
return args[value["__args__"]]
except IndexError:
if "default" in value:
print >>sys.stderr, "Positional argument", value["__args__"], "not specified; using default", value["default"]
return value["default"]
else:
print >>sys.stderr, "Mandatory argument", value["__args__"], "not specified."
sys.exit(-1)
def resolve_env(value):
try:
return os.environ[value["__env__"]]
except KeyError:
if "default" in value:
print >>sys.stderr, "Environment variable", value["__env__"], "not specified; using default", value["default"]
return value["default"]
else:
print >>sys.stderr, "Mandatory environment variable", value["__env__"], "not specified."
sys.exit(-1)
env_and_args_callbacks = {"__args__": resolve_arg,
"__env__": resolve_env}
package_dict = resolve_vars(package_dict, env_and_args_callbacks)
start_args = resolve_vars(start_args, env_and_args_callbacks)
submit_package_dict = dict([(k, ref_of_object(k, v, package_path, master_uri)) for (k, v) in package_dict.items()])
#for key, ref in submit_package_dict.items():
# print >>sys.stderr, key, '-->', simplejson.dumps(ref, cls=SWReferenceJSONEncoder)
package_ref = ref_of_string(pickle.dumps(submit_package_dict), master_uri)
resolved_args = resolve_vars(start_args, {"__package__": lambda x: submit_package_dict[x["__package__"]]})
return build_init_descriptor(start_handler, resolved_args, package_ref, master_uri, ref_of_string)
def jar(my_args=sys.argv):
parser = OptionParser(usage='Usage: ciel jar [options] JAR_FILE CLASS_NAME [args...]')
parser.add_option("-m", "--master", action="store", dest="master", help="URI of the cluster master", metavar="MASTER", default=ciel.config.get('cluster', 'master', 'http://localhost:8000'))
parser.add_option("-j", "--extra-jar", action="append", dest="extra_jars", help="Filename of additional JAR to load", metavar="JAR_FILE", default=[])
parser.add_option("-P", "--package", action="append", dest="package", help="Additional file to upload", metavar="ID=FILENAME", default=[])
parser.add_option("-n", "--num-outputs", action="store", dest="num_outputs", help="Number of outputs for root task", type="int", metavar="N", default=1)
parser.add_option("-L", "--jar-lib", action="store", dest="jar_lib", help="Directory containing CIEL bindings JARs", type="str", metavar="PATH", default=ciel.config.get('java', 'jar_lib'))
(options, args) = parser.parse_args(args=my_args)
master_uri = options.master
if master_uri is None or master_uri == "":
print >>sys.stderr, ("Must specify a master with -m or `ciel config --set cluster.master URL`")
sys.exit(-1)
elif len(args) < 2:
print >>sys.stderr, "Must specify a fully-qualified class to run"
parser.print_help()
sys.exit(-1)
jars = options.extra_jars + [args[1]]
# Consult the config to see where the standard JARs are installed.
jar_path = options.jar_lib
if jar_path is None:
print >>sys.stderr, "Could not find CIEL bindings. Set the JAR libary path using one of:"
print >>sys.stderr, "\tciel jar (--jar-lib|-L) PATH ..."
print >>sys.stderr, "\tciel config --set java.jar_lib PATH"
sys.exit(-1)
for jar_file in glob.glob(os.path.join(jar_path, '*.jar')):
jars = jars + [jar_file]
class_name = args[2]
args = args[3:]
def upload_jar(filename):
with open(filename, 'r') as infile:
return ref_of_string(infile.read(), master_uri)
jar_refs = [upload_jar(j) for j in jars]
package_dict = {}
for binding in options.package:
id, filename = binding.split("=", 2)
with open(filename, 'r') as infile:
package_dict[id] = ref_of_string(infile.read(), master_uri)
package_ref = ref_of_string(pickle.dumps(package_dict), master_uri)
args = {'jar_lib' : jar_refs,
'class_name' : class_name,
'args' : args,
'n_outputs' : options.num_outputs}
init_descriptor = build_init_descriptor("java2", args, package_ref, master_uri, ref_of_string)
job_descriptor = submit_job_for_task(init_descriptor, master_uri)
job_url = urlparse.urljoin(master_uri, "control/browse/job/%s" % job_descriptor['job_id'])
result = await_job(job_descriptor['job_id'], master_uri)
try:
reflist = simple_retrieve_object_for_ref(result, "json", job_descriptor['job_id'], master_uri)
except:
print >>sys.stderr, "Error getting list of references as a result."
try:
j_return = retrieve_object_for_ref(reflist[0], "json", None)
except:
try:
j_return = retrieve_object_for_ref(reflist[0], "noop", None)
except:
print >>sys.stderr, "Error parsing job result."
sys.exit(-1)
print j_return
def jar(my_args=sys.argv):
parser = OptionParser(
usage='Usage: ciel jar [options] JAR_FILE CLASS_NAME [args...]')
parser.add_option("-m",
"--master",
action="store",
dest="master",
help="URI of the cluster master",
metavar="MASTER",
default=ciel.config.get('cluster', 'master',
'http://localhost:8000'))
parser.add_option("-j",
"--extra-jar",
action="append",
dest="extra_jars",
help="Filename of additional JAR to load",
metavar="JAR_FILE",
default=[])
parser.add_option("-P",
"--package",
action="append",
dest="package",
help="Additional file to upload",
metavar="ID=FILENAME",
default=[])
parser.add_option("-n",
"--num-outputs",
action="store",
dest="num_outputs",
help="Number of outputs for root task",
type="int",
metavar="N",
default=1)
parser.add_option("-L",
"--jar-lib",
action="store",
dest="jar_lib",
help="Directory containing CIEL bindings JARs",
type="str",
metavar="PATH",
default=ciel.config.get('java', 'jar_lib'))
(options, args) = parser.parse_args(args=my_args)
master_uri = options.master
if master_uri is None or master_uri == "":
print >> sys.stderr, (
"Must specify a master with -m or `ciel config --set cluster.master URL`"
)
sys.exit(-1)
elif len(args) < 2:
print >> sys.stderr, "Must specify a fully-qualified class to run"
parser.print_help()
sys.exit(-1)
jars = options.extra_jars + [args[1]]
# Consult the config to see where the standard JARs are installed.
jar_path = options.jar_lib
if jar_path is None:
print >> sys.stderr, "Could not find CIEL bindings. Set the JAR libary path using one of:"
print >> sys.stderr, "\tciel jar (--jar-lib|-L) PATH ..."
print >> sys.stderr, "\tciel config --set java.jar_lib PATH"
sys.exit(-1)
for jar_file in glob.glob(os.path.join(jar_path, '*.jar')):
jars = jars + [jar_file]
class_name = args[2]
args = args[3:]
def upload_jar(filename):
with open(filename, 'r') as infile:
return ref_of_string(infile.read(), master_uri)
jar_refs = [upload_jar(j) for j in jars]
package_dict = {}
for binding in options.package:
id, filename = binding.split("=", 2)
with open(filename, 'r') as infile:
package_dict[id] = ref_of_string(infile.read(), master_uri)
package_ref = ref_of_string(pickle.dumps(package_dict), master_uri)
args = {
'jar_lib': jar_refs,
'class_name': class_name,
'args': args,
'n_outputs': options.num_outputs
}
init_descriptor = build_init_descriptor("java2", args, package_ref,
master_uri, ref_of_string)
job_descriptor = submit_job_for_task(init_descriptor, master_uri)
job_url = urlparse.urljoin(
master_uri, "control/browse/job/%s" % job_descriptor['job_id'])
result = await_job(job_descriptor['job_id'], master_uri)
try:
reflist = simple_retrieve_object_for_ref(result, "json",
job_descriptor['job_id'],
master_uri)
except:
print >> sys.stderr, "Error getting list of references as a result."
try:
j_return = retrieve_object_for_ref(reflist[0], "json", None)
except:
try:
j_return = retrieve_object_for_ref(reflist[0], "noop", None)
except:
print >> sys.stderr, "Error parsing job result."
sys.exit(-1)
print j_return