def main(my_args=sys.argv):
parser = OptionParser(usage='Usage: ciel sw [options] SW_SCRIPT [args...]')
parser.add_option("-m", "--master", action="store", dest="master", help="Master URI", metavar="MASTER", default=ciel.config.get('cluster', 'master', 'http://localhost:8000'))
parser.add_option("-i", "--id", action="store", dest="id", help="Job ID", metavar="ID", default="default")
parser.add_option("-e", "--env", action="store_true", dest="send_env", help="Set this flag to send the current environment with the script as _env", default=False)
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="Set this flag to enable verbose output", default=False)
parser.add_option("-p", "--package-file", action="append", type="string", dest="package_files", help="Specify files to be included as package inputs", metavar="KEY FILENAME", nargs=2, default=[])
(options, args) = parser.parse_args(my_args)
if not options.master:
print >> sys.stderr, "Must specify master URI with -m or `ciel config --set cluster.master URI`"
parser.print_help()
sys.exit(-1)
if len(args) != 2:
print >> sys.stderr, "Must specify one script file to execute, as argument"
parser.print_help()
sys.exit(-1)
script_name = args[1]
master_uri = options.master
id = options.id
if options.verbose:
print id, "STARTED", now_as_timestamp()
swi_package = {"swimain": {"filename": script_name}}
for key, filename in options.package_files:
swi_package[key] = {"filename": filename}
swi_args = {"sw_file_ref": {"__package__": "swimain"}, "start_args": args}
if options.send_env:
swi_args["start_env"] = dict(os.environ)
else:
swi_args["start_env"] = {}
new_job = ciel.runtime.util.start_job.submit_job_with_package(swi_package, "swi", swi_args, {}, os.getcwd(), master_uri, args)
result = ciel.runtime.util.start_job.await_job(new_job["job_id"], master_uri)
try:
reflist = retrieve_object_for_ref(result, "json", None)
sw_return = retrieve_object_for_ref(reflist[0], "json", None)
except ErrorReferenceError, ere:
print >>sys.stderr, 'Task failed with an error'
print >>sys.stderr, '%s: "%s"' % (ere.ref.reason, ere.ref.details)
sys.exit(-2)
def _run(self, task_private, task_descriptor, task_record):
self.task_record = task_record
self.task_id = task_descriptor["task_id"]
self.output_ids = task_descriptor["expected_outputs"]
self.output_refs = [None for _ in range(len(self.output_ids))]
self.succeeded = False
self.args = retrieve_object_for_ref(task_private["simple_exec_args"],
"pickle", self.task_record)
try:
self.debug_opts = self.args['debug_options']
except KeyError:
self.debug_opts = []
self.resolve_required_refs(self.args)
try:
self._execute()
for ref in self.output_refs:
if ref is not None:
self.task_record.publish_ref(ref)
else:
ciel.log.error(
"Executor failed to define output %s" % ref.id, "EXEC",
logging.WARNING)
self.succeeded = True
except:
ciel.log.error("Task execution failed", "EXEC", logging.ERROR,
True)
raise
finally:
self.cleanup_task()
def _run(self, task_private, task_descriptor, task_record):
self.task_record = task_record
self.task_id = task_descriptor["task_id"]
self.output_ids = task_descriptor["expected_outputs"]
self.output_refs = [None for _ in range(len(self.output_ids))]
self.succeeded = False
self.args = retrieve_object_for_ref(task_private["simple_exec_args"], "pickle", self.task_record)
try:
self.debug_opts = self.args['debug_options']
except KeyError:
self.debug_opts = []
self.resolve_required_refs(self.args)
try:
self._execute()
for ref in self.output_refs:
if ref is not None:
self.task_record.publish_ref(ref)
else:
ciel.log.error("Executor failed to define output %s" % ref.id, "EXEC", logging.WARNING)
self.succeeded = True
except:
ciel.log.error("Task execution failed", "EXEC", logging.ERROR, True)
raise
finally:
self.cleanup_task()
def main():
parser = OptionParser()
parser.add_option("-m", "--master", action="store", dest="master", help="Master URI", metavar="MASTER", default=os.getenv("SW_MASTER"))
parser.add_option("-s", "--skypy-stub", action="store", dest="skypy_stub", help="Path to Skypy stub.py", metavar="PATH", default=None)
(options, args) = parser.parse_args()
if not options.master:
parser.print_help()
print >> sys.stderr, "Must specify master URI with --master"
sys.exit(1)
master_uri = options.master
script_name = args[0]
script_args = args[1:]
sp_package = {"skypymain": {"filename": script_name}}
sp_args = {"pyfile_ref": {"__package__": "skypymain"}, "entry_point": "skypy_main", "entry_args": script_args}
new_job = ciel.runtime.util.start_job.submit_job_with_package(sp_package, "skypy", sp_args, os.getcwd(), master_uri, args)
result = ciel.runtime.util.start_job.await_job(new_job["job_id"], master_uri)
reflist = retrieve_object_for_ref(result, "json", None)
return reflist[0]
def prepare_task_descriptor_for_execute(cls, task_descriptor, task_record, block_store):
# Convert task_private from a reference to an object in here.
try:
task_descriptor["task_private"] = retrieve_object_for_ref(task_descriptor["task_private"], BaseExecutor.TASK_PRIVATE_ENCODING, task_record)
except:
ciel.log('Error retrieving task_private reference from task', 'BASE_EXECUTOR', logging.WARN, True)
raise
def package_lookup(task_record, block_store, key):
if task_record.package_ref is None:
ciel.log.error("Package lookup for %s in task without package" % key, "EXEC", logging.WARNING)
return None
package_dict = retrieve_object_for_ref(task_record.package_ref, "pickle", task_record)
try:
return package_dict[key]
except KeyError:
ciel.log.error("Package lookup for %s: no such key" % key, "EXEC", logging.WARNING)
return None
def main():
parser = OptionParser()
parser.add_option("-m", "--master", action="store", dest="master", help="Master URI", metavar="MASTER", default=os.getenv("SW_MASTER"))
parser.add_option("-r", "--refs", action="store_true", dest="refs", help="Set this option to look up reference names in the master", default=False)
parser.add_option("-j", "--json", action="store_true", dest="json", help="Set this option to use JSON pretty printing", default=False)
(options, args) = parser.parse_args()
if options.refs:
ref_ids = args
for ref_id in ref_ids:
# Fetch information about the ref from the master.
h = httplib2.Http()
_, content = h.request(urljoin(options.master, '/refs/%s' % ref_id), 'GET')
ref_info = simplejson.loads(content, object_hook=json_decode_object_hook)
ref = ref_info['ref']
if options.json:
obj = retrieve_object_for_ref(ref, 'json', None)
simplejson.dump(obj, sys.stdout, cls=SWReferenceJSONEncoder, indent=4)
print
else:
fh = retrieve_object_for_ref(ref, 'handle', None)
for line in fh:
sys.stdout.write(line)
fh.close()
else:
urls = args
for url in urls:
if options.json:
obj = retrieve_object_for_ref(SWURLReference([url]), 'json', None)
simplejson.dump(obj, sys.stdout, cls=SWReferenceJSONEncoder, indent=4)
print
else:
fh = retrieve_object_for_ref(SWURLReference([url]), 'handle', None)
print fh
for line in fh:
sys.stdout.write(line)
fh.close()
def prepare_task_descriptor_for_execute(cls, task_descriptor, task_record,
block_store):
# Convert task_private from a reference to an object in here.
try:
task_descriptor["task_private"] = retrieve_object_for_ref(
task_descriptor["task_private"],
BaseExecutor.TASK_PRIVATE_ENCODING, task_record)
except:
ciel.log('Error retrieving task_private reference from task',
'BASE_EXECUTOR', logging.WARN, True)
raise
def simple_retrieve_object_for_ref(ref, decoder, jobid, master_uri):
if isinstance(ref, SWErrorReference):
raise Exception("Can't decode %s" % ref)
if isinstance(ref, SW2_FutureReference) or isinstance(ref, SW2_StreamReference) or isinstance(ref, SW2_SocketStreamReference):
ref = external_get_real_ref(ref, jobid, master_uri)
if isinstance(ref, SWDataValue):
return retrieve_object_for_ref(ref, decoder, None)
elif isinstance(ref, SW2_ConcreteReference):
urls = get_fetch_urls_for_ref(ref)
_, content = httplib2.Http().request(urls[0])
return decoders[decoder](StringIO(content))
else:
raise Exception("Don't know how to retrieve a %s" % ref)
def package_lookup(task_record, block_store, key):
if task_record.package_ref is None:
ciel.log.error("Package lookup for %s in task without package" % key,
"EXEC", logging.WARNING)
return None
package_dict = retrieve_object_for_ref(task_record.package_ref, "pickle",
task_record)
try:
return package_dict[key]
except KeyError:
ciel.log.error("Package lookup for %s: no such key" % key, "EXEC",
logging.WARNING)
return None
def simple_retrieve_object_for_ref(ref, decoder, jobid, master_uri):
if isinstance(ref, SWErrorReference):
raise Exception("Can't decode %s" % ref)
if isinstance(ref, SW2_FutureReference) or isinstance(ref, SW2_StreamReference) or isinstance(ref, SW2_SocketStreamReference):
ref = external_get_real_ref(ref, jobid, master_uri)
if isinstance(ref, SWDataValue):
return retrieve_object_for_ref(ref, decoder, None)
elif isinstance(ref, SW2_ConcreteReference):
urls = get_fetch_urls_for_ref(ref)
_, content = httplib2.Http().request(urls[0])
return decoders[decoder](StringIO(content))
else:
raise Exception("Don't know how to retrieve a %s" % ref)
def main():
parser = OptionParser()
parser.add_option("-m",
"--master",
action="store",
dest="master",
help="Master URI",
metavar="MASTER",
default=os.getenv("SW_MASTER"))
parser.add_option("-s",
"--skypy-stub",
action="store",
dest="skypy_stub",
help="Path to Skypy stub.py",
metavar="PATH",
default=None)
(options, args) = parser.parse_args()
if not options.master:
parser.print_help()
print >> sys.stderr, "Must specify master URI with --master"
sys.exit(1)
master_uri = options.master
script_name = args[0]
script_args = args[1:]
sp_package = {"skypymain": {"filename": script_name}}
sp_args = {
"pyfile_ref": {
"__package__": "skypymain"
},
"entry_point": "skypy_main",
"entry_args": script_args
}
new_job = ciel.runtime.util.start_job.submit_job_with_package(
sp_package, "skypy", sp_args, os.getcwd(), master_uri, args)
result = ciel.runtime.util.start_job.await_job(new_job["job_id"],
master_uri)
reflist = retrieve_object_for_ref(result, "json", None)
return reflist[0]
def jar(my_args=sys.argv):
parser = OptionParser(usage='Usage: ciel jar [options] JAR_FILE CLASS_NAME [args...]')
parser.add_option("-m", "--master", action="store", dest="master", help="URI of the cluster master", metavar="MASTER", default=ciel.config.get('cluster', 'master', 'http://localhost:8000'))
parser.add_option("-j", "--extra-jar", action="append", dest="extra_jars", help="Filename of additional JAR to load", metavar="JAR_FILE", default=[])
parser.add_option("-P", "--package", action="append", dest="package", help="Additional file to upload", metavar="ID=FILENAME", default=[])
parser.add_option("-n", "--num-outputs", action="store", dest="num_outputs", help="Number of outputs for root task", type="int", metavar="N", default=1)
parser.add_option("-L", "--jar-lib", action="store", dest="jar_lib", help="Directory containing CIEL bindings JARs", type="str", metavar="PATH", default=ciel.config.get('java', 'jar_lib'))
(options, args) = parser.parse_args(args=my_args)
master_uri = options.master
if master_uri is None or master_uri == "":
print >>sys.stderr, ("Must specify a master with -m or `ciel config --set cluster.master URL`")
sys.exit(-1)
elif len(args) < 2:
print >>sys.stderr, "Must specify a fully-qualified class to run"
parser.print_help()
sys.exit(-1)
jars = options.extra_jars + [args[1]]
# Consult the config to see where the standard JARs are installed.
jar_path = options.jar_lib
if jar_path is None:
print >>sys.stderr, "Could not find CIEL bindings. Set the JAR libary path using one of:"
print >>sys.stderr, "\tciel jar (--jar-lib|-L) PATH ..."
print >>sys.stderr, "\tciel config --set java.jar_lib PATH"
sys.exit(-1)
for jar_file in glob.glob(os.path.join(jar_path, '*.jar')):
jars = jars + [jar_file]
class_name = args[2]
args = args[3:]
def upload_jar(filename):
with open(filename, 'r') as infile:
return ref_of_string(infile.read(), master_uri)
jar_refs = [upload_jar(j) for j in jars]
package_dict = {}
for binding in options.package:
id, filename = binding.split("=", 2)
with open(filename, 'r') as infile:
package_dict[id] = ref_of_string(infile.read(), master_uri)
package_ref = ref_of_string(pickle.dumps(package_dict), master_uri)
args = {'jar_lib' : jar_refs,
'class_name' : class_name,
'args' : args,
'n_outputs' : options.num_outputs}
init_descriptor = build_init_descriptor("java2", args, package_ref, master_uri, ref_of_string)
job_descriptor = submit_job_for_task(init_descriptor, master_uri)
job_url = urlparse.urljoin(master_uri, "control/browse/job/%s" % job_descriptor['job_id'])
result = await_job(job_descriptor['job_id'], master_uri)
try:
reflist = simple_retrieve_object_for_ref(result, "json", job_descriptor['job_id'], master_uri)
except:
print >>sys.stderr, "Error getting list of references as a result."
try:
j_return = retrieve_object_for_ref(reflist[0], "json", None)
except:
try:
j_return = retrieve_object_for_ref(reflist[0], "noop", None)
except:
print >>sys.stderr, "Error parsing job result."
sys.exit(-1)
print j_return
def main():
parser = OptionParser()
parser.add_option("-m",
"--master",
action="store",
dest="master",
help="Master URI",
metavar="MASTER",
default=os.getenv("CIEL_MASTER"))
parser.add_option("-i",
"--id",
action="store",
dest="id",
help="Job ID",
metavar="ID",
default="default")
parser.add_option(
"-e",
"--env",
action="store_true",
dest="send_env",
help=
"Set this flag to send the current environment with the script as _env",
default=False)
parser.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
help="Set this flag to enable verbose output",
default=False)
parser.add_option("-p",
"--package-file",
action="append",
type="string",
dest="package_files",
help="Specify files to be included as package inputs",
metavar="KEY FILENAME",
nargs=2,
default=[])
(options, args) = parser.parse_args()
if not options.master:
parser.print_help()
print >> sys.stderr, "Must specify master URI with --master"
sys.exit(1)
if len(args) != 1:
parser.print_help()
print >> sys.stderr, "Must specify one script file to execute, as argument"
sys.exit(1)
script_name = args[0]
master_uri = options.master
id = options.id
if options.verbose:
print id, "STARTED", now_as_timestamp()
swi_package = {"swimain": {"filename": script_name}}
for key, filename in options.package_files:
swi_package[key] = {"filename": filename}
swi_args = {"sw_file_ref": {"__package__": "swimain"}, "start_args": args}
if options.send_env:
swi_args["start_env"] = dict(os.environ)
else:
swi_args["start_env"] = {}
new_job = ciel.runtime.util.start_job.submit_job_with_package(
swi_package, "swi", swi_args, {}, os.getcwd(), master_uri, args)
result = ciel.runtime.util.start_job.await_job(new_job["job_id"],
master_uri)
try:
reflist = retrieve_object_for_ref(result, "json", None)
sw_return = retrieve_object_for_ref(reflist[0], "json", None)
except ErrorReferenceError, ere:
print >> sys.stderr, 'Task failed with an error'
print >> sys.stderr, '%s: "%s"' % (ere.ref.reason, ere.ref.details)
sys.exit(-2)
def main():
parser = OptionParser()
parser.add_option("-m",
"--master",
action="store",
dest="master",
help="Master URI",
metavar="MASTER",
default=os.getenv("SW_MASTER"))
parser.add_option(
"-r",
"--refs",
action="store_true",
dest="refs",
help="Set this option to look up reference names in the master",
default=False)
parser.add_option("-j",
"--json",
action="store_true",
dest="json",
help="Set this option to use JSON pretty printing",
default=False)
(options, args) = parser.parse_args()
if options.refs:
ref_ids = args
for ref_id in ref_ids:
# Fetch information about the ref from the master.
h = httplib2.Http()
_, content = h.request(
urljoin(options.master, '/refs/%s' % ref_id), 'GET')
ref_info = simplejson.loads(content,
object_hook=json_decode_object_hook)
ref = ref_info['ref']
if options.json:
obj = retrieve_object_for_ref(ref, 'json', None)
simplejson.dump(obj,
sys.stdout,
cls=SWReferenceJSONEncoder,
indent=4)
print
else:
fh = retrieve_object_for_ref(ref, 'handle', None)
for line in fh:
sys.stdout.write(line)
fh.close()
else:
urls = args
for url in urls:
if options.json:
obj = retrieve_object_for_ref(SWURLReference([url]), 'json',
None)
simplejson.dump(obj,
sys.stdout,
cls=SWReferenceJSONEncoder,
indent=4)
print
else:
fh = retrieve_object_for_ref(SWURLReference([url]), 'handle',
None)
print fh
for line in fh:
sys.stdout.write(line)
fh.close()
def jar(my_args=sys.argv):
parser = OptionParser(
usage='Usage: ciel jar [options] JAR_FILE CLASS_NAME [args...]')
parser.add_option("-m",
"--master",
action="store",
dest="master",
help="URI of the cluster master",
metavar="MASTER",
default=ciel.config.get('cluster', 'master',
'http://localhost:8000'))
parser.add_option("-j",
"--extra-jar",
action="append",
dest="extra_jars",
help="Filename of additional JAR to load",
metavar="JAR_FILE",
default=[])
parser.add_option("-P",
"--package",
action="append",
dest="package",
help="Additional file to upload",
metavar="ID=FILENAME",
default=[])
parser.add_option("-n",
"--num-outputs",
action="store",
dest="num_outputs",
help="Number of outputs for root task",
type="int",
metavar="N",
default=1)
parser.add_option("-L",
"--jar-lib",
action="store",
dest="jar_lib",
help="Directory containing CIEL bindings JARs",
type="str",
metavar="PATH",
default=ciel.config.get('java', 'jar_lib'))
(options, args) = parser.parse_args(args=my_args)
master_uri = options.master
if master_uri is None or master_uri == "":
print >> sys.stderr, (
"Must specify a master with -m or `ciel config --set cluster.master URL`"
)
sys.exit(-1)
elif len(args) < 2:
print >> sys.stderr, "Must specify a fully-qualified class to run"
parser.print_help()
sys.exit(-1)
jars = options.extra_jars + [args[1]]
# Consult the config to see where the standard JARs are installed.
jar_path = options.jar_lib
if jar_path is None:
print >> sys.stderr, "Could not find CIEL bindings. Set the JAR libary path using one of:"
print >> sys.stderr, "\tciel jar (--jar-lib|-L) PATH ..."
print >> sys.stderr, "\tciel config --set java.jar_lib PATH"
sys.exit(-1)
for jar_file in glob.glob(os.path.join(jar_path, '*.jar')):
jars = jars + [jar_file]
class_name = args[2]
args = args[3:]
def upload_jar(filename):
with open(filename, 'r') as infile:
return ref_of_string(infile.read(), master_uri)
jar_refs = [upload_jar(j) for j in jars]
package_dict = {}
for binding in options.package:
id, filename = binding.split("=", 2)
with open(filename, 'r') as infile:
package_dict[id] = ref_of_string(infile.read(), master_uri)
package_ref = ref_of_string(pickle.dumps(package_dict), master_uri)
args = {
'jar_lib': jar_refs,
'class_name': class_name,
'args': args,
'n_outputs': options.num_outputs
}
init_descriptor = build_init_descriptor("java2", args, package_ref,
master_uri, ref_of_string)
job_descriptor = submit_job_for_task(init_descriptor, master_uri)
job_url = urlparse.urljoin(
master_uri, "control/browse/job/%s" % job_descriptor['job_id'])
result = await_job(job_descriptor['job_id'], master_uri)
try:
reflist = simple_retrieve_object_for_ref(result, "json",
job_descriptor['job_id'],
master_uri)
except:
print >> sys.stderr, "Error getting list of references as a result."
try:
j_return = retrieve_object_for_ref(reflist[0], "json", None)
except:
try:
j_return = retrieve_object_for_ref(reflist[0], "noop", None)
except:
print >> sys.stderr, "Error parsing job result."
sys.exit(-1)
print j_return
