def main(my_args=sys.argv): parser = OptionParser(usage='Usage: ciel sw [options] SW_SCRIPT [args...]') parser.add_option("-m", "--master", action="store", dest="master", help="Master URI", metavar="MASTER", default=ciel.config.get('cluster', 'master', 'http://localhost:8000')) parser.add_option("-i", "--id", action="store", dest="id", help="Job ID", metavar="ID", default="default") parser.add_option("-e", "--env", action="store_true", dest="send_env", help="Set this flag to send the current environment with the script as _env", default=False) parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="Set this flag to enable verbose output", default=False) parser.add_option("-p", "--package-file", action="append", type="string", dest="package_files", help="Specify files to be included as package inputs", metavar="KEY FILENAME", nargs=2, default=[]) (options, args) = parser.parse_args(my_args) if not options.master: print >> sys.stderr, "Must specify master URI with -m or `ciel config --set cluster.master URI`" parser.print_help() sys.exit(-1) if len(args) != 2: print >> sys.stderr, "Must specify one script file to execute, as argument" parser.print_help() sys.exit(-1) script_name = args[1] master_uri = options.master id = options.id if options.verbose: print id, "STARTED", now_as_timestamp() swi_package = {"swimain": {"filename": script_name}} for key, filename in options.package_files: swi_package[key] = {"filename": filename} swi_args = {"sw_file_ref": {"__package__": "swimain"}, "start_args": args} if options.send_env: swi_args["start_env"] = dict(os.environ) else: swi_args["start_env"] = {} new_job = ciel.runtime.util.start_job.submit_job_with_package(swi_package, "swi", swi_args, {}, os.getcwd(), master_uri, args) result = ciel.runtime.util.start_job.await_job(new_job["job_id"], master_uri) try: reflist = retrieve_object_for_ref(result, "json", None) sw_return = retrieve_object_for_ref(reflist[0], "json", None) except ErrorReferenceError, ere: print >>sys.stderr, 'Task failed with an error' print >>sys.stderr, '%s: "%s"' % (ere.ref.reason, ere.ref.details) sys.exit(-2)
def _run(self, task_private, task_descriptor, task_record): self.task_record = task_record self.task_id = task_descriptor["task_id"] self.output_ids = task_descriptor["expected_outputs"] self.output_refs = [None for _ in range(len(self.output_ids))] self.succeeded = False self.args = retrieve_object_for_ref(task_private["simple_exec_args"], "pickle", self.task_record) try: self.debug_opts = self.args['debug_options'] except KeyError: self.debug_opts = [] self.resolve_required_refs(self.args) try: self._execute() for ref in self.output_refs: if ref is not None: self.task_record.publish_ref(ref) else: ciel.log.error( "Executor failed to define output %s" % ref.id, "EXEC", logging.WARNING) self.succeeded = True except: ciel.log.error("Task execution failed", "EXEC", logging.ERROR, True) raise finally: self.cleanup_task()
def _run(self, task_private, task_descriptor, task_record): self.task_record = task_record self.task_id = task_descriptor["task_id"] self.output_ids = task_descriptor["expected_outputs"] self.output_refs = [None for _ in range(len(self.output_ids))] self.succeeded = False self.args = retrieve_object_for_ref(task_private["simple_exec_args"], "pickle", self.task_record) try: self.debug_opts = self.args['debug_options'] except KeyError: self.debug_opts = [] self.resolve_required_refs(self.args) try: self._execute() for ref in self.output_refs: if ref is not None: self.task_record.publish_ref(ref) else: ciel.log.error("Executor failed to define output %s" % ref.id, "EXEC", logging.WARNING) self.succeeded = True except: ciel.log.error("Task execution failed", "EXEC", logging.ERROR, True) raise finally: self.cleanup_task()
def main(): parser = OptionParser() parser.add_option("-m", "--master", action="store", dest="master", help="Master URI", metavar="MASTER", default=os.getenv("SW_MASTER")) parser.add_option("-s", "--skypy-stub", action="store", dest="skypy_stub", help="Path to Skypy stub.py", metavar="PATH", default=None) (options, args) = parser.parse_args() if not options.master: parser.print_help() print >> sys.stderr, "Must specify master URI with --master" sys.exit(1) master_uri = options.master script_name = args[0] script_args = args[1:] sp_package = {"skypymain": {"filename": script_name}} sp_args = {"pyfile_ref": {"__package__": "skypymain"}, "entry_point": "skypy_main", "entry_args": script_args} new_job = ciel.runtime.util.start_job.submit_job_with_package(sp_package, "skypy", sp_args, os.getcwd(), master_uri, args) result = ciel.runtime.util.start_job.await_job(new_job["job_id"], master_uri) reflist = retrieve_object_for_ref(result, "json", None) return reflist[0]
def prepare_task_descriptor_for_execute(cls, task_descriptor, task_record, block_store): # Convert task_private from a reference to an object in here. try: task_descriptor["task_private"] = retrieve_object_for_ref(task_descriptor["task_private"], BaseExecutor.TASK_PRIVATE_ENCODING, task_record) except: ciel.log('Error retrieving task_private reference from task', 'BASE_EXECUTOR', logging.WARN, True) raise
def package_lookup(task_record, block_store, key): if task_record.package_ref is None: ciel.log.error("Package lookup for %s in task without package" % key, "EXEC", logging.WARNING) return None package_dict = retrieve_object_for_ref(task_record.package_ref, "pickle", task_record) try: return package_dict[key] except KeyError: ciel.log.error("Package lookup for %s: no such key" % key, "EXEC", logging.WARNING) return None
def main(): parser = OptionParser() parser.add_option("-m", "--master", action="store", dest="master", help="Master URI", metavar="MASTER", default=os.getenv("SW_MASTER")) parser.add_option("-r", "--refs", action="store_true", dest="refs", help="Set this option to look up reference names in the master", default=False) parser.add_option("-j", "--json", action="store_true", dest="json", help="Set this option to use JSON pretty printing", default=False) (options, args) = parser.parse_args() if options.refs: ref_ids = args for ref_id in ref_ids: # Fetch information about the ref from the master. h = httplib2.Http() _, content = h.request(urljoin(options.master, '/refs/%s' % ref_id), 'GET') ref_info = simplejson.loads(content, object_hook=json_decode_object_hook) ref = ref_info['ref'] if options.json: obj = retrieve_object_for_ref(ref, 'json', None) simplejson.dump(obj, sys.stdout, cls=SWReferenceJSONEncoder, indent=4) print else: fh = retrieve_object_for_ref(ref, 'handle', None) for line in fh: sys.stdout.write(line) fh.close() else: urls = args for url in urls: if options.json: obj = retrieve_object_for_ref(SWURLReference([url]), 'json', None) simplejson.dump(obj, sys.stdout, cls=SWReferenceJSONEncoder, indent=4) print else: fh = retrieve_object_for_ref(SWURLReference([url]), 'handle', None) print fh for line in fh: sys.stdout.write(line) fh.close()
def prepare_task_descriptor_for_execute(cls, task_descriptor, task_record, block_store): # Convert task_private from a reference to an object in here. try: task_descriptor["task_private"] = retrieve_object_for_ref( task_descriptor["task_private"], BaseExecutor.TASK_PRIVATE_ENCODING, task_record) except: ciel.log('Error retrieving task_private reference from task', 'BASE_EXECUTOR', logging.WARN, True) raise
def simple_retrieve_object_for_ref(ref, decoder, jobid, master_uri): if isinstance(ref, SWErrorReference): raise Exception("Can't decode %s" % ref) if isinstance(ref, SW2_FutureReference) or isinstance(ref, SW2_StreamReference) or isinstance(ref, SW2_SocketStreamReference): ref = external_get_real_ref(ref, jobid, master_uri) if isinstance(ref, SWDataValue): return retrieve_object_for_ref(ref, decoder, None) elif isinstance(ref, SW2_ConcreteReference): urls = get_fetch_urls_for_ref(ref) _, content = httplib2.Http().request(urls[0]) return decoders[decoder](StringIO(content)) else: raise Exception("Don't know how to retrieve a %s" % ref)
def main(): parser = OptionParser() parser.add_option("-m", "--master", action="store", dest="master", help="Master URI", metavar="MASTER", default=os.getenv("SW_MASTER")) parser.add_option("-s", "--skypy-stub", action="store", dest="skypy_stub", help="Path to Skypy stub.py", metavar="PATH", default=None) (options, args) = parser.parse_args() if not options.master: parser.print_help() print >> sys.stderr, "Must specify master URI with --master" sys.exit(1) master_uri = options.master script_name = args[0] script_args = args[1:] sp_package = {"skypymain": {"filename": script_name}} sp_args = { "pyfile_ref": { "__package__": "skypymain" }, "entry_point": "skypy_main", "entry_args": script_args } new_job = ciel.runtime.util.start_job.submit_job_with_package( sp_package, "skypy", sp_args, os.getcwd(), master_uri, args) result = ciel.runtime.util.start_job.await_job(new_job["job_id"], master_uri) reflist = retrieve_object_for_ref(result, "json", None) return reflist[0]
def jar(my_args=sys.argv): parser = OptionParser(usage='Usage: ciel jar [options] JAR_FILE CLASS_NAME [args...]') parser.add_option("-m", "--master", action="store", dest="master", help="URI of the cluster master", metavar="MASTER", default=ciel.config.get('cluster', 'master', 'http://localhost:8000')) parser.add_option("-j", "--extra-jar", action="append", dest="extra_jars", help="Filename of additional JAR to load", metavar="JAR_FILE", default=[]) parser.add_option("-P", "--package", action="append", dest="package", help="Additional file to upload", metavar="ID=FILENAME", default=[]) parser.add_option("-n", "--num-outputs", action="store", dest="num_outputs", help="Number of outputs for root task", type="int", metavar="N", default=1) parser.add_option("-L", "--jar-lib", action="store", dest="jar_lib", help="Directory containing CIEL bindings JARs", type="str", metavar="PATH", default=ciel.config.get('java', 'jar_lib')) (options, args) = parser.parse_args(args=my_args) master_uri = options.master if master_uri is None or master_uri == "": print >>sys.stderr, ("Must specify a master with -m or `ciel config --set cluster.master URL`") sys.exit(-1) elif len(args) < 2: print >>sys.stderr, "Must specify a fully-qualified class to run" parser.print_help() sys.exit(-1) jars = options.extra_jars + [args[1]] # Consult the config to see where the standard JARs are installed. jar_path = options.jar_lib if jar_path is None: print >>sys.stderr, "Could not find CIEL bindings. Set the JAR libary path using one of:" print >>sys.stderr, "\tciel jar (--jar-lib|-L) PATH ..." print >>sys.stderr, "\tciel config --set java.jar_lib PATH" sys.exit(-1) for jar_file in glob.glob(os.path.join(jar_path, '*.jar')): jars = jars + [jar_file] class_name = args[2] args = args[3:] def upload_jar(filename): with open(filename, 'r') as infile: return ref_of_string(infile.read(), master_uri) jar_refs = [upload_jar(j) for j in jars] package_dict = {} for binding in options.package: id, filename = binding.split("=", 2) with open(filename, 'r') as infile: package_dict[id] = ref_of_string(infile.read(), master_uri) package_ref = ref_of_string(pickle.dumps(package_dict), master_uri) args = {'jar_lib' : jar_refs, 'class_name' : class_name, 'args' : args, 'n_outputs' : options.num_outputs} init_descriptor = build_init_descriptor("java2", args, package_ref, master_uri, ref_of_string) job_descriptor = submit_job_for_task(init_descriptor, master_uri) job_url = urlparse.urljoin(master_uri, "control/browse/job/%s" % job_descriptor['job_id']) result = await_job(job_descriptor['job_id'], master_uri) try: reflist = simple_retrieve_object_for_ref(result, "json", job_descriptor['job_id'], master_uri) except: print >>sys.stderr, "Error getting list of references as a result." try: j_return = retrieve_object_for_ref(reflist[0], "json", None) except: try: j_return = retrieve_object_for_ref(reflist[0], "noop", None) except: print >>sys.stderr, "Error parsing job result." sys.exit(-1) print j_return
def main(): parser = OptionParser() parser.add_option("-m", "--master", action="store", dest="master", help="Master URI", metavar="MASTER", default=os.getenv("CIEL_MASTER")) parser.add_option("-i", "--id", action="store", dest="id", help="Job ID", metavar="ID", default="default") parser.add_option( "-e", "--env", action="store_true", dest="send_env", help= "Set this flag to send the current environment with the script as _env", default=False) parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="Set this flag to enable verbose output", default=False) parser.add_option("-p", "--package-file", action="append", type="string", dest="package_files", help="Specify files to be included as package inputs", metavar="KEY FILENAME", nargs=2, default=[]) (options, args) = parser.parse_args() if not options.master: parser.print_help() print >> sys.stderr, "Must specify master URI with --master" sys.exit(1) if len(args) != 1: parser.print_help() print >> sys.stderr, "Must specify one script file to execute, as argument" sys.exit(1) script_name = args[0] master_uri = options.master id = options.id if options.verbose: print id, "STARTED", now_as_timestamp() swi_package = {"swimain": {"filename": script_name}} for key, filename in options.package_files: swi_package[key] = {"filename": filename} swi_args = {"sw_file_ref": {"__package__": "swimain"}, "start_args": args} if options.send_env: swi_args["start_env"] = dict(os.environ) else: swi_args["start_env"] = {} new_job = ciel.runtime.util.start_job.submit_job_with_package( swi_package, "swi", swi_args, {}, os.getcwd(), master_uri, args) result = ciel.runtime.util.start_job.await_job(new_job["job_id"], master_uri) try: reflist = retrieve_object_for_ref(result, "json", None) sw_return = retrieve_object_for_ref(reflist[0], "json", None) except ErrorReferenceError, ere: print >> sys.stderr, 'Task failed with an error' print >> sys.stderr, '%s: "%s"' % (ere.ref.reason, ere.ref.details) sys.exit(-2)
def main(): parser = OptionParser() parser.add_option("-m", "--master", action="store", dest="master", help="Master URI", metavar="MASTER", default=os.getenv("SW_MASTER")) parser.add_option( "-r", "--refs", action="store_true", dest="refs", help="Set this option to look up reference names in the master", default=False) parser.add_option("-j", "--json", action="store_true", dest="json", help="Set this option to use JSON pretty printing", default=False) (options, args) = parser.parse_args() if options.refs: ref_ids = args for ref_id in ref_ids: # Fetch information about the ref from the master. h = httplib2.Http() _, content = h.request( urljoin(options.master, '/refs/%s' % ref_id), 'GET') ref_info = simplejson.loads(content, object_hook=json_decode_object_hook) ref = ref_info['ref'] if options.json: obj = retrieve_object_for_ref(ref, 'json', None) simplejson.dump(obj, sys.stdout, cls=SWReferenceJSONEncoder, indent=4) print else: fh = retrieve_object_for_ref(ref, 'handle', None) for line in fh: sys.stdout.write(line) fh.close() else: urls = args for url in urls: if options.json: obj = retrieve_object_for_ref(SWURLReference([url]), 'json', None) simplejson.dump(obj, sys.stdout, cls=SWReferenceJSONEncoder, indent=4) print else: fh = retrieve_object_for_ref(SWURLReference([url]), 'handle', None) print fh for line in fh: sys.stdout.write(line) fh.close()
def jar(my_args=sys.argv): parser = OptionParser( usage='Usage: ciel jar [options] JAR_FILE CLASS_NAME [args...]') parser.add_option("-m", "--master", action="store", dest="master", help="URI of the cluster master", metavar="MASTER", default=ciel.config.get('cluster', 'master', 'http://localhost:8000')) parser.add_option("-j", "--extra-jar", action="append", dest="extra_jars", help="Filename of additional JAR to load", metavar="JAR_FILE", default=[]) parser.add_option("-P", "--package", action="append", dest="package", help="Additional file to upload", metavar="ID=FILENAME", default=[]) parser.add_option("-n", "--num-outputs", action="store", dest="num_outputs", help="Number of outputs for root task", type="int", metavar="N", default=1) parser.add_option("-L", "--jar-lib", action="store", dest="jar_lib", help="Directory containing CIEL bindings JARs", type="str", metavar="PATH", default=ciel.config.get('java', 'jar_lib')) (options, args) = parser.parse_args(args=my_args) master_uri = options.master if master_uri is None or master_uri == "": print >> sys.stderr, ( "Must specify a master with -m or `ciel config --set cluster.master URL`" ) sys.exit(-1) elif len(args) < 2: print >> sys.stderr, "Must specify a fully-qualified class to run" parser.print_help() sys.exit(-1) jars = options.extra_jars + [args[1]] # Consult the config to see where the standard JARs are installed. jar_path = options.jar_lib if jar_path is None: print >> sys.stderr, "Could not find CIEL bindings. Set the JAR libary path using one of:" print >> sys.stderr, "\tciel jar (--jar-lib|-L) PATH ..." print >> sys.stderr, "\tciel config --set java.jar_lib PATH" sys.exit(-1) for jar_file in glob.glob(os.path.join(jar_path, '*.jar')): jars = jars + [jar_file] class_name = args[2] args = args[3:] def upload_jar(filename): with open(filename, 'r') as infile: return ref_of_string(infile.read(), master_uri) jar_refs = [upload_jar(j) for j in jars] package_dict = {} for binding in options.package: id, filename = binding.split("=", 2) with open(filename, 'r') as infile: package_dict[id] = ref_of_string(infile.read(), master_uri) package_ref = ref_of_string(pickle.dumps(package_dict), master_uri) args = { 'jar_lib': jar_refs, 'class_name': class_name, 'args': args, 'n_outputs': options.num_outputs } init_descriptor = build_init_descriptor("java2", args, package_ref, master_uri, ref_of_string) job_descriptor = submit_job_for_task(init_descriptor, master_uri) job_url = urlparse.urljoin( master_uri, "control/browse/job/%s" % job_descriptor['job_id']) result = await_job(job_descriptor['job_id'], master_uri) try: reflist = simple_retrieve_object_for_ref(result, "json", job_descriptor['job_id'], master_uri) except: print >> sys.stderr, "Error getting list of references as a result." try: j_return = retrieve_object_for_ref(reflist[0], "json", None) except: try: j_return = retrieve_object_for_ref(reflist[0], "noop", None) except: print >> sys.stderr, "Error parsing job result." sys.exit(-1) print j_return