def run_task(self, fw_spec): # print("entering run_task: %s " % str(self)) # Initialize the Flow from the filepath. # Most of the serious stuff is delegated to flow. # Note that FireTask converts all the objects to strings # hence self["flow"] is the workdir of the flow from which # we can reconstruct our object with pickle. print("In MyTask with flow %s" % self["flow"]) workdir = self["flow"]["workdir"] flow = Flow.pickle_load(workdir) flow.build_and_pickle_dump() try: all_ok = fw_spec["all_ok"] except KeyError: all_ok = False if all_ok: print("all_ok will stop") return FWAction() else: print("launched %s" % flow.rapidfire(check_status=True)) all_ok = flow.all_ok new_fw = Firework(FireTaskWithFlow(flow=flow), {'all_ok': all_ok}) return FWAction(stored_data={'all_ok': all_ok}, additions=new_fw)
def main(): def str_examples(): examples = """ Usage example:\n\n runall.py => Run all scripts. """ return examples def show_examples_and_exit(err_msg=None, error_code=1): """Display the usage of the script.""" sys.stderr.write(str_examples()) if err_msg: sys.stderr.write("Fatal Error\n" + err_msg + "\n") sys.exit(error_code) parser = argparse.ArgumentParser( epilog=str_examples(), formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-m', '--mode', type=str, default="sequential", help="execution mode. Default is sequential.") parser.add_argument('-e', '--exclude', type=str, default="", help="Exclude scripts.") parser.add_argument('--keep-dirs', action="store_true", default=False, help="Do not remove flowdirectories.") parser.add_argument('-b', '--bail-on-failure', default=False, help="Exit at the first error.") #parser.add_argument("scripts", nargs="+",help="List of scripts to be executed") options = parser.parse_args() # Find scripts. if options.exclude: options.exclude = options.exclude.split() print("Will exclude:\n", options.exclude) dir = os.path.join(os.path.dirname(__file__)) scripts = [] for fname in os.listdir(dir): if fname in options.exclude: continue if fname.endswith(".py") and fname.startswith("run_"): path = os.path.join(dir, fname) if path != __file__: scripts.append(path) # Run scripts according to mode. dirpaths, retcode = [], 0 if options.mode in ["s", "sequential"]: for script in scripts: # flow will be produced in a temporary workdir. workdir = tempfile.mkdtemp(prefix='flow_' + os.path.basename(script)) ret = call(["python", script, "--workdir", workdir]) retcode += ret if ret != 0: print("retcode %d while running %s" % (ret, script)) if options.bail_on_failure: break dirpaths.append(workdir) execute_flow = False # Comment this line to execute the flow #execute_flow = True if execute_flow: flow = Flow.pickle_load(workdir) print(flow) flow.make_scheduler().start() # Remove directories. if not options.keep_dirs: for dirpath in dirpaths: try: shutil.rmtree(dirpath, ignore_errors=False) except OSError: print("Exception while removing %s" % dirpath) else: show_examples_and_exit(err_msg="Wrong value for mode: %s" % options.mode) print("retcode %d" % retcode) return retcode
def main(): def str_examples(): examples = """ Usage example:\n\n runall.py => Run all scripts. """ return examples def show_examples_and_exit(err_msg=None, error_code=1): """Display the usage of the script.""" sys.stderr.write(str_examples()) if err_msg: sys.stderr.write("Fatal Error\n" + err_msg + "\n") sys.exit(error_code) parser = argparse.ArgumentParser(epilog=str_examples(),formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-V', '--version', action='version', version="%(prog)s version " + __version__) parser.add_argument('--loglevel', default="ERROR", type=str, help="set the loglevel. Possible values: CRITICAL, ERROR (default), WARNING, INFO, DEBUG") parser.add_argument('-m', '--mode', type=str, default="sequential", help="execution mode. Default is sequential.") parser.add_argument('-e', '--exclude', type=str, default="", help="Exclude scripts. Comma-separated names") parser.add_argument('-x', '--execute', default=False, action="store_true", help="Execute flows.") parser.add_argument('--keep-dirs', action="store_true", default=False, help="Do not remove flowdirectories.") parser.add_argument('-b', '--bail-on-failure', default=False, help="Exit at the first error.") #parser.add_argument("scripts", nargs="+",help="List of scripts to be executed") options = parser.parse_args() # loglevel is bound to the string value obtained from the command line argument. # Convert to upper case to allow the user to specify --loglevel=DEBUG or --loglevel=debug import logging numeric_level = getattr(logging, options.loglevel.upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: %s' % options.loglevel) logging.basicConfig(level=numeric_level) # Find scripts. if options.exclude: options.exclude = options.exclude.split(",") print("Will exclude:\n", options.exclude) dir = os.path.join(os.path.dirname(__file__)) scripts = [] for fname in os.listdir(dir): if fname in options.exclude: continue if fname.endswith(".py") and fname.startswith("run_"): path = os.path.join(dir, fname) if path != __file__: scripts.append(path) # Run scripts according to mode. dirpaths, errors, retcode, cnt = [], [], 0, 0 if options.mode in ["s", "sequential"]: for script in scripts: # flow will be produced in a temporary workdir. workdir = tempfile.mkdtemp(prefix='flow_' + os.path.basename(script)) ret = call(["python", script, "--workdir", workdir]) retcode += ret if ret != 0: e = "python %s returned retcode !=0" % script print(e) errors.append(e) if options.bail_on_failure: print("Exiting now since bail_on_failure") break dirpaths.append(workdir) # Here we execute the flow if options.execute: cnt += 1 ret = 0 try: flow = Flow.pickle_load(workdir) flow.make_scheduler().start() if not flow.all_ok: retcode += 1 except Exception as exc: ret += 1 s = "Exception raised during flow execution: %s\n:%s" % (flow, exc) print(s) errors.append(s) if options.bail_on_failure: print("Exiting now since bail_on_failure") break retcode += ret # Remove directories. if not options.keep_dirs: for dirpath in dirpaths: try: shutil.rmtree(dirpath, ignore_errors=False) except OSError: print("Exception while removing %s" % dirpath) else: show_examples_and_exit(err_msg="Wrong value for mode: %s" % options.mode) if errors: for i, err in enumerate(errors): print(92 * "=") print("[%d] %s" % (i, err)) print(92 * "=") print("Number of scripts executed %d/%d" % (cnt, len(scripts)), ", final retcode %d" % retcode) return retcode
def main(): def str_examples(): examples = """ Usage example:\n\n runall.py => Run all scripts. """ return examples def show_examples_and_exit(err_msg=None, error_code=1): """Display the usage of the script.""" sys.stderr.write(str_examples()) if err_msg: sys.stderr.write("Fatal Error\n" + err_msg + "\n") sys.exit(error_code) parser = argparse.ArgumentParser(epilog=str_examples(),formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-m', '--mode', type=str, default="sequential", help="execution mode. Default is sequential.") parser.add_argument('-e', '--exclude', type=str, default="", help="Exclude scripts.") parser.add_argument('--keep-dirs', action="store_true", default=False, help="Do not remove flowdirectories.") parser.add_argument('-b', '--bail-on-failure', default=False, help="Exit at the first error.") #parser.add_argument("scripts", nargs="+",help="List of scripts to be executed") options = parser.parse_args() # Find scripts. if options.exclude: options.exclude = options.exclude.split() print("Will exclude:\n", options.exclude) dir = os.path.join(os.path.dirname(__file__)) scripts = [] for fname in os.listdir(dir): if fname in options.exclude: continue if fname.endswith(".py") and fname.startswith("run_"): path = os.path.join(dir, fname) if path != __file__: scripts.append(path) # Run scripts according to mode. dirpaths, retcode = [], 0 if options.mode in ["s", "sequential"]: for script in scripts: # flow will be produced in a temporary workdir. workdir = tempfile.mkdtemp(prefix='flow_' + os.path.basename(script)) ret = call(["python", script, "--workdir", workdir]) retcode += ret if ret != 0: print("retcode %d while running %s" % (ret, script)) if options.bail_on_failure: break dirpaths.append(workdir) execute_flow = False # Comment this line to execute the flow #execute_flow = True if execute_flow: flow = Flow.pickle_load(workdir) print(flow) flow.make_scheduler().start() # Remove directories. if not options.keep_dirs: for dirpath in dirpaths: try: shutil.rmtree(dirpath, ignore_errors=False) except OSError: print("Exception while removing %s" % dirpath) else: show_examples_and_exit(err_msg="Wrong value for mode: %s" % options.mode) print("retcode %d" % retcode) return retcode
def main(): def str_examples(): examples = """ Usage example:\n\n runall.py => Run all scripts. """ return examples def show_examples_and_exit(err_msg=None, error_code=1): """Display the usage of the script.""" sys.stderr.write(str_examples()) if err_msg: sys.stderr.write("Fatal Error\n" + err_msg + "\n") sys.exit(error_code) parser = argparse.ArgumentParser( epilog=str_examples(), formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-V', '--version', action='version', version="%(prog)s version " + __version__) parser.add_argument( '--loglevel', default="ERROR", type=str, help= "set the loglevel. Possible values: CRITICAL, ERROR (default), WARNING, INFO, DEBUG" ) parser.add_argument('-m', '--mode', type=str, default="sequential", help="execution mode. Default is sequential.") parser.add_argument('-e', '--exclude', type=str, default="", help="Exclude scripts. Comma-separated names") parser.add_argument('-x', '--execute', default=False, action="store_true", help="Execute flows.") parser.add_argument('--keep-dirs', action="store_true", default=False, help="Do not remove flowdirectories.") parser.add_argument('-b', '--bail-on-failure', default=False, help="Exit at the first error.") #parser.add_argument("scripts", nargs="+",help="List of scripts to be executed") options = parser.parse_args() # loglevel is bound to the string value obtained from the command line argument. # Convert to upper case to allow the user to specify --loglevel=DEBUG or --loglevel=debug import logging numeric_level = getattr(logging, options.loglevel.upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: %s' % options.loglevel) logging.basicConfig(level=numeric_level) # Find scripts. if options.exclude: options.exclude = options.exclude.split(",") print("Will exclude:\n", options.exclude) dir = os.path.join(os.path.dirname(__file__)) scripts = [] for fname in os.listdir(dir): if fname in options.exclude: continue if fname.endswith(".py") and fname.startswith("run_"): path = os.path.join(dir, fname) if path != __file__: scripts.append(path) # Run scripts according to mode. dirpaths, errors, retcode, cnt = [], [], 0, 0 if options.mode in ["s", "sequential"]: for script in scripts: # flow will be produced in a temporary workdir. workdir = tempfile.mkdtemp(prefix='flow_' + os.path.basename(script)) ret = call(["python", script, "--workdir", workdir]) retcode += ret if ret != 0: e = "python %s returned retcode !=0" % script print(e) errors.append(e) if options.bail_on_failure: print("Exiting now since bail_on_failure") break dirpaths.append(workdir) # Here we execute the flow if options.execute: cnt += 1 ret = 0 try: flow = Flow.pickle_load(workdir) flow.make_scheduler().start() if not flow.all_ok: retcode += 1 except Exception as exc: ret += 1 s = "Exception raised during flow execution: %s\n:%s" % ( flow, exc) print(s) errors.append(s) if options.bail_on_failure: print("Exiting now since bail_on_failure") break retcode += ret # Remove directories. if not options.keep_dirs: for dirpath in dirpaths: try: shutil.rmtree(dirpath, ignore_errors=False) except OSError: print("Exception while removing %s" % dirpath) else: show_examples_and_exit(err_msg="Wrong value for mode: %s" % options.mode) if errors: for i, err in enumerate(errors): print(92 * "=") print("[%d] %s" % (i, err)) print(92 * "=") print("Number of scripts executed %d/%d" % (cnt, len(scripts)), ", final retcode %d" % retcode) return retcode