Ejemplo n.º 1
0
    def run_task(self, fw_spec):
        # print("entering run_task: %s " % str(self))

        # Initialize the Flow from the filepath.
        # Most of the serious stuff is delegated to flow.
        # Note that FireTask converts all the objects to strings
        # hence self["flow"] is the workdir of the flow from which
        # we can reconstruct our object with pickle.
        print("In MyTask with flow %s" % self["flow"])
        workdir = self["flow"]["workdir"]
        flow = Flow.pickle_load(workdir)

        flow.build_and_pickle_dump()
        try:
            all_ok = fw_spec["all_ok"]
        except KeyError:
            all_ok = False

        if all_ok:
            print("all_ok will stop")
            return FWAction()
        else:
            print("launched %s" % flow.rapidfire(check_status=True))
            all_ok = flow.all_ok
            new_fw = Firework(FireTaskWithFlow(flow=flow), {'all_ok': all_ok})
            return FWAction(stored_data={'all_ok': all_ok}, additions=new_fw)
Ejemplo n.º 2
0
    def run_task(self, fw_spec):
        # print("entering run_task: %s " % str(self))

        # Initialize the Flow from the filepath.
        # Most of the serious stuff is delegated to flow.
        # Note that FireTask converts all the objects to strings
        # hence self["flow"] is the workdir of the flow from which
        # we can reconstruct our object with pickle.
        print("In MyTask with flow %s" % self["flow"])
        workdir = self["flow"]["workdir"]
        flow = Flow.pickle_load(workdir)

        flow.build_and_pickle_dump()
        try:
            all_ok = fw_spec["all_ok"]
        except KeyError:
            all_ok = False

        if all_ok:
            print("all_ok will stop")
            return FWAction()
        else:
            print("launched %s" % flow.rapidfire(check_status=True))
            all_ok = flow.all_ok
            new_fw = Firework(FireTaskWithFlow(flow=flow), {'all_ok': all_ok})
            return FWAction(stored_data={'all_ok': all_ok}, additions=new_fw)
Ejemplo n.º 3
0
def main():
    def str_examples():
        examples = """
          Usage example:\n\n
          runall.py => Run all scripts.
        """
        return examples

    def show_examples_and_exit(err_msg=None, error_code=1):
        """Display the usage of the script."""
        sys.stderr.write(str_examples())
        if err_msg:
            sys.stderr.write("Fatal Error\n" + err_msg + "\n")
        sys.exit(error_code)

    parser = argparse.ArgumentParser(
        epilog=str_examples(),
        formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument('-m',
                        '--mode',
                        type=str,
                        default="sequential",
                        help="execution mode. Default is sequential.")

    parser.add_argument('-e',
                        '--exclude',
                        type=str,
                        default="",
                        help="Exclude scripts.")

    parser.add_argument('--keep-dirs',
                        action="store_true",
                        default=False,
                        help="Do not remove flowdirectories.")

    parser.add_argument('-b',
                        '--bail-on-failure',
                        default=False,
                        help="Exit at the first error.")

    #parser.add_argument("scripts", nargs="+",help="List of scripts to be executed")

    options = parser.parse_args()

    # Find scripts.
    if options.exclude:
        options.exclude = options.exclude.split()
        print("Will exclude:\n", options.exclude)

    dir = os.path.join(os.path.dirname(__file__))
    scripts = []
    for fname in os.listdir(dir):
        if fname in options.exclude: continue
        if fname.endswith(".py") and fname.startswith("run_"):
            path = os.path.join(dir, fname)
            if path != __file__:
                scripts.append(path)

    # Run scripts according to mode.
    dirpaths, retcode = [], 0
    if options.mode in ["s", "sequential"]:
        for script in scripts:
            # flow will be produced in a temporary workdir.
            workdir = tempfile.mkdtemp(prefix='flow_' +
                                       os.path.basename(script))
            ret = call(["python", script, "--workdir", workdir])
            retcode += ret

            if ret != 0:
                print("retcode %d while running %s" % (ret, script))
                if options.bail_on_failure: break

            dirpaths.append(workdir)

            execute_flow = False
            # Comment this line to execute the flow
            #execute_flow = True
            if execute_flow:
                flow = Flow.pickle_load(workdir)
                print(flow)
                flow.make_scheduler().start()

        # Remove directories.
        if not options.keep_dirs:
            for dirpath in dirpaths:
                try:
                    shutil.rmtree(dirpath, ignore_errors=False)
                except OSError:
                    print("Exception while removing %s" % dirpath)

    else:
        show_examples_and_exit(err_msg="Wrong value for mode: %s" %
                               options.mode)

    print("retcode %d" % retcode)
    return retcode
Ejemplo n.º 4
0
def main():
    def str_examples():
        examples = """
          Usage example:\n\n
          runall.py => Run all scripts.
        """
        return examples

    def show_examples_and_exit(err_msg=None, error_code=1):
        """Display the usage of the script."""
        sys.stderr.write(str_examples())
        if err_msg: 
            sys.stderr.write("Fatal Error\n" + err_msg + "\n")
        sys.exit(error_code)

    parser = argparse.ArgumentParser(epilog=str_examples(),formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument('-V', '--version', action='version', version="%(prog)s version " + __version__)
    parser.add_argument('--loglevel', default="ERROR", type=str,
                        help="set the loglevel. Possible values: CRITICAL, ERROR (default), WARNING, INFO, DEBUG")

    parser.add_argument('-m', '--mode', type=str, default="sequential", help="execution mode. Default is sequential.")

    parser.add_argument('-e', '--exclude', type=str, default="", help="Exclude scripts. Comma-separated names")

    parser.add_argument('-x', '--execute', default=False, action="store_true", help="Execute flows.")

    parser.add_argument('--keep-dirs', action="store_true", default=False,
                        help="Do not remove flowdirectories.")

    parser.add_argument('-b', '--bail-on-failure', default=False, help="Exit at the first error.")

    #parser.add_argument("scripts", nargs="+",help="List of scripts to be executed")

    options = parser.parse_args()

    # loglevel is bound to the string value obtained from the command line argument. 
    # Convert to upper case to allow the user to specify --loglevel=DEBUG or --loglevel=debug
    import logging
    numeric_level = getattr(logging, options.loglevel.upper(), None)
    if not isinstance(numeric_level, int):
        raise ValueError('Invalid log level: %s' % options.loglevel)
    logging.basicConfig(level=numeric_level)

    # Find scripts.
    if options.exclude:
        options.exclude = options.exclude.split(",")
        print("Will exclude:\n", options.exclude)

    dir = os.path.join(os.path.dirname(__file__))
    scripts = []
    for fname in os.listdir(dir):
        if fname in options.exclude: continue
        if fname.endswith(".py") and fname.startswith("run_"):
            path = os.path.join(dir, fname)
            if path != __file__:
                scripts.append(path)

    # Run scripts according to mode.
    dirpaths, errors, retcode, cnt = [], [], 0, 0
    if options.mode in ["s", "sequential"]:
        for script in scripts:
            # flow will be produced in a temporary workdir.
            workdir = tempfile.mkdtemp(prefix='flow_' + os.path.basename(script))
            ret = call(["python", script, "--workdir", workdir])
            retcode += ret

            if ret != 0: 
                e = "python %s returned retcode !=0" % script
                print(e)
                errors.append(e)
                if options.bail_on_failure: 
                    print("Exiting now since bail_on_failure")
                    break

            dirpaths.append(workdir)

            # Here we execute the flow
            if options.execute:
                cnt += 1
                ret = 0
                try:
                    flow = Flow.pickle_load(workdir)
                    flow.make_scheduler().start()
                    if not flow.all_ok: retcode += 1

                except Exception as exc:
                    ret += 1
                    s = "Exception raised during flow execution: %s\n:%s" % (flow, exc)
                    print(s)
                    errors.append(s)
                    if options.bail_on_failure: 
                        print("Exiting now since bail_on_failure")
                        break
                    retcode += ret

        # Remove directories.
        if not options.keep_dirs:
            for dirpath in dirpaths:
                try:
                    shutil.rmtree(dirpath, ignore_errors=False)
                except OSError:
                    print("Exception while removing %s" % dirpath)

    else:
        show_examples_and_exit(err_msg="Wrong value for mode: %s" % options.mode)

    if errors:
        for i, err in enumerate(errors):
            print(92 * "=")
            print("[%d] %s" % (i, err))
            print(92 * "=")

    print("Number of scripts executed %d/%d" % (cnt, len(scripts)), ", final retcode %d" % retcode)
    return retcode
Ejemplo n.º 5
0
def main():
    def str_examples():
        examples = """
          Usage example:\n\n
          runall.py => Run all scripts.
        """
        return examples

    def show_examples_and_exit(err_msg=None, error_code=1):
        """Display the usage of the script."""
        sys.stderr.write(str_examples())
        if err_msg: 
            sys.stderr.write("Fatal Error\n" + err_msg + "\n")
        sys.exit(error_code)

    parser = argparse.ArgumentParser(epilog=str_examples(),formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument('-m', '--mode', type=str, default="sequential",
                        help="execution mode. Default is sequential.")

    parser.add_argument('-e', '--exclude', type=str, default="", help="Exclude scripts.")

    parser.add_argument('--keep-dirs', action="store_true", default=False,
                        help="Do not remove flowdirectories.")

    parser.add_argument('-b', '--bail-on-failure', default=False, help="Exit at the first error.")

    #parser.add_argument("scripts", nargs="+",help="List of scripts to be executed")

    options = parser.parse_args()

    # Find scripts.
    if options.exclude:
        options.exclude = options.exclude.split()
        print("Will exclude:\n", options.exclude)

    dir = os.path.join(os.path.dirname(__file__))
    scripts = []
    for fname in os.listdir(dir):
        if fname in options.exclude: continue
        if fname.endswith(".py") and fname.startswith("run_"):
            path = os.path.join(dir, fname)
            if path != __file__:
                scripts.append(path)

    # Run scripts according to mode.
    dirpaths, retcode = [], 0
    if options.mode in ["s", "sequential"]:
        for script in scripts:
            # flow will be produced in a temporary workdir.
            workdir = tempfile.mkdtemp(prefix='flow_' + os.path.basename(script))
            ret = call(["python", script, "--workdir", workdir])
            retcode += ret

            if ret != 0: 
                print("retcode %d while running %s" % (ret, script))
                if options.bail_on_failure: break

            dirpaths.append(workdir)

            execute_flow = False
            # Comment this line to execute the flow
            #execute_flow = True
            if execute_flow:
                flow = Flow.pickle_load(workdir)
                print(flow)
                flow.make_scheduler().start()

        # Remove directories.
        if not options.keep_dirs:
            for dirpath in dirpaths:
                try:
                    shutil.rmtree(dirpath, ignore_errors=False)
                except OSError:
                    print("Exception while removing %s" % dirpath)

    else:
        show_examples_and_exit(err_msg="Wrong value for mode: %s" % options.mode)

    print("retcode %d" % retcode)
    return retcode
Ejemplo n.º 6
0
def main():
    def str_examples():
        examples = """
          Usage example:\n\n
          runall.py => Run all scripts.
        """
        return examples

    def show_examples_and_exit(err_msg=None, error_code=1):
        """Display the usage of the script."""
        sys.stderr.write(str_examples())
        if err_msg:
            sys.stderr.write("Fatal Error\n" + err_msg + "\n")
        sys.exit(error_code)

    parser = argparse.ArgumentParser(
        epilog=str_examples(),
        formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument('-V',
                        '--version',
                        action='version',
                        version="%(prog)s version " + __version__)
    parser.add_argument(
        '--loglevel',
        default="ERROR",
        type=str,
        help=
        "set the loglevel. Possible values: CRITICAL, ERROR (default), WARNING, INFO, DEBUG"
    )

    parser.add_argument('-m',
                        '--mode',
                        type=str,
                        default="sequential",
                        help="execution mode. Default is sequential.")

    parser.add_argument('-e',
                        '--exclude',
                        type=str,
                        default="",
                        help="Exclude scripts. Comma-separated names")

    parser.add_argument('-x',
                        '--execute',
                        default=False,
                        action="store_true",
                        help="Execute flows.")

    parser.add_argument('--keep-dirs',
                        action="store_true",
                        default=False,
                        help="Do not remove flowdirectories.")

    parser.add_argument('-b',
                        '--bail-on-failure',
                        default=False,
                        help="Exit at the first error.")

    #parser.add_argument("scripts", nargs="+",help="List of scripts to be executed")

    options = parser.parse_args()

    # loglevel is bound to the string value obtained from the command line argument.
    # Convert to upper case to allow the user to specify --loglevel=DEBUG or --loglevel=debug
    import logging
    numeric_level = getattr(logging, options.loglevel.upper(), None)
    if not isinstance(numeric_level, int):
        raise ValueError('Invalid log level: %s' % options.loglevel)
    logging.basicConfig(level=numeric_level)

    # Find scripts.
    if options.exclude:
        options.exclude = options.exclude.split(",")
        print("Will exclude:\n", options.exclude)

    dir = os.path.join(os.path.dirname(__file__))
    scripts = []
    for fname in os.listdir(dir):
        if fname in options.exclude: continue
        if fname.endswith(".py") and fname.startswith("run_"):
            path = os.path.join(dir, fname)
            if path != __file__:
                scripts.append(path)

    # Run scripts according to mode.
    dirpaths, errors, retcode, cnt = [], [], 0, 0
    if options.mode in ["s", "sequential"]:
        for script in scripts:
            # flow will be produced in a temporary workdir.
            workdir = tempfile.mkdtemp(prefix='flow_' +
                                       os.path.basename(script))
            ret = call(["python", script, "--workdir", workdir])
            retcode += ret

            if ret != 0:
                e = "python %s returned retcode !=0" % script
                print(e)
                errors.append(e)
                if options.bail_on_failure:
                    print("Exiting now since bail_on_failure")
                    break

            dirpaths.append(workdir)

            # Here we execute the flow
            if options.execute:
                cnt += 1
                ret = 0
                try:
                    flow = Flow.pickle_load(workdir)
                    flow.make_scheduler().start()
                    if not flow.all_ok: retcode += 1

                except Exception as exc:
                    ret += 1
                    s = "Exception raised during flow execution: %s\n:%s" % (
                        flow, exc)
                    print(s)
                    errors.append(s)
                    if options.bail_on_failure:
                        print("Exiting now since bail_on_failure")
                        break
                    retcode += ret

        # Remove directories.
        if not options.keep_dirs:
            for dirpath in dirpaths:
                try:
                    shutil.rmtree(dirpath, ignore_errors=False)
                except OSError:
                    print("Exception while removing %s" % dirpath)

    else:
        show_examples_and_exit(err_msg="Wrong value for mode: %s" %
                               options.mode)

    if errors:
        for i, err in enumerate(errors):
            print(92 * "=")
            print("[%d] %s" % (i, err))
            print(92 * "=")

    print("Number of scripts executed %d/%d" % (cnt, len(scripts)),
          ", final retcode %d" % retcode)
    return retcode