Пример #1
0
        def put(self):
            """
            Queue the specific pipeline
            """

            data   = request.get_json(force=True)
            config = data.get('config')
            user   = data.get('user')

            errors = {}
            step = Step.load_step(config)
            errors = step.validate_config(config)
            if not step.output_dir:
                errors['output_dir'] = 'missing value'

            if not errors:
                # Get id from DB
                db_info = dbmodel.PipelineDb(config['name'], config, [step.name], user, output_dir=step.output_dir)
                config['run_id'] = db_info.run_id

                ut.pretty_print("Submitting step %s (ID %d) for user %s" % (config['name'], config['run_id'], user))
                return pm.add_step(config, user)
            else:
                return errors, 400
Пример #2
0
    doc="""
    Execute a step class
    """

    parser = argparse.ArgumentParser(description=doc,
                                     formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('cfg_file',
                        type=str,
                        help='config file of the step')

    args = parser.parse_args()
    user = getpass.getuser()

    # create a step    
    step = Step.load_step(args.cfg_file)
    if not os.path.exists(step.output_dir):
        os.makedirs(step.output_dir, 0775)

    # remove existing files, except step config and condor files
    full_list = glob.glob(step.output_dir + "/*")
    regex = re.compile("(job\.*|condor\.*|.*\.cfg)")
    to_remove = filter(lambda f: not regex.search(f), full_list)
    for entry in to_remove:
        cmd = ['rm', '-rvf', entry]
        (ec, err, out) = run_as(cmd=cmd, user=user)
        if ec:
            print "WARNING: failed to remove file %s: %s, %s" % (entry, err, out)
        else:
            print "Removed %s" % entry
Пример #3
0
    args = parser.parse_args()


    is_step = False
    try:
        config = Pipeline.load_cfg(args.config_file)
        if 'sys_path' not in config:
            config['sys_path'] = os.path.dirname(os.path.realpath(args.config_file))
    except Exception as e1:
        raise e1
        config = Step.load_cfg(args.config_file)
        if 'sys_path' not in config:
            config['sys_path'] = os.path.dirname(os.path.realpath(args.config_file))  

        step = Step.load_step(config)
        is_step = True


    if args.custom:
        apply_custom(config,args.custom)

    if args.local:
        ut.pretty_print("Instantiating the Pipeline...")
        p = Pipeline(config, user=args.user, db=args.db, schedname='SCHED_LOCAL')
        ut.pretty_print("Running the pipeline...")
        p.run(local=True, verbose=args.verbose)
    elif args.interactive:
        global pi
        ut.pretty_print("Instantiating the Pipeline...")
        pi = Pipeline(config, user=args.user)
Пример #4
0
if __name__ == '__main__':

    doc = """
    Execute a step class
    """

    parser = argparse.ArgumentParser(
        description=doc, formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('cfg_file', type=str, help='config file of the step')

    args = parser.parse_args()
    user = getpass.getuser()

    # create a step
    step = Step.load_step(args.cfg_file)
    if not os.path.exists(step.output_dir):
        os.makedirs(step.output_dir, 0775)

    # remove existing files, except step config and condor files
    full_list = glob.glob(step.output_dir + "/*")
    regex = re.compile("(job\.*|condor\.*|.*\.cfg)")
    to_remove = filter(lambda f: not regex.search(f), full_list)
    for entry in to_remove:
        cmd = ['rm', '-rvf', entry]
        (ec, err, out) = run_as(cmd=cmd, user=user)
        if ec:
            print "WARNING: failed to remove file %s: %s, %s" % (entry, err,
                                                                 out)
        else:
            print "Removed %s" % entry
Пример #5
0
    args = parser.parse_args()

    is_step = False
    try:
        config = Pipeline.load_cfg(args.config_file)
        if 'sys_path' not in config:
            config['sys_path'] = os.path.dirname(
                os.path.realpath(args.config_file))
    except Exception as e1:
        raise e1
        config = Step.load_cfg(args.config_file)
        if 'sys_path' not in config:
            config['sys_path'] = os.path.dirname(
                os.path.realpath(args.config_file))

        step = Step.load_step(config)
        is_step = True

    if args.custom:
        apply_custom(config, args.custom)

    if args.local:
        ut.pretty_print("Instantiating the Pipeline...")
        p = Pipeline(config,
                     user=args.user,
                     db=args.db,
                     schedname='SCHED_LOCAL')
        ut.pretty_print("Running the pipeline...")
        p.run(local=True, verbose=args.verbose)
    elif args.interactive:
        global pi