def put(self):
"""
Queue the specific pipeline
"""
data = request.get_json(force=True)
config = data.get('config')
user = data.get('user')
errors = {}
step = Step.load_step(config)
errors = step.validate_config(config)
if not step.output_dir:
errors['output_dir'] = 'missing value'
if not errors:
# Get id from DB
db_info = dbmodel.PipelineDb(config['name'], config, [step.name], user, output_dir=step.output_dir)
config['run_id'] = db_info.run_id
ut.pretty_print("Submitting step %s (ID %d) for user %s" % (config['name'], config['run_id'], user))
return pm.add_step(config, user)
else:
return errors, 400
doc="""
Execute a step class
"""
parser = argparse.ArgumentParser(description=doc,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('cfg_file',
type=str,
help='config file of the step')
args = parser.parse_args()
user = getpass.getuser()
# create a step
step = Step.load_step(args.cfg_file)
if not os.path.exists(step.output_dir):
os.makedirs(step.output_dir, 0775)
# remove existing files, except step config and condor files
full_list = glob.glob(step.output_dir + "/*")
regex = re.compile("(job\.*|condor\.*|.*\.cfg)")
to_remove = filter(lambda f: not regex.search(f), full_list)
for entry in to_remove:
cmd = ['rm', '-rvf', entry]
(ec, err, out) = run_as(cmd=cmd, user=user)
if ec:
print "WARNING: failed to remove file %s: %s, %s" % (entry, err, out)
else:
print "Removed %s" % entry
args = parser.parse_args()
is_step = False
try:
config = Pipeline.load_cfg(args.config_file)
if 'sys_path' not in config:
config['sys_path'] = os.path.dirname(os.path.realpath(args.config_file))
except Exception as e1:
raise e1
config = Step.load_cfg(args.config_file)
if 'sys_path' not in config:
config['sys_path'] = os.path.dirname(os.path.realpath(args.config_file))
step = Step.load_step(config)
is_step = True
if args.custom:
apply_custom(config,args.custom)
if args.local:
ut.pretty_print("Instantiating the Pipeline...")
p = Pipeline(config, user=args.user, db=args.db, schedname='SCHED_LOCAL')
ut.pretty_print("Running the pipeline...")
p.run(local=True, verbose=args.verbose)
elif args.interactive:
global pi
ut.pretty_print("Instantiating the Pipeline...")
pi = Pipeline(config, user=args.user)
if __name__ == '__main__':
doc = """
Execute a step class
"""
parser = argparse.ArgumentParser(
description=doc, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('cfg_file', type=str, help='config file of the step')
args = parser.parse_args()
user = getpass.getuser()
# create a step
step = Step.load_step(args.cfg_file)
if not os.path.exists(step.output_dir):
os.makedirs(step.output_dir, 0775)
# remove existing files, except step config and condor files
full_list = glob.glob(step.output_dir + "/*")
regex = re.compile("(job\.*|condor\.*|.*\.cfg)")
to_remove = filter(lambda f: not regex.search(f), full_list)
for entry in to_remove:
cmd = ['rm', '-rvf', entry]
(ec, err, out) = run_as(cmd=cmd, user=user)
if ec:
print "WARNING: failed to remove file %s: %s, %s" % (entry, err,
out)
else:
print "Removed %s" % entry
args = parser.parse_args()
is_step = False
try:
config = Pipeline.load_cfg(args.config_file)
if 'sys_path' not in config:
config['sys_path'] = os.path.dirname(
os.path.realpath(args.config_file))
except Exception as e1:
raise e1
config = Step.load_cfg(args.config_file)
if 'sys_path' not in config:
config['sys_path'] = os.path.dirname(
os.path.realpath(args.config_file))
step = Step.load_step(config)
is_step = True
if args.custom:
apply_custom(config, args.custom)
if args.local:
ut.pretty_print("Instantiating the Pipeline...")
p = Pipeline(config,
user=args.user,
db=args.db,
schedname='SCHED_LOCAL')
ut.pretty_print("Running the pipeline...")
p.run(local=True, verbose=args.verbose)
elif args.interactive:
global pi