def main(arguments=None):
args = arvrun_parser.parse_args(arguments)
if len(args.args) == 0:
arvrun_parser.print_help()
return
starting_args = args.args
reading_into = 2
# Parse the command arguments into 'slots'.
# All words following '>' are output arguments and are collected into slots[0].
# All words following '<' are input arguments and are collected into slots[1].
# slots[2..] store the parameters of each command in the pipeline.
#
# e.g. arv-run foo arg1 arg2 '|' bar arg3 arg4 '<' input1 input2 input3 '>' output.txt
# will be parsed into:
# [['output.txt'],
# ['input1', 'input2', 'input3'],
# ['foo', 'arg1', 'arg2'],
# ['bar', 'arg3', 'arg4']]
slots = [[], [], []]
for c in args.args:
if c.startswith('>'):
reading_into = 0
if len(c) > 1:
slots[reading_into].append(c[1:])
elif c.startswith('<'):
reading_into = 1
if len(c) > 1:
slots[reading_into].append(c[1:])
elif c == '|':
reading_into = len(slots)
slots.append([])
else:
slots[reading_into].append(c)
if slots[0] and len(slots[0]) > 1:
logger.error("Can only specify a single stdout file (run-command substitutions are permitted)")
return
if not args.dry_run:
api = arvados.api('v1')
if args.project_uuid:
project = args.project_uuid
else:
project = determine_project(os.getcwd(), api.users().current().execute()["uuid"])
# Identify input files. Look at each parameter and test to see if there is
# a file by that name. This uses 'patterns' to look for within
# command line arguments, such as --foo=file.txt or -lfile.txt
patterns = [re.compile("([^=]+=)(.*)"),
re.compile("(-[A-Za-z])(.+)")]
for j, command in enumerate(slots[1:]):
for i, a in enumerate(command):
if j > 0 and i == 0:
# j == 0 is stdin, j > 0 is commands
# always skip program executable (i == 0) in commands
pass
elif a.startswith('\\'):
# if it starts with a \ then don't do any interpretation
command[i] = a[1:]
else:
# See if it looks like a file
command[i] = statfile('', a)
# If a file named command[i] was found, it would now be an
# ArvFile or UploadFile. If command[i] is a basestring, that
# means it doesn't correspond exactly to a file, so do some
# pattern matching.
if isinstance(command[i], basestring):
for p in patterns:
m = p.match(a)
if m:
command[i] = statfile(m.group(1), m.group(2))
break
files = [c for command in slots[1:] for c in command if isinstance(c, UploadFile)]
if files:
uploadfiles(files, api, dry_run=args.dry_run, num_retries=args.retries, project=project)
for i in range(1, len(slots)):
slots[i] = [("%s%s" % (c.prefix, c.fn)) if isinstance(c, ArvFile) else c for c in slots[i]]
component = {
"script": "run-command",
"script_version": args.script_version,
"repository": args.repository,
"script_parameters": {
},
"runtime_constraints": {}
}
if args.docker_image:
component["runtime_constraints"]["docker_image"] = args.docker_image
task_foreach = []
group_parser = argparse.ArgumentParser()
group_parser.add_argument('-b', '--batch-size', type=int)
group_parser.add_argument('args', nargs=argparse.REMAINDER)
for s in range(2, len(slots)):
for i in range(0, len(slots[s])):
if slots[s][i] == '--':
inp = "input%i" % (s-2)
groupargs = group_parser.parse_args(slots[2][i+1:])
if groupargs.batch_size:
component["script_parameters"][inp] = {"value": {"batch":groupargs.args, "size":groupargs.batch_size}}
slots[s] = slots[s][0:i] + [{"foreach": inp, "command": "$(%s)" % inp}]
else:
component["script_parameters"][inp] = groupargs.args
slots[s] = slots[s][0:i] + ["$(%s)" % inp]
task_foreach.append(inp)
break
if slots[s][i] == '\--':
slots[s][i] = '--'
if slots[0]:
component["script_parameters"]["task.stdout"] = slots[0][0]
if slots[1]:
task_foreach.append("stdin")
component["script_parameters"]["stdin"] = slots[1]
component["script_parameters"]["task.stdin"] = "$(stdin)"
if task_foreach:
component["script_parameters"]["task.foreach"] = task_foreach
component["script_parameters"]["command"] = slots[2:]
if args.ignore_rcode:
component["script_parameters"]["task.ignore_rcode"] = args.ignore_rcode
pipeline = {
"name": "arv-run " + " | ".join([s[0] for s in slots[2:]]),
"description": "@" + " ".join(starting_args) + "@",
"components": {
"command": component
},
"state": "RunningOnClient" if args.local else "RunningOnServer"
}
if args.dry_run:
print(json.dumps(pipeline, indent=4))
else:
pipeline["owner_uuid"] = project
pi = api.pipeline_instances().create(body=pipeline, ensure_unique_name=True).execute()
logger.info("Running pipeline %s", pi["uuid"])
if args.local:
subprocess.call(["arv-run-pipeline-instance", "--instance", pi["uuid"], "--run-jobs-here"] + (["--no-reuse"] if args.no_reuse else []))
elif not args.no_wait:
ws.main(["--pipeline", pi["uuid"]])
pi = api.pipeline_instances().get(uuid=pi["uuid"]).execute()
logger.info("Pipeline is %s", pi["state"])
if "output_uuid" in pi["components"]["command"]:
logger.info("Output is %s", pi["components"]["command"]["output_uuid"])
else:
logger.info("No output")
def main(arguments=None):
args = arvrun_parser.parse_args(arguments)
if len(args.args) == 0:
arvrun_parser.print_help()
return
starting_args = args.args
reading_into = 2
# Parse the command arguments into 'slots'.
# All words following '>' are output arguments and are collected into slots[0].
# All words following '<' are input arguments and are collected into slots[1].
# slots[2..] store the parameters of each command in the pipeline.
#
# e.g. arv-run foo arg1 arg2 '|' bar arg3 arg4 '<' input1 input2 input3 '>' output.txt
# will be parsed into:
# [['output.txt'],
# ['input1', 'input2', 'input3'],
# ['foo', 'arg1', 'arg2'],
# ['bar', 'arg3', 'arg4']]
slots = [[], [], []]
for c in args.args:
if c.startswith('>'):
reading_into = 0
if len(c) > 1:
slots[reading_into].append(c[1:])
elif c.startswith('<'):
reading_into = 1
if len(c) > 1:
slots[reading_into].append(c[1:])
elif c == '|':
reading_into = len(slots)
slots.append([])
else:
slots[reading_into].append(c)
if slots[0] and len(slots[0]) > 1:
logger.error(
"Can only specify a single stdout file (run-command substitutions are permitted)"
)
return
if not args.dry_run:
api = arvados.api('v1')
if args.project_uuid:
project = args.project_uuid
else:
project = determine_project(
os.getcwd(),
api.users().current().execute()["uuid"])
# Identify input files. Look at each parameter and test to see if there is
# a file by that name. This uses 'patterns' to look for within
# command line arguments, such as --foo=file.txt or -lfile.txt
patterns = [re.compile("([^=]+=)(.*)"), re.compile("(-[A-Za-z])(.+)")]
for j, command in enumerate(slots[1:]):
for i, a in enumerate(command):
if j > 0 and i == 0:
# j == 0 is stdin, j > 0 is commands
# always skip program executable (i == 0) in commands
pass
elif a.startswith('\\'):
# if it starts with a \ then don't do any interpretation
command[i] = a[1:]
else:
# See if it looks like a file
command[i] = statfile('', a)
# If a file named command[i] was found, it would now be an
# ArvFile or UploadFile. If command[i] is a basestring, that
# means it doesn't correspond exactly to a file, so do some
# pattern matching.
if isinstance(command[i], basestring):
for p in patterns:
m = p.match(a)
if m:
command[i] = statfile(m.group(1), m.group(2))
break
files = [
c for command in slots[1:] for c in command
if isinstance(c, UploadFile)
]
if files:
uploadfiles(files,
api,
dry_run=args.dry_run,
num_retries=args.retries,
project=project)
for i in xrange(1, len(slots)):
slots[i] = [
("%s%s" % (c.prefix, c.fn)) if isinstance(c, ArvFile) else c
for c in slots[i]
]
component = {
"script": "run-command",
"script_version": args.script_version,
"repository": args.repository,
"script_parameters": {},
"runtime_constraints": {}
}
if args.docker_image:
component["runtime_constraints"]["docker_image"] = args.docker_image
task_foreach = []
group_parser = argparse.ArgumentParser()
group_parser.add_argument('-b', '--batch-size', type=int)
group_parser.add_argument('args', nargs=argparse.REMAINDER)
for s in xrange(2, len(slots)):
for i in xrange(0, len(slots[s])):
if slots[s][i] == '--':
inp = "input%i" % (s - 2)
groupargs = group_parser.parse_args(slots[2][i + 1:])
if groupargs.batch_size:
component["script_parameters"][inp] = {
"value": {
"batch": groupargs.args,
"size": groupargs.batch_size
}
}
slots[s] = slots[s][0:i] + [{
"foreach": inp,
"command": "$(%s)" % inp
}]
else:
component["script_parameters"][inp] = groupargs.args
slots[s] = slots[s][0:i] + ["$(%s)" % inp]
task_foreach.append(inp)
break
if slots[s][i] == '\--':
slots[s][i] = '--'
if slots[0]:
component["script_parameters"]["task.stdout"] = slots[0][0]
if slots[1]:
task_foreach.append("stdin")
component["script_parameters"]["stdin"] = slots[1]
component["script_parameters"]["task.stdin"] = "$(stdin)"
if task_foreach:
component["script_parameters"]["task.foreach"] = task_foreach
component["script_parameters"]["command"] = slots[2:]
if args.ignore_rcode:
component["script_parameters"]["task.ignore_rcode"] = args.ignore_rcode
pipeline = {
"name": "arv-run " + " | ".join([s[0] for s in slots[2:]]),
"description": "@" + " ".join(starting_args) + "@",
"components": {
"command": component
},
"state": "RunningOnClient" if args.local else "RunningOnServer"
}
if args.dry_run:
print(json.dumps(pipeline, indent=4))
else:
pipeline["owner_uuid"] = project
pi = api.pipeline_instances().create(
body=pipeline, ensure_unique_name=True).execute()
logger.info("Running pipeline %s", pi["uuid"])
if args.local:
subprocess.call([
"arv-run-pipeline-instance", "--instance", pi["uuid"],
"--run-jobs-here"
] + (["--no-reuse"] if args.no_reuse else []))
elif not args.no_wait:
ws.main(["--pipeline", pi["uuid"]])
pi = api.pipeline_instances().get(uuid=pi["uuid"]).execute()
logger.info("Pipeline is %s", pi["state"])
if "output_uuid" in pi["components"]["command"]:
logger.info("Output is %s",
pi["components"]["command"]["output_uuid"])
else:
logger.info("No output")
def run_ws(self, args):
return arv_ws.main(args)
def run_ws(self, args):
return arv_ws.main(args)
def main(arguments=None):
args = arvrun_parser.parse_args(arguments)
if len(args.args) == 0:
arvrun_parser.print_help()
return
starting_args = args.args
reading_into = 2
# Parse the command arguments into 'slots'.
# All words following '>' are output arguments and are collected into slots[0].
# All words following '<' are input arguments and are collected into slots[1].
# slots[2..] store the parameters of each command in the pipeline.
#
# e.g. arv-run foo arg1 arg2 '|' bar arg3 arg4 '<' input1 input2 input3 '>' output.txt
# will be parsed into:
# [['output.txt'],
# ['input1', 'input2', 'input3'],
# ['foo', 'arg1', 'arg2'],
# ['bar', 'arg3', 'arg4']]
slots = [[], [], []]
for c in args.args:
if c.startswith('>'):
reading_into = 0
if len(c) > 1:
slots[reading_into].append(c[1:])
elif c.startswith('<'):
reading_into = 1
if len(c) > 1:
slots[reading_into].append(c[1:])
elif c == '|':
reading_into = len(slots)
slots.append([])
else:
slots[reading_into].append(c)
if slots[0] and len(slots[0]) > 1:
logger.error("Can only specify a single stdout file (run-command substitutions are permitted)")
return
if not args.dry_run:
api = arvados.api('v1')
if args.project_uuid:
project = args.project_uuid
else:
project = determine_project(os.getcwd(), api.users().current().execute()["uuid"])
# Identify input files. Look at each parameter and test to see if there is
# a file by that name. This uses 'patterns' to look for within
# command line arguments, such as --foo=file.txt or -lfile.txt
patterns = [re.compile("([^=]+=)(.*)"),
re.compile("(-[A-Za-z])(.+)")]
for j, command in enumerate(slots[1:]):
for i, a in enumerate(command):
if j > 0 and i == 0:
# j == 0 is stdin, j > 0 is commands
# always skip program executable (i == 0) in commands
pass
elif a.startswith('\\'):
# if it starts with a \ then don't do any interpretation
command[i] = a[1:]
else:
# See if it looks like a file
command[i] = statfile('', a)
# If a file named command[i] was found, it would now be an
# ArvFile or UploadFile. If command[i] is a basestring, that
# means it doesn't correspond exactly to a file, so do some
# pattern matching.
if isinstance(command[i], basestring):
for p in patterns:
m = p.match(a)
if m:
command[i] = statfile(m.group(1), m.group(2))
break
n = True
pathprefix = "/"
files = [c for command in slots[1:] for c in command if isinstance(c, UploadFile)]
if len(files) > 0:
# Find the smallest path prefix that includes all the files that need to be uploaded.
# This starts at the root and iteratively removes common parent directory prefixes
# until all file pathes no longer have a common parent.
while n:
pathstep = None
for c in files:
if pathstep is None:
sp = c.fn.split('/')
if len(sp) < 2:
# no parent directories left
n = False
break
# path step takes next directory
pathstep = sp[0] + "/"
else:
# check if pathstep is common prefix for all files
if not c.fn.startswith(pathstep):
n = False
break
if n:
# pathstep is common parent directory for all files, so remove the prefix
# from each path
pathprefix += pathstep
for c in files:
c.fn = c.fn[len(pathstep):]
orgdir = os.getcwd()
os.chdir(pathprefix)
print("Upload local files: \"%s\"" % '" "'.join([c.fn for c in files]))
if args.dry_run:
print("$(input) is %s" % pathprefix.rstrip('/'))
pdh = "$(input)"
else:
files = sorted(files, key=lambda x: x.fn)
collection = arvados.CollectionWriter(api, num_retries=args.retries)
stream = None
for f in files:
sp = os.path.split(f.fn)
if sp[0] != stream:
stream = sp[0]
collection.start_new_stream(stream)
collection.write_file(f.fn, sp[1])
item = api.collections().create(body={"owner_uuid": project, "manifest_text": collection.manifest_text()}).execute()
pdh = item["portable_data_hash"]
print "Uploaded to %s" % item["uuid"]
for c in files:
c.fn = "$(file %s/%s)" % (pdh, c.fn)
os.chdir(orgdir)
for i in xrange(1, len(slots)):
slots[i] = [("%s%s" % (c.prefix, c.fn)) if isinstance(c, ArvFile) else c for c in slots[i]]
component = {
"script": "run-command",
"script_version": args.script_version,
"repository": args.repository,
"script_parameters": {
},
"runtime_constraints": {
"docker_image": args.docker_image
}
}
task_foreach = []
group_parser = argparse.ArgumentParser()
group_parser.add_argument('-b', '--batch-size', type=int)
group_parser.add_argument('args', nargs=argparse.REMAINDER)
for s in xrange(2, len(slots)):
for i in xrange(0, len(slots[s])):
if slots[s][i] == '--':
inp = "input%i" % (s-2)
groupargs = group_parser.parse_args(slots[2][i+1:])
if groupargs.batch_size:
component["script_parameters"][inp] = {"value": {"batch":groupargs.args, "size":groupargs.batch_size}}
slots[s] = slots[s][0:i] + [{"foreach": inp, "command": "$(%s)" % inp}]
else:
component["script_parameters"][inp] = groupargs.args
slots[s] = slots[s][0:i] + ["$(%s)" % inp]
task_foreach.append(inp)
break
if slots[s][i] == '\--':
slots[s][i] = '--'
if slots[0]:
component["script_parameters"]["task.stdout"] = slots[0][0]
if slots[1]:
task_foreach.append("stdin")
component["script_parameters"]["stdin"] = slots[1]
component["script_parameters"]["task.stdin"] = "$(stdin)"
if task_foreach:
component["script_parameters"]["task.foreach"] = task_foreach
component["script_parameters"]["command"] = slots[2:]
if args.ignore_rcode:
component["script_parameters"]["task.ignore_rcode"] = args.ignore_rcode
pipeline = {
"name": "arv-run " + " | ".join([s[0] for s in slots[2:]]),
"description": "@" + " ".join(starting_args) + "@",
"components": {
"command": component
},
"state": "RunningOnClient" if args.local else "RunningOnServer"
}
if args.dry_run:
print(json.dumps(pipeline, indent=4))
else:
pipeline["owner_uuid"] = project
pi = api.pipeline_instances().create(body=pipeline, ensure_unique_name=True).execute()
print "Running pipeline %s" % pi["uuid"]
if args.local:
subprocess.call(["arv-run-pipeline-instance", "--instance", pi["uuid"], "--run-jobs-here"] + (["--no-reuse"] if args.no_reuse else []))
elif not args.no_wait:
ws.main(["--pipeline", pi["uuid"]])
pi = api.pipeline_instances().get(uuid=pi["uuid"]).execute()
print "Pipeline is %s" % pi["state"]
if "output_uuid" in pi["components"]["command"]:
print "Output is %s" % pi["components"]["command"]["output_uuid"]
else:
print "No output"
