def main():
cmd = "{0} 'ls {1}'".format(ssh, dir_)
p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
out, _ = p.communicate()
out = out.decode('utf-8').strip().split(linesep)
fullpaths = [op.join(dir_, p) for p in out]
print(fullpaths)
# Start radical entk pipeline
p = Pipeline()
for i in range(iterations):
s = Stage()
for fp in fullpaths:
t = Task()
t.name = 'Incrementation {}'.format(i)
t.pre_exec = [
'source /home/vhayot/miniconda3/etc/profile.d/conda.sh',
'conda activate radenv'
]
t.executable = 'python /home/vhayot/inc.py'
if i == 0:
t.arguments = [fp, out_dir, i]
else:
# Note: assuming all data is accessible through shared dir
# radical entk functions without sharedfs, however
t.arguments = [
op.join(out_dir,
"it-{0}-{1}".format(i - 1, op.basename(fp))),
out_dir, i
]
s.add_tasks(t)
# Create a new stage everytime there's a dependency
p.add_stages(s)
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = {
'resource': 'xsede.bridges',
'walltime': 20,
'cpus': 5,
'project': 'mc3bggp',
'schema': 'gsissh'
}
appman.workflow = set([p])
appman.run()
