def test():
# create an remote Parsl file
inp = File('ftp://www.iana.org/pub/mirror/rirstats/arin/ARIN-STATS-FORMAT-CHANGE.txt')
# create a local Parsl file
out = File('file:///tmp/ARIN-STATS-FORMAT-CHANGE.txt')
# call the convert app with the Parsl file
f = convert(inputs=[inp], outputs=[out])
f.result()
def test_parallel_for(n=3):
"""Testing a simple parallel for loop
"""
outdir = os.path.abspath('outputs')
if not os.path.exists(outdir):
os.makedirs(outdir)
else:
shutil.rmtree(outdir)
os.makedirs(outdir)
d = {}
start = time.time()
for i in range(0, n):
d[i] = echo_to_file(
inputs=['Hello World {0}'.format(i)],
outputs=[File('{0}/out.{1}.txt'.format(outdir, i))],
stdout='{0}/std.{1}.out'.format(outdir, i),
stderr='{0}/std.{1}.err'.format(outdir, i),
)
assert len(d.keys()) == n, "Only {0}/{1} keys in dict".format(
len(d.keys()), n)
[d[i].result() for i in d]
print("Duration : {0}s".format(time.time() - start))
stdout_file_count = len(
[item for item in os.listdir(outdir) if item.endswith('.out')])
assert stdout_file_count == n, "Only {0}/{1} files in '{2}' ".format(
len(os.listdir('outputs/')), n, outdir)
print("[TEST STATUS] test_parallel_for [SUCCESS]")
return d
def test():
# create a test file
open('/tmp/test.txt', 'w').write('Hello\n')
# create the Parsl file
parsl_file = File('file:///tmp/test.txt')
# call the cat app with the Parsl file
cat(inputs=[parsl_file])
def test_bash_memoization_keywords(n=2):
"""Testing bash memoization
"""
temp_filename = "test.memoization.tmp"
temp_file = File("test.memoization.tmp")
if os.path.exists(temp_filename):
os.remove(temp_filename)
temp_file = File(temp_filename)
print("Launching: ", n)
x = fail_on_presence_kw(outputs=[temp_file], foo={"a": 1, "b": 2})
x.result()
d = {}
for i in range(0, n):
d[i] = fail_on_presence_kw(outputs=[temp_file], foo={"b": 2, "a": 1})
for i in d:
assert d[i].exception() is None
def test_bash_memoization(n=2):
"""Testing bash memoization
"""
temp_filename = "test.memoization.tmp"
temp_file = File(temp_filename)
if os.path.exists(temp_filename):
os.remove(temp_filename)
temp_file = File(temp_filename)
print("Launching: ", n)
x = fail_on_presence(outputs=[temp_file])
x.result()
d = {}
for i in range(0, n):
d[i] = fail_on_presence(outputs=[temp_file])
for i in d:
assert d[i].exception() is None
def test_launch_apps(n=2, outdir='outputs'):
if not os.path.exists(outdir):
os.makedirs(outdir)
else:
shutil.rmtree(outdir)
os.makedirs(outdir)
print('outdir is ', outdir)
all_futs = []
for i in range(n):
fus = double(i, outputs=[File('{0}/{1}.txt'.format(outdir, i))])
all_futs.append(fus)
wait(all_futs)
stdout_file_count = len(
[item for item in os.listdir(outdir) if item.endswith('.txt')])
assert stdout_file_count == n, "Only {}/{} files in '{}' ".format(
len(os.listdir('outputs/')), n, os.listdir(outdir))
import parsl
from parsl import python_app, bash_app, File
#from parsl.data_provider.files import File
from config import config
@bash_app
def compress(inputs=[], decompress=""):
return f"pigz {decompress} {inputs[0]}"
@bash_app
def write_hello(inputs=[], outputs=[], stdout="~/test.out"):
print(outputs)
return f"cat {inputs[0]} > {outputs[0]}"
# @bash_app
# def echo_hello(msg, stdout="hello.out", stderr="hello.err"):
# return f"echo {msg}"
if __name__ == "__main__":
parsl.set_stream_logger()
parsl.load(config)
print(os.getcwd())
out = [File(os.path.join(os.getcwd(), "test.txt"))]
in_ = [File(os.path.join(os.getcwd(), "hw.txt"))]
wh = write_hello(inputs=in_, outputs=out)
with open(wh.outputs[0].result(), 'r') as f:
print(f.read())
compress(inputs=wh.outputs)