def test_increment_slow(depth=5, dur=0.5):
"""Test simple pipeline slow (sleep.5) A->B...->N
"""
# Create the first file
open("test0.txt", 'w').write('0\n')
prev = File("test0.txt")
# Create the first entry in the dictionary holding the futures
futs = {}
print("**************TYpe : ", type(dur), dur)
for i in range(1, depth):
print("Launching {0} with {1}".format(i, prev))
output = File("test{0}.txt".format(i))
fu = slow_increment(
dur,
# Depend on the future from previous call
inputs=[prev],
# Name the file to be created here
outputs=[output],
stdout="incr{0}.out".format(i),
stderr="incr{0}.err".format(i))
[prev] = fu.outputs
futs[i] = prev
print(prev.filepath)
for key in futs:
if key > 0:
fu = futs[key]
data = open(fu.result().filepath, 'r').read().strip()
assert data == str(
key), "[TEST] incr failed for key:{0} got:{1}".format(
key, data)
def test_stage_in_out_globus():
"""Test stage-in then stage-out to/from Globus
Prerequisite:
unsorted.txt must already exist at the specified endpoint
the specified output endpoint must be writeable
"""
unsorted_file = File('globus://03d7d06a-cb6b-11e8-8c6a-0a1d4c5c824a/unsorted.txt')
# Create a local file for output data
sorted_file = File(remote_writeable + "/sorted.txt")
f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file])
# wait for both the app to complete, and the stageout DataFuture to complete.
# It isn't clearly defined whether we need to wait for both, or whether
# waiting for one is sufficient, but at time of writing this test,
# neither is sufficient (!) - see issue #778 - and instead this test will
# sometimes pass even though stageout is not working.
f.result()
result_file = f.outputs[0].result()
assert unsorted_file.local_path is None, "Input file on local side has overridden local_path, file: {}".format(repr(unsorted_file))
# use 'is' rather than '==' because specifically want to check
# object identity rather than value equality
assert sorted_file is result_file, "Result file is not the specified input-output file"
assert result_file.local_path is None, "Result file on local side has overridden local_path, file: {}".format(repr(result_file))
def test_regression_stage_out_does_not_stage_in():
no_stageout_config = Config(
executors=[
ThreadPoolExecutor(
label='local_threads',
storage_access=[NoOpTestingFileStaging(allow_stage_in=False)]
)
]
)
parsl.load(no_stageout_config)
# Test that the helper app runs with no staging
touch("test.1", outputs=[]).result()
# Test with stage-out, checking that provider stage in is never
# invoked. If stage-in is invoked, the the NoOpTestingFileStaging
# provider will raise an exception, which should propagate to
# .result() here.
touch("test.2", outputs=[File("test.2")]).result()
# Test that stage-in exceptions propagate out to user code.
with pytest.raises(NoOpError):
touch("test.3", inputs=[File("test.3")]).result()
parsl.dfk().cleanup()
parsl.clear()
def test_fut_case_4():
"""Testing the behavior of DataFutures where there are dependencies
The first call has a delay of 0.5s, and the second call depends on the first
"""
"""Testing the behavior of DataFutures where there are no dependencies
"""
output_f1 = 'test_fut_case_4_f1.txt'
output_f2 = 'test_fut_case_4_f2.txt'
app_1 = delay_incr(1, delay=0.5, outputs=[File(output_f1)])
app_1.outputs[0]
app_2 = delay_incr(app_1, delay=0.5, outputs=[File(output_f2)])
data_2 = app_2.outputs[0]
status = data_2.done()
result = data_2.result().filepath
print("App_fu : ", app_2)
print("Data_fu : ", data_2)
print("Status : ", status)
print("Result : ", result)
assert os.path.basename(result) == output_f2, \
"DataFuture did not return the filename, got : {0}".format(result)
contents = get_contents(result)
assert contents == '3', 'Output does not match expected "3", got: "{0}"'.format(
result)
return True
def test_regression_stage_in_does_not_stage_out():
no_stageout_config = Config(
executors=[
ThreadPoolExecutor(
label='local_threads',
storage_access=[NoOpTestingFileStaging(allow_stage_out=False)]
)
],
)
parsl.load(no_stageout_config)
f = open("test.4", "a")
f.write("test")
f.close()
# Test that stage in does not invoke stage out. If stage out is
# attempted, then the NoOpTestingFileStaging provider will raise
# an exception which should propagate here.
app_test_in(File("test.4")).result()
# Test that stage out exceptions propagate to user code.
with pytest.raises(NoOpError):
touch("test.5", outputs=[File("test.5")]).result()
parsl.dfk().cleanup()
parsl.clear()
def test_increment(depth=5):
"""Test simple pipeline A->B...->N
"""
# Create the first file
open("test0.txt", 'w').write('0\n')
# Create the first entry in the dictionary holding the futures
prev = File("test0.txt")
futs = {}
for i in range(1, depth):
if os.path.exists('test{0}.txt'.format(i)):
os.remove('test{0}.txt'.format(i))
print("Launching {0} with {1}".format(i, prev))
fu = increment(
inputs=[prev], # Depend on the future from previous call
# Name the file to be created here
outputs=[File("test{0}.txt".format(i))],
stdout="incr{0}.out".format(i),
stderr="incr{0}.err".format(i))
[prev] = fu.outputs
futs[i] = prev
print(prev.filepath)
for key in futs:
if key > 0:
fu = futs[key]
data = open(fu.result().filepath, 'r').read().strip()
assert data == str(
key), "[TEST] incr failed for key:{0} got:{1}".format(
key, data)
def test_implicit_staging_https_args():
unsorted_file = File('https://testbed.petrel.host/test/public/unsorted.txt')
# Create a local file for output data
sorted_file = File('sorted.txt')
f = sort_strings_arg(unsorted_file, outputs=[sorted_file])
f.result()
def perform_ingest(configuration, logdir, rerun):
pipe_scripts_dir = configuration.root_softs + "/ImageProcessingPipelines/workflows/srs/pipe_scripts/"
ingest_file = File("ingest.list")
ingest_filtered_file = File("ingest_filtered.list")
truncatedFileList = File("ingest_filtered_truncated.list")
ingest_fut = create_ingest_file_list(pipe_scripts_dir,
configuration.ingest_source,
outputs=[ingest_file],
stdout=logdir+"/create_ingest_file_list.stdout",
stderr=logdir+"/create_ingest_file_list.stderr",
wrap=configuration.wrap)
# on dev repo, this gives about 45000 files listed in ingest_file
ingest_file_output_file = ingest_fut.outputs[0]
# heather then advises cutting out two specific files - although I only
# see the second at the # moment so I'm only filtering that out...
# see:
# https://github.com/LSSTDESC/DC2-production/issues/359#issuecomment-521330263
# Two centroid files were identified as failing the centroid check and
# will be omitted from processing:
# 00458564 (R32 S21) and 00466748 (R43 S21)
filtered_ingest_list_future = filter_in_place(ingest_file_output_file,
outputs=[ingest_filtered_file],
stdout=logdir+"/filter_in_place.stdout",
stderr=logdir+"/filter_in_place.stderr",
wrap=configuration.wrap)
filtered_ingest_file_output_file = filtered_ingest_list_future.outputs[0]
truncated_ingest_list = truncate_ingest_list(filtered_ingest_file_output_file,
configuration.trim_ingest_list,
outputs=[truncatedFileList],
stdout=logdir+"/truncate_ingest_list.stdout",
stderr=logdir+"/truncate_ingest_list.stderr")
truncatedFileList_output_future = truncated_ingest_list.outputs[0]
# parsl discussion: the UI is awkward that we can make a truncatedFileList
# File but then we need to extract out the datafuture that contains
# "the same" # file to get dependency ordering.
ingest_future = ingest(truncatedFileList_output_future,
configuration.repo_dir,
rerun,
stdout=logdir+"/ingest.stdout",
stderr=logdir+"/ingest.stderr",
wrap=configuration.wrap)
return ingest_future
def test_implicit_staging_https_args():
# unsorted_file = File('https://testbed.petrel.host/test/public/unsorted.txt')
unsorted_file = File('https://gist.githubusercontent.com/yadudoc/7f21dd15e64a421990a46766bfa5359c/'
'raw/7fe04978ea44f807088c349f6ecb0f6ee350ec49/unsorted.txt')
# Create a local file for output data
sorted_file = File('sorted.txt')
f = sort_strings_arg(unsorted_file, outputs=[sorted_file])
f.result()
def test_files():
if os.path.exists('cat_out.txt'):
os.remove('cat_out.txt')
fs = [File('data/' + f) for f in os.listdir('data')]
x = cat(inputs=fs, outputs=[File('cat_out.txt')],
stdout='f_app.out', stderr='f_app.err')
d_x = x.outputs[0]
print(x.result())
print(d_x, type(d_x))
def test_explicit_staging():
unsorted_file = File(
"globus://03d7d06a-cb6b-11e8-8c6a-0a1d4c5c824a/unsorted.txt")
print("File plain ", unsorted_file)
print("Filepath before stage_in ", unsorted_file.filepath)
dfu = unsorted_file.stage_in()
dfu.result()
print("DFU result: ", dfu.result())
print(unsorted_file.filepath)
def test_explicit_staging():
unsorted_file = File(
"globus://037f054a-15cf-11e8-b611-0ac6873fc732/unsorted.txt")
print("File plain ", unsorted_file)
print("Filepath before stage_in ", unsorted_file.filepath)
dfu = unsorted_file.stage_in()
dfu.result()
print("DFU result: ", dfu.result())
print(unsorted_file.filepath)
def test_implicit_staging_https():
"""Test implicit staging for an ftp file
Create a remote input file (https) that points to unsorted.txt.
"""
unsorted_file = File('https://testbed.petrel.host/test/public/unsorted.txt')
# Create a local file for output data
sorted_file = File('sorted.txt')
f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file])
f.result()
def test_implicit_staging_ftp():
"""Test implicit staging for an ftp file
Create a remote input file (ftp) that points to file_test_cpt.txt.
"""
unsorted_file = File('ftp://www.iana.org/pub/mirror/rirstats/arin/ARIN-STATS-FORMAT-CHANGE.txt')
# Create a local file for output data
sorted_file = File('sorted.txt')
f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file])
f.result()
def test_regression_200():
"""Regression test for #200. Pickleablility of Files"""
with open("test.txt", 'w') as f:
f.write("Hello World")
fu = cat(inputs=[File("test.txt")],
outputs=[File("test_output.txt")])
fu.result()
with open(fu.outputs[0].result(), 'r') as f:
data = f.readlines()
assert "Hello World" in data, "Missed data"
def test_implicit_staging_globus():
"""Test implicit staging for an ftp file
Create a remote input file (globus) that points to unsorted.txt.
"""
unsorted_file = File('globus://03d7d06a-cb6b-11e8-8c6a-0a1d4c5c824a/unsorted.txt')
# Create a local file for output data
sorted_file = File('sorted.txt')
f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file])
f.result()
def test_increment(depth=5):
"""Test simple pipeline A->B...->N
"""
cleanup_work(depth)
# Create the first file
open("test0.txt", 'w').write('0\n')
# Create the first entry in the dictionary holding the futures
prev = File("test0.txt")
futs = {}
for i in range(1, depth):
print("Launching {0} with {1}".format(i, prev))
assert (isinstance(prev, DataFuture) or isinstance(prev, File))
output = File("test{0}.txt".format(i))
fu = increment(
inputs=[prev], # Depend on the future from previous call
# Name the file to be created here
outputs=[output],
stdout="incr{0}.out".format(i),
stderr="incr{0}.err".format(i))
[prev] = fu.outputs
futs[i] = prev
print(prev.filepath)
assert (isinstance(prev, DataFuture))
for key in futs:
if key > 0:
fu = futs[key]
file = fu.result()
filename = file.filepath
# this test is a bit close to a test of the specific implementation
# of File
assert not hasattr(
file, 'local_path'
), "File on local side has overridden local_path, file: {}".format(
repr(file))
assert file.filepath == "test{0}.txt".format(
key
), "Submit side filepath has not been preserved over execution"
data = open(filename, 'r').read().strip()
assert data == str(
key
), "[TEST] incr failed for key: {0} got data: {1} from filename {2}".format(
key, data, filename)
cleanup_work(depth)
def test_implicit_staging_https():
"""Test implicit staging for an ftp file
Create a remote input file (globus) that points to unsorted.txt.
"""
unsorted_file = File(
'globus://037f054a-15cf-11e8-b611-0ac6873fc732/unsorted.txt')
# Create a local file for output data
sorted_file = File('sorted.txt')
f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file])
f.result()
def test_5():
"""Testing behavior of outputs """
# Call echo specifying the outputfile
hello = echo("Hello World!", outputs=[File('hello1.txt')])
# the outputs attribute of the AppFuture is a list of DataFutures
print(hello.outputs)
# This step *cat*s hello1.txt to hello2.txt
hello2 = cat(inputs=[hello.outputs[0]], outputs=[File('hello2.txt')])
hello2.result()
with open(hello2.outputs[0].result().filepath, 'r') as f:
print(f.read())
def test_implicit_staging_https():
"""Test implicit staging for an ftp file
Create a remote input file (https) that points to unsorted.txt.
"""
# unsorted_file = File('https://testbed.petrel.host/test/public/unsorted.txt')
unsorted_file = File('https://gist.githubusercontent.com/yadudoc/7f21dd15e64a421990a46766bfa5359c/'
'raw/7fe04978ea44f807088c349f6ecb0f6ee350ec49/unsorted.txt')
# Create a local file for output data
sorted_file = File('sorted.txt')
f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file])
f.result()
def test_stage_in_globus():
"""Test stage-in for a file coming from a remote Globus endpoint
Prerequisite:
unsorted.txt must already exist at the specified endpoint
"""
unsorted_file = File('globus://03d7d06a-cb6b-11e8-8c6a-0a1d4c5c824a/unsorted.txt')
# Create a local file for output data
sorted_file = File('sorted.txt')
f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file])
f.result()
def __init__(self, fut, file_obj, tid=None):
"""Construct the DataFuture object.
If the file_obj is a string convert to a File.
Args:
- fut (AppFuture) : AppFuture that this DataFuture will track
- file_obj (string/File obj) : Something representing file(s)
Kwargs:
- tid (task_id) : Task id that this DataFuture tracks
"""
super().__init__()
self._tid = tid
if isinstance(file_obj, str):
self.file_obj = File(file_obj)
elif isinstance(file_obj, File):
self.file_obj = file_obj
else:
raise ValueError("DataFuture must be initialized with a str or File")
self.parent = fut
if fut is None:
logger.debug("Setting result to filepath since no future was passed")
self.set_result(self.file_obj)
else:
if isinstance(fut, Future):
self.parent.add_done_callback(self.parent_callback)
else:
raise NotFutureError("DataFuture can be created only with a FunctionFuture on None")
logger.debug("Creating DataFuture with parent: %s", self.parent)
logger.debug("Filepath: %s", self.filepath)
def test_files():
fp = os.path.abspath('test_file.py')
strings = [
{
'f': 'file:///test_file.py',
'scheme': 'file',
'path': '/test_file.py'
},
{
'f': './test_file.py',
'scheme': 'file',
'path': './test_file.py'
},
{
'f': fp,
'scheme': 'file',
'path': fp
},
]
for test in strings:
x = File(test['f'])
assert x.scheme == test[
'scheme'], "[TEST] Scheme error. Expected {0} Got {1}".format(
test['scheme'], x.scheme)
assert x.filepath == test[
'path'], "[TEST] Path error. Expected {0} Got {1}".format(
test['path'], x.path)
def test_files():
fp = os.path.abspath('test_file.py')
strings = [
{
'f': 'file://test_file.py',
'protocol': 'file',
'path': 'test_file.py'
},
{
'f': './test_file.py',
'protocol': 'file',
'path': './test_file.py'
},
{
'f': fp,
'protocol': 'file',
'path': fp
},
]
for test in strings:
x = File(test['f'])
assert x.protocol == test[
'protocol'], "[TEST] Protocol error. Expected {0} Got {1}".format(
test['protocol'], x.protocol)
assert x.filepath == test[
'path'], "[TEST] Path error. Expected {0} Got {1}".format(
test['path'], x.filepath)
def test_regression_200():
"""Regression test for #200. Pickleablility of Files"""
if os.path.exists('test_output.txt'):
os.remove('test_output.txt')
with open("test.txt", 'w') as f:
f.write("Hello World")
fu = cat(inputs=[File("test.txt")], outputs=[File("test_output.txt")])
fu.result()
fi = fu.outputs[0].result()
print("type of fi: {}".format(type(fi)))
with open(str(fi), 'r') as f:
data = f.readlines()
assert "Hello World" in data, "Missed data"
def test_open():
with open('test-open.txt', 'w') as tfile:
tfile.write('Hello')
pfile = File('test-open.txt')
with open(str(pfile), 'r') as opfile:
assert (opfile.readlines()[0] == 'Hello')
def find_inputs(self, stage, data_elements):
inputs = []
for inp in stage.input_tags():
item = data_elements[inp]
if isinstance(item,str):
item = File(item)
inputs.append(item)
return inputs
def test_implicit_staging_https_additional_executor():
"""Test implicit staging for an ftp file
Create a remote input file (https) that points to unsorted.txt.
"""
# unsorted_file = File('https://testbed.petrel.host/test/public/unsorted.txt')
unsorted_file = File('https://gist.githubusercontent.com/yadudoc/7f21dd15e64a421990a46766bfa5359c/'
'raw/7fe04978ea44f807088c349f6ecb0f6ee350ec49/unsorted.txt')
# Create a local file for output data
sorted_file = File('sorted.txt')
other_executor = parsl.ThreadPoolExecutor(label='other')
parsl.dfk().add_executors([other_executor])
f = sort_strings_additional_executor(inputs=[unsorted_file], outputs=[sorted_file])
f.result()
def test_parallel_dataflow():
"""Test parallel dataflow from docs on Composing workflows
"""
if os.path.exists('all.txt'):
os.remove('all.txt')
# create 5 files with random numbers
output_files = []
for i in range(5):
if os.path.exists('random-%s.txt' % i):
os.remove('random-%s.txt' % i)
output_files.append(generate(outputs=[File('random-%s.txt' % i)]))
# concatenate the files into a single file
cc = concat(inputs=[i.outputs[0] for i in output_files],
outputs=[File("all.txt")])
# calculate the average of the random numbers
totals = total(inputs=[cc.outputs[0]])
print(totals.result())
def test_slides():
"""Testing code snippet from slides """
if os.path.exists('hello1.txt'):
os.remove('hello1.txt')
hello = echo("Hello World!", outputs=[File('hello1.txt')])
message = cat(inputs=[hello.outputs[0]])
# Waits. This need not be in the slides.
print(hello.result())
print(message.result())
