def test_increment_slow(depth=5, dur=0.5): """Test simple pipeline slow (sleep.5) A->B...->N """ # Create the first file open("test0.txt", 'w').write('0\n') prev = File("test0.txt") # Create the first entry in the dictionary holding the futures futs = {} print("**************TYpe : ", type(dur), dur) for i in range(1, depth): print("Launching {0} with {1}".format(i, prev)) output = File("test{0}.txt".format(i)) fu = slow_increment( dur, # Depend on the future from previous call inputs=[prev], # Name the file to be created here outputs=[output], stdout="incr{0}.out".format(i), stderr="incr{0}.err".format(i)) [prev] = fu.outputs futs[i] = prev print(prev.filepath) for key in futs: if key > 0: fu = futs[key] data = open(fu.result().filepath, 'r').read().strip() assert data == str( key), "[TEST] incr failed for key:{0} got:{1}".format( key, data)
def test_stage_in_out_globus(): """Test stage-in then stage-out to/from Globus Prerequisite: unsorted.txt must already exist at the specified endpoint the specified output endpoint must be writeable """ unsorted_file = File('globus://03d7d06a-cb6b-11e8-8c6a-0a1d4c5c824a/unsorted.txt') # Create a local file for output data sorted_file = File(remote_writeable + "/sorted.txt") f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file]) # wait for both the app to complete, and the stageout DataFuture to complete. # It isn't clearly defined whether we need to wait for both, or whether # waiting for one is sufficient, but at time of writing this test, # neither is sufficient (!) - see issue #778 - and instead this test will # sometimes pass even though stageout is not working. f.result() result_file = f.outputs[0].result() assert unsorted_file.local_path is None, "Input file on local side has overridden local_path, file: {}".format(repr(unsorted_file)) # use 'is' rather than '==' because specifically want to check # object identity rather than value equality assert sorted_file is result_file, "Result file is not the specified input-output file" assert result_file.local_path is None, "Result file on local side has overridden local_path, file: {}".format(repr(result_file))
def test_regression_stage_out_does_not_stage_in(): no_stageout_config = Config( executors=[ ThreadPoolExecutor( label='local_threads', storage_access=[NoOpTestingFileStaging(allow_stage_in=False)] ) ] ) parsl.load(no_stageout_config) # Test that the helper app runs with no staging touch("test.1", outputs=[]).result() # Test with stage-out, checking that provider stage in is never # invoked. If stage-in is invoked, the the NoOpTestingFileStaging # provider will raise an exception, which should propagate to # .result() here. touch("test.2", outputs=[File("test.2")]).result() # Test that stage-in exceptions propagate out to user code. with pytest.raises(NoOpError): touch("test.3", inputs=[File("test.3")]).result() parsl.dfk().cleanup() parsl.clear()
def test_fut_case_4(): """Testing the behavior of DataFutures where there are dependencies The first call has a delay of 0.5s, and the second call depends on the first """ """Testing the behavior of DataFutures where there are no dependencies """ output_f1 = 'test_fut_case_4_f1.txt' output_f2 = 'test_fut_case_4_f2.txt' app_1 = delay_incr(1, delay=0.5, outputs=[File(output_f1)]) app_1.outputs[0] app_2 = delay_incr(app_1, delay=0.5, outputs=[File(output_f2)]) data_2 = app_2.outputs[0] status = data_2.done() result = data_2.result().filepath print("App_fu : ", app_2) print("Data_fu : ", data_2) print("Status : ", status) print("Result : ", result) assert os.path.basename(result) == output_f2, \ "DataFuture did not return the filename, got : {0}".format(result) contents = get_contents(result) assert contents == '3', 'Output does not match expected "3", got: "{0}"'.format( result) return True
def test_regression_stage_in_does_not_stage_out(): no_stageout_config = Config( executors=[ ThreadPoolExecutor( label='local_threads', storage_access=[NoOpTestingFileStaging(allow_stage_out=False)] ) ], ) parsl.load(no_stageout_config) f = open("test.4", "a") f.write("test") f.close() # Test that stage in does not invoke stage out. If stage out is # attempted, then the NoOpTestingFileStaging provider will raise # an exception which should propagate here. app_test_in(File("test.4")).result() # Test that stage out exceptions propagate to user code. with pytest.raises(NoOpError): touch("test.5", outputs=[File("test.5")]).result() parsl.dfk().cleanup() parsl.clear()
def test_increment(depth=5): """Test simple pipeline A->B...->N """ # Create the first file open("test0.txt", 'w').write('0\n') # Create the first entry in the dictionary holding the futures prev = File("test0.txt") futs = {} for i in range(1, depth): if os.path.exists('test{0}.txt'.format(i)): os.remove('test{0}.txt'.format(i)) print("Launching {0} with {1}".format(i, prev)) fu = increment( inputs=[prev], # Depend on the future from previous call # Name the file to be created here outputs=[File("test{0}.txt".format(i))], stdout="incr{0}.out".format(i), stderr="incr{0}.err".format(i)) [prev] = fu.outputs futs[i] = prev print(prev.filepath) for key in futs: if key > 0: fu = futs[key] data = open(fu.result().filepath, 'r').read().strip() assert data == str( key), "[TEST] incr failed for key:{0} got:{1}".format( key, data)
def test_implicit_staging_https_args(): unsorted_file = File('https://testbed.petrel.host/test/public/unsorted.txt') # Create a local file for output data sorted_file = File('sorted.txt') f = sort_strings_arg(unsorted_file, outputs=[sorted_file]) f.result()
def perform_ingest(configuration, logdir, rerun): pipe_scripts_dir = configuration.root_softs + "/ImageProcessingPipelines/workflows/srs/pipe_scripts/" ingest_file = File("ingest.list") ingest_filtered_file = File("ingest_filtered.list") truncatedFileList = File("ingest_filtered_truncated.list") ingest_fut = create_ingest_file_list(pipe_scripts_dir, configuration.ingest_source, outputs=[ingest_file], stdout=logdir+"/create_ingest_file_list.stdout", stderr=logdir+"/create_ingest_file_list.stderr", wrap=configuration.wrap) # on dev repo, this gives about 45000 files listed in ingest_file ingest_file_output_file = ingest_fut.outputs[0] # heather then advises cutting out two specific files - although I only # see the second at the # moment so I'm only filtering that out... # see: # https://github.com/LSSTDESC/DC2-production/issues/359#issuecomment-521330263 # Two centroid files were identified as failing the centroid check and # will be omitted from processing: # 00458564 (R32 S21) and 00466748 (R43 S21) filtered_ingest_list_future = filter_in_place(ingest_file_output_file, outputs=[ingest_filtered_file], stdout=logdir+"/filter_in_place.stdout", stderr=logdir+"/filter_in_place.stderr", wrap=configuration.wrap) filtered_ingest_file_output_file = filtered_ingest_list_future.outputs[0] truncated_ingest_list = truncate_ingest_list(filtered_ingest_file_output_file, configuration.trim_ingest_list, outputs=[truncatedFileList], stdout=logdir+"/truncate_ingest_list.stdout", stderr=logdir+"/truncate_ingest_list.stderr") truncatedFileList_output_future = truncated_ingest_list.outputs[0] # parsl discussion: the UI is awkward that we can make a truncatedFileList # File but then we need to extract out the datafuture that contains # "the same" # file to get dependency ordering. ingest_future = ingest(truncatedFileList_output_future, configuration.repo_dir, rerun, stdout=logdir+"/ingest.stdout", stderr=logdir+"/ingest.stderr", wrap=configuration.wrap) return ingest_future
def test_implicit_staging_https_args(): # unsorted_file = File('https://testbed.petrel.host/test/public/unsorted.txt') unsorted_file = File('https://gist.githubusercontent.com/yadudoc/7f21dd15e64a421990a46766bfa5359c/' 'raw/7fe04978ea44f807088c349f6ecb0f6ee350ec49/unsorted.txt') # Create a local file for output data sorted_file = File('sorted.txt') f = sort_strings_arg(unsorted_file, outputs=[sorted_file]) f.result()
def test_files(): if os.path.exists('cat_out.txt'): os.remove('cat_out.txt') fs = [File('data/' + f) for f in os.listdir('data')] x = cat(inputs=fs, outputs=[File('cat_out.txt')], stdout='f_app.out', stderr='f_app.err') d_x = x.outputs[0] print(x.result()) print(d_x, type(d_x))
def test_explicit_staging(): unsorted_file = File( "globus://03d7d06a-cb6b-11e8-8c6a-0a1d4c5c824a/unsorted.txt") print("File plain ", unsorted_file) print("Filepath before stage_in ", unsorted_file.filepath) dfu = unsorted_file.stage_in() dfu.result() print("DFU result: ", dfu.result()) print(unsorted_file.filepath)
def test_explicit_staging(): unsorted_file = File( "globus://037f054a-15cf-11e8-b611-0ac6873fc732/unsorted.txt") print("File plain ", unsorted_file) print("Filepath before stage_in ", unsorted_file.filepath) dfu = unsorted_file.stage_in() dfu.result() print("DFU result: ", dfu.result()) print(unsorted_file.filepath)
def test_implicit_staging_https(): """Test implicit staging for an ftp file Create a remote input file (https) that points to unsorted.txt. """ unsorted_file = File('https://testbed.petrel.host/test/public/unsorted.txt') # Create a local file for output data sorted_file = File('sorted.txt') f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file]) f.result()
def test_implicit_staging_ftp(): """Test implicit staging for an ftp file Create a remote input file (ftp) that points to file_test_cpt.txt. """ unsorted_file = File('ftp://www.iana.org/pub/mirror/rirstats/arin/ARIN-STATS-FORMAT-CHANGE.txt') # Create a local file for output data sorted_file = File('sorted.txt') f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file]) f.result()
def test_regression_200(): """Regression test for #200. Pickleablility of Files""" with open("test.txt", 'w') as f: f.write("Hello World") fu = cat(inputs=[File("test.txt")], outputs=[File("test_output.txt")]) fu.result() with open(fu.outputs[0].result(), 'r') as f: data = f.readlines() assert "Hello World" in data, "Missed data"
def test_implicit_staging_globus(): """Test implicit staging for an ftp file Create a remote input file (globus) that points to unsorted.txt. """ unsorted_file = File('globus://03d7d06a-cb6b-11e8-8c6a-0a1d4c5c824a/unsorted.txt') # Create a local file for output data sorted_file = File('sorted.txt') f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file]) f.result()
def test_increment(depth=5): """Test simple pipeline A->B...->N """ cleanup_work(depth) # Create the first file open("test0.txt", 'w').write('0\n') # Create the first entry in the dictionary holding the futures prev = File("test0.txt") futs = {} for i in range(1, depth): print("Launching {0} with {1}".format(i, prev)) assert (isinstance(prev, DataFuture) or isinstance(prev, File)) output = File("test{0}.txt".format(i)) fu = increment( inputs=[prev], # Depend on the future from previous call # Name the file to be created here outputs=[output], stdout="incr{0}.out".format(i), stderr="incr{0}.err".format(i)) [prev] = fu.outputs futs[i] = prev print(prev.filepath) assert (isinstance(prev, DataFuture)) for key in futs: if key > 0: fu = futs[key] file = fu.result() filename = file.filepath # this test is a bit close to a test of the specific implementation # of File assert not hasattr( file, 'local_path' ), "File on local side has overridden local_path, file: {}".format( repr(file)) assert file.filepath == "test{0}.txt".format( key ), "Submit side filepath has not been preserved over execution" data = open(filename, 'r').read().strip() assert data == str( key ), "[TEST] incr failed for key: {0} got data: {1} from filename {2}".format( key, data, filename) cleanup_work(depth)
def test_implicit_staging_https(): """Test implicit staging for an ftp file Create a remote input file (globus) that points to unsorted.txt. """ unsorted_file = File( 'globus://037f054a-15cf-11e8-b611-0ac6873fc732/unsorted.txt') # Create a local file for output data sorted_file = File('sorted.txt') f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file]) f.result()
def test_5(): """Testing behavior of outputs """ # Call echo specifying the outputfile hello = echo("Hello World!", outputs=[File('hello1.txt')]) # the outputs attribute of the AppFuture is a list of DataFutures print(hello.outputs) # This step *cat*s hello1.txt to hello2.txt hello2 = cat(inputs=[hello.outputs[0]], outputs=[File('hello2.txt')]) hello2.result() with open(hello2.outputs[0].result().filepath, 'r') as f: print(f.read())
def test_implicit_staging_https(): """Test implicit staging for an ftp file Create a remote input file (https) that points to unsorted.txt. """ # unsorted_file = File('https://testbed.petrel.host/test/public/unsorted.txt') unsorted_file = File('https://gist.githubusercontent.com/yadudoc/7f21dd15e64a421990a46766bfa5359c/' 'raw/7fe04978ea44f807088c349f6ecb0f6ee350ec49/unsorted.txt') # Create a local file for output data sorted_file = File('sorted.txt') f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file]) f.result()
def test_stage_in_globus(): """Test stage-in for a file coming from a remote Globus endpoint Prerequisite: unsorted.txt must already exist at the specified endpoint """ unsorted_file = File('globus://03d7d06a-cb6b-11e8-8c6a-0a1d4c5c824a/unsorted.txt') # Create a local file for output data sorted_file = File('sorted.txt') f = sort_strings(inputs=[unsorted_file], outputs=[sorted_file]) f.result()
def __init__(self, fut, file_obj, tid=None): """Construct the DataFuture object. If the file_obj is a string convert to a File. Args: - fut (AppFuture) : AppFuture that this DataFuture will track - file_obj (string/File obj) : Something representing file(s) Kwargs: - tid (task_id) : Task id that this DataFuture tracks """ super().__init__() self._tid = tid if isinstance(file_obj, str): self.file_obj = File(file_obj) elif isinstance(file_obj, File): self.file_obj = file_obj else: raise ValueError("DataFuture must be initialized with a str or File") self.parent = fut if fut is None: logger.debug("Setting result to filepath since no future was passed") self.set_result(self.file_obj) else: if isinstance(fut, Future): self.parent.add_done_callback(self.parent_callback) else: raise NotFutureError("DataFuture can be created only with a FunctionFuture on None") logger.debug("Creating DataFuture with parent: %s", self.parent) logger.debug("Filepath: %s", self.filepath)
def test_files(): fp = os.path.abspath('test_file.py') strings = [ { 'f': 'file:///test_file.py', 'scheme': 'file', 'path': '/test_file.py' }, { 'f': './test_file.py', 'scheme': 'file', 'path': './test_file.py' }, { 'f': fp, 'scheme': 'file', 'path': fp }, ] for test in strings: x = File(test['f']) assert x.scheme == test[ 'scheme'], "[TEST] Scheme error. Expected {0} Got {1}".format( test['scheme'], x.scheme) assert x.filepath == test[ 'path'], "[TEST] Path error. Expected {0} Got {1}".format( test['path'], x.path)
def test_files(): fp = os.path.abspath('test_file.py') strings = [ { 'f': 'file://test_file.py', 'protocol': 'file', 'path': 'test_file.py' }, { 'f': './test_file.py', 'protocol': 'file', 'path': './test_file.py' }, { 'f': fp, 'protocol': 'file', 'path': fp }, ] for test in strings: x = File(test['f']) assert x.protocol == test[ 'protocol'], "[TEST] Protocol error. Expected {0} Got {1}".format( test['protocol'], x.protocol) assert x.filepath == test[ 'path'], "[TEST] Path error. Expected {0} Got {1}".format( test['path'], x.filepath)
def test_regression_200(): """Regression test for #200. Pickleablility of Files""" if os.path.exists('test_output.txt'): os.remove('test_output.txt') with open("test.txt", 'w') as f: f.write("Hello World") fu = cat(inputs=[File("test.txt")], outputs=[File("test_output.txt")]) fu.result() fi = fu.outputs[0].result() print("type of fi: {}".format(type(fi))) with open(str(fi), 'r') as f: data = f.readlines() assert "Hello World" in data, "Missed data"
def test_open(): with open('test-open.txt', 'w') as tfile: tfile.write('Hello') pfile = File('test-open.txt') with open(str(pfile), 'r') as opfile: assert (opfile.readlines()[0] == 'Hello')
def find_inputs(self, stage, data_elements): inputs = [] for inp in stage.input_tags(): item = data_elements[inp] if isinstance(item,str): item = File(item) inputs.append(item) return inputs
def test_implicit_staging_https_additional_executor(): """Test implicit staging for an ftp file Create a remote input file (https) that points to unsorted.txt. """ # unsorted_file = File('https://testbed.petrel.host/test/public/unsorted.txt') unsorted_file = File('https://gist.githubusercontent.com/yadudoc/7f21dd15e64a421990a46766bfa5359c/' 'raw/7fe04978ea44f807088c349f6ecb0f6ee350ec49/unsorted.txt') # Create a local file for output data sorted_file = File('sorted.txt') other_executor = parsl.ThreadPoolExecutor(label='other') parsl.dfk().add_executors([other_executor]) f = sort_strings_additional_executor(inputs=[unsorted_file], outputs=[sorted_file]) f.result()
def test_parallel_dataflow(): """Test parallel dataflow from docs on Composing workflows """ if os.path.exists('all.txt'): os.remove('all.txt') # create 5 files with random numbers output_files = [] for i in range(5): if os.path.exists('random-%s.txt' % i): os.remove('random-%s.txt' % i) output_files.append(generate(outputs=[File('random-%s.txt' % i)])) # concatenate the files into a single file cc = concat(inputs=[i.outputs[0] for i in output_files], outputs=[File("all.txt")]) # calculate the average of the random numbers totals = total(inputs=[cc.outputs[0]]) print(totals.result())
def test_slides(): """Testing code snippet from slides """ if os.path.exists('hello1.txt'): os.remove('hello1.txt') hello = echo("Hello World!", outputs=[File('hello1.txt')]) message = cat(inputs=[hello.outputs[0]]) # Waits. This need not be in the slides. print(hello.result()) print(message.result())