def test_beyond_filename(self):
base = "/path/with.dot/filename.ext"
sec = "^^.txt"
self.assertEqual(
"/path/with.dot/filename.txt",
apply_secondary_file_format_to_filename(base, sec),
)
def test_none(self):
base = None
sec = ".tsv"
self.assertIsNone(apply_secondary_file_format_to_filename(base, sec))
def test_way_too_many_extensions(self):
base = "filename.ext"
sec = "^^^^^^^^.tsv"
self.assertEqual("filename.tsv",
apply_secondary_file_format_to_filename(base, sec))
def test_one_extension(self):
base = "filename.ext"
sec = "^.tsv"
self.assertEqual("filename.tsv",
apply_secondary_file_format_to_filename(base, sec))
def test_append(self):
base = "filename.ext"
sec = ".tsv"
self.assertEqual("filename.ext.tsv",
apply_secondary_file_format_to_filename(base, sec))
def copy_output(
self,
fs: FileScheme,
outputid,
prefix,
tag,
secondaries,
extension,
iscopyable,
engine_output: Union[WorkflowOutputModel, Any, List[Any]],
shard=None,
):
# the output_folder is an array of an array, for each
if isinstance(engine_output, list):
outs = []
nshards = len(engine_output)
prev_shards = shard or []
# This is a little complicated, we want to output the set of tags that have the same length as we
# we have shards. We'll only let this work if there's one element in the array with the appropriate amount
# of shards.
# find the index
def find_element_where_length_is(iterable, n):
if iterable is None:
return None
for i in range(len(iterable)):
ii = iterable[i]
if isinstance(ii, list) and len(ii) == n:
return i
return None
def explode_at_index(iterable, index_to_explode, index_to_select):
ar = iterable[:index_to_explode] + [
iterable[index_to_explode][index_to_select]
]
if index_to_explode + 1 < len(iterable):
ar.extend(iterable[1 + tag_index_to_explode :])
return ar
tag_index_to_explode = find_element_where_length_is(tag, nshards)
for i in range(nshards):
eout = engine_output[i]
new_shard = [*prev_shards, i]
# choose tag
new_prefix = prefix
if isinstance(new_prefix, list) and len(new_prefix) > 1:
new_prefix = new_prefix[i]
new_shard = new_shard[min(len(new_shard), 1) :]
new_tag = tag
if tag_index_to_explode is not None:
new_tag = explode_at_index(tag, tag_index_to_explode, i)
new_shard = new_shard[min(len(new_shard), 1) :]
outs.append(
self.copy_output(
fs,
outputid=outputid,
tag=new_tag,
prefix=new_prefix,
engine_output=eout,
shard=new_shard,
secondaries=secondaries,
extension=extension,
iscopyable=iscopyable,
)
)
return [o[0] for o in outs], [o[1] for o in outs]
final_tags = tag
outfn = outputid
if final_tags and any(isinstance(t, list) for t in final_tags):
Logger.critical(
f"One of the final output tags {str(final_tags)} was still an array, outputs will be written directly into the output directory"
)
final_tags = None
if prefix:
if isinstance(prefix, list):
if len(prefix) > 1:
Logger.critical(
f"Expected only one output_name for this copy, but found ({', '.join(prefix)}) [{len(prefix)}], using the first outputname"
)
else:
outfn = prefix[0]
else:
outfn = prefix
if final_tags is None:
final_tags = []
outdir = os.path.join(self.path, "/".join(final_tags))
fs.mkdirs(outdir)
if shard is not None:
for s in shard:
outfn += f"_shard-{s}"
# copy output
original_filepath = None
newoutputfilepath = os.path.join(outdir, outfn)
if isinstance(engine_output, WorkflowOutputModel):
original_filepath = engine_output.originalpath
if original_filepath and iscopyable:
ext = extension or get_extension(engine_output.originalpath)
if ext:
dot = "" if ext[0] == "." else "."
outfn += dot + ext
newoutputfilepath += dot + ext
fs.cp_from(engine_output.originalpath, newoutputfilepath, force=True)
elif engine_output.value:
if isinstance(fs, LocalFileScheme):
# Write engine_output to outpath
with open(newoutputfilepath, "w+") as outfile:
outfile.write(str(engine_output.value))
else:
original_filepath = engine_output
if isinstance(fs, LocalFileScheme):
# Write engine_output to outpath
with open(newoutputfilepath, "w+") as outfile:
outfile.write(str(engine_output))
for sec in secondaries or []:
frompath = apply_secondary_file_format_to_filename(original_filepath, sec)
tofn = apply_secondary_file_format_to_filename(outfn, sec)
topath = os.path.join(outdir, tofn)
fs.cp_from(frompath, topath, force=True)
return [original_filepath, newoutputfilepath]