def write_special_types(wf_params, a, b, c, d, e):
blob = Types.Blob()
with blob as w:
w.write("hello I'm a blob".encode('utf-8'))
csv = Types.CSV()
with csv as w:
w.write("hello,i,iz,blob")
mpcsv = Types.MultiPartCSV()
with mpcsv.create_part('000000') as w:
w.write("hello,i,iz,blob")
with mpcsv.create_part('000001') as w:
w.write("hello,i,iz,blob2")
mpblob = Types.MultiPartBlob()
with mpblob.create_part('000000') as w:
w.write("hello I'm a mp blob".encode('utf-8'))
with mpblob.create_part('000001') as w:
w.write("hello I'm a mp blob too".encode('utf-8'))
schema = Types.Schema([('a', Types.Integer), ('b', Types.Integer)])()
with schema as w:
w.write(_pd.DataFrame.from_dict({'a': [1, 2, 3], 'b': [4, 5, 6]}))
w.write(_pd.DataFrame.from_dict({'a': [3, 2, 1], 'b': [6, 5, 4]}))
a.set(blob)
b.set(csv)
c.set(mpcsv)
d.set(mpblob)
e.set(schema)
def collect_blobs(folder_path):
onlyfiles = [
join(folder_path, f) for f in sorted(listdir(folder_path))
if isfile(join(folder_path, f))
]
my_blobs = []
file_names = []
for local_filepath in onlyfiles:
my_blob = Types.Blob()
with my_blob as fileobj:
with open(local_filepath,
mode="rb") as file: # b is important -> binary
fileobj.write(file.read())
my_blobs.append(my_blob)
file_names.append(basename(local_filepath))
return my_blobs, file_names
def test_blob_passing():
@inputs(a=Types.Blob)
@outputs(b=Types.Blob)
@python_task
def test_pass(wf_params, a, b):
b.set(a)
b = Types.Blob()
with b as w:
w.write("Hello world".encode("utf-8"))
out = test_pass.unit_test(a=b)
assert len(out) == 1
with out["b"] as r:
assert r.read().decode("utf-8") == "Hello world"
out = test_pass.unit_test(a=out["b"])
assert len(out) == 1
with out["b"] as r:
assert r.read().decode("utf-8") == "Hello world"
def test_write(wf_params, a):
b = Types.Blob()
with b as w:
w.write("Hello world".encode("utf-8"))
a.set(b)
