def test_image_save(self, tmpdir):
folder = tmpdir
img_url = 'https://www.w3schools.com/css/img_fjords.jpg'
pipeline = Pipeline(
tasks.fetch_url(),
tasks.get_filename(),
tasks.save_file(folder=folder.strpath, default_extension='.jpg'),
url=img_url
)
pipeline.run(wait=True)
filename = folder.join('img_fjords.jpg')
assert filename.read() == fetch_file(img_url)
def spawn_pipeline(url):
return Pipeline(
tasks.fetch_url(),
tasks.get_filename(),
tasks.save_file(folder=tmpdir.strpath),
url=url
)
def start_pipeline(input_file, folder):
def file_processing_pipeline(url):
return Pipeline(tasks.fetch_url(),
tasks.get_filename(),
tasks.save_file(folder=folder),
url=url)
Pipeline(tasks.read_file(input_file=input_file),
tasks.foreach(file_processing_pipeline)).run(wait=True)
def test_foreach(self):
mock_func1 = MagicMock()
mock_func2 = MagicMock()
@task()
def test_func(context):
cb = context.get('cb')
cb()
def create_pipeline(item):
return Pipeline(test_func(), cb=item)
Pipeline(tasks.foreach(create_pipeline),
items=[mock_func1, mock_func2]).run(wait=True)
assert mock_func1.called
assert mock_func2.called
def test_multipipeline(self, tmpdir):
@task(depends=['items'])
def foreach(context, func):
items = context.get('items')
pipelines = []
for item in items:
pipeline = func(item)
pipelines.append(pipeline)
pipeline.run()
context.current_pipeline.wait_for(pipelines)
def spawn_pipeline(url):
return Pipeline(
tasks.fetch_url(),
tasks.get_filename(),
tasks.save_file(folder=tmpdir.strpath),
url=url
)
Pipeline(
foreach(spawn_pipeline),
items=[
'http://i.imgur.com/VGHoi.jpg',
'http://i.imgur.com/UIcZyA4.png',
'http://i.imgur.com/kCXV8yz.jpg',
]
).run(wait=True)
f1 = tmpdir.join('VGHoi.jpg')
f2 = tmpdir.join('UIcZyA4.png')
f3 = tmpdir.join('kCXV8yz.jpg')
assert f1.read() == fetch_file('http://i.imgur.com/VGHoi.jpg')
assert f2.read() == fetch_file('http://i.imgur.com/UIcZyA4.png')
assert f3.read() == fetch_file('http://i.imgur.com/kCXV8yz.jpg')
def create(self, pipeline_config: dict) -> Pipeline:
return Pipeline(pipeline_config["name"], pipeline_config["schedule"],
self._steps_factory.create(pipeline_config["steps"]))
def file_processing_pipeline(url):
return Pipeline(tasks.fetch_url(),
tasks.get_filename(),
tasks.save_file(folder=folder),
url=url)
def create_pipeline(item):
return Pipeline(test_func(), cb=item)