def docker_dps():
print("Now, lets download, process and save each one.")
make_content = DockerStart(blobs=1, manifests=1, manifest_lists=1)
download_content = ConcurrentRunner(DownloadContent())
process = ProcessContent()
save_stage = SaveContent()
print_end = PrintStage()
stages = [make_content, download_content, process, save_stage, print_end]
run_pipeline(stages)
print("Well, that may not have been what we intended...")
print("The manifest lists are all DPS, but manifests and blobs aren't.")
print(
"This is because each item only passes through the pipeline once,"
" manifests and blobs were created in the ProcessContent stage, which "
"simulates seeing this content in a manifest list file.")
print()
print()
print(
"To solve this problem, we can just repeat the stages a couple times to"
" handle the nested content as well.")
make_content = DockerStart(blobs=1, manifests=1, manifest_lists=1)
download_content = ConcurrentRunner(DownloadContent())
process = ProcessContent()
save_stage = SaveContent()
print_end = PrintStage()
stages = [
make_content, download_content, process, save_stage, download_content,
process, save_stage, download_content, process, save_stage, print_end
]
run_pipeline(stages)
print("Hooray! That did it.")
def docker_concurrent_wait_groups():
print("So, how can we make our concurrency work better, reduce memory, "
"and not mess up our many-to-many relationships?")
print("Since StageGroups are treated as regular Stages, we can use the "
"ConcurrentRunner with a whole group.")
print("We also know that nested content is created from content higher up "
"in the heirarchy. So, related content can be kept separate, running "
" inside of a stage group, allowing the wait stage to only wait on the "
" content that we know will be related.")
print("We just need to shift our stages around a little.")
make_content = DockerStart(blobs=1, manifests=1, manifest_lists=1)
download_content = ConcurrentRunner(DownloadContent())
process = ProcessContent()
save_stage = SaveContent()
handle_content = StageGroup([download_content, process, save_stage])
wait_stage = WaitUntilComplete()
relate_content = ContentRelations()
content_stream = ConcurrentRunner(
StageGroup([handle_content, handle_content, handle_content, wait_stage, relate_content])
)
print_end = PrintStage()
stages = [make_content, content_stream, print_end]
run_pipeline(stages)
print("We still had to do some waiting, but items were able to pass through in chuncks, "
"rather than all at once.")
def docker_manymany_problem():
print(
"We have another step, and it presents a problem. Each of these content types"
" are related to each other. To make matters more difficult, they have"
" many-to-many relationships. This is a problem for our current setup because"
" each item streams through individually.")
print()
print()
print(
"To demo the problem, I've written a mock many-to-many that is used by the "
"ContentRelations Stage. All it does is let us know if both of the objects we "
"are trying to fake relate have already been fake saved to the db.")
print(
"Sometimes, we will get lucky and objects just happen to pass through in the right "
"order, but most of the many-to-many writes will fail.")
make_content = DockerStart(blobs=1, manifests=1, manifest_lists=1)
download_content = ConcurrentRunner(DownloadContent())
process = ProcessContent()
save_stage = SaveContent()
handle_content = StageGroup([download_content, process, save_stage])
relate_content = ContentRelations()
print_end = PrintStage()
stages = [
make_content, handle_content, handle_content, handle_content,
relate_content, print_end
]
run_pipeline(stages)
def hello_world_demo():
"""
Here, we use create 2 simple stages, and show that they work together.
"""
# pass "hello" and "world" into out_q.
heya = HelloStage()
print_stuff = PrintStage()
stages = [heya, print_stuff]
run_pipeline(stages)
def concurrent_runner_demo():
print(
"Each fake download takes time. We can improve performance by fake downloading"
" more than one blob at a time. To do this, use the ConcurrentRunner Stage to wrap"
" your download stage.")
make_blobs = DockerStart(blobs=3)
download_blobs = ConcurrentRunner(DownloadContent())
print_them = PrintStage()
stages = [make_blobs, download_blobs, print_them]
run_pipeline(stages)
def docker_dl_intro():
print(
"Just like in the runner and concurrent runner demos, we need to download files."
)
make_content = DockerStart(blobs=1, manifests=1, manifest_lists=1)
print_and_pass = PrintAndPassStage()
download_content = DownloadContent()
print_end = PrintStage()
stages = [make_content, print_and_pass, download_content, print_end]
run_pipeline(stages)
def stage_group_demo():
"""
Stage groups are treated as Stages, and creates an inner pipeline of its own.
"""
print("Stage Group demo.")
print(
"Stage groups can help you to abstract groups that are used together.")
heya = HelloStage(sleep=True)
print_stuff = PrintStage()
stages = [heya, print_stuff]
stage_group = StageGroup(stages)
run_pipeline([stage_group, stage_group])
def wait_demo():
"""
"""
print("Wait demo.")
print("Wait stages can make sure that tasks are ready to be executed.")
print("Notice that the order is changed from the stage_group_demo.")
heya = HelloStage(sleep=True)
print_stuff = PrintStage()
wait = WaitUntilComplete()
stages = [heya, wait, print_stuff]
stage_group = StageGroup(stages)
run_pipeline([stage_group, stage_group])
def runner_demo():
"""
So far, each stage we created is only run 1 at a time. asyncio allows us to run all of the
stages at once, but at any time, only 1 instance of each stage is running.
For stages that use a lot of io, it is useful to run many instances of a stage concurrently.
To do this, we use a Runner Stage.
"""
print("Make a few fake blobs, and then fake download them in serial.")
print("To simulate io, I've put a sleep statement in download.")
make_blobs = DockerStart(blobs=3)
download_blobs = DownloadContent()
print_them = PrintStage()
stages = [make_blobs, download_blobs, print_them]
run_pipeline(stages)
def docker_manymany_solution():
print("So, how can we ensure that all the models are saved before we try to write "
"a many-to-many relationship?")
print("Let's start with the easy way, but it's not ideal.")
print("Just add a new custom Stage, WaitUntilComplete.")
print("Now, all the writes should work.")
make_content = DockerStart(blobs=1, manifests=1, manifest_lists=1)
download_content = ConcurrentRunner(DownloadContent())
process = ProcessContent()
save_stage = SaveContent()
handle_content = StageGroup([download_content, process, save_stage])
wait_stage = WaitUntilComplete()
relate_content = ContentRelations()
print_end = PrintStage()
stages = [make_content, handle_content, handle_content, handle_content,
wait_stage, relate_content, print_end]
run_pipeline(stages)
print("... Unfortunately, every single content stayed in memory until the whole task "
"completed. This pattern limits works against good asynchronous design.")
def docker_grouped_dps():
print(
"Can that be simpler? Yup, let's group the repeated stage into a group."
)
print(
"The results should be exactly the same, though possibly in a different order."
)
make_content = DockerStart(blobs=1, manifests=1, manifest_lists=1)
download_content = ConcurrentRunner(DownloadContent())
process = ProcessContent()
save_stage = SaveContent()
handle_content = StageGroup([download_content, process, save_stage])
print_end = PrintStage()
stages = [
make_content, handle_content, handle_content, handle_content, print_end
]
run_pipeline(stages)
print(
"If you thought 'Why not just loop?', I had the same thought. See `apx_1_loops.py`."
)
