def run_prefetched_series(argv):
"""
Args:
argv:
:return:
"""
patient_series = SeriesFilter.batch_series_studies_by_patient()
batch_size = 15
i = 0
lines = []
for p in patient_series:
if (len(lines) + len(p)) > batch_size:
run_pipeline(
argv, construct_sequence_pipeline(argv,
{"SPECIFIC_GCS": lines}))
client = storage.Client()
bucket = client.get_bucket("ispy_dataquery")
for b in os.listdir("output/"):
name = f"output/small_{b}_{i}"
blob = bucket.blob(name)
blob.upload_from_filename(filename=f"output/{b}")
os.remove(f"output/{b}")
i += 1
lines = []
else:
# Else merely add to series
lines.extend(p)
print(lines)
def run_pipeline_with_gcs(argv, lines, i):
run_pipeline(argv,
construct_sequence_pipeline(argv, {"SPECIFIC_GCS": lines}))
client = storage.Client()
bucket = client.get_bucket("ispy_dataquery")
for b in os.listdir("data/"):
os.rename(f"data/{b}", f"data/{i}_{b}")
name = f"output/{i}_{b}"
blob = bucket.blob(name)
blob.upload_from_filename(filename=f"output/{b}")
os.remove(f"output/{b}")
def main(argv: List[str]) -> int:
"""Main program run for data processing."""
return run_pipeline(argv, construct_main_pipeline)
def run_pipeline_with_locally(argv, lines, i):
run_pipeline(argv,
construct_sequence_pipeline(argv, {"SPECIFIC_GCS": lines}))
for b in os.listdir("data/"):
os.rename(f"data/{b}", f"data/{i}_{b}")
import argparse
import sys
import apache_beam as beam
import pipeline.constants as constants
from pipeline.studies_pipeline import construct
import util
def construct_studies_test_pipeline(parsed_args: argparse.Namespace,
p: beam.Pipeline):
""" Runs a manual test of the Series Pipeline.
"""
args = vars(parsed_args)
series = util.get_series_pipeline(args[constants.STUDIES_PATH])(p, args)
studies = construct(series)
_ = studies | "Print Results" >> beam.Map(
lambda x: print(f"Element: {str(x)}"))
if __name__ == "__main__":
if "--test" in sys.argv:
util.run_pipeline(sys.argv, construct_studies_test_pipeline)
else:
print(
"Currently, can only run Series pipeline as test using `--test`.")
import argparse
import sys
import apache_beam as beam
from pipeline.patient_pipeline import construct
from util import run_pipeline
def construct_patient_test_pipeline(parsed_args: argparse.Namespace,
p: beam.Pipeline):
""" Runs a manual test of the Patient Pipeline. Merely prints each element to STDOUT.
"""
output_patient_pipeline = construct(p, vars(parsed_args))
_ = output_patient_pipeline | "Print Results" >> beam.Map(
lambda x: print(f"Element: {str(x)}"))
if __name__ == "__main__":
if "--test" in sys.argv:
run_pipeline(sys.argv, construct_patient_test_pipeline)
else:
print(
"Currently, can only run Patient pipeline as test using `--test`.")