from kaapana.operators.LocalGetInputDataOperator import LocalGetInputDataOperator
from kaapana.operators.LocalMinioOperator import LocalMinioOperator
from kaapana.operators.LocalWorkflowCleanerOperator import LocalWorkflowCleanerOperator
from example.ExtractStudyIdOperator import ExtractStudyIdOperator
from example.PoolJsonsOperator import PoolJsonsOperator
log = LoggingMixin().log
args = {
'ui_visible': True,
'owner': 'kaapana',
'start_date': days_ago(0),
'retries': 0,
'retry_delay': timedelta(seconds=30)
}
dag = DAG(dag_id='example-dcm-pool-study-ids',
default_args=args,
schedule_interval=None)
get_input = LocalGetInputDataOperator(dag=dag)
extract = ExtractStudyIdOperator(dag=dag)
pool_jsons = PoolJsonsOperator(dag=dag, input_operator=extract)
put_to_minio = LocalMinioOperator(dag=dag,
action='put',
action_operators=[pool_jsons])
clean = LocalWorkflowCleanerOperator(dag=dag, clean_workflow_dir=True)
get_input >> extract >> pool_jsons >> put_to_minio >> clean
args = {
'ui_visible': True,
'owner': 'kaapana',
'start_date': days_ago(0),
'retries': 2,
'retry_delay': timedelta(seconds=30),
}
dag = DAG(dag_id='collect-metadata',
default_args=args,
concurrency=50,
max_active_runs=50,
schedule_interval=None)
get_input = LocalGetInputDataOperator(dag=dag)
anonymizer = LocalDcmAnonymizerOperator(dag=dag, single_slice=True)
extract_metadata = LocalDcm2JsonOperator(dag=dag,
input_operator=anonymizer,
delete_private_tags=False)
concat_metadata = LocalConcatJsonOperator(dag=dag,
name='concatenated-metadata',
input_operator=extract_metadata)
put_to_minio = LocalMinioOperator(dag=dag,
action='put',
action_operators=[concat_metadata],
bucket_name="downloads",
zip_files=True)
clean = LocalWorkflowCleanerOperator(dag=dag)
get_input >> anonymizer >> extract_metadata >> concat_metadata >> put_to_minio >> clean