if [ $status -eq 124 ] #timed out
then
exit 0
fi
exit $status
"""
)
###
# rsync the globbed files over and store on target without hierachy
###
t_rsync = RsyncOperator( task_id='rsync_data',
dry_run=True,
source=args['source_directory'],
target="{{ ti.xcom_pull(task_ids='parse_config',key='experiment_directory') }}",
includes=args['source_includes'],
prune_empty_dirs=True,
flatten=True,
)
###
# delete files large file over a certain amount of time
###
t_remove = BashOperator( task_id='remove_old_source_files',
bash_command="find {{ params.source_directory }} -name \"{{ params.file_glob }}\" -type f -mmin +{{ params.age }} -size {{ params.size }} -exec echo rm -vf '{}' +",
params={
'source_directory': args['source_directory'],
'file_glob': 'FoilHole_*_Data_*.mrc',
'age': args['remove_files_after'],
'size': '+100M',
}
""",
params={
'directory':
args['destination_directory'] + '/.daq/',
'prefix': args['tem'] + '_sync_'
})
###
# rsync the globbed files over and store on target without hierachy
###
rsync = RsyncOperator(
task_id='rsync',
dry_run=str(args['dry_run']),
source=args['source_directory'] + '/',
target="{{ ti.xcom_pull(task_ids='sample_directory') }}",
excludes=args['source_excludes'],
prune_empty_dirs=True,
# chmod='ug+x,u+rw,g+r,g-w,o-rwx',
flatten=False,
priority_weight=50,
newer="{{ ti.xcom_pull(task_ids='last_rsync') }}")
gain_refs = RsyncOperator(
task_id='gain_refs',
dry_run=str(args['dry_run']),
source='%s-gainrefs/' % args['source_directory'],
target="{{ ti.xcom_pull(task_ids='sample_directory') }}/GainRefs/",
includes="*.dm4",
chmod='ug+x,u+rw,g+r,g-w,o-rwx',
flatten=False,
priority_weight=50,