def build_cwltool_cmd1(**kwargs):
ctx = kwargs["dag_run"].conf
run_id = kwargs["run_id"]
tmpdir = utils.get_tmp_dir_path(run_id)
print("tmpdir: ", tmpdir)
data_dirs = ctx["parent_lz_path"]
data_dirs = [data_dirs] if isinstance(data_dirs,
str) else data_dirs
print("data_dirs: ", data_dirs)
command = [
*get_cwltool_base_cmd(tmpdir),
"--relax-path-checks",
"--debug",
"--outdir",
tmpdir / "cwl_out",
"--parallel",
cwl_workflows[0],
"--assay",
params.assay,
"--threads",
THREADS,
]
for data_dir in data_dirs:
command.append("--fastq_dir")
command.append(data_dir)
return join_quote_command_str(command)
def build_cwltool_cmd2(**kwargs):
ctx = kwargs['dag_run'].conf
run_id = kwargs['run_id']
#tmpdir is temp directory in /hubmap-tmp
tmpdir = utils.get_tmp_dir_path(run_id)
print('tmpdir: ', tmpdir)
#get data directory
parent_data_dir = ctx['parent_lz_path']
print('parent_data_dir: ', parent_data_dir)
data_dir = os.path.join(
tmpdir, 'cwl_out',
'ometiff-pyramids') # This stage reads input from stage 1
print('data_dir: ', data_dir)
#this is the call to the CWL
command = [
*get_cwltool_base_cmd(tmpdir),
cwl_workflows[1],
'--input_directory',
'./ometiff-pyramids',
]
return join_quote_command_str(command)
def build_cwltool_cmd2(**kwargs):
ctx = kwargs["dag_run"].conf
run_id = kwargs["run_id"]
tmpdir = utils.get_tmp_dir_path(run_id)
print("tmpdir: ", tmpdir)
data_dir = ctx["parent_lz_path"]
print("data_dir: ", data_dir)
command = [
*get_cwltool_base_cmd(tmpdir),
cwl_workflows[1],
"--input_dir",
".",
]
return join_quote_command_str(command)
def build_cwltool_cwl_cytokit(**kwargs):
ctx = kwargs['dag_run'].conf
run_id = kwargs['run_id']
tmpdir = utils.get_tmp_dir_path(run_id)
print('tmpdir: ', tmpdir)
data_dir = ctx['parent_lz_path']
print('data_dir: ', data_dir)
command = [
*get_cwltool_base_cmd(tmpdir),
cwl_workflows['cytokit'],
'--gpus=0,1',
'--data_dir',
data_dir,
]
return join_quote_command_str(command)
def build_cwltool_cmd3(**kwargs):
ctx = kwargs["dag_run"].conf
run_id = kwargs["run_id"]
tmpdir = utils.get_tmp_dir_path(run_id)
print("tmpdir: ", tmpdir)
data_dir = ctx["parent_lz_path"]
print("data_dir: ", data_dir)
command = [
*get_cwltool_base_cmd(tmpdir),
cwl_workflows[2],
"--input_dir",
# This pipeline invocation runs in a 'hubmap_ui' subdirectory,
# so use the parent directory as input
"..",
]
return join_quote_command_str(command)
def build_cwltool_cmd_sprm_to_anndata(**kwargs):
ctx = kwargs['dag_run'].conf
run_id = kwargs['run_id']
tmpdir = utils.get_tmp_dir_path(run_id)
print('tmpdir: ', tmpdir)
parent_data_dir = ctx['parent_lz_path']
print('parent_data_dir: ', parent_data_dir)
data_dir = tmpdir / 'cwl_out' # This stage reads input from stage 1
print('data_dir: ', data_dir)
command = [
*get_cwltool_base_cmd(tmpdir),
cwl_workflows['sprm_to_anndata'],
'--input_dir',
data_dir / 'sprm_outputs',
]
return join_quote_command_str(command)
def build_cwltool_cmd_ome_tiff_offsets(**kwargs):
ctx = kwargs['dag_run'].conf
run_id = kwargs['run_id']
tmpdir = utils.get_tmp_dir_path(run_id)
print('tmpdir: ', tmpdir)
parent_data_dir = ctx['parent_lz_path']
print('parent_data_dir: ', parent_data_dir)
data_dir = tmpdir / 'cwl_out'
print('data_dir: ', data_dir)
command = [
*get_cwltool_base_cmd(tmpdir),
cwl_workflows['ome_tiff_offsets'],
'--input_dir',
data_dir / 'stitched/expressions',
]
return join_quote_command_str(command)
def build_cwltool_cmd1(**kwargs):
ctx = kwargs['dag_run'].conf
run_id = kwargs['run_id']
tmpdir = utils.get_tmp_dir_path(run_id)
data_dir = ctx['parent_lz_path']
command = [
*get_cwltool_base_cmd(tmpdir),
'--outdir',
tmpdir / 'cwl_out',
'--parallel',
cwl_workflows[0],
'--fastq_dir',
data_dir,
'--threads',
THREADS,
]
return join_quote_command_str(command)
def build_cwltool_cmd_create_vis_symlink_archive(**kwargs):
ctx = kwargs['dag_run'].conf
run_id = kwargs['run_id']
tmpdir = utils.get_tmp_dir_path(run_id)
print('tmpdir: ', tmpdir)
parent_data_dir = ctx['parent_lz_path']
print('parent_data_dir: ', parent_data_dir)
data_dir = tmpdir / 'cwl_out'
print('data_dir: ', data_dir)
command = [
*get_cwltool_base_cmd(tmpdir),
cwl_workflows['create_vis_symlink_archive'],
'--ometiff_dir',
data_dir / 'stitched',
'--sprm_output',
data_dir / 'sprm_outputs',
]
return join_quote_command_str(command)
def build_cwltool_cmd1(**kwargs):
ctx = kwargs['dag_run'].conf
run_id = kwargs['run_id']
#tmpdir is temp directory in /hubmap-tmp
tmpdir = utils.get_tmp_dir_path(run_id)
print('tmpdir: ', tmpdir)
#data directory is input directory in /hubmap-data
data_dir = ctx['parent_lz_path']
print('data_dir: ', data_dir)
#this is the call to the CWL
command = [
*get_cwltool_base_cmd(tmpdir),
cwl_workflows[0],
'--ometiff_directory',
data_dir,
]
return join_quote_command_str(command)
def build_cwltool_cwl_ome_tiff_pyramid(**kwargs):
ctx = kwargs['dag_run'].conf
run_id = kwargs['run_id']
#tmpdir is temp directory in /hubmap-tmp
tmpdir = utils.get_tmp_dir_path(run_id)
print('tmpdir: ', tmpdir)
#data directory is the stitched images, which are found in tmpdir
data_dir = ctx['parent_lz_path']
print('data_dir: ', data_dir)
#this is the call to the CWL
command = [
*get_cwltool_base_cmd(tmpdir),
"--relax-path-checks",
cwl_workflows['ome_tiff_pyramid'],
'--ometiff_directory',
'.',
]
return join_quote_command_str(command)
def build_cwltool_cmd_sprm(**kwargs):
ctx = kwargs['dag_run'].conf
run_id = kwargs['run_id']
tmpdir = utils.get_tmp_dir_path(run_id)
print('tmpdir: ', tmpdir)
parent_data_dir = ctx['parent_lz_path']
print('parent_data_dir: ', parent_data_dir)
data_dir = tmpdir / 'cwl_out'
print('data_dir: ', data_dir)
command = [
*get_cwltool_base_cmd(tmpdir),
cwl_workflows['sprm'],
'--enable_manhole',
'--image_dir',
data_dir / 'stitched/expressions',
'--mask_dir',
data_dir / 'stitched/mask',
]
return join_quote_command_str(command)
def build_cwltool_cmd1(**kwargs):
ctx = kwargs['dag_run'].conf
run_id = kwargs['run_id']
tmpdir = utils.get_tmp_dir_path(run_id)
data_dirs = ctx['parent_lz_path']
data_dirs = [data_dirs] if isinstance(data_dirs, str) else data_dirs
command = [
*get_cwltool_base_cmd(tmpdir),
'--outdir',
tmpdir / 'cwl_out',
'--parallel',
cwl_workflows[0],
'--threads',
THREADS,
]
for data_dir in data_dirs:
command.append('--sequence_directory')
command.append(data_dir)
return join_quote_command_str(command)
def build_cwltool_cmd1(**kwargs):
ctx = kwargs['dag_run'].conf
run_id = kwargs['run_id']
tmpdir = utils.get_tmp_dir_path(run_id)
tmp_subdir = tmpdir / 'cwl_out'
data_dir = ctx['parent_lz_path']
try:
delay_sec = int(ctx['metadata']['delay_sec'])
except ValueError:
print("Could not parse delay_sec "
"{} ; defaulting to 30 sec".format(
ctx['metadata']['delay_sec']))
delay_sec = 30
for fname in ctx['metadata']['files_to_copy']:
print(fname)
commands = [
[f'tmp_dir={tmpdir}'],
['sleep', delay_sec],
['cd', data_dir],
['mkdir', '-p', tmp_subdir],
]
if ctx['metadata']['files_to_copy']:
commands.append(
['cp', *ctx['metadata']['files_to_copy'], tmp_subdir])
print('command list:')
pprint(commands)
command_strs = [
join_quote_command_str(command) for command in commands
]
command_str = ' ; '.join(command_strs)
print('overall command_str:', command_str)
return command_str