def _copy_alignment_file(align_step, in_step, files_to_proc, partitions_obj):
# Note: RAxML supports relaxed PHYLIP format
# From manual:
# The input alignment format of RAxML is relaxed interleaved or sequential PHYLIP or
# FASTA. Relaxed means that sequence names can be of variable length between 1 up to 256 characters
a_f = in_step.step_file('alignment.phy')
orig_phy = align_step.get_phylip_file()
alignment = read_alignment(orig_phy)
partitions = partitions_obj.create_raxml_partitions(
align_step, in_step.step_file('partitions.ind'))
#
if max(len(seq_ident) for seq_ident in align_step.all_sequences()) == 10:
# If max length of ident is 10, than RAxML is confused about real ident length
# Add trailing space to idents
for seq in alignment:
seq.id += ' '
from ..utils.import_methods import import_bio_align_io
with open(a_f, "w") as handle:
p_format = 'phylip-relaxed' if any(
len(s) > 10 for s in alignment) else 'phylip'
count = import_bio_align_io().write(alignment, handle, p_format)
else:
copy_file(orig_phy, a_f)
#
files_to_proc.append(
dict(filename=a_f,
short=align_step.is_short(),
length=len(alignment[0]),
partitions=partitions,
seed=str(random.randint(1000, 10000000))))
def run(self, step_data):
from .steps import AnnotationsStep
from common_utils.file_utils import copy_file
step = AnnotationsStep(self.project, step_data, remove_data=True)
for s in self.args.steps:
a_step = self.project.read_step(s, check_data_type='annotations')
for seq_ident in a_step.all_sequences():
step._sequences.add(seq_ident)
copy_file(a_step.get_sequence_filename(seq_ident),
step.step_file(seq_ident + '.gb'))
step.save()
return step
def save_to_dir(self,
json_save_dir: str,
src_img_dir: str,
dst_img_dir: str = None,
overwrite: bool = False,
show_pbar: bool = False):
"""Saves NDDS_Frame_Handler object to a directory path.
Arguments:
json_save_dir {str} -- [Path to directory where you want to save the NDDS annotation json files.]
src_img_dir {str} -- [Path to directory where the original NDDS images are saved.]
Keyword Arguments:
dst_img_dir {str} -- [Path to directory where you want to copy the original NDDS images.] (default: {None})
overwrite {bool} -- [Whether or not you would like to overwrite existing files/directories.] (default: {False})
show_pbar {bool} -- [Whether or not you would like to show the progress bar.] (default: {False})
"""
self._check_paths_valid(src_img_dir=src_img_dir)
make_dir_if_not_exists(json_save_dir)
delete_all_files_in_dir(json_save_dir, ask_permission=not overwrite)
if dst_img_dir is not None:
make_dir_if_not_exists(dst_img_dir)
delete_all_files_in_dir(dst_img_dir, ask_permission=not overwrite)
if show_pbar:
pbar = tqdm(total=len(self), unit='ann(s)', leave=True)
pbar.set_description(f'Saving {self.__class__.__name__}')
for frame in self:
save_path = f'{json_save_dir}/{get_rootname_from_path(frame.img_path)}.json'
if dst_img_dir is not None:
copy_file(
src_path=f'{src_img_dir}/{get_filename(frame.img_path)}',
dest_path=f'{dst_img_dir}/{get_filename(frame.img_path)}',
silent=True)
if frame.cs_img_path:
copy_file(
src_path=
f'{src_img_dir}/{get_filename(frame.cs_img_path)}',
dest_path=
f'{dst_img_dir}/{get_filename(frame.cs_img_path)}',
silent=True)
if frame.depth_img_path:
copy_file(
src_path=
f'{src_img_dir}/{get_filename(frame.depth_img_path)}',
dest_path=
f'{dst_img_dir}/{get_filename(frame.depth_img_path)}',
silent=True)
if frame.is_img_path:
copy_file(
src_path=
f'{src_img_dir}/{get_filename(frame.is_img_path)}',
dest_path=
f'{dst_img_dir}/{get_filename(frame.is_img_path)}',
silent=True)
frame.ndds_ann.save_to_path(save_path=save_path)
else:
frame.ndds_ann.save_to_path(save_path=save_path)
if show_pbar:
pbar.update()
def create_new_hybrids_data(project, step_data, params):
# Check input files
if not os.path.isfile(params.data_file):
raise ZCItoolsValueError(
f"Input data file {params.data_file} doesn't exist!")
if not os.path.isfile(params.gtyp_cat_file):
raise ZCItoolsValueError(
f"Input genotype category probabilities {params.gtyp_cat_file} doesn't exist!"
)
data_file = os.path.basename(params.data_file)
gtyp_cat_file = os.path.basename(params.gtyp_cat_file)
step = NewHybridsStep(project, step_data, remove_data=True)
step.set_data(data_file, gtyp_cat_file, params.theta_prior,
params.pi_prior, params.burn_in, params.num_sweeps)
# Copy input files
files_to_zip = [step.step_file(data_file), step.step_file(gtyp_cat_file)]
copy_file(params.data_file, files_to_zip[0])
copy_file(params.gtyp_cat_file, files_to_zip[1])
# Create run directories
seeds = random.sample(
list(itertools.product(range(1, _MAX_SMALL_NUMBER + 1), repeat=2)),
params.num_runs)
for seed in seeds:
files_to_zip.append(step.step_file(step.seed_dir(seed)))
ensure_directory(files_to_zip[-1])
files_to_zip.append(step.step_file('finish.yml'))
write_yaml(
dict(data_file=data_file,
gtyp_cat_file=gtyp_cat_file,
theta_prior=params.theta_prior,
pi_prior=params.pi_prior,
burn_in=params.burn_in,
num_sweeps=params.num_sweeps), files_to_zip[-1])
# Stores description.yml
step.save(completed=params.run)
# Run or set instructions
if params.run:
run_module_script(run_new_hybrids, step)
else:
set_run_instructions(run_new_hybrids, step, files_to_zip,
_instructions)
#
return step
def save_to_dir(self, json_save_dir: str, src_img_dir: str, overwrite: bool=False, dst_img_dir: str=None):
self._check_paths_valid(src_img_dir=src_img_dir)
make_dir_if_not_exists(json_save_dir)
delete_all_files_in_dir(json_save_dir, ask_permission=not overwrite)
if dst_img_dir is not None:
make_dir_if_not_exists(dst_img_dir)
delete_all_files_in_dir(dst_img_dir, ask_permission=not overwrite)
for ann in tqdm(self, total=len(self), unit='ann', leave=True):
save_path = f'{json_save_dir}/{get_rootname_from_path(ann.img_path)}.json'
src_img_path = f'{src_img_dir}/{get_filename(ann.img_path)}'
if dst_img_dir is not None:
dst_img_path = f'{dst_img_dir}/{get_filename(ann.img_path)}'
copy_file(src_path=src_img_path, dest_path=dst_img_path, silent=True)
ann.save_to_path(save_path=save_path, img_path=dst_img_path)
else:
ann.save_to_path(save_path=save_path, img_path=src_img_path)
def copy_image(self, coco_image: COCO_Image, dest_img_dir: str, new_img_filename: str=None, verbose: bool=False):
coco_url = coco_image.coco_url
found, src_img_path = find_img_path(coco_url=coco_url, src_root_dir=self.src_root_dir)
if not found:
logger.error(f"Couldn't find any permutation of coco_url={coco_url} under {self.src_root_dir}")
raise Exception
if new_img_filename is None:
img_filename = get_filename(path=coco_url)
if img_filename != coco_image.file_name:
logger.error(f"coco_url filename doesn't match coco json file_name")
logger.error(f"coco_url: {coco_url}")
logger.error(f"file_name: {coco_image.file_name}")
raise Exception
else:
img_filename = new_img_filename
dest_img_path = f"{dest_img_dir}/{img_filename}"
silent = not verbose
copy_file(src_path=src_img_path, dest_path=dest_img_path, silent=silent)
def save_to_dir(self,
json_save_dir: str,
src_img_dir: str,
overwrite: bool = False,
dst_img_dir: str = None,
show_pbar: bool = True):
self._check_paths_valid(src_img_dir=src_img_dir)
make_dir_if_not_exists(json_save_dir)
delete_all_files_in_dir(json_save_dir, ask_permission=not overwrite)
if dst_img_dir is not None:
make_dir_if_not_exists(dst_img_dir)
delete_all_files_in_dir(dst_img_dir, ask_permission=not overwrite)
if show_pbar:
pbar = tqdm(total=len(self), unit='ann(s)', leave=True)
pbar.set_description(f'Saving {self.__class__.__name__}')
for frame in self:
save_path = f'{json_save_dir}/{get_rootname_from_path(frame.img_path)}.json'
if dst_img_dir is not None:
copy_file(
src_path=f'{src_img_dir}/{get_filename(frame.img_path)}',
dest_path=f'{dst_img_dir}/{get_filename(frame.img_path)}',
silent=True)
if frame.cs_img_path:
copy_file(
src_path=
f'{src_img_dir}/{get_filename(frame.cs_img_path)}',
dest_path=
f'{dst_img_dir}/{get_filename(frame.cs_img_path)}',
silent=True)
if frame.depth_img_path:
copy_file(
src_path=
f'{src_img_dir}/{get_filename(frame.depth_img_path)}',
dest_path=
f'{dst_img_dir}/{get_filename(frame.depth_img_path)}',
silent=True)
if frame.is_img_path:
copy_file(
src_path=
f'{src_img_dir}/{get_filename(frame.is_img_path)}',
dest_path=
f'{dst_img_dir}/{get_filename(frame.is_img_path)}',
silent=True)
frame.ndds_ann.save_to_path(save_path=save_path)
else:
frame.ndds_ann.save_to_path(save_path=save_path)
if show_pbar:
pbar.update()
def save_to_dir(self, json_save_dir: str, src_img_dir: str, overwrite: bool=False, dst_img_dir: str=None, show_pbar: bool=True):
self._check_paths_valid(src_img_dir=src_img_dir)
make_dir_if_not_exists(json_save_dir)
delete_all_files_in_dir(json_save_dir, ask_permission=not overwrite)
if dst_img_dir is not None:
make_dir_if_not_exists(dst_img_dir)
delete_all_files_in_dir(dst_img_dir, ask_permission=not overwrite)
pbar = tqdm(total=len(self), unit='annotation(s)', leave=True) if show_pbar else None
if pbar is not None:
pbar.set_description('Writing Labelme Annotations')
for ann in self:
save_path = f'{json_save_dir}/{get_rootname_from_path(ann.img_path)}.json'
src_img_path = f'{src_img_dir}/{get_filename(ann.img_path)}'
if dst_img_dir is not None:
dst_img_path = f'{dst_img_dir}/{get_filename(ann.img_path)}'
copy_file(src_path=src_img_path, dest_path=dst_img_path, silent=True)
ann.save_to_path(save_path=save_path, img_path=dst_img_path)
else:
ann.save_to_path(save_path=save_path, img_path=src_img_path)
if pbar is not None:
pbar.update()
if pbar is not None:
pbar.close()
def create_permutations(project,
step_data,
raw_file,
permutations,
num_traits=None,
run=False):
# Check input files
map_file = raw_file.replace('.raw', '.map')
data_dir, base_raw_file = os.path.split(raw_file)
tmp_files = ('tmp.00m', 'tmp.00c', 'tmp.00r')
for mf in (raw_file, map_file):
if not os.path.isfile(mf):
raise ZCItoolsValueError(
f"Input MapMaker file {mf} doesn't exist!")
for qf in tmp_files:
f = os.path.join(data_dir, qf)
if not os.path.isfile(f):
raise ZCItoolsValueError(
f"Input Windows QTL Cartographer file {qf} doesn't exist!")
#
step = QTLCartStep(project, step_data, remove_data=True)
step.set_data(num_traits, permutations)
# Copy input files
files_to_zip = []
for qf in tmp_files:
files_to_zip.append(step.step_file(qf))
copy_file(os.path.join(data_dir, qf), files_to_zip[-1])
# Create trait directories
# ToDo: find max traits and fix it/set default
assert num_traits and num_traits > 0, num_traits
trait_dirs = []
for t_idx in range(1, num_traits + 1):
trait_dirs.append(step.trait_dir(t_idx))
t_dir = step.step_file(trait_dirs[-1])
ensure_directory(t_dir)
files_to_zip.append(os.path.join(t_dir, 'qtlcart.rc'))
write_str_in_file(
files_to_zip[-1],
_qtlcart_rc.format(trait=t_idx, num_traits=num_traits))
# # Create links to input files
# for qf in tmp_files:
# link_file(os.path.join('..', qf), os.path.join(t_dir, qf))
#
files_to_zip.append(step.step_file('finish.yml'))
write_yaml(dict(permutations=permutations, trait_dirs=trait_dirs),
files_to_zip[-1])
# Stores description.yml
step.save(completed=run)
# Run or set instructions
if run:
run_module_script(run_qtl_cart_perm, step)
else:
set_run_instructions(run_qtl_cart_perm, step, files_to_zip,
_instructions)
#
return step
def move(self,
dst_dataroot: str,
include_depth: bool = True,
include_RT: bool = False,
camera_path: str = None,
fps_path: str = None,
preserve_filename: bool = False,
use_softlink: bool = False,
ask_permission_on_delete: bool = True,
show_pbar: bool = True):
make_dir_if_not_exists(dst_dataroot)
delete_all_files_in_dir(dst_dataroot,
ask_permission=ask_permission_on_delete,
verbose=False)
processed_image_id_list = []
pbar = tqdm(total=len(self.annotations),
unit='annotation(s)',
leave=True) if show_pbar else None
if pbar is not None:
pbar.set_description('Moving Linemod Dataset Data')
for linemod_ann in self.annotations:
if not dir_exists(linemod_ann.data_root):
raise FileNotFoundError(
f"Couldn't find data_root at {linemod_ann.data_root}")
# Images
linemod_image = self.images.get(id=linemod_ann.image_id)[0]
if linemod_image.id not in processed_image_id_list:
img_path = f'{linemod_ann.data_root}/{get_filename(linemod_image.file_name)}'
if not file_exists(img_path):
raise FileNotFoundError(
f"Couldn't find image at {img_path}")
if preserve_filename:
dst_img_path = f'{dst_dataroot}/{get_filename(linemod_image.file_name)}'
if file_exists(dst_img_path):
raise FileExistsError(f"""
Image already exists at {dst_img_path}
Hint: Use preserve_filename=False to bypass this error.
""")
else:
dst_filename = f'{linemod_image.id}.{get_extension_from_filename(linemod_image.file_name)}'
linemod_image.file_name = dst_filename
dst_img_path = f'{dst_dataroot}/{dst_filename}'
if not use_softlink:
copy_file(src_path=img_path,
dest_path=dst_img_path,
silent=True)
else:
create_softlink(src_path=rel_to_abs_path(img_path),
dst_path=rel_to_abs_path(dst_img_path))
processed_image_id_list.append(linemod_image.id)
# Masks
if not file_exists(linemod_ann.mask_path):
raise FileNotFoundError(
f"Couldn't find mask at {linemod_ann.mask_path}")
mask_path = linemod_ann.mask_path
if preserve_filename:
dst_mask_path = f'{dst_dataroot}/{get_filename(linemod_ann.mask_path)}'
if file_exists(dst_mask_path):
raise FileExistsError(f"""
Mask already exists at {dst_mask_path}
Hint: Use preserve_filename=False to bypass this error.
""")
else:
mask_filename = get_filename(linemod_ann.mask_path)
dst_filename = f'{linemod_ann.id}_mask.{get_extension_from_filename(mask_filename)}'
dst_mask_path = f'{dst_dataroot}/{dst_filename}'
linemod_ann.mask_path = dst_mask_path
if not use_softlink:
copy_file(src_path=mask_path,
dest_path=dst_mask_path,
silent=True)
else:
create_softlink(src_path=rel_to_abs_path(mask_path),
dst_path=rel_to_abs_path(dst_mask_path))
# Depth
if include_depth and linemod_ann.depth_path is not None:
if not file_exists(linemod_ann.depth_path):
raise FileNotFoundError(
f"Couldn't find depth at {linemod_ann.depth_path}")
depth_path = linemod_ann.depth_path
if preserve_filename:
dst_depth_path = f'{dst_dataroot}/{get_filename(linemod_ann.depth_path)}'
if file_exists(dst_depth_path):
raise FileExistsError(f"""
Depth already exists at {dst_depth_path}
Hint: Use preserve_filename=False to bypass this error.
""")
else:
depth_filename = get_filename(linemod_ann.depth_path)
dst_filename = f'{linemod_ann.id}_depth.{get_extension_from_filename(depth_filename)}'
dst_depth_path = f'{dst_dataroot}/{dst_filename}'
linemod_ann.depth_path = dst_depth_path
if not use_softlink:
copy_file(src_path=depth_path,
dest_path=dst_depth_path,
silent=True)
else:
create_softlink(src_path=rel_to_abs_path(depth_path),
dst_path=rel_to_abs_path(dst_depth_path))
# RT pickle files
if include_RT:
rootname = get_rootname_from_path(mask_path)
if rootname.endswith('_mask'):
rootname = rootname.replace('_mask', '')
rt_filename = f'{rootname}_RT.pkl'
rt_path = f'{linemod_ann.data_root}/{rt_filename}'
if not file_exists(rt_path):
raise FileNotFoundError(
f"Couldn't find RT pickle file at {rt_path}")
if preserve_filename:
dst_rt_path = f'{dst_dataroot}/{rt_filename}'
if file_exists(dst_depth_path):
raise FileExistsError(f"""
RT pickle file already exists at {dst_rt_path}
Hint: Use preserve_filename=False to bypass this error.
""")
else:
dst_rt_filename = f'{linemod_ann.id}_RT.pkl'
dst_rt_path = f'{dst_dataroot}/{dst_rt_filename}'
if not use_softlink:
copy_file(src_path=rt_path,
dest_path=dst_rt_path,
silent=True)
else:
create_softlink(src_path=rel_to_abs_path(rt_path),
dst_path=rel_to_abs_path(dst_rt_path))
if pbar is not None:
pbar.update()
# Camera setting
if camera_path is not None:
if not file_exists(camera_path):
raise FileNotFoundError(
f"Couldn't find camera settings at {camera_path}")
dst_camera_path = f'{dst_dataroot}/{get_filename(camera_path)}'
if file_exists(dst_camera_path):
raise FileExistsError(
f'Camera settings already saved at {dst_camera_path}')
if not use_softlink:
copy_file(src_path=camera_path,
dest_path=dst_camera_path,
silent=True)
else:
create_softlink(src_path=rel_to_abs_path(camera_path),
dst_path=rel_to_abs_path(dst_camera_path))
# FPS setting
if fps_path is not None:
if not file_exists(fps_path):
raise FileNotFoundError(
f"Couldn't find FPS settings at {fps_path}")
dst_fps_path = f'{dst_dataroot}/{get_filename(fps_path)}'
if file_exists(dst_fps_path):
raise FileExistsError(
f'FPS settings already saved at {dst_fps_path}')
if not use_softlink:
copy_file(src_path=fps_path,
dest_path=dst_fps_path,
silent=True)
else:
create_softlink(src_path=rel_to_abs_path(fps_path),
dst_path=rel_to_abs_path(dst_fps_path))
if pbar is not None:
pbar.close()