def _create_binning_gs(self, list_of_output_gsa):
"""
Create binning gold standard without anonymization first
@param list_of_output_gsa: List of file paths of assemblies
@type list_of_output_gsa: list[str|unicode]
@param file_path_output_gsa_pooled: file paths of assembly from all samples
@type file_path_output_gsa_pooled: str | unicode
@rtype: None
"""
gff = GoldStandardFileFormat(logfile = self._logfile, verbose = self._verbose)
# read-based binning
file_path_metadata = self._project_file_folder_handler.get_genome_metadata_file_path()
file_path_genome_locations = self._project_file_folder_handler.get_genome_location_file_path()
dict_sequence_to_genome_id = gff.get_dict_sequence_to_genome_id(file_path_genome_locations)
dict_genome_id_to_tax_id = gff.get_dict_genome_id_to_tax_id(file_path_metadata)
directories_fastq_dir_in = [
self._project_file_folder_handler.get_reads_dir(True, str(sample_index))
for sample_index in range(self._number_of_samples)]
if (self._read_simulator_type == "art" or self._read_simulator_type == "wgsim"):
paired_end = True
else:
paired_end = False
for sample_index in range(self._number_of_samples):
sample_id = str(sample_index)
readfiles = directories_fastq_dir_in[sample_index]
if self._phase_compress:
file_path_gs_mapping = tempfile.mktemp(
dir=self._project_file_folder_handler.get_tmp_wd(),
prefix="gs_mapping")
else:
file_path_gs_mapping = self._project_file_folder_handler.get_anonymous_reads_map_file_path(sample_id)
samtools = SamtoolsWrapper(
file_path_samtools=self._executable_samtools,
max_processes=self._max_processors,
tmp_dir=self._project_file_folder_handler.get_tmp_wd(),
logfile=self._logfile,
verbose=self._verbose,
debug=self._debug
)
list_file_paths_read_positions = [
samtools.read_start_positions_from_dir_of_bam(self._project_file_folder_handler.get_bam_dir(sample_id))
]
dict_original_seq_pos = gff.get_dict_sequence_name_to_positions(list_file_paths_read_positions)
with open(file_path_gs_mapping, 'w') as stream_output:
row_format = "{aid}\t{gid}\t{tid}\t{sid}\n"
line = '#' + row_format.format(
aid="anonymous_read_id",
gid="genome_id",
tid="tax_id",
sid="read_id")
stream_output.write(line)
for read in dict_original_seq_pos:
seq_id = read.strip().split(' ')[0]
gen_id = read.strip().split('-')[0]
genome_id = dict_sequence_to_genome_id[gen_id]
tax_id = dict_genome_id_to_tax_id[genome_id]
line = row_format.format(
aid=seq_id,
gid=genome_id,
tid=tax_id,
sid=seq_id,
)
stream_output.write(line)
if self._phase_compress:
self._list_tuple_archive_files.append(
(file_path_gs_mapping, self._project_file_folder_handler.get_anonymous_reads_map_file_path(sample_id)+".gz"))
if self._phase_compress:
file_path_gsa_mapping = tempfile.mktemp(
dir=self._project_file_folder_handler.get_tmp_wd(),
prefix="anonymous_gsa_mapping")
else:
file_path_gsa_mapping = self._project_file_folder_handler.get_anonymous_gsa_map_file_path(sample_id)
samtools = SamtoolsWrapper(
file_path_samtools=self._executable_samtools,
max_processes=self._max_processors,
tmp_dir=self._project_file_folder_handler.get_tmp_wd(),
logfile=self._logfile,
verbose=self._verbose,
debug=self._debug
)
list_file_paths_read_positions = [
samtools.read_start_positions_from_dir_of_bam(self._project_file_folder_handler.get_bam_dir(sample_id))
]
dict_original_seq_pos = gff.get_dict_sequence_name_to_positions(list_file_paths_read_positions)
file_path_output_anonymous_gsa_out = self._project_file_folder_handler.get_anonymous_gsa_file_path(sample_id)
gsa = list_of_output_gsa[sample_index]
with open(gsa, 'r') as gs:
with open(file_path_gsa_mapping, 'w') as stream_output:
row_format = "{name}\t{genome_id}\t{tax_id}\t{length}\n"
stream_output.write("@@SEQUENCEID\tBINID\tTAXID\t_LENGTH\n")
for seq_id in gs:
if not seq_id.startswith(">"):
continue
seq_id = seq_id[1:].strip()
seq_info = seq_id.rsplit("_from_", 1)
# print(seq_info)
sequence_id = seq_info[0]
# pos_start, pos_end = re.findall(r'\d+', seq_info[1])[:2]
pos_start = int(seq_info[1].split("_", 1)[0])
pos_end = int(seq_info[1].split("_to_", 1)[1].split("_", 1)[0])
genome_id = dict_sequence_to_genome_id[sequence_id]
tax_id = dict_genome_id_to_tax_id[genome_id]
stream_output.write(row_format.format(
name=seq_id,
genome_id=genome_id,
tax_id=tax_id,
length=str(pos_end-pos_start+1)
)
)
if self._phase_compress:
self._list_tuple_archive_files.append(
(file_path_gsa_mapping, self._project_file_folder_handler.get_anonymous_gsa_map_file_path(sample_id)))
else:
shutil.move(file_path_gsa_mapping, file_path_output_anonymous_gsa_out)
def _anonymize_data(self, list_of_output_gsa, file_path_output_gsa_pooled):
"""
Anonymize reads and assemblies.
@param list_of_output_gsa: List of file paths of assemblies
@type list_of_output_gsa: list[str|unicode]
@param file_path_output_gsa_pooled: file paths of assembly from all samples
@type file_path_output_gsa_pooled: str | unicode
@rtype: None
"""
gs_mapping = GoldStandardFileFormat(
column_name_gid=self._column_name_genome_id,
column_name_ncbi=self._column_name_ncbi,
separator=self._separator,
logfile=self._logfile,
verbose=self._verbose
)
file_path_metadata = self._project_file_folder_handler.get_genome_metadata_file_path()
directories_fastq_dir_in = [
self._project_file_folder_handler.get_reads_dir(True, str(sample_index))
for sample_index in range(self._number_of_samples)]
if (self._read_simulator_type == "art" or self._read_simulator_type == "wgsim"):
paired_end = True
else:
paired_end = False
file_path_genome_locations = self._project_file_folder_handler.get_genome_location_file_path()
for sample_index in range(self._number_of_samples):
file_path_anonymous_reads_tmp, file_path_anonymous_mapping_tmp = self._anonymize_reads(
directories_fastq_dir_in[sample_index],
"S{}R".format(sample_index),
paired_end)
sample_id = str(sample_index)
file_path_anonymous_reads_out = self._project_file_folder_handler.get_anonymous_reads_file_path(sample_id)
file_path_anonymous_gs_mapping_out = self._project_file_folder_handler.get_anonymous_reads_map_file_path(sample_id)
if self._phase_compress:
file_path_anonymous_gs_mapping = tempfile.mktemp(
dir=self._project_file_folder_handler.get_tmp_wd(),
prefix="anonymous_gs_mapping")
else:
file_path_anonymous_gs_mapping = self._project_file_folder_handler.get_anonymous_reads_map_file_path(sample_id)
with open(file_path_anonymous_gs_mapping, 'w') as stream_output:
gs_mapping.gs_read_mapping(
file_path_genome_locations, file_path_metadata, file_path_anonymous_mapping_tmp, stream_output
)
if self._phase_compress:
self._list_tuple_archive_files.append(
(file_path_anonymous_reads_tmp, file_path_anonymous_reads_out+".gz"))
self._list_tuple_archive_files.append(
(file_path_anonymous_gs_mapping, file_path_anonymous_gs_mapping_out+".gz"))
else:
shutil.move(file_path_anonymous_reads_tmp, file_path_anonymous_reads_out)
if not self._phase_gsa and not self._phase_pooled_gsa:
return
samtools = SamtoolsWrapper(
file_path_samtools=self._executable_samtools,
max_processes=self._max_processors,
tmp_dir=self._project_file_folder_handler.get_tmp_wd(),
logfile=self._logfile,
verbose=self._verbose,
debug=self._debug
)
if self._phase_gsa:
for sample_index in range(self._number_of_samples):
file_path_output_anonymous_gsa, file_path_anonymous_mapping_tmp = self._anonymize_gsa(
list_of_output_gsa[sample_index],
"S{}C".format(sample_index))
sample_id = str(sample_index)
file_path_output_anonymous_gsa_out = self._project_file_folder_handler.get_anonymous_gsa_file_path(sample_id)
file_path_anonymous_gsa_mapping_out = self._project_file_folder_handler.get_anonymous_gsa_map_file_path(sample_id)
if self._phase_compress:
file_path_anonymous_gsa_mapping = tempfile.mktemp(
dir=self._project_file_folder_handler.get_tmp_wd(),
prefix="anonymous_gsa_mapping")
else:
file_path_anonymous_gsa_mapping = self._project_file_folder_handler.get_anonymous_gsa_map_file_path(sample_id)
list_file_paths_read_positions = [
samtools.read_start_positions_from_dir_of_bam(self._project_file_folder_handler.get_bam_dir(sample_id))
]
with open(file_path_anonymous_gsa_mapping, 'w') as stream_output:
gs_mapping.gs_contig_mapping(
file_path_genome_locations, file_path_metadata, file_path_anonymous_mapping_tmp,
list_file_paths_read_positions, stream_output
)
if self._phase_compress:
self._list_tuple_archive_files.append(
(file_path_output_anonymous_gsa, file_path_output_anonymous_gsa_out+".gz"))
self._list_tuple_archive_files.append(
(file_path_anonymous_gsa_mapping, file_path_anonymous_gsa_mapping_out+".gz"))
else:
shutil.move(file_path_output_anonymous_gsa, file_path_output_anonymous_gsa_out)
if self._phase_pooled_gsa:
file_path_output_anonymous, file_path_anonymous_mapping_tmp = self._anonymize_pooled_gsa(
file_path_output_gsa_pooled,
"PC")
file_path_output_anonymous_out = self._project_file_folder_handler.get_anonymous_gsa_pooled_file_path()
file_path_anonymous_gsa_mapping_out = self._project_file_folder_handler.get_anonymous_gsa_pooled_map_file_path()
if self._phase_compress:
file_path_anonymous_gsa_mapping = tempfile.mktemp(
dir=self._project_file_folder_handler.get_tmp_wd(),
prefix="anonymous_gsa_pooled_mapping")
else:
file_path_anonymous_gsa_mapping = self._project_file_folder_handler.get_anonymous_gsa_pooled_map_file_path()
list_file_paths_read_positions = [
samtools.read_start_positions_from_dir_of_bam(self._project_file_folder_handler.get_bam_dir(str(sample_index)))
for sample_index in range(self._number_of_samples)
]
with open(file_path_anonymous_gsa_mapping, 'w') as stream_output:
gs_mapping.gs_contig_mapping(
file_path_genome_locations, file_path_metadata, file_path_anonymous_mapping_tmp,
list_file_paths_read_positions, stream_output
)
if self._phase_compress:
self._list_tuple_archive_files.append(
(file_path_output_anonymous, file_path_output_anonymous_out+".gz"))
self._list_tuple_archive_files.append(
(file_path_anonymous_gsa_mapping, file_path_anonymous_gsa_mapping_out+".gz"))
else:
shutil.move(file_path_output_anonymous, file_path_output_anonymous_out)
def _simulate_reads(self, file_path_distribution, sample_index):
"""
Start the simulation of illumina reads
@param file_path_distribution: File path to a distribution
@type file_path_distribution: str | unicode
@param sample_index: Sample index
@type sample_index: int | long
@rtype: None
"""
self._project_file_folder_handler._location_reads = [True, True] # TODO write public method for this
sample_id = str(sample_index)
directory_output_tmp = self._project_file_folder_handler.get_reads_dir(True, sample_id)
directory_bam = self._project_file_folder_handler.get_bam_dir(sample_id)
# directory_script = os.path.dirname(__file__)
# file_path_executable = os.path.join(directory_script, "tools", "readsimulator", "art_illumina")
# directory_error_profiles = os.path.join(directory_script, "tools", "readsimulator", "profile")
if self._read_simulator_type not in dict_of_read_simulators:
raise ValueError("Read simulator type '{}' not supported.".format(self._read_simulator_type))
simulator = dict_of_read_simulators[self._read_simulator_type](
file_path_executable=self._executable_readsim,
directory_error_profiles=self._directory_error_profiles,
separator=self._separator,
max_processes=self._max_processors,
logfile=self._logfile,
verbose=self._verbose,
debug=self._debug,
seed=None, # todo: setting seed here would cause the same seed used for every simulation
tmp_dir=self._project_file_folder_handler.get_tmp_wd())
file_path_genome_locations = self._project_file_folder_handler.get_genome_location_file_path()
simulator.simulate(
file_path_distribution=file_path_distribution,
file_path_genome_locations=file_path_genome_locations,
directory_output=directory_output_tmp,
total_size=self._sample_size_in_base_pairs,
profile=self._error_profile,
fragment_size_mean=self._fragments_size_mean_in_bp,
fragment_size_standard_deviation=self._fragment_size_standard_deviation_in_bp)
# convert sam to bam
samtools = SamtoolsWrapper(
file_path_samtools=self._executable_samtools,
max_processes=self._max_processors,
tmp_dir=self._project_file_folder_handler.get_tmp_wd(),
logfile=self._logfile,
verbose=self._verbose,
debug=self._debug
)
directory_sam = directory_output_tmp
samtools.convert_sam_to_bam(directory_sam, directory_bam)
if not self._phase_anonymize:
list_of_file_path = self._validator.get_files_in_directory(directory_output_tmp, extension="fq")
directory_output_fastq = self._project_file_folder_handler.get_reads_dir(False, sample_id)
if self._phase_compress:
for file_path in list_of_file_path:
self._list_tuple_archive_files.append((file_path, directory_output_fastq))
else:
for file_path in list_of_file_path:
shutil.move(file_path, directory_output_fastq)