def load_info(info_filename):
sys.path.append("./ext/src/python_libs/")
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
info = pyyaml.load(open(info_filename, 'r'))
return info
def run_hammer(corrected_dataset_yaml_filename, configs_dir, execution_home, cfg,
not_used_dataset_data, ext_python_modules_home, log):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
dst_configs = os.path.join(cfg.output_dir, "configs")
if os.path.exists(dst_configs):
shutil.rmtree(dst_configs)
if cfg.iontorrent:
dir_util.copy_tree(os.path.join(configs_dir, "ionhammer"), dst_configs, preserve_times=False)
cfg_file_name = os.path.join(dst_configs, "ionhammer.cfg")
else:
dir_util.copy_tree(os.path.join(configs_dir, "hammer"), dst_configs, preserve_times=False)
cfg_file_name = os.path.join(dst_configs, "config.info")
# removing template configs
for root, dirs, files in os.walk(dst_configs):
for cfg_file in files:
cfg_file = os.path.join(root, cfg_file)
if cfg_file.endswith('.template'):
if os.path.isfile(cfg_file.split('.template')[0]):
os.remove(cfg_file)
else:
os.rename(cfg_file, cfg_file.split('.template')[0])
cfg.tmp_dir = support.get_tmp_dir(prefix="hammer_")
if cfg.iontorrent:
prepare_config_ih(cfg_file_name, cfg, ext_python_modules_home)
binary_name = "ionhammer"
else:
prepare_config_bh(cfg_file_name, cfg, log)
binary_name = "hammer"
command = [os.path.join(execution_home, binary_name),
os.path.abspath(cfg_file_name)]
log.info("\n== Running read error correction tool: " + ' '.join(command) + "\n")
support.sys_call(command, log)
if not os.path.isfile(corrected_dataset_yaml_filename):
support.error("read error correction finished abnormally: " + corrected_dataset_yaml_filename + " not found!")
corrected_dataset_data = pyyaml.load(open(corrected_dataset_yaml_filename, 'r'))
remove_not_corrected_reads(cfg.output_dir)
is_changed = False
if cfg.gzip_output:
is_changed = True
compress_dataset_files(corrected_dataset_data, ext_python_modules_home, cfg.max_threads, log)
if not_used_dataset_data:
is_changed = True
corrected_dataset_data += not_used_dataset_data
if is_changed:
pyyaml.dump(corrected_dataset_data, open(corrected_dataset_yaml_filename, 'w'))
log.info("\n== Dataset description file was created: " + corrected_dataset_yaml_filename + "\n")
if os.path.isdir(cfg.tmp_dir):
shutil.rmtree(cfg.tmp_dir)
def run_hammer(corrected_dataset_yaml_filename, configs_dir, execution_home, cfg,
not_used_dataset_data, ext_python_modules_home, log):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
dst_configs = os.path.join(cfg.output_dir, "configs")
if os.path.exists(dst_configs):
shutil.rmtree(dst_configs)
if cfg.iontorrent:
dir_util.copy_tree(os.path.join(configs_dir, "ionhammer"), dst_configs, preserve_times=False)
cfg_file_name = os.path.join(dst_configs, "ionhammer.cfg")
else:
dir_util.copy_tree(os.path.join(configs_dir, "hammer"), dst_configs, preserve_times=False)
cfg_file_name = os.path.join(dst_configs, "config.info")
# removing template configs
for root, dirs, files in os.walk(dst_configs):
for cfg_file in files:
cfg_file = os.path.join(root, cfg_file)
if cfg_file.endswith('.template'):
if os.path.isfile(cfg_file.split('.template')[0]):
os.remove(cfg_file)
else:
os.rename(cfg_file, cfg_file.split('.template')[0])
cfg.tmp_dir = support.get_tmp_dir(prefix="hammer_")
if cfg.iontorrent:
prepare_config_ih(cfg_file_name, cfg, ext_python_modules_home)
binary_name = "ionhammer"
else:
prepare_config_bh(cfg_file_name, cfg, log)
binary_name = "hammer"
command = [os.path.join(execution_home, binary_name),
os.path.abspath(cfg_file_name)]
log.info("\n== Running read error correction tool: " + ' '.join(command) + "\n")
support.sys_call(command, log)
if not os.path.isfile(corrected_dataset_yaml_filename):
support.error("read error correction finished abnormally: " + corrected_dataset_yaml_filename + " not found!")
corrected_dataset_data = pyyaml.load(open(corrected_dataset_yaml_filename, 'r'))
remove_not_corrected_reads(cfg.output_dir)
is_changed = False
if cfg.gzip_output:
is_changed = True
compress_dataset_files(corrected_dataset_data, ext_python_modules_home, cfg.max_threads, log)
if not_used_dataset_data:
is_changed = True
corrected_dataset_data += not_used_dataset_data
if is_changed:
pyyaml.dump(corrected_dataset_data, open(corrected_dataset_yaml_filename, 'w'))
log.info("\n== Dataset description file was created: " + corrected_dataset_yaml_filename + "\n")
if os.path.isdir(cfg.tmp_dir):
shutil.rmtree(cfg.tmp_dir)
def __init__(self, cfg, *args):
super(BeforeStartStage, self).__init__(*args)
output_dir = cfg["common"].output_dir
self.tmp_files = []
if (os.path.isfile(os.path.join(output_dir, "run_spades.yaml"))):
previous_pipeline = pyyaml.load(
open(os.path.join(output_dir, "run_spades.yaml")))
for previous_stage in previous_pipeline:
self.tmp_files += previous_stage["del_after"]
def get_first_incomplete_command(filename):
with open(filename) as stream:
old_pipeline = pyyaml.load(stream)
first_incomplete_stage_id = 0
while first_incomplete_stage_id < len(old_pipeline):
stage_filename = options_storage.get_stage_filename(first_incomplete_stage_id, old_pipeline[first_incomplete_stage_id]["short_name"])
if not os.path.isfile(stage_filename):
return old_pipeline[first_incomplete_stage_id]
first_incomplete_stage_id += 1
return None
def read_commands_from_yaml(yaml_fpath):
if sys.version.startswith("2."):
import pyyaml2 as yaml
elif sys.version.startswith("3."):
import pyyaml3 as yaml
with open(yaml_fpath) as stream:
data = yaml.load(stream)
commands = []
for kwargs in data:
commands.append(Command(**kwargs))
return commands
def run_bh(result_filename, configs_dir, execution_home, cfg,
ext_python_modules_home, log):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
dst_configs = os.path.join(cfg.output_dir, "configs")
if os.path.exists(dst_configs):
shutil.rmtree(dst_configs)
shutil.copytree(os.path.join(configs_dir, "hammer"), dst_configs)
cfg_file_name = os.path.join(dst_configs, "config.info")
# removing template configs
for root, dirs, files in os.walk(dst_configs):
for cfg_file in files:
cfg_file = os.path.join(root, cfg_file)
if cfg_file.endswith('.info.template'):
if os.path.isfile(cfg_file.split('.template')[0]):
os.remove(cfg_file)
else:
os.rename(cfg_file, cfg_file.split('.template')[0])
prepare_config_bh(cfg_file_name, cfg, log)
command = [
os.path.join(execution_home, "hammer"),
os.path.abspath(cfg_file_name)
]
log.info("\n== Running read error correction tool: " + ' '.join(command) +
"\n")
support.sys_call(command, log)
corrected_dataset_yaml_filename = os.path.join(cfg.tmp_dir,
"corrected.yaml")
if not os.path.isfile(corrected_dataset_yaml_filename):
support.error("read error correction finished abnormally: " +
corrected_dataset_yaml_filename + " not found!")
corrected_dataset_data = pyyaml.load(
open(corrected_dataset_yaml_filename, 'r'))
if cfg.gzip_output:
log.info("\n== Compressing corrected reads (with gzip)")
move_dataset_files(corrected_dataset_data, cfg.output_dir,
ext_python_modules_home, cfg.max_threads, log,
cfg.gzip_output)
corrected_dataset_yaml_filename = result_filename
pyyaml.dump(corrected_dataset_data,
open(corrected_dataset_yaml_filename, 'w'))
log.info("\n== Dataset description file created: " +
corrected_dataset_yaml_filename + "\n")
shutil.rmtree(cfg.tmp_dir)
def prepare_config_ih(filename, cfg, ext_python_modules_home):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
data = pyyaml.load(open(filename, 'r'))
data["dataset"] = cfg.dataset_yaml_filename
data["working_dir"] = cfg.tmp_dir
data["output_dir"] = cfg.output_dir
data["hard_memory_limit"] = cfg.max_memory
data["max_nthreads"] = cfg.max_threads
pyyaml.dump(data, open(filename, 'w'))
def prepare_config_ih(filename, cfg, ext_python_modules_home):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
data = pyyaml.load(open(filename, 'r'))
data["dataset"] = cfg.dataset_yaml_filename
data["working_dir"] = cfg.tmp_dir
data["output_dir"] = cfg.output_dir
data["hard_memory_limit"] = cfg.max_memory
data["max_nthreads"] = cfg.max_threads
pyyaml.dump(data, open(filename, 'w'))
def get_read_length(output_dir, K, ext_python_modules_home, log):
est_params_filename = os.path.join(output_dir, "K%d" % K, "final.lib_data")
max_read_length = 0
if os.path.isfile(est_params_filename):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
est_params_data = pyyaml.load(open(est_params_filename, 'r'))
max_read_length = int(est_params_data['nomerge max read length'])
log.info("Max read length detected as %d" % max_read_length)
if max_read_length == 0:
support.error("Failed to estimate maximum read length! File with estimated params: " + est_params_filename, log)
return max_read_length
def get_first_incomplete_command(filename):
with open(filename) as stream:
old_pipeline = pyyaml.load(stream)
first_incomplete_stage_id = 0
while first_incomplete_stage_id < len(old_pipeline):
stage_name = "stage_%d_%s" % (
first_incomplete_stage_id,
old_pipeline[first_incomplete_stage_id]["short_name"])
if not os.path.isfile(
os.path.join(get_stage.cfg["common"].output_dir, stage_name)):
return old_pipeline[first_incomplete_stage_id]
first_incomplete_stage_id += 1
return None
def get_read_length(output_dir, K, ext_python_modules_home, log):
est_params_filename = os.path.join(output_dir, "K%d" % K, "final.lib_data")
max_read_length = 0
if os.path.isfile(est_params_filename):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
est_params_data = pyyaml.load(open(est_params_filename, 'r'))
max_read_length = int(est_params_data['nomerge max read length'])
log.info("Max read length detected as %d" % max_read_length)
if max_read_length == 0:
support.error("Failed to estimate maximum read length! File with estimated params: " + est_params_filename, log)
return max_read_length
def prepare_config_corr(filename, cfg, ext_python_modules_home):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
data = pyyaml.load(open(filename, 'r'))
data["dataset"] = cfg.dataset
data["output_dir"] = cfg.output_dir
data["work_dir"] = os.path.join(cfg.output_dir, 'tmp')
#data["hard_memory_limit"] = cfg.max_memory
data["max_nthreads"] = cfg.max_threads
data["bwa"] = cfg.bwa
file_c = open(filename, 'w')
pyyaml.dump(data, file_c)
file_c.close()
def prepare_config_corr(filename, cfg, ext_python_modules_home):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
data = pyyaml.load(open(filename, 'r'))
data["dataset"] = cfg.dataset
data["output_dir"] = cfg.output_dir
data["work_dir"] = process_cfg.process_spaces(cfg.tmp_dir)
#data["hard_memory_limit"] = cfg.max_memory
data["max_nthreads"] = cfg.max_threads
data["bwa"] = cfg.bwa
file_c = open(filename, 'w')
pyyaml.dump(data, file_c, default_flow_style = False, default_style='"', width=100500)
file_c.close()
def prepare_config_corr(filename, cfg, ext_python_modules_home):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
data = pyyaml.load(open(filename, 'r'))
data["dataset"] = cfg.dataset
data["output_dir"] = cfg.output_dir
data["work_dir"] = os.path.join(cfg.output_dir, 'tmp')
#data["hard_memory_limit"] = cfg.max_memory
data["max_nthreads"] = cfg.max_threads
data["bwa"] = cfg.bwa
file_c = open(filename, 'w')
pyyaml.dump(data, file_c)
file_c.close()
def get_read_length(output_dir, K, ext_python_modules_home, log):
est_params_filename = os.path.join(output_dir, "K%d" % K, "final.lib_data")
max_read_length = 0
if os.path.isfile(est_params_filename):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
est_params_data = pyyaml.load(open(est_params_filename, 'r'))
for reads_library in est_params_data:
if reads_library['type'] in READS_TYPES_USED_IN_CONSTRUCTION:
if int(reads_library["read length"]) > max_read_length:
max_read_length = int(reads_library["read length"])
if max_read_length == 0:
support.error("Failed to estimate maximum read length! File with estimated params: " + est_params_filename, log)
return max_read_length
def prepare_config_corr(filename, cfg, ext_python_modules_home):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
data = pyyaml.load(open(filename, 'r'))
data["dataset"] = cfg.dataset
data["output_dir"] = cfg.output_dir
data["work_dir"] = cfg.tmp_dir
#data["hard_memory_limit"] = cfg.max_memory
data["max_nthreads"] = cfg.max_threads
data["bwa"] = cfg.bwa
file_c = open(filename, 'w')
pyyaml.dump(data, file_c,
default_flow_style=False, default_style='"', width=float("inf"))
file_c.close()
def clear_configs(cfg, log, command_before_restart_from,
stage_id_before_restart_from):
def matches_with_restart_from_arg(stage, restart_from_arg):
return stage["short_name"].startswith(restart_from_arg.split(":")[0])
spades_commands_fpath = os.path.join(cfg["common"].output_dir,
"run_spades.yaml")
with open(spades_commands_fpath) as stream:
old_pipeline = pyyaml.load(stream)
restart_from_stage_id = None
for num in range(len(old_pipeline)):
stage = old_pipeline[num]
if matches_with_restart_from_arg(stage,
options_storage.args.restart_from):
restart_from_stage_id = num
break
if command_before_restart_from is not None and \
old_pipeline[stage_id_before_restart_from]["short_name"] != command_before_restart_from.short_name:
support.error(
"new and old pipelines have difference before %s" %
options_storage.args.restart_from, log)
if command_before_restart_from is None:
first_del = 0
else:
first_del = stage_id_before_restart_from + 1
if restart_from_stage_id is not None:
stage_filename = options_storage.get_stage_filename(
restart_from_stage_id,
old_pipeline[restart_from_stage_id]["short_name"])
if os.path.isfile(stage_filename):
os.remove(stage_filename)
for delete_id in range(first_del, len(old_pipeline)):
stage_filename = options_storage.get_stage_filename(
delete_id, old_pipeline[delete_id]["short_name"])
if os.path.isfile(stage_filename):
os.remove(stage_filename)
cfg_dir = old_pipeline[delete_id]["config_dir"]
if cfg_dir != "" and os.path.isdir(
os.path.join(cfg["common"].output_dir, cfg_dir)):
shutil.rmtree(os.path.join(cfg["common"].output_dir, cfg_dir))
def prepare_config_ih(self, filename, cfg, ext_python_modules_home):
addsitedir(ext_python_modules_home)
if sys.version.startswith("2."):
import pyyaml2 as pyyaml
elif sys.version.startswith("3."):
import pyyaml3 as pyyaml
data = pyyaml.load(open(filename))
data["dataset"] = cfg.dataset_yaml_filename
data["working_dir"] = cfg.tmp_dir
data["output_dir"] = cfg.output_dir
data["hard_memory_limit"] = cfg.max_memory
data["max_nthreads"] = cfg.max_threads
with open(filename, 'w') as f:
pyyaml.dump(data,
f,
default_flow_style=False,
default_style='"',
width=float("inf"))
def main():
args = parse_args()
# create logger
log = logging.getLogger("Postprocessing")
log.setLevel(logging.DEBUG)
console = logging.StreamHandler(sys.stdout)
console.setFormatter(logging.Formatter("%(message)s"))
console.setLevel(logging.DEBUG)
log.addHandler(console)
addsitedir(args.ext_python_modules_home)
# save dataset from yaml
if sys.version.startswith("2."):
import pyyaml2 as pyyaml
elif sys.version.startswith("3."):
import pyyaml3 as pyyaml
dataset_data = pyyaml.load(open(args.dataset_yaml_file))
# init python_lib_folder
python_modules_home = abspath(dirname(realpath(__file__)))
source_dirs = ["..", "../truspades", "../common", "../executors"]
for dir_name in source_dirs:
sys.path.append(join(python_modules_home, dir_name))
# import alignment and molecule_postprocassing
import alignment
import moleculo_postprocessing
# run command
if isfile(args.result_scaffolds_filename):
shutil.move(args.result_scaffolds_filename,
args.assembled_scaffolds_filename)
alignment_bin = os.path.join(args.bin_home, "spades-bwa")
alignment_dir = os.path.join(args.output_dir, "alignment")
sam_files = alignment.align_bwa(alignment_bin,
args.assembled_scaffolds_filename,
dataset_data, alignment_dir, log,
args.threads)
moleculo_postprocessing.moleculo_postprocessing(
args.assembled_scaffolds_filename, args.truseq_long_reads_file_base,
sam_files, log)
def run_bh(result_filename, configs_dir, execution_home, cfg, ext_python_modules_home, log):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
dst_configs = os.path.join(cfg.output_dir, "configs")
if os.path.exists(dst_configs):
shutil.rmtree(dst_configs)
shutil.copytree(os.path.join(configs_dir, "hammer"), dst_configs)
cfg_file_name = os.path.join(dst_configs, "config.info")
# removing template configs
for root, dirs, files in os.walk(dst_configs):
for cfg_file in files:
cfg_file = os.path.join(root, cfg_file)
if cfg_file.endswith('.info.template'):
if os.path.isfile(cfg_file.split('.template')[0]):
os.remove(cfg_file)
else:
os.rename(cfg_file, cfg_file.split('.template')[0])
prepare_config_bh(cfg_file_name, cfg, log)
command = [os.path.join(execution_home, "hammer"),
os.path.abspath(cfg_file_name)]
log.info("\n== Running read error correction tool: " + ' '.join(command) + "\n")
support.sys_call(command, log)
corrected_dataset_yaml_filename = os.path.join(cfg.tmp_dir, "corrected.yaml")
if not os.path.isfile(corrected_dataset_yaml_filename):
support.error("read error correction finished abnormally: " + corrected_dataset_yaml_filename + " not found!")
corrected_dataset_data = pyyaml.load(open(corrected_dataset_yaml_filename, 'r'))
if cfg.gzip_output:
log.info("\n== Compressing corrected reads (with gzip)")
move_dataset_files(corrected_dataset_data, cfg.output_dir, ext_python_modules_home, cfg.max_threads, log, cfg.gzip_output)
corrected_dataset_yaml_filename = result_filename
pyyaml.dump(corrected_dataset_data, open(corrected_dataset_yaml_filename, 'w'))
log.info("\n== Dataset description file created: " + corrected_dataset_yaml_filename + "\n")
shutil.rmtree(cfg.tmp_dir)
def print_used_values(cfg, log):
def print_value(cfg, section, param, pretty_param="", margin=" "):
if not pretty_param:
pretty_param = param.capitalize().replace('_', ' ')
line = margin + pretty_param
if param in cfg[section].__dict__:
line += ": " + str(cfg[section].__dict__[param])
else:
if param.find("offset") != -1:
line += " will be auto-detected"
log.info(line)
log.info("")
# system info
log.info("System information:")
try:
log.info(" SPAdes version: " + str(spades_version).strip())
log.info(" Python version: " +
".".join(map(str, sys.version_info[0:3])))
# for more details: '[' + str(sys.version_info) + ']'
log.info(" OS: " + platform.platform())
# for more deatils: '[' + str(platform.uname()) + ']'
except Exception:
log.info(" Problem occurred when getting system information")
log.info("")
# main
print_value(cfg, "common", "output_dir", "", "")
if ("error_correction" in cfg) and (not "assembly" in cfg):
log.info("Mode: ONLY read error correction (without assembling)")
elif (not "error_correction" in cfg) and ("assembly" in cfg):
log.info("Mode: ONLY assembling (without read error correction)")
else:
log.info("Mode: read error correction and assembling")
if ("common" in cfg) and ("developer_mode" in cfg["common"].__dict__):
if cfg["common"].developer_mode:
log.info("Debug mode turned ON")
else:
log.info("Debug mode turned OFF")
log.info("")
# dataset
if "dataset" in cfg:
log.info("Dataset parameters:")
if cfg["dataset"].single_cell:
log.info(" Single-cell mode")
else:
log.info(" Multi-cell mode (you should set '--sc' flag if input data"\
" was obtained with MDA (single-cell) technology")
log.info(" Reads:")
dataset_data = pyyaml.load(open(cfg["dataset"].yaml_filename, 'r'))
dataset_data = support.relative2abs_paths(
dataset_data, os.path.dirname(cfg["dataset"].yaml_filename))
support.pretty_print_reads(dataset_data, log)
# error correction
if "error_correction" in cfg:
log.info("Read error correction parameters:")
print_value(cfg, "error_correction", "tmp_dir", "Dir for temp files")
print_value(cfg, "error_correction", "max_iterations", "Iterations")
print_value(cfg, "error_correction", "qvoffset", "PHRED offset")
if cfg["error_correction"].gzip_output:
log.info(" Corrected reads will be compressed (with gzip)")
else:
log.info(" Corrected reads will NOT be compressed (with gzip)")
# assembly
if "assembly" in cfg:
log.info("Assembly parameters:")
print_value(cfg, "assembly", "iterative_K", "k")
log.info("Other parameters:")
print_value(cfg, "common", "max_threads", "Threads")
print_value(cfg, "common", "max_memory", "Memory limit (in Gb)", " ")
log.info("")
def main(args):
os.environ["LC_ALL"] = "C"
if len(args) == 1:
show_usage(0)
log = logging.getLogger('spades')
log.setLevel(logging.DEBUG)
console = logging.StreamHandler(sys.stdout)
console.setFormatter(logging.Formatter('%(message)s'))
console.setLevel(logging.DEBUG)
log.addHandler(console)
support.check_binaries(bin_home, log)
# parse options and safe all parameters to cfg
options = args
cfg, dataset_data = fill_cfg(options, log)
if options_storage.continue_mode:
cmd_line, options = get_options_from_params(os.path.join(options_storage.output_dir, "params.txt"), args[0])
if not options:
support.error("failed to parse command line of the previous run! Please restart from the beginning or specify another output directory.")
cfg, dataset_data = fill_cfg(options, log, secondary_filling=True)
if options_storage.restart_from:
check_cfg_for_partial_run(cfg, type='restart-from')
options_storage.continue_mode = True
if options_storage.stop_after:
check_cfg_for_partial_run(cfg, type='stop-after')
log_filename = os.path.join(cfg["common"].output_dir, "spades.log")
if options_storage.continue_mode:
log_handler = logging.FileHandler(log_filename, mode='a')
else:
log_handler = logging.FileHandler(log_filename, mode='w')
log.addHandler(log_handler)
if options_storage.continue_mode:
log.info("\n======= SPAdes pipeline continued. Log can be found here: " + log_filename + "\n")
log.info("Restored from " + cmd_line)
if options_storage.restart_from:
updated_params = ""
skip_next = False
for v in args[1:]:
if v == '-o' or v == '--restart-from':
skip_next = True
continue
if skip_next or v.startswith('--restart-from='): # you can specify '--restart-from=k33' but not '-o=out_dir'
skip_next = False
continue
updated_params += " " + v
updated_params = updated_params.strip()
log.info("with updated parameters: " + updated_params)
cmd_line += " " + updated_params
log.info("")
params_filename = os.path.join(cfg["common"].output_dir, "params.txt")
params_handler = logging.FileHandler(params_filename, mode='w')
log.addHandler(params_handler)
if options_storage.continue_mode:
log.info(cmd_line)
else:
command = "Command line: "
for v in args:
# substituting relative paths with absolute ones (read paths, output dir path, etc)
v, prefix = support.get_option_prefix(v)
if v in options_storage.dict_of_rel2abs.keys():
v = options_storage.dict_of_rel2abs[v]
if prefix:
command += prefix + ":"
command += v + " "
log.info(command)
# special case
# if "mismatch_corrector" in cfg and not support.get_lib_ids_by_type(dataset_data, 'paired-end'):
# support.warning('cannot perform mismatch correction without at least one paired-end library! Skipping this step.', log)
# del cfg["mismatch_corrector"]
print_used_values(cfg, log)
log.removeHandler(params_handler)
support.check_single_reads_in_options(options, log)
if not options_storage.continue_mode:
log.info("\n======= SPAdes pipeline started. Log can be found here: " + log_filename + "\n")
# splitting interlaced reads and processing Ns in additional contigs if needed
if support.dataset_has_interlaced_reads(dataset_data) or support.dataset_has_additional_contigs(dataset_data)\
or support.dataset_has_nxmate_reads(dataset_data):
dir_for_split_reads = os.path.join(options_storage.output_dir, 'split_input')
if support.dataset_has_interlaced_reads(dataset_data) or support.dataset_has_nxmate_reads(dataset_data):
if not os.path.isdir(dir_for_split_reads):
os.makedirs(dir_for_split_reads)
if support.dataset_has_interlaced_reads(dataset_data):
dataset_data = support.split_interlaced_reads(dataset_data, dir_for_split_reads, log)
if support.dataset_has_nxmate_reads(dataset_data):
dataset_data = support.process_nxmate_reads(dataset_data, dir_for_split_reads, log)
if support.dataset_has_additional_contigs(dataset_data):
dataset_data = support.process_Ns_in_additional_contigs(dataset_data, dir_for_split_reads, log)
options_storage.dataset_yaml_filename = os.path.join(options_storage.output_dir, "input_dataset.yaml")
pyyaml.dump(dataset_data, open(options_storage.dataset_yaml_filename, 'w'))
cfg["dataset"].yaml_filename = options_storage.dataset_yaml_filename
try:
# copying configs before all computations (to prevent its changing at run time)
tmp_configs_dir = os.path.join(cfg["common"].output_dir, "configs")
if os.path.isdir(tmp_configs_dir) and not options_storage.continue_mode:
shutil.rmtree(tmp_configs_dir)
if not os.path.isdir(tmp_configs_dir):
if options_storage.configs_dir:
dir_util.copy_tree(options_storage.configs_dir, tmp_configs_dir, preserve_times=False)
else:
dir_util.copy_tree(os.path.join(spades_home, "configs"), tmp_configs_dir, preserve_times=False)
corrected_dataset_yaml_filename = ''
if "error_correction" in cfg:
STAGE_NAME = "Read error correction"
bh_cfg = merge_configs(cfg["error_correction"], cfg["common"])
corrected_dataset_yaml_filename = os.path.join(bh_cfg.output_dir, "corrected.yaml")
if os.path.isfile(corrected_dataset_yaml_filename) and options_storage.continue_mode \
and not options_storage.restart_from == "ec":
log.info("\n===== Skipping %s (already processed). \n" % STAGE_NAME)
else:
support.continue_from_here(log)
if "HEAPCHECK" in os.environ:
del os.environ["HEAPCHECK"]
if "heap_check" in bh_cfg.__dict__:
os.environ["HEAPCHECK"] = bh_cfg.heap_check
if os.path.exists(bh_cfg.output_dir):
shutil.rmtree(bh_cfg.output_dir)
os.makedirs(bh_cfg.output_dir)
if support.get_lib_ids_by_type(dataset_data, options_storage.LONG_READS_TYPES):
not_used_dataset_data = support.get_libs_by_type(dataset_data, options_storage.LONG_READS_TYPES)
to_correct_dataset_data = support.rm_libs_by_type(dataset_data, options_storage.LONG_READS_TYPES)
to_correct_dataset_yaml_filename = os.path.join(bh_cfg.output_dir, "to_correct.yaml")
pyyaml.dump(to_correct_dataset_data, open(to_correct_dataset_yaml_filename, 'w'))
bh_cfg.__dict__["dataset_yaml_filename"] = to_correct_dataset_yaml_filename
else:
not_used_dataset_data = None
bh_cfg.__dict__["dataset_yaml_filename"] = cfg["dataset"].yaml_filename
log.info("\n===== %s started. \n" % STAGE_NAME)
hammer_logic.run_hammer(corrected_dataset_yaml_filename, tmp_configs_dir, bin_home, bh_cfg, not_used_dataset_data,
ext_python_modules_home, log)
log.info("\n===== %s finished. \n" % STAGE_NAME)
if options_storage.stop_after == 'ec':
support.finish_here(log)
result_contigs_filename = os.path.join(cfg["common"].output_dir, "contigs.fasta")
result_scaffolds_filename = os.path.join(cfg["common"].output_dir, "scaffolds.fasta")
result_assembly_graph_filename = os.path.join(cfg["common"].output_dir, "assembly_graph.fastg")
truseq_long_reads_file_base = os.path.join(cfg["common"].output_dir, "truseq_long_reads")
truseq_long_reads_file = truseq_long_reads_file_base + ".fasta"
misc_dir = os.path.join(cfg["common"].output_dir, "misc")
### if mismatch correction is enabled then result contigs are copied to misc directory
assembled_contigs_filename = os.path.join(misc_dir, "assembled_contigs.fasta")
assembled_scaffolds_filename = os.path.join(misc_dir, "assembled_scaffolds.fasta")
if "assembly" in cfg and not options_storage.run_completed:
STAGE_NAME = "Assembling"
spades_cfg = merge_configs(cfg["assembly"], cfg["common"])
spades_cfg.__dict__["result_contigs"] = result_contigs_filename
spades_cfg.__dict__["result_scaffolds"] = result_scaffolds_filename
spades_cfg.__dict__["result_graph"] = result_assembly_graph_filename
if options_storage.continue_mode and (os.path.isfile(spades_cfg.result_contigs)
or ("mismatch_corrector" in cfg and
os.path.isfile(assembled_contigs_filename)))\
and not options_storage.restart_from == 'as' \
and not options_storage.restart_from == 'scc' \
and not (options_storage.restart_from and options_storage.restart_from.startswith('k')):
log.info("\n===== Skipping %s (already processed). \n" % STAGE_NAME)
# calculating latest_dir for the next stages
latest_dir = support.get_latest_dir(os.path.join(spades_cfg.output_dir, "K*"))
if not latest_dir:
support.error("failed to continue the previous run! Please restart from previous stages or from the beginning.", log)
else:
old_result_files = [result_contigs_filename, result_scaffolds_filename,
assembled_contigs_filename, assembled_scaffolds_filename]
for old_result_file in old_result_files:
if os.path.isfile(old_result_file):
os.remove(old_result_file)
if options_storage.restart_from == 'as':
support.continue_from_here(log)
if os.path.isfile(corrected_dataset_yaml_filename):
dataset_data = pyyaml.load(open(corrected_dataset_yaml_filename, 'r'))
dataset_data = support.relative2abs_paths(dataset_data, os.path.dirname(corrected_dataset_yaml_filename))
if spades_cfg.disable_rr:
spades_cfg.__dict__["rr_enable"] = False
else:
spades_cfg.__dict__["rr_enable"] = True
if "HEAPCHECK" in os.environ:
del os.environ["HEAPCHECK"]
if "heap_check" in spades_cfg.__dict__:
os.environ["HEAPCHECK"] = spades_cfg.heap_check
log.info("\n===== %s started.\n" % STAGE_NAME)
# creating dataset
dataset_filename = os.path.join(spades_cfg.output_dir, "dataset.info")
if not os.path.isfile(dataset_filename) or not options_storage.continue_mode:
dataset_file = open(dataset_filename, 'w')
import process_cfg
dataset_file.write("single_cell" + '\t' + process_cfg.bool_to_str(cfg["dataset"].single_cell) + '\n')
dataset_file.write("meta" + '\t' + process_cfg.bool_to_str(cfg["dataset"].meta) + '\n')
dataset_file.write("moleculo" + '\t' + process_cfg.bool_to_str(cfg["dataset"].truseq) + '\n')
if os.path.isfile(corrected_dataset_yaml_filename):
dataset_file.write("reads" + '\t' + process_cfg.process_spaces(corrected_dataset_yaml_filename) + '\n')
else:
dataset_file.write("reads" + '\t' + process_cfg.process_spaces(cfg["dataset"].yaml_filename) + '\n')
if spades_cfg.developer_mode and "reference" in cfg["dataset"].__dict__:
dataset_file.write("reference_genome" + '\t')
dataset_file.write(process_cfg.process_spaces(cfg["dataset"].reference) + '\n')
dataset_file.close()
spades_cfg.__dict__["dataset"] = dataset_filename
latest_dir = spades_logic.run_spades(tmp_configs_dir, bin_home, spades_cfg, dataset_data, ext_python_modules_home, log)
if os.path.isdir(misc_dir) and not options_storage.continue_mode:
shutil.rmtree(misc_dir)
if not os.path.isdir(misc_dir):
os.makedirs(misc_dir)
if options_storage.continue_mode and options_storage.restart_from and options_storage.restart_from.startswith('k'):
k_str = options_storage.restart_from[1:]
if k_str.find(":") != -1:
k_str = k_str[:k_str.find(":")]
support.error("failed to continue from K=%s because this K was not processed in the original run!" % k_str, log)
log.info("\n===== %s finished. \n" % STAGE_NAME)
if not options_storage.run_completed:
if options_storage.stop_after == 'as' or options_storage.stop_after == 'scc' or (options_storage.stop_after and options_storage.stop_after.startswith('k')):
support.finish_here(log)
#postprocessing
if cfg["run_truseq_postprocessing"] and not options_storage.run_completed:
if options_storage.continue_mode and os.path.isfile(truseq_long_reads_file_base + ".fastq") and not options_storage.restart_from == 'tpp':
log.info("\n===== Skipping %s (already processed). \n" % "TruSeq postprocessing")
else:
support.continue_from_here(log)
if os.path.isfile(result_scaffolds_filename):
shutil.move(result_scaffolds_filename, assembled_scaffolds_filename)
reads_library = dataset_data[0]
alignment_bin = os.path.join(bin_home, "bwa-spades")
alignment_dir = os.path.join(cfg["common"].output_dir, "alignment")
sam_files = alignment.align_bwa(alignment_bin, assembled_scaffolds_filename, dataset_data, alignment_dir, log, options_storage.threads)
moleculo_postprocessing.moleculo_postprocessing(assembled_scaffolds_filename, truseq_long_reads_file_base, sam_files, log)
if options_storage.stop_after == 'tpp':
support.finish_here(log)
#corrector
if "mismatch_corrector" in cfg and not options_storage.run_completed and \
(os.path.isfile(result_contigs_filename) or
(options_storage.continue_mode and os.path.isfile(assembled_contigs_filename))):
STAGE_NAME = "Mismatch correction"
to_correct = dict()
to_correct["contigs"] = (result_contigs_filename, assembled_contigs_filename)
if os.path.isfile(result_scaffolds_filename) or (options_storage.continue_mode and
os.path.isfile(assembled_scaffolds_filename)):
to_correct["scaffolds"] = (result_scaffolds_filename, assembled_scaffolds_filename)
# moving assembled contigs (scaffolds) to misc dir
for assembly_type, (old, new) in to_correct.items():
if options_storage.continue_mode and os.path.isfile(new):
continue
if os.path.isfile(old):
shutil.move(old, new)
if options_storage.continue_mode and os.path.isfile(result_contigs_filename) and \
(os.path.isfile(result_scaffolds_filename) or not os.path.isfile(assembled_scaffolds_filename)) \
and not options_storage.restart_from == 'mc':
log.info("\n===== Skipping %s (already processed). \n" % STAGE_NAME)
else:
if options_storage.restart_from == 'mc':
support.continue_from_here(log)
log.info("\n===== %s started." % STAGE_NAME)
# detecting paired-end library with the largest insert size
cfg["mismatch_corrector"].__dict__["dataset"] = cfg["dataset"].yaml_filename
#TODO: add reads orientation
import corrector_logic
corrector_cfg = cfg["mismatch_corrector"]
# processing contigs and scaffolds (or only contigs)
for assembly_type, (corrected, assembled) in to_correct.items():
if options_storage.continue_mode and os.path.isfile(corrected):
log.info("\n== Skipping processing of " + assembly_type + " (already processed)\n")
continue
support.continue_from_here(log)
log.info("\n== Processing of " + assembly_type + "\n")
tmp_dir_for_corrector = os.path.join(cfg["common"].output_dir, "mismatch_corrector", assembly_type)
cfg["mismatch_corrector"].__dict__["output_dir"] = tmp_dir_for_corrector
# correcting
corr_cfg = merge_configs(cfg["mismatch_corrector"], cfg["common"])
result_corrected_filename = os.path.join(tmp_dir_for_corrector, "corrected_contigs.fasta")
corrector_logic.run_corrector( tmp_configs_dir, bin_home, corr_cfg,
ext_python_modules_home, log, assembled, result_corrected_filename)
if os.path.isfile(result_corrected_filename):
shutil.copyfile(result_corrected_filename, corrected)
tmp_d = os.path.join(tmp_dir_for_corrector, "tmp")
if os.path.isdir(tmp_d) and not cfg["common"].developer_mode:
shutil.rmtree(tmp_d)
log.info("\n===== %s finished.\n" % STAGE_NAME)
if options_storage.stop_after == 'mc':
support.finish_here(log)
if not cfg["common"].developer_mode and os.path.isdir(tmp_configs_dir):
shutil.rmtree(tmp_configs_dir)
if not options_storage.run_completed:
#log.info("")
if "error_correction" in cfg and os.path.isdir(os.path.dirname(corrected_dataset_yaml_filename)):
log.info(" * Corrected reads are in " + support.process_spaces(os.path.dirname(corrected_dataset_yaml_filename) + "/"))
if "assembly" in cfg and os.path.isfile(result_contigs_filename):
message = " * Assembled contigs are in " + support.process_spaces(result_contigs_filename)
log.info(message)
if "assembly" in cfg and os.path.isfile(result_scaffolds_filename):
message = " * Assembled scaffolds are in " + support.process_spaces(result_scaffolds_filename)
log.info(message)
if "assembly" in cfg and os.path.isfile(result_assembly_graph_filename):
message = " * Assembly graph is in " + support.process_spaces(result_assembly_graph_filename)
log.info(message)
#log.info("")
#breaking scaffolds
if os.path.isfile(result_scaffolds_filename):
if not os.path.isdir(misc_dir):
os.makedirs(misc_dir)
result_broken_scaffolds = os.path.join(misc_dir, "broken_scaffolds.fasta")
if not os.path.isfile(result_broken_scaffolds) or not options_storage.continue_mode:
modified, broken_scaffolds = support.break_scaffolds(result_scaffolds_filename,
options_storage.THRESHOLD_FOR_BREAKING_SCAFFOLDS)
if modified:
support.write_fasta(result_broken_scaffolds, broken_scaffolds)
#log.info(" * Scaffolds broken by " + str(options_storage.THRESHOLD_FOR_BREAKING_SCAFFOLDS) +
# " Ns are in " + result_broken_scaffolds)
### printing WARNINGS SUMMARY
if not support.log_warnings(log):
log.info("\n======= SPAdes pipeline finished.") # otherwise it finished WITH WARNINGS
if options_storage.test_mode:
if options_storage.truseq_mode:
if not os.path.isfile(truseq_long_reads_file):
support.error("TEST FAILED: %s does not exist!" % truseq_long_reads_file)
else:
for result_filename in [result_contigs_filename, result_scaffolds_filename]:
if os.path.isfile(result_filename):
result_fasta = list(support.read_fasta(result_filename))
# correctness check: should be one contig of length 1000 bp
correct_number = 1
correct_length = 1000
if not len(result_fasta):
support.error("TEST FAILED: %s does not contain contigs!" % result_filename)
elif len(result_fasta) > correct_number:
support.error("TEST FAILED: %s contains more than %d contig (%d)!" %
(result_filename, correct_number, len(result_fasta)))
elif len(result_fasta[0][1]) != correct_length:
if len(result_fasta[0][1]) > correct_length:
relation = "more"
else:
relation = "less"
support.error("TEST FAILED: %s contains %s than %d bp (%d bp)!" %
(result_filename, relation, correct_length, len(result_fasta[0][1])))
else:
support.error("TEST FAILED: " + result_filename + " does not exist!")
log.info("\n========= TEST PASSED CORRECTLY.")
log.info("\nSPAdes log can be found here: " + log_filename)
log.info("")
log.info("Thank you for using SPAdes!")
log.removeHandler(log_handler)
except Exception:
exc_type, exc_value, _ = sys.exc_info()
if exc_type == SystemExit:
sys.exit(exc_value)
else:
if exc_type == OSError and exc_value.errno == errno.ENOEXEC: # Exec format error
support.error("It looks like you are using SPAdes binaries for another platform.\n" +
support.get_spades_binaries_info_message())
else:
log.exception(exc_value)
support.error("exception caught: %s" % exc_type, log)
except BaseException: # since python 2.5 system-exiting exceptions (e.g. KeyboardInterrupt) are derived from BaseException
exc_type, exc_value, _ = sys.exc_info()
if exc_type == SystemExit:
sys.exit(exc_value)
else:
log.exception(exc_value)
support.error("exception caught: %s" % exc_type, log)
def print_used_values(cfg, log):
def print_value(cfg, section, param, pretty_param="", margin=" "):
if not pretty_param:
pretty_param = param.capitalize().replace('_', ' ')
line = margin + pretty_param
if param in cfg[section].__dict__:
line += ": " + str(cfg[section].__dict__[param])
else:
if "offset" in param:
line += " will be auto-detected"
log.info(line)
log.info("")
# system info
log.info("System information:")
try:
log.info(" SPAdes version: " + str(spades_version).strip())
log.info(" Python version: " + ".".join(map(str, sys.version_info[0:3])))
# for more details: '[' + str(sys.version_info) + ']'
log.info(" OS: " + platform.platform())
# for more details: '[' + str(platform.uname()) + ']'
except Exception:
log.info(" Problem occurred when getting system information")
log.info("")
# main
print_value(cfg, "common", "output_dir", "", "")
if ("error_correction" in cfg) and (not "assembly" in cfg):
log.info("Mode: ONLY read error correction (without assembling)")
elif (not "error_correction" in cfg) and ("assembly" in cfg):
log.info("Mode: ONLY assembling (without read error correction)")
else:
log.info("Mode: read error correction and assembling")
if ("common" in cfg) and ("developer_mode" in cfg["common"].__dict__):
if cfg["common"].developer_mode:
log.info("Debug mode is turned ON")
else:
log.info("Debug mode is turned OFF")
log.info("")
# dataset
if "dataset" in cfg:
log.info("Dataset parameters:")
if options_storage.args.iontorrent:
log.info(" IonTorrent data")
if options_storage.args.bio:
log.info(" BiosyntheticSPAdes mode")
if options_storage.args.meta:
log.info(" Metagenomic mode")
elif options_storage.args.large_genome:
log.info(" Large genome mode")
elif options_storage.args.truseq_mode:
log.info(" Illumina TruSeq mode")
elif options_storage.args.isolate:
log.info(" Isolate mode")
elif options_storage.args.rna:
log.info(" RNA-seq mode")
elif options_storage.args.single_cell:
log.info(" Single-cell mode")
else:
log.info(" Standard mode")
log.info(" For multi-cell/isolate data we recommend to use '--isolate' option;" \
" for single-cell MDA data use '--sc';" \
" for metagenomic data use '--meta';" \
" for RNA-Seq use '--rna'.")
log.info(" Reads:")
dataset_data = pyyaml.load(open(cfg["dataset"].yaml_filename))
dataset_data = support.relative2abs_paths(dataset_data, os.path.dirname(cfg["dataset"].yaml_filename))
support.pretty_print_reads(dataset_data, log)
# error correction
if "error_correction" in cfg:
log.info("Read error correction parameters:")
print_value(cfg, "error_correction", "max_iterations", "Iterations")
print_value(cfg, "error_correction", "qvoffset", "PHRED offset")
if cfg["error_correction"].gzip_output:
log.info(" Corrected reads will be compressed")
else:
log.info(" Corrected reads will NOT be compressed")
# assembly
if "assembly" in cfg:
log.info("Assembly parameters:")
if options_storage.auto_K_allowed():
log.info(" k: automatic selection based on read length")
else:
print_value(cfg, "assembly", "iterative_K", "k")
if options_storage.args.plasmid:
log.info(" Plasmid mode is turned ON")
if cfg["assembly"].disable_rr:
log.info(" Repeat resolution is DISABLED")
else:
log.info(" Repeat resolution is enabled")
if options_storage.args.careful:
log.info(" Mismatch careful mode is turned ON")
else:
log.info(" Mismatch careful mode is turned OFF")
if "mismatch_corrector" in cfg:
log.info(" MismatchCorrector will be used")
else:
log.info(" MismatchCorrector will be SKIPPED")
if cfg["assembly"].cov_cutoff == "off":
log.info(" Coverage cutoff is turned OFF")
elif cfg["assembly"].cov_cutoff == "auto":
log.info(" Coverage cutoff is turned ON and threshold will be auto-detected")
else:
log.info(" Coverage cutoff is turned ON and threshold is %f" % cfg["assembly"].cov_cutoff)
log.info("Other parameters:")
print_value(cfg, "common", "tmp_dir", "Dir for temp files")
print_value(cfg, "common", "max_threads", "Threads")
print_value(cfg, "common", "max_memory", "Memory limit (in Gb)", " ")
log.info("")
def main(args):
os.environ["LC_ALL"] = "C"
if len(args) == 1:
options_storage.usage(spades_version)
sys.exit(0)
log = logging.getLogger('spades')
log.setLevel(logging.DEBUG)
console = logging.StreamHandler(sys.stdout)
console.setFormatter(logging.Formatter('%(message)s'))
console.setLevel(logging.DEBUG)
log.addHandler(console)
support.check_binaries(bin_home, log)
# parse options and safe all parameters to cfg
options = args
cfg, dataset_data = fill_cfg(options, log)
if options_storage.continue_mode:
cmd_line, options = get_options_from_params(
os.path.join(options_storage.output_dir, "params.txt"), args[0])
if not options:
support.error(
"failed to parse command line of the previous run! Please restart from the beginning or specify another output directory."
)
cfg, dataset_data = fill_cfg(options, log)
if options_storage.restart_from:
check_cfg_for_restart_from(cfg)
options_storage.continue_mode = True
log_filename = os.path.join(cfg["common"].output_dir, "spades.log")
if options_storage.continue_mode:
log_handler = logging.FileHandler(log_filename, mode='a')
else:
log_handler = logging.FileHandler(log_filename, mode='w')
log.addHandler(log_handler)
if options_storage.continue_mode:
log.info(
"\n======= SPAdes pipeline continued. Log can be found here: " +
log_filename + "\n")
log.info("Restored from " + cmd_line)
if options_storage.restart_from:
updated_params = ""
flag = False
for v in args[1:]:
if v == '-o' or v == '--restart-from':
flag = True
continue
if flag:
flag = False
continue
updated_params += " " + v
updated_params = updated_params.strip()
log.info("with updated parameters: " + updated_params)
cmd_line += " " + updated_params
log.info("")
params_filename = os.path.join(cfg["common"].output_dir, "params.txt")
params_handler = logging.FileHandler(params_filename, mode='w')
log.addHandler(params_handler)
if options_storage.continue_mode:
log.info(cmd_line)
else:
command = "Command line:"
for v in args:
command += " " + v
log.info(command)
# special case
# if "mismatch_corrector" in cfg and not support.get_lib_ids_by_type(dataset_data, 'paired-end'):
# support.warning('cannot perform mismatch correction without at least one paired-end library! Skipping this step.', log)
# del cfg["mismatch_corrector"]
print_used_values(cfg, log)
log.removeHandler(params_handler)
support.check_single_reads_in_options(options, log)
if not options_storage.continue_mode:
log.info("\n======= SPAdes pipeline started. Log can be found here: " +
log_filename + "\n")
# splitting interlaced reads and processing Ns in additional contigs if needed
if support.dataset_has_interlaced_reads(dataset_data) or support.dataset_has_additional_contigs(dataset_data)\
or support.dataset_has_nxmate_reads(dataset_data):
dir_for_split_reads = os.path.join(options_storage.output_dir,
'split_input')
if support.dataset_has_interlaced_reads(
dataset_data) or support.dataset_has_nxmate_reads(
dataset_data):
if not os.path.isdir(dir_for_split_reads):
os.makedirs(dir_for_split_reads)
if support.dataset_has_interlaced_reads(dataset_data):
dataset_data = support.split_interlaced_reads(
dataset_data, dir_for_split_reads, log)
if support.dataset_has_nxmate_reads(dataset_data):
dataset_data = support.process_nxmate_reads(
dataset_data, dir_for_split_reads, log)
if support.dataset_has_additional_contigs(dataset_data):
dataset_data = support.process_Ns_in_additional_contigs(
dataset_data, dir_for_split_reads, log)
options_storage.dataset_yaml_filename = os.path.join(
options_storage.output_dir, "input_dataset.yaml")
pyyaml.dump(dataset_data,
open(options_storage.dataset_yaml_filename, 'w'))
cfg["dataset"].yaml_filename = options_storage.dataset_yaml_filename
try:
# copying configs before all computations (to prevent its changing at run time)
tmp_configs_dir = os.path.join(cfg["common"].output_dir, "configs")
if os.path.isdir(
tmp_configs_dir) and not options_storage.continue_mode:
shutil.rmtree(tmp_configs_dir)
if not os.path.isdir(tmp_configs_dir):
if options_storage.configs_dir:
dir_util.copy_tree(options_storage.configs_dir,
tmp_configs_dir,
preserve_times=False)
else:
dir_util.copy_tree(os.path.join(spades_home, "configs"),
tmp_configs_dir,
preserve_times=False)
corrected_dataset_yaml_filename = ''
if "error_correction" in cfg:
STAGE_NAME = "Read error correction"
bh_cfg = merge_configs(cfg["error_correction"], cfg["common"])
corrected_dataset_yaml_filename = os.path.join(
bh_cfg.output_dir, "corrected.yaml")
if os.path.isfile(corrected_dataset_yaml_filename) and options_storage.continue_mode \
and not options_storage.restart_from == "ec":
log.info("\n===== Skipping %s (already processed). \n" %
STAGE_NAME)
else:
support.continue_from_here(log)
if "HEAPCHECK" in os.environ:
del os.environ["HEAPCHECK"]
if "heap_check" in bh_cfg.__dict__:
os.environ["HEAPCHECK"] = bh_cfg.heap_check
if os.path.exists(bh_cfg.output_dir):
shutil.rmtree(bh_cfg.output_dir)
os.makedirs(bh_cfg.output_dir)
if support.get_lib_ids_by_type(
dataset_data, options_storage.LONG_READS_TYPES):
not_used_dataset_data = support.get_libs_by_type(
dataset_data, options_storage.LONG_READS_TYPES)
to_correct_dataset_data = support.rm_libs_by_type(
dataset_data, options_storage.LONG_READS_TYPES)
to_correct_dataset_yaml_filename = os.path.join(
bh_cfg.output_dir, "to_correct.yaml")
pyyaml.dump(to_correct_dataset_data,
open(to_correct_dataset_yaml_filename, 'w'))
bh_cfg.__dict__[
"dataset_yaml_filename"] = to_correct_dataset_yaml_filename
else:
not_used_dataset_data = None
bh_cfg.__dict__["dataset_yaml_filename"] = cfg[
"dataset"].yaml_filename
log.info("\n===== %s started. \n" % STAGE_NAME)
hammer_logic.run_hammer(corrected_dataset_yaml_filename,
tmp_configs_dir, bin_home, bh_cfg,
not_used_dataset_data,
ext_python_modules_home, log)
log.info("\n===== %s finished. \n" % STAGE_NAME)
result_contigs_filename = os.path.join(cfg["common"].output_dir,
"contigs.fasta")
result_scaffolds_filename = os.path.join(cfg["common"].output_dir,
"scaffolds.fasta")
misc_dir = os.path.join(cfg["common"].output_dir, "misc")
### if mismatch correction is enabled then result contigs are copied to misc directory
assembled_contigs_filename = os.path.join(misc_dir,
"assembled_contigs.fasta")
assembled_scaffolds_filename = os.path.join(
misc_dir, "assembled_scaffolds.fasta")
if "assembly" in cfg:
STAGE_NAME = "Assembling"
spades_cfg = merge_configs(cfg["assembly"], cfg["common"])
spades_cfg.__dict__["result_contigs"] = result_contigs_filename
spades_cfg.__dict__["result_scaffolds"] = result_scaffolds_filename
if options_storage.continue_mode and (os.path.isfile(spades_cfg.result_contigs)
or ("mismatch_corrector" in cfg and
os.path.isfile(assembled_contigs_filename)))\
and not options_storage.restart_from == 'as' \
and not (options_storage.restart_from and options_storage.restart_from.startswith('k')):
log.info("\n===== Skipping %s (already processed). \n" %
STAGE_NAME)
# calculating latest_dir for the next stages
latest_dir = support.get_latest_dir(
os.path.join(spades_cfg.output_dir, "K*"))
if not latest_dir:
support.error(
"failed to continue the previous run! Please restart from previous stages or from the beginning.",
log)
else:
old_result_files = [
result_contigs_filename, result_scaffolds_filename,
assembled_contigs_filename, assembled_scaffolds_filename
]
for format in [".fasta", ".fastg"]:
for old_result_file in old_result_files:
if os.path.isfile(old_result_file[:-6] + format):
os.remove(old_result_file[:-6] + format)
if options_storage.restart_from == 'as':
support.continue_from_here(log)
if os.path.isfile(corrected_dataset_yaml_filename):
dataset_data = pyyaml.load(
open(corrected_dataset_yaml_filename, 'r'))
dataset_data = support.relative2abs_paths(
dataset_data,
os.path.dirname(corrected_dataset_yaml_filename))
if spades_cfg.disable_rr:
spades_cfg.__dict__["rr_enable"] = False
else:
spades_cfg.__dict__["rr_enable"] = True
if "HEAPCHECK" in os.environ:
del os.environ["HEAPCHECK"]
if "heap_check" in spades_cfg.__dict__:
os.environ["HEAPCHECK"] = spades_cfg.heap_check
log.info("\n===== %s started.\n" % STAGE_NAME)
# creating dataset
dataset_filename = os.path.join(spades_cfg.output_dir,
"dataset.info")
if not os.path.isfile(
dataset_filename) or not options_storage.continue_mode:
dataset_file = open(dataset_filename, 'w')
import process_cfg
dataset_file.write(
"single_cell" + '\t' +
process_cfg.bool_to_str(cfg["dataset"].single_cell) +
'\n')
if os.path.isfile(corrected_dataset_yaml_filename):
dataset_file.write(
"reads" + '\t' + process_cfg.process_spaces(
corrected_dataset_yaml_filename) + '\n')
else:
dataset_file.write("reads" + '\t' +
process_cfg.process_spaces(
cfg["dataset"].yaml_filename) +
'\n')
if spades_cfg.developer_mode and "reference" in cfg[
"dataset"].__dict__:
dataset_file.write("reference_genome" + '\t')
dataset_file.write(
process_cfg.process_spaces(
cfg["dataset"].reference) + '\n')
dataset_file.close()
spades_cfg.__dict__["dataset"] = dataset_filename
latest_dir = spades_logic.run_spades(tmp_configs_dir, bin_home,
spades_cfg, dataset_data,
ext_python_modules_home,
log)
if os.path.isdir(
misc_dir) and not options_storage.continue_mode:
shutil.rmtree(misc_dir)
if not os.path.isdir(misc_dir):
os.makedirs(misc_dir)
if options_storage.continue_mode and options_storage.restart_from and options_storage.restart_from.startswith(
'k'):
k_str = options_storage.restart_from[1:]
if k_str.find(":") != -1:
k_str = k_str[:k_str.find(":")]
support.error(
"failed to continue from K=%s because this K was not processed in the original run!"
% k_str, log)
log.info("\n===== %s finished. \n" % STAGE_NAME)
#corrector
if "mismatch_corrector" in cfg and (
os.path.isfile(result_contigs_filename) or
(options_storage.continue_mode
and os.path.isfile(assembled_contigs_filename))):
STAGE_NAME = "Mismatch correction"
to_correct = dict()
to_correct["contigs"] = (result_contigs_filename,
assembled_contigs_filename)
if os.path.isfile(result_scaffolds_filename) or (
options_storage.continue_mode
and os.path.isfile(assembled_scaffolds_filename)):
to_correct["scaffolds"] = (result_scaffolds_filename,
assembled_scaffolds_filename)
# moving assembled contigs (scaffolds) to misc dir
for assembly_type, (old, new) in to_correct.items():
if options_storage.continue_mode and os.path.isfile(new):
continue
for format in [".fasta", ".fastg"]:
if os.path.isfile(old[:-6] + format):
shutil.move(old[:-6] + format, new[:-6] + format)
if options_storage.continue_mode and os.path.isfile(result_contigs_filename) and \
(os.path.isfile(result_scaffolds_filename) or not os.path.isfile(assembled_scaffolds_filename)) \
and not options_storage.restart_from == 'mc':
log.info("\n===== Skipping %s (already processed). \n" %
STAGE_NAME)
else:
if options_storage.restart_from == 'mc':
support.continue_from_here(log)
log.info("\n===== %s started." % STAGE_NAME)
# detecting paired-end library with the largest insert size
cfg["mismatch_corrector"].__dict__["dataset"] = cfg[
"dataset"].yaml_filename
#TODO: add reads orientation
import corrector_logic
corrector_cfg = cfg["mismatch_corrector"]
# processing contigs and scaffolds (or only contigs)
for assembly_type, (corrected,
assembled) in to_correct.items():
if options_storage.continue_mode and os.path.isfile(
corrected):
log.info("\n== Skipping processing of " +
assembly_type + " (already processed)\n")
continue
support.continue_from_here(log)
log.info("\n== Processing of " + assembly_type + "\n")
tmp_dir_for_corrector = os.path.join(
cfg["common"].output_dir, "mismatch_corrector",
assembly_type)
cfg["mismatch_corrector"].__dict__[
"output_dir"] = tmp_dir_for_corrector
# correcting
corr_cfg = merge_configs(cfg["mismatch_corrector"],
cfg["common"])
result_corrected_filename = os.path.join(
tmp_dir_for_corrector, "corrected_contigs.fasta")
corrector_logic.run_corrector(
tmp_configs_dir, bin_home, corr_cfg,
ext_python_modules_home, log, assembled,
result_corrected_filename)
if os.path.isfile(result_corrected_filename):
shutil.copyfile(result_corrected_filename,
corrected)
tmp_d = os.path.join(tmp_dir_for_corrector, "tmp")
if os.path.isdir(
tmp_d) and not cfg["common"].developer_mode:
shutil.rmtree(tmp_d)
assembled_fastg = assembled[:-6] + ".fastg"
if os.path.isfile(assembled_fastg):
support.create_fastg_from_fasta(
corrected, assembled_fastg, log)
log.info("\n===== %s finished.\n" % STAGE_NAME)
if not cfg["common"].developer_mode and os.path.isdir(tmp_configs_dir):
shutil.rmtree(tmp_configs_dir)
#log.info("")
if "error_correction" in cfg and os.path.isdir(
os.path.dirname(corrected_dataset_yaml_filename)):
log.info(" * Corrected reads are in " + support.process_spaces(
os.path.dirname(corrected_dataset_yaml_filename) + "/"))
if "assembly" in cfg and os.path.isfile(result_contigs_filename):
message = " * Assembled contigs are in " + support.process_spaces(
result_contigs_filename)
if os.path.isfile(result_contigs_filename[:-6] + ".fastg"):
message += " (" + os.path.basename(
result_contigs_filename[:-6] + ".fastg") + ")"
log.info(message)
if "assembly" in cfg and os.path.isfile(result_scaffolds_filename):
message = " * Assembled scaffolds are in " + support.process_spaces(
result_scaffolds_filename)
if os.path.isfile(result_scaffolds_filename[:-6] + ".fastg"):
message += " (" + os.path.basename(
result_scaffolds_filename[:-6] + ".fastg") + ")"
log.info(message)
#log.info("")
#breaking scaffolds
if os.path.isfile(result_scaffolds_filename):
if not os.path.isdir(misc_dir):
os.makedirs(misc_dir)
result_broken_scaffolds = os.path.join(misc_dir,
"broken_scaffolds.fasta")
if not os.path.isfile(result_broken_scaffolds
) or not options_storage.continue_mode:
modified, broken_scaffolds = support.break_scaffolds(
result_scaffolds_filename,
options_storage.THRESHOLD_FOR_BREAKING_SCAFFOLDS)
if modified:
support.write_fasta(result_broken_scaffolds,
broken_scaffolds)
#log.info(" * Scaffolds broken by " + str(options_storage.THRESHOLD_FOR_BREAKING_SCAFFOLDS) +
# " Ns are in " + result_broken_scaffolds)
### printing WARNINGS SUMMARY
if not support.log_warnings(log):
log.info("\n======= SPAdes pipeline finished."
) # otherwise it finished WITH WARNINGS
if options_storage.test_mode:
for result_filename in [
result_contigs_filename, result_scaffolds_filename
]:
if os.path.isfile(result_filename):
result_fasta = list(support.read_fasta(result_filename))
# correctness check: should be one contig of length 1000 bp
correct_number = 1
correct_length = 1000
if not len(result_fasta):
support.error(
"TEST FAILED: %s does not contain contigs!" %
result_filename)
elif len(result_fasta) > correct_number:
support.error(
"TEST FAILED: %s contains more than %d contig (%d)!"
% (result_filename, correct_number,
len(result_fasta)))
elif len(result_fasta[0][1]) != correct_length:
if len(result_fasta[0][1]) > correct_length:
relation = "more"
else:
relation = "less"
support.error(
"TEST FAILED: %s contains %s than %d bp (%d bp)!" %
(result_filename, relation, correct_length,
len(result_fasta[0][1])))
else:
support.error("TEST FAILED: " + result_filename +
" does not exist!")
log.info("\n========= TEST PASSED CORRECTLY.")
log.info("\nSPAdes log can be found here: " + log_filename)
log.info("")
log.info("Thank you for using SPAdes!")
log.removeHandler(log_handler)
except Exception:
exc_type, exc_value, _ = sys.exc_info()
if exc_type == SystemExit:
sys.exit(exc_value)
else:
if exc_type == OSError and exc_value.errno == errno.ENOEXEC: # Exec format error
support.error(
"It looks like you are using SPAdes binaries for another platform.\n"
+ support.get_spades_binaries_info_message())
else:
log.exception(exc_value)
support.error("exception caught: %s" % exc_type, log)
except BaseException: # since python 2.5 system-exiting exceptions (e.g. KeyboardInterrupt) are derived from BaseException
exc_type, exc_value, _ = sys.exc_info()
if exc_type == SystemExit:
sys.exit(exc_value)
else:
log.exception(exc_value)
support.error("exception caught: %s" % exc_type, log)
def fill_cfg(options_to_parse, log):
try:
options, not_options = getopt.gnu_getopt(options_to_parse,
options_storage.short_options,
options_storage.long_options)
except getopt.GetoptError:
_, exc, _ = sys.exc_info()
sys.stderr.write(str(exc) + "\n")
sys.stderr.flush()
options_storage.usage(spades_version)
sys.exit(1)
if not options:
options_storage.usage(spades_version)
sys.exit(1)
if len(not_options) > 1:
for opt, arg in options:
if opt == "-k" and arg.strip().endswith(','):
support.error(
"Do not put spaces after commas in the list of k-mers sizes! Correct example: -k 21,33,55",
log)
support.error(
"Please specify option (e.g. -1, -2, -s, etc) for the following paths: "
+ ", ".join(not_options[1:]) + "\n", log)
# all parameters are stored here
cfg = dict()
# dataset is stored here. We are prepared for up to MAX_LIBS_NUMBER for each type of short-reads libs
dataset_data = [
{} for i in range(options_storage.MAX_LIBS_NUMBER *
len(options_storage.SHORT_READS_TYPES.keys()) +
len(options_storage.LONG_READS_TYPES))
] # "[{}]*num" doesn't work here!
# for parsing options from "previous run command"
options_storage.continue_mode = False
options_storage.k_mers = None
for opt, arg in options:
if opt == '-o':
options_storage.output_dir = os.path.abspath(arg)
elif opt == "--tmp-dir":
options_storage.tmp_dir = os.path.abspath(arg)
elif opt == "--configs-dir":
options_storage.configs_dir = support.check_dir_existence(arg)
elif opt == "--reference":
options_storage.reference = support.check_file_existence(
arg, 'reference', log)
elif opt == "--dataset":
options_storage.dataset_yaml_filename = support.check_file_existence(
arg, 'dataset', log)
elif opt in options_storage.reads_options:
support.add_to_dataset(opt, arg, dataset_data)
elif opt == '-k':
if arg == 'auto':
options_storage.k_mers = arg
else:
options_storage.k_mers = list(map(int, arg.split(",")))
for k in options_storage.k_mers:
if k < options_storage.MIN_K or k > options_storage.MAX_K:
support.error(
'wrong k value ' + str(k) +
': all k values should be between %d and %d' %
(options_storage.MIN_K, options_storage.MAX_K),
log)
if k % 2 == 0:
support.error(
'wrong k value ' + str(k) +
': all k values should be odd', log)
elif opt == "--sc":
options_storage.single_cell = True
elif opt == "--iontorrent":
options_storage.iontorrent = True
elif opt == "--disable-gzip-output":
options_storage.disable_gzip_output = True
elif opt == "--disable-gzip-output:false":
options_storage.disable_gzip_output = False
elif opt == "--disable-rr":
options_storage.disable_rr = True
elif opt == "--disable-rr:false":
options_storage.disable_rr = False
elif opt == "--only-error-correction":
if options_storage.only_assembler:
support.error(
'you cannot specify --only-error-correction and --only-assembler simultaneously'
)
options_storage.only_error_correction = True
elif opt == "--only-assembler":
if options_storage.only_error_correction:
support.error(
'you cannot specify --only-error-correction and --only-assembler simultaneously'
)
options_storage.only_assembler = True
elif opt == "--read-buffer-size":
options_storage.read_buffer_size = int(arg)
elif opt == "--bh-heap-check":
options_storage.bh_heap_check = arg
elif opt == "--spades-heap-check":
options_storage.spades_heap_check = arg
elif opt == "--continue":
options_storage.continue_mode = True
elif opt == "--restart-from":
if arg not in ['ec', 'as', 'mc'] and not arg.startswith('k'):
support.error(
"wrong value for --restart-from option: " + arg +
" (should be 'ec', 'as', 'k<int>', or 'mc'", log)
options_storage.continue_mode = True
options_storage.restart_from = arg
elif opt == '-t' or opt == "--threads":
options_storage.threads = int(arg)
elif opt == '-m' or opt == "--memory":
options_storage.memory = int(arg)
elif opt == "--phred-offset":
if arg == 'auto':
options_storage.qvoffset = arg
elif arg in ['33', '64']:
options_storage.qvoffset = int(arg)
else:
support.error(
'wrong PHRED quality offset value: ' + arg +
' (should be either 33, 64, or \'auto\')', log)
elif opt == "--cov-cutoff":
if arg == 'auto' or arg == 'off':
options_storage.cov_cutoff = arg
elif support.is_float(arg) and float(arg) > 0.0:
options_storage.cov_cutoff = float(arg)
else:
support.error(
'wrong value for --cov-cutoff option: ' + arg +
' (should be a positive float number, or \'auto\', or \'off\')',
log)
elif opt == '-i' or opt == "--iterations":
options_storage.iterations = int(arg)
elif opt == "--debug":
options_storage.developer_mode = True
elif opt == "--debug:false":
options_storage.developer_mode = False
#corrector
elif opt == "--mismatch-correction":
options_storage.mismatch_corrector = True
elif opt == "--mismatch-correction:false":
options_storage.mismatch_corrector = False
elif opt == "--careful":
options_storage.mismatch_corrector = True
options_storage.careful = True
elif opt == "--careful:false":
options_storage.mismatch_corrector = False
options_storage.careful = False
elif opt == '-h' or opt == "--help":
options_storage.usage(spades_version)
sys.exit(0)
elif opt == "--help-hidden":
options_storage.usage(spades_version, True)
sys.exit(0)
elif opt == "--test":
options_storage.set_test_options()
support.add_to_dataset(
'-1', os.path.join(spades_home,
"test_dataset/ecoli_1K_1.fq.gz"),
dataset_data)
support.add_to_dataset(
'-2', os.path.join(spades_home,
"test_dataset/ecoli_1K_2.fq.gz"),
dataset_data)
#break
elif opt == "--diploid":
options_storage.diploid_mode = True
else:
raise ValueError
if not options_storage.output_dir:
support.error(
"the output_dir is not set! It is a mandatory parameter (-o output_dir).",
log)
if not os.path.isdir(options_storage.output_dir):
if options_storage.continue_mode:
support.error(
"the output_dir should exist for --continue and for --restart-from!",
log)
os.makedirs(options_storage.output_dir)
if options_storage.restart_from:
if options_storage.continue_mode: # saving parameters specified with --restart-from
if not support.dataset_is_empty(dataset_data):
support.error(
"you cannot specify reads with --restart-from option!",
log)
options_storage.save_restart_options(log)
else: # overriding previous run parameters
options_storage.load_restart_options()
if options_storage.continue_mode:
return None, None
if options_storage.dataset_yaml_filename:
try:
dataset_data = pyyaml.load(
open(options_storage.dataset_yaml_filename, 'r'))
except pyyaml.YAMLError:
_, exc, _ = sys.exc_info()
support.error('exception caught while parsing YAML file (' +
options_storage.dataset_yaml_filename + '):\n' +
str(exc))
dataset_data = support.relative2abs_paths(
dataset_data,
os.path.dirname(options_storage.dataset_yaml_filename))
else:
dataset_data = support.correct_dataset(dataset_data)
dataset_data = support.relative2abs_paths(dataset_data, os.getcwd())
options_storage.dataset_yaml_filename = os.path.join(
options_storage.output_dir, "input_dataset.yaml")
pyyaml.dump(dataset_data,
open(options_storage.dataset_yaml_filename, 'w'))
support.check_dataset_reads(dataset_data, options_storage.only_assembler,
log)
if not support.get_lib_ids_by_type(
dataset_data, spades_logic.READS_TYPES_USED_IN_CONSTRUCTION):
support.error(
'you should specify at least one unpaired, paired-end, or high-quality mate-pairs library!'
)
options_storage.set_default_values()
### FILLING cfg
cfg["common"] = empty_config()
cfg["dataset"] = empty_config()
if not options_storage.only_assembler:
cfg["error_correction"] = empty_config()
if not options_storage.only_error_correction:
cfg["assembly"] = empty_config()
# common
cfg["common"].__dict__["output_dir"] = options_storage.output_dir
cfg["common"].__dict__["tmp_dir"] = options_storage.tmp_dir
cfg["common"].__dict__["max_threads"] = options_storage.threads
cfg["common"].__dict__["max_memory"] = options_storage.memory
cfg["common"].__dict__["developer_mode"] = options_storage.developer_mode
# dataset section
cfg["dataset"].__dict__["single_cell"] = options_storage.single_cell
cfg["dataset"].__dict__["iontorrent"] = options_storage.iontorrent
cfg["dataset"].__dict__[
"yaml_filename"] = options_storage.dataset_yaml_filename
if options_storage.developer_mode and options_storage.reference:
cfg["dataset"].__dict__["reference"] = options_storage.reference
# error correction
if (not options_storage.only_assembler) and (options_storage.iterations >
0):
cfg["error_correction"].__dict__["output_dir"] = os.path.join(
cfg["common"].output_dir, "corrected")
cfg["error_correction"].__dict__[
"max_iterations"] = options_storage.iterations
cfg["error_correction"].__dict__[
"gzip_output"] = not options_storage.disable_gzip_output
if options_storage.qvoffset:
cfg["error_correction"].__dict__[
"qvoffset"] = options_storage.qvoffset
if options_storage.bh_heap_check:
cfg["error_correction"].__dict__[
"heap_check"] = options_storage.bh_heap_check
cfg["error_correction"].__dict__[
"iontorrent"] = options_storage.iontorrent
# assembly
if not options_storage.only_error_correction:
if options_storage.k_mers:
cfg["assembly"].__dict__["iterative_K"] = options_storage.k_mers
else:
cfg["assembly"].__dict__[
"iterative_K"] = options_storage.K_MERS_SHORT
cfg["assembly"].__dict__["careful"] = options_storage.careful
cfg["assembly"].__dict__["disable_rr"] = options_storage.disable_rr
cfg["assembly"].__dict__["diploid_mode"] = options_storage.diploid_mode
cfg["assembly"].__dict__["cov_cutoff"] = options_storage.cov_cutoff
if options_storage.spades_heap_check:
cfg["assembly"].__dict__[
"heap_check"] = options_storage.spades_heap_check
if options_storage.read_buffer_size:
cfg["assembly"].__dict__[
"read_buffer_size"] = options_storage.read_buffer_size
#corrector can work only if contigs exist (not only error correction)
if (not options_storage.only_error_correction
) and options_storage.mismatch_corrector:
cfg["mismatch_corrector"] = empty_config()
cfg["mismatch_corrector"].__dict__["skip-masked"] = None
cfg["mismatch_corrector"].__dict__["bwa"] = os.path.join(
bin_home, "bwa-spades")
cfg["mismatch_corrector"].__dict__["threads"] = options_storage.threads
cfg["mismatch_corrector"].__dict__[
"output-dir"] = options_storage.output_dir
return cfg, dataset_data
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if make_latest_symlink:
latest_symlink = 'latest'
if os.path.islink(latest_symlink):
os.remove(latest_symlink)
os.symlink(output_dir, latest_symlink)
datasets_dict = dict()
print("Analyzing datasets")
for dataset in datasets:
try:
dataset_data = pyyaml.load(file(dataset, 'r'))
except pyyaml.YAMLError, exc:
support.warning('skipping ' + dataset +
': exception caught while parsing YAML file (' +
options_storage.dataset_yaml_filename + '):\n' +
str(exc))
continue
dataset_data = support.correct_dataset(dataset_data)
for id, library in enumerate(dataset_data):
print("processing lib#" + str(id) + " of " + dataset)
basename = os.path.splitext(os.path.basename(dataset))[0]
cur_key = basename
i = 1
while datasets_dict.has_key(cur_key):
cur_key = basename + "_" + str(i)
def run_hammer(corrected_dataset_yaml_filename, configs_dir, execution_home, cfg,
dataset_data, ext_python_modules_home, only_compressing_is_needed, log):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
# not all reads need processing
if support.get_lib_ids_by_type(dataset_data, options_storage.LONG_READS_TYPES):
not_used_dataset_data = support.get_libs_by_type(dataset_data, options_storage.LONG_READS_TYPES)
to_correct_dataset_data = support.rm_libs_by_type(dataset_data, options_storage.LONG_READS_TYPES)
to_correct_dataset_yaml_filename = os.path.join(cfg.output_dir, "to_correct.yaml")
pyyaml.dump(to_correct_dataset_data, open(to_correct_dataset_yaml_filename, 'w'), default_flow_style = False, default_style='"', width=100500)
cfg.dataset_yaml_filename = to_correct_dataset_yaml_filename
else:
not_used_dataset_data = None
if not only_compressing_is_needed:
dst_configs = os.path.join(cfg.output_dir, "configs")
if os.path.exists(dst_configs):
shutil.rmtree(dst_configs)
if cfg.iontorrent:
dir_util.copy_tree(os.path.join(configs_dir, "ionhammer"), dst_configs, preserve_times=False)
cfg_file_name = os.path.join(dst_configs, "ionhammer.cfg")
else:
dir_util.copy_tree(os.path.join(configs_dir, "hammer"), dst_configs, preserve_times=False)
cfg_file_name = os.path.join(dst_configs, "config.info")
cfg.tmp_dir = support.get_tmp_dir(prefix="hammer_")
if cfg.iontorrent:
prepare_config_ih(cfg_file_name, cfg, ext_python_modules_home)
binary_name = "ionhammer"
else:
prepare_config_bh(cfg_file_name, cfg, log)
binary_name = "hammer"
command = [os.path.join(execution_home, binary_name),
os.path.abspath(cfg_file_name)]
log.info("\n== Running read error correction tool: " + ' '.join(command) + "\n")
support.sys_call(command, log)
if not os.path.isfile(corrected_dataset_yaml_filename):
support.error("read error correction finished abnormally: " + corrected_dataset_yaml_filename + " not found!")
else:
log.info("\n===== Skipping %s (already processed). \n" % "read error correction tool")
support.continue_from_here(log)
corrected_dataset_data = pyyaml.load(open(corrected_dataset_yaml_filename, 'r'))
remove_not_corrected_reads(cfg.output_dir)
is_changed = False
if cfg.gzip_output:
is_changed = True
compress_dataset_files(corrected_dataset_data, ext_python_modules_home, cfg.max_threads, log)
if not_used_dataset_data:
is_changed = True
corrected_dataset_data += not_used_dataset_data
if is_changed:
pyyaml.dump(corrected_dataset_data, open(corrected_dataset_yaml_filename, 'w'), default_flow_style = False, default_style='"', width=100500)
log.info("\n== Dataset description file was created: " + corrected_dataset_yaml_filename + "\n")
if os.path.isdir(cfg.tmp_dir):
shutil.rmtree(cfg.tmp_dir)
def main(args):
os.environ["LC_ALL"] = "C"
if len(args) == 1:
options_storage.usage(spades_version)
sys.exit(0)
log = logging.getLogger('spades')
log.setLevel(logging.DEBUG)
console = logging.StreamHandler(sys.stdout)
console.setFormatter(logging.Formatter('%(message)s'))
console.setLevel(logging.DEBUG)
log.addHandler(console)
support.check_binaries(bin_home, log)
# parse options and safe all parameters to cfg
cfg, dataset_data = fill_cfg(args, log)
if options_storage.continue_mode:
cmd_line, options = get_options_from_params(os.path.join(options_storage.output_dir, "params.txt"))
if not options:
support.error("failed to parse command line of the previous run! Please restart from the beginning or specify another output directory.")
cfg, dataset_data = fill_cfg(options, log)
if options_storage.restart_from:
check_cfg_for_restart_from(cfg)
options_storage.continue_mode = True
log_filename = os.path.join(cfg["common"].output_dir, "spades.log")
if options_storage.continue_mode:
log_handler = logging.FileHandler(log_filename, mode='a')
else:
log_handler = logging.FileHandler(log_filename, mode='w')
log.addHandler(log_handler)
if options_storage.continue_mode:
log.info("\n======= SPAdes pipeline continued. Log can be found here: " + log_filename + "\n")
log.info("Restored from " + cmd_line)
if options_storage.restart_from:
updated_params = ""
flag = False
for v in args[1:]:
if v == '-o' or v == '--restart-from':
flag = True
continue
if flag:
flag = False
continue
updated_params += " " + v
updated_params = updated_params.strip()
log.info("with updated parameters: " + updated_params)
cmd_line += " " + updated_params
log.info("")
params_filename = os.path.join(cfg["common"].output_dir, "params.txt")
params_handler = logging.FileHandler(params_filename, mode='w')
log.addHandler(params_handler)
if options_storage.continue_mode:
log.info(cmd_line)
else:
command = "Command line:"
for v in args:
command += " " + v
log.info(command)
# special case
if "mismatch_corrector" in cfg and not support.get_lib_ids_by_type(dataset_data, 'paired-end'):
support.warning('cannot perform mismatch correction without at least one paired-end library! Skipping this step.', log)
del cfg["mismatch_corrector"]
print_used_values(cfg, log)
log.removeHandler(params_handler)
if not options_storage.continue_mode:
log.info("\n======= SPAdes pipeline started. Log can be found here: " + log_filename + "\n")
# splitting interlaced reads and processing Ns in additional contigs if needed
if support.dataset_has_interlaced_reads(dataset_data) or support.dataset_has_additional_contigs(dataset_data):
dir_for_split_reads = os.path.join(options_storage.output_dir, 'split_input')
if support.dataset_has_interlaced_reads(dataset_data):
if not os.path.isdir(dir_for_split_reads):
os.makedirs(dir_for_split_reads)
dataset_data = support.split_interlaced_reads(dataset_data, dir_for_split_reads, log)
if support.dataset_has_additional_contigs(dataset_data):
dataset_data = support.process_Ns_in_additional_contigs(dataset_data, dir_for_split_reads, log)
options_storage.dataset_yaml_filename = os.path.join(options_storage.output_dir, "input_dataset.yaml")
pyyaml.dump(dataset_data, open(options_storage.dataset_yaml_filename, 'w'))
cfg["dataset"].yaml_filename = options_storage.dataset_yaml_filename
try:
# copying configs before all computations (to prevent its changing at run time)
tmp_configs_dir = os.path.join(cfg["common"].output_dir, "configs")
if os.path.isdir(tmp_configs_dir) and not options_storage.continue_mode:
shutil.rmtree(tmp_configs_dir)
if not os.path.isdir(tmp_configs_dir):
dir_util.copy_tree(os.path.join(spades_home, "configs"), tmp_configs_dir, preserve_times=False)
corrected_dataset_yaml_filename = ''
if "error_correction" in cfg:
STAGE_NAME = "Read error correction"
bh_cfg = merge_configs(cfg["error_correction"], cfg["common"])
corrected_dataset_yaml_filename = os.path.join(bh_cfg.output_dir, "corrected.yaml")
if os.path.isfile(corrected_dataset_yaml_filename) and options_storage.continue_mode \
and not options_storage.restart_from == "ec":
log.info("\n===== Skipping %s (already processed). \n" % STAGE_NAME)
else:
support.continue_from_here(log)
if "HEAPCHECK" in os.environ:
del os.environ["HEAPCHECK"]
if "heap_check" in bh_cfg.__dict__:
os.environ["HEAPCHECK"] = bh_cfg.heap_check
if os.path.exists(bh_cfg.output_dir):
shutil.rmtree(bh_cfg.output_dir)
os.makedirs(bh_cfg.output_dir)
if support.get_lib_ids_by_type(dataset_data, options_storage.LONG_READS_TYPES):
not_used_dataset_data = support.get_libs_by_type(dataset_data, options_storage.LONG_READS_TYPES)
to_correct_dataset_data = support.rm_libs_by_type(dataset_data, options_storage.LONG_READS_TYPES)
to_correct_dataset_yaml_filename = os.path.join(bh_cfg.output_dir, "to_correct.yaml")
pyyaml.dump(to_correct_dataset_data, open(to_correct_dataset_yaml_filename, 'w'))
bh_cfg.__dict__["dataset_yaml_filename"] = to_correct_dataset_yaml_filename
else:
not_used_dataset_data = None
bh_cfg.__dict__["dataset_yaml_filename"] = cfg["dataset"].yaml_filename
log.info("\n===== %s started. \n" % STAGE_NAME)
hammer_logic.run_hammer(corrected_dataset_yaml_filename, tmp_configs_dir, bin_home, bh_cfg, not_used_dataset_data,
ext_python_modules_home, log)
log.info("\n===== %s finished. \n" % STAGE_NAME)
result_contigs_filename = os.path.join(cfg["common"].output_dir, "contigs.fasta")
result_scaffolds_filename = os.path.join(cfg["common"].output_dir, "scaffolds.fasta")
misc_dir = os.path.join(cfg["common"].output_dir, "misc")
### if mismatch correction is enabled then result contigs are copied to misc directory
assembled_contigs_filename = os.path.join(misc_dir, "assembled_contigs.fasta")
assembled_scaffolds_filename = os.path.join(misc_dir, "assembled_scaffolds.fasta")
if "assembly" in cfg:
STAGE_NAME = "Assembling"
spades_cfg = merge_configs(cfg["assembly"], cfg["common"])
spades_cfg.__dict__["result_contigs"] = result_contigs_filename
spades_cfg.__dict__["result_scaffolds"] = result_scaffolds_filename
if options_storage.continue_mode and (os.path.isfile(spades_cfg.result_contigs)
or ("mismatch_corrector" in cfg and
os.path.isfile(assembled_contigs_filename)))\
and not options_storage.restart_from == 'as' \
and not (options_storage.restart_from and options_storage.restart_from.startswith('k')):
log.info("\n===== Skipping %s (already processed). \n" % STAGE_NAME)
# calculating latest_dir for the next stages
latest_dir = support.get_latest_dir(os.path.join(spades_cfg.output_dir, "K*"))
if not latest_dir:
support.error("failed to continue the previous run! Please restart from previous stages or from the beginning.", log)
else:
old_result_files = [result_contigs_filename, result_scaffolds_filename,
assembled_contigs_filename, assembled_scaffolds_filename]
for format in [".fasta", ".fastg"]:
for old_result_file in old_result_files:
if os.path.isfile(old_result_file[:-6] + format):
os.remove(old_result_file[:-6] + format)
if options_storage.restart_from == 'as':
support.continue_from_here(log)
if os.path.isfile(corrected_dataset_yaml_filename):
dataset_data = pyyaml.load(open(corrected_dataset_yaml_filename, 'r'))
dataset_data = support.relative2abs_paths(dataset_data, os.path.dirname(corrected_dataset_yaml_filename))
if spades_cfg.disable_rr:
spades_cfg.__dict__["rr_enable"] = False
else:
spades_cfg.__dict__["rr_enable"] = True
if "HEAPCHECK" in os.environ:
del os.environ["HEAPCHECK"]
if "heap_check" in spades_cfg.__dict__:
os.environ["HEAPCHECK"] = spades_cfg.heap_check
log.info("\n===== %s started.\n" % STAGE_NAME)
# creating dataset
dataset_filename = os.path.join(spades_cfg.output_dir, "dataset.info")
if not os.path.isfile(dataset_filename) or not options_storage.continue_mode:
dataset_file = open(dataset_filename, 'w')
import process_cfg
dataset_file.write("single_cell" + '\t' + process_cfg.bool_to_str(cfg["dataset"].single_cell) + '\n')
if os.path.isfile(corrected_dataset_yaml_filename):
dataset_file.write("reads" + '\t' + process_cfg.process_spaces(corrected_dataset_yaml_filename) + '\n')
else:
dataset_file.write("reads" + '\t' + process_cfg.process_spaces(cfg["dataset"].yaml_filename) + '\n')
if spades_cfg.developer_mode and "reference" in cfg["dataset"].__dict__:
dataset_file.write("reference_genome" + '\t')
dataset_file.write(process_cfg.process_spaces(cfg["dataset"].reference) + '\n')
dataset_file.close()
spades_cfg.__dict__["dataset"] = dataset_filename
latest_dir = spades_logic.run_spades(tmp_configs_dir, bin_home, spades_cfg, dataset_data, ext_python_modules_home, log)
if os.path.isdir(misc_dir) and not options_storage.continue_mode:
shutil.rmtree(misc_dir)
if not os.path.isdir(misc_dir):
os.makedirs(misc_dir)
if options_storage.continue_mode and options_storage.restart_from and options_storage.restart_from.startswith('k'):
k_str = options_storage.restart_from[1:]
if k_str.find(":") != -1:
k_str = k_str[:k_str.find(":")]
support.error("failed to continue from K=%s because this K was not processed in the original run!" % k_str, log)
log.info("\n===== %s finished. \n" % STAGE_NAME)
#corrector
if "mismatch_corrector" in cfg and (os.path.isfile(result_contigs_filename) or
(options_storage.continue_mode and os.path.isfile(assembled_contigs_filename))):
STAGE_NAME = "Mismatch correction"
to_correct = dict()
to_correct["contigs"] = (result_contigs_filename, assembled_contigs_filename)
if os.path.isfile(result_scaffolds_filename) or (options_storage.continue_mode and
os.path.isfile(assembled_scaffolds_filename)):
to_correct["scaffolds"] = (result_scaffolds_filename, assembled_scaffolds_filename)
# moving assembled contigs (scaffolds) to misc dir
for assembly_type, (old, new) in to_correct.items():
if options_storage.continue_mode and os.path.isfile(new):
continue
for format in [".fasta", ".fastg"]:
if os.path.isfile(old[:-6] + format):
shutil.move(old[:-6] + format, new[:-6] + format)
if options_storage.continue_mode and os.path.isfile(result_contigs_filename) and \
(os.path.isfile(result_scaffolds_filename) or not os.path.isfile(assembled_scaffolds_filename)) \
and not options_storage.restart_from == 'mc':
log.info("\n===== Skipping %s (already processed). \n" % STAGE_NAME)
else:
if options_storage.restart_from == 'mc':
support.continue_from_here(log)
log.info("\n===== %s started." % STAGE_NAME)
# detecting paired-end library with the largest insert size
est_params_data = pyyaml.load(open(os.path.join(latest_dir, "final.lib_data"), 'r'))
max_IS_library = None
for reads_library in est_params_data:
if reads_library['type'] == 'paired-end':
if not max_IS_library or float(reads_library["insert size mean"]) > float(max_IS_library["insert size mean"]):
max_IS_library = reads_library
if not max_IS_library:
support.error('Mismatch correction cannot be performed without at least one paired-end library!', log)
if not max_IS_library["insert size mean"]:
support.warning('Failed to estimate insert size for all paired-end libraries. Starting Mismatch correction'
' based on the first paired-end library and with default insert size.', log)
else:
cfg["mismatch_corrector"].__dict__["insert-size"] = round(max_IS_library["insert size mean"])
yaml_dirname = os.path.dirname(options_storage.dataset_yaml_filename)
cfg["mismatch_corrector"].__dict__["1"] = list(map(lambda x: os.path.join(yaml_dirname, x),
max_IS_library['left reads']))
cfg["mismatch_corrector"].__dict__["2"] = list(map(lambda x: os.path.join(yaml_dirname, x),
max_IS_library['right reads']))
#TODO: add reads orientation
import corrector
corrector_cfg = cfg["mismatch_corrector"]
args = []
for key, values in corrector_cfg.__dict__.items():
if key == "output-dir":
continue
# for processing list of reads
if not isinstance(values, list):
values = [values]
for value in values:
if len(key) == 1:
args.append('-' + key)
else:
args.append('--' + key)
if value is not None:
args.append(value)
# processing contigs and scaffolds (or only contigs)
for assembly_type, (corrected, assembled) in to_correct.items():
if options_storage.continue_mode and os.path.isfile(corrected):
log.info("\n== Skipping processing of " + assembly_type + " (already processed)\n")
continue
support.continue_from_here(log)
log.info("\n== Processing of " + assembly_type + "\n")
cur_args = args[:]
cur_args += ['-c', assembled]
tmp_dir_for_corrector = support.get_tmp_dir(prefix="mis_cor_%s_" % assembly_type)
cur_args += ['--output-dir', tmp_dir_for_corrector]
# correcting
corrector.main(cur_args, ext_python_modules_home, log)
result_corrected_filename = os.path.join(tmp_dir_for_corrector, "corrected_contigs.fasta")
# moving corrected contigs (scaffolds) to SPAdes output dir
if os.path.isfile(result_corrected_filename):
shutil.move(result_corrected_filename, corrected)
if os.path.isdir(tmp_dir_for_corrector):
shutil.rmtree(tmp_dir_for_corrector)
assembled_fastg = assembled[:-6] + ".fastg"
if os.path.isfile(assembled_fastg):
support.create_fastg_from_fasta(corrected, assembled_fastg, log)
log.info("\n===== %s finished.\n" % STAGE_NAME)
if not cfg["common"].developer_mode and os.path.isdir(tmp_configs_dir):
shutil.rmtree(tmp_configs_dir)
#log.info("")
if "error_correction" in cfg and os.path.isdir(os.path.dirname(corrected_dataset_yaml_filename)):
log.info(" * Corrected reads are in " + support.process_spaces(os.path.dirname(corrected_dataset_yaml_filename) + "/"))
if "assembly" in cfg and os.path.isfile(result_contigs_filename):
message = " * Assembled contigs are in " + support.process_spaces(result_contigs_filename)
if os.path.isfile(result_contigs_filename[:-6] + ".fastg"):
message += " (" + os.path.basename(result_contigs_filename[:-6] + ".fastg") + ")"
log.info(message)
if "assembly" in cfg and os.path.isfile(result_scaffolds_filename):
message = " * Assembled scaffolds are in " + support.process_spaces(result_scaffolds_filename)
if os.path.isfile(result_scaffolds_filename[:-6] + ".fastg"):
message += " (" + os.path.basename(result_scaffolds_filename[:-6] + ".fastg") + ")"
log.info(message)
#log.info("")
#breaking scaffolds
if os.path.isfile(result_scaffolds_filename):
if not os.path.isdir(misc_dir):
os.makedirs(misc_dir)
result_broken_scaffolds = os.path.join(misc_dir, "broken_scaffolds.fasta")
if not os.path.isfile(result_broken_scaffolds) or not options_storage.continue_mode:
modified, broken_scaffolds = support.break_scaffolds(result_scaffolds_filename,
options_storage.THRESHOLD_FOR_BREAKING_SCAFFOLDS)
if modified:
support.write_fasta(result_broken_scaffolds, broken_scaffolds)
#log.info(" * Scaffolds broken by " + str(options_storage.THRESHOLD_FOR_BREAKING_SCAFFOLDS) +
# " Ns are in " + result_broken_scaffolds)
### printing WARNINGS SUMMARY
if not support.log_warnings(log):
log.info("\n======= SPAdes pipeline finished.") # otherwise it finished WITH WARNINGS
if options_storage.test_mode:
for result_filename in [result_contigs_filename, result_scaffolds_filename]:
if os.path.isfile(result_filename):
result_fasta = list(support.read_fasta(result_filename))
# correctness check: should be one contig of length 1000 bp
correct_number = 1
correct_length = 1000
if not len(result_fasta):
support.error("TEST FAILED: %s does not contain contigs!" % result_filename)
elif len(result_fasta) > correct_number:
support.error("TEST FAILED: %s contains more than %d contig (%d)!" %
(result_filename, correct_number, len(result_fasta)))
elif len(result_fasta[0][1]) != correct_length:
if len(result_fasta[0][1]) > correct_length:
relation = "more"
else:
relation = "less"
support.error("TEST FAILED: %s contains %s than %d bp (%d bp)!" %
(result_filename, relation, correct_length, len(result_fasta[0][1])))
else:
support.error("TEST FAILED: " + result_filename + " does not exist!")
log.info("\n========= TEST PASSED CORRECTLY.")
log.info("\nSPAdes log can be found here: " + log_filename)
log.info("")
log.info("Thank you for using SPAdes!")
log.removeHandler(log_handler)
except Exception:
exc_type, exc_value, _ = sys.exc_info()
if exc_type == SystemExit:
sys.exit(exc_value)
else:
if exc_type == OSError and exc_value.errno == errno.ENOEXEC: # Exec format error
support.error("It looks like you are using SPAdes binaries for another platform.\n" +
support.get_spades_binaries_info_message())
else:
log.exception(exc_value)
support.error("exception caught: %s" % exc_type, log)
except BaseException: # since python 2.5 system-exiting exceptions (e.g. KeyboardInterrupt) are derived from BaseException
exc_type, exc_value, _ = sys.exc_info()
if exc_type == SystemExit:
sys.exit(exc_value)
else:
log.exception(exc_value)
support.error("exception caught: %s" % exc_type, log)
def fill_cfg(options_to_parse, log):
try:
options, not_options = getopt.gnu_getopt(options_to_parse,
options_storage.short_options,
options_storage.long_options)
except getopt.GetoptError:
_, exc, _ = sys.exc_info()
sys.stderr.write(str(exc) + "\n")
sys.stderr.flush()
options_storage.usage(spades_version)
sys.exit(1)
if not options:
options_storage.usage(spades_version)
sys.exit(1)
# all parameters are stored here
cfg = dict()
# dataset is stored here. We are prepared for up to MAX_LIBS_NUMBER paired-end libs and MAX_LIBS_NUMBER mate-pair libs
dataset_data = [{} for i in range(options_storage.MAX_LIBS_NUMBER * 2)]
options_storage.continue_mode = False
for opt, arg in options:
if opt == '-o':
options_storage.output_dir = arg
elif opt == "--tmp-dir":
options_storage.tmp_dir = arg
elif opt == "--reference":
options_storage.reference = support.check_file_existence(
arg, 'reference', log)
elif opt == "--dataset":
options_storage.dataset_yaml_filename = support.check_file_existence(
arg, 'dataset', log)
elif opt in options_storage.reads_options:
support.add_to_dataset(opt, arg, dataset_data)
elif opt == '-k':
options_storage.k_mers = list(map(int, arg.split(",")))
for k in options_storage.k_mers:
if k > 127:
support.error(
'wrong k value ' + str(k) +
': all k values should be less than 128', log)
if k % 2 == 0:
support.error(
'wrong k value ' + str(k) +
': all k values should be odd', log)
elif opt == "--sc":
options_storage.single_cell = True
elif opt == "--disable-gzip-output":
options_storage.disable_gzip_output = True
elif opt == "--only-error-correction":
if options_storage.only_assembler:
support.error(
'you cannot specify --only-error-correction and --only-assembler simultaneously'
)
options_storage.only_error_correction = True
elif opt == "--only-assembler":
if options_storage.only_error_correction:
support.error(
'you cannot specify --only-error-correction and --only-assembler simultaneously'
)
options_storage.only_assembler = True
elif opt == "--bh-heap-check":
options_storage.bh_heap_check = arg
elif opt == "--spades-heap-check":
options_storage.spades_heap_check = arg
elif opt == "--continue":
options_storage.continue_mode = True
elif opt == '-t' or opt == "--threads":
options_storage.threads = int(arg)
elif opt == '-m' or opt == "--memory":
options_storage.memory = int(arg)
elif opt == "--phred-offset":
if int(arg) in [33, 64]:
options_storage.qvoffset = int(arg)
else:
support.error(
'wrong PHRED quality offset value ' + str(arg) +
': should be either 33 or 64', log)
elif opt == '-i' or opt == "--iterations":
options_storage.iterations = int(arg)
elif opt == "--debug":
options_storage.developer_mode = True
elif opt == "--rectangles":
options_storage.rectangles = True
#corrector
elif opt == "--mismatch-correction":
options_storage.mismatch_corrector = True
elif opt == "--careful":
options_storage.mismatch_corrector = True
options_storage.careful = True
elif opt == '-h' or opt == "--help":
options_storage.usage(spades_version)
sys.exit(0)
elif opt == "--help-hidden":
options_storage.usage(spades_version, True)
sys.exit(0)
elif opt == "--test":
options_storage.set_test_options()
support.add_to_dataset(
'-1', os.path.join(spades_home,
"test_dataset/ecoli_1K_1.fq.gz"),
dataset_data)
support.add_to_dataset(
'-2', os.path.join(spades_home,
"test_dataset/ecoli_1K_2.fq.gz"),
dataset_data)
#break
else:
raise ValueError
if not options_storage.output_dir:
support.error(
"the output_dir is not set! It is a mandatory parameter (-o output_dir).",
log)
if not os.path.isdir(options_storage.output_dir):
if options_storage.continue_mode:
support.error("the output_dir should exist for --continue!", log)
os.makedirs(options_storage.output_dir)
if options_storage.continue_mode:
return None, None
if options_storage.dataset_yaml_filename:
try:
dataset_data = pyyaml.load(
open(options_storage.dataset_yaml_filename, 'r'))
except pyyaml.YAMLError:
_, exc, _ = sys.exc_info()
support.error('exception caught while parsing YAML file (' +
options_storage.dataset_yaml_filename + '):\n' +
str(exc))
dataset_data = support.relative2abs_paths(
dataset_data,
os.path.dirname(options_storage.dataset_yaml_filename))
else:
dataset_data = support.correct_dataset(dataset_data)
dataset_data = support.relative2abs_paths(dataset_data, os.getcwd())
options_storage.dataset_yaml_filename = os.path.join(
options_storage.output_dir, "input_dataset.yaml")
pyyaml.dump(dataset_data,
open(options_storage.dataset_yaml_filename, 'w'))
support.check_dataset_reads(dataset_data, options_storage.only_assembler,
log)
if support.dataset_has_only_mate_pairs_libraries(dataset_data):
support.error(
'you should specify at least one paired-end or unpaired library (only mate-pairs libraries were found)!'
)
if options_storage.rectangles and (len(dataset_data) > 1):
support.error(
'rectangle graph algorithm for repeat resolution cannot work with multiple libraries!'
)
### FILLING cfg
cfg["common"] = empty_config()
cfg["dataset"] = empty_config()
if not options_storage.only_assembler:
cfg["error_correction"] = empty_config()
if not options_storage.only_error_correction:
cfg["assembly"] = empty_config()
# common
cfg["common"].__dict__["output_dir"] = os.path.abspath(
options_storage.output_dir)
cfg["common"].__dict__["max_threads"] = options_storage.threads
cfg["common"].__dict__["max_memory"] = options_storage.memory
cfg["common"].__dict__["developer_mode"] = options_storage.developer_mode
# dataset section
cfg["dataset"].__dict__["single_cell"] = options_storage.single_cell
cfg["dataset"].__dict__["yaml_filename"] = os.path.abspath(
options_storage.dataset_yaml_filename)
if options_storage.developer_mode and options_storage.reference:
cfg["dataset"].__dict__["reference"] = options_storage.reference
# error correction
if (not options_storage.only_assembler) and (options_storage.iterations >
0):
cfg["error_correction"].__dict__["output_dir"] = os.path.join(
cfg["common"].output_dir, "corrected")
cfg["error_correction"].__dict__[
"max_iterations"] = options_storage.iterations
cfg["error_correction"].__dict__[
"gzip_output"] = not options_storage.disable_gzip_output
if options_storage.qvoffset:
cfg["error_correction"].__dict__[
"qvoffset"] = options_storage.qvoffset
if options_storage.bh_heap_check:
cfg["error_correction"].__dict__[
"heap_check"] = options_storage.bh_heap_check
if options_storage.tmp_dir:
cfg["error_correction"].__dict__[
"tmp_dir"] = options_storage.tmp_dir
else:
cfg["error_correction"].__dict__["tmp_dir"] = cfg[
"error_correction"].output_dir
cfg["error_correction"].tmp_dir = os.path.join(
os.path.abspath(cfg["error_correction"].tmp_dir), 'tmp')
# assembly
if not options_storage.only_error_correction:
if options_storage.k_mers:
cfg["assembly"].__dict__["iterative_K"] = options_storage.k_mers
else:
cfg["assembly"].__dict__[
"iterative_K"] = options_storage.k_mers_short
cfg["assembly"].__dict__["careful"] = options_storage.careful
if options_storage.spades_heap_check:
cfg["assembly"].__dict__[
"heap_check"] = options_storage.spades_heap_check
#corrector can work only if contigs exist (not only error correction)
if (not options_storage.only_error_correction
) and options_storage.mismatch_corrector:
cfg["mismatch_corrector"] = empty_config()
cfg["mismatch_corrector"].__dict__["skip-masked"] = ""
cfg["mismatch_corrector"].__dict__["bwa"] = os.path.join(
bin_home, "bwa-spades")
cfg["mismatch_corrector"].__dict__["threads"] = options_storage.threads
cfg["mismatch_corrector"].__dict__[
"output-dir"] = options_storage.output_dir
return cfg, dataset_data
def print_used_values(cfg, log):
def print_value(cfg, section, param, pretty_param="", margin=" "):
if not pretty_param:
pretty_param = param.capitalize().replace('_', ' ')
line = margin + pretty_param
if param in cfg[section].__dict__:
line += ": " + str(cfg[section].__dict__[param])
else:
if param.find("offset") != -1:
line += " will be auto-detected"
log.info(line)
log.info("")
# system info
log.info("System information:")
try:
log.info(" SPAdes version: " + str(spades_version).strip())
log.info(" Python version: " + ".".join(map(str, sys.version_info[0:3])))
# for more details: '[' + str(sys.version_info) + ']'
log.info(" OS: " + platform.platform())
# for more deatils: '[' + str(platform.uname()) + ']'
except Exception:
log.info(" Problem occurred when getting system information")
log.info("")
# main
print_value(cfg, "common", "output_dir", "", "")
if ("error_correction" in cfg) and (not "assembly" in cfg):
log.info("Mode: ONLY read error correction (without assembling)")
elif (not "error_correction" in cfg) and ("assembly" in cfg):
log.info("Mode: ONLY assembling (without read error correction)")
else:
log.info("Mode: read error correction and assembling")
if ("common" in cfg) and ("developer_mode" in cfg["common"].__dict__):
if cfg["common"].developer_mode:
log.info("Debug mode is turned ON")
else:
log.info("Debug mode is turned OFF")
log.info("")
# dataset
if "dataset" in cfg:
log.info("Dataset parameters:")
if cfg["dataset"].single_cell:
log.info(" Single-cell mode")
else:
log.info(" Multi-cell mode (you should set '--sc' flag if input data"\
" was obtained with MDA (single-cell) technology")
if cfg["dataset"].iontorrent:
log.info(" IonTorrent data")
log.info(" Reads:")
dataset_data = pyyaml.load(open(cfg["dataset"].yaml_filename, 'r'))
dataset_data = support.relative2abs_paths(dataset_data, os.path.dirname(cfg["dataset"].yaml_filename))
support.pretty_print_reads(dataset_data, log)
# error correction
if "error_correction" in cfg:
log.info("Read error correction parameters:")
print_value(cfg, "error_correction", "max_iterations", "Iterations")
print_value(cfg, "error_correction", "qvoffset", "PHRED offset")
if cfg["error_correction"].gzip_output:
log.info(" Corrected reads will be compressed (with gzip)")
else:
log.info(" Corrected reads will NOT be compressed (with gzip)")
# assembly
if "assembly" in cfg:
log.info("Assembly parameters:")
if options_storage.auto_K_allowed():
log.info(" k: automatic selection based on read length")
else:
print_value(cfg, "assembly", "iterative_K", "k")
if cfg["assembly"].careful:
log.info(" Mismatch careful mode is turned ON")
else:
log.info(" Mismatch careful mode is turned OFF")
if cfg["assembly"].disable_rr:
log.info(" Repeat resolution is DISABLED")
else:
log.info(" Repeat resolution is enabled")
if "mismatch_corrector" in cfg:
log.info(" MismatchCorrector will be used")
else:
log.info(" MismatchCorrector will be SKIPPED")
log.info("Other parameters:")
print_value(cfg, "common", "tmp_dir", "Dir for temp files")
print_value(cfg, "common", "max_threads", "Threads")
print_value(cfg, "common", "max_memory", "Memory limit (in Gb)", " ")
log.info("")
def fill_cfg(options_to_parse, log):
try:
options, not_options = getopt.gnu_getopt(options_to_parse, options_storage.short_options, options_storage.long_options)
except getopt.GetoptError:
_, exc, _ = sys.exc_info()
sys.stderr.write(str(exc) + "\n")
sys.stderr.flush()
options_storage.usage(spades_version)
sys.exit(1)
if not options:
options_storage.usage(spades_version)
sys.exit(1)
if len(not_options) > 1:
for opt, arg in options:
if opt == "-k" and arg.strip().endswith(','):
support.error("Do not put spaces after commas in the list of k-mers sizes! Correct example: -k 21,33,55", log)
support.error("Please specify option (e.g. -1, -2, -s, etc) for the following paths: " + ", ".join(not_options[1:]) + "\n", log)
# all parameters are stored here
cfg = dict()
# dataset is stored here. We are prepared for up to MAX_LIBS_NUMBER for each type of short-reads libs
dataset_data = [{} for i in range(options_storage.MAX_LIBS_NUMBER * len(options_storage.SHORT_READS_TYPES.keys()))] # "[{}] * num" doesn't work here!
# for parsing options from "previous run command"
options_storage.continue_mode = False
options_storage.k_mers = None
for opt, arg in options:
if opt == '-o':
options_storage.output_dir = os.path.abspath(arg)
elif opt == "--tmp-dir":
options_storage.tmp_dir = os.path.abspath(arg)
elif opt == "--reference":
options_storage.reference = support.check_file_existence(arg, 'reference', log)
elif opt == "--dataset":
options_storage.dataset_yaml_filename = support.check_file_existence(arg, 'dataset', log)
elif opt in options_storage.reads_options:
support.add_to_dataset(opt, arg, dataset_data)
elif opt == '-k':
if arg == 'auto':
options_storage.k_mers = arg
else:
options_storage.k_mers = list(map(int, arg.split(",")))
for k in options_storage.k_mers:
if k < options_storage.MIN_K or k > options_storage.MAX_K:
support.error('wrong k value ' + str(k) + ': all k values should be between %d and %d' %
(options_storage.MIN_K, options_storage.MAX_K), log)
if k % 2 == 0:
support.error('wrong k value ' + str(k) + ': all k values should be odd', log)
elif opt == "--sc":
options_storage.single_cell = True
elif opt == "--iontorrent":
options_storage.iontorrent = True
elif opt == "--disable-gzip-output":
options_storage.disable_gzip_output = True
elif opt == "--disable-gzip-output:false":
options_storage.disable_gzip_output = False
elif opt == "--disable-rr":
options_storage.disable_rr = True
elif opt == "--disable-rr:false":
options_storage.disable_rr = False
elif opt == "--only-error-correction":
if options_storage.only_assembler:
support.error('you cannot specify --only-error-correction and --only-assembler simultaneously')
options_storage.only_error_correction = True
elif opt == "--only-assembler":
if options_storage.only_error_correction:
support.error('you cannot specify --only-error-correction and --only-assembler simultaneously')
options_storage.only_assembler = True
elif opt == "--read-buffer-size":
options_storage.read_buffer_size = int(arg)
elif opt == "--bh-heap-check":
options_storage.bh_heap_check = arg
elif opt == "--spades-heap-check":
options_storage.spades_heap_check = arg
elif opt == "--continue":
options_storage.continue_mode = True
elif opt == "--restart-from":
if arg not in ['ec', 'as', 'mc'] and not arg.startswith('k'):
support.error("wrong value for --restart-from option: " + arg + " (only 'ec', 'as', 'k<int>', 'mc' are available)", log)
options_storage.continue_mode = True
options_storage.restart_from = arg
elif opt == '-t' or opt == "--threads":
options_storage.threads = int(arg)
elif opt == '-m' or opt == "--memory":
options_storage.memory = int(arg)
elif opt == "--phred-offset":
if arg == 'auto':
options_storage.qvoffset = arg
elif arg in ['33', '64']:
options_storage.qvoffset = int(arg)
else:
support.error('wrong PHRED quality offset value ' + str(arg) + ': should be either 33 or 64', log)
elif opt == '-i' or opt == "--iterations":
options_storage.iterations = int(arg)
elif opt == "--debug":
options_storage.developer_mode = True
elif opt == "--debug:false":
options_storage.developer_mode = False
#corrector
elif opt == "--mismatch-correction":
options_storage.mismatch_corrector = True
elif opt == "--mismatch-correction:false":
options_storage.mismatch_corrector = False
elif opt == "--careful":
options_storage.mismatch_corrector = True
options_storage.careful = True
elif opt == "--careful:false":
options_storage.mismatch_corrector = False
options_storage.careful = False
elif opt == '-h' or opt == "--help":
options_storage.usage(spades_version)
sys.exit(0)
elif opt == "--help-hidden":
options_storage.usage(spades_version, True)
sys.exit(0)
elif opt == "--test":
options_storage.set_test_options()
support.add_to_dataset('-1', os.path.join(spades_home, "test_dataset/ecoli_1K_1.fq.gz"), dataset_data)
support.add_to_dataset('-2', os.path.join(spades_home, "test_dataset/ecoli_1K_2.fq.gz"), dataset_data)
#break
elif opt == "--diploid":
options_storage.diploid_mode = True
else:
raise ValueError
if not options_storage.output_dir:
support.error("the output_dir is not set! It is a mandatory parameter (-o output_dir).", log)
if not os.path.isdir(options_storage.output_dir):
if options_storage.continue_mode:
support.error("the output_dir should exist for --continue and for --restart-from!", log)
os.makedirs(options_storage.output_dir)
if options_storage.restart_from:
if options_storage.continue_mode: # saving parameters specified with --restart-from
if not support.dataset_is_empty(dataset_data):
support.error("you cannot specify reads with --restart-from option!", log)
options_storage.save_restart_options(log)
else: # overriding previous run parameters
options_storage.load_restart_options()
if options_storage.continue_mode:
return None, None
if options_storage.dataset_yaml_filename:
try:
dataset_data = pyyaml.load(open(options_storage.dataset_yaml_filename, 'r'))
except pyyaml.YAMLError:
_, exc, _ = sys.exc_info()
support.error('exception caught while parsing YAML file (' + options_storage.dataset_yaml_filename + '):\n' + str(exc))
dataset_data = support.relative2abs_paths(dataset_data, os.path.dirname(options_storage.dataset_yaml_filename))
else:
dataset_data = support.correct_dataset(dataset_data)
dataset_data = support.relative2abs_paths(dataset_data, os.getcwd())
options_storage.dataset_yaml_filename = os.path.join(options_storage.output_dir, "input_dataset.yaml")
pyyaml.dump(dataset_data, open(options_storage.dataset_yaml_filename, 'w'))
support.check_dataset_reads(dataset_data, options_storage.only_assembler, log)
if not support.get_lib_ids_by_type(dataset_data, spades_logic.READS_TYPES_USED_IN_CONSTRUCTION):
support.error('you should specify at least one unpaired, paired-end, or high-quality mate-pairs library!')
options_storage.set_default_values()
### FILLING cfg
cfg["common"] = empty_config()
cfg["dataset"] = empty_config()
if not options_storage.only_assembler:
cfg["error_correction"] = empty_config()
if not options_storage.only_error_correction:
cfg["assembly"] = empty_config()
# common
cfg["common"].__dict__["output_dir"] = options_storage.output_dir
cfg["common"].__dict__["tmp_dir"] = options_storage.tmp_dir
cfg["common"].__dict__["max_threads"] = options_storage.threads
cfg["common"].__dict__["max_memory"] = options_storage.memory
cfg["common"].__dict__["developer_mode"] = options_storage.developer_mode
# dataset section
cfg["dataset"].__dict__["single_cell"] = options_storage.single_cell
cfg["dataset"].__dict__["iontorrent"] = options_storage.iontorrent
cfg["dataset"].__dict__["yaml_filename"] = options_storage.dataset_yaml_filename
if options_storage.developer_mode and options_storage.reference:
cfg["dataset"].__dict__["reference"] = options_storage.reference
# error correction
if (not options_storage.only_assembler) and (options_storage.iterations > 0):
cfg["error_correction"].__dict__["output_dir"] = os.path.join(cfg["common"].output_dir, "corrected")
cfg["error_correction"].__dict__["max_iterations"] = options_storage.iterations
cfg["error_correction"].__dict__["gzip_output"] = not options_storage.disable_gzip_output
if options_storage.qvoffset:
cfg["error_correction"].__dict__["qvoffset"] = options_storage.qvoffset
if options_storage.bh_heap_check:
cfg["error_correction"].__dict__["heap_check"] = options_storage.bh_heap_check
cfg["error_correction"].__dict__["iontorrent"] = options_storage.iontorrent
# assembly
if not options_storage.only_error_correction:
if options_storage.k_mers:
cfg["assembly"].__dict__["iterative_K"] = options_storage.k_mers
else:
cfg["assembly"].__dict__["iterative_K"] = options_storage.K_MERS_SHORT
cfg["assembly"].__dict__["careful"] = options_storage.careful
cfg["assembly"].__dict__["disable_rr"] = options_storage.disable_rr
cfg["assembly"].__dict__["diploid_mode"] = options_storage.diploid_mode
if options_storage.spades_heap_check:
cfg["assembly"].__dict__["heap_check"] = options_storage.spades_heap_check
if options_storage.read_buffer_size:
cfg["assembly"].__dict__["read_buffer_size"] = options_storage.read_buffer_size
#corrector can work only if contigs exist (not only error correction)
if (not options_storage.only_error_correction) and options_storage.mismatch_corrector:
cfg["mismatch_corrector"] = empty_config()
cfg["mismatch_corrector"].__dict__["skip-masked"] = None
cfg["mismatch_corrector"].__dict__["bwa"] = os.path.join(bin_home, "bwa-spades")
cfg["mismatch_corrector"].__dict__["threads"] = options_storage.threads
cfg["mismatch_corrector"].__dict__["output-dir"] = options_storage.output_dir
return cfg, dataset_data
def compress_dataset_files(input_file, ext_python_modules_home, max_threads,
log, not_used_yaml_file, output_dir, gzip_output):
addsitedir(ext_python_modules_home)
if sys.version.startswith("2."):
import pyyaml2 as pyyaml
from joblib2 import Parallel, delayed
elif sys.version.startswith("3."):
import pyyaml3 as pyyaml
from joblib3 import Parallel, delayed
dataset_data = pyyaml.load(open(input_file))
remove_not_corrected_reads(output_dir)
is_changed = False
if gzip_output:
is_changed = True
pigz_path = support.which("pigz")
if pigz_path:
compressor = "pigz"
else:
compressor = "gzip"
log.info("\n== Compressing corrected reads (with %s)" % compressor)
to_compress = []
for reads_library in dataset_data:
for key, value in reads_library.items():
if key.endswith("reads"):
compressed_reads_filenames = []
for reads_file in value:
compressed_reads_filenames.append(reads_file + ".gz")
to_compress.append(reads_file)
reads_library[key] = compressed_reads_filenames
if len(to_compress):
for reads_file in to_compress:
if not isfile(reads_file):
support.error(
"something went wrong and file with corrected reads (%s) is missing!"
% reads_file, log)
if pigz_path:
for reads_file in to_compress:
support.sys_call([
pigz_path, "-f", "-7", "-p",
str(max_threads), reads_file
], log)
else:
n_jobs = min(len(to_compress), max_threads)
outputs = Parallel(n_jobs=n_jobs)(
delayed(support.sys_call)(["gzip", "-f", "-7", reads_file])
for reads_file in to_compress)
for output in outputs:
if output:
log.info(output)
if not_used_yaml_file != "":
is_changed = True
not_used_dataset_data = pyyaml.load(open(not_used_yaml_file))
dataset_data += not_used_dataset_data
if is_changed:
with open(input_file, 'w') as f:
pyyaml.dump(dataset_data,
f,
default_flow_style=False,
default_style='"',
width=float("inf"))
def main():
os.environ["LC_ALL"] = "C"
if len(sys.argv) == 1:
options_storage.usage(spades_version)
sys.exit(0)
log = logging.getLogger('spades')
log.setLevel(logging.DEBUG)
console = logging.StreamHandler(sys.stdout)
console.setFormatter(logging.Formatter('%(message)s'))
console.setLevel(logging.DEBUG)
log.addHandler(console)
check_binaries(bin_home, log)
# parse options and safe all parameters to cfg
cfg, dataset_data = fill_cfg(sys.argv, log)
if options_storage.continue_mode:
cmd_line, options = get_options_from_params(os.path.join(options_storage.output_dir, "params.txt"))
if not options:
support.error("failed to parse command line of the previous run! Please restart from the beginning.")
cfg, dataset_data = fill_cfg(options, log)
options_storage.continue_mode = True
log_filename = os.path.join(cfg["common"].output_dir, "spades.log")
if options_storage.continue_mode:
log_handler = logging.FileHandler(log_filename, mode='a')
else:
log_handler = logging.FileHandler(log_filename, mode='w')
log.addHandler(log_handler)
if options_storage.continue_mode:
log.info("\n======= SPAdes pipeline continued. Log can be found here: " + log_filename + "\n")
log.info("Restored from " + cmd_line)
else:
params_filename = os.path.join(cfg["common"].output_dir, "params.txt")
params_handler = logging.FileHandler(params_filename, mode='w')
log.addHandler(params_handler)
command = "Command line:"
for v in sys.argv:
command += " " + v
log.info(command)
print_used_values(cfg, log)
log.removeHandler(params_handler)
log.info("\n======= SPAdes pipeline started. Log can be found here: " + log_filename + "\n")
# splitting interlaced reads if needed
if support.dataset_has_interlaced_reads(dataset_data):
dir_for_split_reads = os.path.join(os.path.abspath(options_storage.output_dir), 'split_reads')
if not os.path.isdir(dir_for_split_reads):
os.makedirs(dir_for_split_reads)
dataset_data = support.split_interlaced_reads(dataset_data, dir_for_split_reads, log)
options_storage.dataset_yaml_filename = os.path.join(options_storage.output_dir, "input_dataset.yaml")
pyyaml.dump(dataset_data, open(options_storage.dataset_yaml_filename, 'w'))
cfg["dataset"].yaml_filename = os.path.abspath(options_storage.dataset_yaml_filename)
try:
# copying configs before all computations (to prevent its changing at run time)
tmp_configs_dir = os.path.join(cfg["common"].output_dir, "configs")
if os.path.isdir(tmp_configs_dir) and not options_storage.continue_mode:
shutil.rmtree(tmp_configs_dir)
if not os.path.isdir(tmp_configs_dir):
shutil.copytree(os.path.join(spades_home, "configs"), tmp_configs_dir)
corrected_dataset_yaml_filename = ''
if "error_correction" in cfg:
bh_cfg = merge_configs(cfg["error_correction"], cfg["common"])
bh_cfg.__dict__["dataset_yaml_filename"] = cfg["dataset"].yaml_filename
corrected_dataset_yaml_filename = os.path.join(bh_cfg.output_dir, "corrected.yaml")
if os.path.isfile(corrected_dataset_yaml_filename) and options_storage.continue_mode:
log.info("\n===== Skipping read error correction (already processed). \n")
else:
options_storage.continue_mode = False # continue from here
if "HEAPCHECK" in os.environ:
del os.environ["HEAPCHECK"]
if "heap_check" in bh_cfg.__dict__:
os.environ["HEAPCHECK"] = bh_cfg.heap_check
if os.path.exists(bh_cfg.output_dir):
shutil.rmtree(bh_cfg.output_dir)
os.makedirs(bh_cfg.output_dir)
if not os.path.exists(bh_cfg.tmp_dir):
os.makedirs(bh_cfg.tmp_dir)
log.info("\n===== Read error correction started. \n")
bh_logic.run_bh(corrected_dataset_yaml_filename, tmp_configs_dir, bin_home, bh_cfg,
ext_python_modules_home, log)
log.info("\n===== Read error correction finished. \n")
result_contigs_filename = os.path.join(cfg["common"].output_dir, "contigs.fasta")
result_scaffolds_filename = os.path.join(cfg["common"].output_dir, "scaffolds.fasta")
misc_dir = os.path.join(cfg["common"].output_dir, "misc")
### if mismatch correction is enabled then result contigs are copied to misc directory
assembled_contigs_filename = os.path.join(misc_dir, "assembled_contigs.fasta")
assembled_scaffolds_filename = os.path.join(misc_dir, "assembled_scaffolds.fasta")
if "assembly" in cfg:
spades_cfg = merge_configs(cfg["assembly"], cfg["common"])
spades_cfg.__dict__["result_contigs"] = result_contigs_filename
spades_cfg.__dict__["result_scaffolds"] = result_scaffolds_filename
spades_cfg.__dict__["additional_contigs"] = os.path.join(spades_cfg.output_dir, "simplified_contigs.fasta")
if options_storage.continue_mode and (os.path.isfile(spades_cfg.result_contigs)
or ("mismatch_corrector" in cfg and
os.path.isfile(assembled_contigs_filename))):
log.info("\n===== Skipping assembling (already processed). \n")
# calculating latest_dir for the next stages
latest_dir = support.get_latest_dir(os.path.join(spades_cfg.output_dir, "K*"))
if not latest_dir:
support.error("failed to continue the previous run! Please restart from the beginning.")
else:
if os.path.isfile(corrected_dataset_yaml_filename):
dataset_data = pyyaml.load(open(corrected_dataset_yaml_filename, 'r'))
dataset_data = support.relative2abs_paths(dataset_data, os.path.dirname(corrected_dataset_yaml_filename))
if support.dataset_has_paired_reads(dataset_data):
spades_cfg.__dict__["paired_mode"] = True
else:
spades_cfg.__dict__["paired_mode"] = False
if options_storage.rectangles:
spades_cfg.__dict__["resolving_mode"] = "rectangles"
if "HEAPCHECK" in os.environ:
del os.environ["HEAPCHECK"]
if "heap_check" in spades_cfg.__dict__:
os.environ["HEAPCHECK"] = spades_cfg.heap_check
log.info("\n===== Assembling started.\n")
# creating dataset
dataset_filename = os.path.join(spades_cfg.output_dir, "dataset.info")
if not os.path.isfile(dataset_filename) or not options_storage.continue_mode:
dataset_file = open(dataset_filename, 'w')
import process_cfg
dataset_file.write("single_cell" + '\t' + process_cfg.bool_to_str(cfg["dataset"].single_cell) + '\n')
if os.path.isfile(corrected_dataset_yaml_filename):
dataset_file.write("reads" + '\t' + process_cfg.process_spaces(corrected_dataset_yaml_filename) + '\n')
else:
dataset_file.write("reads" + '\t' + process_cfg.process_spaces(cfg["dataset"].yaml_filename) + '\n')
if spades_cfg.developer_mode and "reference" in cfg["dataset"].__dict__:
dataset_file.write("reference_genome" + '\t')
dataset_file.write(process_cfg.process_spaces(os.path.abspath(cfg["dataset"].reference)) + '\n')
dataset_file.close()
spades_cfg.__dict__["dataset"] = dataset_filename
latest_dir = spades_logic.run_spades(tmp_configs_dir, bin_home, spades_cfg, log)
#rectangles
if spades_cfg.paired_mode and options_storage.rectangles:
if options_storage.continue_mode: # TODO: continue mode
support.warning("sorry, --continue doesn't work with --rectangles yet. Skipping repeat resolving.")
else:
sys.path.append(os.path.join(python_modules_home, "rectangles"))
import rrr
rrr_input_dir = os.path.join(latest_dir, "saves")
rrr_outpath = os.path.join(spades_cfg.output_dir, "rectangles")
if not os.path.exists(rrr_outpath):
os.mkdir(rrr_outpath)
rrr_reference_information_file = os.path.join(rrr_input_dir,
"late_pair_info_counted_etalon_distance.txt")
rrr_test_util = rrr.TestUtils(rrr_reference_information_file,
os.path.join(rrr_outpath, "rectangles.log"))
rrr.resolve(rrr_input_dir, rrr_outpath, rrr_test_util, "", cfg["dataset"].single_cell, spades_cfg.careful)
shutil.copyfile(os.path.join(rrr_outpath, "rectangles_extend_before_scaffold.fasta"), spades_cfg.result_contigs)
shutil.copyfile(os.path.join(rrr_outpath, "rectangles_extend.fasta"), spades_cfg.result_scaffolds)
if not spades_cfg.developer_mode:
if os.path.exists(rrr_input_dir):
shutil.rmtree(rrr_input_dir)
if os.path.exists(rrr_outpath):
shutil.rmtree(rrr_outpath, True)
if os.path.exists(rrr_outpath):
os.system('rm -r ' + rrr_outpath)
#EOR
if os.path.isdir(misc_dir) and not options_storage.continue_mode:
shutil.rmtree(misc_dir)
if not os.path.isdir(misc_dir):
os.makedirs(misc_dir)
if os.path.isfile(spades_cfg.additional_contigs):
shutil.move(spades_cfg.additional_contigs, misc_dir)
log.info("\n===== Assembling finished. \n")
#corrector
if "mismatch_corrector" in cfg and (os.path.isfile(result_contigs_filename) or
(options_storage.continue_mode and os.path.isfile(assembled_contigs_filename))):
to_correct = dict()
to_correct["contigs"] = (result_contigs_filename, assembled_contigs_filename)
if os.path.isfile(result_scaffolds_filename) or (options_storage.continue_mode and
os.path.isfile(assembled_scaffolds_filename)):
to_correct["scaffolds"] = (result_scaffolds_filename, assembled_scaffolds_filename)
# moving assembled contigs (scaffolds) to misc dir
for k, (old, new) in to_correct.items():
if options_storage.continue_mode and os.path.isfile(new):
continue
shutil.move(old, new)
if options_storage.continue_mode and os.path.isfile(result_contigs_filename) and \
(os.path.isfile(result_scaffolds_filename) or not os.path.isfile(assembled_scaffolds_filename)):
log.info("\n===== Skipping mismatch correction (already processed). \n")
else:
log.info("\n===== Mismatch correction started.")
# detecting paired-end library with the largest insert size
dataset_data = pyyaml.load(open(options_storage.dataset_yaml_filename, 'r')) ### initial dataset, i.e. before error correction
dataset_data = support.relative2abs_paths(dataset_data, os.path.dirname(options_storage.dataset_yaml_filename))
paired_end_libraries_ids = []
for id, reads_library in enumerate(dataset_data):
if reads_library['type'] == 'paired-end':
paired_end_libraries_ids.append(id)
if not len(paired_end_libraries_ids):
support.error('Mismatch correction cannot be performed without at least one paired-end library!')
estimated_params = load_config_from_file(os.path.join(latest_dir, "_est_params.info"))
max_insert_size = -1
target_paired_end_library_id = -1
for id in paired_end_libraries_ids:
if float(estimated_params.__dict__["insert_size_" + str(id)]) > max_insert_size:
max_insert_size = float(estimated_params.__dict__["insert_size_" + str(id)])
target_paired_end_library_id = id
yaml_dirname = os.path.dirname(options_storage.dataset_yaml_filename)
cfg["mismatch_corrector"].__dict__["1"] = list(map(lambda x: os.path.join(yaml_dirname, x),
dataset_data[target_paired_end_library_id]['left reads']))
cfg["mismatch_corrector"].__dict__["2"] = list(map(lambda x: os.path.join(yaml_dirname, x),
dataset_data[target_paired_end_library_id]['right reads']))
cfg["mismatch_corrector"].__dict__["insert-size"] = round(max_insert_size)
#TODO: add reads orientation
import corrector
corrector_cfg = cfg["mismatch_corrector"]
args = []
for key, values in corrector_cfg.__dict__.items():
if key == "output-dir":
continue
# for processing list of reads
if not isinstance(values, list):
values = [values]
for value in values:
if len(key) == 1:
args.append('-' + key)
else:
args.append('--' + key)
if value:
args.append(value)
# processing contigs and scaffolds (or only contigs)
for k, (corrected, assembled) in to_correct.items():
if options_storage.continue_mode and os.path.isfile(corrected):
log.info("\n== Skipping processing of " + k + " (already processed)\n")
continue
options_storage.continue_mode = False
log.info("\n== Processing of " + k + "\n")
cur_args = args[:]
cur_args += ['-c', assembled]
tmp_dir_for_corrector = os.path.join(corrector_cfg.__dict__["output-dir"], "mismatch_corrector_" + k)
cur_args += ['--output-dir', tmp_dir_for_corrector]
# correcting
corrector.main(cur_args, ext_python_modules_home, log)
result_corrected_filename = os.path.abspath(os.path.join(tmp_dir_for_corrector, "corrected_contigs.fasta"))
# moving corrected contigs (scaffolds) to SPAdes output dir
if os.path.isfile(result_corrected_filename):
shutil.move(result_corrected_filename, corrected)
if os.path.isdir(tmp_dir_for_corrector):
shutil.rmtree(tmp_dir_for_corrector)
log.info("\n===== Mismatch correction finished.\n")
if not cfg["common"].developer_mode and os.path.isdir(tmp_configs_dir):
shutil.rmtree(tmp_configs_dir)
#log.info("")
if os.path.isdir(os.path.dirname(corrected_dataset_yaml_filename)):
log.info(" * Corrected reads are in " + os.path.dirname(corrected_dataset_yaml_filename) + "/")
if os.path.isfile(result_contigs_filename):
log.info(" * Assembled contigs are in " + result_contigs_filename)
if os.path.isfile(result_scaffolds_filename):
log.info(" * Assembled scaffolds are in " + result_scaffolds_filename)
#log.info("")
#breaking scaffolds
if os.path.isfile(result_scaffolds_filename):
if not os.path.isdir(misc_dir):
os.makedirs(misc_dir)
result_broken_scaffolds = os.path.join(misc_dir, "broken_scaffolds.fasta")
threshold = 3
if not os.path.isfile(result_broken_scaffolds) or not options_storage.continue_mode:
support.break_scaffolds(result_scaffolds_filename, threshold, result_broken_scaffolds)
#log.info(" * Scaffolds broken by " + str(threshold) + " Ns are in " + result_broken_scaffolds)
### printing WARNINGS SUMMARY
if not support.log_warnings(log):
log.info("\n======= SPAdes pipeline finished.") # otherwise it finished WITH WARNINGS
log.info("\nSPAdes log can be found here: " + log_filename)
log.info("")
log.info("Thank you for using SPAdes!")
log.removeHandler(log_handler)
except Exception:
_, exc, _ = sys.exc_info()
log.exception(exc)
support.error("exception caught", log)
def run_hammer(corrected_dataset_yaml_filename, configs_dir, execution_home,
cfg, dataset_data, ext_python_modules_home,
only_compressing_is_needed, log):
addsitedir(ext_python_modules_home)
if sys.version.startswith('2.'):
import pyyaml2 as pyyaml
elif sys.version.startswith('3.'):
import pyyaml3 as pyyaml
# not all reads need processing
if support.get_lib_ids_by_type(dataset_data,
options_storage.LONG_READS_TYPES):
not_used_dataset_data = support.get_libs_by_type(
dataset_data, options_storage.LONG_READS_TYPES)
to_correct_dataset_data = support.rm_libs_by_type(
dataset_data, options_storage.LONG_READS_TYPES)
to_correct_dataset_yaml_filename = os.path.join(
cfg.output_dir, "to_correct.yaml")
pyyaml.dump(to_correct_dataset_data,
open(to_correct_dataset_yaml_filename, 'w'),
default_flow_style=False,
default_style='"',
width=float("inf"))
cfg.dataset_yaml_filename = to_correct_dataset_yaml_filename
else:
not_used_dataset_data = None
if not only_compressing_is_needed:
dst_configs = os.path.join(cfg.output_dir, "configs")
if os.path.exists(dst_configs):
shutil.rmtree(dst_configs)
if cfg.iontorrent:
dir_util.copy_tree(os.path.join(configs_dir, "ionhammer"),
dst_configs,
preserve_times=False)
cfg_file_name = os.path.join(dst_configs, "ionhammer.cfg")
else:
dir_util.copy_tree(os.path.join(configs_dir, "hammer"),
dst_configs,
preserve_times=False)
cfg_file_name = os.path.join(dst_configs, "config.info")
cfg.tmp_dir = support.get_tmp_dir(prefix="hammer_")
if cfg.iontorrent:
prepare_config_ih(cfg_file_name, cfg, ext_python_modules_home)
binary_name = "spades-ionhammer"
else:
prepare_config_bh(cfg_file_name, cfg, log)
binary_name = "spades-hammer"
command = [
os.path.join(execution_home, binary_name),
os.path.abspath(cfg_file_name)
]
log.info("\n== Running read error correction tool: " +
' '.join(command) + "\n")
support.sys_call(command, log)
if not os.path.isfile(corrected_dataset_yaml_filename):
support.error("read error correction finished abnormally: " +
corrected_dataset_yaml_filename + " not found!")
else:
log.info("\n===== Skipping %s (already processed). \n" %
"read error correction tool")
support.continue_from_here(log)
corrected_dataset_data = pyyaml.load(
open(corrected_dataset_yaml_filename, 'r'))
remove_not_corrected_reads(cfg.output_dir)
is_changed = False
if cfg.gzip_output:
is_changed = True
compress_dataset_files(corrected_dataset_data, ext_python_modules_home,
cfg.max_threads, log)
if not_used_dataset_data:
is_changed = True
corrected_dataset_data += not_used_dataset_data
if is_changed:
pyyaml.dump(corrected_dataset_data,
open(corrected_dataset_yaml_filename, 'w'),
default_flow_style=False,
default_style='"',
width=float("inf"))
log.info("\n== Dataset description file was created: " +
corrected_dataset_yaml_filename + "\n")
if os.path.isdir(cfg.tmp_dir):
shutil.rmtree(cfg.tmp_dir)
def main():
os.environ["LC_ALL"] = "C"
if len(sys.argv) == 1:
options_storage.usage(spades_version)
sys.exit(0)
log = logging.getLogger('spades')
log.setLevel(logging.DEBUG)
console = logging.StreamHandler(sys.stdout)
console.setFormatter(logging.Formatter('%(message)s'))
console.setLevel(logging.DEBUG)
log.addHandler(console)
check_binaries(bin_home, log)
# parse options and safe all parameters to cfg
cfg, dataset_data = fill_cfg(sys.argv, log)
if options_storage.continue_mode:
cmd_line, options = get_options_from_params(
os.path.join(options_storage.output_dir, "params.txt"))
if not options:
support.error(
"failed to parse command line of the previous run! Please restart from the beginning."
)
cfg, dataset_data = fill_cfg(options, log)
options_storage.continue_mode = True
log_filename = os.path.join(cfg["common"].output_dir, "spades.log")
if options_storage.continue_mode:
log_handler = logging.FileHandler(log_filename, mode='a')
else:
log_handler = logging.FileHandler(log_filename, mode='w')
log.addHandler(log_handler)
if options_storage.continue_mode:
log.info(
"\n======= SPAdes pipeline continued. Log can be found here: " +
log_filename + "\n")
log.info("Restored from " + cmd_line)
else:
params_filename = os.path.join(cfg["common"].output_dir, "params.txt")
params_handler = logging.FileHandler(params_filename, mode='w')
log.addHandler(params_handler)
command = "Command line:"
for v in sys.argv:
command += " " + v
log.info(command)
print_used_values(cfg, log)
log.removeHandler(params_handler)
log.info("\n======= SPAdes pipeline started. Log can be found here: " +
log_filename + "\n")
# splitting interlaced reads if needed
if support.dataset_has_interlaced_reads(dataset_data):
dir_for_split_reads = os.path.join(
os.path.abspath(options_storage.output_dir), 'split_reads')
if not os.path.isdir(dir_for_split_reads):
os.makedirs(dir_for_split_reads)
dataset_data = support.split_interlaced_reads(dataset_data,
dir_for_split_reads, log)
options_storage.dataset_yaml_filename = os.path.join(
options_storage.output_dir, "input_dataset.yaml")
pyyaml.dump(dataset_data,
open(options_storage.dataset_yaml_filename, 'w'))
cfg["dataset"].yaml_filename = os.path.abspath(
options_storage.dataset_yaml_filename)
try:
# copying configs before all computations (to prevent its changing at run time)
tmp_configs_dir = os.path.join(cfg["common"].output_dir, "configs")
if os.path.isdir(
tmp_configs_dir) and not options_storage.continue_mode:
shutil.rmtree(tmp_configs_dir)
if not os.path.isdir(tmp_configs_dir):
shutil.copytree(os.path.join(spades_home, "configs"),
tmp_configs_dir)
corrected_dataset_yaml_filename = ''
if "error_correction" in cfg:
bh_cfg = merge_configs(cfg["error_correction"], cfg["common"])
bh_cfg.__dict__["dataset_yaml_filename"] = cfg[
"dataset"].yaml_filename
corrected_dataset_yaml_filename = os.path.join(
bh_cfg.output_dir, "corrected.yaml")
if os.path.isfile(corrected_dataset_yaml_filename
) and options_storage.continue_mode:
log.info(
"\n===== Skipping read error correction (already processed). \n"
)
else:
options_storage.continue_mode = False # continue from here
if "HEAPCHECK" in os.environ:
del os.environ["HEAPCHECK"]
if "heap_check" in bh_cfg.__dict__:
os.environ["HEAPCHECK"] = bh_cfg.heap_check
if os.path.exists(bh_cfg.output_dir):
shutil.rmtree(bh_cfg.output_dir)
os.makedirs(bh_cfg.output_dir)
if not os.path.exists(bh_cfg.tmp_dir):
os.makedirs(bh_cfg.tmp_dir)
log.info("\n===== Read error correction started. \n")
bh_logic.run_bh(corrected_dataset_yaml_filename,
tmp_configs_dir, bin_home, bh_cfg,
ext_python_modules_home, log)
log.info("\n===== Read error correction finished. \n")
result_contigs_filename = os.path.join(cfg["common"].output_dir,
"contigs.fasta")
result_scaffolds_filename = os.path.join(cfg["common"].output_dir,
"scaffolds.fasta")
misc_dir = os.path.join(cfg["common"].output_dir, "misc")
### if mismatch correction is enabled then result contigs are copied to misc directory
assembled_contigs_filename = os.path.join(misc_dir,
"assembled_contigs.fasta")
assembled_scaffolds_filename = os.path.join(
misc_dir, "assembled_scaffolds.fasta")
if "assembly" in cfg:
spades_cfg = merge_configs(cfg["assembly"], cfg["common"])
spades_cfg.__dict__["result_contigs"] = result_contigs_filename
spades_cfg.__dict__["result_scaffolds"] = result_scaffolds_filename
spades_cfg.__dict__["additional_contigs"] = os.path.join(
spades_cfg.output_dir, "simplified_contigs.fasta")
if options_storage.continue_mode and (
os.path.isfile(spades_cfg.result_contigs) or
("mismatch_corrector" in cfg
and os.path.isfile(assembled_contigs_filename))):
log.info("\n===== Skipping assembling (already processed). \n")
# calculating latest_dir for the next stages
latest_dir = support.get_latest_dir(
os.path.join(spades_cfg.output_dir, "K*"))
if not latest_dir:
support.error(
"failed to continue the previous run! Please restart from the beginning."
)
else:
if os.path.isfile(corrected_dataset_yaml_filename):
dataset_data = pyyaml.load(
open(corrected_dataset_yaml_filename, 'r'))
dataset_data = support.relative2abs_paths(
dataset_data,
os.path.dirname(corrected_dataset_yaml_filename))
if support.dataset_has_paired_reads(dataset_data):
spades_cfg.__dict__["paired_mode"] = True
else:
spades_cfg.__dict__["paired_mode"] = False
if options_storage.rectangles:
spades_cfg.__dict__["resolving_mode"] = "rectangles"
if "HEAPCHECK" in os.environ:
del os.environ["HEAPCHECK"]
if "heap_check" in spades_cfg.__dict__:
os.environ["HEAPCHECK"] = spades_cfg.heap_check
log.info("\n===== Assembling started.\n")
# creating dataset
dataset_filename = os.path.join(spades_cfg.output_dir,
"dataset.info")
if not os.path.isfile(
dataset_filename) or not options_storage.continue_mode:
dataset_file = open(dataset_filename, 'w')
import process_cfg
dataset_file.write(
"single_cell" + '\t' +
process_cfg.bool_to_str(cfg["dataset"].single_cell) +
'\n')
if os.path.isfile(corrected_dataset_yaml_filename):
dataset_file.write(
"reads" + '\t' + process_cfg.process_spaces(
corrected_dataset_yaml_filename) + '\n')
else:
dataset_file.write("reads" + '\t' +
process_cfg.process_spaces(
cfg["dataset"].yaml_filename) +
'\n')
if spades_cfg.developer_mode and "reference" in cfg[
"dataset"].__dict__:
dataset_file.write("reference_genome" + '\t')
dataset_file.write(
process_cfg.process_spaces(
os.path.abspath(cfg["dataset"].reference)) +
'\n')
dataset_file.close()
spades_cfg.__dict__["dataset"] = dataset_filename
latest_dir = spades_logic.run_spades(tmp_configs_dir, bin_home,
spades_cfg, log)
#rectangles
if spades_cfg.paired_mode and options_storage.rectangles:
if options_storage.continue_mode: # TODO: continue mode
support.warning(
"sorry, --continue doesn't work with --rectangles yet. Skipping repeat resolving."
)
else:
sys.path.append(
os.path.join(python_modules_home, "rectangles"))
import rrr
rrr_input_dir = os.path.join(latest_dir, "saves")
rrr_outpath = os.path.join(spades_cfg.output_dir,
"rectangles")
if not os.path.exists(rrr_outpath):
os.mkdir(rrr_outpath)
rrr_reference_information_file = os.path.join(
rrr_input_dir,
"late_pair_info_counted_etalon_distance.txt")
rrr_test_util = rrr.TestUtils(
rrr_reference_information_file,
os.path.join(rrr_outpath, "rectangles.log"))
rrr.resolve(rrr_input_dir, rrr_outpath, rrr_test_util,
"", cfg["dataset"].single_cell,
spades_cfg.careful)
shutil.copyfile(
os.path.join(
rrr_outpath,
"rectangles_extend_before_scaffold.fasta"),
spades_cfg.result_contigs)
shutil.copyfile(
os.path.join(rrr_outpath,
"rectangles_extend.fasta"),
spades_cfg.result_scaffolds)
if not spades_cfg.developer_mode:
if os.path.exists(rrr_input_dir):
shutil.rmtree(rrr_input_dir)
if os.path.exists(rrr_outpath):
shutil.rmtree(rrr_outpath, True)
if os.path.exists(rrr_outpath):
os.system('rm -r ' + rrr_outpath)
#EOR
if os.path.isdir(
misc_dir) and not options_storage.continue_mode:
shutil.rmtree(misc_dir)
if not os.path.isdir(misc_dir):
os.makedirs(misc_dir)
if os.path.isfile(spades_cfg.additional_contigs):
shutil.move(spades_cfg.additional_contigs, misc_dir)
log.info("\n===== Assembling finished. \n")
#corrector
if "mismatch_corrector" in cfg and (
os.path.isfile(result_contigs_filename) or
(options_storage.continue_mode
and os.path.isfile(assembled_contigs_filename))):
to_correct = dict()
to_correct["contigs"] = (result_contigs_filename,
assembled_contigs_filename)
if os.path.isfile(result_scaffolds_filename) or (
options_storage.continue_mode
and os.path.isfile(assembled_scaffolds_filename)):
to_correct["scaffolds"] = (result_scaffolds_filename,
assembled_scaffolds_filename)
# moving assembled contigs (scaffolds) to misc dir
for k, (old, new) in to_correct.items():
if options_storage.continue_mode and os.path.isfile(new):
continue
shutil.move(old, new)
if options_storage.continue_mode and os.path.isfile(result_contigs_filename) and \
(os.path.isfile(result_scaffolds_filename) or not os.path.isfile(assembled_scaffolds_filename)):
log.info(
"\n===== Skipping mismatch correction (already processed). \n"
)
else:
log.info("\n===== Mismatch correction started.")
# detecting paired-end library with the largest insert size
dataset_data = pyyaml.load(
open(options_storage.dataset_yaml_filename, 'r')
) ### initial dataset, i.e. before error correction
dataset_data = support.relative2abs_paths(
dataset_data,
os.path.dirname(options_storage.dataset_yaml_filename))
paired_end_libraries_ids = []
for id, reads_library in enumerate(dataset_data):
if reads_library['type'] == 'paired-end':
paired_end_libraries_ids.append(id)
if not len(paired_end_libraries_ids):
support.error(
'Mismatch correction cannot be performed without at least one paired-end library!'
)
estimated_params = load_config_from_file(
os.path.join(latest_dir, "_est_params.info"))
max_insert_size = -1
target_paired_end_library_id = -1
for id in paired_end_libraries_ids:
if float(estimated_params.__dict__[
"insert_size_" + str(id)]) > max_insert_size:
max_insert_size = float(
estimated_params.__dict__["insert_size_" +
str(id)])
target_paired_end_library_id = id
yaml_dirname = os.path.dirname(
options_storage.dataset_yaml_filename)
cfg["mismatch_corrector"].__dict__["1"] = list(
map(
lambda x: os.path.join(yaml_dirname, x),
dataset_data[target_paired_end_library_id]
['left reads']))
cfg["mismatch_corrector"].__dict__["2"] = list(
map(
lambda x: os.path.join(yaml_dirname, x),
dataset_data[target_paired_end_library_id]
['right reads']))
cfg["mismatch_corrector"].__dict__["insert-size"] = round(
max_insert_size)
#TODO: add reads orientation
import corrector
corrector_cfg = cfg["mismatch_corrector"]
args = []
for key, values in corrector_cfg.__dict__.items():
if key == "output-dir":
continue
# for processing list of reads
if not isinstance(values, list):
values = [values]
for value in values:
if len(key) == 1:
args.append('-' + key)
else:
args.append('--' + key)
if value:
args.append(value)
# processing contigs and scaffolds (or only contigs)
for k, (corrected, assembled) in to_correct.items():
if options_storage.continue_mode and os.path.isfile(
corrected):
log.info("\n== Skipping processing of " + k +
" (already processed)\n")
continue
options_storage.continue_mode = False
log.info("\n== Processing of " + k + "\n")
cur_args = args[:]
cur_args += ['-c', assembled]
tmp_dir_for_corrector = os.path.join(
corrector_cfg.__dict__["output-dir"],
"mismatch_corrector_" + k)
cur_args += ['--output-dir', tmp_dir_for_corrector]
# correcting
corrector.main(cur_args, ext_python_modules_home, log)
result_corrected_filename = os.path.abspath(
os.path.join(tmp_dir_for_corrector,
"corrected_contigs.fasta"))
# moving corrected contigs (scaffolds) to SPAdes output dir
if os.path.isfile(result_corrected_filename):
shutil.move(result_corrected_filename, corrected)
if os.path.isdir(tmp_dir_for_corrector):
shutil.rmtree(tmp_dir_for_corrector)
log.info("\n===== Mismatch correction finished.\n")
if not cfg["common"].developer_mode and os.path.isdir(tmp_configs_dir):
shutil.rmtree(tmp_configs_dir)
#log.info("")
if os.path.isdir(os.path.dirname(corrected_dataset_yaml_filename)):
log.info(" * Corrected reads are in " +
os.path.dirname(corrected_dataset_yaml_filename) + "/")
if os.path.isfile(result_contigs_filename):
log.info(" * Assembled contigs are in " + result_contigs_filename)
if os.path.isfile(result_scaffolds_filename):
log.info(" * Assembled scaffolds are in " +
result_scaffolds_filename)
#log.info("")
#breaking scaffolds
if os.path.isfile(result_scaffolds_filename):
if not os.path.isdir(misc_dir):
os.makedirs(misc_dir)
result_broken_scaffolds = os.path.join(misc_dir,
"broken_scaffolds.fasta")
threshold = 3
if not os.path.isfile(result_broken_scaffolds
) or not options_storage.continue_mode:
support.break_scaffolds(result_scaffolds_filename, threshold,
result_broken_scaffolds)
#log.info(" * Scaffolds broken by " + str(threshold) + " Ns are in " + result_broken_scaffolds)
### printing WARNINGS SUMMARY
if not support.log_warnings(log):
log.info("\n======= SPAdes pipeline finished."
) # otherwise it finished WITH WARNINGS
log.info("\nSPAdes log can be found here: " + log_filename)
log.info("")
log.info("Thank you for using SPAdes!")
log.removeHandler(log_handler)
except Exception:
_, exc, _ = sys.exc_info()
log.exception(exc)
support.error("exception caught", log)
def fill_cfg(options_to_parse, log):
try:
options, not_options = getopt.gnu_getopt(options_to_parse, options_storage.short_options, options_storage.long_options)
except getopt.GetoptError:
_, exc, _ = sys.exc_info()
sys.stderr.write(str(exc) + "\n")
sys.stderr.flush()
options_storage.usage(spades_version)
sys.exit(1)
if not options:
options_storage.usage(spades_version)
sys.exit(1)
# all parameters are stored here
cfg = dict()
# dataset is stored here. We are prepared for up to MAX_LIBS_NUMBER paired-end libs and MAX_LIBS_NUMBER mate-pair libs
dataset_data = [{} for i in range(options_storage.MAX_LIBS_NUMBER * 2)]
options_storage.continue_mode = False
for opt, arg in options:
if opt == '-o':
options_storage.output_dir = arg
elif opt == "--tmp-dir":
options_storage.tmp_dir = arg
elif opt == "--reference":
options_storage.reference = support.check_file_existence(arg, 'reference', log)
elif opt == "--dataset":
options_storage.dataset_yaml_filename = support.check_file_existence(arg, 'dataset', log)
elif opt in options_storage.reads_options:
support.add_to_dataset(opt, arg, dataset_data)
elif opt == '-k':
options_storage.k_mers = list(map(int, arg.split(",")))
for k in options_storage.k_mers:
if k > 127:
support.error('wrong k value ' + str(k) + ': all k values should be less than 128', log)
if k % 2 == 0:
support.error('wrong k value ' + str(k) + ': all k values should be odd', log)
elif opt == "--sc":
options_storage.single_cell = True
elif opt == "--disable-gzip-output":
options_storage.disable_gzip_output = True
elif opt == "--only-error-correction":
if options_storage.only_assembler:
support.error('you cannot specify --only-error-correction and --only-assembler simultaneously')
options_storage.only_error_correction = True
elif opt == "--only-assembler":
if options_storage.only_error_correction:
support.error('you cannot specify --only-error-correction and --only-assembler simultaneously')
options_storage.only_assembler = True
elif opt == "--bh-heap-check":
options_storage.bh_heap_check = arg
elif opt == "--spades-heap-check":
options_storage.spades_heap_check = arg
elif opt == "--continue":
options_storage.continue_mode = True
elif opt == '-t' or opt == "--threads":
options_storage.threads = int(arg)
elif opt == '-m' or opt == "--memory":
options_storage.memory = int(arg)
elif opt == "--phred-offset":
if int(arg) in [33, 64]:
options_storage.qvoffset = int(arg)
else:
support.error('wrong PHRED quality offset value ' + str(arg) + ': should be either 33 or 64', log)
elif opt == '-i' or opt == "--iterations":
options_storage.iterations = int(arg)
elif opt == "--debug":
options_storage.developer_mode = True
elif opt == "--rectangles":
options_storage.rectangles = True
#corrector
elif opt == "--mismatch-correction":
options_storage.mismatch_corrector = True
elif opt == "--careful":
options_storage.mismatch_corrector = True
options_storage.careful = True
elif opt == '-h' or opt == "--help":
options_storage.usage(spades_version)
sys.exit(0)
elif opt == "--help-hidden":
options_storage.usage(spades_version, True)
sys.exit(0)
elif opt == "--test":
options_storage.set_test_options()
support.add_to_dataset('-1', os.path.join(spades_home, "test_dataset/ecoli_1K_1.fq.gz"), dataset_data)
support.add_to_dataset('-2', os.path.join(spades_home, "test_dataset/ecoli_1K_2.fq.gz"), dataset_data)
#break
else:
raise ValueError
if not options_storage.output_dir:
support.error("the output_dir is not set! It is a mandatory parameter (-o output_dir).", log)
if not os.path.isdir(options_storage.output_dir):
if options_storage.continue_mode:
support.error("the output_dir should exist for --continue!", log)
os.makedirs(options_storage.output_dir)
if options_storage.continue_mode:
return None, None
if options_storage.dataset_yaml_filename:
try:
dataset_data = pyyaml.load(open(options_storage.dataset_yaml_filename, 'r'))
except pyyaml.YAMLError:
_, exc, _ = sys.exc_info()
support.error('exception caught while parsing YAML file (' + options_storage.dataset_yaml_filename + '):\n' + str(exc))
dataset_data = support.relative2abs_paths(dataset_data, os.path.dirname(options_storage.dataset_yaml_filename))
else:
dataset_data = support.correct_dataset(dataset_data)
dataset_data = support.relative2abs_paths(dataset_data, os.getcwd())
options_storage.dataset_yaml_filename = os.path.join(options_storage.output_dir, "input_dataset.yaml")
pyyaml.dump(dataset_data, open(options_storage.dataset_yaml_filename, 'w'))
support.check_dataset_reads(dataset_data, options_storage.only_assembler, log)
if support.dataset_has_only_mate_pairs_libraries(dataset_data):
support.error('you should specify at least one paired-end or unpaired library (only mate-pairs libraries were found)!')
if options_storage.rectangles and (len(dataset_data) > 1):
support.error('rectangle graph algorithm for repeat resolution cannot work with multiple libraries!')
### FILLING cfg
cfg["common"] = empty_config()
cfg["dataset"] = empty_config()
if not options_storage.only_assembler:
cfg["error_correction"] = empty_config()
if not options_storage.only_error_correction:
cfg["assembly"] = empty_config()
# common
cfg["common"].__dict__["output_dir"] = os.path.abspath(options_storage.output_dir)
cfg["common"].__dict__["max_threads"] = options_storage.threads
cfg["common"].__dict__["max_memory"] = options_storage.memory
cfg["common"].__dict__["developer_mode"] = options_storage.developer_mode
# dataset section
cfg["dataset"].__dict__["single_cell"] = options_storage.single_cell
cfg["dataset"].__dict__["yaml_filename"] = os.path.abspath(options_storage.dataset_yaml_filename)
if options_storage.developer_mode and options_storage.reference:
cfg["dataset"].__dict__["reference"] = options_storage.reference
# error correction
if (not options_storage.only_assembler) and (options_storage.iterations > 0):
cfg["error_correction"].__dict__["output_dir"] = os.path.join(cfg["common"].output_dir, "corrected")
cfg["error_correction"].__dict__["max_iterations"] = options_storage.iterations
cfg["error_correction"].__dict__["gzip_output"] = not options_storage.disable_gzip_output
if options_storage.qvoffset:
cfg["error_correction"].__dict__["qvoffset"] = options_storage.qvoffset
if options_storage.bh_heap_check:
cfg["error_correction"].__dict__["heap_check"] = options_storage.bh_heap_check
if options_storage.tmp_dir:
cfg["error_correction"].__dict__["tmp_dir"] = options_storage.tmp_dir
else:
cfg["error_correction"].__dict__["tmp_dir"] = cfg["error_correction"].output_dir
cfg["error_correction"].tmp_dir = os.path.join(os.path.abspath(cfg["error_correction"].tmp_dir), 'tmp')
# assembly
if not options_storage.only_error_correction:
if options_storage.k_mers:
cfg["assembly"].__dict__["iterative_K"] = options_storage.k_mers
else:
cfg["assembly"].__dict__["iterative_K"] = options_storage.k_mers_short
cfg["assembly"].__dict__["careful"] = options_storage.careful
if options_storage.spades_heap_check:
cfg["assembly"].__dict__["heap_check"] = options_storage.spades_heap_check
#corrector can work only if contigs exist (not only error correction)
if (not options_storage.only_error_correction) and options_storage.mismatch_corrector:
cfg["mismatch_corrector"] = empty_config()
cfg["mismatch_corrector"].__dict__["skip-masked"] = ""
cfg["mismatch_corrector"].__dict__["bwa"] = os.path.join(bin_home, "bwa-spades")
cfg["mismatch_corrector"].__dict__["threads"] = options_storage.threads
cfg["mismatch_corrector"].__dict__["output-dir"] = options_storage.output_dir
return cfg, dataset_data
