def main():
parser = argparse.ArgumentParser('miRNA harmonization')
# Logging flag
parser.add_argument(
'-d',
'--debug',
action='store_const',
const=logging.DEBUG,
dest='level',
help='Enable debug logging.',
)
parser.set_defaults(level=logging.INFO)
# Required flags
parser.add_argument(
'-r',
'--reference_fasta_path',
required=False,
help='Reference fasta path.',
)
parser.add_argument(
'-b',
'--bam_path',
nargs='?',
default=[sys.stdin],
help='Source bam path.',
)
parser.add_argument(
'-l',
'--log_dir',
required=False,
type=is_dir,
help='Log file directory.',
)
parser.add_argument(
'-u',
'--uuid',
required=False,
help='analysis_id string',
)
args = parser.parse_args()
reference_fasta_path = args.reference_fasta_path
preharmonized_bam_path = args.bam_path
log_dir = args.log_dir
uuid = args.uuid
# Logging Setup
logging.basicConfig(
filename=os.path.join(log_dir, 'aln_' + uuid + '.log'),
filemode='a',
level=args.level,
format='%(asctime)s %(levelname)s %(message)s',
datefmt='%Y-%m-%d_%H:%M:%S_%Z',
)
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
logger = logging.getLogger(__name__)
hostname = os.uname()[1]
logger.info('hostname=%s' % hostname)
logger.info('preharmonized_bam_path=%s' % preharmonized_bam_path)
engine_path = 'sqlite:///' + os.path.join(log_dir, uuid + '_harmonize.db')
engine = sqlalchemy.create_engine(engine_path,
isolation_level='SERIALIZABLE')
bam_validate.bam_validate(uuid, preharmonized_bam_path, engine, logger)
bam_stats.do_samtools_flagstat(uuid, preharmonized_bam_path,
reference_fasta_path, engine, logger)
readgroup_path_dict = bam_util.write_readgroups(uuid,
preharmonized_bam_path,
engine, logger)
bam_util.bam_to_fastq(uuid, preharmonized_bam_path, engine, logger)
top_dir = os.path.dirname(preharmonized_bam_path)
fastq_dir = os.path.join(top_dir, 'fastq')
fastq_validate.fastq_validate(uuid, fastq_dir, engine, logger)
# Harmonization
be_lenient = False
harmonized_readgroup_bam_path_list = bam_util.bwa(uuid,
preharmonized_bam_path,
reference_fasta_path,
readgroup_path_dict,
engine, logger)
fastq_list = fastq_util.buildfastqlist(fastq_dir, logger)
fastq_path_list = [os.path.join(fastq_dir, fastq) for fastq in fastq_list]
for harmonized_readgroup_bam_path in harmonized_readgroup_bam_path_list:
if pipe_util.is_aln_bam(harmonized_readgroup_bam_path, logger):
be_lenient = True
harmonized_sorted_bam_path_list = picard_bam_sort.bam_sort(
uuid, preharmonized_bam_path, harmonized_readgroup_bam_path_list,
reference_fasta_path, engine, logger, be_lenient)
harmonized_bam_merge_path = picard_bam_merge.bam_merge(
uuid, preharmonized_bam_path, harmonized_sorted_bam_path_list, engine,
logger, be_lenient)
bam_validate.bam_validate(uuid, harmonized_bam_merge_path, engine, logger)
bam_stats.do_samtools_flagstat(uuid, harmonized_bam_merge_path,
reference_fasta_path, engine, logger)
def main():
parser = argparse.ArgumentParser('miRNA harmonization')
# Logging flag
parser.add_argument('-d', '--debug',
action = 'store_const',
const = logging.DEBUG,
dest = 'level',
help = 'Enable debug logging.',
)
parser.set_defaults(level = logging.INFO)
# Required flags
parser.add_argument('-r', '--reference_fasta_path',
required = False,
help = 'Reference fasta path.',
)
parser.add_argument('-b', '--bam_path',
nargs = '?',
default = [sys.stdin],
help = 'Source bam path.',
)
parser.add_argument('-l', '--log_dir',
required = False,
type = is_dir,
help = 'Log file directory.',
)
parser.add_argument('-u', '--uuid',
required = False,
help = 'analysis_id string',
)
args = parser.parse_args()
reference_fasta_path = args.reference_fasta_path
preharmonized_bam_path = args.bam_path
log_dir = args.log_dir
uuid = args.uuid
# Logging Setup
logging.basicConfig(
filename = os.path.join(log_dir, 'aln_' + uuid + '.log'),
filemode = 'a',
level = args.level,
format='%(asctime)s %(levelname)s %(message)s',
datefmt='%Y-%m-%d_%H:%M:%S_%Z',
)
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
logger = logging.getLogger(__name__)
hostname = os.uname()[1]
logger.info('hostname=%s' % hostname)
logger.info('preharmonized_bam_path=%s' % preharmonized_bam_path)
engine_path = 'sqlite:///' + os.path.join(log_dir, uuid + '_harmonize.db')
engine = sqlalchemy.create_engine(engine_path, isolation_level='SERIALIZABLE')
bam_validate.bam_validate(uuid, preharmonized_bam_path, engine, logger)
bam_stats.do_samtools_flagstat(uuid, preharmonized_bam_path, reference_fasta_path, engine, logger)
readgroup_path_dict = bam_util.write_readgroups(uuid, preharmonized_bam_path, engine, logger)
bam_util.bam_to_fastq(uuid, preharmonized_bam_path, engine, logger)
top_dir = os.path.dirname(preharmonized_bam_path)
fastq_dir = os.path.join(top_dir, 'fastq')
fastq_validate.fastq_validate(uuid, fastq_dir, engine, logger)
# Harmonization
be_lenient = False
harmonized_readgroup_bam_path_list = bam_util.bwa(uuid, preharmonized_bam_path, reference_fasta_path, readgroup_path_dict, engine, logger)
fastq_list = fastq_util.buildfastqlist(fastq_dir, logger)
fastq_path_list = [os.path.join(fastq_dir, fastq) for fastq in fastq_list]
for harmonized_readgroup_bam_path in harmonized_readgroup_bam_path_list:
if pipe_util.is_aln_bam(harmonized_readgroup_bam_path, logger):
be_lenient = True
harmonized_sorted_bam_path_list = picard_bam_sort.bam_sort(uuid, preharmonized_bam_path, harmonized_readgroup_bam_path_list, reference_fasta_path, engine, logger, be_lenient)
harmonized_bam_merge_path = picard_bam_merge.bam_merge(uuid, preharmonized_bam_path, harmonized_sorted_bam_path_list, engine, logger, be_lenient)
bam_validate.bam_validate(uuid, harmonized_bam_merge_path, engine, logger)
bam_stats.do_samtools_flagstat(uuid, harmonized_bam_merge_path, reference_fasta_path, engine, logger)
def main():
parser = argparse.ArgumentParser('harmonization pipeline')
# Logging flags.
parser.add_argument('-d','--debug',
action='store_const',
const=logging.DEBUG,
dest='level',
help='Enable debug logging.',
)
parser.set_defaults(level=logging.INFO)
# Required flags.
parser.add_argument('-g','--s3_reference_bucket',
required=True,
help='S3 bucket name containing reference fasta.',
)
parser.add_argument('-b','--s3_bam_bucket',
required=True,
help='S3 bucket name containing source bam file.',
)
parser.add_argument('-r','--reference_fasta_name',
required=True,
help='Reference fasta S3 key name.',
)
parser.add_argument('-a','--bam_analysis_id',
required=True,
help='Source bam file S3 key name.',
)
parser.add_argument('-s','--scratch_dir',
required=True,
type=is_dir,
help='Scratch file directory.',
)
parser.add_argument('-l','--log_dir',
required=True,
type=is_dir,
help='Log file directory.',
)
parser.add_argument('-f','--s3cfg_dir',
required=True,
type=is_dir,
help='.s3cfg file directory.',
)
parser.add_argument('-t','--thread_count',
required=True,
type=is_nat,
help='Maximum number of threads for execution.',
)
parser.add_argument('-c','--csv_stats',
required=True,
help='Write to csv rather than postgres'
)
parser.add_argument('-u','--postgres_user',
required=False,
help='postgres username'
)
parser.add_argument('-p','--postgres_password',
required=False,
help='postgres password'
)
parser.add_argument('-n','--postgres_hostname',
required=False,
help='postgres hostname'
)
args = parser.parse_args()
s3_reference_bucket = args.s3_reference_bucket
s3_bam_bucket = args.s3_bam_bucket
reference_fasta_name = args.reference_fasta_name
reference_fasta_name = reference_fasta_name.strip('/')
bam_analysis_id = args.bam_analysis_id
bam_analysis_id = bam_analysis_id.strip('/')
scratch_dir = args.scratch_dir
log_dir = args.log_dir
s3cfg_dir= args.s3cfg_dir
thread_count = str(args.thread_count)
csv_stats = literal_eval(args.csv_stats)
postgres_user = args.postgres_user
postgres_password = args.postgres_password
postgres_hostname = args.postgres_hostname
##logging
uuid=pipe_util.get_uuid_from_path(bam_analysis_id)
logging.basicConfig(
filename=os.path.join(log_dir,'aln_'+uuid+'.log'), #/host for docker
level=args.level,
filemode='a',
format='%(asctime)s %(levelname)s %(message)s',
datefmt='%Y-%m-%d_%H:%M:%S_%Z',
)
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
logger=logging.getLogger(__name__)
hostname=os.uname()[1]
logger.info('hostname=%s' % hostname)
##open stats and timing db
if not csv_stats:
if (postgres_user is None) or (postgres_password is None) or (postgres_hostname is None):
logger.debug('must enter postgres user, password and hostname if not writing to csv (or use -c True)')
sys.exit(1)
engine_path='postgresql://'+postgres_user+':'+postgres_password+'@'+postgres_hostname+'/gdc_harmonize'
engine=sqlalchemy.create_engine(engine_path,isolation_level='SERIALIZABLE')
##get reference genome
get_s3_objects(uuid,s3_reference_bucket,reference_fasta_name,scratch_dir,s3cfg_dir,engine,logger)
reference_fasta_path=os.path.join(scratch_dir,reference_fasta_name)
logger.info('scratch_dir=%s' % scratch_dir)
logger.info('reference_fasta_name=%s' % reference_fasta_name)
logger.info('reference_fasta_path=%s' % reference_fasta_path)
##get bam to be harmonized
get_s3_objects(uuid,s3_bam_bucket,bam_analysis_id,scratch_dir,s3cfg_dir,engine,logger)
bam_path=bam_util.get_bam_path(bam_analysis_id,scratch_dir,logger)
#get original reference genome for stats
original_fasta_name=bam_util.get_bam_reference(bam_path,logger)
if not (original_fasta_name.endswith('.fa') or original_fasta_name.endswith('.fasta')):
original_fasta_name+='.fa'
original_fasta_path=os.path.join(scratch_dir,original_fasta_name)
get_s3_objects(uuid,s3_reference_bucket,original_fasta_name,scratch_dir,s3cfg_dir,engine,logger)
##pipeline
#verify_util.verify_cgquery_md5(reference_fasta_name,logger)
verify_util.verify_cgquery(uuid,bam_path,engine,logger)
bam_validate.bam_validate(uuid,bam_path,engine,logger)
bam_stats.bam_stats(uuid,bam_path,original_fasta_path,engine,logger)
bam_util.bam_to_fastq(uuid,bam_path,engine,logger)
fastq_length=fastq_validate.fastqc_validate(uuid,bam_path,thread_count,engine,logger)
readgroup_path_dict=bam_util.write_readgroups(uuid,bam_path,engine,logger)#to file and db
#MEM_ALN_CUTOFF=70
#if fastq_length<MEM_ALN_CUTOFF: # do bwa aln...
# bam_path_list=bwa.bwa(uuid,bam_path,reference_fasta_path,readgroup_path_dict,thread_count,engine,logger)
# bam_sort_path_list=bam_sort.bam_sort(uuid,bam_path,bam_path_list,reference_fasta_path,thread_count,engine,logger)
# bam_merge_path=bam_merge.bam_merge(uuid,bam_path,bam_sort_path_list,engine,logger)
# bam_md_path=bam_mark_duplicates.bam_mark_duplicates(uuid,bam_merge_path,thread_count,engine,logger)
# bam_validate.bam_validate(uuid,bam_md_path,engine,logger)
# bam_stats.bam_stats(uuid,bam_md_path,reference_fasta_path,engine,logger)
# verify_util.store_md5_size(uuid,bam_md_path,engine,logger)
# bai_md_path=bam_md_path+'.bai'
# verify_util.store_md5_size(uuid,bai_md_path,engine,logger)
# ...and do bwa mem
bam_path_list=bwa_mem.bwa_mem(uuid,bam_path,reference_fasta_path,readgroup_path_dict,thread_count,engine,logger)
bam_sort_path_list=bam_sort.bam_sort(uuid,bam_path,bam_path_list,reference_fasta_path,thread_count,engine,logger)
bam_merge_path=bam_merge.bam_merge(uuid,bam_path,bam_sort_path_list,engine,logger)
bam_md_path=bam_mark_duplicates.bam_mark_duplicates(uuid,bam_merge_path,thread_count,engine,logger)
bam_validate.bam_validate(uuid,bam_md_path,engine,logger)
#bam_stats.bam_stats(uuid,bam_md_path,reference_fasta_path,engine,logger)
verify_util.store_md5_size(uuid,bam_md_path,engine,logger)
bai_md_path=bam_md_path+'.bai'
verify_util.store_md5_size(uuid,bai_md_path,engine,logger)