def test_find_fastq_and_build_db_collection(self):
ci = Collect_seqrun_fastq_to_db(
fastq_dir=self.fastq_dir,
session_class=self.session_class,
seqrun_igf_id=self.seqrun_igf_id,
flowcell_id=self.flowcell_id,
model_name=self.model_name,
file_location=self.file_location,
samplesheet_file=self.samplesheet_file,
manifest_name=self.manifest_name,
)
ci.find_fastq_and_build_db_collection()
ca = CollectionAdaptor(**{'session_class': self.session_class})
ca.start_session()
file_path = 'data/collect_fastq_dir/sc_1_8/IGFP0001_test_22-8-2017_rna_sc/IGF00001/IGF00001-1_S1_L003_R1_001.fastq.gz'
(name,
type) = ca.fetch_collection_name_and_table_from_file_path(file_path)
ca.close_session()
self.assertEqual(name, 'IGF00001_NEXTSEQ_TESTABC_3')
def test_find_fastq_and_build_db_collection(self):
ci = Collect_seqrun_fastq_to_db(
fastq_dir=self.fastq_dir,
session_class=self.session_class,
seqrun_igf_id=self.seqrun_igf_id,
flowcell_id=self.flowcell_id,
model_name=self.model_name,
file_location=self.file_location,
samplesheet_file=self.samplesheet_file,
manifest_name=self.manifest_name,
)
ci.find_fastq_and_build_db_collection()
ca = CollectionAdaptor(**{'session_class': self.session_class})
ca.start_session()
query = ca.session.query(Collection).filter(
Collection.name == 'IGF00001_MISEQ_000000000-D0YLK_1')
file_path = 'data/collect_fastq_dir/1_16/IGFP0001_test_22-8-2017_rna/IGF00002/IGF00002-2_S1_L001_R1_001.fastq.gz'
(name,
type) = ca.fetch_collection_name_and_table_from_file_path(file_path)
ca.close_session()
self.assertEqual(name, 'IGF00002_MISEQ_000000000-D0YLK_1')
def run(self):
try:
fastq_file = self.param_required('fastq_file')
fastq_dir = self.param_required('fastq_dir')
igf_session_class = self.param_required('igf_session_class')
fastqc_exe = self.param_required('fastqc_exe')
tag = self.param_required('tag')
seqrun_igf_id = self.param_required('seqrun_igf_id')
seqrun_date = self.param_required('seqrun_date')
flowcell_id = self.param_required('flowcell_id')
fastqc_options = self.param('fastqc_options')
base_results_dir = self.param_required('base_results_dir')
project_name = self.param_required('project_name')
force_overwrite = self.param('force_overwrite')
fastqc_dir_label = self.param('fastqc_dir_label')
required_collection_table = self.param('required_collection_table')
sample_name = self.param('sample_name')
hpc_location = self.param('hpc_location')
fastqc_collection_type = self.param('fastqc_collection_type')
use_ephemeral_space = self.param('use_ephemeral_space')
store_file = self.param('store_file')
lane_index_info = os.path.basename(fastq_dir) # get the lane and index length info
fastq_file_label = os.path.basename(fastq_file).replace('.fastq.gz','')
collection_name = None
collection_table = None
if tag=='known' and store_file: # fetch sample name for known fastq, if its not defined
base = BaseAdaptor(**{'session_class':igf_session_class})
base.start_session() # connect to db
ca = CollectionAdaptor(**{'session':base.session})
(collection_name,collection_table) = \
ca.fetch_collection_name_and_table_from_file_path(\
file_path=fastq_file) # fetch collection name and table info
if collection_table != required_collection_table:
raise ValueError(
'Expected collection table {0} and got {1}, {2}'.\
format(
required_collection_table,
collection_table,
fastq_file))
ra = RunAdaptor(**{'session':base.session})
sample = ra.fetch_sample_info_for_run(run_igf_id=collection_name)
sample_name = sample['sample_igf_id']
base.close_session()
fastqc_result_dir = \
os.path.join(\
base_results_dir,
project_name,
seqrun_date,
flowcell_id,
lane_index_info,
tag) # result dir path is generic
if sample_name is not None:
fastqc_result_dir = \
os.path.join(\
fastqc_result_dir,
sample_name) # add sample name to dir path if its available
fastqc_result_dir = \
os.path.join(\
fastqc_result_dir,
fastq_file_label,
fastqc_dir_label) # keep multiple files under same dir
if os.path.exists(fastqc_result_dir) and force_overwrite:
remove_dir(fastqc_result_dir) # remove existing output dir if force_overwrite is true
if not os.path.exists(fastqc_result_dir):
os.makedirs(fastqc_result_dir,mode=0o775) # create output dir if its not present
temp_work_dir = \
get_temp_dir(use_ephemeral_space=use_ephemeral_space) # get a temp work dir
if not os.path.exists(fastq_file):
raise IOError('fastq file {0} not readable'.format(fastq_file)) # raise if fastq file path is not readable
fastqc_output = \
os.path.join(\
temp_work_dir,
fastq_file_label)
os.mkdir(fastqc_output) # create fastqc output dir
fastqc_param = \
self.format_tool_options(fastqc_options) # format fastqc params
fastqc_cmd = \
[fastqc_exe, '-o',fastqc_output, '-d',temp_work_dir ] # fastqc base parameters
fastqc_cmd.extend(fastqc_param) # add additional parameters
fastqc_cmd.append(fastq_file) # fastqc input file
subprocess.check_call(' '.join(fastqc_cmd),shell=True) # run fastqc
fastqc_zip = None
fastqc_html = None
for root, _, files in os.walk(top=fastqc_output):
for file in files:
if fnmatch.fnmatch(file, '*.zip'):
input_fastqc_zip = os.path.join(root,file)
copy2(input_fastqc_zip,fastqc_result_dir)
fastqc_zip = os.path.join(fastqc_result_dir,file)
if fnmatch.fnmatch(file, '*.html'):
input_fastqc_html = os.path.join(root,file)
copy2(input_fastqc_html,fastqc_result_dir)
fastqc_html = os.path.join(fastqc_result_dir,file)
if fastqc_html is None or fastqc_zip is None:
raise ValueError('Missing required values, fastqc zip: {0}, fastqc html: {1}'.\
format(fastqc_zip,fastqc_html))
if tag=='known' and store_file:
if collection_name is None:
raise ValueError('couldn\'t retrieve collection name for {0}'.\
format(fastq_file))
fastqc_files = \
[{'name':collection_name,
'type':fastqc_collection_type,
'table':required_collection_table,
'file_path':fastqc_zip,
'location':hpc_location},
{'name':collection_name,
'type':fastqc_collection_type,
'table':required_collection_table,
'file_path':fastqc_html,
'location':hpc_location},
]
ca = CollectionAdaptor(**{'session_class':igf_session_class})
ca.start_session()
ca.load_file_and_create_collection(data=fastqc_files) # store fastqc files to db
ca.close_session()
self.param('dataflow_params',
{'fastqc_html':fastqc_html,
'lane_index_info':lane_index_info,
'sample_name':sample_name,
'fastqc':{'fastq_dir':fastq_dir,
'fastqc_zip':fastqc_zip,
'fastqc_html':fastqc_html}}) # set dataflow params
except Exception as e:
message = \
'seqrun: {2}, Error in {0}: {1}'.\
format(\
self.__class__.__name__,
e,
seqrun_igf_id)
self.warning(message)
self.post_message_to_slack(message,reaction='fail') # post msg to slack for failed jobs
raise
def _process_samples_data(self):
'''
An internal method for processing samples data
'''
try:
fastq_dir = self.param_required('fastq_dir')
qc_files = self.param_required('qc_files')
samplesheet_filename = self.param('samplesheet_filename')
igf_session_class = self.param_required('igf_session_class')
remote_project_path = self.param_required('remote_project_path')
project_name = self.param_required('project_name')
seqrun_date = self.param_required('seqrun_date')
flowcell_id = self.param_required('flowcell_id')
lane_index_info = self.param_required('lane_index_info')
singlecell_tag = self.param('singlecell_tag')
remote_path = \
os.path.join(\
remote_project_path,
project_name,
seqrun_date,
flowcell_id,
lane_index_info) # get remote base path
base = BaseAdaptor(**{'session_class': igf_session_class})
base.start_session() # connect to db
ca = CollectionAdaptor(**{'session': base.session})
ra = RunAdaptor(**{'session': base.session})
fastqc_data = list()
for fastqc_file in qc_files[
'fastqc']: # get fastqc files for fastq_dir
fastqc_zip = fastqc_file['fastqc_zip']
fastq_file = fastqc_file['fastq_file']
qc_fastq_dir = fastqc_file['fastq_dir']
if qc_fastq_dir == fastq_dir: # check for fastq dir
remote_fastqc_path = fastqc_file['remote_fastqc_path']
remote_fastqc_path = \
os.path.relpath(\
remote_fastqc_path,
start=remote_path) # get relative path
(total_reads, _) = \
get_fastq_info_from_fastq_zip(fastqc_zip)
(collection_name,_) = \
ca.fetch_collection_name_and_table_from_file_path(\
file_path=fastq_file) # fetch collection name and table info
sample = ra.fetch_sample_info_for_run(
run_igf_id=collection_name)
sample_name = sample['sample_igf_id']
fastqc_data.\
append(\
{'Sample_ID':sample_name,
'Fastqc':remote_fastqc_path,
'FastqFile':fastq_file,
'TotalReads':total_reads})
base.close_session() # close db connection
fastqs_data = list()
for fastqs_file in qc_files[
'fastqscreen']: # get fastqs files for fastq_dir
fastq_file = fastqs_file['fastq_file']
remote_fastqs_path = fastqs_file['remote_fastqscreen_path']
qs_fastq_dir = fastqc_file['fastq_dir']
if qs_fastq_dir == fastq_dir: # check for accu data
remote_fastqs_path = \
os.path.relpath(\
remote_fastqs_path,
start=remote_path) # get relative path
fastqs_data.\
append(\
{'Fastqscreen':remote_fastqs_path,
'FastqFile':fastq_file})
if len(fastqc_data) == 0 or len(fastqs_data) == 0:
raise ValueError('Value not found for fastqc: {0} or fastqscreen:{1}'.\
format(len(fastqc_data), len(fastqs_data)))
fastqc_data = pd.DataFrame(fastqc_data)
fastqs_data = pd.DataFrame(fastqs_data).set_index(
'FastqFile') # convert to dataframe
merged_qc_info = \
fastqc_data.\
join(\
fastqs_data,
how='inner',
on='FastqFile',
lsuffix='',
rsuffix='_s'
) # merge fastqc and fastqscreen info
if len(merged_qc_info) == 0:
raise ValueError('No QC data found for merging, fastqc:{0}, fastqscreen: {1}'.\
format(len(fastqc_data), len(fastqs_data)))
samplesheet_file = \
os.path.join(\
fastq_dir,
samplesheet_filename)
if not os.path.exists(samplesheet_file):
raise IOError('samplesheet file {0} not found'.\
format(samplesheet_file))
final_samplesheet_data = list()
samplesheet_sc = SampleSheet(
infile=samplesheet_file
) # read samplesheet for single cell check
samplesheet_sc.\
filter_sample_data(\
condition_key='Description',
condition_value=singlecell_tag,
method='include') # keep only single cell samples
if len(samplesheet_sc._data) > 0:
sc_data = \
pd.DataFrame(samplesheet_sc._data).\
drop(['Sample_ID','Sample_Name','index'],axis=1).\
drop_duplicates().\
rename(columns={'Original_Sample_ID':'Sample_ID',
'Original_Sample_Name':'Sample_Name',
'Original_index':'index'}).\
to_dict(orient='region') # restructure single cell data. sc data doesn't have index2
final_samplesheet_data.extend(
sc_data) # add single cell samples to final data
sa = SampleSheet(infile=samplesheet_file)
sa.filter_sample_data(\
condition_key='Description',
condition_value=singlecell_tag,
method='exclude') # remove only single cell samples
if len(sa._data) > 0:
final_samplesheet_data.extend(
sa._data) # add non single cell samples info to final data
sample_data = \
pd.DataFrame(final_samplesheet_data).\
set_index('Sample_ID') # get sample info from final data
merged_data = \
merged_qc_info.\
join(\
sample_data,
how='inner',
on='Sample_ID',
lsuffix='',
rsuffix='_sa') # merge sample data with qc data
required_headers = \
['Sample_ID',
'Sample_Name',
'FastqFile',
'TotalReads',
'index']
if 'index2' in list(sample_data.columns):
required_headers.append('index2')
required_headers.\
extend(\
['Fastqc',
'Fastqscreen']) # create header order
merged_data['FastqFile'] = \
merged_data['FastqFile'].\
map(lambda path: os.path.basename(path)) # keep only fastq filename
qc_merged_data = \
merged_data.loc[:,required_headers].\
to_dict(orient='records') # extract final data
return required_headers, qc_merged_data
except:
raise