def release_run(self, run_name):
os.chdir(os.path.join(self.config['root_dir'], run_name, 'TEMP'))
fastq_files = glob.glob('*_sequence.txt.gz')
released_files = []
removed_files = []
for fastq_file in fastq_files:
status = self.get_release_status_for_file(filename=fastq_file)
if status == 1 or status == 0:
os.remove(fastq_file)
removed_files.append(fastq_file)
elif status == 2:
released_files.append(fastq_file)
os.chdir('..')
os.rename('TEMP', 'COMPLETED')
os.mknod(os.path.join('COMPLETED', 'sync.me'))
hostname = socket.gethostname()
subject = '({}) Run {} released'.format(hostname, run_name)
message = '''Run {run_name} on {hostname} has been released.
These files have been marked for import:
{imported}
These files will not be released:
{removed}
'''
message = message.format(run_name=run_name, hostname=hostname,
imported='\n'.join(sorted(released_files)),
removed='\n'.join(sorted(removed_files)))
emailer.send_mail(api_key=self.config['email']['EMAIL_HOST_PASSWORD'],
to=self.config['addresses']['to'],
cc=self.config['addresses']['cc'],
reply_to=self.config['addresses']['reply-to'],
subject=subject,
content=message)
self.update_run_status(run_name, '6')
def process_run(run_config=None, config=None):
# send email announcing preprocessing initiation
subject = "{} - Run {} Preprocessing Initiated".format(socket.gethostname(), run_config["run_name"])
message = "Preprocessing has been started for run\n{}".format(run_config["run_name"])
emailer.send_mail(
api_key=config["email"]["EMAIL_HOST_PASSWORD"],
to=config["addresses"]["to"],
cc=config["addresses"]["cc"],
reply_to=config["addresses"]["reply-to"],
subject=subject,
content=message,
)
response = requests.get(os.path.join(config["seqConfig"]["URL_set_run_status"], run_config["run_name"], "3"))
print response.text
# collect the lanes by barcode. Each barcode length requires its own process.
lane_collections = collect_lanes_by_barcode_len(run_config=run_config)
base_calls_dir = os.path.join(config["root_dir"], run_config["run_name"], config["BaseCalls_dir"])
# TEMP directory will hold all symlinks to processed data
try:
os.mkdir(os.path.join(config["root_dir"], run_config["run_name"], "TEMP"))
except OSError:
print "TEMP dir already exists. continuing"
for barcode_length in lane_collections:
"""
The conversion software must run once for each barcode length,
requiring newly generated SampleSheet.csv files each time
"""
# build samplesheet
generate_support_files(
run_config=run_config, config=config, lanes=lane_collections[barcode_length], path=base_calls_dir
)
# execute configureBclToFastq and make -j num_cores
bcl_to_fastq(run_config=run_config, config=config, barcode_len=barcode_length)
# Post-demultiplexing steps:
# link sequence.txt.gz files
link_files(run_config=run_config)
# update the run status on seq-config
response = requests.get(os.path.join(config["seqConfig"]["URL_set_run_status"], run_config["run_name"], "4"))
print response.text
# grab all the demux summary files, email them and send to seq-config
demultiplex_files = find_demultiplex_files()
post_demultiplex_files(config=config, run_name=run_config["run_name"])
subj = "Preprocessing complete: {}".format(run_config["run_name"])
message = """
Preprocessing complete for run: {}
Server: {}
"""
message = message.format(run_config["run_name"], socket.gethostname())
emailer.send_mail(
api_key=config["email"]["EMAIL_HOST_PASSWORD"],
to=config["addresses"]["to"],
cc=config["addresses"]["cc"],
reply_to=config["addresses"]["reply-to"],
subject=subj,
content=message,
html_files=demultiplex_files,
)
# tasks that can run in the meantime:
fastqc.run_fastqc(config=config, run_config=run_config)
fastqc.send_fastqc_to_server(config=config, run_config=run_config)
def bcl_to_fastq(run_config=None, config=None, barcode_len=None):
"""
This needs to be performed in the /raid/illumina_production/<run_name> dir
runs the BCLToFastq Illumina software in 2 stages:
1. configuration - build directory structure, and supporting files
2. run process described in Makefile with 'make' over the available cores
Also sends announcement emails if processes have errored out
:param run_config:
:param config:
:param barcode_len:
:return:
"""
cpu_count = str(multiprocessing.cpu_count())
os.chdir(os.path.join(config["root_dir"], run_config["run_name"]))
cmd = build_configureBclToFastq_command(run_config=run_config, config=config, barcode_len=barcode_len)
try:
with open(
os.path.join(config["root_dir"], run_config["run_name"], "configureBclToFastq.log.out"), "w"
) as out, open(
os.path.join(config["root_dir"], run_config["run_name"], "configureBclToFastq.log.err"), "w"
) as err:
print >> out, "{}".format(cmd)
proc = subprocess.Popen(cmd, stdout=out, stderr=err, shell=True)
proc.communicate()
exit_code = proc.returncode
if exit_code > 0:
print >> sys.stderr, "ERROR [{}]: configureBclToFastq.pl command failed!!!\nExiting.".format(exit_code)
server_name = get_hostname()
subj = "Error processing {} : {}".format(server_name, run_config["run_name"])
content = "Server: {}\nRun name: {}\n Error running 'configureBclToFastq.pl'".format(
server_name, run_config["run_name"]
)
emailer.send_mail(
api_key=config["email"]["EMAIL_HOST_PASSWORD"],
to=config["addresses"]["to"],
cc=config["addresses"]["cc"],
reply_to=config["addresses"]["reply-to"],
subject=subj,
content=content,
)
sys.exit(1)
except:
print >> sys.stderr, "ERROR: unable to run configureBclToFastq. Exiting."
raise
os.chdir("Data/Intensities/BaseCalls/Unaligned")
try:
with open("BclToFastq.log.out", "w") as out, open("BclToFastq.log.err", "w") as err:
proc = subprocess.Popen(["make -j " + cpu_count], stdout=out, stderr=err, shell=True)
proc.communicate()
exit_code = proc.wait()
if exit_code > 0:
print >> sys.stderr, "ERROR: make command failed!!!\nExiting."
server_name = get_hostname()
subj = "Error processing {} : {}".format(server_name, run_config["run_name"])
content = "Server: {}\nRun name: {}\n Error running 'make'".format(server_name, run_config["run_name"])
emailer.send_mail(
api_key=config["email"]["EMAIL_HOST_PASSWORD"],
to=config["addresses"]["to"],
cc=config["addresses"]["cc"],
reply_to=config["addresses"]["reply-to"],
subject=subj,
content=content,
)
sys.exit(1)
except:
print >> sys.stderr, "ERROR: unable to run 'make' ({})".format(run_config["run_name"])
raise
os.chdir("..")
try:
os.rename("Unaligned", "Unaligned" + str(barcode_len)) # in case multiple demux cycles are required
except OSError:
print >> sys.stderr, "Warning! moving Unaligned{} directory to Unaligned{}.old".format(
str(barcode_len), str(barcode_len)
)
os.rename("Unaligned" + str(barcode_len), "old.Unaligned" + str(barcode_len))
os.rename("Unaligned", "Unaligned" + str(barcode_len))
os.rename("SampleSheet.csv", "SampleSheet" + str(barcode_len) + ".csv")
os.chdir(os.path.join(config["root_dir"], run_config["run_name"]))