def results(lims, process_id):
"""Upload bioanalyzer results to artifacts."""
process = Process(lims, id=process_id)
sample_measurements = {}
# Parse File
for output in process.all_outputs(unique=True):
if output.name == 'Bioanalyzer Output':
bioanalyzer_result_file = output.files[0]
for line in lims.get_file_contents(
bioanalyzer_result_file.id).split('\n'):
if line.startswith('Sample Name'):
sample = line.rstrip().split(',')[1]
elif line.startswith('Region 1'):
line = re.sub(
r'"([0-9]+),([0-9\.]+)"', r'\1\2', line
) # Fix remove thousands seperator (,) and quotes ("")
size = line.rstrip().split(',')[5]
sample_measurements[sample] = int(size)
# Set UDF
for artifact in process.all_outputs():
if artifact.name in sample_measurements:
artifact.udf['Dx Fragmentlengte (bp)'] = sample_measurements[
artifact.name]
artifact.put()
def results(lims, process_id):
"""Upload tapestation results to artifacts."""
process = Process(lims, id=process_id)
sample_size_measurements = {}
sample_concentration_measurements = {}
# Parse File
for output in process.all_outputs(unique=True):
if output.name == 'TapeStation Output':
tapestation_result_file = output.files[0]
for line in lims.get_file_contents(
tapestation_result_file.id).split('\n'):
if line.startswith('FileName'):
header = line.split(',')
if 'Size [bp]' in header: # Tapestation compact peak table
size_index = header.index('Size [bp]')
concentration_index = None
else: # Tapestation compact region table
size_index = header.index('Average Size [bp]')
try:
concentration_index = header.index(
u'Conc. [pg/\xb5l]') # micro sign
concentration_correction = 1000 # Used to transform pg/ul to ng/ul
except ValueError:
concentration_index = header.index(
u'Conc. [ng/\xb5l]') # micro sign
concentration_correction = 1
sample_index = header.index('Sample Description')
elif line:
data = line.split(',')
sample = data[sample_index]
if sample != 'Ladder':
if data[size_index]:
size = int(data[size_index])
sample_size_measurements[sample] = size
if concentration_index and data[concentration_index]:
# Correct concentration
concentration = float(data[concentration_index]
) / concentration_correction
sample_concentration_measurements[
sample] = concentration
# Set UDF
for artifact in process.all_outputs():
if artifact.name not in [
'TapeStation Output', 'TapeStation Samplesheet',
'TapeStation Sampleplots PDF'
]:
sample_name = artifact.name.split('_')[0]
if sample_name in sample_size_measurements:
artifact.udf[
'Dx Fragmentlengte (bp)'] = sample_size_measurements[
sample_name]
if sample_name in sample_concentration_measurements:
artifact.udf[
'Dx Concentratie fluorescentie (ng/ul)'] = sample_concentration_measurements[
sample_name]
artifact.put()
def main(lims, args):
p = Process(lims, id=args.pid)
log = []
datamap = {}
wsname = None
username = "******".format(p.technician.first_name,
p.technician.last_name)
user_email = p.technician.email
for art in p.all_inputs():
if len(art.samples) != 1:
log.append(
"Warning : artifact {0} has more than one sample".format(
art.id))
for sample in art.samples:
#take care of lamda DNA
if sample.project:
if sample.project.id not in datamap:
datamap[sample.project.id] = [sample.name]
else:
datamap[sample.project.id].append(sample.name)
for art in p.all_outputs():
try:
wsname = art.location[0].name
break
except:
pass
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
for pid in datamap:
pj = Project(lims, id=pid)
if len(datamap[pid]) > 1:
rnt = "{0} samples planned for {1}".format(len(datamap[pid]),
wsname)
else:
rnt = "{0} sample planned for {1}".format(len(datamap[pid]),
wsname)
running_note = {
"note": rnt,
"user": username,
"email": user_email,
"category": "Workset"
}
write_note_to_couch(pid, now, running_note, lims.get_uri())
log.append(
"Updated project {0} : {1}, {2} samples in this workset".format(
pid, pj.name, len(datamap[pid])))
with open("EPP_Notes.log", "w") as flog:
flog.write("\n".join(log))
for out in p.all_outputs():
#attach the log file
if out.name == "RNotes Log":
attach_file(os.path.join(os.getcwd(), "EPP_Notes.log"), out)
sys.stderr.write("Updated {0} projects successfully".format(
len(list(datamap.keys()))))
def main(lims, args):
p=Process(lims, id=args.pid)
log=[]
datamap={}
wsname=None
username="******".format(p.technician.first_name, p.technician.last_name)
user_email=p.technician.email
for art in p.all_inputs():
if len(art.samples)!=1:
log.append("Warning : artifact {0} has more than one sample".format(art.id))
for sample in art.samples:
#take care of lamda DNA
if sample.project:
if sample.project.id not in datamap:
datamap[sample.project.id]=[sample.name]
else:
datamap[sample.project.id].append(sample.name)
for art in p.all_outputs():
try:
wsname=art.location[0].name
break
except:
pass
now=datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
for pid in datamap:
pj=Project(lims, id=pid)
running_notes=json.loads(pj.udf['Running Notes'])
if len(datamap[pid]) > 1:
rnt="{0} samples planned for {1}".format(len(datamap[pid]), wsname)
else:
rnt="{0} sample planned for {1}".format(len(datamap[pid]), wsname)
running_notes[now]={"note": rnt, "user" : username, "email":user_email, "category":"Workset"}
pj.udf['Running Notes']=json.dumps(running_notes)
pj.put()
log.append("Updated project {0} : {1}, {2} samples in this workset".format(pid,pj.name, len(datamap[pid])))
with open("EPP_Notes.log", "w") as flog:
flog.write("\n".join(log))
for out in p.all_outputs():
#attach the log file
if out.name=="RNotes Log":
attach_file(os.path.join(os.getcwd(), "EPP_Notes.log"), out)
sys.stderr.write("Updated {0} projects successfully".format(len(datamap.keys())))
def main(lims,args,epp_logger):
p = Process(lims,id = args.pid)
udf_factor1 = 'Concentration (ng/ul)'
result_udf = 'Concentration nM'
udf_factor2 = 'Size (bp)'
if args.aggregate:
artifacts = p.all_inputs(unique=True)
else:
all_artifacts = p.all_outputs(unique=True)
artifacts = filter(lambda a: a.output_type == "Analyte", all_artifacts)
# print rrtifacts
correct_artifacts, wrong_factor1 = check_udf_is_defined(artifacts, udf_factor1)
correct_artifacts, wrong_factor2 = check_udf_is_defined(correct_artifacts, udf_factor2)
f = open(args.res, "a")
if correct_artifacts:
apply_calculations(lims, correct_artifacts, udf_factor1,
udf_factor2, result_udf, epp_logger, f)
f.close()
d = {'ca': len(correct_artifacts),
'ia': len(wrong_factor1)+ len(wrong_factor2) }
abstract = ("Updated {ca} artifact(s), skipped {ia} artifact(s) with "
"wrong and/or blank values for some udfs.").format(**d)
print >> sys.stderr, abstract # stderr will be logged and printed in GUI
def main(lims, args, epp_logger):
p = Process(lims, id=args.pid)
udf_check = "Conc. Units"
value_check = "ng/ul"
udf_factor1 = "Concentration"
udf_factor2 = "Volume (ul)"
result_udf = "Amount (ng)"
if args.aggregate:
artifacts = p.all_inputs(unique=True)
else:
all_artifacts = p.all_outputs(unique=True)
artifacts = filter(lambda a: a.output_type == "ResultFile", all_artifacts)
correct_artifacts, wrong_factor1 = check_udf_is_defined(artifacts, udf_factor1)
correct_artifacts, wrong_factor2 = check_udf_is_defined(correct_artifacts, udf_factor2)
correct_artifacts, wrong_value = check_udf_has_value(correct_artifacts, udf_check, value_check)
if correct_artifacts:
apply_calculations(lims, correct_artifacts, udf_factor1, "*", udf_factor2, result_udf, epp_logger, p)
d = {"ca": len(correct_artifacts), "ia": len(wrong_factor1) + len(wrong_factor2) + len(wrong_value)}
abstract = (
"Updated {ca} artifact(s), skipped {ia} artifact(s) with " "wrong and/or blank values for some udfs."
).format(**d)
print >>sys.stderr, abstract # stderr will be logged and printed in GUI
def main(lims, args, epp_logger):
p = Process(lims, id=args.pid)
udf_check = 'Conc. Units'
value_check = 'ng/ul'
concentration_udf = 'Concentration'
size_udf = 'Size (bp)'
if args.aggregate:
artifacts = p.all_inputs(unique=True)
else:
all_artifacts = p.all_outputs(unique=True)
artifacts = filter(lambda a: a.output_type == "ResultFile",
all_artifacts)
correct_artifacts, no_concentration = check_udf_is_defined(
artifacts, concentration_udf)
correct_artifacts, no_size = check_udf_is_defined(correct_artifacts,
size_udf)
correct_artifacts, wrong_value = check_udf_has_value(
correct_artifacts, udf_check, value_check)
apply_calculations(lims, correct_artifacts, concentration_udf, size_udf,
udf_check, epp_logger)
d = {
'ca': len(correct_artifacts),
'ia': len(wrong_value) + len(no_size) + len(no_concentration)
}
abstract = ("Updated {ca} artifact(s), skipped {ia} artifact(s) with "
"wrong and/or blank values for some udfs.").format(**d)
print >> sys.stderr, abstract # stderr will be logged and printed in GUI
def main(lims, args):
currentStep = Process(lims, id=args.pid)
driver_file_out = None
driver = []
ar_driver = {}
valid_cols = set()
for output in currentStep.all_outputs():
if output.name == "Driver File":
driver_file_out = output
elif output.output_type == "ResultFile":
location_ar = output.location[1].split(":")
valid_cols.add(location_ar[0])
#idx = (ord(location_ar[0])-65)*12 + int(location_ar[1])-1
ar_driver[output.location[1].replace(":",
"")] = output.samples[0].name
col_idx = -1
for column in sorted(list(valid_cols)):
col_idx += 1
for i in xrange(1, 13):
location = "{}{}".format(column, i)
driver.append((col_idx * 12 + i, location,
ar_driver.get(location,
"ladder" if i == 12 else "")))
with open("frag_an_driver.csv", "w") as f:
for line in driver:
f.write("{0},{1},{2}\n".format(line[0], line[1], line[2]))
lims.upload_new_file(driver_file_out, "frag_an_driver.csv")
def main(lims, args, epp_logger):
p = Process(lims, id=args.pid)
udf_check = "Conc. Units"
value_check = "ng/ul"
concentration_udf = "Concentration"
size_udf = "Size (bp)"
if args.aggregate:
artifacts = p.all_inputs(unique=True)
else:
all_artifacts = p.all_outputs(unique=True)
artifacts = filter(lambda a: a.output_type == "ResultFile", all_artifacts)
correct_artifacts, no_concentration = check_udf_is_defined(artifacts, concentration_udf)
correct_artifacts, no_size = check_udf_is_defined(correct_artifacts, size_udf)
correct_artifacts, wrong_value = check_udf_has_value(correct_artifacts, udf_check, value_check)
apply_calculations(lims, correct_artifacts, concentration_udf, size_udf, udf_check, epp_logger)
d = {"ca": len(correct_artifacts), "ia": len(wrong_value) + len(no_size) + len(no_concentration)}
abstract = (
"Updated {ca} artifact(s), skipped {ia} artifact(s) with " "wrong and/or blank values for some udfs."
).format(**d)
print >>sys.stderr, abstract # stderr will be logged and printed in GUI
class File2UDF():
def __init__(self, lims, pid, inp_is_source):
self.lims = lims
self.source_is_input_artifact = inp_is_source
self.process = Process(lims, id=pid)
self.input_output_maps = self.process.input_output_maps
self.artifacts = {}
self.failed_arts = False
self.passed_arts = 0
self.all_arts = 0
self.result_file = None
def get_artifacts(self):
"""Gets input and output artifacts from input_output_maps.
Gets the location from the inart, but saves the outart in the artifacts dictionary."""
for inp, outp in self.input_output_maps:
if outp.get("output-generation-type") == "PerAllInputs":
continue
in_art = Artifact(self.lims, id=inp['limsid'])
out_art = Artifact(self.lims, id=outp['limsid'])
source_art = in_art if self.source_is_input_artifact == True else out_art
col, row = source_art.location[1].split(':')
well = col + row
self.artifacts[well] = out_art
self.all_arts += 1
def get_result_file(self, result_file):
"""Reads file from args if present. Otherwise searches for file in outarts with
name Tapestation CSV"""
if result_file and os.path.isfile(result_file):
self.result_file = result_file
else:
files = filter(lambda a: a.name in ["Tapestation CSV"],
self.process.all_outputs())
if len(files) > 1:
sys.exit('more than one Qubit Result File')
else:
self.result_file = files[0].files[0].content_location.split(
'scilifelab.se')[1]
def set_udfs(self):
"""Reads the csv and sets the average size bp for each sample"""
with open(self.result_file) as f:
d = DictReader(f, delimiter=',')
l = list(d)
for sample in l:
well = sample.get('WellId')
size = sample.get('Average Size [bp]')
if size and well in self.artifacts:
art = self.artifacts[well]
art.udf['Size (bp)'] = int(size)
art.put()
self.passed_arts += 1
else:
self.failed_arts = True
def main(lims, args):
process = Process(lims, id=args.pid)
artifacts = process.all_outputs()
updated_arts = 0
for art in artifacts:
updated_arts += get_buffer(art, args.process_types)
print >> sys.stderr, 'Updated ' + str(
updated_arts) + ' samples with volume from Buffer step.'
def main(args):
log = []
lims = Lims(BASEURI, USERNAME, PASSWORD)
process = Process(lims, id=args.pid)
for io in process.input_output_maps:
if io[1]['output-generation-type'] != 'PerInput':
continue
try:
starting_amount = obtain_amount(io[0]['uri'])
except Exception as e:
log.append(str(e))
starting_amount = 0
log.append("Starting amount of {} : {} ng".format(
io[0]['uri'].samples[0].name, starting_amount))
current_amount = starting_amount
#preps
preps = lims.get_processes(
inputartifactlimsid=io[0]['uri'].id,
type=["Setup Workset/Plate", "Amount confirmation QC"])
for pro in preps:
if pro.id == args.pid:
continue # skip the current step
for prepio in pro.input_output_maps:
if prepio[1]['output-generation-type'] == 'PerInput' and prepio[
0]['uri'].id == io[0]['uri'].id:
if "Amount taken (ng)" in prepio[1][
'uri'].udf: #should always be true
prep_amount = prepio[1]['uri'].udf["Amount taken (ng)"]
log.append(
"Removing {} ng for prep {} for sample {}".format(
prep_amount, pro.id,
io[0]['uri'].samples[0].name))
current_amount = current_amount - prep_amount
else:
log.append(
"No Amount Taken found for prep {} of sample {}".
format(pro.id, io[0]['uri'].samples[0].name))
if current_amount < 0:
log.append(
"Estimated amount for sample {} is {}, correcting to zero".
format(io[0]['uri'].samples[0].name, current_amount))
current_amount = 0
update_output_values(io[0]['uri'], io[1]['uri'], current_amount)
with open("amount_check_log.txt", "w") as f:
f.write("\n".join(log))
for out in process.all_outputs():
if out.name == "QC Assignment Log File":
for f in out.files:
lims.request_session.delete(f.uri)
lims.upload_new_file(out, "amount_check_log.txt")
def main(lims, args, logger):
logger.debug("Getting Process with ID %s", args.pid)
p = Process(lims, id=args.pid)
logger.debug(p)
# Precompute lookup dictionary for output artifacts
output_artifacts = {
artifact.id: artifact
for artifact in p.all_outputs(unique=True)
}
logger.debug(output_artifacts)
logger.debug(p.input_output_maps)
input_output_map = {}
for input_, output_ in p.input_output_maps:
if output_["output-generation-type"] == "PerInput":
input_output_map[input_["limsid"]] = output_["limsid"]
logger.debug("output_artifacts: %s", output_artifacts)
logger.debug("input_output_map: %s", input_output_map)
tapestation_file = get_tapestation_file(p, args.tapestation_csv)
if not tapestation_file:
raise (RuntimeError(
"Cannot find the TapeStation csv file, are you sure it has been uploaded?"
))
logger.debug(tapestation_file)
outputs = []
measured_peaks = parse_tapestation_csv(tapestation_file.splitlines(),
args.min_fragsize,
args.max_fragsize)
for well, peaks in measured_peaks.items():
fragment_size = -1
if len(peaks) == 1:
fragment_size = peaks[0].Size
logger.debug([well, peaks, fragment_size])
# Find input artifact, this has well information
artifact = find_input_in_well(well, p)
# Find output artifact, this has the UDF where we store the peak size
output = output_artifacts[input_output_map[artifact.id]]
logger.debug("Output artifact: %s", output)
logger.debug("Modifying UDF '%s' of artifact '%s'", args.udf_fragsize,
artifact)
output.udf[args.udf_fragsize] = fragment_size
outputs.append(output)
for out in outputs:
out.put()
def main(lims, args):
process = Process(lims, id=args.pid)
# Read in run recipe file
for outart in process.all_outputs():
if outart.type == 'ResultFile' and outart.name == 'Run Recipe':
try:
fid = outart.files[0].id
file_name = outart.files[0].original_location
content = lims.get_file_contents(id=fid).read()
except:
raise(RuntimeError("Cannot access the run recipe file."))
break
with open("/srv/mfs/NovaSeq_data/gls_recipe_novaseq/{}".format(file_name), 'w') as sf:
sf.write(content)
def main(lims, args):
process = Process(lims, id=args.pid)
# Read in run recipe file
for outart in process.all_outputs():
if outart.type == 'ResultFile' and outart.name == 'Run Recipe':
try:
fid = outart.files[0].id
file_name = outart.files[0].original_location
content = lims.get_file_contents(id=fid).read()
if isinstance(content, bytes):
content = content.decode('utf-8')
except:
raise RuntimeError("Cannot access the run recipe file.")
break
with open("/srv/mfs/NovaSeq_data/gls_recipe_novaseq/{}".format(file_name),
'w') as sf:
sf.write(content)
def set_qc_flag(lims, process_id):
"""Set qc flags based on Dx Fragmentlengte (bp) udf and criterea set by users."""
process = Process(lims, id=process_id)
min_size = process.udf['Minimale fragmentlengte (bp)']
max_size = process.udf['Maximale fragmentlengte (bp)']
for artifact in process.all_outputs():
try:
size = artifact.udf['Dx Fragmentlengte (bp)']
if size >= min_size and size <= max_size:
artifact.qc_flag = 'PASSED'
else:
artifact.qc_flag = 'FAILED'
except KeyError:
artifact.qc_flag = 'FAILED'
finally:
artifact.put()
def main(lims, args, epp_logger):
p = Process(lims, id=args.pid)
if p.type.name == 'Aggregate QC (Library Validation) 4.0':
udf_check = 'Conc. Units'
value_check = ['nM', 'pM']
udf_factor1 = 'Concentration'
udf_factor2 = 'Volume (ul)'
result_udf = 'Amount (fmol)'
else:
udf_check = 'Conc. Units'
value_check = ['ng/ul', 'ng/uL']
udf_factor1 = 'Concentration'
udf_factor2 = 'Volume (ul)'
result_udf = 'Amount (ng)'
if args.aggregate:
artifacts = p.all_inputs(unique=True)
else:
all_artifacts = p.all_outputs(unique=True)
artifacts = [a for a in all_artifacts if a.output_type == "ResultFile"]
correct_artifacts, wrong_factor1 = check_udf_is_defined(
artifacts, udf_factor1)
correct_artifacts, wrong_factor2 = check_udf_is_defined(
correct_artifacts, udf_factor2)
correct_artifacts, wrong_value = check_udf_has_value(
correct_artifacts, udf_check, value_check)
if correct_artifacts:
apply_calculations(lims, correct_artifacts, udf_factor1, '*',
udf_factor2, result_udf, epp_logger, p)
d = {
'ca': len(correct_artifacts),
'ia': len(wrong_factor1) + len(wrong_factor2) + len(wrong_value)
}
abstract = ("Updated {ca} artifact(s), skipped {ia} artifact(s) with "
"wrong and/or blank values for some udfs.").format(**d)
print(abstract,
file=sys.stderr) # stderr will be logged and printed in GUI
def main(args):
log = []
lims = Lims(BASEURI,USERNAME,PASSWORD)
process = Process(lims, id=args.pid)
for io in process.input_output_maps:
if io[1]['output-generation-type'] != 'PerInput':
continue
try:
starting_amount = obtain_amount(io[0]['uri'])
except Exception as e:
log.append(str(e))
starting_amount = 0
log.append("Starting amount of {} : {} ng".format(io[0]['uri'].samples[0].name, starting_amount))
current_amount = starting_amount
#preps
preps = lims.get_processes(inputartifactlimsid=io[0]['uri'].id, type=["Setup Workset/Plate", "Amount confirmation QC"])
for pro in preps:
if pro.id == args.pid:
continue # skip the current step
for prepio in pro.input_output_maps:
if prepio[1]['output-generation-type'] == 'PerInput' and prepio[0]['uri'].id == io[0]['uri'].id:
if "Amount taken (ng)" in prepio[1]['uri'].udf: #should always be true
prep_amount = prepio[1]['uri'].udf["Amount taken (ng)"]
log.append("Removing {} ng for prep {} for sample {}".format(prep_amount, pro.id, io[0]['uri'].samples[0].name))
current_amount = current_amount - prep_amount
else:
log.append("No Amount Taken found for prep {} of sample {}".format(pro.id, io[0]['uri'].samples[0].name))
if current_amount < 0:
log.append("Estimated amount for sample {} is {}, correcting to zero".format(io[0]['uri'].samples[0].name, current_amount))
current_amount = 0
update_output_values(io[0]['uri'], io[1]['uri'], current_amount)
with open("amount_check_log.txt", "w") as f:
f.write("\n".join(log))
for out in process.all_outputs():
if out.name == "QC Assignment Log File" :
for f in out.files:
lims.request_session.delete(f.uri)
lims.upload_new_file(out, "amount_check_log.txt")
def main(lims, args):
process = Process(lims, id=args.pid)
pools = filter(lambda a: a.output_type == "Analyte",
process.all_outputs(unique=True))
CFP = CalculationsForPools(process, pools)
CFP.calculate_volumes_for_pooling()
CFP.calculate_volume_wather()
abstract = ''
if CFP.amount_fail:
abstract += 'Input amount low for samples in pool. Generate placement map for more info. '
if CFP.failed:
missing = ', '.join(list(set(CFP.missing_udfs)))
abstract += 'Failed to perform calculations for ' + str(
CFP.failed
) + ' pools. Some of the following udfs are invalid or missing: ' + missing + '. '
if CFP.okej:
abstract += 'Performed calculations for ' + str(CFP.okej) + ' pools.'
if CFP.failed or CFP.amount_fail:
sys.exit(abstract)
else:
print >> sys.stderr, abstract
def main(lims, args):
log = []
content = None
process = Process(lims, id=args.pid)
# Copy Read and index parameter from the step "Load to Flowcell (NovaSeq 6000 v2.0)"
UDF_to_copy = [
'Read 1 Cycles', 'Read 2 Cycles', 'Index Read 1', 'Index Read 2'
]
for i in UDF_to_copy:
if process.parent_processes()[0].udf.get(i):
process.udf[i] = process.parent_processes()[0].udf[i]
process.put()
# Fetch Flowcell ID
FCID = process.parent_processes()[0].output_containers()[0].name
for outart in process.all_outputs():
if outart.type == 'ResultFile' and outart.name == 'Run Info':
try:
lims.upload_new_file(
outart,
max(glob.glob(
'/srv/mfs/NovaSeq_data/*{}/RunInfo.xml'.format(FCID)),
key=os.path.getctime))
except:
raise RuntimeError("No RunInfo.xml Found!")
elif outart.type == 'ResultFile' and outart.name == 'Run Parameters':
try:
lims.upload_new_file(
outart,
max(glob.glob(
'/srv/mfs/NovaSeq_data/*{}/RunParameters.xml'.format(
FCID)),
key=os.path.getctime))
except:
raise RuntimeError("No RunParameters.xml Found!")
def main(lims, args, epp_logger):
p = Process(lims, id=args.pid)
udf_check = 'Conc. Units'
value_check = 'ng/ul'
udf_factor1 = 'Concentration'
udf_factor2 = 'Volume (ul)'
result_udf = 'Amount (ng)'
if args.aggregate:
artifacts = p.all_inputs(unique=True)
else:
all_artifacts = p.all_outputs(unique=True)
artifacts = filter(lambda a: a.output_type == "ResultFile",
all_artifacts)
correct_artifacts, wrong_factor1 = check_udf_is_defined(
artifacts, udf_factor1)
correct_artifacts, wrong_factor2 = check_udf_is_defined(
correct_artifacts, udf_factor2)
correct_artifacts, wrong_value = check_udf_has_value(
correct_artifacts, udf_check, value_check)
if correct_artifacts:
apply_calculations(lims, correct_artifacts, udf_factor1, '*',
udf_factor2, result_udf, epp_logger, p)
d = {
'ca': len(correct_artifacts),
'ia': len(wrong_factor1) + len(wrong_factor2) + len(wrong_value)
}
abstract = ("Updated {ca} artifact(s), skipped {ia} artifact(s) with "
"wrong and/or blank values for some udfs.").format(**d)
print >> sys.stderr, abstract # stderr will be logged and printed in GUI
def main(lims, args):
log=[]
thisyear=datetime.now().year
content = None
if args.mytest:
test()
else:
process = Process(lims, id=args.pid)
if process.type.name == 'Cluster Generation (HiSeq X) 1.0':
header = gen_X_header(process)
reads = gen_X_reads_info(process)
(data, obj) = gen_X_lane_data(process)
check_index_distance(obj, log)
content = "{}{}{}".format(header, reads, data)
if os.path.exists("/srv/mfs/samplesheets/HiSeqX/{}".format(thisyear)):
try:
with open("/srv/mfs/samplesheets/HiSeqX/{}/{}.csv".format(thisyear, obj[0]['fc']), 'w') as sf:
sf.write(content)
os.chmod("/srv/mfs/samplesheets/HiSeqX/{}/{}.csv".format(thisyear, obj[0]['fc']), 0664)
except Exception as e:
log.append(e)
elif process.type.name == 'Cluster Generation (Illumina SBS) 4.0':
(content, obj) = gen_Hiseq_lane_data(process)
check_index_distance(obj, log)
if os.path.exists("/srv/mfs/samplesheets/{}".format(thisyear)):
try:
with open("/srv/mfs/samplesheets/{}/{}.csv".format(thisyear, obj[0]['fc']), 'w') as sf:
sf.write(content)
os.chmod("/srv/mfs/samplesheets/{}/{}.csv".format(thisyear, obj[0]['fc']), 0664)
except Exception as e:
log.append(e)
elif process.type.name == 'Denature, Dilute and Load Sample (MiSeq) 4.0':
header = gen_Miseq_header(process)
reads = gen_Miseq_reads(process)
settings = gen_Miseq_settings(process)
(data, obj) = gen_Miseq_data(process)
check_index_distance(obj, log)
content = "{}{}{}{}".format(header, reads, settings, data)
if not args.test:
for out in process.all_outputs():
if out.name == "Scilifelab SampleSheet" :
ss_art = out
elif out.name == "Scilifelab Log" :
log_id= out.id
elif out.type == "Analyte":
fc_name = out.location[0].name
with open("{}.csv".format(fc_name), "w", 0o664) as f:
f.write(content)
os.chmod("{}.csv".format(fc_name),0664)
for f in ss_art.files:
lims.request_session.delete(f.uri)
lims.upload_new_file(ss_art, "{}.csv".format(fc_name))
if log:
with open("{}_{}_Error.log".format(log_id, fc_name), "w") as f:
f.write('\n'.join(log))
sys.stderr.write("Errors were met, check the log.")
sys.exit(1)
else:
print content
print log
def samplesheet_multiplex_library_pool(lims, process_id, output_file):
"""Create manual pipetting samplesheet for multiplexing(pooling) samples."""
process = Process(lims, id=process_id)
inputs = list(set(process.all_inputs()))
outputs = list(set(process.all_outputs()))
sample_concentration = {}
sample_size = {}
trio_statuses = {}
ul_sample = {}
ng_sample = {}
udf_output = []
udf_ul_sample = {}
udf_name_ul_sample = {}
plate_id = {}
well_id = {}
pools_not_3 = []
order = [
'A1', 'B1', 'C1', 'D1', 'E1', 'F1', 'G1', 'H1', 'A2', 'B2', 'C2', 'D2',
'E2', 'F2', 'G2', 'H2', 'A3', 'B3', 'C3', 'D3', 'E3', 'F3', 'G3', 'H3',
'A4', 'B4', 'C4', 'D4', 'E4', 'F4', 'G4', 'H4', 'A5', 'B5', 'C5', 'D5',
'E5', 'F5', 'G5', 'H5', 'A6', 'B6', 'C6', 'D6', 'E6', 'F6', 'G6', 'H6',
'A7', 'B7', 'C7', 'D7', 'E7', 'F7', 'G7', 'H7', 'A8', 'B8', 'C8', 'D8',
'E8', 'F8', 'G8', 'H8', 'A9', 'B9', 'C9', 'D9', 'E9', 'F9', 'G9', 'H9',
'A10', 'B10', 'C10', 'D10', 'E10', 'F10', 'G10', 'H10', 'A11', 'B11',
'C11', 'D11', 'E11', 'F11', 'G11', 'H11', 'A12', 'B12', 'C12', 'D12',
'E12', 'F12', 'G12', 'H12'
]
order = dict(zip(order, range(len(order))))
well_order = {}
sample_well_pool = []
# get input udfs 'Dx sample volume ul' and 'Dx Samplenaam' per output analyte
for output in outputs:
if output.type == 'Analyte':
if 'Dx sample volume (ul)' in output.udf and 'Dx Samplenaam' in output.udf:
udf_ul_sample[
output.name] = output.udf['Dx sample volume (ul)']
# if samplename is complete sequencename take only monsternummer
if re.search(r'U\d{6}\D{2}', output.udf['Dx Samplenaam']):
udf_name_ul_sample[
output.name] = output.udf['Dx Samplenaam'][9:]
else:
udf_name_ul_sample[
output.name] = output.udf['Dx Samplenaam']
udf_output.append(output.name)
# get concentration, size, containername and well per input artifact
for input in inputs:
sample = input.samples[0]
samplename = sample.name
if 'Dx Concentratie fluorescentie (ng/ul)' in input.udf:
measurement = input.udf['Dx Concentratie fluorescentie (ng/ul)']
qcflag = input.qc_flag
if qcflag == 'UNKNOWN' or 'PASSED':
sample_concentration[samplename] = measurement
if 'Dx Fragmentlengte (bp)' in input.udf:
measurement = input.udf['Dx Fragmentlengte (bp)']
qcflag = input.qc_flag
if qcflag == 'UNKNOWN' or 'PASSED':
sample_size[samplename] = measurement
plate_id[samplename] = input.container.name
placement = input.location[1]
placement = ''.join(placement.split(':'))
well_id[samplename] = placement
well_order[sample.name] = order[placement]
# get familystatus per sample in output analyte and determine trio composition if number of samples in pool = 3
for output in outputs:
if output.type == 'Analyte':
sample_given_ul = ''
if len(output.samples) == 3:
samplestatus = []
for sample in output.samples:
# First check GIAB controls
if 'CFGIAB' in sample.name.upper():
sample.udf['Dx Familie status'] = 'Kind'
elif 'PFGIAB' in sample.name.upper(
) or 'PMGIAB' in sample.name.upper():
sample.udf['Dx Familie status'] = 'Ouder'
if 'Dx Onderzoeksreden' in sample.udf and sample.udf[
'Dx Onderzoeksreden'] == 'Research':
samplestatus.append('Kind')
else:
samplestatus.append(sample.udf['Dx Familie status'])
if samplestatus == ['Kind'] * 3 or samplestatus == ['Ouder'
] * 3:
trio_statuses[output.name] = 'CCC'
elif sorted(samplestatus) == ['Kind', 'Ouder', 'Ouder']:
trio_statuses[output.name] = 'CPP'
elif sorted(samplestatus) == ['Kind', 'Kind', 'Ouder']:
trio_statuses[output.name] = 'CCP'
# if udfs 'Dx sample volume ul' and 'Dx Samplenaam' are not empty change trio status and do pre-calculation
if output.name in udf_output:
trio_statuses[output.name] = 'adapted'
for sample in output.samples:
if sample.name == udf_name_ul_sample[output.name]:
sample_given_ul = sample
ng_sample[
sample.
name] = library_dilution_calculator_fixed_volume(
sample_concentration[sample.name],
sample_size[sample.name],
udf_ul_sample[output.name])
for sample in output.samples:
if sample.name != udf_name_ul_sample[output.name]:
ng_sample[
sample.
name] = library_dilution_calculator_fixed_ng(
sample_concentration[sample.name],
sample_size[sample.name],
sample.udf['Dx Familie status'],
ng_sample[udf_name_ul_sample[output.name]],
sample_given_ul.udf['Dx Familie status'])
output.udf['Dx input pool (ng)'] = round(
ng_sample[output.samples[0].name] +
ng_sample[output.samples[1].name] +
ng_sample[output.samples[2].name], 2)
output.put()
else:
output.udf['Dx input pool (ng)'] = 750
output.put()
# if number of samples in pool is not 3 set trio status and prepare error warning output file
else:
trio_statuses[output.name] = 'not_3'
pools_not_3.append(output.name)
# calculation if udfs 'Dx sample volume ul' and 'Dx Samplenaam' are empty and not empty
if not sample_given_ul:
for sample in output.samples:
if 'Dx Onderzoeksreden' in sample.udf and sample.udf[
'Dx Onderzoeksreden'] == 'Research':
sample_pedigree = 'Kind'
else:
sample_pedigree = sample.udf['Dx Familie status']
ul_sample[sample.name] = library_dilution_calculator(
concentration=sample_concentration[sample.name],
size=sample_size[sample.name],
trio=trio_statuses[output.name],
pedigree=sample_pedigree,
ng=0)
else:
for sample in output.samples:
if sample.udf['Dx Onderzoeksreden'] == 'Research':
sample_pedigree = 'Kind'
else:
sample_pedigree = sample.udf['Dx Familie status']
ul_sample[sample.name] = library_dilution_calculator(
concentration=sample_concentration[sample.name],
size=sample_size[sample.name],
trio=trio_statuses[output.name],
pedigree=sample_pedigree,
ng=ng_sample[sample.name])
# sorting pools then wells for output file
sort_pool_name = output.name
if re.search(r'#\d_', sort_pool_name):
sort_pool_name = re.sub('#', '#0', sort_pool_name)
for sample in output.samples:
sample_well_pool.append([
sample, well_order[sample.name], sort_pool_name,
output.name
])
sorted_samples = sorted(sample_well_pool,
key=lambda sample: (sample[2], sample[1]))
# write output file per output analyte sorted on pool number
output_file.write('Sample\tul Sample\tPlaat_id\twell_id\tpool\n')
if pools_not_3:
output_file.write(
'De volgende pool(s) hebben een ander aantal samples dan 3: {pools}\n'
.format(pools=pools_not_3))
for sorted_sample in sorted_samples:
sample = sorted_sample[0]
output_file.write(
'{sample}\t{ul_sample:.2f}\t{plate_id}\t{well_id}\t{pool}\n'.
format(sample=sample.name,
ul_sample=ul_sample[sample.name],
plate_id=plate_id[sample.name],
well_id=well_id[sample.name],
pool=sorted_sample[3]))
def main(lims, args, logger):
p = Process(lims, id=args.pid)
# Precompute lookup dictionaries for output artifacts and input_output_maps
output_artifacts = {
artifact.id: artifact
for artifact in p.all_outputs(unique=True)
}
input_output_map = {}
for input_, output_ in p.input_output_maps:
if output_["output-generation-type"] == "PerInput":
input_output_map[input_["limsid"]] = output_["limsid"]
logger.info("output_artifacts: %s", output_artifacts)
logger.info("input_output_map: %s", input_output_map)
sparkfile = get_spark_file(p, args.sparkOutputFilename)
if not sparkfile:
raise (RuntimeError(
"Cannot find the Spark output file, are you sure it has been uploaded?"
))
workbook = xlrd.open_workbook(file_contents=sparkfile.read())
sheet = workbook.sheet_by_index(0)
well_re = re.compile("[A-Z][0-9]{1,2}")
if args.convertToNm:
fragment_size = format_fragment_size(args.fragmentSize)
outputs = []
for row_i in range(0, sheet.nrows):
if is_well(sheet.cell(row_i, 0).value, well_re):
well = sheet.cell(row_i, 0).value
if args.wellFromOutput:
artifact = find_output_in_well(well, p)
else:
artifact = find_input_in_well(well, p)
if not artifact:
raise (RuntimeError(
"Error! Cannot find sample at well position %s, row %s" %
(well, row_i)))
logger.info("Input artifact: %s", artifact)
if sheet.ncols > 2: # some files may be missing the "NoCalc" column
concentration = sheet.cell(row_i, 2).value
else:
concentration = sheet.cell(row_i, 1).value
if concentration == "NoCalc":
concentration = sheet.cell(row_i, 1).value
concentration = format_concentration(concentration)
logger.info("concentration: %s", concentration)
# Find output artifact
output = output_artifacts[input_output_map[artifact.id]]
logger.info("Output artifact: %s", output)
output.udf[args.concentrationUdf] = concentration
outputs.append(output)
if args.convertToNm:
concentration_nm = convert_to_nm(concentration, fragment_size)
output.udf[args.concentrationUdfNm] = concentration_nm
for out in outputs:
out.put()
def results(lims, process_id):
"""Upload tecan results to artifacts."""
process = Process(lims, id=process_id)
concentration_range = map(float, re.findall('[\d\.]+', process.udf['Concentratiebereik (ng/ul)']))
# Parse output file
for output in process.all_outputs(unique=True):
if output.name == 'Tecan Spark Output':
tecan_result_file = output.files[0]
tecan_file_order = ['Dx Fluorescentie (nM)', 'sample_name']
tecan_file_part = -1
measurements = {}
sample_measurements = {}
for line in lims.get_file_contents(tecan_result_file.id).data.split('\n'):
if not line.startswith('<>'):
data = line.rstrip().split('\t')
for index, value in enumerate(data[1:]):
value = value.rstrip()
if value:
coordinate = '{0}{1}'.format(data[0], str(index))
if tecan_file_order[tecan_file_part] == 'Dx Fluorescentie (nM)':
measurements[coordinate] = float(value)
elif tecan_file_order[tecan_file_part] == 'sample_name':
if value not in sample_measurements:
sample_measurements[value] = [measurements[coordinate]]
else:
sample_measurements[value].append(measurements[coordinate])
else:
tecan_file_part += 1
# Calculate linear regression for concentration
# Assumes no std duplicates
baseline_fluorescence = sample_measurements['Dx Tecan std 1'][0]
fluorescence_values = [
sample_measurements['Dx Tecan std 1'][0] - baseline_fluorescence,
sample_measurements['Dx Tecan std 2'][0] - baseline_fluorescence,
sample_measurements['Dx Tecan std 3'][0] - baseline_fluorescence,
sample_measurements['Dx Tecan std 4'][0] - baseline_fluorescence,
sample_measurements['Dx Tecan std 5'][0] - baseline_fluorescence,
sample_measurements['Dx Tecan std 6'][0] - baseline_fluorescence,
]
if process.udf['Reagentia kit'] == 'Quant-iT High-Sensitivity dsDNA kit':
ng_values = [0, 5, 10, 20, 40, 60, 80, 100]
fluorescence_values.append(sample_measurements['Dx Tecan std 7'][0] - baseline_fluorescence)
fluorescence_values.append(sample_measurements['Dx Tecan std 8'][0] - baseline_fluorescence)
elif process.udf['Reagentia kit'] == 'Quant-iT Broad Range dsDNA kit':
ng_values = [0, 50, 100, 200, 400, 600]
regression_slope = sum([x*y for x, y in zip(fluorescence_values, ng_values)]) / sum([x**2 for x in fluorescence_values])
rsquared = 1 - (sum([(y - x*regression_slope)**2 for x, y in zip(fluorescence_values, ng_values)]) / sum([y**2 for y in ng_values]))
# Set udf values
process.udf['R-squared waarde'] = rsquared
process.put()
artifact_count = {}
for artifact in process.all_outputs():
if artifact.name not in ['Tecan Spark Output', 'Tecan Spark Samplesheet', 'check gemiddelde concentratie', 'Label plaat']:
if len(artifact.samples) == 1: # Remove 'meet_id' from artifact name if artifact is not a pool
artifact_name = artifact.name.split('_')[0]
else:
artifact_name = artifact.name
# Set Average Concentratie fluorescentie
sample_fluorescence = sum(sample_measurements[artifact_name]) / float(len(sample_measurements[artifact_name]))
sample_concentration = ((sample_fluorescence - baseline_fluorescence) * regression_slope) / 2.0
artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = sample_concentration
# Set artifact Concentratie fluorescentie
# Get artifact index == count
if artifact_name not in artifact_count:
artifact_count[artifact_name] = 0
else:
artifact_count[artifact_name] += 1
artifact_fluorescence = sample_measurements[artifact_name][artifact_count[artifact_name]]
artifact_concentration = ((artifact_fluorescence - baseline_fluorescence) * regression_slope) / 2.0
artifact.udf['Dx Conc. goedgekeurde meting (ng/ul)'] = artifact_concentration
# Set QC flags
if artifact_name.startswith('Dx Tecan std'):
artifact.qc_flag = 'PASSED'
std_number = int(artifact_name.split(' ')[3])
artifact.udf['Dx Conc. goedgekeurde meting (ng/ul)'] = ng_values[std_number - 1]
artifact.udf['Dx Concentratie fluorescentie (ng/ul)'] = ng_values[std_number - 1]
else:
# Calculate measurement deviation from average.
if concentration_range[0] <= sample_concentration <= concentration_range[1]:
if len(sample_measurements[artifact_name]) == 1:
artifact.qc_flag = 'PASSED'
elif len(sample_measurements[artifact_name]) == 2:
artifact_fluorescence_difference = abs(sample_measurements[artifact_name][0] - sample_measurements[artifact_name][1])
artifact_fluorescence_deviation = artifact_fluorescence_difference / sample_fluorescence
if artifact_fluorescence_deviation <= 0.1:
artifact.qc_flag = 'PASSED'
else:
artifact.qc_flag = 'FAILED'
else:
artifact.qc_flag = 'FAILED'
artifact.put()
def main(lims, args):
log = []
thisyear = datetime.now().year
process = Process(lims, id=args.pid)
# Fetch FC ID
for out in process.all_outputs():
try:
if out.type == "Analyte":
fc_name = out.location[0].name
except Exception as e:
log.append(str(e))
# Fetch required run step UDFs
run_mode = process.udf.get('Run Mode','')
sample_loading_type = "NovaSeqXp" if process.udf.get('Loading Workflow Type') == "NovaSeq Xp" else "NovaSeqStandard"
workflow_type = process.udf.get('Workflow Type','').replace(' ','')
librarytube_ID = process.udf.get('Library Tube Barcode','')
paired_end = True if process.udf.get('Paired End') == "True" else False
read1 = process.udf.get('Read 1 Cycles',0)
read2 = process.udf.get('Read 2 Cycles',0)
index_read1 = process.udf.get('Index Read 1',0)
index_read2 = process.udf.get('Index Read 2',0)
output_folder = "\\\\172.16.1.6\\novaseqdata\\Runs\\"
attachment = "\\\\172.16.1.6\\samplesheets\\novaseq\\{}\\\\{}.csv".format(thisyear, fc_name)
basespace_mode = process.udf.get('BaseSpace Sequence Hub Configuration')
if basespace_mode == "Not Used":
use_basespace = False
else:
use_basespace = True
use_custom_read1_primer = process.udf.get('Use Custom Read 1 Primer')
use_custom_read2_primer = process.udf.get('Use Custom Read 2 Primer')
use_custom_index_read1_primer = process.udf.get('Use Custom Index Read 1 Primer')
# Prepare json file
output = {
"run_name":fc_name,
"run_mode":workflow_type,
"workflow_type":workflow_type,
"sample_loading_type":sample_loading_type,
"librarytube_ID":librarytube_ID,
"flowcell_ID":fc_name,
"rehyb":False,
"paired_end":paired_end,
"read1":read1,
"read2":read2,
"index_read1":index_read1,
"index_read2":index_read2,
"output_folder":output_folder,
"attachment":attachment,
"use_basespace":use_basespace,
"basespace_mode":basespace_mode,
"use_custom_read1_primer":use_custom_read1_primer,
"use_custom_read2_primer":use_custom_read2_primer,
"use_custom_index_read1_primer":use_custom_index_read1_primer
}
# Write json file
if os.path.exists("/srv/mfs/NovaSeq_data/gls_recipe_novaseq/"):
try:
with open("/srv/mfs/NovaSeq_data/gls_recipe_novaseq/{}.json".format(fc_name), 'w') as sf:
json.dump(output,sf,separators=(',',':'))
except Exception as e:
log.append(str(e))
for out in process.all_outputs():
if out.name == "Run Recipe":
ss_art = out
elif out.name == "Run Recipe Log":
log_id = out.id
with open("{}.json".format(fc_name), "w", 0o664) as sf:
json.dump(output,sf,separators=(',',':'))
os.chmod("{}.json".format(fc_name),0o664)
for f in ss_art.files:
lims.request_session.delete(f.uri)
lims.upload_new_file(ss_art, "{}.json".format(fc_name))
# Write log
if log:
with open("{}_{}_Error.log".format(log_id,fc_name), "w") as f:
f.write('\n'.join(log))
sys.stderr.write("Errors were met, check the log.")
sys.exit(1)
def main(lims, args):
log = []
thisyear = datetime.now().year
content = None
if args.mytest:
test()
else:
process = Process(lims, id=args.pid)
if process.type.name == 'Cluster Generation (HiSeq X) 1.0':
header = gen_X_header(process)
reads = gen_X_reads_info(process)
(data, obj) = gen_X_lane_data(process)
check_index_distance(obj, log)
content = "{}{}{}".format(header, reads, data)
if os.path.exists(
"/srv/mfs/samplesheets/HiSeqX/{}".format(thisyear)):
try:
with open(
"/srv/mfs/samplesheets/HiSeqX/{}/{}.csv".format(
thisyear, obj[0]['fc']), 'w') as sf:
sf.write(content)
except Exception as e:
log.append(str(e))
elif process.type.name == 'Cluster Generation (Illumina SBS) 4.0':
(content, obj) = gen_Hiseq_lane_data(process)
check_index_distance(obj, log)
if os.path.exists("/srv/mfs/samplesheets/{}".format(thisyear)):
try:
with open(
"/srv/mfs/samplesheets/{}/{}.csv".format(
thisyear, obj[0]['fc']), 'w') as sf:
sf.write(content)
except Exception as e:
log.append(str(e))
elif process.type.name == 'Load to Flowcell (NovaSeq 6000 v2.0)':
(content, obj) = gen_Novaseq_lane_data(process)
check_index_distance(obj, log)
if os.path.exists(
"/srv/mfs/samplesheets/novaseq/{}".format(thisyear)):
try:
with open(
"/srv/mfs/samplesheets/novaseq/{}/{}.csv".format(
thisyear, obj[0]['fc']), 'w') as sf:
sf.write(content)
except Exception as e:
log.append(str(e))
elif process.type.name == 'Denature, Dilute and Load Sample (MiSeq) 4.0':
header = gen_Miseq_header(process)
reads = gen_Miseq_reads(process)
settings = gen_Miseq_settings(process)
(data, obj) = gen_Miseq_data(process)
check_index_distance(obj, log)
content = "{}{}{}{}".format(header, reads, settings, data)
elif process.type.name == 'Load to Flowcell (NextSeq v1.0)':
(content, obj) = gen_Nextseq_lane_data(process)
check_index_distance(obj, log)
nextseq_fc = process.udf['Experiment Name'] if process.udf[
'Experiment Name'] else obj[0]['fc']
if os.path.exists(
"/srv/mfs/samplesheets/nextseq/{}".format(thisyear)):
try:
with open(
"/srv/mfs/samplesheets/nextseq/{}/{}.csv".format(
thisyear, nextseq_fc), 'w') as sf:
sf.write(content)
except Exception as e:
log.append(str(e))
elif process.type.name in [
'MinION QC', 'Load Sample and Sequencing (MinION) 1.0'
]:
content = gen_MinION_QC_data(process)
run_type = 'QC' if process.type.name == 'MinION QC' else 'DELIVERY'
fc_name = run_type + "_" + process.udf[
'Nanopore Kit'] + "_" + process.udf['Flowcell ID'].upper(
) + "_" + "Samplesheet" + "_" + process.id
if os.path.exists(
"/srv/mfs/samplesheets/nanopore/{}".format(thisyear)):
try:
with open(
"/srv/mfs/samplesheets/nanopore/{}/{}.csv".format(
thisyear, fc_name), 'w') as sf:
sf.write(content)
except Exception as e:
log.append(str(e))
if not args.test:
for out in process.all_outputs():
if out.name == "Scilifelab SampleSheet":
ss_art = out
elif out.name == "Scilifelab Log":
log_id = out.id
elif out.type == "Analyte":
if process.type.name == 'Load to Flowcell (NextSeq v1.0)':
fc_name = process.udf['Experiment Name'] if process.udf[
'Experiment Name'] else out.location[0].name
else:
fc_name = out.location[0].name
elif process.type.name in [
'MinION QC', 'Load Sample and Sequencing (MinION) 1.0'
]:
run_type = 'QC' if process.type.name == 'MinION QC' else 'DELIVERY'
fc_name = run_type + "_" + process.udf[
'Nanopore Kit'] + "_" + process.udf[
'Flowcell ID'].upper(
) + "_" + "Samplesheet" + "_" + process.id
else:
fc_name = "Samplesheet" + "_" + process.id
with open("{}.csv".format(fc_name), "w", 0o664) as f:
f.write(content)
os.chmod("{}.csv".format(fc_name), 0o664)
for f in ss_art.files:
lims.request_session.delete(f.uri)
lims.upload_new_file(ss_art, "{}.csv".format(fc_name))
if log:
with open("{}_{}_Error.log".format(log_id, fc_name), "w") as f:
f.write('\n'.join(log))
sys.stderr.write("Errors were met, check the log.")
sys.exit(1)
else:
print(content)
print(log)
def main(lims, args):
conc_is_local = True
process = Process(lims, id=args.pid)
log_art = None
log = []
fid = None
# first, read the fragment analyzer results
for o in process.all_outputs():
if o.name == 'CSV Result File':
try:
fid = o.files[0].id
except:
sys.exit("Please upload a CSV result file.")
if o.name == 'Calculation Log':
log_art = o
file_contents = lims.get_file_contents(id=fid)
frag_data = {}
keys = []
for line in file_contents.splitlines():
if not keys:
keys = line.split(',')
else:
values = line.split(',')
frag_data[values[0]] = {}
for i in xrange(1, len(values)):
frag_data[values[0]][keys[i]] = values[i]
# Then, read the concentration from the step defined in the process udf
try:
conc_process_name = process.udf['Concentration Source']
conc_is_local = False
except KeyError:
conc_is_local = True
for io in process.input_output_maps:
if 'Fragment Analyzer' in io[1]['uri'].name and io[1]['output-generation-type']== 'PerInput':
base_concentration = None
base_conc_unit = None
well = io[1]['uri'].location[1].replace(":", "")
if conc_is_local:
base_concentration = float(frag_data[well]['ng/uL'])
base_conc_unit = 'ng/uL'
else:
try:
concentration_step = lims.get_processes(type=conc_process_name, inputartifactlimsid=io[0]['limsid'])[0]
except IndexError:
log.append("Cannot find a {} step starting with {}".format(conc_process_name, io[0]['limsid']))
else:
for io2 in concentration_step.input_output_maps:
if io2[0]['limsid'] == io[0]['limsid'] and "Concentration" in io2[1]['uri'].udf:
base_concentration = io2[1]['uri'].udf['Concentration']
base_conc_unit = io2[1]['uri'].udf['Conc. Units']
try:
io[1]['uri'].udf['Min Size (bp)'] = int(frag_data[well]['Range'].split('to')[0].split('bp')[0].strip())
io[1]['uri'].udf['Max Size (bp)'] = int(frag_data[well]['Range'].split('to')[1].split('bp')[0].strip())
if 'Ratio (%)' not in io[1]['uri'].udf:
io[1]['uri'].udf['Ratio (%)'] = float(frag_data[well]['% Total'])
io[1]['uri'].udf['Size (bp)'] = int(frag_data[well]['Avg. Size'])
io[1]['uri'].put()
if base_concentration and base_conc_unit:
if conc_is_local:
io[1]['uri'].udf['Concentration'] = base_concentration
else:
io[1]['uri'].udf['Concentration'] = base_concentration * (float(io[1]['uri'].udf['Ratio (%)']) / 100.0)
io[1]['uri'].udf['Conc. Units'] = base_conc_unit
io[1]['uri'].put()
log.append("Updated values for output {}".format(io[1]['uri'].name))
else:
log.append("Failed to update the concentration of output {}".format(io[1]['uri'].name))
except Exception as e:
log.append("Error updating {} with fragment analyzer data : {}".format(io[1]['uri'].name, e))
if log:
with open("{}_frag_analyzer.log".format(log_art.id), "w") as logContext:
logContext.write("\n".join(log))
def main(lims, args):
log=[]
thisyear=datetime.now().year
content = None
if args.mytest:
test()
else:
process = Process(lims, id=args.pid)
if process.type.name == 'Cluster Generation (HiSeq X) 1.0':
header = gen_X_header(process)
reads = gen_X_reads_info(process)
(data, obj) = gen_X_lane_data(process)
check_index_distance(obj, log)
content = "{}{}{}".format(header, reads, data)
if os.path.exists("/srv/mfs/samplesheets/HiSeqX/{}".format(thisyear)):
try:
with open("/srv/mfs/samplesheets/HiSeqX/{}/{}.csv".format(thisyear, obj[0]['fc']), 'w') as sf:
sf.write(content)
os.chmod("/srv/mfs/samplesheets/HiSeqX/{}/{}.csv".format(thisyear, obj[0]['fc']), 0664)
except Exception as e:
log.append(str(e))
elif process.type.name == 'Cluster Generation (Illumina SBS) 4.0':
(content, obj) = gen_Hiseq_lane_data(process)
check_index_distance(obj, log)
if os.path.exists("/srv/mfs/samplesheets/{}".format(thisyear)):
try:
with open("/srv/mfs/samplesheets/{}/{}.csv".format(thisyear, obj[0]['fc']), 'w') as sf:
sf.write(content)
os.chmod("/srv/mfs/samplesheets/{}/{}.csv".format(thisyear, obj[0]['fc']), 0664)
except Exception as e:
log.append(str(e))
elif process.type.name == 'Denature, Dilute and Load Sample (MiSeq) 4.0':
header = gen_Miseq_header(process)
reads = gen_Miseq_reads(process)
settings = gen_Miseq_settings(process)
(data, obj) = gen_Miseq_data(process)
check_index_distance(obj, log)
content = "{}{}{}{}".format(header, reads, settings, data)
if not args.test:
for out in process.all_outputs():
if out.name == "Scilifelab SampleSheet" :
ss_art = out
elif out.name == "Scilifelab Log" :
log_id= out.id
elif out.type == "Analyte":
fc_name = out.location[0].name
with open("{}.csv".format(fc_name), "w", 0o664) as f:
f.write(content)
os.chmod("{}.csv".format(fc_name),0664)
for f in ss_art.files:
lims.request_session.delete(f.uri)
lims.upload_new_file(ss_art, "{}.csv".format(fc_name))
if log:
with open("{}_{}_Error.log".format(log_id, fc_name), "w") as f:
f.write('\n'.join(log))
sys.stderr.write("Errors were met, check the log.")
sys.exit(1)
else:
print content
print log
def main(lims, args):
conc_is_local = True
process = Process(lims, id=args.pid)
log_art = None
log = []
fid = None
# first, read the fragment analyzer results
for o in process.all_outputs():
if o.name == 'CSV Result File':
try:
fid = o.files[0].id
except:
sys.exit("Please upload a CSV result file.")
if o.name == 'Calculation Log':
log_art = o
file_contents = lims.get_file_contents(id=fid)
if isinstance(file_contents, bytes):
file_contents = file_contents.decode('utf-8')
frag_data = {}
keys = []
for line in file_contents.splitlines():
if not keys:
keys = line.split(',')
else:
values = line.split(',')
frag_data[values[0]] = {}
for i in range(1, len(values)):
frag_data[values[0]][keys[i]] = values[i]
# Then, read the concentration from the step defined in the process udf
try:
conc_process_name = process.udf['Concentration Source']
conc_is_local = False
except KeyError:
conc_is_local = True
for io in process.input_output_maps:
if 'Fragment Analyzer' in io[1]['uri'].name and io[1]['output-generation-type']== 'PerInput':
base_concentration = None
base_conc_unit = None
well = io[1]['uri'].location[1].replace(":", "")
if conc_is_local:
base_concentration = float(frag_data[well]['ng/uL'])
base_conc_unit = 'ng/uL'
else:
try:
concentration_step = lims.get_processes(type=conc_process_name, inputartifactlimsid=io[0]['limsid'])[0]
except IndexError:
log.append("Cannot find a {} step starting with {}".format(conc_process_name, io[0]['limsid']))
else:
for io2 in concentration_step.input_output_maps:
if io2[0]['limsid'] == io[0]['limsid'] and "Concentration" in io2[1]['uri'].udf:
base_concentration = io2[1]['uri'].udf['Concentration']
base_conc_unit = io2[1]['uri'].udf['Conc. Units']
try:
io[1]['uri'].udf['Min Size (bp)'] = int(frag_data[well]['Range'].split('to')[0].split('bp')[0].strip())
io[1]['uri'].udf['Max Size (bp)'] = int(frag_data[well]['Range'].split('to')[1].split('bp')[0].strip())
if 'Ratio (%)' not in io[1]['uri'].udf:
io[1]['uri'].udf['Ratio (%)'] = float(frag_data[well]['% Total'])
io[1]['uri'].udf['Size (bp)'] = int(frag_data[well]['Avg. Size'])
io[1]['uri'].put()
if base_concentration and base_conc_unit:
if conc_is_local:
io[1]['uri'].udf['Concentration'] = base_concentration
else:
io[1]['uri'].udf['Concentration'] = base_concentration * (float(io[1]['uri'].udf['Ratio (%)']) / 100.0)
io[1]['uri'].udf['Conc. Units'] = base_conc_unit
io[1]['uri'].put()
log.append("Updated values for output {}".format(io[1]['uri'].name))
else:
log.append("Failed to update the concentration of output {}".format(io[1]['uri'].name))
except Exception as e:
log.append("Error updating {} with fragment analyzer data : {}".format(io[1]['uri'].name, e))
if log:
with open("{}_frag_analyzer.log".format(log_art.id), "w") as logContext:
logContext.write("\n".join(log))
