def submit_project(self,
project_name: str,
samples: List[dict],
researcher_id: str = '3'):
"""Parse Scout project."""
containers = self.prepare(samples)
lims_project = Project.create(
self,
researcher=Researcher(self, id=researcher_id),
name=project_name,
)
LOG.info("%s: created new LIMS project", lims_project.id)
containers_data = [
batch.build_container(
name=container['name'],
con_type=Containertype(lims=self, id=container['type']),
) for container in containers
]
container_details = batch.build_container_batch(containers_data)
LOG.debug("%s: saving containers", lims_project.name)
container_map = self.save_containers(container_details)
reagentlabel_samples = [
sample for container in containers
for sample in container['samples'] if sample['index_sequence']
]
samples_data = []
for container in containers:
for sample in container['samples']:
LOG.debug("%s: adding sample to container: %s", sample['name'],
container['name'])
lims_container = container_map[container['name']]
sample_data = batch.build_sample(
name=sample['name'],
project=lims_project,
container=lims_container,
location=sample['location'],
udfs=sample['udfs'],
)
samples_data.append(sample_data)
sample_details = batch.build_sample_batch(samples_data)
process_reagentlabels = len(reagentlabel_samples) > 0
sample_map = self.save_samples(sample_details,
map_samples=process_reagentlabels)
if process_reagentlabels:
artifacts_data = [
batch.build_artifact(
artifact=sample_map[sample['name']].artifact,
reagent_label=sample['index_sequence'],
) for sample in reagentlabel_samples
]
artifact_details = batch.build_artifact_batch(artifacts_data)
self.update_artifacts(artifact_details)
lims_project_data = self._export_project(lims_project)
return lims_project_data
def submit_dx_samples():
form = SubmitDXSampleForm()
if form.validate_on_submit():
container_type = Containertype(lims, id='2') # Tube
workflow = Workflow(lims, id=app.config['LIMS_DX_SAMPLE_SUBMIT_WORKFLOW'])
for sample_name in form.parsed_samples:
# Get or create project
lims_projects = lims.get_projects(name=form.parsed_samples[sample_name]['project'])
if not lims_projects:
lims_project = Project.create(lims, name=form.parsed_samples[sample_name]['project'], researcher=form.researcher, udf={'Application': 'DX'})
else:
lims_project = lims_projects[0]
# Set sample udf data
udf_data = form.parsed_worklist[sample_name]
udf_data['Sample Type'] = form.parsed_samples[sample_name]['type']
udf_data['Dx Fragmentlengte (bp) Externe meting'] = form.pool_fragment_length.data
udf_data['Dx Conc. (ng/ul) Externe meting'] = form.pool_concentration.data
udf_data['Dx Exoomequivalent'] = form.parsed_samples[sample_name]['exome_count']
# Create sample
container = Container.create(lims, type=container_type, name=udf_data['Dx Fractienummer'])
sample = Sample.create(lims, container=container, position='1:1', project=lims_project, name=sample_name, udf=udf_data)
print sample.name, sample.artifact.name
# Add reagent label (barcode)
artifact = sample.artifact
artifact_xml_dom = minidom.parseString(artifact.xml())
for artifact_name_node in artifact_xml_dom.getElementsByTagName('name'):
parent = artifact_name_node.parentNode
reagent_label = artifact_xml_dom.createElement('reagent-label')
reagent_label.setAttribute('name', form.parsed_samples[sample_name]['barcode'])
parent.appendChild(reagent_label)
lims.put(artifact.uri, artifact_xml_dom.toxml(encoding='utf-8'))
lims.route_artifacts([sample.artifact], workflow_uri=workflow.uri)
return render_template('submit_dx_samples_done.html', title='Submit DX samples', project_name=lims_project.name, form=form)
return render_template('submit_dx_samples.html', title='Submit DX samples', form=form)
def from_helix(lims, email_settings, input_file):
"""Upload samples from helix export file."""
project_name = 'Dx {filename}'.format(filename=input_file.name.rstrip('.csv').split('/')[-1])
helix_initials = project_name.split('_')[-1]
# Try lims connection
try:
lims.check_version()
except ConnectionError:
subject = "ERROR Lims Helix Upload: {0}".format(project_name)
message = "Can't connect to lims server, please contact a lims administrator."
send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message)
sys.exit(message)
# Get researcher using helix initials
for researcher in lims.get_researchers():
if researcher.fax == helix_initials: # Use FAX as intials field as the lims initials field can't be edited via the 5.0 web interface.
email_settings['to_import_helix'].append(researcher.email)
break
else: # No researcher found
subject = "ERROR Lims Helix Upload: {0}".format(project_name)
message = "Can't find researcher with initials: {0}.".format(helix_initials)
send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message)
sys.exit(message)
# Create project
if not lims.get_projects(name=project_name):
project = Project.create(lims, name=project_name, researcher=researcher, udf={'Application': 'DX'})
else:
subject = "ERROR Lims Helix Upload: {0}".format(project_name)
message = "Duplicate project / werklijst. Samples not loaded."
send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message)
sys.exit(message)
container_type = Containertype(lims, id='2') # Tube
# match header and udf fields
udf_column = {
'Dx Onderzoeknummer': {'column': 'Onderzoeknummer'},
'Dx Fractienummer': {'column': 'Fractienummer'},
'Dx Monsternummer': {'column': 'Monsternummer'},
'Dx Concentratie (ng/ul)': {'column': 'Concentratie (ng/ul)'},
'Dx Materiaal type': {'column': 'Materiaal'},
'Dx Foetus': {'column': 'Foetus'},
'Dx Foetus ID': {'column': 'Foet_id'},
'Dx Foetus geslacht': {'column': 'Foetus_geslacht'},
'Dx Overleden': {'column': 'Overleden'},
'Dx Opslaglocatie': {'column': 'Opslagpositie'},
'Dx Spoed': {'column': 'Spoed'},
'Dx NICU Spoed': {'column': 'NICU Spoed'},
'Dx Persoons ID': {'column': 'Persoons_id'},
'Dx Werklijstnummer': {'column': 'Werklijstnummer'},
'Dx Familienummer': {'column': 'Familienummer'},
'Dx Geslacht': {'column': 'Geslacht'},
'Dx Geboortejaar': {'column': 'Geboortejaar'},
'Dx Meet ID': {'column': 'Stof_meet_id'},
'Dx Stoftest code': {'column': 'Stoftestcode'},
'Dx Stoftest omschrijving': {'column': 'Stoftestomschrijving'},
'Dx Onderzoeksindicatie': {'column': 'Onderzoeksindicatie'},
'Dx Onderzoeksreden': {'column': 'Onderzoeksreden'},
'Dx Protocolcode': {'column': 'Protocolcode'},
'Dx Protocolomschrijving': {'column': 'Protocolomschrijving'},
'Dx Einddatum': {'column': 'Einddatum'},
'Dx Gerelateerde onderzoeken': {'column': 'Gerelateerde onderzoeken'},
}
header = input_file.readline().rstrip().split(',') # expect header on first line
for udf in udf_column:
udf_column[udf]['index'] = header.index(udf_column[udf]['column'])
# Setup email
subject = "Lims Helix Upload: {0}".format(project_name)
message = "Project: {0}\n\nSamples:\n".format(project_name)
# Parse samples
for line in input_file:
data = line.rstrip().strip('"').split('","')
udf_data = {'Sample Type': 'DNA isolated', 'Dx Import warning': ''} # required lims input
for udf in udf_column:
# Transform specific udf
if udf in ['Dx Overleden', 'Dx Spoed', 'Dx NICU Spoed']:
udf_data[udf] = clarity_epp.upload.utils.char_to_bool(data[udf_column[udf]['index']])
elif udf in ['Dx Geslacht', 'Dx Foetus geslacht']:
udf_data[udf] = clarity_epp.upload.utils.transform_sex(data[udf_column[udf]['index']])
elif udf == 'Dx Foetus':
udf_data[udf] = bool(data[udf_column[udf]['index']].strip())
elif udf == 'Dx Concentratie (ng/ul)':
udf_data[udf] = data[udf_column[udf]['index']].replace(',', '.')
elif udf in ['Dx Monsternummer', 'Dx Fractienummer']:
udf_data[udf] = clarity_epp.upload.utils.transform_sample_name(data[udf_column[udf]['index']])
elif udf == 'Dx Gerelateerde onderzoeken':
udf_data[udf] = data[udf_column[udf]['index']].replace(',', ';')
elif udf == 'Dx Einddatum':
date = datetime.strptime(data[udf_column[udf]['index']], '%d-%m-%Y') # Helix format (14-01-2021)
udf_data[udf] = date.strftime('%Y-%m-%d') # LIMS format (2021-01-14)
else:
udf_data[udf] = data[udf_column[udf]['index']]
sample_name = udf_data['Dx Monsternummer']
# Set 'Dx Handmatig' udf
if udf_data['Dx Foetus'] or udf_data['Dx Overleden'] or udf_data['Dx Materiaal type'] not in ['BL', 'BLHEP', 'BM', 'BMEDTA']:
udf_data['Dx Handmatig'] = True
else:
udf_data['Dx Handmatig'] = False
# Set 'Dx Familie status' udf
if udf_data['Dx Onderzoeksreden'] == 'Bevestiging diagnose':
udf_data['Dx Familie status'] = 'Kind'
elif udf_data['Dx Onderzoeksreden'] == 'Prenataal onderzoek':
udf_data['Dx Familie status'] = 'Kind'
elif udf_data['Dx Onderzoeksreden'] == 'Eerstegraads-verwantenond':
udf_data['Dx Familie status'] = 'Kind'
elif udf_data['Dx Onderzoeksreden'] == 'Partneronderzoek':
udf_data['Dx Familie status'] = 'Kind'
elif udf_data['Dx Onderzoeksreden'] == 'Informativiteitstest':
udf_data['Dx Familie status'] = 'Ouder'
else:
udf_data['Dx Import warning'] = ';'.join(['Onbekende onderzoeksreden, familie status niet ingevuld.', udf_data['Dx Import warning']])
# Set 'Dx Geslacht' and 'Dx Geboortejaar' with 'Foetus' information if 'Dx Foetus == True'
if udf_data['Dx Foetus']:
udf_data['Dx Geslacht'] = udf_data['Dx Foetus geslacht']
udf_data['Dx Geboortejaar'] = ''
# Set 'Dx Geslacht = Onbekend' if 'Dx Onderzoeksindicatie == DSD00'
if udf_data['Dx Onderzoeksindicatie'] == 'DSD00' and udf_data['Dx Familie status'] == 'Kind':
udf_data['Dx Geslacht'] = 'Onbekend'
# Check 'Dx Familienummer' and correct
if '/' in udf_data['Dx Familienummer']:
udf_data['Dx Import warning'] = ';'.join([
'Meerdere familienummers, laatste wordt gebruikt. ({0})'.format(udf_data['Dx Familienummer']),
udf_data['Dx Import warning']
])
udf_data['Dx Familienummer'] = udf_data['Dx Familienummer'].split('/')[-1].strip(' ')
sample_list = lims.get_samples(name=sample_name)
if sample_list:
sample = sample_list[0]
if udf_data['Dx Protocolomschrijving'] in sample.udf['Dx Protocolomschrijving']:
message += "{0}\tERROR: Duplicate sample and Protocolomschrijving code: {1}.\n".format(sample_name, udf_data['Dx Protocolomschrijving'])
else:
# Update existing sample if new Protocolomschrijving and thus workflow.
# Append udf fields
append_udf = [
'Dx Onderzoeknummer', 'Dx Onderzoeksindicatie', 'Dx Onderzoeksreden', 'Dx Werklijstnummer', 'Dx Protocolcode', 'Dx Protocolomschrijving',
'Dx Meet ID', 'Dx Stoftest code', 'Dx Stoftest omschrijving'
]
for udf in append_udf:
sample.udf[udf] = ';'.join([udf_data[udf], str(sample.udf[udf])])
# Update udf fields
update_udf = ['Dx Overleden', 'Dx Spoed', 'Dx NICU Spoed', 'Dx Handmatig', 'Dx Opslaglocatie', 'Dx Import warning']
for udf in update_udf:
sample.udf[udf] = udf_data[udf]
# Add to new workflow
workflow = clarity_epp.upload.utils.stoftestcode_to_workflow(lims, udf_data['Dx Stoftest code'])
if workflow:
sample.put()
lims.route_artifacts([sample.artifact], workflow_uri=workflow.uri)
message += "{0}\tUpdated and added to workflow: {1}.\n".format(sample.name, workflow.name)
else:
message += "{0}\tERROR: Stoftest code {1} is not linked to a workflow.\n".format(sample.name, udf_data['Dx Stoftest code'])
else:
# Check other samples from patient
sample_list = lims.get_samples(udf={'Dx Persoons ID': udf_data['Dx Persoons ID']})
for sample in sample_list:
if sample.udf['Dx Protocolomschrijving'] == udf_data['Dx Protocolomschrijving'] and sample.udf['Dx Foetus'] == udf_data['Dx Foetus']:
udf_data['Dx Import warning'] = ';'.join(['Onderzoek reeds uitgevoerd.', udf_data['Dx Import warning']])
# Add sample to workflow
workflow = clarity_epp.upload.utils.stoftestcode_to_workflow(lims, udf_data['Dx Stoftest code'])
if workflow:
container = Container.create(lims, type=container_type, name=udf_data['Dx Fractienummer'])
sample = Sample.create(lims, container=container, position='1:1', project=project, name=sample_name, udf=udf_data)
lims.route_artifacts([sample.artifact], workflow_uri=workflow.uri)
message += "{0}\tCreated and added to workflow: {1}.\n".format(sample.name, workflow.name)
else:
message += "{0}\tERROR: Stoftest code {1} is not linked to a workflow.\n".format(sample_name, udf_data['Dx Stoftest code'])
# Send final email
send_email(email_settings['server'], email_settings['from'], email_settings['to_import_helix'], subject, message)
def submit_samples():
form = SubmitSampleForm()
if form.validate_on_submit():
# Create lims project
lims_project = Project.create(
lims,
name=app.config['LIMS_INDICATIONS'][form.indicationcode.data]['project_name_prefix'],
researcher=form.researcher,
udf={'Application': form.indicationcode.data}
)
lims_project.name = '{0}_{1}'.format(lims_project.name, lims_project.id)
lims_project.put()
# Save attachment
attachment = form.attachment.data
if attachment:
temp_dir = mkdtemp()
attachment_path = path.join(temp_dir, secure_filename(attachment.filename))
attachment.save(attachment_path)
print attachment_path
lims.upload_new_file(lims_project, attachment_path)
rmtree(temp_dir)
# Create Samples
lims_container_type = Containertype(lims, id='2') # Tube
sample_artifacts = []
for sample in form.parsed_samples:
lims_container = Container.create(lims, type=lims_container_type, name=sample['name'])
sample_udf_data = {
'Sample Type': 'DNA library',
'Dx Fragmentlengte (bp) Externe meting': form.pool_fragment_length.data,
'Dx Conc. (ng/ul) Externe meting': form.pool_concentration.data,
'Dx Exoomequivalent': sample['exome_count'],
}
lims_sample = Sample.create(lims, container=lims_container, position='1:1', project=lims_project, name=sample['name'], udf=sample_udf_data)
print lims_sample.name, lims_sample.artifact.name
artifact = lims_sample.artifact
sample_artifacts.append(artifact)
# Add reagent label (barcode)
artifact_xml_dom = minidom.parseString(artifact.xml())
for artifact_name_node in artifact_xml_dom.getElementsByTagName('name'):
parent = artifact_name_node.parentNode
reagent_label = artifact_xml_dom.createElement('reagent-label')
reagent_label.setAttribute('name', sample['barcode'])
parent.appendChild(reagent_label)
lims.put(artifact.uri, artifact_xml_dom.toxml(encoding='utf-8'))
# Route artifacts to workflow
workflow = Workflow(lims, id=app.config['LIMS_INDICATIONS'][form.indicationcode.data]['workflow_id'])
lims.route_artifacts(sample_artifacts, workflow_uri=workflow.uri)
# Send email
subject = "Clarity Portal Sample Upload - {0}".format(lims_project.name)
message = "Gebruikersnaam\t{0}\n".format(form.username.data)
message += "Indicatie code\t{0}\n".format(form.indicationcode.data)
message += "Lims Project naam\t{0}\n".format(lims_project.name)
message += "Pool - Fragment lengte\t{0}\n".format(form.pool_fragment_length.data)
message += "Pool - Concentratie\t{0}\n".format(form.pool_concentration.data)
message += "Pool - Exoom equivalenten\t{0}\n\n".format(form.sum_exome_count)
message += "Sample naam\tBarcode\tExome equivalenten\tSample type\n"
for sample in form.parsed_samples:
message += "{0}\t{1}\t{2}\t{3}\n".format(sample['name'], sample['barcode'], sample['exome_count'], sample['type'])
send_email(app.config['EMAIL_FROM'], app.config['LIMS_INDICATIONS'][form.indicationcode.data]['email_to'], subject, message)
return render_template('submit_samples_done.html', title='Submit samples', project_name=lims_project.name, form=form)
return render_template('submit_samples.html', title='Submit samples', form=form)
