def main(args):
log = lutils.setupLog('bioinfologger', args.logfile)
lims = Lims(BASEURI, USERNAME, PASSWORD)
with open(args.conf) as conf_file:
conf = yaml.safe_load(conf_file)
bioinfodb = lutils.setupServer(conf)['bioinfo_analysis']
open_projects = bioinfodb.view('latest_data/sample_id_open')
for row in open_projects.rows:
project_id = row.key[0]
sample_id = row.key[3]
close_date = None
try:
close_date = Project(lims=lims, id=project_id).close_date
except HTTPError as e:
if '404: Project not found' in e.message:
log.error('Project '+project_id+' not found in LIMS')
continue
if close_date is not None:
try:
doc = bioinfodb.get(row.id)
except Exception as e:
log.error(e + 'in Project '+project_id+ ' Sample '+sample_id+ ' while accessing doc from statusdb')
doc['project_closed'] = True
try:
bioinfodb.save(doc)
log.info('Updated Project '+project_id+ ' Sample '+sample_id)
except Exception as e:
log.error(e + 'in Project '+project_id+ ' Sample '+sample_id+ ' while saving to statusdb')
def main(args):
lims_db = get_session()
lims = Lims(BASEURI, USERNAME, PASSWORD)
with open(args.conf) as cf:
db_conf = yaml.load(cf)
couch = setupServer(db_conf)
db = couch["expected_yields"]
postgres_string = "{} hours".format(args.hours)
project_ids = get_last_modified_projectids(lims_db, postgres_string)
for project in [Project(lims, id=x) for x in project_ids]:
samples_count = 0
samples = lims.get_samples(projectname=project.name)
for sample in samples:
if not ("Status (manual)" in sample.udf
and sample.udf["Status (manual)"] == "Aborted"):
samples_count += 1
try:
lanes_ordered = project.udf['Sequence units ordered (lanes)']
key = parse_sequencing_platform(project.udf['Sequencing platform'])
except:
continue
for row in db.view("yields/min_yield"):
db_key = [x.lower() if x else None for x in row.key]
if db_key == key:
try:
project.udf['Reads Min'] = float(
row.value) * lanes_ordered / samples_count
project.put()
except ZeroDivisionError:
pass
def test_route_artifact(self, mocked_post):
lims = Lims(self.url, username=self.username, password=self.password)
artifact = Mock(uri=self.url + "/artifact/2")
lims.route_artifacts(artifact_list=[artifact],
workflow_uri=self.url +
'/api/v2/configuration/workflows/1')
assert mocked_post.call_count == 1
def test_upload_new_file(self, mocked_open, mocked_isfile):
lims = Lims(self.url, username=self.username, password=self.password)
xml_intro = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>"""
file_start = """<file:file xmlns:file="http://genologics.com/ri/file">"""
file_start2 = """<file:file xmlns:file="http://genologics.com/ri/file" uri="{url}/api/v2/files/40-3501" limsid="40-3501">"""
attached = """ <attached-to>{url}/api/v2/samples/test_sample</attached-to>"""
upload = """ <original-location>filename_to_upload</original-location>"""
content_loc = """ <content-location>sftp://{url}/opt/gls/clarity/users/glsftp/clarity/samples/test_sample/test</content-location>"""
file_end = """</file:file>"""
glsstorage_xml = '\n'.join(
[xml_intro, file_start, attached, upload, content_loc,
file_end]).format(url=self.url)
file_post_xml = '\n'.join(
[xml_intro, file_start2, attached, upload, content_loc,
file_end]).format(url=self.url)
with patch('requests.post',
side_effect=[
Mock(content=glsstorage_xml, status_code=200),
Mock(content=file_post_xml, status_code=200),
Mock(content="", status_code=200)
]):
file = lims.upload_new_file(
Mock(uri=self.url + "/api/v2/samples/test_sample"),
'filename_to_upload')
assert file.id == "40-3501"
with patch('requests.post',
side_effect=[Mock(content=self.error_xml,
status_code=400)]):
self.assertRaises(
HTTPError, lims.upload_new_file,
Mock(uri=self.url + "/api/v2/samples/test_sample"),
'filename_to_upload')
def main(args):
log = []
lims = Lims(BASEURI, USERNAME, PASSWORD)
process = Process(lims, id=args.pid)
for swp_iomap in process.input_output_maps:
if swp_iomap[1]['output-generation-type'] != 'PerInput':
continue
inp_artifact = swp_iomap[0]['uri']
amount_check_pros = lims.get_processes(
type='Amount confirmation QC', inputartifactlimsid=inp_artifact.id)
amount_check_pros.sort(reverse=True, key=lambda x: x.date_run)
try:
correct_amount_check_pro = amount_check_pros[0]
except KeyError:
sys.exit(
"Cannot find an Amount Confirmation QC step for artifact {}".
format(inp_artifact.id))
else:
for iomap in correct_amount_check_pro.input_output_maps:
if iomap[1]['output-generation-type'] != 'PerInput':
continue
if iomap[0]['limsid'] == inp_artifact.id:
for udf_name in [
'Concentration', 'Conc. Units', 'Total Volume (uL)'
]:
try:
swp_iomap[1]['uri'].udf[udf_name] = iomap[1][
'uri'].udf[udf_name]
except:
import pdb
pdb.set_trace()
swp_iomap[1]['uri'].udf['Amount taken (ng)'] = iomap[1][
'uri'].udf['Amount to take (ng)']
swp_iomap[1]['uri'].put()
def generate_output(project_id, dest_plate_list, best_sample_struct,total_lanes, req_lanes, lane_maps, rounded_ratios,
target_clusters, clusters_per_lane, extra_lanes, lane_volume, pool_excess, final_pool_sizes, volume_ratios, desired_ratios):
""""Gathers the container id and well name for all samples in project"""
timestamp = datetime.fromtimestamp(time()).strftime('%Y-%m-%d_%H:%M')
#Cred to Denis for providing a base epp
location = dict()
lims = Lims(BASEURI, USERNAME, PASSWORD)
allProjects = lims.get_projects()
for proj in allProjects:
if proj.id == project_id:
projName = proj.name
break
#Sets up source id
#All normalization processes for project
norms=['Library Normalization (MiSeq) 4.0', 'Library Normalization (Illumina SBS) 4.0','Library Normalization (HiSeq X) 1.0']
pros=lims.get_processes(type=norms, projectname=projName)
#For all processes
for p in pros:
#For all artifacts in process
for o in p.all_outputs():
#If artifact is analyte type and has project name in sample
if o.type=="Analyte" and project_id in o.name:
location[o.name.split()[0]] = list()
location[o.name.split()[0]].append(o.location[0].id)
location[o.name.split()[0]].append(o.location[1])
#Continue coding from here
generate_summary(projName, best_sample_struct, timestamp, project_id, dest_plate_list, total_lanes, req_lanes,
lane_maps, rounded_ratios, target_clusters, clusters_per_lane, extra_lanes, volume_ratios, desired_ratios, lane_volume, pool_excess)
generate_csv(projName, timestamp, location, dest_plate_list, total_lanes, best_sample_struct, rounded_ratios, lane_volume, pool_excess, final_pool_sizes)
generate_dumpfile(projName, timestamp, location, dest_plate_list, total_lanes, best_sample_struct, rounded_ratios, lane_volume, pool_excess, final_pool_sizes)
def test_A(server_test1):
# GIVEN: A lims with a sample with the udf 'customer': 'cust002'
# WHEN creating a genologics Sample entity of that sample
lims = Lims("http://127.0.0.1:8000", 'dummy', 'dummy')
s = Sample(lims, id='ACC2351A1')
# THEN the sample instance should have the udf
assert s.udf['customer'] == 'cust002'
def test_B(server_test1):
# GIVEN: A lims with a process with the udf 'Instrument Used': 'Cava'
# WHEN creating a genologics Process entity of that process
lims = Lims("http://127.0.0.1:8000", 'dummy', 'dummy')
p = Process(lims, id='24-168017')
# THEN the process instance should have the udf
assert p.udf['Instrument Used'] == 'Cava'
def setUp(self):
self.et = ElementTree.fromstring("""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry>
<artifact uri="http://testgenologics.com:4040/api/v2/artifacts/a1"></artifact>
</test-entry>
""")
self.lims = Lims('http://testgenologics.com:4040', username='******', password='******')
self.a1 = Artifact(self.lims, id='a1')
self.a2 = Artifact(self.lims, id='a2')
self.instance = Mock(root=self.et, lims=self.lims)
def main(args):
log = []
lims = Lims(BASEURI, USERNAME, PASSWORD)
process = Process(lims, id=args.pid)
for io in process.input_output_maps:
if io[1]['output-generation-type'] != 'PerInput':
continue
try:
starting_amount = obtain_amount(io[0]['uri'])
except Exception as e:
log.append(str(e))
starting_amount = 0
log.append("Starting amount of {} : {} ng".format(
io[0]['uri'].samples[0].name, starting_amount))
current_amount = starting_amount
#preps
preps = lims.get_processes(
inputartifactlimsid=io[0]['uri'].id,
type=["Setup Workset/Plate", "Amount confirmation QC"])
for pro in preps:
if pro.id == args.pid:
continue # skip the current step
for prepio in pro.input_output_maps:
if prepio[1]['output-generation-type'] == 'PerInput' and prepio[
0]['uri'].id == io[0]['uri'].id:
if "Amount taken (ng)" in prepio[1][
'uri'].udf: #should always be true
prep_amount = prepio[1]['uri'].udf["Amount taken (ng)"]
log.append(
"Removing {} ng for prep {} for sample {}".format(
prep_amount, pro.id,
io[0]['uri'].samples[0].name))
current_amount = current_amount - prep_amount
else:
log.append(
"No Amount Taken found for prep {} of sample {}".
format(pro.id, io[0]['uri'].samples[0].name))
if current_amount < 0:
log.append(
"Estimated amount for sample {} is {}, correcting to zero".
format(io[0]['uri'].samples[0].name, current_amount))
current_amount = 0
update_output_values(io[0]['uri'], io[1]['uri'], current_amount)
with open("amount_check_log.txt", "w") as f:
f.write("\n".join(log))
for out in process.all_outputs():
if out.name == "QC Assignment Log File":
for f in out.files:
lims.request_session.delete(f.uri)
lims.upload_new_file(out, "amount_check_log.txt")
def test_D(server_test1):
# GIVEN: A lims with a sample with:
# name: 'maya'
# Udf "Source": "blood", "Reads missing (M)": 0
# WHEN creating a genologics Lims object and filtering on the fields.
lims = Lims("http://127.0.0.1:8000", 'dummy', 'dummy')
samples = lims.get_samples(udf={"Source": "blood", "Reads missing (M)": 0}, name='maya')
# Then the sample should be found
assert samples == [Sample(lims, id='ACC2351A2')]
def test_tostring(self):
lims = Lims(self.url, username=self.username, password=self.password)
from xml.etree import ElementTree as ET
a = ET.Element('a')
b = ET.SubElement(a, 'b')
c = ET.SubElement(a, 'c')
d = ET.SubElement(c, 'd')
etree = ET.ElementTree(a)
expected_string = b"""<?xml version='1.0' encoding='utf-8'?>
<a><b /><c><d /></c></a>"""
string = lims.tostring(etree)
assert string == expected_string
def test_C(server_test1):
# GIVEN: A lims with a process with:
# process type: 'CG002 - qPCR QC (Library Validation) (Dev)'
# input artifact: '2-1155237'
# Udf 'Instrument Used': 'Cava'
# WHEN creating a genologics Lims object and filtering on the fields.
lims = Lims("http://127.0.0.1:8000", 'dummy', 'dummy')
processes = lims.get_processes(type=['CG002 - qPCR QC (Library Validation) (Dev)'], inputartifactlimsid=['2-1155237'], udf={'Instrument Used': 'Cava'})
# Then the process should be found
assert processes == [Process(lims, id='24-168017')]
def main(args):
lims = Lims(BASEURI, USERNAME, PASSWORD)
CC = CheckConfigurations(lims)
if args.udf_preset:
udf_results = CC.search_steps_for_udf(search_by=args.udf_preset)
elif args.automation_string:
search_results = CC.search_automations(search_by=args.automation_string)
elif args.print_all_bash:
CC.print_all_bash()
elif args.all_active:
CC.serarch_all_active()
def test_parse_response(self):
lims = Lims(self.url, username=self.username, password=self.password)
r = Mock(content=self.sample_xml, status_code=200)
pr = lims.parse_response(r)
assert pr is not None
assert callable(pr.find)
assert hasattr(pr.attrib, '__getitem__')
r = Mock(content=self.error_xml, status_code=400)
self.assertRaises(HTTPError, lims.parse_response, r)
r = Mock(content=self.error_no_msg_xml, status_code=400)
self.assertRaises(HTTPError, lims.parse_response, r)
def main(args):
lims = Lims(BASEURI, USERNAME, PASSWORD)
conts = lims.get_containers(name=args.flowcell)
print "found {} containers with name {}".format(len(conts), args.flowcell)
for cont in conts:
if cont.type.name not in args.types:
print "Container {} is a {} and will be renamed".format(
cont.id, cont.type.name)
cont.name = args.name
cont.put()
else:
print "Container {} is a {} and will NOT be renamed".format(
cont.id, cont.type.name)
def diff_project_objects(pj_id, couch, logfile, new=True):
proj_db = couch['projects']
samp_db = couch['samples']
log = setupLog('diff - {}'.format(pj_id), logfile)
lims = Lims(BASEURI, USERNAME, PASSWORD)
view = proj_db.view('projects/lims_followed')
try:
old_project_couchid = view[pj_id].rows[0].value
except KeyError, IndexError:
log.error("No such project {}".format(pj_id))
def main(args):
lims = Lims(BASEURI,USERNAME,PASSWORD)
process = Process(lims, id=args.pid)
for io in process.input_output_maps:
if io[1]['output-generation-type'] != 'PerInput':
continue
for idx, field in enumerate(args.fields):
if field in io[0]['uri'].udf:
if args.destfields:
io[1]['uri'].udf[args.destfields[idx]] = io[0]['uri'].udf[field]
else:
io[1]['uri'].udf[field] = io[0]['uri'].udf[field]
io[1]['uri'].put()
def test_get(self, mocked_instance):
lims = Lims(self.url, username=self.username, password=self.password)
r = lims.get('{url}/api/v2/artifacts?sample_name=test_sample'.format(
url=self.url))
assert r is not None
assert callable(r.find)
assert hasattr(r.attrib, '__getitem__')
assert mocked_instance.call_count == 1
mocked_instance.assert_called_with(
'http://testgenologics.com:4040/api/v2/artifacts?sample_name=test_sample',
timeout=16,
headers={'accept': 'application/xml'},
params={},
auth=('test', 'password'))
def main(args):
lims = Lims(BASEURI, USERNAME, PASSWORD)
process = Process(lims, id=args.pid)
for io in process.input_output_maps:
if io[1]['output-generation-type'] != 'PerInput':
continue
if "Amount taken (ng)" in io[1]['uri'].udf:
if "Amount left (ng)" in io[0]['uri'].udf:
io[0]['uri'].udf["Amount left (ng)"] = io[0]['uri'].udf[
"Amount left (ng)"] - io[1]['uri'].udf["Amount taken (ng)"]
else:
io[0]['uri'].udf["Amount left (ng)"] = io[0]['uri'].udf[
"Amount(ng)"] - io[1]['uri'].udf["Amount taken (ng)"]
io[0]['uri'].put()
def main(options):
conf = options.conf
output_f = options.output_f
couch = load_couch_server(conf)
mainlims = Lims(BASEURI, USERNAME, PASSWORD)
lims_db = get_session()
mainlog = logging.getLogger('psullogger')
mainlog.setLevel(level=logging.INFO)
mfh = logging.handlers.RotatingFileHandler(options.logfile,
maxBytes=209715200,
backupCount=5)
mft = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
mfh.setFormatter(mft)
mainlog.addHandler(mfh)
# try getting orderportal config
oconf = None
try:
with open(options.oconf, 'r') as ocf:
oconf = yaml.load(ocf, Loader=yaml.SafeLoader)['order_portal']
except Exception as e:
mainlog.warn(
"Loading orderportal config {} failed due to {}, so order information "
"for project will not be updated".format(options.oconf, e))
if options.project_name:
host = get_configuration()['url']
pj_id = lims_db.query(DBProject.luid).filter(
DBProject.name == options.project_name).scalar()
if not pj_id:
pj_id = options.project_name
P = ProjectSQL(lims_db, mainlog, pj_id, host, couch, oconf)
if options.upload:
P.save(
update_modification_time=not options.no_new_modification_time)
else:
if output_f is not None:
with open(output_f, 'w') as f:
f.write(json.dumps(P.obj))
else:
print(json.dumps(P.obj))
else:
projects = create_projects_list(options, lims_db, mainlims, mainlog)
masterProcess(options, projects, mainlims, mainlog, oconf)
lims_db.commit()
lims_db.close()
def __init__(self, current_step, logger=None, cache=False):
"""
Initializes the context.
:param current_step: The step from which the extension is running
:param logger: A logger instance
:param cache: Set to True to use the cache folder (.cache) for downloaded files
"""
lims = Lims(BASEURI, USERNAME, PASSWORD)
lims.check_version()
self.advanced = Advanced(lims)
self.current_step = Process(lims, id=current_step)
self.logger = logger or logging.getLogger(__name__)
self._local_shared_files = []
self.cache = cache
def test_post(self):
lims = Lims(self.url, username=self.username, password=self.password)
uri = '{url}/api/v2/samples'.format(url=self.url)
with patch('requests.post',
return_value=Mock(content=self.sample_xml,
status_code=200)) as mocked_put:
response = lims.post(uri=uri, data=self.sample_xml)
assert mocked_put.call_count == 1
with patch('requests.post',
return_value=Mock(content=self.error_xml,
status_code=400)) as mocked_put:
self.assertRaises(HTTPError,
lims.post,
uri=uri,
data=self.sample_xml)
assert mocked_put.call_count == 1
def __init__(self):
self.log = logging.getLogger('multiqc')
# Check that this plugin hasn't been disabled
if config.kwargs.get('disable_clarity', False) is True:
self.log.info(
"Skipping MultiQC_Clarity as disabled on command line")
return None
if getattr(config, 'disable_clarity', False) is True:
self.log.debug(
"Skipping MultiQC_Clarity as specified in config file")
return None
super(MultiQC_clarity_metadata, self).__init__(
name='Clarity',
anchor='clarity',
href='https://github.com/Galithil/MultiQC_Clarity',
info="fetches data from your Basespace Clarity LIMS instance.")
self.lims = Lims(BASEURI, USERNAME, PASSWORD)
self.metadata = {}
self.header_metadata = {}
self.general_metadata = {}
self.tab_metadata = {}
self.samples = []
self.sections = []
self.schema = getattr(config, 'clarity', None)
if self.schema is None:
self.log.warn("No config found for MultiQC_Clarity")
return None
self.get_samples()
self.get_metadata('Header')
self.get_metadata('General Statistics')
self.get_metadata('Clarity Tab')
self.update_multiqc_report()
self.make_sections()
report.modules_output.append(self)
def __init__(self):
self.log = logging.getLogger('multiqc')
# Check that this plugin hasn't been disabled
if config.kwargs.get('disable_clarity', False) is True:
self.log.info(
"Skipping MultiQC_Clarity as disabled on command line")
return None
if getattr(config, 'disable_clarity', False) is True:
self.log.debug(
"Skipping MultiQC_Clarity as specified in config file")
return None
super(MultiQC_clarity_metadata, self).__init__(name='Clarity LIMS',
anchor='clarity')
self.intro = '''<p>The <a href="https://github.com/MultiQC/MultiQC_Clarity" target="_blank">MultiQC_Clarity</a>
plugin fetches data from a specified
<a href="https://www.genologics.com/clarity-lims/" target="_blank">Basespace Clarity LIMS</a> instance.</p>'''
self.lims = Lims(BASEURI, USERNAME, PASSWORD)
self.metadata = {}
self.header_metadata = {}
self.general_metadata = {}
self.tab_metadata = {}
self.samples = []
self.schema = getattr(config, 'clarity', None)
if self.schema is None:
self.log.debug("No config found for MultiQC_Clarity")
return None
self.get_samples()
self.get_metadata('report_header_info')
self.get_metadata('general_stats')
self.get_metadata('clarity_module')
self.update_multiqc_report()
self.make_sections()
report.modules_output.append(self)
def namesetter(PID):
lims = Lims(BASEURI, USERNAME, PASSWORD)
lims.check_version()
#Find LIMS entry with same PID
allProjects = lims.get_projects()
for proj in allProjects:
if proj.id == PID:
limsproject = proj.name
break
#Error handling
if not 'limsproject' in locals():
print("{} not available in LIMS.".format(PID))
return None
#Enter project summary process
stepname=['Project Summary 1.3']
process=lims.get_processes(type=stepname, projectname=limsproject)
#Error handling
if process == []:
print("{} for {} is not available in LIMS.".format(stepname, limsproject))
return None
loop = True
while loop:
if "Bioinfo responsible" in process[0].udf:
response = process[0].udf["Bioinfo responsible"]
else:
response = "Unassigned"
print("Existing Bioinfo responsible for project {} aka {} is: {}".format(limsproject, PID, response.encode('utf-8')))
#Checks for valid name
in_responsibles = False
config_responsibles =Udfconfig(lims, id="1128")
while not in_responsibles:
if sys.version_info[0] == 3:
newname = input("Enter name of new Bioinfo responsible: ")
elif sys.version_info[0] == 2:
newname = raw_input("Enter name of new Bioinfo responsible: ")
for names in config_responsibles.presets:
if newname in names:
in_responsibles = True
newname = names
if not in_responsibles:
print("Subset {} not found in accepted Bioinfo responsible list.".format(newname))
else:
print("Suggested name is {}".format(newname))
if sys.version_info[0] == 3:
confirmation = input("Project {} aka {} will have {} as new Bioinfo responsible, is this correct (Y/N)? ".format(limsproject, PID, newname))
elif sys.version_info[0] == 2:
confirmation = raw_input("Project {} aka {} will have {} as new Bioinfo responsible, is this correct (Y/N)? ".format(limsproject, PID, newname))
if confirmation == 'Y' or confirmation == 'y':
try:
newname.encode('ascii')
process[0].udf["Bioinfo responsible"] = str(newname)
process[0].put()
print("Project {} aka {} assigned to {}".format(limsproject, PID, newname))
return None
except (UnicodeDecodeError, UnicodeEncodeError):
#Weird solution due to put function
process[0].udf["Bioinfo responsible"] = response
print("ERROR: You tried to use a special character, didn't you? Don't do that. New standards and stuff...")
elif confirmation == 'N' or confirmation == 'n':
loop = False
else:
print("Invalid answer.")
x for x in io if x[1]['output-generation-type'] == 'PerInput'
]
# Fetch the first input-output artifact pair
(input, output) = io_filtered[0]
# Instantiate the output artifact
output_artifact = Artifact(output['limsid'])
# Attach the file
attach_file(args.file, output_artifact)
if __name__ == "__main__":
parser = ArgumentParser()
# Arguments that are useful in all EPP scripts
parser.add_argument("--log", default=sys.stdout, help="Log file")
# Arguments specific for this scripts task
parser.add_argument("--pid", help="Process id")
parser.add_argument("--file", help="File to upload")
args = parser.parse_args()
# Log everything to log argument
with EppLogger(args.log):
lims = Lims(BASEURI, USERNAME, PASSWORD)
lims.check_version()
main(lims, args.pid, args.file)
def __init__(self, *args, **kwargs):
self.lims = Lims('https://test.claritylims.com', 'user', 'password')
super(TestExample, self).__init__(*args, **kwargs)
def setUp(self):
self.lims = Lims(url, username='******', password='******')
from flask import Flask
from flask_bootstrap import Bootstrap
from flask_wtf.csrf import CSRFProtect
from genologics.lims import Lims
app = Flask(__name__)
app.config.from_object('config')
lims = Lims(app.config['BASEURI'], app.config['USERNAME'], app.config['PASSWORD'])
bootstrap = Bootstrap(app)
csrf = CSRFProtect(app)
from . import views
