def test_upload_new_file(self, mocked_open, mocked_isfile): lims = Lims(self.url, username=self.username, password=self.password) xml_intro = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>""" file_start = """<file:file xmlns:file="http://pyclarity_lims.com/ri/file">""" file_start2 = """<file:file xmlns:file="http://pyclarity_lims.com/ri/file" uri="{url}/api/v2/files/40-3501" limsid="40-3501">""" attached = """ <attached-to>{url}/api/v2/samples/test_sample</attached-to>""" upload = """ <original-location>filename_to_upload</original-location>""" content_loc = """ <content-location>sftp://{url}/opt/gls/clarity/users/glsftp/clarity/samples/test_sample/test</content-location>""" file_end = """</file:file>""" glsstorage_xml = '\n'.join( [xml_intro, file_start, attached, upload, content_loc, file_end]).format(url=self.url) file_post_xml = '\n'.join( [xml_intro, file_start2, attached, upload, content_loc, file_end]).format(url=self.url) with patch('requests.post', side_effect=[ Mock(content=glsstorage_xml, status_code=200), Mock(content=file_post_xml, status_code=200), Mock(content="", status_code=200) ]): file = lims.upload_new_file( Mock(uri=self.url + "/api/v2/samples/test_sample"), 'filename_to_upload') assert file.id == "40-3501" with patch('requests.post', side_effect=[Mock(content=self.error_xml, status_code=400)]): self.assertRaises( HTTPError, lims.upload_new_file, Mock(uri=self.url + "/api/v2/samples/test_sample"), 'filename_to_upload')
def test_route_artifact(self, mocked_post): lims = Lims(self.url, username=self.username, password=self.password) artifact = Mock(uri=self.url + "/artifact/2") lims.route_artifacts(artifact_list=[artifact], workflow_uri=self.url + '/api/v2/configuration/workflows/1') assert mocked_post.call_count == 1
def test_get_instances(self): lims = Lims(self.url, username=self.username, password=self.password) sample_xml_template = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <smp:samples xmlns:smp="http://pyclarity_lims.com/ri/sample"> <sample uri="{url}/api/v2/samples/{s1}" limsid="{s1}"/> <sample uri="{url}/api/v2/samples/{s2}" limsid="{s2}"/> {next_page} </smp:samples> """ sample_xml1 = sample_xml_template.format( s1='sample1', s2='sample2', url=self.url, next_page='<next-page uri="{url}/api/v2/samples?start-index=3"/>'. format(url=self.url)) sample_xml2 = sample_xml_template.format( s1='sample3', s2='sample4', url=self.url, next_page='<next-page uri="{url}/api/v2/samples?start-index=5"/>'. format(url=self.url)) sample_xml3 = sample_xml_template.format(s1='sample5', s2='sample6', url=self.url, next_page='') get_returns = [ Mock(content=sample_xml1, status_code=200), Mock(content=sample_xml2, status_code=200), Mock(content=sample_xml3, status_code=200) ] with patch('requests.Session.get', side_effect=get_returns) as mget: samples = lims._get_instances(Sample, nb_pages=2, params={'projectname': 'p1'}) assert len(samples) == 4 assert mget.call_count == 2 mget.assert_any_call( 'http://testgenologics.com:4040/api/v2/samples', auth=('test', 'password'), headers={'accept': 'application/xml'}, params={'projectname': 'p1'}, timeout=16) mget.assert_called_with( 'http://testgenologics.com:4040/api/v2/samples?start-index=3', auth=('test', 'password'), headers={'accept': 'application/xml'}, params={'projectname': 'p1'}, timeout=16) with patch('requests.Session.get', side_effect=get_returns) as mget: samples = lims._get_instances(Sample, nb_pages=0) assert len(samples) == 6 assert mget.call_count == 3 with patch('requests.Session.get', side_effect=get_returns) as mget: samples = lims._get_instances(Sample, nb_pages=-1) assert len(samples) == 6 assert mget.call_count == 3
def test_tostring(self): lims = Lims(self.url, username=self.username, password=self.password) from xml.etree import ElementTree as ET a = ET.Element('a') b = ET.SubElement(a, 'b') c = ET.SubElement(a, 'c') d = ET.SubElement(c, 'd') etree = ET.ElementTree(a) expected_string = b"""<?xml version='1.0' encoding='utf-8'?> <a><b /><c><d /></c></a>""" string = lims.tostring(etree) assert string == expected_string
def test_parse_response(self): lims = Lims(self.url, username=self.username, password=self.password) r = Mock(content=self.sample_xml, status_code=200) pr = lims.parse_response(r) assert pr is not None assert callable(pr.find) assert hasattr(pr.attrib, '__getitem__') r = Mock(content=self.error_xml, status_code=400) self.assertRaises(HTTPError, lims.parse_response, r) r = Mock(content=self.error_no_msg_xml, status_code=400) self.assertRaises(HTTPError, lims.parse_response, r)
def test_get(self, mocked_instance): lims = Lims(self.url, username=self.username, password=self.password) r = lims.get('{url}/api/v2/artifacts?sample_name=test_sample'.format( url=self.url)) assert r is not None assert callable(r.find) assert hasattr(r.attrib, '__getitem__') assert mocked_instance.call_count == 1 mocked_instance.assert_called_with( 'http://testgenologics.com:4040/api/v2/artifacts?sample_name=test_sample', timeout=16, headers={'accept': 'application/xml'}, params={}, auth=('test', 'password'))
def connection(new=False, **kwargs): global _lims if not _lims or new: param = cfg.get('clarity') or {} param.update(kwargs) _lims = Lims(**param) return _lims
def setUp(self): et = ElementTree.fromstring( """<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <test-entry> <artifact uri="http://testgenologics.com:4040/api/v2/artifacts/a1"></artifact> <artifact uri="http://testgenologics.com:4040/api/v2/artifacts/a2"></artifact> <other>thing</other> </test-entry> """) self.lims = Lims('http://testgenologics.com:4040', username='******', password='******') self.a1 = Artifact(self.lims, id='a1') self.a2 = Artifact(self.lims, id='a2') self.instance1 = Mock(root=et, lims=self.lims) et = ElementTree.fromstring( """<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <test-entry> <nesting> <artifact uri="http://testgenologics.com:4040/api/v2/artifacts/a1"></artifact> <artifact uri="http://testgenologics.com:4040/api/v2/artifacts/a2"></artifact> </nesting> </test-entry> """) self.instance2 = Mock(root=et, lims=self.lims)
def setUp(self): et = ElementTree.fromstring( '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <test-entry> <pooled-inputs> <pool output-uri="{uri}/out1" name="pool1"> <input uri="{uri}/in1"/> <input uri="{uri}/in2"/> </pool> <pool output-uri="{uri}/out2" name="pool2"> <input uri="{uri}/in3"/> <input uri="{uri}/in4"/> </pool> </pooled-inputs> </test-entry>'''.format(uri='http://testgenologics.com:4040')) self.lims = Lims('http://testgenologics.com:4040', username='******', password='******') self.instance1 = Mock(root=et, lims=self.lims) self.dict1 = XmlPooledInputDict(self.instance1) self.out1 = Artifact(self.lims, uri='http://testgenologics.com:4040/out1') self.in1 = Artifact(self.lims, uri='http://testgenologics.com:4040/in1') self.in2 = Artifact(self.lims, uri='http://testgenologics.com:4040/in2')
def setUp(self): et = ElementTree.fromstring( '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <test-entry> <artifacts> <artifact uri="{url}/artifacts/a1"> <queue-time>2011-12-25T01:10:10.050+00:00</queue-time> <location> <container uri="{url}/containers/c1"/> <value>A:1</value> </location> </artifact> <artifact uri="{url}/artifacts/a2"> <queue-time>2011-12-25T01:10:10.200+01:00</queue-time> <location> <container uri="{url}/containers/c1"/> <value>A:2</value> </location> </artifact> <artifact uri="{url}/artifacts/a3"> <queue-time>2011-12-25T01:10:10.050-01:00</queue-time> <location> <container uri="{url}/containers/c1"/> <value>A:3</value> </location> </artifact> </artifacts> </test-entry>'''.format( url='http://testgenologics.com:4040/api/v2')) self.lims = Lims('http://testgenologics.com:4040', username='******', password='******') self.instance1 = Mock(root=et, lims=self.lims)
def test_post(self): lims = Lims(self.url, username=self.username, password=self.password) uri = '{url}/api/v2/samples'.format(url=self.url) with patch('requests.post', return_value=Mock(content=self.sample_xml, status_code=200)) as mocked_put: lims.post(uri=uri, data=self.sample_xml) assert mocked_put.call_count == 1 with patch('requests.post', return_value=Mock(content=self.error_xml, status_code=400)) as mocked_put: self.assertRaises(HTTPError, lims.post, uri=uri, data=self.sample_xml) assert mocked_put.call_count == 1
class TestEntities(TestCase): dummy_xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <dummy></dummy>""" def setUp(self): self.lims = Lims(url, username='******', password='******') def _tostring(self, entity): return self.lims.tostring(ElementTree.ElementTree( entity.root)).decode("utf-8")
def setUp(self): et = ElementTree.fromstring( '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <test-entry> <reagent-label name="label name"/> </test-entry>''') self.lims = Lims('http://testgenologics.com:4040', username='******', password='******') self.instance1 = Mock(root=et, lims=self.lims)
def setUp(self): et = ElementTree.fromstring( """<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <test-entry> <ri:externalid xmlns:ri="http://genologics.com/ri" id="1" uri="http://testgenologics.com:4040/api/v2/external/1" /> <ri:externalid xmlns:ri="http://genologics.com/ri" id="2" uri="http://testgenologics.com:4040/api/v2/external/2" /> </test-entry> """) self.lims = Lims('http://testgenologics.com:4040', username='******', password='******') self.instance1 = Mock(root=et, lims=self.lims)
def setUp(self): et = ElementTree.fromstring( """<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <test-entry> <test-tags> <test-tag attrib1="value1" attrib2="value2"/> <test-tag attrib1="value11" attrib2="value12" attrib3="value13"/> </test-tags> </test-entry> """) self.lims = Lims('http://testgenologics.com:4040', username='******', password='******') self.instance1 = Mock(root=et, lims=self.lims)
def setUp(self): et = ElementTree.fromstring( """<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <test-entry xmlns:udf="http://genologics.com/ri/userdefined"> <placement uri="http://testgenologics.com:4040/api/v2/artifacts/a1" limsid="a1"> <value>A:1</value> </placement> <other>thing</other> </test-entry>""") self.lims = Lims('http://testgenologics.com:4040', username='******', password='******') self.instance1 = Mock(root=et, lims=self.lims) self.dict1 = PlacementDictionary(self.instance1) self.art1 = Artifact(lims=self.lims, id='a1')
def setUp(self): et = ElementTree.fromstring( """<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <test-entry xmlns:udf="http://genologics.com/ri/userdefined"> <test-tag attrib1="value1" attrib2="value2"/> <test-tag attrib1="value11" attrib2="value12" attrib3="value13"/> </test-entry>""") self.lims = Lims('http://testgenologics.com:4040', username='******', password='******') self.instance1 = Mock(root=et, lims=self.lims) self.dict1 = XmlElementAttributeDict(self.instance1, tag='test-tag', position=0) self.dict2 = XmlElementAttributeDict(self.instance1, tag='test-tag', position=1)
def setUp(self): et = ElementTree.fromstring( '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <test-entry> <next-action step-uri="{url}/prt/1/stp/1" action="nextstep" artifact-uri="{url}/arts/a1"/> </test-entry>'''.format(url='http://testgenologics.com:4040')) et1 = ElementTree.fromstring( '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <test-entry> <next-action artifact-uri="{url}/arts/a1"/> </test-entry>'''.format(url='http://testgenologics.com:4040')) self.lims = Lims('http://testgenologics.com:4040', username='******', password='******') self.instance1 = Mock(root=et, lims=self.lims) self.instance_empty = Mock(root=et1, lims=self.lims)
def setUp(self): et = ElementTree.fromstring( """<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <test-entry xmlns:udf="http://genologics.com/ri/userdefined"> <test-tag> <key1>value1</key1> </test-tag> </test-entry>""") self.lims = Lims('http://testgenologics.com:4040', username='******', password='******') self.instance1 = Mock(root=et, lims=self.lims) self.dict1 = SubTagDictionary(self.instance1, tag='test-tag') et = ElementTree.fromstring( """<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <test-entry xmlns:udf="http://genologics.com/ri/userdefined"> </test-entry>""") self.instance2 = Mock(root=et, lims=self.lims) self.dict2 = SubTagDictionary(self.instance2, tag='test-tag')
def main(): a = ArgumentParser() a.add_argument('--url', type=str) a.add_argument('--username', type=str) a.add_argument('--password', type=str) a.add_argument('--config', type=str, required=True) a.add_argument('--generate_config', action='store_true') args = a.parse_args() config = {} if os.path.isfile(args.config): with open(args.config, 'r') as open_file: config = yaml.load(open_file) url = args.url or config.get('clarity', {}).get('url') if not url: print('Specify url on the command line') a.print_help() return 1 username = args.username or config.get('clarity', {}).get('username') if not username: print('Specify username on the command line') a.print_help() return 1 password = args.password or config.get('clarity', {}).get('password') if not password: print('Specify password on the command line') a.print_help() return 1 lims = Lims(baseuri=url, username=username, password=password) if args.generate_config: config['entities'] = generate_entities_expected_output(lims) with open(args.config, 'w') as open_file: yaml.dump(config, open_file, width=180, indent=4) else: test_all_entities(lims, config.get('entities', {}))
def test_get_file_contents(self): lims = Lims(self.url, username=self.username, password=self.password) lims.validate_response = Mock() lims.request_session = Mock(get=Mock( return_value=Mock(encoding=None, text='some data\r\n'))) exp_url = self.url + '/api/v2/files/an_id/download' assert lims.get_file_contents(uri=self.url + '/api/v2/files/an_id') == 'some data\r\n' assert lims.request_session.get.return_value.encoding is None lims.request_session.get.assert_called_with(exp_url, auth=(self.username, self.password), timeout=16) assert lims.get_file_contents(id='an_id', encoding='utf-16', crlf=True) == 'some data\n' assert lims.request_session.get.return_value.encoding == 'utf-16' lims.request_session.get.assert_called_with(exp_url, auth=(self.username, self.password), timeout=16)
def test_get_uri(self): lims = Lims(self.url, username=self.username, password=self.password) assert lims.get_uri( 'artifacts', sample_name='test_sample' ) == '{url}/api/v2/artifacts?sample_name=test_sample'.format( url=self.url)
def setUp(self): self.lims = Lims(url, username='******', password='******')
class MyLims: def __init__(self): self.lims = Lims(*get_config()) self.lims.check_version() def get_project_names_slow(self, *proj_nums): # Get the list of all projects. projects = self.lims.get_projects() print('{} projects in total'.format(len(projects))) # Unfortunately this doesn't work... # projects = lims.get_projects(name="10657_%") # and calling .name on every project is slow as it entails an extra GET # Maybe we need to bypass Clarity and hit PostgreSQL? #Yes I'm scanning the list many times, but the key thing is I only fetch it once. res = [] projects = filter_names(( p.name for p in projects )) for p_num in proj_nums: p_prefix = str(p_num) + '_' p_name = None for pn in projects: if pn.startswith(p_prefix): if p_name: raise LookupError("More than one project found with prefix " + p_prefix) else: p_name = pn res.append(p_name) return res def get_project_names(self, *proj_nums): #Quickly get the list of all project names. lims = self.lims projects = [] proot = lims.get(lims.get_uri('projects')) while True: # Loop over all pages. for node in proot.findall('.//name'): projects.append(node.text) next_page = proot.find('next-page') if next_page is None: break proot = lims.get(next_page.attrib['uri']) #Yes I'm scanning the list many times, but the key thing is I only fetch it once. res = [] projects = filter_names(projects) for p_num in proj_nums: p_prefix = str(p_num) + '_' p_name = None for proj in projects: if proj.startswith(p_prefix): if p_name: raise LookupError("More than one project found with prefix " + p_prefix) else: p_name = proj res.append(p_name) return res
def __init__(self): self.lims = Lims(*get_config()) self.lims.check_version()
def main(args): if args.debug: L.basicConfig(format='{name:s} {levelname:s}: {message:s}', level=L.DEBUG, style='{') # But silence chatter from pyclarity_lims L.getLogger("requests").propagate = False L.getLogger("urllib3").propagate = False else: L.basicConfig(format='{message:s}', level=L.INFO, style='{') # Connect to ze lims... lims = Lims(**get_config()) # How to do this? # 1 - Firstly look to see if the name is already known. If so, report and exit. # 2 - Then split out the flowcell name and look for a container with that name. # 3 - If none, exit. If several, pick the latest (highest number) # 4 - Find the first analyte and thus the step (see notes) # 5 - Load the step and set the values (if not set already or forced) # Sanity check, since I accidentally set several run IDs to # /lustre/fastqdata/180416_M05898_0001_000000000-D42P7 etc. by foolish use of # a shell loop. assert re.match(r"^[0-9]{6}_[A-Za-z0-9_-]+", args.runid), \ "Run ID does not look right - {}".format(args.runid) # 1 existing_proc = lims.get_processes(type=QC_PROCESS, udf={'RunID': args.runid}) if len(existing_proc): L.info("Run {} is already in the LIMS database...".format(args.runid)) # There should just be one! for ep in existing_proc: L.info(" " + ep.uri) if args.close and not args.no_act: try: L.info(" Completing the existing step...") ep.step.get() ep.step.advance() L.info(" completed") except Exception as e: L.info(" " + str(e)) if args.debug: # It's useful to be able to get the parent process because that's what gets fed # into the sample sheet generator. for epp_uri in sorted( set(epp.uri for epp in ep.parent_processes())): L.debug(" parent --> " + epp_uri) exit( 0 ) # Not an error, but FIXME I should allow the ID to be transferred to a new process. # 2 if args.container_name: container_name = args.container_name else: container_name = re.split('[_-]', args.runid)[-1] # Prune off the stage letter. mo = re.match(r'[AB](.........)', container_name) if mo: container_name = mo.group(1) existing_containers = lims.get_containers(name=container_name) # If the container name is not in all upper case then try making it so... if container_name != container_name.upper(): existing_containers.extend( lims.get_containers(name=container_name.upper())) # 3 if len(existing_containers) == 0: L.warning("No container found with name {}.".format(container_name)) exit(1) existing_container = sorted(existing_containers, key=lambda c: c.uri)[-1] if len(existing_containers) > 1: L.warning( "Multiple containers found with name {}. The latest ({}) will be used." .format(container_name, existing_container.id)) L.info("Finding QC step for container {}.".format(existing_container.uri)) # 4 - HiSeq lane 1 is 1:1 while MiSeq lane is 'A:1' for some reson. We should see on eor the other. analyte1, = [ existing_container.placements.get(loc) for loc in ['1:1', 'A:1'] if loc in existing_container.placements ] L.debug("Examining analyte {}".format(analyte1.uri)) # https://clarity.genomics.ed.ac.uk/api/v2/processes?type=Flow%20Cell%20Lane%20QC%20EG%201.0%20ST&inputartifactlimsid=2-126766 # I need to handle the error if there is not one process - older runs lack the step, for example. # In the development LIMS we also seem to have multiple QC processes per flowcell. As before I think we need to go for # the latest one. # Also I've seen a case where a stage was queued and removed (ie. not completed). This shows up on the artifact page but # not in the list when I run the get_processes search below. try: qc_proc = sorted(lims.get_processes(type=QC_PROCESS, inputartifactlimsid=analyte1.id), key=lambda p: p.id)[-1] except IndexError: L.error("Could not find a QC step for this container.") exit(1) # 5 - it seems Clarity likes to set UDFs on a step not a process(?) qc_step = qc_proc.step.details L.info("Setting UDFs on {}".format(qc_step.uri)) if qc_step.udf.get('RunID'): if args.force: L.info( "Forcing new RunID for step {} in place of '{}' as you requested." .format(qc_step.id, qc_step.udf.get('RunID'))) else: L.info("RunID for step {} is already set to '{}'.".format( qc_step.id, qc_step.udf.get('RunID'))) exit(1) #So, set it... qc_step.udf['RunID'] = args.runid qc_step.udf['Run Report'] = REPORT_URL_TEMPLATE.format(args.runid) L.debug(pformat(qc_step.udf)) if args.no_act: L.info("Nothing will be saved as no_act was set.") else: qc_step.put() if args.close: try: L.info("Completing the step...") qc_proc.step.get(force=True) qc_proc.step.advance() L.info("completed") except Exception as e: L.info(str(e)) L.info("DONE")