def test_upload_new_file(self, mocked_open, mocked_isfile):
lims = Lims(self.url, username=self.username, password=self.password)
xml_intro = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>"""
file_start = """<file:file xmlns:file="http://pyclarity_lims.com/ri/file">"""
file_start2 = """<file:file xmlns:file="http://pyclarity_lims.com/ri/file" uri="{url}/api/v2/files/40-3501" limsid="40-3501">"""
attached = """ <attached-to>{url}/api/v2/samples/test_sample</attached-to>"""
upload = """ <original-location>filename_to_upload</original-location>"""
content_loc = """ <content-location>sftp://{url}/opt/gls/clarity/users/glsftp/clarity/samples/test_sample/test</content-location>"""
file_end = """</file:file>"""
glsstorage_xml = '\n'.join(
[xml_intro, file_start, attached, upload, content_loc,
file_end]).format(url=self.url)
file_post_xml = '\n'.join(
[xml_intro, file_start2, attached, upload, content_loc,
file_end]).format(url=self.url)
with patch('requests.post',
side_effect=[
Mock(content=glsstorage_xml, status_code=200),
Mock(content=file_post_xml, status_code=200),
Mock(content="", status_code=200)
]):
file = lims.upload_new_file(
Mock(uri=self.url + "/api/v2/samples/test_sample"),
'filename_to_upload')
assert file.id == "40-3501"
with patch('requests.post',
side_effect=[Mock(content=self.error_xml,
status_code=400)]):
self.assertRaises(
HTTPError, lims.upload_new_file,
Mock(uri=self.url + "/api/v2/samples/test_sample"),
'filename_to_upload')
def test_route_artifact(self, mocked_post):
lims = Lims(self.url, username=self.username, password=self.password)
artifact = Mock(uri=self.url + "/artifact/2")
lims.route_artifacts(artifact_list=[artifact],
workflow_uri=self.url +
'/api/v2/configuration/workflows/1')
assert mocked_post.call_count == 1
def test_get_instances(self):
lims = Lims(self.url, username=self.username, password=self.password)
sample_xml_template = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<smp:samples xmlns:smp="http://pyclarity_lims.com/ri/sample">
<sample uri="{url}/api/v2/samples/{s1}" limsid="{s1}"/>
<sample uri="{url}/api/v2/samples/{s2}" limsid="{s2}"/>
{next_page}
</smp:samples>
"""
sample_xml1 = sample_xml_template.format(
s1='sample1',
s2='sample2',
url=self.url,
next_page='<next-page uri="{url}/api/v2/samples?start-index=3"/>'.
format(url=self.url))
sample_xml2 = sample_xml_template.format(
s1='sample3',
s2='sample4',
url=self.url,
next_page='<next-page uri="{url}/api/v2/samples?start-index=5"/>'.
format(url=self.url))
sample_xml3 = sample_xml_template.format(s1='sample5',
s2='sample6',
url=self.url,
next_page='')
get_returns = [
Mock(content=sample_xml1, status_code=200),
Mock(content=sample_xml2, status_code=200),
Mock(content=sample_xml3, status_code=200)
]
with patch('requests.Session.get', side_effect=get_returns) as mget:
samples = lims._get_instances(Sample,
nb_pages=2,
params={'projectname': 'p1'})
assert len(samples) == 4
assert mget.call_count == 2
mget.assert_any_call(
'http://testgenologics.com:4040/api/v2/samples',
auth=('test', 'password'),
headers={'accept': 'application/xml'},
params={'projectname': 'p1'},
timeout=16)
mget.assert_called_with(
'http://testgenologics.com:4040/api/v2/samples?start-index=3',
auth=('test', 'password'),
headers={'accept': 'application/xml'},
params={'projectname': 'p1'},
timeout=16)
with patch('requests.Session.get', side_effect=get_returns) as mget:
samples = lims._get_instances(Sample, nb_pages=0)
assert len(samples) == 6
assert mget.call_count == 3
with patch('requests.Session.get', side_effect=get_returns) as mget:
samples = lims._get_instances(Sample, nb_pages=-1)
assert len(samples) == 6
assert mget.call_count == 3
def test_tostring(self):
lims = Lims(self.url, username=self.username, password=self.password)
from xml.etree import ElementTree as ET
a = ET.Element('a')
b = ET.SubElement(a, 'b')
c = ET.SubElement(a, 'c')
d = ET.SubElement(c, 'd')
etree = ET.ElementTree(a)
expected_string = b"""<?xml version='1.0' encoding='utf-8'?>
<a><b /><c><d /></c></a>"""
string = lims.tostring(etree)
assert string == expected_string
def test_parse_response(self):
lims = Lims(self.url, username=self.username, password=self.password)
r = Mock(content=self.sample_xml, status_code=200)
pr = lims.parse_response(r)
assert pr is not None
assert callable(pr.find)
assert hasattr(pr.attrib, '__getitem__')
r = Mock(content=self.error_xml, status_code=400)
self.assertRaises(HTTPError, lims.parse_response, r)
r = Mock(content=self.error_no_msg_xml, status_code=400)
self.assertRaises(HTTPError, lims.parse_response, r)
def test_get(self, mocked_instance):
lims = Lims(self.url, username=self.username, password=self.password)
r = lims.get('{url}/api/v2/artifacts?sample_name=test_sample'.format(
url=self.url))
assert r is not None
assert callable(r.find)
assert hasattr(r.attrib, '__getitem__')
assert mocked_instance.call_count == 1
mocked_instance.assert_called_with(
'http://testgenologics.com:4040/api/v2/artifacts?sample_name=test_sample',
timeout=16,
headers={'accept': 'application/xml'},
params={},
auth=('test', 'password'))
def connection(new=False, **kwargs):
global _lims
if not _lims or new:
param = cfg.get('clarity') or {}
param.update(kwargs)
_lims = Lims(**param)
return _lims
def setUp(self):
et = ElementTree.fromstring(
"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry>
<artifact uri="http://testgenologics.com:4040/api/v2/artifacts/a1"></artifact>
<artifact uri="http://testgenologics.com:4040/api/v2/artifacts/a2"></artifact>
<other>thing</other>
</test-entry>
""")
self.lims = Lims('http://testgenologics.com:4040',
username='******',
password='******')
self.a1 = Artifact(self.lims, id='a1')
self.a2 = Artifact(self.lims, id='a2')
self.instance1 = Mock(root=et, lims=self.lims)
et = ElementTree.fromstring(
"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry>
<nesting>
<artifact uri="http://testgenologics.com:4040/api/v2/artifacts/a1"></artifact>
<artifact uri="http://testgenologics.com:4040/api/v2/artifacts/a2"></artifact>
</nesting>
</test-entry>
""")
self.instance2 = Mock(root=et, lims=self.lims)
def setUp(self):
et = ElementTree.fromstring(
'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry>
<pooled-inputs>
<pool output-uri="{uri}/out1" name="pool1">
<input uri="{uri}/in1"/>
<input uri="{uri}/in2"/>
</pool>
<pool output-uri="{uri}/out2" name="pool2">
<input uri="{uri}/in3"/>
<input uri="{uri}/in4"/>
</pool>
</pooled-inputs>
</test-entry>'''.format(uri='http://testgenologics.com:4040'))
self.lims = Lims('http://testgenologics.com:4040',
username='******',
password='******')
self.instance1 = Mock(root=et, lims=self.lims)
self.dict1 = XmlPooledInputDict(self.instance1)
self.out1 = Artifact(self.lims,
uri='http://testgenologics.com:4040/out1')
self.in1 = Artifact(self.lims,
uri='http://testgenologics.com:4040/in1')
self.in2 = Artifact(self.lims,
uri='http://testgenologics.com:4040/in2')
def setUp(self):
et = ElementTree.fromstring(
'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry>
<artifacts>
<artifact uri="{url}/artifacts/a1">
<queue-time>2011-12-25T01:10:10.050+00:00</queue-time>
<location>
<container uri="{url}/containers/c1"/>
<value>A:1</value>
</location>
</artifact>
<artifact uri="{url}/artifacts/a2">
<queue-time>2011-12-25T01:10:10.200+01:00</queue-time>
<location>
<container uri="{url}/containers/c1"/>
<value>A:2</value>
</location>
</artifact>
<artifact uri="{url}/artifacts/a3">
<queue-time>2011-12-25T01:10:10.050-01:00</queue-time>
<location>
<container uri="{url}/containers/c1"/>
<value>A:3</value>
</location>
</artifact>
</artifacts>
</test-entry>'''.format(
url='http://testgenologics.com:4040/api/v2'))
self.lims = Lims('http://testgenologics.com:4040',
username='******',
password='******')
self.instance1 = Mock(root=et, lims=self.lims)
def test_post(self):
lims = Lims(self.url, username=self.username, password=self.password)
uri = '{url}/api/v2/samples'.format(url=self.url)
with patch('requests.post',
return_value=Mock(content=self.sample_xml,
status_code=200)) as mocked_put:
lims.post(uri=uri, data=self.sample_xml)
assert mocked_put.call_count == 1
with patch('requests.post',
return_value=Mock(content=self.error_xml,
status_code=400)) as mocked_put:
self.assertRaises(HTTPError,
lims.post,
uri=uri,
data=self.sample_xml)
assert mocked_put.call_count == 1
class TestEntities(TestCase):
dummy_xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<dummy></dummy>"""
def setUp(self):
self.lims = Lims(url, username='******', password='******')
def _tostring(self, entity):
return self.lims.tostring(ElementTree.ElementTree(
entity.root)).decode("utf-8")
def setUp(self):
et = ElementTree.fromstring(
'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry>
<reagent-label name="label name"/>
</test-entry>''')
self.lims = Lims('http://testgenologics.com:4040',
username='******',
password='******')
self.instance1 = Mock(root=et, lims=self.lims)
def setUp(self):
et = ElementTree.fromstring(
"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry>
<ri:externalid xmlns:ri="http://genologics.com/ri" id="1" uri="http://testgenologics.com:4040/api/v2/external/1" />
<ri:externalid xmlns:ri="http://genologics.com/ri" id="2" uri="http://testgenologics.com:4040/api/v2/external/2" />
</test-entry>
""")
self.lims = Lims('http://testgenologics.com:4040',
username='******',
password='******')
self.instance1 = Mock(root=et, lims=self.lims)
def setUp(self):
et = ElementTree.fromstring(
"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry>
<test-tags>
<test-tag attrib1="value1" attrib2="value2"/>
<test-tag attrib1="value11" attrib2="value12" attrib3="value13"/>
</test-tags>
</test-entry>
""")
self.lims = Lims('http://testgenologics.com:4040',
username='******',
password='******')
self.instance1 = Mock(root=et, lims=self.lims)
def setUp(self):
et = ElementTree.fromstring(
"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry xmlns:udf="http://genologics.com/ri/userdefined">
<placement uri="http://testgenologics.com:4040/api/v2/artifacts/a1" limsid="a1">
<value>A:1</value>
</placement>
<other>thing</other>
</test-entry>""")
self.lims = Lims('http://testgenologics.com:4040',
username='******',
password='******')
self.instance1 = Mock(root=et, lims=self.lims)
self.dict1 = PlacementDictionary(self.instance1)
self.art1 = Artifact(lims=self.lims, id='a1')
def setUp(self):
et = ElementTree.fromstring(
"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry xmlns:udf="http://genologics.com/ri/userdefined">
<test-tag attrib1="value1" attrib2="value2"/>
<test-tag attrib1="value11" attrib2="value12" attrib3="value13"/>
</test-entry>""")
self.lims = Lims('http://testgenologics.com:4040',
username='******',
password='******')
self.instance1 = Mock(root=et, lims=self.lims)
self.dict1 = XmlElementAttributeDict(self.instance1,
tag='test-tag',
position=0)
self.dict2 = XmlElementAttributeDict(self.instance1,
tag='test-tag',
position=1)
def setUp(self):
et = ElementTree.fromstring(
'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry>
<next-action step-uri="{url}/prt/1/stp/1" action="nextstep" artifact-uri="{url}/arts/a1"/>
</test-entry>'''.format(url='http://testgenologics.com:4040'))
et1 = ElementTree.fromstring(
'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry>
<next-action artifact-uri="{url}/arts/a1"/>
</test-entry>'''.format(url='http://testgenologics.com:4040'))
self.lims = Lims('http://testgenologics.com:4040',
username='******',
password='******')
self.instance1 = Mock(root=et, lims=self.lims)
self.instance_empty = Mock(root=et1, lims=self.lims)
def setUp(self):
et = ElementTree.fromstring(
"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry xmlns:udf="http://genologics.com/ri/userdefined">
<test-tag>
<key1>value1</key1>
</test-tag>
</test-entry>""")
self.lims = Lims('http://testgenologics.com:4040',
username='******',
password='******')
self.instance1 = Mock(root=et, lims=self.lims)
self.dict1 = SubTagDictionary(self.instance1, tag='test-tag')
et = ElementTree.fromstring(
"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<test-entry xmlns:udf="http://genologics.com/ri/userdefined">
</test-entry>""")
self.instance2 = Mock(root=et, lims=self.lims)
self.dict2 = SubTagDictionary(self.instance2, tag='test-tag')
def main():
a = ArgumentParser()
a.add_argument('--url', type=str)
a.add_argument('--username', type=str)
a.add_argument('--password', type=str)
a.add_argument('--config', type=str, required=True)
a.add_argument('--generate_config', action='store_true')
args = a.parse_args()
config = {}
if os.path.isfile(args.config):
with open(args.config, 'r') as open_file:
config = yaml.load(open_file)
url = args.url or config.get('clarity', {}).get('url')
if not url:
print('Specify url on the command line')
a.print_help()
return 1
username = args.username or config.get('clarity', {}).get('username')
if not username:
print('Specify username on the command line')
a.print_help()
return 1
password = args.password or config.get('clarity', {}).get('password')
if not password:
print('Specify password on the command line')
a.print_help()
return 1
lims = Lims(baseuri=url, username=username, password=password)
if args.generate_config:
config['entities'] = generate_entities_expected_output(lims)
with open(args.config, 'w') as open_file:
yaml.dump(config, open_file, width=180, indent=4)
else:
test_all_entities(lims, config.get('entities', {}))
def test_get_file_contents(self):
lims = Lims(self.url, username=self.username, password=self.password)
lims.validate_response = Mock()
lims.request_session = Mock(get=Mock(
return_value=Mock(encoding=None, text='some data\r\n')))
exp_url = self.url + '/api/v2/files/an_id/download'
assert lims.get_file_contents(uri=self.url +
'/api/v2/files/an_id') == 'some data\r\n'
assert lims.request_session.get.return_value.encoding is None
lims.request_session.get.assert_called_with(exp_url,
auth=(self.username,
self.password),
timeout=16)
assert lims.get_file_contents(id='an_id', encoding='utf-16',
crlf=True) == 'some data\n'
assert lims.request_session.get.return_value.encoding == 'utf-16'
lims.request_session.get.assert_called_with(exp_url,
auth=(self.username,
self.password),
timeout=16)
def test_get_uri(self):
lims = Lims(self.url, username=self.username, password=self.password)
assert lims.get_uri(
'artifacts', sample_name='test_sample'
) == '{url}/api/v2/artifacts?sample_name=test_sample'.format(
url=self.url)
def setUp(self):
self.lims = Lims(url, username='******', password='******')
class MyLims:
def __init__(self):
self.lims = Lims(*get_config())
self.lims.check_version()
def get_project_names_slow(self, *proj_nums):
# Get the list of all projects.
projects = self.lims.get_projects()
print('{} projects in total'.format(len(projects)))
# Unfortunately this doesn't work...
# projects = lims.get_projects(name="10657_%")
# and calling .name on every project is slow as it entails an extra GET
# Maybe we need to bypass Clarity and hit PostgreSQL?
#Yes I'm scanning the list many times, but the key thing is I only fetch it once.
res = []
projects = filter_names(( p.name for p in projects ))
for p_num in proj_nums:
p_prefix = str(p_num) + '_'
p_name = None
for pn in projects:
if pn.startswith(p_prefix):
if p_name:
raise LookupError("More than one project found with prefix " + p_prefix)
else:
p_name = pn
res.append(p_name)
return res
def get_project_names(self, *proj_nums):
#Quickly get the list of all project names.
lims = self.lims
projects = []
proot = lims.get(lims.get_uri('projects'))
while True: # Loop over all pages.
for node in proot.findall('.//name'):
projects.append(node.text)
next_page = proot.find('next-page')
if next_page is None: break
proot = lims.get(next_page.attrib['uri'])
#Yes I'm scanning the list many times, but the key thing is I only fetch it once.
res = []
projects = filter_names(projects)
for p_num in proj_nums:
p_prefix = str(p_num) + '_'
p_name = None
for proj in projects:
if proj.startswith(p_prefix):
if p_name:
raise LookupError("More than one project found with prefix " + p_prefix)
else:
p_name = proj
res.append(p_name)
return res
def __init__(self):
self.lims = Lims(*get_config())
self.lims.check_version()
def main(args):
if args.debug:
L.basicConfig(format='{name:s} {levelname:s}: {message:s}',
level=L.DEBUG,
style='{')
# But silence chatter from pyclarity_lims
L.getLogger("requests").propagate = False
L.getLogger("urllib3").propagate = False
else:
L.basicConfig(format='{message:s}', level=L.INFO, style='{')
# Connect to ze lims...
lims = Lims(**get_config())
# How to do this?
# 1 - Firstly look to see if the name is already known. If so, report and exit.
# 2 - Then split out the flowcell name and look for a container with that name.
# 3 - If none, exit. If several, pick the latest (highest number)
# 4 - Find the first analyte and thus the step (see notes)
# 5 - Load the step and set the values (if not set already or forced)
# Sanity check, since I accidentally set several run IDs to
# /lustre/fastqdata/180416_M05898_0001_000000000-D42P7 etc. by foolish use of
# a shell loop.
assert re.match(r"^[0-9]{6}_[A-Za-z0-9_-]+", args.runid), \
"Run ID does not look right - {}".format(args.runid)
# 1
existing_proc = lims.get_processes(type=QC_PROCESS,
udf={'RunID': args.runid})
if len(existing_proc):
L.info("Run {} is already in the LIMS database...".format(args.runid))
# There should just be one!
for ep in existing_proc:
L.info(" " + ep.uri)
if args.close and not args.no_act:
try:
L.info(" Completing the existing step...")
ep.step.get()
ep.step.advance()
L.info(" completed")
except Exception as e:
L.info(" " + str(e))
if args.debug:
# It's useful to be able to get the parent process because that's what gets fed
# into the sample sheet generator.
for epp_uri in sorted(
set(epp.uri for epp in ep.parent_processes())):
L.debug(" parent --> " + epp_uri)
exit(
0
) # Not an error, but FIXME I should allow the ID to be transferred to a new process.
# 2
if args.container_name:
container_name = args.container_name
else:
container_name = re.split('[_-]', args.runid)[-1]
# Prune off the stage letter.
mo = re.match(r'[AB](.........)', container_name)
if mo:
container_name = mo.group(1)
existing_containers = lims.get_containers(name=container_name)
# If the container name is not in all upper case then try making it so...
if container_name != container_name.upper():
existing_containers.extend(
lims.get_containers(name=container_name.upper()))
# 3
if len(existing_containers) == 0:
L.warning("No container found with name {}.".format(container_name))
exit(1)
existing_container = sorted(existing_containers, key=lambda c: c.uri)[-1]
if len(existing_containers) > 1:
L.warning(
"Multiple containers found with name {}. The latest ({}) will be used."
.format(container_name, existing_container.id))
L.info("Finding QC step for container {}.".format(existing_container.uri))
# 4 - HiSeq lane 1 is 1:1 while MiSeq lane is 'A:1' for some reson. We should see on eor the other.
analyte1, = [
existing_container.placements.get(loc) for loc in ['1:1', 'A:1']
if loc in existing_container.placements
]
L.debug("Examining analyte {}".format(analyte1.uri))
# https://clarity.genomics.ed.ac.uk/api/v2/processes?type=Flow%20Cell%20Lane%20QC%20EG%201.0%20ST&inputartifactlimsid=2-126766
# I need to handle the error if there is not one process - older runs lack the step, for example.
# In the development LIMS we also seem to have multiple QC processes per flowcell. As before I think we need to go for
# the latest one.
# Also I've seen a case where a stage was queued and removed (ie. not completed). This shows up on the artifact page but
# not in the list when I run the get_processes search below.
try:
qc_proc = sorted(lims.get_processes(type=QC_PROCESS,
inputartifactlimsid=analyte1.id),
key=lambda p: p.id)[-1]
except IndexError:
L.error("Could not find a QC step for this container.")
exit(1)
# 5 - it seems Clarity likes to set UDFs on a step not a process(?)
qc_step = qc_proc.step.details
L.info("Setting UDFs on {}".format(qc_step.uri))
if qc_step.udf.get('RunID'):
if args.force:
L.info(
"Forcing new RunID for step {} in place of '{}' as you requested."
.format(qc_step.id, qc_step.udf.get('RunID')))
else:
L.info("RunID for step {} is already set to '{}'.".format(
qc_step.id, qc_step.udf.get('RunID')))
exit(1)
#So, set it...
qc_step.udf['RunID'] = args.runid
qc_step.udf['Run Report'] = REPORT_URL_TEMPLATE.format(args.runid)
L.debug(pformat(qc_step.udf))
if args.no_act:
L.info("Nothing will be saved as no_act was set.")
else:
qc_step.put()
if args.close:
try:
L.info("Completing the step...")
qc_proc.step.get(force=True)
qc_proc.step.advance()
L.info("completed")
except Exception as e:
L.info(str(e))
L.info("DONE")