def icatconfig(setupicat, testConfig, request):
client, conf, config = getConfig(ids="mandatory")
mainbase = request.config.getini('mainstoragebase')
archivebase = request.config.getini('archivestoragebase')
conf.cmdargs.append("--mainStorageBase=%s" % mainbase)
conf.cmdargs.append("--archiveStorageBase=%s" % archivebase)
incomingbase = request.config.getini('incomingbase')
proposaldir = os.path.join(incomingbase, testProposalNo.replace('/', '_'))
os.mkdir(proposaldir)
conf.proposaldir = proposaldir
def cleanup():
shutil.rmtree(proposaldir)
client.login(conf.auth, conf.credentials)
query = Query(client,
"Dataset",
conditions={"name": "LIKE '%s-%%'" % testDatasetPrefix})
wipe_data(client, query)
query.setLimit((0, 500))
while True:
objs = client.search(query)
if not objs:
break
client.deleteMany(objs)
client.logout()
if testConfig.cleanup:
request.addfinalizer(cleanup)
return (conf, config)
def test_download(tmpdirsec, client, case, method):
conf = getConfig(confSection=case['dluser'])
if len(case['dfs']) > 1:
zfname = os.path.join(tmpdirsec.dir, "%s.zip" % case['dsname'])
print("\nDownload %s to file %s" % (case['dsname'], zfname))
args = conf.cmdargs + [ '--outputfile', zfname,
case['invname'], case['dsname'], method ]
callscript("downloaddata.py", args)
zf = zipfile.ZipFile(zfname, 'r')
zinfos = zf.infolist()
assert len(zinfos) == len(case['dfs'])
for df in case['dfs']:
zi = None
for i in zinfos:
if i.filename.endswith(df['dfname']):
zi = i
break
assert zi is not None
assert "%x" % (zi.CRC & 0xffffffff) == df['testfile'].crc32
assert zi.file_size == df['testfile'].size
elif len(case['dfs']) == 1:
df = case['dfs'][0]
dfname = os.path.join(tmpdirsec.dir, "dl_%s" % df['dfname'])
print("\nDownload %s to file %s" % (case['dsname'], dfname))
args = conf.cmdargs + [ '--outputfile', dfname,
case['invname'], case['dsname'], method ]
callscript("downloaddata.py", args)
assert filecmp.cmp(df['testfile'].fname, dfname)
else:
raise RuntimeError("No datafiles for dataset %s" % case['dsname'])
def test_add_relateddatafile(data, user, rdfname):
client, conf = getConfig(confSection=user)
client.login(conf.auth, conf.credentials)
rdfdata = data['related_datafiles'][rdfname]
rdf = client.new("relatedDatafile")
initobj(rdf, rdfdata)
rdf.sourceDatafile = get_datafile(client, rdfdata['source'])
rdf.destDatafile = create_datafile(client, data, rdfdata['dest'])
rdf.create()
def test_add_publication(data, user, pubname):
client, conf = getConfig(confSection=user)
client.login(conf.auth, conf.credentials)
pubdata = data['publications'][pubname]
publication = client.new("publication")
initobj(publication, pubdata)
query = "Investigation [name='%s']" % pubdata['investigation']
publication.investigation = client.assertedSearch(query)[0]
publication.create()
def client(setupicat, request):
conf = getConfig(ids="mandatory")
client = icat.Client(conf.url, **conf.client_kwargs)
client.login(conf.auth, conf.credentials)
def cleanup():
query = "SELECT df FROM Datafile df WHERE df.location IS NOT NULL"
client.deleteData(client.search(query))
request.addfinalizer(cleanup)
return client
def test_add_relateddatafile(data, user, rdfname):
conf = getConfig(confSection=user)
client = icat.Client(conf.url, **conf.client_kwargs)
client.login(conf.auth, conf.credentials)
rdfdata = data['related_datafiles'][rdfname]
rdf = client.new("relatedDatafile")
initobj(rdf, rdfdata)
rdf.sourceDatafile = get_datafile(client, rdfdata['source'])
rdf.destDatafile = create_datafile(client, data, rdfdata['dest'])
rdf.create()
def test_clone_ids(setupicat):
"""Same as above, but configure ids this time.
"""
client, conf = getConfig(ids="mandatory")
clone = client.clone()
assert isinstance(clone, icat.client.Client)
assert clone.url == client.url
assert clone.ids.url == client.ids.url
assert clone.kwargs == client.kwargs
assert clone.apiversion == client.apiversion
def test_clone_minimal(setupicat):
"""Clone a simple client. Not logged in, no ids.
"""
client, conf = getConfig(ids=False)
clone = client.clone()
assert isinstance(clone, icat.client.Client)
assert clone.url == client.url
assert clone.ids is None
assert clone.kwargs == client.kwargs
assert clone.apiversion == client.apiversion
def test_add_publication(data, user, pubname):
conf = getConfig(confSection=user)
client = icat.Client(conf.url, **conf.client_kwargs)
client.login(conf.auth, conf.credentials)
pubdata = data['publications'][pubname]
publication = client.new("publication")
initobj(publication, pubdata)
query = "Investigation [name='%s']" % pubdata['investigation']
publication.investigation = client.assertedSearch(query)[0]
publication.create()
def client(setupicat, request):
conf = getConfig(ids="mandatory")
client = icat.Client(conf.url, **conf.client_kwargs)
client.login(conf.auth, conf.credentials)
def cleanup():
query = "SELECT df FROM Datafile df WHERE df.location IS NOT NULL"
client.deleteData(client.search(query))
request.addfinalizer(cleanup)
return client
def test_check_summary_user(tmpdirsec, user):
"""Check the number of objects from a user's point of view.
This checks which objects a given user may see and thus whether
the (read) access rules work as expected.
"""
summary = os.path.join(tmpdirsec, "summary.%s" % user)
ref = refsummary[user]
_, conf = getConfig(confSection=user)
with open(summary, "wt") as out:
callscript("icatsummary.py", conf.cmdargs, stdout=out)
assert filecmp.cmp(ref, summary), "ICAT content was not as expected"
def test_client_sslContext_kwarg(setupicat):
"""Set the `sslContext` keyword argument to the Client constructor.
Issue #34.
"""
_, conf = getConfig()
kwargs = getClientKWargs(conf)
sslverify = kwargs.pop('checkCert', True)
cafile = kwargs.pop('caFile', None)
capath = kwargs.pop('caPath', None)
kwargs['sslContext'] = create_ssl_context(sslverify, cafile, capath)
client = icat.Client(conf.url, **kwargs)
client.login(conf.auth, conf.credentials)
def test_check_summary_user(tmpdirsec, user):
"""Check the number of objects from a user's point of view.
This checks which objects a given user may see and thus whether
the (read) access rules work as expected.
"""
summary = os.path.join(tmpdirsec.dir, "summary.%s" % user)
ref = refsummary[user]
conf = getConfig(confSection=user)
with open(summary, "wt") as out:
callscript("icatsummary.py", conf.cmdargs, stdout=out)
assert filecmp.cmp(ref, summary), "ICAT content was not as expected"
def test_get_icat_version():
"""Query the version from the test ICAT server.
This implicitly tests that the test ICAT server is properly
configured and that we can connect to it.
"""
client, conf, _ = getConfig(needlogin=False, ids=False)
# python-icat supports ICAT server 4.2 or newer. But actually, we
# just want to check that client.apiversion is set and supports
# comparison with version strings.
assert client.apiversion >= '4.2'
print("\nConnect to %s\nICAT version %s\n" % (conf.url, client.apiversion))
def client(setupicat, testConfig, request):
client, conf, _ = getConfig(ids="mandatory")
client.login(conf.auth, conf.credentials)
def cleanup():
query = Query(client, "Dataset", conditions={
"name": "LIKE '%s-%%'" % testDatasetName
})
wipe_data(client, query)
client.deleteMany(client.search(query))
client.logout()
if testConfig.cleanup:
request.addfinalizer(cleanup)
createDatasets(client, testConfig)
return client
def test_add_study(data, user, studyname):
client, conf = getConfig(confSection=user)
client.login(conf.auth, conf.credentials)
studydata = data['studies'][studyname]
study = client.new("study")
initobj(study, studydata)
query = "User [name='%s']" % data['users'][studydata['user']]['name']
study.user = client.assertedSearch(query)[0]
for invname in studydata['investigations']:
query = "Investigation [name='%s']" % invname
si = client.new("studyInvestigation")
si.investigation = client.assertedSearch(query)[0]
study.studyInvestigations.append(si)
study.create()
def test_get_icat_version():
"""Query the version from the test ICAT server.
This implicitly tests that the test ICAT server is properly
configured and that we can connect to it.
"""
conf = getConfig(needlogin=False)
client = icat.Client(conf.url, **conf.client_kwargs)
# python-icat supports ICAT server 4.2 or newer. But actually, we
# just want to check that client.apiversion is set and supports
# comparison with version strings.
assert client.apiversion >= '4.2'
print("\nConnect to %s\nICAT version %s\n" % (conf.url, client.apiversion))
def test_get_ids_version():
"""Query the version from the test IDS server.
This implicitly tests that the test ICAT and IDS servers are
properly configured and that we can connect to them.
"""
client, conf, _ = getConfig(needlogin=False, ids="mandatory")
# python-icat supports all publicly released IDS server version,
# e.g. 1.0.0 or newer. But actually, we just want to check that
# client.apiversion is set and supports comparison with version
# strings.
assert client.ids.apiversion >= '1.0.0'
print("\nConnect to %s\nIDS version %s\n"
% (conf.idsurl, client.ids.apiversion))
def test_add_study(data, user, studyname):
pytest.skip("Study disabled, see Issue icatproject/icat.server#155")
conf = getConfig(confSection=user)
client = icat.Client(conf.url, **conf.client_kwargs)
client.login(conf.auth, conf.credentials)
studydata = data['studies'][studyname]
study = client.new("study")
initobj(study, studydata)
query = "User [name='%s']" % studydata['user']
study.user = client.assertedSearch(query)[0]
for invname in studydata['investigations']:
query = "Investigation [name='%s']" % invname
si = client.new("studyInvestigation")
si.investigation = client.assertedSearch(query)[0]
study.studyInvestigations.append(si)
study.create()
def test_client_set_transport(setupicat):
"""Try setting a custom transport in the client using set_options().
See Issue #33 why this is relevant.
"""
_, conf = getConfig()
kwargs = getClientKWargs(conf)
client = icat.Client(conf.url, **kwargs)
proxy = {}
if conf.http_proxy:
proxy['http'] = config.http_proxy
if conf.https_proxy:
proxy['https'] = config.https_proxy
transport = MyHTTPSTransport(client.sslContext, proxy=proxy)
client.set_options(transport=transport)
client.login(conf.auth, conf.credentials)
assert transport.sendCounter >= 1
def test_get_ids_version():
"""Query the version from the test IDS server.
This implicitly tests that the test ICAT and IDS servers are
properly configured and that we can connect to them.
"""
conf = getConfig(needlogin=False, ids="mandatory")
client = icat.Client(conf.url, **conf.client_kwargs)
# python-icat supports all publicly released IDS server version,
# e.g. 1.0.0 or newer. But actually, we just want to check that
# client.apiversion is set and supports comparison with version
# strings.
assert client.ids.apiversion >= '1.0.0'
print("\nConnect to %s\nIDS version %s\n"
% (conf.idsurl, client.ids.apiversion))
def test_add_study(data, user, studyname):
pytest.skip("Study disabled, see Issue icatproject/icat.server#155")
conf = getConfig(confSection=user)
client = icat.Client(conf.url, **conf.client_kwargs)
client.login(conf.auth, conf.credentials)
studydata = data['studies'][studyname]
study = client.new("study")
initobj(study, studydata)
query = "User [name='%s']" % studydata['user']
study.user = client.assertedSearch(query)[0]
for invname in studydata['investigations']:
query = "Investigation [name='%s']" % invname
si = client.new("studyInvestigation")
si.investigation = client.assertedSearch(query)[0]
study.studyInvestigations.append(si)
study.create()
def test_equality_client(client):
"""Test that objects that belong to different clients are never equal.
There used to be a bug such that the client was not taken into
account, fixed in c9a1be6.
"""
# Get a second client that is connected to the same server.
client2, _ = getConfig(needlogin=False)
u1 = client.new("user", id=728, name="u_a")
u2 = client2.new("user", id=728, name="u_a")
# u1 and u2 have all attributes, including the id the same.
assert u1.id == u2.id
assert u1.name == u2.name
# But they belong to different client instances and thus, they are
# still not equal.
assert u1 != u2
assert not (u1 == u2)
def test_equality_client(client):
"""Test that objects that belong to different clients are never equal.
There used to be a bug such that the client was not taken into
account, fixed in c9a1be6.
"""
# Get a second client that is connected to the same server.
conf = getConfig(needlogin=False)
client2 = icat.Client(conf.url, **conf.client_kwargs)
u1 = client.new("user", id=728, name="u_a")
u2 = client2.new("user", id=728, name="u_a")
# u1 and u2 have all attributes, including the id the same.
assert u1.id == u2.id
assert u1.name == u2.name
# But they belong to different client instances and thus, they are
# still not equal.
assert u1 != u2
assert not (u1 == u2)
def icatconfig(setupicat, testConfig, request):
client, conf, config = getConfig(ids="mandatory")
client.login(conf.auth, conf.credentials)
mainbase = request.config.getini('mainstoragebase')
archivebase = request.config.getini('archivestoragebase')
conf.cmdargs.append("--mainStorageBase=%s" % mainbase)
conf.cmdargs.append("--archiveStorageBase=%s" % archivebase)
incomingbase = request.config.getini('incomingbase')
proposaldir = os.path.join(incomingbase, testProposalNo.replace('/', '_'))
os.mkdir(proposaldir)
conf.proposaldir = proposaldir
def cleanup():
os.rmdir(proposaldir)
client.logout()
if testConfig.cleanup:
request.addfinalizer(cleanup)
return (client, conf, config)
def test_login(user):
"""Login to the ICAT server.
"""
client, conf = getConfig(confSection=user)
sessionId = client.login(conf.auth, conf.credentials)
assert sessionId
assert sessionId == client.sessionId
username = client.getUserName()
assert username == "%s/%s" % (conf.auth, user)
print("\nLogged in as %s to %s." % (user, conf.url))
client.logout()
assert client.sessionId is None
# Verify that the logout was effective, e.g. that the sessionId is
# invalidated.
with tmpSessionId(client, sessionId):
with pytest.raises(icat.exception.ICATSessionError):
username = client.getUserName()
def test_upload(tmpdirsec, client, case):
f = DummyDatafile(tmpdirsec.dir,
case['dfname'], case['size'], case['mtime'])
print("\nUpload file %s" % case['dfname'])
conf = getConfig(confSection=case['uluser'])
args = conf.cmdargs + [case['invname'], case['dsname'],
case['dfformat'], f.fname]
callscript("addfile.py", args)
query = Query(client, "Datafile", conditions={
"name": "= '%s'" % case['dfname'],
"dataset.name": "= '%s'" % case['dsname'],
"dataset.investigation.name": "= '%s'" % case['invname'],
})
df = client.assertedSearch(query)[0]
assert df.location is not None
assert df.fileSize == f.size
assert df.checksum == f.crc32
if f.mtime:
assert df.datafileModTime == f.mtime
case['testfile'] = f
def icatconfig(setupicat, testConfig, request):
client, conf, config = getConfig(ids="mandatory")
def cleanup():
client.login(conf.auth, conf.credentials)
query = Query(client,
"Dataset",
conditions={"name": "LIKE '%s-%%'" % testDatasetName})
wipe_data(client, query)
query.setLimit((0, 500))
while True:
objs = client.search(query)
if not objs:
break
client.deleteMany(objs)
client.logout()
if testConfig.cleanup:
request.addfinalizer(cleanup)
return (client, conf, config)
def test_login(user):
"""Login to the ICAT server.
"""
conf = getConfig(confSection=user)
client = icat.Client(conf.url, **conf.client_kwargs)
sessionId = client.login(conf.auth, conf.credentials)
assert sessionId
assert sessionId == client.sessionId
username = client.getUserName()
assert username == user
print("\nLogged in as %s to %s." % (user, conf.url))
client.logout()
assert client.sessionId is None
# Verify that the logout was effective, e.g. that the sessionId is
# invalidated.
with tmpSessionId(client, sessionId):
with pytest.raises(icat.exception.ICATSessionError):
username = client.getUserName()
def test_upload(tmpdirsec, client, case):
f = DummyDatafile(tmpdirsec.dir, case['dfname'], case['size'],
case['mtime'])
print("\nUpload file %s" % case['dfname'])
conf = getConfig(confSection=case['uluser'])
args = conf.cmdargs + [
case['invname'], case['dsname'], case['dfformat'], f.fname
]
callscript("addfile.py", args)
query = Query(client,
"Datafile",
conditions={
"name": "= '%s'" % case['dfname'],
"dataset.name": "= '%s'" % case['dsname'],
"dataset.investigation.name": "= '%s'" % case['invname'],
})
df = client.assertedSearch(query)[0]
assert df.location is not None
assert df.fileSize == f.size
assert df.checksum == f.crc32
if f.mtime:
assert df.datafileModTime == f.mtime
case['testfile'] = f
def test_clone_login(setupicat):
"""Clone a client that is logged in.
The clone should not share the session. Original client and clone
should be able to login and out without interfering the other.
"""
client, conf = getConfig()
client.login(conf.auth, conf.credentials)
clone = client.clone()
assert clone.url == client.url
assert clone.kwargs == client.kwargs
assert clone.apiversion == client.apiversion
assert clone.sessionId is None, "the clone must not inherit the session"
# The clone may start it's own session
clone.login(conf.auth, conf.credentials)
assert clone.sessionId
assert clone.sessionId != client.sessionId
# both are still logged in as the same user
assert clone.getUserName() == client.getUserName()
# Now logout the clone. This must not affect the client's session.
clone.logout()
assert clone.sessionId is None
assert client.sessionId
def test_addjob(user, jobname):
_, conf = getConfig(confSection=user)
args = conf.cmdargs + [testinput, jobname]
callscript("add-job.py", args)
def test_addinvdata(user, invname):
_, conf = getConfig(confSection=user)
args = conf.cmdargs + [testinput, invname]
callscript("add-investigation-data.py", args)
def test_create_sampletype(user, sample):
_, conf = getConfig(confSection=user)
args = conf.cmdargs + [testinput, sample]
callscript("create-sampletype.py", args)
def client():
conf = getConfig(needlogin=False)
client = icat.Client(conf.url, **conf.client_kwargs)
return client
assert df.location is not None
assert df.fileSize == f.size
assert df.checksum == f.crc32
if f.mtime:
assert df.datafileModTime == f.mtime
case['testfile'] = f
@pytest.fixture(scope='function', params=["getData", "getPreparedData"])
def method(request):
return request.param
@pytest.mark.parametrize(("case"), markeddatasets)
def test_download(tmpdirsec, client, case, method):
conf = getConfig(confSection=case['dluser'])
if len(case['dfs']) > 1:
zfname = os.path.join(tmpdirsec.dir, "%s.zip" % case['dsname'])
print("\nDownload %s to file %s" % (case['dsname'], zfname))
args = conf.cmdargs + [
'--outputfile', zfname, case['invname'], case['dsname'], method
]
callscript("downloaddata.py", args)
zf = zipfile.ZipFile(zfname, 'r')
zinfos = zf.infolist()
assert len(zinfos) == len(case['dfs'])
for df in case['dfs']:
zi = None
for i in zinfos:
if i.filename.endswith(df['dfname']):
zi = i
def test_create_investigation(invname):
_, conf = getConfig(confSection="useroffice")
args = conf.cmdargs + [testinput, invname]
callscript("create-investigation.py", args)
def test_create_investigation(invname):
conf = getConfig(confSection="useroffice")
args = conf.cmdargs + [testinput, invname]
callscript("create-investigation.py", args)
def client():
conf = getConfig(needlogin=False)
client = icat.Client(conf.url, **conf.client_kwargs)
return client
def test_addjob(user, jobname):
conf = getConfig(confSection=user)
args = conf.cmdargs + [testinput, jobname]
callscript("add-job.py", args)
def client(setupicat):
client, conf = getConfig()
client.login(conf.auth, conf.credentials)
return client
def client(setupicat):
client, conf = getConfig(confSection="nbour")
client.login(conf.auth, conf.credentials)
return client
def test_getversion():
"""Get version info from the ICAT server.
"""
conf = getConfig(needlogin=False)
callscript("getversion.py", conf.cmdargs)
def test_login(user):
"""Login to the ICAT server.
"""
conf = getConfig(confSection=user)
callscript("login.py", conf.cmdargs)
def test_addinvdata(user, invname):
conf = getConfig(confSection=user)
args = conf.cmdargs + [testinput, invname]
callscript("add-investigation-data.py", args)
def test_getversion():
"""Get version info from the ICAT server.
"""
_, conf = getConfig(needlogin=False)
callscript("getversion.py", conf.cmdargs)
def test_create_sampletype(user, sample):
conf = getConfig(confSection=user)
args = conf.cmdargs + [testinput, sample]
callscript("create-sampletype.py", args)
def test_login(user):
"""Login to the ICAT server.
"""
_, conf = getConfig(confSection=user)
callscript("login.py", conf.cmdargs)
def conf(setupicat):
return getConfig(confSection="acord", ids="mandatory")
def conf(setupicat):
return getConfig(confSection="acord", ids="mandatory")
def client(setupicat):
conf = getConfig()
client = icat.Client(conf.url, **conf.client_kwargs)
client.login(conf.auth, conf.credentials)
return client
def client(setupicat):
client, conf = getConfig(confSection="acord", ids="mandatory")
client.login(conf.auth, conf.credentials)
return client
def cmdargs(setupicat):
_, conf = getConfig(confSection="acord", ids="mandatory")
return conf.cmdargs + ["-f", "XML"]
def checker():
conf = getConfig(needlogin=False)
client = icat.Client(conf.url, **conf.client_kwargs)
return ICATChecker(client)
