def test_ingest_datafiles_upload(tmpdirsec, client, dataset, cmdargs):
"""Upload datafiles to IDS from icatingest.
Same as last test, but set the --upload-datafiles flag so that
icatingest will not create the datafiles as objects in the ICAT,
but upload the files to IDS instead.
"""
dummyfiles = [ DummyDatafile(tmpdirsec.dir,
f['dfname'], f['size'], f['mtime'])
for f in testdatafiles ]
args = cmdargs + ["-i", datafiles, "--upload-datafiles",
"--datafile-dir", tmpdirsec.dir]
callscript("icatingest.py", args)
# Verify that the datafiles have been uploaded.
dataset = client.searchMatching(dataset)
for f in dummyfiles:
query = Query(client, "Datafile", conditions={
"name": "= '%s'" % f.name,
"dataset.id": "= %d" % dataset.id,
})
df = client.assertedSearch(query)[0]
assert df.location is not None
assert df.fileSize == f.size
assert df.checksum == f.crc32
if f.mtime:
assert df.datafileModTime == f.mtime
def test_download(tmpdirsec, client, case, method):
conf = getConfig(confSection=case['dluser'])
if len(case['dfs']) > 1:
zfname = os.path.join(tmpdirsec.dir, "%s.zip" % case['dsname'])
print("\nDownload %s to file %s" % (case['dsname'], zfname))
args = conf.cmdargs + [ '--outputfile', zfname,
case['invname'], case['dsname'], method ]
callscript("downloaddata.py", args)
zf = zipfile.ZipFile(zfname, 'r')
zinfos = zf.infolist()
assert len(zinfos) == len(case['dfs'])
for df in case['dfs']:
zi = None
for i in zinfos:
if i.filename.endswith(df['dfname']):
zi = i
break
assert zi is not None
assert "%x" % (zi.CRC & 0xffffffff) == df['testfile'].crc32
assert zi.file_size == df['testfile'].size
elif len(case['dfs']) == 1:
df = case['dfs'][0]
dfname = os.path.join(tmpdirsec.dir, "dl_%s" % df['dfname'])
print("\nDownload %s to file %s" % (case['dsname'], dfname))
args = conf.cmdargs + [ '--outputfile', dfname,
case['invname'], case['dsname'], method ]
callscript("downloaddata.py", args)
assert filecmp.cmp(df['testfile'].fname, dfname)
else:
raise RuntimeError("No datafiles for dataset %s" % case['dsname'])
def test_create(imgdir, monkeypatch):
"""Create the index.
"""
monkeypatch.chdir(str(imgdir))
callscript("photoidx.py", ["create"])
idxfile = str(imgdir / ".index.yaml")
assert filecmp.cmp(refindex, idxfile), "index file differs from reference"
def test_ingest_datafiles_upload(tmpdirsec, client, dataset, cmdargs):
"""Upload datafiles to IDS from icatingest.
Same as last test, but set the --upload-datafiles flag so that
icatingest will not create the datafiles as objects in the ICAT,
but upload the files to IDS instead.
"""
dummyfiles = [ DummyDatafile(tmpdirsec, f['dfname'], f['size'], f['mtime'])
for f in testdatafiles ]
args = cmdargs + ["-i", datafiles, "--upload-datafiles",
"--datafile-dir", tmpdirsec]
callscript("icatingest.py", args)
# Verify that the datafiles have been uploaded.
dataset = client.searchMatching(dataset)
for f in dummyfiles:
query = Query(client, "Datafile", conditions={
"name": "= '%s'" % f.name,
"dataset.id": "= %d" % dataset.id,
})
df = client.assertedSearch(query)[0]
assert df.location is not None
assert df.fileSize == f.size
assert df.checksum == f.crc32
if f.mtime:
assert df.datafileModTime == f.mtime
def test_create(imgdir, monkeypatch):
"""Create the index.
"""
monkeypatch.chdir(imgdir)
callscript("photoidx.py", ["create"])
idxfile = os.path.join(imgdir, ".index.yaml")
assert filecmp.cmp(refindex, idxfile), "index file differs from reference"
def test_ingest_xml(standardConfig):
"""Restore the ICAT content from a XML dumpfile.
"""
callscript("wipeicat.py", standardConfig.cmdargs)
refdump = backends["XML"]['refdump']
args = standardConfig.cmdargs + ["-f", "XML", "-i", refdump]
callscript("icatingest.py", args)
def test_check_summary_root_yaml(standardConfig, tmpdirsec):
"""Check the number of objects for each class at the ICAT server.
"""
summary = os.path.join(tmpdirsec.dir, "summary")
ref = refsummary["root"]
with open(summary, "wt") as out:
callscript("icatsummary.py", standardConfig.cmdargs, stdout=out)
assert filecmp.cmp(ref, summary), "ICAT content was not as expected"
def test_check_summary_root(standardCmdArgs, tmpdirsec):
"""Check the number of objects for each class at the ICAT server.
"""
summary = os.path.join(tmpdirsec, "summary")
ref = refsummary["root"]
with open(summary, "wt") as out:
callscript("icatsummary.py", standardCmdArgs, stdout=out)
assert filecmp.cmp(ref, summary), "ICAT content was not as expected"
def test_ls_all(imgdir):
"""List all images.
"""
fname = imgdir / "out"
with fname.open("wt") as f:
callscript("photoidx.py", ["-d", str(imgdir), "ls"], stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == testimgs
def test_ls_all(imgdir):
"""List all images.
"""
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
callscript("photoidx.py", ["-d", imgdir, "ls"], stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == testimgs
def test_ls_by_single_tag(imgdir):
"""Select by one single tag.
"""
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
args = ["-d", imgdir, "ls", "--tags", "Shinto_shrine"]
callscript("photoidx.py", args, stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == ["dsc_4664.jpg", "dsc_4831.jpg"]
def test_lstags_all(imgdir):
"""List tags.
"""
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["-d", str(imgdir), "lstags"]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == ["Hakone", "Shinto_shrine", "Tokyo"]
def test_lstags_all(imgdir):
"""List tags.
"""
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
args = ["-d", imgdir, "lstags"]
callscript("photoidx.py", args, stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == ["Hakone", "Shinto_shrine", "Tokyo"]
def test_ls_by_single_tag(imgdir):
"""Select by one single tag.
"""
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["-d", str(imgdir), "ls", "--tags", "Shinto_shrine"]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == ["dsc_4664.jpg", "dsc_4831.jpg"]
def ingest(self, conf):
args = conf.cmdargs + [str(self.proposal), self.name]
args.extend(f.path for f in self.files)
start = timer()
callscript("addfile-archive.py", args)
end = timer()
elapsed = Time(end - start)
log.info("Ingest %s to dataset %s in %s (%s/s)", self.size, self.name,
elapsed, MemorySpace(self.size / elapsed))
return StatItem("ingest", self.name, int(self.size), float(elapsed))
def test_lstags_by_tags(imgdir, monkeypatch):
"""List tags selected by tags.
"""
monkeypatch.chdir(str(imgdir))
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["lstags", "--tags", "Tokyo"]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == ["Shinto_shrine", "Tokyo"]
def test_ingest_dataset_params(client, dataset, cmdargs):
"""Ingest a file setting some dataset parameters.
"""
dataset.create()
args = cmdargs + ["-i", ds_params]
callscript("icatingest.py", args)
verify_dataset_params(client, dataset, {
("Magnetic field", 5.3, "T"),
("Reactor power", 10.0, "MW"),
("Sample temperature", 293.15, "K")
})
def test_check_content(standardConfig, tmpdirsec):
"""Dump the resulting content and compare with a reference dump.
"""
dump = os.path.join(tmpdirsec.dir, "dump.yaml")
fdump = os.path.join(tmpdirsec.dir, "dump-filter.yaml")
reffdump = os.path.join(tmpdirsec.dir, "dump-filter-ref.yaml")
filter_file(refdump, reffdump, *yaml_filter)
args = standardConfig.cmdargs + ["-f", "YAML", "-o", dump]
callscript("icatdump.py", args)
filter_file(dump, fdump, *yaml_filter)
assert filecmp.cmp(reffdump, fdump), "content of ICAT was not as expected"
def test_check_content(standardConfig, tmpdirsec):
"""Dump the resulting content and compare with a reference dump.
"""
dump = os.path.join(tmpdirsec.dir, "dump.yaml")
fdump = os.path.join(tmpdirsec.dir, "dump-filter.yaml")
reffdump = os.path.join(tmpdirsec.dir, "dump-filter-ref.yaml")
filter_file(refdump, reffdump, *yaml_filter)
args = standardConfig.cmdargs + ["-f", "YAML", "-o", dump]
callscript("icatdump.py", args)
filter_file(dump, fdump, *yaml_filter)
assert filecmp.cmp(reffdump, fdump), "content of ICAT was not as expected"
def test_ingest_dataset_params(client, dataset, cmdargs):
"""Ingest a file setting some dataset parameters.
"""
dataset.create()
args = cmdargs + ["-i", ds_params]
callscript("icatingest.py", args)
verify_dataset_params(client, dataset, {
("Magnetic field", 5.3, "T"),
("Reactor power", 10.0, "MW"),
("Sample temperature", 293.15, "K")
})
def test_lstags_by_tags(imgdir, monkeypatch):
"""List tags selected by tags.
"""
monkeypatch.chdir(imgdir)
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
args = ["lstags", "--tags", "Tokyo"]
callscript("photoidx.py", args, stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == ["Shinto_shrine", "Tokyo"]
def test_ls_by_empty_tag(imgdir):
"""Select by empty tags.
The option tags with empty value selects images having no tag.
"""
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
args = ["-d", imgdir, "ls", "--tags", ""]
callscript("photoidx.py", args, stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == ["dsc_5126.jpg", "dsc_5167.jpg"]
def test_addtag_by_date(imgdir):
"""Select by date.
"""
args = ["-d", str(imgdir), "addtag", "--date", "2016-03-05", "Hakone"]
callscript("photoidx.py", args)
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["-d", str(imgdir), "ls", "--tags", "Hakone"]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == ["dsc_4831.jpg"]
def test_addtag_all(imgdir):
"""Tag all images.
"""
args = ["-d", str(imgdir), "addtag", "all"]
callscript("photoidx.py", args)
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["-d", str(imgdir), "ls", "--tags", "all"]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == testimgs
def test_ingest_datafiles(tmpdirsec, client, dataset, cmdargs):
"""Ingest a dataset with some datafiles.
"""
dummyfiles = [f["dfname"] for f in testdatafiles]
args = cmdargs + ["-i", datafiles]
callscript("icatingest.py", args)
# Verify that the datafiles have been uploaded.
dataset = client.searchMatching(dataset)
for fname in dummyfiles:
query = Query(client, "Datafile", conditions={"name": "= '%s'" % fname, "dataset.id": "= %d" % dataset.id})
df = client.assertedSearch(query)[0]
assert df.location is None
def test_check_summary_user(tmpdirsec, user):
"""Check the number of objects from a user's point of view.
This checks which objects a given user may see and thus whether
the (read) access rules work as expected.
"""
summary = os.path.join(tmpdirsec.dir, "summary.%s" % user)
ref = refsummary[user]
conf = getConfig(confSection=user)
with open(summary, "wt") as out:
callscript("icatsummary.py", conf.cmdargs, stdout=out)
assert filecmp.cmp(ref, summary), "ICAT content was not as expected"
def test_addtag_all(imgdir):
"""Tag all images.
"""
args = ["-d", imgdir, "addtag", "all"]
callscript("photoidx.py", args)
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
args = ["-d", imgdir, "ls", "--tags", "all"]
callscript("photoidx.py", args, stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == testimgs
def test_check_content(standardCmdArgs, tmpdirsec):
"""Dump the resulting content and compare with a reference dump.
"""
require_icat_version("4.6.0", "Issue icatproject/icat.server#155")
dump = os.path.join(tmpdirsec, "dump.yaml")
fdump = os.path.join(tmpdirsec, "dump-filter.yaml")
reffdump = os.path.join(tmpdirsec, "dump-filter-ref.yaml")
filter_file(refdump, reffdump, *yaml_filter)
args = standardCmdArgs + ["-f", "YAML", "-o", dump]
callscript("icatdump.py", args)
filter_file(dump, fdump, *yaml_filter)
assert filecmp.cmp(reffdump, fdump), "content of ICAT was not as expected"
def test_check_summary_user(tmpdirsec, user):
"""Check the number of objects from a user's point of view.
This checks which objects a given user may see and thus whether
the (read) access rules work as expected.
"""
summary = os.path.join(tmpdirsec, "summary.%s" % user)
ref = refsummary[user]
_, conf = getConfig(confSection=user)
with open(summary, "wt") as out:
callscript("icatsummary.py", conf.cmdargs, stdout=out)
assert filecmp.cmp(ref, summary), "ICAT content was not as expected"
def test_addtag_by_date(imgdir):
"""Select by date.
"""
args = ["-d", imgdir, "addtag", "--date", "2016-03-05", "Hakone"]
callscript("photoidx.py", args)
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
args = ["-d", imgdir, "ls", "--tags", "Hakone"]
callscript("photoidx.py", args, stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == ["dsc_4831.jpg"]
def test_ls_by_empty_tag(imgdir):
"""Select by empty tags.
The option tags with empty value selects images having no tag.
"""
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["-d", str(imgdir), "ls", "--tags", ""]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == ["dsc_5126.jpg", "dsc_5167.jpg"]
def test_ls_by_mult_tags(imgdir):
"""Select by multiple tags.
Combining multiple tags acts like an and, it selects only images
having all the tags set.
"""
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
args = ["-d", imgdir, "ls", "--tags", "Tokyo,Shinto_shrine"]
callscript("photoidx.py", args, stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == ["dsc_4664.jpg"]
def test_select_by_tag(imgdir, monkeypatch):
"""Select by tag.
"""
monkeypatch.chdir(str(imgdir))
args = ["select", "--tags", "Shinto_shrine"]
callscript("photoidx.py", args)
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["ls", "--selected"]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == ["dsc_4664.jpg", "dsc_4831.jpg", "dsc_5126.jpg"]
def test_rmtag_all(imgdir, monkeypatch):
"""Remove a tag from all images.
"""
monkeypatch.chdir(imgdir)
args = ["-d", imgdir, "rmtag", "all"]
callscript("photoidx.py", args)
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
args = ["-d", imgdir, "ls", "--tags", "all"]
callscript("photoidx.py", args, stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == []
def test_ls_by_mult_tags(imgdir):
"""Select by multiple tags.
Combining multiple tags acts like an and, it selects only images
having all the tags set.
"""
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["-d", str(imgdir), "ls", "--tags", "Tokyo,Shinto_shrine"]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == ["dsc_4664.jpg"]
def test_deselect_by_files(imgdir, monkeypatch):
"""Deselect by file names.
"""
monkeypatch.chdir(str(imgdir))
args = ["deselect", "dsc_4831.jpg"]
callscript("photoidx.py", args)
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["ls", "--selected"]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == ["dsc_4664.jpg", "dsc_5126.jpg"]
def test_check_content_xml(standardConfig, tmpdirsec, backend):
"""Dump the content and check that we get the reference dump file back.
"""
refdump = backends[backend]['refdump']
fileext = backends[backend]['fileext']
dump = os.path.join(tmpdirsec.dir, "dump" + fileext)
fdump = os.path.join(tmpdirsec.dir, "dump-filter" + fileext)
reffdump = os.path.join(tmpdirsec.dir, "dump-filter-ref" + fileext)
filter_file(refdump, reffdump, *backends[backend]['filter'])
args = standardConfig.cmdargs + ["-f", backend, "-o", dump]
callscript("icatdump.py", args)
filter_file(dump, fdump, *backends[backend]['filter'])
assert filecmp.cmp(reffdump, fdump), "content of ICAT was not as expected"
def test_addtag_by_files(imgdir, monkeypatch):
"""Select by file names.
"""
monkeypatch.chdir(str(imgdir))
args = ["addtag", "Shinto_shrine", "dsc_4664.jpg", "dsc_4831.jpg"]
callscript("photoidx.py", args)
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["ls", "--tags", "Shinto_shrine"]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == ["dsc_4664.jpg", "dsc_4831.jpg"]
def test_ls_by_date_and_tag(imgdir):
"""Select by date and tags.
Multiple selection criteria, such as date and tags may be
combined.
"""
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
args = ["-d", imgdir, "ls", "--tags", "Tokyo", "--date", "2016-02-28"]
callscript("photoidx.py", args, stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == ["dsc_4623.jpg"]
def test_rmtag_all(imgdir, monkeypatch):
"""Remove a tag from all images.
"""
monkeypatch.chdir(str(imgdir))
args = ["-d", str(imgdir), "rmtag", "all"]
callscript("photoidx.py", args)
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["-d", str(imgdir), "ls", "--tags", "all"]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == []
def test_deselect_by_files(imgdir, monkeypatch):
"""Deselect by file names.
"""
monkeypatch.chdir(imgdir)
args = ["deselect", "dsc_4831.jpg"]
callscript("photoidx.py", args)
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
args = ["ls", "--selected"]
callscript("photoidx.py", args, stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == ["dsc_4664.jpg", "dsc_5126.jpg"]
def test_select_by_tag(imgdir, monkeypatch):
"""Select by tag.
"""
monkeypatch.chdir(imgdir)
args = ["select", "--tags", "Shinto_shrine"]
callscript("photoidx.py", args)
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
args = ["ls", "--selected"]
callscript("photoidx.py", args, stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == ["dsc_4664.jpg", "dsc_4831.jpg", "dsc_5126.jpg"]
def test_ls_md5(imgdir, monkeypatch):
"""List with --checksum=md5 option.
"""
md5sum = "/usr/bin/md5sum"
if not Path(md5sum).is_file():
pytest.skip("md5sum not found.")
monkeypatch.chdir(str(imgdir))
fname = imgdir / "md5"
with fname.open("wt") as f:
callscript("photoidx.py", ["ls", "--checksum=md5"], stdout=f)
with fname.open("rt") as f:
cmd = [md5sum, "-c"]
print(">", *cmd)
subprocess.check_call(cmd, stdin=f)
def test_ingest_duplicate_check_err(client, dataset, cmdargs):
"""Ingest with a collision of a duplicate object.
Same test as above, but use CHECK which fails due to mismatch.
"""
dataset.create()
ptype = client.assertedSearch("ParameterType [name='Reactor power']")[0]
p = client.new("datasetParameter", numericValue=5.0, dataset=dataset, type=ptype)
p.create()
args = cmdargs + ["-i", ds_params, "--duplicate", "CHECK"]
# FIXME: should inspect stderr and verify ICATObjectExistsError.
with pytest.raises(CalledProcessError) as err:
callscript("icatingest.py", args)
verify_dataset_params(client, dataset, {("Magnetic field", 5.3, "T"), ("Reactor power", 5.0, "MW")})
def test_addtag_by_gpspos(imgdir, monkeypatch):
"""Select by GPS position.
"""
monkeypatch.chdir(imgdir)
args = ["addtag", "--gpspos", "35.6883 N, 139.7544 E",
"--gpsradius", "20.0", "Tokyo"]
callscript("photoidx.py", args)
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
args = ["ls", "--tags", "Tokyo"]
callscript("photoidx.py", args, stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == ["dsc_4623.jpg", "dsc_4664.jpg"]
def test_ingest(ingestcase, standardCmdArgs):
"""Restore the ICAT content from a dumpfile.
"""
backend, filetype = ingestcase
refdump = backends[backend]['refdump']
if filetype == 'FILE':
args = standardCmdArgs + ["-f", backend, "-i", refdump]
callscript("icatingest.py", args)
elif filetype == 'STDINOUT':
args = standardCmdArgs + ["-f", backend]
with open(refdump, "rt") as infile:
callscript("icatingest.py", args, stdin=infile)
else:
raise RuntimeError("Invalid file type %s" % filetype)
def test_ls_by_neg_tags(imgdir, monkeypatch):
"""Select by negating tags.
Prepending a tag by an exclamation mark selects the images having
the tag not set.
"""
monkeypatch.chdir(imgdir)
fname = os.path.join(imgdir, "out")
with open(fname, "wt") as f:
args = ["ls", "--tags", "Tokyo,!Shinto_shrine"]
callscript("photoidx.py", args, stdout=f)
with open(fname, "rt") as f:
out = f.read().split()
assert out == ["dsc_4623.jpg"]
def test_ls_md5(imgdir, monkeypatch):
"""List with --checksum=md5 option.
"""
md5sum = "/usr/bin/md5sum"
if not os.path.isfile(md5sum):
pytest.skip("md5sum not found.")
monkeypatch.chdir(imgdir)
fname = os.path.join(imgdir, "md5")
with open(fname, "wt") as f:
callscript("photoidx.py", ["ls", "--checksum=md5"], stdout=f)
with open(fname, "rt") as f:
cmd = [md5sum, "-c"]
print(">", *cmd)
subprocess.check_call(cmd, stdin=f)
def test_ls_by_neg_tags(imgdir, monkeypatch):
"""Select by negating tags.
Prepending a tag by an exclamation mark selects the images having
the tag not set.
"""
monkeypatch.chdir(str(imgdir))
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["ls", "--tags", "Tokyo,!Shinto_shrine"]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == ["dsc_4623.jpg"]
def test_ingest_datafiles(tmpdirsec, client, dataset, cmdargs):
"""Ingest a dataset with some datafiles.
"""
dummyfiles = [ f['dfname'] for f in testdatafiles ]
args = cmdargs + ["-i", datafiles]
callscript("icatingest.py", args)
# Verify that the datafiles have been uploaded.
dataset = client.searchMatching(dataset)
for fname in dummyfiles:
query = Query(client, "Datafile", conditions={
"name": "= '%s'" % fname,
"dataset.id": "= %d" % dataset.id,
})
df = client.assertedSearch(query)[0]
assert df.location is None
def test_ingest_duplicate_overwrite(client, dataset, cmdargs):
"""Ingest with a collision of a duplicate object.
Same test as above, but now overwrite the old value.
"""
dataset.create()
ptype = client.assertedSearch("ParameterType [name='Reactor power']")[0]
p = client.new("datasetParameter", numericValue=5.0, dataset=dataset, type=ptype)
p.create()
args = cmdargs + ["-i", ds_params, "--duplicate", "OVERWRITE"]
callscript("icatingest.py", args)
verify_dataset_params(
client,
dataset,
{("Magnetic field", 5.3, "T"), ("Reactor power", 10.0, "MW"), ("Sample temperature", 293.15, "K")},
)
def test_ls_by_date_and_tag(imgdir):
"""Select by date and tags.
Multiple selection criteria, such as date and tags may be
combined.
"""
fname = imgdir / "out"
with fname.open("wt") as f:
args = [
"-d",
str(imgdir), "ls", "--tags", "Tokyo", "--date", "2016-02-28"
]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == ["dsc_4623.jpg"]
def test_addtag_by_gpspos(imgdir, monkeypatch):
"""Select by GPS position.
"""
monkeypatch.chdir(str(imgdir))
args = [
"addtag", "--gpspos", "35.6883 N, 139.7544 E", "--gpsradius", "20.0",
"Tokyo"
]
callscript("photoidx.py", args)
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["ls", "--tags", "Tokyo"]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
out = f.read().split()
assert out == ["dsc_4623.jpg", "dsc_4664.jpg"]
def test_ingest_duplicate_check_ok(client, dataset, cmdargs):
"""Ingest with a collision of a duplicate object.
Same test as above, but now it matches, so CHECK should return ok.
"""
dataset.create()
ptype = client.assertedSearch("ParameterType [name='Reactor power']")[0]
p = client.new("datasetParameter", numericValue=10.0,
dataset=dataset, type=ptype)
p.create()
args = cmdargs + ["-i", ds_params, "--duplicate", "CHECK"]
callscript("icatingest.py", args)
verify_dataset_params(client, dataset, {
("Magnetic field", 5.3, "T"),
("Reactor power", 10.0, "MW"),
("Sample temperature", 293.15, "K")
})
def test_ingest_duplicate_overwrite(client, dataset, cmdargs):
"""Ingest with a collision of a duplicate object.
Same test as above, but now overwrite the old value.
"""
dataset.create()
ptype = client.assertedSearch("ParameterType [name='Reactor power']")[0]
p = client.new("datasetParameter", numericValue=5.0,
dataset=dataset, type=ptype)
p.create()
args = cmdargs + ["-i", ds_params, "--duplicate", "OVERWRITE"]
callscript("icatingest.py", args)
verify_dataset_params(client, dataset, {
("Magnetic field", 5.3, "T"),
("Reactor power", 10.0, "MW"),
("Sample temperature", 293.15, "K")
})
def test_ingest_duplicate_check_err(client, dataset, cmdargs):
"""Ingest with a collision of a duplicate object.
Same test as above, but use CHECK which fails due to mismatch.
"""
dataset.create()
ptype = client.assertedSearch("ParameterType [name='Reactor power']")[0]
p = client.new("datasetParameter", numericValue=5.0,
dataset=dataset, type=ptype)
p.create()
args = cmdargs + ["-i", ds_params, "--duplicate", "CHECK"]
# FIXME: should inspect stderr and verify ICATObjectExistsError.
with pytest.raises(CalledProcessError) as err:
callscript("icatingest.py", args)
verify_dataset_params(client, dataset, {
("Magnetic field", 5.3, "T"),
("Reactor power", 5.0, "MW")
})
def test_ingest_duplicate_check_types(tmpdirsec, dataset, cmdargs, inputdata):
"""Ingest with a collision of a duplicate object.
Similar to test_ingest_duplicate_check_ok(), but trying several
input datasets that test different data types. Issue #9.
"""
# Most input data create a datafile or a dataset parameter related
# to dataset and thus assume the dataset to already exist. Only
# ingest_data_boolean creates the dataset itself.
if inputdata is not ingest_data_boolean:
dataset.create()
# We simply ingest twice the same data, using duplicate=CHECK the
# second time. This obviously leads to matching duplicates.
inpfile = os.path.join(tmpdirsec.dir, "ingest.xml")
with open(inpfile, "wt") as f:
f.write(inputdata)
args = cmdargs + ["-i", inpfile]
callscript("icatingest.py", args)
callscript("icatingest.py", args + ["--duplicate", "CHECK"])
def test_stats_filtered(imgdir):
"""Show stats on a selection.
"""
fname = imgdir / "out"
with fname.open("wt") as f:
args = ["-d", str(imgdir), "stats", "--tags", "Tokyo"]
callscript("photoidx.py", args, stdout=f)
with fname.open("rt") as f:
stats = yaml.safe_load(f)
assert stats["Count"] == 2
assert stats["Oldest"] == datetime.datetime(2016, 2, 28, 17, 26, 39)
assert stats["Newest"] == datetime.datetime(2016, 2, 29, 11, 37, 51)
assert stats["By date"] == {
datetime.date(2016, 2, 28): 1,
datetime.date(2016, 2, 29): 1,
}
assert stats["By tag"] == {
"Shinto_shrine": 1,
"Tokyo": 2,
}
