def test_tag_ref(imgdir): idxfname = str(imgdir / ".index.yaml") reffname = str(imgdir / "index-ref.yaml") shutil.copy(gettestdata("index-create.yaml"), idxfname) with photo.index.Index(idxfile=imgdir) as idx: taglist = [ "Japan", "Tokyo", "Hakone", "Kyoto", "Ginza", "Shinto_shrine", "Geisha", "Ryoan-ji" ] for t in taglist: idxfilter = photo.idxfilter.IdxFilter(files=tags[t]) for i in idxfilter.filter(idx): i.tags.add(t) idx.write() shutil.copy(idxfname, reffname)
def test_tag_ref(imgdir, argparser): idxfname = os.path.join(imgdir, ".index.yaml") reffname = os.path.join(imgdir, "index-ref.yaml") shutil.copy(gettestdata("index-create.yaml"), idxfname) idx = photo.index.Index(idxfile=imgdir) taglist = [ "Japan", "Tokyo", "Hakone", "Kyoto", "Ginza", "Shinto_shrine", "Geisha", "Ryoan-ji" ] for t in taglist: args = argparser.parse_args(tags[t]) idxfilter = photo.idxfilter.IdxFilter(args) for i in idxfilter.filter(idx): i.tags.add(t) idx.write() shutil.copy(idxfname, reffname)
def test_tag_shuffle(imgdir): """Same as test_tag_ref(), only the order of setting the tags differ. """ idxfname = str(imgdir / ".index.yaml") reffname = str(imgdir / "index-ref.yaml") shutil.copy(gettestdata("index-create.yaml"), idxfname) with photo.index.Index(idxfile=imgdir) as idx: taglist = [ "Ginza", "Hakone", "Japan", "Geisha", "Shinto_shrine", "Tokyo", "Kyoto", "Ryoan-ji" ] for t in taglist: idxfilter = photo.idxfilter.IdxFilter(files=tags[t]) for i in idxfilter.filter(idx): i.tags.add(t) idx.write() assert filecmp.cmp(idxfname, reffname), "index file differs from reference"
def test_tag_shuffle(imgdir, argparser): """Same as test_tag_ref(), only the order of setting the tags differ. """ idxfname = os.path.join(imgdir, ".index.yaml") reffname = os.path.join(imgdir, "index-ref.yaml") shutil.copy(gettestdata("index-create.yaml"), idxfname) idx = photo.index.Index(idxfile=imgdir) taglist = [ "Ginza", "Hakone", "Japan", "Geisha", "Shinto_shrine", "Tokyo", "Kyoto", "Ryoan-ji" ] for t in taglist: args = argparser.parse_args(tags[t]) idxfilter = photo.idxfilter.IdxFilter(args) for i in idxfilter.filter(idx): i.tags.add(t) idx.write() assert filecmp.cmp(idxfname, reffname), "index file differs from reference"
def test_tag_remove(imgdir): """First set all tags on all images, then remove the wrong ones. """ idxfname = str(imgdir / ".index.yaml") reffname = str(imgdir / "index-ref.yaml") shutil.copy(gettestdata("index-create.yaml"), idxfname) with photo.index.Index(idxfile=imgdir) as idx: taglist = [ "Tokyo", "Shinto_shrine", "Ginza", "Geisha", "Japan", "Ryoan-ji", "Hakone", "Kyoto" ] for t in taglist: for i in idx: i.tags.add(t) for t in taglist: for i in idx: if str(i.filename) not in tags[t]: i.tags.remove(t) idx.write() assert filecmp.cmp(idxfname, reffname), "index file differs from reference"
def test_tag_remove(imgdir): """First set all tags on all images, then remove the wrong ones. """ idxfname = os.path.join(imgdir, ".index.yaml") reffname = os.path.join(imgdir, "index-ref.yaml") shutil.copy(gettestdata("index-create.yaml"), idxfname) idx = photo.index.Index(idxfile=imgdir) taglist = [ "Tokyo", "Shinto_shrine", "Ginza", "Geisha", "Japan", "Ryoan-ji", "Hakone", "Kyoto" ] for t in taglist: for i in idx: i.tags.add(t) for t in taglist: for i in idx: if i.filename not in tags[t]: i.tags.remove(t) idx.write() assert filecmp.cmp(idxfname, reffname), "index file differs from reference"
def test_tag_extra(imgdir): """Add a spurious extra tag first and remove it later. """ idxfname = str(imgdir / ".index.yaml") reffname = str(imgdir / "index-ref.yaml") shutil.copy(gettestdata("index-create.yaml"), idxfname) with photo.index.Index(idxfile=imgdir) as idx: taglist = [ "Japan", "Tokyo", "Hakone", "Kyoto", "Ginza", "Shinto_shrine", "Geisha", "Ryoan-ji" ] for i in idx: i.tags.add("extra") for t in taglist: idxfilter = photo.idxfilter.IdxFilter(files=tags[t]) for i in idxfilter.filter(idx): i.tags.add(t) for i in idx: i.tags.remove("extra") idx.write() assert filecmp.cmp(idxfname, reffname), "index file differs from reference"
def test_tag_extra(imgdir, argparser): """Add a spurious extra tag first and remove it later. """ idxfname = os.path.join(imgdir, ".index.yaml") reffname = os.path.join(imgdir, "index-ref.yaml") shutil.copy(gettestdata("index-create.yaml"), idxfname) idx = photo.index.Index(idxfile=imgdir) taglist = [ "Japan", "Tokyo", "Hakone", "Kyoto", "Ginza", "Shinto_shrine", "Geisha", "Ryoan-ji" ] for i in idx: i.tags.add("extra") for t in taglist: args = argparser.parse_args(tags[t]) idxfilter = photo.idxfilter.IdxFilter(args) for i in idxfilter.filter(idx): i.tags.add(t) for i in idx: i.tags.remove("extra") idx.write() assert filecmp.cmp(idxfname, reffname), "index file differs from reference"
""" import os.path import argparse import shutil import filecmp import pytest import photo.index import photo.idxfilter from conftest import tmpdir, gettestdata testimgs = [ "dsc_4623.jpg", "dsc_4664.jpg", "dsc_4831.jpg", "dsc_5126.jpg", "dsc_5167.jpg" ] testimgfiles = [ gettestdata(i) for i in testimgs ] @pytest.fixture(scope="module") def imgdir(tmpdir): for fname in testimgfiles: shutil.copy(fname, tmpdir) return tmpdir @pytest.fixture(scope="module") def argparser(): parser = argparse.ArgumentParser() photo.idxfilter.addFilterArguments(parser) return parser # Each test ends up adding the same set of tags to each image. But # the order in which the tags are added differs between tests and some
"""Add images to an index. """ import filecmp import shutil import pytest import photo.index from conftest import tmpdir, gettestdata testimgs = [ "dsc_4623.jpg", "dsc_4664.jpg", "dsc_4831.jpg", "dsc_5126.jpg", "dsc_5167.jpg" ] testimgfiles = [gettestdata(i) for i in testimgs] refindex = gettestdata("index-create.yaml") def test_createupdate(tmpdir): for fname in testimgfiles[:3]: shutil.copy(fname, str(tmpdir)) with photo.index.Index(imgdir=tmpdir) as idx: idx.write() for fname in testimgfiles[3:]: shutil.copy(fname, str(tmpdir)) with photo.index.Index(idxfile=tmpdir, imgdir=tmpdir) as idx: idx.write() idxfile = str(tmpdir / ".index.yaml") assert filecmp.cmp(refindex, idxfile), "index file differs from reference"
The prefix 'pidx:' for tags is reserved for internal use in photo-tools. It should be removed when reading an index file. """ import os.path import shutil import filecmp import pytest import photo.index from conftest import tmpdir, gettestdata testimgs = [ "dsc_4623.jpg", "dsc_4664.jpg", "dsc_4831.jpg", "dsc_5126.jpg", "dsc_5167.jpg" ] testimgfiles = [ gettestdata(i) for i in testimgs ] invindex = gettestdata("index-reserved-tags.yaml") refindex = gettestdata("index-tagged.yaml") def test_reserved_tags_convert(tmpdir): for fname in testimgfiles: shutil.copy(fname, tmpdir) idxfile = os.path.join(tmpdir, ".index.yaml") shutil.copy(invindex, idxfile) # reading and writing the index transparantly filters out tags # using the reserved prefix. idx = photo.index.Index(idxfile=tmpdir) idx.write() assert filecmp.cmp(idxfile, refindex), "index file differs from reference"
"""Check checksums. """ from pathlib import Path import shutil import subprocess import pytest import photo.index from conftest import tmpdir, gettestdata testimgs = [ "dsc_4623.jpg", "dsc_4664.jpg", "dsc_4831.jpg", "dsc_5126.jpg", "dsc_5167.jpg" ] testimgfiles = [gettestdata(i) for i in testimgs] hashalg = { "md5": "/usr/bin/md5sum", "sha1": "/usr/bin/sha1sum", "sha224": "/usr/bin/sha224sum", "sha256": "/usr/bin/sha256sum", "sha384": "/usr/bin/sha384sum", "sha512": "/usr/bin/sha512sum", } @pytest.mark.dependency() def test_create_checksum(tmpdir): for fname in testimgfiles: shutil.copy(fname, str(tmpdir)) with photo.index.Index(imgdir=tmpdir, hashalg=hashalg.keys()) as idx:
import icat.config from icat.query import Query from icat.dumpfile import open_dumpfile import icat.dumpfile_xml import icat.dumpfile_yaml from icat.dump_queries import * from conftest import getConfig, require_icat_version from conftest import gettestdata, callscript from conftest import filter_file, yaml_filter, xml_filter require_icat_version("4.4.0", "need InvestigationGroup") backends = { 'XML': { 'refdump': gettestdata("icatdump.xml"), 'fileext': '.xml', 'filter': xml_filter, }, 'YAML': { 'refdump': gettestdata("icatdump.yaml"), 'fileext': '.yaml', 'filter': yaml_filter, }, } assert backends.keys() == icat.dumpfile.Backends.keys() # The following cases are tuples of a backend and a file type (regular # file, stdin/stdout, in-memory stream). They are used for both, # input and output. We test all combinations, e.g. reading from a XML # file and writing it back as YAML to stdout.
"""Read and write an index having optional name attributes in it. """ import filecmp import shutil import pytest import photo.index from conftest import tmpdir, gettestdata refindex = gettestdata("index-name.yaml") @pytest.fixture(scope="module") def imgdir(tmpdir): shutil.copy(refindex, str(tmpdir / ".index.yaml")) return tmpdir def test_readwrite(imgdir): """Read the index file and write it out again. """ with photo.index.Index(idxfile=imgdir) as idx: assert idx[0].name == "ginza.jpg" assert idx[1].name is None assert idx[3].name == "geisha.jpg" idx.write() idxfile = str(imgdir / ".index.yaml") assert filecmp.cmp(refindex, idxfile), "index file differs from reference"
#12). As a consequence, the index file format is changed. photo-tools still reads legacy files and transparently converts them to the new format. This feature is tested in this module. """ import os.path import shutil import filecmp import pytest import photo.index from conftest import tmpdir, gettestdata testimgs = [ "dsc_4623.jpg", "dsc_4664.jpg", "dsc_4831.jpg", "dsc_5126.jpg", "dsc_5167.jpg" ] testimgfiles = [ gettestdata(i) for i in testimgs ] legacyindex = gettestdata("index-legacy.yaml") refindex = gettestdata("index-create.yaml") def test_legacyconvert(tmpdir): for fname in testimgfiles: shutil.copy(fname, tmpdir) idxfile = os.path.join(tmpdir, ".index.yaml") shutil.copy(legacyindex, idxfile) # reading and writing the index transparantly converts it. idx = photo.index.Index(idxfile=tmpdir) idx.write() assert filecmp.cmp(idxfile, refindex), "index file differs from reference"
"""Call the command line script photoidx.py. """ from __future__ import print_function import os.path import shutil import filecmp import subprocess import pytest from conftest import tmpdir, gettestdata, callscript testimgs = [ "dsc_4623.jpg", "dsc_4664.jpg", "dsc_4831.jpg", "dsc_5126.jpg", "dsc_5167.jpg" ] testimgfiles = [ gettestdata(i) for i in testimgs ] refindex = gettestdata("index-create.yaml") @pytest.fixture(scope="module") def imgdir(tmpdir): for fname in testimgfiles: shutil.copy(fname, tmpdir) return tmpdir # Note: the default value for the "-d" option to photoidx is the # current working directory. So changing to imgdir before calling # photoidx without the "-d" option should be equivalent to calling # "photoidx -d imgdir". We more or less try both variants at random # in the tests.
"""Test icatdump and icatingest. """ from __future__ import print_function import os.path import pytest from subprocess import CalledProcessError import icat import icat.config from icat.query import Query from conftest import DummyDatafile, gettestdata, getConfig, callscript # Test input ds_params = gettestdata("ingest-ds-params.xml") datafiles = gettestdata("ingest-datafiles.xml") @pytest.fixture(scope="module") def conf(setupicat): return getConfig(confSection="acord", ids="mandatory") @pytest.fixture(scope="module") def client(conf): client = icat.Client(conf.url, **conf.client_kwargs) client.login(conf.auth, conf.credentials) return client @pytest.fixture(scope="module") def cmdargs(conf): return conf.cmdargs + ["-f", "XML"]
def imgdir(tmpdir): for fname in testimgfiles: shutil.copy(fname, tmpdir) shutil.copy(gettestdata("index-tagged.yaml"), os.path.join(tmpdir, ".index.yaml")) return tmpdir
def imgdir(tmpdir): for fname in testimgfiles: shutil.copy(fname, str(tmpdir)) shutil.copy(gettestdata("index-tagged.yaml"), str(tmpdir / ".index.yaml")) return tmpdir
#12). As a consequence, the index file format is changed. photo-tools still reads legacy files and transparently converts them to the new format. This feature is tested in this module. """ import filecmp import shutil import pytest import photo.index from conftest import tmpdir, gettestdata testimgs = [ "dsc_4623.jpg", "dsc_4664.jpg", "dsc_4831.jpg", "dsc_5126.jpg", "dsc_5167.jpg" ] testimgfiles = [gettestdata(i) for i in testimgs] legacyindex = gettestdata("index-legacy.yaml") refindex = gettestdata("index-create.yaml") def test_legacyconvert(tmpdir): for fname in testimgfiles: shutil.copy(fname, str(tmpdir)) idxfile = str(tmpdir / ".index.yaml") shutil.copy(legacyindex, idxfile) # reading and writing the index transparantly converts it. with photo.index.Index(idxfile=tmpdir) as idx: idx.write() assert filecmp.cmp(idxfile, refindex), "index file differs from reference"
import re import filecmp import pytest import icat import icat.config from conftest import getConfig, require_icat_version from conftest import gettestdata, callscript from conftest import filter_file, yaml_filter, xml_filter # test content has InvestigationGroup objects. require_icat_version("4.4.0") backends = { 'XML': { 'refdump': gettestdata("icatdump.xml"), 'fileext': '.xml', 'filter': xml_filter, }, 'YAML': { 'refdump': gettestdata("icatdump.yaml"), 'fileext': '.yaml', 'filter': yaml_filter, }, } users = [ "acord", "ahau", "jbotu", "jdoe", "nbour", "rbeck" ] refsummary = { "root": gettestdata("summary") } for u in users: refsummary[u] = gettestdata("summary.%s" % u) # Read permission on DataCollection, DataCollectionDatafile,
When tags are set as unicode while the content is pure ASCII, PyYAML marks them as `!!python/unicode 'tag'`. """ import filecmp import shutil import pytest import photo.index from conftest import tmpdir, gettestdata testimgs = [ "dsc_4623.jpg", "dsc_4664.jpg", "dsc_4831.jpg", "dsc_5126.jpg", "dsc_5167.jpg" ] testimgfiles = [ gettestdata(i) for i in testimgs ] baseindex = gettestdata("index-create.yaml") refindex = gettestdata("index-unicode-tags.yaml") tags = { "dsc_4623.jpg": [ "T\u014Dky\u014D", "Ginza" ], "dsc_4664.jpg": [ "T\u014Dky\u014D", "Meiji-jing\u016B", "Shint\u014D" ], "dsc_4831.jpg": [ "Hakone", "Shint\u014D" ], "dsc_5126.jpg": [ "Ky\u014Dto", "Gion" ], "dsc_5167.jpg": [ "Ky\u014Dto", "Ry\u014Dan-ji", "Buddha" ], } @pytest.fixture(scope="module") def imgdir(tmpdir): for fname in testimgfiles:
We will need multiple processes to test this. """ import filecmp from multiprocessing import Process, Queue import shutil import pytest import photo.index import photo.idxfilter from conftest import tmpdir, gettestdata testimgs = [ "dsc_4623.jpg", "dsc_4664.jpg", "dsc_4831.jpg", "dsc_5126.jpg", "dsc_5167.jpg" ] testimgfiles = [gettestdata(i) for i in testimgs] refindex = gettestdata("index-tagged.yaml") # tags to test and expected results. tags = { "Shinto_shrine": ["dsc_4664.jpg", "dsc_4831.jpg"], "Tokyo": ["dsc_4623.jpg", "dsc_4664.jpg"], } @pytest.fixture(scope="module") def imgdir(tmpdir): for fname in testimgfiles: shutil.copy(fname, str(tmpdir)) shutil.copy(refindex, str(tmpdir / ".index.yaml")) return tmpdir
""" import os.path import filecmp import yaml import pytest import icat import icat.config from icat.query import Query from conftest import getConfig, require_icat_version from conftest import gettestdata, callscript, filter_file, yaml_filter # wipeicat uses JPQL search syntax. require_icat_version("4.3.0") testinput = gettestdata("example_data.yaml") refdump = gettestdata("icatdump.yaml") users = ["acord", "ahau", "jbotu", "jdoe", "nbour", "rbeck"] refsummary = {"root": gettestdata("summary")} for u in users: refsummary[u] = gettestdata("summary.%s" % u) # Labels used in test dependencies. alldata = [ "init", "sample_durol", "sample_nimnga", "sample_nio", "inv_081", "inv_101", "inv_121", "invdata_081", "invdata_101", "invdata_121", "job1", "rdf1", "pub1" ] @pytest.fixture(scope="module") def data():
The prefix 'pidx:' for tags is reserved for internal use in photo-tools. It should be removed when reading an index file. """ import filecmp import shutil import pytest import photo.index from conftest import tmpdir, gettestdata testimgs = [ "dsc_4623.jpg", "dsc_4664.jpg", "dsc_4831.jpg", "dsc_5126.jpg", "dsc_5167.jpg" ] testimgfiles = [gettestdata(i) for i in testimgs] invindex = gettestdata("index-reserved-tags.yaml") refindex = gettestdata("index-tagged.yaml") def test_reserved_tags_convert(tmpdir): for fname in testimgfiles: shutil.copy(fname, str(tmpdir)) idxfile = str(tmpdir / ".index.yaml") shutil.copy(invindex, idxfile) # reading and writing the index transparantly filters out tags # using the reserved prefix. with photo.index.Index(idxfile=tmpdir) as idx: idx.write() assert filecmp.cmp(idxfile, refindex), "index file differs from reference"
"""Test different options to filter images by date. This is essentially the same as test_01_filter_date, but using IdxFilter.from_args() rather then the constructor directly. So it mostly test the date parser in idxfilter. """ import argparse import pytest import photo.index import photo.idxfilter from conftest import gettestdata testimgs = ["dsc_%04d.jpg" % i for i in range(1, 13)] indexfile = gettestdata("index-date.yaml") @pytest.fixture(scope="module") def argparser(): parser = argparse.ArgumentParser() photo.idxfilter.addFilterArguments(parser) return parser def test_single_date(argparser): """Select by single date. """ with photo.index.Index(idxfile=indexfile) as idx: args = argparser.parse_args(["--date=2016-02-29"]) idxfilter = photo.idxfilter.IdxFilter.from_args(args) fnames = [str(i.filename) for i in idxfilter.filter(idx)]
def legacy_1_0_archive(): return gettestdata("mailarchive-legacy-1_0.tar.xz")
"""Test different options to filter images by date. """ import argparse import pytest import photo.index import photo.idxfilter from conftest import gettestdata testimgs = [ "dsc_%04d.jpg" % i for i in range(1,13) ] indexfile = gettestdata("index-date.yaml") @pytest.fixture(scope="module") def argparser(): parser = argparse.ArgumentParser() photo.idxfilter.addFilterArguments(parser) return parser def test_single_date(argparser): """Select by single date. """ idx = photo.index.Index(idxfile=indexfile) args = argparser.parse_args(["--date=2016-02-29"]) idxfilter = photo.idxfilter.IdxFilter(args) fnames = [ i.filename for i in idxfilter.filter(idx) ] assert fnames == testimgs[1:4] def test_interval_date_date(argparser): """Select by an interval between two dates.
""" import os.path import filecmp import yaml import pytest import icat import icat.config from icat.query import Query from conftest import getConfig, require_icat_version from conftest import gettestdata, callscript, filter_file, yaml_filter require_icat_version("4.3.0", "need JPQL query syntax") testinput = gettestdata("example_data.yaml") refdump = gettestdata("icatdump.yaml") users = [ "acord", "ahau", "jbotu", "jdoe", "nbour", "rbeck" ] refsummary = { "root": gettestdata("summary") } for u in users: refsummary[u] = gettestdata("summary.%s" % u) # Labels used in test dependencies. alldata = ["init", "sample_durol", "sample_nimnga", "sample_nio", "inv_081", "inv_101", "inv_121", "invdata_081", "invdata_101", "invdata_121", "job1", "rdf1", "study1", "pub1"] @pytest.fixture(scope="module") def data(): with open(testinput, 'r') as f: return yaml.load(f)
"""Test creating a mail archive and check its content. """ import datetime import email import pytest from pytest_dependency import depends import yaml from archive import Archive from archive.mailarchive import MailIndex, MailArchive from conftest import gettestdata testdata = gettestdata("mails.tar.gz") testmails = [] def getmsgs(): """Yield a couple of test mails along with appropriate folder names. """ mails = Archive().open(testdata) idx = yaml.safe_load(mails.get_metadata(".index.yaml").fileobj) for folder in sorted(idx.keys()): for msg_path in idx[folder]: msgbytes = mails._file.extractfile(msg_path).read() msg = email.message_from_bytes(msgbytes) testmails.append((folder, msg)) yield (folder, msgbytes) @pytest.fixture(scope="module", params=["abs", "rel"]) def testcase(request):
"""Test icatdump and icatingest. """ from __future__ import print_function import os.path import pytest from subprocess import CalledProcessError import icat import icat.config from icat.query import Query from conftest import DummyDatafile, gettestdata, getConfig, callscript # Test input ds_params = gettestdata("ingest-ds-params.xml") datafiles = gettestdata("ingest-datafiles.xml") @pytest.fixture(scope="module") def client(setupicat): client, conf = getConfig(confSection="acord", ids="mandatory") client.login(conf.auth, conf.credentials) return client @pytest.fixture(scope="module") def cmdargs(setupicat): _, conf = getConfig(confSection="acord", ids="mandatory") return conf.cmdargs + ["-f", "XML"] @pytest.fixture(scope="function") def dataset(client): """A dataset to be used in the test.
"""Read an image index from the index file and write it back. """ import filecmp import shutil import pytest import photo.index from conftest import tmpdir, gettestdata testimgs = [ "dsc_4623.jpg", "dsc_4664.jpg", "dsc_4831.jpg", "dsc_5126.jpg", "dsc_5167.jpg" ] testimgfiles = [gettestdata(i) for i in testimgs] refindex = gettestdata("index-tagged.yaml") refindexu = gettestdata("index-unicode-tags.yaml") @pytest.fixture(scope="module") def imgdir(tmpdir): for fname in testimgfiles: shutil.copy(fname, str(tmpdir)) return tmpdir def test_read_non_existent(imgdir): """Try to read an index file that does not exist. """ with pytest.raises(OSError): with photo.index.Index(idxfile=imgdir) as idx: