def test_ifneeded_parse(self):
@ifneeded("parse")
def testfunc(repo, basefile):
repo.called = True
# mockdoc = Mock()
# mockdoc.basefile="1234"
mockbasefile = "1234"
mockrepo = Mock()
mockrepo.store.needed = DocumentStore(datadir='fake').needed
mockrepo.called = False
mockrepo.config.force = False
# test 1: Outfile is newer - the ifneeded decorator should
# make sure the actual testfunc code is never reached
with patch('ferenda.util.outfile_is_newer', return_value=True):
testfunc(mockrepo, mockbasefile)
self.assertFalse(mockrepo.called)
mockrepo.called = False
# test 2: Outfile is older
with patch('ferenda.util.outfile_is_newer', return_value=False):
testfunc(mockrepo, mockbasefile)
self.assertTrue(mockrepo.called)
mockrepo.called = False
# test 3: Outfile is newer, but the global force option was set
mockrepo.config.force = True
with patch('ferenda.util.outfile_is_newer', return_value=True):
testfunc(mockrepo, mockbasefile)
self.assertTrue(mockrepo.called)
mockrepo.config.force = None
mockrepo.called = False
class Compression(unittest.TestCase):
compression = None
expected_suffix = ""
expected_mimetype = "text/plain"
dummytext = """For applications that require data compression, the functions in this module allow compression and decompression, using the zlib library. The zlib library has its own home page at http://www.zlib.net. There are known incompatibilities between the Python module and versions of the zlib library earlier than 1.1.3; 1.1.3 has a security vulnerability, so we recommend using 1.1.4 or later.
zlib’s functions have many options and often need to be used in a particular order. This documentation doesn’t attempt to cover all of the permutations; consult the zlib manual at http://www.zlib.net/manual.html for authoritative information.
For reading and writing .gz files see the gzip module.
"""
def p(self, path):
path = self.datadir + "/" + path
return path.replace('/', '\\') if os.sep == '\\' else path
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.store = DocumentStore(self.datadir, compression=self.compression)
def test_intermediate_path(self):
self.assertEqual(
self.p("intermediate/123/a.xml" + self.expected_suffix),
self.store.intermediate_path('123/a'))
def test_intermediate_path_selectsuffix(self):
self.store.intermediate_suffixes = [".html", ".xhtml"]
util.writefile(self.p("intermediate/123/a.html"), self.dummytext)
self.assertEqual(
self.p("intermediate/123/a.html") + self.expected_suffix,
self.store.intermediate_path('123/a'))
def test_open_intermediate_path(self):
self.store.intermediate_suffixes = [".html", ".xhtml"]
with self.store.open_intermediate("123/a", mode="w",
suffix=".xhtml") as fp:
fp.write(self.dummytext)
filename = self.p("intermediate/123/a.xhtml" + self.expected_suffix)
self.assertTrue(os.path.exists(filename))
mimetype = util.runcmd("file -b --mime-type %s" % filename)[1]
self.assertIn(mimetype.strip(), self.expected_mimetype)
with self.store.open_intermediate("123/a") as fp:
# note, open_intermediate should open the file with the
# the .xhtml suffix automatically
self.assertEqual(self.dummytext, fp.read())
def test_ifneeded_relate(self):
@ifneeded("relate")
def testfunc(repo, basefile, needed):
repo.called = True
repo.needed = needed
try:
datadir = tempfile.mkdtemp()
mockbasefile = "1234"
mockrepo = Mock()
mockrepo.store = DocumentStore(datadir=datadir)
mockrepo.called = False
mockrepo.config.force = False
# create some docentry file in a good place
de = DocumentEntry(mockrepo.store.documententry_path("1234"))
now = datetime.datetime.now()
de.indexed_ts = now + datetime.timedelta(seconds=3600)
de.indexed_ft = now + datetime.timedelta(seconds=-3600)
de.indexed_dep = now + datetime.timedelta(seconds=-3600)
de.save()
# test 1: Outfile is newer - the ifneeded decorator should
# make sure the actual testfunc code is never reached
# NOTE: the "relate" branch of DocumentStore.needed
# doesn't use outfile_is_newer, so we can't patch that, we
# have to create actual files
parsedpath = mockrepo.store.parsed_path("1234")
util.writefile(parsedpath, "dummy")
os.utime(parsedpath, (now.timestamp(), now.timestamp() - 7200))
testfunc(mockrepo, mockbasefile)
self.assertFalse(mockrepo.called)
mockrepo.called = False
# test 2: Outfile is older than the information in the documententry file
os.utime(parsedpath, (now.timestamp(), now.timestamp()))
testfunc(mockrepo, mockbasefile)
self.assertTrue(mockrepo.called)
self.assertTrue(mockrepo.needed)
self.assertFalse(mockrepo.needed.triples)
self.assertFalse(mockrepo.needed.dependencies)
self.assertTrue(mockrepo.needed.fulltext)
mockrepo.called = False
# test 3: Outfile is newer, but the global force option was set
os.utime(parsedpath, (now.timestamp(), now.timestamp() - 7200))
mockrepo.config.force = True
testfunc(mockrepo, mockbasefile)
self.assertTrue(mockrepo.called)
mockrepo.config.force = None
mockrepo.called = False
finally:
if os.path.exists(datadir):
shutil.rmtree(datadir)
class Compression(unittest.TestCase):
compression = None
expected_suffix = ""
expected_mimetype = "text/plain"
dummytext = """For applications that require data compression, the functions in this module allow compression and decompression, using the zlib library. The zlib library has its own home page at http://www.zlib.net. There are known incompatibilities between the Python module and versions of the zlib library earlier than 1.1.3; 1.1.3 has a security vulnerability, so we recommend using 1.1.4 or later.
zlib’s functions have many options and often need to be used in a particular order. This documentation doesn’t attempt to cover all of the permutations; consult the zlib manual at http://www.zlib.net/manual.html for authoritative information.
For reading and writing .gz files see the gzip module.
"""
def p(self,path):
path = self.datadir+"/"+path
return path.replace('/', '\\') if os.sep == '\\' else path
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.store = DocumentStore(self.datadir, compression=self.compression)
def test_intermediate_path(self):
self.assertEqual(self.p("intermediate/123/a.xml" + self.expected_suffix),
self.store.intermediate_path('123/a'))
def test_intermediate_path_selectsuffix(self):
self.store.intermediate_suffixes = [".html", ".xhtml"]
util.writefile(self.p("intermediate/123/a.html"), self.dummytext)
self.assertEqual(self.p("intermediate/123/a.html") + self.expected_suffix,
self.store.intermediate_path('123/a'))
def test_open_intermediate_path(self):
self.store.intermediate_suffixes = [".html", ".xhtml"]
with self.store.open_intermediate("123/a", mode="w", suffix=".xhtml") as fp:
fp.write(self.dummytext)
filename = self.p("intermediate/123/a.xhtml" + self.expected_suffix)
self.assertTrue(os.path.exists(filename))
mimetype = util.runcmd("file -b --mime-type %s" % filename)[1]
self.assertIn(mimetype.strip(), self.expected_mimetype)
with self.store.open_intermediate("123/a") as fp:
# note, open_intermediate should open the file with the
# the .xhtml suffix automatically
self.assertEqual(self.dummytext, fp.read())
def test_ifneeded_generate(self):
@ifneeded("generate")
def testfunc(repo, basefile):
repo.called = True
mockbasefile = "1234"
mockrepo = Mock()
mockrepo.store.needed = DocumentStore(datadir='fake').needed
mockrepo.called = False
mockrepo.config.force = False
# test 1: Outfile is newer - the ifneeded decorator should
# make sure the actual testfunc code is never reached
with patch('ferenda.util.outfile_is_newer', return_value=True):
testfunc(mockrepo, mockbasefile)
self.assertFalse(mockrepo.called)
mockrepo.called = False
# test 2: Outfile is older than source file
with patch('ferenda.util.outfile_is_newer', return_value=False):
testfunc(mockrepo, mockbasefile)
self.assertTrue(mockrepo.called)
mockrepo.called = False
# FIXME: we could add more tests, eg create a dependency file
# and make sure an arbitrary file named in that depfile is
# newer then outfile. but the tests in testDocStore.Needed
# should cover that pretty well.
# test 3: Outfile is newer, but the global force option was set
mockrepo.config.force = True
with patch('ferenda.util.outfile_is_newer', return_value=True):
testfunc(mockrepo, mockbasefile)
self.assertTrue(mockrepo.called)
mockrepo.config.force = None
mockrepo.called = False
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.basefile = "1"
self.store = DocumentStore(self.datadir + "/base")
self.d = Devel()
self.globalconf = LayeredConfig(Defaults({
'datadir': self.datadir,
'patchdir': self.datadir,
'download_text': None,
'intermediate_text': None,
'devel': {
'class': 'ferenda.Devel'
},
'base': {
'class': 'testDevel.MockRepo'
},
'koi8': {
'class': 'testDevel.Koi8Repo'
}
}),
cascade=True)
self.d.config = self.globalconf.devel
self.d.config.download_text
self.d.config.download_text = "what"
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.store = DocumentStore(self.datadir)
class Needed(unittest.TestCase):
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.store = DocumentStore(self.datadir)
def tearDown(self):
shutil.rmtree(self.datadir)
def create_file(self, path, timestampoffset=0, content="dummy"):
util.ensure_dir(path)
with open(path, "w") as fp:
fp.write(content)
if timestampoffset:
os.utime(path, (time.time(), time.time() + timestampoffset))
def test_parse_not_needed(self):
self.create_file(self.store.downloaded_path("a"))
self.create_file(self.store.parsed_path("a"), 3600)
self.assertFalse(self.store.needed("a", "parse"))
def test_parse_needed(self):
self.create_file(self.store.downloaded_path("a"))
res = self.store.needed("a", "parse")
self.assertTrue(res)
self.assertIn("outfile doesn't exist", res.reason)
def test_parse_needed_outdated(self):
self.create_file(self.store.downloaded_path("a"))
self.create_file(self.store.parsed_path("a"), -3600)
res = self.store.needed("a", "parse")
self.assertTrue(res)
self.assertIn("is newer than outfile", res.reason)
def create_entry(self, basefile, timestampoffset=0):
# create a entry file with indexed_{ft,ts,dep} set to the
# current time with optional offset. Also
# .status['generated']['date'], to test needed(...,
# 'transformlinks')
de = DocumentEntry(self.store.documententry_path(basefile))
delta = timedelta(seconds=timestampoffset)
ts = datetime.now() + delta
de.indexed_ts = ts
de.indexed_ft = ts
de.indexed_dep = ts
de.updated = ts
de.status['generate'] = {'date': ts}
de.save()
def test_relate_not_needed(self):
self.create_entry("a")
self.create_file(self.store.distilled_path("a"), -3600)
self.create_file(self.store.parsed_path("a"), -3600)
self.create_file(self.store.dependencies_path("a"), -3600)
self.assertFalse(self.store.needed("a", "relate"))
def test_relate_needed_ts(self):
self.create_entry("a", -3600)
self.create_file(self.store.distilled_path("a"))
self.create_file(self.store.parsed_path("a"), -7200)
self.create_file(self.store.dependencies_path("a"), -7200)
res = self.store.needed("a", "relate")
self.assertTrue(res)
self.assertIn("is newer than indexed_ts in documententry", res.reason)
def test_relate_needed_ft(self):
self.create_entry("a", -3600)
self.create_file(self.store.distilled_path("a"), -7200)
self.create_file(self.store.parsed_path("a"))
self.create_file(self.store.dependencies_path("a"), -7200)
res = self.store.needed("a", "relate")
self.assertTrue(res)
self.assertIn("is newer than indexed_ft in documententry", res.reason)
def test_relate_needed_dep(self):
self.create_entry("a", -3600)
self.create_file(self.store.distilled_path("a"), -7200)
self.create_file(self.store.parsed_path("a"), -7200)
self.create_file(self.store.dependencies_path("a"))
res = self.store.needed("a", "relate")
self.assertTrue(res)
self.assertIn("is newer than indexed_dep in documententry", res.reason)
def test_generate_not_needed(self):
self.create_file(self.store.parsed_path("a"))
self.create_file(self.store.generated_path("a"), 3600)
self.assertFalse(self.store.needed("a", "generate"))
def test_generate_not_needed_404(self):
self.create_file(self.store.parsed_path("a"))
self.create_file(self.store.generated_path("a") + ".404", 3600)
self.assertFalse(self.store.needed("a", "generate"))
def test_generate_needed(self):
self.create_file(self.store.parsed_path("a"))
res = self.store.needed("a", "generate")
self.assertTrue(res)
self.assertIn("outfile doesn't exist", res.reason)
def test_generate_needed_outdated(self):
self.create_file(self.store.parsed_path("a"))
self.create_file(self.store.generated_path("a"), -3600)
res = self.store.needed("a", "generate")
self.assertTrue(res)
self.assertIn("is newer than outfile", res.reason)
def test_generate_needed_outdated_dep(self):
self.create_file(self.store.parsed_path("a"), -3600)
self.create_file(self.store.generated_path("a"), -1800)
dependency_path = self.store.datadir + os.sep + "blahonga.txt"
self.create_file(self.store.dependencies_path("a"), -3600, dependency_path)
# create a arbitrary dependency file which is newer than outfile
self.create_file(dependency_path)
res = self.store.needed("a", "generate")
self.assertTrue(res)
self.assertIn("is newer than outfile", res.reason)
def test_transformlinks_needed(self):
self.create_file(self.store.generated_path("a"))
self.create_entry("a")
res = self.store.needed("a", "transformlinks")
self.assertTrue(res)
self.assertIn("has not been modified after generate", res.reason)
def test_transformlinks_not_needed(self):
self.create_entry("a", -3600)
self.create_file(self.store.generated_path("a"))
self.assertFalse(self.store.needed("a", "transformlinks"))
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.store = DocumentStore(self.datadir, compression=self.compression)
class Store(unittest.TestCase):
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.store = DocumentStore(self.datadir)
def tearDown(self):
shutil.rmtree(self.datadir)
def p(self,path):
path = self.datadir+"/"+path
return path.replace('/', '\\') if os.sep == '\\' else path
def test_open(self):
wanted_filename = self.store.path("basefile", "maindir", ".suffix")
with self.store.open("basefile", "maindir", ".suffix", "w") as fp:
self.assertNotEqual(fp.name, wanted_filename)
self.assertEqual(fp.realname, wanted_filename)
fp.write("This is the data")
self.assertEqual(util.readfile(wanted_filename),
"This is the data")
mtime = os.stat(wanted_filename).st_mtime
# make sure that the open method also can be used
with self.store.open("basefile", "maindir", ".suffix") as fp:
self.assertEqual("This is the data",
fp.read())
# make sure writing identical content does not actually write
# a new file
time.sleep(.1) # just to get a different mtime
with self.store.open("basefile", "maindir", ".suffix", "w") as fp:
fp.write("This is the data")
self.assertEqual(os.stat(wanted_filename).st_mtime,
mtime)
# make sure normal
fp = self.store.open("basefile", "maindir", ".suffix", "w")
fp.write("This is the new data")
fp.close()
self.assertEqual(util.readfile(wanted_filename),
"This is the new data")
def test_open_binary(self):
wanted_filename = self.store.path("basefile", "maindir", ".suffix")
# the smallest possible PNG image
bindata = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\nIDATx\x9cc\x00\x01\x00\x00\x05\x00\x01\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82'
with self.store.open("basefile", "maindir", ".suffix", "wb") as fp:
fp.write(bindata)
mimetype = util.runcmd("file -b --mime-type %s" % wanted_filename)[1]
self.assertEqual("image/png", mimetype.strip())
# make sure that the open method also can be used
with self.store.open("basefile", "maindir", ".suffix", "rb") as fp:
self.assertEqual(bindata, fp.read())
def test_path(self):
self.assertEqual(self.store.path("123","foo", ".bar"),
self.p("foo/123.bar"))
self.assertEqual(self.store.path("123/a","foo", ".bar"),
self.p("foo/123/a.bar"))
self.assertEqual(self.store.path("123:a","foo", ".bar"),
self.p("foo/123/%3Aa.bar"))
realsep = os.sep
try:
os.sep = "\\"
self.assertEqual(self.store.path("123", "foo", ".bar"),
self.datadir.replace("/", os.sep) + "\\foo\\123.bar")
finally:
os.sep = realsep
def test_path_version(self):
eq = self.assertEqual
eq(self.store.path("123","foo", ".bar", version="42"),
self.p("archive/foo/123/.versions/42.bar"))
eq(self.store.path("123/a","foo", ".bar", version="42"),
self.p("archive/foo/123/a/.versions/42.bar"))
eq(self.store.path("123:a","foo", ".bar", version="42"),
self.p("archive/foo/123/%3Aa/.versions/42.bar"))
eq(self.store.path("123:a","foo", ".bar", version="42:1"),
self.p("archive/foo/123/%3Aa/.versions/42/%3A1.bar"))
self.store.storage_policy = "dir"
eq(self.store.path("123","foo", ".bar", version="42"),
self.p("archive/foo/123/.versions/42/index.bar"))
eq(self.store.path("123/a","foo", ".bar", version="42"),
self.p("archive/foo/123/a/.versions/42/index.bar"))
eq(self.store.path("123:a","foo", ".bar", version="42"),
self.p("archive/foo/123/%3Aa/.versions/42/index.bar"))
eq(self.store.path("123:a","foo", ".bar", version="42:1"),
self.p("archive/foo/123/%3Aa/.versions/42/%3A1/index.bar"))
def test_path_attachment(self):
eq = self.assertEqual
repo = self.store # to shorten lines < 80 chars
repo.storage_policy = "dir" # attachments require this
eq(repo.path("123","foo", None, attachment="external.foo"),
self.p("foo/123/external.foo"))
eq(repo.path("123/a","foo", None, attachment="external.foo"),
self.p("foo/123/a/external.foo"))
eq(repo.path("123:a","foo", None, attachment="external.foo"),
self.p("foo/123/%3Aa/external.foo"))
with self.assertRaises(AttachmentNameError):
repo.path("123:a","foo", None,
attachment="invalid:attachment")
with self.assertRaises(AttachmentNameError):
repo.path("123:a","foo", None,
attachment="invalid/attachment"),
repo.storage_policy = "file"
with self.assertRaises(AttachmentPolicyError):
repo.path("123:a","foo", None,
attachment="external.foo"),
def test_path_version_attachment(self):
eq = self.assertEqual
self.store.storage_policy = "dir"
eq(self.store.path("123","foo", None,
version="42", attachment="external.foo"),
self.p("archive/foo/123/.versions/42/external.foo"))
eq(self.store.path("123/a","foo", None,
version="42", attachment="external.foo"),
self.p("archive/foo/123/a/.versions/42/external.foo"))
eq(self.store.path("123:a","foo", None,
version="42", attachment="external.foo"),
self.p("archive/foo/123/%3Aa/.versions/42/external.foo"))
def test_specific_path_methods(self):
self.assertEqual(self.store.downloaded_path('123/a'),
self.p("downloaded/123/a.html"))
self.assertEqual(self.store.downloaded_path('123/a', version="1"),
self.p("archive/downloaded/123/a/.versions/1.html"))
self.assertEqual(self.store.parsed_path('123/a', version="1"),
self.p("archive/parsed/123/a/.versions/1.xhtml"))
self.assertEqual(self.store.generated_path('123/a', version="1"),
self.p("archive/generated/123/a/.versions/1.html"))
self.store.storage_policy = "dir"
self.assertEqual(self.store.downloaded_path('123/a'),
self.p("downloaded/123/a/index.html"))
self.assertEqual(self.store.downloaded_path('123/a', version="1"),
self.p("archive/downloaded/123/a/.versions/1/index.html"))
self.assertEqual(self.store.parsed_path('123/a', version="1"),
self.p("archive/parsed/123/a/.versions/1/index.xhtml"))
self.assertEqual(self.store.generated_path('123/a', version="1"),
self.p("archive/generated/123/a/.versions/1/index.html"))
def test_basefile_to_pathfrag(self):
self.assertEqual(self.store.basefile_to_pathfrag("123-a"), "123-a")
self.assertEqual(self.store.basefile_to_pathfrag("123/a"), "123/a")
self.assertEqual(self.store.basefile_to_pathfrag("123:a"), "123"+os.sep+"%3Aa")
def test_pathfrag_to_basefile(self):
self.assertEqual(self.store.pathfrag_to_basefile("123-a"), "123-a")
self.assertEqual(self.store.pathfrag_to_basefile("123/a"), "123/a")
self.assertEqual(self.store.pathfrag_to_basefile("123/%3Aa"), "123:a")
try:
# make sure the pathfrag method works as expected even when os.sep is not "/"
realsep = os.sep
os.sep = "\\"
self.assertEqual(self.store.pathfrag_to_basefile("123\\a"), "123/a")
finally:
os.sep = realsep
def test_list_basefiles_file(self):
files = ["downloaded/123/a.html",
"downloaded/123/b.html",
"downloaded/124/a.html",
"downloaded/124/b.html"]
basefiles = ["124/b", "124/a", "123/b", "123/a"]
for f in files:
util.writefile(self.p(f),"Nonempty")
self.assertEqual(list(self.store.list_basefiles_for("parse")),
basefiles)
def test_list_basefiles_parse_dir(self):
files = ["downloaded/123/a/index.html",
"downloaded/123/b/index.html",
"downloaded/124/a/index.html",
"downloaded/124/b/index.html"]
basefiles = ["124/b", "124/a", "123/b", "123/a"]
self.store.storage_policy = "dir"
for f in files:
p = self.p(f)
util.writefile(p,"nonempty")
self.assertEqual(list(self.store.list_basefiles_for("parse")),
basefiles)
def test_list_basefiles_generate_dir(self):
files = ["parsed/123/a/index.xhtml",
"parsed/123/b/index.xhtml",
"parsed/124/a/index.xhtml",
"parsed/124/b/index.xhtml"]
basefiles = ["124/b", "124/a", "123/b", "123/a"]
self.store.storage_policy = "dir"
for f in files:
util.writefile(self.p(f),"nonempty")
self.assertEqual(list(self.store.list_basefiles_for("generate")),
basefiles)
def test_list_basefiles_postgenerate_file(self):
files = ["generated/123/a.html",
"generated/123/b.html",
"generated/124/a.html",
"generated/124/b.html"]
basefiles = ["124/b", "124/a", "123/b", "123/a"]
for f in files:
util.writefile(self.p(f),"nonempty")
self.assertEqual(list(self.store.list_basefiles_for("_postgenerate")),
basefiles)
def test_list_basefiles_invalid(self):
with self.assertRaises(ValueError):
list(self.store.list_basefiles_for("invalid_action"))
def test_list_versions_file(self):
files = ["archive/downloaded/123/a/.versions/1.html",
"archive/downloaded/123/a/.versions/2.html",
"archive/downloaded/123/a/.versions/2bis.html",
"archive/downloaded/123/a/.versions/10.html"]
versions = ["1","2", "2bis", "10"]
for f in files:
util.writefile(self.p(f),"nonempty")
# list_versions(action, basefile)
self.assertEqual(list(self.store.list_versions("123/a","downloaded")),
versions)
def test_list_versions_dir(self):
files = ["archive/downloaded/123/a/.versions/1/index.html",
"archive/downloaded/123/a/.versions/2/index.html",
"archive/downloaded/123/a/.versions/2bis/index.html",
"archive/downloaded/123/a/.versions/10/index.html"]
basefiles = ['123/a']
versions = ["1","2", "2bis", "10"]
for f in files:
util.writefile(self.p(f),"nonempty")
self.store.storage_policy = "dir"
self.assertEqual(list(self.store.list_versions("123/a", "downloaded")),
versions)
def test_list_complicated_versions(self):
# the test here is that basefile + version might be ambigious
# as to where to split unless we add the reserved .versions
# directory
a_files = ["archive/downloaded/123/.versions/a/27.html",
"archive/downloaded/123/.versions/a/27/b"]
b_files = ["archive/downloaded/123/b/.versions/27.html",
"archive/downloaded/123/b/.versions/27/b.html"]
a_versions = ["a/27", "a/27/b"]
b_versions = ["27", "27/b"]
for f in a_files + b_files:
util.writefile(self.p(f),"nonempty")
self.assertEqual(list(self.store.list_versions("123","downloaded")),
a_versions)
self.assertEqual(list(self.store.list_versions("123/b","downloaded")),
b_versions)
def test_list_attachments(self):
self.store.storage_policy = "dir" # attachments require this
files = ["downloaded/123/a/index.html",
"downloaded/123/a/attachment.html",
"downloaded/123/a/appendix.pdf",
"downloaded/123/a/other.txt"]
basefiles = ['123/a']
attachments = ['appendix.pdf', 'attachment.html', 'other.txt']
for f in files:
util.writefile(self.p(f),"nonempty")
# list_attachments(action, basefile, version=None)
self.assertEqual(list(self.store.list_attachments("123/a", "downloaded")),
attachments)
def test_list_invalid_attachments(self):
# test that files with an invalid suffix (in
# store.invalid_suffixes) is not listed
self.store.storage_policy = "dir" # attachments require this
files = ["downloaded/123/a/index.html",
"downloaded/123/a/index.invalid",
"downloaded/123/a/other.invalid",
"downloaded/123/a/other.txt"]
basefiles = ['123/a']
attachments = ['other.txt']
for f in files:
util.writefile(self.p(f),"nonempty")
# list_attachments(action, basefile, version=None)
self.assertEqual(list(self.store.list_attachments("123/a", "downloaded")),
attachments)
def test_list_attachments_version(self):
self.store.storage_policy = "dir" # attachments require this
files = ["archive/downloaded/123/a/.versions/1/index.html",
"archive/downloaded/123/a/.versions/1/attachment.txt",
"archive/downloaded/123/a/.versions/2/index.html",
"archive/downloaded/123/a/.versions/2/attachment.txt",
"archive/downloaded/123/a/.versions/2/other.txt"]
basefiles = ['123/a']
versions = ['1','2']
attachments_1 = ['attachment.txt']
attachments_2 = ['attachment.txt', 'other.txt']
for f in files:
util.writefile(self.p(f),"nonempty")
self.assertEqual(list(self.store.list_attachments("123/a","downloaded",
"1")),
attachments_1)
self.assertEqual(list(self.store.list_attachments("123/a","downloaded",
"2")),
attachments_2)
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.store = DocumentStore(self.datadir)
class Store(unittest.TestCase):
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.store = DocumentStore(self.datadir)
def tearDown(self):
shutil.rmtree(self.datadir)
def p(self, path):
path = self.datadir + "/" + path
return path.replace("/", "\\") if os.sep == "\\" else path
def test_open(self):
wanted_filename = self.store.path("basefile", "maindir", ".suffix")
with self.store.open("basefile", "maindir", ".suffix", "w") as fp:
self.assertNotEqual(fp.name, wanted_filename)
self.assertEqual(fp.realname, wanted_filename)
fp.write("This is the data")
self.assertEqual(util.readfile(wanted_filename), "This is the data")
mtime = os.stat(wanted_filename).st_mtime
# make sure writing identical content does not actually write
# a new file
time.sleep(0.1) # just to get a different mtime
with self.store.open("basefile", "maindir", ".suffix", "w") as fp:
fp.write("This is the data")
self.assertEqual(os.stat(wanted_filename).st_mtime, mtime)
def test_path(self):
self.assertEqual(self.store.path("123", "foo", ".bar"), self.p("foo/123.bar"))
self.assertEqual(self.store.path("123/a", "foo", ".bar"), self.p("foo/123/a.bar"))
self.assertEqual(self.store.path("123:a", "foo", ".bar"), self.p("foo/123/%3Aa.bar"))
realsep = os.sep
try:
os.sep = "\\"
self.assertEqual(
self.store.path("123", "foo", ".bar"), self.datadir.replace("/", os.sep) + "\\foo\\123.bar"
)
finally:
os.sep = realsep
def test_path_version(self):
eq = self.assertEqual
eq(self.store.path("123", "foo", ".bar", version="42"), self.p("archive/foo/123/42.bar"))
eq(self.store.path("123/a", "foo", ".bar", version="42"), self.p("archive/foo/123/a/42.bar"))
eq(self.store.path("123:a", "foo", ".bar", version="42"), self.p("archive/foo/123/%3Aa/42.bar"))
eq(self.store.path("123:a", "foo", ".bar", version="42:1"), self.p("archive/foo/123/%3Aa/42/%3A1.bar"))
self.store.storage_policy = "dir"
eq(self.store.path("123", "foo", ".bar", version="42"), self.p("archive/foo/123/42/index.bar"))
eq(self.store.path("123/a", "foo", ".bar", version="42"), self.p("archive/foo/123/a/42/index.bar"))
eq(self.store.path("123:a", "foo", ".bar", version="42"), self.p("archive/foo/123/%3Aa/42/index.bar"))
eq(self.store.path("123:a", "foo", ".bar", version="42:1"), self.p("archive/foo/123/%3Aa/42/%3A1/index.bar"))
def test_path_attachment(self):
eq = self.assertEqual
repo = self.store # to shorten lines < 80 chars
repo.storage_policy = "dir" # attachments require this
eq(repo.path("123", "foo", None, attachment="external.foo"), self.p("foo/123/external.foo"))
eq(repo.path("123/a", "foo", None, attachment="external.foo"), self.p("foo/123/a/external.foo"))
eq(repo.path("123:a", "foo", None, attachment="external.foo"), self.p("foo/123/%3Aa/external.foo"))
with self.assertRaises(AttachmentNameError):
repo.path("123:a", "foo", None, attachment="invalid:attachment")
with self.assertRaises(AttachmentNameError):
repo.path("123:a", "foo", None, attachment="invalid/attachment"),
repo.storage_policy = "file"
with self.assertRaises(AttachmentPolicyError):
repo.path("123:a", "foo", None, attachment="external.foo"),
def test_path_version_attachment(self):
eq = self.assertEqual
self.store.storage_policy = "dir"
eq(
self.store.path("123", "foo", None, version="42", attachment="external.foo"),
self.p("archive/foo/123/42/external.foo"),
)
eq(
self.store.path("123/a", "foo", None, version="42", attachment="external.foo"),
self.p("archive/foo/123/a/42/external.foo"),
)
eq(
self.store.path("123:a", "foo", None, version="42", attachment="external.foo"),
self.p("archive/foo/123/%3Aa/42/external.foo"),
)
def test_specific_path_methods(self):
self.assertEqual(self.store.downloaded_path("123/a"), self.p("downloaded/123/a.html"))
self.assertEqual(self.store.downloaded_path("123/a", version="1"), self.p("archive/downloaded/123/a/1.html"))
self.assertEqual(self.store.parsed_path("123/a", version="1"), self.p("archive/parsed/123/a/1.xhtml"))
self.assertEqual(self.store.generated_path("123/a", version="1"), self.p("archive/generated/123/a/1.html"))
self.store.storage_policy = "dir"
self.assertEqual(self.store.downloaded_path("123/a"), self.p("downloaded/123/a/index.html"))
self.assertEqual(
self.store.downloaded_path("123/a", version="1"), self.p("archive/downloaded/123/a/1/index.html")
)
self.assertEqual(self.store.parsed_path("123/a", version="1"), self.p("archive/parsed/123/a/1/index.xhtml"))
self.assertEqual(
self.store.generated_path("123/a", version="1"), self.p("archive/generated/123/a/1/index.html")
)
def test_basefile_to_pathfrag(self):
self.assertEqual(self.store.basefile_to_pathfrag("123-a"), "123-a")
self.assertEqual(self.store.basefile_to_pathfrag("123/a"), "123/a")
self.assertEqual(self.store.basefile_to_pathfrag("123:a"), "123" + os.sep + "%3Aa")
def test_pathfrag_to_basefile(self):
self.assertEqual(self.store.pathfrag_to_basefile("123-a"), "123-a")
self.assertEqual(self.store.pathfrag_to_basefile("123/a"), "123/a")
self.assertEqual(self.store.pathfrag_to_basefile("123/%3Aa"), "123:a")
try:
# make sure the pathfrag method works as expected even when os.sep is not "/"
realsep = os.sep
os.sep = "\\"
self.assertEqual(self.store.pathfrag_to_basefile("123\\a"), "123/a")
finally:
os.sep = realsep
def test_list_basefiles_file(self):
files = ["downloaded/123/a.html", "downloaded/123/b.html", "downloaded/124/a.html", "downloaded/124/b.html"]
basefiles = ["124/b", "124/a", "123/b", "123/a"]
for f in files:
util.writefile(self.p(f), "Nonempty")
self.assertEqual(list(self.store.list_basefiles_for("parse")), basefiles)
def test_list_basefiles_parse_dir(self):
files = [
"downloaded/123/a/index.html",
"downloaded/123/b/index.html",
"downloaded/124/a/index.html",
"downloaded/124/b/index.html",
]
basefiles = ["124/b", "124/a", "123/b", "123/a"]
self.store.storage_policy = "dir"
for f in files:
util.writefile(self.p(f), "nonempty")
self.assertEqual(list(self.store.list_basefiles_for("parse")), basefiles)
def test_list_basefiles_generate_dir(self):
files = [
"parsed/123/a/index.xhtml",
"parsed/123/b/index.xhtml",
"parsed/124/a/index.xhtml",
"parsed/124/b/index.xhtml",
]
basefiles = ["124/b", "124/a", "123/b", "123/a"]
self.store.storage_policy = "dir"
for f in files:
util.writefile(self.p(f), "nonempty")
self.assertEqual(list(self.store.list_basefiles_for("generate")), basefiles)
def test_list_basefiles_postgenerate_file(self):
files = ["generated/123/a.html", "generated/123/b.html", "generated/124/a.html", "generated/124/b.html"]
basefiles = ["124/b", "124/a", "123/b", "123/a"]
for f in files:
util.writefile(self.p(f), "nonempty")
self.assertEqual(list(self.store.list_basefiles_for("_postgenerate")), basefiles)
def test_list_basefiles_invalid(self):
with self.assertRaises(ValueError):
list(self.store.list_basefiles_for("invalid_action"))
def test_list_versions_file(self):
files = [
"archive/downloaded/123/a/1.html",
"archive/downloaded/123/a/2.html",
"archive/downloaded/123/a/2bis.html",
"archive/downloaded/123/a/10.html",
]
versions = ["1", "2", "2bis", "10"]
for f in files:
util.writefile(self.p(f), "nonempty")
# list_versions(action, basefile)
self.assertEqual(list(self.store.list_versions("123/a", "downloaded")), versions)
def test_list_versions_dir(self):
files = [
"archive/downloaded/123/a/1/index.html",
"archive/downloaded/123/a/2/index.html",
"archive/downloaded/123/a/2bis/index.html",
"archive/downloaded/123/a/10/index.html",
]
basefiles = ["123/a"]
versions = ["1", "2", "2bis", "10"]
for f in files:
util.writefile(self.p(f), "nonempty")
self.store.storage_policy = "dir"
self.assertEqual(list(self.store.list_versions("123/a", "downloaded")), versions)
def test_list_attachments(self):
files = [
"downloaded/123/a/index.html",
"downloaded/123/a/attachment.html",
"downloaded/123/a/appendix.pdf",
"downloaded/123/a/other.txt",
]
basefiles = ["123/a"]
attachments = ["appendix.pdf", "attachment.html", "other.txt"]
for f in files:
util.writefile(self.p(f), "nonempty")
# list_attachments(action, basefile, version=None)
self.assertEqual(list(self.store.list_attachments("123/a", "downloaded")), attachments)
def test_list_attachments_version(self):
files = [
"archive/downloaded/123/a/1/index.html",
"archive/downloaded/123/a/1/attachment.txt",
"archive/downloaded/123/a/2/index.html",
"archive/downloaded/123/a/2/attachment.txt",
"archive/downloaded/123/a/2/other.txt",
]
basefiles = ["123/a"]
versions = ["1", "2"]
attachments_1 = ["attachment.txt"]
attachments_2 = ["attachment.txt", "other.txt"]
for f in files:
util.writefile(self.p(f), "nonempty")
self.assertEqual(list(self.store.list_attachments("123/a", "downloaded", "1")), attachments_1)
self.assertEqual(list(self.store.list_attachments("123/a", "downloaded", "2")), attachments_2)
class Needed(unittest.TestCase):
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.store = DocumentStore(self.datadir)
def tearDown(self):
shutil.rmtree(self.datadir)
def create_file(self, path, timestampoffset=0, content="dummy"):
util.ensure_dir(path)
with open(path, "w") as fp:
fp.write(content)
if timestampoffset:
os.utime(path, (time.time(), time.time() + timestampoffset))
def test_parse_not_needed(self):
self.create_file(self.store.downloaded_path("a"))
self.create_file(self.store.parsed_path("a"), 3600)
self.assertFalse(self.store.needed("a", "parse"))
def test_parse_needed(self):
self.create_file(self.store.downloaded_path("a"))
res = self.store.needed("a", "parse")
self.assertTrue(res)
self.assertIn("outfile doesn't exist", res.reason)
def test_parse_needed_outdated(self):
self.create_file(self.store.downloaded_path("a"))
self.create_file(self.store.parsed_path("a"), -3600)
res = self.store.needed("a", "parse")
self.assertTrue(res)
self.assertIn("is newer than outfile", res.reason)
def create_entry(self, basefile, timestampoffset=0):
# create a entry file with indexed_{ft,ts,dep} set to the
# current time with optional offset. Also
# .status['generated']['date'], to test needed(...,
# 'transformlinks')
de = DocumentEntry(self.store.documententry_path(basefile))
delta = timedelta(seconds=timestampoffset)
ts = datetime.now() + delta
de.indexed_ts = ts
de.indexed_ft = ts
de.indexed_dep = ts
de.updated = ts
de.status['generate'] = {'date': ts}
de.save()
def test_relate_not_needed(self):
self.create_entry("a")
self.create_file(self.store.distilled_path("a"), -3600)
self.create_file(self.store.parsed_path("a"), -3600)
self.create_file(self.store.dependencies_path("a"), -3600)
self.assertFalse(self.store.needed("a", "relate"))
def test_relate_needed_ts(self):
self.create_entry("a", -3600)
self.create_file(self.store.distilled_path("a"))
self.create_file(self.store.parsed_path("a"), -7200)
self.create_file(self.store.dependencies_path("a"), -7200)
res = self.store.needed("a", "relate")
self.assertTrue(res)
self.assertIn("is newer than indexed_ts in documententry", res.reason)
def test_relate_needed_ft(self):
self.create_entry("a", -3600)
self.create_file(self.store.distilled_path("a"), -7200)
self.create_file(self.store.parsed_path("a"))
self.create_file(self.store.dependencies_path("a"), -7200)
res = self.store.needed("a", "relate")
self.assertTrue(res)
self.assertIn("is newer than indexed_ft in documententry", res.reason)
def test_relate_needed_dep(self):
self.create_entry("a", -3600)
self.create_file(self.store.distilled_path("a"), -7200)
self.create_file(self.store.parsed_path("a"), -7200)
self.create_file(self.store.dependencies_path("a"))
res = self.store.needed("a", "relate")
self.assertTrue(res)
self.assertIn("is newer than indexed_dep in documententry", res.reason)
def test_generate_not_needed(self):
self.create_file(self.store.parsed_path("a"))
self.create_file(self.store.generated_path("a"), 3600)
self.assertFalse(self.store.needed("a", "generate"))
def test_generate_not_needed_404(self):
self.create_file(self.store.parsed_path("a"))
self.create_file(self.store.generated_path("a") + ".404", 3600)
self.assertFalse(self.store.needed("a", "generate"))
def test_generate_needed(self):
self.create_file(self.store.parsed_path("a"))
res = self.store.needed("a", "generate")
self.assertTrue(res)
self.assertIn("outfile doesn't exist", res.reason)
def test_generate_needed_outdated(self):
self.create_file(self.store.parsed_path("a"))
self.create_file(self.store.generated_path("a"), -3600)
res = self.store.needed("a", "generate")
self.assertTrue(res)
self.assertIn("is newer than outfile", res.reason)
def test_generate_needed_outdated_dep(self):
self.create_file(self.store.parsed_path("a"), -3600)
self.create_file(self.store.generated_path("a"), -1800)
dependency_path = self.store.datadir + os.sep + "blahonga.txt"
self.create_file(self.store.dependencies_path("a"), -3600, dependency_path)
# create a arbitrary dependency file which is newer than outfile
self.create_file(dependency_path)
res = self.store.needed("a", "generate")
self.assertTrue(res)
self.assertIn("is newer than outfile", res.reason)
def test_transformlinks_needed(self):
self.create_file(self.store.generated_path("a"))
self.create_entry("a")
res = self.store.needed("a", "transformlinks")
self.assertTrue(res)
self.assertIn("has not been modified after generate", res.reason)
def test_transformlinks_not_needed(self):
self.create_entry("a", -3600)
self.create_file(self.store.generated_path("a"))
self.assertFalse(self.store.needed("a", "transformlinks"))
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.store = DocumentStore(self.datadir, compression=self.compression)
class Store(unittest.TestCase):
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.store = DocumentStore(self.datadir)
def tearDown(self):
shutil.rmtree(self.datadir)
def p(self,path):
path = self.datadir+"/"+path
return path.replace('/', '\\') if os.sep == '\\' else path
def test_open(self):
wanted_filename = self.store.path("basefile", "maindir", ".suffix")
with self.store.open("basefile", "maindir", ".suffix", "w") as fp:
self.assertNotEqual(fp.name, wanted_filename)
self.assertEqual(fp.realname, wanted_filename)
fp.write("This is the data")
self.assertEqual(util.readfile(wanted_filename),
"This is the data")
mtime = os.stat(wanted_filename).st_mtime
# make sure that the open method also can be used
with self.store.open("basefile", "maindir", ".suffix") as fp:
self.assertEqual("This is the data",
fp.read())
# make sure writing identical content does not actually write
# a new file
time.sleep(.1) # just to get a different mtime
with self.store.open("basefile", "maindir", ".suffix", "w") as fp:
fp.write("This is the data")
self.assertEqual(os.stat(wanted_filename).st_mtime,
mtime)
# make sure normal
fp = self.store.open("basefile", "maindir", ".suffix", "w")
fp.write("This is the new data")
fp.close()
self.assertEqual(util.readfile(wanted_filename),
"This is the new data")
def test_open_binary(self):
wanted_filename = self.store.path("basefile", "maindir", ".suffix")
# the smallest possible PNG image
bindata = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\nIDATx\x9cc\x00\x01\x00\x00\x05\x00\x01\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82'
with self.store.open("basefile", "maindir", ".suffix", "wb") as fp:
fp.write(bindata)
mimetype = util.runcmd("file -b --mime-type %s" % wanted_filename)[1]
self.assertEqual("image/png", mimetype.strip())
# make sure that the open method also can be used
with self.store.open("basefile", "maindir", ".suffix", "rb") as fp:
self.assertEqual(bindata, fp.read())
def test_path(self):
self.assertEqual(self.store.path("123","foo", ".bar"),
self.p("foo/123.bar"))
self.assertEqual(self.store.path("123/a","foo", ".bar"),
self.p("foo/123/a.bar"))
self.assertEqual(self.store.path("123:a","foo", ".bar"),
self.p("foo/123/%3Aa.bar"))
realsep = os.sep
try:
os.sep = "\\"
self.assertEqual(self.store.path("123", "foo", ".bar"),
self.datadir.replace("/", os.sep) + "\\foo\\123.bar")
finally:
os.sep = realsep
def test_path_version(self):
eq = self.assertEqual
eq(self.store.path("123","foo", ".bar", version="42"),
self.p("archive/foo/123/42.bar"))
eq(self.store.path("123/a","foo", ".bar", version="42"),
self.p("archive/foo/123/a/42.bar"))
eq(self.store.path("123:a","foo", ".bar", version="42"),
self.p("archive/foo/123/%3Aa/42.bar"))
eq(self.store.path("123:a","foo", ".bar", version="42:1"),
self.p("archive/foo/123/%3Aa/42/%3A1.bar"))
self.store.storage_policy = "dir"
eq(self.store.path("123","foo", ".bar", version="42"),
self.p("archive/foo/123/42/index.bar"))
eq(self.store.path("123/a","foo", ".bar", version="42"),
self.p("archive/foo/123/a/42/index.bar"))
eq(self.store.path("123:a","foo", ".bar", version="42"),
self.p("archive/foo/123/%3Aa/42/index.bar"))
eq(self.store.path("123:a","foo", ".bar", version="42:1"),
self.p("archive/foo/123/%3Aa/42/%3A1/index.bar"))
def test_path_attachment(self):
eq = self.assertEqual
repo = self.store # to shorten lines < 80 chars
repo.storage_policy = "dir" # attachments require this
eq(repo.path("123","foo", None, attachment="external.foo"),
self.p("foo/123/external.foo"))
eq(repo.path("123/a","foo", None, attachment="external.foo"),
self.p("foo/123/a/external.foo"))
eq(repo.path("123:a","foo", None, attachment="external.foo"),
self.p("foo/123/%3Aa/external.foo"))
with self.assertRaises(AttachmentNameError):
repo.path("123:a","foo", None,
attachment="invalid:attachment")
with self.assertRaises(AttachmentNameError):
repo.path("123:a","foo", None,
attachment="invalid/attachment"),
repo.storage_policy = "file"
with self.assertRaises(AttachmentPolicyError):
repo.path("123:a","foo", None,
attachment="external.foo"),
def test_path_version_attachment(self):
eq = self.assertEqual
self.store.storage_policy = "dir"
eq(self.store.path("123","foo", None,
version="42", attachment="external.foo"),
self.p("archive/foo/123/42/external.foo"))
eq(self.store.path("123/a","foo", None,
version="42", attachment="external.foo"),
self.p("archive/foo/123/a/42/external.foo"))
eq(self.store.path("123:a","foo", None,
version="42", attachment="external.foo"),
self.p("archive/foo/123/%3Aa/42/external.foo"))
def test_specific_path_methods(self):
self.assertEqual(self.store.downloaded_path('123/a'),
self.p("downloaded/123/a.html"))
self.assertEqual(self.store.downloaded_path('123/a', version="1"),
self.p("archive/downloaded/123/a/1.html"))
self.assertEqual(self.store.parsed_path('123/a', version="1"),
self.p("archive/parsed/123/a/1.xhtml"))
self.assertEqual(self.store.generated_path('123/a', version="1"),
self.p("archive/generated/123/a/1.html"))
self.store.storage_policy = "dir"
self.assertEqual(self.store.downloaded_path('123/a'),
self.p("downloaded/123/a/index.html"))
self.assertEqual(self.store.downloaded_path('123/a', version="1"),
self.p("archive/downloaded/123/a/1/index.html"))
self.assertEqual(self.store.parsed_path('123/a', version="1"),
self.p("archive/parsed/123/a/1/index.xhtml"))
self.assertEqual(self.store.generated_path('123/a', version="1"),
self.p("archive/generated/123/a/1/index.html"))
def test_basefile_to_pathfrag(self):
self.assertEqual(self.store.basefile_to_pathfrag("123-a"), "123-a")
self.assertEqual(self.store.basefile_to_pathfrag("123/a"), "123/a")
self.assertEqual(self.store.basefile_to_pathfrag("123:a"), "123"+os.sep+"%3Aa")
def test_pathfrag_to_basefile(self):
self.assertEqual(self.store.pathfrag_to_basefile("123-a"), "123-a")
self.assertEqual(self.store.pathfrag_to_basefile("123/a"), "123/a")
self.assertEqual(self.store.pathfrag_to_basefile("123/%3Aa"), "123:a")
try:
# make sure the pathfrag method works as expected even when os.sep is not "/"
realsep = os.sep
os.sep = "\\"
self.assertEqual(self.store.pathfrag_to_basefile("123\\a"), "123/a")
finally:
os.sep = realsep
def test_list_basefiles_file(self):
files = ["downloaded/123/a.html",
"downloaded/123/b.html",
"downloaded/124/a.html",
"downloaded/124/b.html"]
basefiles = ["124/b", "124/a", "123/b", "123/a"]
for f in files:
util.writefile(self.p(f),"Nonempty")
self.assertEqual(list(self.store.list_basefiles_for("parse")),
basefiles)
def test_list_basefiles_parse_dir(self):
files = ["downloaded/123/a/index.html",
"downloaded/123/b/index.html",
"downloaded/124/a/index.html",
"downloaded/124/b/index.html"]
basefiles = ["124/b", "124/a", "123/b", "123/a"]
self.store.storage_policy = "dir"
for f in files:
p = self.p(f)
util.writefile(p,"nonempty")
self.assertEqual(list(self.store.list_basefiles_for("parse")),
basefiles)
def test_list_basefiles_generate_dir(self):
files = ["parsed/123/a/index.xhtml",
"parsed/123/b/index.xhtml",
"parsed/124/a/index.xhtml",
"parsed/124/b/index.xhtml"]
basefiles = ["124/b", "124/a", "123/b", "123/a"]
self.store.storage_policy = "dir"
for f in files:
util.writefile(self.p(f),"nonempty")
self.assertEqual(list(self.store.list_basefiles_for("generate")),
basefiles)
def test_list_basefiles_postgenerate_file(self):
files = ["generated/123/a.html",
"generated/123/b.html",
"generated/124/a.html",
"generated/124/b.html"]
basefiles = ["124/b", "124/a", "123/b", "123/a"]
for f in files:
util.writefile(self.p(f),"nonempty")
self.assertEqual(list(self.store.list_basefiles_for("_postgenerate")),
basefiles)
def test_list_basefiles_invalid(self):
with self.assertRaises(ValueError):
list(self.store.list_basefiles_for("invalid_action"))
def test_list_versions_file(self):
files = ["archive/downloaded/123/a/1.html",
"archive/downloaded/123/a/2.html",
"archive/downloaded/123/a/2bis.html",
"archive/downloaded/123/a/10.html"]
versions = ["1","2", "2bis", "10"]
for f in files:
util.writefile(self.p(f),"nonempty")
# list_versions(action, basefile)
self.assertEqual(list(self.store.list_versions("123/a","downloaded")),
versions)
def test_list_versions_dir(self):
files = ["archive/downloaded/123/a/1/index.html",
"archive/downloaded/123/a/2/index.html",
"archive/downloaded/123/a/2bis/index.html",
"archive/downloaded/123/a/10/index.html"]
basefiles = ['123/a']
versions = ["1","2", "2bis", "10"]
for f in files:
util.writefile(self.p(f),"nonempty")
self.store.storage_policy = "dir"
self.assertEqual(list(self.store.list_versions("123/a", "downloaded")),
versions)
def test_list_attachments(self):
self.store.storage_policy = "dir" # attachments require this
files = ["downloaded/123/a/index.html",
"downloaded/123/a/attachment.html",
"downloaded/123/a/appendix.pdf",
"downloaded/123/a/other.txt"]
basefiles = ['123/a']
attachments = ['appendix.pdf', 'attachment.html', 'other.txt']
for f in files:
util.writefile(self.p(f),"nonempty")
# list_attachments(action, basefile, version=None)
self.assertEqual(list(self.store.list_attachments("123/a", "downloaded")),
attachments)
def test_list_invalid_attachments(self):
# test that files with an invalid suffix (in
# store.invalid_suffixes) is not listed
self.store.storage_policy = "dir" # attachments require this
files = ["downloaded/123/a/index.html",
"downloaded/123/a/index.invalid",
"downloaded/123/a/other.invalid",
"downloaded/123/a/other.txt"]
basefiles = ['123/a']
attachments = ['other.txt']
for f in files:
util.writefile(self.p(f),"nonempty")
# list_attachments(action, basefile, version=None)
self.assertEqual(list(self.store.list_attachments("123/a", "downloaded")),
attachments)
def test_list_attachments_version(self):
self.store.storage_policy = "dir" # attachments require this
files = ["archive/downloaded/123/a/1/index.html",
"archive/downloaded/123/a/1/attachment.txt",
"archive/downloaded/123/a/2/index.html",
"archive/downloaded/123/a/2/attachment.txt",
"archive/downloaded/123/a/2/other.txt"]
basefiles = ['123/a']
versions = ['1','2']
attachments_1 = ['attachment.txt']
attachments_2 = ['attachment.txt', 'other.txt']
for f in files:
util.writefile(self.p(f),"nonempty")
self.assertEqual(list(self.store.list_attachments("123/a","downloaded",
"1")),
attachments_1)
self.assertEqual(list(self.store.list_attachments("123/a","downloaded",
"2")),
attachments_2)
def test_mkpatch(self):
tempdir = tempfile.mkdtemp()
basefile = "1"
# Test 1: A repo which do not use any intermediate files. In
# this case, the user edits the downloaded file, then runs
# mkpatch, which saves the edited file, re-downloads the file,
# and computes the diff.
store = DocumentStore(tempdir + "/base")
downloaded_path = store.downloaded_path(basefile)
def my_download_single(self):
# this function simulates downloading
with open(downloaded_path, "wb") as fp:
fp.write(
"""This is a file.
It has been downloaded.
""".encode()
)
repo = DocumentRepository(datadir=tempdir)
with repo.store.open_downloaded(basefile, "wb") as fp:
fp.write(
"""This is a file.
It has been patched.
""".encode()
)
d = Devel()
globalconf = LayeredConfig(
{
"datadir": tempdir,
"patchdir": tempdir,
"devel": {"class": "ferenda.Devel"},
"base": {"class": "ferenda.DocumentRepository"},
},
cascade=True,
)
d.config = globalconf.devel
with patch("ferenda.DocumentRepository.download_single") as mock:
mock.side_effect = my_download_single
patchpath = d.mkpatch("base", basefile, "Example patch")
patchcontent = util.readfile(patchpath)
self.assertIn("Example patch", patchcontent)
self.assertIn("@@ -1,2 +1,2 @@", patchcontent)
self.assertIn("-It has been downloaded.", patchcontent)
self.assertIn("+It has been patched.", patchcontent)
# test 2: Same, but with a multi-line desc
with repo.store.open_downloaded(basefile, "wb") as fp:
fp.write(
"""This is a file.
It has been patched.
""".encode()
)
longdesc = """A longer comment
spanning
several lines"""
with patch("ferenda.DocumentRepository.download_single") as mock:
mock.side_effect = my_download_single
patchpath = d.mkpatch("base", basefile, longdesc)
patchcontent = util.readfile(patchpath)
desccontent = util.readfile(patchpath.replace(".patch", ".desc"))
self.assertEqual(longdesc, desccontent)
self.assertFalse("A longer comment" in patchcontent)
self.assertIn("@@ -1,2 +1,2 @@", patchcontent)
self.assertIn("-It has been downloaded.", patchcontent)
self.assertIn("+It has been patched.", patchcontent)
# test 3: If intermediate file exists, patch that one
intermediate_path = store.intermediate_path(basefile)
util.ensure_dir(intermediate_path)
with open(intermediate_path, "wb") as fp:
fp.write(
"""This is a intermediate file.
It has been patched.
""".encode()
)
intermediate_path = store.intermediate_path(basefile)
def my_parse(self, basefile=None):
# this function simulates downloading
with open(intermediate_path, "wb") as fp:
fp.write(
"""This is a intermediate file.
It has been processed.
""".encode()
)
with patch("ferenda.DocumentRepository.parse") as mock:
mock.side_effect = my_parse
patchpath = d.mkpatch("base", basefile, "Example patch")
patchcontent = util.readfile(patchpath)
self.assertIn("@@ -1,2 +1,2 @@ Example patch", patchcontent)
self.assertIn(" This is a intermediate file", patchcontent)
self.assertIn("-It has been processed.", patchcontent)
self.assertIn("+It has been patched.", patchcontent)
