def test_save_meta_files01(self):
self.create_general_config()
book = Book(Host(self.test_root))
book.meta = {
'20200101000000000': {
'title': 'Dummy 1 中文'
},
'20200101000000001': {
'title': 'Dummy 2 中文'
},
}
book.save_meta_files()
with open(os.path.join(self.test_root, 'tree', 'meta.js'),
encoding='UTF-8') as fh:
self.assertEqual(
fh.read(), """/**
* Feel free to edit this file, but keep data code valid JSON format.
*/
scrapbook.meta({
"20200101000000000": {
"title": "Dummy 1 中文"
},
"20200101000000001": {
"title": "Dummy 2 中文"
}
})""")
def test_get_lock02(self, mock_filelock):
"""With parameters"""
host = Host(self.test_root)
host.get_lock('test',
timeout=10, stale=120, poll_interval=0.3, assume_acquired=True)
mock_filelock.assert_called_once_with(host, 'test',
timeout=10, stale=120, poll_interval=0.3, assume_acquired=True)
def test_save_toc_files02(self):
self.create_general_config()
book = Book(Host(self.test_root))
book.toc = {
'root': [
'20200101000000000',
'20200101000000001',
'20200101000000002',
'20200101000000003',
'20200101000000004',
],
'20200101000000001': [
'20200101000000011'
],
'20200101000000002': [
'20200101000000021'
],
'20200101000000003': [
'20200101000000031',
'20200101000000032'
],
}
book.save_toc_files()
with open(os.path.join(self.test_root, 'tree', 'toc.js'), encoding='UTF-8') as fh:
self.assertEqual(fh.read(), """/**
* Feel free to edit this file, but keep data code valid JSON format.
*/
scrapbook.toc({
"root": [
"20200101000000000",
"20200101000000001",
"20200101000000002",
"20200101000000003",
"20200101000000004"
]
})""")
with open(os.path.join(self.test_root, 'tree', 'toc1.js'), encoding='UTF-8') as fh:
self.assertEqual(fh.read(), """/**
* Feel free to edit this file, but keep data code valid JSON format.
*/
scrapbook.toc({
"20200101000000001": [
"20200101000000011"
],
"20200101000000002": [
"20200101000000021"
]
})""")
with open(os.path.join(self.test_root, 'tree', 'toc2.js'), encoding='UTF-8') as fh:
self.assertEqual(fh.read(), """/**
* Feel free to edit this file, but keep data code valid JSON format.
*/
scrapbook.toc({
"20200101000000003": [
"20200101000000031",
"20200101000000032"
]
})""")
def test_save_fulltext_files01(self):
self.create_general_config()
book = Book(Host(self.test_root))
book.fulltext = {
"20200101000000000": {
'index.html': {
'content': 'dummy text 1 中文',
}
},
"20200101000000001": {
'index.html': {
'content': 'dummy text 2 中文',
}
},
}
book.save_fulltext_files()
with open(os.path.join(self.test_root, 'tree', 'fulltext.js'), encoding='UTF-8') as fh:
self.assertEqual(fh.read(), """/**
* This file is generated by WebScrapBook and is not intended to be edited.
*/
scrapbook.fulltext({
"20200101000000000": {
"index.html": {
"content": "dummy text 1 中文"
}
},
"20200101000000001": {
"index.html": {
"content": "dummy text 2 中文"
}
}
})""")
def test_load_tree_file01(self):
"""Test normal loading"""
self.create_general_config()
with open(os.path.join(self.test_root, 'meta.js'), 'w', encoding='UTF-8') as f:
f.write("""/**
* This file is generated by WebScrapBook and is not intended to be edited.
*/
scrapbook.meta({
"20200101000000000": {
"index": "20200101000000000/index.html",
"title": "Dummy",
"type": "",
"create": "20200101000000000",
"modify": "20200101000000000"
}
})""")
book = Book(Host(self.test_root))
self.assertEqual(
book.load_tree_file(os.path.join(self.test_root, 'meta.js')), {
'20200101000000000': {
'index': '20200101000000000/index.html',
'title': 'Dummy',
'type': '',
'create': '20200101000000000',
'modify': '20200101000000000',
},
})
def test_get_tree_file(self):
self.create_general_config()
book = Book(Host(self.test_root))
self.assertEqual(book.get_tree_file('meta'),
os.path.join(self.test_root, 'tree', 'meta.js'))
self.assertEqual(book.get_tree_file('toc', 1),
os.path.join(self.test_root, 'tree', 'toc1.js'))
def test_meta_modify02(self):
"""Don't take mtime of index.html if not newer."""
index_file = os.path.join(self.test_input, 'data', '20200102030405',
'index.html')
os.makedirs(os.path.dirname(index_file))
with open(index_file, 'w', encoding='UTF-8') as fh:
pass
t = time.mktime((2020, 1, 1, 0, 0, 0, 0, 0, -1))
os.utime(index_file, (t, t))
with open(self.test_input_rdf, 'w', encoding='UTF-8') as fh:
fh.write("""\
<?xml version="1.0"?>
<RDF:RDF xmlns:NS1="http://amb.vis.ne.jp/mozilla/scrapbook-rdf#"
xmlns:NC="http://home.netscape.com/NC-rdf#"
xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<RDF:Description RDF:about="urn:scrapbook:item20200102030405"
NS1:id="20200102030405"
NS1:type=""
NS1:create="20200102030406"
NS1:modify="20200102030407" />
</RDF:RDF>
""")
for info in sb2wsb.run(self.test_input, self.test_output):
pass
book = Host(self.test_output).books['']
book.load_meta_files()
self.assertEqual(
book.meta['20200102030405']['modify'],
util.datetime_to_id(util.id_to_datetime_legacy('20200102030407')))
def test_init03(self):
"""Check modified path"""
with open(self.test_config, 'w', encoding='UTF-8') as f:
f.write("""[app]
root = public
[book ""]
name = scrapbook
top_dir = sb
data_dir = data
tree_dir = tree
index = tree/map.html
no_tree = false
""")
host = Host(self.test_root)
book = Book(host)
self.assertEqual(book.host, host)
self.assertEqual(book.id, '')
self.assertEqual(book.name, 'scrapbook')
self.assertEqual(book.root, self.test_root)
self.assertEqual(book.top_dir, os.path.join(self.test_root, 'public', 'sb'))
self.assertEqual(book.data_dir, os.path.join(self.test_root, 'public', 'sb', 'data'))
self.assertEqual(book.tree_dir, os.path.join(self.test_root, 'public', 'sb', 'tree'))
self.assertFalse(book.no_tree)
def test_backup05(self):
"""Pass if file outside the host root."""
book = Book(Host(self.test_root))
book.init_backup()
book.backup(__file__)
self.assertListEqual(os.listdir(self.test_wsbdir), [])
def test_meta_type02(self):
"""Use "site" if it's "marked" and has sitemap.xml"""
index_file = os.path.join(self.test_input, 'data', '20200101010101',
'index.html')
sitemap_file = os.path.join(self.test_input, 'data', '20200101010101',
'sitemap.xml')
os.makedirs(os.path.dirname(sitemap_file))
with open(index_file, 'w', encoding='UTF-8') as fh:
pass
with open(sitemap_file, 'w', encoding='UTF-8') as fh:
pass
with open(self.test_input_rdf, 'w', encoding='UTF-8') as fh:
fh.write("""\
<?xml version="1.0"?>
<RDF:RDF xmlns:NS1="http://amb.vis.ne.jp/mozilla/scrapbook-rdf#"
xmlns:NC="http://home.netscape.com/NC-rdf#"
xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<RDF:Description RDF:about="urn:scrapbook:item20200101010101"
NS1:type="marked" />
</RDF:RDF>
""")
for info in sb2wsb.run(self.test_input, self.test_output):
pass
book = Host(self.test_output).books['']
book.load_meta_files()
self.assertEqual(book.meta['20200101010101']['type'], 'site')
self.assertTrue(book.meta['20200101010101']['marked'])
def test_meta_basic03(self):
"""Default value for missing keys.
- type: ""
- create: infer from id
- modify: infer from create (and then id)
"""
with open(self.test_input_rdf, 'w', encoding='UTF-8') as fh:
fh.write("""\
<?xml version="1.0"?>
<RDF:RDF xmlns:NS1="http://amb.vis.ne.jp/mozilla/scrapbook-rdf#"
xmlns:NC="http://home.netscape.com/NC-rdf#"
xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<RDF:Description RDF:about="urn:scrapbook:item20200102030405" />
</RDF:RDF>
""")
for info in sb2wsb.run(self.test_input, self.test_output):
pass
book = Host(self.test_output).books['']
book.load_meta_files()
self.assertDictEqual(book.meta['20200102030405'], {
'index': '20200102030405/index.html',
'type': '',
'create': util.datetime_to_id(util.id_to_datetime_legacy('20200102030405')),
'modify': util.datetime_to_id(util.id_to_datetime_legacy('20200102030405')),
})
def test_meta_basic02(self):
"""A typical item sample of legacy ScrapBook."""
with open(self.test_input_rdf, 'w', encoding='UTF-8') as fh:
fh.write("""\
<?xml version="1.0"?>
<RDF:RDF xmlns:NS1="http://amb.vis.ne.jp/mozilla/scrapbook-rdf#"
xmlns:NC="http://home.netscape.com/NC-rdf#"
xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<RDF:Description RDF:about="urn:scrapbook:item20200102030405"
NS1:id="20200102030405"
NS1:type=""
NS1:title="dummy title"
NS1:chars=""
NS1:comment="dummy comment __BR__ 2nd line"
NS1:icon="favicon.ico"
NS1:source="http://example.com/foo" />
</RDF:RDF>
""")
for info in sb2wsb.run(self.test_input, self.test_output):
pass
book = Host(self.test_output).books['']
book.load_meta_files()
self.assertDictEqual(book.meta['20200102030405'], {
'index': '20200102030405/index.html',
'title': 'dummy title',
'type': '',
'create': util.datetime_to_id(util.id_to_datetime_legacy('20200102030405')),
'modify': util.datetime_to_id(util.id_to_datetime_legacy('20200102030405')),
'source': 'http://example.com/foo',
'icon': 'favicon.ico',
'comment': 'dummy comment\n2nd line',
})
def test_get_tree_lock02(self, mock_get_lock):
"""With parameters"""
self.create_general_config()
host = Host(self.test_root)
book = Book(host)
book.get_tree_lock(timeout=10, stale=120, poll_interval=0.3, assume_acquired=True)
mock_get_lock.assert_called_once_with('tree',
timeout=10, stale=120, poll_interval=0.3, assume_acquired=True)
def test_persist03(self):
"""Lock file missing (or inaccessible)."""
lock_file = os.path.join(self.test_root, WSB_DIR, 'locks', '098f6bcd4621d373cade4e832627b4f6.lock')
host = Host(self.test_root)
with self.assertRaises(wsb_host.LockPersistOSError):
wsb_host.FileLock(host, 'test', persist='dummy')
def test_acquire06(self):
"""Occupied by a directory"""
lock = Host(self.test_root).get_lock('test')
os.makedirs(lock.file)
with self.assertRaises(wsb_host.LockGenerateError):
lock.acquire()
def test_backup04(self):
"""Pass if file not exist."""
test_file = os.path.join(self.test_wsbdir, 'icon', 'test.txt')
book = Book(Host(self.test_root))
book.init_backup()
book.backup(test_file)
self.assertFalse(os.path.lexists(os.path.join(book.backup_dir, WSB_DIR, 'icon', 'test.txt')))
def test_keep02(self):
"""Lock should be auto-extended until released."""
lock_file = os.path.join(self.test_root, WSB_DIR, 'locks', '098f6bcd4621d373cade4e832627b4f6.lock')
lock = Host(self.test_root).get_lock('test', stale=0.01)
with lock.acquire() as lh:
mtime = os.stat(lock_file).st_mtime
time.sleep(0.005)
self.assertGreater(os.stat(lock_file).st_mtime, mtime)
def test_get_static_file03(self):
"""Lookup static file from local themes"""
other_static = os.path.join(self.test_root, WSB_DIR, 'themes', 'default', 'static', 'test.txt')
os.makedirs(os.path.dirname(other_static))
with open(other_static, 'w') as fh:
pass
host = Host(self.test_root)
self.assertEqual(host.get_static_file('test.txt'), other_static)
def test_get_index_paths01(self):
self.create_general_config()
book = Book(Host(self.test_root))
self.assertEqual(book.get_index_paths('20200101000000000/index.html'),
['index.html'])
self.assertEqual(book.get_index_paths('20200101000000000.html'),
['20200101000000000.html'])
self.assertEqual(book.get_index_paths('20200101000000000.htz'),
['index.html'])
def test_release03(self):
"""File not exist"""
lock = Host(self.test_root).get_lock('test')
lock.acquire()
os.remove(lock.file)
with self.assertRaises(wsb_host.LockReleaseNotFoundError):
lock.release()
def test_acquire01(self):
"""Normal case"""
lock = Host(self.test_root).get_lock('test')
lock.acquire()
with open(lock.file) as fh:
self.assertTrue(fh.read(), lock.id)
self.assertTrue(lock.locked)
def test_release01(self):
"""Nnormal case"""
lock = Host(self.test_root).get_lock('test')
lock.acquire()
lock.release()
self.assertFalse(os.path.lexists(lock.file))
self.assertFalse(lock.locked)
def test_acquire05(self):
"""Unable to generate upper directory"""
lock = Host(self.test_root).get_lock('test')
with open(os.path.join(self.test_root, WSB_DIR, 'locks'), 'wb') as fh:
pass
with self.assertRaises(wsb_host.LockGenerateError):
lock.acquire()
def test_backup04(self):
"""Pass if file outside the host root."""
test_backup_dir = os.path.join(self.test_root, 'backup')
os.makedirs(test_backup_dir)
host = Host(self.test_root)
host.backup(__file__, test_backup_dir)
self.assertListEqual(os.listdir(test_backup_dir), [])
def test_get_static_file01(self):
"""Lookup static file from built-in themes"""
host = Host(self.test_root)
self.assertEqual(
os.path.normcase(host.get_static_file('index.css')),
os.path.normcase(
os.path.abspath(
os.path.join(wsb_host.__file__, '..', '..', 'themes',
'default', 'static', 'index.css'))))
def test_path03(self):
"""<input dir>/index.html should be indexed as single html page
"""
index_file = os.path.join(self.test_input, 'index.html')
with open(index_file, 'w', encoding='UTF-8') as fh:
fh.write("""\
<!DOCTYPE html>
<html
data-scrapbook-create="20200101000000000"
data-scrapbook-modify="20200101000000000"
data-scrapbook-source="http://example.com">
<head>
<meta charset="UTF-8">
<title>MyTitle 中文</title>
</head>
<body>
page content
</body>
</html>
""")
for info in file2wsb.run(self.test_input, self.test_output):
pass
book = Host(self.test_output).books['']
book.load_meta_files()
book.load_toc_files()
ids = list(book.meta.keys())
id_item = ids[0]
self.assertDictEqual(
book.meta, {
id_item: {
'title': 'MyTitle 中文',
'type': '',
'index': f'{id_item}.html',
'create': '20200101000000000',
'modify': '20200101000000000',
'source': 'http://example.com',
'icon': '',
'comment': '',
},
})
self.assertDictEqual(book.toc, {
'root': [
id_item,
],
})
self.assertEqual(
set(
glob.iglob(os.path.join(self.test_output, '**'),
recursive=True)),
{
os.path.join(self.test_output, ''),
os.path.join(self.test_output, f'{id_item}.html'),
})
def test_save_toc_files03(self):
self.create_general_config()
os.makedirs(os.path.join(self.test_root, 'tree'))
with open(os.path.join(self.test_root, 'tree', 'toc.js'),
'w',
encoding='UTF-8') as fh:
fh.write('dummy')
with open(os.path.join(self.test_root, 'tree', 'toc1.js'),
'w',
encoding='UTF-8') as fh:
fh.write('dummy1')
with open(os.path.join(self.test_root, 'tree', 'toc2.js'),
'w',
encoding='UTF-8') as fh:
fh.write('dummy2')
with open(os.path.join(self.test_root, 'tree', 'toc4.js'),
'w',
encoding='UTF-8') as fh:
fh.write('dummy4')
book = Book(Host(self.test_root))
book.toc = {
'root': [
'20200101000000000',
'20200101000000001',
'20200101000000002',
],
'20200101000000000': ['20200101000000003']
}
book.save_toc_files()
with open(os.path.join(self.test_root, 'tree', 'toc.js'),
encoding='UTF-8') as fh:
self.assertEqual(
fh.read(), """/**
* Feel free to edit this file, but keep data code valid JSON format.
*/
scrapbook.toc({
"root": [
"20200101000000000",
"20200101000000001",
"20200101000000002"
],
"20200101000000000": [
"20200101000000003"
]
})""")
self.assertFalse(
os.path.exists(os.path.join(self.test_root, 'tree', 'toc1.js')))
self.assertFalse(
os.path.exists(os.path.join(self.test_root, 'tree', 'toc2.js')))
self.assertFalse(
os.path.exists(os.path.join(self.test_root, 'tree', 'toc3.js')))
self.assertTrue(
os.path.exists(os.path.join(self.test_root, 'tree', 'toc4.js')))
def test_acquire03(self):
"""Already exists, timeout as acquire param"""
lock = Host(self.test_root).get_lock('test')
os.makedirs(os.path.dirname(lock.file))
with open(lock.file, 'w') as fh:
pass
with self.assertRaises(wsb_host.LockTimeoutError):
lock.acquire(timeout=0)
def test_get_index_paths04(self):
"""MAFF with no page"""
self.create_general_config()
os.makedirs(os.path.join(self.test_root, 'data'))
archive_file = os.path.join(self.test_root, 'data', '20200101000000000.maff')
with zipfile.ZipFile(archive_file, 'w') as zh:
pass
book = Book(Host(self.test_root))
self.assertEqual(book.get_index_paths('20200101000000000.maff'), [])
def test_get_index_paths02(self):
"""MAFF with single page"""
self.create_general_config()
os.makedirs(os.path.join(self.test_root, 'data'))
archive_file = os.path.join(self.test_root, 'data', '20200101000000000.maff')
with zipfile.ZipFile(archive_file, 'w') as zh:
zh.writestr('20200101000000000/index.html', """dummy""")
book = Book(Host(self.test_root))
self.assertEqual(book.get_index_paths('20200101000000000.maff'), ['20200101000000000/index.html'])