def test_acquire03(self):
"""Already exists, timeout as acquire param"""
lock = Host(self.test_root).get_lock('test')
os.makedirs(os.path.dirname(lock.file))
with open(lock.file, 'w') as fh:
pass
with self.assertRaises(wsb_host.LockTimeoutError):
lock.acquire(timeout=0)
def test_get_static_file03(self):
"""Lookup static file from local themes"""
other_static = os.path.join(self.test_root, WSB_DIR, 'themes',
'default', 'static', 'test.txt')
os.makedirs(os.path.dirname(other_static))
with open(other_static, 'w') as fh:
pass
host = Host(self.test_root)
self.assertEqual(host.get_static_file('test.txt'), other_static)
def test_toc(self):
with open(self.test_input_rdf, 'w', encoding='UTF-8') as fh:
fh.write("""\
<?xml version="1.0"?>
<RDF:RDF xmlns:NS1="http://amb.vis.ne.jp/mozilla/scrapbook-rdf#"
xmlns:NC="http://home.netscape.com/NC-rdf#"
xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<RDF:Seq RDF:about="urn:scrapbook:root">
<RDF:li RDF:resource="urn:scrapbook:item20200101000000"/>
<RDF:li RDF:resource="urn:scrapbook:item20200101000001"/>
<RDF:li RDF:resource="urn:scrapbook:item20200101000002"/>
<RDF:li RDF:resource="urn:scrapbook:item20200101000003"/>
<RDF:li RDF:resource="urn:scrapbook:item20200101000004"/>
</RDF:Seq>
<RDF:Seq RDF:about="urn:scrapbook:item20200101000001">
<RDF:li RDF:resource="urn:scrapbook:item20200101000005"/>
<RDF:li RDF:resource="urn:scrapbook:item20200101000006"/>
</RDF:Seq>
<RDF:Seq RDF:about="urn:scrapbook:item20200101000002">
<RDF:li RDF:resource="urn:scrapbook:item20200101000007"/>
<RDF:li RDF:resource="urn:scrapbook:item20200101000008"/>
</RDF:Seq>
<RDF:Seq RDF:about="urn:scrapbook:item20200101000003">
<RDF:li RDF:resource="urn:scrapbook:item20200101000009"/>
</RDF:Seq>
</RDF:RDF>
""")
for info in sb2wsb.run(self.test_input, self.test_output):
pass
book = Host(self.test_output).books['']
book.load_toc_files()
self.assertEqual(
book.toc, {
'root': [
'20200101000000',
'20200101000001',
'20200101000002',
'20200101000003',
'20200101000004',
],
'20200101000001': [
'20200101000005',
'20200101000006',
],
'20200101000002': [
'20200101000007',
'20200101000008',
],
'20200101000003': [
'20200101000009',
],
})
def test_acquire_with(self):
"""Lock should be released after an with statement."""
lock = Host(self.test_root).get_lock('test')
with lock.acquire() as lh:
self.assertTrue(os.path.isfile(lock.file))
self.assertTrue(lock.locked)
self.assertEqual(lh, lock)
self.assertFalse(os.path.exists(lock.file))
self.assertFalse(lock.locked)
def test_meta_basic01(self):
"""A typical item sample of legacy ScrapBook X."""
with open(self.test_input_rdf, 'w', encoding='UTF-8') as fh:
fh.write("""\
<?xml version="1.0"?>
<RDF:RDF xmlns:NS1="http://amb.vis.ne.jp/mozilla/scrapbook-rdf#"
xmlns:NC="http://home.netscape.com/NC-rdf#"
xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<RDF:Description RDF:about="urn:scrapbook:item20200102030405"
NS1:id="20200102030405"
NS1:create="20200102030406"
NS1:modify="20200102030407"
NS1:type=""
NS1:title="dummy title"
NS1:chars="UTF-8"
NS1:icon="favicon.ico"
NS1:source="http://example.com/foo"
NS1:comment="dummy comment __BR__ 2nd line"
NS1:lock="true" />
</RDF:RDF>
""")
for info in sb2wsb.run(self.test_input, self.test_output):
pass
book = Host(self.test_output).books['']
book.load_meta_files()
self.assertDictEqual(
book.meta['20200102030405'], {
'index':
'20200102030405/index.html',
'title':
'dummy title',
'type':
'',
'create':
util.datetime_to_id(
util.id_to_datetime_legacy('20200102030406')),
'modify':
util.datetime_to_id(
util.id_to_datetime_legacy('20200102030407')),
'source':
'http://example.com/foo',
'icon':
'favicon.ico',
'comment':
'dummy comment\n2nd line',
'charset':
'UTF-8',
'locked':
True,
})
def test_backup03(self):
"""Pass if file not exist."""
test_backup_dir = os.path.join(self.test_root, 'backup')
os.makedirs(test_backup_dir)
test_file = os.path.join(self.test_wsbdir, 'icon', 'nonexist.txt')
host = Host(self.test_root)
host.backup(test_file, test_backup_dir)
self.assertFalse(
os.path.lexists(
os.path.join(test_backup_dir, WSB_DIR, 'icon',
'nonexist.txt')))
def test_acquire04(self):
"""Stale lock should be regenerated"""
lock = Host(self.test_root).get_lock('test', timeout=1, stale=0)
os.makedirs(os.path.dirname(lock.file))
with open(lock.file, 'w') as fh:
fh.write('oldid')
lock.acquire()
with open(lock.file) as fh:
self.assertTrue(fh.read(), lock.id)
self.assertNotEqual(lock.id, 'oldid')
self.assertTrue(lock.locked)
def test_get_lock02(self, mock_filelock):
"""With parameters"""
host = Host(self.test_root)
host.get_lock('test',
timeout=10,
stale=120,
poll_interval=0.3,
assume_acquired=True)
mock_filelock.assert_called_once_with(host,
'test',
timeout=10,
stale=120,
poll_interval=0.3,
assume_acquired=True)
def test_backup05(self):
"""Test base param."""
test_backup_dir = os.path.join(self.test_root, 'backup')
os.makedirs(test_backup_dir)
test_base_dir = os.path.join(test_root, 'backup')
host = Host(self.test_root)
host.backup(os.path.join(test_base_dir, 'test.txt'),
test_backup_dir,
base=test_base_dir)
with open(os.path.join(test_backup_dir, 'test.txt'),
encoding='UTF-8') as fh:
self.assertEqual(fh.read(), 'ABC123')
def test_backup01(self):
"""A common case."""
test_backup_dir = os.path.join(self.test_root, 'backup')
os.makedirs(test_backup_dir)
test_file = os.path.join(self.test_root, 'tree', 'meta.js')
os.makedirs(os.path.dirname(test_file))
with open(test_file, 'w', encoding='UTF-8') as fh:
fh.write('abc')
host = Host(self.test_root)
host.backup(test_file, test_backup_dir)
with open(os.path.join(test_backup_dir, 'tree', 'meta.js'),
encoding='UTF-8') as fh:
self.assertEqual(fh.read(), 'abc')
def test_meta_basic04(self):
"""Should work correctly for another NS name
(seen in some scrapbook.rdf files)
"""
with open(self.test_input_rdf, 'w', encoding='UTF-8') as fh:
fh.write("""\
<?xml version="1.0"?>
<RDF:RDF xmlns:NS2="http://amb.vis.ne.jp/mozilla/scrapbook-rdf#"
xmlns:NC="http://home.netscape.com/NC-rdf#"
xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<RDF:Description RDF:about="urn:scrapbook:item20200102030405"
NS2:id="20200102030405"
NS2:type=""
NS2:title="title"
NS2:comment="comment"
NS2:icon="favicon.ico"
NS2:source="http://example.com" />
</RDF:RDF>
""")
for info in sb2wsb.run(self.test_input, self.test_output):
pass
book = Host(self.test_output).books['']
book.load_meta_files()
self.assertDictEqual(
book.meta['20200102030405'], {
'index':
'20200102030405/index.html',
'type':
'',
'create':
util.datetime_to_id(
util.id_to_datetime_legacy('20200102030405')),
'modify':
util.datetime_to_id(
util.id_to_datetime_legacy('20200102030405')),
'title':
'title',
'comment':
'comment',
'icon':
'favicon.ico',
'source':
'http://example.com',
})
def test_save_meta_files01(self):
self.create_general_config()
book = Book(Host(self.test_root))
book.meta = {
'20200101000000000': {
'title': 'Dummy 1 中文'
},
'20200101000000001': {
'title': 'Dummy 2 中文'
},
}
book.save_meta_files()
with open(os.path.join(self.test_root, 'tree', 'meta.js'),
encoding='UTF-8') as fh:
self.assertEqual(
fh.read(), """/**
* Feel free to edit this file, but keep data code valid JSON format.
*/
scrapbook.meta({
"20200101000000000": {
"title": "Dummy 1 中文"
},
"20200101000000001": {
"title": "Dummy 2 中文"
}
})""")
def test_load_tree_file01(self):
"""Test normal loading"""
self.create_general_config()
with open(os.path.join(self.test_root, 'meta.js'), 'w', encoding='UTF-8') as f:
f.write("""/**
* This file is generated by WebScrapBook and is not intended to be edited.
*/
scrapbook.meta({
"20200101000000000": {
"index": "20200101000000000/index.html",
"title": "Dummy",
"type": "",
"create": "20200101000000000",
"modify": "20200101000000000"
}
})""")
book = Book(Host(self.test_root))
self.assertEqual(
book.load_tree_file(os.path.join(self.test_root, 'meta.js')), {
'20200101000000000': {
'index': '20200101000000000/index.html',
'title': 'Dummy',
'type': '',
'create': '20200101000000000',
'modify': '20200101000000000',
},
})
def test_save_fulltext_files01(self):
self.create_general_config()
book = Book(Host(self.test_root))
book.fulltext = {
"20200101000000000": {
'index.html': {
'content': 'dummy text 1 中文',
}
},
"20200101000000001": {
'index.html': {
'content': 'dummy text 2 中文',
}
},
}
book.save_fulltext_files()
with open(os.path.join(self.test_root, 'tree', 'fulltext.js'), encoding='UTF-8') as fh:
self.assertEqual(fh.read(), """/**
* This file is generated by WebScrapBook and is not intended to be edited.
*/
scrapbook.fulltext({
"20200101000000000": {
"index.html": {
"content": "dummy text 1 中文"
}
},
"20200101000000001": {
"index.html": {
"content": "dummy text 2 中文"
}
}
})""")
def test_init03(self):
"""Check modified path"""
with open(self.test_config, 'w', encoding='UTF-8') as f:
f.write("""[app]
root = public
[book ""]
name = scrapbook
top_dir = sb
data_dir = data
tree_dir = tree
index = tree/map.html
no_tree = false
""")
host = Host(self.test_root)
book = Book(host)
self.assertEqual(book.host, host)
self.assertEqual(book.id, '')
self.assertEqual(book.name, 'scrapbook')
self.assertEqual(book.root, self.test_root)
self.assertEqual(book.top_dir, os.path.join(self.test_root, 'public', 'sb'))
self.assertEqual(book.data_dir, os.path.join(self.test_root, 'public', 'sb', 'data'))
self.assertEqual(book.tree_dir, os.path.join(self.test_root, 'public', 'sb', 'tree'))
self.assertFalse(book.no_tree)
def test_backup05(self):
"""Pass if file outside the host root."""
book = Book(Host(self.test_root))
book.init_backup()
book.backup(__file__)
self.assertListEqual(os.listdir(self.test_wsbdir), [])
def test_save_toc_files02(self):
self.create_general_config()
book = Book(Host(self.test_root))
book.toc = {
'root': [
'20200101000000000',
'20200101000000001',
'20200101000000002',
'20200101000000003',
'20200101000000004',
],
'20200101000000001': [
'20200101000000011'
],
'20200101000000002': [
'20200101000000021'
],
'20200101000000003': [
'20200101000000031',
'20200101000000032'
],
}
book.save_toc_files()
with open(os.path.join(self.test_root, 'tree', 'toc.js'), encoding='UTF-8') as fh:
self.assertEqual(fh.read(), """/**
* Feel free to edit this file, but keep data code valid JSON format.
*/
scrapbook.toc({
"root": [
"20200101000000000",
"20200101000000001",
"20200101000000002",
"20200101000000003",
"20200101000000004"
]
})""")
with open(os.path.join(self.test_root, 'tree', 'toc1.js'), encoding='UTF-8') as fh:
self.assertEqual(fh.read(), """/**
* Feel free to edit this file, but keep data code valid JSON format.
*/
scrapbook.toc({
"20200101000000001": [
"20200101000000011"
],
"20200101000000002": [
"20200101000000021"
]
})""")
with open(os.path.join(self.test_root, 'tree', 'toc2.js'), encoding='UTF-8') as fh:
self.assertEqual(fh.read(), """/**
* Feel free to edit this file, but keep data code valid JSON format.
*/
scrapbook.toc({
"20200101000000003": [
"20200101000000031",
"20200101000000032"
]
})""")
def test_get_tree_file(self):
self.create_general_config()
book = Book(Host(self.test_root))
self.assertEqual(book.get_tree_file('meta'),
os.path.join(self.test_root, 'tree', 'meta.js'))
self.assertEqual(book.get_tree_file('toc', 1),
os.path.join(self.test_root, 'tree', 'toc1.js'))
def test_backup07(self):
"""A common case."""
test_file = os.path.join(self.test_root, 'tree', 'meta.js')
os.makedirs(os.path.dirname(test_file))
with open(test_file, 'w', encoding='UTF-8') as fh:
fh.write('abc')
host = Host(self.test_root)
host.backup(test_file)
backup_dirname = os.listdir(os.path.join(self.test_wsbdir,
'backup'))[0]
self.assertRegex(backup_dirname, r'^\d{17}$')
with open(os.path.join(self.test_wsbdir, 'backup', backup_dirname,
'tree', 'meta.js'),
encoding='UTF-8') as fh:
self.assertEqual(fh.read(), 'abc')
def test_persist03(self):
"""Lock file missing (or inaccessible)."""
lock_file = os.path.join(self.test_root, WSB_DIR, 'locks', '098f6bcd4621d373cade4e832627b4f6.lock')
host = Host(self.test_root)
with self.assertRaises(wsb_host.LockPersistOSError):
wsb_host.FileLock(host, 'test', persist='dummy')
def test_get_tree_lock02(self, mock_get_lock):
"""With parameters"""
self.create_general_config()
host = Host(self.test_root)
book = Book(host)
book.get_tree_lock(timeout=10, stale=120, poll_interval=0.3, assume_acquired=True)
mock_get_lock.assert_called_once_with('tree',
timeout=10, stale=120, poll_interval=0.3, assume_acquired=True)
def test_backup04(self):
"""Pass if file not exist."""
test_file = os.path.join(self.test_wsbdir, 'icon', 'test.txt')
book = Book(Host(self.test_root))
book.init_backup()
book.backup(test_file)
self.assertFalse(os.path.lexists(os.path.join(book.backup_dir, WSB_DIR, 'icon', 'test.txt')))
def test_get_index_paths01(self):
self.create_general_config()
book = Book(Host(self.test_root))
self.assertEqual(book.get_index_paths('20200101000000000/index.html'),
['index.html'])
self.assertEqual(book.get_index_paths('20200101000000000.html'),
['20200101000000000.html'])
self.assertEqual(book.get_index_paths('20200101000000000.htz'),
['index.html'])
def test_keep01(self):
"""Lock should be auto-extended until released."""
lock_file = os.path.join(self.test_root, WSB_DIR, 'locks', '098f6bcd4621d373cade4e832627b4f6.lock')
lock = Host(self.test_root).get_lock('test', stale=0.01)
lock.acquire()
lock.keep()
mtime = os.stat(lock_file).st_mtime
time.sleep(0.005)
self.assertGreater(os.stat(lock_file).st_mtime, mtime)
lock.release()
def test_save_toc_files03(self):
self.create_general_config()
os.makedirs(os.path.join(self.test_root, 'tree'))
with open(os.path.join(self.test_root, 'tree', 'toc.js'),
'w',
encoding='UTF-8') as fh:
fh.write('dummy')
with open(os.path.join(self.test_root, 'tree', 'toc1.js'),
'w',
encoding='UTF-8') as fh:
fh.write('dummy1')
with open(os.path.join(self.test_root, 'tree', 'toc2.js'),
'w',
encoding='UTF-8') as fh:
fh.write('dummy2')
with open(os.path.join(self.test_root, 'tree', 'toc4.js'),
'w',
encoding='UTF-8') as fh:
fh.write('dummy4')
book = Book(Host(self.test_root))
book.toc = {
'root': [
'20200101000000000',
'20200101000000001',
'20200101000000002',
],
'20200101000000000': ['20200101000000003']
}
book.save_toc_files()
with open(os.path.join(self.test_root, 'tree', 'toc.js'),
encoding='UTF-8') as fh:
self.assertEqual(
fh.read(), """/**
* Feel free to edit this file, but keep data code valid JSON format.
*/
scrapbook.toc({
"root": [
"20200101000000000",
"20200101000000001",
"20200101000000002"
],
"20200101000000000": [
"20200101000000003"
]
})""")
self.assertFalse(
os.path.exists(os.path.join(self.test_root, 'tree', 'toc1.js')))
self.assertFalse(
os.path.exists(os.path.join(self.test_root, 'tree', 'toc2.js')))
self.assertFalse(
os.path.exists(os.path.join(self.test_root, 'tree', 'toc3.js')))
self.assertTrue(
os.path.exists(os.path.join(self.test_root, 'tree', 'toc4.js')))
def test_get_index_paths04(self):
"""MAFF with no page"""
self.create_general_config()
os.makedirs(os.path.join(self.test_root, 'data'))
archive_file = os.path.join(self.test_root, 'data', '20200101000000000.maff')
with zipfile.ZipFile(archive_file, 'w') as zh:
pass
book = Book(Host(self.test_root))
self.assertEqual(book.get_index_paths('20200101000000000.maff'), [])
def test_get_index_paths02(self):
"""MAFF with single page"""
self.create_general_config()
os.makedirs(os.path.join(self.test_root, 'data'))
archive_file = os.path.join(self.test_root, 'data', '20200101000000000.maff')
with zipfile.ZipFile(archive_file, 'w') as zh:
zh.writestr('20200101000000000/index.html', """dummy""")
book = Book(Host(self.test_root))
self.assertEqual(book.get_index_paths('20200101000000000.maff'), ['20200101000000000/index.html'])
def test_meta_icon05(self):
"""Handle special chars."""
with open(self.test_input_rdf, 'w', encoding='UTF-8') as fh:
fh.write("""\
<?xml version="1.0"?>
<RDF:RDF xmlns:NS1="http://amb.vis.ne.jp/mozilla/scrapbook-rdf#"
xmlns:NC="http://home.netscape.com/NC-rdf#"
xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<RDF:Description RDF:about="urn:scrapbook:item20200102030405"
NS1:icon="resource://scrapbook/%E4%B8%AD%E6%96%87%25%23abc.png" />
</RDF:RDF>
""")
for info in sb2wsb.run(self.test_input, self.test_output):
pass
book = Host(self.test_output).books['']
book.load_meta_files()
self.assertEqual(book.meta['20200102030405']['icon'], '../../%E4%B8%AD%E6%96%87%25%23abc.png')
def test_meta_icon01(self):
"""Resolve resource://scrapbook/data/<self-id>/..."""
with open(self.test_input_rdf, 'w', encoding='UTF-8') as fh:
fh.write("""\
<?xml version="1.0"?>
<RDF:RDF xmlns:NS1="http://amb.vis.ne.jp/mozilla/scrapbook-rdf#"
xmlns:NC="http://home.netscape.com/NC-rdf#"
xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<RDF:Description RDF:about="urn:scrapbook:item20200102030405"
NS1:icon="resource://scrapbook/data/20200102030405/favicon.ico" />
</RDF:RDF>
""")
for info in sb2wsb.run(self.test_input, self.test_output):
pass
book = Host(self.test_output).books['']
book.load_meta_files()
self.assertEqual(book.meta['20200102030405']['icon'], 'favicon.ico')
def test_persist02(self):
"""Wrong ID."""
lock_file = os.path.join(self.test_root, WSB_DIR, 'locks', '098f6bcd4621d373cade4e832627b4f6.lock')
os.makedirs(os.path.dirname(lock_file))
with open(lock_file, 'w', encoding='UTF-8') as fh:
fh.write('oldid')
host = Host(self.test_root)
with self.assertRaises(wsb_host.LockPersistUnmatchError):
wsb_host.FileLock(host, 'test', persist='dummy')
