def test_get_tree_file(self): self.create_general_config() book = Book(Host(self.test_root)) self.assertEqual(book.get_tree_file('meta'), os.path.join(self.test_root, 'tree', 'meta.js')) self.assertEqual(book.get_tree_file('toc', 1), os.path.join(self.test_root, 'tree', 'toc1.js'))
def test_load_tree_file01(self): """Test normal loading""" self.create_general_config() with open(os.path.join(self.test_root, 'meta.js'), 'w', encoding='UTF-8') as f: f.write("""/** * This file is generated by WebScrapBook and is not intended to be edited. */ scrapbook.meta({ "20200101000000000": { "index": "20200101000000000/index.html", "title": "Dummy", "type": "", "create": "20200101000000000", "modify": "20200101000000000" } })""") book = Book(Host(self.test_root)) self.assertEqual( book.load_tree_file(os.path.join(self.test_root, 'meta.js')), { '20200101000000000': { 'index': '20200101000000000/index.html', 'title': 'Dummy', 'type': '', 'create': '20200101000000000', 'modify': '20200101000000000', }, })
def test_save_toc_files02(self): self.create_general_config() book = Book(Host(self.test_root)) book.toc = { 'root': [ '20200101000000000', '20200101000000001', '20200101000000002', '20200101000000003', '20200101000000004', ], '20200101000000001': [ '20200101000000011' ], '20200101000000002': [ '20200101000000021' ], '20200101000000003': [ '20200101000000031', '20200101000000032' ], } book.save_toc_files() with open(os.path.join(self.test_root, 'tree', 'toc.js'), encoding='UTF-8') as fh: self.assertEqual(fh.read(), """/** * Feel free to edit this file, but keep data code valid JSON format. */ scrapbook.toc({ "root": [ "20200101000000000", "20200101000000001", "20200101000000002", "20200101000000003", "20200101000000004" ] })""") with open(os.path.join(self.test_root, 'tree', 'toc1.js'), encoding='UTF-8') as fh: self.assertEqual(fh.read(), """/** * Feel free to edit this file, but keep data code valid JSON format. */ scrapbook.toc({ "20200101000000001": [ "20200101000000011" ], "20200101000000002": [ "20200101000000021" ] })""") with open(os.path.join(self.test_root, 'tree', 'toc2.js'), encoding='UTF-8') as fh: self.assertEqual(fh.read(), """/** * Feel free to edit this file, but keep data code valid JSON format. */ scrapbook.toc({ "20200101000000003": [ "20200101000000031", "20200101000000032" ] })""")
def test_save_meta_files01(self): self.create_general_config() book = Book(Host(self.test_root)) book.meta = { '20200101000000000': { 'title': 'Dummy 1 中文' }, '20200101000000001': { 'title': 'Dummy 2 中文' }, } book.save_meta_files() with open(os.path.join(self.test_root, 'tree', 'meta.js'), encoding='UTF-8') as fh: self.assertEqual( fh.read(), """/** * Feel free to edit this file, but keep data code valid JSON format. */ scrapbook.meta({ "20200101000000000": { "title": "Dummy 1 中文" }, "20200101000000001": { "title": "Dummy 2 中文" } })""")
def test_save_fulltext_files01(self): self.create_general_config() book = Book(Host(self.test_root)) book.fulltext = { "20200101000000000": { 'index.html': { 'content': 'dummy text 1 中文', } }, "20200101000000001": { 'index.html': { 'content': 'dummy text 2 中文', } }, } book.save_fulltext_files() with open(os.path.join(self.test_root, 'tree', 'fulltext.js'), encoding='UTF-8') as fh: self.assertEqual(fh.read(), """/** * This file is generated by WebScrapBook and is not intended to be edited. */ scrapbook.fulltext({ "20200101000000000": { "index.html": { "content": "dummy text 1 中文" } }, "20200101000000001": { "index.html": { "content": "dummy text 2 中文" } } })""")
def test_get_tree_lock02(self, mock_get_lock): """With parameters""" self.create_general_config() host = Host(self.test_root) book = Book(host) book.get_tree_lock(timeout=10, stale=120, poll_interval=0.3, assume_acquired=True) mock_get_lock.assert_called_once_with('tree', timeout=10, stale=120, poll_interval=0.3, assume_acquired=True)
def test_get_index_paths01(self): self.create_general_config() book = Book(Host(self.test_root)) self.assertEqual(book.get_index_paths('20200101000000000/index.html'), ['index.html']) self.assertEqual(book.get_index_paths('20200101000000000.html'), ['20200101000000000.html']) self.assertEqual(book.get_index_paths('20200101000000000.htz'), ['index.html'])
def test_save_toc_files03(self): self.create_general_config() os.makedirs(os.path.join(self.test_root, 'tree')) with open(os.path.join(self.test_root, 'tree', 'toc.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy') with open(os.path.join(self.test_root, 'tree', 'toc1.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy1') with open(os.path.join(self.test_root, 'tree', 'toc2.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy2') with open(os.path.join(self.test_root, 'tree', 'toc4.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy4') book = Book(Host(self.test_root)) book.toc = { 'root': [ '20200101000000000', '20200101000000001', '20200101000000002', ], '20200101000000000': ['20200101000000003'] } book.save_toc_files() with open(os.path.join(self.test_root, 'tree', 'toc.js'), encoding='UTF-8') as fh: self.assertEqual( fh.read(), """/** * Feel free to edit this file, but keep data code valid JSON format. */ scrapbook.toc({ "root": [ "20200101000000000", "20200101000000001", "20200101000000002" ], "20200101000000000": [ "20200101000000003" ] })""") self.assertFalse( os.path.exists(os.path.join(self.test_root, 'tree', 'toc1.js'))) self.assertFalse( os.path.exists(os.path.join(self.test_root, 'tree', 'toc2.js'))) self.assertFalse( os.path.exists(os.path.join(self.test_root, 'tree', 'toc3.js'))) self.assertTrue( os.path.exists(os.path.join(self.test_root, 'tree', 'toc4.js')))
def test_get_index_paths02(self): """MAFF with single page""" self.create_general_config() os.makedirs(os.path.join(self.test_root, 'data')) archive_file = os.path.join(self.test_root, 'data', '20200101000000000.maff') with zipfile.ZipFile(archive_file, 'w') as zh: zh.writestr('20200101000000000/index.html', """dummy""") book = Book(Host(self.test_root)) self.assertEqual(book.get_index_paths('20200101000000000.maff'), ['20200101000000000/index.html'])
def test_get_index_paths04(self): """MAFF with no page""" self.create_general_config() os.makedirs(os.path.join(self.test_root, 'data')) archive_file = os.path.join(self.test_root, 'data', '20200101000000000.maff') with zipfile.ZipFile(archive_file, 'w') as zh: pass book = Book(Host(self.test_root)) self.assertEqual(book.get_index_paths('20200101000000000.maff'), [])
def test_save_meta_files03(self): self.create_general_config() os.makedirs(os.path.join(self.test_root, 'tree')) with open(os.path.join(self.test_root, 'tree', 'meta.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy') with open(os.path.join(self.test_root, 'tree', 'meta1.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy1') with open(os.path.join(self.test_root, 'tree', 'meta2.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy2') with open(os.path.join(self.test_root, 'tree', 'meta3.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy3') book = Book(Host(self.test_root)) book.meta = { '20200101000000000': { 'title': 'Dummy 1 中文' }, '20200101000000001': { 'title': 'Dummy 2 中文' }, } book.save_meta_files() with open(os.path.join(self.test_root, 'tree', 'meta.js'), encoding='UTF-8') as fh: self.assertEqual( fh.read(), """/** * Feel free to edit this file, but keep data code valid JSON format. */ scrapbook.meta({ "20200101000000000": { "title": "Dummy 1 中文" }, "20200101000000001": { "title": "Dummy 2 中文" } })""") self.assertFalse( os.path.exists(os.path.join(self.test_root, 'tree', 'meta1.js'))) self.assertFalse( os.path.exists(os.path.join(self.test_root, 'tree', 'meta2.js'))) self.assertFalse( os.path.exists(os.path.join(self.test_root, 'tree', 'meta3.js'))) self.assertFalse( os.path.exists(os.path.join(self.test_root, 'tree', 'meta4.js')))
def test_load_tree_file06(self): """Test empty file should not error out.""" self.create_general_config() with open(os.path.join(self.test_root, 'meta.js'), 'w', encoding='UTF-8') as f: f.write('') book = Book(Host(self.test_root)) self.assertEqual( book.load_tree_file(os.path.join(self.test_root, 'meta.js')), {})
def test_backup03(self): """Pass if backup_dir not set.""" test_file = os.path.join(self.test_wsbdir, 'icon', 'test.txt') os.makedirs(os.path.dirname(test_file)) with open(test_file, 'w', encoding='UTF-8') as fh: fh.write('abc') test_stat = os.stat(test_file) book = Book(Host(self.test_root)) book.backup(test_file) self.assertListEqual(os.listdir(self.test_wsbdir), ['icon'])
def test_load_note_file03(self): """Return original text if malformatted.""" test_file = os.path.join(self.test_root, 'index.html') html = """\ <html><head><meta http-equiv="Content-Type" content="text/html;Charset=UTF-8"></head><body> Note content 2nd line 3rd line </body></html>""" with open(test_file, 'w', encoding='UTF-8') as f: f.write(html) book = Book(Host(self.test_root)) content = book.load_note_file(test_file) self.assertEqual(content, html)
def test_iter_tree_files02(self): """Break since nonexisting index""" self.create_general_config() os.makedirs(os.path.join(self.test_root, 'tree')) with open(os.path.join(self.test_root, 'tree', 'meta.js'), 'w', encoding='UTF-8') as f: pass with open(os.path.join(self.test_root, 'tree', 'meta1.js'), 'w', encoding='UTF-8') as f: pass with open(os.path.join(self.test_root, 'tree', 'meta3.js'), 'w', encoding='UTF-8') as f: pass book = Book(Host(self.test_root)) self.assertEqual(list(book.iter_tree_files('meta')), [ os.path.join(self.test_root, 'tree', 'meta.js'), os.path.join(self.test_root, 'tree', 'meta1.js'), ])
def test_init03(self): """Check modified path""" with open(self.test_config, 'w', encoding='UTF-8') as f: f.write("""[app] root = public [book ""] name = scrapbook top_dir = sb data_dir = data tree_dir = tree index = tree/map.html no_tree = false """) host = Host(self.test_root) book = Book(host) self.assertEqual(book.host, host) self.assertEqual(book.id, '') self.assertEqual(book.name, 'scrapbook') self.assertEqual(book.root, self.test_root) self.assertEqual(book.top_dir, os.path.join(self.test_root, 'public', 'sb')) self.assertEqual(book.data_dir, os.path.join(self.test_root, 'public', 'sb', 'data')) self.assertEqual(book.tree_dir, os.path.join(self.test_root, 'public', 'sb', 'tree')) self.assertFalse(book.no_tree)
def test_load_note_file02(self): """Test for common legacy note file wrapper.""" test_file = os.path.join(self.test_root, 'index.html') with open(test_file, 'w', encoding='UTF-8') as f: f.write("""\ <html><head><meta http-equiv="Content-Type" content="text/html;Charset=UTF-8"></head><body><pre> Note content 2nd line 3rd line </pre></body></html>""") book = Book(Host(self.test_root)) content = book.load_note_file(test_file) self.assertEqual(content, """\ Note content 2nd line 3rd line""")
def test_load_note_file01(self): """Test for common note file wrapper.""" test_file = os.path.join(self.test_root, 'index.html') with open(test_file, 'w', encoding='UTF-8') as f: f.write("""\ <!DOCTYPE html><html><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width"><style>pre{white-space: pre-wrap; overflow-wrap: break-word;}</style></head><body><pre> Note content 2nd line 3rd line </pre></body></html>""") book = Book(Host(self.test_root)) content = book.load_note_file(test_file) self.assertEqual(content, """\ Note content 2nd line 3rd line""")
def test_load_tree_file04(self): """Test malformed wrapping""" self.create_general_config() with open(os.path.join(self.test_root, 'meta.js'), 'w', encoding='UTF-8') as f: f.write("""({ "20200101000000000": { "index": "20200101000000000/index.html", "title": "Dummy", "type": "", "create": "20200101000000000", "modify": "20200101000000000" } })""") book = Book(Host(self.test_root)) with self.assertRaises(wsb_book.TreeFileMalformedWrappingError): book.load_tree_file(os.path.join(self.test_root, 'meta.js'))
def test_load_tree_file05(self): """Test malformed JSON""" self.create_general_config() with open(os.path.join(self.test_root, 'meta.js'), 'w', encoding='UTF-8') as f: f.write(""" scrapbook.meta({ '20200101000000000': { index: '20200101000000000/index.html', title: 'Dummy', type: '', create: '20200101000000000', modify: '20200101000000000' } })""") book = Book(Host(self.test_root)) with self.assertRaises(wsb_book.TreeFileMalformedJsonError): book.load_tree_file(os.path.join(self.test_root, 'meta.js'))
def test_backup05(self): """Pass if file outside the host root.""" book = Book(Host(self.test_root)) book.init_backup() book.backup(__file__) self.assertListEqual(os.listdir(self.test_wsbdir), [])
def test_save_note_file01(self): """Test saving. Enforce LF linefeeds.""" test_file = os.path.join(self.test_root, 'index.html') book = Book(Host(self.test_root)) book.save_note_file(test_file, """\ Note content 2nd line 3rd line""") with open(test_file, encoding='UTF-8', newline='') as fh: self.assertEqual( fh.read(), """\ <!DOCTYPE html><html><head>\ <meta charset="UTF-8">\ <meta name="viewport" content="width=device-width">\ <style>pre { white-space: pre-wrap; overflow-wrap: break-word; }</style>\ </head><body><pre> Note content 2nd line 3rd line </pre></body></html>""")
def test_backup04(self): """Pass if file not exist.""" test_file = os.path.join(self.test_wsbdir, 'icon', 'test.txt') book = Book(Host(self.test_root)) book.init_backup() book.backup(test_file) self.assertFalse(os.path.lexists(os.path.join(book.backup_dir, WSB_DIR, 'icon', 'test.txt')))
def test_init_backup(self): book = Book(Host(self.test_root)) book.init_backup(True) self.assertRegex( book.backup_dir, r'^' + re.escape(os.path.join(self.test_root, WSB_DIR, 'backup', '')) + r'\d{17}$', ) ts = util.datetime_to_id() book.init_backup(ts) self.assertEqual( book.backup_dir, os.path.join(self.test_root, WSB_DIR, 'backup', ts), ) book.init_backup(False) self.assertIsNone(book.backup_dir)
def test_backup01(self): """A common case.""" test_file = os.path.join(self.test_root, 'tree', 'meta.js') os.makedirs(os.path.dirname(test_file)) with open(test_file, 'w', encoding='UTF-8') as fh: fh.write('abc') book = Book(Host(self.test_root)) book.init_backup() book.backup(test_file) with open(os.path.join(book.backup_dir, 'tree', 'meta.js'), encoding='UTF-8') as fh: self.assertEqual(fh.read(), 'abc')
def test_backup02(self): """A common directory case.""" test_dir = os.path.join(self.test_root, 'tree') os.makedirs(test_dir) with open(os.path.join(test_dir, 'meta.js'), 'w', encoding='UTF-8') as fh: fh.write('abc') with open(os.path.join(test_dir, 'toc.js'), 'w', encoding='UTF-8') as fh: fh.write('def') book = Book(Host(self.test_root)) book.init_backup() book.backup(test_dir) with open(os.path.join(book.backup_dir, 'tree', 'meta.js'), encoding='UTF-8') as fh: self.assertEqual(fh.read(), 'abc') with open(os.path.join(book.backup_dir, 'tree', 'toc.js'), encoding='UTF-8') as fh: self.assertEqual(fh.read(), 'def')
def test_init02(self): """Check book_id param""" with open(self.test_config, 'w', encoding='UTF-8') as f: f.write("""[book "book1"] name = scrapbook1 top_dir = data_dir = tree_dir = .wsb/tree index = .wsb/tree/map.html no_tree = false """) host = Host(self.test_root) book = Book(host, 'book1') self.assertEqual(book.host, host) self.assertEqual(book.id, 'book1') self.assertEqual(book.name, 'scrapbook1') self.assertEqual(book.root, self.test_root) self.assertEqual(book.top_dir, self.test_root) self.assertEqual(book.data_dir, self.test_root) self.assertEqual(book.tree_dir, os.path.join(self.test_root, '.wsb', 'tree')) self.assertFalse(book.no_tree)
def test_get_lock01(self, mock_filelock): self.create_general_config() host = Host(self.test_root) book = Book(host) book.get_lock('test') mock_filelock.assert_called_once_with(host, 'book--test')
def test_load_fulltext_files03(self, mock_func): book = Book(Host(self.test_root)) book.fulltext = {} book.load_fulltext_files(refresh=True) mock_func.assert_called_once_with('fulltext')
def test_save_fulltext_files03(self): self.create_general_config() os.makedirs(os.path.join(self.test_root, 'tree')) with open(os.path.join(self.test_root, 'tree', 'fulltext.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy') with open(os.path.join(self.test_root, 'tree', 'fulltext1.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy1') with open(os.path.join(self.test_root, 'tree', 'fulltext2.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy2') with open(os.path.join(self.test_root, 'tree', 'fulltext3.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy3') with open(os.path.join(self.test_root, 'tree', 'fulltext4.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy4') with open(os.path.join(self.test_root, 'tree', 'fulltext6.js'), 'w', encoding='UTF-8') as fh: fh.write('dummy6') book = Book(Host(self.test_root)) book.fulltext = { "20200101000000000": { 'index.html': { 'content': 'dummy text 1 中文', }, 'frame.html': { 'content': 'frame page content', }, }, "20200101000000001": { 'index.html': { 'content': 'dummy text 2 中文', }, }, } book.save_fulltext_files() with open(os.path.join(self.test_root, 'tree', 'fulltext.js'), encoding='UTF-8') as fh: self.assertEqual(fh.read(), """/** * This file is generated by WebScrapBook and is not intended to be edited. */ scrapbook.fulltext({ "20200101000000000": { "index.html": { "content": "dummy text 1 中文" }, "frame.html": { "content": "frame page content" } } })""") with open(os.path.join(self.test_root, 'tree', 'fulltext1.js'), encoding='UTF-8') as fh: self.assertEqual(fh.read(), """/** * This file is generated by WebScrapBook and is not intended to be edited. */ scrapbook.fulltext({ "20200101000000001": { "index.html": { "content": "dummy text 2 中文" } } })""") self.assertFalse(os.path.exists(os.path.join(self.test_root, 'tree', 'fulltext2.js'))) self.assertFalse(os.path.exists(os.path.join(self.test_root, 'tree', 'fulltext3.js'))) self.assertFalse(os.path.exists(os.path.join(self.test_root, 'tree', 'fulltext4.js'))) self.assertFalse(os.path.exists(os.path.join(self.test_root, 'tree', 'fulltext5.js'))) self.assertTrue(os.path.exists(os.path.join(self.test_root, 'tree', 'fulltext6.js')))