def test_lock_one_change(mocker, tmpworkdir):
yml = """\
download_root: droot
download:
'http://wherever.com/file.js': file.js
'http://wherever.com/file2.js': file2.js"""
mktree(tmpworkdir, {'grablib.yml': yml})
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.return_value = MockResponse()
Grab().download()
assert mock_requests_get.call_count == 2
assert gettree(tmpworkdir, max_len=None) == {
'grablib.yml': yml,
'droot': {'file.js': 'response text', 'file2.js': 'response text'},
'.grablib.lock': """\
b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js file.js
b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file2.js file2.js\n"""
}
tmpworkdir.join('droot/file.js').remove()
print('##################')
Grab().download()
assert mock_requests_get.call_count == 3
assert gettree(tmpworkdir, max_len=None) == {
'grablib.yml': yml,
'droot': {'file.js': 'response text', 'file2.js': 'response text'},
'.grablib.lock': """\
b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js file.js
b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file2.js file2.js\n"""
}
def test_lock_local_file_changes(mocker, tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': "download:\n 'http://wherever.com/file.js': x"
})
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.return_value = MockResponse()
Grab(download_root='test-download-dir').download()
assert mock_requests_get.call_count == 1
assert gettree(tmpworkdir) == {
'grablib.yml': "download:\n 'http://wherever.com/file.js': x",
'test-download-dir': {'x': 'response text'},
'.grablib.lock': 'b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js x\n'
}
mktree(tmpworkdir, {
'grablib.yml': "download:\n 'http://wherever.com/file.js': x2"
})
Grab(download_root='test-download-dir').download()
assert mock_requests_get.call_count == 2
assert gettree(tmpworkdir, max_len=0) == {
'grablib.yml': "download:\n 'http://wherever.com/file.js': x2",
'test-download-dir': {'x2': 'response text'},
'.grablib.lock': 'b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js x2\n'
'# "stale" files which grablib should delete where found, '
'you can delete these once everyone has run grablib\n'
'b5a3344a4b3651ebd60a1e15309d737c :stale x\n'
}
def test_simple_render(tmpdir):
foo_page = '# hello\n\nthis is a test foo: {{ foo }}'
mktree(
tmpdir, {
'pages': {
'foobar.md': foo_page,
'spam.html': '# SPAM',
'favicon.ico': '*',
},
'theme/templates/main.jinja': 'main, content:\n\n{{ content }}',
'tmp/content/foobar.md': foo_page,
})
config = Config(
source_dir=str(tmpdir),
tmp_dir=str(tmpdir.join('tmp')),
foo='bar',
)
som = {
'pages': {
'foobar.md': {
'uri': '/foobar.html',
'infile': config.pages_dir / 'foobar.md',
'template': 'main.jinja',
'content_template': Path('content') / 'foobar.md',
},
'favicon.ico': {
'uri': '/favicon.ico',
'infile': config.pages_dir / 'favicon.ico',
},
}
}
expected_tree = {
'foobar.html': ('main, content:\n\n'
'<h1 id="1-hello">hello</h1>\n\n'
'<p>this is a test foo: </p>\n'),
'favicon.ico':
'*',
}
assert not tmpdir.join('dist').check()
assert render_pages(config, som) is None
assert gettree(tmpdir.join('dist')) == expected_tree
tmpdir.join('dist').remove(rec=1)
assert not tmpdir.join('dist').check()
cache = render_pages(config, som, {})
assert gettree(tmpdir.join('dist')) == expected_tree
assert len(cache) == 2
tmpdir.join('dist').remove(rec=1)
assert not tmpdir.join('dist').check()
render_pages(config, som, cache)
assert gettree(tmpdir.join('dist')) != expected_tree
def test_copy_assets_prod(tmpdir):
mktree(
tmpdir, {
'pages/foobar.md': '# hello',
'theme': {
'templates': {
'main.jinja': 'main:\n {{ content }}'
},
'assets': {
'image.png': '*',
'favicon.ico': '*',
'move': {
'foobar.svg': 'x'
}
}
},
})
config = get_config(str(tmpdir))
config.mode = Mode.production
copy_assets(config)
assert gettree(tmpdir.join('dist')) == {
'image.3389dae.png': '*',
'favicon.ico': '*',
'move': {
'foobar.9dd4e46.svg': 'x'
}
}
def test_dev_simple(tmpdir, mocker, loop):
async def awatch_alt(*args, **kwargs):
yield {(Change.modified, str(tmpdir.join('pages/foobar.md')))}
yield {(Change.modified, str(tmpdir.join('pages/features/whatever.md')))}
yield {(Change.modified, str(tmpdir.join('harrier.yml')))}
yield {(Change.added, str(tmpdir.join('theme/sass/main.scss')))}
tmpdir.join('harrier.yml').write('foo: 2')
yield {(Change.modified, str(tmpdir.join('harrier.yml')))}
asyncio.set_event_loop(loop)
mktree(tmpdir, {
'pages': {
'foobar.md': '# hello\n {{ config.foo }}',
'features/whatever.md': '## Foo',
},
'harrier.yml': 'foo: 1'
})
mocker.patch('harrier.dev.awatch', side_effect=awatch_alt)
assert not tmpdir.join('dist').check()
dev(str(tmpdir), 25698)
# debug(gettree(tmpdir.join('dist')))
assert gettree(tmpdir.join('dist')) == {
'foobar': {
'index.html': '<h1 id="1-hello">hello</h1>\n\n<p>2</p>\n',
},
'features': {
'whatever': {
'index.html': '<h2 id="2-foo">Foo</h2>\n',
},
},
}
def test_resolve_sass_path_dev(tmpdir):
mktree(
tmpdir, {
'pages/foobar.md': '# hello',
'theme': {
'assets/assets/image.png':
'*',
'sass/main.scss':
'body {content: resolve_path("/assets/image.png")}',
},
})
config = get_config(str(tmpdir))
config.mode = Mode.development
assets_grablib(config)
mtime = tmpdir.join('theme/sass/main.scss').stat().mtime
assert gettree(tmpdir.join('dist')) == {
'assets': {
'image.png': '*',
},
'theme': {
'main.css.map':
RegexStr('{.*'),
'main.css': ("body {\n"
" content: '/assets/image.png?t=%0.0f'; }\n"
"\n"
"/*# sourceMappingURL=main.css.map */") % mtime,
'.src': {
'main.scss':
'body {content: resolve_path("/assets/image.png")}',
},
},
}
def test_dev_data(tmpdir, mocker, loop):
async def awatch_alt(*args, **kwargs):
tmpdir.join('data/foobar.yml').write('a: 2')
yield {(Change.modified, str(tmpdir.join('data/foobar.yml')))}
asyncio.set_event_loop(loop)
mktree(tmpdir, {
'pages': {
'foobar.html': '{{ data.foobar.a }}',
},
'data/foobar.yml': 'a: 1'
})
mocker.patch('harrier.dev.awatch', side_effect=awatch_alt)
mocker.patch('harrier.dev.Server', return_value=MockServer())
assert not tmpdir.join('dist').check()
dev(str(tmpdir), 8000)
# debug(gettree(tmpdir.join('dist')))
assert gettree(tmpdir.join('dist')) == {
'foobar': {
'index.html': '2\n',
},
}
def test_ignored_directory(tmpdir, mocker, loop):
async def awatch_alt(*args, **kwargs):
yield {(Change.modified, str(tmpdir.join('pages/ignored.html')))}
asyncio.set_event_loop(loop)
mktree(tmpdir, {
'pages': {
'foobar.html': '1',
'ignored.html': '2'
},
'harrier.yaml': (
'ignore:\n'
'- /ignored.html'
)
})
mocker.patch('harrier.dev.awatch', side_effect=awatch_alt)
mocker.patch('harrier.dev.Server', return_value=MockServer())
assert not tmpdir.join('dist').check()
dev(str(tmpdir), 8000)
# debug(gettree(tmpdir.join('dist')))
assert gettree(tmpdir.join('dist')) == {
'foobar': {
'index.html': '1\n',
},
}
def test_grablib(tmpdir):
mktree(
tmpdir, {
'pages/foobar.md':
'# hello',
'theme': {
'templates': {
'main.jinja': 'main:\n {{ content }}'
},
'sass/main.scss': ('@import "DL/demo";'
'body {background: $foo}'),
},
'harrier.yml':
(f'download:\n'
f" 'https://gist.githubusercontent.com/samuelcolvin/ae6d04dadbb4d552d365f440d3ac8015/"
f"raw/cda04f66c71e4a5f418e78d111d651ae3a2e3784/demo.scss': '_demo.scss'"
)
})
config = get_config(str(tmpdir))
run_grablib(config)
assert gettree(tmpdir.join('dist')) == {
'theme': {
'main.7cc3e19.css': 'body{background:#BAD}\n',
},
}
def test_ignore_no_template(tmpdir):
mktree(
tmpdir, {
'pages': {
'ignore_this.md':
'this file is ignored',
'normal.md':
'hello this is normal',
'no_template.md':
'this should be passed through as-is',
'normal_but_no_output.md': ('---\n'
'output: false\n'
'---\n'
'hello this is normal\n')
},
'theme': {
'templates/foobar.jinja': 'rendered {{ content }}',
},
'harrier.yml': ('default_template: "foobar.jinja"\n'
'ignore:\n'
'- "**/ignore*"\n'
'defaults:\n'
' "/no_temp*":\n'
' pass_through: true\n')
})
build(tmpdir, mode=Mode.production)
assert gettree(tmpdir.join('dist')) == {
'no_template': {
'index.html': 'this should be passed through as-is',
},
'normal': {
'index.html': 'rendered <p>hello this is normal</p>\n',
},
}
def test_dev_delete(tmpdir, mocker, loop):
async def awatch_alt(*args, **kwargs):
yield {(Change.deleted, str(tmpdir.join('pages/features/whatever.md')))}
asyncio.set_event_loop(loop)
mktree(tmpdir, {
'pages': {
'foobar.md': 'hello',
'features/whatever.md': 'Foo',
},
})
mocker.patch('harrier.dev.awatch', side_effect=awatch_alt)
mocker.patch('harrier.dev.Server', return_value=MockServer())
assert not tmpdir.join('dist').check()
assert dev(str(tmpdir), 8000) == 0
# debug(gettree(tmpdir.join('dist')))
assert gettree(tmpdir.join('dist')) == {
'foobar': {
'index.html': '<p>hello</p>\n',
},
'features': {
'whatever': {},
},
}
def test_inline_css_dev(tmpdir):
mktree(
tmpdir, {
'pages': {
'foo.html': '{{inline_css("theme/main.css")}}',
'bar.html': "{{ url('theme/main.css') }}",
},
'theme': {
'sass/main.scss': 'body {width: 10px + 10px;}',
},
})
build(tmpdir, mode=Mode.development)
assert gettree(tmpdir.join('dist')) == {
'foo': {
'index.html': ('body {\n'
' width: 20px; }\n'
'\n'
'/*# sourceMappingURL=/theme/main.css.map */\n'),
},
'bar': {
'index.html': RegexStr(r'/theme/main.css\?t=\d+\n'),
},
'theme': {
'main.css.map':
RegexStr('{.*'),
'main.css': ('body {\n'
' width: 20px; }\n'
'\n'
'/*# sourceMappingURL=main.css.map */'),
'.src': {
'main.scss': 'body {width: 10px + 10px;}',
},
},
}
def test_full_build(tmpdir):
mktree(
tmpdir, {
'pages': {
'foobar.html': ('{{ url("foobar.png") }}\n'
'{{ resolve_url("theme/main.css") }}\n'
'{{ resolve_url("another") }}\n'),
'another.md':
'# Hello',
'third.yaml':
'content: "hello there"',
},
'theme': {
'sass/main.scss': 'body {width: 10px + 10px;}',
'assets/foobar.png': '*',
},
})
build(tmpdir, mode=Mode.production)
assert gettree(tmpdir.join('dist')) == {
'foobar': {
'index.html': ('/foobar.3389dae.png\n'
'/theme/main.a1ac3a7.css\n'
'/another/\n'),
},
'another': {
'index.html': '<h1 id="1-hello">Hello</h1>\n'
},
'theme': {
'main.a1ac3a7.css': 'body{width:20px}\n',
},
'third': {
'index.html': 'hello there\n'
},
'foobar.3389dae.png': '*',
}
def test_sass_clever_import_node_modules(tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': """
build_root: built_at
debug: true
build:
sass:
css: sass_dir
""",
'sass_dir': {
'foo.scss': "@import 'NM/foobar/bar';",
},
'node_modules': {
'foobar/_bar.scss': 'a {color: black;}'
}
})
Grab().build()
tree = gettree(tmpworkdir.join('built_at/css'))
assert {
'foo.css': 'a {\n'
' color: black; }\n\n'
'/*# sourceMappingURL=foo.css.map */',
'foo.css.map': RegexStr('{.*'),
'.src': {
'foo.scss': "@import 'NM/foobar/bar';"
}
} == tree
def test_extensions_error(tmpdir, mocker, loop):
async def awatch_alt(*args, **kwargs):
tmpdir.join('extensions.py').write('print(xxx)')
yield {(Change.modified, str(tmpdir.join('extensions.py')))}
asyncio.set_event_loop(loop)
mktree(tmpdir, {
'pages': {
'foobar.md': '**hello**',
},
'theme/templates/main.jinja': 'main:\n {{ content }}',
'harrier.yml': 'default_template: main.jinja',
'extensions.py': 'x = 1'
})
mocker.patch('harrier.dev.awatch', side_effect=awatch_alt)
mocker.patch('harrier.dev.Server', return_value=MockServer())
assert not tmpdir.join('dist').check()
assert dev(str(tmpdir), 8000) == 1
assert gettree(tmpdir.join('dist')) == {
'foobar': {
'index.html': 'main:\n <p><strong>hello</strong></p>\n',
},
}
def test_sass(tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': """
build_root: "built_at"
build:
sass:
"css": "sass_dir"
""",
'sass_dir': {
'not_css.txt': 'not included',
'adir/_mixin.scss': 'a { color: red};',
'foo.scss': """
@import 'adir/mixin';
.foo {
.bar {
color: black;
width: (60px / 6);
}
}
"""
}
})
Grab().build()
assert gettree(tmpworkdir.join('built_at')) == {
'css': {
'foo.css': 'a{color:red}.foo .bar{color:black;width:10px}\n'
}
}
def test_sass_custom_functions(tmpdir):
mktree(tmpdir, {
'sass': {
'foo.scss': (
".foo {color: waffle(100px, 'foobar.png')}"
)
},
'downloads': {}
})
args = None
def waffle(a, b):
nonlocal args
args = f'{a.value:0.0f}{a.unit}', b
return '#G00D'
sass_gen = SassGenerator(
input_dir=Path(tmpdir.join('sass')),
output_dir=Path(tmpdir.join('output')),
download_root=Path(tmpdir.join('downloads')),
custom_functions={waffle},
)
sass_gen()
assert gettree(tmpdir.join('output')) == {
'foo.css': '.foo{color:#G00D}\n'
}
assert args == ('100px', 'foobar.png')
def test_rm_some(tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': """
build_root: "built_at"
build:
wipe:
- boom.txt
- another_dir.*
""",
'built_at': {
'foo/bar.js': 'x',
'boom.txt': 'x',
'another_dir': {
'a.txt': 'x',
'b.txt': 'x',
},
'remain.txt': 'y',
}
})
setup_logging('DEBUG')
Grab().build()
assert {
'foo': {
'bar.js': 'x'
},
'remain.txt': 'y',
} == gettree(tmpworkdir.join('built_at'))
def test_stale_lock(mocker, tmpworkdir):
gl = """\
download:
'http://wherever.com/file.js': foo
"""
lock = (
'# this is a comment\n'
'b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js foo\n'
' #so is this\n'
'b5a3344a4b3651ebd60a1e15309d737c :stale to_delete\n'
)
mktree(tmpworkdir, {
'grablib.yml': gl,
'.grablib.lock': lock,
'test-download-dir': {'foo': 'response text', 'to_delete': 'response text'},
})
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.return_value = MockResponse()
grab = Grab(download_root='test-download-dir')
grab.download()
assert gettree(tmpworkdir, max_len=0) == {
'grablib.yml': gl,
'.grablib.lock': 'b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js foo\n'
'# "stale" files which grablib should delete where found, '
'you can delete these once everyone has run grablib\n'
'b5a3344a4b3651ebd60a1e15309d737c :stale to_delete\n',
'test-download-dir': {'foo': 'response text'},
}
def test_lock_zip(mocker, tmpworkdir):
mktree(tmpworkdir, {'grablib.yml': zip_dowload_yml})
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.side_effect = request_fixture
Grab().download()
assert mock_requests_get.call_count == 1
assert zip_downloaded_directory == gettree(tmpworkdir, max_len=None)
Grab().download()
assert mock_requests_get.call_count == 1
assert zip_downloaded_directory == gettree(tmpworkdir, max_len=None)
Grab().download()
assert mock_requests_get.call_count == 1
assert zip_downloaded_directory == gettree(tmpworkdir, max_len=None)
tmpworkdir.join('droot/subdirectory/a.txt').remove()
Grab().download()
assert mock_requests_get.call_count == 2
assert zip_downloaded_directory == gettree(tmpworkdir, max_len=None)
def test_extensions_ok(tmpdir):
mktree(
tmpdir, {
'pages': {
'foo.md': '# foo',
'bar.html': '{{ 4|add_one }} {{ dynamic }}',
'spam.html': 'before',
'splat.html': '{{ 3 is two }} {{ 2 is two }}',
},
'theme/templates/main.jinja':
'{{ content }}',
'extensions.py':
"""
from harrier.extensions import modify, template
@modify.pages('/foo.*')
def modify_foo(page, config):
page['content'] += ' changed by extension'
return page
@modify.som
def add_var(site):
site['dynamic'] = 42
return site
@modify.som
def change_pages(site):
site['pages']['/spam.html']['content'] = 'after'
return site
@template.filter
def add_one(v):
return v + 1
@template.test
def two(v):
return v == 2
"""
})
build(str(tmpdir))
assert gettree(tmpdir.join('dist')) == {
'foo': {
'index.html':
'<h1 id="1-foo-changed-by-extension">foo changed by extension</h1>\n',
},
'bar': {
'index.html': '5 42\n',
},
'spam': {
'index.html': 'after\n',
},
'splat': {
'index.html': 'False True\n',
},
}
def test_no_lock(mocker, tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': "lock: null\ndownload:\n 'http://wherever.com/file.js': x"
})
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.return_value = MockResponse()
grab = Grab(download_root='s')
grab.download()
assert gettree(tmpworkdir) == {
'grablib.yml': "lock: null\ndownload:\n 'http://wherever.com/file.js': x",
's': {'x': 'response text'},
}
def test_zip_null(mocker, tmpworkdir):
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.side_effect = request_fixture
mktree(tmpworkdir, {
'grablib.yaml': """
"download":
"https://any-old-url.com/test_assets.zip":
"test_assets/assets/a.txt": null
"test_assets/assets/(.+)": "subdirectory/{filename}"
"""})
Grab(download_root='test-download-dir').download()
assert gettree(tmpworkdir.join('test-download-dir')) == {'subdirectory': {'b.txt': 'b\n'}}
def test_build_no_templates(tmpdir):
mktree(tmpdir, {
'pages': {
'foobar.md': ('### Whatever'),
},
})
build(tmpdir, mode=Mode.production)
assert gettree(tmpdir.join('dist')) == {
'foobar': {
'index.html': ('<h3 id="3-whatever">Whatever</h3>\n'),
},
}
def test_xml_no_front_matter(tmpdir):
mktree(
tmpdir, {
'pages': {
'foobar.xml': ('<x><y>{{ config.whatever }}</y></x>'),
},
'harrier.yml': 'whatever: 123'
})
build(tmpdir, mode=Mode.production)
assert gettree(tmpdir.join('dist')) == {
'foobar.xml': '<x><y>{{ config.whatever }}</y></x>'
}
def test_zip_double(mocker, tmpworkdir):
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.side_effect = request_fixture
mktree(tmpworkdir, {
'grablib.yaml': """
'download':
'https://any-old-url.com/test_assets.zip':
'test_assets/assets/a.txt':
- a.txt
- a_again.txt
'test_assets/assets/(.+)': '{filename}'
"""})
Grab(download_root='test-download-dir').download()
assert gettree(tmpworkdir.join('test-download-dir')) == {'b.txt': 'b\n', 'a.txt': 'a\n', 'a_again.txt': 'a\n'}
def test_lock_unchanged(mocker, tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': "download:\n 'http://wherever.com/file.js': x",
'.grablib.lock': 'b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js x\n',
})
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.return_value = MockResponse()
grab = Grab(download_root='test-download-dir')
grab.download()
assert gettree(tmpworkdir) == {
'grablib.yml': "download:\n 'http://wherever.com/file.js': x",
'test-download-dir': {'x': 'response text'},
'.grablib.lock': 'b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js x\n'
}
def test_rm_all(tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': """
build_root: "built_at"
build:
wipe: '.*'
""",
'built_at': {
'foo/bar.js': 'x',
'boom.txt': 'x',
}
})
Grab().build()
assert gettree(tmpworkdir.join('built_at')) == {}
def test_jinja_format(tmpdir):
mktree(
tmpdir, {
'pages': {
'index.html':
'{{"{:0.2f}"|format(3.1415)}}',
'2032-06-02-date.txt':
'---\nx: 1\n---\n{{ "{:%b %d, %Y}"|format(page.created) }}',
},
})
build(tmpdir, mode=Mode.production)
assert gettree(tmpdir.join('dist')) == {
'index.html': '3.14\n',
'date.txt': 'Jun 02, 2032\n'
}
def test_simple_lock(mocker, tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': "lock: the.lock\ndownload:\n 'http://wherever.com/file.js': x"
})
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.return_value = MockResponse()
Grab(download_root='static').download()
assert mock_requests_get.call_count == 1
assert gettree(tmpworkdir) == {
'grablib.yml': "lock: the.lock\ndownload:\n 'http://wherever.com/file.js': x",
'static': {'x': 'response text'},
'the.lock': 'b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js x\n'
}
Grab(download_root='static').download()
assert mock_requests_get.call_count == 1
def test_no_trailing_slash(tmpdir):
mktree(
tmpdir, {
'pages': {
'index.html': '<a href="{{ url("other") }}">link to other</a>',
'other.html': 'xxx'
},
'harrier.yml': 'apply_trailing_slash: false'
})
build(tmpdir, mode=Mode.production)
assert gettree(tmpdir.join('dist')) == {
'other': {
'index.html': 'xxx\n',
},
'index.html': '<a href="/other">link to other</a>\n',
}
def test_lock_zip_remote_changed(mocker, tmpworkdir):
mktree(tmpworkdir, {'grablib.yml': zip_dowload_yml})
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
zip_r = request_fixture('https://any-old-url.com/test_assets.zip')
mock_requests_get.side_effect = [
MockResponse(content=zip_r.content, headers={'content-type': 'application/zip'}),
MockResponse(content=zip_r.content + b'x', headers={'content-type': 'application/zip'}),
]
Grab().download()
assert mock_requests_get.call_count == 1
assert zip_downloaded_directory == gettree(tmpworkdir, max_len=None)
tmpworkdir.join('droot/subdirectory/a.txt').remove()
with pytest.raises(GrablibError):
Grab().download()
assert mock_requests_get.call_count == 2
def test_generate_pages(tmpdir):
mktree(
tmpdir, {
'pages': {
'index.html': 'hello',
},
'extensions.py':
"""
from pathlib import Path
from harrier.extensions import modify
THIS_DIR = Path(__file__).parent.resolve()
@modify.generate_pages
def add_extra_pages(som):
config: Config = som['config']
yield {
'path': Path('extra/index.md'),
'content': '# this is a test\\n\\nwith of generating pages dynamically',
}
yield {
'path': Path('more/index.html'),
'content': 'testing {{ page.x }}',
'data': {
'uri': '/foo-bar-whatever',
'x': 123,
}
}
(THIS_DIR / 'pages' / 'binary_file').write_bytes(b'xxx')
yield {
'path': 'binary_file',
'content': None,
}
"""
})
build(str(tmpdir))
assert gettree(tmpdir.join('dist')) == {
'extra': {
'index.html': ('<h1 id="1-this-is-a-test">this is a test</h1>\n'
'\n'
'<p>with of generating pages dynamically</p>\n'),
},
'foo-bar-whatever': {
'index.html': 'testing 123\n'
},
'index.html': 'hello\n',
'binary_file': 'xxx',
}
def test_just_build(tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': """
build_root: "built_at"
build:
cat:
"libraries.js":
- "./foo.js"
""",
'foo.js': 'var v = "foo js";',
})
result = CliRunner().invoke(cli, ['download'])
assert result.exit_code == 0
assert tmpworkdir.join('built_at').check() is False
result = CliRunner().invoke(cli, ['build'])
assert result.exit_code == 0
assert gettree(tmpworkdir.join('built_at')) == {'libraries.js': '/* === foo.js === */\nvar v="foo js";\n'}
def test_already_deleted(mocker, tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': 'download: {}',
'.grablib.lock': 'b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js path/to/x\n',
'test-download-dir': {'foo': 'bar'},
})
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.return_value = MockResponse()
grab = Grab(download_root='test-download-dir')
grab.download()
assert gettree(tmpworkdir, max_len=0) == {
'grablib.yml': 'download: {}',
'test-download-dir': {'foo': 'bar'},
'.grablib.lock': '# "stale" files which grablib should delete where found, '
'you can delete these once everyone has run grablib\n'
'b5a3344a4b3651ebd60a1e15309d737c :stale path/to/x\n'
}
def test_yaml_render(tmpdir):
mktree(
tmpdir, {
'pages': {
'test.yml': ('template: foobar.jinja\n'
'foo:\n'
' bar: 42\n'),
},
'theme': {
'templates/foobar.jinja': 'foo bar: {{ page.foo.bar }}',
},
})
build(tmpdir, mode=Mode.production)
assert gettree(tmpdir.join('dist')) == {
'test': {
'index.html': 'foo bar: 42\n'
}
}
def test_lock_remote_file_changes(mocker, tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': "download:\n 'http://wherever.com/file.js': x"
})
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.side_effect = [
MockResponse(),
MockResponse(content=b'changed')
]
Grab(download_root='s').download()
assert mock_requests_get.call_count == 1
assert gettree(tmpworkdir) == {
'grablib.yml': "download:\n 'http://wherever.com/file.js': x",
's': {'x': 'response text'},
'.grablib.lock': 'b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js x\n'
}
tmpworkdir.join('s/x').remove()
with pytest.raises(GrablibError):
Grab(download_root='s').download()
def test_changed_file_url(mocker, tmpworkdir):
gl = """\
download:
'http://wherever.com/file_different.js': x
"""
mktree(tmpworkdir, {
'grablib.yml': gl,
'.grablib.lock': 'b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js x\n',
'test-download-dir': {'x': 'response text'},
})
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.return_value = MockResponse()
grab = Grab(download_root='test-download-dir')
grab.download()
assert gettree(tmpworkdir, max_len=0) == {
'grablib.yml': gl,
'test-download-dir': {'x': 'response text'},
'.grablib.lock': 'b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file_different.js x\n'
}
def test_cat_regex(tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': """
build_root: "built_at"
build:
cat:
"f.min.js":
- src: "./foo.js"
replace:
"change": "!new_value"
""",
'foo.js': 'var v = "change js";',
})
Grab().build()
assert gettree(tmpworkdir.join('built_at')) == {
'f.min.js':
'/* === foo.js === */\n'
'var v="!new_value js";\n'
}
def test_with_aliases(mocker, tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': """\
download_root: download_to
aliases:
WHATEVER: https://www.whatever.com
download:
"WHATEVER/foo.js": "js/"
"""
})
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.return_value = MockResponse()
Grab().download()
mock_requests_get.assert_called_with('https://www.whatever.com/foo.js')
assert gettree(tmpworkdir.join('download_to')) == {
'js': {
'foo.js': 'response text'
}
}
def test_inline_css_prod(tmpdir):
mktree(
tmpdir, {
'pages': {
'foobar.html': '{{inline_css("theme/main.css")}}'
},
'theme': {
'sass/main.scss': 'body {width: 10px + 10px;}',
},
})
build(tmpdir, mode=Mode.production)
assert gettree(tmpdir.join('dist')) == {
'foobar': {
'index.html': ('body{width:20px}\n'),
},
'theme': {
'main.a1ac3a7.css': 'body{width:20px}\n',
},
}
def test_raw_sass_prod(tmpdir):
mktree(tmpdir, {
'sass': {
'adir/_mixin.scss': '$my_colour: #G00D;',
'foo.scss': (
"@import 'adir/mixin';\n"
".foo {color: $my_colour}"
)
},
'downloads': {}
})
sass_gen = SassGenerator(
input_dir=Path(tmpdir.join('sass')),
output_dir=Path(tmpdir.join('output')),
download_root=Path(tmpdir.join('downloads')),
)
sass_gen()
assert gettree(tmpdir.join('output')) == {
'foo.css': '.foo{color:#G00D}\n'
}
def test_stale_changed(mocker, tmpworkdir):
gl = """\
download:
'http://wherever.com/file.js': x
"""
mktree(tmpworkdir, {
'grablib.yml': gl,
'.grablib.lock': 'b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js old\n',
'test-download-dir': {'old': 'response text - different'},
})
mock_requests_get = mocker.patch('grablib.download.requests.Session.get')
mock_requests_get.return_value = MockResponse()
grab = Grab(download_root='test-download-dir')
with pytest.raises(GrablibError):
grab.download()
assert gettree(tmpworkdir) == {
'grablib.yml': gl,
'.grablib.lock': 'b5a3344a4b3651ebd60a1e15309d737c http://wherever.com/file.js old\n',
'test-download-dir': {'old': 'response text - different', 'x': 'response text'},
}
def test_frontmatter_maybe(tmpdir):
mktree(
tmpdir, {
'pages': {
'foobar.xml': ('---\n'
'foo: bar\n'
'---\n'
'<x><y>{{ config.whatever }}</y></x>'),
'foobar.txt': ('---\n'
'x: 1\n'
'---\n'
'{{ page.x }}'),
},
'harrier.yml': 'whatever: 123'
})
build(tmpdir, mode=Mode.production)
assert gettree(tmpdir.join('dist')) == {
'foobar.xml': '<x><y>123</y></x>\n',
'foobar.txt': '1\n',
}
def test_sass_exclude(tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': """
build_root: 'built_at'
build:
sass:
css:
src: sass_dir
exclude: 'adir/.*$'
""",
'sass_dir': {
'adir/bar.scss': '.bar { color: red};',
'foo.scss': '.foo { color: black;}'
}
})
Grab().build()
assert gettree(tmpworkdir.join('built_at')) == {
'css': {
'foo.css': '.foo{color:black}\n'
}
}
def test_grablib_pygments(tmpdir):
mktree(
tmpdir, {
'pages/foobar.md': '# hello',
'theme': {
'sass': {
'_other.scss':
'div {colour: red}',
'main.scss': ('@import "other";\n'
'@import "pygments/default";\n'),
},
}
})
config = get_config(str(tmpdir))
run_grablib(config)
assert gettree(tmpdir.join('dist')) == {
'theme': {
'main.9a9caa6.css': RegexStr('div{colour:red}\.hi \.hll.*'),
},
}
def test_cat_download_root(tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': """
download_root: downloaded2
build_root: "built_at"
build:
cat:
"f.min.js":
- "./foo.js"
- "DL/bar.js"
""",
'foo.js': 'var v = "foo js";',
'downloaded2/bar.js': 'var v = "bar js";',
})
Grab().build()
assert gettree(tmpworkdir.join('built_at')) == {
'f.min.js':
'/* === foo.js === */\n'
'var v="foo js";\n'
'/* === bar.js === */\n'
'var v="bar js";\n'
}
def test_sass_debug(tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': """
build_root: "built_at"
debug: true
build:
sass:
"css": "sass_dir"
""",
'sass_dir': {
'foo.scss': '.foo { .bar {color: black;}}'
}
})
Grab().build()
tree = gettree(tmpworkdir.join('built_at/css'))
assert {
'foo.css': '.foo .bar {\n color: black; }\n\n/*# sourceMappingURL=foo.css.map */',
'foo.css.map': RegexStr('{\n\t"version": 3,\n\t"file": "\.src/foo.css".*'),
'.src': {
'foo.scss': '.foo { .bar {color: black;}}'
}
} == tree
def test_cat_debug(tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': """
build_root: "built_at"
build:
cat:
"libraries.js":
- "./foo.js"
- "./bar.js"
""",
'foo.js': 'var v = "foo js";\n vindent = true;',
'bar.js': 'var v = "bar js";',
})
Grab(debug=True).build()
assert {
'libraries.js':
'/* === foo.js === */\n'
'var v = "foo js";\n'
' vindent = true;\n'
'/* === bar.js === */\n'
'var v = "bar js";\n'
} == gettree(tmpworkdir.join('built_at'))
def test_resolve_sass_path(tmpdir):
mktree(
tmpdir, {
'pages/foobar.md': '# hello',
'theme': {
'assets/assets/image.png':
'*',
'sass/main.scss':
'body {content: resolve_path("/assets/image.png")}',
},
})
config = get_config(str(tmpdir))
assets_grablib(config)
assert gettree(tmpdir.join('dist')) == {
'theme': {
'main.d024f29.css': "body{content:'/assets/image.3389dae.png'}\n",
},
'assets': {
'image.3389dae.png': '*',
},
}
def test_sass_clever_import(tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': """
download_root: downloaded2
build_root: built_at
debug: false
build:
sass:
css: sass_dir
""",
'downloaded2': {
'x.css': '.x {width:100px}'
},
'sass_dir': {
'foo.scss': "@import 'SRC/path/to/bar';\n@import 'DL/x';",
'path/to/_bar.scss': 'a {color: black;}'
}
})
Grab().build()
assert {
'foo.css': 'a{color:black}.x{width:100px}\n',
} == gettree(tmpworkdir.join('built_at/css'))
def test_sass_log_repeat(tmpworkdir, smart_caplog):
mktree(tmpworkdir, {
'grablib.yml': """
build_root: "built_at"
build:
sass:
"css": "sass_dir"
""",
'sass_dir': {
'foo.scss': 'span {color: white}'
}
})
Grab().build()
assert gettree(tmpworkdir.join('built_at')) == {'css': {'foo.css': 'span{color:white}\n'}}
assert '18B' in smart_caplog
assert '%' not in smart_caplog
Grab().build()
assert '%' not in smart_caplog
tmpworkdir.join('sass_dir/foo.scss').write('span {color: white};\ndiv {color: red}')
Grab().build()
assert '32B' in smart_caplog
assert '+78%' in smart_caplog
def test_dev_extensions(tmpdir, mocker, loop):
async def awatch_alt(*args, **kwargs):
yield {(Change.modified, str(tmpdir.join('pages/foobar.html')))}
yield {(Change.modified, str(tmpdir.join('pages/foobar.html')))}
asyncio.set_event_loop(loop)
mktree(tmpdir, {
'pages': {
'foobar.html': 'before',
},
'call': '0',
'extensions.py': """
from pathlib import Path
from harrier.extensions import modify, template
p = Path(__file__).parent / 'call'
@modify.som
def change_pages(site):
v = int(p.read_text())
v += 1
p.write_text(str(v))
site['pages']['/foobar.html']['content'] = str(v)
return site
"""
})
mocker.patch('harrier.dev.awatch', side_effect=awatch_alt)
mocker.patch('harrier.dev.Server', return_value=MockServer())
assert not tmpdir.join('dist').check()
dev(str(tmpdir), 8000)
# debug(gettree(tmpdir.join('dist')))
assert gettree(tmpdir.join('dist')) == {
'foobar': {
'index.html': '3\n',
},
}
def test_smart_sass_url(tmpdir):
mktree(
tmpdir, {
'pages/foobar.md': '# hello',
'theme': {
'assets/assets/image.png':
'*',
'sass/main.scss':
'body {background: smart_url("assets/image.png")}',
},
})
config = get_config(str(tmpdir))
assets_grablib(config)
assert gettree(tmpdir.join('dist')) == {
'theme': {
'main.28d2724.css':
"body{background:url('/assets/image.3389dae.png')}\n",
},
'assets': {
'image.3389dae.png': '*',
},
}
def test_cat(tmpworkdir):
mktree(tmpworkdir, {
'grablib.yml': """
download:
http://whatever.com/file: file.txt
build_root: "built_at"
build:
cat:
"libs.min.js":
- "./foo.js"
- "./bar.js"
""",
'foo.js': 'var v = "foo js";\n vindent = true;',
'bar.js': 'var v = "bar js";',
})
Grab().build()
assert gettree(tmpworkdir.join('built_at')) == {
'libs.min.js':
'/* === foo.js === */\n'
'var v="foo js";vindent=true;\n'
'/* === bar.js === */\n'
'var v="bar js";\n'
}
def test_copy_extensions(tmpdir):
mktree(
tmpdir, {
'pages': {
'index.html': 'hello',
},
'theme/assets': {
'image1.png': 'a',
'image2.png': 'c',
'foo/bar.svg': 'c',
},
'extensions.py':
"""
from harrier.extensions import modify, template
@modify.copy('/foo/*')
def modify_foo(in_path, out_path, config):
out_path.write_text(f'{in_path.name} {in_path.read_text()} custom')
return 1 # prevent default copy
@modify.copy('/image2.png')
def print_in_path(in_path, out_path, config):
out_path.with_name(out_path.name + '.alt').write_bytes(in_path.read_bytes() + b'2')
# return nothing so normal copy also happens
"""
})
build(str(tmpdir))
assert gettree(tmpdir.join('dist')) == {
'index.html': 'hello\n',
'foo': {
'bar.4a8a08f.svg': 'bar.svg c custom',
},
'image1.0cc175b.png': 'a',
'image2.4a8a08f.png': 'c',
'image2.4a8a08f.png.alt': 'c2',
}
def test_gettree(tmpdir):
tmpdir.join('foo').write_text('bar', 'utf8')
assert {'foo': 'bar'} == gettree(tmpdir)
def test_gettree_truncate(tmpdir):
tmpdir.join('foo').write_text('123456789', 'utf8')
assert {'foo': '12...'} == gettree(tmpdir, max_len=5)
def test_build_multi_part(tmpdir):
mktree(
tmpdir, {
'pages': {
'multipart_list.md': ('---\n'
'uri: /list_md.html\n'
'template: list.jinja\n'
'---\n'
'part 1\n'
'--- . ---\n'
'part **2**\n'
'---.---\n'
'this is part *3*\n'),
'multipart_dict.md': ('---\n'
'uri: /dict_md.html\n'
'template: dict.jinja\n'
'---\n'
'the main **section**\n'
'--- other ---\n'
'part *2*\n'),
'multipart_list.html': ('---\n'
'uri: /list_html.html\n'
'template: list.jinja\n'
'---\n'
'part 1\n'
'--- . ---\n'
'part 2\n'
'---.---\n'
'this is part 3\n'),
'multipart_dict.html': ('---\n'
'uri: /dict_html.html\n'
'template: dict.jinja\n'
'---\n'
'the main section\n'
'--- other ---\n'
'part 2\n'),
},
'theme': {
'templates/': {
'list.jinja': ('{% for v in content %}\n'
' {{ v.content }}\n'
'{% endfor %}\n'),
'dict.jinja': ('{{ content.main.content }}\n'
'{{ content.other.content }}\n'),
},
},
})
build(tmpdir, mode=Mode.production)
assert gettree(tmpdir.join('dist')) == {
'list_md.html': ('\n'
' <p>part 1</p>\n'
'\n\n'
' <p>part <strong>2</strong></p>\n'
'\n\n'
' <p>this is part <em>3</em></p>\n'),
'dict_md.html': ('<p>the main <strong>section</strong></p>\n'
'\n'
'<p>part <em>2</em></p>\n'),
'list_html.html': ('\n'
' part 1\n'
'\n'
' part 2\n'
'\n'
' this is part 3\n'),
'dict_html.html': ('the main section\n'
'part 2\n'),
}
def test_mock_executor(tmpdir, mocker):
foobar_path = str(tmpdir.join('pages/foobar.md'))
async def awatch_alt(*args, **kwargs):
yield {(Change.modified, str(tmpdir.join('harrier.yml')))}
yield {(Change.modified, foobar_path)}
yield {(Change.modified, str(tmpdir.join('theme/assets/main.png')))}
yield {(Change.modified, str(tmpdir.join('theme/sass/main.scss')))}
yield {(Change.modified, str(tmpdir.join('theme/templates/main.jinja')))}
yield {(Change.modified, str(tmpdir.join('extensions.py')))}
yield {(Change.modified, str(tmpdir.join('data/foobar.yml')))}
mktree(tmpdir, {
'pages/foobar.md': '# hello',
'theme/templates/main.jinja': 'main:\n {{ content }}',
'harrier.yml': 'foo: bar',
})
mocker.patch('harrier.dev.awatch', side_effect=awatch_alt)
mocker.patch('harrier.dev.Server', return_value=MockServer())
loop = asyncio.new_event_loop()
f = asyncio.Future(loop=loop)
f.set_result(None)
mock_run_in_executor = mocker.patch.object(loop, 'run_in_executor', return_value=f)
asyncio.set_event_loop(loop)
assert not tmpdir.join('dist').check()
dev(str(tmpdir), 8000)
assert gettree(tmpdir.join('dist')) == {}
assert [c[0][2].dict(exclude={'config_path'}) for c in mock_run_in_executor.call_args_list] == [
{
'pages': '__FB__',
'assets': False,
'sass': False,
'templates': False,
'data': False,
'extensions': False,
'update_config': False,
},
{
'pages': set(),
'assets': False,
'sass': False,
'templates': False,
'data': False,
'extensions': False,
'update_config': True,
},
{
'pages': {(Change.modified, Path(foobar_path))},
'assets': False,
'sass': False,
'templates': False,
'data': False,
'extensions': False,
'update_config': False,
},
{
'pages': set(),
'assets': True,
'sass': False,
'templates': False,
'data': False,
'extensions': False,
'update_config': False,
},
{
'pages': set(),
'assets': False,
'sass': True,
'templates': False,
'data': False,
'extensions': False,
'update_config': False,
},
{
'pages': set(),
'assets': False,
'sass': False,
'templates': True,
'data': False,
'extensions': False,
'update_config': False,
},
{
'pages': set(),
'assets': False,
'sass': False,
'templates': False,
'data': False,
'extensions': True,
'update_config': False,
},
{
'pages': set(),
'assets': False,
'sass': False,
'templates': False,
'data': True,
'extensions': False,
'update_config': False,
},
]
