def test_freezeinfo_add_404_url(reason, expected_reasons):
hook_called = False
def start_hook(freezeinfo):
nonlocal hook_called
hook_called = True
freezeinfo.add_url(
'http://example.com/404-page.html',
reason=reason,
)
with context_for_test('app_with_extra_page') as module:
config = {
'output': {
'type': 'dict'
},
'prefix': 'http://example.com/',
'hooks': {
'start': [start_hook]
},
}
with raises_multierror_with_one_exception(UnexpectedStatus) as e:
freeze(module.app, config)
assert e.freezeyt_task.reasons == expected_reasons
def test_add_hook():
recorded_tasks = {}
def register_hook(freeze_info):
freeze_info.add_hook('page_frozen', record_page)
def record_page(task_info):
recorded_tasks[task_info.get_a_url()] = task_info
with context_for_test('app_2pages') as module:
config = {
'output': {
'type': 'dict'
},
'prefix': 'http://example.com/',
'hooks': {
'start': [register_hook]
},
}
freeze(module.app, config)
print(recorded_tasks)
assert len(recorded_tasks) == 2
info = recorded_tasks['http://example.com:80/']
assert info.path == 'index.html'
info = recorded_tasks['http://example.com:80/second_page.html']
assert info.path == 'second_page.html'
def test_modified_mime_db_file(tmp_path):
"""Integration test with modified mime-db, where is purposely
set wrong extensions for MIME image/png to check if our db was used.
"""
MIME_DB_TO_JSON = {
"image/png": {
"source": "iana",
"compressible": False,
"extensions": ["Jpeg","jPg"]
},
'text/html': {
"extensions": ["htmL"]
}
}
builddir = tmp_path / 'build'
db_path = tmp_path / "mime_db.json"
with open(db_path, mode="w") as mime_db:
json.dump(MIME_DB_TO_JSON, mime_db)
with context_for_test('app_wrong_mimetype') as module:
freeze_config = {
'output': str(builddir),
'mime_db_file': str(db_path)
}
freeze(module.app, freeze_config)
assert (builddir / 'index.html').exists()
# 'image.jpg' exists because we linked jpg extension with MIME 'image/png'
assert (builddir / 'image.jpg').exists()
def test_multiple_hooks():
recorded_tasks = {}
counter = 0
def record_page(task_info):
recorded_tasks[task_info.get_a_url()] = task_info
def increment_counter(task_info):
nonlocal counter
counter += 1
with context_for_test('app_2pages') as module:
config = {
'output': {
'type': 'dict'
},
'prefix': 'http://example.com/',
'hooks': {
'page_frozen': [record_page, increment_counter]
},
}
freeze(module.app, config)
print(recorded_tasks)
assert len(recorded_tasks) == 2
assert counter == 2
info = recorded_tasks['http://example.com:80/']
assert info.path == 'index.html'
info = recorded_tasks['http://example.com:80/second_page.html']
assert info.path == 'second_page.html'
def test_cli_with_config_variable(tmp_path):
app_name = 'app_with_extra_files'
build_dir = tmp_path / 'build'
cli_args = ['app', str(build_dir), '--import-config', 'app:freeze_config']
with context_for_test(app_name):
run_and_check(cli_args, app_name, build_dir)
def test_task_counts_extra_page():
recorded_done_counts = []
recorded_paths = set()
expected_total = 3
def record_start(freeze_info):
check_task_counts(freeze_info, expected_total)
assert freeze_info.done_task_count == 0
def record_page(task_info):
check_task_counts(task_info.freeze_info, expected_total)
recorded_done_counts.append(task_info.freeze_info.done_task_count)
recorded_paths.add(task_info.path)
with context_for_test('app_with_extra_page_deep') as module:
config = {
**module.freeze_config,
'output': {
'type': 'dict'
},
'prefix': 'http://example.com/',
'hooks': {
'start': [record_start],
'page_frozen': [record_page],
},
}
freeze(module.app, config)
assert recorded_done_counts == [1, 2, 3]
assert recorded_paths == {
'index.html',
'extra/index.html',
'extra/extra_deep/index.html',
}
def test_page_frozen_hook_with_redirects(policy):
recorded_tasks = []
def record_page(task_info):
recorded_tasks.append(task_info.path)
with context_for_test('app_redirects') as module:
config = dict(module.freeze_config)
config['output'] = {'type': 'dict'}
config['hooks'] = {'page_frozen': [record_page]}
config['redirect_policy'] = policy
output = freeze(module.app, config)
output_paths = []
def add_output_to_output_paths(dict_to_add, path_so_far):
for key, value in dict_to_add.items():
if isinstance(value, bytes):
output_paths.append(path_so_far + key)
else:
add_output_to_output_paths(value, path_so_far + key + '/')
add_output_to_output_paths(output, '')
recorded_tasks.sort()
output_paths.sort()
assert recorded_tasks == output_paths
def test_taskinfo_has_freezeinfo():
freezeinfos = []
def start_hook(freezeinfo):
freezeinfos.append(freezeinfo)
def page_frozen_hook(pageinfo):
freezeinfos.append(pageinfo.freeze_info)
with context_for_test('app_simple') as module:
config = {
'output': {
'type': 'dict'
},
'prefix': 'http://example.com/',
'hooks': {
'start': [start_hook],
'page_frozen': [page_frozen_hook],
},
}
freeze(module.app, config)
a, b = freezeinfos
assert a is b
def test_circular_redirect(tmp_path, monkeypatch):
with context_for_test('circular_redirect') as module:
freeze_config = module.freeze_config
freeze_config['output'] = {'type': 'dir', 'dir': tmp_path}
freeze_config['status_handlers'] = {'3xx': 'follow'}
with pytest.raises(InfiniteRedirection):
freeze(module.app, freeze_config)
def test_simple_output_specification(tmp_path):
expected = FIXTURES_PATH / 'app_2pages' / 'test_expected_output'
with context_for_test('app_2pages') as module:
freeze_config = {'output': str(tmp_path)}
freeze(module.app, freeze_config)
assert_dirs_same(tmp_path, expected)
def test_cli_cleanup_config_works(tmp_path):
app_name = 'app_cleanup_config'
build_dir = tmp_path / 'build'
cli_args = ['app', str(build_dir), '--import-config', 'app:freeze_config']
with context_for_test(app_name):
run_and_check(cli_args, app_name, build_dir)
assert build_dir.exists()
assert (build_dir / 'index.html').exists()
def test_redirect_policy_follow(tmp_path, monkeypatch):
with context_for_test('app_redirects') as module:
freeze_config = module.freeze_config
freeze_config['output'] = {'type': 'dict'}
freeze_config['status_handlers'] = {'3xx': 'follow'}
result = freeze(module.app, freeze_config) # freeze content to dict
assert result == module.expected_dict_follow
def test_cli_with_prefix_option(tmp_path):
app_name = 'app_url_for_prefix'
build_dir = tmp_path / 'build'
with context_for_test(app_name, ) as module:
freeze_config = getattr(module, 'freeze_config')
prefix = freeze_config['prefix']
cli_args = ['app', str(build_dir), '--prefix', prefix]
run_and_check(cli_args, app_name, build_dir)
def test_external_extra_files(tmp_path):
with context_for_test('app_2pages') as module:
freeze_config = {
'output': {
'type': 'dict'
},
'extra_pages': ['http://external.example/foo.html'],
}
with pytest.raises(ExternalURLError):
freeze(module.app, freeze_config)
def test_output(tmp_path, monkeypatch, app_name):
app_path = FIXTURES_PATH / app_name
error_path = app_path / 'error.txt'
with context_for_test(app_name) as module:
module = importlib.import_module('app')
app = module.app
freeze_config = getattr(module, 'freeze_config', {})
expected_dict = getattr(module, 'expected_dict', None)
expecting_dir = not getattr(module, 'no_expected_directory', False)
freeze_config['output'] = {'type': 'dir', 'dir': tmp_path}
if error_path.exists():
with pytest.raises(ValueError) as exc:
freeze(app, freeze_config)
exception_name = exc.type.__name__
expected_name = error_path.read_text().strip()
assert exception_name == expected_name
else:
# Non error app.
# We want to check against expected data stored in a directory
# and/or in a dict. At least one of those must exist.
if not expecting_dir and expected_dict is None:
raise AssertionError(f'({app_name}) is not contain any' +
'expected output (dict or dir)')
if expecting_dir:
# test the output saved in dir 'test_expected_output'
freeze(app, freeze_config) # freeze content to tmp_path
expected = app_path / 'test_expected_output'
if not expected.exists():
make_output = os.environ.get('TEST_CREATE_EXPECTED_OUTPUT')
if make_output == '1':
shutil.copytree(tmp_path, expected)
else:
raise AssertionError(
f'Expected output directory ({expected}) does not exist. '
+
'Run with TEST_CREATE_EXPECTED_OUTPUT=1 to create it'
)
assert_dirs_same(tmp_path, expected)
if expected_dict is not None:
# test the output saved in dictionary
freeze_config['output'] = {'type': 'dict'}
result = freeze(app, freeze_config) # freeze content to dict
assert result == expected_dict
def test_cli_cleanup_command_line_has_higher_priority(tmp_path):
app_name = 'app_cleanup_config'
build_dir = tmp_path / 'build'
cli_args = [
'app',
str(build_dir), '--cleanup', '--import-config', 'app:freeze_config'
]
with context_for_test(app_name):
run_and_check(cli_args, app_name, build_dir)
assert not build_dir.exists()
def test_cli_with_extra_page_option(tmp_path):
app_name = 'app_with_extra_page_deep'
build_dir = tmp_path / 'build'
cli_args = ['app', str(build_dir)]
with context_for_test(app_name) as module:
freeze_config = getattr(module, 'freeze_config')
for extra_page in freeze_config['extra_pages']:
cli_args.extend(['--extra-page', extra_page])
run_and_check(cli_args, app_name, build_dir)
def test_except_star():
with context_for_test('app_various_errors') as module:
app = module.app
config = {
'output': {'type': 'dict'},
}
try:
freeze(app, config)
except* TypeError as type_errors:
assert len(type_errors.exceptions) == 2
except* UnexpectedStatus as status_errors:
assert len(status_errors.exceptions) == 1
def test_multierror(tmp_path, monkeypatch):
with context_for_test('app_2_broken_links') as module:
freeze_config = {
'output': {'type': 'dir', 'dir': tmp_path},
}
with pytest.raises(MultiError) as excinfo:
freeze(module.app, freeze_config)
multierror = excinfo.value
assert len(multierror.exceptions) == 2
for exception in multierror.exceptions:
with pytest.raises(UnexpectedStatus):
raise exception
def test_page_frozen_hook_by_name():
with context_for_test('app_2pages') as module:
config = {
'output': {
'type': 'dict'
},
'prefix': 'http://example.com/',
'hooks': {
'page_frozen': [f'{__name__}:hook']
},
}
_recorded_hook_calls.clear()
freeze(module.app, config)
assert len(_recorded_hook_calls) == 2
def test_get_no_links(tmp_path):
"""Test configuration of no url_finders.
We should get just root page.
"""
builddir = tmp_path / 'build'
with context_for_test('app_2pages') as module:
freeze_config = {
'output': str(builddir),
'url_finders': {},
}
freeze(module.app, freeze_config)
assert (builddir / 'index.html').exists()
assert not (builddir / 'second_page.html').exists()
def test_succesfully_loaded_get_mimetype_config(tmp_path, get_mimetypeID):
"""Test if user configuration of external functions get_mimetype
is loaded and used during web app freezing.
"""
builddir = tmp_path / 'build'
with context_for_test('default_mimetype_plain') as module:
freeze_config = {
'output': str(builddir),
'get_mimetype': GET_MIMETYPES[get_mimetypeID],
}
freeze(module.app, freeze_config)
assert (builddir / 'index.html').exists()
assert (builddir / 'textfile').exists()
def test_cli_with_extra_page_option(tmp_path):
app_name = 'app_with_extra_page_deep'
build_dir = tmp_path / 'build'
with context_for_test(app_name) as module:
cli_args = ['app', str(build_dir)]
freeze_config = getattr(module, 'freeze_config')
extra_pages = []
for extra in freeze_config['extra_pages']:
extra_pages.append('--extra-page')
extra_pages.append(extra)
cli_args.extend(extra_pages)
run_and_check(cli_args, app_name, build_dir)
def test_cli_prefix_conflict(tmp_path):
app_name = 'app_url_for_prefix'
config_file = tmp_path / 'config.yaml'
build_dir = tmp_path / 'build'
with context_for_test(app_name) as module:
freeze_config = getattr(module, 'freeze_config')
prefix = freeze_config['prefix']
cli_args = ['app', str(build_dir), '--prefix', prefix]
data = {'prefix': 'http://pyladies.cz/lessons/'}
with open(config_file, mode='w') as file:
safe_dump(data, stream=file)
cli_args.extend(['--config', config_file])
run_and_check(cli_args, app_name, build_dir)
def test_get_url_finder_by_name_defined_by_user(tmp_path):
"""Test if we freezer parse url_finder inserted as func type.
"""
builddir = tmp_path / 'build'
with context_for_test('app_3pages_deep') as module:
freeze_config = {
'output': str(builddir),
'url_finders': {'text/html': f'{__name__}:url_finder'},
}
freeze(module.app, freeze_config)
assert (builddir / 'index.html').exists()
assert not (builddir / 'second_page.html').exists()
assert not (builddir / 'third_page.html').exists()
def test_page_failed_hook():
records = []
expected_total = 3
def record_frozen(task_info):
check_task_counts(task_info.freeze_info, expected_total)
records.append((
'frozen',
task_info.path,
task_info.freeze_info.total_task_count,
task_info.freeze_info.done_task_count,
task_info.freeze_info.failed_task_count,
))
assert task_info.exception == None
def record_fail(task_info):
check_task_counts(task_info.freeze_info, expected_total)
records.append((
'failed',
task_info.path,
task_info.freeze_info.total_task_count,
task_info.freeze_info.done_task_count,
task_info.freeze_info.failed_task_count,
))
assert isinstance(task_info.exception, UnexpectedStatus)
with context_for_test('app_2_broken_links') as module:
config = {
'output': {
'type': 'dict'
},
'prefix': 'http://example.com/',
'hooks': {
'page_frozen': [record_frozen],
'page_failed': [record_fail],
},
}
with pytest.raises(MultiError):
freeze(module.app, config)
assert records == [
('frozen', 'index.html', 3, 1, 0),
('failed', 'nowhere', 3, 2, 1),
('failed', 'also_nowhere', 3, 3, 2),
]
def test_cli_with_fixtures_output(tmp_path, app_name):
config_file = tmp_path / 'config.yaml'
build_dir = tmp_path / 'build'
with context_for_test(app_name) as module:
cli_args = ['app', str(build_dir)]
freeze_config = getattr(module, 'freeze_config', None)
if not getattr(module, 'config_is_serializable', True):
pytest.skip('Config is not serializable')
if freeze_config != None:
with open(config_file, mode='w') as file:
safe_dump(freeze_config, stream=file)
cli_args.extend(['--config', config_file])
run_and_check(cli_args, app_name, build_dir)
def test_simple_plugin():
recorded_calls = []
def record_call(freeze_info):
recorded_calls.append(freeze_info)
with context_for_test('app_2pages') as module:
config = {
'output': {
'type': 'dict'
},
'prefix': 'http://example.com/',
'plugins': [record_call],
}
freeze(module.app, config)
assert len(recorded_calls) == 1
def test_get_url_finder_callable_defined_by_user(found_url, tmp_path):
"""Test if we freezer parse url_finder inserted as func type by user.
"""
def my_url_finder(page_content, base_url, headers=None):
return [found_url]
builddir = tmp_path / 'build'
with context_for_test('app_3pages_deep') as module:
freeze_config = {
'output': str(builddir),
'url_finders': {'text/html': my_url_finder},
}
freeze(module.app, freeze_config)
assert (builddir / 'index.html').exists()
assert not (builddir / 'second_page.html').exists()
assert (builddir / 'third_page.html').exists()
def test_task_counts():
recorded_counts = []
expected_total = 2
def record_start(freeze_info):
check_task_counts(freeze_info, expected_total)
recorded_counts.append((
'(start)',
freeze_info.total_task_count,
freeze_info.done_task_count,
))
def record_page(task_info):
check_task_counts(task_info.freeze_info, expected_total)
recorded_counts.append((
task_info.path,
task_info.freeze_info.total_task_count,
task_info.freeze_info.done_task_count,
))
with context_for_test('app_2pages') as module:
config = {
'output': {
'type': 'dict'
},
'prefix': 'http://example.com/',
'hooks': {
'start': [record_start],
'page_frozen': [record_page],
},
}
freeze(module.app, config)
print(recorded_counts)
assert recorded_counts == [
('(start)', 1, 0),
('index.html', 2, 1),
('second_page.html', 2, 2),
]
