def _add_extra_pages(self, prefix, extras):
"""Add URLs of extra pages from config.
Handles both literal URLs and generators.
"""
for extra in extras:
if isinstance(extra, dict):
try:
generator = extra['generator']
except KeyError:
raise ValueError(
'extra_pages must be strings or dicts with ' +
f'a "generator" key, not `{extra}`')
if isinstance(generator, str):
generator = import_variable_from_module(generator)
self._add_extra_pages(prefix, generator(self.app))
elif isinstance(extra, str):
url = parse_absolute_url(
urljoin(prefix, decode_input_path(extra)))
try:
self.add_task(
url,
reason='extra page',
)
except ExternalURLError:
raise ExternalURLError(
f'External URL specified in extra_pages: {url}')
else:
generator = extra
self._add_extra_pages(prefix, generator(self.app))
def parse_handlers(handlers: Mapping,
default_module: Optional[str] = None) -> Mapping:
result = {}
for key, handler_or_name in handlers.items():
if isinstance(handler_or_name, str):
handler = import_variable_from_module(
handler_or_name, default_module_name=default_module)
else:
handler = handler_or_name
if not callable(handler):
raise TypeError(
"Handler for {key!r} in configuration must be a string or a callable,"
+ f" not {type(handler)}!")
result[key] = handler
return result
def test_empty():
with pytest.raises(ValueError):
import_variable_from_module("")
def test_missing_variable_with_default():
with pytest.raises(ValueError):
import_variable_from_module("math:", default_variable_name='cos')
def test_missing_module():
with pytest.raises(ValueError):
import_variable_from_module(":sin")
def test_dotted_variable():
joinpath = import_variable_from_module("pathlib:Path.joinpath")
assert joinpath.__name__ == 'joinpath'
def test_dotted_module():
urlparse = import_variable_from_module("urllib.parse:urlparse")
assert urlparse.__name__ == 'urlparse'
def test_overridden_default():
sin = import_variable_from_module("math:sin", default_variable_name='cos')
assert sin.__name__ == 'sin'
def __init__(self, app, config):
self.app = app
self.config = config
self.check_version(self.config.get('version'))
self.freeze_info = hooks.FreezeInfo(self)
CONFIG_DATA = (('extra_pages', ()), ('extra_files', None),
('default_mimetype', 'application/octet-stream'),
('get_mimetype', default_mimetype),
('mime_db_file', None), ('url_to_path',
default_url_to_path))
for attr_name, default in CONFIG_DATA:
setattr(self, attr_name, config.get(attr_name, default))
if self.mime_db_file:
with open(self.mime_db_file) as file:
mime_db = json.load(file)
mime_db = convert_mime_db(mime_db)
self.get_mimetype = functools.partial(mime_db_mimetype, mime_db)
if isinstance(self.get_mimetype, str):
self.get_mimetype = import_variable_from_module(self.get_mimetype)
if isinstance(self.url_to_path, str):
self.url_to_path = import_variable_from_module(self.url_to_path)
if config.get('use_default_url_finders', True):
_url_finders = dict(DEFAULT_URL_FINDERS,
**config.get('url_finders', {}))
else:
_url_finders = config.get('url_finders', {})
self.url_finders = parse_handlers(
_url_finders, default_module='freezeyt.url_finders')
_status_handlers = dict(DEFAULT_STATUS_HANDLERS,
**config.get('status_handlers', {}))
self.status_handlers = parse_handlers(
_status_handlers, default_module='freezeyt.status_handlers')
for key in self.status_handlers:
if not STATUS_KEY_RE.fullmatch(key):
raise ValueError(
'Status descriptions must be strings with 3 digits or one '
+ f'digit and "xx", got f{key!r}')
prefix = config.get('prefix', 'http://localhost:8000/')
# Decode path in the prefix URL.
# Save the parsed version of prefix as self.prefix
prefix_parsed = parse_absolute_url(prefix)
decoded_path = decode_input_path(prefix_parsed.path)
if not decoded_path.endswith('/'):
raise ValueError('prefix must end with /')
self.prefix = prefix_parsed.replace(path=decoded_path)
output = config['output']
if isinstance(output, str):
output = {'type': 'dir', 'dir': output}
if output['type'] == 'dict':
self.saver = DictSaver(self.prefix)
elif output['type'] == 'dir':
try:
output_dir = output['dir']
except KeyError:
raise ValueError("output directory not specified")
self.saver = FileSaver(Path(output_dir), self.prefix)
else:
raise ValueError(f"unknown output type {output['type']}")
# The tasks for individual pages are tracked in the followng sets
# (actually dictionaries: {task.path: task})
# Each task must be in exactly in one of these.
self.done_tasks = {}
self.redirecting_tasks = {}
self.inprogress_tasks = {}
self.failed_tasks = {}
self.task_queues = {
TaskStatus.DONE: self.done_tasks,
TaskStatus.REDIRECTING: self.redirecting_tasks,
TaskStatus.IN_PROGRESS: self.inprogress_tasks,
TaskStatus.FAILED: self.failed_tasks,
}
try:
self.add_task(prefix_parsed, reason='site root (homepage)')
self._add_extra_files()
self._add_extra_pages(prefix, self.extra_pages)
self.hooks = {}
for name, funcs in config.get('hooks', {}).items():
for func in funcs:
if isinstance(func, str):
func = import_variable_from_module(func)
self.add_hook(name, func)
for plugin in config.get('plugins', {}):
if isinstance(plugin, str):
plugin = import_variable_from_module(plugin)
plugin(self.freeze_info)
self.semaphore = asyncio.Semaphore(MAX_RUNNING_TASKS)
except:
self.cancel_tasks()
raise
def test_errors(testname):
name, kwargs, error_message = INPUT_ERROR_DATA[testname]
with pytest.raises(ValueError) as excinfo:
import_variable_from_module(name, **kwargs)
assert excinfo.value.args[0] == error_message
def test_valid_data(testname):
name, kwargs, expected = INPUT_DATA[testname]
imported = import_variable_from_module(name, **kwargs)
assert imported.__name__ == expected
def __init__(self, app, config):
self.app = app
self.config = config
self.freeze_info = hooks.FreezeInfo(self)
self.extra_pages = config.get('extra_pages', ())
self.extra_files = config.get('extra_files', None)
self.url_finders = parse_handlers(
config.get('url_finders', DEFAULT_URL_FINDERS),
default_module='freezeyt.url_finders')
_status_handlers = dict(DEFAULT_STATUS_HANDLERS,
**config.get('status_handlers', {}))
self.status_handlers = parse_handlers(
_status_handlers, default_module='freezeyt.status_handlers')
prefix = config.get('prefix', 'http://localhost:8000/')
# Decode path in the prefix URL.
# Save the parsed version of prefix as self.prefix
prefix_parsed = parse_absolute_url(prefix)
decoded_path = decode_input_path(prefix_parsed.path)
if not decoded_path.endswith('/'):
raise ValueError('prefix must end with /')
self.prefix = prefix_parsed.replace(path=decoded_path)
output = config['output']
if isinstance(output, str):
output = {'type': 'dir', 'dir': output}
if output['type'] == 'dict':
self.saver = DictSaver(self.prefix)
elif output['type'] == 'dir':
try:
output_dir = output['dir']
except KeyError:
raise ValueError("output directory not specified")
self.saver = FileSaver(Path(output_dir), self.prefix)
else:
raise ValueError(f"unknown output type {output['type']}")
self.url_to_path = config.get('url_to_path', default_url_to_path)
if isinstance(self.url_to_path, str):
self.url_to_path = import_variable_from_module(self.url_to_path)
# The tasks for individual pages are tracked in the followng sets
# (actually dictionaries: {task.path: task})
# Each task must be in exactly in one of these.
self.done_tasks = {}
self.redirecting_tasks = {}
self.pending_tasks = {}
self.inprogress_tasks = {}
self.task_queues = {
TaskStatus.PENDING: self.pending_tasks,
TaskStatus.DONE: self.done_tasks,
TaskStatus.REDIRECTING: self.redirecting_tasks,
TaskStatus.IN_PROGRESS: self.inprogress_tasks,
}
self.add_task(prefix_parsed, reason='site root (homepage)')
self._add_extra_pages(prefix, self.extra_pages)
self.hooks = {}
for name, func in config.get('hooks', {}).items():
if isinstance(func, str):
func = import_variable_from_module(func)
self.hooks[name] = func
def main(
module_name,
dest_path,
prefix,
extra_pages,
config_file,
config_var,
progress,
cleanup,
):
"""
MODULE_NAME
Name of the Python web app module which will be frozen.
DEST_PATH
Absolute or relative path to the directory to which the files
will be frozen.
Example use:
python -m freezeyt demo_app build --prefix 'http://localhost:8000/' --extra-page /extra/
python -m freezeyt demo_app build -c config.yaml
"""
if config_file and config_var:
raise click.UsageError(
"Can't pass configuration both in a file and in a variable.")
elif config_file != None:
config = yaml.safe_load(config_file)
if not isinstance(config, dict):
raise SyntaxError(
f'File {config_file.name} is not a YAML dictionary.')
elif config_var is not None:
config = import_variable_from_module(config_var)
else:
config = {}
if extra_pages:
config.setdefault('extra_pages', []).extend(extra_pages)
if prefix != None:
config['prefix'] = prefix
if 'output' in config:
if dest_path is not None:
raise click.UsageError(
'DEST_PATH argument is not needed if output is configured from file'
)
else:
if dest_path is None:
raise click.UsageError('DEST_PATH argument is required')
config['output'] = {'type': 'dir', 'dir': dest_path}
if progress is None:
if sys.stdout.isatty():
progress = 'bar'
else:
progress = 'log'
if progress == 'bar':
config.setdefault('plugins',
[]).append('freezeyt.progressbar:ProgressBarPlugin')
if progress in ('log', 'bar'):
# The 'log' plugin is activated both with --progress=log and
# --progress=bar.
config.setdefault('plugins',
[]).append('freezeyt.progressbar:LogPlugin')
if cleanup is not None:
config['cleanup'] = cleanup
app = import_variable_from_module(
module_name,
default_variable_name='app',
)
try:
freeze(app, config)
except MultiError as multierr:
if sys.stderr.isatty():
cols, lines = shutil.get_terminal_size()
message = f' {multierr} '
click.echo(file=sys.stderr)
click.secho(message.center(cols, '='), file=sys.stderr, fg='red')
for task in multierr.tasks:
message = str(task.exception)
if message:
# Separate the error type and value by a semicolon
# (only if there is a value)
message = ': ' + message
err_type = click.style(type(task.exception).__name__, fg='red')
path = click.style(task.path, fg='cyan')
click.echo(f'{err_type}{message}', file=sys.stderr)
click.echo(f' in {path}', file=sys.stderr)
for reason in task.reasons:
click.echo(f' {reason}', file=sys.stderr)
exit(1)
def test_basic():
sin = import_variable_from_module("math:sin")
assert sin.__name__ == 'sin'
def test_default():
cos = import_variable_from_module("math", default_variable_name='cos')
assert cos.__name__ == 'cos'
def main(module_name, dest_path, prefix, extra_pages, config_file, config_var):
"""
MODULE_NAME
Name of the Python web app module which will be frozen.
DEST_PATH
Absolute or relative path to the directory to which the files
will be frozen.
--prefix
URL, where we want to deploy our static site
--extra-page
Path to page without any link in application
-c / --config
Path to configuration YAML file
-C / --import-config
Dictionary with the configuration
Example use:
python -m freezeyt demo_app build --prefix 'http://localhost:8000/' --extra-page /extra/
python -m freezeyt demo_app build -c config.yaml
"""
if config_file and config_var:
raise click.UsageError(
"Can't pass configuration both in a file and in a variable.")
elif config_file != None:
config = yaml.safe_load(config_file)
if not isinstance(config, dict):
raise SyntaxError(
f'File {config_file.name} is not a YAML dictionary.')
elif config_var is not None:
config = import_variable_from_module(config_var)
else:
config = {}
if extra_pages:
config.setdefault('extra_pages', []).extend(extra_pages)
if prefix != None:
config['prefix'] = prefix
if 'output' in config:
if dest_path is not None:
raise click.UsageError(
'DEST_PATH argument is not needed if output is configured from file'
)
else:
if dest_path is None:
raise click.UsageError('DEST_PATH argument is required')
config['output'] = {'type': 'dir', 'dir': dest_path}
app = import_variable_from_module(
module_name,
default_variable_name='app',
)
freeze(app, config)
def main(module_name, dest_path, prefix, extra_page, config):
"""
MODULE_NAME
Name of the Python web app module which will be frozen.
DEST_PATH
Absolute or relative path to the directory to which the files
will be frozen.
--prefix
URL, where we want to deploy our static site
--extra-page
Path to page without any link in application
-c / --config
Path to configuration YAML file
Example use:
python -m freezeyt demo_app build --prefix 'http://localhost:8000/' --extra-page /extra/
python -m freezeyt demo_app build -c config.yaml
"""
cli_params = {'extra_pages': list(extra_page)}
if prefix != None:
cli_params['prefix'] = prefix
if config != None:
file_config = yaml.safe_load(config)
if not isinstance(file_config, dict):
raise SyntaxError(
f'File {config.name} is not prepared as YAML dictionary.')
else:
print("Loading config YAML file was successful")
if (file_config.get('prefix', None) != None) and (prefix is None):
cli_params['prefix'] = file_config['prefix']
cli_params['extra_pages'].extend(file_config.get(
'extra_pages', []))
cli_params['extra_files'] = file_config.get('extra_files', None)
if 'output' in file_config:
cli_params['output'] = file_config['output']
if 'output' in cli_params:
if dest_path is not None:
raise click.UsageError(
'DEST_PATH argument is not needed if output is configured from file'
)
else:
if dest_path is None:
raise click.UsageError('DEST_PATH argument is required')
cli_params['output'] = {'type': 'dir', 'dir': dest_path}
app = import_variable_from_module(
module_name,
default_variable_name='app',
)
freeze(app, cli_params)
