def prompt_overwrite_json(original, new, target_path, dumps=json_dumps):
"""
Prompt end user with a diff of original and new json that may
overwrite the file at the target_path. This function only displays
a confirmation prompt and it is up to the caller to implement the
actual functionality. Optionally, a custom json.dumps method can
also be passed in for output generation.
"""
# generate compacted ndiff output.
diff = '\n'.join(l for l in (line.rstrip() for line in difflib.ndiff(
json_dumps(original).splitlines(),
json_dumps(new).splitlines(),
)) if l[:1] in '?+-' or l[-1:] in '{}' or l[-2:] == '},')
basename_target = basename(target_path)
return prompt(
"Generated '%(basename_target)s' differs with '%(target_path)s'.\n\n"
"The following is a compacted list of changes required:\n"
"%(diff)s\n\n"
"Overwrite '%(target_path)s'?" % locals(),
choices=(
('Yes', True),
('No', False),
),
default_key=1,
)
def nunjucks_precompile(path, name):
require_stmt = 'var nunjucks = require("nunjucks");\n'
stdout, stderr = node(
'%sprocess.stdout.write(nunjucks.precompile(%s, {"name": %s}));' %
(require_stmt, json_dumps(path), json_dumps(name)))
if stderr:
logger.error("failed to precompile '%s'\n%s'", path, stderr)
return None
else:
return stdout
def assemble(self, spec):
"""
Assemble the library by compiling everything and generate the
required files for the final bundling.
"""
def generate_alias(prefix):
alias = {}
key = prefix + self.targetpath_suffix
for modname, target in spec.get(key, {}).items():
# the alias must point to the full path.
alias[modname] = join(spec[BUILD_DIR], *target.split('/'))
return alias
# the build config is the file that will be passed to webpack for
# building the final bundle.
webpack_config = WebpackConfig({
'mode':
spec.get(WEBPACK_MODE, DEFAULT_WEBPACK_MODE),
'devtool':
spec.get(WEBPACK_DEVTOOL, DEFAULT_WEBPACK_DEVTOOL),
'output': {
'path': dirname(spec[EXPORT_TARGET]),
'filename': basename(spec[EXPORT_TARGET]),
# TODO determine if publicPath is needed.
# TODO these are using magic values. The library target
# should be configured, along with umdNamedDefine also
# when the way to expose the relevant options as proper
# sets are determined.
'libraryTarget': 'umd',
'umdNamedDefine': True,
},
'resolve': {},
# TODO should omit this if no alias are found
'resolveLoader': {
'alias': spec.get(WEBPACK_RESOLVELOADER_ALIAS, {}),
},
'externals':
spec.get(WEBPACK_EXTERNALS, {}),
'module': {},
})
if spec.get(WEBPACK_OPTIMIZE_MINIMIZE):
webpack_config['optimization'] = {'minimize': True}
if WEBPACK_OUTPUT_LIBRARY in spec:
webpack_config['output']['library'] = spec[WEBPACK_OUTPUT_LIBRARY]
version = get_bin_version(
spec[self.webpack_bin_key],
kw={
'env':
webpack_env(pathsep.join(self.find_node_modules_basedir())),
})
logger.debug("found webpack at '%s' to be version '%s'",
spec[self.webpack_bin_key], version)
webpack_config['__webpack_target__'] = version
# set up alias lookup mapping.
webpack_config['resolve']['alias'] = alias = {}
# generate the aliases - yes, we include the bundled sources to
# be explicit as there are cases where an alternative bundle may
# be specified using optional advices.
main_prefixes = (
'transpiled',
'bundled',
)
source_alias = {}
for prefix in main_prefixes:
source_alias.update(generate_alias(prefix))
# Do the same for loaders, but keep it in a separate mapping for
# now.
other_prefixes = ('loaderplugins', )
other_alias = {}
for prefix in other_prefixes:
other_alias.update(generate_alias(prefix))
# It is assumed that if WEBPACK_ENTRY_POINT is defined, it will
# resolve into a target through the generated alias mapping.
# Otherwise, assume one of the default calmjs style exports.
if (spec.get(WEBPACK_ENTRY_POINT,
DEFAULT_BOOTSTRAP_EXPORT) == DEFAULT_BOOTSTRAP_EXPORT):
# now resolve whether the webpack.externals has been defined
# in a manner that requires the complete lookup module
logger.info("spec webpack_entry_point defined to be '%s'",
DEFAULT_BOOTSTRAP_EXPORT)
if (webpack_config['externals'].get(DEFAULT_BOOTSTRAP_EXPORT) ==
DEFAULT_BOOTSTRAP_EXPORT_CONFIG):
logger.info(
"webpack.externals defined '%s' with value that enables "
"the calmjs webpack bootstrap module; generating module "
"with the complete bootstrap template",
DEFAULT_BOOTSTRAP_EXPORT)
# assign the internal loader to the alias
# TODO check that the default loader not being passed
# through check_all_alias_declared is not an issue.
alias[DEFAULT_CALMJS_EXPORT_NAME] = self.write_lookup_module(
spec, _DEFAULT_LOADER_FILENAME,
*_WEBPACK_CALMJS_MODULE_LOADER_TEMPLATE)
# the bootstrap module will be the entry point in this
# case.
webpack_config['entry'] = self.write_bootstrap_module(spec)
if (spec.get(WEBPACK_OUTPUT_LIBRARY) !=
DEFAULT_BOOTSTRAP_EXPORT):
# a simple warning will do, as this may only be an
# inconvenience.
logger.warning(
"exporting complete calmjs bootstrap module with "
"webpack.output.library as '%s' (expected '%s')",
spec.get(WEBPACK_OUTPUT_LIBRARY),
DEFAULT_BOOTSTRAP_EXPORT,
)
else:
logger.info(
"webpack.externals does not have '%s' defined for "
"the complete calmjs webpack bootstrap module; "
"generating simple export of modules",
DEFAULT_BOOTSTRAP_EXPORT)
# one key bit: to avoid the default webpack behavior of
# potentially overriding values on the root node (say,
# there exists window.modules already by some other
# package), only use the template that exports to module
# if that is defined; otherwise simply use the load only
# template that will just require the selected modules.
if spec.get(WEBPACK_OUTPUT_LIBRARY):
template = _WEBPACK_ENTRY_CALMJS_EXPORT_ONLY_TEMPLATE
else:
template = _WEBPACK_ENTRY_CALMJS_LOAD_ONLY_TEMPLATE
# the resulting file is the entry point.
webpack_config['entry'] = self.write_lookup_module(
spec, _DEFAULT_BOOTSTRAP_FILENAME, *template)
if (spec.get(WEBPACK_OUTPUT_LIBRARY) ==
DEFAULT_BOOTSTRAP_EXPORT):
logger.critical(
"cowardly aborting export to webpack.output.library "
"as '%s' without the complete bootstrap; generating "
"module with export only template",
DEFAULT_BOOTSTRAP_EXPORT,
)
raise ValueError(
"aborting export of webpack.output.library as '%s' "
"with incomplete settings and bootstrap module" %
DEFAULT_BOOTSTRAP_EXPORT)
else:
# need to manually resolve the entry
# if externals has been defined, use the complete lookup module
# otherwise, use the simplified version.
wp_ep = spec[WEBPACK_ENTRY_POINT]
if wp_ep not in source_alias:
msg = "'%s' not found in the source alias map" % wp_ep
logger.error(msg)
logger.info(
'source alias map {alias: targetpath}: %s',
json_dumps(source_alias),
)
raise ToolchainAbort(msg)
webpack_config['entry'] = source_alias[wp_ep]
# merge all aliases for writing of configuration file
alias.update(source_alias)
alias.update(other_alias)
# record the webpack config to the spec
spec[WEBPACK_CONFIG] = webpack_config
if spec.get(VERIFY_IMPORTS, True):
missing = self.check_all_alias_declared(
source_alias,
partial(
check_name_declared,
webpack_config['resolve']['alias'],
webpack_config['resolveLoader']['alias'],
webpack_config['externals'],
spec.get(CALMJS_LOADERPLUGIN_REGISTRY),
))
if missing:
logger.warning(
"source file(s) referenced modules that are not in alias "
"or externals: %s",
', '.join(sorted(repr_compat(m) for m in missing)))
update_spec_webpack_loaders_modules(spec, alias)
webpack_config['module']['rules'] = spec.get(WEBPACK_MODULE_RULES, [])
# write the configuration file, after everything is checked.
self.write_webpack_config(spec, webpack_config)
def karma_requirejs(spec):
"""
An advice for the karma runtime before execution of karma that is
needed for integrating the requirejs framework for testing into
karma; needed when RJSToolchain was used for artifact generation.
This advice should be registered to BEFORE_KARMA by RJSToolchain.
This will modify the related items in spec for the generation of the
karma.conf.js to ensure compatibility with requirejs idioms for the
execution of the tests through karma.
"""
# Importing this here as these modules may not be available, so to
# avoid potential issues, import them within the scope of this
# function; this function should never be called if the calmjs.dev
# python package is not available for import (and the setup should
# not add this to a valid advice).
try:
from calmjs.dev import karma
except ImportError:
logger.error(
"package 'calmjs.dev' not available; cannot apply requirejs "
"specific information without karma being available.")
return
required_keys = [karma.KARMA_CONFIG, BUILD_DIR]
for key in required_keys:
if key not in spec:
logger.error(
"'%s' not provided by spec; aborting configuration for karma "
"test runner", key)
raise ToolchainAbort("spec missing key '%s'" % key)
config = spec.get(karma.KARMA_CONFIG)
config_files = config.get('files', [])
build_dir = spec.get(BUILD_DIR)
plugin_registry = spec.get(RJS_LOADER_PLUGIN_REGISTRY)
if not plugin_registry:
logger.warning(
'no rjs loader plugin registry provided in spec; '
"falling back to default registry '%s'",
RJS_LOADER_PLUGIN_REGISTRY_NAME)
plugin_registry = get(RJS_LOADER_PLUGIN_REGISTRY_NAME)
test_module_paths_map = spec.get(TEST_MODULE_PATHS_MAP, {})
test_conf = plugin_registry.modname_target_mapping_to_config_paths(
test_module_paths_map)
# Ensure '/absolute' is prefixed like so to eliminate spurious error
# messages in the test runner, simply because the requirejs plugin
# will try to go through this mechanism to find a timestamp and fail
# to find its expected path, triggering the unwanted messages. This
# naive prefixing is actually consistent for all platforms including
# Windows...
new_paths = {
# however, the actual path fragments need to be split and joined
# with the web standard '/' separator.
k: '/absolute' + '/'.join(v.split(sep))
for k, v in test_conf['paths'].items()
}
test_conf['paths'] = new_paths
test_config_path = spec['karma_requirejs_test_config'] = join(
build_dir, 'requirejs_test_config.js')
with open(test_config_path, 'w') as fd:
fd.write(UMD_REQUIREJS_JSON_EXPORT_HEADER)
json_dump(test_conf, fd)
fd.write(UMD_REQUIREJS_JSON_EXPORT_FOOTER)
# Export all the module dependencies first so they get pre-loaded
# and thus be able to be loaded synchronously by test modules.
deps = sorted(spec.get('export_module_names', []))
if spec.get(ARTIFACT_PATHS):
# TODO have a flag of some sort for flagging this as optional.
deps.extend(process_artifacts(spec.get(ARTIFACT_PATHS)))
test_prefix = spec.get(TEST_FILENAME_PREFIX, TEST_FILENAME_PREFIX_DEFAULT)
tests = []
# Process tests separately; include them iff the filename starts
# with test, otherwise they are just provided as dependency modules.
for k, v in test_module_paths_map.items():
if basename(v).startswith(test_prefix):
tests.append(k)
else:
deps.append(k)
test_script_path = spec['karma_requirejs_test_script'] = join(
build_dir, 'karma_test_init.js')
with open(test_script_path, 'w') as fd:
fd.write(TEST_SCRIPT_TEMPLATE % (json_dumps(deps), json_dumps(tests)))
frameworks = ['requirejs']
frameworks.extend(config['frameworks'])
config['frameworks'] = frameworks
# rebuild the files listing in specific ordering as the load order
# matters from within a test browser spawned by karma.
files = []
# first include the configuration files
files.extend(spec.get(CONFIG_JS_FILES, []))
# then append the test configuration path
files.append(test_config_path)
# then the script
files.append(test_script_path)
# then extend the configured paths but do not auto-include them.
files.extend({'pattern': f, 'included': False} for f in config_files)
# update the file listing with modifications; this will be written
# out as part of karma.conf.js by the KarmaRuntime.
config['files'] = files
def karma_requirejs(spec):
"""
An advice for the karma runtime before execution of karma that is
needed for integrating the requirejs framework for testing into
karma; needed when RJSToolchain was used for artifact generation.
This advice should be registered to BEFORE_KARMA by RJSToolchain.
This will modify the related items in spec for the generation of the
karma.conf.js to ensure compatibility with requirejs idioms for the
execution of the tests through karma.
"""
# Importing this here as these modules may not be available, so to
# avoid potential issues, import them within the scope of this
# function; this function should never be called if the calmjs.dev
# python package is not available for import (and the setup should
# not add this to a valid advice).
try:
from calmjs.dev import karma
except ImportError:
logger.error(
"package 'calmjs.dev' not available; cannot apply requirejs "
"specific information without karma being available."
)
return
required_keys = [karma.KARMA_CONFIG, BUILD_DIR]
for key in required_keys:
if key not in spec:
logger.error(
"'%s' not provided by spec; aborting configuration for karma "
"test runner", key
)
raise ToolchainAbort("spec missing key '%s'" % key)
config = spec.get(karma.KARMA_CONFIG)
config_files = config.get('files', [])
build_dir = spec.get(BUILD_DIR)
spec_update_loaderplugin_registry(
spec, default=get(RJS_LOADER_PLUGIN_REGISTRY_NAME))
plugin_registry = spec[CALMJS_LOADERPLUGIN_REGISTRY]
test_module_paths_map = spec.get(TEST_MODULE_PATHS_MAP, {})
test_conf = plugin_registry.modname_targetpath_mapping_to_config_paths(
test_module_paths_map)
# Ensure '/absolute' is prefixed like so to eliminate spurious error
# messages in the test runner, simply because the requirejs plugin
# will try to go through this mechanism to find a timestamp and fail
# to find its expected path, triggering the unwanted messages. This
# naive prefixing is actually consistent for all platforms including
# Windows...
new_paths = {
# however, the actual path fragments need to be split and joined
# with the web standard '/' separator.
k: '/absolute' + '/'.join(v.split(sep))
for k, v in test_conf['paths'].items()
}
test_conf['paths'] = new_paths
test_config_path = spec['karma_requirejs_test_config'] = join(
build_dir, 'requirejs_test_config.js')
with open(test_config_path, 'w') as fd:
fd.write(UMD_REQUIREJS_JSON_EXPORT_HEADER)
json_dump(test_conf, fd)
fd.write(UMD_REQUIREJS_JSON_EXPORT_FOOTER)
# Export all the module dependencies first so they get pre-loaded
# and thus be able to be loaded synchronously by test modules.
deps = sorted(spec.get('export_module_names', []))
if spec.get(ARTIFACT_PATHS):
# TODO have a flag of some sort for flagging this as optional.
deps.extend(process_artifacts(spec.get(ARTIFACT_PATHS)))
test_prefix = spec.get(TEST_FILENAME_PREFIX, TEST_FILENAME_PREFIX_DEFAULT)
tests = []
# Process tests separately; include them iff the filename starts
# with test, otherwise they are just provided as dependency modules.
for k, v in test_module_paths_map.items():
if basename(v).startswith(test_prefix):
tests.append(k)
else:
deps.append(k)
test_script_path = spec['karma_requirejs_test_script'] = join(
build_dir, 'karma_test_init.js')
with open(test_script_path, 'w') as fd:
fd.write(TEST_SCRIPT_TEMPLATE % (json_dumps(deps), json_dumps(tests)))
frameworks = ['requirejs']
frameworks.extend(config['frameworks'])
config['frameworks'] = frameworks
# rebuild the files listing in specific ordering as the load order
# matters from within a test browser spawned by karma.
files = []
# first include the configuration files
files.extend(spec.get(CONFIG_JS_FILES, []))
# then append the test configuration path
files.append(test_config_path)
# then the script
files.append(test_script_path)
# then extend the configured paths but do not auto-include them.
files.extend({'pattern': f, 'included': False} for f in config_files)
# update the file listing with modifications; this will be written
# out as part of karma.conf.js by the KarmaRuntime.
config['files'] = files
def test_dumps(self):
self.assertEqual(
json_dumps({'a': 'b', 'c': 'd'}),
'{\n "a": "b",\n "c": "d"\n}'
)