def test_repr_compat(self):
class fake_unicode(object):
def __repr__(self):
return "u'fake'"
previous = utils.unicode
self.addCleanup(setattr, utils, 'unicode', previous)
utils.unicode = fake_unicode
self.assertEqual("'fake'", utils.repr_compat(fake_unicode()))
utils.unicode = None
self.assertEqual("u'fake'", utils.repr_compat(fake_unicode()))
def t_error(self, token):
if self.cur_token:
# TODO make use of the extended calling signature when done
raise ECMASyntaxError('Illegal character %s at %s:%s after %s' % (
repr_compat(token.value[0]),
token.lineno,
self._get_colno(token),
format_lex_token(self.cur_token),
))
else:
raise ECMASyntaxError('Illegal character %s at %s:%s' % (
repr_compat(token.value[0]),
token.lineno,
self._get_colno(token),
))
def broken_string_token_handler(lexer, token):
match = PATT_BROKEN_STRING.match(token.value)
if not match:
return
# update the error token value to only include what was matched here
# as this will be the actual token that "failed"
token.value = match.group()
# calculate colno for current token colno before...
colno = lexer._get_colno(token)
# updating the newline indexes for the error reporting for raw
# lexpos
lexer._update_newline_idx(token)
# probe for the next values (which no valid rules will match)
position = lexer.lexer.lexpos + len(token.value)
failure = lexer.lexer.lexdata[position:position + 2]
if failure and failure[0] == '\\':
type_ = {'x': 'hexadecimal', 'u': 'unicode'}[failure[1]]
seq = re.match(r'\\[xu][0-9-a-f-A-F]*',
lexer.lexer.lexdata[position:]).group()
raise ECMASyntaxError(
"Invalid %s escape sequence '%s' at %s:%s" %
(type_, seq, lexer.lineno, lexer._get_colno_lexpos(position)))
tl = 16 # truncate length
raise ECMASyntaxError(
'Unterminated string literal %s at %s:%s' %
(repr_compat(token.value[:tl].strip() +
(token.value[tl:] and '...')), token.lineno, colno))
def walk(self,
node,
omit=('lexpos', 'lineno', 'colno', 'rowno'),
indent=0,
depth=-1,
pos=False,
_level=0):
"""
Accepts the standard node argument, along with an optional omit
flag - it should be an iterable that lists out all attributes
that should be omitted from the repr output.
"""
if not depth:
return '<%s ...>' % node.__class__.__name__
attrs = []
children = node.children()
ids = {id(child) for child in children}
indentation = ' ' * (indent * (_level + 1))
header = '\n' + indentation if indent else ''
joiner = ',\n' + indentation if indent else ', '
tailer = '\n' + ' ' * (indent * _level) if indent else ''
for k, v in vars(node).items():
if k.startswith('_'):
continue
if id(v) in ids:
ids.remove(id(v))
if isinstance(v, Node):
attrs.append(
(k, self.walk(v, omit, indent, depth - 1, pos, _level)))
elif isinstance(v, list):
items = []
for i in v:
if id(i) in ids:
ids.remove(id(i))
items.append(
self.walk(i, omit, indent, depth - 1, pos, _level + 1))
attrs.append(
(k, '[' + header + joiner.join(items) + tailer + ']'))
else:
attrs.append((k, repr_compat(v)))
if ids:
# for unnamed child nodes.
attrs.append(('?children', '[' + header + joiner.join(
self.walk(child, omit, indent, depth - 1, pos, _level + 1)
for child in children if id(child) in ids) + tailer + ']'))
position = ('@%s:%s ' % (
'?' if node.lineno is None else node.lineno,
'?' if node.colno is None else node.colno,
) if pos else '')
omit_keys = () if not omit else set(omit)
return '<%s %s%s>' % (node.__class__.__name__, position, ', '.join(
'%s=%s' % (k, v) for k, v in sorted(attrs) if k not in omit_keys))
def read(parser, stream):
"""
Return an AST from the input ES5 stream.
Arguments
parser
A parser instance.
stream
Either a stream object or a callable that produces one. The
stream object to read from; its 'read' method will be invoked.
If a callable was provided, the 'close' method on its return
value will be called to close the stream.
"""
source = stream() if callable(stream) else stream
try:
text = source.read()
stream_name = getattr(source, 'name', None)
try:
result = parser(text)
except ECMASyntaxError as e:
error_name = repr_compat(stream_name or source)
raise type(e)('%s in %s' % (str(e), error_name))
finally:
if callable(stream):
source.close()
result.sourcepath = stream_name
return result
def t_BROKEN_STRING(self, token):
# calculate colno for current token colno before...
colno = self._get_colno(token)
# updating the newline indexes for the error reporting for raw
# lexpos
self._update_newline_idx(token)
# probe for the next values (which no valid rules will match)
failure = self.lexer.lexdata[self.lexer.lexpos:self.lexer.lexpos + 2]
if failure and failure[0] == '\\':
type_ = {'x': 'hexadecimal', 'u': 'unicode'}[failure[1]]
seq = re.match(r'\\[xu][0-9-a-f-A-F]*',
self.lexer.lexdata[self.lexer.lexpos:]).group()
raise ECMASyntaxError("Invalid %s escape sequence '%s' at %s:%s" %
(type_, seq, self.lineno,
self._get_colno_lexpos(self.lexer.lexpos)))
tl = 16 # truncate length
raise ECMASyntaxError(
'Unterminated string literal %s at %s:%s' %
(repr_compat(token.value[:tl].strip() +
(token.value[tl:] and '...')), token.lineno, colno))
def assemble(self, spec):
"""
Assemble the library by compiling everything and generate the
required files for the final bundling.
"""
def generate_alias(prefix):
alias = {}
key = prefix + self.targetpath_suffix
for modname, target in spec.get(key, {}).items():
# the alias must point to the full path.
alias[modname] = join(spec[BUILD_DIR], *target.split('/'))
return alias
# the build config is the file that will be passed to webpack for
# building the final bundle.
webpack_config = WebpackConfig({
'mode':
spec.get(WEBPACK_MODE, DEFAULT_WEBPACK_MODE),
'devtool':
spec.get(WEBPACK_DEVTOOL, DEFAULT_WEBPACK_DEVTOOL),
'output': {
'path': dirname(spec[EXPORT_TARGET]),
'filename': basename(spec[EXPORT_TARGET]),
# TODO determine if publicPath is needed.
# TODO these are using magic values. The library target
# should be configured, along with umdNamedDefine also
# when the way to expose the relevant options as proper
# sets are determined.
'libraryTarget': 'umd',
'umdNamedDefine': True,
},
'resolve': {},
# TODO should omit this if no alias are found
'resolveLoader': {
'alias': spec.get(WEBPACK_RESOLVELOADER_ALIAS, {}),
},
'externals':
spec.get(WEBPACK_EXTERNALS, {}),
'module': {},
})
if spec.get(WEBPACK_OPTIMIZE_MINIMIZE):
webpack_config['optimization'] = {'minimize': True}
if WEBPACK_OUTPUT_LIBRARY in spec:
webpack_config['output']['library'] = spec[WEBPACK_OUTPUT_LIBRARY]
version = get_bin_version(
spec[self.webpack_bin_key],
kw={
'env':
webpack_env(pathsep.join(self.find_node_modules_basedir())),
})
logger.debug("found webpack at '%s' to be version '%s'",
spec[self.webpack_bin_key], version)
webpack_config['__webpack_target__'] = version
# set up alias lookup mapping.
webpack_config['resolve']['alias'] = alias = {}
# generate the aliases - yes, we include the bundled sources to
# be explicit as there are cases where an alternative bundle may
# be specified using optional advices.
main_prefixes = (
'transpiled',
'bundled',
)
source_alias = {}
for prefix in main_prefixes:
source_alias.update(generate_alias(prefix))
# Do the same for loaders, but keep it in a separate mapping for
# now.
other_prefixes = ('loaderplugins', )
other_alias = {}
for prefix in other_prefixes:
other_alias.update(generate_alias(prefix))
# It is assumed that if WEBPACK_ENTRY_POINT is defined, it will
# resolve into a target through the generated alias mapping.
# Otherwise, assume one of the default calmjs style exports.
if (spec.get(WEBPACK_ENTRY_POINT,
DEFAULT_BOOTSTRAP_EXPORT) == DEFAULT_BOOTSTRAP_EXPORT):
# now resolve whether the webpack.externals has been defined
# in a manner that requires the complete lookup module
logger.info("spec webpack_entry_point defined to be '%s'",
DEFAULT_BOOTSTRAP_EXPORT)
if (webpack_config['externals'].get(DEFAULT_BOOTSTRAP_EXPORT) ==
DEFAULT_BOOTSTRAP_EXPORT_CONFIG):
logger.info(
"webpack.externals defined '%s' with value that enables "
"the calmjs webpack bootstrap module; generating module "
"with the complete bootstrap template",
DEFAULT_BOOTSTRAP_EXPORT)
# assign the internal loader to the alias
# TODO check that the default loader not being passed
# through check_all_alias_declared is not an issue.
alias[DEFAULT_CALMJS_EXPORT_NAME] = self.write_lookup_module(
spec, _DEFAULT_LOADER_FILENAME,
*_WEBPACK_CALMJS_MODULE_LOADER_TEMPLATE)
# the bootstrap module will be the entry point in this
# case.
webpack_config['entry'] = self.write_bootstrap_module(spec)
if (spec.get(WEBPACK_OUTPUT_LIBRARY) !=
DEFAULT_BOOTSTRAP_EXPORT):
# a simple warning will do, as this may only be an
# inconvenience.
logger.warning(
"exporting complete calmjs bootstrap module with "
"webpack.output.library as '%s' (expected '%s')",
spec.get(WEBPACK_OUTPUT_LIBRARY),
DEFAULT_BOOTSTRAP_EXPORT,
)
else:
logger.info(
"webpack.externals does not have '%s' defined for "
"the complete calmjs webpack bootstrap module; "
"generating simple export of modules",
DEFAULT_BOOTSTRAP_EXPORT)
# one key bit: to avoid the default webpack behavior of
# potentially overriding values on the root node (say,
# there exists window.modules already by some other
# package), only use the template that exports to module
# if that is defined; otherwise simply use the load only
# template that will just require the selected modules.
if spec.get(WEBPACK_OUTPUT_LIBRARY):
template = _WEBPACK_ENTRY_CALMJS_EXPORT_ONLY_TEMPLATE
else:
template = _WEBPACK_ENTRY_CALMJS_LOAD_ONLY_TEMPLATE
# the resulting file is the entry point.
webpack_config['entry'] = self.write_lookup_module(
spec, _DEFAULT_BOOTSTRAP_FILENAME, *template)
if (spec.get(WEBPACK_OUTPUT_LIBRARY) ==
DEFAULT_BOOTSTRAP_EXPORT):
logger.critical(
"cowardly aborting export to webpack.output.library "
"as '%s' without the complete bootstrap; generating "
"module with export only template",
DEFAULT_BOOTSTRAP_EXPORT,
)
raise ValueError(
"aborting export of webpack.output.library as '%s' "
"with incomplete settings and bootstrap module" %
DEFAULT_BOOTSTRAP_EXPORT)
else:
# need to manually resolve the entry
# if externals has been defined, use the complete lookup module
# otherwise, use the simplified version.
wp_ep = spec[WEBPACK_ENTRY_POINT]
if wp_ep not in source_alias:
msg = "'%s' not found in the source alias map" % wp_ep
logger.error(msg)
logger.info(
'source alias map {alias: targetpath}: %s',
json_dumps(source_alias),
)
raise ToolchainAbort(msg)
webpack_config['entry'] = source_alias[wp_ep]
# merge all aliases for writing of configuration file
alias.update(source_alias)
alias.update(other_alias)
# record the webpack config to the spec
spec[WEBPACK_CONFIG] = webpack_config
if spec.get(VERIFY_IMPORTS, True):
missing = self.check_all_alias_declared(
source_alias,
partial(
check_name_declared,
webpack_config['resolve']['alias'],
webpack_config['resolveLoader']['alias'],
webpack_config['externals'],
spec.get(CALMJS_LOADERPLUGIN_REGISTRY),
))
if missing:
logger.warning(
"source file(s) referenced modules that are not in alias "
"or externals: %s",
', '.join(sorted(repr_compat(m) for m in missing)))
update_spec_webpack_loaders_modules(spec, alias)
webpack_config['module']['rules'] = spec.get(WEBPACK_MODULE_RULES, [])
# write the configuration file, after everything is checked.
self.write_webpack_config(spec, webpack_config)
def __repr__(self):
return str('<%s value=%s>' %
(type(self).__name__, repr_compat(self.value)))
