def _prepare_wrapper(self):
"""Prepares an output wrapper and returns a list of command line arguments
for Closure Compiler to use it."""
# Load the wrapper and use Closure to strip whitespace and comments.
# This requires %output% in the template to be protected, so Closure doesn't
# fail to parse it.
wrapper_input_path = _get_source_path('build/wrapper.template.js')
wrapper_output_path = _get_source_path('dist/wrapper.js')
with shakaBuildHelpers.open_file(wrapper_input_path, 'r') as f:
wrapper_code = f.read().replace('%output%', '"%output%"')
jar = _get_source_path(
'node_modules/google-closure-compiler-java/compiler.jar')
cmd_line = ['java', '-jar', jar, '-O', 'WHITESPACE_ONLY']
proc = shakaBuildHelpers.execute_subprocess(cmd_line,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stripped_wrapper_code = proc.communicate(
wrapper_code.encode('utf8'))[0]
if proc.returncode != 0:
raise RuntimeError('Failed to strip whitespace from wrapper!')
with shakaBuildHelpers.open_file(wrapper_output_path, 'w') as f:
code = stripped_wrapper_code.decode('utf8')
f.write(code.replace('"%output%"', '%output%'))
return ['--output_wrapper_file=%s' % wrapper_output_path]
def process(text, options):
"""Decodes a JSON string containing source map data.
Args:
text: A JSON string containing source map data.
options: An object containing the command-line options.
"""
# The spec allows a map file to start with )]} to prevent javascript from
# including it.
if text.startswith(')]}\'\n') or text.startswith(')]}\n'):
_, text = text.split('\n', 1)
# Decode the JSON data and get the parts we need.
data = json.loads(text)
# Paths are relative to the output directory.
base = os.path.join(shakaBuildHelpers.get_source_base(), 'dist')
with shakaBuildHelpers.open_file(os.path.join(base, data['file']),
'r') as f:
file_lines = f.readlines()
names = data['names']
mappings = data['mappings']
tokens = decode_mappings(mappings, names)
sizes = process_sizes(tokens, file_lines)
# Print out one of the results.
if options.all_tokens:
print_tokens(tokens, file_lines, sizes)
elif options.function_sizes:
print_sizes(sizes)
elif options.function_deps or options.class_deps:
temp = process_deps(tokens, file_lines, options.class_deps)
print_deps(temp, options.dot_format)
def player_version():
"""Gets the version of the library from player.js."""
path = os.path.join(shakaBuildHelpers.get_source_base(), 'lib',
'player.js')
with shakaBuildHelpers.open_file(path, 'r') as f:
match = re.search(r'shaka\.Player\.version = \'(.*)\'', f.read())
return match.group(1) if match else ''
def check_spelling(_):
"""Checks that source files don't have any common misspellings."""
logging.info('Checking for common misspellings...')
complete = build.Build()
# Normally we don't need to include @core, but because we look at the build
# object directly, we need to include it here. When using main(), it will
# call addCore which will ensure core is included.
if not complete.parse_build(['+@complete', '+@core'], os.getcwd()):
logging.error('Error parsing complete build')
return False
base = shakaBuildHelpers.get_source_base()
complete.include.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'test'),
re.compile(r'.*\.js$')))
complete.include.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'demo'),
re.compile(r'.*\.js$')))
complete.include.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'build'),
re.compile(r'.*\.(js|py)$')))
with shakaBuildHelpers.open_file(
os.path.join(base, 'build', 'misspellings.txt')) as f:
misspellings = ast.literal_eval(f.read())
has_error = False
for path in complete.include:
with shakaBuildHelpers.open_file(path) as f:
for i, line in enumerate(f):
for regex, replace_pattern in misspellings.items():
for match in re.finditer(regex, line):
repl = match.expand(replace_pattern)
if match.group(0).lower() == repl:
continue # No-op suggestion
if not has_error:
logging.error(
'The following file(s) have misspellings:')
logging.error(
' %s:%d:%d: Did you mean %r?' % (os.path.relpath(
path, base), i + 1, match.start() + 1, repl))
has_error = True
return not has_error
def check_spelling(_):
"""Checks that source files don't have any common misspellings."""
logging.info('Checking for common misspellings...')
complete_build = complete_build_files()
if not complete_build:
return False
base = shakaBuildHelpers.get_source_base()
complete_build.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'test'),
re.compile(r'.*\.js$')))
complete_build.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'demo'),
re.compile(r'.*\.js$')))
complete_build.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'build'),
re.compile(r'.*\.(js|py)$')))
with shakaBuildHelpers.open_file(
os.path.join(base, 'build', 'misspellings.txt')) as f:
misspellings = ast.literal_eval(f.read())
has_error = False
for path in complete_build:
with shakaBuildHelpers.open_file(path) as f:
for i, line in enumerate(f):
for regex, replace_pattern in misspellings.items():
for match in re.finditer(regex, line):
repl = match.expand(replace_pattern).lower()
if match.group(0).lower() == repl:
continue # No-op suggestion
if not has_error:
logging.error(
'The following file(s) have misspellings:')
logging.error(
' %s:%d:%d: Did you mean %r?' % (os.path.relpath(
path, base), i + 1, match.start() + 1, repl))
has_error = True
return not has_error
def _locales_changed(self):
# If locales is None, it means we are being called by a caller who doesn't
# care what locales are in use. This is true, for example, when we are
# running a compiler pass over the tests.
if self.locales is None:
return False
# Find out what locales we used before. If they have changed, we must
# regenerate the output.
last_locales = None
try:
prefix = '// LOCALES: '
with shakaBuildHelpers.open_file(self.output, 'r') as f:
for line in f:
if line.startswith(prefix):
last_locales = line.replace(prefix, '').strip().split(', ')
except IOError:
# The file wasn't found or couldn't be read, so it needs to be redone.
return True
return set(last_locales) != set(self.locales)
def check_eslint_disable(_):
"""Checks that source files correctly use "eslint-disable".
- Rules are disabled/enabled in nested blocks.
- Rules are not disabled multiple times.
- Rules are enabled again by the end of the file.
Returns:
True on success, False on failure.
"""
logging.info('Checking correct usage of eslint-disable...')
complete_build = complete_build_files()
if not complete_build:
return False
base = shakaBuildHelpers.get_source_base()
complete_build.update(shakaBuildHelpers.get_all_js_files('test'))
complete_build.update(shakaBuildHelpers.get_all_js_files('demo'))
has_error = False
for path in complete_build:
# The stack of rules that are disabled.
disabled = []
with shakaBuildHelpers.open_file(path, 'r') as f:
rel_path = os.path.relpath(path, base)
for i, line in enumerate(f):
match = re.match(
r'^\s*/\* eslint-(disable|enable) ([\w-]*) \*/$', line)
if match:
if match.group(1) == 'disable':
# |line| disables a rule; validate it isn't already disabled.
if match.group(2) in disabled:
logging.error('%s:%d Rule %r already disabled',
rel_path, i + 1, match.group(2))
has_error = True
else:
disabled.append(match.group(2))
else:
# |line| enabled a rule; validate it's already disabled and it's
# enabled in the correct order.
if not disabled or match.group(2) not in disabled:
logging.error("%s:%d Rule %r isn't disabled",
rel_path, i + 1, match.group(2))
has_error = True
elif disabled[-1] != match.group(2):
logging.error('%s:%d Rule %r enabled out of order',
rel_path, i + 1, match.group(2))
has_error = True
disabled = [
x for x in disabled if x != match.group(2)
]
else:
disabled = disabled[:-1]
else:
# |line| is not a normal eslint-disable or eslint-enable line. Verify
# we don't have this text elsewhere where eslint will ignore it.
if re.search(r'eslint-(disable|enable)(?!-(next-)?line)',
line):
logging.error('%s:%d Invalid eslint-disable', rel_path,
i + 1)
has_error = True
for rule in disabled:
logging.error('%s:%d Rule %r still disabled at end of file',
rel_path, i + 1, rule)
has_error = True
return not has_error
def compile(self, options, force=False):
"""Builds the files in |self.source_files| using the given Closure
command-line options.
Args:
options: An array of options to give to Closure.
force: Generate the output even if the inputs have not changed.
Returns:
True on success; False on failure.
"""
if not force:
if self.timestamp_file:
if not _must_build(self.timestamp_file, self.source_files):
return True
else:
if not _must_build(self.compiled_js_path, self.source_files):
return True
jar = _get_source_path(
'node_modules/google-closure-compiler-java/compiler.jar')
output_options = []
if self.output_compiled_bundle:
output_options += [
'--js_output_file',
self.compiled_js_path,
]
if self.add_source_map:
source_base = _get_source_path('')
output_options += [
'--create_source_map',
self.source_map_path,
# This uses a simple string replacement to create relative paths.
# "source|replacement".
'--source_map_location_mapping',
source_base + '|../',
]
if shakaBuildHelpers.is_windows(
) or shakaBuildHelpers.is_cygwin():
output_options += [
# On Windows, the source map needs to use '/' for paths, so we
# need to have this mapping so it creates the correct relative
# paths. For some reason, we still need the mapping above for
# other parts of the source map.
'--source_map_location_mapping',
source_base.replace('\\', '/') + '|../',
]
if self.add_wrapper:
output_options += self._prepare_wrapper()
cmd_line = ['java', '-jar', jar] + output_options + options
cmd_line += self.source_files
if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
logging.error('Build failed')
return False
if self.output_compiled_bundle and self.add_source_map:
# Add a special source-mapping comment so that Chrome and Firefox can map
# line and character numbers from the compiled library back to the
# original source locations.
with shakaBuildHelpers.open_file(self.compiled_js_path, 'a') as f:
f.write('//# sourceMappingURL=%s' %
os.path.basename(self.source_map_path))
if self.timestamp_file:
_update_timestamp(self.timestamp_file)
return True
def parse_build(self, lines, root):
"""Parses a Build object from the given lines of commands.
This will recursively read and parse builds.
Args:
lines: An array of strings defining commands.
root: The full path to the base directory.
Returns:
True on success, False otherwise.
"""
for line in lines:
# Strip comments
try:
line = line[:line.index('#')]
except ValueError:
pass
# Strip whitespace and ignore empty lines.
line = line.strip()
if not line:
continue
if line[0] == '+':
is_neg = False
line = line[1:].strip()
elif line[0] == '-':
is_neg = True
line = line[1:].strip()
else:
logging.error('Operation (+/-) required')
return False
if line[0] == '@':
line = line[1:].strip()
build_path = self._get_build_file_path(line, root)
if not build_path:
return False
lines = shakaBuildHelpers.open_file(build_path).readlines()
sub_root = os.path.dirname(build_path)
# If this is a build file, then recurse and combine the builds.
sub_build = Build()
if not sub_build.parse_build(lines, sub_root):
return False
if is_neg:
self._combine(sub_build.reverse())
else:
self._combine(sub_build)
else:
if not os.path.isabs(line):
line = os.path.abspath(os.path.join(root, line))
if not os.path.isfile(line):
logging.error('Unable to find file: %s', line)
return False
if is_neg:
self.include.discard(line)
self.exclude.add(line)
else:
self.include.add(line)
self.exclude.discard(line)
return True
def main(args):
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-d',
'--dot-format',
action='store_true',
help='Prints in DOT format.')
parser.add_argument('source_map',
nargs='?',
default='shaka-player.compiled.map',
help='The source map or the name of the build to use.')
print_types = parser.add_mutually_exclusive_group(required=True)
print_types.add_argument('-c',
'--class-deps',
action='store_true',
help='Prints the class dependencies.')
print_types.add_argument('-f',
'--function-deps',
action='store_true',
help='Prints the function dependencies.')
print_types.add_argument(
'-s',
'--function-sizes',
action='store_true',
help='Prints the function sizes (in number of characters).')
print_types.add_argument('-t',
'--all-tokens',
action='store_true',
help='Prints all tokens in the source map.')
options = parser.parse_args(args)
# Verify arguments are correct.
if (options.dot_format and not options.function_deps
and not options.class_deps):
parser.error('--dot-format only valid with --function-deps or '
'--class-deps.')
# Try to find the file
name = options.source_map
if not os.path.isfile(name):
# Get the source code base directory
base = shakaBuildHelpers.get_source_base()
# Supports the following searches:
# * File name given, map in dist/
# * Type given, map in working directory
# * Type given, map in dist/
if os.path.isfile(os.path.join(base, 'dist', name)):
name = os.path.join(base, 'dist', name)
elif os.path.isfile(os.path.join('shaka-player.' + name +
'.debug.map')):
name = os.path.join('shaka-player.' + name + '.debug.map')
elif os.path.isfile(
os.path.join(base, 'dist',
'shaka-player.' + name + '.debug.map')):
name = os.path.join(base, 'dist',
'shaka-player.' + name + '.debug.map')
else:
logging.error('"%s" not found; build Shaka first.', name)
return 1
with shakaBuildHelpers.open_file(name, 'r') as f:
process(f.read(), options)
return 0
def check_eslint_disable(_):
"""Checks that source files correctly use "eslint-disable".
- Rules are disabled/enabled in nested blocks.
- Rules are not disabled multiple times.
- Rules are enabled again by the end of the file.
Returns:
True on success, False on failure.
"""
logging.info('Checking correct usage of eslint-disable...')
complete = build.Build()
# Normally we don't need to include @core, but because we look at the build
# object directly, we need to include it here. When using main(), it will
# call addCore which will ensure core is included.
if not complete.parse_build(['+@complete', '+@core'], os.getcwd()):
logging.error('Error parsing complete build')
return False
base = shakaBuildHelpers.get_source_base()
complete.include.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'test'),
re.compile(r'.*\.js$')))
complete.include.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'demo'),
re.compile(r'.*\.js$')))
has_error = False
for path in complete.include:
# The stack of rules that are disabled.
disabled = []
with shakaBuildHelpers.open_file(path, 'r') as f:
rel_path = os.path.relpath(path, base)
for i, line in enumerate(f):
match = re.match(
r'^\s*/\* eslint-(disable|enable) ([\w-]*) \*/$', line)
if match:
if match.group(1) == 'disable':
# |line| disables a rule; validate it isn't already disabled.
if match.group(2) in disabled:
logging.error('%s:%d Rule %r already disabled',
rel_path, i + 1, match.group(2))
has_error = True
else:
disabled.append(match.group(2))
else:
# |line| enabled a rule; validate it's already disabled and it's
# enabled in the correct order.
if not disabled or match.group(2) not in disabled:
logging.error("%s:%d Rule %r isn't disabled",
rel_path, i + 1, match.group(2))
has_error = True
elif disabled[-1] != match.group(2):
logging.error('%s:%d Rule %r enabled out of order',
rel_path, i + 1, match.group(2))
has_error = True
disabled = [
x for x in disabled if x != match.group(2)
]
else:
disabled = disabled[:-1]
else:
# |line| is not a normal eslint-disable or eslint-enable line. Verify
# we don't have this text elsewhere where eslint will ignore it.
if re.search(r'eslint-(disable|enable)(?!-(next-)?line)',
line):
logging.error('%s:%d Invalid eslint-disable', rel_path,
i + 1)
has_error = True
for rule in disabled:
logging.error('%s:%d Rule %r still disabled at end of file',
rel_path, i + 1, rule)
has_error = True
return not has_error
def changelog_version():
"""Gets the version of the library from the CHANGELOG."""
path = os.path.join(shakaBuildHelpers.get_source_base(), 'CHANGELOG.md')
with shakaBuildHelpers.open_file(path, 'r') as f:
match = re.search(r'## (.*) \(', f.read())
return match.group(1) if match else ''
