def check_complete(_):
"""Checks whether the 'complete' build references every file.
This is used by the build script to ensure that every file is included in at
least one build type.
Returns:
True on success, False on failure.
"""
logging.info('Checking that the build files are complete...')
complete_build = complete_build_files()
if not complete_build:
return False
match = re.compile(r'.*\.js$')
base = shakaBuildHelpers.get_source_base()
all_files = set()
all_files.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'lib'), match))
all_files.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'ui'), match))
missing_files = all_files - complete_build
if missing_files:
logging.error('There are files missing from the complete build:')
for missing in missing_files:
# Convert to a path relative to source base.
logging.error(' ' + os.path.relpath(missing, base))
return False
return True
def check_complete(_):
"""Checks whether the 'complete' build references every file.
This is used by the build script to ensure that every file is included in at
least one build type.
Returns:
True on success, False on failure.
"""
logging.info('Checking that the build files are complete...')
complete = build.Build()
# Normally we don't need to include @core, but because we look at the build
# object directly, we need to include it here. When using main(), it will
# call addCore which will ensure core is included.
if not complete.parse_build(['+@complete', '+@core'], os.getcwd()):
logging.error('Error parsing complete build')
return False
match = re.compile(r'.*\.js$')
base = shakaBuildHelpers.get_source_base()
all_files = set()
all_files.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'lib'), match))
all_files.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'ui'), match))
missing_files = all_files - complete.include
if missing_files:
logging.error('There are files missing from the complete build:')
for missing in missing_files:
# Convert to a path relative to source base.
logging.error(' ' + os.path.relpath(missing, base))
return False
return True
def check_complete(_):
"""Checks whether the 'complete' build references every file.
This is used by the build script to ensure that every file is included in at
least one build type.
Returns:
True on success, False on failure.
"""
logging.info('Checking that the build files are complete...')
complete = build.Build()
# Normally we don't need to include @core, but because we look at the build
# object directly, we need to include it here. When using main(), it will
# call addCore which will ensure core is included.
if not complete.parse_build(['+@complete', '+@core'], os.getcwd()):
logging.error('Error parsing complete build')
return False
match = re.compile(r'.*\.js$')
base = shakaBuildHelpers.get_source_base()
all_files = set()
all_files.update(shakaBuildHelpers.get_all_files(os.path.join(base, 'lib'), match))
all_files.update(shakaBuildHelpers.get_all_files(os.path.join(base, 'ui'), match))
missing_files = all_files - complete.include
if missing_files:
logging.error('There are files missing from the complete build:')
for missing in missing_files:
# Convert to a path relative to source base.
logging.error(' ' + os.path.relpath(missing, base))
return False
return True
def add_closure(self):
"""Adds the closure library and externs."""
# Add externs and closure dependencies.
source_base = shakaBuildHelpers.get_source_base()
match = re.compile(r'.*\.js$')
self.include |= set(
shakaBuildHelpers.get_all_files(
os.path.join(source_base, 'externs'), match) +
shakaBuildHelpers.get_all_files(
os.path.join(source_base, 'third_party', 'closure'), match))
def add_closure(self):
"""Adds the closure library and externs."""
# Add externs and closure dependencies.
source_base = shakaBuildHelpers.get_source_base()
match = re.compile(r'.*\.js$')
self.include |= set(
shakaBuildHelpers.get_all_files(
os.path.join(source_base, 'externs'), match) +
shakaBuildHelpers.get_all_files(
os.path.join(source_base, 'third_party', 'closure'), match))
def add_core(self):
"""Adds the core library."""
# Add externs and closure dependencies.
source_base = shakaBuildHelpers.get_source_base()
match = re.compile(r".*\.js$")
self.include |= set(
shakaBuildHelpers.get_all_files(os.path.join(source_base, "externs"), match)
+ shakaBuildHelpers.get_all_files(os.path.join(source_base, "third_party", "closure"), match)
)
# Check that there are no files in 'core' that are removed
core_build = Build()
core_build.parse_build(["+@core"], os.getcwd())
core_files = core_build.include
if self.exclude & core_files:
print >> sys.stderr, "Cannot exclude files from core"
self.include |= core_files
def add_core(self):
"""Adds the core library."""
# Add externs and closure dependencies.
source_base = shakaBuildHelpers.get_source_base()
match = re.compile(r'.*\.js$')
self.include |= set(
shakaBuildHelpers.get_all_files(
os.path.join(source_base, 'externs'), match) +
shakaBuildHelpers.get_all_files(
os.path.join(source_base, 'third_party', 'closure'), match))
# Check that there are no files in 'core' that are removed
core_build = Build()
core_build.parse_build(['+@core'], os.getcwd())
core_files = core_build.include
if self.exclude & core_files:
logging.error('Cannot exclude files from core')
self.include |= core_files
def add_core(self):
"""Adds the core library."""
# Add externs and closure dependencies.
source_base = shakaBuildHelpers.get_source_base()
match = re.compile(r'.*\.js$')
self.include |= set(
shakaBuildHelpers.get_all_files(
os.path.join(source_base, 'externs'), match) +
shakaBuildHelpers.get_all_files(
os.path.join(source_base, 'third_party', 'closure'), match))
# Check that there are no files in 'core' that are removed
core_build = Build()
core_build.parse_build(['+@core'], os.getcwd())
core_files = core_build.include
if self.exclude & core_files:
logging.error('Cannot exclude files from core')
self.include |= core_files
def check_spelling(_):
"""Checks that source files don't have any common misspellings."""
logging.info('Checking for common misspellings...')
complete = build.Build()
# Normally we don't need to include @core, but because we look at the build
# object directly, we need to include it here. When using main(), it will
# call addCore which will ensure core is included.
if not complete.parse_build(['+@complete', '+@core'], os.getcwd()):
logging.error('Error parsing complete build')
return False
base = shakaBuildHelpers.get_source_base()
complete.include.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'test'),
re.compile(r'.*\.js$')))
complete.include.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'demo'),
re.compile(r'.*\.js$')))
complete.include.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'build'),
re.compile(r'.*\.(js|py)$')))
with shakaBuildHelpers.open_file(
os.path.join(base, 'build', 'misspellings.txt')) as f:
misspellings = ast.literal_eval(f.read())
has_error = False
for path in complete.include:
with shakaBuildHelpers.open_file(path) as f:
for i, line in enumerate(f):
for regex, replace_pattern in misspellings.items():
for match in re.finditer(regex, line):
repl = match.expand(replace_pattern)
if match.group(0).lower() == repl:
continue # No-op suggestion
if not has_error:
logging.error(
'The following file(s) have misspellings:')
logging.error(
' %s:%d:%d: Did you mean %r?' % (os.path.relpath(
path, base), i + 1, match.start() + 1, repl))
has_error = True
return not has_error
def compile_less(path_name, main_file_name, parsed_args):
match = re.compile(r'.*\.less$')
base = shakaBuildHelpers.get_source_base()
main_less_src = os.path.join(base, path_name, main_file_name + '.less')
all_less_srcs = shakaBuildHelpers.get_all_files(
os.path.join(base, path_name), match)
output = os.path.join(base, 'dist', main_file_name + '.css')
less = compiler.Less(main_less_src, all_less_srcs, output)
return less.compile(parsed_args.force)
def compile_less(path_name, main_file_name, parsed_args):
match = re.compile(r'.*\.less$')
base = shakaBuildHelpers.get_source_base()
main_less_src = os.path.join(base, path_name, main_file_name + '.less')
all_less_srcs = shakaBuildHelpers.get_all_files(
os.path.join(base, path_name), match)
output = os.path.join(base, 'dist', main_file_name + '.css')
less = compiler.Less(main_less_src, all_less_srcs, output)
return less.compile(parsed_args.force)
def check_spelling(_):
"""Checks that source files don't have any common misspellings."""
logging.info('Checking for common misspellings...')
complete_build = complete_build_files()
if not complete_build:
return False
base = shakaBuildHelpers.get_source_base()
complete_build.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'test'),
re.compile(r'.*\.js$')))
complete_build.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'demo'),
re.compile(r'.*\.js$')))
complete_build.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'build'),
re.compile(r'.*\.(js|py)$')))
with shakaBuildHelpers.open_file(
os.path.join(base, 'build', 'misspellings.txt')) as f:
misspellings = ast.literal_eval(f.read())
has_error = False
for path in complete_build:
with shakaBuildHelpers.open_file(path) as f:
for i, line in enumerate(f):
for regex, replace_pattern in misspellings.items():
for match in re.finditer(regex, line):
repl = match.expand(replace_pattern).lower()
if match.group(0).lower() == repl:
continue # No-op suggestion
if not has_error:
logging.error(
'The following file(s) have misspellings:')
logging.error(
' %s:%d:%d: Did you mean %r?' % (os.path.relpath(
path, base), i + 1, match.start() + 1, repl))
has_error = True
return not has_error
def __init__(self, config_path):
self.config_path = config_path
self.source_files = shakaBuildHelpers.get_all_files(
_get_source_path('docs/tutorials'))
self.source_files += shakaBuildHelpers.get_all_files(
_get_source_path('docs/jsdoc-template'))
self.source_files += [
_get_source_path('docs/jsdoc-plugin.js'),
_get_source_path('docs/api-mainpage.md'),
]
# Just one of many output files, used to check the freshness of the docs.
self.output = _get_source_path('docs/api/index.html')
# To avoid getting out of sync with the source files jsdoc actually reads,
# parse the config file and locate all source files based on that.
with open(self.config_path, 'r') as f:
config = json.load(f)
for path in config['source']['include']:
full_path = _get_source_path(path)
self.source_files += shakaBuildHelpers.get_all_js_files(full_path)
def __init__(self, config_path):
self.config_path = config_path
self.source_files = []
# Just one of many output files, used to check the freshness of the docs.
self.output = _get_source_path('docs/api/index.html')
# To avoid getting out of sync with the source files jsdoc actually reads,
# parse the config file and locate all source files based on that.
match = re.compile(r'.*\.js$')
with open(self.config_path, 'rb') as f:
config = json.load(f)
for path in config['source']['include']:
full_path = _get_source_path(path)
self.source_files += shakaBuildHelpers.get_all_files(
full_path, match)
def check_externs():
"""Runs an extra compile pass over the generated externs to ensure that they
are usable.
Returns:
True on success, False on failure.
"""
logging.info('Checking the usability of generated externs...')
# Create a complete "build" object.
externs_build = build.Build()
if not externs_build.parse_build(['+@complete'], os.getcwd()):
return False
externs_build.add_core()
# Use it to generate externs for the next check.
if not externs_build.generate_externs('check'):
return False
# Create a custom "build" object, add all manually-written externs, then add
# the generated externs we just generated.
source_base = shakaBuildHelpers.get_source_base()
manual_externs = shakaBuildHelpers.get_all_files(
os.path.join(source_base, 'externs'), re.compile(r'.*\.js$'))
generated_externs = os.path.join(source_base, 'dist',
'shaka-player.check.externs.js')
check_build = build.Build()
check_build.include = set(manual_externs)
check_build.include.add(generated_externs)
# Build with the complete set of externs, but without any application code.
# This will help find issues in the generated externs, independent of the app.
# Since we have no app, don't use the defines. Unused defines cause a
# compilation error.
closure_opts = build.common_closure_opts + build.debug_closure_opts + [
'--checks-only', '-O', 'SIMPLE'
]
ok = check_build.build_raw(closure_opts)
# Clean up the temporary externs we just generated.
os.unlink(generated_externs)
# Return the success/failure of the build above.
return ok
def check_externs(_):
"""Runs an extra compile pass over the generated externs to ensure that they
are usable.
Returns:
True on success, False on failure.
"""
logging.info('Checking the usability of generated externs...')
# Create a complete "build" object.
externs_build = build.Build()
if not externs_build.parse_build(['+@complete'], os.getcwd()):
return False
externs_build.add_core()
# Use it to generate externs for the next check.
if not externs_build.generate_externs('check'):
return False
# Create a custom "build" object, add all manually-written externs, then add
# the generated externs we just generated.
source_base = shakaBuildHelpers.get_source_base()
manual_externs = shakaBuildHelpers.get_all_files(
os.path.join(source_base, 'externs'), re.compile(r'.*\.js$'))
generated_externs = os.path.join(
source_base, 'dist', 'shaka-player.check.externs.js')
check_build = build.Build()
check_build.include = set(manual_externs)
check_build.include.add(generated_externs)
# Build with the complete set of externs, but without any application code.
# This will help find issues in the generated externs, independent of the app.
# Since we have no app, don't use the defines. Unused defines cause a
# compilation error.
closure_opts = build.common_closure_opts + build.debug_closure_opts + [
'--checks-only', '-O', 'SIMPLE'
]
ok = check_build.build_raw(closure_opts)
# Clean up the temporary externs we just generated.
os.unlink(generated_externs)
# Return the success/failure of the build above.
return ok
def get(*args):
return shakaBuildHelpers.get_all_files(os.path.join(base, *args),
match)
def main(args):
parser = argparse.ArgumentParser(
description='User facing build script for building the Shaka'
' Player Project.')
parser.add_argument('--fix',
help='Automatically fix style violations.',
action='store_true')
parser.add_argument(
'--force',
'-f',
help='Force building the library even if no files have changed.',
action='store_true')
parser.add_argument(
'--debug',
help='Limit which build types to build. Will at least build the debug '
'version.',
action='store_true')
parser.add_argument(
'--release',
help='Limit which build types to build. Will at least build the '
'release version.',
action='store_true')
parsed_args = parser.parse_args(args)
if gendeps.main([]) != 0:
return 1
check_args = []
if parsed_args.fix:
check_args += ['--fix']
if parsed_args.force:
check_args += ['--force']
if check.main(check_args) != 0:
return 1
docs_args = []
if parsed_args.force:
docs_args += ['--force']
if docs.main(docs_args) != 0:
return 1
match = re.compile(r'.*\.less$')
base = shakaBuildHelpers.get_source_base()
main_less_src = os.path.join(base, 'ui', 'controls.less')
all_less_srcs = shakaBuildHelpers.get_all_files(os.path.join(base, 'ui'),
match)
output = os.path.join(base, 'dist', 'controls.css')
less = compiler.Less(main_less_src, all_less_srcs, output)
if not less.compile(parsed_args.force):
return 1
build_args_with_ui = ['--name', 'ui', '+@complete']
build_args_without_ui = ['--name', 'compiled', '+@complete', '-@ui']
if parsed_args.force:
build_args_with_ui += ['--force']
build_args_without_ui += ['--force']
# Create the list of build modes to build with. If the list is empty
# by the end, then populate it with every mode.
modes = []
modes += ['debug'] if parsed_args.debug else []
modes += ['release'] if parsed_args.release else []
# If --debug or --release are not given, build with everything.
if not modes:
modes += ['debug', 'release']
for mode in modes:
# Complete build includes the UI library, but it is optional and player lib
# should build and work without it as well.
# First, build the full build (UI included) and then build excluding UI.
for build_args in [build_args_with_ui, build_args_without_ui]:
if build.main(build_args + ['--mode', mode]) != 0:
return 1
is_debug = mode == 'debug'
if not apps.build_all(parsed_args.force, is_debug):
return 1
return 0
def get(*path_components):
return shakaBuildHelpers.get_all_files(
os.path.join(base, *path_components), match)
def check_eslint_disable(_):
"""Checks that source files correctly use "eslint-disable".
- Rules are disabled/enabled in nested blocks.
- Rules are not disabled multiple times.
- Rules are enabled again by the end of the file.
Returns:
True on success, False on failure.
"""
logging.info('Checking correct usage of eslint-disable...')
complete = build.Build()
# Normally we don't need to include @core, but because we look at the build
# object directly, we need to include it here. When using main(), it will
# call addCore which will ensure core is included.
if not complete.parse_build(['+@complete', '+@core'], os.getcwd()):
logging.error('Error parsing complete build')
return False
base = shakaBuildHelpers.get_source_base()
complete.include.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'test'),
re.compile(r'.*\.js$')))
complete.include.update(
shakaBuildHelpers.get_all_files(os.path.join(base, 'demo'),
re.compile(r'.*\.js$')))
has_error = False
for path in complete.include:
# The stack of rules that are disabled.
disabled = []
with open(path, 'r') as f:
rel_path = os.path.relpath(path, base)
for i, line in enumerate(f):
match = re.match(
r'^\s*/\* eslint-(disable|enable) ([\w-]*) \*/$', line)
if match:
if match.group(1) == 'disable':
# |line| disables a rule; validate it isn't already disabled.
if match.group(2) in disabled:
logging.error('%s:%d Rule %r already disabled',
rel_path, i + 1, match.group(2))
has_error = True
else:
disabled.append(match.group(2))
else:
# |line| enabled a rule; validate it's already disabled and it's
# enabled in the correct order.
if not disabled or match.group(2) not in disabled:
logging.error("%s:%d Rule %r isn't disabled",
rel_path, i + 1, match.group(2))
has_error = True
elif disabled[-1] != match.group(2):
logging.error('%s:%d Rule %r enabled out of order',
rel_path, i + 1, match.group(2))
has_error = True
disabled = [
x for x in disabled if x != match.group(2)
]
else:
disabled = disabled[:-1]
else:
# |line| is not a normal eslint-disable or eslint-enable line. Verify
# we don't have this text elsewhere where eslint will ignore it.
if re.search(r'eslint-(disable|enable)(?!-(next-)?line)',
line):
logging.error('%s:%d Invalid eslint-disable', rel_path,
i + 1)
has_error = True
for rule in disabled:
logging.error('%s:%d Rule %r still disabled at end of file',
rel_path, i + 1, rule)
has_error = True
return not has_error
def get(path):
return shakaBuildHelpers.get_all_files(os.path.join(base, path), match)
def main(args):
parser = argparse.ArgumentParser(
description='User facing build script for building the Shaka'
' Player Project.')
parser.add_argument(
'--locales',
type=str,
nargs='+',
default=generateLocalizations.DEFAULT_LOCALES,
help='The list of locales to compile in (default %(default)r)')
parser.add_argument('--fix',
help='Automatically fix style violations.',
action='store_true')
parser.add_argument(
'--force',
'-f',
help='Force building the library even if no files have changed.',
action='store_true')
parser.add_argument(
'--debug',
help='Limit which build types to build. Will at least build the debug '
'version.',
action='store_true')
parser.add_argument(
'--release',
help='Limit which build types to build. Will at least build the '
'release version.',
action='store_true')
parsed_args = parser.parse_args(args)
# Make the dist/ folder, ignore errors.
base = shakaBuildHelpers.get_source_base()
try:
os.mkdir(os.path.join(base, 'dist'))
except OSError:
pass
# Generate localizations before running gendeps, so the output is available
# to the deps system.
# TODO(#1858): It might be time to look at a third-party build system.
localizations = compiler.GenerateLocalizations(parsed_args.locales)
if not localizations.generate(parsed_args.force):
return 1
if gendeps.main([]) != 0:
return 1
check_args = []
if parsed_args.fix:
check_args += ['--fix']
if parsed_args.force:
check_args += ['--force']
if check.main(check_args) != 0:
return 1
docs_args = []
if parsed_args.force:
docs_args += ['--force']
if docs.main(docs_args) != 0:
return 1
match = re.compile(r'.*\.less$')
main_less_src = os.path.join(base, 'ui', 'controls.less')
all_less_srcs = shakaBuildHelpers.get_all_files(os.path.join(base, 'ui'),
match)
output = os.path.join(base, 'dist', 'controls.css')
less = compiler.Less(main_less_src, all_less_srcs, output)
if not less.compile(parsed_args.force):
return 1
build_args_with_ui = ['--name', 'ui', '+@complete']
build_args_with_ui += ['--locales'] + parsed_args.locales
build_args_without_ui = ['--name', 'compiled', '+@complete', '-@ui']
if parsed_args.force:
build_args_with_ui += ['--force']
build_args_without_ui += ['--force']
# Create the list of build modes to build with. If the list is empty
# by the end, then populate it with every mode.
modes = []
modes += ['debug'] if parsed_args.debug else []
modes += ['release'] if parsed_args.release else []
# If --debug or --release are not given, build with everything.
if not modes:
modes += ['debug', 'release']
for mode in modes:
# Complete build includes the UI library, but it is optional and player lib
# should build and work without it as well.
# First, build the full build (UI included) and then build excluding UI.
for build_args in [build_args_with_ui, build_args_without_ui]:
if build.main(build_args + ['--mode', mode]) != 0:
return 1
is_debug = mode == 'debug'
if not apps.build_all(parsed_args.force, is_debug):
return 1
return 0
def get(*args): return shakaBuildHelpers.get_all_files(os.path.join(base, *args), match)
def get(*path_components):
return shakaBuildHelpers.get_all_files(
os.path.join(base, *path_components), match)
def __init__(self, locales):
self.locales = locales
self.source_files = shakaBuildHelpers.get_all_files(
_get_source_path('ui/locales'))
self.output = _get_source_path('dist/locales.js')