def _prepare_wrapper(self): """Prepares an output wrapper and returns a list of command line arguments for Closure Compiler to use it.""" # Load the wrapper and use Closure to strip whitespace and comments. # This requires %output% in the template to be protected, so Closure doesn't # fail to parse it. wrapper_input_path = _get_source_path('build/wrapper.template.js') wrapper_output_path = _get_source_path('dist/wrapper.js') with shakaBuildHelpers.open_file(wrapper_input_path, 'r') as f: wrapper_code = f.read().replace('%output%', '"%output%"') jar = _get_source_path( 'node_modules/google-closure-compiler-java/compiler.jar') cmd_line = ['java', '-jar', jar, '-O', 'WHITESPACE_ONLY'] proc = shakaBuildHelpers.execute_subprocess(cmd_line, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stripped_wrapper_code = proc.communicate( wrapper_code.encode('utf8'))[0] if proc.returncode != 0: raise RuntimeError('Failed to strip whitespace from wrapper!') with shakaBuildHelpers.open_file(wrapper_output_path, 'w') as f: code = stripped_wrapper_code.decode('utf8') f.write(code.replace('"%output%"', '%output%')) return ['--output_wrapper_file=%s' % wrapper_output_path]
def process(text, options): """Decodes a JSON string containing source map data. Args: text: A JSON string containing source map data. options: An object containing the command-line options. """ # The spec allows a map file to start with )]} to prevent javascript from # including it. if text.startswith(')]}\'\n') or text.startswith(')]}\n'): _, text = text.split('\n', 1) # Decode the JSON data and get the parts we need. data = json.loads(text) # Paths are relative to the output directory. base = os.path.join(shakaBuildHelpers.get_source_base(), 'dist') with shakaBuildHelpers.open_file(os.path.join(base, data['file']), 'r') as f: file_lines = f.readlines() names = data['names'] mappings = data['mappings'] tokens = decode_mappings(mappings, names) sizes = process_sizes(tokens, file_lines) # Print out one of the results. if options.all_tokens: print_tokens(tokens, file_lines, sizes) elif options.function_sizes: print_sizes(sizes) elif options.function_deps or options.class_deps: temp = process_deps(tokens, file_lines, options.class_deps) print_deps(temp, options.dot_format)
def player_version(): """Gets the version of the library from player.js.""" path = os.path.join(shakaBuildHelpers.get_source_base(), 'lib', 'player.js') with shakaBuildHelpers.open_file(path, 'r') as f: match = re.search(r'shaka\.Player\.version = \'(.*)\'', f.read()) return match.group(1) if match else ''
def check_spelling(_): """Checks that source files don't have any common misspellings.""" logging.info('Checking for common misspellings...') complete = build.Build() # Normally we don't need to include @core, but because we look at the build # object directly, we need to include it here. When using main(), it will # call addCore which will ensure core is included. if not complete.parse_build(['+@complete', '+@core'], os.getcwd()): logging.error('Error parsing complete build') return False base = shakaBuildHelpers.get_source_base() complete.include.update( shakaBuildHelpers.get_all_files(os.path.join(base, 'test'), re.compile(r'.*\.js$'))) complete.include.update( shakaBuildHelpers.get_all_files(os.path.join(base, 'demo'), re.compile(r'.*\.js$'))) complete.include.update( shakaBuildHelpers.get_all_files(os.path.join(base, 'build'), re.compile(r'.*\.(js|py)$'))) with shakaBuildHelpers.open_file( os.path.join(base, 'build', 'misspellings.txt')) as f: misspellings = ast.literal_eval(f.read()) has_error = False for path in complete.include: with shakaBuildHelpers.open_file(path) as f: for i, line in enumerate(f): for regex, replace_pattern in misspellings.items(): for match in re.finditer(regex, line): repl = match.expand(replace_pattern) if match.group(0).lower() == repl: continue # No-op suggestion if not has_error: logging.error( 'The following file(s) have misspellings:') logging.error( ' %s:%d:%d: Did you mean %r?' % (os.path.relpath( path, base), i + 1, match.start() + 1, repl)) has_error = True return not has_error
def check_spelling(_): """Checks that source files don't have any common misspellings.""" logging.info('Checking for common misspellings...') complete_build = complete_build_files() if not complete_build: return False base = shakaBuildHelpers.get_source_base() complete_build.update( shakaBuildHelpers.get_all_files(os.path.join(base, 'test'), re.compile(r'.*\.js$'))) complete_build.update( shakaBuildHelpers.get_all_files(os.path.join(base, 'demo'), re.compile(r'.*\.js$'))) complete_build.update( shakaBuildHelpers.get_all_files(os.path.join(base, 'build'), re.compile(r'.*\.(js|py)$'))) with shakaBuildHelpers.open_file( os.path.join(base, 'build', 'misspellings.txt')) as f: misspellings = ast.literal_eval(f.read()) has_error = False for path in complete_build: with shakaBuildHelpers.open_file(path) as f: for i, line in enumerate(f): for regex, replace_pattern in misspellings.items(): for match in re.finditer(regex, line): repl = match.expand(replace_pattern).lower() if match.group(0).lower() == repl: continue # No-op suggestion if not has_error: logging.error( 'The following file(s) have misspellings:') logging.error( ' %s:%d:%d: Did you mean %r?' % (os.path.relpath( path, base), i + 1, match.start() + 1, repl)) has_error = True return not has_error
def _locales_changed(self): # If locales is None, it means we are being called by a caller who doesn't # care what locales are in use. This is true, for example, when we are # running a compiler pass over the tests. if self.locales is None: return False # Find out what locales we used before. If they have changed, we must # regenerate the output. last_locales = None try: prefix = '// LOCALES: ' with shakaBuildHelpers.open_file(self.output, 'r') as f: for line in f: if line.startswith(prefix): last_locales = line.replace(prefix, '').strip().split(', ') except IOError: # The file wasn't found or couldn't be read, so it needs to be redone. return True return set(last_locales) != set(self.locales)
def check_eslint_disable(_): """Checks that source files correctly use "eslint-disable". - Rules are disabled/enabled in nested blocks. - Rules are not disabled multiple times. - Rules are enabled again by the end of the file. Returns: True on success, False on failure. """ logging.info('Checking correct usage of eslint-disable...') complete_build = complete_build_files() if not complete_build: return False base = shakaBuildHelpers.get_source_base() complete_build.update(shakaBuildHelpers.get_all_js_files('test')) complete_build.update(shakaBuildHelpers.get_all_js_files('demo')) has_error = False for path in complete_build: # The stack of rules that are disabled. disabled = [] with shakaBuildHelpers.open_file(path, 'r') as f: rel_path = os.path.relpath(path, base) for i, line in enumerate(f): match = re.match( r'^\s*/\* eslint-(disable|enable) ([\w-]*) \*/$', line) if match: if match.group(1) == 'disable': # |line| disables a rule; validate it isn't already disabled. if match.group(2) in disabled: logging.error('%s:%d Rule %r already disabled', rel_path, i + 1, match.group(2)) has_error = True else: disabled.append(match.group(2)) else: # |line| enabled a rule; validate it's already disabled and it's # enabled in the correct order. if not disabled or match.group(2) not in disabled: logging.error("%s:%d Rule %r isn't disabled", rel_path, i + 1, match.group(2)) has_error = True elif disabled[-1] != match.group(2): logging.error('%s:%d Rule %r enabled out of order', rel_path, i + 1, match.group(2)) has_error = True disabled = [ x for x in disabled if x != match.group(2) ] else: disabled = disabled[:-1] else: # |line| is not a normal eslint-disable or eslint-enable line. Verify # we don't have this text elsewhere where eslint will ignore it. if re.search(r'eslint-(disable|enable)(?!-(next-)?line)', line): logging.error('%s:%d Invalid eslint-disable', rel_path, i + 1) has_error = True for rule in disabled: logging.error('%s:%d Rule %r still disabled at end of file', rel_path, i + 1, rule) has_error = True return not has_error
def compile(self, options, force=False): """Builds the files in |self.source_files| using the given Closure command-line options. Args: options: An array of options to give to Closure. force: Generate the output even if the inputs have not changed. Returns: True on success; False on failure. """ if not force: if self.timestamp_file: if not _must_build(self.timestamp_file, self.source_files): return True else: if not _must_build(self.compiled_js_path, self.source_files): return True jar = _get_source_path( 'node_modules/google-closure-compiler-java/compiler.jar') output_options = [] if self.output_compiled_bundle: output_options += [ '--js_output_file', self.compiled_js_path, ] if self.add_source_map: source_base = _get_source_path('') output_options += [ '--create_source_map', self.source_map_path, # This uses a simple string replacement to create relative paths. # "source|replacement". '--source_map_location_mapping', source_base + '|../', ] if shakaBuildHelpers.is_windows( ) or shakaBuildHelpers.is_cygwin(): output_options += [ # On Windows, the source map needs to use '/' for paths, so we # need to have this mapping so it creates the correct relative # paths. For some reason, we still need the mapping above for # other parts of the source map. '--source_map_location_mapping', source_base.replace('\\', '/') + '|../', ] if self.add_wrapper: output_options += self._prepare_wrapper() cmd_line = ['java', '-jar', jar] + output_options + options cmd_line += self.source_files if shakaBuildHelpers.execute_get_code(cmd_line) != 0: logging.error('Build failed') return False if self.output_compiled_bundle and self.add_source_map: # Add a special source-mapping comment so that Chrome and Firefox can map # line and character numbers from the compiled library back to the # original source locations. with shakaBuildHelpers.open_file(self.compiled_js_path, 'a') as f: f.write('//# sourceMappingURL=%s' % os.path.basename(self.source_map_path)) if self.timestamp_file: _update_timestamp(self.timestamp_file) return True
def parse_build(self, lines, root): """Parses a Build object from the given lines of commands. This will recursively read and parse builds. Args: lines: An array of strings defining commands. root: The full path to the base directory. Returns: True on success, False otherwise. """ for line in lines: # Strip comments try: line = line[:line.index('#')] except ValueError: pass # Strip whitespace and ignore empty lines. line = line.strip() if not line: continue if line[0] == '+': is_neg = False line = line[1:].strip() elif line[0] == '-': is_neg = True line = line[1:].strip() else: logging.error('Operation (+/-) required') return False if line[0] == '@': line = line[1:].strip() build_path = self._get_build_file_path(line, root) if not build_path: return False lines = shakaBuildHelpers.open_file(build_path).readlines() sub_root = os.path.dirname(build_path) # If this is a build file, then recurse and combine the builds. sub_build = Build() if not sub_build.parse_build(lines, sub_root): return False if is_neg: self._combine(sub_build.reverse()) else: self._combine(sub_build) else: if not os.path.isabs(line): line = os.path.abspath(os.path.join(root, line)) if not os.path.isfile(line): logging.error('Unable to find file: %s', line) return False if is_neg: self.include.discard(line) self.exclude.add(line) else: self.include.add(line) self.exclude.discard(line) return True
def main(args): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-d', '--dot-format', action='store_true', help='Prints in DOT format.') parser.add_argument('source_map', nargs='?', default='shaka-player.compiled.map', help='The source map or the name of the build to use.') print_types = parser.add_mutually_exclusive_group(required=True) print_types.add_argument('-c', '--class-deps', action='store_true', help='Prints the class dependencies.') print_types.add_argument('-f', '--function-deps', action='store_true', help='Prints the function dependencies.') print_types.add_argument( '-s', '--function-sizes', action='store_true', help='Prints the function sizes (in number of characters).') print_types.add_argument('-t', '--all-tokens', action='store_true', help='Prints all tokens in the source map.') options = parser.parse_args(args) # Verify arguments are correct. if (options.dot_format and not options.function_deps and not options.class_deps): parser.error('--dot-format only valid with --function-deps or ' '--class-deps.') # Try to find the file name = options.source_map if not os.path.isfile(name): # Get the source code base directory base = shakaBuildHelpers.get_source_base() # Supports the following searches: # * File name given, map in dist/ # * Type given, map in working directory # * Type given, map in dist/ if os.path.isfile(os.path.join(base, 'dist', name)): name = os.path.join(base, 'dist', name) elif os.path.isfile(os.path.join('shaka-player.' + name + '.debug.map')): name = os.path.join('shaka-player.' + name + '.debug.map') elif os.path.isfile( os.path.join(base, 'dist', 'shaka-player.' + name + '.debug.map')): name = os.path.join(base, 'dist', 'shaka-player.' + name + '.debug.map') else: logging.error('"%s" not found; build Shaka first.', name) return 1 with shakaBuildHelpers.open_file(name, 'r') as f: process(f.read(), options) return 0
def check_eslint_disable(_): """Checks that source files correctly use "eslint-disable". - Rules are disabled/enabled in nested blocks. - Rules are not disabled multiple times. - Rules are enabled again by the end of the file. Returns: True on success, False on failure. """ logging.info('Checking correct usage of eslint-disable...') complete = build.Build() # Normally we don't need to include @core, but because we look at the build # object directly, we need to include it here. When using main(), it will # call addCore which will ensure core is included. if not complete.parse_build(['+@complete', '+@core'], os.getcwd()): logging.error('Error parsing complete build') return False base = shakaBuildHelpers.get_source_base() complete.include.update( shakaBuildHelpers.get_all_files(os.path.join(base, 'test'), re.compile(r'.*\.js$'))) complete.include.update( shakaBuildHelpers.get_all_files(os.path.join(base, 'demo'), re.compile(r'.*\.js$'))) has_error = False for path in complete.include: # The stack of rules that are disabled. disabled = [] with shakaBuildHelpers.open_file(path, 'r') as f: rel_path = os.path.relpath(path, base) for i, line in enumerate(f): match = re.match( r'^\s*/\* eslint-(disable|enable) ([\w-]*) \*/$', line) if match: if match.group(1) == 'disable': # |line| disables a rule; validate it isn't already disabled. if match.group(2) in disabled: logging.error('%s:%d Rule %r already disabled', rel_path, i + 1, match.group(2)) has_error = True else: disabled.append(match.group(2)) else: # |line| enabled a rule; validate it's already disabled and it's # enabled in the correct order. if not disabled or match.group(2) not in disabled: logging.error("%s:%d Rule %r isn't disabled", rel_path, i + 1, match.group(2)) has_error = True elif disabled[-1] != match.group(2): logging.error('%s:%d Rule %r enabled out of order', rel_path, i + 1, match.group(2)) has_error = True disabled = [ x for x in disabled if x != match.group(2) ] else: disabled = disabled[:-1] else: # |line| is not a normal eslint-disable or eslint-enable line. Verify # we don't have this text elsewhere where eslint will ignore it. if re.search(r'eslint-(disable|enable)(?!-(next-)?line)', line): logging.error('%s:%d Invalid eslint-disable', rel_path, i + 1) has_error = True for rule in disabled: logging.error('%s:%d Rule %r still disabled at end of file', rel_path, i + 1, rule) has_error = True return not has_error
def changelog_version(): """Gets the version of the library from the CHANGELOG.""" path = os.path.join(shakaBuildHelpers.get_source_base(), 'CHANGELOG.md') with shakaBuildHelpers.open_file(path, 'r') as f: match = re.search(r'## (.*) \(', f.read()) return match.group(1) if match else ''