def install_dependency(call, output_dir): """Installs a dependency into a directory. Assumes that there is no output on stdout if installation fails. Args: call: List of arguments to call to install the dependency, e.g. ['npm', 'install', 'bower']. output_dir: Directory to install into. Raises: InstallationError """ popen = subprocess.Popen(call, cwd=output_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = popen.communicate() # Pass info and errors through to the debug log. for line in surrogateescape.decode(stdout).split('\n'): if line: logging.debug('%s: %s', call[0], line) for line in surrogateescape.decode(stderr).split('\n'): if line: logging.debug('%s err: %s', call[0], line) # If installation failed, stdout will be empty. if not stdout: raise InstallationError('Failed to install with command: `{}`.'.format( ' '.join(call)))
def install_dependency(call, output_dir): """Installs a dependency into a directory. Assumes that there is no output on stdout if installation fails. Args: call: List of arguments to call to install the dependency, e.g. ['npm', 'install', 'bower']. output_dir: Directory to install into. Raises: InstallationError """ popen = subprocess.Popen(call, cwd=output_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = popen.communicate() # Pass info and errors through to the debug log. for line in surrogateescape.decode(stdout).split('\n'): if line: logging.debug('%s: %s', call[0], line) for line in surrogateescape.decode(stderr).split('\n'): if line: logging.debug('%s err: %s', call[0], line) # If installation failed, stdout will be empty. if not stdout: raise InstallationError( 'Failed to install with command: `{}`.'.format(' '.join(call)))
def insert_todos_into_file(js_path): """Inserts TODO comments in a JavaScript file. The TODO comments inserted should draw attention to places in the converted app that the developer will need to edit to finish converting their app. Args: js_path: Path to JavaScript file. """ with open(js_path) as in_js_file: # This search is very naïve and will only check line-by-line if there # are easily spotted Chrome Apps API function calls. out_js_lines = [] for line_no, line in enumerate(in_js_file): line = surrogateescape.decode(line) api_call = chrome_app.apis.api_member_used(line) if api_call is not None: # Construct a TODO comment. newline = '\r\n' if line.endswith('\r\n') else '\n' todo = '// TODO(Caterpillar): Check usage of {}.{}'.format( api_call, newline) logging.debug('Inserting TODO in `%s:%d`:\n\t%s', js_path, line_no, todo) out_js_lines.append(todo) out_js_lines.append(line) with open(js_path, 'w') as out_js_file: logging.debug('Writing modified file `%s`.', js_path) out_js = surrogateescape.encode(''.join(out_js_lines)) out_js_file.write(out_js)
def edit_code(output_dir, required_js_paths, chrome_app_manifest, config): """Directly edits the code of the output web app. All editing of user code should be called from this function. Args: output_dir: Path to web app. required_js_paths: Paths of scripts to be included in the web app, relative to Caterpillar's boilerplate directory in the output web app. chrome_app_manifest: Manifest dictionary of the _Chrome App_. config: Configuration dictionary. """ logging.debug('Editing web app code.') # Walk the app for JS and HTML. # Insert TODOs into JS. # Inject script and meta tags into HTML. dirwalk = os.walk(output_dir) for (dirpath, _, filenames) in dirwalk: for filename in filenames: path = os.path.join(dirpath, filename) root_path = os.path.relpath(output_dir, dirpath) if filename.endswith('.js'): insert_todos_into_file(path) elif filename.endswith('.html'): logging.debug('Editing `%s`.', path) with open(path) as in_html_file: soup = bs4.BeautifulSoup( surrogateescape.decode(in_html_file.read()), 'html.parser') inject_script_tags( soup, required_js_paths, root_path, config['boilerplate_dir'], path) inject_misc_tags(soup, chrome_app_manifest, root_path, path) logging.debug('Writing edited and prettified `%s`.', path) with open(path, 'w') as out_html_file: out_html_file.write(surrogateescape.encode(soup.prettify()))
def insert_todos_into_file(js_path): """Inserts TODO comments in a JavaScript file. The TODO comments inserted should draw attention to places in the converted app that the developer will need to edit to finish converting their app. Args: js_path: Path to JavaScript file. """ with open(js_path) as in_js_file: # This search is very naïve and will only check line-by-line if there # are easily spotted Chrome Apps API function calls. out_js_lines = [] for line_no, line in enumerate(in_js_file): line = surrogateescape.decode(line) api_call = chrome_app.apis.api_member_used(line) if api_call is not None: # Construct a TODO comment. newline = '\r\n' if line.endswith('\r\n') else '\n' todo = '// TODO(Caterpillar): Check usage of {}.{}'.format(api_call, newline) logging.debug('Inserting TODO in `%s:%d`:\n\t%s', js_path, line_no, todo) out_js_lines.append(todo) out_js_lines.append(line) with open(js_path, 'w') as out_js_file: logging.debug('Writing modified file `%s`.', js_path) out_js = surrogateescape.encode(''.join(out_js_lines)) out_js_file.write(out_js)
def test_surrogate_decode(self): bs = b'latin-1: caf\xe9; utf-8: caf\xc3\xa9' s = bs.decode('utf-8', errors='surrogateescape') self.assertIsInstance(s, unicode) self.assertEqual(s, u'latin-1: caf\udce9; utf-8: caf\xe9') s = surrogateescape.decode(bs) self.assertIsInstance(s, unicode) self.assertEqual(s, u'latin-1: caf\udce9; utf-8: caf\xe9')
def usage(apis, directory, context_size=2, ignore_dirs=None): """Gets information about the usage of Chrome Apps APIs in an app directory. Args: apis: List of API names. directory: Path to app directory. context_size: Number of lines either side of each API usage to consider part of the context for that usage. Default is 2. ignore_dirs: Set of absolute directory paths to ignore. Optional. Returns: Dictionary mapping API names to dictionaries. These dictionaries then map member names to (filepath, linenum, context, context_linenum) tuples. - linenum is the line number of the API usage. - context is a string containing the lines of code surrounding references to the API usage. - context_linenum is the line number that the context starts on. """ if ignore_dirs is None: ignore_dirs = set() # Maps API names to dictionaries that map API members to contexts usage_data = {api: collections.defaultdict(list) for api in apis} api_regexes = { api: re.compile(r'chrome\.{}((?:\.\w+)+)'.format(api)) for api in apis } for js_path in walk.all_paths(directory, extension='js', ignore_dirs=ignore_dirs): with open(js_path, 'rU') as js_file: lines = [surrogateescape.decode(line) for line in js_file] for line_num, line in enumerate(lines): for api, regex in api_regexes.items(): match = regex.search(line) if match: context_linenum = max(0, line_num - context_size) member = match.group(1)[ 1:] # Slice to remove the leading dot. context = lines[context_linenum:line_num + context_size + 1] rel_path = os.path.relpath(js_path, directory) member_usage = (rel_path, line_num, ''.join(context), context_linenum) usage_data[api][member].append(member_usage) return usage_data
def usage(apis, directory, context_size=2, ignore_dirs=None): """Gets information about the usage of Chrome Apps APIs in an app directory. Args: apis: List of API names. directory: Path to app directory. context_size: Number of lines either side of each API usage to consider part of the context for that usage. Default is 2. ignore_dirs: Set of absolute directory paths to ignore. Optional. Returns: Dictionary mapping API names to dictionaries. These dictionaries then map member names to (filepath, linenum, context, context_linenum) tuples. - linenum is the line number of the API usage. - context is a string containing the lines of code surrounding references to the API usage. - context_linenum is the line number that the context starts on. """ if ignore_dirs is None: ignore_dirs = set() # Maps API names to dictionaries that map API members to contexts usage_data = {api: collections.defaultdict(list) for api in apis} api_regexes = {api: re.compile(r'chrome\.{}((?:\.\w+)+)'.format(api)) for api in apis} for js_path in walk.all_paths( directory, extension='js', ignore_dirs=ignore_dirs): with open(js_path, 'rU') as js_file: lines = [surrogateescape.decode(line) for line in js_file] for line_num, line in enumerate(lines): for api, regex in api_regexes.items(): match = regex.search(line) if match: context_linenum = max(0, line_num - context_size) member = match.group(1)[1:] # Slice to remove the leading dot. context = lines[context_linenum:line_num + context_size + 1] rel_path = os.path.relpath(js_path, directory) member_usage = (rel_path, line_num, ''.join(context), context_linenum) usage_data[api][member].append(member_usage) return usage_data
def app_apis(directory): """Returns a set of Chrome APIs used in a given app directory. Args: directory: App directory to search for Chrome APIs. Returns: A sorted list of Chrome API names. """ # This will be done really naively by searching for 'chrome.*'. # Note that this fails for cases like 'var a = chrome; a.tts;'. # For each js file in the directory, add all of the Chrome APIs being used to # a set of Chrome API names. apis = set() for js_path in walk.all_paths(directory, extension='js'): with open(js_path, 'rU') as js_file: js = surrogateescape.decode(js_file.read()) for api_match in CHROME_API_REGEX.finditer(js): apis.add(api_match.group(1)) return sorted(apis)
def edit_code(output_dir, required_js_paths, chrome_app_manifest, config): """Directly edits the code of the output web app. All editing of user code should be called from this function. Args: output_dir: Path to web app. required_js_paths: Paths of scripts to be included in the web app, relative to Caterpillar's boilerplate directory in the output web app. chrome_app_manifest: Manifest dictionary of the _Chrome App_. config: Configuration dictionary. """ logging.debug('Editing web app code.') # Walk the app for JS and HTML. # Insert TODOs into JS. # Inject script and meta tags into HTML. dirwalk = os.walk(output_dir) for (dirpath, _, filenames) in dirwalk: for filename in filenames: path = os.path.join(dirpath, filename) root_path = os.path.relpath(output_dir, dirpath) if filename.endswith('.js'): insert_todos_into_file(path) elif filename.endswith('.html'): logging.debug('Editing `%s`.', path) with open(path) as in_html_file: soup = bs4.BeautifulSoup( surrogateescape.decode(in_html_file.read()), 'html.parser') inject_script_tags(soup, required_js_paths, root_path, config['boilerplate_dir'], path) inject_misc_tags(soup, chrome_app_manifest, root_path, path) logging.debug('Writing edited and prettified `%s`.', path) with open(path, 'w') as out_html_file: out_html_file.write(surrogateescape.encode( soup.prettify()))