def install_dependency(call, output_dir):
"""Installs a dependency into a directory.
Assumes that there is no output on stdout if installation fails.
Args:
call: List of arguments to call to install the dependency, e.g.
['npm', 'install', 'bower'].
output_dir: Directory to install into.
Raises:
InstallationError
"""
popen = subprocess.Popen(call,
cwd=output_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = popen.communicate()
# Pass info and errors through to the debug log.
for line in surrogateescape.decode(stdout).split('\n'):
if line:
logging.debug('%s: %s', call[0], line)
for line in surrogateescape.decode(stderr).split('\n'):
if line:
logging.debug('%s err: %s', call[0], line)
# If installation failed, stdout will be empty.
if not stdout:
raise InstallationError('Failed to install with command: `{}`.'.format(
' '.join(call)))
def install_dependency(call, output_dir):
"""Installs a dependency into a directory.
Assumes that there is no output on stdout if installation fails.
Args:
call: List of arguments to call to install the dependency, e.g.
['npm', 'install', 'bower'].
output_dir: Directory to install into.
Raises:
InstallationError
"""
popen = subprocess.Popen(call, cwd=output_dir, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = popen.communicate()
# Pass info and errors through to the debug log.
for line in surrogateescape.decode(stdout).split('\n'):
if line:
logging.debug('%s: %s', call[0], line)
for line in surrogateescape.decode(stderr).split('\n'):
if line:
logging.debug('%s err: %s', call[0], line)
# If installation failed, stdout will be empty.
if not stdout:
raise InstallationError(
'Failed to install with command: `{}`.'.format(' '.join(call)))
def insert_todos_into_file(js_path):
"""Inserts TODO comments in a JavaScript file.
The TODO comments inserted should draw attention to places in the converted
app that the developer will need to edit to finish converting their app.
Args:
js_path: Path to JavaScript file.
"""
with open(js_path) as in_js_file:
# This search is very naïve and will only check line-by-line if there
# are easily spotted Chrome Apps API function calls.
out_js_lines = []
for line_no, line in enumerate(in_js_file):
line = surrogateescape.decode(line)
api_call = chrome_app.apis.api_member_used(line)
if api_call is not None:
# Construct a TODO comment.
newline = '\r\n' if line.endswith('\r\n') else '\n'
todo = '// TODO(Caterpillar): Check usage of {}.{}'.format(
api_call, newline)
logging.debug('Inserting TODO in `%s:%d`:\n\t%s', js_path,
line_no, todo)
out_js_lines.append(todo)
out_js_lines.append(line)
with open(js_path, 'w') as out_js_file:
logging.debug('Writing modified file `%s`.', js_path)
out_js = surrogateescape.encode(''.join(out_js_lines))
out_js_file.write(out_js)
def edit_code(output_dir, required_js_paths, chrome_app_manifest, config):
"""Directly edits the code of the output web app.
All editing of user code should be called from this function.
Args:
output_dir: Path to web app.
required_js_paths: Paths of scripts to be included in the web app, relative
to Caterpillar's boilerplate directory in the output web app.
chrome_app_manifest: Manifest dictionary of the _Chrome App_.
config: Configuration dictionary.
"""
logging.debug('Editing web app code.')
# Walk the app for JS and HTML.
# Insert TODOs into JS.
# Inject script and meta tags into HTML.
dirwalk = os.walk(output_dir)
for (dirpath, _, filenames) in dirwalk:
for filename in filenames:
path = os.path.join(dirpath, filename)
root_path = os.path.relpath(output_dir, dirpath)
if filename.endswith('.js'):
insert_todos_into_file(path)
elif filename.endswith('.html'):
logging.debug('Editing `%s`.', path)
with open(path) as in_html_file:
soup = bs4.BeautifulSoup(
surrogateescape.decode(in_html_file.read()), 'html.parser')
inject_script_tags(
soup, required_js_paths, root_path, config['boilerplate_dir'], path)
inject_misc_tags(soup, chrome_app_manifest, root_path, path)
logging.debug('Writing edited and prettified `%s`.', path)
with open(path, 'w') as out_html_file:
out_html_file.write(surrogateescape.encode(soup.prettify()))
def insert_todos_into_file(js_path):
"""Inserts TODO comments in a JavaScript file.
The TODO comments inserted should draw attention to places in the converted
app that the developer will need to edit to finish converting their app.
Args:
js_path: Path to JavaScript file.
"""
with open(js_path) as in_js_file:
# This search is very naïve and will only check line-by-line if there
# are easily spotted Chrome Apps API function calls.
out_js_lines = []
for line_no, line in enumerate(in_js_file):
line = surrogateescape.decode(line)
api_call = chrome_app.apis.api_member_used(line)
if api_call is not None:
# Construct a TODO comment.
newline = '\r\n' if line.endswith('\r\n') else '\n'
todo = '// TODO(Caterpillar): Check usage of {}.{}'.format(api_call,
newline)
logging.debug('Inserting TODO in `%s:%d`:\n\t%s', js_path, line_no,
todo)
out_js_lines.append(todo)
out_js_lines.append(line)
with open(js_path, 'w') as out_js_file:
logging.debug('Writing modified file `%s`.', js_path)
out_js = surrogateescape.encode(''.join(out_js_lines))
out_js_file.write(out_js)
def test_surrogate_decode(self):
bs = b'latin-1: caf\xe9; utf-8: caf\xc3\xa9'
s = bs.decode('utf-8', errors='surrogateescape')
self.assertIsInstance(s, unicode)
self.assertEqual(s, u'latin-1: caf\udce9; utf-8: caf\xe9')
s = surrogateescape.decode(bs)
self.assertIsInstance(s, unicode)
self.assertEqual(s, u'latin-1: caf\udce9; utf-8: caf\xe9')
def test_surrogate_decode(self):
bs = b'latin-1: caf\xe9; utf-8: caf\xc3\xa9'
s = bs.decode('utf-8', errors='surrogateescape')
self.assertIsInstance(s, unicode)
self.assertEqual(s, u'latin-1: caf\udce9; utf-8: caf\xe9')
s = surrogateescape.decode(bs)
self.assertIsInstance(s, unicode)
self.assertEqual(s, u'latin-1: caf\udce9; utf-8: caf\xe9')
def usage(apis, directory, context_size=2, ignore_dirs=None):
"""Gets information about the usage of Chrome Apps APIs in an app directory.
Args:
apis: List of API names.
directory: Path to app directory.
context_size: Number of lines either side of each API usage to consider part
of the context for that usage. Default is 2.
ignore_dirs: Set of absolute directory paths to ignore. Optional.
Returns:
Dictionary mapping API names to dictionaries. These dictionaries then map
member names to (filepath, linenum, context, context_linenum) tuples.
- linenum is the line number of the API usage.
- context is a string containing the lines of code surrounding references
to the API usage.
- context_linenum is the line number that the context starts on.
"""
if ignore_dirs is None:
ignore_dirs = set()
# Maps API names to dictionaries that map API members to contexts
usage_data = {api: collections.defaultdict(list) for api in apis}
api_regexes = {
api: re.compile(r'chrome\.{}((?:\.\w+)+)'.format(api))
for api in apis
}
for js_path in walk.all_paths(directory,
extension='js',
ignore_dirs=ignore_dirs):
with open(js_path, 'rU') as js_file:
lines = [surrogateescape.decode(line) for line in js_file]
for line_num, line in enumerate(lines):
for api, regex in api_regexes.items():
match = regex.search(line)
if match:
context_linenum = max(0, line_num - context_size)
member = match.group(1)[
1:] # Slice to remove the leading dot.
context = lines[context_linenum:line_num + context_size +
1]
rel_path = os.path.relpath(js_path, directory)
member_usage = (rel_path, line_num, ''.join(context),
context_linenum)
usage_data[api][member].append(member_usage)
return usage_data
def usage(apis, directory, context_size=2, ignore_dirs=None):
"""Gets information about the usage of Chrome Apps APIs in an app directory.
Args:
apis: List of API names.
directory: Path to app directory.
context_size: Number of lines either side of each API usage to consider part
of the context for that usage. Default is 2.
ignore_dirs: Set of absolute directory paths to ignore. Optional.
Returns:
Dictionary mapping API names to dictionaries. These dictionaries then map
member names to (filepath, linenum, context, context_linenum) tuples.
- linenum is the line number of the API usage.
- context is a string containing the lines of code surrounding references
to the API usage.
- context_linenum is the line number that the context starts on.
"""
if ignore_dirs is None:
ignore_dirs = set()
# Maps API names to dictionaries that map API members to contexts
usage_data = {api: collections.defaultdict(list) for api in apis}
api_regexes = {api: re.compile(r'chrome\.{}((?:\.\w+)+)'.format(api))
for api in apis}
for js_path in walk.all_paths(
directory, extension='js', ignore_dirs=ignore_dirs):
with open(js_path, 'rU') as js_file:
lines = [surrogateescape.decode(line) for line in js_file]
for line_num, line in enumerate(lines):
for api, regex in api_regexes.items():
match = regex.search(line)
if match:
context_linenum = max(0, line_num - context_size)
member = match.group(1)[1:] # Slice to remove the leading dot.
context = lines[context_linenum:line_num + context_size + 1]
rel_path = os.path.relpath(js_path, directory)
member_usage = (rel_path, line_num, ''.join(context), context_linenum)
usage_data[api][member].append(member_usage)
return usage_data
def app_apis(directory):
"""Returns a set of Chrome APIs used in a given app directory.
Args:
directory: App directory to search for Chrome APIs.
Returns:
A sorted list of Chrome API names.
"""
# This will be done really naively by searching for 'chrome.*'.
# Note that this fails for cases like 'var a = chrome; a.tts;'.
# For each js file in the directory, add all of the Chrome APIs being used to
# a set of Chrome API names.
apis = set()
for js_path in walk.all_paths(directory, extension='js'):
with open(js_path, 'rU') as js_file:
js = surrogateescape.decode(js_file.read())
for api_match in CHROME_API_REGEX.finditer(js):
apis.add(api_match.group(1))
return sorted(apis)
def app_apis(directory):
"""Returns a set of Chrome APIs used in a given app directory.
Args:
directory: App directory to search for Chrome APIs.
Returns:
A sorted list of Chrome API names.
"""
# This will be done really naively by searching for 'chrome.*'.
# Note that this fails for cases like 'var a = chrome; a.tts;'.
# For each js file in the directory, add all of the Chrome APIs being used to
# a set of Chrome API names.
apis = set()
for js_path in walk.all_paths(directory, extension='js'):
with open(js_path, 'rU') as js_file:
js = surrogateescape.decode(js_file.read())
for api_match in CHROME_API_REGEX.finditer(js):
apis.add(api_match.group(1))
return sorted(apis)
def edit_code(output_dir, required_js_paths, chrome_app_manifest, config):
"""Directly edits the code of the output web app.
All editing of user code should be called from this function.
Args:
output_dir: Path to web app.
required_js_paths: Paths of scripts to be included in the web app, relative
to Caterpillar's boilerplate directory in the output web app.
chrome_app_manifest: Manifest dictionary of the _Chrome App_.
config: Configuration dictionary.
"""
logging.debug('Editing web app code.')
# Walk the app for JS and HTML.
# Insert TODOs into JS.
# Inject script and meta tags into HTML.
dirwalk = os.walk(output_dir)
for (dirpath, _, filenames) in dirwalk:
for filename in filenames:
path = os.path.join(dirpath, filename)
root_path = os.path.relpath(output_dir, dirpath)
if filename.endswith('.js'):
insert_todos_into_file(path)
elif filename.endswith('.html'):
logging.debug('Editing `%s`.', path)
with open(path) as in_html_file:
soup = bs4.BeautifulSoup(
surrogateescape.decode(in_html_file.read()),
'html.parser')
inject_script_tags(soup, required_js_paths, root_path,
config['boilerplate_dir'], path)
inject_misc_tags(soup, chrome_app_manifest, root_path, path)
logging.debug('Writing edited and prettified `%s`.', path)
with open(path, 'w') as out_html_file:
out_html_file.write(surrogateescape.encode(
soup.prettify()))
