def verify_node_install(self): # check if Node is installed sys.path.append(mozpath.join(self.topsrcdir, "tools", "lint", "eslint")) import setup_helper with silence(): node_valid = setup_helper.check_node_executables_valid() if not node_valid: # running again to get details printed out setup_helper.check_node_executables_valid() raise ValueError("Can't find Node. did you run ./mach bootstrap ?") return True
def setup(root, **lintargs): setup_helper.set_project_root(root) if not setup_helper.check_node_executables_valid(): return 1 return setup_helper.eslint_maybe_setup()
def _setup_node_packages(self, package_json_path): # Install the browsertime Node.js requirements. sys.path.append(mozpath.join(self.topsrcdir, "tools", "lint", "eslint")) import setup_helper if not setup_helper.check_node_executables_valid(): return should_clobber = self.get_arg("clobber") # To use a custom `geckodriver`, set # os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url) # to an endpoint with binaries named like # https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31. if AUTOMATION: os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true" os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true" self.info( "Installing browsertime node module from {package_json}", package_json=package_json_path, ) install_url = self.get_arg("install-url") setup_helper.package_setup( self.state_path, "browsertime", should_update=install_url is not None, should_clobber=should_clobber, no_optional=install_url or AUTOMATION, )
def runESLint(print_func, files): """Runs ESLint on the files that are passed. Keyword arguments: print_func -- A function to call to print the output. files -- A list of files to be checked. """ try: basepath = setup_helper.get_project_root() if not basepath: return False if not setup_helper.check_node_executables_valid(): return False if setup_helper.eslint_module_needs_setup(): setup_helper.eslint_setup() dir = os.path.join(basepath, "node_modules", ".bin") eslint_path = os.path.join(dir, "eslint") if os.path.exists(os.path.join(dir, "eslint.cmd")): eslint_path = os.path.join(dir, "eslint.cmd") output = check_output( [eslint_path, "--format", "json", "--plugin", "html"] + files, cwd=basepath) display(print_func, output) return True except CalledProcessError as ex: display(print_func, ex.output) return False
def runESLint(print_func, files): """Runs ESLint on the files that are passed. Keyword arguments: print_func -- A function to call to print the output. files -- A list of files to be checked. """ try: basepath = setup_helper.get_project_root() if not basepath: return False if not setup_helper.check_node_executables_valid(): return False if setup_helper.eslint_module_needs_setup(): setup_helper.eslint_setup() dir = os.path.join(basepath, "node_modules", ".bin") eslint_path = os.path.join(dir, "eslint") if os.path.exists(os.path.join(dir, "eslint.cmd")): eslint_path = os.path.join(dir, "eslint.cmd") output = check_output([eslint_path, "--format", "json", "--plugin", "html"] + files, cwd=basepath) display(print_func, output) return True except CalledProcessError as ex: display(print_func, ex.output) return False
def setup(self, should_clobber=False): r'''Install browsertime and visualmetrics.py requirements.''' from mozbuild.action.tooltool import unpack_file from mozbuild.artifact_cache import ArtifactCache sys.path.append(mozpath.join(self.topsrcdir, 'tools', 'lint', 'eslint')) import setup_helper # Download the visualmetrics.py requirements. artifact_cache = ArtifactCache(self.artifact_cache_path, log=self.log, skip_cache=False) fetches = host_fetches[host_platform()] for tool, fetch in sorted(fetches.items()): archive = artifact_cache.fetch(fetch['url']) # TODO: assert type, verify sha256 (and size?). if fetch.get('unpack', True): cwd = os.getcwd() try: os.chdir(self.state_path) self.log( logging.INFO, 'browsertime', {'path': archive}, 'Unpacking temporary location {path}') unpack_file(archive) finally: os.chdir(cwd) # Install the browsertime Node.js requirements. if not setup_helper.check_node_executables_valid(): return 1 self.log( logging.INFO, 'browsertime', {'package_json': mozpath.join(BROWSERTIME_ROOT, 'package.json')}, 'Installing browsertime node module from {package_json}') status = setup_helper.package_setup( BROWSERTIME_ROOT, 'browsertime', should_clobber=should_clobber) if status: return status return self.check()
def _verify_node_install(self): # check if Node is installed sys.path.append(mozpath.join(self.topsrcdir, 'tools', 'lint', 'eslint')) import setup_helper with silence(): node_valid = setup_helper.check_node_executables_valid() if not node_valid: print("Can't find Node. did you run ./mach bootstrap ?") return False # check if the browsertime package has been deployed correctly # for this we just check for the browsertime directory presence if not os.path.exists(browsertime_path()): print("Could not find browsertime.js, try ./mach browsertime --setup") print("If that still fails, try ./mach browsertime --setup --clobber") return False return True
def build_docs( self, path=None, fmt="html", outdir=None, auto_open=True, serve=True, http=None, archive=False, upload=False, jobs=None, write_url=None, verbose=None, ): # TODO: Bug 1704891 - move the ESLint setup tools to a shared place. sys.path.append(mozpath.join(self.topsrcdir, "tools", "lint", "eslint")) import setup_helper setup_helper.set_project_root(self.topsrcdir) if not setup_helper.check_node_executables_valid(): return 1 setup_helper.eslint_maybe_setup() # Set the path so that Sphinx can find jsdoc, unfortunately there isn't # a way to pass this to Sphinx itself at the moment. os.environ["PATH"] = ( mozpath.join(self.topsrcdir, "node_modules", ".bin") + os.pathsep + self._node_path() + os.pathsep + os.environ["PATH"]) self.activate_virtualenv() self.virtualenv_manager.install_pip_requirements( os.path.join(here, "requirements.txt")) import webbrowser from livereload import Server from moztreedocs.package import create_tarball unique_id = "%s/%s" % (self.project, str(uuid.uuid1())) outdir = outdir or os.path.join(self.topobjdir, "docs") savedir = os.path.join(outdir, fmt) path = path or self.topsrcdir path = os.path.normpath(os.path.abspath(path)) docdir = self._find_doc_dir(path) if not docdir: print(self._dump_sphinx_backtrace()) return die("failed to generate documentation:\n" "%s: could not find docs at this location" % path) result = self._run_sphinx(docdir, savedir, fmt=fmt, jobs=jobs, verbose=verbose) if result != 0: print(self._dump_sphinx_backtrace()) return die("failed to generate documentation:\n" "%s: sphinx return code %d" % (path, result)) else: print("\nGenerated documentation:\n%s" % savedir) # Upload the artifact containing the link to S3 # This would be used by code-review to post the link to Phabricator if write_url is not None: unique_link = BASE_LINK + unique_id + "/index.html" with open(write_url, "w") as fp: fp.write(unique_link) fp.flush() print("Generated " + write_url) if archive: archive_path = os.path.join(outdir, "%s.tar.gz" % self.project) create_tarball(archive_path, savedir) print("Archived to %s" % archive_path) if upload: self._s3_upload(savedir, self.project, unique_id, self.version) if not serve: index_path = os.path.join(savedir, "index.html") if auto_open and os.path.isfile(index_path): webbrowser.open(index_path) return # Create livereload server. Any files modified in the specified docdir # will cause a re-build and refresh of the browser (if open). try: host, port = http.split(":", 1) port = int(port) except ValueError: return die("invalid address: %s" % http) server = Server() sphinx_trees = self.manager.trees or {savedir: docdir} for _, src in sphinx_trees.items(): run_sphinx = partial(self._run_sphinx, src, savedir, fmt=fmt, jobs=jobs, verbose=verbose) server.watch(src, run_sphinx) server.serve( host=host, port=port, root=savedir, open_url_delay=0.1 if auto_open else None, )
def setup(self, should_clobber=False, new_upstream_url=""): r"""Install browsertime and visualmetrics.py prerequisites and the Node.js package.""" sys.path.append(mozpath.join(self.topsrcdir, "tools", "lint", "eslint")) import setup_helper if not new_upstream_url: self.setup_prerequisites() if new_upstream_url: package_json_path = os.path.join(BROWSERTIME_ROOT, "package.json") self.log( logging.INFO, "browsertime", { "new_upstream_url": new_upstream_url, "package_json_path": package_json_path, }, "Updating browsertime node module version in {package_json_path} " "to {new_upstream_url}", ) if not re.search("/tarball/[a-f0-9]{40}$", new_upstream_url): raise ValueError( "New upstream URL does not end with /tarball/[a-f0-9]{40}: '%s'" % new_upstream_url) with open(package_json_path) as f: existing_body = json.loads( f.read(), object_pairs_hook=collections.OrderedDict) existing_body["devDependencies"]["browsertime"] = new_upstream_url updated_body = json.dumps(existing_body) with open(package_json_path, "w") as f: f.write(updated_body) # Install the browsertime Node.js requirements. if not setup_helper.check_node_executables_valid(): return 1 # To use a custom `geckodriver`, set # os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url) # to an endpoint with binaries named like # https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31. if AUTOMATION: os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true" os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true" self.log( logging.INFO, "browsertime", {"package_json": mozpath.join(BROWSERTIME_ROOT, "package.json")}, "Installing browsertime node module from {package_json}", ) status = setup_helper.package_setup( BROWSERTIME_ROOT, "browsertime", should_update=new_upstream_url != "", should_clobber=should_clobber, no_optional=new_upstream_url or AUTOMATION, ) if status: return status if new_upstream_url or AUTOMATION: return 0 return self.check()
def setup(self, should_clobber=False): r'''Install browsertime and visualmetrics.py requirements.''' automation = bool(os.environ.get('MOZ_AUTOMATION')) from mozbuild.action.tooltool import unpack_file from mozbuild.artifact_cache import ArtifactCache sys.path.append(mozpath.join(self.topsrcdir, 'tools', 'lint', 'eslint')) import setup_helper if not os.environ.get('MOZ_AUTOMATION') and host_platform().startswith('linux'): # On Linux ImageMagick needs to be installed manually, and `mach bootstrap` doesn't # do that (yet). Provide some guidance. try: from shutil import which except ImportError: from shutil_which import which im_programs = ('compare', 'convert', 'mogrify') for im_program in im_programs: prog = which(im_program) if not prog: print('Error: On Linux, ImageMagick must be on the PATH. ' 'Install ImageMagick manually and try again (or update PATH). ' 'On Ubuntu and Debian, try `sudo apt-get install imagemagick`. ' 'On Fedora, try `sudo dnf install imagemagick`. ' 'On CentOS, try `sudo yum install imagemagick`.') return 1 # Download the visualmetrics.py requirements. artifact_cache = ArtifactCache(self.artifact_cache_path, log=self.log, skip_cache=False) fetches = host_fetches[host_platform()] for tool, fetch in sorted(fetches.items()): archive = artifact_cache.fetch(fetch['url']) # TODO: assert type, verify sha256 (and size?). if fetch.get('unpack', True): cwd = os.getcwd() try: mkdir(self.state_path) os.chdir(self.state_path) self.log( logging.INFO, 'browsertime', {'path': archive}, 'Unpacking temporary location {path}') unpack_file(archive) # Make sure the expected path exists after extraction path = os.path.join(self.state_path, fetch.get('path')) if not os.path.exists(path): raise Exception("Cannot find an extracted directory: %s" % path) try: # Some archives provide binaries that don't have the # executable bit set so we need to set it here for root, dirs, files in os.walk(path): for edir in dirs: loc_to_change = os.path.join(root, edir) st = os.stat(loc_to_change) os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC) for efile in files: loc_to_change = os.path.join(root, efile) st = os.stat(loc_to_change) os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC) except Exception as e: raise Exception( "Could not set executable bit in %s, error: %s" % (path, str(e)) ) finally: os.chdir(cwd) # Install the browsertime Node.js requirements. if not setup_helper.check_node_executables_valid(): return 1 if 'GECKODRIVER_BASE_URL' not in os.environ: # Use custom `geckodriver` with pre-release Android support. url = 'https://github.com/ncalexan/geckodriver/releases/download/v0.24.0-android/' os.environ['GECKODRIVER_BASE_URL'] = url self.log( logging.INFO, 'browsertime', {'package_json': mozpath.join(BROWSERTIME_ROOT, 'package.json')}, 'Installing browsertime node module from {package_json}') status = setup_helper.package_setup( BROWSERTIME_ROOT, 'browsertime', should_clobber=should_clobber, no_optional=automation) if status: return status if automation: return 0 return self.check()
def setup(self, should_clobber=False): r'''Install browsertime and visualmetrics.py requirements.''' from mozbuild.action.tooltool import unpack_file from mozbuild.artifact_cache import ArtifactCache sys.path.append(mozpath.join(self.topsrcdir, 'tools', 'lint', 'eslint')) import setup_helper if host_platform().startswith('linux'): # On Linux ImageMagick needs to be installed manually, and `mach bootstrap` doesn't # do that (yet). Provide some guidance. import which im_programs = ('compare', 'convert', 'mogrify') try: for im_program in im_programs: which.which(im_program) except which.WhichError as e: print( 'Error: {} On Linux, ImageMagick must be on the PATH. ' 'Install ImageMagick manually and try again (or update PATH). ' 'On Ubuntu and Debian, try `sudo apt-get install imagemagick`. ' 'On Fedora, try `sudo dnf install imagemagick`. ' 'On CentOS, try `sudo yum install imagemagick`.'.format(e)) return 1 # Download the visualmetrics.py requirements. artifact_cache = ArtifactCache(self.artifact_cache_path, log=self.log, skip_cache=False) fetches = host_fetches[host_platform()] for tool, fetch in sorted(fetches.items()): archive = artifact_cache.fetch(fetch['url']) # TODO: assert type, verify sha256 (and size?). if fetch.get('unpack', True): cwd = os.getcwd() try: mkdir(self.state_path) os.chdir(self.state_path) self.log(logging.INFO, 'browsertime', {'path': archive}, 'Unpacking temporary location {path}') unpack_file(archive) finally: os.chdir(cwd) # Install the browsertime Node.js requirements. if not setup_helper.check_node_executables_valid(): return 1 if 'GECKODRIVER_BASE_URL' not in os.environ: # Use custom `geckodriver` with pre-release Android support. url = 'https://github.com/ncalexan/geckodriver/releases/download/v0.24.0-android/' os.environ['GECKODRIVER_BASE_URL'] = url self.log( logging.INFO, 'browsertime', {'package_json': mozpath.join(BROWSERTIME_ROOT, 'package.json')}, 'Installing browsertime node module from {package_json}') status = setup_helper.package_setup(BROWSERTIME_ROOT, 'browsertime', should_clobber=should_clobber) if status: return status return self.check()
def lint(paths, config, binary=None, fix=None, setup=None, **lintargs): """Run eslint.""" setup_helper.set_project_root(lintargs['root']) module_path = setup_helper.get_project_root() if not setup_helper.check_node_executables_valid(): return 1 if setup: return setup_helper.eslint_setup() setup_helper.eslint_maybe_setup() # Valid binaries are: # - Any provided by the binary argument. # - Any pointed at by the ESLINT environmental variable. # - Those provided by mach eslint --setup. # # eslint --setup installs some mozilla specific plugins and installs # all node modules locally. This is the preferred method of # installation. if not binary: binary = os.environ.get('ESLINT', None) if not binary: binary = os.path.join(module_path, "node_modules", ".bin", "eslint") if not os.path.isfile(binary): binary = None if not binary: print(ESLINT_NOT_FOUND_MESSAGE) return 1 extra_args = lintargs.get('extra_args') or [] cmd_args = [binary, # Enable the HTML plugin. # We can't currently enable this in the global config file # because it has bad interactions with the SublimeText # ESLint plugin (bug 1229874). '--plugin', 'html', # This keeps ext as a single argument. '--ext', '[{}]'.format(','.join(config['extensions'])), '--format', 'json', ] + extra_args + paths # eslint requires that --fix be set before the --ext argument. if fix: cmd_args.insert(1, '--fix') shell = False if os.environ.get('MSYSTEM') in ('MINGW32', 'MINGW64'): # The eslint binary needs to be run from a shell with msys shell = True orig = signal.signal(signal.SIGINT, signal.SIG_IGN) proc = ProcessHandler(cmd_args, env=os.environ, stream=None, shell=shell) proc.run() signal.signal(signal.SIGINT, orig) try: proc.wait() except KeyboardInterrupt: proc.kill() return [] if not proc.output: return [] # no output means success try: jsonresult = json.loads(proc.output[0]) except ValueError: print(ESLINT_ERROR_MESSAGE.format("\n".join(proc.output))) return 1 results = [] for obj in jsonresult: errors = obj['messages'] for err in errors: err.update({ 'hint': err.get('fix'), 'level': 'error' if err['severity'] == 2 else 'warning', 'lineno': err.get('line'), 'path': obj['filePath'], 'rule': err.get('ruleId'), }) results.append(result.from_config(config, **err)) return results
def setup(self, should_clobber=False, new_upstream_url=''): r'''Install browsertime and visualmetrics.py prerequisites and the Node.js package.''' sys.path.append(mozpath.join(self.topsrcdir, 'tools', 'lint', 'eslint')) import setup_helper if not new_upstream_url: self.setup_prerequisites() if new_upstream_url: package_json_path = os.path.join(BROWSERTIME_ROOT, 'package.json') self.log( logging.INFO, 'browsertime', {'new_upstream_url': new_upstream_url, 'package_json_path': package_json_path}, 'Updating browsertime node module version in {package_json_path} ' 'to {new_upstream_url}') if not re.search('/tarball/[a-f0-9]{40}$', new_upstream_url): raise ValueError("New upstream URL does not end with /tarball/[a-f0-9]{40}: '{}'" .format(new_upstream_url)) with open(package_json_path) as f: existing_body = json.loads(f.read(), object_pairs_hook=collections.OrderedDict) existing_body['devDependencies']['browsertime'] = new_upstream_url updated_body = json.dumps(existing_body) with open(package_json_path, 'w') as f: f.write(updated_body) # Install the browsertime Node.js requirements. if not setup_helper.check_node_executables_valid(): return 1 # To use a custom `geckodriver`, set # os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url) # to an endpoint with binaries named like # https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31. if AUTOMATION: os.environ[b"CHROMEDRIVER_SKIP_DOWNLOAD"] = b"true" os.environ[b"GECKODRIVER_SKIP_DOWNLOAD"] = b"true" self.log( logging.INFO, 'browsertime', {'package_json': mozpath.join(BROWSERTIME_ROOT, 'package.json')}, 'Installing browsertime node module from {package_json}') status = setup_helper.package_setup( BROWSERTIME_ROOT, 'browsertime', should_update=new_upstream_url != '', should_clobber=should_clobber, no_optional=new_upstream_url or AUTOMATION) if status: return status if new_upstream_url or AUTOMATION: return 0 return self.check()
def lint(paths, config, binary=None, fix=None, setup=None, **lintargs): """Run eslint.""" global project_root setup_helper.set_project_root(lintargs['root']) module_path = setup_helper.get_project_root() if not setup_helper.check_node_executables_valid(): return 1 if setup: return setup_helper.eslint_setup() if setup_helper.eslint_module_needs_setup(): setup_helper.eslint_setup() # Valid binaries are: # - Any provided by the binary argument. # - Any pointed at by the ESLINT environmental variable. # - Those provided by mach eslint --setup. # # eslint --setup installs some mozilla specific plugins and installs # all node modules locally. This is the preferred method of # installation. if not binary: binary = os.environ.get('ESLINT', None) if not binary: binary = os.path.join(module_path, "node_modules", ".bin", "eslint") if not os.path.isfile(binary): binary = None if not binary: print(ESLINT_NOT_FOUND_MESSAGE) return 1 extra_args = lintargs.get('extra_args') or [] cmd_args = [ binary, # Enable the HTML plugin. # We can't currently enable this in the global config file # because it has bad interactions with the SublimeText # ESLint plugin (bug 1229874). '--plugin', 'html', # This keeps ext as a single argument. '--ext', '[{}]'.format(','.join(config['extensions'])), '--format', 'json', ] + extra_args + paths # eslint requires that --fix be set before the --ext argument. if fix: cmd_args.insert(1, '--fix') shell = False if os.environ.get('MSYSTEM') in ('MINGW32', 'MINGW64'): # The eslint binary needs to be run from a shell with msys shell = True orig = signal.signal(signal.SIGINT, signal.SIG_IGN) proc = ProcessHandler(cmd_args, env=os.environ, stream=None, shell=shell) proc.run() signal.signal(signal.SIGINT, orig) try: proc.wait() except KeyboardInterrupt: proc.kill() return [] if not proc.output: return [] # no output means success try: jsonresult = json.loads(proc.output[0]) except ValueError: print(ESLINT_ERROR_MESSAGE.format("\n".join(proc.output))) return 1 results = [] for obj in jsonresult: errors = obj['messages'] for err in errors: err.update({ 'hint': err.get('fix'), 'level': 'error' if err['severity'] == 2 else 'warning', 'lineno': err.get('line'), 'path': obj['filePath'], 'rule': err.get('ruleId'), }) results.append(result.from_config(config, **err)) return results
def setup(self, should_clobber=False, new_upstream_url=""): """Install browsertime and visualmetrics.py prerequisites and the Node.js package. """ super(BrowsertimeRunner, self).setup() # installing Python deps on the fly for dep in ("Pillow==%s" % PILLOW_VERSION, "pyssim==%s" % PYSSIM_VERSION): if self._need_install(dep): self.virtualenv_manager._run_pip(["install", dep]) # check if the browsertime package has been deployed correctly # for this we just check for the browsertime directory presence if os.path.exists(self.browsertime_js): return sys.path.append(mozpath.join(self.topsrcdir, "tools", "lint", "eslint")) import setup_helper if not new_upstream_url: self.setup_prerequisites() # preparing ~/.mozbuild/browsertime for file in ("package.json", "package-lock.json"): src = mozpath.join(BROWSERTIME_SRC_ROOT, file) target = mozpath.join(self.state_path, file) if not os.path.exists(target): shutil.copyfile(src, target) package_json_path = mozpath.join(self.state_path, "package.json") if new_upstream_url: self.info( "Updating browsertime node module version in {package_json_path} " "to {new_upstream_url}", new_upstream_url=new_upstream_url, package_json_path=package_json_path, ) if not re.search("/tarball/[a-f0-9]{40}$", new_upstream_url): raise ValueError( "New upstream URL does not end with /tarball/[a-f0-9]{40}: '{}'" .format(new_upstream_url)) with open(package_json_path) as f: existing_body = json.loads( f.read(), object_pairs_hook=collections.OrderedDict) existing_body["devDependencies"]["browsertime"] = new_upstream_url updated_body = json.dumps(existing_body) with open(package_json_path, "w") as f: f.write(updated_body) # Install the browsertime Node.js requirements. if not setup_helper.check_node_executables_valid(): return # To use a custom `geckodriver`, set # os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url) # to an endpoint with binaries named like # https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31. if AUTOMATION: os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true" os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true" self.info( "Installing browsertime node module from {package_json}", package_json=package_json_path, ) setup_helper.package_setup( self.state_path, "browsertime", should_update=new_upstream_url != "", should_clobber=should_clobber, no_optional=new_upstream_url or AUTOMATION, )