def tooltool_fetch(self, manifest, bootstrap_cmd=None, output_dir=None, privileged=False, cache=None): """docstring for tooltool_fetch""" tooltool = self.query_exe("tooltool.py", return_type="list") cmd = tooltool # get the tooltools servers from configuration default_urls = self.config["tooltool_servers"] proxxy = Proxxy(self.config, self.log_obj) proxxy_urls = proxxy.get_proxies_and_urls(default_urls) for proxyied_url in proxxy_urls: cmd.extend(["--url", proxyied_url]) cmd.extend(["fetch", "-m", manifest, "-o"]) if cache: cmd.extend(["-c", cache]) self.retry( self.run_command, args=(cmd,), kwargs={"cwd": output_dir, "error_list": TooltoolErrorList, "privileged": privileged}, good_statuses=(0,), error_message="Tooltool %s fetch failed!" % manifest, error_level=FATAL, ) if bootstrap_cmd is not None: error_message = "Tooltool bootstrap %s failed!" % str(bootstrap_cmd) self.retry( self.run_command, args=(bootstrap_cmd,), kwargs={"cwd": output_dir, "error_list": TooltoolErrorList, "privileged": privileged}, good_statuses=(0,), error_message=error_message, error_level=FATAL, )
def tooltool_fetch(self, manifest, output_dir=None, privileged=False, cache=None): """docstring for tooltool_fetch""" # Use vendored tooltool.py if available. if self.topsrcdir: cmd = [sys.executable, os.path.join(self.topsrcdir, "testing", "docker", "recipes", "tooltool.py")] elif self.config.get("download_tooltool"): cmd = [sys.executable, self._fetch_tooltool_py()] else: cmd = self.query_exe("tooltool.py", return_type="list") # get the tooltool servers from configuration default_urls = self.config.get("tooltool_servers", TOOLTOOL_SERVERS) # add slashes (bug 1155630) def add_slash(url): return url if url.endswith("/") else (url + "/") default_urls = [add_slash(u) for u in default_urls] # proxxy-ify proxxy = Proxxy(self.config, self.log_obj) proxxy_urls = proxxy.get_proxies_and_urls(default_urls) for proxyied_url in proxxy_urls: cmd.extend(["--url", proxyied_url]) # handle authentication file, if given auth_file = self._get_auth_file() if auth_file and os.path.exists(auth_file): cmd.extend(["--authentication-file", auth_file]) cmd.extend(["fetch", "-m", manifest, "-o"]) if cache: cmd.extend(["-c", cache]) # when mock is enabled run tooltool in mock. We can't use # run_command_m in all cases because it won't exist unless # MockMixin is used on the parent class if self.config.get("mock_target"): cmd_runner = self.run_command_m else: cmd_runner = self.run_command timeout = self.config.get("tooltool_timeout", 10 * 60) self.retry( cmd_runner, args=(cmd,), kwargs={ "cwd": output_dir, "error_list": TooltoolErrorList, "privileged": privileged, "output_timeout": timeout, }, good_statuses=(0,), error_message="Tooltool %s fetch failed!" % manifest, error_level=FATAL, )
def tooltool_fetch(self, manifest, output_dir=None, privileged=False, cache=None): """docstring for tooltool_fetch""" if self.config.get("download_tooltool"): cmd = [sys.executable, self._fetch_tooltool_py()] else: cmd = self.query_exe('tooltool.py', return_type='list') # get the tooltool servers from configuration default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS) # add slashes (bug 1155630) def add_slash(url): return url if url.endswith('/') else (url + '/') default_urls = [add_slash(u) for u in default_urls] # proxxy-ify proxxy = Proxxy(self.config, self.log_obj) proxxy_urls = proxxy.get_proxies_and_urls(default_urls) for proxyied_url in proxxy_urls: cmd.extend(['--url', proxyied_url]) # handle authentication file, if given auth_file = self._get_auth_file() if auth_file and os.path.exists(auth_file): cmd.extend(['--authentication-file', auth_file]) cmd.extend(['fetch', '-m', manifest, '-o']) if cache: cmd.extend(['-c', cache]) # when mock is enabled run tooltool in mock. We can't use # run_command_m in all cases because it won't exist unless # MockMixin is used on the parent class if self.config.get('mock_target'): cmd_runner = self.run_command_m else: cmd_runner = self.run_command self.retry( cmd_runner, args=(cmd, ), kwargs={ 'cwd': output_dir, 'error_list': TooltoolErrorList, 'privileged': privileged, }, good_statuses=(0, ), error_message="Tooltool %s fetch failed!" % manifest, error_level=FATAL, )
def tooltool_fetch(self, manifest, output_dir=None, privileged=False, cache=None): """docstring for tooltool_fetch""" if self.config.get("download_tooltool"): cmd = [sys.executable, self._fetch_tooltool_py()] else: cmd = self.query_exe('tooltool.py', return_type='list') # get the tooltool servers from configuration default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS) # add slashes (bug 1155630) def add_slash(url): return url if url.endswith('/') else (url + '/') default_urls = [add_slash(u) for u in default_urls] # proxxy-ify proxxy = Proxxy(self.config, self.log_obj) proxxy_urls = proxxy.get_proxies_and_urls(default_urls) for proxyied_url in proxxy_urls: cmd.extend(['--url', proxyied_url]) # handle authentication file, if given auth_file = self._get_auth_file() if auth_file and os.path.exists(auth_file): cmd.extend(['--authentication-file', auth_file]) cmd.extend(['fetch', '-m', manifest, '-o']) if cache: cmd.extend(['-c', cache]) # when mock is enabled run tooltool in mock. We can't use # run_command_m in all cases because it won't exist unless # MockMixin is used on the parent class if self.config.get('mock_target'): cmd_runner = self.run_command_m else: cmd_runner = self.run_command timeout = self.config.get('tooltool_timeout', 10 * 60) self.retry( cmd_runner, args=(cmd, ), kwargs={'cwd': output_dir, 'error_list': TooltoolErrorList, 'privileged': privileged, 'output_timeout': timeout, }, good_statuses=(0, ), error_message="Tooltool %s fetch failed!" % manifest, error_level=FATAL, )
def tooltool_fetch(self, manifest, bootstrap_cmd=None, output_dir=None, privileged=False, cache=None): """docstring for tooltool_fetch""" tooltool = self.query_exe('tooltool.py', return_type='list') if self.config.get("developer_mode"): if not os.path.exists(str(tooltool)): tooltool = self._fetch_tooltool_py() cmd = [tooltool, "--authentication-file", get_credentials_path()] else: cmd = tooltool # get the tooltools servers from configuration default_urls = self.config['tooltool_servers'] # add slashes (bug 1155630) def add_slash(url): return url if url.endswith('/') else (url + '/') default_urls = [add_slash(u) for u in default_urls] proxxy = Proxxy(self.config, self.log_obj) proxxy_urls = proxxy.get_proxies_and_urls(default_urls) for proxyied_url in proxxy_urls: cmd.extend(['--url', proxyied_url]) cmd.extend(['fetch', '-m', manifest, '-o']) if cache: cmd.extend(['-c', cache]) self.retry( self.run_command, args=(cmd, ), kwargs={'cwd': output_dir, 'error_list': TooltoolErrorList, 'privileged': privileged, }, good_statuses=(0, ), error_message="Tooltool %s fetch failed!" % manifest, error_level=FATAL, ) if bootstrap_cmd is not None: error_message = "Tooltool bootstrap %s failed!" % str(bootstrap_cmd) self.retry( self.run_command, args=(bootstrap_cmd, ), kwargs={'cwd': output_dir, 'error_list': TooltoolErrorList, 'privileged': privileged, }, good_statuses=(0, ), error_message=error_message, error_level=FATAL, )
def tooltool_fetch(self, manifest, output_dir=None, privileged=False, cache=None): """docstring for tooltool_fetch""" tooltool = self.query_exe('tooltool.py', return_type='list') if self.config.get("download_tooltool"): tooltool = [bin for bin in tooltool if os.path.exists(bin)] if tooltool: cmd = [tooltool[0]] else: cmd = [self._fetch_tooltool_py()] else: cmd = tooltool # get the tooltool servers from configuration default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS) # add slashes (bug 1155630) def add_slash(url): return url if url.endswith('/') else (url + '/') default_urls = [add_slash(u) for u in default_urls] # proxxy-ify proxxy = Proxxy(self.config, self.log_obj) proxxy_urls = proxxy.get_proxies_and_urls(default_urls) for proxyied_url in proxxy_urls: cmd.extend(['--url', proxyied_url]) # handle authentication file, if given auth_file = self._get_auth_file() if auth_file and os.path.exists(auth_file): cmd.extend(['--authentication-file', auth_file]) cmd.extend(['fetch', '-m', manifest, '-o']) if cache: cmd.extend(['-c', cache]) self.retry( self.run_command, args=(cmd, ), kwargs={'cwd': output_dir, 'error_list': TooltoolErrorList, 'privileged': privileged, }, good_statuses=(0, ), error_message="Tooltool %s fetch failed!" % manifest, error_level=FATAL, )
def tooltool_fetch(self, manifest, bootstrap_cmd=None, output_dir=None, privileged=False, cache=None): """docstring for tooltool_fetch""" tooltool = self.query_exe('tooltool.py', return_type='list') cmd = tooltool # get the tooltools servers from configuration default_urls = self.config['tooltool_servers'] proxxy = Proxxy(self.config, self.log_obj) proxxy_urls = proxxy.get_proxies_and_urls(default_urls) for proxyied_url in proxxy_urls: cmd.extend(['--url', proxyied_url]) cmd.extend(['fetch', '-m', manifest, '-o']) if cache: cmd.extend(['-c', cache]) self.retry( self.run_command, args=(cmd, ), kwargs={ 'cwd': output_dir, 'error_list': TooltoolErrorList, 'privileged': privileged, }, good_statuses=(0, ), error_message="Tooltool %s fetch failed!" % manifest, error_level=FATAL, ) if bootstrap_cmd is not None: error_message = "Tooltool bootstrap %s failed!" % str( bootstrap_cmd) self.retry( self.run_command, args=(bootstrap_cmd, ), kwargs={ 'cwd': output_dir, 'error_list': TooltoolErrorList, 'privileged': privileged, }, good_statuses=(0, ), error_message=error_message, error_level=FATAL, )
def _query_proxxy(self): if not self.proxxy: # we don't have a proxxy object. Create it. # By default, Proxxy accepts a full configuration looks for the # 'proxxy' element. If 'proxxy' is not defined it uses PROXXY_CONFIG # For GaiaTest, if there's no proxxy element, don't use a proxxy at # all. To do this we must pass a special configuraion proxxy_conf = {'proxxy': self.config.get('proxxy', {})} proxxy = Proxxy(proxxy_conf, self.log_obj) self.proxxy = proxxy return self.proxxy
def tooltool_fetch(self, manifest, bootstrap_cmd=None, output_dir=None, privileged=False): """docstring for tooltool_fetch""" tooltool = self.query_exe('tooltool.py', return_type='list') cmd = tooltool # get the tooltools servers from configuration default_urls = self.config['tooltool_servers'] proxxy = Proxxy(self.config, self.log_obj) proxxy_urls = proxxy.get_proxies_and_urls(default_urls) for proxyied_url in proxxy_urls: cmd.extend(['--url', proxyied_url]) cmd.extend(['fetch', '-m', manifest, '-o']) self.retry( self.run_command, args=(cmd, ), kwargs={'cwd': output_dir, 'error_list': TooltoolErrorList, 'privileged': privileged, }, good_statuses=(0, ), error_message="Tooltool %s fetch failed!" % manifest, error_level=FATAL, ) if bootstrap_cmd is not None: error_message = "Tooltool bootstrap %s failed!" % str(bootstrap_cmd) self.retry( self.run_command, args=(bootstrap_cmd, ), kwargs={'cwd': output_dir, 'error_list': TooltoolErrorList, 'privileged': privileged, }, good_statuses=(0, ), error_message=error_message, error_level=FATAL, )
def _query_proxxy(self): """manages the proxxy""" if not self.proxxy: self.proxxy = Proxxy(self.config, self.log_obj) return self.proxxy
def install_module(self, module=None, module_url=None, install_method=None, requirements=(), optional=False, global_options=[], no_deps=False, editable=False): """ Install module via pip. module_url can be a url to a python package tarball, a path to a directory containing a setup.py (absolute or relative to work_dir) or None, in which case it will default to the module name. requirements is a list of pip requirements files. If specified, these will be combined with the module_url (if any), like so: pip install -r requirements1.txt -r requirements2.txt module_url """ c = self.config dirs = self.query_abs_dirs() env = self.query_env() venv_path = self.query_virtualenv_path() self.info("Installing %s into virtualenv %s" % (module, venv_path)) if not module_url: module_url = module if install_method in (None, 'pip'): if not module_url and not requirements: self.fatal("Must specify module and/or requirements") pip = self.query_python_path("pip") if c.get("verbose_pip"): command = [pip, "-v", "install"] else: command = [pip, "install"] if no_deps: command += ["--no-deps"] # To avoid timeouts with our pypi server, increase default timeout: # https://bugzilla.mozilla.org/show_bug.cgi?id=1007230#c802 command += ['--timeout', str(c.get('pip_timeout', 120))] for requirement in requirements: command += ["-r", requirement] if c.get('find_links') and not c["pip_index"]: command += ['--no-index'] for opt in global_options: command += ["--global-option", opt] elif install_method == 'easy_install': if not module: self.fatal("module parameter required with install_method='easy_install'") if requirements: # Install pip requirements files separately, since they're # not understood by easy_install. self.install_module(requirements=requirements, install_method='pip') # Allow easy_install to be overridden by # self.config['exes']['easy_install'] default = 'easy_install' if self._is_windows(): # Don't invoke `easy_install` directly on windows since # the 'install' in the executable name hits UAC # - http://answers.microsoft.com/en-us/windows/forum/windows_7-security/uac-message-do-you-want-to-allow-the-following/bea30ad8-9ef8-4897-aab4-841a65f7af71 # - https://bugzilla.mozilla.org/show_bug.cgi?id=791840 default = [self.query_python_path(), self.query_python_path('easy_install-script.py')] command = self.query_exe('easy_install', default=default, return_type="list") else: self.fatal("install_module() doesn't understand an install_method of %s!" % install_method) # Add --find-links pages to look at. Add --trusted-host automatically if # the host isn't secure. This allows modern versions of pip to connect # without requiring an override. proxxy = Proxxy(self.config, self.log_obj) trusted_hosts = set() for link in proxxy.get_proxies_and_urls(c.get('find_links', [])): parsed = urlparse.urlparse(link) try: socket.gethostbyname(parsed.hostname) except socket.gaierror as e: self.info('error resolving %s (ignoring): %s' % (parsed.hostname, e.message)) continue command.extend(["--find-links", link]) if parsed.scheme != 'https': trusted_hosts.add(parsed.hostname) if (install_method != 'easy_install' and self.pip_version >= distutils.version.LooseVersion('6.0')): for host in sorted(trusted_hosts): command.extend(['--trusted-host', host]) # module_url can be None if only specifying requirements files if module_url: if editable: if install_method in (None, 'pip'): command += ['-e'] else: self.fatal("editable installs not supported for install_method %s" % install_method) command += [module_url] # If we're only installing a single requirements file, use # the file's directory as cwd, so relative paths work correctly. cwd = dirs['abs_work_dir'] if not module and len(requirements) == 1: cwd = os.path.dirname(requirements[0]) quoted_command = subprocess.list2cmdline(command) # Allow for errors while building modules, but require a # return status of 0. self.retry( self.run_command, # None will cause default value to be used attempts=1 if optional else None, good_statuses=(0,), error_level=WARNING if optional else FATAL, error_message='Could not install python package: ' + quoted_command + ' failed after %(attempts)d tries!', args=[command, ], kwargs={ 'error_list': VirtualenvErrorList, 'cwd': cwd, 'env': env, # WARNING only since retry will raise final FATAL if all # retry attempts are unsuccessful - and we only want # an ERROR of FATAL if *no* retry attempt works 'error_level': WARNING, } )
def tooltool_fetch(self, manifest, output_dir=None, privileged=False, cache=None): """docstring for tooltool_fetch""" for d in (output_dir, cache): if d is not None and not os.path.exists(d): self.mkdir_p(d) if self.topsrcdir: cmd = [ sys.executable, '-u', os.path.join(self.topsrcdir, 'mach'), 'artifact', 'toolchain', '-v', ] else: cmd = [ sys.executable, '-u', os.path.join(_external_tools_path, 'tooltool.py'), ] # get the tooltool servers from configuration default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS) # add slashes (bug 1155630) def add_slash(url): return url if url.endswith('/') else (url + '/') default_urls = [add_slash(u) for u in default_urls] # proxxy-ify proxxy = Proxxy(self.config, self.log_obj) proxxy_urls = proxxy.get_proxies_and_urls(default_urls) for proxyied_url in proxxy_urls: cmd.extend([ '--tooltool-url' if self.topsrcdir else '--url', proxyied_url ]) # handle authentication file, if given auth_file = self._get_auth_file() if auth_file and os.path.exists(auth_file): cmd.extend(['--authentication-file', auth_file]) if self.topsrcdir: cmd.extend(['--tooltool-manifest', manifest]) cmd.extend([ '--artifact-manifest', os.path.join(self.topsrcdir, 'toolchains.json') ]) else: cmd.extend(['fetch', '-m', manifest, '-o']) if cache: cmd.extend(['--cache-dir' if self.topsrcdir else '-c', cache]) toolchains = os.environ.get('MOZ_TOOLCHAINS') if toolchains: cmd.extend(toolchains.split()) # when mock is enabled run tooltool in mock. We can't use # run_command_m in all cases because it won't exist unless # MockMixin is used on the parent class if self.config.get('mock_target'): cmd_runner = self.run_command_m else: cmd_runner = self.run_command timeout = self.config.get('tooltool_timeout', 10 * 60) self.retry( cmd_runner, args=(cmd, ), kwargs={ 'cwd': output_dir, 'error_list': TooltoolErrorList, 'privileged': privileged, 'output_timeout': timeout, }, good_statuses=(0, ), error_message="Tooltool %s fetch failed!" % manifest, error_level=FATAL, )
def install_module(self, module=None, module_url=None, install_method=None, requirements=(), optional=False, global_options=[], no_deps=False, editable=False): """ Install module via pip. module_url can be a url to a python package tarball, a path to a directory containing a setup.py (absolute or relative to work_dir) or None, in which case it will default to the module name. requirements is a list of pip requirements files. If specified, these will be combined with the module_url (if any), like so: pip install -r requirements1.txt -r requirements2.txt module_url """ c = self.config dirs = self.query_abs_dirs() env = self.query_env() venv_path = self.query_virtualenv_path() self.info("Installing %s into virtualenv %s" % (module, venv_path)) if not module_url: module_url = module if install_method in (None, 'pip'): if not module_url and not requirements: self.fatal("Must specify module and/or requirements") pip = self.query_python_path("pip") if c.get("verbose_pip"): command = [pip, "-v", "install"] else: command = [pip, "install"] if no_deps: command += ["--no-deps"] virtualenv_cache_dir = c.get("virtualenv_cache_dir", os.path.join(venv_path, "cache")) if virtualenv_cache_dir: command += ["--download-cache", virtualenv_cache_dir] # To avoid timeouts with our pypi server, increase default timeout: # https://bugzilla.mozilla.org/show_bug.cgi?id=1007230#c802 command += ['--timeout', str(c.get('pip_timeout', 120))] for requirement in requirements: command += ["-r", requirement] if c.get('find_links') and not c["pip_index"]: command += ['--no-index'] for opt in global_options: command += ["--global-option", opt] elif install_method == 'easy_install': if not module: self.fatal( "module parameter required with install_method='easy_install'" ) if requirements: # Install pip requirements files separately, since they're # not understood by easy_install. self.install_module(requirements=requirements, install_method='pip') # Allow easy_install to be overridden by # self.config['exes']['easy_install'] default = 'easy_install' if self._is_windows(): # Don't invoke `easy_install` directly on windows since # the 'install' in the executable name hits UAC # - http://answers.microsoft.com/en-us/windows/forum/windows_7-security/uac-message-do-you-want-to-allow-the-following/bea30ad8-9ef8-4897-aab4-841a65f7af71 # - https://bugzilla.mozilla.org/show_bug.cgi?id=791840 default = [ self.query_python_path(), self.query_python_path('easy_install-script.py') ] command = self.query_exe('easy_install', default=default, return_type="list") else: self.fatal( "install_module() doesn't understand an install_method of %s!" % install_method) # Add --find-links pages to look at proxxy = Proxxy(self.config, self.log_obj) for link in proxxy.get_proxies_and_urls(c.get('find_links', [])): command.extend(["--find-links", link]) # module_url can be None if only specifying requirements files if module_url: if editable: if install_method in (None, 'pip'): command += ['-e'] else: self.fatal( "editable installs not supported for install_method %s" % install_method) command += [module_url] # If we're only installing a single requirements file, use # the file's directory as cwd, so relative paths work correctly. cwd = dirs['abs_work_dir'] if not module and len(requirements) == 1: cwd = os.path.dirname(requirements[0]) quoted_command = subprocess.list2cmdline(command) # Allow for errors while building modules, but require a # return status of 0. self.retry( self.run_command, # None will cause default value to be used attempts=1 if optional else None, good_statuses=(0, ), error_level=WARNING if optional else FATAL, error_message='Could not install python package: ' + quoted_command + ' failed after %(attempts)d tries!', args=[ command, ], kwargs={ 'error_list': VirtualenvErrorList, 'cwd': cwd, 'env': env, # WARNING only since retry will raise final FATAL if all # retry attempts are unsuccessful - and we only want # an ERROR of FATAL if *no* retry attempt works 'error_level': WARNING, })
def install_module(self, module=None, module_url=None, install_method=None, requirements=(), optional=False, global_options=[], no_deps=False, editable=False): """ Install module via pip. module_url can be a url to a python package tarball, a path to a directory containing a setup.py (absolute or relative to work_dir) or None, in which case it will default to the module name. requirements is a list of pip requirements files. If specified, these will be combined with the module_url (if any), like so: pip install -r requirements1.txt -r requirements2.txt module_url """ c = self.config dirs = self.query_abs_dirs() env = self.query_env() venv_path = self.query_virtualenv_path() self.info("Installing %s into virtualenv %s" % (module, venv_path)) if not module_url: module_url = module if install_method in (None, 'pip'): if not module_url and not requirements: self.fatal("Must specify module and/or requirements") pip = self.query_python_path("pip") if c.get("verbose_pip"): command = [pip, "-v", "install"] else: command = [pip, "install"] if no_deps: command += ["--no-deps"] # To avoid timeouts with our pypi server, increase default timeout: # https://bugzilla.mozilla.org/show_bug.cgi?id=1007230#c802 command += ['--timeout', str(c.get('pip_timeout', 120))] for requirement in requirements: command += ["-r", requirement] if c.get('find_links') and not c["pip_index"]: command += ['--no-index'] for opt in global_options: command += ["--global-option", opt] elif install_method == 'easy_install': if not module: self.fatal("module parameter required with install_method='easy_install'") if requirements: # Install pip requirements files separately, since they're # not understood by easy_install. self.install_module(requirements=requirements, install_method='pip') # Allow easy_install to be overridden by # self.config['exes']['easy_install'] default = 'easy_install' command = self.query_exe('easy_install', default=default, return_type="list") else: self.fatal("install_module() doesn't understand an install_method of %s!" % install_method) # Add --find-links pages to look at. Add --trusted-host automatically if # the host isn't secure. This allows modern versions of pip to connect # without requiring an override. proxxy = Proxxy(self.config, self.log_obj) trusted_hosts = set() for link in proxxy.get_proxies_and_urls(c.get('find_links', [])): parsed = urlparse.urlparse(link) try: socket.gethostbyname(parsed.hostname) except socket.gaierror as e: self.info('error resolving %s (ignoring): %s' % (parsed.hostname, e.message)) continue command.extend(["--find-links", link]) if parsed.scheme != 'https': trusted_hosts.add(parsed.hostname) if install_method != 'easy_install': for host in sorted(trusted_hosts): command.extend(['--trusted-host', host]) # module_url can be None if only specifying requirements files if module_url: if editable: if install_method in (None, 'pip'): command += ['-e'] else: self.fatal("editable installs not supported for install_method %s" % install_method) command += [module_url] # If we're only installing a single requirements file, use # the file's directory as cwd, so relative paths work correctly. cwd = dirs['abs_work_dir'] if not module and len(requirements) == 1: cwd = os.path.dirname(requirements[0]) quoted_command = subprocess.list2cmdline(command) # Allow for errors while building modules, but require a # return status of 0. self.retry( self.run_command, # None will cause default value to be used attempts=1 if optional else None, good_statuses=(0,), error_level=WARNING if optional else FATAL, error_message='Could not install python package: ' + quoted_command + ' failed after %(attempts)d tries!', args=[command, ], kwargs={ 'error_list': VirtualenvErrorList, 'cwd': cwd, 'env': env, # WARNING only since retry will raise final FATAL if all # retry attempts are unsuccessful - and we only want # an ERROR of FATAL if *no* retry attempt works 'error_level': WARNING, } )
def tooltool_fetch(self, manifest, bootstrap_cmd=None, output_dir=None, privileged=False, cache=None): """docstring for tooltool_fetch""" tooltool = self.query_exe('tooltool.py', return_type='list') if self.config.get("developer_mode"): if not os.path.exists(str(tooltool)): tooltool = self._fetch_tooltool_py() cmd = [tooltool, "--authentication-file", get_credentials_path()] else: cmd = tooltool # get the tooltools servers from configuration default_urls = self.config['tooltool_servers'] # add slashes (bug 1155630) def add_slash(url): return url if url.endswith('/') else (url + '/') default_urls = [add_slash(u) for u in default_urls] proxxy = Proxxy(self.config, self.log_obj) proxxy_urls = proxxy.get_proxies_and_urls(default_urls) for proxyied_url in proxxy_urls: cmd.extend(['--url', proxyied_url]) cmd.extend(['fetch', '-m', manifest, '-o']) if cache: cmd.extend(['-c', cache]) self.retry( self.run_command, args=(cmd, ), kwargs={ 'cwd': output_dir, 'error_list': TooltoolErrorList, 'privileged': privileged, }, good_statuses=(0, ), error_message="Tooltool %s fetch failed!" % manifest, error_level=FATAL, ) if bootstrap_cmd is not None: error_message = "Tooltool bootstrap %s failed!" % str( bootstrap_cmd) self.retry( self.run_command, args=(bootstrap_cmd, ), kwargs={ 'cwd': output_dir, 'error_list': TooltoolErrorList, 'privileged': privileged, }, good_statuses=(0, ), error_message=error_message, error_level=FATAL, )
def install_module( self, module=None, module_url=None, install_method=None, requirements=(), optional=False, global_options=[], no_deps=False, editable=False, ): """ Install module via pip. module_url can be a url to a python package tarball, a path to a directory containing a setup.py (absolute or relative to work_dir) or None, in which case it will default to the module name. requirements is a list of pip requirements files. If specified, these will be combined with the module_url (if any), like so: pip install -r requirements1.txt -r requirements2.txt module_url """ c = self.config dirs = self.query_abs_dirs() env = self.query_env() venv_path = self.query_virtualenv_path() self.info("Installing %s into virtualenv %s" % (module, venv_path)) if not module_url: module_url = module if install_method in (None, "pip"): if not module_url and not requirements: self.fatal("Must specify module and/or requirements") pip = self.query_python_path("pip") if c.get("verbose_pip"): command = [pip, "-v", "install"] else: command = [pip, "install"] if no_deps: command += ["--no-deps"] # To avoid timeouts with our pypi server, increase default timeout: # https://bugzilla.mozilla.org/show_bug.cgi?id=1007230#c802 command += ["--timeout", str(c.get("pip_timeout", 120))] for requirement in requirements: command += ["-r", requirement] if c.get("find_links") and not c["pip_index"]: command += ["--no-index"] for opt in global_options: command += ["--global-option", opt] elif install_method == "easy_install": if not module: self.fatal("module parameter required with install_method='easy_install'") if requirements: # Install pip requirements files separately, since they're # not understood by easy_install. self.install_module(requirements=requirements, install_method="pip") # Allow easy_install to be overridden by # self.config['exes']['easy_install'] default = "easy_install" if self._is_windows(): # Don't invoke `easy_install` directly on windows since # the 'install' in the executable name hits UAC # - http://answers.microsoft.com/en-us/windows/forum/windows_7-security/uac-message-do-you-want-to-allow-the-following/bea30ad8-9ef8-4897-aab4-841a65f7af71 # - https://bugzilla.mozilla.org/show_bug.cgi?id=791840 default = [self.query_python_path(), self.query_python_path("easy_install-script.py")] command = self.query_exe("easy_install", default=default, return_type="list") else: self.fatal("install_module() doesn't understand an install_method of %s!" % install_method) # Add --find-links pages to look at proxxy = Proxxy(self.config, self.log_obj) for link in proxxy.get_proxies_and_urls(c.get("find_links", [])): command.extend(["--find-links", link]) # module_url can be None if only specifying requirements files if module_url: if editable: if install_method in (None, "pip"): command += ["-e"] else: self.fatal("editable installs not supported for install_method %s" % install_method) command += [module_url] # If we're only installing a single requirements file, use # the file's directory as cwd, so relative paths work correctly. cwd = dirs["abs_work_dir"] if not module and len(requirements) == 1: cwd = os.path.dirname(requirements[0]) quoted_command = subprocess.list2cmdline(command) # Allow for errors while building modules, but require a # return status of 0. self.retry( self.run_command, # None will cause default value to be used attempts=1 if optional else None, good_statuses=(0,), error_level=WARNING if optional else FATAL, error_message="Could not install python package: " + quoted_command + " failed after %(attempts)d tries!", args=[command], kwargs={ "error_list": VirtualenvErrorList, "cwd": cwd, "env": env, # WARNING only since retry will raise final FATAL if all # retry attempts are unsuccessful - and we only want # an ERROR of FATAL if *no* retry attempt works "error_level": WARNING, }, )