def tooltool_fetch(self, manifest, bootstrap_cmd=None, output_dir=None, privileged=False, cache=None):
"""docstring for tooltool_fetch"""
tooltool = self.query_exe("tooltool.py", return_type="list")
cmd = tooltool
# get the tooltools servers from configuration
default_urls = self.config["tooltool_servers"]
proxxy = Proxxy(self.config, self.log_obj)
proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
for proxyied_url in proxxy_urls:
cmd.extend(["--url", proxyied_url])
cmd.extend(["fetch", "-m", manifest, "-o"])
if cache:
cmd.extend(["-c", cache])
self.retry(
self.run_command,
args=(cmd,),
kwargs={"cwd": output_dir, "error_list": TooltoolErrorList, "privileged": privileged},
good_statuses=(0,),
error_message="Tooltool %s fetch failed!" % manifest,
error_level=FATAL,
)
if bootstrap_cmd is not None:
error_message = "Tooltool bootstrap %s failed!" % str(bootstrap_cmd)
self.retry(
self.run_command,
args=(bootstrap_cmd,),
kwargs={"cwd": output_dir, "error_list": TooltoolErrorList, "privileged": privileged},
good_statuses=(0,),
error_message=error_message,
error_level=FATAL,
)
def tooltool_fetch(self, manifest, output_dir=None, privileged=False, cache=None):
"""docstring for tooltool_fetch"""
# Use vendored tooltool.py if available.
if self.topsrcdir:
cmd = [sys.executable, os.path.join(self.topsrcdir, "testing", "docker", "recipes", "tooltool.py")]
elif self.config.get("download_tooltool"):
cmd = [sys.executable, self._fetch_tooltool_py()]
else:
cmd = self.query_exe("tooltool.py", return_type="list")
# get the tooltool servers from configuration
default_urls = self.config.get("tooltool_servers", TOOLTOOL_SERVERS)
# add slashes (bug 1155630)
def add_slash(url):
return url if url.endswith("/") else (url + "/")
default_urls = [add_slash(u) for u in default_urls]
# proxxy-ify
proxxy = Proxxy(self.config, self.log_obj)
proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
for proxyied_url in proxxy_urls:
cmd.extend(["--url", proxyied_url])
# handle authentication file, if given
auth_file = self._get_auth_file()
if auth_file and os.path.exists(auth_file):
cmd.extend(["--authentication-file", auth_file])
cmd.extend(["fetch", "-m", manifest, "-o"])
if cache:
cmd.extend(["-c", cache])
# when mock is enabled run tooltool in mock. We can't use
# run_command_m in all cases because it won't exist unless
# MockMixin is used on the parent class
if self.config.get("mock_target"):
cmd_runner = self.run_command_m
else:
cmd_runner = self.run_command
timeout = self.config.get("tooltool_timeout", 10 * 60)
self.retry(
cmd_runner,
args=(cmd,),
kwargs={
"cwd": output_dir,
"error_list": TooltoolErrorList,
"privileged": privileged,
"output_timeout": timeout,
},
good_statuses=(0,),
error_message="Tooltool %s fetch failed!" % manifest,
error_level=FATAL,
)
def tooltool_fetch(self,
manifest,
output_dir=None,
privileged=False,
cache=None):
"""docstring for tooltool_fetch"""
if self.config.get("download_tooltool"):
cmd = [sys.executable, self._fetch_tooltool_py()]
else:
cmd = self.query_exe('tooltool.py', return_type='list')
# get the tooltool servers from configuration
default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS)
# add slashes (bug 1155630)
def add_slash(url):
return url if url.endswith('/') else (url + '/')
default_urls = [add_slash(u) for u in default_urls]
# proxxy-ify
proxxy = Proxxy(self.config, self.log_obj)
proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
for proxyied_url in proxxy_urls:
cmd.extend(['--url', proxyied_url])
# handle authentication file, if given
auth_file = self._get_auth_file()
if auth_file and os.path.exists(auth_file):
cmd.extend(['--authentication-file', auth_file])
cmd.extend(['fetch', '-m', manifest, '-o'])
if cache:
cmd.extend(['-c', cache])
# when mock is enabled run tooltool in mock. We can't use
# run_command_m in all cases because it won't exist unless
# MockMixin is used on the parent class
if self.config.get('mock_target'):
cmd_runner = self.run_command_m
else:
cmd_runner = self.run_command
self.retry(
cmd_runner,
args=(cmd, ),
kwargs={
'cwd': output_dir,
'error_list': TooltoolErrorList,
'privileged': privileged,
},
good_statuses=(0, ),
error_message="Tooltool %s fetch failed!" % manifest,
error_level=FATAL,
)
def tooltool_fetch(self, manifest,
output_dir=None, privileged=False, cache=None):
"""docstring for tooltool_fetch"""
if self.config.get("download_tooltool"):
cmd = [sys.executable, self._fetch_tooltool_py()]
else:
cmd = self.query_exe('tooltool.py', return_type='list')
# get the tooltool servers from configuration
default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS)
# add slashes (bug 1155630)
def add_slash(url):
return url if url.endswith('/') else (url + '/')
default_urls = [add_slash(u) for u in default_urls]
# proxxy-ify
proxxy = Proxxy(self.config, self.log_obj)
proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
for proxyied_url in proxxy_urls:
cmd.extend(['--url', proxyied_url])
# handle authentication file, if given
auth_file = self._get_auth_file()
if auth_file and os.path.exists(auth_file):
cmd.extend(['--authentication-file', auth_file])
cmd.extend(['fetch', '-m', manifest, '-o'])
if cache:
cmd.extend(['-c', cache])
# when mock is enabled run tooltool in mock. We can't use
# run_command_m in all cases because it won't exist unless
# MockMixin is used on the parent class
if self.config.get('mock_target'):
cmd_runner = self.run_command_m
else:
cmd_runner = self.run_command
timeout = self.config.get('tooltool_timeout', 10 * 60)
self.retry(
cmd_runner,
args=(cmd, ),
kwargs={'cwd': output_dir,
'error_list': TooltoolErrorList,
'privileged': privileged,
'output_timeout': timeout,
},
good_statuses=(0, ),
error_message="Tooltool %s fetch failed!" % manifest,
error_level=FATAL,
)
def tooltool_fetch(self, manifest, bootstrap_cmd=None,
output_dir=None, privileged=False, cache=None):
"""docstring for tooltool_fetch"""
tooltool = self.query_exe('tooltool.py', return_type='list')
if self.config.get("developer_mode"):
if not os.path.exists(str(tooltool)):
tooltool = self._fetch_tooltool_py()
cmd = [tooltool,
"--authentication-file",
get_credentials_path()]
else:
cmd = tooltool
# get the tooltools servers from configuration
default_urls = self.config['tooltool_servers']
# add slashes (bug 1155630)
def add_slash(url):
return url if url.endswith('/') else (url + '/')
default_urls = [add_slash(u) for u in default_urls]
proxxy = Proxxy(self.config, self.log_obj)
proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
for proxyied_url in proxxy_urls:
cmd.extend(['--url', proxyied_url])
cmd.extend(['fetch', '-m', manifest, '-o'])
if cache:
cmd.extend(['-c', cache])
self.retry(
self.run_command,
args=(cmd, ),
kwargs={'cwd': output_dir,
'error_list': TooltoolErrorList,
'privileged': privileged,
},
good_statuses=(0, ),
error_message="Tooltool %s fetch failed!" % manifest,
error_level=FATAL,
)
if bootstrap_cmd is not None:
error_message = "Tooltool bootstrap %s failed!" % str(bootstrap_cmd)
self.retry(
self.run_command,
args=(bootstrap_cmd, ),
kwargs={'cwd': output_dir,
'error_list': TooltoolErrorList,
'privileged': privileged,
},
good_statuses=(0, ),
error_message=error_message,
error_level=FATAL,
)
def tooltool_fetch(self, manifest,
output_dir=None, privileged=False, cache=None):
"""docstring for tooltool_fetch"""
tooltool = self.query_exe('tooltool.py', return_type='list')
if self.config.get("download_tooltool"):
tooltool = [bin for bin in tooltool if os.path.exists(bin)]
if tooltool:
cmd = [tooltool[0]]
else:
cmd = [self._fetch_tooltool_py()]
else:
cmd = tooltool
# get the tooltool servers from configuration
default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS)
# add slashes (bug 1155630)
def add_slash(url):
return url if url.endswith('/') else (url + '/')
default_urls = [add_slash(u) for u in default_urls]
# proxxy-ify
proxxy = Proxxy(self.config, self.log_obj)
proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
for proxyied_url in proxxy_urls:
cmd.extend(['--url', proxyied_url])
# handle authentication file, if given
auth_file = self._get_auth_file()
if auth_file and os.path.exists(auth_file):
cmd.extend(['--authentication-file', auth_file])
cmd.extend(['fetch', '-m', manifest, '-o'])
if cache:
cmd.extend(['-c', cache])
self.retry(
self.run_command,
args=(cmd, ),
kwargs={'cwd': output_dir,
'error_list': TooltoolErrorList,
'privileged': privileged,
},
good_statuses=(0, ),
error_message="Tooltool %s fetch failed!" % manifest,
error_level=FATAL,
)
def tooltool_fetch(self,
manifest,
bootstrap_cmd=None,
output_dir=None,
privileged=False,
cache=None):
"""docstring for tooltool_fetch"""
tooltool = self.query_exe('tooltool.py', return_type='list')
cmd = tooltool
# get the tooltools servers from configuration
default_urls = self.config['tooltool_servers']
proxxy = Proxxy(self.config, self.log_obj)
proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
for proxyied_url in proxxy_urls:
cmd.extend(['--url', proxyied_url])
cmd.extend(['fetch', '-m', manifest, '-o'])
if cache:
cmd.extend(['-c', cache])
self.retry(
self.run_command,
args=(cmd, ),
kwargs={
'cwd': output_dir,
'error_list': TooltoolErrorList,
'privileged': privileged,
},
good_statuses=(0, ),
error_message="Tooltool %s fetch failed!" % manifest,
error_level=FATAL,
)
if bootstrap_cmd is not None:
error_message = "Tooltool bootstrap %s failed!" % str(
bootstrap_cmd)
self.retry(
self.run_command,
args=(bootstrap_cmd, ),
kwargs={
'cwd': output_dir,
'error_list': TooltoolErrorList,
'privileged': privileged,
},
good_statuses=(0, ),
error_message=error_message,
error_level=FATAL,
)
def _query_proxxy(self):
if not self.proxxy:
# we don't have a proxxy object. Create it.
# By default, Proxxy accepts a full configuration looks for the
# 'proxxy' element. If 'proxxy' is not defined it uses PROXXY_CONFIG
# For GaiaTest, if there's no proxxy element, don't use a proxxy at
# all. To do this we must pass a special configuraion
proxxy_conf = {'proxxy': self.config.get('proxxy', {})}
proxxy = Proxxy(proxxy_conf, self.log_obj)
self.proxxy = proxxy
return self.proxxy
def tooltool_fetch(self, manifest, bootstrap_cmd=None,
output_dir=None, privileged=False):
"""docstring for tooltool_fetch"""
tooltool = self.query_exe('tooltool.py', return_type='list')
cmd = tooltool
# get the tooltools servers from configuration
default_urls = self.config['tooltool_servers']
proxxy = Proxxy(self.config, self.log_obj)
proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
for proxyied_url in proxxy_urls:
cmd.extend(['--url', proxyied_url])
cmd.extend(['fetch', '-m', manifest, '-o'])
self.retry(
self.run_command,
args=(cmd, ),
kwargs={'cwd': output_dir,
'error_list': TooltoolErrorList,
'privileged': privileged,
},
good_statuses=(0, ),
error_message="Tooltool %s fetch failed!" % manifest,
error_level=FATAL,
)
if bootstrap_cmd is not None:
error_message = "Tooltool bootstrap %s failed!" % str(bootstrap_cmd)
self.retry(
self.run_command,
args=(bootstrap_cmd, ),
kwargs={'cwd': output_dir,
'error_list': TooltoolErrorList,
'privileged': privileged,
},
good_statuses=(0, ),
error_message=error_message,
error_level=FATAL,
)
def _query_proxxy(self):
"""manages the proxxy"""
if not self.proxxy:
self.proxxy = Proxxy(self.config, self.log_obj)
return self.proxxy
def install_module(self, module=None, module_url=None, install_method=None,
requirements=(), optional=False, global_options=[],
no_deps=False, editable=False):
"""
Install module via pip.
module_url can be a url to a python package tarball, a path to
a directory containing a setup.py (absolute or relative to work_dir)
or None, in which case it will default to the module name.
requirements is a list of pip requirements files. If specified, these
will be combined with the module_url (if any), like so:
pip install -r requirements1.txt -r requirements2.txt module_url
"""
c = self.config
dirs = self.query_abs_dirs()
env = self.query_env()
venv_path = self.query_virtualenv_path()
self.info("Installing %s into virtualenv %s" % (module, venv_path))
if not module_url:
module_url = module
if install_method in (None, 'pip'):
if not module_url and not requirements:
self.fatal("Must specify module and/or requirements")
pip = self.query_python_path("pip")
if c.get("verbose_pip"):
command = [pip, "-v", "install"]
else:
command = [pip, "install"]
if no_deps:
command += ["--no-deps"]
# To avoid timeouts with our pypi server, increase default timeout:
# https://bugzilla.mozilla.org/show_bug.cgi?id=1007230#c802
command += ['--timeout', str(c.get('pip_timeout', 120))]
for requirement in requirements:
command += ["-r", requirement]
if c.get('find_links') and not c["pip_index"]:
command += ['--no-index']
for opt in global_options:
command += ["--global-option", opt]
elif install_method == 'easy_install':
if not module:
self.fatal("module parameter required with install_method='easy_install'")
if requirements:
# Install pip requirements files separately, since they're
# not understood by easy_install.
self.install_module(requirements=requirements,
install_method='pip')
# Allow easy_install to be overridden by
# self.config['exes']['easy_install']
default = 'easy_install'
if self._is_windows():
# Don't invoke `easy_install` directly on windows since
# the 'install' in the executable name hits UAC
# - http://answers.microsoft.com/en-us/windows/forum/windows_7-security/uac-message-do-you-want-to-allow-the-following/bea30ad8-9ef8-4897-aab4-841a65f7af71
# - https://bugzilla.mozilla.org/show_bug.cgi?id=791840
default = [self.query_python_path(), self.query_python_path('easy_install-script.py')]
command = self.query_exe('easy_install', default=default, return_type="list")
else:
self.fatal("install_module() doesn't understand an install_method of %s!" % install_method)
# Add --find-links pages to look at. Add --trusted-host automatically if
# the host isn't secure. This allows modern versions of pip to connect
# without requiring an override.
proxxy = Proxxy(self.config, self.log_obj)
trusted_hosts = set()
for link in proxxy.get_proxies_and_urls(c.get('find_links', [])):
parsed = urlparse.urlparse(link)
try:
socket.gethostbyname(parsed.hostname)
except socket.gaierror as e:
self.info('error resolving %s (ignoring): %s' %
(parsed.hostname, e.message))
continue
command.extend(["--find-links", link])
if parsed.scheme != 'https':
trusted_hosts.add(parsed.hostname)
if (install_method != 'easy_install' and
self.pip_version >= distutils.version.LooseVersion('6.0')):
for host in sorted(trusted_hosts):
command.extend(['--trusted-host', host])
# module_url can be None if only specifying requirements files
if module_url:
if editable:
if install_method in (None, 'pip'):
command += ['-e']
else:
self.fatal("editable installs not supported for install_method %s" % install_method)
command += [module_url]
# If we're only installing a single requirements file, use
# the file's directory as cwd, so relative paths work correctly.
cwd = dirs['abs_work_dir']
if not module and len(requirements) == 1:
cwd = os.path.dirname(requirements[0])
quoted_command = subprocess.list2cmdline(command)
# Allow for errors while building modules, but require a
# return status of 0.
self.retry(
self.run_command,
# None will cause default value to be used
attempts=1 if optional else None,
good_statuses=(0,),
error_level=WARNING if optional else FATAL,
error_message='Could not install python package: ' + quoted_command + ' failed after %(attempts)d tries!',
args=[command, ],
kwargs={
'error_list': VirtualenvErrorList,
'cwd': cwd,
'env': env,
# WARNING only since retry will raise final FATAL if all
# retry attempts are unsuccessful - and we only want
# an ERROR of FATAL if *no* retry attempt works
'error_level': WARNING,
}
)
def tooltool_fetch(self,
manifest,
output_dir=None,
privileged=False,
cache=None):
"""docstring for tooltool_fetch"""
for d in (output_dir, cache):
if d is not None and not os.path.exists(d):
self.mkdir_p(d)
if self.topsrcdir:
cmd = [
sys.executable,
'-u',
os.path.join(self.topsrcdir, 'mach'),
'artifact',
'toolchain',
'-v',
]
else:
cmd = [
sys.executable,
'-u',
os.path.join(_external_tools_path, 'tooltool.py'),
]
# get the tooltool servers from configuration
default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS)
# add slashes (bug 1155630)
def add_slash(url):
return url if url.endswith('/') else (url + '/')
default_urls = [add_slash(u) for u in default_urls]
# proxxy-ify
proxxy = Proxxy(self.config, self.log_obj)
proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
for proxyied_url in proxxy_urls:
cmd.extend([
'--tooltool-url' if self.topsrcdir else '--url', proxyied_url
])
# handle authentication file, if given
auth_file = self._get_auth_file()
if auth_file and os.path.exists(auth_file):
cmd.extend(['--authentication-file', auth_file])
if self.topsrcdir:
cmd.extend(['--tooltool-manifest', manifest])
cmd.extend([
'--artifact-manifest',
os.path.join(self.topsrcdir, 'toolchains.json')
])
else:
cmd.extend(['fetch', '-m', manifest, '-o'])
if cache:
cmd.extend(['--cache-dir' if self.topsrcdir else '-c', cache])
toolchains = os.environ.get('MOZ_TOOLCHAINS')
if toolchains:
cmd.extend(toolchains.split())
# when mock is enabled run tooltool in mock. We can't use
# run_command_m in all cases because it won't exist unless
# MockMixin is used on the parent class
if self.config.get('mock_target'):
cmd_runner = self.run_command_m
else:
cmd_runner = self.run_command
timeout = self.config.get('tooltool_timeout', 10 * 60)
self.retry(
cmd_runner,
args=(cmd, ),
kwargs={
'cwd': output_dir,
'error_list': TooltoolErrorList,
'privileged': privileged,
'output_timeout': timeout,
},
good_statuses=(0, ),
error_message="Tooltool %s fetch failed!" % manifest,
error_level=FATAL,
)
def install_module(self,
module=None,
module_url=None,
install_method=None,
requirements=(),
optional=False,
global_options=[],
no_deps=False,
editable=False):
"""
Install module via pip.
module_url can be a url to a python package tarball, a path to
a directory containing a setup.py (absolute or relative to work_dir)
or None, in which case it will default to the module name.
requirements is a list of pip requirements files. If specified, these
will be combined with the module_url (if any), like so:
pip install -r requirements1.txt -r requirements2.txt module_url
"""
c = self.config
dirs = self.query_abs_dirs()
env = self.query_env()
venv_path = self.query_virtualenv_path()
self.info("Installing %s into virtualenv %s" % (module, venv_path))
if not module_url:
module_url = module
if install_method in (None, 'pip'):
if not module_url and not requirements:
self.fatal("Must specify module and/or requirements")
pip = self.query_python_path("pip")
if c.get("verbose_pip"):
command = [pip, "-v", "install"]
else:
command = [pip, "install"]
if no_deps:
command += ["--no-deps"]
virtualenv_cache_dir = c.get("virtualenv_cache_dir",
os.path.join(venv_path, "cache"))
if virtualenv_cache_dir:
command += ["--download-cache", virtualenv_cache_dir]
# To avoid timeouts with our pypi server, increase default timeout:
# https://bugzilla.mozilla.org/show_bug.cgi?id=1007230#c802
command += ['--timeout', str(c.get('pip_timeout', 120))]
for requirement in requirements:
command += ["-r", requirement]
if c.get('find_links') and not c["pip_index"]:
command += ['--no-index']
for opt in global_options:
command += ["--global-option", opt]
elif install_method == 'easy_install':
if not module:
self.fatal(
"module parameter required with install_method='easy_install'"
)
if requirements:
# Install pip requirements files separately, since they're
# not understood by easy_install.
self.install_module(requirements=requirements,
install_method='pip')
# Allow easy_install to be overridden by
# self.config['exes']['easy_install']
default = 'easy_install'
if self._is_windows():
# Don't invoke `easy_install` directly on windows since
# the 'install' in the executable name hits UAC
# - http://answers.microsoft.com/en-us/windows/forum/windows_7-security/uac-message-do-you-want-to-allow-the-following/bea30ad8-9ef8-4897-aab4-841a65f7af71
# - https://bugzilla.mozilla.org/show_bug.cgi?id=791840
default = [
self.query_python_path(),
self.query_python_path('easy_install-script.py')
]
command = self.query_exe('easy_install',
default=default,
return_type="list")
else:
self.fatal(
"install_module() doesn't understand an install_method of %s!"
% install_method)
# Add --find-links pages to look at
proxxy = Proxxy(self.config, self.log_obj)
for link in proxxy.get_proxies_and_urls(c.get('find_links', [])):
command.extend(["--find-links", link])
# module_url can be None if only specifying requirements files
if module_url:
if editable:
if install_method in (None, 'pip'):
command += ['-e']
else:
self.fatal(
"editable installs not supported for install_method %s"
% install_method)
command += [module_url]
# If we're only installing a single requirements file, use
# the file's directory as cwd, so relative paths work correctly.
cwd = dirs['abs_work_dir']
if not module and len(requirements) == 1:
cwd = os.path.dirname(requirements[0])
quoted_command = subprocess.list2cmdline(command)
# Allow for errors while building modules, but require a
# return status of 0.
self.retry(
self.run_command,
# None will cause default value to be used
attempts=1 if optional else None,
good_statuses=(0, ),
error_level=WARNING if optional else FATAL,
error_message='Could not install python package: ' +
quoted_command + ' failed after %(attempts)d tries!',
args=[
command,
],
kwargs={
'error_list': VirtualenvErrorList,
'cwd': cwd,
'env': env,
# WARNING only since retry will raise final FATAL if all
# retry attempts are unsuccessful - and we only want
# an ERROR of FATAL if *no* retry attempt works
'error_level': WARNING,
})
def install_module(self, module=None, module_url=None, install_method=None,
requirements=(), optional=False, global_options=[],
no_deps=False, editable=False):
"""
Install module via pip.
module_url can be a url to a python package tarball, a path to
a directory containing a setup.py (absolute or relative to work_dir)
or None, in which case it will default to the module name.
requirements is a list of pip requirements files. If specified, these
will be combined with the module_url (if any), like so:
pip install -r requirements1.txt -r requirements2.txt module_url
"""
c = self.config
dirs = self.query_abs_dirs()
env = self.query_env()
venv_path = self.query_virtualenv_path()
self.info("Installing %s into virtualenv %s" % (module, venv_path))
if not module_url:
module_url = module
if install_method in (None, 'pip'):
if not module_url and not requirements:
self.fatal("Must specify module and/or requirements")
pip = self.query_python_path("pip")
if c.get("verbose_pip"):
command = [pip, "-v", "install"]
else:
command = [pip, "install"]
if no_deps:
command += ["--no-deps"]
# To avoid timeouts with our pypi server, increase default timeout:
# https://bugzilla.mozilla.org/show_bug.cgi?id=1007230#c802
command += ['--timeout', str(c.get('pip_timeout', 120))]
for requirement in requirements:
command += ["-r", requirement]
if c.get('find_links') and not c["pip_index"]:
command += ['--no-index']
for opt in global_options:
command += ["--global-option", opt]
elif install_method == 'easy_install':
if not module:
self.fatal("module parameter required with install_method='easy_install'")
if requirements:
# Install pip requirements files separately, since they're
# not understood by easy_install.
self.install_module(requirements=requirements,
install_method='pip')
# Allow easy_install to be overridden by
# self.config['exes']['easy_install']
default = 'easy_install'
command = self.query_exe('easy_install', default=default, return_type="list")
else:
self.fatal("install_module() doesn't understand an install_method of %s!" % install_method)
# Add --find-links pages to look at. Add --trusted-host automatically if
# the host isn't secure. This allows modern versions of pip to connect
# without requiring an override.
proxxy = Proxxy(self.config, self.log_obj)
trusted_hosts = set()
for link in proxxy.get_proxies_and_urls(c.get('find_links', [])):
parsed = urlparse.urlparse(link)
try:
socket.gethostbyname(parsed.hostname)
except socket.gaierror as e:
self.info('error resolving %s (ignoring): %s' %
(parsed.hostname, e.message))
continue
command.extend(["--find-links", link])
if parsed.scheme != 'https':
trusted_hosts.add(parsed.hostname)
if install_method != 'easy_install':
for host in sorted(trusted_hosts):
command.extend(['--trusted-host', host])
# module_url can be None if only specifying requirements files
if module_url:
if editable:
if install_method in (None, 'pip'):
command += ['-e']
else:
self.fatal("editable installs not supported for install_method %s" % install_method)
command += [module_url]
# If we're only installing a single requirements file, use
# the file's directory as cwd, so relative paths work correctly.
cwd = dirs['abs_work_dir']
if not module and len(requirements) == 1:
cwd = os.path.dirname(requirements[0])
quoted_command = subprocess.list2cmdline(command)
# Allow for errors while building modules, but require a
# return status of 0.
self.retry(
self.run_command,
# None will cause default value to be used
attempts=1 if optional else None,
good_statuses=(0,),
error_level=WARNING if optional else FATAL,
error_message='Could not install python package: ' + quoted_command + ' failed after %(attempts)d tries!',
args=[command, ],
kwargs={
'error_list': VirtualenvErrorList,
'cwd': cwd,
'env': env,
# WARNING only since retry will raise final FATAL if all
# retry attempts are unsuccessful - and we only want
# an ERROR of FATAL if *no* retry attempt works
'error_level': WARNING,
}
)
def tooltool_fetch(self,
manifest,
bootstrap_cmd=None,
output_dir=None,
privileged=False,
cache=None):
"""docstring for tooltool_fetch"""
tooltool = self.query_exe('tooltool.py', return_type='list')
if self.config.get("developer_mode"):
if not os.path.exists(str(tooltool)):
tooltool = self._fetch_tooltool_py()
cmd = [tooltool, "--authentication-file", get_credentials_path()]
else:
cmd = tooltool
# get the tooltools servers from configuration
default_urls = self.config['tooltool_servers']
# add slashes (bug 1155630)
def add_slash(url):
return url if url.endswith('/') else (url + '/')
default_urls = [add_slash(u) for u in default_urls]
proxxy = Proxxy(self.config, self.log_obj)
proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
for proxyied_url in proxxy_urls:
cmd.extend(['--url', proxyied_url])
cmd.extend(['fetch', '-m', manifest, '-o'])
if cache:
cmd.extend(['-c', cache])
self.retry(
self.run_command,
args=(cmd, ),
kwargs={
'cwd': output_dir,
'error_list': TooltoolErrorList,
'privileged': privileged,
},
good_statuses=(0, ),
error_message="Tooltool %s fetch failed!" % manifest,
error_level=FATAL,
)
if bootstrap_cmd is not None:
error_message = "Tooltool bootstrap %s failed!" % str(
bootstrap_cmd)
self.retry(
self.run_command,
args=(bootstrap_cmd, ),
kwargs={
'cwd': output_dir,
'error_list': TooltoolErrorList,
'privileged': privileged,
},
good_statuses=(0, ),
error_message=error_message,
error_level=FATAL,
)
def install_module(
self,
module=None,
module_url=None,
install_method=None,
requirements=(),
optional=False,
global_options=[],
no_deps=False,
editable=False,
):
"""
Install module via pip.
module_url can be a url to a python package tarball, a path to
a directory containing a setup.py (absolute or relative to work_dir)
or None, in which case it will default to the module name.
requirements is a list of pip requirements files. If specified, these
will be combined with the module_url (if any), like so:
pip install -r requirements1.txt -r requirements2.txt module_url
"""
c = self.config
dirs = self.query_abs_dirs()
env = self.query_env()
venv_path = self.query_virtualenv_path()
self.info("Installing %s into virtualenv %s" % (module, venv_path))
if not module_url:
module_url = module
if install_method in (None, "pip"):
if not module_url and not requirements:
self.fatal("Must specify module and/or requirements")
pip = self.query_python_path("pip")
if c.get("verbose_pip"):
command = [pip, "-v", "install"]
else:
command = [pip, "install"]
if no_deps:
command += ["--no-deps"]
# To avoid timeouts with our pypi server, increase default timeout:
# https://bugzilla.mozilla.org/show_bug.cgi?id=1007230#c802
command += ["--timeout", str(c.get("pip_timeout", 120))]
for requirement in requirements:
command += ["-r", requirement]
if c.get("find_links") and not c["pip_index"]:
command += ["--no-index"]
for opt in global_options:
command += ["--global-option", opt]
elif install_method == "easy_install":
if not module:
self.fatal("module parameter required with install_method='easy_install'")
if requirements:
# Install pip requirements files separately, since they're
# not understood by easy_install.
self.install_module(requirements=requirements, install_method="pip")
# Allow easy_install to be overridden by
# self.config['exes']['easy_install']
default = "easy_install"
if self._is_windows():
# Don't invoke `easy_install` directly on windows since
# the 'install' in the executable name hits UAC
# - http://answers.microsoft.com/en-us/windows/forum/windows_7-security/uac-message-do-you-want-to-allow-the-following/bea30ad8-9ef8-4897-aab4-841a65f7af71
# - https://bugzilla.mozilla.org/show_bug.cgi?id=791840
default = [self.query_python_path(), self.query_python_path("easy_install-script.py")]
command = self.query_exe("easy_install", default=default, return_type="list")
else:
self.fatal("install_module() doesn't understand an install_method of %s!" % install_method)
# Add --find-links pages to look at
proxxy = Proxxy(self.config, self.log_obj)
for link in proxxy.get_proxies_and_urls(c.get("find_links", [])):
command.extend(["--find-links", link])
# module_url can be None if only specifying requirements files
if module_url:
if editable:
if install_method in (None, "pip"):
command += ["-e"]
else:
self.fatal("editable installs not supported for install_method %s" % install_method)
command += [module_url]
# If we're only installing a single requirements file, use
# the file's directory as cwd, so relative paths work correctly.
cwd = dirs["abs_work_dir"]
if not module and len(requirements) == 1:
cwd = os.path.dirname(requirements[0])
quoted_command = subprocess.list2cmdline(command)
# Allow for errors while building modules, but require a
# return status of 0.
self.retry(
self.run_command,
# None will cause default value to be used
attempts=1 if optional else None,
good_statuses=(0,),
error_level=WARNING if optional else FATAL,
error_message="Could not install python package: " + quoted_command + " failed after %(attempts)d tries!",
args=[command],
kwargs={
"error_list": VirtualenvErrorList,
"cwd": cwd,
"env": env,
# WARNING only since retry will raise final FATAL if all
# retry attempts are unsuccessful - and we only want
# an ERROR of FATAL if *no* retry attempt works
"error_level": WARNING,
},
)
