def _extract(self, src, dest):
"""extract source distribution into vendor directory"""
finder = FileFinder(src)
for path, _ in finder.find('*'):
base, ext = os.path.splitext(path)
if ext == '.whl':
# Wheels would extract into a directory with the name of the package, but
# we want the platform signifiers, minus the version number.
# Wheel filenames look like:
# {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}
bits = base.split('-')
# Remove the version number.
bits.pop(1)
target = os.path.join(dest, '-'.join(bits))
mozfile.remove(
target) # remove existing version of vendored package
os.mkdir(target)
mozfile.extract(os.path.join(finder.base, path), target)
else:
# packages extract into package-version directory name and we strip the version
tld = mozfile.extract(os.path.join(finder.base, path), dest)[0]
target = os.path.join(dest, tld.rpartition('-')[0])
mozfile.remove(
target) # remove existing version of vendored package
mozfile.move(tld, target)
# If any files inside the vendored package were symlinks, turn them into normal files
# because hg.mozilla.org forbids symlinks in the repository.
link_finder = FileFinder(target)
for _, f in link_finder.find('**'):
if os.path.islink(f.path):
link_target = os.path.realpath(f.path)
os.unlink(f.path)
shutil.copyfile(link_target, f.path)
def _prepare_tests(self, revision, tests_url):
use_cache = settings['MAX_TESTS_CACHE_SIZE'] > 0
if use_cache and revision in tests_cache:
# the tests bundle is possibly being downloaded by another thread,
# wait a bit before downloading ourselves.
timeout = 300 # 5 minutes
start = datetime.datetime.now()
while datetime.datetime.now() - start < datetime.timedelta(seconds=timeout):
if tests_cache[revision] != None:
logger.debug("using pre-downloaded tests bundle for revision '{}'".format(revision))
# another thread has already downloaded the bundle for this revision, woohoo!
return tests_cache[revision]
time.sleep(1)
logger.debug("downloading tests bundle for revision '{}'".format(revision))
if use_cache:
# let other threads know we are already downloading this rev
tests_cache[revision] = None
if len(tests_cache) >= settings['MAX_TESTS_CACHE_SIZE']:
# clean up the oldest revision, it most likely isn't needed anymore
mozfile.remove(tests_cache.popitem(last=False)[1]) # FIFO
tf = mozfile.NamedTemporaryFile(suffix='.zip')
with open(tf.name, 'wb') as f:
f.write(self._download(tests_url))
tests_path = tempfile.mkdtemp()
mozfile.extract(tf.name, tests_path)
if use_cache:
tests_cache[revision] = tests_path
return tests_path
def _get_harness_root():
# Check if there is a local build
harness_root = os.path.join(build.topobjdir, '_tests', 'testing',
'mochitest')
if os.path.isdir(harness_root):
return harness_root
# Check if it was previously set up by another test
harness_root = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests',
'mochitest')
if os.path.isdir(harness_root):
return harness_root
# Check if there is a test package to download
if 'GECKO_INSTALLER_URL' in os.environ:
base_url = os.environ['GECKO_INSTALLER_URL'].rsplit('/', 1)[0]
test_packages = requests.get(base_url +
'/target.test_packages.json').json()
dest = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests')
for name in test_packages['mochitest']:
url = base_url + '/' + name
bundle = os.path.join(os.environ['PYTHON_TEST_TMP'], name)
r = requests.get(url, stream=True)
with open(bundle, 'w+b') as fh:
for chunk in r.iter_content(chunk_size=1024):
fh.write(chunk)
mozfile.extract(bundle, dest)
return os.path.join(dest, 'mochitest')
# Couldn't find a harness root, let caller do error handling.
return None
def _get_test_harness(suite, install_dir):
# Check if there is a local build
harness_root = os.path.join(build.topobjdir, '_tests', install_dir)
if os.path.isdir(harness_root):
return harness_root
# Check if it was previously set up by another test
harness_root = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests', suite)
if os.path.isdir(harness_root):
return harness_root
# Check if there is a test package to download
if 'GECKO_INSTALLER_URL' in os.environ:
base_url = os.environ['GECKO_INSTALLER_URL'].rsplit('/', 1)[0]
test_packages = requests.get(base_url + '/target.test_packages.json').json()
dest = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests')
for name in test_packages[suite]:
url = base_url + '/' + name
bundle = os.path.join(os.environ['PYTHON_TEST_TMP'], name)
r = requests.get(url, stream=True)
with open(bundle, 'w+b') as fh:
for chunk in r.iter_content(chunk_size=1024):
fh.write(chunk)
mozfile.extract(bundle, dest)
return os.path.join(dest, suite)
# Couldn't find a harness root, let caller do error handling.
return None
def prepare_symbols(self):
url = self.get_resource_url(lambda x: x.startswith('b2g') and
x.endswith('crashreporter-symbols.zip'))
file_name = self.download_file(url)
extract_dir = os.path.join(os.path.dirname(file_name), 'symbols')
if os.path.isdir(extract_dir):
shutil.rmtree(extract_dir)
mozfile.extract(file_name, extract_dir)
os.remove(file_name)
def prepare_symbols(self):
url = self.get_resource_url(lambda x: x.startswith('b2g') and x.
endswith('crashreporter-symbols.zip'))
file_name = self.download_file(url)
extract_dir = os.path.join(os.path.dirname(file_name), 'symbols')
if os.path.isdir(extract_dir):
shutil.rmtree(extract_dir)
mozfile.extract(file_name, extract_dir)
os.remove(file_name)
def prepare_tests(self):
url = self.get_resource_url(lambda x: x.startswith(self.device) and
x.endswith('%s.tests.zip' % self.platform))
file_name = self.download_file(url)
path = os.path.join(self.metadata['workdir'], 'tests')
if os.path.isdir(path):
shutil.rmtree(path)
mozfile.extract(file_name, path)
os.remove(file_name)
def _prepare_device(self, device):
url = self.get_resource_url(lambda x: x == '%s.zip' % device)
file_name = self.download_file(url)
mozfile.extract(file_name)
os.remove(file_name)
url = self.get_resource_url(lambda x: x == 'build.prop')
self.download_file(url)
url = self.get_resource_url(lambda x: x == 'sources.xml')
self.download_file(url)
def _prepare_device(self, device):
url = self.get_resource_url(lambda x: x == '%s.zip' % device)
file_name = self.download_file(url)
mozfile.extract(file_name)
os.remove(file_name)
url = self.get_resource_url(lambda x: x == 'build.prop')
self.download_file(url)
url = self.get_resource_url(lambda x: x == 'sources.xml')
self.download_file(url)
def install(src, dest):
"""Install a zip, exe, tar.gz, tar.bz2 or dmg file, and return the path of
the installation folder.
:param src: Path to the install file
:param dest: Path to install to (to ensure we do not overwrite any existent
files the folder should not exist yet)
"""
src = os.path.realpath(src)
dest = os.path.realpath(dest)
if not is_installer(src):
raise InvalidSource(src + ' is not valid installer file.')
if not os.path.exists(dest):
os.makedirs(dest)
trbk = None
try:
install_dir = None
if zipfile.is_zipfile(src) or tarfile.is_tarfile(src):
install_dir = mozfile.extract(src, dest)[0]
elif src.lower().endswith('.dmg'):
install_dir = _install_dmg(src, dest)
elif src.lower().endswith('.exe'):
install_dir = _install_exe(src, dest)
return install_dir
except Exception, ex:
cls, exc, trbk = sys.exc_info()
error = InstallError('Failed to install "%s (%s)"' % (src, str(ex)))
raise InstallError, error, trbk
def _extract(self, src, dest, keep_extra_files=False):
"""extract source distribution into vendor directory"""
ignore = ()
if not keep_extra_files:
ignore = (
"*/doc",
"*/docs",
"*/test",
"*/tests",
)
finder = FileFinder(src)
for path, _ in finder.find("*"):
base, ext = os.path.splitext(path)
# packages extract into package-version directory name and we strip the version
tld = mozfile.extract(os.path.join(finder.base, path),
dest,
ignore=ignore)[0]
target = os.path.join(dest, tld.rpartition("-")[0])
mozfile.remove(
target) # remove existing version of vendored package
mozfile.move(tld, target)
# If any files inside the vendored package were symlinks, turn them into normal files
# because hg.mozilla.org forbids symlinks in the repository.
link_finder = FileFinder(target)
for _, f in link_finder.find("**"):
if os.path.islink(f.path):
link_target = os.path.realpath(f.path)
os.unlink(f.path)
shutil.copyfile(link_target, f.path)
def install(src, dest):
"""Install a zip, exe, tar.gz, tar.bz2 or dmg file, and return the path of
the installation folder.
:param src: Path to the install file
:param dest: Path to install to (to ensure we do not overwrite any existent
files the folder should not exist yet)
"""
src = os.path.realpath(src)
dest = os.path.realpath(dest)
if not is_installer(src):
raise InvalidSource(src + ' is not valid installer file.')
if not os.path.exists(dest):
os.makedirs(dest)
trbk = None
try:
install_dir = None
if zipfile.is_zipfile(src) or tarfile.is_tarfile(src):
install_dir = mozfile.extract(src, dest)[0]
elif src.lower().endswith('.dmg'):
install_dir = _install_dmg(src, dest)
elif src.lower().endswith('.exe'):
install_dir = _install_exe(src, dest)
return install_dir
except Exception, ex:
cls, exc, trbk = sys.exc_info()
error = InstallError('Failed to install "%s (%s)"' % (src, str(ex)))
raise InstallError, error, trbk
def prepare_emulator(self):
if self.metadata.get('branch', self._base_branch) != self._base_branch:
q = "The emulators on releases.mozilla.org are based on %s, but you specified '%s'. Do want to switch to %s instead?"
if prompt(q % (self._base_branch, self.metadata['branch'], self._base_branch)) == 'y':
self.metadata['branch'] = self._base_branch
else:
self.metadata['branch'] = self._base_branch
url = '%s/%s/' % (self._base_url, '%s')
url = self.get_date_url(url, lambda x: x.string.startswith('emulator-arm')
and x.string.endswith('tar.gz'))
file_name = self.download_file(url)
extract_dir = os.path.join(self.metadata['workdir'], 'b2g-distro')
if os.path.isdir(extract_dir):
shutil.rmtree(extract_dir)
mozfile.extract(file_name)
os.remove(file_name)
def test_extract(self):
"""test the generalized extract function"""
# test extracting a tarball
tarball = self.create_tarball()
self.assertTrue(os.path.exists(tarball))
try:
dest = tempfile.mkdtemp()
try:
mozfile.extract(tarball, dest)
self.ensure_directory_contents(dest)
finally:
shutil.rmtree(dest)
finally:
os.remove(tarball)
# test extracting a zipfile
_zipfile = self.create_zip()
self.assertTrue(os.path.exists(_zipfile))
try:
dest = tempfile.mkdtemp()
try:
mozfile.extract_zip(_zipfile, dest)
self.ensure_directory_contents(dest)
finally:
shutil.rmtree(dest)
finally:
os.remove(_zipfile)
# test extracting some non-archive; this should fail
fd, filename = tempfile.mkstemp()
os.write(fd, b'This is not a zipfile or tarball')
os.close(fd)
exception = None
try:
dest = tempfile.mkdtemp()
mozfile.extract(filename, dest)
except Exception as exc:
exception = exc
finally:
os.remove(filename)
os.rmdir(dest)
self.assertTrue(isinstance(exception, Exception))
def test_extract_non_archive(tarpath, zippath):
"""test the generalized extract function"""
# test extracting some non-archive; this should fail
fd, filename = tempfile.mkstemp()
os.write(fd, b'This is not a zipfile or tarball')
os.close(fd)
exception = None
try:
dest = tempfile.mkdtemp()
mozfile.extract(filename, dest)
except Exception as exc:
exception = exc
finally:
os.remove(filename)
os.rmdir(dest)
assert isinstance(exception, Exception)
def _extract(self, src, dest, keep_extra_files=False):
"""extract source distribution into vendor directory"""
ignore = ()
if not keep_extra_files:
ignore = ("*/doc", "*/docs", "*/test", "*/tests")
finder = FileFinder(src)
for archive, _ in finder.find("*"):
_, ext = os.path.splitext(archive)
archive_path = os.path.join(finder.base, archive)
if ext == ".whl":
# Archive is named like "$package-name-1.0-py2.py3-none-any.whl", and should
# have four dashes that aren't part of the package name.
package_name, version, spec, abi, platform_and_suffix = archive.rsplit(
"-", 4
)
target_package_dir = os.path.join(dest, package_name)
mozfile.remove(target_package_dir)
os.mkdir(target_package_dir)
# Extract all the contents of the wheel into the package subdirectory.
# We're expecting at least a code directory and a ".dist-info" directory,
# though there may be a ".data" directory as well.
mozfile.extract(archive_path, target_package_dir, ignore=ignore)
_denormalize_symlinks(target_package_dir)
else:
# Archive is named like "$package-name-1.0.tar.gz", and the rightmost
# dash should separate the package name from the rest of the archive
# specifier.
package_name, archive_postfix = archive.rsplit("-", 1)
package_dir = os.path.join(dest, package_name)
mozfile.remove(package_dir)
# The archive should only contain one top-level directory, which has
# the source files. We extract this directory directly to
# the vendor directory.
extracted_files = mozfile.extract(archive_path, dest, ignore=ignore)
assert len(extracted_files) == 1
extracted_package_dir = extracted_files[0]
# The extracted package dir includes the version in the name,
# which we don't we don't want.
mozfile.move(extracted_package_dir, package_dir)
_denormalize_symlinks(package_dir)
def prepare_gecko(self):
url = self.get_resource_url(lambda x: x.startswith('b2g') and
x.endswith('.tar.gz'))
file_name = self.download_file(url)
files = mozfile.extract(file_name)
os.remove(file_name)
extract_dir = os.path.join(self.metadata['workdir'], 'gecko')
if os.path.isdir(extract_dir):
shutil.rmtree(extract_dir)
shutil.move(files[0], extract_dir)
def prepare_emulator(self):
if self.metadata.get('branch', self._base_branch) != self._base_branch:
q = "The emulators on releases.mozilla.org are based on %s, but you specified '%s'. Do want to switch to %s instead?"
if prompt(q % (self._base_branch, self.metadata['branch'],
self._base_branch)) == 'y':
self.metadata['branch'] = self._base_branch
else:
self.metadata['branch'] = self._base_branch
url = '%s/%s/' % (self._base_url, '%s')
url = self.get_date_url(
url, lambda x: x.string.startswith('emulator-arm') and x.string.
endswith('tar.gz'))
file_name = self.download_file(url)
extract_dir = os.path.join(self.metadata['workdir'], 'b2g-distro')
if os.path.isdir(extract_dir):
shutil.rmtree(extract_dir)
mozfile.extract(file_name)
os.remove(file_name)
def test_extract(self):
"""test the generalized extract function"""
# test extracting a tarball
tarball = self.create_tarball()
self.assertTrue(os.path.exists(tarball))
try:
dest = tempfile.mkdtemp()
try:
mozfile.extract(tarball, dest)
self.ensure_directory_contents(dest)
finally:
shutil.rmtree(dest)
finally:
os.remove(tarball)
# test extracting a zipfile
_zipfile = self.create_zip()
self.assertTrue(os.path.exists(_zipfile))
try:
dest = tempfile.mkdtemp()
try:
mozfile.extract_zip(_zipfile, dest)
self.ensure_directory_contents(dest)
finally:
shutil.rmtree(dest)
finally:
os.remove(_zipfile)
# test extracting some non-archive; this should fail
fd, filename = tempfile.mkstemp()
os.write(fd, "This is not a zipfile or tarball")
os.close(fd)
exception = None
try:
dest = tempfile.mkdtemp()
mozfile.extract(filename, dest)
except Exception as exception:
pass
finally:
os.remove(filename)
os.rmdir(dest)
self.assertTrue(isinstance(exception, Exception))
def prepare_gecko(self):
url = self.get_resource_url(
lambda x: x.startswith('b2g') and x.endswith('.tar.gz'))
file_name = self.download_file(url)
files = mozfile.extract(file_name)
os.remove(file_name)
extract_dir = os.path.join(self.metadata['workdir'], 'gecko')
if os.path.isdir(extract_dir):
shutil.rmtree(extract_dir)
shutil.move(files[0], extract_dir)
def install(src, dest):
"""Install a zip, exe, tar.gz, tar.bz2 or dmg file, and return the path of
the installation folder.
:param src: Path to the install file
:param dest: Path to install to (to ensure we do not overwrite any existent
files the folder should not exist yet)
"""
src = os.path.realpath(src)
dest = os.path.realpath(dest)
if not is_installer(src):
raise InvalidSource(src + ' is not valid installer file.')
did_we_create = False
if not os.path.exists(dest):
did_we_create = True
os.makedirs(dest)
trbk = None
try:
install_dir = None
if src.lower().endswith('.dmg'):
install_dir = _install_dmg(src, dest)
elif src.lower().endswith('.exe'):
install_dir = _install_exe(src, dest)
elif zipfile.is_zipfile(src) or tarfile.is_tarfile(src):
install_dir = mozfile.extract(src, dest)[0]
return install_dir
except:
cls, exc, trbk = sys.exc_info()
if did_we_create:
try:
# try to uninstall this properly
uninstall(dest)
except:
# uninstall may fail, let's just try to clean the folder
# in this case
try:
mozfile.remove(dest)
except:
pass
if issubclass(cls, Exception):
error = InstallError('Failed to install "%s (%s)"' %
(src, str(exc)))
raise InstallError, error, trbk
# any other kind of exception like KeyboardInterrupt is just re-raised.
raise cls, exc, trbk
finally:
# trbk won't get GC'ed due to circular reference
# http://docs.python.org/library/sys.html#sys.exc_info
del trbk
def install(src, dest):
"""Install a zip, exe, tar.gz, tar.bz2 or dmg file, and return the path of
the installation folder.
:param src: Path to the install file
:param dest: Path to install to (to ensure we do not overwrite any existent
files the folder should not exist yet)
"""
src = os.path.realpath(src)
dest = os.path.realpath(dest)
if not is_installer(src):
raise InvalidSource(src + ' is not valid installer file.')
did_we_create = False
if not os.path.exists(dest):
did_we_create = True
os.makedirs(dest)
trbk = None
try:
install_dir = None
if src.lower().endswith('.dmg'):
install_dir = _install_dmg(src, dest)
elif src.lower().endswith('.exe'):
install_dir = _install_exe(src, dest)
elif zipfile.is_zipfile(src) or tarfile.is_tarfile(src):
install_dir = mozfile.extract(src, dest)[0]
return install_dir
except:
cls, exc, trbk = sys.exc_info()
if did_we_create:
try:
# try to uninstall this properly
uninstall(dest)
except:
# uninstall may fail, let's just try to clean the folder
# in this case
try:
mozfile.remove(dest)
except:
pass
if issubclass(cls, Exception):
error = InstallError('Failed to install "%s (%s)"' % (src, str(exc)))
raise InstallError, error, trbk
# any other kind of exception like KeyboardInterrupt is just re-raised.
raise cls, exc, trbk
finally:
# trbk won't get GC'ed due to circular reference
# http://docs.python.org/library/sys.html#sys.exc_info
del trbk
def prepare_xre(self, url=None):
"""
Prepares the xre directory
"""
url = url or self._default_xre_url
file_name = self.download_file(url)
path = os.path.join(self.metadata['workdir'], 'xre')
if os.path.isdir(path):
shutil.rmtree(path)
files = mozfile.extract(file_name)
os.remove(file_name)
def prepare_xre(self, url=None):
"""
Prepares the xre directory
"""
url = url or self._default_xre_url
file_name = self.download_file(url)
path = os.path.join(self.metadata['workdir'], 'xre')
if os.path.isdir(path):
shutil.rmtree(path)
files = mozfile.extract(file_name)
os.remove(file_name)
def prepare_unagi(self):
url = self.get_resource_url(
lambda x: x.startswith('b2g') and x.endswith('.tar.gz'))
file_name = self.download_file(url)
files = mozfile.extract(file_name)
os.remove(file_name)
mvdir = os.path.join(self.metadata['workdir'], 'gecko')
if os.path.isdir(mvdir):
shutil.rmtree(mvdir)
shutil.move(files[0], mvdir)
url = self.get_resource_url(lambda x: x == 'unagi.zip')
file_name = self.download_file(url)
mozfile.extract(file_name)
os.remove(file_name)
url = self.get_resource_url(lambda x: x == 'build.prop')
self.download_file(url)
url = self.get_resource_url(lambda x: x == 'sources.xml')
self.download_file(url)
def prepare_unagi(self):
url = self.get_resource_url(lambda x: x.startswith('b2g') and
x.endswith('.tar.gz'))
file_name = self.download_file(url)
files = mozfile.extract(file_name)
os.remove(file_name)
mvdir = os.path.join(self.metadata['workdir'], 'gecko')
if os.path.isdir(mvdir):
shutil.rmtree(mvdir)
shutil.move(files[0], mvdir)
url = self.get_resource_url(lambda x: x == 'unagi.zip')
file_name = self.download_file(url)
mozfile.extract(file_name)
os.remove(file_name)
url = self.get_resource_url(lambda x: x == 'build.prop')
self.download_file(url)
url = self.get_resource_url(lambda x: x == 'sources.xml')
self.download_file(url)
def _download_fetches(self):
# TODO: make sure fetch-content script is available everywhere
# so this isn't needed
fetches_dir = os.environ['MOZ_FETCHES_DIR']
fetches = json.loads(os.environ.get('MOZ_FETCHES', '{}'))
for fetch in fetches:
extdir = fetches_dir
if 'dest' in 'fetch':
extdir = os.path.join(extdir, fetch['dest'])
artifact = fetch['artifact']
if not artifact.startswith('public/'):
raise Exception(
'Private artifacts in `MOZ_FETCHES` not supported.')
url = ARTIFACT_URL.format(artifact=artifact, task=fetch['task'])
path = self.download_file(url, parent_dir=extdir)
if fetch['extract']:
mozfile.extract(path, extdir)
os.remove(path)
def prepare_panda(self):
url = self.get_resource_url(lambda x: x == 'boot.tar.bz2')
file_name = self.download_file(url)
mozfile.extract(file_name)
os.remove(file_name)
url = self.get_resource_url(lambda x: x == 'system.tar.bz2')
file_name = self.download_file(url)
mozfile.extract(file_name)
os.remove(file_name)
url = self.get_resource_url(lambda x: x == 'userdata.tar.bz2')
file_name = self.download_file(url)
mozfile.extract(file_name)
os.remove(file_name)
url = self.get_resource_url(lambda x: x == 'gaia-tests.zip')
test_dir = os.path.join(self.metadata['workdir'], 'tests')
if os.path.isdir(test_dir):
shutil.rmtree(test_dir)
os.makedirs(test_dir)
file_name = self.download_file(url, 'tests')
mozfile.extract(file_name)
os.remove(file_name)
url = self.get_resource_url(lambda x: x == 'build.prop')
self.download_file(url)
url = self.get_resource_url(lambda x: x == 'sources.xml')
self.download_file(url)
# license
doc = self.download_file(self.url, tempfile.mkstemp()[1], silent=True)
soup = BeautifulSoup(open(doc, 'r'))
os.remove(doc)
text = soup.find_all('pre')[-1].string
license = open(os.path.join(self.metadata['workdir'], 'license.txt'),
'w')
license.write(text)
license.close()
def prepare_panda(self):
url = self.get_resource_url(lambda x: x == 'boot.tar.bz2')
file_name = self.download_file(url)
mozfile.extract(file_name)
os.remove(file_name)
url = self.get_resource_url(lambda x: x == 'system.tar.bz2')
file_name = self.download_file(url)
mozfile.extract(file_name)
os.remove(file_name)
url = self.get_resource_url(lambda x: x == 'userdata.tar.bz2')
file_name = self.download_file(url)
mozfile.extract(file_name)
os.remove(file_name)
url = self.get_resource_url(lambda x: x == 'gaia-tests.zip')
test_dir = os.path.join(self.metadata['workdir'], 'tests')
if os.path.isdir(test_dir):
shutil.rmtree(test_dir)
os.makedirs(test_dir)
file_name = self.download_file(url, 'tests')
mozfile.extract(file_name)
os.remove(file_name)
url = self.get_resource_url(lambda x: x == 'build.prop')
self.download_file(url)
url = self.get_resource_url(lambda x: x == 'sources.xml')
self.download_file(url)
# license
doc = self.download_file(self.url, tempfile.mkstemp()[1], silent=True)
soup = BeautifulSoup(open(doc, 'r'))
os.remove(doc)
text = soup.find_all('pre')[-1].string
license = open(os.path.join(self.metadata['workdir'], 'license.txt'), 'w')
license.write(text)
license.close()
def install(src, dest):
"""Install a zip, exe, tar.gz, tar.bz2 or dmg file, and return the path of
the installation folder.
:param src: Path to the install file
:param dest: Path to install to (to ensure we do not overwrite any existent
files the folder should not exist yet)
"""
src = os.path.realpath(src)
dest = os.path.realpath(dest)
if not is_installer(src):
raise InvalidSource(src + ' is not valid installer file.')
if not os.path.exists(dest):
os.makedirs(dest)
trbk = None
try:
install_dir = None
if zipfile.is_zipfile(src) or tarfile.is_tarfile(src):
install_dir = mozfile.extract(src, dest)[0]
elif src.lower().endswith('.dmg'):
install_dir = _install_dmg(src, dest)
elif src.lower().endswith('.exe'):
install_dir = _install_exe(src, dest)
return install_dir
except Exception:
cls, exc, trbk = sys.exc_info()
error = InstallError('Failed to install "%s"' % src)
raise InstallError, error, trbk
finally:
# trbk won't get GC'ed due to circular reference
# http://docs.python.org/library/sys.html#sys.exc_info
del trbk
def install(src, dest):
"""Install a zip, exe, tar.gz, tar.bz2 or dmg file, and return the path of
the installation folder.
:param src: Path to the install file
:param dest: Path to install to (to ensure we do not overwrite any existent
files the folder should not exist yet)
"""
src = os.path.realpath(src)
dest = os.path.realpath(dest)
if not is_installer(src):
raise InvalidSource(src + ' is not valid installer file.')
if not os.path.exists(dest):
os.makedirs(dest)
trbk = None
try:
install_dir = None
if zipfile.is_zipfile(src) or tarfile.is_tarfile(src):
install_dir = mozfile.extract(src, dest)[0]
elif src.lower().endswith('.dmg'):
install_dir = _install_dmg(src, dest)
elif src.lower().endswith('.exe'):
install_dir = _install_exe(src, dest)
return install_dir
except Exception:
cls, exc, trbk = sys.exc_info()
error = InstallError('Failed to install "%s"' % src)
raise InstallError, error, trbk
finally:
# trbk won't get GC'ed due to circular reference
# http://docs.python.org/library/sys.html#sys.exc_info
del trbk
def download_extract(self, url, file_name=None, silent=False, outdir=None):
file_name = self.download_file(url, file_name=file_name, silent=silent)
files = mozfile.extract(file_name)
os.remove(file_name)
if not outdir and len(files) > 1:
name = os.path.basename(file_name)
for suffix in ('.zip', '.tar.gz', '.tar.bz2'):
if name.endswith(suffix):
outdir = name[:-len(suffix)]
break
if outdir:
outdir = os.path.join(self.metadata['workdir'], outdir)
if os.path.isdir(outdir):
shutil.rmtree(outdir)
if len(files) == 1:
shutil.move(files[0], outdir)
elif len(files) > 1:
os.makedirs(outdir)
for f in files:
shutil.move(f, os.path.join(outdir, os.path.basename(f)))
log.debug("extracted '%s' to '%s'" % (file_name, outdir or files[0]))
self.is_addon(os.path.join(path, x))]
addons.sort()
# install each addon
for addon in addons:
# determine the addon id
addon_details = self.addon_details(addon)
addon_id = addon_details.get('id')
# if the add-on has to be unpacked force it now
# note: we might want to let Firefox do it in case of addon details
orig_path = None
if os.path.isfile(addon) and (unpack or addon_details['unpack']):
orig_path = addon
addon = tempfile.mkdtemp()
mozfile.extract(orig_path, addon)
# copy the addon to the profile
extensions_path = os.path.join(self.profile, 'extensions', 'staged')
addon_path = os.path.join(extensions_path, addon_id)
if os.path.isfile(addon):
addon_path += '.xpi'
# move existing xpi file to backup location to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
shutil.move(addon_path, self.backup_dir)
# copy new add-on to the extension folder
if not os.path.exists(extensions_path):
def test_extract_ignore(tmpdir, bundlepath):
dest = tmpdir.mkdir("dest").strpath
ignore = ("foo", "**/fleem.txt", "read*.txt")
mozfile.extract(bundlepath, dest, ignore=ignore)
assert sorted(os.listdir(dest)) == ["bar.txt", "foo.txt"]
def install_from_path(self, path, unpack=False):
"""
Installs addon from a filepath, url or directory of addons in the profile.
:param path: url, path to .xpi, or directory of addons
:param unpack: whether to unpack unless specified otherwise in the install.rdf
"""
# if the addon is a URL, download it
# note that this won't work with protocols urllib2 doesn't support
if mozfile.is_url(path):
path = self.download(path)
self.downloaded_addons.append(path)
addons = [path]
# if path is not an add-on, try to install all contained add-ons
try:
self.addon_details(path)
except AddonFormatError as e:
module_logger.warning('Could not install %s: %s' % (path, str(e)))
# If the path doesn't exist, then we don't really care, just return
if not os.path.isdir(path):
return
addons = [os.path.join(path, x) for x in os.listdir(path) if
self.is_addon(os.path.join(path, x))]
addons.sort()
# install each addon
for addon in addons:
# determine the addon id
addon_details = self.addon_details(addon)
addon_id = addon_details.get('id')
# if the add-on has to be unpacked force it now
# note: we might want to let Firefox do it in case of addon details
orig_path = None
if os.path.isfile(addon) and (unpack or addon_details['unpack']):
orig_path = addon
addon = tempfile.mkdtemp()
mozfile.extract(orig_path, addon)
# copy the addon to the profile
extensions_path = os.path.join(self.profile, 'extensions', 'staged')
addon_path = os.path.join(extensions_path, addon_id)
if os.path.isfile(addon):
addon_path += '.xpi'
# move existing xpi file to backup location to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
shutil.move(addon_path, self.backup_dir)
# copy new add-on to the extension folder
if not os.path.exists(extensions_path):
os.makedirs(extensions_path)
shutil.copy(addon, addon_path)
else:
# move existing folder to backup location to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
shutil.move(addon_path, self.backup_dir)
# copy new add-on to the extension folder
shutil.copytree(addon, addon_path, symlinks=True)
# if we had to extract the addon, remove the temporary directory
if orig_path:
mozfile.remove(addon)
addon = orig_path
self._addons.append(addon_id)
self.installed_addons.append(addon)
def install_from_path(self, path, unpack=False):
"""
Installs addon from a filepath, url or directory of addons in the profile.
:param path: url, path to .xpi, or directory of addons
:param unpack: whether to unpack unless specified otherwise in the install.rdf
"""
# if the addon is a URL, download it
# note that this won't work with protocols urllib2 doesn't support
if mozfile.is_url(path):
response = urllib2.urlopen(path)
fd, path = tempfile.mkstemp(suffix='.xpi')
os.write(fd, response.read())
os.close(fd)
self.downloaded_addons.append(path)
addons = [path]
# if path is not an add-on, try to install all contained add-ons
if not self.is_addon(path):
# If the path doesn't exist, then we don't really care, just return
if not os.path.isdir(path):
return
addons = [
os.path.join(path, x) for x in os.listdir(path)
if self.is_addon(os.path.join(path, x))
]
addons.sort()
# install each addon
for addon in addons:
# determine the addon id
addon_details = self.addon_details(addon)
addon_id = addon_details.get('id')
assert addon_id, 'The addon id could not be found: %s' % addon
# if the add-on has to be unpacked force it now
# note: we might want to let Firefox do it in case of addon details
orig_path = None
if os.path.isfile(addon) and (unpack or addon_details['unpack']):
orig_path = addon
addon = tempfile.mkdtemp()
mozfile.extract(orig_path, addon)
# copy the addon to the profile
extensions_path = os.path.join(self.profile, 'extensions',
'staged')
addon_path = os.path.join(extensions_path, addon_id)
if os.path.isfile(addon):
addon_path += '.xpi'
# save existing xpi file to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
shutil.copy(addon_path, self.backup_dir)
if not os.path.exists(extensions_path):
os.makedirs(extensions_path)
shutil.copy(addon, addon_path)
else:
# save existing dir to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
dest = os.path.join(self.backup_dir,
os.path.basename(addon_path))
dir_util.copy_tree(addon_path, dest, preserve_symlinks=1)
dir_util.copy_tree(addon, addon_path, preserve_symlinks=1)
# if we had to extract the addon, remove the temporary directory
if orig_path:
dir_util.remove_tree(addon)
addon = orig_path
self._addons.append(addon_id)
self.installed_addons.append(addon)
def _install_addon(self, path, unpack=False):
addons = [path]
# if path is not an add-on, try to install all contained add-ons
try:
self.addon_details(path)
except AddonFormatError as e:
module_logger.warning('Could not install %s: %s' % (path, str(e)))
# If the path doesn't exist, then we don't really care, just return
if not os.path.isdir(path):
return
addons = [
os.path.join(path, x) for x in os.listdir(path)
if self.is_addon(os.path.join(path, x))
]
addons.sort()
# install each addon
for addon in addons:
# determine the addon id
addon_details = self.addon_details(addon)
addon_id = addon_details.get('id')
# if the add-on has to be unpacked force it now
# note: we might want to let Firefox do it in case of addon details
orig_path = None
if os.path.isfile(addon) and (unpack or addon_details['unpack']):
orig_path = addon
addon = tempfile.mkdtemp()
mozfile.extract(orig_path, addon)
# copy the addon to the profile
extensions_path = os.path.join(self.profile, 'extensions')
addon_path = os.path.join(extensions_path, addon_id)
if os.path.isfile(addon):
addon_path += '.xpi'
# move existing xpi file to backup location to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
shutil.move(addon_path, self.backup_dir)
# copy new add-on to the extension folder
if not os.path.exists(extensions_path):
os.makedirs(extensions_path)
shutil.copy(addon, addon_path)
else:
# move existing folder to backup location to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
shutil.move(addon_path, self.backup_dir)
# copy new add-on to the extension folder
shutil.copytree(addon, addon_path, symlinks=True)
# if we had to extract the addon, remove the temporary directory
if orig_path:
mozfile.remove(addon)
addon = orig_path
self._addons.append(addon_id)
self.installed_addons.append(addon)
def test_extract_ignore(tmpdir, bundlepath):
dest = tmpdir.mkdir('dest').strpath
ignore = ('foo', '**/fleem.txt', 'read*.txt')
mozfile.extract(bundlepath, dest, ignore=ignore)
assert sorted(os.listdir(dest)) == ['bar.txt', 'foo.txt']
def test_extract(tmpdir, bundlepath, ensure_directory_contents):
"""test extracting a zipfile"""
dest = tmpdir.mkdir('dest').strpath
mozfile.extract(bundlepath, dest)
ensure_directory_contents(dest)
]
addons.sort()
# install each addon
for addon in addons:
# determine the addon id
addon_details = self.addon_details(addon)
addon_id = addon_details.get('id')
# if the add-on has to be unpacked force it now
# note: we might want to let Firefox do it in case of addon details
orig_path = None
if os.path.isfile(addon) and (unpack or addon_details['unpack']):
orig_path = addon
addon = tempfile.mkdtemp()
mozfile.extract(orig_path, addon)
# copy the addon to the profile
extensions_path = os.path.join(self.profile, 'extensions',
'staged')
addon_path = os.path.join(extensions_path, addon_id)
if os.path.isfile(addon):
addon_path += '.xpi'
# move existing xpi file to backup location to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
shutil.move(addon_path, self.backup_dir)
# copy new add-on to the extension folder
def install_from_path(self, path, unpack=False):
"""
Installs addon from a filepath, url or directory of addons in the profile.
:param path: url, path to .xpi, or directory of addons
:param unpack: whether to unpack unless specified otherwise in the install.rdf
"""
# if the addon is a URL, download it
# note that this won't work with protocols urllib2 doesn't support
if mozfile.is_url(path):
response = urllib2.urlopen(path)
fd, path = tempfile.mkstemp(suffix=".xpi")
os.write(fd, response.read())
os.close(fd)
self.downloaded_addons.append(path)
addons = [path]
# if path is not an add-on, try to install all contained add-ons
if not self.is_addon(path):
# If the path doesn't exist, then we don't really care, just return
if not os.path.isdir(path):
return
addons = [os.path.join(path, x) for x in os.listdir(path) if self.is_addon(os.path.join(path, x))]
addons.sort()
# install each addon
for addon in addons:
# determine the addon id
addon_details = self.addon_details(addon)
addon_id = addon_details.get("id")
assert addon_id, "The addon id could not be found: %s" % addon
# if the add-on has to be unpacked force it now
# note: we might want to let Firefox do it in case of addon details
orig_path = None
if os.path.isfile(addon) and (unpack or addon_details["unpack"]):
orig_path = addon
addon = tempfile.mkdtemp()
mozfile.extract(orig_path, addon)
# copy the addon to the profile
extensions_path = os.path.join(self.profile, "extensions", "staged")
addon_path = os.path.join(extensions_path, addon_id)
if os.path.isfile(addon):
addon_path += ".xpi"
# save existing xpi file to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
shutil.copy(addon_path, self.backup_dir)
if not os.path.exists(extensions_path):
os.makedirs(extensions_path)
shutil.copy(addon, addon_path)
else:
# save existing dir to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
dest = os.path.join(self.backup_dir, os.path.basename(addon_path))
dir_util.copy_tree(addon_path, dest, preserve_symlinks=1)
dir_util.copy_tree(addon, addon_path, preserve_symlinks=1)
# if we had to extract the addon, remove the temporary directory
if orig_path:
dir_util.remove_tree(addon)
addon = orig_path
self._addons.append(addon_id)
self.installed_addons.append(addon)
