def parse_xpi(xpi, addon=None, check=True):
"""Extract and parse an XPI."""
# Extract to /tmp
path = tempfile.mkdtemp()
try:
xpi = get_file(xpi)
extract_xpi(xpi, path)
xpi_info = Extractor.parse(path)
except forms.ValidationError:
raise
except IOError as e:
if len(e.args) < 2:
errno, strerror = None, e[0]
else:
errno, strerror = e
log.error('I/O error({0}): {1}'.format(errno, strerror))
raise forms.ValidationError(
ugettext('Could not parse the manifest file.'))
except Exception:
log.error('XPI parse error', exc_info=True)
raise forms.ValidationError(
ugettext('Could not parse the manifest file.'))
finally:
rm_local_tmp_dir(path)
if check:
return check_xpi_info(xpi_info, addon)
else:
return xpi_info
def test_storage_walk():
tmp = tempfile.mkdtemp(dir=settings.TMP_PATH)
jn = partial(os.path.join, tmp)
try:
storage.save(jn('file1.txt'), ContentFile(''))
storage.save(jn('one/file1.txt'), ContentFile(''))
storage.save(jn('one/file2.txt'), ContentFile(''))
storage.save(jn('one/two/file1.txt'), ContentFile(''))
storage.save(jn('one/three/file1.txt'), ContentFile(''))
storage.save(jn('four/five/file1.txt'), ContentFile(''))
storage.save(jn(u'four/kristi\u2603/kristi\u2603.txt'),
ContentFile(''))
results = [(dir, set(subdirs), set(files))
for dir, subdirs, files in sorted(walk_storage(tmp))]
assert results.pop(0) == (tmp, set(['four',
'one']), set(['file1.txt']))
assert results.pop(0) == (jn('four'),
set(['five', 'kristi\xe2\x98\x83']), set([]))
assert results.pop(0) == (jn('four/five'), set([]), set(['file1.txt']))
assert results.pop(0) == (jn('four/kristi\xe2\x98\x83'), set([]),
set(['kristi\xe2\x98\x83.txt']))
assert results.pop(0) == (jn('one'), set(['three', 'two']),
set(['file1.txt', 'file2.txt']))
assert results.pop(0) == (jn('one/three'), set([]), set(['file1.txt']))
assert results.pop(0) == (jn('one/two'), set([]), set(['file1.txt']))
assert len(results) == 0
finally:
rm_local_tmp_dir(tmp)
def test_storage_walk():
tmp = tempfile.mkdtemp(dir=settings.TMP_PATH)
jn = partial(os.path.join, tmp)
try:
storage.save(jn('file1.txt'), ContentFile(''))
storage.save(jn('one/file1.txt'), ContentFile(''))
storage.save(jn('one/file2.txt'), ContentFile(''))
storage.save(jn('one/two/file1.txt'), ContentFile(''))
storage.save(jn('one/three/file1.txt'), ContentFile(''))
storage.save(jn('four/five/file1.txt'), ContentFile(''))
storage.save(jn(u'four/kristi\u2603/kristi\u2603.txt'),
ContentFile(''))
results = [(dir, set(subdirs), set(files))
for dir, subdirs, files in sorted(walk_storage(tmp))]
assert results.pop(0) == (
tmp, set(['four', 'one']), set(['file1.txt']))
assert results.pop(0) == (
jn('four'), set(['five', 'kristi\xe2\x98\x83']), set([]))
assert results.pop(0) == (
jn('four/five'), set([]), set(['file1.txt']))
assert results.pop(0) == (
jn('four/kristi\xe2\x98\x83'), set([]),
set(['kristi\xe2\x98\x83.txt']))
assert results.pop(0) == (
jn('one'), set(['three', 'two']), set(['file1.txt', 'file2.txt']))
assert results.pop(0) == (
jn('one/three'), set([]), set(['file1.txt']))
assert results.pop(0) == (
jn('one/two'), set([]), set(['file1.txt']))
assert len(results) == 0
finally:
rm_local_tmp_dir(tmp)
def test_storage_walk():
tmp = force_str(tempfile.mkdtemp(dir=settings.TMP_PATH))
jn = partial(os.path.join, tmp)
storage = SafeStorage(user_media='tmp')
try:
storage.save(jn('file1.txt'), ContentFile(''))
storage.save(jn('one/file1.txt'), ContentFile(''))
storage.save(jn('one/file2.txt'), ContentFile(''))
storage.save(jn('one/two/file1.txt'), ContentFile(''))
storage.save(jn('one/three/file1.txt'), ContentFile(''))
storage.save(jn('four/five/file1.txt'), ContentFile(''))
storage.save(jn('four/kristi\u2603/kristi\u2603.txt'), ContentFile(''))
results = [(dir, set(subdirs), set(files))
for dir, subdirs, files in sorted(storage.walk(tmp))]
assert results.pop(0) == (tmp, {'four', 'one'}, {'file1.txt'})
assert results.pop(0) == (jn('four'), {'five', 'kristi\u2603'}, set())
assert results.pop(0) == (jn('four/five'), set(), {'file1.txt'})
assert results.pop(0) == (
jn('four/kristi\u2603'),
set(),
{'kristi\u2603.txt'},
)
assert results.pop(0) == (
jn('one'),
{'three', 'two'},
{'file1.txt', 'file2.txt'},
)
assert results.pop(0) == (jn('one/three'), set(), {'file1.txt'})
assert results.pop(0) == (jn('one/two'), set(), {'file1.txt'})
assert len(results) == 0
finally:
rm_local_tmp_dir(tmp)
def parse_xpi(xpi, addon=None, check=True):
"""Extract and parse an XPI."""
# Extract to /tmp
path = tempfile.mkdtemp()
try:
xpi = get_file(xpi)
extract_xpi(xpi, path)
xpi_info = Extractor.parse(path)
except forms.ValidationError:
raise
except IOError as e:
if len(e.args) < 2:
errno, strerror = None, e[0]
else:
errno, strerror = e
log.error('I/O error({0}): {1}'.format(errno, strerror))
raise forms.ValidationError(_('Could not parse the manifest file.'))
except Exception:
log.error('XPI parse error', exc_info=True)
raise forms.ValidationError(_('Could not parse the manifest file.'))
finally:
rm_local_tmp_dir(path)
if check:
return check_xpi_info(xpi_info, addon)
else:
return xpi_info
def extract_zip(source, remove=False, force_fsync=False):
"""Extracts the zip file. If remove is given, removes the source file."""
tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH)
try:
zip_file = SafeZip(source, force_fsync=force_fsync)
zip_file.extract_to_dest(tempdir)
except Exception:
rm_local_tmp_dir(tempdir)
raise
if remove:
os.remove(source)
return tempdir
def extract_zip(source, remove=False, force_fsync=False):
"""Extracts the zip file. If remove is given, removes the source file."""
tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH)
try:
zip_file = SafeZip(source, force_fsync=force_fsync)
zip_file.extract_to_dest(tempdir)
except Exception:
rm_local_tmp_dir(tempdir)
raise
if remove:
os.remove(source)
return tempdir
def extract_zip(source, remove=False, fatal=True):
"""Extracts the zip file. If remove is given, removes the source file."""
tempdir = tempfile.mkdtemp()
zip_file = SafeUnzip(source)
try:
if zip_file.is_valid(fatal):
zip_file.extract_to_dest(tempdir)
except:
rm_local_tmp_dir(tempdir)
raise
if remove:
os.remove(source)
return tempdir
def extract_zip(source, remove=False, raise_on_failure=True):
"""Extracts the zip file. If remove is given, removes the source file."""
tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH)
zip_file = SafeZip(source, raise_on_failure=raise_on_failure)
try:
if zip_file.is_valid():
zip_file.extract_to_dest(tempdir)
except Exception:
rm_local_tmp_dir(tempdir)
raise
if remove:
os.remove(source)
return tempdir
def test_zip(self):
# This zip contains just one file chrome/ that we expect
# to be unzipped as a directory, not a file.
xpi = self.xpi_path('directory-test')
# This is to work around: http://bugs.python.org/issue4710
# which was fixed in Python 2.6.2. If the required version
# of Python for zamboni goes to 2.6.2 or above, this can
# be removed.
try:
dest = tempfile.mkdtemp()
zipfile.ZipFile(xpi).extractall(dest)
assert os.path.isdir(os.path.join(dest, 'chrome'))
finally:
rm_local_tmp_dir(dest)
def extract_zip(source, remove=False, raise_on_failure=True):
"""Extracts the zip file. If remove is given, removes the source file."""
tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH)
zip_file = SafeZip(source, raise_on_failure=raise_on_failure)
try:
if zip_file.is_valid():
zip_file.extract_to_dest(tempdir)
except Exception:
rm_local_tmp_dir(tempdir)
raise
if remove:
os.remove(source)
return tempdir
def extract_zip(source, remove=False, fatal=True):
"""Extracts the zip file. If remove is given, removes the source file."""
tempdir = tempfile.mkdtemp()
zip_file = SafeUnzip(source)
try:
if zip_file.is_valid(fatal):
zip_file.extract_to_dest(tempdir)
except:
rm_local_tmp_dir(tempdir)
raise
if remove:
os.remove(source)
return tempdir
def test_zip(self):
# This zip contains just one file chrome/ that we expect
# to be unzipped as a directory, not a file.
xpi = self.xpi_path('directory-test')
# This is to work around: http://bugs.python.org/issue4710
# which was fixed in Python 2.6.2. If the required version
# of Python for zamboni goes to 2.6.2 or above, this can
# be removed.
try:
dest = tempfile.mkdtemp()
zipfile.ZipFile(xpi).extractall(dest)
assert os.path.isdir(os.path.join(dest, 'chrome'))
finally:
rm_local_tmp_dir(dest)
def extract_extension_to_dest(source, dest=None, force_fsync=False):
"""Extract `source` to `dest`.
`source` is the path to an extension or extension source, which can be a
zip or a compressed tar (gzip, bzip)
Note that this doesn't verify the contents of `source` except for
that it requires something valid to be extracted.
:returns: Extraction target directory, if `dest` is `None` it'll be a
temporary directory.
:raises FileNotFoundError: if the source file is not found on the filestem
:raises forms.ValidationError: if the zip is invalid
"""
target, tempdir = None, None
if dest is None:
target = tempdir = tempfile.mkdtemp(dir=settings.TMP_PATH)
else:
target = dest
try:
source = force_str(source)
if source.endswith(('.zip', '.xpi')):
with open(source, 'rb') as source_file:
zip_file = SafeZip(source_file, force_fsync=force_fsync)
zip_file.extract_to_dest(target)
elif source.endswith(('.tar.gz', '.tar.bz2', '.tgz')):
tarfile_class = tarfile.TarFile if not force_fsync else FSyncedTarFile
with tarfile_class.open(source) as archive:
archive.extractall(target)
else:
raise FileNotFoundError # Unsupported file, shouldn't be reached
except (zipfile.BadZipFile, tarfile.ReadError, OSError,
forms.ValidationError) as e:
if tempdir is not None:
rm_local_tmp_dir(tempdir)
if isinstance(e, (FileNotFoundError, forms.ValidationError)):
# We let FileNotFoundError (which are a subclass of IOError, or
# rather OSError but that's an alias) and ValidationError be
# raised, the caller will have to deal with it.
raise
# Any other exceptions we caught, we raise a generic ValidationError
# instead.
raise forms.ValidationError(gettext('Invalid or broken archive.'))
return target
def test_rm_stored_dir():
tmp = tempfile.mkdtemp(dir=settings.TMP_PATH)
jn = partial(os.path.join, tmp)
try:
storage.save(jn('file1.txt'), ContentFile('<stuff>'))
storage.save(jn('one/file1.txt'), ContentFile(''))
storage.save(jn('one/two/file1.txt'), ContentFile('moar stuff'))
storage.save(jn(u'one/kristi\u0107/kristi\u0107.txt'), ContentFile(''))
rm_stored_dir(jn('one'))
assert not storage.exists(jn('one'))
assert not storage.exists(jn('one/file1.txt'))
assert not storage.exists(jn('one/two'))
assert not storage.exists(jn('one/two/file1.txt'))
assert not storage.exists(jn(u'one/kristi\u0107/kristi\u0107.txt'))
assert storage.exists(jn('file1.txt'))
finally:
rm_local_tmp_dir(tmp)
def test_rm_stored_dir():
tmp = tempfile.mkdtemp()
jn = partial(os.path.join, tmp)
try:
storage.save(jn("file1.txt"), ContentFile("<stuff>"))
storage.save(jn("one/file1.txt"), ContentFile(""))
storage.save(jn("one/two/file1.txt"), ContentFile("moar stuff"))
storage.save(jn(u"one/kristi\u0107/kristi\u0107.txt"), ContentFile(""))
rm_stored_dir(jn("one"))
yield (eq_, storage.exists(jn("one")), False)
yield (eq_, storage.exists(jn("one/file1.txt")), False)
yield (eq_, storage.exists(jn("one/two")), False)
yield (eq_, storage.exists(jn("one/two/file1.txt")), False)
yield (eq_, storage.exists(jn(u"one/kristi\u0107/kristi\u0107.txt")), False)
yield (eq_, storage.exists(jn("file1.txt")), True)
finally:
rm_local_tmp_dir(tmp)
def repack(xpi_path, raise_on_failure=True):
"""Unpack the XPI, yield the temp folder, and repack on exit.
Usage:
with repack('foo.xpi') as temp_folder:
# 'foo.xpi' files are extracted to the temp_folder.
modify_files(temp_folder) # Modify the files in the temp_folder.
# The 'foo.xpi' extension is now repacked, with the file changes.
"""
# Unpack.
tempdir = extract_zip(xpi_path, remove=False, fatal=raise_on_failure)
yield tempdir
try:
# Repack.
repacked = u'{0}.repacked'.format(xpi_path) # Temporary file.
zip_folder_content(tempdir, repacked)
# Overwrite the initial file with the repacked one.
shutil.move(repacked, xpi_path)
finally:
rm_local_tmp_dir(tempdir)
def test_rm_stored_dir():
tmp = tempfile.mkdtemp(dir=settings.TMP_PATH)
jn = partial(os.path.join, tmp)
try:
storage.save(jn('file1.txt'), ContentFile('<stuff>'))
storage.save(jn('one/file1.txt'), ContentFile(''))
storage.save(jn('one/two/file1.txt'), ContentFile('moar stuff'))
storage.save(jn(u'one/kristi\u0107/kristi\u0107.txt'),
ContentFile(''))
rm_stored_dir(jn('one'))
assert not storage.exists(jn('one'))
assert not storage.exists(jn('one/file1.txt'))
assert not storage.exists(jn('one/two'))
assert not storage.exists(jn('one/two/file1.txt'))
assert not storage.exists(jn(u'one/kristi\u0107/kristi\u0107.txt'))
assert storage.exists(jn('file1.txt'))
finally:
rm_local_tmp_dir(tmp)
def repack(xpi_path, raise_on_failure=True):
"""Unpack the XPI, yield the temp folder, and repack on exit.
Usage:
with repack('foo.xpi') as temp_folder:
# 'foo.xpi' files are extracted to the temp_folder.
modify_files(temp_folder) # Modify the files in the temp_folder.
# The 'foo.xpi' extension is now repacked, with the file changes.
"""
# Unpack.
tempdir = extract_zip(xpi_path, remove=False, fatal=raise_on_failure)
yield tempdir
try:
# Repack.
repacked = u'{0}.repacked'.format(xpi_path) # Temporary file.
zip_folder_content(tempdir, repacked)
# Overwrite the initial file with the repacked one.
shutil.move(repacked, xpi_path)
finally:
rm_local_tmp_dir(tempdir)
def extract_strict_compatibility_value_for_addon(addon):
strict_compatibility = None # We don't know yet.
extracted_dir = None
try:
# We take a shortcut here and only look at the first file we
# find...
# Note that we can't use parse_addon() wrapper because it no longer
# exposes the real value of `strictCompatibility`...
path = addon.current_version.all_files[0].file_path
with storage.open(path) as file_:
extracted_dir = extract_zip(file_)
parser = RDFExtractor(extracted_dir)
strict_compatibility = parser.find('strictCompatibility') == 'true'
except Exception as exp:
# A number of things can go wrong: missing file, path somehow not
# existing, etc. In any case, that means the add-on is in a weird
# state and should be ignored (this is a one off task).
log.exception(u'bump_appver_for_legacy_addons: ignoring addon %d, '
u'received %s when extracting.', addon.pk, unicode(exp))
finally:
if extracted_dir:
rm_local_tmp_dir(extracted_dir)
return strict_compatibility
def parse_xpi(xpi, addon=None, minimal=False):
"""Extract and parse an XPI. Returns a dict with various properties
describing the xpi.
Will raise ValidationError if something went wrong while parsing.
If minimal is True, it avoids validation as much as possible (still raising
ValidationError for hard errors like I/O or invalid json/rdf) and returns
only the minimal set of properties needed to decide what to do with the
add-on: guid, version and is_webextension.
"""
# Extract to /tmp
path = tempfile.mkdtemp()
try:
xpi = get_file(xpi)
extract_xpi(xpi, path)
xpi_info = Extractor.parse(path, minimal=minimal)
except forms.ValidationError:
raise
except IOError as e:
if len(e.args) < 2:
errno, strerror = None, e[0]
else:
errno, strerror = e
log.error('I/O error({0}): {1}'.format(errno, strerror))
raise forms.ValidationError(ugettext(
'Could not parse the manifest file.'))
except Exception:
log.error('XPI parse error', exc_info=True)
raise forms.ValidationError(ugettext(
'Could not parse the manifest file.'))
finally:
rm_local_tmp_dir(path)
if minimal:
return xpi_info
return check_xpi_info(xpi_info, addon)
def parse_xpi(xpi, addon=None, minimal=False):
"""Extract and parse an XPI. Returns a dict with various properties
describing the xpi.
Will raise ValidationError if something went wrong while parsing.
If minimal is True, it avoids validation as much as possible (still raising
ValidationError for hard errors like I/O or invalid json/rdf) and returns
only the minimal set of properties needed to decide what to do with the
add-on: guid, version and is_webextension.
"""
# Extract to /tmp
path = tempfile.mkdtemp()
try:
xpi = get_file(xpi)
extract_xpi(xpi, path)
xpi_info = Extractor.parse(path, minimal=minimal)
except forms.ValidationError:
raise
except IOError as e:
if len(e.args) < 2:
errno, strerror = None, e[0]
else:
errno, strerror = e
log.error('I/O error({0}): {1}'.format(errno, strerror))
raise forms.ValidationError(
ugettext('Could not parse the manifest file.'))
except Exception:
log.error('XPI parse error', exc_info=True)
raise forms.ValidationError(
ugettext('Could not parse the manifest file.'))
finally:
rm_local_tmp_dir(path)
if minimal:
return xpi_info
return check_xpi_info(xpi_info, addon)
def test_storage_walk():
tmp = tempfile.mkdtemp()
jn = partial(os.path.join, tmp)
try:
storage.save(jn("file1.txt"), ContentFile(""))
storage.save(jn("one/file1.txt"), ContentFile(""))
storage.save(jn("one/file2.txt"), ContentFile(""))
storage.save(jn("one/two/file1.txt"), ContentFile(""))
storage.save(jn("one/three/file1.txt"), ContentFile(""))
storage.save(jn("four/five/file1.txt"), ContentFile(""))
storage.save(jn(u"four/kristi\u2603/kristi\u2603.txt"), ContentFile(""))
results = [(dir, set(subdirs), set(files)) for dir, subdirs, files in sorted(walk_storage(tmp))]
yield (eq_, results.pop(0), (tmp, set(["four", "one"]), set(["file1.txt"])))
yield (eq_, results.pop(0), (jn("four"), set(["five", "kristi\xe2\x98\x83"]), set([])))
yield (eq_, results.pop(0), (jn("four/five"), set([]), set(["file1.txt"])))
yield (eq_, results.pop(0), (jn("four/kristi\xe2\x98\x83"), set([]), set(["kristi\xe2\x98\x83.txt"])))
yield (eq_, results.pop(0), (jn("one"), set(["three", "two"]), set(["file1.txt", "file2.txt"])))
yield (eq_, results.pop(0), (jn("one/three"), set([]), set(["file1.txt"])))
yield (eq_, results.pop(0), (jn("one/two"), set([]), set(["file1.txt"])))
yield (eq_, len(results), 0)
finally:
rm_local_tmp_dir(tmp)
def tearDown(self):
rm_local_tmp_dir(self.temp_dir)
super().tearDown()
def tearDown(self):
rm_local_tmp_dir(self.temp_dir)
super(TestIconForm, self).tearDown()
def tearDown(self):
rm_local_tmp_dir(self.temp_dir)
super(TestIconForm, self).tearDown()
def tearDown(self):
rm_local_tmp_dir(self.tmp)
super(TestFileOps, self).tearDown()
def tearDown(self):
rm_local_tmp_dir(self.tmp)
super(TestFileOps, self).tearDown()
def tearDown(self):
rm_local_tmp_dir(self.tmp)
super(TestLocalFileStorage, self).tearDown()
def tearDown(self):
rm_local_tmp_dir(self.tmp)
super(TestLocalFileStorage, self).tearDown()
def cleanup(self):
if os.path.exists(self.dest):
rm_local_tmp_dir(self.dest)
def cleanup(self):
if os.path.exists(self.dest):
rm_local_tmp_dir(self.dest)
