def check_for_api_keys_in_file(results, upload):
if upload.addon:
users = upload.addon.authors.all()
else:
users = [upload.user] if upload.user else []
keys = []
for user in users:
try:
key = APIKey.get_jwt_key(user_id=user.id)
keys.append(key)
except APIKey.DoesNotExist:
pass
if len(keys) > 0:
zipfile = SafeZip(source=upload.path)
zipfile.is_valid()
for zipinfo in zipfile.info_list:
if zipinfo.file_size >= 64:
file_ = zipfile.read(zipinfo)
for key in keys:
if key.secret in file_.decode(encoding='unicode-escape',
errors="ignore"):
log.info('Developer API key for user %s found in '
'submission.' % key.user)
if key.user == upload.user:
msg = ugettext('Your developer API key was found '
'in the submitted file. To protect '
'your account, the key will be '
'revoked.')
else:
msg = ugettext('The developer API key of a '
'coauthor was found in the '
'submitted file. To protect your '
'add-on, the key will be revoked.')
insert_validation_message(results,
type_='error',
message=msg,
msg_id='api_key_detected',
compatibility_type=None)
# Revoke after 2 minutes to allow the developer to
# fetch the validation results
revoke_api_key.apply_async(kwargs={'key_id': key.id},
countdown=120)
zipfile.close()
return results
def rezip_file(response, pk):
# An .xpi does not have a directory inside the zip, yet zips from github
# do, so we'll need to rezip the file before passing it through to the
# validator.
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
old_filename = '{}_github_webhook.zip'.format(pk)
old_path = os.path.join(loc, old_filename)
with storage.open(old_path, 'wb') as old:
old.write(response.content)
new_filename = '{}_github_webhook.xpi'.format(pk)
new_path = os.path.join(loc, new_filename)
old_zip = SafeZip(old_path)
if not old_zip.is_valid():
raise
with storage.open(new_path, 'w') as new:
new_zip = zipfile.ZipFile(new, 'w')
for obj in old_zip.filelist:
# Basically strip off the leading directory.
new_filename = obj.filename.partition('/')[-1]
if not new_filename:
continue
new_zip.writestr(new_filename, old_zip.read(obj.filename))
new_zip.close()
old_zip.close()
return new_path
def rezip_file(response, pk):
# An .xpi does not have a directory inside the zip, yet zips from github
# do, so we'll need to rezip the file before passing it through to the
# validator.
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
old_filename = '{}_github_webhook.zip'.format(pk)
old_path = os.path.join(loc, old_filename)
with storage.open(old_path, 'wb') as old:
old.write(response.content)
new_filename = '{}_github_webhook.xpi'.format(pk)
new_path = os.path.join(loc, new_filename)
old_zip = SafeZip(old_path)
if not old_zip.is_valid():
raise
with storage.open(new_path, 'w') as new:
new_zip = zipfile.ZipFile(new, 'w')
for obj in old_zip.filelist:
# Basically strip off the leading directory.
new_filename = obj.filename.partition('/')[-1]
if not new_filename:
continue
new_zip.writestr(new_filename, old_zip.read(obj.filename))
new_zip.close()
old_zip.close()
return new_path
def build_webext_dictionary_from_legacy(addon, destination):
"""Create a webext package of a legacy dictionary `addon`, and put it in
`destination` path."""
from olympia.files.utils import SafeZip # Avoid circular import.
old_path = addon.current_version.all_files[0].file_path
old_zip = SafeZip(old_path)
if not old_zip.is_valid():
raise ValidationError('Current dictionary xpi is not valid')
if not addon.target_locale:
raise ValidationError('Addon has no target_locale')
dictionary_path = ''
with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as new_zip:
for obj in old_zip.filelist:
splitted = obj.filename.split('/')
# Ignore useless directories and files.
if splitted[0] in ('META-INF', '__MACOSX', 'chrome',
'chrome.manifest', 'install.rdf'):
continue
# Also ignore javascript (regardless of where they are, not just at
# the root), since dictionaries should not contain any code.
if splitted[-1].endswith('.js'):
continue
# Store the path of the last .dic file we find. It can be inside a
# directory.
if (splitted[-1].endswith('.dic')):
dictionary_path = obj.filename
new_zip.writestr(obj.filename, old_zip.read(obj.filename))
# Now that all files we want from the old zip are copied, build and
# add manifest.json.
if not dictionary_path:
# This should not happen... It likely means it's an invalid
# dictionary to begin with, or one that has its .dic file in a
# chrome/ directory for some reason. Abort!
raise ValidationError('Current dictionary xpi has no .dic file')
# Dumb version number increment. This will be invalid in some cases,
# but some of the dictionaries we have currently already have wild
# version numbers anyway.
version_number = addon.current_version.version
if version_number.endswith('.1-typefix'):
version_number = version_number.replace('.1-typefix', '.2webext')
else:
version_number = '%s.1webext' % version_number
manifest = {
'manifest_version': 2,
'name': unicode(addon.name),
'version': version_number,
'dictionaries': {addon.target_locale: dictionary_path},
}
# Write manifest.json we just build.
new_zip.writestr('manifest.json', json.dumps(manifest))
def check_for_api_keys_in_file(results, upload):
if upload.addon:
users = upload.addon.authors.all()
else:
users = [upload.user] if upload.user else []
keys = []
for user in users:
try:
key = APIKey.get_jwt_key(user_id=user.id)
keys.append(key)
except APIKey.DoesNotExist:
pass
if len(keys) > 0:
zipfile = SafeZip(source=upload.path)
zipfile.is_valid()
for zipinfo in zipfile.info_list:
if zipinfo.file_size >= 64:
file_ = zipfile.read(zipinfo)
for key in keys:
if key.secret in file_.decode(encoding='unicode-escape',
errors="ignore"):
log.info('Developer API key for user %s found in '
'submission.' % key.user)
if key.user == upload.user:
msg = ugettext('Your developer API key was found '
'in the submitted file. To protect '
'your account, the key will be '
'revoked.')
else:
msg = ugettext('The developer API key of a '
'coauthor was found in the '
'submitted file. To protect your '
'add-on, the key will be revoked.')
insert_validation_message(
results, type_='error',
message=msg, msg_id='api_key_detected',
compatibility_type=None)
# Revoke after 2 minutes to allow the developer to
# fetch the validation results
revoke_api_key.apply_async(
kwargs={'key_id': key.id}, countdown=120)
zipfile.close()
return results
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, raise_on_failure=False)
if not zip.is_valid():
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError, e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, raise_on_failure=False)
if not zip.is_valid():
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError, e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, validate=False)
try:
is_valid = zip.is_valid()
except (zipfile.BadZipfile, IOError):
is_valid = False
if not is_valid:
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError as e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
res = self._get_localepicker.search(manifest)
if not res:
log.error('Locale browser not in chrome.manifest: %s' % self.pk)
return ''
try:
p = res.groups()[1]
if 'localepicker.properties' not in p:
p = os.path.join(p, 'localepicker.properties')
res = zip.extract_from_manifest(p)
except (zipfile.BadZipfile, IOError) as e:
log.error('Error unzipping: %s, %s in file: %s' % (p, e, self.pk))
return ''
except (ValueError, KeyError) as e:
log.error('No file named: %s in file: %s' % (e, self.pk))
return ''
end = time.time() - start
log.info('Extracted localepicker file: %s in %.2fs' %
(self.pk, end))
statsd.timing('files.extract.localepicker', (end * 1000))
return res
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, validate=False)
try:
is_valid = zip.is_valid()
except (zipfile.BadZipfile, IOError):
is_valid = False
if not is_valid:
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError as e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
res = self._get_localepicker.search(manifest)
if not res:
log.error('Locale browser not in chrome.manifest: %s' % self.pk)
return ''
try:
p = res.groups()[1]
if 'localepicker.properties' not in p:
p = os.path.join(p, 'localepicker.properties')
res = zip.extract_from_manifest(p)
except (zipfile.BadZipfile, IOError) as e:
log.error('Error unzipping: %s, %s in file: %s' % (p, e, self.pk))
return ''
except (ValueError, KeyError) as e:
log.error('No file named: %s in file: %s' % (e, self.pk))
return ''
end = time.time() - start
log.info('Extracted localepicker file: %s in %.2fs' %
(self.pk, end))
statsd.timing('files.extract.localepicker', (end * 1000))
return res
def test_is_broken(self):
zip_file = SafeZip(self.xpi_path('signed'))
zip_file.is_valid()
zip_file.info_list[2].filename = 'META-INF/foo.sf'
assert not zip_file.is_signed()
def test_is_secure(self):
zip_file = SafeZip(self.xpi_path('signed'))
zip_file.is_valid()
assert zip_file.is_signed()
def test_not_secure(self):
zip_file = SafeZip(self.xpi_path('extension'))
zip_file.is_valid()
assert not zip_file.is_signed()
def test_read(self):
zip_file = SafeZip(self.xpi_path('langpack-localepicker'))
assert zip_file.is_valid()
assert 'locale browser de' in zip_file.read('chrome.manifest')
def test_unzip_not_fatal(self):
zip_file = SafeZip(self.xpi_path('search.xml'), raise_on_failure=False)
assert not zip_file.is_valid()
def test_unzip_not_fatal(self):
zip_file = SafeZip(self.xpi_path('search.xml'), raise_on_failure=False)
assert not zip_file.is_valid()
def test_read(self):
zip_file = SafeZip(self.xpi_path('langpack-localepicker'))
assert zip_file.is_valid()
assert 'locale browser de' in zip_file.read('chrome.manifest')
def test_is_broken(self):
zip_file = SafeZip(self.xpi_path('signed'))
zip_file.is_valid()
zip_file.info_list[2].filename = 'META-INF/foo.sf'
assert not zip_file.is_signed()
def test_is_secure(self):
zip_file = SafeZip(self.xpi_path('signed'))
zip_file.is_valid()
assert zip_file.is_signed()
def test_not_secure(self):
zip_file = SafeZip(self.xpi_path('extension'))
zip_file.is_valid()
assert not zip_file.is_signed()
