def _run_yara_for_path(scanner_result, path, definition=None):
with statsd.timer('devhub.yara'):
if definition is None:
# Retrieve then concatenate all the active/valid Yara rules.
definition = '\n'.join(
ScannerRule.objects.filter(
scanner=YARA, is_active=True, definition__isnull=False
).values_list('definition', flat=True)
)
rules = yara.compile(source=definition)
zip_file = SafeZip(source=path)
for zip_info in zip_file.info_list:
if not zip_info.is_dir():
file_content = zip_file.read(zip_info).decode(
errors='ignore'
)
for match in rules.match(data=file_content):
# Add the filename to the meta dict.
meta = {**match.meta, 'filename': zip_info.filename}
scanner_result.add_yara_result(
rule=match.rule,
tags=match.tags,
meta=meta
)
zip_file.close()
def run_yara(results, upload_pk):
"""
Apply a set of Yara rules on a FileUpload and store the Yara results
(matches).
This task is intended to be run as part of the submission process only.
When a version is created from a FileUpload, the files are removed. In
addition, we usually delete old FileUpload entries after 180 days.
- `results` are the validation results passed in the validation chain. This
task is a validation task, which is why it must receive the validation
results as first argument.
- `upload_pk` is the FileUpload ID.
"""
log.info('Starting yara task for FileUpload %s.', upload_pk)
if not results['metadata']['is_webextension']:
log.info('Not running yara for FileUpload %s, it is not a '
'webextension.', upload_pk)
return results
upload = FileUpload.objects.get(pk=upload_pk)
try:
scanner_result = ScannerResult(upload=upload, scanner=YARA)
with statsd.timer('devhub.yara'):
rules = yara.compile(filepath=settings.YARA_RULES_FILEPATH)
zip_file = SafeZip(source=upload.path)
for zip_info in zip_file.info_list:
if not zip_info.is_dir():
file_content = zip_file.read(zip_info).decode(
errors='ignore'
)
for match in rules.match(data=file_content):
# Add the filename to the meta dict.
meta = {**match.meta, 'filename': zip_info.filename}
scanner_result.add_yara_result(
rule=match.rule,
tags=match.tags,
meta=meta
)
zip_file.close()
scanner_result.save()
if scanner_result.has_matches:
statsd.incr('devhub.yara.has_matches')
statsd.incr('devhub.yara.success')
log.info('Ending scanner "yara" task for FileUpload %s.', upload_pk)
except Exception:
statsd.incr('devhub.yara.failure')
# We log the exception but we do not raise to avoid perturbing the
# submission flow.
log.exception('Error in scanner "yara" task for FileUpload %s.',
upload_pk)
return results
def build_webext_dictionary_from_legacy(addon, destination):
"""Create a webext package of a legacy dictionary `addon`, and put it in
`destination` path."""
from olympia.files.utils import SafeZip # Avoid circular import.
old_path = addon.current_version.all_files[0].file_path
old_zip = SafeZip(old_path)
if not old_zip.is_valid():
raise ValidationError('Current dictionary xpi is not valid')
if not addon.target_locale:
raise ValidationError('Addon has no target_locale')
dictionary_path = ''
with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as new_zip:
for obj in old_zip.filelist:
splitted = obj.filename.split('/')
# Ignore useless directories and files.
if splitted[0] in ('META-INF', '__MACOSX', 'chrome',
'chrome.manifest', 'install.rdf'):
continue
# Also ignore javascript (regardless of where they are, not just at
# the root), since dictionaries should not contain any code.
if splitted[-1].endswith('.js'):
continue
# Store the path of the last .dic file we find. It can be inside a
# directory.
if (splitted[-1].endswith('.dic')):
dictionary_path = obj.filename
new_zip.writestr(obj.filename, old_zip.read(obj.filename))
# Now that all files we want from the old zip are copied, build and
# add manifest.json.
if not dictionary_path:
# This should not happen... It likely means it's an invalid
# dictionary to begin with, or one that has its .dic file in a
# chrome/ directory for some reason. Abort!
raise ValidationError('Current dictionary xpi has no .dic file')
# Dumb version number increment. This will be invalid in some cases,
# but some of the dictionaries we have currently already have wild
# version numbers anyway.
version_number = addon.current_version.version
if version_number.endswith('.1-typefix'):
version_number = version_number.replace('.1-typefix', '.2webext')
else:
version_number = '%s.1webext' % version_number
manifest = {
'manifest_version': 2,
'name': unicode(addon.name),
'version': version_number,
'dictionaries': {addon.target_locale: dictionary_path},
}
# Write manifest.json we just build.
new_zip.writestr('manifest.json', json.dumps(manifest))
def rezip_file(response, pk):
# An .xpi does not have a directory inside the zip, yet zips from github
# do, so we'll need to rezip the file before passing it through to the
# validator.
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
old_filename = '{}_github_webhook.zip'.format(pk)
old_path = os.path.join(loc, old_filename)
with storage.open(old_path, 'wb') as old:
old.write(response.content)
new_filename = '{}_github_webhook.xpi'.format(pk)
new_path = os.path.join(loc, new_filename)
old_zip = SafeZip(old_path)
if not old_zip.is_valid:
raise
with storage.open(new_path, 'w') as new:
new_zip = zipfile.ZipFile(new, 'w')
for obj in old_zip.filelist:
# Basically strip off the leading directory.
new_filename = obj.filename.partition('/')[-1]
if not new_filename:
continue
new_zip.writestr(new_filename, old_zip.read(obj.filename))
new_zip.close()
old_zip.close()
return new_path
def rezip_file(response, pk):
# An .xpi does not have a directory inside the zip, yet zips from github
# do, so we'll need to rezip the file before passing it through to the
# validator.
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
old_filename = '{}_github_webhook.zip'.format(pk)
old_path = os.path.join(loc, old_filename)
with storage.open(old_path, 'wb') as old:
old.write(response.content)
new_filename = '{}_github_webhook.xpi'.format(pk)
new_path = os.path.join(loc, new_filename)
old_zip = SafeZip(old_path)
if not old_zip.is_valid:
raise
with storage.open(new_path, 'w') as new:
new_zip = zipfile.ZipFile(new, 'w')
for obj in old_zip.filelist:
# Basically strip off the leading directory.
new_filename = obj.filename.partition('/')[-1]
if not new_filename:
continue
new_zip.writestr(new_filename, old_zip.read(obj.filename))
new_zip.close()
old_zip.close()
return new_path
def check_for_api_keys_in_file(results, upload_pk):
upload = FileUpload.objects.get(pk=upload_pk)
if upload.addon:
users = upload.addon.authors.all()
else:
users = [upload.user] if upload.user else []
keys = []
for user in users:
try:
key = APIKey.get_jwt_key(user_id=user.id)
keys.append(key)
except APIKey.DoesNotExist:
pass
try:
if len(keys) > 0:
zipfile = SafeZip(source=upload.path)
for zipinfo in zipfile.info_list:
if zipinfo.file_size >= 64:
file_ = zipfile.read(zipinfo)
for key in keys:
if key.secret in file_.decode(errors='ignore'):
log.info('Developer API key for user %s found in '
'submission.' % key.user)
if key.user == upload.user:
msg = gettext('Your developer API key was '
'found in the submitted file. '
'To protect your account, the '
'key will be revoked.')
else:
msg = gettext('The developer API key of a '
'coauthor was found in the '
'submitted file. To protect '
'your add-on, the key will be '
'revoked.')
annotations.insert_validation_message(
results,
type_='error',
message=msg,
msg_id='api_key_detected',
compatibility_type=None,
)
# Revoke after 2 minutes to allow the developer to
# fetch the validation results
revoke_api_key.apply_async(
kwargs={'key_id': key.id}, countdown=120)
zipfile.close()
except (ValidationError, BadZipFile, IOError):
pass
return results
def run_yara(upload_pk):
"""
Apply a set of Yara rules on a FileUpload and store the results.
This task is intended to be run as part of the submission process only.
When a version is created from a FileUpload, the files are removed. In
addition, we usually delete old FileUpload entries after 180 days.
"""
log.info('Starting yara task for FileUpload %s.', upload_pk)
upload = FileUpload.objects.get(pk=upload_pk)
if not upload.path.endswith('.xpi'):
log.info('Not running yara for FileUpload %s, it is not a xpi file.',
upload_pk)
return
try:
result = YaraResult()
result.upload = upload
with statsd.timer('devhub.yara'):
rules = yara.compile(filepath=settings.YARA_RULES_FILEPATH)
zip_file = SafeZip(source=upload.path)
for zip_info in zip_file.info_list:
if not zip_info.is_dir():
file_content = zip_file.read(zip_info).decode(
errors='ignore'
)
for match in rules.match(data=file_content):
# Add the filename to the meta dict.
meta = {**match.meta, 'filename': zip_info.filename}
result.add_match(
rule=match.rule,
tags=match.tags,
meta=meta
)
zip_file.close()
result.save()
statsd.incr('devhub.yara.success')
log.info('Ending yara task for FileUpload %s.', upload_pk)
except Exception:
statsd.incr('devhub.yara.failure')
# We log the exception but we do not raise to avoid perturbing the
# submission flow.
log.exception('Error in yara task for FileUpload %s.', upload_pk)
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, raise_on_failure=False)
if not zip.is_valid():
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError, e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, raise_on_failure=False)
if not zip.is_valid():
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError, e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
def annotate_legacy_addon_restrictions(path, results, parsed_data, error=True):
"""
Annotate validation results to restrict uploads of legacy
(non-webextension) add-ons.
"""
# We can be broad here. Search plugins are not validated through this
# path and as of right now (Jan 2019) there aren't any legacy type
# add-ons allowed to submit anymore.
msg = ugettext(u'Legacy extensions are no longer supported in Firefox.')
description = ugettext(
u'Add-ons for Thunderbird and SeaMonkey are now listed and '
u'maintained on addons.thunderbird.net. You can use the same '
u'account to update your add-ons on the new site.')
# `parsed_data` only contains the most minimal amount of data because
# we aren't in the right context. Let's explicitly fetch the add-ons
# apps so that we can adjust the messaging to the user.
xpi = get_file(path)
extractor = RDFExtractor(SafeZip(xpi))
targets_thunderbird_or_seamonkey = False
thunderbird_or_seamonkey = {amo.THUNDERBIRD.guid, amo.SEAMONKEY.guid}
for ctx in extractor.rdf.objects(None, extractor.uri('targetApplication')):
if extractor.find('id', ctx) in thunderbird_or_seamonkey:
targets_thunderbird_or_seamonkey = True
description = description if targets_thunderbird_or_seamonkey else []
insert_validation_message(results,
type_='error' if error else 'warning',
message=msg,
description=description,
msg_id='legacy_addons_unsupported')
def clean_source(self):
source = self.cleaned_data.get('source')
if source:
try:
if source.name.endswith('.zip'):
zip_file = SafeZip(source)
# testzip() returns None if there are no broken CRCs.
if zip_file.zip_file.testzip() is not None:
raise zipfile.BadZipfile()
elif source.name.endswith(('.tar.gz', '.tar.bz2')):
# For tar files we need to do a little more work.
# Fortunately tarfile.open() already handles compression
# formats for us automatically.
with tarfile.open(fileobj=source) as archive:
archive_members = archive.getmembers()
for member in archive_members:
archive_member_validator(archive, member)
else:
valid_extensions_string = u'(%s)' % u', '.join(
VALID_SOURCE_EXTENSIONS)
raise forms.ValidationError(
ugettext(
'Unsupported file type, please upload an archive '
'file {extensions}.'.format(
extensions=valid_extensions_string)))
except (zipfile.BadZipfile, tarfile.ReadError, IOError):
raise forms.ValidationError(
ugettext('Invalid or broken archive.'))
return source
def test_invalid_zip_encoding(self):
with pytest.raises(forms.ValidationError) as exc:
SafeZip(self.xpi_path('invalid-cp437-encoding.xpi'))
assert isinstance(exc.value, forms.ValidationError)
assert exc.value.message.endswith(
'Please make sure all filenames are utf-8 or latin1 encoded.')
def to_internal_value(self, data):
data = super().to_internal_value(data)
# Ensure the file type is one we support.
if not data.name.endswith(VALID_SOURCE_EXTENSIONS):
error_msg = (
'Unsupported file type, please upload an archive file ({extensions}).'
)
raise exceptions.ValidationError(
error_msg.format(
extensions=(', '.join(VALID_SOURCE_EXTENSIONS))))
# Check inside to see if the file extension matches the content.
try:
_, ext = os.path.splitext(data.name)
if ext == '.zip':
# testzip() returns None if there are no broken CRCs.
if SafeZip(data).zip_file.testzip() is not None:
raise zipfile.BadZipFile()
else:
# For tar files we need to do a little more work.
mode = 'r:bz2' if ext == '.bz2' else 'r:gz'
with tarfile.open(mode=mode, fileobj=data) as archive:
for member in archive.getmembers():
archive_member_validator(archive, member)
except (zipfile.BadZipFile, tarfile.ReadError, OSError, EOFError):
raise exceptions.ValidationError('Invalid or broken archive.')
return data
def check_for_api_keys_in_file(results, upload):
if upload.addon:
users = upload.addon.authors.all()
else:
users = [upload.user] if upload.user else []
keys = []
for user in users:
try:
key = APIKey.get_jwt_key(user_id=user.id)
keys.append(key)
except APIKey.DoesNotExist:
pass
if len(keys) > 0:
zipfile = SafeZip(source=upload.path)
for zipinfo in zipfile.info_list:
if zipinfo.file_size >= 64:
file_ = zipfile.read(zipinfo)
for key in keys:
if key.secret in file_.decode(encoding='unicode-escape',
errors="ignore"):
log.info('Developer API key for user %s found in '
'submission.' % key.user)
if key.user == upload.user:
msg = ugettext('Your developer API key was found '
'in the submitted file. To protect '
'your account, the key will be '
'revoked.')
else:
msg = ugettext('The developer API key of a '
'coauthor was found in the '
'submitted file. To protect your '
'add-on, the key will be revoked.')
insert_validation_message(
results, type_='error',
message=msg, msg_id='api_key_detected',
compatibility_type=None)
# Revoke after 2 minutes to allow the developer to
# fetch the validation results
revoke_api_key.apply_async(
kwargs={'key_id': key.id}, countdown=120)
zipfile.close()
return results
def _run_yara_for_path(scanner_result, path, definition=None):
"""
Inner function to run yara on a particular path and add results to the
given scanner_result. The caller is responsible for saving the
scanner_result to the database.
Takes an optional definition to run a single arbitrary yara rule, otherwise
uses all active yara ScannerRules.
"""
with statsd.timer('devhub.yara'):
if definition is None:
# Retrieve then concatenate all the active/valid Yara rules.
definition = '\n'.join(
ScannerRule.objects.filter(
scanner=YARA, is_active=True, definition__isnull=False
).values_list('definition', flat=True)
)
# Initialize external variables so that compilation works, we'll
# override them later when matching.
externals = ScannerRule.get_yara_externals()
rules = yara.compile(source=definition, externals=externals)
zip_file = SafeZip(source=path)
for zip_info in zip_file.info_list:
if not zip_info.is_dir():
file_content = zip_file.read(zip_info).decode(
errors='ignore'
)
filename = zip_info.filename
# Fill externals variable for this file.
externals['is_json_file'] = filename.endswith('.json')
externals['is_manifest_file'] = filename == 'manifest.json'
externals['is_locale_file'] = (
filename.startswith('_locales/') and
filename.endswith('/messages.json')
)
for match in rules.match(
data=file_content, externals=externals):
# Also add the filename to the meta dict in results.
meta = {**match.meta, 'filename': filename}
scanner_result.add_yara_result(
rule=match.rule,
tags=match.tags,
meta=meta
)
zip_file.close()
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, validate=False)
try:
is_valid = zip.is_valid()
except (zipfile.BadZipfile, IOError):
is_valid = False
if not is_valid:
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError as e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
res = self._get_localepicker.search(manifest)
if not res:
log.error('Locale browser not in chrome.manifest: %s' % self.pk)
return ''
try:
p = res.groups()[1]
if 'localepicker.properties' not in p:
p = os.path.join(p, 'localepicker.properties')
res = zip.extract_from_manifest(p)
except (zipfile.BadZipfile, IOError) as e:
log.error('Error unzipping: %s, %s in file: %s' % (p, e, self.pk))
return ''
except (ValueError, KeyError) as e:
log.error('No file named: %s in file: %s' % (e, self.pk))
return ''
end = time.time() - start
log.info('Extracted localepicker file: %s in %.2fs' %
(self.pk, end))
statsd.timing('files.extract.localepicker', (end * 1000))
return res
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, validate=False)
try:
is_valid = zip.is_valid()
except (zipfile.BadZipfile, IOError):
is_valid = False
if not is_valid:
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError as e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
res = self._get_localepicker.search(manifest)
if not res:
log.error('Locale browser not in chrome.manifest: %s' % self.pk)
return ''
try:
p = res.groups()[1]
if 'localepicker.properties' not in p:
p = os.path.join(p, 'localepicker.properties')
res = zip.extract_from_manifest(p)
except (zipfile.BadZipfile, IOError) as e:
log.error('Error unzipping: %s, %s in file: %s' % (p, e, self.pk))
return ''
except (ValueError, KeyError) as e:
log.error('No file named: %s in file: %s' % (e, self.pk))
return ''
end = time.time() - start
log.info('Extracted localepicker file: %s in %.2fs' %
(self.pk, end))
statsd.timing('files.extract.localepicker', (end * 1000))
return res
def extract_strict_compatibility_value_for_addon(addon):
strict_compatibility = None # We don't know yet.
try:
# We take a shortcut here and only look at the first file we
# find...
# Note that we can't use parse_addon() wrapper because it no longer
# exposes the real value of `strictCompatibility`...
path = addon.current_version.all_files[0].file_path
zip_file = SafeZip(get_file(path))
parser = RDFExtractor(zip_file)
strict_compatibility = parser.find('strictCompatibility') == 'true'
except Exception as exp:
# A number of things can go wrong: missing file, path somehow not
# existing, etc. In any case, that means the add-on is in a weird
# state and should be ignored (this is a one off task).
log.exception(u'bump_appver_for_legacy_addons: ignoring addon %d, '
u'received %s when extracting.', addon.pk, unicode(exp))
return strict_compatibility
def test_is_broken(self):
zip_file = SafeZip(self.xpi_path('signed'))
zip_file.info_list[2].filename = 'META-INF/foo.sf'
assert not zip_file.is_signed()
def test_is_secure(self):
zip_file = SafeZip(self.xpi_path('signed'))
assert zip_file.is_signed()
def test_not_secure(self):
zip_file = SafeZip(self.xpi_path('extension'))
assert not zip_file.is_signed()
def test_unzip_not_fatal(self):
zip_file = SafeZip(self.xpi_path('search.xml'), raise_on_failure=False)
assert not zip_file.is_valid()
def test_read(self):
zip_file = SafeZip(self.xpi_path('langpack-localepicker'))
assert zip_file.is_valid
assert b'locale browser de' in zip_file.read('chrome.manifest')
def test_unzip_not_fatal(self):
zip_file = SafeZip(self.xpi_path('search.xml'), raise_on_failure=False)
assert not zip_file.is_valid()
def extract_and_commit_from_file_obj(cls, file_obj, channel, author=None):
"""Extract all files from `file_obj` and comit them.
This is doing the following:
* Create a temporary `git worktree`_
* Remove all files in that worktree
* Extract the zip behind `file_obj` into the worktree
* Commit all files
Kinda like doing::
$ workdir_name=$(uuid)
$ mkdir /tmp/$workdir_name
$ git worktree add /tmp/$workdir_name
Preparing worktree (new branch 'af4172e4-d8c7…')
HEAD is now at 8c5223e Initial commit
$ git worktree list
/tmp/addon-repository 8c5223e [master]
/tmp/af4172e4-d8c7-4486-a5f2-316458da91ff 8c5223e [af4172e4-d8c7…]
$ unzip dingrafowl-falcockalo-lockapionk.zip -d /tmp/$workdir_name
Archive: dingrafowl-falcockalo-lockapionk.zip
extracting: /tmp/af4172e4-d8c7…/manifest.json
$ pushd /tmp/$workdir_name
/tmp/af4172e4-d8c7-4486-a5f2-316458da91ff /tmp/addon-repository
$ git status
On branch af4172e4-d8c7-4486-a5f2-316458da91ff
Untracked files:
(use "git add <file>..." to include in what will be committed)
manifest.json
$ git add *
$ git commit -a -m "Creating new version"
[af4172e4-d8c7-4486-a5f2-316458da91ff c4285f8] Creating new version
…
$ cd addon-repository
$ git checkout -b listed
Switched to a new branch 'listed'
# We don't technically do a full cherry-pick but it's close enough
# and does almost what we do. We are technically commiting
# directly on top of the branch as if we checked out the branch
# in the worktree (via -b) but pygit doesn't properly support that
# so we "simply" set the parents correctly.
$ git cherry-pick c4285f8
[listed a4d0f63] Creating new version…
This ignores the fact that there may be a race-condition of two
versions being created at the same time. Since all relevant file based
work is done in a temporary worktree there won't be any conflicts and
usually the last upload simply wins the race and we're setting the
HEAD of the branch (listed/unlisted) to that specific commit.
.. _`git worktree`: https://git-scm.com/docs/git-worktree
"""
# Make sure we're always using the en-US locale by default
translation.activate('en-US')
addon = file_obj.version.addon
repo = cls(addon.id)
branch = repo.find_or_create_branch(BRANCHES[channel])
# Create a temporary worktree that we can use to unpack the zip
# without disturbing the current git workdir since it creates a new
# temporary directory where we extract to.
with TemporaryWorktree(repo.git_repository) as worktree:
# Now extract the zip to the workdir
zip_file = SafeZip(file_obj.current_file_path, force_fsync=True)
zip_file.extract_to_dest(worktree.path)
# Stage changes, `TemporaryWorktree` always cleans the whole
# directory so we can simply add all changes and have the correct
# state.
# Add all changes to the index (git add ...)
worktree.repo.index.add_all()
worktree.repo.index.write()
tree = worktree.repo.index.write_tree()
# Now create an commit directly on top of the respective branch
message = (
'Create new version {version} ({version_id}) for '
'{addon} from {file_obj}'.format(
version=repr(file_obj.version),
version_id=file_obj.version.id,
addon=repr(addon),
file_obj=repr(file_obj)))
oid = worktree.repo.create_commit(
None,
# author, using the actual uploading user
repo.get_author(author),
# committer, using addons-robot because that's the user
# actually doing the commit.
repo.get_author(), # commiter, using addons-robot
message,
tree,
# Set the current branch HEAD as the parent of this commit
# so that it'll go straight into the branches commit log
[branch.target]
)
# Fetch the commit object
commit = worktree.repo.get(oid)
# And set the commit we just created as HEAD of the relevant
# branch, and updates the reflog. This does not require any
# merges.
branch.set_target(commit.hex)
# Set the latest git hash on the related version.
file_obj.version.update(git_hash=commit.hex)
return repo
def test_read(self):
zip_file = SafeZip(self.xpi_path('langpack-localepicker'))
assert zip_file.is_valid()
assert 'locale browser de' in zip_file.read('chrome.manifest')
def test_not_secure(self):
zip_file = SafeZip(self.xpi_path('extension'))
zip_file.is_valid()
assert not zip_file.is_signed()
def test_is_secure(self):
zip_file = SafeZip(self.xpi_path('signed'))
zip_file.is_valid()
assert zip_file.is_signed()
def test_is_broken(self):
zip_file = SafeZip(self.xpi_path('signed'))
zip_file.is_valid()
zip_file.info_list[2].filename = 'META-INF/foo.sf'
assert not zip_file.is_signed()
def build_webext_dictionary_from_legacy(addon, destination):
"""Create a webext package of a legacy dictionary `addon`, and put it in
`destination` path."""
from olympia.files.utils import SafeZip # Avoid circular import.
old_path = addon.current_version.all_files[0].file_path
old_zip = SafeZip(old_path)
if not old_zip.is_valid:
raise ValidationError('Current dictionary xpi is not valid')
dictionary_path = ''
with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as new_zip:
for obj in old_zip.filelist:
splitted = obj.filename.split('/')
# Ignore useless directories and files.
if splitted[0] in ('META-INF', '__MACOSX', 'chrome',
'chrome.manifest', 'install.rdf'):
continue
# Also ignore javascript (regardless of where they are, not just at
# the root), since dictionaries should not contain any code.
if splitted[-1].endswith('.js'):
continue
# Store the path of the last .dic file we find. It can be inside a
# directory.
if (splitted[-1].endswith('.dic')):
dictionary_path = obj.filename
new_zip.writestr(obj.filename, old_zip.read(obj.filename))
# Now that all files we want from the old zip are copied, build and
# add manifest.json.
if not dictionary_path:
# This should not happen... It likely means it's an invalid
# dictionary to begin with, or one that has its .dic file in a
# chrome/ directory for some reason. Abort!
raise ValidationError('Current dictionary xpi has no .dic file')
if addon.target_locale:
target_language = addon.target_locale
else:
# Guess target_locale since we don't have one already. Note that
# for extra confusion, target_locale is a language, not a locale.
target_language = to_language(
os.path.splitext(os.path.basename(dictionary_path))[0])
if target_language not in settings.AMO_LANGUAGES:
# We couldn't find that language in the list we support. Let's
# try with just the prefix.
target_language = target_language.split('-')[0]
if target_language not in settings.AMO_LANGUAGES:
# We tried our best.
raise ValidationError(u'Addon has no target_locale and we'
u' could not guess one from the xpi')
# Dumb version number increment. This will be invalid in some cases,
# but some of the dictionaries we have currently already have wild
# version numbers anyway.
version_number = addon.current_version.version
if version_number.endswith('.1-typefix'):
version_number = version_number.replace('.1-typefix', '.2webext')
else:
version_number = '%s.1webext' % version_number
manifest = {
'manifest_version': 2,
'name': unicode(addon.name),
'browser_specific_settings': {
'gecko': {
'id': addon.guid,
},
},
'version': version_number,
'dictionaries': {
target_language: dictionary_path
},
}
# Write manifest.json we just build.
new_zip.writestr('manifest.json', json.dumps(manifest))
def test_unzip_limit(self):
with pytest.raises(forms.ValidationError):
SafeZip(self.xpi_path('langpack-localepicker'))
def test_unzip_fatal(self):
with pytest.raises(zipfile.BadZipfile):
SafeZip(self.xpi_path('search.xml'))
def build_webext_dictionary_from_legacy(addon, destination):
"""Create a webext package of a legacy dictionary `addon`, and put it in
`destination` path."""
from olympia.files.utils import SafeZip # Avoid circular import.
old_path = addon.current_version.all_files[0].file_path
old_zip = SafeZip(old_path)
if not old_zip.is_valid:
raise ValidationError('Current dictionary xpi is not valid')
dictionary_path = ''
with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as new_zip:
for obj in old_zip.filelist:
splitted = obj.filename.split('/')
# Ignore useless directories and files.
if splitted[0] in ('META-INF', '__MACOSX', 'chrome',
'chrome.manifest', 'install.rdf'):
continue
# Also ignore javascript (regardless of where they are, not just at
# the root), since dictionaries should not contain any code.
if splitted[-1].endswith('.js'):
continue
# Store the path of the last .dic file we find. It can be inside a
# directory.
if (splitted[-1].endswith('.dic')):
dictionary_path = obj.filename
new_zip.writestr(obj.filename, old_zip.read(obj.filename))
# Now that all files we want from the old zip are copied, build and
# add manifest.json.
if not dictionary_path:
# This should not happen... It likely means it's an invalid
# dictionary to begin with, or one that has its .dic file in a
# chrome/ directory for some reason. Abort!
raise ValidationError('Current dictionary xpi has no .dic file')
if addon.target_locale:
target_language = addon.target_locale
else:
# Guess target_locale since we don't have one already. Note that
# for extra confusion, target_locale is a language, not a locale.
target_language = to_language(os.path.splitext(
os.path.basename(dictionary_path))[0])
if target_language not in settings.AMO_LANGUAGES:
# We couldn't find that language in the list we support. Let's
# try with just the prefix.
target_language = target_language.split('-')[0]
if target_language not in settings.AMO_LANGUAGES:
# We tried our best.
raise ValidationError(u'Addon has no target_locale and we'
u' could not guess one from the xpi')
# Dumb version number increment. This will be invalid in some cases,
# but some of the dictionaries we have currently already have wild
# version numbers anyway.
version_number = addon.current_version.version
if version_number.endswith('.1-typefix'):
version_number = version_number.replace('.1-typefix', '.2webext')
else:
version_number = '%s.1webext' % version_number
manifest = {
'manifest_version': 2,
'name': unicode(addon.name),
'applications': {
'gecko': {
'id': addon.guid,
},
},
'version': version_number,
'dictionaries': {target_language: dictionary_path},
}
# Write manifest.json we just build.
new_zip.writestr('manifest.json', json.dumps(manifest))
def extract_and_commit_from_file_obj(cls, file_obj, channel, author=None):
"""Extract all files from `file_obj` and comit them.
This is doing the following:
* Create a temporary `git worktree`_
* Remove all files in that worktree
* Extract the zip behind `file_obj` into the worktree
* Commit all files
Kinda like doing::
$ workdir_name=$(uuid)
$ mkdir /tmp/$workdir_name
$ git worktree add /tmp/$workdir_name
Preparing worktree (new branch 'af4172e4-d8c7…')
HEAD is now at 8c5223e Initial commit
$ git worktree list
/tmp/addon-repository 8c5223e [master]
/tmp/af4172e4-d8c7-4486-a5f2-316458da91ff 8c5223e [af4172e4-d8c7…]
$ unzip dingrafowl-falcockalo-lockapionk.zip -d /tmp/$workdir_name
Archive: dingrafowl-falcockalo-lockapionk.zip
extracting: /tmp/af4172e4-d8c7…/manifest.json
$ pushd /tmp/$workdir_name
/tmp/af4172e4-d8c7-4486-a5f2-316458da91ff /tmp/addon-repository
$ git status
On branch af4172e4-d8c7-4486-a5f2-316458da91ff
Untracked files:
(use "git add <file>..." to include in what will be committed)
manifest.json
$ git add *
$ git commit -a -m "Creating new version"
[af4172e4-d8c7-4486-a5f2-316458da91ff c4285f8] Creating new version
…
$ cd addon-repository
$ git checkout -b listed
Switched to a new branch 'listed'
# We don't technically do a full cherry-pick but it's close enough
# and does almost what we do. We are technically commiting
# directly on top of the branch as if we checked out the branch
# in the worktree (via -b) but pygit doesn't properly support that
# so we "simply" set the parents correctly.
$ git cherry-pick c4285f8
[listed a4d0f63] Creating new version…
This ignores the fact that there may be a race-condition of two
versions being created at the same time. Since all relevant file based
work is done in a temporary worktree there won't be any conflicts and
usually the last upload simply wins the race and we're setting the
HEAD of the branch (listed/unlisted) to that specific commit.
.. _`git worktree`: https://git-scm.com/docs/git-worktree
"""
# Make sure we're always using the en-US locale by default
translation.activate('en-US')
addon = file_obj.version.addon
repo = cls(addon.id)
branch = repo.find_or_create_branch(BRANCHES[channel])
# Create a temporary worktree that we can use to unpack the zip
# without disturbing the current git workdir since it creates a new
# temporary directory where we extract to.
with TemporaryWorktree(repo.git_repository) as worktree:
# Now extract the zip to the workdir
zip_file = SafeZip(file_obj.current_file_path, force_fsync=True)
zip_file.extract_to_dest(worktree.path)
# Stage changes, `TemporaryWorktree` always cleans the whole
# directory so we can simply add all changes and have the correct
# state.
# Add all changes to the index (git add ...)
worktree.repo.index.add_all()
worktree.repo.index.write()
tree = worktree.repo.index.write_tree()
# Now create an commit directly on top of the respective branch
message = ('Create new version {version} ({version_id}) for '
'{addon} from {file_obj}'.format(
version=repr(file_obj.version),
version_id=file_obj.version.id,
addon=repr(addon),
file_obj=repr(file_obj)))
oid = worktree.repo.create_commit(
None,
# author, using the actual uploading user
repo.get_author(author),
# committer, using addons-robot because that's the user
# actually doing the commit.
repo.get_author(), # commiter, using addons-robot
message,
tree,
# Set the current branch HEAD as the parent of this commit
# so that it'll go straight into the branches commit log
[branch.target])
# Fetch the commit object
commit = worktree.repo.get(oid)
# And set the commit we just created as HEAD of the relevant
# branch, and updates the reflog. This does not require any
# merges.
branch.set_target(commit.hex)
# Set the latest git hash on the related version.
file_obj.version.update(git_hash=commit.hex)
return repo
