def _run_yara_for_path(scanner_result, path, definition=None):
with statsd.timer('devhub.yara'):
if definition is None:
# Retrieve then concatenate all the active/valid Yara rules.
definition = '\n'.join(
ScannerRule.objects.filter(
scanner=YARA, is_active=True, definition__isnull=False
).values_list('definition', flat=True)
)
rules = yara.compile(source=definition)
zip_file = SafeZip(source=path)
for zip_info in zip_file.info_list:
if not zip_info.is_dir():
file_content = zip_file.read(zip_info).decode(
errors='ignore'
)
for match in rules.match(data=file_content):
# Add the filename to the meta dict.
meta = {**match.meta, 'filename': zip_info.filename}
scanner_result.add_yara_result(
rule=match.rule,
tags=match.tags,
meta=meta
)
zip_file.close()
def rezip_file(response, pk):
# An .xpi does not have a directory inside the zip, yet zips from github
# do, so we'll need to rezip the file before passing it through to the
# validator.
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
old_filename = '{}_github_webhook.zip'.format(pk)
old_path = os.path.join(loc, old_filename)
with storage.open(old_path, 'wb') as old:
old.write(response.content)
new_filename = '{}_github_webhook.xpi'.format(pk)
new_path = os.path.join(loc, new_filename)
old_zip = SafeZip(old_path)
if not old_zip.is_valid:
raise
with storage.open(new_path, 'w') as new:
new_zip = zipfile.ZipFile(new, 'w')
for obj in old_zip.filelist:
# Basically strip off the leading directory.
new_filename = obj.filename.partition('/')[-1]
if not new_filename:
continue
new_zip.writestr(new_filename, old_zip.read(obj.filename))
new_zip.close()
old_zip.close()
return new_path
def rezip_file(response, pk):
# An .xpi does not have a directory inside the zip, yet zips from github
# do, so we'll need to rezip the file before passing it through to the
# validator.
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
old_filename = '{}_github_webhook.zip'.format(pk)
old_path = os.path.join(loc, old_filename)
with storage.open(old_path, 'wb') as old:
old.write(response.content)
new_filename = '{}_github_webhook.xpi'.format(pk)
new_path = os.path.join(loc, new_filename)
old_zip = SafeZip(old_path)
if not old_zip.is_valid():
raise
with storage.open(new_path, 'w') as new:
new_zip = zipfile.ZipFile(new, 'w')
for obj in old_zip.filelist:
# Basically strip off the leading directory.
new_filename = obj.filename.partition('/')[-1]
if not new_filename:
continue
new_zip.writestr(new_filename, old_zip.read(obj.filename))
new_zip.close()
old_zip.close()
return new_path
def build_webext_dictionary_from_legacy(addon, destination):
"""Create a webext package of a legacy dictionary `addon`, and put it in
`destination` path."""
from olympia.files.utils import SafeZip # Avoid circular import.
old_path = addon.current_version.all_files[0].file_path
old_zip = SafeZip(old_path)
if not old_zip.is_valid():
raise ValidationError('Current dictionary xpi is not valid')
if not addon.target_locale:
raise ValidationError('Addon has no target_locale')
dictionary_path = ''
with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as new_zip:
for obj in old_zip.filelist:
splitted = obj.filename.split('/')
# Ignore useless directories and files.
if splitted[0] in ('META-INF', '__MACOSX', 'chrome',
'chrome.manifest', 'install.rdf'):
continue
# Also ignore javascript (regardless of where they are, not just at
# the root), since dictionaries should not contain any code.
if splitted[-1].endswith('.js'):
continue
# Store the path of the last .dic file we find. It can be inside a
# directory.
if (splitted[-1].endswith('.dic')):
dictionary_path = obj.filename
new_zip.writestr(obj.filename, old_zip.read(obj.filename))
# Now that all files we want from the old zip are copied, build and
# add manifest.json.
if not dictionary_path:
# This should not happen... It likely means it's an invalid
# dictionary to begin with, or one that has its .dic file in a
# chrome/ directory for some reason. Abort!
raise ValidationError('Current dictionary xpi has no .dic file')
# Dumb version number increment. This will be invalid in some cases,
# but some of the dictionaries we have currently already have wild
# version numbers anyway.
version_number = addon.current_version.version
if version_number.endswith('.1-typefix'):
version_number = version_number.replace('.1-typefix', '.2webext')
else:
version_number = '%s.1webext' % version_number
manifest = {
'manifest_version': 2,
'name': unicode(addon.name),
'version': version_number,
'dictionaries': {addon.target_locale: dictionary_path},
}
# Write manifest.json we just build.
new_zip.writestr('manifest.json', json.dumps(manifest))
def run_yara(results, upload_pk):
"""
Apply a set of Yara rules on a FileUpload and store the Yara results
(matches).
This task is intended to be run as part of the submission process only.
When a version is created from a FileUpload, the files are removed. In
addition, we usually delete old FileUpload entries after 180 days.
- `results` are the validation results passed in the validation chain. This
task is a validation task, which is why it must receive the validation
results as first argument.
- `upload_pk` is the FileUpload ID.
"""
log.info('Starting yara task for FileUpload %s.', upload_pk)
if not results['metadata']['is_webextension']:
log.info('Not running yara for FileUpload %s, it is not a '
'webextension.', upload_pk)
return results
upload = FileUpload.objects.get(pk=upload_pk)
try:
scanner_result = ScannerResult(upload=upload, scanner=YARA)
with statsd.timer('devhub.yara'):
rules = yara.compile(filepath=settings.YARA_RULES_FILEPATH)
zip_file = SafeZip(source=upload.path)
for zip_info in zip_file.info_list:
if not zip_info.is_dir():
file_content = zip_file.read(zip_info).decode(
errors='ignore'
)
for match in rules.match(data=file_content):
# Add the filename to the meta dict.
meta = {**match.meta, 'filename': zip_info.filename}
scanner_result.add_yara_result(
rule=match.rule,
tags=match.tags,
meta=meta
)
zip_file.close()
scanner_result.save()
if scanner_result.has_matches:
statsd.incr('devhub.yara.has_matches')
statsd.incr('devhub.yara.success')
log.info('Ending scanner "yara" task for FileUpload %s.', upload_pk)
except Exception:
statsd.incr('devhub.yara.failure')
# We log the exception but we do not raise to avoid perturbing the
# submission flow.
log.exception('Error in scanner "yara" task for FileUpload %s.',
upload_pk)
return results
def check_for_api_keys_in_file(results, upload_pk):
upload = FileUpload.objects.get(pk=upload_pk)
if upload.addon:
users = upload.addon.authors.all()
else:
users = [upload.user] if upload.user else []
keys = []
for user in users:
try:
key = APIKey.get_jwt_key(user_id=user.id)
keys.append(key)
except APIKey.DoesNotExist:
pass
try:
if len(keys) > 0:
zipfile = SafeZip(source=upload.path)
for zipinfo in zipfile.info_list:
if zipinfo.file_size >= 64:
file_ = zipfile.read(zipinfo)
for key in keys:
if key.secret in file_.decode(errors='ignore'):
log.info('Developer API key for user %s found in '
'submission.' % key.user)
if key.user == upload.user:
msg = gettext('Your developer API key was '
'found in the submitted file. '
'To protect your account, the '
'key will be revoked.')
else:
msg = gettext('The developer API key of a '
'coauthor was found in the '
'submitted file. To protect '
'your add-on, the key will be '
'revoked.')
annotations.insert_validation_message(
results,
type_='error',
message=msg,
msg_id='api_key_detected',
compatibility_type=None,
)
# Revoke after 2 minutes to allow the developer to
# fetch the validation results
revoke_api_key.apply_async(
kwargs={'key_id': key.id}, countdown=120)
zipfile.close()
except (ValidationError, BadZipFile, IOError):
pass
return results
def run_yara(upload_pk):
"""
Apply a set of Yara rules on a FileUpload and store the results.
This task is intended to be run as part of the submission process only.
When a version is created from a FileUpload, the files are removed. In
addition, we usually delete old FileUpload entries after 180 days.
"""
log.info('Starting yara task for FileUpload %s.', upload_pk)
upload = FileUpload.objects.get(pk=upload_pk)
if not upload.path.endswith('.xpi'):
log.info('Not running yara for FileUpload %s, it is not a xpi file.',
upload_pk)
return
try:
result = YaraResult()
result.upload = upload
with statsd.timer('devhub.yara'):
rules = yara.compile(filepath=settings.YARA_RULES_FILEPATH)
zip_file = SafeZip(source=upload.path)
for zip_info in zip_file.info_list:
if not zip_info.is_dir():
file_content = zip_file.read(zip_info).decode(
errors='ignore'
)
for match in rules.match(data=file_content):
# Add the filename to the meta dict.
meta = {**match.meta, 'filename': zip_info.filename}
result.add_match(
rule=match.rule,
tags=match.tags,
meta=meta
)
zip_file.close()
result.save()
statsd.incr('devhub.yara.success')
log.info('Ending yara task for FileUpload %s.', upload_pk)
except Exception:
statsd.incr('devhub.yara.failure')
# We log the exception but we do not raise to avoid perturbing the
# submission flow.
log.exception('Error in yara task for FileUpload %s.', upload_pk)
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, raise_on_failure=False)
if not zip.is_valid():
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError, e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, raise_on_failure=False)
if not zip.is_valid():
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError, e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
def check_for_api_keys_in_file(results, upload):
if upload.addon:
users = upload.addon.authors.all()
else:
users = [upload.user] if upload.user else []
keys = []
for user in users:
try:
key = APIKey.get_jwt_key(user_id=user.id)
keys.append(key)
except APIKey.DoesNotExist:
pass
if len(keys) > 0:
zipfile = SafeZip(source=upload.path)
zipfile.is_valid()
for zipinfo in zipfile.info_list:
if zipinfo.file_size >= 64:
file_ = zipfile.read(zipinfo)
for key in keys:
if key.secret in file_.decode(encoding='unicode-escape',
errors="ignore"):
log.info('Developer API key for user %s found in '
'submission.' % key.user)
if key.user == upload.user:
msg = ugettext('Your developer API key was found '
'in the submitted file. To protect '
'your account, the key will be '
'revoked.')
else:
msg = ugettext('The developer API key of a '
'coauthor was found in the '
'submitted file. To protect your '
'add-on, the key will be revoked.')
insert_validation_message(
results, type_='error',
message=msg, msg_id='api_key_detected',
compatibility_type=None)
# Revoke after 2 minutes to allow the developer to
# fetch the validation results
revoke_api_key.apply_async(
kwargs={'key_id': key.id}, countdown=120)
zipfile.close()
return results
def _run_yara_for_path(scanner_result, path, definition=None):
"""
Inner function to run yara on a particular path and add results to the
given scanner_result. The caller is responsible for saving the
scanner_result to the database.
Takes an optional definition to run a single arbitrary yara rule, otherwise
uses all active yara ScannerRules.
"""
with statsd.timer('devhub.yara'):
if definition is None:
# Retrieve then concatenate all the active/valid Yara rules.
definition = '\n'.join(
ScannerRule.objects.filter(
scanner=YARA, is_active=True, definition__isnull=False
).values_list('definition', flat=True)
)
# Initialize external variables so that compilation works, we'll
# override them later when matching.
externals = ScannerRule.get_yara_externals()
rules = yara.compile(source=definition, externals=externals)
zip_file = SafeZip(source=path)
for zip_info in zip_file.info_list:
if not zip_info.is_dir():
file_content = zip_file.read(zip_info).decode(
errors='ignore'
)
filename = zip_info.filename
# Fill externals variable for this file.
externals['is_json_file'] = filename.endswith('.json')
externals['is_manifest_file'] = filename == 'manifest.json'
externals['is_locale_file'] = (
filename.startswith('_locales/') and
filename.endswith('/messages.json')
)
for match in rules.match(
data=file_content, externals=externals):
# Also add the filename to the meta dict in results.
meta = {**match.meta, 'filename': filename}
scanner_result.add_yara_result(
rule=match.rule,
tags=match.tags,
meta=meta
)
zip_file.close()
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, validate=False)
try:
is_valid = zip.is_valid()
except (zipfile.BadZipfile, IOError):
is_valid = False
if not is_valid:
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError as e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
res = self._get_localepicker.search(manifest)
if not res:
log.error('Locale browser not in chrome.manifest: %s' % self.pk)
return ''
try:
p = res.groups()[1]
if 'localepicker.properties' not in p:
p = os.path.join(p, 'localepicker.properties')
res = zip.extract_from_manifest(p)
except (zipfile.BadZipfile, IOError) as e:
log.error('Error unzipping: %s, %s in file: %s' % (p, e, self.pk))
return ''
except (ValueError, KeyError) as e:
log.error('No file named: %s in file: %s' % (e, self.pk))
return ''
end = time.time() - start
log.info('Extracted localepicker file: %s in %.2fs' %
(self.pk, end))
statsd.timing('files.extract.localepicker', (end * 1000))
return res
def get_localepicker(self):
"""
For a file that is part of a language pack, extract
the chrome/localepicker.properties file and return as
a string.
"""
start = time.time()
zip = SafeZip(self.file_path, validate=False)
try:
is_valid = zip.is_valid()
except (zipfile.BadZipfile, IOError):
is_valid = False
if not is_valid:
return ''
try:
manifest = zip.read('chrome.manifest')
except KeyError as e:
log.info('No file named: chrome.manifest in file: %s' % self.pk)
return ''
res = self._get_localepicker.search(manifest)
if not res:
log.error('Locale browser not in chrome.manifest: %s' % self.pk)
return ''
try:
p = res.groups()[1]
if 'localepicker.properties' not in p:
p = os.path.join(p, 'localepicker.properties')
res = zip.extract_from_manifest(p)
except (zipfile.BadZipfile, IOError) as e:
log.error('Error unzipping: %s, %s in file: %s' % (p, e, self.pk))
return ''
except (ValueError, KeyError) as e:
log.error('No file named: %s in file: %s' % (e, self.pk))
return ''
end = time.time() - start
log.info('Extracted localepicker file: %s in %.2fs' %
(self.pk, end))
statsd.timing('files.extract.localepicker', (end * 1000))
return res
def build_webext_dictionary_from_legacy(addon, destination):
"""Create a webext package of a legacy dictionary `addon`, and put it in
`destination` path."""
from olympia.files.utils import SafeZip # Avoid circular import.
old_path = addon.current_version.all_files[0].file_path
old_zip = SafeZip(old_path)
if not old_zip.is_valid:
raise ValidationError('Current dictionary xpi is not valid')
dictionary_path = ''
with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as new_zip:
for obj in old_zip.filelist:
splitted = obj.filename.split('/')
# Ignore useless directories and files.
if splitted[0] in ('META-INF', '__MACOSX', 'chrome',
'chrome.manifest', 'install.rdf'):
continue
# Also ignore javascript (regardless of where they are, not just at
# the root), since dictionaries should not contain any code.
if splitted[-1].endswith('.js'):
continue
# Store the path of the last .dic file we find. It can be inside a
# directory.
if (splitted[-1].endswith('.dic')):
dictionary_path = obj.filename
new_zip.writestr(obj.filename, old_zip.read(obj.filename))
# Now that all files we want from the old zip are copied, build and
# add manifest.json.
if not dictionary_path:
# This should not happen... It likely means it's an invalid
# dictionary to begin with, or one that has its .dic file in a
# chrome/ directory for some reason. Abort!
raise ValidationError('Current dictionary xpi has no .dic file')
if addon.target_locale:
target_language = addon.target_locale
else:
# Guess target_locale since we don't have one already. Note that
# for extra confusion, target_locale is a language, not a locale.
target_language = to_language(os.path.splitext(
os.path.basename(dictionary_path))[0])
if target_language not in settings.AMO_LANGUAGES:
# We couldn't find that language in the list we support. Let's
# try with just the prefix.
target_language = target_language.split('-')[0]
if target_language not in settings.AMO_LANGUAGES:
# We tried our best.
raise ValidationError(u'Addon has no target_locale and we'
u' could not guess one from the xpi')
# Dumb version number increment. This will be invalid in some cases,
# but some of the dictionaries we have currently already have wild
# version numbers anyway.
version_number = addon.current_version.version
if version_number.endswith('.1-typefix'):
version_number = version_number.replace('.1-typefix', '.2webext')
else:
version_number = '%s.1webext' % version_number
manifest = {
'manifest_version': 2,
'name': unicode(addon.name),
'applications': {
'gecko': {
'id': addon.guid,
},
},
'version': version_number,
'dictionaries': {target_language: dictionary_path},
}
# Write manifest.json we just build.
new_zip.writestr('manifest.json', json.dumps(manifest))
def test_read(self):
zip_file = SafeZip(self.xpi_path('langpack-localepicker'))
assert zip_file.is_valid
assert b'locale browser de' in zip_file.read('chrome.manifest')
def test_read(self):
zip_file = SafeZip(self.xpi_path('langpack-localepicker'))
assert zip_file.is_valid()
assert 'locale browser de' in zip_file.read('chrome.manifest')
def build_webext_dictionary_from_legacy(addon, destination):
"""Create a webext package of a legacy dictionary `addon`, and put it in
`destination` path."""
from olympia.files.utils import SafeZip # Avoid circular import.
old_path = addon.current_version.all_files[0].file_path
old_zip = SafeZip(old_path)
if not old_zip.is_valid:
raise ValidationError('Current dictionary xpi is not valid')
dictionary_path = ''
with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as new_zip:
for obj in old_zip.filelist:
splitted = obj.filename.split('/')
# Ignore useless directories and files.
if splitted[0] in ('META-INF', '__MACOSX', 'chrome',
'chrome.manifest', 'install.rdf'):
continue
# Also ignore javascript (regardless of where they are, not just at
# the root), since dictionaries should not contain any code.
if splitted[-1].endswith('.js'):
continue
# Store the path of the last .dic file we find. It can be inside a
# directory.
if (splitted[-1].endswith('.dic')):
dictionary_path = obj.filename
new_zip.writestr(obj.filename, old_zip.read(obj.filename))
# Now that all files we want from the old zip are copied, build and
# add manifest.json.
if not dictionary_path:
# This should not happen... It likely means it's an invalid
# dictionary to begin with, or one that has its .dic file in a
# chrome/ directory for some reason. Abort!
raise ValidationError('Current dictionary xpi has no .dic file')
if addon.target_locale:
target_language = addon.target_locale
else:
# Guess target_locale since we don't have one already. Note that
# for extra confusion, target_locale is a language, not a locale.
target_language = to_language(
os.path.splitext(os.path.basename(dictionary_path))[0])
if target_language not in settings.AMO_LANGUAGES:
# We couldn't find that language in the list we support. Let's
# try with just the prefix.
target_language = target_language.split('-')[0]
if target_language not in settings.AMO_LANGUAGES:
# We tried our best.
raise ValidationError(u'Addon has no target_locale and we'
u' could not guess one from the xpi')
# Dumb version number increment. This will be invalid in some cases,
# but some of the dictionaries we have currently already have wild
# version numbers anyway.
version_number = addon.current_version.version
if version_number.endswith('.1-typefix'):
version_number = version_number.replace('.1-typefix', '.2webext')
else:
version_number = '%s.1webext' % version_number
manifest = {
'manifest_version': 2,
'name': unicode(addon.name),
'browser_specific_settings': {
'gecko': {
'id': addon.guid,
},
},
'version': version_number,
'dictionaries': {
target_language: dictionary_path
},
}
# Write manifest.json we just build.
new_zip.writestr('manifest.json', json.dumps(manifest))
