def rezip_file(response, pk):
# An .xpi does not have a directory inside the zip, yet zips from github
# do, so we'll need to rezip the file before passing it through to the
# validator.
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
old_filename = '{}_github_webhook.zip'.format(pk)
old_path = os.path.join(loc, old_filename)
with storage.open(old_path, 'wb') as old:
old.write(response.content)
new_filename = '{}_github_webhook.xpi'.format(pk)
new_path = os.path.join(loc, new_filename)
old_zip = SafeZip(old_path)
if not old_zip.is_valid:
raise
with storage.open(new_path, 'w') as new:
new_zip = zipfile.ZipFile(new, 'w')
for obj in old_zip.filelist:
# Basically strip off the leading directory.
new_filename = obj.filename.partition('/')[-1]
if not new_filename:
continue
new_zip.writestr(new_filename, old_zip.read(obj.filename))
new_zip.close()
old_zip.close()
return new_path
def _run_yara_for_path(scanner_result, path, definition=None):
with statsd.timer('devhub.yara'):
if definition is None:
# Retrieve then concatenate all the active/valid Yara rules.
definition = '\n'.join(
ScannerRule.objects.filter(
scanner=YARA, is_active=True, definition__isnull=False
).values_list('definition', flat=True)
)
rules = yara.compile(source=definition)
zip_file = SafeZip(source=path)
for zip_info in zip_file.info_list:
if not zip_info.is_dir():
file_content = zip_file.read(zip_info).decode(
errors='ignore'
)
for match in rules.match(data=file_content):
# Add the filename to the meta dict.
meta = {**match.meta, 'filename': zip_info.filename}
scanner_result.add_yara_result(
rule=match.rule,
tags=match.tags,
meta=meta
)
zip_file.close()
def run_yara(results, upload_pk):
"""
Apply a set of Yara rules on a FileUpload and store the Yara results
(matches).
This task is intended to be run as part of the submission process only.
When a version is created from a FileUpload, the files are removed. In
addition, we usually delete old FileUpload entries after 180 days.
- `results` are the validation results passed in the validation chain. This
task is a validation task, which is why it must receive the validation
results as first argument.
- `upload_pk` is the FileUpload ID.
"""
log.info('Starting yara task for FileUpload %s.', upload_pk)
if not results['metadata']['is_webextension']:
log.info('Not running yara for FileUpload %s, it is not a '
'webextension.', upload_pk)
return results
upload = FileUpload.objects.get(pk=upload_pk)
try:
scanner_result = ScannerResult(upload=upload, scanner=YARA)
with statsd.timer('devhub.yara'):
rules = yara.compile(filepath=settings.YARA_RULES_FILEPATH)
zip_file = SafeZip(source=upload.path)
for zip_info in zip_file.info_list:
if not zip_info.is_dir():
file_content = zip_file.read(zip_info).decode(
errors='ignore'
)
for match in rules.match(data=file_content):
# Add the filename to the meta dict.
meta = {**match.meta, 'filename': zip_info.filename}
scanner_result.add_yara_result(
rule=match.rule,
tags=match.tags,
meta=meta
)
zip_file.close()
scanner_result.save()
if scanner_result.has_matches:
statsd.incr('devhub.yara.has_matches')
statsd.incr('devhub.yara.success')
log.info('Ending scanner "yara" task for FileUpload %s.', upload_pk)
except Exception:
statsd.incr('devhub.yara.failure')
# We log the exception but we do not raise to avoid perturbing the
# submission flow.
log.exception('Error in scanner "yara" task for FileUpload %s.',
upload_pk)
return results
def rezip_file(response, pk):
# An .xpi does not have a directory inside the zip, yet zips from github
# do, so we'll need to rezip the file before passing it through to the
# validator.
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
old_filename = '{}_github_webhook.zip'.format(pk)
old_path = os.path.join(loc, old_filename)
with storage.open(old_path, 'wb') as old:
old.write(response.content)
new_filename = '{}_github_webhook.xpi'.format(pk)
new_path = os.path.join(loc, new_filename)
old_zip = SafeZip(old_path)
if not old_zip.is_valid():
raise
with storage.open(new_path, 'w') as new:
new_zip = zipfile.ZipFile(new, 'w')
for obj in old_zip.filelist:
# Basically strip off the leading directory.
new_filename = obj.filename.partition('/')[-1]
if not new_filename:
continue
new_zip.writestr(new_filename, old_zip.read(obj.filename))
new_zip.close()
old_zip.close()
return new_path
def check_for_api_keys_in_file(results, upload_pk):
upload = FileUpload.objects.get(pk=upload_pk)
if upload.addon:
users = upload.addon.authors.all()
else:
users = [upload.user] if upload.user else []
keys = []
for user in users:
try:
key = APIKey.get_jwt_key(user_id=user.id)
keys.append(key)
except APIKey.DoesNotExist:
pass
try:
if len(keys) > 0:
zipfile = SafeZip(source=upload.path)
for zipinfo in zipfile.info_list:
if zipinfo.file_size >= 64:
file_ = zipfile.read(zipinfo)
for key in keys:
if key.secret in file_.decode(errors='ignore'):
log.info('Developer API key for user %s found in '
'submission.' % key.user)
if key.user == upload.user:
msg = gettext('Your developer API key was '
'found in the submitted file. '
'To protect your account, the '
'key will be revoked.')
else:
msg = gettext('The developer API key of a '
'coauthor was found in the '
'submitted file. To protect '
'your add-on, the key will be '
'revoked.')
annotations.insert_validation_message(
results,
type_='error',
message=msg,
msg_id='api_key_detected',
compatibility_type=None,
)
# Revoke after 2 minutes to allow the developer to
# fetch the validation results
revoke_api_key.apply_async(
kwargs={'key_id': key.id}, countdown=120)
zipfile.close()
except (ValidationError, BadZipFile, IOError):
pass
return results
def run_yara(upload_pk):
"""
Apply a set of Yara rules on a FileUpload and store the results.
This task is intended to be run as part of the submission process only.
When a version is created from a FileUpload, the files are removed. In
addition, we usually delete old FileUpload entries after 180 days.
"""
log.info('Starting yara task for FileUpload %s.', upload_pk)
upload = FileUpload.objects.get(pk=upload_pk)
if not upload.path.endswith('.xpi'):
log.info('Not running yara for FileUpload %s, it is not a xpi file.',
upload_pk)
return
try:
result = YaraResult()
result.upload = upload
with statsd.timer('devhub.yara'):
rules = yara.compile(filepath=settings.YARA_RULES_FILEPATH)
zip_file = SafeZip(source=upload.path)
for zip_info in zip_file.info_list:
if not zip_info.is_dir():
file_content = zip_file.read(zip_info).decode(
errors='ignore'
)
for match in rules.match(data=file_content):
# Add the filename to the meta dict.
meta = {**match.meta, 'filename': zip_info.filename}
result.add_match(
rule=match.rule,
tags=match.tags,
meta=meta
)
zip_file.close()
result.save()
statsd.incr('devhub.yara.success')
log.info('Ending yara task for FileUpload %s.', upload_pk)
except Exception:
statsd.incr('devhub.yara.failure')
# We log the exception but we do not raise to avoid perturbing the
# submission flow.
log.exception('Error in yara task for FileUpload %s.', upload_pk)
def check_for_api_keys_in_file(results, upload):
if upload.addon:
users = upload.addon.authors.all()
else:
users = [upload.user] if upload.user else []
keys = []
for user in users:
try:
key = APIKey.get_jwt_key(user_id=user.id)
keys.append(key)
except APIKey.DoesNotExist:
pass
if len(keys) > 0:
zipfile = SafeZip(source=upload.path)
zipfile.is_valid()
for zipinfo in zipfile.info_list:
if zipinfo.file_size >= 64:
file_ = zipfile.read(zipinfo)
for key in keys:
if key.secret in file_.decode(encoding='unicode-escape',
errors="ignore"):
log.info('Developer API key for user %s found in '
'submission.' % key.user)
if key.user == upload.user:
msg = ugettext('Your developer API key was found '
'in the submitted file. To protect '
'your account, the key will be '
'revoked.')
else:
msg = ugettext('The developer API key of a '
'coauthor was found in the '
'submitted file. To protect your '
'add-on, the key will be revoked.')
insert_validation_message(
results, type_='error',
message=msg, msg_id='api_key_detected',
compatibility_type=None)
# Revoke after 2 minutes to allow the developer to
# fetch the validation results
revoke_api_key.apply_async(
kwargs={'key_id': key.id}, countdown=120)
zipfile.close()
return results
def _run_yara_for_path(scanner_result, path, definition=None):
"""
Inner function to run yara on a particular path and add results to the
given scanner_result. The caller is responsible for saving the
scanner_result to the database.
Takes an optional definition to run a single arbitrary yara rule, otherwise
uses all active yara ScannerRules.
"""
with statsd.timer('devhub.yara'):
if definition is None:
# Retrieve then concatenate all the active/valid Yara rules.
definition = '\n'.join(
ScannerRule.objects.filter(
scanner=YARA, is_active=True, definition__isnull=False
).values_list('definition', flat=True)
)
# Initialize external variables so that compilation works, we'll
# override them later when matching.
externals = ScannerRule.get_yara_externals()
rules = yara.compile(source=definition, externals=externals)
zip_file = SafeZip(source=path)
for zip_info in zip_file.info_list:
if not zip_info.is_dir():
file_content = zip_file.read(zip_info).decode(
errors='ignore'
)
filename = zip_info.filename
# Fill externals variable for this file.
externals['is_json_file'] = filename.endswith('.json')
externals['is_manifest_file'] = filename == 'manifest.json'
externals['is_locale_file'] = (
filename.startswith('_locales/') and
filename.endswith('/messages.json')
)
for match in rules.match(
data=file_content, externals=externals):
# Also add the filename to the meta dict in results.
meta = {**match.meta, 'filename': filename}
scanner_result.add_yara_result(
rule=match.rule,
tags=match.tags,
meta=meta
)
zip_file.close()