def _run_yara_for_path(scanner_result, path, definition=None): with statsd.timer('devhub.yara'): if definition is None: # Retrieve then concatenate all the active/valid Yara rules. definition = '\n'.join( ScannerRule.objects.filter( scanner=YARA, is_active=True, definition__isnull=False ).values_list('definition', flat=True) ) rules = yara.compile(source=definition) zip_file = SafeZip(source=path) for zip_info in zip_file.info_list: if not zip_info.is_dir(): file_content = zip_file.read(zip_info).decode( errors='ignore' ) for match in rules.match(data=file_content): # Add the filename to the meta dict. meta = {**match.meta, 'filename': zip_info.filename} scanner_result.add_yara_result( rule=match.rule, tags=match.tags, meta=meta ) zip_file.close()
def rezip_file(response, pk): # An .xpi does not have a directory inside the zip, yet zips from github # do, so we'll need to rezip the file before passing it through to the # validator. loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex) old_filename = '{}_github_webhook.zip'.format(pk) old_path = os.path.join(loc, old_filename) with storage.open(old_path, 'wb') as old: old.write(response.content) new_filename = '{}_github_webhook.xpi'.format(pk) new_path = os.path.join(loc, new_filename) old_zip = SafeZip(old_path) if not old_zip.is_valid: raise with storage.open(new_path, 'w') as new: new_zip = zipfile.ZipFile(new, 'w') for obj in old_zip.filelist: # Basically strip off the leading directory. new_filename = obj.filename.partition('/')[-1] if not new_filename: continue new_zip.writestr(new_filename, old_zip.read(obj.filename)) new_zip.close() old_zip.close() return new_path
def rezip_file(response, pk): # An .xpi does not have a directory inside the zip, yet zips from github # do, so we'll need to rezip the file before passing it through to the # validator. loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex) old_filename = '{}_github_webhook.zip'.format(pk) old_path = os.path.join(loc, old_filename) with storage.open(old_path, 'wb') as old: old.write(response.content) new_filename = '{}_github_webhook.xpi'.format(pk) new_path = os.path.join(loc, new_filename) old_zip = SafeZip(old_path) if not old_zip.is_valid(): raise with storage.open(new_path, 'w') as new: new_zip = zipfile.ZipFile(new, 'w') for obj in old_zip.filelist: # Basically strip off the leading directory. new_filename = obj.filename.partition('/')[-1] if not new_filename: continue new_zip.writestr(new_filename, old_zip.read(obj.filename)) new_zip.close() old_zip.close() return new_path
def build_webext_dictionary_from_legacy(addon, destination): """Create a webext package of a legacy dictionary `addon`, and put it in `destination` path.""" from olympia.files.utils import SafeZip # Avoid circular import. old_path = addon.current_version.all_files[0].file_path old_zip = SafeZip(old_path) if not old_zip.is_valid(): raise ValidationError('Current dictionary xpi is not valid') if not addon.target_locale: raise ValidationError('Addon has no target_locale') dictionary_path = '' with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as new_zip: for obj in old_zip.filelist: splitted = obj.filename.split('/') # Ignore useless directories and files. if splitted[0] in ('META-INF', '__MACOSX', 'chrome', 'chrome.manifest', 'install.rdf'): continue # Also ignore javascript (regardless of where they are, not just at # the root), since dictionaries should not contain any code. if splitted[-1].endswith('.js'): continue # Store the path of the last .dic file we find. It can be inside a # directory. if (splitted[-1].endswith('.dic')): dictionary_path = obj.filename new_zip.writestr(obj.filename, old_zip.read(obj.filename)) # Now that all files we want from the old zip are copied, build and # add manifest.json. if not dictionary_path: # This should not happen... It likely means it's an invalid # dictionary to begin with, or one that has its .dic file in a # chrome/ directory for some reason. Abort! raise ValidationError('Current dictionary xpi has no .dic file') # Dumb version number increment. This will be invalid in some cases, # but some of the dictionaries we have currently already have wild # version numbers anyway. version_number = addon.current_version.version if version_number.endswith('.1-typefix'): version_number = version_number.replace('.1-typefix', '.2webext') else: version_number = '%s.1webext' % version_number manifest = { 'manifest_version': 2, 'name': unicode(addon.name), 'version': version_number, 'dictionaries': {addon.target_locale: dictionary_path}, } # Write manifest.json we just build. new_zip.writestr('manifest.json', json.dumps(manifest))
def run_yara(results, upload_pk): """ Apply a set of Yara rules on a FileUpload and store the Yara results (matches). This task is intended to be run as part of the submission process only. When a version is created from a FileUpload, the files are removed. In addition, we usually delete old FileUpload entries after 180 days. - `results` are the validation results passed in the validation chain. This task is a validation task, which is why it must receive the validation results as first argument. - `upload_pk` is the FileUpload ID. """ log.info('Starting yara task for FileUpload %s.', upload_pk) if not results['metadata']['is_webextension']: log.info('Not running yara for FileUpload %s, it is not a ' 'webextension.', upload_pk) return results upload = FileUpload.objects.get(pk=upload_pk) try: scanner_result = ScannerResult(upload=upload, scanner=YARA) with statsd.timer('devhub.yara'): rules = yara.compile(filepath=settings.YARA_RULES_FILEPATH) zip_file = SafeZip(source=upload.path) for zip_info in zip_file.info_list: if not zip_info.is_dir(): file_content = zip_file.read(zip_info).decode( errors='ignore' ) for match in rules.match(data=file_content): # Add the filename to the meta dict. meta = {**match.meta, 'filename': zip_info.filename} scanner_result.add_yara_result( rule=match.rule, tags=match.tags, meta=meta ) zip_file.close() scanner_result.save() if scanner_result.has_matches: statsd.incr('devhub.yara.has_matches') statsd.incr('devhub.yara.success') log.info('Ending scanner "yara" task for FileUpload %s.', upload_pk) except Exception: statsd.incr('devhub.yara.failure') # We log the exception but we do not raise to avoid perturbing the # submission flow. log.exception('Error in scanner "yara" task for FileUpload %s.', upload_pk) return results
def check_for_api_keys_in_file(results, upload_pk): upload = FileUpload.objects.get(pk=upload_pk) if upload.addon: users = upload.addon.authors.all() else: users = [upload.user] if upload.user else [] keys = [] for user in users: try: key = APIKey.get_jwt_key(user_id=user.id) keys.append(key) except APIKey.DoesNotExist: pass try: if len(keys) > 0: zipfile = SafeZip(source=upload.path) for zipinfo in zipfile.info_list: if zipinfo.file_size >= 64: file_ = zipfile.read(zipinfo) for key in keys: if key.secret in file_.decode(errors='ignore'): log.info('Developer API key for user %s found in ' 'submission.' % key.user) if key.user == upload.user: msg = gettext('Your developer API key was ' 'found in the submitted file. ' 'To protect your account, the ' 'key will be revoked.') else: msg = gettext('The developer API key of a ' 'coauthor was found in the ' 'submitted file. To protect ' 'your add-on, the key will be ' 'revoked.') annotations.insert_validation_message( results, type_='error', message=msg, msg_id='api_key_detected', compatibility_type=None, ) # Revoke after 2 minutes to allow the developer to # fetch the validation results revoke_api_key.apply_async( kwargs={'key_id': key.id}, countdown=120) zipfile.close() except (ValidationError, BadZipFile, IOError): pass return results
def run_yara(upload_pk): """ Apply a set of Yara rules on a FileUpload and store the results. This task is intended to be run as part of the submission process only. When a version is created from a FileUpload, the files are removed. In addition, we usually delete old FileUpload entries after 180 days. """ log.info('Starting yara task for FileUpload %s.', upload_pk) upload = FileUpload.objects.get(pk=upload_pk) if not upload.path.endswith('.xpi'): log.info('Not running yara for FileUpload %s, it is not a xpi file.', upload_pk) return try: result = YaraResult() result.upload = upload with statsd.timer('devhub.yara'): rules = yara.compile(filepath=settings.YARA_RULES_FILEPATH) zip_file = SafeZip(source=upload.path) for zip_info in zip_file.info_list: if not zip_info.is_dir(): file_content = zip_file.read(zip_info).decode( errors='ignore' ) for match in rules.match(data=file_content): # Add the filename to the meta dict. meta = {**match.meta, 'filename': zip_info.filename} result.add_match( rule=match.rule, tags=match.tags, meta=meta ) zip_file.close() result.save() statsd.incr('devhub.yara.success') log.info('Ending yara task for FileUpload %s.', upload_pk) except Exception: statsd.incr('devhub.yara.failure') # We log the exception but we do not raise to avoid perturbing the # submission flow. log.exception('Error in yara task for FileUpload %s.', upload_pk)
def get_localepicker(self): """ For a file that is part of a language pack, extract the chrome/localepicker.properties file and return as a string. """ start = time.time() zip = SafeZip(self.file_path, raise_on_failure=False) if not zip.is_valid(): return '' try: manifest = zip.read('chrome.manifest') except KeyError, e: log.info('No file named: chrome.manifest in file: %s' % self.pk) return ''
def get_localepicker(self): """ For a file that is part of a language pack, extract the chrome/localepicker.properties file and return as a string. """ start = time.time() zip = SafeZip(self.file_path, raise_on_failure=False) if not zip.is_valid(): return '' try: manifest = zip.read('chrome.manifest') except KeyError, e: log.info('No file named: chrome.manifest in file: %s' % self.pk) return ''
def check_for_api_keys_in_file(results, upload): if upload.addon: users = upload.addon.authors.all() else: users = [upload.user] if upload.user else [] keys = [] for user in users: try: key = APIKey.get_jwt_key(user_id=user.id) keys.append(key) except APIKey.DoesNotExist: pass if len(keys) > 0: zipfile = SafeZip(source=upload.path) zipfile.is_valid() for zipinfo in zipfile.info_list: if zipinfo.file_size >= 64: file_ = zipfile.read(zipinfo) for key in keys: if key.secret in file_.decode(encoding='unicode-escape', errors="ignore"): log.info('Developer API key for user %s found in ' 'submission.' % key.user) if key.user == upload.user: msg = ugettext('Your developer API key was found ' 'in the submitted file. To protect ' 'your account, the key will be ' 'revoked.') else: msg = ugettext('The developer API key of a ' 'coauthor was found in the ' 'submitted file. To protect your ' 'add-on, the key will be revoked.') insert_validation_message( results, type_='error', message=msg, msg_id='api_key_detected', compatibility_type=None) # Revoke after 2 minutes to allow the developer to # fetch the validation results revoke_api_key.apply_async( kwargs={'key_id': key.id}, countdown=120) zipfile.close() return results
def _run_yara_for_path(scanner_result, path, definition=None): """ Inner function to run yara on a particular path and add results to the given scanner_result. The caller is responsible for saving the scanner_result to the database. Takes an optional definition to run a single arbitrary yara rule, otherwise uses all active yara ScannerRules. """ with statsd.timer('devhub.yara'): if definition is None: # Retrieve then concatenate all the active/valid Yara rules. definition = '\n'.join( ScannerRule.objects.filter( scanner=YARA, is_active=True, definition__isnull=False ).values_list('definition', flat=True) ) # Initialize external variables so that compilation works, we'll # override them later when matching. externals = ScannerRule.get_yara_externals() rules = yara.compile(source=definition, externals=externals) zip_file = SafeZip(source=path) for zip_info in zip_file.info_list: if not zip_info.is_dir(): file_content = zip_file.read(zip_info).decode( errors='ignore' ) filename = zip_info.filename # Fill externals variable for this file. externals['is_json_file'] = filename.endswith('.json') externals['is_manifest_file'] = filename == 'manifest.json' externals['is_locale_file'] = ( filename.startswith('_locales/') and filename.endswith('/messages.json') ) for match in rules.match( data=file_content, externals=externals): # Also add the filename to the meta dict in results. meta = {**match.meta, 'filename': filename} scanner_result.add_yara_result( rule=match.rule, tags=match.tags, meta=meta ) zip_file.close()
def get_localepicker(self): """ For a file that is part of a language pack, extract the chrome/localepicker.properties file and return as a string. """ start = time.time() zip = SafeZip(self.file_path, validate=False) try: is_valid = zip.is_valid() except (zipfile.BadZipfile, IOError): is_valid = False if not is_valid: return '' try: manifest = zip.read('chrome.manifest') except KeyError as e: log.info('No file named: chrome.manifest in file: %s' % self.pk) return '' res = self._get_localepicker.search(manifest) if not res: log.error('Locale browser not in chrome.manifest: %s' % self.pk) return '' try: p = res.groups()[1] if 'localepicker.properties' not in p: p = os.path.join(p, 'localepicker.properties') res = zip.extract_from_manifest(p) except (zipfile.BadZipfile, IOError) as e: log.error('Error unzipping: %s, %s in file: %s' % (p, e, self.pk)) return '' except (ValueError, KeyError) as e: log.error('No file named: %s in file: %s' % (e, self.pk)) return '' end = time.time() - start log.info('Extracted localepicker file: %s in %.2fs' % (self.pk, end)) statsd.timing('files.extract.localepicker', (end * 1000)) return res
def get_localepicker(self): """ For a file that is part of a language pack, extract the chrome/localepicker.properties file and return as a string. """ start = time.time() zip = SafeZip(self.file_path, validate=False) try: is_valid = zip.is_valid() except (zipfile.BadZipfile, IOError): is_valid = False if not is_valid: return '' try: manifest = zip.read('chrome.manifest') except KeyError as e: log.info('No file named: chrome.manifest in file: %s' % self.pk) return '' res = self._get_localepicker.search(manifest) if not res: log.error('Locale browser not in chrome.manifest: %s' % self.pk) return '' try: p = res.groups()[1] if 'localepicker.properties' not in p: p = os.path.join(p, 'localepicker.properties') res = zip.extract_from_manifest(p) except (zipfile.BadZipfile, IOError) as e: log.error('Error unzipping: %s, %s in file: %s' % (p, e, self.pk)) return '' except (ValueError, KeyError) as e: log.error('No file named: %s in file: %s' % (e, self.pk)) return '' end = time.time() - start log.info('Extracted localepicker file: %s in %.2fs' % (self.pk, end)) statsd.timing('files.extract.localepicker', (end * 1000)) return res
def build_webext_dictionary_from_legacy(addon, destination): """Create a webext package of a legacy dictionary `addon`, and put it in `destination` path.""" from olympia.files.utils import SafeZip # Avoid circular import. old_path = addon.current_version.all_files[0].file_path old_zip = SafeZip(old_path) if not old_zip.is_valid: raise ValidationError('Current dictionary xpi is not valid') dictionary_path = '' with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as new_zip: for obj in old_zip.filelist: splitted = obj.filename.split('/') # Ignore useless directories and files. if splitted[0] in ('META-INF', '__MACOSX', 'chrome', 'chrome.manifest', 'install.rdf'): continue # Also ignore javascript (regardless of where they are, not just at # the root), since dictionaries should not contain any code. if splitted[-1].endswith('.js'): continue # Store the path of the last .dic file we find. It can be inside a # directory. if (splitted[-1].endswith('.dic')): dictionary_path = obj.filename new_zip.writestr(obj.filename, old_zip.read(obj.filename)) # Now that all files we want from the old zip are copied, build and # add manifest.json. if not dictionary_path: # This should not happen... It likely means it's an invalid # dictionary to begin with, or one that has its .dic file in a # chrome/ directory for some reason. Abort! raise ValidationError('Current dictionary xpi has no .dic file') if addon.target_locale: target_language = addon.target_locale else: # Guess target_locale since we don't have one already. Note that # for extra confusion, target_locale is a language, not a locale. target_language = to_language(os.path.splitext( os.path.basename(dictionary_path))[0]) if target_language not in settings.AMO_LANGUAGES: # We couldn't find that language in the list we support. Let's # try with just the prefix. target_language = target_language.split('-')[0] if target_language not in settings.AMO_LANGUAGES: # We tried our best. raise ValidationError(u'Addon has no target_locale and we' u' could not guess one from the xpi') # Dumb version number increment. This will be invalid in some cases, # but some of the dictionaries we have currently already have wild # version numbers anyway. version_number = addon.current_version.version if version_number.endswith('.1-typefix'): version_number = version_number.replace('.1-typefix', '.2webext') else: version_number = '%s.1webext' % version_number manifest = { 'manifest_version': 2, 'name': unicode(addon.name), 'applications': { 'gecko': { 'id': addon.guid, }, }, 'version': version_number, 'dictionaries': {target_language: dictionary_path}, } # Write manifest.json we just build. new_zip.writestr('manifest.json', json.dumps(manifest))
def test_read(self): zip_file = SafeZip(self.xpi_path('langpack-localepicker')) assert zip_file.is_valid assert b'locale browser de' in zip_file.read('chrome.manifest')
def test_read(self): zip_file = SafeZip(self.xpi_path('langpack-localepicker')) assert zip_file.is_valid() assert 'locale browser de' in zip_file.read('chrome.manifest')
def build_webext_dictionary_from_legacy(addon, destination): """Create a webext package of a legacy dictionary `addon`, and put it in `destination` path.""" from olympia.files.utils import SafeZip # Avoid circular import. old_path = addon.current_version.all_files[0].file_path old_zip = SafeZip(old_path) if not old_zip.is_valid: raise ValidationError('Current dictionary xpi is not valid') dictionary_path = '' with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as new_zip: for obj in old_zip.filelist: splitted = obj.filename.split('/') # Ignore useless directories and files. if splitted[0] in ('META-INF', '__MACOSX', 'chrome', 'chrome.manifest', 'install.rdf'): continue # Also ignore javascript (regardless of where they are, not just at # the root), since dictionaries should not contain any code. if splitted[-1].endswith('.js'): continue # Store the path of the last .dic file we find. It can be inside a # directory. if (splitted[-1].endswith('.dic')): dictionary_path = obj.filename new_zip.writestr(obj.filename, old_zip.read(obj.filename)) # Now that all files we want from the old zip are copied, build and # add manifest.json. if not dictionary_path: # This should not happen... It likely means it's an invalid # dictionary to begin with, or one that has its .dic file in a # chrome/ directory for some reason. Abort! raise ValidationError('Current dictionary xpi has no .dic file') if addon.target_locale: target_language = addon.target_locale else: # Guess target_locale since we don't have one already. Note that # for extra confusion, target_locale is a language, not a locale. target_language = to_language( os.path.splitext(os.path.basename(dictionary_path))[0]) if target_language not in settings.AMO_LANGUAGES: # We couldn't find that language in the list we support. Let's # try with just the prefix. target_language = target_language.split('-')[0] if target_language not in settings.AMO_LANGUAGES: # We tried our best. raise ValidationError(u'Addon has no target_locale and we' u' could not guess one from the xpi') # Dumb version number increment. This will be invalid in some cases, # but some of the dictionaries we have currently already have wild # version numbers anyway. version_number = addon.current_version.version if version_number.endswith('.1-typefix'): version_number = version_number.replace('.1-typefix', '.2webext') else: version_number = '%s.1webext' % version_number manifest = { 'manifest_version': 2, 'name': unicode(addon.name), 'browser_specific_settings': { 'gecko': { 'id': addon.guid, }, }, 'version': version_number, 'dictionaries': { target_language: dictionary_path }, } # Write manifest.json we just build. new_zip.writestr('manifest.json', json.dumps(manifest))