def _run_yara_for_path(scanner_result, path, definition=None): with statsd.timer('devhub.yara'): if definition is None: # Retrieve then concatenate all the active/valid Yara rules. definition = '\n'.join( ScannerRule.objects.filter( scanner=YARA, is_active=True, definition__isnull=False ).values_list('definition', flat=True) ) rules = yara.compile(source=definition) zip_file = SafeZip(source=path) for zip_info in zip_file.info_list: if not zip_info.is_dir(): file_content = zip_file.read(zip_info).decode( errors='ignore' ) for match in rules.match(data=file_content): # Add the filename to the meta dict. meta = {**match.meta, 'filename': zip_info.filename} scanner_result.add_yara_result( rule=match.rule, tags=match.tags, meta=meta ) zip_file.close()
def run_yara(results, upload_pk): """ Apply a set of Yara rules on a FileUpload and store the Yara results (matches). This task is intended to be run as part of the submission process only. When a version is created from a FileUpload, the files are removed. In addition, we usually delete old FileUpload entries after 180 days. - `results` are the validation results passed in the validation chain. This task is a validation task, which is why it must receive the validation results as first argument. - `upload_pk` is the FileUpload ID. """ log.info('Starting yara task for FileUpload %s.', upload_pk) if not results['metadata']['is_webextension']: log.info('Not running yara for FileUpload %s, it is not a ' 'webextension.', upload_pk) return results upload = FileUpload.objects.get(pk=upload_pk) try: scanner_result = ScannerResult(upload=upload, scanner=YARA) with statsd.timer('devhub.yara'): rules = yara.compile(filepath=settings.YARA_RULES_FILEPATH) zip_file = SafeZip(source=upload.path) for zip_info in zip_file.info_list: if not zip_info.is_dir(): file_content = zip_file.read(zip_info).decode( errors='ignore' ) for match in rules.match(data=file_content): # Add the filename to the meta dict. meta = {**match.meta, 'filename': zip_info.filename} scanner_result.add_yara_result( rule=match.rule, tags=match.tags, meta=meta ) zip_file.close() scanner_result.save() if scanner_result.has_matches: statsd.incr('devhub.yara.has_matches') statsd.incr('devhub.yara.success') log.info('Ending scanner "yara" task for FileUpload %s.', upload_pk) except Exception: statsd.incr('devhub.yara.failure') # We log the exception but we do not raise to avoid perturbing the # submission flow. log.exception('Error in scanner "yara" task for FileUpload %s.', upload_pk) return results
def build_webext_dictionary_from_legacy(addon, destination): """Create a webext package of a legacy dictionary `addon`, and put it in `destination` path.""" from olympia.files.utils import SafeZip # Avoid circular import. old_path = addon.current_version.all_files[0].file_path old_zip = SafeZip(old_path) if not old_zip.is_valid(): raise ValidationError('Current dictionary xpi is not valid') if not addon.target_locale: raise ValidationError('Addon has no target_locale') dictionary_path = '' with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as new_zip: for obj in old_zip.filelist: splitted = obj.filename.split('/') # Ignore useless directories and files. if splitted[0] in ('META-INF', '__MACOSX', 'chrome', 'chrome.manifest', 'install.rdf'): continue # Also ignore javascript (regardless of where they are, not just at # the root), since dictionaries should not contain any code. if splitted[-1].endswith('.js'): continue # Store the path of the last .dic file we find. It can be inside a # directory. if (splitted[-1].endswith('.dic')): dictionary_path = obj.filename new_zip.writestr(obj.filename, old_zip.read(obj.filename)) # Now that all files we want from the old zip are copied, build and # add manifest.json. if not dictionary_path: # This should not happen... It likely means it's an invalid # dictionary to begin with, or one that has its .dic file in a # chrome/ directory for some reason. Abort! raise ValidationError('Current dictionary xpi has no .dic file') # Dumb version number increment. This will be invalid in some cases, # but some of the dictionaries we have currently already have wild # version numbers anyway. version_number = addon.current_version.version if version_number.endswith('.1-typefix'): version_number = version_number.replace('.1-typefix', '.2webext') else: version_number = '%s.1webext' % version_number manifest = { 'manifest_version': 2, 'name': unicode(addon.name), 'version': version_number, 'dictionaries': {addon.target_locale: dictionary_path}, } # Write manifest.json we just build. new_zip.writestr('manifest.json', json.dumps(manifest))
def rezip_file(response, pk): # An .xpi does not have a directory inside the zip, yet zips from github # do, so we'll need to rezip the file before passing it through to the # validator. loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex) old_filename = '{}_github_webhook.zip'.format(pk) old_path = os.path.join(loc, old_filename) with storage.open(old_path, 'wb') as old: old.write(response.content) new_filename = '{}_github_webhook.xpi'.format(pk) new_path = os.path.join(loc, new_filename) old_zip = SafeZip(old_path) if not old_zip.is_valid: raise with storage.open(new_path, 'w') as new: new_zip = zipfile.ZipFile(new, 'w') for obj in old_zip.filelist: # Basically strip off the leading directory. new_filename = obj.filename.partition('/')[-1] if not new_filename: continue new_zip.writestr(new_filename, old_zip.read(obj.filename)) new_zip.close() old_zip.close() return new_path
def check_for_api_keys_in_file(results, upload_pk): upload = FileUpload.objects.get(pk=upload_pk) if upload.addon: users = upload.addon.authors.all() else: users = [upload.user] if upload.user else [] keys = [] for user in users: try: key = APIKey.get_jwt_key(user_id=user.id) keys.append(key) except APIKey.DoesNotExist: pass try: if len(keys) > 0: zipfile = SafeZip(source=upload.path) for zipinfo in zipfile.info_list: if zipinfo.file_size >= 64: file_ = zipfile.read(zipinfo) for key in keys: if key.secret in file_.decode(errors='ignore'): log.info('Developer API key for user %s found in ' 'submission.' % key.user) if key.user == upload.user: msg = gettext('Your developer API key was ' 'found in the submitted file. ' 'To protect your account, the ' 'key will be revoked.') else: msg = gettext('The developer API key of a ' 'coauthor was found in the ' 'submitted file. To protect ' 'your add-on, the key will be ' 'revoked.') annotations.insert_validation_message( results, type_='error', message=msg, msg_id='api_key_detected', compatibility_type=None, ) # Revoke after 2 minutes to allow the developer to # fetch the validation results revoke_api_key.apply_async( kwargs={'key_id': key.id}, countdown=120) zipfile.close() except (ValidationError, BadZipFile, IOError): pass return results
def run_yara(upload_pk): """ Apply a set of Yara rules on a FileUpload and store the results. This task is intended to be run as part of the submission process only. When a version is created from a FileUpload, the files are removed. In addition, we usually delete old FileUpload entries after 180 days. """ log.info('Starting yara task for FileUpload %s.', upload_pk) upload = FileUpload.objects.get(pk=upload_pk) if not upload.path.endswith('.xpi'): log.info('Not running yara for FileUpload %s, it is not a xpi file.', upload_pk) return try: result = YaraResult() result.upload = upload with statsd.timer('devhub.yara'): rules = yara.compile(filepath=settings.YARA_RULES_FILEPATH) zip_file = SafeZip(source=upload.path) for zip_info in zip_file.info_list: if not zip_info.is_dir(): file_content = zip_file.read(zip_info).decode( errors='ignore' ) for match in rules.match(data=file_content): # Add the filename to the meta dict. meta = {**match.meta, 'filename': zip_info.filename} result.add_match( rule=match.rule, tags=match.tags, meta=meta ) zip_file.close() result.save() statsd.incr('devhub.yara.success') log.info('Ending yara task for FileUpload %s.', upload_pk) except Exception: statsd.incr('devhub.yara.failure') # We log the exception but we do not raise to avoid perturbing the # submission flow. log.exception('Error in yara task for FileUpload %s.', upload_pk)
def get_localepicker(self): """ For a file that is part of a language pack, extract the chrome/localepicker.properties file and return as a string. """ start = time.time() zip = SafeZip(self.file_path, raise_on_failure=False) if not zip.is_valid(): return '' try: manifest = zip.read('chrome.manifest') except KeyError, e: log.info('No file named: chrome.manifest in file: %s' % self.pk) return ''
def annotate_legacy_addon_restrictions(path, results, parsed_data, error=True): """ Annotate validation results to restrict uploads of legacy (non-webextension) add-ons. """ # We can be broad here. Search plugins are not validated through this # path and as of right now (Jan 2019) there aren't any legacy type # add-ons allowed to submit anymore. msg = ugettext(u'Legacy extensions are no longer supported in Firefox.') description = ugettext( u'Add-ons for Thunderbird and SeaMonkey are now listed and ' u'maintained on addons.thunderbird.net. You can use the same ' u'account to update your add-ons on the new site.') # `parsed_data` only contains the most minimal amount of data because # we aren't in the right context. Let's explicitly fetch the add-ons # apps so that we can adjust the messaging to the user. xpi = get_file(path) extractor = RDFExtractor(SafeZip(xpi)) targets_thunderbird_or_seamonkey = False thunderbird_or_seamonkey = {amo.THUNDERBIRD.guid, amo.SEAMONKEY.guid} for ctx in extractor.rdf.objects(None, extractor.uri('targetApplication')): if extractor.find('id', ctx) in thunderbird_or_seamonkey: targets_thunderbird_or_seamonkey = True description = description if targets_thunderbird_or_seamonkey else [] insert_validation_message(results, type_='error' if error else 'warning', message=msg, description=description, msg_id='legacy_addons_unsupported')
def clean_source(self): source = self.cleaned_data.get('source') if source: try: if source.name.endswith('.zip'): zip_file = SafeZip(source) # testzip() returns None if there are no broken CRCs. if zip_file.zip_file.testzip() is not None: raise zipfile.BadZipfile() elif source.name.endswith(('.tar.gz', '.tar.bz2')): # For tar files we need to do a little more work. # Fortunately tarfile.open() already handles compression # formats for us automatically. with tarfile.open(fileobj=source) as archive: archive_members = archive.getmembers() for member in archive_members: archive_member_validator(archive, member) else: valid_extensions_string = u'(%s)' % u', '.join( VALID_SOURCE_EXTENSIONS) raise forms.ValidationError( ugettext( 'Unsupported file type, please upload an archive ' 'file {extensions}.'.format( extensions=valid_extensions_string))) except (zipfile.BadZipfile, tarfile.ReadError, IOError): raise forms.ValidationError( ugettext('Invalid or broken archive.')) return source
def test_invalid_zip_encoding(self): with pytest.raises(forms.ValidationError) as exc: SafeZip(self.xpi_path('invalid-cp437-encoding.xpi')) assert isinstance(exc.value, forms.ValidationError) assert exc.value.message.endswith( 'Please make sure all filenames are utf-8 or latin1 encoded.')
def to_internal_value(self, data): data = super().to_internal_value(data) # Ensure the file type is one we support. if not data.name.endswith(VALID_SOURCE_EXTENSIONS): error_msg = ( 'Unsupported file type, please upload an archive file ({extensions}).' ) raise exceptions.ValidationError( error_msg.format( extensions=(', '.join(VALID_SOURCE_EXTENSIONS)))) # Check inside to see if the file extension matches the content. try: _, ext = os.path.splitext(data.name) if ext == '.zip': # testzip() returns None if there are no broken CRCs. if SafeZip(data).zip_file.testzip() is not None: raise zipfile.BadZipFile() else: # For tar files we need to do a little more work. mode = 'r:bz2' if ext == '.bz2' else 'r:gz' with tarfile.open(mode=mode, fileobj=data) as archive: for member in archive.getmembers(): archive_member_validator(archive, member) except (zipfile.BadZipFile, tarfile.ReadError, OSError, EOFError): raise exceptions.ValidationError('Invalid or broken archive.') return data
def check_for_api_keys_in_file(results, upload): if upload.addon: users = upload.addon.authors.all() else: users = [upload.user] if upload.user else [] keys = [] for user in users: try: key = APIKey.get_jwt_key(user_id=user.id) keys.append(key) except APIKey.DoesNotExist: pass if len(keys) > 0: zipfile = SafeZip(source=upload.path) for zipinfo in zipfile.info_list: if zipinfo.file_size >= 64: file_ = zipfile.read(zipinfo) for key in keys: if key.secret in file_.decode(encoding='unicode-escape', errors="ignore"): log.info('Developer API key for user %s found in ' 'submission.' % key.user) if key.user == upload.user: msg = ugettext('Your developer API key was found ' 'in the submitted file. To protect ' 'your account, the key will be ' 'revoked.') else: msg = ugettext('The developer API key of a ' 'coauthor was found in the ' 'submitted file. To protect your ' 'add-on, the key will be revoked.') insert_validation_message( results, type_='error', message=msg, msg_id='api_key_detected', compatibility_type=None) # Revoke after 2 minutes to allow the developer to # fetch the validation results revoke_api_key.apply_async( kwargs={'key_id': key.id}, countdown=120) zipfile.close() return results
def _run_yara_for_path(scanner_result, path, definition=None): """ Inner function to run yara on a particular path and add results to the given scanner_result. The caller is responsible for saving the scanner_result to the database. Takes an optional definition to run a single arbitrary yara rule, otherwise uses all active yara ScannerRules. """ with statsd.timer('devhub.yara'): if definition is None: # Retrieve then concatenate all the active/valid Yara rules. definition = '\n'.join( ScannerRule.objects.filter( scanner=YARA, is_active=True, definition__isnull=False ).values_list('definition', flat=True) ) # Initialize external variables so that compilation works, we'll # override them later when matching. externals = ScannerRule.get_yara_externals() rules = yara.compile(source=definition, externals=externals) zip_file = SafeZip(source=path) for zip_info in zip_file.info_list: if not zip_info.is_dir(): file_content = zip_file.read(zip_info).decode( errors='ignore' ) filename = zip_info.filename # Fill externals variable for this file. externals['is_json_file'] = filename.endswith('.json') externals['is_manifest_file'] = filename == 'manifest.json' externals['is_locale_file'] = ( filename.startswith('_locales/') and filename.endswith('/messages.json') ) for match in rules.match( data=file_content, externals=externals): # Also add the filename to the meta dict in results. meta = {**match.meta, 'filename': filename} scanner_result.add_yara_result( rule=match.rule, tags=match.tags, meta=meta ) zip_file.close()
def get_localepicker(self): """ For a file that is part of a language pack, extract the chrome/localepicker.properties file and return as a string. """ start = time.time() zip = SafeZip(self.file_path, validate=False) try: is_valid = zip.is_valid() except (zipfile.BadZipfile, IOError): is_valid = False if not is_valid: return '' try: manifest = zip.read('chrome.manifest') except KeyError as e: log.info('No file named: chrome.manifest in file: %s' % self.pk) return '' res = self._get_localepicker.search(manifest) if not res: log.error('Locale browser not in chrome.manifest: %s' % self.pk) return '' try: p = res.groups()[1] if 'localepicker.properties' not in p: p = os.path.join(p, 'localepicker.properties') res = zip.extract_from_manifest(p) except (zipfile.BadZipfile, IOError) as e: log.error('Error unzipping: %s, %s in file: %s' % (p, e, self.pk)) return '' except (ValueError, KeyError) as e: log.error('No file named: %s in file: %s' % (e, self.pk)) return '' end = time.time() - start log.info('Extracted localepicker file: %s in %.2fs' % (self.pk, end)) statsd.timing('files.extract.localepicker', (end * 1000)) return res
def extract_strict_compatibility_value_for_addon(addon): strict_compatibility = None # We don't know yet. try: # We take a shortcut here and only look at the first file we # find... # Note that we can't use parse_addon() wrapper because it no longer # exposes the real value of `strictCompatibility`... path = addon.current_version.all_files[0].file_path zip_file = SafeZip(get_file(path)) parser = RDFExtractor(zip_file) strict_compatibility = parser.find('strictCompatibility') == 'true' except Exception as exp: # A number of things can go wrong: missing file, path somehow not # existing, etc. In any case, that means the add-on is in a weird # state and should be ignored (this is a one off task). log.exception(u'bump_appver_for_legacy_addons: ignoring addon %d, ' u'received %s when extracting.', addon.pk, unicode(exp)) return strict_compatibility
def test_is_broken(self): zip_file = SafeZip(self.xpi_path('signed')) zip_file.info_list[2].filename = 'META-INF/foo.sf' assert not zip_file.is_signed()
def test_is_secure(self): zip_file = SafeZip(self.xpi_path('signed')) assert zip_file.is_signed()
def test_not_secure(self): zip_file = SafeZip(self.xpi_path('extension')) assert not zip_file.is_signed()
def test_unzip_not_fatal(self): zip_file = SafeZip(self.xpi_path('search.xml'), raise_on_failure=False) assert not zip_file.is_valid()
def test_read(self): zip_file = SafeZip(self.xpi_path('langpack-localepicker')) assert zip_file.is_valid assert b'locale browser de' in zip_file.read('chrome.manifest')
def extract_and_commit_from_file_obj(cls, file_obj, channel, author=None): """Extract all files from `file_obj` and comit them. This is doing the following: * Create a temporary `git worktree`_ * Remove all files in that worktree * Extract the zip behind `file_obj` into the worktree * Commit all files Kinda like doing:: $ workdir_name=$(uuid) $ mkdir /tmp/$workdir_name $ git worktree add /tmp/$workdir_name Preparing worktree (new branch 'af4172e4-d8c7…') HEAD is now at 8c5223e Initial commit $ git worktree list /tmp/addon-repository 8c5223e [master] /tmp/af4172e4-d8c7-4486-a5f2-316458da91ff 8c5223e [af4172e4-d8c7…] $ unzip dingrafowl-falcockalo-lockapionk.zip -d /tmp/$workdir_name Archive: dingrafowl-falcockalo-lockapionk.zip extracting: /tmp/af4172e4-d8c7…/manifest.json $ pushd /tmp/$workdir_name /tmp/af4172e4-d8c7-4486-a5f2-316458da91ff /tmp/addon-repository $ git status On branch af4172e4-d8c7-4486-a5f2-316458da91ff Untracked files: (use "git add <file>..." to include in what will be committed) manifest.json $ git add * $ git commit -a -m "Creating new version" [af4172e4-d8c7-4486-a5f2-316458da91ff c4285f8] Creating new version … $ cd addon-repository $ git checkout -b listed Switched to a new branch 'listed' # We don't technically do a full cherry-pick but it's close enough # and does almost what we do. We are technically commiting # directly on top of the branch as if we checked out the branch # in the worktree (via -b) but pygit doesn't properly support that # so we "simply" set the parents correctly. $ git cherry-pick c4285f8 [listed a4d0f63] Creating new version… This ignores the fact that there may be a race-condition of two versions being created at the same time. Since all relevant file based work is done in a temporary worktree there won't be any conflicts and usually the last upload simply wins the race and we're setting the HEAD of the branch (listed/unlisted) to that specific commit. .. _`git worktree`: https://git-scm.com/docs/git-worktree """ # Make sure we're always using the en-US locale by default translation.activate('en-US') addon = file_obj.version.addon repo = cls(addon.id) branch = repo.find_or_create_branch(BRANCHES[channel]) # Create a temporary worktree that we can use to unpack the zip # without disturbing the current git workdir since it creates a new # temporary directory where we extract to. with TemporaryWorktree(repo.git_repository) as worktree: # Now extract the zip to the workdir zip_file = SafeZip(file_obj.current_file_path, force_fsync=True) zip_file.extract_to_dest(worktree.path) # Stage changes, `TemporaryWorktree` always cleans the whole # directory so we can simply add all changes and have the correct # state. # Add all changes to the index (git add ...) worktree.repo.index.add_all() worktree.repo.index.write() tree = worktree.repo.index.write_tree() # Now create an commit directly on top of the respective branch message = ( 'Create new version {version} ({version_id}) for ' '{addon} from {file_obj}'.format( version=repr(file_obj.version), version_id=file_obj.version.id, addon=repr(addon), file_obj=repr(file_obj))) oid = worktree.repo.create_commit( None, # author, using the actual uploading user repo.get_author(author), # committer, using addons-robot because that's the user # actually doing the commit. repo.get_author(), # commiter, using addons-robot message, tree, # Set the current branch HEAD as the parent of this commit # so that it'll go straight into the branches commit log [branch.target] ) # Fetch the commit object commit = worktree.repo.get(oid) # And set the commit we just created as HEAD of the relevant # branch, and updates the reflog. This does not require any # merges. branch.set_target(commit.hex) # Set the latest git hash on the related version. file_obj.version.update(git_hash=commit.hex) return repo
def test_read(self): zip_file = SafeZip(self.xpi_path('langpack-localepicker')) assert zip_file.is_valid() assert 'locale browser de' in zip_file.read('chrome.manifest')
def test_not_secure(self): zip_file = SafeZip(self.xpi_path('extension')) zip_file.is_valid() assert not zip_file.is_signed()
def test_is_secure(self): zip_file = SafeZip(self.xpi_path('signed')) zip_file.is_valid() assert zip_file.is_signed()
def test_is_broken(self): zip_file = SafeZip(self.xpi_path('signed')) zip_file.is_valid() zip_file.info_list[2].filename = 'META-INF/foo.sf' assert not zip_file.is_signed()
def build_webext_dictionary_from_legacy(addon, destination): """Create a webext package of a legacy dictionary `addon`, and put it in `destination` path.""" from olympia.files.utils import SafeZip # Avoid circular import. old_path = addon.current_version.all_files[0].file_path old_zip = SafeZip(old_path) if not old_zip.is_valid: raise ValidationError('Current dictionary xpi is not valid') dictionary_path = '' with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as new_zip: for obj in old_zip.filelist: splitted = obj.filename.split('/') # Ignore useless directories and files. if splitted[0] in ('META-INF', '__MACOSX', 'chrome', 'chrome.manifest', 'install.rdf'): continue # Also ignore javascript (regardless of where they are, not just at # the root), since dictionaries should not contain any code. if splitted[-1].endswith('.js'): continue # Store the path of the last .dic file we find. It can be inside a # directory. if (splitted[-1].endswith('.dic')): dictionary_path = obj.filename new_zip.writestr(obj.filename, old_zip.read(obj.filename)) # Now that all files we want from the old zip are copied, build and # add manifest.json. if not dictionary_path: # This should not happen... It likely means it's an invalid # dictionary to begin with, or one that has its .dic file in a # chrome/ directory for some reason. Abort! raise ValidationError('Current dictionary xpi has no .dic file') if addon.target_locale: target_language = addon.target_locale else: # Guess target_locale since we don't have one already. Note that # for extra confusion, target_locale is a language, not a locale. target_language = to_language( os.path.splitext(os.path.basename(dictionary_path))[0]) if target_language not in settings.AMO_LANGUAGES: # We couldn't find that language in the list we support. Let's # try with just the prefix. target_language = target_language.split('-')[0] if target_language not in settings.AMO_LANGUAGES: # We tried our best. raise ValidationError(u'Addon has no target_locale and we' u' could not guess one from the xpi') # Dumb version number increment. This will be invalid in some cases, # but some of the dictionaries we have currently already have wild # version numbers anyway. version_number = addon.current_version.version if version_number.endswith('.1-typefix'): version_number = version_number.replace('.1-typefix', '.2webext') else: version_number = '%s.1webext' % version_number manifest = { 'manifest_version': 2, 'name': unicode(addon.name), 'browser_specific_settings': { 'gecko': { 'id': addon.guid, }, }, 'version': version_number, 'dictionaries': { target_language: dictionary_path }, } # Write manifest.json we just build. new_zip.writestr('manifest.json', json.dumps(manifest))
def test_unzip_limit(self): with pytest.raises(forms.ValidationError): SafeZip(self.xpi_path('langpack-localepicker'))
def test_unzip_fatal(self): with pytest.raises(zipfile.BadZipfile): SafeZip(self.xpi_path('search.xml'))
def build_webext_dictionary_from_legacy(addon, destination): """Create a webext package of a legacy dictionary `addon`, and put it in `destination` path.""" from olympia.files.utils import SafeZip # Avoid circular import. old_path = addon.current_version.all_files[0].file_path old_zip = SafeZip(old_path) if not old_zip.is_valid: raise ValidationError('Current dictionary xpi is not valid') dictionary_path = '' with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as new_zip: for obj in old_zip.filelist: splitted = obj.filename.split('/') # Ignore useless directories and files. if splitted[0] in ('META-INF', '__MACOSX', 'chrome', 'chrome.manifest', 'install.rdf'): continue # Also ignore javascript (regardless of where they are, not just at # the root), since dictionaries should not contain any code. if splitted[-1].endswith('.js'): continue # Store the path of the last .dic file we find. It can be inside a # directory. if (splitted[-1].endswith('.dic')): dictionary_path = obj.filename new_zip.writestr(obj.filename, old_zip.read(obj.filename)) # Now that all files we want from the old zip are copied, build and # add manifest.json. if not dictionary_path: # This should not happen... It likely means it's an invalid # dictionary to begin with, or one that has its .dic file in a # chrome/ directory for some reason. Abort! raise ValidationError('Current dictionary xpi has no .dic file') if addon.target_locale: target_language = addon.target_locale else: # Guess target_locale since we don't have one already. Note that # for extra confusion, target_locale is a language, not a locale. target_language = to_language(os.path.splitext( os.path.basename(dictionary_path))[0]) if target_language not in settings.AMO_LANGUAGES: # We couldn't find that language in the list we support. Let's # try with just the prefix. target_language = target_language.split('-')[0] if target_language not in settings.AMO_LANGUAGES: # We tried our best. raise ValidationError(u'Addon has no target_locale and we' u' could not guess one from the xpi') # Dumb version number increment. This will be invalid in some cases, # but some of the dictionaries we have currently already have wild # version numbers anyway. version_number = addon.current_version.version if version_number.endswith('.1-typefix'): version_number = version_number.replace('.1-typefix', '.2webext') else: version_number = '%s.1webext' % version_number manifest = { 'manifest_version': 2, 'name': unicode(addon.name), 'applications': { 'gecko': { 'id': addon.guid, }, }, 'version': version_number, 'dictionaries': {target_language: dictionary_path}, } # Write manifest.json we just build. new_zip.writestr('manifest.json', json.dumps(manifest))
def extract_and_commit_from_file_obj(cls, file_obj, channel, author=None): """Extract all files from `file_obj` and comit them. This is doing the following: * Create a temporary `git worktree`_ * Remove all files in that worktree * Extract the zip behind `file_obj` into the worktree * Commit all files Kinda like doing:: $ workdir_name=$(uuid) $ mkdir /tmp/$workdir_name $ git worktree add /tmp/$workdir_name Preparing worktree (new branch 'af4172e4-d8c7…') HEAD is now at 8c5223e Initial commit $ git worktree list /tmp/addon-repository 8c5223e [master] /tmp/af4172e4-d8c7-4486-a5f2-316458da91ff 8c5223e [af4172e4-d8c7…] $ unzip dingrafowl-falcockalo-lockapionk.zip -d /tmp/$workdir_name Archive: dingrafowl-falcockalo-lockapionk.zip extracting: /tmp/af4172e4-d8c7…/manifest.json $ pushd /tmp/$workdir_name /tmp/af4172e4-d8c7-4486-a5f2-316458da91ff /tmp/addon-repository $ git status On branch af4172e4-d8c7-4486-a5f2-316458da91ff Untracked files: (use "git add <file>..." to include in what will be committed) manifest.json $ git add * $ git commit -a -m "Creating new version" [af4172e4-d8c7-4486-a5f2-316458da91ff c4285f8] Creating new version … $ cd addon-repository $ git checkout -b listed Switched to a new branch 'listed' # We don't technically do a full cherry-pick but it's close enough # and does almost what we do. We are technically commiting # directly on top of the branch as if we checked out the branch # in the worktree (via -b) but pygit doesn't properly support that # so we "simply" set the parents correctly. $ git cherry-pick c4285f8 [listed a4d0f63] Creating new version… This ignores the fact that there may be a race-condition of two versions being created at the same time. Since all relevant file based work is done in a temporary worktree there won't be any conflicts and usually the last upload simply wins the race and we're setting the HEAD of the branch (listed/unlisted) to that specific commit. .. _`git worktree`: https://git-scm.com/docs/git-worktree """ # Make sure we're always using the en-US locale by default translation.activate('en-US') addon = file_obj.version.addon repo = cls(addon.id) branch = repo.find_or_create_branch(BRANCHES[channel]) # Create a temporary worktree that we can use to unpack the zip # without disturbing the current git workdir since it creates a new # temporary directory where we extract to. with TemporaryWorktree(repo.git_repository) as worktree: # Now extract the zip to the workdir zip_file = SafeZip(file_obj.current_file_path, force_fsync=True) zip_file.extract_to_dest(worktree.path) # Stage changes, `TemporaryWorktree` always cleans the whole # directory so we can simply add all changes and have the correct # state. # Add all changes to the index (git add ...) worktree.repo.index.add_all() worktree.repo.index.write() tree = worktree.repo.index.write_tree() # Now create an commit directly on top of the respective branch message = ('Create new version {version} ({version_id}) for ' '{addon} from {file_obj}'.format( version=repr(file_obj.version), version_id=file_obj.version.id, addon=repr(addon), file_obj=repr(file_obj))) oid = worktree.repo.create_commit( None, # author, using the actual uploading user repo.get_author(author), # committer, using addons-robot because that's the user # actually doing the commit. repo.get_author(), # commiter, using addons-robot message, tree, # Set the current branch HEAD as the parent of this commit # so that it'll go straight into the branches commit log [branch.target]) # Fetch the commit object commit = worktree.repo.get(oid) # And set the commit we just created as HEAD of the relevant # branch, and updates the reflog. This does not require any # merges. branch.set_target(commit.hex) # Set the latest git hash on the related version. file_obj.version.update(git_hash=commit.hex) return repo