def test_extract_version_source_to_git(): addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'}) # Generate source file source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH) with zipfile.ZipFile(source, 'w') as zip_file: zip_file.writestr('manifest.json', '{}') source.seek(0) addon.current_version.update(source=source) extract_version_source_to_git(addon.current_version.pk) repo = AddonGitRepository(addon.pk, package_type='source') assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source') assert os.listdir(repo.git_repository_path) == ['.git']
def migrate_webextensions_to_git_storage(ids, **kw): # recursive imports... from olympia.versions.tasks import (extract_version_to_git, extract_version_source_to_git) log.info('Migrating add-ons to git storage %d-%d [%d].', ids[0], ids[-1], len(ids)) addons = Addon.unfiltered.filter(id__in=ids) for addon in addons: # Filter out versions that are already present in the git # storage. versions = addon.versions.filter(git_hash='').order_by('created') for version in versions: # Back in the days an add-on was able to have multiple files # per version. That changed, we are very naive here and extracting # simply the first file in the list. For WebExtensions there is # only a very very small number that have different files for # a single version. unique_file_hashes = set( [x.original_hash for x in version.all_files]) if len(unique_file_hashes) > 1: # Log actually different hashes so that we can clean them # up manually and work together with developers later. log.info( 'Version {version} of {addon} has more than one uploaded ' 'file'.format(version=repr(version), addon=repr(addon))) if not unique_file_hashes: log.info('No files found for {version} from {addon}'.format( version=repr(version), addon=repr(addon))) continue # Don't call the task as a task but do the extraction in process # this makes sure we don't overwhelm the storage and also makes # sure we don't end up with tasks committing at random times but # correctly in-order instead. try: file_id = version.all_files[0].pk log.info('Extracting file {file_id} to git storage'.format( file_id=file_id)) extract_version_to_git(version.pk) if version.source: extract_version_source_to_git(version.pk) log.info( 'Extraction of file {file_id} into git storage succeeded'. format(file_id=file_id)) except Exception: log.exception('Extraction of file {file_id} from {version} ' '({addon}) failed'.format( file_id=version.all_files[0], version=repr(version), addon=repr(addon))) continue
def migrate_webextensions_to_git_storage(ids, **kw): # recursive imports... from olympia.versions.tasks import ( extract_version_to_git, extract_version_source_to_git) log.info( 'Migrating add-ons to git storage %d-%d [%d].', ids[0], ids[-1], len(ids)) addons = Addon.unfiltered.filter(id__in=ids) for addon in addons: # Filter out versions that are already present in the git # storage. versions = addon.versions.filter(git_hash='').order_by('created') for version in versions: # Back in the days an add-on was able to have multiple files # per version. That changed, we are very naive here and extracting # simply the first file in the list. For WebExtensions there is # only a very very small number that have different files for # a single version. unique_file_hashes = set([ x.original_hash for x in version.all_files ]) if len(unique_file_hashes) > 1: # Log actually different hashes so that we can clean them # up manually and work together with developers later. log.info( 'Version {version} of {addon} has more than one uploaded ' 'file'.format(version=repr(version), addon=repr(addon))) if not unique_file_hashes: log.info('No files found for {version} from {addon}'.format( version=repr(version), addon=repr(addon))) continue # Don't call the task as a task but do the extraction in process # this makes sure we don't overwhelm the storage and also makes # sure we don't end up with tasks committing at random times but # correctly in-order instead. try: file_id = version.all_files[0].pk log.info('Extracting file {file_id} to git storage'.format( file_id=file_id)) extract_version_to_git(version.pk) if version.source: extract_version_source_to_git(version.pk) log.info( 'Extraction of file {file_id} into git storage succeeded' .format(file_id=file_id)) except Exception: log.exception( 'Extraction of file {file_id} from {version} ' '({addon}) failed'.format( file_id=version.all_files[0], version=repr(version), addon=repr(addon))) continue