def test_extract_version_source_to_git():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
source.seek(0)
addon.current_version.update(source=source)
extract_version_source_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk, package_type='source')
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source')
assert os.listdir(repo.git_repository_path) == ['.git']
def test_extract_version_source_to_git():
addon = addon_factory(file_kw={'filename': 'webextension_no_id.xpi'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
source.seek(0)
addon.current_version.update(source=source)
extract_version_source_to_git(addon.current_version.pk)
repo = AddonGitRepository(addon.pk, package_type='source')
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source')
assert os.listdir(repo.git_repository_path) == ['.git']
def migrate_webextensions_to_git_storage(ids, **kw):
# recursive imports...
from olympia.versions.tasks import (extract_version_to_git,
extract_version_source_to_git)
log.info('Migrating add-ons to git storage %d-%d [%d].', ids[0], ids[-1],
len(ids))
addons = Addon.unfiltered.filter(id__in=ids)
for addon in addons:
# Filter out versions that are already present in the git
# storage.
versions = addon.versions.filter(git_hash='').order_by('created')
for version in versions:
# Back in the days an add-on was able to have multiple files
# per version. That changed, we are very naive here and extracting
# simply the first file in the list. For WebExtensions there is
# only a very very small number that have different files for
# a single version.
unique_file_hashes = set(
[x.original_hash for x in version.all_files])
if len(unique_file_hashes) > 1:
# Log actually different hashes so that we can clean them
# up manually and work together with developers later.
log.info(
'Version {version} of {addon} has more than one uploaded '
'file'.format(version=repr(version), addon=repr(addon)))
if not unique_file_hashes:
log.info('No files found for {version} from {addon}'.format(
version=repr(version), addon=repr(addon)))
continue
# Don't call the task as a task but do the extraction in process
# this makes sure we don't overwhelm the storage and also makes
# sure we don't end up with tasks committing at random times but
# correctly in-order instead.
try:
file_id = version.all_files[0].pk
log.info('Extracting file {file_id} to git storage'.format(
file_id=file_id))
extract_version_to_git(version.pk)
if version.source:
extract_version_source_to_git(version.pk)
log.info(
'Extraction of file {file_id} into git storage succeeded'.
format(file_id=file_id))
except Exception:
log.exception('Extraction of file {file_id} from {version} '
'({addon}) failed'.format(
file_id=version.all_files[0],
version=repr(version),
addon=repr(addon)))
continue
def migrate_webextensions_to_git_storage(ids, **kw):
# recursive imports...
from olympia.versions.tasks import (
extract_version_to_git, extract_version_source_to_git)
log.info(
'Migrating add-ons to git storage %d-%d [%d].',
ids[0], ids[-1], len(ids))
addons = Addon.unfiltered.filter(id__in=ids)
for addon in addons:
# Filter out versions that are already present in the git
# storage.
versions = addon.versions.filter(git_hash='').order_by('created')
for version in versions:
# Back in the days an add-on was able to have multiple files
# per version. That changed, we are very naive here and extracting
# simply the first file in the list. For WebExtensions there is
# only a very very small number that have different files for
# a single version.
unique_file_hashes = set([
x.original_hash for x in version.all_files
])
if len(unique_file_hashes) > 1:
# Log actually different hashes so that we can clean them
# up manually and work together with developers later.
log.info(
'Version {version} of {addon} has more than one uploaded '
'file'.format(version=repr(version), addon=repr(addon)))
if not unique_file_hashes:
log.info('No files found for {version} from {addon}'.format(
version=repr(version), addon=repr(addon)))
continue
# Don't call the task as a task but do the extraction in process
# this makes sure we don't overwhelm the storage and also makes
# sure we don't end up with tasks committing at random times but
# correctly in-order instead.
try:
file_id = version.all_files[0].pk
log.info('Extracting file {file_id} to git storage'.format(
file_id=file_id))
extract_version_to_git(version.pk)
if version.source:
extract_version_source_to_git(version.pk)
log.info(
'Extraction of file {file_id} into git storage succeeded'
.format(file_id=file_id))
except Exception:
log.exception(
'Extraction of file {file_id} from {version} '
'({addon}) failed'.format(
file_id=version.all_files[0],
version=repr(version),
addon=repr(addon)))
continue
