def path():
# Check file paths / permissions
read_and_write = (
settings.TMP_PATH,
settings.MEDIA_ROOT,
user_media_path('addons'),
user_media_path('guarded_addons'),
user_media_path('addon_icons'),
user_media_path('previews'),
user_media_path('userpics'),)
read_only = [os.path.join(settings.ROOT, 'locale')]
filepaths = [(path, os.R_OK | os.W_OK, 'We want read + write')
for path in read_and_write]
filepaths += [(path, os.R_OK, 'We want read') for path in read_only]
filepath_results = []
filepath_status = True
for path, perms, notes in filepaths:
path_exists = os.path.exists(path)
path_perms = os.access(path, perms)
filepath_status = filepath_status and path_exists and path_perms
if not isinstance(path, bytes):
notes += ' / should be a bytestring!'
filepath_results.append((path, path_exists, path_perms, notes))
status = filepath_status
status = ''
if not filepath_status:
status = 'check main status page for broken perms / values'
return status, filepath_results
def migrate_legacy_dictionary_to_webextension(addon):
"""Migrate a single legacy dictionary to webextension format, creating a
new package from the current_version, faking an upload to create a new
Version instance."""
user = UserProfile.objects.get(pk=settings.TASK_USER_ID)
now = datetime.now()
# Wrap zip in FileUpload for Version.from_upload() to consume.
upload = FileUpload.objects.create(user=user, valid=True)
destination = os.path.join(user_media_path('addons'), 'temp',
uuid.uuid4().hex + '.xpi')
build_webext_dictionary_from_legacy(addon, destination)
upload.update(path=destination)
parsed_data = parse_addon(upload, user=user)
# Create version.
version = Version.from_upload(upload,
addon,
platforms=[amo.PLATFORM_ALL.id],
channel=amo.RELEASE_CHANNEL_LISTED,
parsed_data=parsed_data)
activity.log_create(amo.LOG.ADD_VERSION, version, addon, user=user)
# Sign the file, and set it to public. That should automatically set
# current_version to the version we created.
file_ = version.all_files[0]
sign_file(file_)
file_.update(datestatuschanged=now, reviewed=now, status=amo.STATUS_PUBLIC)
def migrate_legacy_dictionary_to_webextension(addon):
"""Migrate a single legacy dictionary to webextension format, creating a
new package from the current_version, faking an upload to create a new
Version instance."""
user = UserProfile.objects.get(pk=settings.TASK_USER_ID)
now = datetime.now()
# Wrap zip in FileUpload for Version.from_upload() to consume.
upload = FileUpload.objects.create(
user=user, valid=True)
destination = os.path.join(
user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi')
target_language = build_webext_dictionary_from_legacy(addon, destination)
if not addon.target_locale:
addon.update(target_locale=target_language)
upload.update(path=destination)
parsed_data = parse_addon(upload, addon=addon, user=user)
# Create version.
# WebExtension dictionaries are only compatible with Firefox Desktop
# Firefox for Android uses the OS spellchecking.
version = Version.from_upload(
upload, addon, selected_apps=[amo.FIREFOX.id],
channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data)
activity.log_create(amo.LOG.ADD_VERSION, version, addon, user=user)
# Sign the file, and set it to public. That should automatically set
# current_version to the version we created.
file_ = version.all_files[0]
sign_file(file_)
file_.update(datestatuschanged=now, reviewed=now, status=amo.STATUS_PUBLIC)
def new_theme_version_with_69_properties(old_version):
timer = StopWatch(
'addons.tasks.repack_themes_for_69.new_theme_version.')
timer.start()
author = get_user()
# Wrap zip in FileUpload for Version from_upload to consume.
upload = FileUpload.objects.create(user=author, valid=True)
filename = uuid.uuid4().hex + '.xpi'
destination = os.path.join(user_media_path('addons'), 'temp', filename)
old_xpi = get_filepath(old_version.all_files[0])
build_69_compatible_theme(
old_xpi, destination, get_next_version_number(old_version.addon))
upload.update(path=destination, name=filename)
timer.log_interval('1.build_xpi')
# Create addon + version
parsed_data = parse_addon(upload, addon=old_version.addon, user=author)
timer.log_interval('2.parse_addon')
version = Version.from_upload(
upload, old_version.addon, selected_apps=[amo.FIREFOX.id],
channel=amo.RELEASE_CHANNEL_LISTED,
parsed_data=parsed_data)
timer.log_interval('3.initialize_version')
# And finally sign the files (actually just one)
for file_ in version.all_files:
sign_file(file_)
file_.update(
reviewed=datetime.now(),
status=amo.STATUS_APPROVED)
timer.log_interval('4.sign_files')
return version
def save_theme_reupload(header, footer, addon, **kw):
header_dst = None
footer_dst = None
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
try:
if header:
header = os.path.join(settings.TMP_PATH, 'persona_header', header)
header_dst = os.path.join(dst_root, 'pending_header.png')
save_persona_image(src=header, full_dst=header_dst)
if footer:
footer = os.path.join(settings.TMP_PATH, 'persona_footer', footer)
footer_dst = os.path.join(dst_root, 'pending_footer.png')
save_persona_image(src=footer, full_dst=footer_dst)
except IOError as e:
log.error(str(e))
raise
if header_dst or footer_dst:
theme = addon.persona
header = 'pending_header.png' if header_dst else theme.header
# Theme footer is optional, but can't be None.
footer = theme.footer or ''
if footer_dst:
footer = 'pending_footer.png'
# Store pending header and/or footer file paths for review.
RereviewQueueTheme.objects.filter(theme=theme).delete()
rqt = RereviewQueueTheme(theme=theme, header=header, footer=footer)
rereviewqueuetheme_checksum(rqt=rqt)
rqt.save()
def new_theme_version_with_69_properties(old_version):
timer = StopWatch(
'addons.tasks.repack_themes_for_69.new_theme_version.')
timer.start()
author = get_user()
# Wrap zip in FileUpload for Version from_upload to consume.
upload = FileUpload.objects.create(user=author, valid=True)
filename = uuid.uuid4().hex + '.xpi'
destination = os.path.join(user_media_path('addons'), 'temp', filename)
old_xpi = get_filepath(old_version.all_files[0])
build_69_compatible_theme(
old_xpi, destination, get_next_version_number(old_version.addon))
upload.update(path=destination, name=filename)
timer.log_interval('1.build_xpi')
# Create addon + version
parsed_data = parse_addon(upload, addon=old_version.addon, user=author)
timer.log_interval('2.parse_addon')
version = Version.from_upload(
upload, old_version.addon, selected_apps=[amo.FIREFOX.id],
channel=amo.RELEASE_CHANNEL_LISTED,
parsed_data=parsed_data)
timer.log_interval('3.initialize_version')
# And finally sign the files (actually just one)
for file_ in version.all_files:
sign_file(file_)
file_.update(
reviewed=datetime.now(),
status=amo.STATUS_APPROVED)
timer.log_interval('4.sign_files')
return version
def rezip_file(response, pk):
# An .xpi does not have a directory inside the zip, yet zips from github
# do, so we'll need to rezip the file before passing it through to the
# validator.
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
old_filename = '{}_github_webhook.zip'.format(pk)
old_path = os.path.join(loc, old_filename)
with storage.open(old_path, 'wb') as old:
old.write(response.content)
new_filename = '{}_github_webhook.xpi'.format(pk)
new_path = os.path.join(loc, new_filename)
old_zip = SafeUnzip(old_path)
if not old_zip.is_valid():
raise
with storage.open(new_path, 'w') as new:
new_zip = zipfile.ZipFile(new, 'w')
for obj in old_zip.filelist:
# Basically strip off the leading directory.
new_filename = obj.filename.partition('/')[-1]
if not new_filename:
continue
new_zip.writestr(new_filename, old_zip.read(obj.filename))
new_zip.close()
old_zip.close()
return new_path
def test_reupload(self, save_persona_image_mock,
create_persona_preview_images_mock,
make_checksum_mock):
make_checksum_mock.return_value = 'checksumbeforeyouwrecksome'
data = self.get_dict(header_hash='y0l0', footer_hash='abab')
self.form = EditThemeForm(data, request=self.request,
instance=self.instance)
assert self.form.is_valid()
self.form.save()
dst = os.path.join(user_media_path('addons'), str(self.instance.id))
header_src = os.path.join(settings.TMP_PATH, 'persona_header',
u'y0l0')
footer_src = os.path.join(settings.TMP_PATH, 'persona_footer',
u'abab')
assert save_persona_image_mock.mock_calls == (
[mock.call(src=header_src,
full_dst=os.path.join(dst, 'pending_header.png')),
mock.call(src=footer_src,
full_dst=os.path.join(dst, 'pending_footer.png'))])
rqt = RereviewQueueTheme.objects.filter(theme=self.instance.persona)
assert rqt.count() == 1
assert rqt[0].header == 'pending_header.png'
assert rqt[0].footer == 'pending_footer.png'
assert not rqt[0].dupe_persona
def migrate_legacy_dictionary_to_webextension(addon):
"""Migrate a single legacy dictionary to webextension format, creating a
new package from the current_version, faking an upload to create a new
Version instance."""
user = UserProfile.objects.get(pk=settings.TASK_USER_ID)
now = datetime.now()
# Wrap zip in FileUpload for Version.from_upload() to consume.
upload = FileUpload.objects.create(user=user, valid=True)
destination = os.path.join(user_media_path('addons'), 'temp',
uuid.uuid4().hex + '.xpi')
target_language = build_webext_dictionary_from_legacy(addon, destination)
if not addon.target_locale:
addon.update(target_locale=target_language)
upload.update(path=destination)
parsed_data = parse_addon(upload, addon=addon, user=user)
# Create version.
# WebExtension dictionaries are only compatible with Firefox Desktop
# Firefox for Android uses the OS spellchecking.
version = Version.from_upload(upload,
addon,
selected_apps=[amo.FIREFOX.id],
channel=amo.RELEASE_CHANNEL_LISTED,
parsed_data=parsed_data)
activity.log_create(amo.LOG.ADD_VERSION, version, addon, user=user)
# Sign the file, and set it to public. That should automatically set
# current_version to the version we created.
file_ = version.all_files[0]
sign_file(file_)
file_.update(datestatuschanged=now, reviewed=now, status=amo.STATUS_PUBLIC)
def test_reupload(self, save_persona_image_mock,
create_persona_preview_images_mock, make_checksum_mock):
make_checksum_mock.return_value = 'checksumbeforeyouwrecksome'
data = self.get_dict(header_hash='y0l0', footer_hash='abab')
self.form = EditThemeForm(data,
request=self.request,
instance=self.instance)
assert self.form.is_valid()
self.form.save()
dst = os.path.join(user_media_path('addons'), str(self.instance.id))
header_src = os.path.join(settings.TMP_PATH, 'persona_header', u'y0l0')
footer_src = os.path.join(settings.TMP_PATH, 'persona_footer', u'abab')
assert save_persona_image_mock.mock_calls == ([
mock.call(src=header_src,
full_dst=os.path.join(dst, 'pending_header.png')),
mock.call(src=footer_src,
full_dst=os.path.join(dst, 'pending_footer.png'))
])
rqt = RereviewQueueTheme.objects.filter(theme=self.instance.persona)
assert rqt.count() == 1
assert rqt[0].header == 'pending_header.png'
assert rqt[0].footer == 'pending_footer.png'
assert not rqt[0].dupe_persona
def save_theme_reupload(header, footer, addon, **kw):
header_dst = None
footer_dst = None
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
try:
if header:
header = os.path.join(settings.TMP_PATH, 'persona_header', header)
header_dst = os.path.join(dst_root, 'pending_header.png')
save_persona_image(src=header, full_dst=header_dst)
if footer:
footer = os.path.join(settings.TMP_PATH, 'persona_footer', footer)
footer_dst = os.path.join(dst_root, 'pending_footer.png')
save_persona_image(src=footer, full_dst=footer_dst)
except IOError as e:
log.error(str(e))
raise
if header_dst or footer_dst:
theme = addon.persona
header = 'pending_header.png' if header_dst else theme.header
# Theme footer is optional, but can't be None.
footer = theme.footer or ''
if footer_dst:
footer = 'pending_footer.png'
# Store pending header and/or footer file paths for review.
RereviewQueueTheme.objects.filter(theme=theme).delete()
rqt = RereviewQueueTheme(theme=theme, header=header, footer=footer)
rereviewqueuetheme_checksum(rqt=rqt)
rqt.save()
def rezip_file(response, pk):
# An .xpi does not have a directory inside the zip, yet zips from github
# do, so we'll need to rezip the file before passing it through to the
# validator.
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
old_filename = '{}_github_webhook.zip'.format(pk)
old_path = os.path.join(loc, old_filename)
with storage.open(old_path, 'wb') as old:
old.write(response.content)
new_filename = '{}_github_webhook.xpi'.format(pk)
new_path = os.path.join(loc, new_filename)
old_zip = SafeZip(old_path)
if not old_zip.is_valid:
raise
with storage.open(new_path, 'w') as new:
new_zip = zipfile.ZipFile(new, 'w')
for obj in old_zip.filelist:
# Basically strip off the leading directory.
new_filename = obj.filename.partition('/')[-1]
if not new_filename:
continue
new_zip.writestr(new_filename, old_zip.read(obj.filename))
new_zip.close()
old_zip.close()
return new_path
def test_success(self, save_persona_image_mock,
create_persona_preview_images_mock, make_checksum_mock):
make_checksum_mock.return_value = 'hashyourselfbeforeyoucrashyourself'
self.request.user = UserProfile.objects.get(pk=2519)
data = self.get_dict()
header_url = self.get_img_urls()
# Upload header image.
img = open(get_image_path('persona-header.jpg'), 'rb')
r_ajax = self.client.post(header_url, {'upload_image': img})
content = json.loads(force_text(r_ajax.content))
data.update(header_hash=content['upload_hash'])
# Populate and save form.
self.post()
assert self.form.is_valid(), self.form.errors
self.form.save()
addon = Addon.objects.filter(type=amo.ADDON_PERSONA).order_by('-id')[0]
persona = addon.persona
# Test for correct Addon and Persona values.
assert six.text_type(addon.name) == data['name']
assert addon.slug == data['slug']
self.assertSetEqual(set(addon.categories.values_list('id', flat=True)),
{self.cat.id})
self.assertSetEqual(set(addon.tags.values_list('tag_text', flat=True)),
set(data['tags'].split(', ')))
assert persona.persona_id == 0
assert persona.license == data['license']
assert persona.accentcolor == data['accentcolor'].lstrip('#')
assert persona.textcolor == data['textcolor'].lstrip('#')
assert persona.author == self.request.user.username
assert persona.display_username == self.request.user.name
assert not persona.dupe_persona
v = addon.versions.all()
assert len(v) == 1
assert v[0].version == '0'
# Test for header and preview images.
dst = os.path.join(user_media_path('addons'), str(addon.id))
header_src = os.path.join(settings.TMP_PATH, 'persona_header',
u'b4ll1n')
assert save_persona_image_mock.mock_calls == ([
mock.call(src=header_src, full_dst=os.path.join(dst, 'header.png'))
])
create_persona_preview_images_mock.assert_called_with(
src=header_src,
full_dst=[
os.path.join(dst, 'preview.png'),
os.path.join(dst, 'icon.png')
],
set_modified_on=addon.serializable_reference())
def delete_persona_image(dst, **kw):
log.info('[1@None] Deleting persona image: %s.' % dst)
if not dst.startswith(user_media_path('addons')):
log.error("Someone tried deleting something they shouldn't: %s" % dst)
return
try:
storage.delete(dst)
except Exception, e:
log.error('Error deleting persona image: %s' % e)
def delete_persona_image(dst, **kw):
log.info('[1@None] Deleting persona image: %s.' % dst)
if not dst.startswith(user_media_path('addons')):
log.error("Someone tried deleting something they shouldn't: %s" % dst)
return
try:
storage.delete(dst)
except Exception as e:
log.error('Error deleting persona image: %s' % e)
def picture_dir(self):
from olympia.amo.templatetags.jinja_helpers import user_media_path
split_id = re.match(r'((\d*?)(\d{0,3}?))\d{1,3}$', str(self.id))
return os.path.join(
user_media_path('userpics'),
split_id.group(2) or '0',
split_id.group(1) or '0',
)
def add_static_theme_from_lwt(lwt):
# Try to handle LWT with no authors
author = (lwt.listed_authors or [_get_lwt_default_author()])[0]
# Wrap zip in FileUpload for Addon/Version from_upload to consume.
upload = FileUpload.objects.create(
user=author, valid=True)
destination = os.path.join(
user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi')
build_static_theme_xpi_from_lwt(lwt, destination)
upload.update(path=destination)
# Create addon + version
parsed_data = parse_addon(upload, user=author)
addon = Addon.initialize_addon_from_upload(
parsed_data, upload, amo.RELEASE_CHANNEL_LISTED, author)
# Version.from_upload sorts out platforms for us.
version = Version.from_upload(
upload, addon, platforms=None, channel=amo.RELEASE_CHANNEL_LISTED,
parsed_data=parsed_data)
# Set category
static_theme_categories = CATEGORIES.get(amo.FIREFOX.id, []).get(
amo.ADDON_STATICTHEME, [])
lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat.
lwt_category_slug = lwt_category.slug if lwt_category else 'other'
static_category = static_theme_categories.get(
lwt_category_slug, static_theme_categories.get('other'))
AddonCategory.objects.create(
addon=addon,
category=Category.from_static_category(static_category, True))
# Set license
lwt_license = PERSONA_LICENSES_IDS.get(
lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright
static_license = License.objects.get(builtin=lwt_license.builtin)
version.update(license=static_license)
# Set tags
for addon_tag in AddonTag.objects.filter(addon=lwt):
AddonTag.objects.create(addon=addon, tag=addon_tag.tag)
# Logging
activity.log_create(
amo.LOG.CREATE_STATICTHEME_FROM_PERSONA, addon, user=author)
log.debug('New static theme %r created from %r' % (addon, lwt))
# And finally sign the files (actually just one)
for file_ in version.all_files:
sign_file(file_)
file_.update(
datestatuschanged=datetime.now(),
reviewed=datetime.now(),
status=amo.STATUS_PUBLIC)
addon.update(status=amo.STATUS_PUBLIC)
return addon
def test_success(self, save_persona_image_mock,
create_persona_preview_images_mock, make_checksum_mock):
make_checksum_mock.return_value = 'hashyourselfbeforeyoucrashyourself'
self.request.user = UserProfile.objects.get(pk=2519)
data = self.get_dict()
header_url = self.get_img_urls()
# Upload header image.
img = open(get_image_path('persona-header.jpg'), 'rb')
r_ajax = self.client.post(header_url, {'upload_image': img})
data.update(header_hash=json.loads(r_ajax.content)['upload_hash'])
# Populate and save form.
self.post()
assert self.form.is_valid(), self.form.errors
self.form.save()
addon = Addon.objects.filter(type=amo.ADDON_PERSONA).order_by('-id')[0]
persona = addon.persona
# Test for correct Addon and Persona values.
assert unicode(addon.name) == data['name']
assert addon.slug == data['slug']
self.assertSetEqual(set(addon.categories.values_list('id', flat=True)),
{self.cat.id})
self.assertSetEqual(set(addon.tags.values_list('tag_text', flat=True)),
set(data['tags'].split(', ')))
assert persona.persona_id == 0
assert persona.license == data['license']
assert persona.accentcolor == data['accentcolor'].lstrip('#')
assert persona.textcolor == data['textcolor'].lstrip('#')
assert persona.author == self.request.user.username
assert persona.display_username == self.request.user.name
assert not persona.dupe_persona
v = addon.versions.all()
assert len(v) == 1
assert v[0].version == '0'
# Test for header and preview images.
dst = os.path.join(user_media_path('addons'), str(addon.id))
header_src = os.path.join(settings.TMP_PATH, 'persona_header',
u'b4ll1n')
assert save_persona_image_mock.mock_calls == (
[mock.call(src=header_src,
full_dst=os.path.join(dst, 'header.png'))])
create_persona_preview_images_mock.assert_called_with(
src=header_src,
full_dst=[os.path.join(dst, 'preview.png'),
os.path.join(dst, 'icon.png')],
set_modified_on=addon.serializable_reference())
def _image_path(self, folder, file_ext):
from olympia.amo.templatetags.jinja_helpers import user_media_path
url = os.path.join(
user_media_path(self.media_folder),
folder,
str(self.id // 1000),
f'{self.id}.{file_ext}',
)
return url
def get_background_image_urls(self):
if self.addon.type != amo.ADDON_STATICTHEME:
return []
background_images_folder = os.path.join(
user_media_path('addons'), str(self.addon.id), unicode(self.id))
background_images_url = '/'.join(
(user_media_url('addons'), str(self.addon.id), unicode(self.id)))
out = [
background.replace(background_images_folder, background_images_url)
for background in walkfiles(background_images_folder)]
return out
def get_background_image_urls(self):
if self.addon.type != amo.ADDON_STATICTHEME:
return []
background_images_folder = os.path.join(
user_media_path('addons'), str(self.addon.id), unicode(self.id))
background_images_url = '/'.join(
(user_media_url('addons'), str(self.addon.id), unicode(self.id)))
out = [
background.replace(background_images_folder, background_images_url)
for background in walkfiles(background_images_folder)]
return out
def delete_icon(dst, **kw):
log.info('[1@None] Deleting icon: %s.' % dst)
if not dst.startswith(user_media_path('collection_icons')):
log.error("Someone tried deleting something they shouldn't: %s" % dst)
return
try:
storage.delete(dst)
except Exception, e:
log.error("Error deleting icon: %s" % e)
def delete_icon(dst, **kw):
log.info('[1@None] Deleting icon: %s.' % dst)
if not dst.startswith(user_media_path('collection_icons')):
log.error("Someone tried deleting something they shouldn't: %s" % dst)
return
try:
storage.delete(dst)
except Exception, e:
log.error("Error deleting icon: %s" % e)
def delete_photo(dst, **kw):
task_log.debug('[1@None] Deleting photo: %s.' % dst)
if not dst.startswith(user_media_path('userpics')):
task_log.error("Someone tried deleting something they shouldn't: %s"
% dst)
return
try:
storage.delete(dst)
except Exception, e:
task_log.error("Error deleting userpic: %s" % e)
def delete_photo(dst, **kw):
task_log.info('[1@None] Deleting photo: %s.' % dst)
if not dst.startswith(user_media_path('userpics')):
task_log.error("Someone tried deleting something they shouldn't: %s" %
dst)
return
try:
storage.delete(dst)
except Exception as e:
task_log.error('Error deleting userpic: %s' % e)
def path():
# Check file paths / permissions
rw = (
settings.TMP_PATH,
settings.MEDIA_ROOT,
user_media_path('addons'),
user_media_path('guarded_addons'),
user_media_path('addon_icons'),
user_media_path('previews'),
user_media_path('userpics'),
dump_apps.Command.get_json_path(),
)
r = [
os.path.join(settings.ROOT, 'locale'),
# The deploy process will want write access to this.
# We do not want Django to have write access though.
settings.PROD_DETAILS_DIR
]
filepaths = [(path, os.R_OK | os.W_OK, 'We want read + write')
for path in rw]
filepaths += [(path, os.R_OK, 'We want read') for path in r]
filepath_results = []
filepath_status = True
for path, perms, notes in filepaths:
path_exists = os.path.exists(path)
path_perms = os.access(path, perms)
filepath_status = filepath_status and path_exists and path_perms
if not isinstance(path, str):
notes += ' / should be a bytestring!'
filepath_results.append((path, path_exists, path_perms, notes))
status = filepath_status
status = ''
if not filepath_status:
status = 'check main status page for broken perms / values'
return status, filepath_results
def add_static_theme_from_lwt(lwt):
# Try to handle LWT with no authors
author = (lwt.listed_authors or [_get_lwt_default_author()])[0]
# Wrap zip in FileUpload for Addon/Version from_upload to consume.
upload = FileUpload.objects.create(
user=author, valid=True)
destination = os.path.join(
user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi')
build_static_theme_xpi_from_lwt(lwt, destination)
upload.update(path=destination)
# Create addon + version
parsed_data = parse_addon(upload, user=author)
addon = Addon.initialize_addon_from_upload(
parsed_data, upload, amo.RELEASE_CHANNEL_LISTED, author)
# Version.from_upload sorts out platforms for us.
version = Version.from_upload(
upload, addon, platforms=None, channel=amo.RELEASE_CHANNEL_LISTED,
parsed_data=parsed_data)
# Set category
static_theme_categories = CATEGORIES.get(amo.FIREFOX.id, []).get(
amo.ADDON_STATICTHEME, [])
lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat.
lwt_category_slug = lwt_category.slug if lwt_category else 'other'
static_category = static_theme_categories.get(
lwt_category_slug, static_theme_categories.get('other'))
AddonCategory.objects.create(
addon=addon,
category=Category.from_static_category(static_category, True))
# Set license
lwt_license = PERSONA_LICENSES_IDS.get(
lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright
static_license = License.objects.get(builtin=lwt_license.builtin)
version.update(license=static_license)
# Set tags
for addon_tag in AddonTag.objects.filter(addon=lwt):
AddonTag.objects.create(addon=addon, tag=addon_tag.tag)
# Logging
activity.log_create(
amo.LOG.CREATE_STATICTHEME_FROM_PERSONA, addon, user=author)
log.debug('New static theme %r created from %r' % (addon, lwt))
# And finally update the statuses
version.all_files[0].update(status=amo.STATUS_PUBLIC)
addon.update(status=amo.STATUS_PUBLIC)
return addon
def save_theme(header, addon, **kw):
"""Save theme image and calculates checksum after theme save."""
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
header = os.path.join(settings.TMP_PATH, 'persona_header', header)
header_dst = os.path.join(dst_root, 'header.png')
try:
save_persona_image(src=header, full_dst=header_dst)
create_persona_preview_images(
src=header, full_dst=[os.path.join(dst_root, 'preview.png'),
os.path.join(dst_root, 'icon.png')],
set_modified_on=[addon])
theme_checksum(addon.persona)
except IOError:
addon.delete()
raise
def save_theme_reupload(header, addon_pk, **kw):
addon = Addon.objects.get(pk=addon_pk)
header_dst = None
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
try:
if header:
header = os.path.join(settings.TMP_PATH, 'persona_header', header)
header_dst = os.path.join(dst_root, 'pending_header.png')
save_persona_image(src=header, full_dst=header_dst)
except IOError as e:
log.error(str(e))
raise
if header_dst:
theme = addon.persona
header = 'pending_header.png' if header_dst else theme.header
def save_theme_reupload(header, addon_pk, **kw):
addon = Addon.objects.get(pk=addon_pk)
header_dst = None
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
try:
if header:
header = os.path.join(settings.TMP_PATH, 'persona_header', header)
header_dst = os.path.join(dst_root, 'pending_header.png')
save_persona_image(src=header, full_dst=header_dst)
except IOError as e:
log.error(str(e))
raise
if header_dst:
theme = addon.persona
header = 'pending_header.png' if header_dst else theme.header
def save_theme(header, addon_pk, **kw):
"""Save theme image and calculates checksum after theme save."""
addon = Addon.objects.get(pk=addon_pk)
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
header = os.path.join(settings.TMP_PATH, 'persona_header', header)
header_dst = os.path.join(dst_root, 'header.png')
try:
save_persona_image(src=header, full_dst=header_dst)
create_persona_preview_images(
src=header, full_dst=[os.path.join(dst_root, 'preview.png'),
os.path.join(dst_root, 'icon.png')],
set_modified_on=addon.serializable_reference())
theme_checksum(addon.persona)
except IOError:
addon.delete()
raise
def _uploader(resize_size, final_size):
img = get_image_path('mozilla.png')
original_size = (339, 128)
src = tempfile.NamedTemporaryFile(mode='r+b',
suffix='.png',
delete=False,
dir=settings.TMP_PATH)
if not isinstance(final_size, list):
final_size = [final_size]
resize_size = [resize_size]
uploadto = user_media_path('addon_icons')
try:
os.makedirs(uploadto)
except OSError:
pass
for rsize, expected_size in zip(resize_size, final_size):
# resize_icon moves the original
shutil.copyfile(img, src.name)
src_image = Image.open(src.name)
assert src_image.size == original_size
dest_name = os.path.join(uploadto, '1234')
with mock.patch('olympia.amo.utils.pngcrush_image') as pngcrush_mock:
return_value = tasks.resize_icon(src.name, dest_name, [rsize])
dest_image = '%s-%s.png' % (dest_name, rsize)
assert pngcrush_mock.call_count == 1
assert pngcrush_mock.call_args_list[0][0][0] == dest_image
assert image_size(dest_image) == expected_size
# original should have been moved to -original
orig_image = '%s-original.png' % dest_name
assert os.path.exists(orig_image)
# Return value of the task should be a dict with an icon_hash key
# containing the 8 first chars of the md5 hash of the source file,
# which is bb362450b00f0461c6bddc6b97b3c30b.
assert return_value == {'icon_hash': 'bb362450'}
os.remove(dest_image)
assert not os.path.exists(dest_image)
os.remove(orig_image)
assert not os.path.exists(orig_image)
shutil.rmtree(uploadto)
assert not os.path.exists(src.name)
def add_file(self, chunks, filename, size):
if not self.uuid:
self.uuid = self._meta.get_field('uuid')._create_uuid()
filename = force_text(u'{0}_{1}'.format(self.uuid.hex, filename))
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
base, ext = os.path.splitext(filename)
is_crx = False
# Change a ZIP to an XPI, to maintain backward compatibility
# with older versions of Firefox and to keep the rest of the XPI code
# path as consistent as possible for ZIP uploads.
# See: https://github.com/mozilla/addons-server/pull/2785
if ext == '.zip':
ext = '.xpi'
# If the extension is a CRX, we need to do some actual work to it
# before we just convert it to an XPI. We strip the header from the
# CRX, then it's good; see more about the CRX file format here:
# https://developer.chrome.com/extensions/crx
if ext == '.crx':
ext = '.xpi'
is_crx = True
if ext in amo.VALID_ADDON_FILE_EXTENSIONS:
loc += ext
log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc),
extra={
'email':
(self.user.email if self.user and self.user.email else '')
})
if is_crx:
hash_func = write_crx_as_xpi(chunks, loc)
else:
hash_func = hashlib.sha256()
with storage.open(loc, 'wb') as file_destination:
for chunk in chunks:
hash_func.update(chunk)
file_destination.write(chunk)
self.path = loc
self.name = filename
self.hash = 'sha256:%s' % hash_func.hexdigest()
self.save()
def _uploader(resize_size, final_size):
img = get_image_path('mozilla.png')
original_size = (339, 128)
src = tempfile.NamedTemporaryFile(
mode='r+w+b', suffix='.png', delete=False, dir=settings.TMP_PATH)
if not isinstance(final_size, list):
final_size = [final_size]
resize_size = [resize_size]
uploadto = user_media_path('addon_icons')
try:
os.makedirs(uploadto)
except OSError:
pass
for rsize, expected_size in zip(resize_size, final_size):
# resize_icon moves the original
shutil.copyfile(img, src.name)
src_image = Image.open(src.name)
assert src_image.size == original_size
dest_name = os.path.join(uploadto, '1234')
with mock.patch('olympia.amo.utils.pngcrush_image') as pngcrush_mock:
return_value = tasks.resize_icon(src.name, dest_name, [rsize])
dest_image = '%s-%s.png' % (dest_name, rsize)
assert pngcrush_mock.call_count == 1
assert pngcrush_mock.call_args_list[0][0][0] == dest_image
assert image_size(dest_image) == expected_size
# original should have been moved to -original
orig_image = '%s-original.png' % dest_name
assert os.path.exists(orig_image)
# Return value of the task should be a dict with an icon_hash key
# containing the 8 first chars of the md5 hash of the source file,
# which is bb362450b00f0461c6bddc6b97b3c30b.
assert return_value == {'icon_hash': 'bb362450'}
os.remove(dest_image)
assert not os.path.exists(dest_image)
os.remove(orig_image)
assert not os.path.exists(orig_image)
shutil.rmtree(uploadto)
assert not os.path.exists(src.name)
def create_theme_images(theme, placement, hash_):
"""
Generates 2 images, one in the temp folder and the other in the
user-media one. Both are needed to generate previews for themes.
"""
color = random.choice(ImageColor.colormap.keys())
image = Image.new('RGB', (3000, 200), color)
tmp_path = os.path.join(settings.TMP_PATH,
'persona_{placement}'.format(placement=placement))
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
tmp_loc = os.path.join(tmp_path, hash_)
image.save(tmp_loc, 'jpeg')
media_path = os.path.join(user_media_path('addons'), str(theme.id))
if not os.path.exists(media_path):
os.makedirs(media_path)
media_loc = os.path.join(media_path, hash_)
image.save(media_loc, 'jpeg')
def add_file(self, chunks, filename, size):
if not self.uuid:
self.uuid = self._meta.get_field('uuid')._create_uuid()
_base, ext = os.path.splitext(filename)
was_crx = ext == '.crx'
# Filename we'll expose (but not use for storage).
self.name = force_str('{0}_{1}'.format(self.uuid.hex, filename))
# Final path on our filesystem. If it had a valid extension we change
# it to .xpi (CRX files are converted before validation, so they will
# be treated as normal .xpi for validation). If somehow this is
# not a valid archive or the extension is invalid parse_addon() will
# eventually complain at validation time or before even reaching the
# linter.
if ext in amo.VALID_ADDON_FILE_EXTENSIONS:
ext = '.xpi'
self.path = os.path.join(user_media_path('addons'), 'temp',
'{0}{1}'.format(uuid.uuid4().hex, ext))
hash_obj = None
if was_crx:
try:
hash_obj = write_crx_as_xpi(chunks, self.path)
except InvalidOrUnsupportedCrx:
# We couldn't convert the crx file. Write it to the filesystem
# normally, the validation process should reject this with a
# proper error later.
pass
if hash_obj is None:
hash_obj = self.write_data_to_path(chunks)
self.hash = 'sha256:%s' % hash_obj.hexdigest()
# The following log statement is used by foxsec-pipeline.
log.info(
'UPLOAD: %r (%s bytes) to %r' % (self.name, size, self.path),
extra={
'email':
(self.user.email if self.user and self.user.email else ''),
'upload_hash': self.hash,
},
)
self.save()
def save_theme(header, footer, addon, **kw):
"""Save theme image and calculates checksum after theme save."""
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
header = os.path.join(settings.TMP_PATH, 'persona_header', header)
header_dst = os.path.join(dst_root, 'header.png')
if footer:
footer = os.path.join(settings.TMP_PATH, 'persona_footer', footer)
footer_dst = os.path.join(dst_root, 'footer.png')
try:
save_persona_image(src=header, full_dst=header_dst)
if footer:
save_persona_image(src=footer, full_dst=footer_dst)
create_persona_preview_images(
src=header, full_dst=[os.path.join(dst_root, 'preview.png'),
os.path.join(dst_root, 'icon.png')],
set_modified_on=[addon])
theme_checksum(addon.persona)
except IOError:
addon.delete()
raise
def approve_rereview(theme):
"""Replace original theme with pending theme on filesystem."""
# If reuploaded theme, replace old theme design.
storage = LocalFileStorage()
rereview = theme.rereviewqueuetheme_set.all()
reupload = rereview[0]
if reupload.header_path != reupload.theme.header_path:
create_persona_preview_images(
src=reupload.header_path,
full_dst=[
reupload.theme.thumb_path,
reupload.theme.icon_path],
set_modified_on=[reupload.theme.addon])
if not reupload.theme.is_new():
# Legacy themes also need a preview_large.jpg.
# Modern themes use preview.png for both thumb and preview so there
# is no problem there.
copy_stored_file(reupload.theme.thumb_path,
reupload.theme.preview_path, storage=storage)
move_stored_file(
reupload.header_path, reupload.theme.header_path, storage=storage)
theme = reupload.theme
footer_path = theme.footer_path
if reupload.footer_path != footer_path:
if not footer_path:
dst_root = os.path.join(user_media_path('addons'),
str(theme.addon.id))
footer_path = os.path.join(dst_root, 'footer.png')
theme.footer = 'footer.png'
theme.save()
move_stored_file(
reupload.footer_path, footer_path, storage=storage)
rereview.delete()
theme.addon.increment_theme_version_number()
def add_file(self, chunks, filename, size):
if not self.uuid:
self.uuid = self._meta.get_field('uuid')._create_uuid()
filename = force_bytes(u'{0}_{1}'.format(self.uuid.hex, filename))
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
base, ext = os.path.splitext(force_bytes(filename))
is_crx = False
# Change a ZIP to an XPI, to maintain backward compatibility
# with older versions of Firefox and to keep the rest of the XPI code
# path as consistent as possible for ZIP uploads.
# See: https://github.com/mozilla/addons-server/pull/2785
if ext == '.zip':
ext = '.xpi'
# If the extension is a CRX, we need to do some actual work to it
# before we just convert it to an XPI. We strip the header from the
# CRX, then it's good; see more about the CRX file format here:
# https://developer.chrome.com/extensions/crx
if ext == '.crx':
ext = '.xpi'
is_crx = True
if ext in EXTENSIONS:
loc += ext
log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc))
if is_crx:
hash = write_crx_as_xpi(chunks, storage, loc)
else:
hash = hashlib.sha256()
with storage.open(loc, 'wb') as file_destination:
for chunk in chunks:
hash.update(chunk)
file_destination.write(chunk)
self.path = loc
self.name = filename
self.hash = 'sha256:%s' % hash.hexdigest()
self.save()
def approve_rereview(theme):
"""Replace original theme with pending theme on filesystem."""
# If reuploaded theme, replace old theme design.
storage = LocalFileStorage()
rereview = theme.rereviewqueuetheme_set.all()
reupload = rereview[0]
if reupload.header_path != reupload.theme.header_path:
create_persona_preview_images(
src=reupload.header_path,
full_dst=[
reupload.theme.thumb_path,
reupload.theme.icon_path],
set_modified_on=[reupload.theme.addon])
if not reupload.theme.is_new():
# Legacy themes also need a preview_large.jpg.
# Modern themes use preview.png for both thumb and preview so there
# is no problem there.
copy_stored_file(reupload.theme.thumb_path,
reupload.theme.preview_path, storage=storage)
move_stored_file(
reupload.header_path, reupload.theme.header_path, storage=storage)
theme = reupload.theme
footer_path = theme.footer_path
if reupload.footer_path != footer_path:
if not footer_path:
dst_root = os.path.join(user_media_path('addons'),
str(theme.addon.id))
footer_path = os.path.join(dst_root, 'footer.png')
theme.footer = 'footer.png'
theme.save()
move_stored_file(
reupload.footer_path, footer_path, storage=storage)
rereview.delete()
theme.addon.increment_theme_version_number()
def _uploader(resize_size, final_size):
img = get_image_path('mozilla.png')
original_size = (339, 128)
src = tempfile.NamedTemporaryFile(mode='r+w+b',
suffix=".png",
delete=False)
# resize_icon removes the original
shutil.copyfile(img, src.name)
src_image = Image.open(src.name)
assert src_image.size == original_size
if isinstance(final_size, list):
uploadto = user_media_path('addon_icons')
try:
os.makedirs(uploadto)
except OSError:
pass
for rsize, fsize in zip(resize_size, final_size):
dest_name = os.path.join(uploadto, '1234')
tasks.resize_icon(src.name, dest_name, resize_size, locally=True)
dest_image = Image.open(open('%s-%s.png' % (dest_name, rsize)))
assert dest_image.size == fsize
if os.path.exists(dest_image.filename):
os.remove(dest_image.filename)
assert not os.path.exists(dest_image.filename)
shutil.rmtree(uploadto)
else:
dest = tempfile.mktemp(suffix='.png')
tasks.resize_icon(src.name, dest, resize_size, locally=True)
dest_image = Image.open(dest)
assert dest_image.size == final_size
assert not os.path.exists(src.name)
def save_theme_reupload(header, addon_pk, **kw):
addon = Addon.objects.get(pk=addon_pk)
header_dst = None
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
try:
if header:
header = os.path.join(settings.TMP_PATH, 'persona_header', header)
header_dst = os.path.join(dst_root, 'pending_header.png')
save_persona_image(src=header, full_dst=header_dst)
except IOError as e:
log.error(str(e))
raise
if header_dst:
theme = addon.persona
header = 'pending_header.png' if header_dst else theme.header
# Store pending header file paths for review.
RereviewQueueTheme.objects.filter(theme=theme).delete()
rqt = RereviewQueueTheme(theme=theme, header=header)
rereviewqueuetheme_checksum(rqt=rqt)
rqt.save()
def save_theme_reupload(header, addon_pk, **kw):
addon = Addon.objects.get(pk=addon_pk)
header_dst = None
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
try:
if header:
header = os.path.join(settings.TMP_PATH, 'persona_header', header)
header_dst = os.path.join(dst_root, 'pending_header.png')
save_persona_image(src=header, full_dst=header_dst)
except IOError as e:
log.error(str(e))
raise
if header_dst:
theme = addon.persona
header = 'pending_header.png' if header_dst else theme.header
# Store pending header file paths for review.
RereviewQueueTheme.objects.filter(theme=theme).delete()
rqt = RereviewQueueTheme(theme=theme, header=header)
rereviewqueuetheme_checksum(rqt=rqt)
rqt.save()
def _uploader(resize_size, final_size):
img = get_image_path('mozilla.png')
original_size = (339, 128)
src = tempfile.NamedTemporaryFile(mode='r+w+b', suffix=".png",
delete=False)
# resize_icon removes the original
shutil.copyfile(img, src.name)
src_image = Image.open(src.name)
assert src_image.size == original_size
if isinstance(final_size, list):
uploadto = user_media_path('addon_icons')
try:
os.makedirs(uploadto)
except OSError:
pass
for rsize, fsize in zip(resize_size, final_size):
dest_name = os.path.join(uploadto, '1234')
tasks.resize_icon(src.name, dest_name, resize_size, locally=True)
dest_image = Image.open(open('%s-%s.png' % (dest_name, rsize)))
assert dest_image.size == fsize
if os.path.exists(dest_image.filename):
os.remove(dest_image.filename)
assert not os.path.exists(dest_image.filename)
shutil.rmtree(uploadto)
else:
dest = tempfile.mktemp(suffix='.png')
tasks.resize_icon(src.name, dest, resize_size, locally=True)
dest_image = Image.open(dest)
assert dest_image.size == final_size
assert not os.path.exists(src.name)
def path():
# Check file paths / permissions
rw = (settings.TMP_PATH,
settings.MEDIA_ROOT,
user_media_path('addons'),
user_media_path('guarded_addons'),
user_media_path('addon_icons'),
user_media_path('collection_icons'),
user_media_path('previews'),
user_media_path('userpics'),
user_media_path('reviewer_attachments'),
dump_apps.Command.get_json_path(),)
r = [os.path.join(settings.ROOT, 'locale'),
# The deploy process will want write access to this.
# We do not want Django to have write access though.
settings.PROD_DETAILS_DIR]
filepaths = [(path, os.R_OK | os.W_OK, 'We want read + write')
for path in rw]
filepaths += [(path, os.R_OK, 'We want read') for path in r]
filepath_results = []
filepath_status = True
for path, perms, notes in filepaths:
path_exists = os.path.exists(path)
path_perms = os.access(path, perms)
filepath_status = filepath_status and path_exists and path_perms
if not isinstance(path, str):
notes += ' / should be a bytestring!'
filepath_results.append((path, path_exists, path_perms, notes))
status = filepath_status
status = ''
if not filepath_status:
status = 'check main status page for broken perms / values'
return status, filepath_results
def test_with_settings(self):
path = jinja_helpers.user_media_path('addons')
assert path == '/another/path/'
def test_without_settings(self):
del settings.ADDONS_PATH
path = jinja_helpers.user_media_path('addons')
assert path == '/path/addons'
def guarded_file_path(self):
return os.path.join(user_media_path('guarded_addons'),
str(self.version.addon_id), self.filename)
def gc(test_result=True):
"""Site-wide garbage collections."""
def days_ago(days):
return datetime.today() - timedelta(days=days)
log.debug('Collecting data to delete')
logs = (ActivityLog.objects.filter(created__lt=days_ago(90)).exclude(
action__in=amo.LOG_KEEP).values_list('id', flat=True))
# Paypal only keeps retrying to verify transactions for up to 3 days. If we
# still have an unverified transaction after 6 days, we might as well get
# rid of it.
contributions_to_delete = (Contribution.objects.filter(
transaction_id__isnull=True,
created__lt=days_ago(6)).values_list('id', flat=True))
collections_to_delete = (Collection.objects.filter(
created__lt=days_ago(2),
type=amo.COLLECTION_ANONYMOUS).values_list('id', flat=True))
for chunk in chunked(logs, 100):
tasks.delete_logs.delay(chunk)
for chunk in chunked(contributions_to_delete, 100):
tasks.delete_stale_contributions.delay(chunk)
for chunk in chunked(collections_to_delete, 100):
tasks.delete_anonymous_collections.delay(chunk)
# Incomplete addons cannot be deleted here because when an addon is
# rejected during a review it is marked as incomplete. See bug 670295.
log.debug('Cleaning up test results extraction cache.')
# lol at check for '/'
if settings.MEDIA_ROOT and settings.MEDIA_ROOT != '/':
cmd = ('find', settings.MEDIA_ROOT, '-maxdepth', '1', '-name',
'validate-*', '-mtime', '+7', '-type', 'd', '-exec', 'rm',
'-rf', "{}", ';')
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
else:
log.warning('MEDIA_ROOT not defined.')
if user_media_path('collection_icons'):
log.debug('Cleaning up uncompressed icons.')
cmd = ('find', user_media_path('collection_icons'), '-name',
'*__unconverted', '-mtime', '+1', '-type', 'f', '-exec', 'rm',
'{}', ';')
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
USERPICS_PATH = user_media_path('userpics')
if USERPICS_PATH:
log.debug('Cleaning up uncompressed userpics.')
cmd = ('find', USERPICS_PATH, '-name', '*__unconverted', '-mtime',
'+1', '-type', 'f', '-exec', 'rm', '{}', ';')
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
# Delete stale FileUploads.
stale_uploads = FileUpload.objects.filter(
created__lte=days_ago(180)).order_by('id')
for file_upload in stale_uploads:
log.debug(u'[FileUpload:{uuid}] Removing file: {path}'.format(
uuid=file_upload.uuid, path=file_upload.path))
if file_upload.path:
try:
storage.delete(file_upload.path)
except OSError:
pass
file_upload.delete()
def _image_path(self, url_template):
from olympia.amo.templatetags.jinja_helpers import user_media_path
args = [user_media_path(self.media_folder), self.id // 1000, self.id]
return url_template % tuple(args)
def add_static_theme_from_lwt(lwt):
from olympia.activity.models import AddonLog
olympia.core.set_user(UserProfile.objects.get(pk=settings.TASK_USER_ID))
# Try to handle LWT with no authors
author = (lwt.listed_authors or [_get_lwt_default_author()])[0]
# Wrap zip in FileUpload for Addon/Version from_upload to consume.
upload = FileUpload.objects.create(user=author, valid=True)
destination = os.path.join(user_media_path('addons'), 'temp',
uuid.uuid4().hex + '.xpi')
build_static_theme_xpi_from_lwt(lwt, destination)
upload.update(path=destination)
# Create addon + version
parsed_data = parse_addon(upload, user=author)
addon = Addon.initialize_addon_from_upload(parsed_data, upload,
amo.RELEASE_CHANNEL_LISTED,
author)
addon_updates = {}
# static themes are only compatible with Firefox at the moment,
# not Android
version = Version.from_upload(upload,
addon,
selected_apps=[amo.FIREFOX.id],
channel=amo.RELEASE_CHANNEL_LISTED,
parsed_data=parsed_data)
# Set category
static_theme_categories = CATEGORIES.get(amo.FIREFOX.id,
[]).get(amo.ADDON_STATICTHEME, [])
lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat.
lwt_category_slug = lwt_category.slug if lwt_category else 'other'
static_category = static_theme_categories.get(
lwt_category_slug, static_theme_categories.get('other'))
AddonCategory.objects.create(addon=addon,
category=Category.from_static_category(
static_category, True))
# Set license
lwt_license = PERSONA_LICENSES_IDS.get(
lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright
static_license = License.objects.get(builtin=lwt_license.builtin)
version.update(license=static_license)
# Set tags
for addon_tag in AddonTag.objects.filter(addon=lwt):
AddonTag.objects.create(addon=addon, tag=addon_tag.tag)
# Steal the ratings (even with soft delete they'll be deleted anyway)
addon_updates.update(average_rating=lwt.average_rating,
bayesian_rating=lwt.bayesian_rating,
total_ratings=lwt.total_ratings,
text_ratings_count=lwt.text_ratings_count)
Rating.unfiltered.filter(addon=lwt).update(addon=addon, version=version)
# Modify the activity log entry too.
rating_activity_log_ids = [
l.id for l in amo.LOG if getattr(l, 'action_class', '') == 'review'
]
addonlog_qs = AddonLog.objects.filter(
addon=lwt, activity_log__action__in=rating_activity_log_ids)
[alog.transfer(addon) for alog in addonlog_qs.iterator()]
# Copy the ADU statistics - the raw(ish) daily UpdateCounts for stats
# dashboard and future update counts, and copy the summary numbers for now.
migrate_theme_update_count(lwt, addon)
addon_updates.update(average_daily_users=lwt.persona.popularity or 0,
hotness=lwt.persona.movers or 0)
# Logging
activity.log_create(amo.LOG.CREATE_STATICTHEME_FROM_PERSONA,
addon,
user=author)
# And finally sign the files (actually just one)
for file_ in version.all_files:
sign_file(file_)
file_.update(datestatuschanged=lwt.last_updated,
reviewed=datetime.now(),
status=amo.STATUS_PUBLIC)
addon_updates['status'] = amo.STATUS_PUBLIC
# set the modified and creation dates to match the original.
addon_updates['created'] = lwt.created
addon_updates['modified'] = lwt.modified
addon_updates['last_updated'] = lwt.last_updated
addon.update(**addon_updates)
return addon
def path_prefix(self):
return os.path.join(user_media_path('addons'), str(self.addon_id))
def add_static_theme_from_lwt(lwt):
from olympia.activity.models import AddonLog
olympia.core.set_user(UserProfile.objects.get(pk=settings.TASK_USER_ID))
# Try to handle LWT with no authors
author = (lwt.listed_authors or [_get_lwt_default_author()])[0]
# Wrap zip in FileUpload for Addon/Version from_upload to consume.
upload = FileUpload.objects.create(
user=author, valid=True)
destination = os.path.join(
user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi')
build_static_theme_xpi_from_lwt(lwt, destination)
upload.update(path=destination)
# Create addon + version
parsed_data = parse_addon(upload, user=author)
addon = Addon.initialize_addon_from_upload(
parsed_data, upload, amo.RELEASE_CHANNEL_LISTED, author)
addon_updates = {}
# static themes are only compatible with Firefox at the moment,
# not Android
version = Version.from_upload(
upload, addon, selected_apps=[amo.FIREFOX.id],
channel=amo.RELEASE_CHANNEL_LISTED,
parsed_data=parsed_data)
# Set category
static_theme_categories = CATEGORIES.get(amo.FIREFOX.id, []).get(
amo.ADDON_STATICTHEME, [])
lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat.
lwt_category_slug = lwt_category.slug if lwt_category else 'other'
static_category = static_theme_categories.get(
lwt_category_slug, static_theme_categories.get('other'))
AddonCategory.objects.create(
addon=addon,
category=Category.from_static_category(static_category, True))
# Set license
lwt_license = PERSONA_LICENSES_IDS.get(
lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright
static_license = License.objects.get(builtin=lwt_license.builtin)
version.update(license=static_license)
# Set tags
for addon_tag in AddonTag.objects.filter(addon=lwt):
AddonTag.objects.create(addon=addon, tag=addon_tag.tag)
# Steal the ratings (even with soft delete they'll be deleted anyway)
addon_updates.update(
average_rating=lwt.average_rating,
bayesian_rating=lwt.bayesian_rating,
total_ratings=lwt.total_ratings,
text_ratings_count=lwt.text_ratings_count)
Rating.unfiltered.filter(addon=lwt).update(addon=addon, version=version)
# Modify the activity log entry too.
rating_activity_log_ids = [
l.id for l in amo.LOG if getattr(l, 'action_class', '') == 'review']
addonlog_qs = AddonLog.objects.filter(
addon=lwt, activity_log__action__in=rating_activity_log_ids)
[alog.transfer(addon) for alog in addonlog_qs.iterator()]
# Copy the ADU statistics - the raw(ish) daily UpdateCounts for stats
# dashboard and future update counts, and copy the summary numbers for now.
migrate_theme_update_count(lwt, addon)
addon_updates.update(
average_daily_users=lwt.persona.popularity or 0,
hotness=lwt.persona.movers or 0)
# Logging
activity.log_create(
amo.LOG.CREATE_STATICTHEME_FROM_PERSONA, addon, user=author)
# And finally sign the files (actually just one)
for file_ in version.all_files:
sign_file(file_)
file_.update(
datestatuschanged=lwt.last_updated,
reviewed=datetime.now(),
status=amo.STATUS_PUBLIC)
addon_updates['status'] = amo.STATUS_PUBLIC
# set the modified and creation dates to match the original.
addon_updates['created'] = lwt.created
addon_updates['modified'] = lwt.modified
addon_updates['last_updated'] = lwt.last_updated
addon.update(**addon_updates)
return addon
def picture_dir(self):
from olympia.amo.templatetags.jinja_helpers import user_media_path
split_id = re.match(r'((\d*?)(\d{0,3}?))\d{1,3}$', str(self.id))
return os.path.join(user_media_path('userpics'),
split_id.group(2) or '0',
split_id.group(1) or '0')
def gc(test_result=True):
"""Site-wide garbage collections."""
def days_ago(days):
return datetime.today() - timedelta(days=days)
log.debug('Collecting data to delete')
logs = (ActivityLog.objects.filter(created__lt=days_ago(90))
.exclude(action__in=amo.LOG_KEEP).values_list('id', flat=True))
# Paypal only keeps retrying to verify transactions for up to 3 days. If we
# still have an unverified transaction after 6 days, we might as well get
# rid of it.
contributions_to_delete = (
Contribution.objects
.filter(transaction_id__isnull=True, created__lt=days_ago(6))
.values_list('id', flat=True))
collections_to_delete = (
Collection.objects.filter(created__lt=days_ago(2),
type=amo.COLLECTION_ANONYMOUS)
.values_list('id', flat=True))
for chunk in chunked(logs, 100):
tasks.delete_logs.delay(chunk)
for chunk in chunked(contributions_to_delete, 100):
tasks.delete_stale_contributions.delay(chunk)
for chunk in chunked(collections_to_delete, 100):
tasks.delete_anonymous_collections.delay(chunk)
# Incomplete addons cannot be deleted here because when an addon is
# rejected during a review it is marked as incomplete. See bug 670295.
log.debug('Cleaning up test results extraction cache.')
# lol at check for '/'
if settings.MEDIA_ROOT and settings.MEDIA_ROOT != '/':
cmd = ('find', settings.MEDIA_ROOT, '-maxdepth', '1', '-name',
'validate-*', '-mtime', '+7', '-type', 'd',
'-exec', 'rm', '-rf', "{}", ';')
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
else:
log.warning('MEDIA_ROOT not defined.')
if user_media_path('collection_icons'):
log.debug('Cleaning up uncompressed icons.')
cmd = ('find', user_media_path('collection_icons'),
'-name', '*__unconverted', '-mtime', '+1', '-type', 'f',
'-exec', 'rm', '{}', ';')
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
USERPICS_PATH = user_media_path('userpics')
if USERPICS_PATH:
log.debug('Cleaning up uncompressed userpics.')
cmd = ('find', USERPICS_PATH,
'-name', '*__unconverted', '-mtime', '+1', '-type', 'f',
'-exec', 'rm', '{}', ';')
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
# Delete stale FileUploads.
stale_uploads = FileUpload.objects.filter(
created__lte=days_ago(180)).order_by('id')
for file_upload in stale_uploads:
log.debug(u'[FileUpload:{uuid}] Removing file: {path}'
.format(uuid=file_upload.uuid, path=file_upload.path))
if file_upload.path:
try:
storage.delete(file_upload.path)
except OSError:
pass
file_upload.delete()
def guarded_file_path(self):
return os.path.join(user_media_path('guarded_addons'),
str(self.version.addon_id), self.filename)
def get_img_dir(self):
return os.path.join(user_media_path('collection_icons'),
str(self.id / 1000))
def path_prefix(self):
return os.path.join(user_media_path('addons'), str(self.addon_id))
def from_upload(cls, upload, addon, platforms, channel,
source=None, parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of platform ids, a channel id and the
parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
assert parsed_data is not None
from olympia.addons.models import AddonFeatureCompatibility
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(
channel=channel, exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
version = cls.objects.create(
addon=addon,
version=parsed_data['version'],
license_id=license_id,
source=source,
channel=channel,
)
log.info(
'New version: %r (%s) from %r' % (version, version.id, upload))
activity.log_create(amo.LOG.ADD_VERSION, version, addon)
# Update the add-on e10s compatibility since we're creating a new
# version that may change that.
e10s_compatibility = parsed_data.get('e10s_compatibility')
if e10s_compatibility is not None:
feature_compatibility = (
AddonFeatureCompatibility.objects.get_or_create(addon=addon)[0]
)
feature_compatibility.update(e10s=e10s_compatibility)
compatible_apps = {}
for app in parsed_data.get('apps', []):
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# See #2828: sometimes when we generate the filename(s) below, in
# File.from_upload(), cache-machine is confused and has trouble
# fetching the ApplicationsVersions that were just created. To work
# around this we pre-generate version.compatible_apps and avoid the
# queries completely.
version._compatible_apps = compatible_apps
if addon.type in [amo.ADDON_SEARCH, amo.ADDON_STATICTHEME]:
# Search extensions and static themes are always for all platforms.
platforms = [amo.PLATFORM_ALL.id]
else:
platforms = cls._make_safe_platform_files(platforms)
# Create as many files as we have platforms. Update the all_files
# cached property on the Version while we're at it, because we might
# need it afterwards.
version.all_files = [
File.from_upload(
upload, version, platform, parsed_data=parsed_data)
for platform in platforms]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to all platforms, remove the upload.
storage.delete(upload.path)
version_uploaded.send(sender=version)
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME and
channel == amo.RELEASE_CHANNEL_LISTED):
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
theme_data = parsed_data.get('theme', {})
version_root = os.path.join(dst_root, unicode(version.id))
utils.extract_header_img(
version.all_files[0].file_path, theme_data, version_root)
preview = VersionPreview.objects.create(version=version)
generate_static_theme_preview(
theme_data, version_root, preview.pk)
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'
.format(delta=upload_time, version=version,
created=upload.created, now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
