def save_theme_reupload(header, footer, addon, **kw):
header_dst = None
footer_dst = None
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
try:
if header:
header = os.path.join(settings.TMP_PATH, 'persona_header', header)
header_dst = os.path.join(dst_root, 'pending_header.png')
save_persona_image(src=header, full_dst=header_dst)
if footer:
footer = os.path.join(settings.TMP_PATH, 'persona_footer', footer)
footer_dst = os.path.join(dst_root, 'pending_footer.png')
save_persona_image(src=footer, full_dst=footer_dst)
except IOError as e:
log.error(str(e))
raise
if header_dst or footer_dst:
theme = addon.persona
header = 'pending_header.png' if header_dst else theme.header
# Theme footer is optional, but can't be None.
footer = theme.footer or ''
if footer_dst:
footer = 'pending_footer.png'
# Store pending header and/or footer file paths for review.
RereviewQueueTheme.objects.filter(theme=theme).delete()
rqt = RereviewQueueTheme(theme=theme, header=header, footer=footer)
rereviewqueuetheme_checksum(rqt=rqt)
rqt.save()
def test_edit_media_uploadedicon_noresize(self):
img = "static/img/notifications/error.png"
src_image = open(img, "rb")
data = dict(upload_image=src_image)
response = self.client.post(self.icon_upload, data)
response_json = json.loads(response.content)
addon = self.get_addon()
# Now, save the form so it gets moved properly.
data = dict(icon_type="image/png", icon_upload_hash=response_json["upload_hash"])
data_formset = self.formset_media(**data)
r = self.client.post(self.media_edit_url, data_formset)
eq_(r.context["form"].errors, {})
addon = self.get_addon()
# Unfortunate hardcoding of URL
addon_url = addon.get_icon_url(64).split("?")[0]
assert addon_url.endswith("addon_icons/3/%s-64.png" % addon.id), "Unexpected path: %r" % addon_url
eq_(data["icon_type"], "image/png")
# Check that it was actually uploaded
dirname = os.path.join(user_media_path("addon_icons"), "%s" % (addon.id / 1000))
dest = os.path.join(dirname, "%s-64.png" % addon.id)
assert storage.exists(dest)
eq_(Image.open(storage.open(dest)).size, (48, 48))
def rezip_file(response, pk):
# An .xpi does not have a directory inside the zip, yet zips from github
# do, so we'll need to rezip the file before passing it through to the
# validator.
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
old_filename = '{}_github_webhook.zip'.format(pk)
old_path = os.path.join(loc, old_filename)
with storage.open(old_path, 'wb') as old:
old.write(response.content)
new_filename = '{}_github_webhook.xpi'.format(pk)
new_path = os.path.join(loc, new_filename)
old_zip = SafeUnzip(old_path)
if not old_zip.is_valid():
raise
with storage.open(new_path, 'w') as new:
new_zip = zipfile.ZipFile(new, 'w')
for obj in old_zip.filelist:
# Basically strip off the leading directory.
new_filename = obj.filename.partition('/')[-1]
if not new_filename:
continue
new_zip.writestr(new_filename, old_zip.read(obj.filename))
new_zip.close()
old_zip.close()
return new_path
def add_file(self, chunks, filename, size):
if not self.uuid:
self.uuid = self._meta.get_field('uuid')._create_uuid().hex
filename = smart_str(u'{0}_{1}'.format(self.uuid, filename))
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
base, ext = os.path.splitext(smart_path(filename))
# Change a ZIP to an XPI, to maintain backward compatibility
# with older versions of Firefox and to keep the rest of the XPI code
# path as consistent as possible for ZIP uploads.
# See: https://github.com/mozilla/addons-server/pull/2785
if ext == '.zip':
ext = '.xpi'
if ext in EXTENSIONS:
loc += ext
log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc))
hash = hashlib.sha256()
with storage.open(loc, 'wb') as fd:
for chunk in chunks:
hash.update(chunk)
fd.write(chunk)
self.path = loc
self.name = filename
self.hash = 'sha256:%s' % hash.hexdigest()
self.save()
def test_reupload(self, save_persona_image_mock,
create_persona_preview_images_mock,
make_checksum_mock):
make_checksum_mock.return_value = 'checksumbeforeyouwrecksome'
data = self.get_dict(header_hash='y0l0', footer_hash='abab')
self.form = EditThemeForm(data, request=self.request,
instance=self.instance)
assert self.form.is_valid()
self.form.save()
dst = os.path.join(user_media_path('addons'), str(self.instance.id))
header_src = os.path.join(settings.TMP_PATH, 'persona_header',
u'y0l0')
footer_src = os.path.join(settings.TMP_PATH, 'persona_footer',
u'abab')
assert save_persona_image_mock.mock_calls == (
[mock.call(src=header_src,
full_dst=os.path.join(dst, 'pending_header.png')),
mock.call(src=footer_src,
full_dst=os.path.join(dst, 'pending_footer.png'))])
rqt = RereviewQueueTheme.objects.filter(theme=self.instance.persona)
assert rqt.count() == 1
assert rqt[0].header == 'pending_header.png'
assert rqt[0].footer == 'pending_footer.png'
assert not rqt[0].dupe_persona
def path():
# Check file paths / permissions
rw = (settings.TMP_PATH,
settings.MEDIA_ROOT,
user_media_path('addons'),
user_media_path('guarded_addons'),
user_media_path('addon_icons'),
user_media_path('collection_icons'),
user_media_path('previews'),
user_media_path('userpics'),
user_media_path('reviewer_attachments'),
dump_apps.Command.JSON_PATH,)
r = [os.path.join(settings.ROOT, 'locale'),
# The deploy process will want write access to this.
# We do not want Django to have write access though.
settings.PROD_DETAILS_DIR]
filepaths = [(path, os.R_OK | os.W_OK, "We want read + write")
for path in rw]
filepaths += [(path, os.R_OK, "We want read") for path in r]
filepath_results = []
filepath_status = True
for path, perms, notes in filepaths:
path_exists = os.path.exists(path)
path_perms = os.access(path, perms)
filepath_status = filepath_status and path_exists and path_perms
filepath_results.append((path, path_exists, path_perms, notes))
status = filepath_status
status = ''
if not filepath_status:
status = 'check main status page for broken perms'
return status, filepath_results
def delete_persona_image(dst, **kw):
log.info('[1@None] Deleting persona image: %s.' % dst)
if not dst.startswith(user_media_path('addons')):
log.error("Someone tried deleting something they shouldn't: %s" % dst)
return
try:
storage.delete(dst)
except Exception, e:
log.error('Error deleting persona image: %s' % e)
def delete_icon(dst, **kw):
log.info('[1@None] Deleting icon: %s.' % dst)
if not dst.startswith(user_media_path('collection_icons')):
log.error("Someone tried deleting something they shouldn't: %s" % dst)
return
try:
storage.delete(dst)
except Exception, e:
log.error("Error deleting icon: %s" % e)
def delete_photo(dst, **kw):
task_log.debug('[1@None] Deleting photo: %s.' % dst)
if not dst.startswith(user_media_path('userpics')):
task_log.error("Someone tried deleting something they shouldn't: %s"
% dst)
return
try:
storage.delete(dst)
except Exception, e:
task_log.error("Error deleting userpic: %s" % e)
def add_file(self, chunks, filename, size):
if not self.uuid:
self.uuid = self._meta.get_field('uuid')._create_uuid().hex
filename = smart_str(u'{0}_{1}'.format(self.uuid, filename))
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
base, ext = os.path.splitext(smart_path(filename))
if ext in EXTENSIONS:
loc += ext
log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc))
hash = hashlib.sha256()
with storage.open(loc, 'wb') as fd:
for chunk in chunks:
hash.update(chunk)
fd.write(chunk)
self.path = loc
self.name = filename
self.hash = 'sha256:%s' % hash.hexdigest()
self.save()
def create_theme_images(theme, placement, hash_):
"""
Generates 2 images, one in the temp folder and the other in the
user-media one. Both are needed to generate previews for themes.
"""
color = random.choice(ImageColor.colormap.keys())
image = Image.new('RGB', (3000, 200), color)
tmp_path = os.path.join(settings.TMP_PATH,
'persona_{placement}'.format(placement=placement))
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
tmp_loc = os.path.join(tmp_path, hash_)
image.save(tmp_loc, 'jpeg')
media_path = os.path.join(user_media_path('addons'), str(theme.id))
if not os.path.exists(media_path):
os.makedirs(media_path)
media_loc = os.path.join(media_path, hash_)
image.save(media_loc, 'jpeg')
def _save_file(self, version):
data = self.cleaned_data
xpi = data['xpi']
hash = hashlib.sha256()
f = File(version=version,
platform=amo.PLATFORM_DICT[data['platform']].id,
size=xpi.size)
filename = f.generate_filename()
path = os.path.join(user_media_path('addons'), str(version.addon.id))
with storage.open(os.path.join(path, filename), 'wb') as destination:
for chunk in xpi.chunks():
hash.update(chunk)
destination.write(chunk)
f.hash = 'sha256:%s' % hash.hexdigest()
f.save()
return f
def test_reupload_no_footer(self, save_persona_image_mock, create_persona_preview_images_mock, make_checksum_mock):
make_checksum_mock.return_value = "checksumbeforeyouwrecksome"
data = self.get_dict(header_hash="y0l0", footer_hash="")
self.form = EditThemeForm(data, request=self.request, instance=self.instance)
assert self.form.is_valid()
self.form.save()
dst = os.path.join(user_media_path("addons"), str(self.instance.id))
header_src = os.path.join(settings.TMP_PATH, "persona_header", u"y0l0")
assert save_persona_image_mock.mock_calls == (
[mock.call(src=header_src, full_dst=os.path.join(dst, "pending_header.png"))]
)
rqt = RereviewQueueTheme.objects.filter(theme=self.instance.persona)
assert rqt.count() == 1
assert rqt[0].header == "pending_header.png"
assert rqt[0].footer == ""
assert not rqt[0].dupe_persona
def save_theme(header, footer, addon, **kw):
"""Save theme image and calculates checksum after theme save."""
dst_root = os.path.join(user_media_path('addons'), str(addon.id))
header = os.path.join(settings.TMP_PATH, 'persona_header', header)
header_dst = os.path.join(dst_root, 'header.png')
if footer:
footer = os.path.join(settings.TMP_PATH, 'persona_footer', footer)
footer_dst = os.path.join(dst_root, 'footer.png')
try:
save_persona_image(src=header, full_dst=header_dst)
if footer:
save_persona_image(src=footer, full_dst=footer_dst)
create_persona_preview_images(
src=header, full_dst=[os.path.join(dst_root, 'preview.png'),
os.path.join(dst_root, 'icon.png')],
set_modified_on=[addon])
theme_checksum(addon.persona)
except IOError:
addon.delete()
raise
def approve_rereview(theme):
"""Replace original theme with pending theme on filesystem."""
# If reuploaded theme, replace old theme design.
storage = LocalFileStorage()
rereview = theme.rereviewqueuetheme_set.all()
reupload = rereview[0]
if reupload.header_path != reupload.theme.header_path:
create_persona_preview_images(
src=reupload.header_path,
full_dst=[
reupload.theme.thumb_path,
reupload.theme.icon_path],
set_modified_on=[reupload.theme.addon])
if not reupload.theme.is_new():
# Legacy themes also need a preview_large.jpg.
# Modern themes use preview.png for both thumb and preview so there
# is no problem there.
copy_stored_file(reupload.theme.thumb_path,
reupload.theme.preview_path, storage=storage)
move_stored_file(
reupload.header_path, reupload.theme.header_path, storage=storage)
theme = reupload.theme
footer_path = theme.footer_path
if reupload.footer_path != footer_path:
if not footer_path:
dst_root = os.path.join(user_media_path('addons'),
str(theme.addon.id))
footer_path = os.path.join(dst_root, 'footer.png')
theme.footer = 'footer.png'
theme.save()
move_stored_file(
reupload.footer_path, footer_path, storage=storage)
rereview.delete()
theme.addon.increment_version()
def add_file(self, chunks, filename, size):
if not self.uuid:
self.uuid = self._meta.get_field('uuid')._create_uuid()
filename = force_bytes(u'{0}_{1}'.format(self.uuid.hex, filename))
loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
base, ext = os.path.splitext(force_bytes(filename))
is_crx = False
# Change a ZIP to an XPI, to maintain backward compatibility
# with older versions of Firefox and to keep the rest of the XPI code
# path as consistent as possible for ZIP uploads.
# See: https://github.com/mozilla/addons-server/pull/2785
if ext == '.zip':
ext = '.xpi'
# If the extension is a CRX, we need to do some actual work to it
# before we just convert it to an XPI. We strip the header from the
# CRX, then it's good; see more about the CRX file format here:
# https://developer.chrome.com/extensions/crx
if ext == '.crx':
ext = '.xpi'
is_crx = True
if ext in EXTENSIONS:
loc += ext
log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc))
if is_crx:
hash = write_crx_as_xpi(chunks, storage, loc)
else:
hash = hashlib.sha256()
with storage.open(loc, 'wb') as file_destination:
for chunk in chunks:
hash.update(chunk)
file_destination.write(chunk)
self.path = loc
self.name = filename
self.hash = 'sha256:%s' % hash.hexdigest()
self.save()
def _uploader(resize_size, final_size):
img = get_image_path('mozilla.png')
original_size = (339, 128)
src = tempfile.NamedTemporaryFile(mode='r+w+b', suffix=".png",
delete=False)
# resize_icon removes the original
shutil.copyfile(img, src.name)
src_image = Image.open(src.name)
assert src_image.size == original_size
if isinstance(final_size, list):
uploadto = user_media_path('addon_icons')
try:
os.makedirs(uploadto)
except OSError:
pass
for rsize, fsize in zip(resize_size, final_size):
dest_name = os.path.join(uploadto, '1234')
tasks.resize_icon(src.name, dest_name, resize_size, locally=True)
dest_image = Image.open(open('%s-%s.png' % (dest_name, rsize)))
assert dest_image.size == fsize
if os.path.exists(dest_image.filename):
os.remove(dest_image.filename)
assert not os.path.exists(dest_image.filename)
shutil.rmtree(uploadto)
else:
dest = tempfile.mktemp(suffix='.png')
tasks.resize_icon(src.name, dest, resize_size, locally=True)
dest_image = Image.open(dest)
assert dest_image.size == final_size
assert not os.path.exists(src.name)
def path():
# Check file paths / permissions
rw = (
settings.TMP_PATH,
settings.MEDIA_ROOT,
user_media_path('addons'),
user_media_path('guarded_addons'),
user_media_path('addon_icons'),
user_media_path('collection_icons'),
user_media_path('previews'),
user_media_path('userpics'),
user_media_path('reviewer_attachments'),
dump_apps.Command.JSON_PATH,
)
r = [
os.path.join(settings.ROOT, 'locale'),
# The deploy process will want write access to this.
# We do not want Django to have write access though.
settings.PROD_DETAILS_DIR
]
filepaths = [(path, os.R_OK | os.W_OK, "We want read + write")
for path in rw]
filepaths += [(path, os.R_OK, "We want read") for path in r]
filepath_results = []
filepath_status = True
for path, perms, notes in filepaths:
path_exists = os.path.exists(path)
path_perms = os.access(path, perms)
filepath_status = filepath_status and path_exists and path_perms
filepath_results.append((path, path_exists, path_perms, notes))
status = filepath_status
status = ''
if not filepath_status:
status = 'check main status page for broken perms'
return status, filepath_results
def guarded_file_path(self):
return os.path.join(user_media_path('guarded_addons'),
str(self.version.addon_id), self.filename)
def mirror_file_path(self):
return os.path.join(user_media_path('addons'),
str(self.version.addon_id), self.filename)
def test_without_settings(self):
del settings.ADDONS_PATH
path = helpers.user_media_path('addons')
eq_(path, '/path/addons')
def test_with_settings(self):
path = helpers.user_media_path('addons')
assert path == '/another/path/'
def mirror_path_prefix(self):
return os.path.join(user_media_path('addons'), str(self.addon_id))
def test_success(self, save_persona_image_mock,
create_persona_preview_images_mock, make_checksum_mock):
make_checksum_mock.return_value = 'hashyourselfbeforeyoucrashyourself'
self.request.user = UserProfile.objects.get(pk=2519)
data = self.get_dict()
header_url, footer_url = self.get_img_urls()
# Upload header image.
img = open(get_image_path('persona-header.jpg'), 'rb')
r_ajax = self.client.post(header_url, {'upload_image': img})
data.update(header_hash=json.loads(r_ajax.content)['upload_hash'])
# Upload footer image.
img = open(get_image_path('persona-footer.jpg'), 'rb')
r_ajax = self.client.post(footer_url, {'upload_image': img})
data.update(footer_hash=json.loads(r_ajax.content)['upload_hash'])
# Populate and save form.
self.post()
assert self.form.is_valid(), self.form.errors
self.form.save()
addon = Addon.objects.filter(type=amo.ADDON_PERSONA).order_by('-id')[0]
persona = addon.persona
# Test for correct Addon and Persona values.
assert unicode(addon.name) == data['name']
assert addon.slug == data['slug']
self.assertSetEqual(set(addon.categories.values_list('id', flat=True)),
{self.cat.id})
self.assertSetEqual(set(addon.tags.values_list('tag_text', flat=True)),
set(data['tags'].split(', ')))
assert persona.persona_id == 0
assert persona.license == data['license']
assert persona.accentcolor == data['accentcolor'].lstrip('#')
assert persona.textcolor == data['textcolor'].lstrip('#')
assert persona.author == self.request.user.username
assert persona.display_username == self.request.user.name
assert not persona.dupe_persona
v = addon.versions.all()
assert len(v) == 1
assert v[0].version == '0'
# Test for header, footer, and preview images.
dst = os.path.join(user_media_path('addons'), str(addon.id))
header_src = os.path.join(settings.TMP_PATH, 'persona_header',
u'b4ll1n')
footer_src = os.path.join(settings.TMP_PATH, 'persona_footer',
u'5w4g')
assert save_persona_image_mock.mock_calls == (
[mock.call(src=header_src,
full_dst=os.path.join(dst, 'header.png')),
mock.call(src=footer_src,
full_dst=os.path.join(dst, 'footer.png'))])
create_persona_preview_images_mock.assert_called_with(
src=header_src,
full_dst=[os.path.join(dst, 'preview.png'),
os.path.join(dst, 'icon.png')],
set_modified_on=[addon])
def mirror_file_path(self):
return os.path.join(user_media_path('addons'),
str(self.version.addon_id), self.filename)
def handle(self, *args, **kw):
z = ((user_media_path('collection_icons'), resize_icon),
(user_media_path('userpics'), resize_photo))
for base, task in z:
self.fix(base, task)
def picture_dir(self):
from olympia.amo.helpers import user_media_path
split_id = re.match(r'((\d*?)(\d{0,3}?))\d{1,3}$', str(self.id))
return os.path.join(user_media_path('userpics'),
split_id.group(2) or '0',
split_id.group(1) or '0')
def test_with_settings(self):
path = helpers.user_media_path('addons')
eq_(path, '/another/path/')
def test_with_settings(self):
path = helpers.user_media_path('addons')
eq_(path, '/another/path/')
def gc(test_result=True):
"""Site-wide garbage collections."""
def days_ago(days):
return datetime.today() - timedelta(days=days)
log.debug("Collecting data to delete")
logs = (
ActivityLog.objects.filter(created__lt=days_ago(90))
.exclude(action__in=amo.LOG_KEEP)
.values_list("id", flat=True)
)
# Paypal only keeps retrying to verify transactions for up to 3 days. If we
# still have an unverified transaction after 6 days, we might as well get
# rid of it.
contributions_to_delete = Contribution.objects.filter(
transaction_id__isnull=True, created__lt=days_ago(6)
).values_list("id", flat=True)
collections_to_delete = Collection.objects.filter(
created__lt=days_ago(2), type=amo.COLLECTION_ANONYMOUS
).values_list("id", flat=True)
for chunk in chunked(logs, 100):
tasks.delete_logs.delay(chunk)
for chunk in chunked(contributions_to_delete, 100):
tasks.delete_stale_contributions.delay(chunk)
for chunk in chunked(collections_to_delete, 100):
tasks.delete_anonymous_collections.delay(chunk)
# Incomplete addons cannot be deleted here because when an addon is
# rejected during a review it is marked as incomplete. See bug 670295.
log.debug("Cleaning up test results extraction cache.")
# lol at check for '/'
if settings.MEDIA_ROOT and settings.MEDIA_ROOT != "/":
cmd = (
"find",
settings.MEDIA_ROOT,
"-maxdepth",
"1",
"-name",
"validate-*",
"-mtime",
"+7",
"-type",
"d",
"-exec",
"rm",
"-rf",
"{}",
";",
)
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
else:
log.warning("MEDIA_ROOT not defined.")
if user_media_path("collection_icons"):
log.debug("Cleaning up uncompressed icons.")
cmd = (
"find",
user_media_path("collection_icons"),
"-name",
"*__unconverted",
"-mtime",
"+1",
"-type",
"f",
"-exec",
"rm",
"{}",
";",
)
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
USERPICS_PATH = user_media_path("userpics")
if USERPICS_PATH:
log.debug("Cleaning up uncompressed userpics.")
cmd = ("find", USERPICS_PATH, "-name", "*__unconverted", "-mtime", "+1", "-type", "f", "-exec", "rm", "{}", ";")
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
def handle(self, *args, **kw):
z = ((user_media_path('collection_icons'),
resize_icon),
(user_media_path('userpics'), resize_photo))
for base, task in z:
self.fix(base, task)
def get_img_dir(self):
return os.path.join(user_media_path('collection_icons'),
str(self.id / 1000))
def get_img_dir(self):
return os.path.join(user_media_path('collection_icons'),
str(self.id / 1000))
def picture_dir(self):
from olympia.amo.helpers import user_media_path
split_id = re.match(r'((\d*?)(\d{0,3}?))\d{1,3}$', str(self.id))
return os.path.join(user_media_path('userpics'),
split_id.group(2) or '0',
split_id.group(1) or '0')
def test_without_settings(self):
del settings.ADDONS_PATH
path = helpers.user_media_path('addons')
assert path == '/path/addons'
def path_prefix(self):
return os.path.join(user_media_path('addons'), str(self.addon_id))
def test_with_settings(self):
path = helpers.user_media_path('addons')
assert path == '/another/path/'
def guarded_file_path(self):
return os.path.join(user_media_path('guarded_addons'),
str(self.version.addon_id), self.filename)
def test_image_path(self):
up = self.get_update('en-US', 15663)
up.get_update()
image_path = up.image_path('foo.png')
assert user_media_path('addons') in image_path
def test_image_path(self):
up = self.get_update('en-US', 15663)
up.get_update()
image_path = up.image_path('foo.png')
assert user_media_path('addons') in image_path
def gc(test_result=True):
"""Site-wide garbage collections."""
def days_ago(days):
return datetime.today() - timedelta(days=days)
log.debug('Collecting data to delete')
logs = (ActivityLog.objects.filter(created__lt=days_ago(90))
.exclude(action__in=amo.LOG_KEEP).values_list('id', flat=True))
# Paypal only keeps retrying to verify transactions for up to 3 days. If we
# still have an unverified transaction after 6 days, we might as well get
# rid of it.
contributions_to_delete = (
Contribution.objects
.filter(transaction_id__isnull=True, created__lt=days_ago(6))
.values_list('id', flat=True))
collections_to_delete = (
Collection.objects.filter(created__lt=days_ago(2),
type=amo.COLLECTION_ANONYMOUS)
.values_list('id', flat=True))
for chunk in chunked(logs, 100):
tasks.delete_logs.delay(chunk)
for chunk in chunked(contributions_to_delete, 100):
tasks.delete_stale_contributions.delay(chunk)
for chunk in chunked(collections_to_delete, 100):
tasks.delete_anonymous_collections.delay(chunk)
# Incomplete addons cannot be deleted here because when an addon is
# rejected during a review it is marked as incomplete. See bug 670295.
log.debug('Cleaning up test results extraction cache.')
# lol at check for '/'
if settings.MEDIA_ROOT and settings.MEDIA_ROOT != '/':
cmd = ('find', settings.MEDIA_ROOT, '-maxdepth', '1', '-name',
'validate-*', '-mtime', '+7', '-type', 'd',
'-exec', 'rm', '-rf', "{}", ';')
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
else:
log.warning('MEDIA_ROOT not defined.')
if user_media_path('collection_icons'):
log.debug('Cleaning up uncompressed icons.')
cmd = ('find', user_media_path('collection_icons'),
'-name', '*__unconverted', '-mtime', '+1', '-type', 'f',
'-exec', 'rm', '{}', ';')
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
USERPICS_PATH = user_media_path('userpics')
if USERPICS_PATH:
log.debug('Cleaning up uncompressed userpics.')
cmd = ('find', USERPICS_PATH,
'-name', '*__unconverted', '-mtime', '+1', '-type', 'f',
'-exec', 'rm', '{}', ';')
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
def gc(test_result=True):
"""Site-wide garbage collections."""
def days_ago(days):
return datetime.today() - timedelta(days=days)
log.debug('Collecting data to delete')
logs = (ActivityLog.objects.filter(created__lt=days_ago(90))
.exclude(action__in=amo.LOG_KEEP).values_list('id', flat=True))
# Paypal only keeps retrying to verify transactions for up to 3 days. If we
# still have an unverified transaction after 6 days, we might as well get
# rid of it.
contributions_to_delete = (
Contribution.objects
.filter(transaction_id__isnull=True, created__lt=days_ago(6))
.values_list('id', flat=True))
collections_to_delete = (
Collection.objects.filter(created__lt=days_ago(2),
type=amo.COLLECTION_ANONYMOUS)
.values_list('id', flat=True))
for chunk in chunked(logs, 100):
tasks.delete_logs.delay(chunk)
for chunk in chunked(contributions_to_delete, 100):
tasks.delete_stale_contributions.delay(chunk)
for chunk in chunked(collections_to_delete, 100):
tasks.delete_anonymous_collections.delay(chunk)
# Incomplete addons cannot be deleted here because when an addon is
# rejected during a review it is marked as incomplete. See bug 670295.
log.debug('Cleaning up test results extraction cache.')
# lol at check for '/'
if settings.MEDIA_ROOT and settings.MEDIA_ROOT != '/':
cmd = ('find', settings.MEDIA_ROOT, '-maxdepth', '1', '-name',
'validate-*', '-mtime', '+7', '-type', 'd',
'-exec', 'rm', '-rf', "{}", ';')
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
else:
log.warning('MEDIA_ROOT not defined.')
if user_media_path('collection_icons'):
log.debug('Cleaning up uncompressed icons.')
cmd = ('find', user_media_path('collection_icons'),
'-name', '*__unconverted', '-mtime', '+1', '-type', 'f',
'-exec', 'rm', '{}', ';')
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
USERPICS_PATH = user_media_path('userpics')
if USERPICS_PATH:
log.debug('Cleaning up uncompressed userpics.')
cmd = ('find', USERPICS_PATH,
'-name', '*__unconverted', '-mtime', '+1', '-type', 'f',
'-exec', 'rm', '{}', ';')
output = Popen(cmd, stdout=PIPE).communicate()[0]
for line in output.split("\n"):
log.debug(line)
# Delete stale FileUploads.
stale_uploads = FileUpload.objects.filter(
created__lte=days_ago(180)).order_by('id')
for file_upload in stale_uploads:
log.debug(u'[FileUpload:{uuid}] Removing file: {path}'
.format(uuid=file_upload.uuid, path=file_upload.path))
if file_upload.path:
try:
storage.delete(file_upload.path)
except OSError:
pass
file_upload.delete()
def test_success(self, save_persona_image_mock,
create_persona_preview_images_mock, make_checksum_mock):
if not hasattr(Image.core, 'jpeg_encoder'):
raise SkipTest
make_checksum_mock.return_value = 'hashyourselfbeforeyoucrashyourself'
self.request.user = UserProfile.objects.get(pk=2519)
data = self.get_dict()
header_url, footer_url = self.get_img_urls()
# Upload header image.
img = open(get_image_path('persona-header.jpg'), 'rb')
r_ajax = self.client.post(header_url, {'upload_image': img})
data.update(header_hash=json.loads(r_ajax.content)['upload_hash'])
# Upload footer image.
img = open(get_image_path('persona-footer.jpg'), 'rb')
r_ajax = self.client.post(footer_url, {'upload_image': img})
data.update(footer_hash=json.loads(r_ajax.content)['upload_hash'])
# Populate and save form.
self.post()
eq_(self.form.is_valid(), True, self.form.errors)
self.form.save()
addon = Addon.objects.filter(type=amo.ADDON_PERSONA).order_by('-id')[0]
persona = addon.persona
# Test for correct Addon and Persona values.
eq_(unicode(addon.name), data['name'])
eq_(addon.slug, data['slug'])
self.assertSetEqual(addon.categories.values_list('id', flat=True),
[self.cat.id])
self.assertSetEqual(addon.tags.values_list('tag_text', flat=True),
data['tags'].split(', '))
eq_(persona.persona_id, 0)
eq_(persona.license, data['license'])
eq_(persona.accentcolor, data['accentcolor'].lstrip('#'))
eq_(persona.textcolor, data['textcolor'].lstrip('#'))
eq_(persona.author, self.request.user.username)
eq_(persona.display_username, self.request.user.name)
assert not persona.dupe_persona
v = addon.versions.all()
eq_(len(v), 1)
eq_(v[0].version, '0')
# Test for header, footer, and preview images.
dst = os.path.join(user_media_path('addons'), str(addon.id))
header_src = os.path.join(settings.TMP_PATH, 'persona_header',
u'b4ll1n')
footer_src = os.path.join(settings.TMP_PATH, 'persona_footer', u'5w4g')
eq_(save_persona_image_mock.mock_calls, [
mock.call(src=header_src, full_dst=os.path.join(dst,
'header.png')),
mock.call(src=footer_src, full_dst=os.path.join(dst, 'footer.png'))
])
create_persona_preview_images_mock.assert_called_with(
src=header_src,
full_dst=[
os.path.join(dst, 'preview.png'),
os.path.join(dst, 'icon.png')
],
set_modified_on=[addon])
