Esempio n. 1
0
def save_theme_reupload(header, footer, addon, **kw):
    header_dst = None
    footer_dst = None
    dst_root = os.path.join(user_media_path('addons'), str(addon.id))

    try:
        if header:
            header = os.path.join(settings.TMP_PATH, 'persona_header', header)
            header_dst = os.path.join(dst_root, 'pending_header.png')
            save_persona_image(src=header, full_dst=header_dst)
        if footer:
            footer = os.path.join(settings.TMP_PATH, 'persona_footer', footer)
            footer_dst = os.path.join(dst_root, 'pending_footer.png')
            save_persona_image(src=footer, full_dst=footer_dst)
    except IOError as e:
        log.error(str(e))
        raise

    if header_dst or footer_dst:
        theme = addon.persona
        header = 'pending_header.png' if header_dst else theme.header
        # Theme footer is optional, but can't be None.
        footer = theme.footer or ''
        if footer_dst:
            footer = 'pending_footer.png'

        # Store pending header and/or footer file paths for review.
        RereviewQueueTheme.objects.filter(theme=theme).delete()
        rqt = RereviewQueueTheme(theme=theme, header=header, footer=footer)
        rereviewqueuetheme_checksum(rqt=rqt)
        rqt.save()
    def test_edit_media_uploadedicon_noresize(self):
        img = "static/img/notifications/error.png"
        src_image = open(img, "rb")

        data = dict(upload_image=src_image)

        response = self.client.post(self.icon_upload, data)
        response_json = json.loads(response.content)
        addon = self.get_addon()

        # Now, save the form so it gets moved properly.
        data = dict(icon_type="image/png", icon_upload_hash=response_json["upload_hash"])
        data_formset = self.formset_media(**data)

        r = self.client.post(self.media_edit_url, data_formset)
        eq_(r.context["form"].errors, {})
        addon = self.get_addon()

        # Unfortunate hardcoding of URL
        addon_url = addon.get_icon_url(64).split("?")[0]
        assert addon_url.endswith("addon_icons/3/%s-64.png" % addon.id), "Unexpected path: %r" % addon_url

        eq_(data["icon_type"], "image/png")

        # Check that it was actually uploaded
        dirname = os.path.join(user_media_path("addon_icons"), "%s" % (addon.id / 1000))
        dest = os.path.join(dirname, "%s-64.png" % addon.id)

        assert storage.exists(dest)

        eq_(Image.open(storage.open(dest)).size, (48, 48))
Esempio n. 3
0
def rezip_file(response, pk):
    # An .xpi does not have a directory inside the zip, yet zips from github
    # do, so we'll need to rezip the file before passing it through to the
    # validator.
    loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
    old_filename = '{}_github_webhook.zip'.format(pk)
    old_path = os.path.join(loc, old_filename)

    with storage.open(old_path, 'wb') as old:
        old.write(response.content)

    new_filename = '{}_github_webhook.xpi'.format(pk)
    new_path = os.path.join(loc, new_filename)

    old_zip = SafeUnzip(old_path)
    if not old_zip.is_valid():
        raise

    with storage.open(new_path, 'w') as new:
        new_zip = zipfile.ZipFile(new, 'w')

        for obj in old_zip.filelist:
            # Basically strip off the leading directory.
            new_filename = obj.filename.partition('/')[-1]
            if not new_filename:
                continue
            new_zip.writestr(new_filename, old_zip.read(obj.filename))

        new_zip.close()

    old_zip.close()
    return new_path
Esempio n. 4
0
    def add_file(self, chunks, filename, size):
        if not self.uuid:
            self.uuid = self._meta.get_field('uuid')._create_uuid().hex
        filename = smart_str(u'{0}_{1}'.format(self.uuid, filename))
        loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
        base, ext = os.path.splitext(smart_path(filename))

        # Change a ZIP to an XPI, to maintain backward compatibility
        # with older versions of Firefox and to keep the rest of the XPI code
        # path as consistent as possible for ZIP uploads.
        # See: https://github.com/mozilla/addons-server/pull/2785
        if ext == '.zip':
            ext = '.xpi'

        if ext in EXTENSIONS:
            loc += ext

        log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc))
        hash = hashlib.sha256()
        with storage.open(loc, 'wb') as fd:
            for chunk in chunks:
                hash.update(chunk)
                fd.write(chunk)
        self.path = loc
        self.name = filename
        self.hash = 'sha256:%s' % hash.hexdigest()
        self.save()
Esempio n. 5
0
    def test_reupload(self, save_persona_image_mock,
                      create_persona_preview_images_mock,
                      make_checksum_mock):
        make_checksum_mock.return_value = 'checksumbeforeyouwrecksome'
        data = self.get_dict(header_hash='y0l0', footer_hash='abab')
        self.form = EditThemeForm(data, request=self.request,
                                  instance=self.instance)
        assert self.form.is_valid()
        self.form.save()

        dst = os.path.join(user_media_path('addons'), str(self.instance.id))
        header_src = os.path.join(settings.TMP_PATH, 'persona_header',
                                  u'y0l0')
        footer_src = os.path.join(settings.TMP_PATH, 'persona_footer',
                                  u'abab')

        assert save_persona_image_mock.mock_calls == (
            [mock.call(src=header_src,
                       full_dst=os.path.join(dst, 'pending_header.png')),
             mock.call(src=footer_src,
                       full_dst=os.path.join(dst, 'pending_footer.png'))])

        rqt = RereviewQueueTheme.objects.filter(theme=self.instance.persona)
        assert rqt.count() == 1
        assert rqt[0].header == 'pending_header.png'
        assert rqt[0].footer == 'pending_footer.png'
        assert not rqt[0].dupe_persona
Esempio n. 6
0
def path():
    # Check file paths / permissions
    rw = (settings.TMP_PATH,
          settings.MEDIA_ROOT,
          user_media_path('addons'),
          user_media_path('guarded_addons'),
          user_media_path('addon_icons'),
          user_media_path('collection_icons'),
          user_media_path('previews'),
          user_media_path('userpics'),
          user_media_path('reviewer_attachments'),
          dump_apps.Command.JSON_PATH,)
    r = [os.path.join(settings.ROOT, 'locale'),
         # The deploy process will want write access to this.
         # We do not want Django to have write access though.
         settings.PROD_DETAILS_DIR]
    filepaths = [(path, os.R_OK | os.W_OK, "We want read + write")
                 for path in rw]
    filepaths += [(path, os.R_OK, "We want read") for path in r]
    filepath_results = []
    filepath_status = True

    for path, perms, notes in filepaths:
        path_exists = os.path.exists(path)
        path_perms = os.access(path, perms)
        filepath_status = filepath_status and path_exists and path_perms
        filepath_results.append((path, path_exists, path_perms, notes))

    status = filepath_status
    status = ''
    if not filepath_status:
        status = 'check main status page for broken perms'

    return status, filepath_results
Esempio n. 7
0
def delete_persona_image(dst, **kw):
    log.info('[1@None] Deleting persona image: %s.' % dst)
    if not dst.startswith(user_media_path('addons')):
        log.error("Someone tried deleting something they shouldn't: %s" % dst)
        return
    try:
        storage.delete(dst)
    except Exception, e:
        log.error('Error deleting persona image: %s' % e)
Esempio n. 8
0
def delete_icon(dst, **kw):
    log.info('[1@None] Deleting icon: %s.' % dst)

    if not dst.startswith(user_media_path('collection_icons')):
        log.error("Someone tried deleting something they shouldn't: %s" % dst)
        return

    try:
        storage.delete(dst)
    except Exception, e:
        log.error("Error deleting icon: %s" % e)
Esempio n. 9
0
def delete_photo(dst, **kw):
    task_log.debug('[1@None] Deleting photo: %s.' % dst)

    if not dst.startswith(user_media_path('userpics')):
        task_log.error("Someone tried deleting something they shouldn't: %s"
                       % dst)
        return

    try:
        storage.delete(dst)
    except Exception, e:
        task_log.error("Error deleting userpic: %s" % e)
Esempio n. 10
0
 def add_file(self, chunks, filename, size):
     if not self.uuid:
         self.uuid = self._meta.get_field('uuid')._create_uuid().hex
     filename = smart_str(u'{0}_{1}'.format(self.uuid, filename))
     loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
     base, ext = os.path.splitext(smart_path(filename))
     if ext in EXTENSIONS:
         loc += ext
     log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc))
     hash = hashlib.sha256()
     with storage.open(loc, 'wb') as fd:
         for chunk in chunks:
             hash.update(chunk)
             fd.write(chunk)
     self.path = loc
     self.name = filename
     self.hash = 'sha256:%s' % hash.hexdigest()
     self.save()
Esempio n. 11
0
def create_theme_images(theme, placement, hash_):
    """
    Generates 2 images, one in the temp folder and the other in the
    user-media one. Both are needed to generate previews for themes.

    """
    color = random.choice(ImageColor.colormap.keys())
    image = Image.new('RGB', (3000, 200), color)
    tmp_path = os.path.join(settings.TMP_PATH,
                            'persona_{placement}'.format(placement=placement))
    if not os.path.exists(tmp_path):
        os.makedirs(tmp_path)
    tmp_loc = os.path.join(tmp_path, hash_)
    image.save(tmp_loc, 'jpeg')
    media_path = os.path.join(user_media_path('addons'), str(theme.id))
    if not os.path.exists(media_path):
        os.makedirs(media_path)
    media_loc = os.path.join(media_path, hash_)
    image.save(media_loc, 'jpeg')
Esempio n. 12
0
    def _save_file(self, version):
        data = self.cleaned_data
        xpi = data['xpi']
        hash = hashlib.sha256()

        f = File(version=version,
                 platform=amo.PLATFORM_DICT[data['platform']].id,
                 size=xpi.size)

        filename = f.generate_filename()
        path = os.path.join(user_media_path('addons'), str(version.addon.id))
        with storage.open(os.path.join(path, filename), 'wb') as destination:
            for chunk in xpi.chunks():
                hash.update(chunk)
                destination.write(chunk)

        f.hash = 'sha256:%s' % hash.hexdigest()
        f.save()
        return f
Esempio n. 13
0
    def test_reupload_no_footer(self, save_persona_image_mock, create_persona_preview_images_mock, make_checksum_mock):
        make_checksum_mock.return_value = "checksumbeforeyouwrecksome"
        data = self.get_dict(header_hash="y0l0", footer_hash="")
        self.form = EditThemeForm(data, request=self.request, instance=self.instance)
        assert self.form.is_valid()
        self.form.save()

        dst = os.path.join(user_media_path("addons"), str(self.instance.id))
        header_src = os.path.join(settings.TMP_PATH, "persona_header", u"y0l0")

        assert save_persona_image_mock.mock_calls == (
            [mock.call(src=header_src, full_dst=os.path.join(dst, "pending_header.png"))]
        )

        rqt = RereviewQueueTheme.objects.filter(theme=self.instance.persona)
        assert rqt.count() == 1
        assert rqt[0].header == "pending_header.png"
        assert rqt[0].footer == ""
        assert not rqt[0].dupe_persona
Esempio n. 14
0
def save_theme(header, footer, addon, **kw):
    """Save theme image and calculates checksum after theme save."""
    dst_root = os.path.join(user_media_path('addons'), str(addon.id))
    header = os.path.join(settings.TMP_PATH, 'persona_header', header)
    header_dst = os.path.join(dst_root, 'header.png')
    if footer:
        footer = os.path.join(settings.TMP_PATH, 'persona_footer', footer)
        footer_dst = os.path.join(dst_root, 'footer.png')

    try:
        save_persona_image(src=header, full_dst=header_dst)
        if footer:
            save_persona_image(src=footer, full_dst=footer_dst)
        create_persona_preview_images(
            src=header, full_dst=[os.path.join(dst_root, 'preview.png'),
                                  os.path.join(dst_root, 'icon.png')],
            set_modified_on=[addon])
        theme_checksum(addon.persona)
    except IOError:
        addon.delete()
        raise
Esempio n. 15
0
def approve_rereview(theme):
    """Replace original theme with pending theme on filesystem."""
    # If reuploaded theme, replace old theme design.
    storage = LocalFileStorage()
    rereview = theme.rereviewqueuetheme_set.all()
    reupload = rereview[0]

    if reupload.header_path != reupload.theme.header_path:
        create_persona_preview_images(
            src=reupload.header_path,
            full_dst=[
                reupload.theme.thumb_path,
                reupload.theme.icon_path],
            set_modified_on=[reupload.theme.addon])

        if not reupload.theme.is_new():
            # Legacy themes also need a preview_large.jpg.
            # Modern themes use preview.png for both thumb and preview so there
            # is no problem there.
            copy_stored_file(reupload.theme.thumb_path,
                             reupload.theme.preview_path, storage=storage)

        move_stored_file(
            reupload.header_path, reupload.theme.header_path, storage=storage)

    theme = reupload.theme
    footer_path = theme.footer_path
    if reupload.footer_path != footer_path:
        if not footer_path:
            dst_root = os.path.join(user_media_path('addons'),
                                    str(theme.addon.id))
            footer_path = os.path.join(dst_root, 'footer.png')
            theme.footer = 'footer.png'
            theme.save()

        move_stored_file(
            reupload.footer_path, footer_path, storage=storage)
    rereview.delete()

    theme.addon.increment_version()
Esempio n. 16
0
    def add_file(self, chunks, filename, size):
        if not self.uuid:
            self.uuid = self._meta.get_field('uuid')._create_uuid()

        filename = force_bytes(u'{0}_{1}'.format(self.uuid.hex, filename))
        loc = os.path.join(user_media_path('addons'), 'temp', uuid.uuid4().hex)
        base, ext = os.path.splitext(force_bytes(filename))
        is_crx = False

        # Change a ZIP to an XPI, to maintain backward compatibility
        # with older versions of Firefox and to keep the rest of the XPI code
        # path as consistent as possible for ZIP uploads.
        # See: https://github.com/mozilla/addons-server/pull/2785
        if ext == '.zip':
            ext = '.xpi'

        # If the extension is a CRX, we need to do some actual work to it
        # before we just convert it to an XPI. We strip the header from the
        # CRX, then it's good; see more about the CRX file format here:
        # https://developer.chrome.com/extensions/crx
        if ext == '.crx':
            ext = '.xpi'
            is_crx = True

        if ext in EXTENSIONS:
            loc += ext

        log.info('UPLOAD: %r (%s bytes) to %r' % (filename, size, loc))
        if is_crx:
            hash = write_crx_as_xpi(chunks, storage, loc)
        else:
            hash = hashlib.sha256()
            with storage.open(loc, 'wb') as file_destination:
                for chunk in chunks:
                    hash.update(chunk)
                    file_destination.write(chunk)
        self.path = loc
        self.name = filename
        self.hash = 'sha256:%s' % hash.hexdigest()
        self.save()
Esempio n. 17
0
def _uploader(resize_size, final_size):
    img = get_image_path('mozilla.png')
    original_size = (339, 128)

    src = tempfile.NamedTemporaryFile(mode='r+w+b', suffix=".png",
                                      delete=False)

    # resize_icon removes the original
    shutil.copyfile(img, src.name)

    src_image = Image.open(src.name)
    assert src_image.size == original_size

    if isinstance(final_size, list):
        uploadto = user_media_path('addon_icons')
        try:
            os.makedirs(uploadto)
        except OSError:
            pass
        for rsize, fsize in zip(resize_size, final_size):
            dest_name = os.path.join(uploadto, '1234')

            tasks.resize_icon(src.name, dest_name, resize_size, locally=True)
            dest_image = Image.open(open('%s-%s.png' % (dest_name, rsize)))
            assert dest_image.size == fsize

            if os.path.exists(dest_image.filename):
                os.remove(dest_image.filename)
            assert not os.path.exists(dest_image.filename)
        shutil.rmtree(uploadto)
    else:
        dest = tempfile.mktemp(suffix='.png')
        tasks.resize_icon(src.name, dest, resize_size, locally=True)
        dest_image = Image.open(dest)
        assert dest_image.size == final_size

    assert not os.path.exists(src.name)
Esempio n. 18
0
def path():
    # Check file paths / permissions
    rw = (
        settings.TMP_PATH,
        settings.MEDIA_ROOT,
        user_media_path('addons'),
        user_media_path('guarded_addons'),
        user_media_path('addon_icons'),
        user_media_path('collection_icons'),
        user_media_path('previews'),
        user_media_path('userpics'),
        user_media_path('reviewer_attachments'),
        dump_apps.Command.JSON_PATH,
    )
    r = [
        os.path.join(settings.ROOT, 'locale'),
        # The deploy process will want write access to this.
        # We do not want Django to have write access though.
        settings.PROD_DETAILS_DIR
    ]
    filepaths = [(path, os.R_OK | os.W_OK, "We want read + write")
                 for path in rw]
    filepaths += [(path, os.R_OK, "We want read") for path in r]
    filepath_results = []
    filepath_status = True

    for path, perms, notes in filepaths:
        path_exists = os.path.exists(path)
        path_perms = os.access(path, perms)
        filepath_status = filepath_status and path_exists and path_perms
        filepath_results.append((path, path_exists, path_perms, notes))

    status = filepath_status
    status = ''
    if not filepath_status:
        status = 'check main status page for broken perms'

    return status, filepath_results
Esempio n. 19
0
 def guarded_file_path(self):
     return os.path.join(user_media_path('guarded_addons'),
                         str(self.version.addon_id), self.filename)
Esempio n. 20
0
 def mirror_file_path(self):
     return os.path.join(user_media_path('addons'),
                         str(self.version.addon_id), self.filename)
Esempio n. 21
0
 def test_without_settings(self):
     del settings.ADDONS_PATH
     path = helpers.user_media_path('addons')
     eq_(path, '/path/addons')
Esempio n. 22
0
 def test_with_settings(self):
     path = helpers.user_media_path('addons')
     assert path == '/another/path/'
Esempio n. 23
0
 def mirror_path_prefix(self):
     return os.path.join(user_media_path('addons'), str(self.addon_id))
Esempio n. 24
0
    def test_success(self, save_persona_image_mock,
                     create_persona_preview_images_mock, make_checksum_mock):
        make_checksum_mock.return_value = 'hashyourselfbeforeyoucrashyourself'

        self.request.user = UserProfile.objects.get(pk=2519)

        data = self.get_dict()
        header_url, footer_url = self.get_img_urls()

        # Upload header image.
        img = open(get_image_path('persona-header.jpg'), 'rb')
        r_ajax = self.client.post(header_url, {'upload_image': img})
        data.update(header_hash=json.loads(r_ajax.content)['upload_hash'])

        # Upload footer image.
        img = open(get_image_path('persona-footer.jpg'), 'rb')
        r_ajax = self.client.post(footer_url, {'upload_image': img})
        data.update(footer_hash=json.loads(r_ajax.content)['upload_hash'])

        # Populate and save form.
        self.post()
        assert self.form.is_valid(), self.form.errors
        self.form.save()

        addon = Addon.objects.filter(type=amo.ADDON_PERSONA).order_by('-id')[0]
        persona = addon.persona

        # Test for correct Addon and Persona values.
        assert unicode(addon.name) == data['name']
        assert addon.slug == data['slug']
        self.assertSetEqual(set(addon.categories.values_list('id', flat=True)),
                            {self.cat.id})
        self.assertSetEqual(set(addon.tags.values_list('tag_text', flat=True)),
                            set(data['tags'].split(', ')))
        assert persona.persona_id == 0
        assert persona.license == data['license']
        assert persona.accentcolor == data['accentcolor'].lstrip('#')
        assert persona.textcolor == data['textcolor'].lstrip('#')
        assert persona.author == self.request.user.username
        assert persona.display_username == self.request.user.name
        assert not persona.dupe_persona

        v = addon.versions.all()
        assert len(v) == 1
        assert v[0].version == '0'

        # Test for header, footer, and preview images.
        dst = os.path.join(user_media_path('addons'), str(addon.id))

        header_src = os.path.join(settings.TMP_PATH, 'persona_header',
                                  u'b4ll1n')
        footer_src = os.path.join(settings.TMP_PATH, 'persona_footer',
                                  u'5w4g')

        assert save_persona_image_mock.mock_calls == (
            [mock.call(src=header_src,
                       full_dst=os.path.join(dst, 'header.png')),
             mock.call(src=footer_src,
                       full_dst=os.path.join(dst, 'footer.png'))])

        create_persona_preview_images_mock.assert_called_with(
            src=header_src,
            full_dst=[os.path.join(dst, 'preview.png'),
                      os.path.join(dst, 'icon.png')],
            set_modified_on=[addon])
Esempio n. 25
0
 def mirror_file_path(self):
     return os.path.join(user_media_path('addons'),
                         str(self.version.addon_id), self.filename)
Esempio n. 26
0
 def handle(self, *args, **kw):
     z = ((user_media_path('collection_icons'), resize_icon),
          (user_media_path('userpics'), resize_photo))
     for base, task in z:
         self.fix(base, task)
Esempio n. 27
0
 def picture_dir(self):
     from olympia.amo.helpers import user_media_path
     split_id = re.match(r'((\d*?)(\d{0,3}?))\d{1,3}$', str(self.id))
     return os.path.join(user_media_path('userpics'),
                         split_id.group(2) or '0',
                         split_id.group(1) or '0')
Esempio n. 28
0
 def test_with_settings(self):
     path = helpers.user_media_path('addons')
     eq_(path, '/another/path/')
Esempio n. 29
0
 def test_with_settings(self):
     path = helpers.user_media_path('addons')
     eq_(path, '/another/path/')
Esempio n. 30
0
def gc(test_result=True):
    """Site-wide garbage collections."""

    def days_ago(days):
        return datetime.today() - timedelta(days=days)

    log.debug("Collecting data to delete")

    logs = (
        ActivityLog.objects.filter(created__lt=days_ago(90))
        .exclude(action__in=amo.LOG_KEEP)
        .values_list("id", flat=True)
    )

    # Paypal only keeps retrying to verify transactions for up to 3 days. If we
    # still have an unverified transaction after 6 days, we might as well get
    # rid of it.
    contributions_to_delete = Contribution.objects.filter(
        transaction_id__isnull=True, created__lt=days_ago(6)
    ).values_list("id", flat=True)

    collections_to_delete = Collection.objects.filter(
        created__lt=days_ago(2), type=amo.COLLECTION_ANONYMOUS
    ).values_list("id", flat=True)

    for chunk in chunked(logs, 100):
        tasks.delete_logs.delay(chunk)
    for chunk in chunked(contributions_to_delete, 100):
        tasks.delete_stale_contributions.delay(chunk)
    for chunk in chunked(collections_to_delete, 100):
        tasks.delete_anonymous_collections.delay(chunk)
    # Incomplete addons cannot be deleted here because when an addon is
    # rejected during a review it is marked as incomplete. See bug 670295.

    log.debug("Cleaning up test results extraction cache.")
    # lol at check for '/'
    if settings.MEDIA_ROOT and settings.MEDIA_ROOT != "/":
        cmd = (
            "find",
            settings.MEDIA_ROOT,
            "-maxdepth",
            "1",
            "-name",
            "validate-*",
            "-mtime",
            "+7",
            "-type",
            "d",
            "-exec",
            "rm",
            "-rf",
            "{}",
            ";",
        )

        output = Popen(cmd, stdout=PIPE).communicate()[0]

        for line in output.split("\n"):
            log.debug(line)

    else:
        log.warning("MEDIA_ROOT not defined.")

    if user_media_path("collection_icons"):
        log.debug("Cleaning up uncompressed icons.")

        cmd = (
            "find",
            user_media_path("collection_icons"),
            "-name",
            "*__unconverted",
            "-mtime",
            "+1",
            "-type",
            "f",
            "-exec",
            "rm",
            "{}",
            ";",
        )
        output = Popen(cmd, stdout=PIPE).communicate()[0]

        for line in output.split("\n"):
            log.debug(line)

    USERPICS_PATH = user_media_path("userpics")
    if USERPICS_PATH:
        log.debug("Cleaning up uncompressed userpics.")

        cmd = ("find", USERPICS_PATH, "-name", "*__unconverted", "-mtime", "+1", "-type", "f", "-exec", "rm", "{}", ";")
        output = Popen(cmd, stdout=PIPE).communicate()[0]

        for line in output.split("\n"):
            log.debug(line)
Esempio n. 31
0
 def handle(self, *args, **kw):
     z = ((user_media_path('collection_icons'),
           resize_icon),
          (user_media_path('userpics'), resize_photo))
     for base, task in z:
         self.fix(base, task)
Esempio n. 32
0
 def get_img_dir(self):
     return os.path.join(user_media_path('collection_icons'),
                         str(self.id / 1000))
Esempio n. 33
0
 def get_img_dir(self):
     return os.path.join(user_media_path('collection_icons'),
                         str(self.id / 1000))
Esempio n. 34
0
 def picture_dir(self):
     from olympia.amo.helpers import user_media_path
     split_id = re.match(r'((\d*?)(\d{0,3}?))\d{1,3}$', str(self.id))
     return os.path.join(user_media_path('userpics'),
                         split_id.group(2) or '0',
                         split_id.group(1) or '0')
Esempio n. 35
0
 def test_without_settings(self):
     del settings.ADDONS_PATH
     path = helpers.user_media_path('addons')
     assert path == '/path/addons'
Esempio n. 36
0
 def path_prefix(self):
     return os.path.join(user_media_path('addons'), str(self.addon_id))
Esempio n. 37
0
 def test_with_settings(self):
     path = helpers.user_media_path('addons')
     assert path == '/another/path/'
Esempio n. 38
0
 def guarded_file_path(self):
     return os.path.join(user_media_path('guarded_addons'),
                         str(self.version.addon_id), self.filename)
Esempio n. 39
0
 def test_image_path(self):
     up = self.get_update('en-US', 15663)
     up.get_update()
     image_path = up.image_path('foo.png')
     assert user_media_path('addons') in image_path
Esempio n. 40
0
 def test_image_path(self):
     up = self.get_update('en-US', 15663)
     up.get_update()
     image_path = up.image_path('foo.png')
     assert user_media_path('addons') in image_path
Esempio n. 41
0
def gc(test_result=True):
    """Site-wide garbage collections."""
    def days_ago(days):
        return datetime.today() - timedelta(days=days)

    log.debug('Collecting data to delete')

    logs = (ActivityLog.objects.filter(created__lt=days_ago(90))
            .exclude(action__in=amo.LOG_KEEP).values_list('id', flat=True))

    # Paypal only keeps retrying to verify transactions for up to 3 days. If we
    # still have an unverified transaction after 6 days, we might as well get
    # rid of it.
    contributions_to_delete = (
        Contribution.objects
        .filter(transaction_id__isnull=True, created__lt=days_ago(6))
        .values_list('id', flat=True))

    collections_to_delete = (
        Collection.objects.filter(created__lt=days_ago(2),
                                  type=amo.COLLECTION_ANONYMOUS)
        .values_list('id', flat=True))

    for chunk in chunked(logs, 100):
        tasks.delete_logs.delay(chunk)
    for chunk in chunked(contributions_to_delete, 100):
        tasks.delete_stale_contributions.delay(chunk)
    for chunk in chunked(collections_to_delete, 100):
        tasks.delete_anonymous_collections.delay(chunk)
    # Incomplete addons cannot be deleted here because when an addon is
    # rejected during a review it is marked as incomplete. See bug 670295.

    log.debug('Cleaning up test results extraction cache.')
    # lol at check for '/'
    if settings.MEDIA_ROOT and settings.MEDIA_ROOT != '/':
        cmd = ('find', settings.MEDIA_ROOT, '-maxdepth', '1', '-name',
               'validate-*', '-mtime', '+7', '-type', 'd',
               '-exec', 'rm', '-rf', "{}", ';')

        output = Popen(cmd, stdout=PIPE).communicate()[0]

        for line in output.split("\n"):
            log.debug(line)

    else:
        log.warning('MEDIA_ROOT not defined.')

    if user_media_path('collection_icons'):
        log.debug('Cleaning up uncompressed icons.')

        cmd = ('find', user_media_path('collection_icons'),
               '-name', '*__unconverted', '-mtime', '+1', '-type', 'f',
               '-exec', 'rm', '{}', ';')
        output = Popen(cmd, stdout=PIPE).communicate()[0]

        for line in output.split("\n"):
            log.debug(line)

    USERPICS_PATH = user_media_path('userpics')
    if USERPICS_PATH:
        log.debug('Cleaning up uncompressed userpics.')

        cmd = ('find', USERPICS_PATH,
               '-name', '*__unconverted', '-mtime', '+1', '-type', 'f',
               '-exec', 'rm', '{}', ';')
        output = Popen(cmd, stdout=PIPE).communicate()[0]

        for line in output.split("\n"):
            log.debug(line)
Esempio n. 42
0
def gc(test_result=True):
    """Site-wide garbage collections."""
    def days_ago(days):
        return datetime.today() - timedelta(days=days)

    log.debug('Collecting data to delete')

    logs = (ActivityLog.objects.filter(created__lt=days_ago(90))
            .exclude(action__in=amo.LOG_KEEP).values_list('id', flat=True))

    # Paypal only keeps retrying to verify transactions for up to 3 days. If we
    # still have an unverified transaction after 6 days, we might as well get
    # rid of it.
    contributions_to_delete = (
        Contribution.objects
        .filter(transaction_id__isnull=True, created__lt=days_ago(6))
        .values_list('id', flat=True))

    collections_to_delete = (
        Collection.objects.filter(created__lt=days_ago(2),
                                  type=amo.COLLECTION_ANONYMOUS)
        .values_list('id', flat=True))

    for chunk in chunked(logs, 100):
        tasks.delete_logs.delay(chunk)
    for chunk in chunked(contributions_to_delete, 100):
        tasks.delete_stale_contributions.delay(chunk)
    for chunk in chunked(collections_to_delete, 100):
        tasks.delete_anonymous_collections.delay(chunk)
    # Incomplete addons cannot be deleted here because when an addon is
    # rejected during a review it is marked as incomplete. See bug 670295.

    log.debug('Cleaning up test results extraction cache.')
    # lol at check for '/'
    if settings.MEDIA_ROOT and settings.MEDIA_ROOT != '/':
        cmd = ('find', settings.MEDIA_ROOT, '-maxdepth', '1', '-name',
               'validate-*', '-mtime', '+7', '-type', 'd',
               '-exec', 'rm', '-rf', "{}", ';')

        output = Popen(cmd, stdout=PIPE).communicate()[0]

        for line in output.split("\n"):
            log.debug(line)

    else:
        log.warning('MEDIA_ROOT not defined.')

    if user_media_path('collection_icons'):
        log.debug('Cleaning up uncompressed icons.')

        cmd = ('find', user_media_path('collection_icons'),
               '-name', '*__unconverted', '-mtime', '+1', '-type', 'f',
               '-exec', 'rm', '{}', ';')
        output = Popen(cmd, stdout=PIPE).communicate()[0]

        for line in output.split("\n"):
            log.debug(line)

    USERPICS_PATH = user_media_path('userpics')
    if USERPICS_PATH:
        log.debug('Cleaning up uncompressed userpics.')

        cmd = ('find', USERPICS_PATH,
               '-name', '*__unconverted', '-mtime', '+1', '-type', 'f',
               '-exec', 'rm', '{}', ';')
        output = Popen(cmd, stdout=PIPE).communicate()[0]

        for line in output.split("\n"):
            log.debug(line)

    # Delete stale FileUploads.
    stale_uploads = FileUpload.objects.filter(
        created__lte=days_ago(180)).order_by('id')
    for file_upload in stale_uploads:
        log.debug(u'[FileUpload:{uuid}] Removing file: {path}'
                  .format(uuid=file_upload.uuid, path=file_upload.path))
        if file_upload.path:
            try:
                storage.delete(file_upload.path)
            except OSError:
                pass
        file_upload.delete()
Esempio n. 43
0
    def test_success(self, save_persona_image_mock,
                     create_persona_preview_images_mock, make_checksum_mock):
        if not hasattr(Image.core, 'jpeg_encoder'):
            raise SkipTest
        make_checksum_mock.return_value = 'hashyourselfbeforeyoucrashyourself'

        self.request.user = UserProfile.objects.get(pk=2519)

        data = self.get_dict()
        header_url, footer_url = self.get_img_urls()

        # Upload header image.
        img = open(get_image_path('persona-header.jpg'), 'rb')
        r_ajax = self.client.post(header_url, {'upload_image': img})
        data.update(header_hash=json.loads(r_ajax.content)['upload_hash'])

        # Upload footer image.
        img = open(get_image_path('persona-footer.jpg'), 'rb')
        r_ajax = self.client.post(footer_url, {'upload_image': img})
        data.update(footer_hash=json.loads(r_ajax.content)['upload_hash'])

        # Populate and save form.
        self.post()
        eq_(self.form.is_valid(), True, self.form.errors)
        self.form.save()

        addon = Addon.objects.filter(type=amo.ADDON_PERSONA).order_by('-id')[0]
        persona = addon.persona

        # Test for correct Addon and Persona values.
        eq_(unicode(addon.name), data['name'])
        eq_(addon.slug, data['slug'])
        self.assertSetEqual(addon.categories.values_list('id', flat=True),
                            [self.cat.id])
        self.assertSetEqual(addon.tags.values_list('tag_text', flat=True),
                            data['tags'].split(', '))
        eq_(persona.persona_id, 0)
        eq_(persona.license, data['license'])
        eq_(persona.accentcolor, data['accentcolor'].lstrip('#'))
        eq_(persona.textcolor, data['textcolor'].lstrip('#'))
        eq_(persona.author, self.request.user.username)
        eq_(persona.display_username, self.request.user.name)
        assert not persona.dupe_persona

        v = addon.versions.all()
        eq_(len(v), 1)
        eq_(v[0].version, '0')

        # Test for header, footer, and preview images.
        dst = os.path.join(user_media_path('addons'), str(addon.id))

        header_src = os.path.join(settings.TMP_PATH, 'persona_header',
                                  u'b4ll1n')
        footer_src = os.path.join(settings.TMP_PATH, 'persona_footer', u'5w4g')

        eq_(save_persona_image_mock.mock_calls, [
            mock.call(src=header_src, full_dst=os.path.join(dst,
                                                            'header.png')),
            mock.call(src=footer_src, full_dst=os.path.join(dst, 'footer.png'))
        ])

        create_persona_preview_images_mock.assert_called_with(
            src=header_src,
            full_dst=[
                os.path.join(dst, 'preview.png'),
                os.path.join(dst, 'icon.png')
            ],
            set_modified_on=[addon])