def test_skip_positive():
"""The ``feed_image`` hook can enforce handling in a positive
manner, e.g. by instructing to continue with image processing right
away while skipping other ``feed_image`` callbacks, which then won't
have a chance to do their validation.
"""
addin1 = count_addin()
addin2 = count_addin()
ADDINS = [handle_feed_images, addin1, addin2]
class TestFeed(BaseImageTestFeed):
def pass1(feed):
# first pass, everything is normal
assert addin1.feed_image == 1
assert addin2.feed_image == 1
assert addin1.feed_image_updated == 1
assert addin2.feed_image_updated == 1
# if we instruct addin1's ``feed_image`` hook to
# force a succeed...
addin1.returnvalue["feed_image"] = False
def pass2(feed):
# ...then this time addin2's ``feed_image`` hook will
# never even be called.
assert addin2.feed_image == 1
assert addin1.feed_image == 2
# although the other handlers in both addins are still triggered.
assert addin1.feed_image_updated == 2
assert addin2.feed_image_updated == 2
feedev.testcaller()
def test_original_format():
# handle_feed_images not specified, this tests the dependency
ADDINS = [store_feed_images((tempdir, '%(model_id)s.%(extension)s'))]
class TestImage(feedev.File):
content = ValidPNGImage
url = 'http://bla/stuff.png'
class TestFeed(feedev.Feed):
content = FeedWithImage % TestImage.url
def pass1(feed):
# check that the image was stored
imgfile = path.join(tempdir, '%s.%s' % (feed.id, 'png'))
assert path.exists(imgfile)
# Make sure it was written correctly; this will usually not
# be true if the image was saved using PIL, which is the
# reason why we explicitely specify a url with an extension
# to ``TestImage`` (otherwise, the image would be loaded into
# PIL to defer the extension from the content), and PIL
# subsequently used to save the file.
assert open(imgfile, 'rb').read() == TestImage.content
feedev.testcaller()
def test_restrict_mediatype():
"""Test mime type validation.
"""
counter = image_hook_counter()
ADDINS = [feed_image_restrict_mediatypes(('image/png', 'image/gif')), counter]
class TestFeedImage(feedev.File):
content = ""
def headers(p):
if p == 1: return {'Content-Type': 'text/plain'}
elif p == 2: return {'Content-Type': 'image/jpeg'}
elif p == 3: return {'Content-Type': 'image/png; charset=ISO-8859-1'} # charsets are correctly parsed out
elif p == 4: return {'Content-Type': 'image/png'}
class TestFeed(feedev.Feed):
content = FeedWithImage % (TestFeedImage.url)
def pass1(feed):
assert counter.success == 0
def pass2(feed):
assert counter.success == 0
def pass3(feed):
assert counter.success == 1
def pass4(feed):
assert counter.success == 2
feedev.testcaller()
def test_update_every():
"""Test timed update restriction.
"""
counter = image_hook_counter()
ADDINS = [feed_image_restrict_frequency(delta=20), counter]
class TestFeedImage(feedev.File):
pass
class TestFeed(feedev.Feed):
content = FeedWithImage % TestFeedImage.url
def pass1(feed):
assert counter.success == 1
def pass2(feed):
# we're not yet updating the image again...
assert counter.success == 1
# ...this also doesn't count as a failure (an existing images would be kept)
assert counter.failure == 0
# ...but let's advance the clock for the next pass
datetime.datetime.modify(seconds=30)
def pass3(feed):
assert counter.success == 2
MockDateTime.install()
try:
feedev.testcaller()
finally:
MockDateTime.uninstall()
def test_failure_reset():
"""Image fails if no valid extension (=no image).
"""
counter = image_hook_counter()
ADDINS = [feed_image_restrict_extensions(('png', 'gif')), counter]
class ValidExtensionImage(feedev.File):
url = 'http://images/image.png'
class InvalidExtensionImage(feedev.File):
url = 'http://images/image.jpeg'
class TestFeed(feedev.Feed):
def content(p):
if p == 1: image = ValidExtensionImage
elif p == 2: image = InvalidExtensionImage
return FeedWithImage % saxutils.escape(image.url)
def pass1(feed):
assert counter.failure == 0
def pass2(feed):
# invalid extension cause the image to fail completely
assert counter.failure == 1
feedev.testcaller()
def test_thumbnails():
# handle_feed_images not specified, this tests the dependency
ADDINS = [feed_image_thumbnails(
((100,100),),
(tempdir, '%(model_id)s-thumb-%(size)s.gif'),
format='gif')]
class TestImage(feedev.File):
content = ValidPNGImage
url = 'http://bla/stuff.png'
class TestFeed(feedev.Feed):
content = FeedWithImage % TestImage.url
def pass1(feed):
# check that the image was stored
imgfile = path.join(tempdir, '%s-thumb-%s.%s' % (feed.id, '100x100', 'gif'))
assert path.exists(imgfile)
# check that it was stored as a true gif file, as requested
from PIL import Image
i = Image.open(imgfile)
assert i.format == 'GIF'
# ... with the requested size
assert i.size == (100,100)
feedev.testcaller()
def test_enclosure_addin():
"""Test the problem through and in the context of the enclosure
handling addins (enclosures are one of the problem areas with respect
to this bug.
"""
ADDINS = [
# both plugins separately exhibited the bug in the passed,
# though note that ``collect_enclosure_data`` depends on
# ``store_encloures`` and cannot be used without it
store_enclosures(),
collect_enclosure_data('length', 'type'),
]
class BozoFeed(feedev.Feed):
content = """
<rss>
<channel>
<item>
<guid>item1</guid>
<enclosure href="http://h.com/p/f.mp3" length="10" />
<!-- item closing tag missing -->
</channel></rss>
"""
def pass1(feed):
# If the patch were not applied, we'd expect a Storm
# "unicode expected" exception to be raised during
# addin execution, the assertion here is just for
# completeness sake.
href = feed.items.one().enclosures.one().href
assert href == 'http://h.com/p/f.mp3'
assert type(href) == unicode
feedev.testcaller()
def test_max_size_by_content_length():
"""Test validation against ``Content-Length`` header.
"""
counter = image_hook_counter()
ADDINS = [feed_image_restrict_size(max_size=10), counter]
class TestFeedImage(feedev.File):
content = ""
def headers(p):
if p == 1: return {'Content-Length': '15'} # nose that those are strings, just like real headers # TODO: make the testframework ensure that header values are always strings
else: return {'Content-Length': '5'}
class TestFeed(feedev.Feed):
content = FeedWithImage % (TestFeedImage.url)
def pass1(feed):
# at this point, the image is too large and is ignored
assert counter.success == 0
# also, processing failed (=no image)
assert counter.failure == 1
def pass2(feed):
# it's been fixed, we meet the limit
assert counter.success == 1
feedev.testcaller()
def test_max_size():
"""Test live validation of file size, effective in case a
Content-Length header is missing.
"""
counter = image_hook_counter()
ADDINS = [feed_image_restrict_size(max_size=10), image_reader, counter]
class TestFeedImage(feedev.File):
def content(p):
if p == 1: return "b"*15
else: return "b"*5
class TestFeed(feedev.Feed):
content = FeedWithImage % TestFeedImage.url
def pass1(feed):
# at this point, the image is to large and is ignored
assert counter.success == 0
# also, processing failed (=no image)
assert counter.failure == 1
def pass2(feed):
# it's been fixed, we meet the limit
assert counter.success == 1
feedev.testcaller()
def test_restrict_extensions():
"""Test file extension validation.
"""
counter = image_hook_counter()
ADDINS = [feed_image_restrict_extensions(('png', 'gif')), counter]
class JpegImage(feedev.File):
url = 'http://images/image.jpeg'
class NonImageFile(feedev.File):
url = 'http://documents/stuff.xml'
class PngImage(feedev.File):
url = 'http://images/image.png'
class ImageWithQueryString(feedev.File):
url = 'http://images/image.png?q=x&r=3'
class ImageWitoutExtension(feedev.File):
url = 'http://images/image'
class ImageWithExtensionInQuery(feedev.File):
url = 'http://images/image?x=66&filename=.png'
class ImageWithoutExtensionButValidContent(feedev.File):
url = 'http://images/validimage'
content = ValidPNGImage
class TestFeed(feedev.Feed):
def content(p):
if p == 1: image = JpegImage
elif p == 2: image = NonImageFile
elif p == 3: image = PngImage
elif p == 4: image = ImageWithQueryString
elif p == 5: image = ImageWitoutExtension
elif p == 6: image = ImageWithExtensionInQuery
elif p == 7: image = ImageWithoutExtensionButValidContent
return FeedWithImage % saxutils.escape(image.url)
def pass1(feed):
assert counter.success == 0
def pass2(feed):
assert counter.success == 0
def pass3(feed):
assert counter.success == 1
def pass4(feed):
assert counter.success == 2
def pass5(feed):
# Image has no extension in URL, and extension cannot be
# determined via content type or content (since the former
# is missing and the latter is invalid). In this case,
# we fail, the image is not handled.
assert counter.success == 2
def pass6(feed):
# As in pass5, no extension is available. The extension
# from the querystring is currently ignored.
assert counter.success == 2
def pass7(feed):
# the image is once again lacking an extension, but this
# time the content is valid and the extension can be
# deferred from it.
assert counter.success == 3
feedev.testcaller()
def test_basic():
counter = count_addin()
ADDINS = [handle_feed_images, counter]
class HookTestFeed(BaseImageTestFeed):
content = """
<rss><channel>
<image><url>http://example.org/image.png</url></image>
</channel></rss>
"""
def pass1(feed):
# pass1: test everything ok, no errors
assert counter.feed_image == 1 # always called
assert counter.update_feed_image == 1 # called bc prev success
assert counter.feed_image_updated == 1 # called bc prev success
assert counter.feed_image_failed == 0 # not called on success
# pass2: test failure in feed_image
counter.fail = "feed_image"
def pass2(feed):
assert counter.feed_image == 2 # always called
assert counter.update_feed_image == 1 # unchanged bc error
assert counter.feed_image_updated == 1 # unchanged bc error
assert counter.feed_image_failed == 1 # called due to error
# pass3: test failure in update_feed_image
counter.fail = "update_feed_image"
def pass3(feed):
assert counter.feed_image == 3 # always called
assert counter.update_feed_image == 2 # called bc success
assert counter.feed_image_updated == 1 # unchanged bc error
assert counter.feed_image_failed == 2 # called due to error
# pass4: test failure in feed_image_updated
# (although it really shouldn't happen)
counter.fail = "feed_image_updated"
def pass4(feed):
assert counter.feed_image == 4 # always called
assert counter.update_feed_image == 3 # called bc success
assert counter.feed_image_updated == 2 # called bc success
assert counter.feed_image_failed == 3 # called due to error
# pass5: test feed_image non-error stop
counter.returnvalue["feed_image"] = True
def pass5(feed):
assert counter.feed_image == 5 # always called
assert counter.update_feed_image == 3 # unchanged bc skip
assert counter.feed_image_updated == 2 # unchanged bc skip
assert counter.feed_image_failed == 3 # not called on skip
feedev.testcaller()
def test_basic():
"""
Only the feeds in this local namespace are used, ``GlobalFeed``
is ignored.
"""
class LocalFeed1(GlobalFeed):
pass
class LocalFeed2(GlobalFeed):
pass
feedev.testcaller()
assert called == 2
def test_basic():
ADDINS = [
# some unicode fields, both "special" generated fields
collect_feed_image_data('href', 'title', 'extension', 'filename')
]
class TestFeed(feedev.Feed):
content = """
<rss><channel>
<title>test-feed</title>
<image>
{% =1 %}
<url>http://example.org/blog/image.jpg</url>
<title>Example.org Cover</title>
<link>http://example.org/</link>
{% end %}
{% =2 %}
<url>http://example.org/new-cover.png</url>
{% end %}
</image>
</channel></rss>
"""
def pass1(feed):
# initial values are picked up
assert feed.image_title == 'Example.org Cover'
assert feed.image_href == 'http://example.org/blog/image.jpg'
assert feed.image_extension == 'jpg'
assert feed.image_filename == 'image.jpg'
# [bug] Ensure those fields we are checking are actually,
# really, model fields, avaible in the database, not just
# attributes, while the real fields are using their standard
# name (i.e. "title" instead of "image_title").
assert 'image_title' in [c._detect_attr_name(feed.__class__)
for c in feed._storm_columns.keys()]
def pass2(feed):
# changed values are picked up
assert feed.image_title == ''
assert feed.image_href == 'http://example.org/new-cover.png'
assert feed.image_extension == 'png'
assert feed.image_filename == 'new-cover.png'
feedev.testcaller()
def test_bozo():
ADDINS = [collect_feed_image_data('href')]
class BozoFeed(feedev.Feed):
content = """
<rss><channel>
<title>test-feed</title>
<image>
<url>http://example.org/blog/image.jpg</url>
</channel></rss>
"""
def pass1(feed):
# feed is bozo (image tag not closed), but addin is active
# nevertheless.
assert feed.image_href == 'http://example.org/blog/image.jpg'
feedev.testcaller()
def test_unicode():
"""Regression test for a bug that caused storm to raise an
"Expected unicode" TypeError in certain circumstances (the addin
was trying to assign a str).
"""
ADDINS = [collect_feed_image_data('href', 'extension')]
class ImgWithExtByContent(feedev.File):
url = 'http://images/validimage'
content = ValidPNGImage
class ImgWithExtByHeaders(feedev.File):
url = 'http://images/validimage'
headers = {'Content-Type': 'image/gif; charset=utf8'}
class BozoFeed(feedev.Feed):
content = """
<rss><channel>
<image>
<!-- extension from url is unicode -->
{% =1 %}<url>http://example.org/blog/image.jpg</url>{% end %}
<!-- extension read from headers is unicode -->
{% =2 %}<url>"""+ImgWithExtByHeaders.url+"""</url>{% end %}
<!-- extension from pil content is unicode -->
{% =3 %}<url>"""+ImgWithExtByContent.url+"""</url>{% end %}
</image>
</channel></rss>
"""
# These assertions are here for completion's sake, but
# normally shouldn't ever fail, since the bug we are
# regression here really raises a TypeError.
def pass1(feed):
assert type(feed.image_href) == unicode
assert type(feed.image_extension) == unicode
def pass2(feed):
assert type(feed.image_extension) == unicode
def pass3(feed):
assert type(feed.image_extension) == unicode
feedev.testcaller()
def test_force_format():
ADDINS = [handle_feed_images,
store_feed_images((tempdir, '%(model_id)s.gif'), format='gif')]
class TestImage(feedev.File):
content = ValidPNGImage
class TestFeed(feedev.Feed):
content = FeedWithImage % TestImage.url
def pass1(feed):
# check that the image was stored
imgfile = path.join(tempdir, '%s.%s' % (feed.id, 'gif'))
assert path.exists(imgfile)
# check that it was stored as a true gif file
from PIL import Image
assert Image.open(imgfile).format == 'GIF'
feedev.testcaller()
def test_custom_fieldname():
"""[bug] Make sure that the extended syntax, that allows the collector
to support a "standard field" field by the name X, but map it to a
model field with a different name, works as expected.
In particular we had a bug initially that caused the wrong field to be
added to the generated models.
"""
class my_collector(base_data_collector):
model_name = 'feed'
standard_fields = {
'title': {'target': 'my_title_field', 'field': (Unicode, (), {})}
}
ADDINS = [my_collector('title')]
class MyFeed(test.Feed):
def pass1(feed):
assert hasattr(feed, 'my_title_field')
assert hasattr(db.models.Feed, 'my_title_field')
test.testcaller()
def test_failure_reset():
"""Image fails if no valid extension (=no image).
"""
counter = image_hook_counter()
ADDINS = [feed_image_restrict_mediatypes(('image/png', 'image/gif')), counter]
class TestFeedImage(feedev.File):
content = ""
def headers(p):
if p == 1: return {'Content-Type': 'image/png'}
elif p == 2: return {'Content-Type': 'image/jpeg'}
class TestFeed(feedev.Feed):
content = FeedWithImage % TestFeedImage.url
def pass1(feed):
assert counter.failure == 0
def pass2(feed):
# invalid media types cause the image to fail completely
assert counter.failure == 1
feedev.testcaller()
def test_download_chunk():
"""Sepcial test for the ``feed_image_download_chunk`` hook, that
is triggered while an image is downloaded.
"""
class force_download_dummy(addins.base):
def on_update_feed_image(self, feed, image_dict, image):
image.chunk_size = 10 # read 10 bytes at one time
image.data.read()
counter = count_addin()
ADDINS = [handle_feed_images, force_download_dummy, counter]
class TestImage(feedev.File):
content = "a" * 35 # 35 bytes total size
class TestFeed(feedev.Feed):
content = FeedWithImage % TestImage.url
def pass1(feed):
# 34 bytes ala 10 byte blocks = 4 reads
assert counter.feed_image_download_chunk == 4
feedev.testcaller()
def test_unsupported_extensions():
"""Regression test for a bug that caused a ``KeyError`` to be raised
when the extension of a target filename of a thumbnail did not refer
to a valid image format, as supported by PIL.
Instead, we want the thumbnail to be saved with whatever format the
image originally has.
The same bug also affected normal saving of a feed image (via the
``store_feed_images`` addin) under certain circumstances (when saved
through PIL), and this is tested in ``test_remote_image.py``.
"""
ADDINS = [feed_image_thumbnails(
((100,100),),
(tempdir, '%(model_id)s-thumb'))] # note: without extension
class TestImage(feedev.File):
content = ValidPNGImage
url = 'http://bla/strange-extension.aspx'
class TestFeed(feedev.Feed):
content = FeedWithImage % TestImage.url
def pass1(feed):
# check that the image was stored
imgfile = path.join(tempdir, '%s-thumb' % (feed.id))
assert path.exists(imgfile)
# check that it was stored as a png file, the source image format
from PIL import Image
i = Image.open(imgfile)
assert i.format == 'PNG'
# and most importantly, no exception is raised
feedev.testcaller()
def test_special_fields():
"""Test the special fields that every subclass will be able to provide.
"""
class my_collector(base_data_collector):
model_name = 'feed'
standard_fields = {}
def on_after_parse(self, feed, data_dict):
return self._process(feed, data_dict.feed)
ADDINS = [my_collector(__now='last_processed')]
class BozoFeed(test.Feed):
# make sure the content here is not a valid feed, we want to make
# sure that this works even when feed the is bozo.
content = """<bozo>"""
def pass1(feed):
assert hasattr(feed, 'last_processed')
assert hasattr(db.models.Feed, 'last_processed')
# field should now have a value not too far from right now
assert abs(feed.last_processed - datetime.datetime.utcnow()).seconds < 10
test.testcaller()
def test_failure_reset():
"""Image data is cleared if the image fails to process.
"""
class FailDummy(addins.base):
active = False
def on_update_feed_image(self, *a, **kw):
if self.active:
raise ImageError()
fail_dummy = FailDummy
ADDINS = [collect_feed_image_data('href', ), fail_dummy]
class TestFeed(feedev.Feed):
content = """
<rss><channel>
<title>test-feed</title>
<image>
<url>http://example.org/blog/image.jpg</url>
<title>Example.org Cover</title>
<link>http://example.org/</link>
</image>
</channel></rss>
"""
def pass1(feed):
# initially, the value is picked up
assert feed.image_href == 'http://example.org/blog/image.jpg'
# ...but in the next pass, image handling will fail...
fail_dummy.active = True
def pass2(feed):
# ...and the value is removed.
assert feed.image_href == ''
feedev.testcaller()
def test_basic():
"""Test that hooks are triggered correctly, depending on whether a
feed image is given or not.
This is in addition to the specific tests for each hook.
"""
counter = image_hook_counter()
ADDINS = [handle_feed_images(), counter]
class TestFeed(feedev.Feed):
content = """
<rss><channel>
<title>test-feed</title>
{% =2 %}<image></image>{% end %}
{% =3 %}<image><url></url></image>{% end %}
{% =4 %}<image><url>http://host/image.gif</url></image>{% end %}
</channel></rss>
"""
def pass1(feed):
# no tag at all
assert counter.called == 0
def pass2(feed):
# no url tag
assert counter.called == 0
def pass3(feed):
# empty url
assert counter.called == 0
def pass4(feed):
# finally, everything is ok!
assert counter.called == 1
feedev.testcaller()
def test_no_passes():
try:
feedev.testcaller()
except Exception, e:
assert 'nothing to test' in str(e)