def load(context, url, callback):
enable_http_loader = context.config.get('AWS_ENABLE_HTTP_LOADER',
default=False)
if enable_http_loader and 'http' in url:
return http_loader.load(context, url, callback)
url = urllib2.unquote(url)
if context.config.S3_LOADER_BUCKET:
bucket = context.config.S3_LOADER_BUCKET
else:
bucket, url = _get_bucket(url)
if not _validate_bucket(context, bucket):
return callback(None)
bucket_loader = Bucket(
connection=thumbor_aws.connection.get_connection(context), name=bucket)
file_key = bucket_loader.get_key(url)
if not file_key:
return http_loader.load(context, url, callback)
return callback(file_key.read())
def test_load_with_utf8_url(self):
url = self.get_url(quote(u'/maracujá.jpg'.encode('utf-8')))
config = Config()
ctx = Context(None, config, None)
with expect.error_not_to_happen(UnicodeDecodeError):
loader.load(ctx, url, self.stop)
self.wait()
def test_load_with_utf8_url(self):
url = self.get_url(quote(u'/maracujá.jpg'.encode('utf-8')))
config = Config()
ctx = Context(None, config, None)
with expect.error_not_to_happen(UnicodeDecodeError):
loader.load(ctx, url, self.stop)
self.wait()
def callback_wrapper(result):
if result.successful:
logger.info("Image " + path + " found on C5")
callback(result)
else:
# If file_loader failed try http_loader
logger.info("Image " + path + " NOT found on C5")
http_loader.load(context, path, callback)
def topic(self, callback):
url = self.get_url('/')
loader.http_client = self._http_client
config = Config()
config.HTTP_LOADER_FORWARD_USER_AGENT = True
ctx = Context(None, config, None, HandlerMock({"User-Agent": "test-user-agent"}))
loader.load(ctx, url, callback)
def topic(self, callback):
url = self.get_url('/')
loader.http_client = self._http_client
config = Config()
config.ALLOWED_SOURCES = ['s.glbimg.com']
ctx = Context(None, config, None)
loader.load(ctx, url, callback)
def topic(self, callback):
url = self.get_url('/')
loader.http_client = self._http_client
config = Config()
config.ALLOWED_SOURCES = ['s.glbimg.com']
ctx = Context(None, config, None)
loader.load(ctx, url, callback)
def topic(self, callback):
url = self.get_url('/')
loader.http_client = self._http_client
config = Config()
config.HTTP_LOADER_FORWARD_USER_AGENT = True
ctx = Context(None, config, None, HandlerMock({"User-Agent": "test-user-agent"}))
loader.load(ctx, url, callback)
def topic(self, callback):
url = self.get_url('/')
loader.http_client = self._http_client
config = Config()
config.HTTP_LOADER_FORWARD_USER_AGENT = True
config.HTTP_LOADER_DEFAULT_USER_AGENT = "DEFAULT_USER_AGENT"
ctx = Context(None, config, None, HandlerMock({}))
loader.load(ctx, url, callback)
def test_load_with_callback(self):
url = self.get_url('/')
config = Config()
ctx = Context(None, config, None)
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).to_equal('Hello')
expect(result.successful).to_be_true()
def topic(self):
self.async_topic = ""
def get_contents(contents):
self.async_topic = contents
url = self._get_url("/")
loader.load(url, get_contents)
return self.async_topic
def test_load_with_user_agent(self):
url = self.get_url('/')
config = Config()
config.HTTP_LOADER_FORWARD_USER_AGENT = True
ctx = Context(None, config, None, HandlerMock({"User-Agent": "test-user-agent"}))
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).to_equal('test-user-agent')
def test_load_with_user_agent(self):
url = self.get_url('/')
config = Config()
config.HTTP_LOADER_FORWARD_USER_AGENT = True
ctx = Context(None, config, None, HandlerMock({"User-Agent": "test-user-agent"}))
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).to_equal('test-user-agent')
def topic(self, callback):
url = self.get_url('/')
loader.http_client = self._http_client
config = Config()
config.HTTP_LOADER_FORWARD_USER_AGENT = True
config.HTTP_LOADER_DEFAULT_USER_AGENT = "DEFAULT_USER_AGENT"
ctx = Context(None, config, None, HandlerMock({}))
loader.load(ctx, url, callback)
def test_load_with_callback(self):
url = self.get_url('/')
config = Config()
ctx = Context(None, config, None)
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).to_equal('Hello')
expect(result.successful).to_be_true()
def test_load_with_curl(self):
url = self.get_url('/')
config = Config()
config.HTTP_LOADER_CURL_ASYNC_HTTP_CLIENT = True
ctx = Context(None, config, None)
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).to_equal('Hello')
expect(result.successful).to_be_true()
def test_load_with_curl(self):
url = self.get_url('/')
config = Config()
config.HTTP_LOADER_CURL_ASYNC_HTTP_CLIENT = True
ctx = Context(None, config, None)
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).to_equal('Hello')
expect(result.successful).to_be_true()
def test_load_with_default_user_agent(self):
url = self.get_url("/")
config = Config()
config.HTTP_LOADER_FORWARD_USER_AGENT = True
config.HTTP_LOADER_DEFAULT_USER_AGENT = "DEFAULT_USER_AGENT"
ctx = Context(None, config, None, HandlerMock({}))
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).to_equal("DEFAULT_USER_AGENT")
def test_load_without_curl_but_speed_timeout(self):
url = self.get_url('/')
config = Config()
config.HTTP_LOADER_CURL_LOW_SPEED_TIME = 1
config.HTTP_LOADER_CURL_LOW_SPEED_LIMIT = 1000000000000
ctx = Context(None, config, None)
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).to_equal('Hello')
expect(result.successful).to_be_true()
def callback_wrapper(result):
if result.successful:
callback(result)
else:
# If file_loader failed try http_loader
if (path.find('http') != -1):
http_loader.load(context, path, callback)
else:
result = LoaderResult()
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
callback(result)
def test_load_without_curl_but_speed_timeout(self):
url = self.get_url('/')
config = Config()
config.HTTP_LOADER_CURL_LOW_SPEED_TIME = 1
config.HTTP_LOADER_CURL_LOW_SPEED_LIMIT = 1000000000000
ctx = Context(None, config, None)
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).to_equal('Hello')
expect(result.successful).to_be_true()
def test_load_with_timeout(self):
url = self.get_url('/')
config = Config()
config.HTTP_LOADER_CURL_ASYNC_HTTP_CLIENT = True
config.HTTP_LOADER_REQUEST_TIMEOUT = 1
ctx = Context(None, config, None)
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).to_be_null()
expect(result.successful).to_be_false()
def test_load_with_timeout(self):
url = self.get_url('/')
config = Config()
config.HTTP_LOADER_CURL_ASYNC_HTTP_CLIENT = True
config.HTTP_LOADER_REQUEST_TIMEOUT = 1
ctx = Context(None, config, None)
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).to_be_null()
expect(result.successful).to_be_false()
def test_load_with_all_headers(self):
url = self.get_url('/')
config = Config()
config.HTTP_LOADER_FORWARD_ALL_HEADERS = True
handler_mock_options = {"X-Test": "123", "DNT": "1", "X-Server": "thumbor"}
ctx = Context(None, config, None, HandlerMock(handler_mock_options))
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).to_include("Dnt:1\n")
expect(result.buffer).to_include("X-Server:thumbor\n")
expect(result.buffer).to_include("X-Test:123\n")
def load(context, url, callback):
logger.debug("*** LOADER.load" )
start = datetime.datetime.now()
if HTTP_RE.match(url):
method = 'http'
HttpLoader.load(context, url, callback)
else:
method = 's3'
logger.debug("*** s3")
S3Loader.load(context, url, callback)
finish = datetime.datetime.now()
metric_name = "thumbor.loader.%s.request_time" % method
logger.debug("*** librato submit")
LIBRATO_API.submit(metric_name, (finish - start).total_seconds() * 1000, description="original file request time (ms)")
def test_before_expiration_returns_something(self):
options.STORAGE_EXPIRATION_SECONDS = 1000
image_url = 'www.globo.com/media/globocom/img/sprite1.png?8'
http_loaded = http.load(image_url)
storage = Storage()
storage.put(image_url, http_loaded)
assert storage.get(image_url)
def load(context, url, callback):
enable_http_loader = context.config.get('AWS_ENABLE_HTTP_LOADER', default=False)
if enable_http_loader and 'http' in url:
return http_loader.load(context, url, callback)
url = urllib2.unquote(url)
if context.config.S3_LOADER_BUCKET:
bucket = context.config.S3_LOADER_BUCKET
else:
bucket, url = _get_bucket(url)
if not _validate_bucket(context, bucket):
return callback(None)
bucket_loader = Bucket(
connection=thumbor_aws.connection.get_connection(context),
name=bucket
)
file_key = bucket_loader.get_key(url)
if not file_key:
return callback(None)
return callback(file_key.read())
def test_load_with_empty_accept(self):
url = self.get_url('/')
config = Config()
handler_mock_options = {
"Accept-Encoding": "gzip",
"User-Agent": "Thumbor",
"Host": "localhost",
"Accept": "",
"X-Server": "thumbor"
}
ctx = Context(None, config, None, HandlerMock(handler_mock_options))
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).to_include("Accept:image/*;q=0.9,*/*;q=0.1\n")
def test_loads_image(self):
image_url = 'www.globo.com/media/globocom/img/sprite1.png'
http_loaded = http.load(image_url)
response = urllib2.urlopen('http://' + image_url).read()
self.assertEqual(http_loaded, response,
'http_loaded is not the same as the http request')
def test_load_with_some_excluded_headers(self):
url = self.get_url('/')
config = Config()
handler_mock_options = {
"Accept-Encoding": "gzip",
"User-Agent": "Thumbor",
"Host": "localhost",
"Accept": "*/*",
"X-Server": "thumbor"
}
ctx = Context(None, config, None, HandlerMock(handler_mock_options))
loader.load(ctx, url, self.stop)
result = self.wait()
expect(result).to_be_instance_of(LoaderResult)
expect(result.buffer).Not.to_include("X-Server:thumbor")
def test_should_return_a_future(self):
url = self.get_url('/')
config = Config()
ctx = Context(None, config, None)
future = loader.load(ctx, url)
expect(isinstance(future, Future)).to_be_true()
def test_should_return_a_future(self):
url = self.get_url('/')
config = Config()
ctx = Context(None, config, None)
future = loader.load(ctx, url)
expect(isinstance(future, Future)).to_be_true()
def test_stores_image_does_not_store_crypt_key_if_not_in_options(self):
rm_storage()
options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False
image_url = 'www.globo.com/media/globocom/img/sprite1.png'
http_loaded = http.load(image_url)
storage = Storage()
storage.put(image_url, http_loaded)
assert not collection.find_one({'path': image_url}).has_key('crypto'), 'crypto key should not be found into the storage'
def test_after_expiration_returns_none(self):
options.STORAGE_EXPIRATION_SECONDS = 1
image_url = 'www.globo.com/media/globocom/img/sprite1.png?9'
http_loaded = http.load(image_url)
storage = Storage()
storage.put(image_url, http_loaded)
time.sleep(2)
assert storage.get(image_url) is None
def test_stores_image(self):
rm_storage()
options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False
image_url = 'www.globo.com/media/globocom/img/sprite1.png'
http_loaded = http.load(image_url)
storage = Storage()
storage.put(image_url, http_loaded)
assert collection.find_one({'path': image_url}), 'image not found into the storage'
def test_stores_image(self):
rm_storage()
options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False
image_url = 'www.globo.com/media/globocom/img/sprite1.png'
http_loaded = http.load(image_url)
storage = file_storage.Storage()
storage.put(image_url, http_loaded)
assert exists('/tmp/thumbor/storage/www.globo.com/media/globocom/img/sprite1.png'), 'image not found into the storage'
def test_get_crypto_for_url(self):
image_url = 'www.globo.com/media/globocom/img/sprite1.png?6'
http_loaded = http.load(image_url)
storage = Storage()
storage.put(image_url, http_loaded)
assert storage.get_crypto(image_url) == options.SECURITY_KEY
def test_stores_image_does_not_store_crypt_key_if_not_in_options(self):
rm_storage()
options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False
image_url = 'www.globo.com/media/globocom/img/sprite1.png'
http_loaded = http.load(image_url)
storage = file_storage.Storage()
storage.put(image_url, http_loaded)
assert not exists('/tmp/thumbor/storage/www.globo.com/media/globocom/img/sprite1.txt'), 'crypto key was found into the storage'
def test_stores_image(self):
rm_storage()
options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False
image_url = 'www.globo.com/media/globocom/img/sprite1.png'
http_loaded = http.load(image_url)
storage = Storage()
storage.put(image_url, http_loaded)
assert collection.find_one({'path': image_url
}), 'image not found into the storage'
def test_gets_image(self):
options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False
image_url = 'www.globo.com/media/globocom/img/sprite1.png?2'
http_loaded = http.load(image_url)
storage = Storage()
storage.put(image_url, http_loaded)
stored_image = storage.get(image_url)
assert stored_image and not isinstance(stored_image, dict), 'image not found into the storage'
assert http_loaded == stored_image, 'stored image did not match the loaded one'
def test_stores_image_does_not_store_crypt_key_if_not_in_options(self):
rm_storage()
options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False
image_url = 'www.globo.com/media/globocom/img/sprite1.png'
http_loaded = http.load(image_url)
storage = Storage()
storage.put(image_url, http_loaded)
assert not collection.find_one({
'path': image_url
}).has_key('crypto'), 'crypto key should not be found into the storage'
def test_stores_image(self):
rm_storage()
options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False
image_url = 'www.globo.com/media/globocom/img/sprite1.png'
http_loaded = http.load(image_url)
storage = file_storage.Storage()
storage.put(image_url, http_loaded)
assert exists(
'/tmp/thumbor/storage/www.globo.com/media/globocom/img/sprite1.png'
), 'image not found into the storage'
def test_stores_image_does_not_store_crypt_key_if_not_in_options(self):
rm_storage()
options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False
image_url = 'www.globo.com/media/globocom/img/sprite1.png'
http_loaded = http.load(image_url)
storage = file_storage.Storage()
storage.put(image_url, http_loaded)
assert not exists(
'/tmp/thumbor/storage/www.globo.com/media/globocom/img/sprite1.txt'
), 'crypto key was found into the storage'
def test_stores_image_does_not_store_crypt_key_if_not_in_options(self):
options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False
image_url = 'www.globo.com/media/globocom/img/sprite1.png?3'
http_loaded = http.load(image_url)
storage = Storage()
storage.put(image_url, http_loaded)
image = get_image_by_path(image_url)
assert not image['security_key']
def test_stores_image(self):
options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False
image_url = 'www.globo.com/media/globocom/img/sprite1.png?1'
http_loaded = http.load(image_url)
storage = Storage()
storage.put(image_url, http_loaded)
image = get_image_by_path(image_url)
assert image['url'] == image_url
assert image['contents']
assert image['last_update']
def load(context, url, callback):
url = quote_url(url)
match = S3_RE.match(url)
def callback_wrapper(result):
if result.successful:
callback(result)
else:
logger.info('s3 {0}'.format(
os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')))
)
# If not on efs, try s3
S3Loader.load(context,
os.path.join(match.group('bucket').rstrip('/'),
match.group('path').lstrip('/')),
callback)
# If melody s3 file, first try to load from efs
if match:
logger.info('BOTO {0}'.format(match.group('path')))
# TEMP try s3 direct
S3Loader.load(context,
os.path.join(match.group('bucket').rstrip('/'),
match.group('path').lstrip('/')),
callback)
# FileLoader.load(context, match.group('path'), callback_wrapper)
# else get from the internet
elif HTTP_RE.match(url):
logger.info('WEB {0}'.format(url))
HttpLoader.load(context, url, callback)
else:
logger.info('FILE {0}'.format(url))
result = LoaderResult()
result.successful = False
result.error = LoaderResult.ERROR_NOT_FOUND
# callback(result)
# TEMP enable file loader
FileLoader.load(context, url, callback)
def test_finding_crypto_file(self):
rm_storage()
image_url = 'www.globo.com/media/globocom/img/sprite1.png'
http_loaded = http.load(image_url)
storage = file_storage.Storage()
storage.put(image_url, http_loaded)
crypto_file = '/tmp/thumbor/storage/www.globo.com/media/globocom/img/sprite1.txt'
assert exists(crypto_file), 'crypto key was not found into the storage'
assert file(crypto_file).read() == options.SECURITY_KEY
def test_finding_crypto_file(self):
options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True
options.SECURITY_KEY = 'MY-SECURITY-KEY-5'
image_url = 'www.globo.com/media/globocom/img/sprite1.png?5'
http_loaded = http.load(image_url)
storage = Storage()
storage.put(image_url, http_loaded)
image = get_image_by_path(image_url)
assert image['security_key'] == options.SECURITY_KEY
def test_storing_an_empty_key_raises(self):
options.SECURITY_KEY = False
options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True
image_url = 'www.globo.com/media/globocom/img/sprite1.png?4'
http_loaded = http.load(image_url)
storage = Storage()
try:
storage.put(image_url, http_loaded)
except RuntimeError, err:
assert str(err) == "STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified"
return
def test_storing_an_empty_key_raises(self):
rm_storage()
options.SECURITY_KEY = False
image_url = 'www.globo.com/media/globocom/img/sprite1.png'
http_loaded = http.load(image_url)
storage = file_storage.Storage()
try:
storage.put(image_url, http_loaded)
except RuntimeError, err:
assert str(err) == "STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified"
return
