def load(context, url, callback):
# type: (Context, str, Callable[..., Any]) -> None
source_type, encoded_url = separate_url_and_source_type(url)
actual_url = base64.urlsafe_b64decode(urllib.parse.unquote(encoded_url)).decode('utf-8')
if source_type not in (THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE,
THUMBOR_EXTERNAL_TYPE):
callback(get_not_found_result())
logging.warning('INVALID SOURCE TYPE: ' + source_type)
return
if source_type == THUMBOR_S3_TYPE:
if actual_url.startswith('/user_uploads/'):
actual_url = actual_url[len('/user_uploads/'):]
else:
raise AssertionError("Unexpected s3 file.")
s3_loader.load(context, actual_url, callback)
elif source_type == THUMBOR_LOCAL_FILE_TYPE:
if actual_url.startswith('/user_uploads/'):
actual_url = actual_url[len('/user_uploads/'):]
local_file_path_prefix = 'files/'
else:
raise AssertionError("Unexpected local file.")
patched_local_url = local_file_path_prefix + actual_url
file_loader.load(context, patched_local_url, callback)
elif source_type == THUMBOR_EXTERNAL_TYPE:
https_loader.load(context, actual_url, callback)
def topic(self, callback):
conf = Config()
conf.define('TC_AWS_ALLOWED_BUCKETS', [], '')
context = Context(config=conf)
s3_loader.load(context, '/'.join([s3_bucket, IMAGE_PATH]),
callback)
def load(context, url, callback):
# type: (Context, str, Callable[..., Any]) -> None
source_type, encoded_url = separate_url_and_source_type(url)
actual_url = base64.urlsafe_b64decode(urllib.parse.unquote(encoded_url))
if source_type not in (THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE,
THUMBOR_EXTERNAL_TYPE):
callback(get_not_found_result())
logging.warning('INVALID SOURCE TYPE: ' + source_type)
return
if source_type == THUMBOR_S3_TYPE:
if actual_url.startswith('/user_uploads/'): # type: ignore # python 2 type differs from python 3 type
actual_url = actual_url[len('/user_uploads/'):]
else:
raise AssertionError("Unexpected s3 file.")
s3_loader.load(context, actual_url, callback)
elif source_type == THUMBOR_LOCAL_FILE_TYPE:
if actual_url.startswith('/user_uploads/'): # type: ignore # python 2 type differs from python 3 type
actual_url = actual_url[len('/user_uploads/'):]
local_file_path_prefix = 'files/'
else:
raise AssertionError("Unexpected local file.")
patched_local_url = local_file_path_prefix + actual_url # type: ignore # python 2 type differs from python 3 type
file_loader.load(context, patched_local_url, callback)
elif source_type == THUMBOR_EXTERNAL_TYPE:
https_loader.load(context, actual_url, callback)
def should_not_redirect_to_http_if_not_prefixed_with_scheme(
self, topic, load_sync_patch):
def callback(*args):
pass
s3_loader.load(topic, 'foo.bar', callback)
expect(load_sync_patch.called).to_be_false()
def test_should_use_http_loader(self, load_sync_patch):
def cb(a, b, callback, *args, **kwargs):
callback('foobar')
return None
load_sync_patch.side_effect = cb
conf = Config(TC_AWS_ENABLE_HTTP_LOADER=True)
s3_loader.load(Context(config=conf), 'http://foo.bar')
self.assertTrue(load_sync_patch.called)
def topic(self, callback):
conn = boto.connect_s3()
bucket = conn.create_bucket(s3_bucket)
k = Key(bucket)
k.key = '/'.join(['root_path', IMAGE_PATH])
k.set_contents_from_string(IMAGE_BYTES)
conf = Config()
conf.define('TC_AWS_LOADER_BUCKET', s3_bucket, '')
conf.define('TC_AWS_LOADER_ROOT_PATH', 'root_path', '')
context = Context(config=conf)
s3_loader.load(context, IMAGE_PATH, callback)
def topic(self, callback):
conn = boto.connect_s3()
bucket = conn.create_bucket(s3_bucket)
k = Key(bucket)
k.key = '/'.join(['root_path', IMAGE_PATH])
k.set_contents_from_string(IMAGE_BYTES)
conf = Config()
conf.define('TC_AWS_LOADER_BUCKET', s3_bucket, '')
conf.define('TC_AWS_LOADER_ROOT_PATH', 'root_path', '')
context = Context(config=conf)
s3_loader.load(context, IMAGE_PATH, callback)
def load(context, url, callback):
# type: (Context, str, Callable[..., Any]) -> None
source_type, encoded_url = separate_url_and_source_type(url)
actual_url = base64.urlsafe_b64decode(urllib.parse.unquote(encoded_url))
if source_type not in (THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE,
THUMBOR_EXTERNAL_TYPE):
callback(get_not_found_result())
logging.warning('INVALID SOURCE TYPE: ' + source_type)
return
if source_type == THUMBOR_S3_TYPE:
s3_loader.load(context, actual_url, callback)
elif source_type == THUMBOR_LOCAL_FILE_TYPE:
patched_local_url = 'files/' + actual_url # type: ignore # python 2 type differs from python 3 type
file_loader.load(context, patched_local_url, callback)
elif source_type == THUMBOR_EXTERNAL_TYPE:
https_loader.load(context, actual_url, callback)
def load(context, url, callback):
# type: (Context, str, Callable[..., Any]) -> None
source_type, encoded_url = separate_url_and_source_type(url)
actual_url = base64.urlsafe_b64decode(urllib.parse.unquote(encoded_url))
if source_type not in (THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE,
THUMBOR_EXTERNAL_TYPE):
callback(get_not_found_result())
logging.warning('INVALID SOURCE TYPE: ' + source_type)
return
if source_type == THUMBOR_S3_TYPE:
s3_loader.load(context, actual_url, callback)
elif source_type == THUMBOR_LOCAL_FILE_TYPE:
patched_local_url = 'files/' + actual_url # type: ignore # python 2 type differs from python 3 type
file_loader.load(context, patched_local_url, callback)
elif source_type == THUMBOR_EXTERNAL_TYPE:
https_loader.load(context, actual_url, callback)
def test_can_validate_buckets(self):
conf = Config(
TC_AWS_ALLOWED_BUCKETS=['whitelist_bucket'],
TC_AWS_LOADER_BUCKET=None,
)
image = yield s3_loader.load(Context(config=conf),
'/'.join([s3_bucket, IMAGE_PATH]))
self.assertIsNone(image.buffer)
def load(context, url, callback):
# type: (Context, str, Callable[..., Any]) -> None
url = urllib.parse.unquote(url)
url_params = get_url_params(url)
source_type = url_params.get('source_type')
if not sign_is_valid(url, context) or source_type not in (
THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE):
callback(get_not_found_result())
return
url = url.rsplit('?', 1)[0]
if source_type == THUMBOR_S3_TYPE:
s3_loader.load(context, url, callback)
elif source_type == THUMBOR_LOCAL_FILE_TYPE:
file_loader.load(context, url, callback)
elif source_type == THUMBOR_EXTERNAL_TYPE:
http_loader.load_sync(context,
url,
callback,
normalize_url_func=http_loader._normalize_url)
def load(context, url, callback):
# type: (Context, str, Callable[..., Any]) -> None
url = urllib.parse.unquote(url)
url_params = get_url_params(url)
source_type = url_params.get('source_type')
if not sign_is_valid(url, context) or source_type not in (
THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE):
callback(get_not_found_result())
return
url = url.rsplit('?', 1)[0]
if source_type == THUMBOR_S3_TYPE:
s3_loader.load(context, url, callback)
elif source_type == THUMBOR_LOCAL_FILE_TYPE:
file_loader.load(context, url, callback)
elif source_type == THUMBOR_EXTERNAL_TYPE:
http_loader.load_sync(
context,
url,
callback,
normalize_url_func=http_loader._normalize_url)
def test_can_load_image(self):
client = botocore.session.get_session().create_client('s3')
client.create_bucket(Bucket=s3_bucket)
client.put_object(
Bucket=s3_bucket,
Key=''.join(['root_path', IMAGE_PATH]),
Body=IMAGE_BYTES,
ContentType='image/jpeg',
)
conf = Config(TC_AWS_LOADER_BUCKET=s3_bucket,
TC_AWS_LOADER_ROOT_PATH='root_path')
image = yield s3_loader.load(Context(config=conf), IMAGE_PATH)
self.assertEqual(image, IMAGE_BYTES)
def should_redirect_to_http(self, topic, load_sync_patch):
def callback(*args):
pass
s3_loader.load(topic, 'http://foo.bar', callback)
expect(load_sync_patch.called).to_be_true()
def should_redirect_to_http(self, topic):
with patch(
'thumbor.loaders.http_loader.load_sync') as mock_load_sync:
yield s3_loader.load(topic, 'http://foo.bar')
expect(mock_load_sync.called).to_be_true()
def should_not_redirect_to_http_if_not_prefixed_with_scheme(self, topic, load_sync_patch):
def callback(*args):
pass
s3_loader.load(topic, 'foo.bar', callback)
expect(load_sync_patch.called).to_be_false()
def should_redirect_to_http(self, topic, load_sync_patch):
def callback(*args):
pass
s3_loader.load(topic, 'http://foo.bar', callback)
expect(load_sync_patch.called).to_be_true()
def topic(self, callback):
conf = Config()
conf.define('TC_AWS_ALLOWED_BUCKETS', [], '')
context = Context(config=conf)
s3_loader.load(context, '/'.join([s3_bucket, IMAGE_PATH]), callback)
def test_should_not_use_http_loader_if_not_prefixed_with_scheme(
self, load_sync_patch):
conf = Config(TC_AWS_ENABLE_HTTP_LOADER=True)
yield s3_loader.load(Context(config=conf), 'foo/bar')
self.assertFalse(load_sync_patch.called)
def should_redirect_to_http(self, topic):
with patch('thumbor.loaders.http_loader.load_sync') as mock_load_sync:
yield s3_loader.load(topic, 'http://foo.bar')
expect(mock_load_sync.called).to_be_true()
def should_load_from_s3(self, topic):
image = yield s3_loader.load(topic, '/'.join([s3_bucket, IMAGE_PATH]))
expect(image).to_equal(None)
def should_load_from_s3(self, topic):
image = yield s3_loader.load(topic,
'/'.join([s3_bucket, IMAGE_PATH]))
expect(image).to_equal(None)
def should_load_from_s3(self, topic):
image = yield s3_loader.load(topic, '/'.join(['root_path', IMAGE_PATH]))
expect(image).to_equal(IMAGE_BYTES)
def should_load_from_s3(self, topic):
image = yield s3_loader.load(topic,
'/'.join(['root_path', IMAGE_PATH]))
expect(image).to_equal(IMAGE_BYTES)