def load(context, url, callback): enable_http_loader = context.config.get('AWS_ENABLE_HTTP_LOADER', default=False) if enable_http_loader and 'http' in url: return http_loader.load(context, url, callback) url = urllib2.unquote(url) if context.config.S3_LOADER_BUCKET: bucket = context.config.S3_LOADER_BUCKET else: bucket, url = _get_bucket(url) if not _validate_bucket(context, bucket): return callback(None) bucket_loader = Bucket( connection=thumbor_aws.connection.get_connection(context), name=bucket) file_key = bucket_loader.get_key(url) if not file_key: return http_loader.load(context, url, callback) return callback(file_key.read())
def test_load_with_utf8_url(self): url = self.get_url(quote(u'/maracujá.jpg'.encode('utf-8'))) config = Config() ctx = Context(None, config, None) with expect.error_not_to_happen(UnicodeDecodeError): loader.load(ctx, url, self.stop) self.wait()
def callback_wrapper(result): if result.successful: logger.info("Image " + path + " found on C5") callback(result) else: # If file_loader failed try http_loader logger.info("Image " + path + " NOT found on C5") http_loader.load(context, path, callback)
def topic(self, callback): url = self.get_url('/') loader.http_client = self._http_client config = Config() config.HTTP_LOADER_FORWARD_USER_AGENT = True ctx = Context(None, config, None, HandlerMock({"User-Agent": "test-user-agent"})) loader.load(ctx, url, callback)
def topic(self, callback): url = self.get_url('/') loader.http_client = self._http_client config = Config() config.ALLOWED_SOURCES = ['s.glbimg.com'] ctx = Context(None, config, None) loader.load(ctx, url, callback)
def topic(self, callback): url = self.get_url('/') loader.http_client = self._http_client config = Config() config.HTTP_LOADER_FORWARD_USER_AGENT = True config.HTTP_LOADER_DEFAULT_USER_AGENT = "DEFAULT_USER_AGENT" ctx = Context(None, config, None, HandlerMock({})) loader.load(ctx, url, callback)
def test_load_with_callback(self): url = self.get_url('/') config = Config() ctx = Context(None, config, None) loader.load(ctx, url, self.stop) result = self.wait() expect(result).to_be_instance_of(LoaderResult) expect(result.buffer).to_equal('Hello') expect(result.successful).to_be_true()
def topic(self): self.async_topic = "" def get_contents(contents): self.async_topic = contents url = self._get_url("/") loader.load(url, get_contents) return self.async_topic
def test_load_with_user_agent(self): url = self.get_url('/') config = Config() config.HTTP_LOADER_FORWARD_USER_AGENT = True ctx = Context(None, config, None, HandlerMock({"User-Agent": "test-user-agent"})) loader.load(ctx, url, self.stop) result = self.wait() expect(result).to_be_instance_of(LoaderResult) expect(result.buffer).to_equal('test-user-agent')
def test_load_with_curl(self): url = self.get_url('/') config = Config() config.HTTP_LOADER_CURL_ASYNC_HTTP_CLIENT = True ctx = Context(None, config, None) loader.load(ctx, url, self.stop) result = self.wait() expect(result).to_be_instance_of(LoaderResult) expect(result.buffer).to_equal('Hello') expect(result.successful).to_be_true()
def test_load_with_default_user_agent(self): url = self.get_url("/") config = Config() config.HTTP_LOADER_FORWARD_USER_AGENT = True config.HTTP_LOADER_DEFAULT_USER_AGENT = "DEFAULT_USER_AGENT" ctx = Context(None, config, None, HandlerMock({})) loader.load(ctx, url, self.stop) result = self.wait() expect(result).to_be_instance_of(LoaderResult) expect(result.buffer).to_equal("DEFAULT_USER_AGENT")
def test_load_without_curl_but_speed_timeout(self): url = self.get_url('/') config = Config() config.HTTP_LOADER_CURL_LOW_SPEED_TIME = 1 config.HTTP_LOADER_CURL_LOW_SPEED_LIMIT = 1000000000000 ctx = Context(None, config, None) loader.load(ctx, url, self.stop) result = self.wait() expect(result).to_be_instance_of(LoaderResult) expect(result.buffer).to_equal('Hello') expect(result.successful).to_be_true()
def callback_wrapper(result): if result.successful: callback(result) else: # If file_loader failed try http_loader if (path.find('http') != -1): http_loader.load(context, path, callback) else: result = LoaderResult() result.error = LoaderResult.ERROR_NOT_FOUND result.successful = False callback(result)
def test_load_with_timeout(self): url = self.get_url('/') config = Config() config.HTTP_LOADER_CURL_ASYNC_HTTP_CLIENT = True config.HTTP_LOADER_REQUEST_TIMEOUT = 1 ctx = Context(None, config, None) loader.load(ctx, url, self.stop) result = self.wait() expect(result).to_be_instance_of(LoaderResult) expect(result.buffer).to_be_null() expect(result.successful).to_be_false()
def test_load_with_all_headers(self): url = self.get_url('/') config = Config() config.HTTP_LOADER_FORWARD_ALL_HEADERS = True handler_mock_options = {"X-Test": "123", "DNT": "1", "X-Server": "thumbor"} ctx = Context(None, config, None, HandlerMock(handler_mock_options)) loader.load(ctx, url, self.stop) result = self.wait() expect(result).to_be_instance_of(LoaderResult) expect(result.buffer).to_include("Dnt:1\n") expect(result.buffer).to_include("X-Server:thumbor\n") expect(result.buffer).to_include("X-Test:123\n")
def load(context, url, callback): logger.debug("*** LOADER.load" ) start = datetime.datetime.now() if HTTP_RE.match(url): method = 'http' HttpLoader.load(context, url, callback) else: method = 's3' logger.debug("*** s3") S3Loader.load(context, url, callback) finish = datetime.datetime.now() metric_name = "thumbor.loader.%s.request_time" % method logger.debug("*** librato submit") LIBRATO_API.submit(metric_name, (finish - start).total_seconds() * 1000, description="original file request time (ms)")
def test_before_expiration_returns_something(self): options.STORAGE_EXPIRATION_SECONDS = 1000 image_url = 'www.globo.com/media/globocom/img/sprite1.png?8' http_loaded = http.load(image_url) storage = Storage() storage.put(image_url, http_loaded) assert storage.get(image_url)
def load(context, url, callback): enable_http_loader = context.config.get('AWS_ENABLE_HTTP_LOADER', default=False) if enable_http_loader and 'http' in url: return http_loader.load(context, url, callback) url = urllib2.unquote(url) if context.config.S3_LOADER_BUCKET: bucket = context.config.S3_LOADER_BUCKET else: bucket, url = _get_bucket(url) if not _validate_bucket(context, bucket): return callback(None) bucket_loader = Bucket( connection=thumbor_aws.connection.get_connection(context), name=bucket ) file_key = bucket_loader.get_key(url) if not file_key: return callback(None) return callback(file_key.read())
def test_load_with_empty_accept(self): url = self.get_url('/') config = Config() handler_mock_options = { "Accept-Encoding": "gzip", "User-Agent": "Thumbor", "Host": "localhost", "Accept": "", "X-Server": "thumbor" } ctx = Context(None, config, None, HandlerMock(handler_mock_options)) loader.load(ctx, url, self.stop) result = self.wait() expect(result).to_be_instance_of(LoaderResult) expect(result.buffer).to_include("Accept:image/*;q=0.9,*/*;q=0.1\n")
def test_loads_image(self): image_url = 'www.globo.com/media/globocom/img/sprite1.png' http_loaded = http.load(image_url) response = urllib2.urlopen('http://' + image_url).read() self.assertEqual(http_loaded, response, 'http_loaded is not the same as the http request')
def test_load_with_some_excluded_headers(self): url = self.get_url('/') config = Config() handler_mock_options = { "Accept-Encoding": "gzip", "User-Agent": "Thumbor", "Host": "localhost", "Accept": "*/*", "X-Server": "thumbor" } ctx = Context(None, config, None, HandlerMock(handler_mock_options)) loader.load(ctx, url, self.stop) result = self.wait() expect(result).to_be_instance_of(LoaderResult) expect(result.buffer).Not.to_include("X-Server:thumbor")
def test_should_return_a_future(self): url = self.get_url('/') config = Config() ctx = Context(None, config, None) future = loader.load(ctx, url) expect(isinstance(future, Future)).to_be_true()
def test_stores_image_does_not_store_crypt_key_if_not_in_options(self): rm_storage() options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False image_url = 'www.globo.com/media/globocom/img/sprite1.png' http_loaded = http.load(image_url) storage = Storage() storage.put(image_url, http_loaded) assert not collection.find_one({'path': image_url}).has_key('crypto'), 'crypto key should not be found into the storage'
def test_after_expiration_returns_none(self): options.STORAGE_EXPIRATION_SECONDS = 1 image_url = 'www.globo.com/media/globocom/img/sprite1.png?9' http_loaded = http.load(image_url) storage = Storage() storage.put(image_url, http_loaded) time.sleep(2) assert storage.get(image_url) is None
def test_stores_image(self): rm_storage() options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False image_url = 'www.globo.com/media/globocom/img/sprite1.png' http_loaded = http.load(image_url) storage = Storage() storage.put(image_url, http_loaded) assert collection.find_one({'path': image_url}), 'image not found into the storage'
def test_stores_image(self): rm_storage() options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False image_url = 'www.globo.com/media/globocom/img/sprite1.png' http_loaded = http.load(image_url) storage = file_storage.Storage() storage.put(image_url, http_loaded) assert exists('/tmp/thumbor/storage/www.globo.com/media/globocom/img/sprite1.png'), 'image not found into the storage'
def test_get_crypto_for_url(self): image_url = 'www.globo.com/media/globocom/img/sprite1.png?6' http_loaded = http.load(image_url) storage = Storage() storage.put(image_url, http_loaded) assert storage.get_crypto(image_url) == options.SECURITY_KEY
def test_stores_image_does_not_store_crypt_key_if_not_in_options(self): rm_storage() options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False image_url = 'www.globo.com/media/globocom/img/sprite1.png' http_loaded = http.load(image_url) storage = file_storage.Storage() storage.put(image_url, http_loaded) assert not exists('/tmp/thumbor/storage/www.globo.com/media/globocom/img/sprite1.txt'), 'crypto key was found into the storage'
def test_stores_image(self): rm_storage() options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False image_url = 'www.globo.com/media/globocom/img/sprite1.png' http_loaded = http.load(image_url) storage = Storage() storage.put(image_url, http_loaded) assert collection.find_one({'path': image_url }), 'image not found into the storage'
def test_gets_image(self): options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False image_url = 'www.globo.com/media/globocom/img/sprite1.png?2' http_loaded = http.load(image_url) storage = Storage() storage.put(image_url, http_loaded) stored_image = storage.get(image_url) assert stored_image and not isinstance(stored_image, dict), 'image not found into the storage' assert http_loaded == stored_image, 'stored image did not match the loaded one'
def test_stores_image_does_not_store_crypt_key_if_not_in_options(self): rm_storage() options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False image_url = 'www.globo.com/media/globocom/img/sprite1.png' http_loaded = http.load(image_url) storage = Storage() storage.put(image_url, http_loaded) assert not collection.find_one({ 'path': image_url }).has_key('crypto'), 'crypto key should not be found into the storage'
def test_stores_image(self): rm_storage() options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False image_url = 'www.globo.com/media/globocom/img/sprite1.png' http_loaded = http.load(image_url) storage = file_storage.Storage() storage.put(image_url, http_loaded) assert exists( '/tmp/thumbor/storage/www.globo.com/media/globocom/img/sprite1.png' ), 'image not found into the storage'
def test_stores_image_does_not_store_crypt_key_if_not_in_options(self): rm_storage() options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False image_url = 'www.globo.com/media/globocom/img/sprite1.png' http_loaded = http.load(image_url) storage = file_storage.Storage() storage.put(image_url, http_loaded) assert not exists( '/tmp/thumbor/storage/www.globo.com/media/globocom/img/sprite1.txt' ), 'crypto key was found into the storage'
def test_stores_image_does_not_store_crypt_key_if_not_in_options(self): options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False image_url = 'www.globo.com/media/globocom/img/sprite1.png?3' http_loaded = http.load(image_url) storage = Storage() storage.put(image_url, http_loaded) image = get_image_by_path(image_url) assert not image['security_key']
def test_stores_image(self): options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = False image_url = 'www.globo.com/media/globocom/img/sprite1.png?1' http_loaded = http.load(image_url) storage = Storage() storage.put(image_url, http_loaded) image = get_image_by_path(image_url) assert image['url'] == image_url assert image['contents'] assert image['last_update']
def load(context, url, callback): url = quote_url(url) match = S3_RE.match(url) def callback_wrapper(result): if result.successful: callback(result) else: logger.info('s3 {0}'.format( os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/'))) ) # If not on efs, try s3 S3Loader.load(context, os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')), callback) # If melody s3 file, first try to load from efs if match: logger.info('BOTO {0}'.format(match.group('path'))) # TEMP try s3 direct S3Loader.load(context, os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')), callback) # FileLoader.load(context, match.group('path'), callback_wrapper) # else get from the internet elif HTTP_RE.match(url): logger.info('WEB {0}'.format(url)) HttpLoader.load(context, url, callback) else: logger.info('FILE {0}'.format(url)) result = LoaderResult() result.successful = False result.error = LoaderResult.ERROR_NOT_FOUND # callback(result) # TEMP enable file loader FileLoader.load(context, url, callback)
def test_finding_crypto_file(self): rm_storage() image_url = 'www.globo.com/media/globocom/img/sprite1.png' http_loaded = http.load(image_url) storage = file_storage.Storage() storage.put(image_url, http_loaded) crypto_file = '/tmp/thumbor/storage/www.globo.com/media/globocom/img/sprite1.txt' assert exists(crypto_file), 'crypto key was not found into the storage' assert file(crypto_file).read() == options.SECURITY_KEY
def test_finding_crypto_file(self): options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True options.SECURITY_KEY = 'MY-SECURITY-KEY-5' image_url = 'www.globo.com/media/globocom/img/sprite1.png?5' http_loaded = http.load(image_url) storage = Storage() storage.put(image_url, http_loaded) image = get_image_by_path(image_url) assert image['security_key'] == options.SECURITY_KEY
def test_storing_an_empty_key_raises(self): options.SECURITY_KEY = False options.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True image_url = 'www.globo.com/media/globocom/img/sprite1.png?4' http_loaded = http.load(image_url) storage = Storage() try: storage.put(image_url, http_loaded) except RuntimeError, err: assert str(err) == "STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified" return
def test_storing_an_empty_key_raises(self): rm_storage() options.SECURITY_KEY = False image_url = 'www.globo.com/media/globocom/img/sprite1.png' http_loaded = http.load(image_url) storage = file_storage.Storage() try: storage.put(image_url, http_loaded) except RuntimeError, err: assert str(err) == "STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified" return