def load(context, url, callback):
# type: (Context, str, Callable[..., Any]) -> None
source_type, encoded_url = separate_url_and_source_type(url)
actual_url = base64.urlsafe_b64decode(urllib.parse.unquote(encoded_url))
if source_type not in (THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE,
THUMBOR_EXTERNAL_TYPE):
callback(get_not_found_result())
logging.warning('INVALID SOURCE TYPE: ' + source_type)
return
if source_type == THUMBOR_S3_TYPE:
if actual_url.startswith('/user_uploads/'): # type: ignore # python 2 type differs from python 3 type
actual_url = actual_url[len('/user_uploads/'):]
else:
raise AssertionError("Unexpected s3 file.")
s3_loader.load(context, actual_url, callback)
elif source_type == THUMBOR_LOCAL_FILE_TYPE:
if actual_url.startswith('/user_uploads/'): # type: ignore # python 2 type differs from python 3 type
actual_url = actual_url[len('/user_uploads/'):]
local_file_path_prefix = 'files/'
else:
raise AssertionError("Unexpected local file.")
patched_local_url = local_file_path_prefix + actual_url # type: ignore # python 2 type differs from python 3 type
file_loader.load(context, patched_local_url, callback)
elif source_type == THUMBOR_EXTERNAL_TYPE:
https_loader.load(context, actual_url, callback)
def load(context, path, callback):
file_path = join(context.config.FILE_LOADER_ROOT_PATH.rstrip('/'),
unquote(path).lstrip('/'))
file_path = abspath(file_path)
inside_root_path = file_path.startswith(
context.config.FILE_LOADER_ROOT_PATH)
if inside_root_path and is_video(file_path):
# Extract a frame from the video and load it instead of the original path
logger.warning('processing video... %s', file_path)
with get_video_frame(context, file_path) as image_path:
if image_path:
callback(read_file(image_path))
return
elif inside_root_path and is_pdf(file_path):
# extract first page of pdf and load it
logger.warning('processing pdf... %s', file_path)
with get_pdf_page(context, file_path) as image_path:
if image_path:
callback(read_file(image_path))
return
else:
# First attempt to load with file_loader
file_loader.load(context, path, callback)
return
# If we got here, there was a failure
result = LoaderResult()
result.error = LoaderResult.ERROR_NOT_FOUND
result.successful = False
callback(result)
def load(context, url, callback):
# type: (Context, str, Callable[..., Any]) -> None
source_type, encoded_url = separate_url_and_source_type(url)
actual_url = base64.urlsafe_b64decode(urllib.parse.unquote(encoded_url)).decode('utf-8')
if source_type not in (THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE,
THUMBOR_EXTERNAL_TYPE):
callback(get_not_found_result())
logging.warning('INVALID SOURCE TYPE: ' + source_type)
return
if source_type == THUMBOR_S3_TYPE:
if actual_url.startswith('/user_uploads/'):
actual_url = actual_url[len('/user_uploads/'):]
else:
raise AssertionError("Unexpected s3 file.")
s3_loader.load(context, actual_url, callback)
elif source_type == THUMBOR_LOCAL_FILE_TYPE:
if actual_url.startswith('/user_uploads/'):
actual_url = actual_url[len('/user_uploads/'):]
local_file_path_prefix = 'files/'
else:
raise AssertionError("Unexpected local file.")
patched_local_url = local_file_path_prefix + actual_url
file_loader.load(context, patched_local_url, callback)
elif source_type == THUMBOR_EXTERNAL_TYPE:
https_loader.load(context, actual_url, callback)
def load(context, url, callback):
# type: (Context, str, Callable[..., Any]) -> None
source_type, encoded_url = separate_url_and_source_type(url)
actual_url = base64.urlsafe_b64decode(urllib.parse.unquote(encoded_url))
if source_type not in (THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE,
THUMBOR_EXTERNAL_TYPE):
callback(get_not_found_result())
logging.warning('INVALID SOURCE TYPE: ' + source_type)
return
if source_type == THUMBOR_S3_TYPE:
s3_loader.load(context, actual_url, callback)
elif source_type == THUMBOR_LOCAL_FILE_TYPE:
patched_local_url = 'files/' + actual_url # type: ignore # python 2 type differs from python 3 type
file_loader.load(context, patched_local_url, callback)
elif source_type == THUMBOR_EXTERNAL_TYPE:
https_loader.load(context, actual_url, callback)
def test_loads_image(self):
image_url = 'img/fixture1.png'
file_loaded = file_loader.load(image_url)
response = open(join(fixtures_folder, 'img', 'fixture1.png')).read()
self.assertEqual(
file_loaded, response,
'file_loaded is not the same as the filesystem equivalent')
def load(context, url, callback):
# type: (Context, str, Callable[..., Any]) -> None
source_type, encoded_url = separate_url_and_source_type(url)
actual_url = base64.urlsafe_b64decode(urllib.parse.unquote(encoded_url))
if source_type not in (THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE,
THUMBOR_EXTERNAL_TYPE):
callback(get_not_found_result())
logging.warning('INVALID SOURCE TYPE: ' + source_type)
return
if source_type == THUMBOR_S3_TYPE:
s3_loader.load(context, actual_url, callback)
elif source_type == THUMBOR_LOCAL_FILE_TYPE:
patched_local_url = 'files/' + actual_url # type: ignore # python 2 type differs from python 3 type
file_loader.load(context, patched_local_url, callback)
elif source_type == THUMBOR_EXTERNAL_TYPE:
https_loader.load(context, actual_url, callback)
def load(context, url, callback):
url = quote_url(url)
match = S3_RE.match(url)
def callback_wrapper(result):
if result.successful:
callback(result)
else:
logger.info('s3 {0}'.format(
os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')))
)
# If not on efs, try s3
S3Loader.load(context,
os.path.join(match.group('bucket').rstrip('/'),
match.group('path').lstrip('/')),
callback)
# If melody s3 file, first try to load from efs
if match:
logger.info('BOTO {0}'.format(match.group('path')))
# TEMP try s3 direct
S3Loader.load(context,
os.path.join(match.group('bucket').rstrip('/'),
match.group('path').lstrip('/')),
callback)
# FileLoader.load(context, match.group('path'), callback_wrapper)
# else get from the internet
elif HTTP_RE.match(url):
logger.info('WEB {0}'.format(url))
HttpLoader.load(context, url, callback)
else:
logger.info('FILE {0}'.format(url))
result = LoaderResult()
result.successful = False
result.error = LoaderResult.ERROR_NOT_FOUND
# callback(result)
# TEMP enable file loader
FileLoader.load(context, url, callback)
def load(context, url, callback):
# type: (Context, str, Callable[..., Any]) -> None
url = urllib.parse.unquote(url)
url_params = get_url_params(url)
source_type = url_params.get('source_type')
if not sign_is_valid(url, context) or source_type not in (
THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE):
callback(get_not_found_result())
return
url = url.rsplit('?', 1)[0]
if source_type == THUMBOR_S3_TYPE:
s3_loader.load(context, url, callback)
elif source_type == THUMBOR_LOCAL_FILE_TYPE:
file_loader.load(context, url, callback)
elif source_type == THUMBOR_EXTERNAL_TYPE:
http_loader.load_sync(context,
url,
callback,
normalize_url_func=http_loader._normalize_url)
def load(context, url, callback):
# type: (Context, str, Callable[..., Any]) -> None
url = urllib.parse.unquote(url)
url_params = get_url_params(url)
source_type = url_params.get('source_type')
if not sign_is_valid(url, context) or source_type not in (
THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE):
callback(get_not_found_result())
return
url = url.rsplit('?', 1)[0]
if source_type == THUMBOR_S3_TYPE:
s3_loader.load(context, url, callback)
elif source_type == THUMBOR_LOCAL_FILE_TYPE:
file_loader.load(context, url, callback)
elif source_type == THUMBOR_EXTERNAL_TYPE:
http_loader.load_sync(
context,
url,
callback,
normalize_url_func=http_loader._normalize_url)
def topic(self, callback, context):
load(context, '../file_loader_vows.py', callback)
def topic(self, callback, context):
load(context, 'image_NOT.jpg', callback)
def load_file(self, file_name):
return load(self.ctx, file_name, lambda x: x).result()
def load_file(self, file_name):
return load(self.ctx, file_name, lambda x: x).result()
def test_loads_image(self):
image_url = 'img/fixture1.png'
file_loaded = file_loader.load(image_url)
response = open(join(fixtures_folder, 'img', 'fixture1.png')).read()
self.assertEqual(file_loaded, response, 'file_loaded is not the same as the filesystem equivalent')
def topic(self, callback, context):
load(context, 'image_NOT.jpg', callback)
def topic(self, callback, context):
load(context, '../file_loader_vows.py', callback)
def callback_wrapper(result):
if result:
callback(result)
else:
# Fallback
file_loader.load(context, path, callback)
