def load(context, url, callback): # type: (Context, str, Callable[..., Any]) -> None source_type, encoded_url = separate_url_and_source_type(url) actual_url = base64.urlsafe_b64decode(urllib.parse.unquote(encoded_url)) if source_type not in (THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE): callback(get_not_found_result()) logging.warning('INVALID SOURCE TYPE: ' + source_type) return if source_type == THUMBOR_S3_TYPE: if actual_url.startswith('/user_uploads/'): # type: ignore # python 2 type differs from python 3 type actual_url = actual_url[len('/user_uploads/'):] else: raise AssertionError("Unexpected s3 file.") s3_loader.load(context, actual_url, callback) elif source_type == THUMBOR_LOCAL_FILE_TYPE: if actual_url.startswith('/user_uploads/'): # type: ignore # python 2 type differs from python 3 type actual_url = actual_url[len('/user_uploads/'):] local_file_path_prefix = 'files/' else: raise AssertionError("Unexpected local file.") patched_local_url = local_file_path_prefix + actual_url # type: ignore # python 2 type differs from python 3 type file_loader.load(context, patched_local_url, callback) elif source_type == THUMBOR_EXTERNAL_TYPE: https_loader.load(context, actual_url, callback)
def load(context, path, callback): file_path = join(context.config.FILE_LOADER_ROOT_PATH.rstrip('/'), unquote(path).lstrip('/')) file_path = abspath(file_path) inside_root_path = file_path.startswith( context.config.FILE_LOADER_ROOT_PATH) if inside_root_path and is_video(file_path): # Extract a frame from the video and load it instead of the original path logger.warning('processing video... %s', file_path) with get_video_frame(context, file_path) as image_path: if image_path: callback(read_file(image_path)) return elif inside_root_path and is_pdf(file_path): # extract first page of pdf and load it logger.warning('processing pdf... %s', file_path) with get_pdf_page(context, file_path) as image_path: if image_path: callback(read_file(image_path)) return else: # First attempt to load with file_loader file_loader.load(context, path, callback) return # If we got here, there was a failure result = LoaderResult() result.error = LoaderResult.ERROR_NOT_FOUND result.successful = False callback(result)
def load(context, url, callback): # type: (Context, str, Callable[..., Any]) -> None source_type, encoded_url = separate_url_and_source_type(url) actual_url = base64.urlsafe_b64decode(urllib.parse.unquote(encoded_url)).decode('utf-8') if source_type not in (THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE): callback(get_not_found_result()) logging.warning('INVALID SOURCE TYPE: ' + source_type) return if source_type == THUMBOR_S3_TYPE: if actual_url.startswith('/user_uploads/'): actual_url = actual_url[len('/user_uploads/'):] else: raise AssertionError("Unexpected s3 file.") s3_loader.load(context, actual_url, callback) elif source_type == THUMBOR_LOCAL_FILE_TYPE: if actual_url.startswith('/user_uploads/'): actual_url = actual_url[len('/user_uploads/'):] local_file_path_prefix = 'files/' else: raise AssertionError("Unexpected local file.") patched_local_url = local_file_path_prefix + actual_url file_loader.load(context, patched_local_url, callback) elif source_type == THUMBOR_EXTERNAL_TYPE: https_loader.load(context, actual_url, callback)
def load(context, url, callback): # type: (Context, str, Callable[..., Any]) -> None source_type, encoded_url = separate_url_and_source_type(url) actual_url = base64.urlsafe_b64decode(urllib.parse.unquote(encoded_url)) if source_type not in (THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE): callback(get_not_found_result()) logging.warning('INVALID SOURCE TYPE: ' + source_type) return if source_type == THUMBOR_S3_TYPE: s3_loader.load(context, actual_url, callback) elif source_type == THUMBOR_LOCAL_FILE_TYPE: patched_local_url = 'files/' + actual_url # type: ignore # python 2 type differs from python 3 type file_loader.load(context, patched_local_url, callback) elif source_type == THUMBOR_EXTERNAL_TYPE: https_loader.load(context, actual_url, callback)
def test_loads_image(self): image_url = 'img/fixture1.png' file_loaded = file_loader.load(image_url) response = open(join(fixtures_folder, 'img', 'fixture1.png')).read() self.assertEqual( file_loaded, response, 'file_loaded is not the same as the filesystem equivalent')
def load(context, url, callback): url = quote_url(url) match = S3_RE.match(url) def callback_wrapper(result): if result.successful: callback(result) else: logger.info('s3 {0}'.format( os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/'))) ) # If not on efs, try s3 S3Loader.load(context, os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')), callback) # If melody s3 file, first try to load from efs if match: logger.info('BOTO {0}'.format(match.group('path'))) # TEMP try s3 direct S3Loader.load(context, os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')), callback) # FileLoader.load(context, match.group('path'), callback_wrapper) # else get from the internet elif HTTP_RE.match(url): logger.info('WEB {0}'.format(url)) HttpLoader.load(context, url, callback) else: logger.info('FILE {0}'.format(url)) result = LoaderResult() result.successful = False result.error = LoaderResult.ERROR_NOT_FOUND # callback(result) # TEMP enable file loader FileLoader.load(context, url, callback)
def load(context, url, callback): # type: (Context, str, Callable[..., Any]) -> None url = urllib.parse.unquote(url) url_params = get_url_params(url) source_type = url_params.get('source_type') if not sign_is_valid(url, context) or source_type not in ( THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE): callback(get_not_found_result()) return url = url.rsplit('?', 1)[0] if source_type == THUMBOR_S3_TYPE: s3_loader.load(context, url, callback) elif source_type == THUMBOR_LOCAL_FILE_TYPE: file_loader.load(context, url, callback) elif source_type == THUMBOR_EXTERNAL_TYPE: http_loader.load_sync(context, url, callback, normalize_url_func=http_loader._normalize_url)
def load(context, url, callback): # type: (Context, str, Callable[..., Any]) -> None url = urllib.parse.unquote(url) url_params = get_url_params(url) source_type = url_params.get('source_type') if not sign_is_valid(url, context) or source_type not in ( THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE): callback(get_not_found_result()) return url = url.rsplit('?', 1)[0] if source_type == THUMBOR_S3_TYPE: s3_loader.load(context, url, callback) elif source_type == THUMBOR_LOCAL_FILE_TYPE: file_loader.load(context, url, callback) elif source_type == THUMBOR_EXTERNAL_TYPE: http_loader.load_sync( context, url, callback, normalize_url_func=http_loader._normalize_url)
def topic(self, callback, context): load(context, '../file_loader_vows.py', callback)
def topic(self, callback, context): load(context, 'image_NOT.jpg', callback)
def load_file(self, file_name): return load(self.ctx, file_name, lambda x: x).result()
def test_loads_image(self): image_url = 'img/fixture1.png' file_loaded = file_loader.load(image_url) response = open(join(fixtures_folder, 'img', 'fixture1.png')).read() self.assertEqual(file_loaded, response, 'file_loaded is not the same as the filesystem equivalent')
def callback_wrapper(result): if result: callback(result) else: # Fallback file_loader.load(context, path, callback)