def load(context, url, callback):
url = quote_url(url)
match = S3_RE.match(url)
def callback_wrapper(result):
if result.successful:
callback(result)
else:
logger.info('s3 {0}'.format(
os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')))
)
# If not on efs, try s3
S3Loader.load(context,
os.path.join(match.group('bucket').rstrip('/'),
match.group('path').lstrip('/')),
callback)
# If melody s3 file, first try to load from efs
if match:
logger.info('BOTO {0}'.format(match.group('path')))
# TEMP try s3 direct
S3Loader.load(context,
os.path.join(match.group('bucket').rstrip('/'),
match.group('path').lstrip('/')),
callback)
# FileLoader.load(context, match.group('path'), callback_wrapper)
# else get from the internet
elif HTTP_RE.match(url):
logger.info('WEB {0}'.format(url))
HttpLoader.load(context, url, callback)
else:
logger.info('FILE {0}'.format(url))
result = LoaderResult()
result.successful = False
result.error = LoaderResult.ERROR_NOT_FOUND
# callback(result)
# TEMP enable file loader
FileLoader.load(context, url, callback)
def _normalize_url(url):
url = http_loader.quote_url(url)
return url if url.startswith('http') else 'https://%s' % url
def _normalize_url(url):
url = http_loader.quote_url(unquote(url))
if url.startswith('http:'):
url = url.replace('http:', 'https:', 1)
return url if url.startswith('https://') else 'https://%s' % url
def _normalize_url(url, forwarded_proto=None):
url = http_loader.quote_url(url)
if url.startswith('http:'):
url = url.replace('http:', 'https:', 1)
return url if url.startswith('https://') else 'https://%s' % url
def _normalize_url(url):
url = http_loader.quote_url(url)
return url if url.startswith('http') else 'https://%s' % url
def _normalize_url(url):
url = http_loader.quote_url(unquote(url))
if url.startswith('http:'):
url = url.replace('http:', 'https:', 1)
return url if url.startswith('https://') else 'https://%s' % url
def _normalize_url(url):
url = http_loader.quote_url(url)
return url if url.startswith("http") else f"https://{url}"
def _normalize_url(url, forwarded_proto=None):
url = http_loader.quote_url(url)
return url if url.startswith('http') else 'https://%s' % url
def _normalize_url(url):
url = http_loader.quote_url(url)
if url.startswith("http:"):
url = url.replace("http:", "https:", 1)
return url if url.startswith("https://") else "https://%s" % url