def load(context, url, callback): url = quote_url(url) match = S3_RE.match(url) def callback_wrapper(result): if result.successful: callback(result) else: logger.info('s3 {0}'.format( os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/'))) ) # If not on efs, try s3 S3Loader.load(context, os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')), callback) # If melody s3 file, first try to load from efs if match: logger.info('BOTO {0}'.format(match.group('path'))) # TEMP try s3 direct S3Loader.load(context, os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')), callback) # FileLoader.load(context, match.group('path'), callback_wrapper) # else get from the internet elif HTTP_RE.match(url): logger.info('WEB {0}'.format(url)) HttpLoader.load(context, url, callback) else: logger.info('FILE {0}'.format(url)) result = LoaderResult() result.successful = False result.error = LoaderResult.ERROR_NOT_FOUND # callback(result) # TEMP enable file loader FileLoader.load(context, url, callback)
def _normalize_url(url): url = http_loader.quote_url(url) return url if url.startswith('http') else 'https://%s' % url
def _normalize_url(url): url = http_loader.quote_url(unquote(url)) if url.startswith('http:'): url = url.replace('http:', 'https:', 1) return url if url.startswith('https://') else 'https://%s' % url
def _normalize_url(url, forwarded_proto=None): url = http_loader.quote_url(url) if url.startswith('http:'): url = url.replace('http:', 'https:', 1) return url if url.startswith('https://') else 'https://%s' % url
def _normalize_url(url): url = http_loader.quote_url(url) return url if url.startswith("http") else f"https://{url}"
def _normalize_url(url, forwarded_proto=None): url = http_loader.quote_url(url) return url if url.startswith('http') else 'https://%s' % url
def _normalize_url(url): url = http_loader.quote_url(url) if url.startswith("http:"): url = url.replace("http:", "https:", 1) return url if url.startswith("https://") else "https://%s" % url