Example #1
0
def load(context, url, callback):
    url = quote_url(url)
    match = S3_RE.match(url)

    def callback_wrapper(result):
        if result.successful:
            callback(result)
        else:
            logger.info('s3 {0}'.format(
                os.path.join(match.group('bucket').rstrip('/'), match.group('path').lstrip('/')))
            )

            # If not on efs, try s3
            S3Loader.load(context,
                          os.path.join(match.group('bucket').rstrip('/'),
                                       match.group('path').lstrip('/')),
                          callback)

    # If melody s3 file, first try to load from efs
    if match:
        logger.info('BOTO {0}'.format(match.group('path')))

        # TEMP try s3 direct
        S3Loader.load(context,
                      os.path.join(match.group('bucket').rstrip('/'),
                                   match.group('path').lstrip('/')),
                      callback)

        # FileLoader.load(context, match.group('path'), callback_wrapper)
    # else get from the internet
    elif HTTP_RE.match(url):
        logger.info('WEB {0}'.format(url))
        HttpLoader.load(context, url, callback)
    else:
        logger.info('FILE {0}'.format(url))
        result = LoaderResult()
        result.successful = False
        result.error = LoaderResult.ERROR_NOT_FOUND
        # callback(result)
        # TEMP enable file loader
        FileLoader.load(context, url, callback)
Example #2
0
def _normalize_url(url):
    url = http_loader.quote_url(url)
    return url if url.startswith('http') else 'https://%s' % url
def _normalize_url(url):
    url = http_loader.quote_url(unquote(url))
    if url.startswith('http:'):
        url = url.replace('http:', 'https:', 1)

    return url if url.startswith('https://') else 'https://%s' % url
Example #4
0
def _normalize_url(url, forwarded_proto=None):
    url = http_loader.quote_url(url)
    if url.startswith('http:'):
        url = url.replace('http:', 'https:', 1)

    return url if url.startswith('https://') else 'https://%s' % url
Example #5
0
def _normalize_url(url):
    url = http_loader.quote_url(url)
    return url if url.startswith('http') else 'https://%s' % url
def _normalize_url(url):
    url = http_loader.quote_url(unquote(url))
    if url.startswith('http:'):
        url = url.replace('http:', 'https:', 1)

    return url if url.startswith('https://') else 'https://%s' % url
Example #7
0
def _normalize_url(url):
    url = http_loader.quote_url(url)
    return url if url.startswith("http") else f"https://{url}"
Example #8
0
def _normalize_url(url, forwarded_proto=None):
    url = http_loader.quote_url(url)
    return url if url.startswith('http') else 'https://%s' % url
def _normalize_url(url):
    url = http_loader.quote_url(url)
    if url.startswith("http:"):
        url = url.replace("http:", "https:", 1)

    return url if url.startswith("https://") else "https://%s" % url