def test_resolver_dir_and_file(self): a_file = os.path.realpath(__file__) a_dir = os.path.dirname(a_file) # a file -- assume path index res = DefaultResolverMixin.make_best_resolver(a_file) assert isinstance(res, PathIndexResolver) # a dir -- asume prefix res = DefaultResolverMixin.make_best_resolver(a_dir) assert isinstance(res, PrefixResolver) # not a valid file -- default to prefix res = DefaultResolverMixin.make_best_resolver('file://test/x_invalid') assert isinstance(res, PrefixResolver)
def test_resolver_dir_and_file(self): a_file = os.path.realpath(__file__) a_dir = os.path.dirname(a_file) # a file -- assume path index res = DefaultResolverMixin.make_best_resolver(a_file) assert isinstance(res, PathIndexResolver) # a dir -- assume prefix res = DefaultResolverMixin.make_best_resolver(a_dir) assert isinstance(res, PrefixResolver) # not a valid file -- default to prefix res = DefaultResolverMixin.make_best_resolver('file://test/x_invalid') assert isinstance(res, PrefixResolver)
def test_resolver_http_prefix_not_wildcard(self): resolver = DefaultResolverMixin.make_best_resolver( 'http://example.com/*/') cdx = CDXObject() res = resolver('example.warc.gz', cdx) assert res == 'http://example.com/*/example.warc.gz'
def test_resolver_dir_wildcard(self): resolver = DefaultResolverMixin.make_best_resolver(os.path.join(get_test_dir(), '*', '')) cdx = CDXObject() res = resolver('example.warc.gz', cdx) assert len(res) == 1 assert res[0] == os.path.join(get_test_dir(), 'warcs', 'example.warc.gz')
def test_resolver_dir_wildcard(self): resolver = DefaultResolverMixin.make_best_resolver(os.path.join(get_test_dir(), '*', '')) cdx = CDXObject() res = resolver('example.warc.gz', cdx) assert len(res) == 1 assert res[0] == os.path.join(get_test_dir(), 'warcs', 'example.warc.gz')
def test_resolver_dir_wildcard_as_file_url(self): url = to_file_url(get_test_dir()) + '/*/' resolver = DefaultResolverMixin.make_best_resolver(url) cdx = CDXObject() res = resolver('example.warc.gz', cdx) assert len(res) == 1 assert res[0] == os.path.abspath(os.path.join(get_test_dir(), 'warcs', 'example.warc.gz'))
def test_resolver_dir_wildcard_as_file_url(self): url = to_file_url(get_test_dir()) + '/*/' resolver = DefaultResolverMixin.make_best_resolver(url) cdx = CDXObject() res = resolver('example.warc.gz', cdx) assert len(res) == 1 assert res[0] == os.path.abspath(os.path.join(get_test_dir(), 'warcs', 'example.warc.gz'))
def test_resolver_dir_wildcard_with_coll(self): resolver = DefaultResolverMixin.make_best_resolver('s3://bucket/colls/*/archives/') cdx = CDXObject() cdx['source'] = 'my-coll/indexes/index.cdxj' cdx['source-coll'] = 'my-coll' res = resolver('example.warc.gz', cdx) assert res == 's3://bucket/colls/my-coll/archives/example.warc.gz'
def test_resolver_dir_wildcard_with_coll(self): resolver = DefaultResolverMixin.make_best_resolver('s3://bucket/colls/*/archives/') cdx = CDXObject() cdx['source'] = 'my-coll/indexes/index.cdxj' cdx['source-coll'] = 'my-coll' res = resolver('example.warc.gz', cdx) assert res == 's3://bucket/colls/my-coll/archives/example.warc.gz'
def test_make_best_resolver_pathindex(self): path = os.path.join(get_test_dir(), 'text_content', 'pathindex.txt') res = DefaultResolverMixin.make_best_resolver(path) assert isinstance(res, PathIndexResolver) assert repr(res) == "PathIndexResolver('{0}')".format(path)
def test_make_best_resolver_redis(self): res = DefaultResolverMixin.make_best_resolver( 'redis://myhost.example.com:1234/1') assert isinstance(res, RedisResolver) assert repr( res) == "RedisResolver('redis://myhost.example.com:1234/1')"
def test_make_best_resolver_http(self): res = DefaultResolverMixin.make_best_resolver( 'http://myhost.example.com/warcs/') assert isinstance(res, PrefixResolver) assert repr( res) == "PrefixResolver('http://myhost.example.com/warcs/')"
def test_resolver_http_prefix_not_wildcard(self): resolver = DefaultResolverMixin.make_best_resolver('http://example.com/*/') cdx = CDXObject() res = resolver('example.warc.gz', cdx) assert res == 'http://example.com/*/example.warc.gz'
def test_make_best_resolver_pathindex(self): path = os.path.join(get_test_dir(), 'text_content', 'pathindex.txt') res = DefaultResolverMixin.make_best_resolver(path) assert isinstance(res, PathIndexResolver) assert repr(res) == "PathIndexResolver('{0}')".format(path)
def test_make_best_resolver_redis(self): res = DefaultResolverMixin.make_best_resolver('redis://myhost.example.com:1234/1') assert isinstance(res, RedisResolver) assert repr(res) == "RedisResolver('redis://myhost.example.com:1234/1')"
def test_make_best_resolver_http(self): res = DefaultResolverMixin.make_best_resolver('http://myhost.example.com/warcs/') assert isinstance(res, PrefixResolver) assert repr(res) == "PrefixResolver('http://myhost.example.com/warcs/')"