def test_ascloudpath():
from cloudfiles.paths import ascloudpath, ExtractedPath
pth = ExtractedPath('precomputed', 'gs', 'my_bucket', 'of/heaven', None)
assert ascloudpath(pth) == "precomputed://gs://my_bucket/of/heaven"
pth = ExtractedPath(None, 'gs', 'my_bucket', 'of/heaven', None)
assert ascloudpath(pth) == "gs://my_bucket/of/heaven"
pth = ExtractedPath(None, 'file', 'my_bucket', 'of/heaven', None)
assert ascloudpath(pth) == "file://my_bucket/of/heaven"
pth = ExtractedPath(None, 'mem', 'my_bucket', 'of/heaven', None)
assert ascloudpath(pth) == "mem://my_bucket/of/heaven"
pth = ExtractedPath(None, 'https', 'my_bucket', 'of/heaven', 'https://some.domain.com')
assert ascloudpath(pth) == "https://some.domain.com/my_bucket/of/heaven"
pth = ExtractedPath('graphene', 'https', 'my_bucket', 'of/heaven', 'https://some.domain.com')
assert ascloudpath(pth) == "graphene://https://some.domain.com/my_bucket/of/heaven"
pth = ExtractedPath(None, 's3', 'my_bucket', 'of/heaven', 'https://some.domain.com')
assert ascloudpath(pth) == "s3://https://some.domain.com/my_bucket/of/heaven"
pth = ExtractedPath("precomputed", 's3', 'my_bucket', 'of/heaven', 'https://some.domain.com')
assert ascloudpath(pth) == "precomputed://s3://https://some.domain.com/my_bucket/of/heaven"
def test_to_https_protocol():
from cloudfiles.paths import extract, to_https_protocol, ExtractedPath
pth = to_https_protocol("gs://my_bucket/to/heaven")
assert pth == "https://storage.googleapis.com/my_bucket/to/heaven"
pth = to_https_protocol("s3://my_bucket/to/heaven")
assert pth == "https://s3.amazonaws.com/my_bucket/to/heaven"
pth = to_https_protocol("matrix://my_bucket/to/heaven")
assert pth == "https://s3-hpcrc.rc.princeton.edu/my_bucket/to/heaven"
pth = to_https_protocol("file://my_bucket/to/heaven")
assert pth == "file://my_bucket/to/heaven"
pth = to_https_protocol("mem://my_bucket/to/heaven")
assert pth == "mem://my_bucket/to/heaven"
pth = ExtractedPath('precomputed', 'gs', 'my_bucket', 'to/heaven', None)
pth = to_https_protocol(pth)
assert pth == extract("https://storage.googleapis.com/my_bucket/to/heaven")
def test_path_extraction():
from cloudfiles import paths, exceptions, lib
ExtractedPath = paths.ExtractedPath
def shoulderror(url):
try:
pth = paths.extract(url)
assert False, url
except exceptions.UnsupportedProtocolError:
pass
def okgoogle(url):
path = paths.extract(url)
assert path.protocol == 'gs', url
assert path.bucket == 'bucket', url
assert path.path in ('dataset/layer', 'dataset/layer/'), url
assert path.host is None
assert path.format == 'precomputed', url
okgoogle('gs://bucket/dataset/layer')
shoulderror('s4://dataset/layer')
shoulderror('dataset/layer')
# don't error
assert (paths.extract('graphene://http://localhost:8080/segmentation/1.0/testvol')
== ExtractedPath(
'graphene', 'http', None,
'segmentation/1.0/testvol', 'http://localhost:8080'))
assert (paths.extract('precomputed://gs://fafb-ffn1-1234567')
== ExtractedPath(
'precomputed', 'gs', 'fafb-ffn1-1234567',
'', None))
assert (paths.extract('precomputed://gs://fafb-ffn1-1234567/segmentation')
== ExtractedPath(
'precomputed', 'gs', 'fafb-ffn1-1234567',
'segmentation', None))
firstdir = lambda x: '/' + x.split('/')[1]
homepath = lib.toabs('~')
homerintermediate = homepath.replace(firstdir(homepath), '')[1:]
curpath = lib.toabs('.')
curintermediate = curpath.replace(firstdir(curpath), '')[1:]
match = re.match(r'((?:(?:\w:\\\\)|/).+?)\b', lib.toabs('.'))
bucket, = match.groups()
assert (paths.extract('s3://seunglab-test/intermediate/path/dataset/layer')
== ExtractedPath(
'precomputed', 's3', 'seunglab-test',
'intermediate/path/dataset/layer', None
))
assert (paths.extract('file:///tmp/dataset/layer')
== ExtractedPath(
'precomputed', 'file', None,
"/tmp/dataset/layer", None
))
assert (paths.extract('file://seunglab-test/intermediate/path/dataset/layer')
== ExtractedPath(
'precomputed', 'file', None,
os.path.join(curpath, 'seunglab-test/intermediate/path/dataset/layer'), None
))
assert (paths.extract('gs://seunglab-test/intermediate/path/dataset/layer')
== ExtractedPath(
'precomputed', 'gs', 'seunglab-test',
'intermediate/path/dataset/layer', None
))
assert (paths.extract('file://~/seunglab-test/intermediate/path/dataset/layer')
== ExtractedPath(
'precomputed', 'file', None,
os.path.join(homepath, 'seunglab-test/intermediate/path/dataset/layer'),
None
)
)
assert (paths.extract('file:///User/me/.cloudvolume/cache/gs/bucket/dataset/layer')
== ExtractedPath(
'precomputed', 'file', None,
'/User/me/.cloudvolume/cache/gs/bucket/dataset/layer', None
))
shoulderror('ou3bouqjsa fkj aojsf oaojf ojsaf')
okgoogle('gs://bucket/dataset/layer/')
# shoulderror('gs://bucket/dataset/layer/info')
path = paths.extract('s3://bucketxxxxxx/datasetzzzzz91h8__3/layer1br9bobasjf/')
assert path.format == 'precomputed'
assert path.protocol == 's3'
assert path.bucket == 'bucketxxxxxx'
assert path.path == 'datasetzzzzz91h8__3/layer1br9bobasjf/'
assert path.host is None
path = paths.extract('file:///bucket/dataset/layer/')
assert path.format == 'precomputed'
assert path.protocol == 'file'
assert path.bucket is None
assert path.path == '/bucket/dataset/layer'
assert path.host is None
shoulderror('lucifer://bucket/dataset/layer/')
shoulderror('gs://///')
path = paths.extract('file:///tmp/removeme/layer/')
assert path.format == 'precomputed'
assert path.protocol == 'file'
assert path.bucket is None
assert path.path == '/tmp/removeme/layer'
assert path.host is None
assert (paths.extract('gs://username/a/username2/b/c/d')
== ExtractedPath(
'precomputed', 'gs', 'username',
'a/username2/b/c/d', None
))