def __init__(
self,
fs,
url,
mode="rb",
asynchronous=False,
session=None,
loop=None,
**kwargs
):
path = fs._strip_protocol(url)
url = URL(fs.webdav_url) / path
self.url = url.as_uri()
self.details = {"name": self.url, "size": None}
self.asynchronous = asynchronous
self.session = session
self.loop = loop
super(HTTPStreamFile, self).__init__(
fs=fs,
path=path,
mode=mode,
block_size=0,
cache_type="none",
cache_options={},
**kwargs)
if self.mode == "rb":
self.r = sync(self.loop, self.session.get, self.url, **self.kwargs)
elif self.mode == "wb":
pass
else:
raise ValueError
def __init__(
self,
fs,
url,
mode="rb",
block_size=None,
cache_type="bytes",
cache_options=None,
asynchronous=False,
session=None,
loop=None,
**kwargs
):
path = fs._strip_protocol(url)
url = URL(fs.webdav_url) / path
self.url = url.as_uri()
self.asynchronous = asynchronous
self.session = session
self.loop = loop
if mode not in {"rb", "wb"}:
raise ValueError
super(HTTPFile, self).__init__(
fs=fs,
path=path,
mode=mode,
block_size=block_size,
cache_type=cache_type,
cache_options=cache_options,
**kwargs
)
async def _put_file(self, lpath, rpath, **kwargs):
self.webdav_url = self._get_webdav_url(rpath) or self.webdav_url
path = self._strip_protocol(rpath)
url = URL(self.webdav_url) / path
url = url.as_uri()
kw = self.kwargs.copy()
kw.update(kwargs)
with open(lpath, "rb") as fd:
r = await self.session.put(url, data=fd, **self.kwargs)
r.raise_for_status()
async def _rm_file(self, path, **kwargs):
"""
Remove file or directory (must be empty)
:param path: (str)
"""
url = URL(self.api_url) / 'namespace' / _encode(path)
url = url.as_uri()
kw = self.kwargs.copy()
kw.update(kwargs)
async with self.session.delete(url, **kw) as r:
if r.status == 404:
raise FileNotFoundError(url)
r.raise_for_status()
async def _get_file(self, rpath, lpath, chunk_size=5 * 2 ** 20, **kwargs):
self.webdav_url = self._get_webdav_url(rpath) or self.webdav_url
path = self._strip_protocol(rpath)
url = URL(self.webdav_url) / path
url = url.as_uri()
kw = self.kwargs.copy()
kw.update(kwargs)
async with self.session.get(url, **self.kwargs) as r:
if r.status == 404:
raise FileNotFoundError(rpath)
r.raise_for_status()
with open(lpath, "wb") as fd:
chunk = True
while chunk:
chunk = await r.content.read(chunk_size)
fd.write(chunk)
async def _cat_file(self, url, start=None, end=None, **kwargs):
self.webdav_url = self._get_webdav_url(url) or self.webdav_url
path = self._strip_protocol(url)
url = URL(self.webdav_url) / path
url = url.as_uri()
kw = self.kwargs.copy()
kw.update(kwargs)
if (start is None) ^ (end is None):
raise ValueError("Give start and end or neither")
if start is not None:
headers = kw.pop("headers", {}).copy()
headers["Range"] = "bytes=%i-%i" % (start, end - 1)
kw["headers"] = headers
async with self.session.get(url, **kw) as r:
if r.status == 404:
raise FileNotFoundError(url)
r.raise_for_status()
out = await r.read()
return out
def test_simple(self):
original = 'http://www.example.com/path/to/file.ext?query#fragment'
url = URL(original)
self.assertEqual(str(url), original)
self.assertEqual(url.as_uri(), original)
self.assertEqual(url.as_posix(), original)
self.assertEqual(url.drive, 'http://www.example.com')
self.assertEqual(url.root, '/')
self.assertEqual(url.anchor, 'http://www.example.com/')
self.assertEqual(url.path, '/path/to/file.ext')
self.assertEqual(url.name, 'file.ext')
self.assertEqual(url.suffix, '.ext')
self.assertListEqual(url.suffixes, ['.ext'])
self.assertEqual(url.stem, 'file')
self.assertTupleEqual(url.parts, ('http://www.example.com/', 'path', 'to', 'file.ext'))
self.assertEqual(str(url.parent), 'http://www.example.com/path/to')
self.assertEqual(url.scheme, 'http')
self.assertEqual(url.netloc, 'www.example.com')
self.assertEqual(url.query, 'query')
self.assertEqual(url.fragment, 'fragment')
async def _mv(self, path1, path2, **kwargs):
"""
Rename path1 to path2
:param path1: (str) source path
:param path2: (str) destination path
:param kwargs: (dict) optional arguments passed on to requests
"""
path1 = self._strip_protocol(path1)
path2 = self._strip_protocol(path2)
url = URL(self.api_url) / 'namespace' / _encode(path1)
url = url.as_uri()
data = dict(action='mv', destination=path2)
kw = self.kwargs.copy()
kw.update(kwargs)
async with self.session.post(url, json=data, **kw) as r:
if r.status == 404:
raise FileNotFoundError(url)
r.raise_for_status()
return await r.json()
def test_simple(self):
original = 'http://www.example.com/path/to/file.ext?query#fragment'
url = URL(original)
self.assertEqual(str(url), original)
self.assertEqual(url.as_uri(), original)
self.assertEqual(url.as_posix(), original)
self.assertEqual(url.drive, 'http://www.example.com')
self.assertEqual(url.root, '/')
self.assertEqual(url.anchor, 'http://www.example.com/')
self.assertEqual(url.path, '/path/to/file.ext')
self.assertEqual(url.name, 'file.ext')
self.assertEqual(url.suffix, '.ext')
self.assertListEqual(url.suffixes, ['.ext'])
self.assertEqual(url.stem, 'file')
self.assertTupleEqual(
url.parts, ('http://www.example.com/', 'path', 'to', 'file.ext'))
self.assertEqual(str(url.parent), 'http://www.example.com/path/to')
self.assertEqual(url.scheme, 'http')
self.assertEqual(url.netloc, 'www.example.com')
self.assertEqual(url.query, 'query')
self.assertEqual(url.fragment, 'fragment')
async def _get_info(self, path, children=False, limit=None, **kwargs):
"""
Request file or directory metadata to the API
:param path: (str)
:param children: (bool) if True, return metadata of the children paths
as well
:param limit: (int) if provided and children is True, set limit to the
number of children returned
:param kwargs: (dict) optional arguments passed on to requests
:return (dict) path metadata
"""
url = URL(self.api_url) / 'namespace' / _encode(path)
url = url.with_query(children=children)
if limit is not None and children:
url = url.add_query(limit=f'{limit}')
url = url.as_uri()
kw = self.kwargs.copy()
kw.update(kwargs)
async with self.session.get(url, **kw) as r:
if r.status == 404:
raise FileNotFoundError(url)
r.raise_for_status()
return await r.json()
