def _add_from_url(self, dataset, path, url, nocopy=False, **kwargs):
"""Process an add from url and return the location on disk."""
u = parse.urlparse(url)
if u.scheme not in Dataset.SUPPORTED_SCHEMES:
raise NotImplementedError('{} URLs are not supported'.format(
u.scheme))
dst = path.joinpath(os.path.basename(url)).absolute()
if u.scheme in ('', 'file'):
src = Path(u.path).absolute()
# if we have a directory, recurse
if src.is_dir():
files = {}
os.mkdir(dst)
for f in src.iterdir():
files.update(
self._add_from_url(dataset,
dst,
f.absolute().as_posix(),
nocopy=nocopy))
return files
if nocopy:
try:
os.link(src, dst)
except Exception as e:
raise Exception('Could not create hard link '
'- retry without nocopy.') from e
else:
shutil.copy(src, dst)
# Do not expose local paths.
src = None
else:
try:
response = requests.get(url)
dst.write_bytes(response.content)
except error.HTTPError as e: # pragma nocover
raise e
# make the added file read-only
mode = dst.stat().st_mode & 0o777
dst.chmod(mode & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
self.track_paths_in_storage(dst.relative_to(self.path))
dataset_path = self.path / self.datadir / dataset.name
result = dst.relative_to(dataset_path).as_posix()
return {
result:
DatasetFile(
path=result,
url=url,
authors=dataset.authors,
dataset=dataset.name,
)
}
def _add_from_url(self, dataset, path, url, link=False, **kwargs):
"""Process an add from url and return the location on disk."""
u = parse.urlparse(url)
if u.scheme not in Dataset.SUPPORTED_SCHEMES:
raise NotImplementedError(
'{} URLs are not supported'.format(u.scheme)
)
# Respect the directory struture inside the source path.
relative_to = kwargs.pop('relative_to', None)
if relative_to:
dst_path = Path(u.path).resolve().absolute().relative_to(
Path(relative_to).resolve().absolute()
)
else:
dst_path = os.path.basename(u.path)
dst = path.joinpath(dst_path).absolute()
if u.scheme in ('', 'file'):
src = Path(u.path).absolute()
# if we have a directory, recurse
if src.is_dir():
files = []
dst.mkdir(parents=True, exist_ok=True)
for f in src.iterdir():
files.extend(
self._add_from_url(
dataset,
dst,
f.absolute().as_posix(),
link=link,
**kwargs
)
)
return files
# Make sure the parent directory exists.
dst.parent.mkdir(parents=True, exist_ok=True)
if link:
try:
os.link(str(src), str(dst))
except Exception as e:
raise Exception(
'Could not create hard link '
'- retry without --link.'
) from e
else:
shutil.copy(str(src), str(dst))
# Do not expose local paths.
src = None
else:
try:
response = requests.get(url)
dst.write_bytes(response.content)
except error.HTTPError as e: # pragma nocover
raise e
# make the added file read-only
mode = dst.stat().st_mode & 0o777
dst.chmod(mode & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
self.track_paths_in_storage(str(dst.relative_to(self.path)))
return [{
'path': dst.relative_to(self.path),
'url': url,
'creator': dataset.creator,
'dataset': dataset.name,
'parent': self
}]
