def _unpack_gzip(self, fname, key, url):
logger.debug("Unpacking data file %s", key)
# We have to use a try except block, as this will crash with
# permissions denied on windows, when trying to copy an open file
# here the temporary file
# Therefore we close the file, after copying and then delete it manually
try:
with gzip.open(fname, "rb") as f_in:
with NamedTemporaryFile("wb", delete=False) as f_out:
with tqdm(
# total=f_in.size,
desc="Unpack",
unit="B",
unit_scale=True,
unit_divisor=1024,
) as t:
fobj = CallbackIOWrapper(t.update, f_in, "read")
while True:
chunk = fobj.read(1024)
if not chunk:
break
f_out.write(chunk)
f_out.flush()
t.reset()
import_file_to_cache(url, f_out.name, pkgname=PKGNAME)
finally:
try:
os.remove(f_out.name)
except:
pass
def upload(deposition_id, file, access_token, url):
logging.getLogger('zenodo_upload').info('Uploading {}'.format(file))
file_name = os.path.basename(file)
r = requests.get('{}/api/deposit/depositions/{}'.format(
url, deposition_id),
params={'access_token': access_token})
bucket_url = r.json()['links']['bucket']
with open(file, 'rb') as fp:
with tqdm(total=os.stat(file).st_size,
unit="B",
unit_scale=True,
unit_divisor=1024) as t:
wrapped_file = CallbackIOWrapper(t.update, fp, 'read')
r = requests.put(
'%s/%s' % (bucket_url, file_name),
data=wrapped_file,
params={'access_token': access_token},
)
if r.status_code >= 400:
if r.status_code == 500:
logging.getLogger('zenodo_upload').critical(
'An error occurred while uploading {}. Status code: {}.)'.
format(file, r.status_code))
else:
logging.getLogger('zenodo_upload').critical(
'An error occurred while uploading {}. Status code: {}. {})'.
format(file, r.status_code, r.json()))
exit(1)
logging.getLogger('zenodo_upload').info(
'Check your upload at {}/deposit/{}'.format(url, deposition_id))
def upload_fobj(self, stream, rpath, callback=None, **kwargs):
parent_id = self._get_item_id(self._parent(rpath), create=True)
if callback:
stream = CallbackIOWrapper(callback.relative_update, stream,
"read")
return self.gdrive_upload_fobj(posixpath.basename(rpath.rstrip("/")),
parent_id, stream)
def tdqm_or_callback_wrapped(
fobj, method, total, callback=None, **pbar_kwargs
):
if callback:
from funcy import nullcontext
from tqdm.utils import CallbackIOWrapper
callback.set_size(total)
wrapper = CallbackIOWrapper(callback.relative_update, fobj, method)
return nullcontext(wrapper)
return Tqdm.wrapattr(fobj, method, total=total, bytes=True, **pbar_kwargs)
def upload(
self,
from_info,
to_info,
total=None,
desc=None,
callback=None,
no_progress_bar=False,
**pbar_args,
):
is_file_obj = hasattr(from_info, "read")
method = "upload_fobj" if is_file_obj else "put_file"
if not hasattr(self, method):
raise RemoteActionNotImplemented(method, self.scheme)
if to_info.scheme != self.scheme:
raise NotImplementedError
if not is_file_obj:
desc = desc or from_info.name
stack = contextlib.ExitStack()
if not callback:
pbar = ui.progress(
desc=desc,
disable=no_progress_bar,
bytes=True,
total=total or -1,
**pbar_args,
)
stack.enter_context(pbar)
callback = pbar.as_callback()
if total:
callback.set_size(total)
with stack:
if is_file_obj:
wrapped = CallbackIOWrapper(callback.relative_update,
from_info)
# `size` is used to provide hints to the WebdavFileSystem
# for legacy servers.
# pylint: disable=no-member
return self.upload_fobj(wrapped, to_info, size=total)
if from_info.scheme != "local":
raise NotImplementedError
logger.debug("Uploading '%s' to '%s'", from_info, to_info)
# pylint: disable=no-member
return self.put_file(os.fspath(from_info),
to_info,
callback=callback)
def open(self, full_path, mode='r', encoding='utf-8'):
if not full_path.startswith('oss://'):
return super().open(full_path, mode)
bucket, path = self._split(full_path)
with mute_stderr():
path_exists = bucket.object_exists(path)
if 'w' in mode:
if path_exists:
bucket.delete_object(path)
if 'b' in mode:
return BinaryOSSFile(bucket, path)
return OSSFile(bucket, path)
elif mode == 'a':
position = bucket.head_object(path).content_length if path_exists else 0
return OSSFile(bucket, path, position=position)
else:
if not path_exists:
raise FileNotFoundError(full_path)
obj = bucket.get_object(path)
# # auto cache large files to avoid memory issues
# if obj.content_length > 200 * 1024 ** 2: # 200M
# path = cache_file(full_path)
# return super().open(path, mode)
if obj.content_length > 200 * 1024 ** 2: # 200M
with tqdm(total=obj.content_length, unit='B', unit_scale=True, unit_divisor=1024, leave=False,
desc='reading ' + os.path.basename(full_path)) as t:
obj = CallbackIOWrapper(t.update, obj, "read")
data = obj.read()
else:
import time
data = obj.read()
if mode == 'rb':
return NullContextWrapper(BytesIO(data))
else:
assert mode == 'r'
return NullContextWrapper(StringIO(data.decode()))
def get_file(
self,
from_info,
to_info,
callback=DEFAULT_CALLBACK,
**kwargs,
):
# pylint: disable=protected-access
total: int = self.getsize(from_info)
if total:
callback.set_size(total)
with self.open(from_info, "rb") as fobj, open(to_info, "wb") as fdest:
wrapped = CallbackIOWrapper(callback.relative_update, fobj)
shutil.copyfileobj(wrapped, fdest, length=fobj.blocksize)
def get_file(self,
from_info,
to_file,
callback=DEFAULT_CALLBACK,
**kwargs):
with self.hdfs(from_info) as hdfs:
file_info = hdfs.get_file_info(from_info.path)
total = file_info.size
if total:
callback.set_size(total)
with hdfs.open_input_stream(from_info.path) as sobj:
with open(to_file, "wb+") as fobj:
wrapped = CallbackIOWrapper(callback.relative_update, sobj)
shutil.copyfileobj(wrapped, fobj, self.BLOCK_SIZE)
def get_file(self,
from_info,
to_file,
callback=DEFAULT_CALLBACK,
**kwargs):
import shutil
total = self.getsize(from_info)
if total:
callback.set_size(total)
with self.open(from_info, "rb", **kwargs) as from_fobj:
with open(to_file, "wb+") as to_fobj:
wrapped = CallbackIOWrapper(callback.relative_update,
from_fobj)
shutil.copyfileobj(wrapped, to_fobj)
def put_file(
self,
from_file,
to_info,
callback=DEFAULT_CALLBACK,
**kwargs,
):
"""Add compatibility support for Callback."""
# pylint: disable=protected-access
self.makedirs(to_info.parent)
size = os.path.getsize(from_file)
with open(from_file, "rb") as fobj:
callback.set_size(size)
wrapped = CallbackIOWrapper(callback.relative_update, fobj)
self.upload_fobj(wrapped, to_info)
self.fs.invalidate_cache(self._with_bucket(to_info.parent))
def put_file(self,
from_file,
to_info,
callback=DEFAULT_CALLBACK,
**kwargs):
with self.hdfs(to_info) as hdfs:
hdfs.create_dir(to_info.parent.path)
tmp_file = tmp_fname(to_info.path)
total: int = os.path.getsize(from_file)
callback.set_size(total)
with open(from_file, "rb") as fobj:
wrapped = CallbackIOWrapper(callback.relative_update, fobj)
with hdfs.open_output_stream(tmp_file) as sobj:
shutil.copyfileobj(wrapped, sobj, self.BLOCK_SIZE)
hdfs.move(tmp_file, to_info.path)
def upload(self, path: str, progressbar: bool = False) -> ParseResponse:
"""
Upload a file located in `path` to http://anonfiles.com.
Example
-------
```
from anonfile import AnonFile
anon = AnonFile('my_token')
result = anon.upload('test.txt')
# https://anonfiles.com/9ee1jcu6u9/test_txt
print(result.url.geturl())
```
Note
----
- `AnonFile` offers unlimited bandwidth
- Uploads cannot exceed a file size of 20G
"""
size = os.stat(path).st_size
options = AnonFile.__progressbar_options(None,
f"Upload: {Path(path).name}",
unit='B',
total=size,
disable=progressbar)
with open(path, mode='rb') as file_handler:
with tqdm(**options) as tqdm_handler:
response = self.session.post(urljoin(AnonFile.API, 'upload'),
params={'token': self.token},
files={
'file':
CallbackIOWrapper(
tqdm_handler.update,
file_handler, 'read')
},
timeout=self.timeout,
proxies=getproxies(),
verify=True)
return ParseResponse(response, Path(path))
def _extractall_with_progress_bar(source, dest):
"Unzip source into dest, updating a progress bar as we go."
# Derived from https://stackoverflow.com/a/65513860rchive.extractall(directory)
dest = Path(dest).expanduser()
with zipfile.ZipFile(source) as zipf, tqdm(
desc="Extracting",
unit="iB",
unit_scale=True,
unit_divisor=1024,
total=sum(getattr(i, "file_size", 0) for i in zipf.infolist()),
) as pbar:
os.makedirs(dest, exist_ok=True)
for i in zipf.infolist():
if not getattr(i, "file_size", 0): # directory
zipf.extract(i, os.fspath(dest))
else:
with zipf.open(i) as fi, open(os.fspath(dest / i.filename), "wb") as fo:
copyfileobj(CallbackIOWrapper(pbar.update, fi), fo)
print(f"Extracted internal data files to {dest}", file=sys.stderr)
def do_put_file(args):
if not os.path.exists(args.localfile):
sys.exit(f"Local file '{args.localfile}' does not exist.")
client = ClientThread(args.uri,
connection_refused_delay=None,
connect_timeout_delay=None)
client.start()
remotefile = create_to_path(args.localfile, args.remotefile)
size = os.stat(args.localfile).st_size
with open(args.localfile, 'rb') as fin:
with tqdm(total=size,
unit='B',
unit_scale=True,
unit_divisor=1024) as progress:
client.wait_for_connection()
client.put_file(CallbackIOWrapper(progress.update, fin, 'read'),
size,
remotefile)
def upload_with_progress_bar(filepath, upload_url):
"""
Upload file with a progress bar
Parameters
----------
filepath : str
Path to file to upload
upload_url : str
URL to put the file onto
"""
_LOGGER.info("Uploading %s to %s", filepath, upload_url)
file_size = os.stat(filepath).st_size
with open(filepath, "rb") as file_handle:
with tqdm(total=file_size, **TQDM_KWARGS) as tqdm_bar:
wrapped_file = CallbackIOWrapper(tqdm_bar.update, file_handle,
"read")
requests.put(upload_url, data=wrapped_file)