async def upload_files(request: JsonRpcRequest,
file_group_name: str,
paths: List[str],
full_path: bool = False,
root=None,
flatten=False,
recursive=True,
merge=False):
total_files = count_files_in_paths(paths)
uploaded_files = 0
def __uploadCallback(current, file_size):
"""
Callback from file upload.
"""
nonlocal uploaded_files
if current == file_size:
uploaded_files += 1
request.push_stream(
dict(uploaded=uploaded_files, total=total_files))
print("Uploded {0}/{1}".format(uploaded_files, total_files))
for path in paths:
remote_path = None
local_path = path
is_dir = os.path.isdir(path)
# If fullPath is true, the the prefix becomes the directory path.
# Any existing prefix is overwritten/ignored
if full_path:
remote_path = path
elif not merge and is_dir:
remote_path = os.path.join(root, os.path.basename(path))
else:
remote_path = root
if recursive and is_dir:
local_path = os.path.join(path, SUBDIR_FILTER)
try:
request.push_stream(dict(uploaded=0, total=total_files))
request.auth.client.file.upload(local_path=local_path,
file_group=file_group_name,
remote_path=remote_path,
flatten=flatten,
progress_callback=__uploadCallback)
except ValueError as valueError:
raise error.JsonRpcError(
code=JsonRpcErrorCodes.BATCH_CLIENT_ERROR,
message="Failed to upload files to file group",
data={"original": str(valueError)})
return dict(uploaded=uploaded_files, total=total_files)
def parse_request(self, message: str) -> JsonRpcRequest:
return JsonRpcRequest.from_json(self, message)
async def upload_files(request: JsonRpcRequest,
file_group_name: str,
paths: List[str],
full_path: bool = False,
root=None,
flatten=False,
recursive=True,
merge=False):
total_files = count_files_in_paths(paths)
uploaded_files = 0
def __uploadCallback(current, file_size):
"""
Callback from file upload.
"""
nonlocal uploaded_files
if current == 0:
print("returning")
return
if current == file_size:
uploaded_files += 1
request.push_stream(
dict(uploaded=uploaded_files, total=total_files))
print("Complete uploads {0}/{1}".format(uploaded_files,
total_files))
else:
# Files larger than 64MB trigger a block upload in storage client
percent = round((current / file_size) * 100)
request.push_stream(
dict(uploaded=uploaded_files,
total=total_files,
partial=percent))
print("Partial upload: {0}% of {1}".format(percent, file_size))
for path in paths:
remote_path = None
local_path = path
is_dir = os.path.isdir(path)
# If fullPath is true, the the prefix becomes the directory path.
# Any existing prefix is overwritten/ignored
if full_path:
remote_path = path
elif not merge and is_dir:
remote_path = os.path.join(root, os.path.basename(path))
else:
remote_path = root
if recursive and is_dir:
local_path = os.path.join(path, SUBDIR_FILTER)
try:
request.push_stream(dict(uploaded=0, total=total_files))
request.auth.client.file.upload(local_path=local_path,
file_group=file_group_name,
remote_path=remote_path,
flatten=flatten,
progress_callback=__uploadCallback)
except ValueError as valueError:
logging.error("Failed to upload files to file group. %s",
str(valueError))
raise error.JsonRpcError(code=JsonRpcErrorCodes.BATCH_CLIENT_ERROR,
message=str(valueError))
return dict(uploaded=uploaded_files, total=total_files)