async def process_request(self, request: JsonRpcRequest):
try:
print("< {0} {1}".format(request.request_id, request.method))
result = await app.call_procedure(request)
response = JsonRpcResponse(
request=request,
result=result,
)
await self.send_response(response)
except error.JsonRpcError as rpc_error:
response = JsonRpcResponse(
request=request,
error=rpc_error,
)
await self.send_response(response)
except Exception as e:
print("Error", type(e), e.args)
print(traceback.format_exc())
response = JsonRpcResponse(
request=request,
error=error.JsonRpcError(500, "Server internal error", str(e)),
)
await self.send_response(response)
async def process_request(self, request: JsonRpcRequest):
try:
print("< {0} {1}".format(request.request_id, request.method))
result = await app.call_procedure(request)
if isinstance(result, ResponseStream):
self.__handle_response_stream(request, result)
else:
response = JsonRpcResponse(
request=request,
result=result,
)
await self.send_response(response)
except error.JsonRpcError as rpc_error:
response = JsonRpcResponse(
request=request,
error=rpc_error,
)
await self.send_response(response)
except Exception as e:
response = JsonRpcResponse(
request=request,
error=error.JsonRpcError(500, "Server internal error", str(e)),
)
await self.send_response(response)
raise e
async def upload_files(request: JsonRpcRequest,
file_group_name: str,
paths: List[str],
full_path: bool = False,
root=None,
flatten=False,
recursive=True,
merge=False):
total_files = count_files_in_paths(paths)
uploaded_files = 0
def __uploadCallback(current, file_size):
"""
Callback from file upload.
"""
nonlocal uploaded_files
if current == file_size:
uploaded_files += 1
request.push_stream(
dict(uploaded=uploaded_files, total=total_files))
print("Uploded {0}/{1}".format(uploaded_files, total_files))
for path in paths:
remote_path = None
local_path = path
is_dir = os.path.isdir(path)
# If fullPath is true, the the prefix becomes the directory path.
# Any existing prefix is overwritten/ignored
if full_path:
remote_path = path
elif not merge and is_dir:
remote_path = os.path.join(root, os.path.basename(path))
else:
remote_path = root
if recursive and is_dir:
local_path = os.path.join(path, SUBDIR_FILTER)
try:
request.push_stream(dict(uploaded=0, total=total_files))
request.auth.client.file.upload(local_path=local_path,
file_group=file_group_name,
remote_path=remote_path,
flatten=flatten,
progress_callback=__uploadCallback)
except ValueError as valueError:
raise error.JsonRpcError(
code=JsonRpcErrorCodes.BATCH_CLIENT_ERROR,
message="Failed to upload files to file group",
data={"original": str(valueError)})
return dict(uploaded=uploaded_files, total=total_files)
def create_file_group(request: JsonRpcRequest, name, directory, options):
# default these parameters as they are optional
prefix, flatten, fullPath = [None, False, False]
if options:
if options.get(PARAM_PREFIX):
prefix = options.get(PARAM_PREFIX)
if options.get(PARAM_FULL_PATH):
fullPath = options.get(PARAM_FULL_PATH)
if options.get(PARAM_FLATTEN):
flatten = options.get(PARAM_FLATTEN)
# If fullPath is true, the the prefix becomes the directory path.
# Any existing prefix is overwritten/ignored
if fullPath:
prefix = directory.replace(SUBDIR_FILTER, "")
try:
request.auth.client.file.upload(directory, name, prefix, flatten,
__uploadCallback)
except ValueError as valueError:
raise error.JsonRpcError(
code=JsonRpcErrorCodes.BATCH_CLIENT_ERROR,
message="Failed to upload files to file group",
data={"original": str(valueError)})
# just return this count to the user, can do something better later
# TODO: keep track of uploading files in the callback below and remove
# this code.
uploadCount = 0
for _, _, files in os.walk(directory.replace(SUBDIR_FILTER, "")):
uploadCount += len(files)
return {
"uploaded": uploadCount,
}
def __getRequiredParameterError(parameter):
return error.JsonRpcError(code=JsonRpcErrorCodes.INVALID_PARAMS,
message=ERROR_REQUIRED_PARAM.format(parameter),
data={"status": 400})
async def upload_files(request: JsonRpcRequest,
file_group_name: str,
paths: List[str],
full_path: bool = False,
root=None,
flatten=False,
recursive=True,
merge=False):
total_files = count_files_in_paths(paths)
uploaded_files = 0
def __uploadCallback(current, file_size):
"""
Callback from file upload.
"""
nonlocal uploaded_files
if current == 0:
print("returning")
return
if current == file_size:
uploaded_files += 1
request.push_stream(
dict(uploaded=uploaded_files, total=total_files))
print("Complete uploads {0}/{1}".format(uploaded_files,
total_files))
else:
# Files larger than 64MB trigger a block upload in storage client
percent = round((current / file_size) * 100)
request.push_stream(
dict(uploaded=uploaded_files,
total=total_files,
partial=percent))
print("Partial upload: {0}% of {1}".format(percent, file_size))
for path in paths:
remote_path = None
local_path = path
is_dir = os.path.isdir(path)
# If fullPath is true, the the prefix becomes the directory path.
# Any existing prefix is overwritten/ignored
if full_path:
remote_path = path
elif not merge and is_dir:
remote_path = os.path.join(root, os.path.basename(path))
else:
remote_path = root
if recursive and is_dir:
local_path = os.path.join(path, SUBDIR_FILTER)
try:
request.push_stream(dict(uploaded=0, total=total_files))
request.auth.client.file.upload(local_path=local_path,
file_group=file_group_name,
remote_path=remote_path,
flatten=flatten,
progress_callback=__uploadCallback)
except ValueError as valueError:
logging.error("Failed to upload files to file group. %s",
str(valueError))
raise error.JsonRpcError(code=JsonRpcErrorCodes.BATCH_CLIENT_ERROR,
message=str(valueError))
return dict(uploaded=uploaded_files, total=total_files)