def _upload_loop(file_chunk_stream: FileChunkStream, query_params: dict, **kwargs):
for iteration, chunk in enumerate(file_chunk_stream.generate()):
if "reset" in query_params and iteration != 0:
query_params["reset"] = str(False)
result = _upload_loop_chunk(
chunk, file_chunk_stream, query_params=query_params.copy(), **kwargs
)
_attribute_upload_response_handler(result)
file_chunk_stream.close()
def upload_to_storage(upload_entries, upload_api_fun, upload_tar_api_fun, warn_limit=None, **kwargs):
unique_upload_entries = scan_unique_upload_entries(upload_entries)
progress_indicator = SilentProgressIndicator()
if warn_limit is not None:
total_size = 0
for entry in unique_upload_entries:
if not entry.is_stream():
total_size += os.path.getsize(entry.source_path)
if total_size >= warn_limit:
progress_indicator = LoggingProgressIndicator(total_size)
for package in split_upload_files(unique_upload_entries):
if package.is_empty():
continue
uploading_multiple_entries = package.len > 1
creating_a_single_empty_dir = package.len == 1 and not package.items[0].is_stream() \
and os.path.isdir(package.items[0].source_path)
if uploading_multiple_entries or creating_a_single_empty_dir:
data = compress_to_tar_gz_in_memory(upload_entries=package.items)
upload_tar_api_fun(**dict(kwargs, data=data))
progress_indicator.progress(package.size)
else:
file_chunk_stream = FileChunkStream(package.items[0])
upload_api_fun(**dict(kwargs, data=file_chunk_stream, progress_indicator=progress_indicator))
progress_indicator.complete()
def test_permissions_to_unix_string_for_nonexistent_file(self):
# when
permissions_string = FileChunkStream.permissions_to_unix_string(
'/some/path')
# then
self.assertEqual('-' * 10, permissions_string)
def test_generate_chunks_from_stream(self):
# given
text = u"ABCDEFGHIJKLMNOPRSTUWXYZ"
# when
stream = FileChunkStream(UploadEntry(StringIO(text), "some/path"))
chunks = list()
for chunk in stream.generate(chunk_size=10):
chunks.append(chunk)
# then
self.assertEqual(stream.length, None)
self.assertEqual(chunks, [
FileChunk(b"ABCDEFGHIJ", 0, 10),
FileChunk(b"KLMNOPRSTU", 10, 20),
FileChunk(b"WXYZ", 20, 24)
])
def test_permissions_to_unix_string_for_directory(self, lstat):
# given
lstat.return_value.st_mode = 0o642
# when
permissions_string = FileChunkStream.permissions_to_unix_string('/some/path')
# then
self.assertEqual('drw-r---w-', permissions_string)
def test_permissions_to_unix_string_for_file(self, lstat):
# given
lstat.return_value.st_mode = 0o731
# when
permissions_string = FileChunkStream.permissions_to_unix_string('/some/path')
# then
self.assertEqual('-rwx-wx--x', permissions_string)
def upload_file_attribute(
swagger_client: SwaggerClient,
container_id: str,
attribute: str,
source: Union[str, bytes],
ext: str,
multipart_config: Optional[MultipartConfig],
) -> List[NeptuneException]:
if isinstance(source, str) and not os.path.isfile(source):
return [FileUploadError(source, "Path not found or is a not a file.")]
target = attribute
if ext:
target += "." + ext
try:
upload_entry = UploadEntry(
source if isinstance(source, str) else BytesIO(source), target
)
if multipart_config is None:
# the legacy upload procedure
url = build_operation_url(
swagger_client.swagger_spec.api_url,
swagger_client.api.uploadAttribute.operation.path_name,
)
upload_configuration = DEFAULT_UPLOAD_CONFIG
_upload_loop(
file_chunk_stream=FileChunkStream(upload_entry, upload_configuration),
http_client=swagger_client.swagger_spec.http_client,
url=url,
query_params={
"experimentId": container_id,
"attribute": attribute,
"ext": ext,
},
)
else:
_multichunk_upload(
upload_entry,
query_params={
"experimentIdentifier": container_id,
"attribute": attribute,
"ext": ext,
},
swagger_client=swagger_client,
multipart_config=multipart_config,
target=FileUploadTarget.FILE_ATOM,
)
except MetadataInconsistency as e:
return [e]
def test_generate_chunks_from_stream(self):
# given
text = u"ABCDEFGHIJKLMNOPRSTUWXYZ"
# when
stream = FileChunkStream(
UploadEntry(BytesIO(bytes(text, "utf-8")), "some/path"),
AttributeUploadConfiguration(10),
)
chunks = list()
for chunk in stream.generate():
chunks.append(chunk)
# then
self.assertEqual(stream.length, 24)
self.assertEqual(
chunks,
[
FileChunk(b"ABCDEFGHIJ", 0, 10),
FileChunk(b"KLMNOPRSTU", 10, 20),
FileChunk(b"WXYZ", 20, 24),
],
)
def upload_to_storage(files_list, upload_api_fun, upload_tar_api_fun,
**kwargs):
for package in split_upload_files(files_list):
if package.is_empty():
break
uploading_multiple_entries = package.len > 1
creating_a_single_empty_dir = package.len == 1 and os.path.isdir(
package.items[0].source_path)
if uploading_multiple_entries or creating_a_single_empty_dir:
data = compress_to_tar_gz_in_memory(upload_entries=package.items)
upload_tar_api_fun(**dict(kwargs, data=data))
else:
file_chunk_stream = FileChunkStream(package.items[0])
upload_api_fun(**dict(kwargs, data=file_chunk_stream))
def upload_file_set_attribute(
swagger_client: SwaggerClient,
container_id: str,
attribute: str,
file_globs: Iterable[str],
reset: bool,
multipart_config: Optional[MultipartConfig],
) -> List[NeptuneException]:
unique_upload_entries = get_unique_upload_entries(file_globs)
try:
upload_configuration = DEFAULT_UPLOAD_CONFIG
for package in split_upload_files(
upload_entries=unique_upload_entries,
upload_configuration=upload_configuration,
):
if package.is_empty() and not reset:
continue
uploading_multiple_entries = package.len > 1
creating_a_single_empty_dir = (
package.len == 1
and not package.items[0].is_stream()
and os.path.isdir(package.items[0].source_path)
)
if (
uploading_multiple_entries
or creating_a_single_empty_dir
or package.is_empty()
):
data = compress_to_tar_gz_in_memory(upload_entries=package.items)
url = build_operation_url(
swagger_client.swagger_spec.api_url,
swagger_client.api.uploadFileSetAttributeTar.operation.path_name,
)
result = upload_raw_data(
http_client=swagger_client.swagger_spec.http_client,
url=url,
data=data,
headers={"Content-Type": "application/octet-stream"},
query_params={
"experimentId": container_id,
"attribute": attribute,
"reset": str(reset),
},
)
_attribute_upload_response_handler(result)
else:
upload_entry = package.items[0]
if multipart_config is None:
# the legacy upload procedure
url = build_operation_url(
swagger_client.swagger_spec.api_url,
swagger_client.api.uploadFileSetAttributeChunk.operation.path_name,
)
file_chunk_stream = FileChunkStream(
upload_entry=upload_entry,
upload_configuration=upload_configuration,
)
_upload_loop(
file_chunk_stream=file_chunk_stream,
http_client=swagger_client.swagger_spec.http_client,
url=url,
query_params={
"experimentId": container_id,
"attribute": attribute,
"reset": str(reset),
"path": upload_entry.target_path,
},
)
else:
_multichunk_upload(
upload_entry,
query_params={
"experimentIdentifier": container_id,
"attribute": attribute,
"subPath": upload_entry.target_path,
},
swagger_client=swagger_client,
multipart_config=multipart_config,
target=FileUploadTarget.FILE_SET,
)
reset = False
except MetadataInconsistency as e:
if len(e.args) == 1:
return [e]
else:
return [MetadataInconsistency(desc) for desc in e.args]