def test_upload_files(dataset: RemoteDataset, request_upload_endpoint: str):
request_upload_response = {
"blocked_items": [],
"items": [{"dataset_item_id": 1, "filename": "test.jpg", "path": "/"}],
}
upload_to_s3_endpoint = "https://darwin-data.s3.eu-west-1.amazonaws.com/test.jpg?X-Amz-Signature=abc"
confirm_upload_endpoint = "http://localhost/api/dataset_items/1/confirm_upload"
sign_upload_endpoint = "http://localhost/api/dataset_items/1/sign_upload"
sign_upload_response = {"upload_url": upload_to_s3_endpoint}
responses.add(responses.PUT, request_upload_endpoint, json=request_upload_response, status=200)
responses.add(responses.GET, sign_upload_endpoint, json=sign_upload_response, status=200)
responses.add(responses.PUT, upload_to_s3_endpoint, status=201)
responses.add(responses.PUT, confirm_upload_endpoint, status=200)
Path("test.jpg").touch()
local_file = LocalFile(local_path=Path("test.jpg"))
upload_handler = UploadHandler(dataset, [local_file])
upload_handler.upload()
for file_to_upload in upload_handler.progress:
file_to_upload()
responses.assert_call_count(request_upload_endpoint, 1)
responses.assert_call_count(sign_upload_endpoint, 1)
responses.assert_call_count(upload_to_s3_endpoint, 1)
responses.assert_call_count(confirm_upload_endpoint, 1)
assert upload_handler.error_count == 0
def test_error_count_is_correct(dataset: RemoteDataset, request_upload_endpoint: str):
request_upload_response = {
"blocked_items": [],
"items": [{"dataset_item_id": 1, "filename": "test.jpg", "path": "/"}],
}
sign_upload_endpoint = "http://localhost/api/dataset_items/1/sign_upload"
upload_to_s3_endpoint = "https://darwin-data.s3.eu-west-1.amazonaws.com/test.jpg?X-Amz-Signature=abc"
confirm_upload_endpoint = "http://localhost/api/dataset_items/1/confirm_upload"
responses.add(responses.PUT, request_upload_endpoint, json=request_upload_response, status=200)
responses.add(responses.GET, sign_upload_endpoint, status=500)
local_file = LocalFile(local_path=Path("test.jpg"))
upload_handler = UploadHandler(dataset, [local_file])
upload_handler.upload()
for file_to_upload in upload_handler.progress:
file_to_upload()
responses.assert_call_count(request_upload_endpoint, 1)
responses.assert_call_count(sign_upload_endpoint, 1)
responses.assert_call_count(upload_to_s3_endpoint, 0)
responses.assert_call_count(confirm_upload_endpoint, 0)
assert upload_handler.pending_count == 1
assert upload_handler.error_count == 1
assert upload_handler.blocked_count == 0
error = upload_handler.errors[0]
assert str(error.file_path) == "test.jpg"
assert error.stage == UploadStage.REQUEST_SIGNATURE
def test_pending_count_is_correct(dataset: RemoteDataset, request_upload_endpoint: str):
response = {"blocked_items": [], "items": [{"dataset_item_id": 1, "filename": "test.jpg", "path": "/"}]}
responses.add(responses.PUT, request_upload_endpoint, json=response, status=200)
local_file = LocalFile(local_path=Path("test.jpg"))
upload_handler = UploadHandler(dataset, [local_file])
assert upload_handler.pending_count == 1
assert upload_handler.blocked_count == 0
assert upload_handler.error_count == 0
pending_item = upload_handler.pending_items[0]
assert pending_item.dataset_item_id == 1
assert pending_item.filename == "test.jpg"
assert pending_item.path == "/"
assert pending_item.reason is None
def works_with_local_files_list(remote_dataset: RemoteDataset):
assert_upload_mocks_are_correctly_called(remote_dataset,
[LocalFile("test.jpg")])
def raises_if_both_as_frames_and_local_files_are_given(
remote_dataset: RemoteDataset):
with pytest.raises(ValueError):
remote_dataset.push([LocalFile("test.jpg")], as_frames=True)
def push(
self,
files_to_upload: Optional[List[Union[PathLike, LocalFile]]],
*,
blocking: bool = True,
multi_threaded: bool = True,
fps: int = 0,
as_frames: bool = False,
files_to_exclude: Optional[List[PathLike]] = None,
path: Optional[str] = None,
preserve_folders: bool = False,
progress_callback: Optional[ProgressCallback] = None,
file_upload_callback: Optional[FileUploadCallback] = None,
) -> UploadHandler:
"""Uploads a local dataset (images ONLY) in the datasets directory.
Parameters
----------
files_to_upload : Optional[List[Union[PathLike, LocalFile]]]
List of files to upload. Those can be folders.
blocking : bool
If False, the dataset is not uploaded and a generator function is returned instead.
multi_threaded : bool
Uses multiprocessing to upload the dataset in parallel.
If blocking is False this has no effect.
files_to_exclude : Optional[PathLike]]
Optional list of files to exclude from the file scan. Those can be folders.
fps : int
When the uploading file is a video, specify its framerate.
as_frames: bool
When the uploading file is a video, specify whether it's going to be uploaded as a list of frames.
path: Optional[str]
Optional path to store the files in.
preserve_folders : bool
Specify whether or not to preserve folder paths when uploading
progress_callback: Optional[ProgressCallback]
Optional callback, called every time the progress of an uploading files is reported.
file_upload_callback: Optional[FileUploadCallback]
Optional callback, called every time a file chunk is uploaded.
Returns
-------
handler : UploadHandler
Class for handling uploads, progress and error messages
"""
if files_to_exclude is None:
files_to_exclude = []
if files_to_upload is None:
raise ValueError("No files or directory specified.")
uploading_files = [
item for item in files_to_upload if isinstance(item, LocalFile)
]
search_files = [
item for item in files_to_upload
if not isinstance(item, LocalFile)
]
generic_parameters_specified = path is not None or fps != 0 or as_frames is not False
if uploading_files and generic_parameters_specified:
raise ValueError(
"Cannot specify a path when uploading a LocalFile object.")
for found_file in find_files(search_files,
files_to_exclude=files_to_exclude):
local_path = path
if preserve_folders:
source_files = [
source_file for source_file in search_files
if is_relative_to(found_file, source_file)
]
if source_files:
local_path = str(
found_file.relative_to(source_files[0]).parent)
uploading_files.append(
LocalFile(found_file,
fps=fps,
as_frames=as_frames,
path=local_path))
if not uploading_files:
raise ValueError(
"No files to upload, check your path, exclusion filters and resume flag"
)
handler = UploadHandler(self, uploading_files)
if blocking:
handler.upload(
multi_threaded=multi_threaded,
progress_callback=progress_callback,
file_upload_callback=file_upload_callback,
)
else:
handler.prepare_upload()
return handler