def upload_zipped_flat_media_files(
gc: GirderClient,
manager: JobManager,
folderId: str,
working_directory: Path,
create_subfolder=False,
):
"""Takes a flat folder of media files and/or annotation and generates a dataset from it"""
listOfFileNames = os.listdir(working_directory)
validation = gc.sendRestRequest('POST',
'/dive_dataset/validate_files',
json=listOfFileNames)
root_folderId = folderId
default_fps = gc.getFolder(root_folderId).get(
f"meta.{constants.FPSMarker}", -1)
if validation.get('ok', False):
manager.write(f"Annotations: {validation['annotations']}\n")
manager.write(f"Media: {validation['media']}\n")
dataset_type = validation['type']
manager.write(f"Type: {dataset_type}\n")
if create_subfolder != '':
sub_folder = gc.createFolder(
folderId,
create_subfolder,
reuseExisting=True,
)
root_folderId = str(sub_folder["_id"])
# Upload all resulting items back into the root folder
manager.updateStatus(JobStatus.PUSHING_OUTPUT)
# create a source folder to place the zipFile inside of
gc.upload(f'{working_directory}/*', root_folderId)
if dataset_type == constants.ImageSequenceType and default_fps == -1:
default_fps = 1
gc.addMetadataToFolder(
str(root_folderId),
{
constants.TypeMarker: dataset_type,
constants.FPSMarker: default_fps
},
)
# After uploading the default files we do a the postprocess for video conversion now
gc.sendRestRequest("POST",
f"/dive_rpc/postprocess/{str(root_folderId)}")
else:
manager.write(f"Message: {validation['message']}\n")
raise Exception("Could not Validate media Files")
def test_upload_zip_data(dataset: dict):
user = zipUser
client = GirderClient(apiUrl='http://localhost:8010/api/v1')
client.authenticate(username=user['login'], password=user['password'])
dsPath = localDataRoot / str(dataset['path'])
privateFolder = getTestFolder(client)
newDatasetFolder = client.createFolder(
privateFolder['_id'],
dataset['name'],
metadata={
'fps': dataset['fps'],
'type': dataset['type'],
},
)
if Path(dsPath).is_file():
client.uploadFileToFolder(newDatasetFolder['_id'], str(dsPath))
client.post(f'dive_rpc/postprocess/{newDatasetFolder["_id"]}')
wait_for_jobs(client,
max_wait_timeout=30,
expected_status=dataset['job_status'])
resultFolder = client.getFolder(newDatasetFolder['_id'])
# verify sub datasets if they exist
if dataset.get('subDatasets', False):
folders = list(client.listFolder(newDatasetFolder['_id']))
for item in dataset["subDatasets"]:
matches = [x for x in folders if x["name"] == item["name"]]
if len(matches) > 0:
meta = matches[0].get("meta", {})
assert meta.get("fps", -1) == item["fps"]
assert meta.get("type", "") == item["type"]
assert meta.get("annotate", False)
elif dataset['job_status'] == JobStatus.SUCCESS:
assert resultFolder['meta'].get("annotate", False)
assert type(resultFolder['meta'].get("fps")) in [int, float]
assert type(resultFolder['meta'].get("type")) == str
else:
assert resultFolder['meta'].get("annotate", None) is None
