def upload_zipped_flat_media_files( gc: GirderClient, manager: JobManager, folderId: str, working_directory: Path, create_subfolder=False, ): """Takes a flat folder of media files and/or annotation and generates a dataset from it""" listOfFileNames = os.listdir(working_directory) validation = gc.sendRestRequest('POST', '/dive_dataset/validate_files', json=listOfFileNames) root_folderId = folderId default_fps = gc.getFolder(root_folderId).get( f"meta.{constants.FPSMarker}", -1) if validation.get('ok', False): manager.write(f"Annotations: {validation['annotations']}\n") manager.write(f"Media: {validation['media']}\n") dataset_type = validation['type'] manager.write(f"Type: {dataset_type}\n") if create_subfolder != '': sub_folder = gc.createFolder( folderId, create_subfolder, reuseExisting=True, ) root_folderId = str(sub_folder["_id"]) # Upload all resulting items back into the root folder manager.updateStatus(JobStatus.PUSHING_OUTPUT) # create a source folder to place the zipFile inside of gc.upload(f'{working_directory}/*', root_folderId) if dataset_type == constants.ImageSequenceType and default_fps == -1: default_fps = 1 gc.addMetadataToFolder( str(root_folderId), { constants.TypeMarker: dataset_type, constants.FPSMarker: default_fps }, ) # After uploading the default files we do a the postprocess for video conversion now gc.sendRestRequest("POST", f"/dive_rpc/postprocess/{str(root_folderId)}") else: manager.write(f"Message: {validation['message']}\n") raise Exception("Could not Validate media Files")
def test_upload_zip_data(dataset: dict): user = zipUser client = GirderClient(apiUrl='http://localhost:8010/api/v1') client.authenticate(username=user['login'], password=user['password']) dsPath = localDataRoot / str(dataset['path']) privateFolder = getTestFolder(client) newDatasetFolder = client.createFolder( privateFolder['_id'], dataset['name'], metadata={ 'fps': dataset['fps'], 'type': dataset['type'], }, ) if Path(dsPath).is_file(): client.uploadFileToFolder(newDatasetFolder['_id'], str(dsPath)) client.post(f'dive_rpc/postprocess/{newDatasetFolder["_id"]}') wait_for_jobs(client, max_wait_timeout=30, expected_status=dataset['job_status']) resultFolder = client.getFolder(newDatasetFolder['_id']) # verify sub datasets if they exist if dataset.get('subDatasets', False): folders = list(client.listFolder(newDatasetFolder['_id'])) for item in dataset["subDatasets"]: matches = [x for x in folders if x["name"] == item["name"]] if len(matches) > 0: meta = matches[0].get("meta", {}) assert meta.get("fps", -1) == item["fps"] assert meta.get("type", "") == item["type"] assert meta.get("annotate", False) elif dataset['job_status'] == JobStatus.SUCCESS: assert resultFolder['meta'].get("annotate", False) assert type(resultFolder['meta'].get("fps")) in [int, float] assert type(resultFolder['meta'].get("type")) == str else: assert resultFolder['meta'].get("annotate", None) is None