def validate(self, attrs): instance = StagedFile(**attrs) instance.clean() # This is set in the clean method attrs.update({"file_id": instance.file_id}) return attrs
def _copy_output_files(self, *, container, base_dir: Path): found_files = container.exec_run(f"find {base_dir} -type f") if found_files.exit_code != 0: logger.warning(f"Error listing {base_dir}") return output_files = [ base_dir / Path(f) for f in found_files.output.decode().splitlines() ] if not output_files: logger.warning("Output directory is empty") return # TODO: This thing should not interact with the database result = Result.objects.create(job_id=self._job_id) # Create the upload session but do not save it until we have the # files upload_session = RawImageUploadSession(algorithm_result=result) images = [] for file in output_files: new_uuid = uuid.uuid4() django_file = File(get_file(container=container, src=file)) staged_file = StagedFile( csrf="staging_conversion_csrf", client_id=self._job_id, client_filename=file.name, file_id=new_uuid, timeout=timezone.now() + timedelta(hours=24), start_byte=0, end_byte=django_file.size - 1, total_size=django_file.size, ) staged_file.file.save(f"{uuid.uuid4()}", django_file) staged_file.save() staged_ajax_file = StagedAjaxFile(new_uuid) images.append( RawImageFile( upload_session=upload_session, filename=staged_ajax_file.name, staged_file_id=staged_ajax_file.uuid, ) ) upload_session.save(skip_processing=True) RawImageFile.objects.bulk_create(images) upload_session.process_images()
def create_uploaded_file( content: bytes, chunks=None, user_pk_str="test_user_pk", client_id="test_client_id", client_filename="test_client_filename_{uuid}", timeout=None, init_total_size=True, ) -> uuid.UUID: if chunks is None: chunks = [len(content)] if timeout is None: timeout = timedelta(minutes=1) new_uuid = uuid.uuid4() client_filename = client_filename.format(uuid=new_uuid) start = 0 if type(init_total_size) == int: total_size = init_total_size elif init_total_size: total_size = len(content) else: total_size = None for chunk in chunks: staged_file = StagedFile( user_pk_str=user_pk_str, client_id=client_id, client_filename=client_filename, file_id=new_uuid, timeout=timezone.now() + timeout, start_byte=start, end_byte=chunk - 1, total_size=total_size, ) string_file = BytesIO(content[start:chunk]) django_file = files.File(string_file) staged_file.file.save(f"{client_filename}_{uuid.uuid4()}", django_file) staged_file.save() assert staged_file.file.size == chunk - start start = chunk return new_uuid
def create_uploaded_file( content: bytes, chunks=None, csrf="test_csrf", client_id="test_client_id", client_filename="test_client_filename_{uuid}", timeout=timedelta(minutes=1), init_total_size=True, ) -> uuid.UUID: if chunks is None: chunks = [len(content)] new_uuid = uuid.uuid4() client_filename = client_filename.format(uuid=new_uuid) start = 0 if type(init_total_size) == int: total_size = init_total_size elif init_total_size: total_size = len(content) else: total_size = None for chunk in chunks: staged_file = StagedFile( csrf=csrf, client_id=client_id, client_filename=client_filename, file_id=new_uuid, timeout=timezone.now() + timeout, start_byte=start, end_byte=chunk - 1, total_size=total_size, ) string_file = BytesIO(content[start:chunk]) django_file = files.File(string_file) staged_file.file.save(f"{client_filename}_{uuid.uuid4()}", django_file) staged_file.save() assert staged_file.file.size == chunk - start start = chunk return new_uuid
def _copy_output_files(self, *, container, base_dir: Path): output_files = [ base_dir / Path(f) for f in container.exec_run(f"find {base_dir} -type f") .output.decode() .splitlines() ] if not output_files: raise ValueError("Output directory is empty") # TODO: This thing should not interact with the database job = SubmissionToAnnotationSetJob.objects.get(pk=self._job_id) annotationset = AnnotationSet.objects.create( creator=job.submission.creator, base=job.base, submission=job.submission, kind=AnnotationSet.PREDICTION, ) if self.__was_unzipped: # Create the upload session but do not save it until we have the # files upload_session = RawImageUploadSession(annotationset=annotationset) images = [] for file in output_files: new_uuid = uuid.uuid4() django_file = File(get_file(container=container, src=file)) staged_file = StagedFile( csrf="staging_conversion_csrf", client_id=self._job_id, client_filename=file.name, file_id=new_uuid, timeout=timezone.now() + timedelta(hours=24), start_byte=0, end_byte=django_file.size - 1, total_size=django_file.size, ) staged_file.file.save(f"{uuid.uuid4()}", django_file) staged_file.save() staged_ajax_file = StagedAjaxFile(new_uuid) images.append( RawImageFile( upload_session=upload_session, filename=staged_ajax_file.name, staged_file_id=staged_ajax_file.uuid, ) ) upload_session.save(skip_processing=True) RawImageFile.objects.bulk_create(images) upload_session.process_images() else: assert len(output_files) == 1 f = get_file(container=container, src=output_files[0]) annotationset.labels = process_csv_file(f) annotationset.save()
def _copy_output_files(self, *, container, base_dir: Path): output_files = [ base_dir / Path(f) for f in container.exec_run(f"find {base_dir} -type f") .output.decode() .splitlines() ] if not output_files: raise ValueError("Output directory is empty") # TODO: This thing should not interact with the database job = SubmissionToAnnotationSetJob.objects.get(pk=self._job_id) annotationset = AnnotationSet.objects.create( creator=job.submission.creator, base=job.base, submission=job.submission, kind=AnnotationSet.PREDICTION, ) if self.__was_unzipped: # Create the upload session but do not save it until we have the # files upload_session = RawImageUploadSession(annotationset=annotationset) images = [] for file in output_files: new_uuid = uuid.uuid4() django_file = File(get_file(container=container, src=file)) staged_file = StagedFile( csrf="staging_conversion_csrf", client_id=self._job_id, client_filename=file.name, file_id=new_uuid, timeout=timezone.now() + timedelta(hours=24), start_byte=0, end_byte=django_file.size - 1, total_size=django_file.size, ) staged_file.file.save(f"{uuid.uuid4()}", django_file) staged_file.save() staged_ajax_file = StagedAjaxFile(new_uuid) images.append( RawImageFile( upload_session=upload_session, filename=staged_ajax_file.name, staged_file_id=staged_ajax_file.uuid, ) ) upload_session.save(skip_processing=True) RawImageFile.objects.bulk_create(images) upload_session.process_images() else: assert len(output_files) == 1 f = get_file(container=container, src=output_files[0]) annotationset.labels = process_csv_file(f) annotationset.save()