def validate(self, attrs):
instance = StagedFile(**attrs)
instance.clean()
# This is set in the clean method
attrs.update({"file_id": instance.file_id})
return attrs
def _copy_output_files(self, *, container, base_dir: Path):
found_files = container.exec_run(f"find {base_dir} -type f")
if found_files.exit_code != 0:
logger.warning(f"Error listing {base_dir}")
return
output_files = [
base_dir / Path(f)
for f in found_files.output.decode().splitlines()
]
if not output_files:
logger.warning("Output directory is empty")
return
# TODO: This thing should not interact with the database
result = Result.objects.create(job_id=self._job_id)
# Create the upload session but do not save it until we have the
# files
upload_session = RawImageUploadSession(algorithm_result=result)
images = []
for file in output_files:
new_uuid = uuid.uuid4()
django_file = File(get_file(container=container, src=file))
staged_file = StagedFile(
csrf="staging_conversion_csrf",
client_id=self._job_id,
client_filename=file.name,
file_id=new_uuid,
timeout=timezone.now() + timedelta(hours=24),
start_byte=0,
end_byte=django_file.size - 1,
total_size=django_file.size,
)
staged_file.file.save(f"{uuid.uuid4()}", django_file)
staged_file.save()
staged_ajax_file = StagedAjaxFile(new_uuid)
images.append(
RawImageFile(
upload_session=upload_session,
filename=staged_ajax_file.name,
staged_file_id=staged_ajax_file.uuid,
)
)
upload_session.save(skip_processing=True)
RawImageFile.objects.bulk_create(images)
upload_session.process_images()
def create_uploaded_file(
content: bytes,
chunks=None,
user_pk_str="test_user_pk",
client_id="test_client_id",
client_filename="test_client_filename_{uuid}",
timeout=None,
init_total_size=True,
) -> uuid.UUID:
if chunks is None:
chunks = [len(content)]
if timeout is None:
timeout = timedelta(minutes=1)
new_uuid = uuid.uuid4()
client_filename = client_filename.format(uuid=new_uuid)
start = 0
if type(init_total_size) == int:
total_size = init_total_size
elif init_total_size:
total_size = len(content)
else:
total_size = None
for chunk in chunks:
staged_file = StagedFile(
user_pk_str=user_pk_str,
client_id=client_id,
client_filename=client_filename,
file_id=new_uuid,
timeout=timezone.now() + timeout,
start_byte=start,
end_byte=chunk - 1,
total_size=total_size,
)
string_file = BytesIO(content[start:chunk])
django_file = files.File(string_file)
staged_file.file.save(f"{client_filename}_{uuid.uuid4()}", django_file)
staged_file.save()
assert staged_file.file.size == chunk - start
start = chunk
return new_uuid
def create_uploaded_file(
content: bytes,
chunks=None,
csrf="test_csrf",
client_id="test_client_id",
client_filename="test_client_filename_{uuid}",
timeout=timedelta(minutes=1),
init_total_size=True,
) -> uuid.UUID:
if chunks is None:
chunks = [len(content)]
new_uuid = uuid.uuid4()
client_filename = client_filename.format(uuid=new_uuid)
start = 0
if type(init_total_size) == int:
total_size = init_total_size
elif init_total_size:
total_size = len(content)
else:
total_size = None
for chunk in chunks:
staged_file = StagedFile(
csrf=csrf,
client_id=client_id,
client_filename=client_filename,
file_id=new_uuid,
timeout=timezone.now() + timeout,
start_byte=start,
end_byte=chunk - 1,
total_size=total_size,
)
string_file = BytesIO(content[start:chunk])
django_file = files.File(string_file)
staged_file.file.save(f"{client_filename}_{uuid.uuid4()}", django_file)
staged_file.save()
assert staged_file.file.size == chunk - start
start = chunk
return new_uuid
def _copy_output_files(self, *, container, base_dir: Path):
output_files = [
base_dir / Path(f)
for f in container.exec_run(f"find {base_dir} -type f")
.output.decode()
.splitlines()
]
if not output_files:
raise ValueError("Output directory is empty")
# TODO: This thing should not interact with the database
job = SubmissionToAnnotationSetJob.objects.get(pk=self._job_id)
annotationset = AnnotationSet.objects.create(
creator=job.submission.creator,
base=job.base,
submission=job.submission,
kind=AnnotationSet.PREDICTION,
)
if self.__was_unzipped:
# Create the upload session but do not save it until we have the
# files
upload_session = RawImageUploadSession(annotationset=annotationset)
images = []
for file in output_files:
new_uuid = uuid.uuid4()
django_file = File(get_file(container=container, src=file))
staged_file = StagedFile(
csrf="staging_conversion_csrf",
client_id=self._job_id,
client_filename=file.name,
file_id=new_uuid,
timeout=timezone.now() + timedelta(hours=24),
start_byte=0,
end_byte=django_file.size - 1,
total_size=django_file.size,
)
staged_file.file.save(f"{uuid.uuid4()}", django_file)
staged_file.save()
staged_ajax_file = StagedAjaxFile(new_uuid)
images.append(
RawImageFile(
upload_session=upload_session,
filename=staged_ajax_file.name,
staged_file_id=staged_ajax_file.uuid,
)
)
upload_session.save(skip_processing=True)
RawImageFile.objects.bulk_create(images)
upload_session.process_images()
else:
assert len(output_files) == 1
f = get_file(container=container, src=output_files[0])
annotationset.labels = process_csv_file(f)
annotationset.save()
def _copy_output_files(self, *, container, base_dir: Path):
output_files = [
base_dir / Path(f)
for f in container.exec_run(f"find {base_dir} -type f")
.output.decode()
.splitlines()
]
if not output_files:
raise ValueError("Output directory is empty")
# TODO: This thing should not interact with the database
job = SubmissionToAnnotationSetJob.objects.get(pk=self._job_id)
annotationset = AnnotationSet.objects.create(
creator=job.submission.creator,
base=job.base,
submission=job.submission,
kind=AnnotationSet.PREDICTION,
)
if self.__was_unzipped:
# Create the upload session but do not save it until we have the
# files
upload_session = RawImageUploadSession(annotationset=annotationset)
images = []
for file in output_files:
new_uuid = uuid.uuid4()
django_file = File(get_file(container=container, src=file))
staged_file = StagedFile(
csrf="staging_conversion_csrf",
client_id=self._job_id,
client_filename=file.name,
file_id=new_uuid,
timeout=timezone.now() + timedelta(hours=24),
start_byte=0,
end_byte=django_file.size - 1,
total_size=django_file.size,
)
staged_file.file.save(f"{uuid.uuid4()}", django_file)
staged_file.save()
staged_ajax_file = StagedAjaxFile(new_uuid)
images.append(
RawImageFile(
upload_session=upload_session,
filename=staged_ajax_file.name,
staged_file_id=staged_ajax_file.uuid,
)
)
upload_session.save(skip_processing=True)
RawImageFile.objects.bulk_create(images)
upload_session.process_images()
else:
assert len(output_files) == 1
f = get_file(container=container, src=output_files[0])
annotationset.labels = process_csv_file(f)
annotationset.save()