def test_append_cut_json(self):
content1 = """
{
"a": 5,
"b": "text"
}
{
"a": 1"""
content2 = """55,
"r": "something"
}
{
"a": {
"b": [1, 2, 3]
}
}"""
with create_file(content1) as filename, open(filename, "a") as fp:
splitter = JsonFileSplitter(filename)
self.assertEqual(splitter.get(), {"a": 5, "b": "text"})
self.assertEqual(splitter.get(), None)
fp.write(content2)
fp.flush()
self.assertEqual(splitter.get(), {"a": 155, "r": "something"})
self.assertEqual(splitter.get(), {"a": {"b": [1, 2, 3]}})
self.assertEqual(splitter.get(), None)
splitter.close()
def test_big_json(self):
content = """
{
"a": 5,
"b": "text"
}
{
"a": "%s",
"b": "%s"
}
{}
""".lstrip() % (
"x" * JsonFileSplitter.BUFFER_SIZE * 2,
"y" * JsonFileSplitter.BUFFER_SIZE * 2,
)
with create_file(content) as filename:
splitter = JsonFileSplitter(filename)
self.assertEqual(splitter.get(), {"a": 5, "b": "text"})
self.assertEqual(
splitter.get(),
{
"a": "x" * JsonFileSplitter.BUFFER_SIZE * 2,
"b": "y" * JsonFileSplitter.BUFFER_SIZE * 2,
},
)
self.assertEqual(splitter.get(), {})
self.assertEqual(splitter.get(), None)
splitter.close()
def test_save_download_text_stream_to_given_destination(self):
exp = init(mode="debug", flush_period=0.5)
data = "Some test content of the stream"
exp["some/num/attr_name"] = FileVal.from_stream(StringIO(data))
self.assertIsInstance(exp.get_structure()["some"]["num"]["attr_name"],
File)
with create_file() as temp_filename:
exp["some/num/attr_name"].download(temp_filename)
with open(temp_filename, "rt") as file:
self.assertEqual(file.read(), data)
def test_assign_raise_not_image(self):
# given
path = self._random_path()
op_processor = MagicMock()
exp = self._create_run(processor=op_processor)
attr = FileSeries(exp, path)
file = File.from_content("some text")
with create_file(file.content, binary_mode=True) as tmp_filename:
saved_file = File(tmp_filename)
# when
with self.assertRaises(OperationNotSupported):
attr.assign([file])
with self.assertRaises(OperationNotSupported):
attr.assign([saved_file])
def test_simple_file(self):
content = """
{
"a": 5,
"b": "text"
}
{
"a": 13
}
{}
""".lstrip()
with create_file(content) as filename:
splitter = JsonFileSplitter(filename)
self.assertEqual(splitter.get(), {"a": 5, "b": "text"})
self.assertEqual(splitter.get(), {"a": 13})
self.assertEqual(splitter.get(), {})
self.assertEqual(splitter.get(), None)
splitter.close()
def test_upload_small_file_attribute(self, upload_raw_data):
# given
exp_uuid = str(uuid.uuid4())
swagger_mock = self._get_swagger_mock()
upload_raw_data.return_value = json.dumps(
{
"uploadId": "placeholder",
"errors": [],
}
)
data = b"testdata"
# when
with create_file(content=data, binary_mode=True) as filename:
upload_file_attribute(
swagger_client=swagger_mock,
container_id=exp_uuid,
attribute="target/path.txt",
source=filename,
ext="txt",
multipart_config=self.multipart_config,
)
# then
swagger_mock.api.fileSetFileMultipartUploadStart.assert_not_called()
swagger_mock.api.fileSetFileMultipartUploadFinish.assert_not_called()
swagger_mock.api.fileSetFileMultipartUploadPart.assert_not_called()
swagger_mock.api.fileSetFileUpload.assert_not_called()
swagger_mock.api.fileAtomMultipartUploadStart.assert_not_called()
swagger_mock.api.fileAtomMultipartUploadFinish.assert_not_called()
swagger_mock.api.fileAtomMultipartUploadPart.assert_not_called()
swagger_mock.api.fileAtomUpload.assert_not_called()
upload_raw_data.assert_called_once_with(
data=data,
http_client=swagger_mock.swagger_spec.http_client,
url="https://ui.neptune.ai/attributes/storage/file/upload",
query_params={
"experimentIdentifier": str(exp_uuid),
"attribute": "target/path.txt",
"ext": "txt",
},
)
def test_upload_single_small_file_in_file_set_attribute(self, upload_raw_data):
# given
exp_uuid = uuid.uuid4()
swagger_mock = self._get_swagger_mock()
upload_raw_data.return_value = json.dumps(
{
"errors": [],
}
)
data = b"testdata"
# when
with create_file(content=data, binary_mode=True) as filename:
upload_file_set_attribute(
swagger_client=swagger_mock,
container_id=str(exp_uuid),
attribute="some/attribute",
file_globs=[filename],
reset=True,
multipart_config=self.multipart_config,
)
# then
swagger_mock.api.fileSetFileMultipartUploadStart.assert_not_called()
swagger_mock.api.fileSetFileMultipartUploadFinish.assert_not_called()
swagger_mock.api.fileSetFileMultipartUploadPart.assert_not_called()
swagger_mock.api.fileSetFileUpload.assert_not_called()
swagger_mock.api.fileAtomMultipartUploadStart.assert_not_called()
swagger_mock.api.fileAtomMultipartUploadFinish.assert_not_called()
swagger_mock.api.fileAtomMultipartUploadPart.assert_not_called()
swagger_mock.api.fileAtomUpload.assert_not_called()
upload_raw_data.assert_called_once_with(
data=data,
http_client=swagger_mock.swagger_spec.http_client,
url="https://ui.neptune.ai/attributes/storage/fileset/upload",
query_params={
"subPath": os.path.basename(filename),
"experimentIdentifier": str(exp_uuid),
"attribute": "some/attribute",
},
)
def test_log_path(self):
# given
wait = self._random_wait()
path = self._random_path()
op_processor = MagicMock()
exp = self._create_run(processor=op_processor)
attr = FileSeries(exp, path)
file = File.as_image(numpy.random.rand(10, 10) * 255)
with create_file(file.content, binary_mode=True) as tmp_filename:
saved_file = File(tmp_filename)
# when
attr.log(
saved_file,
step=3,
timestamp=self._now(),
wait=wait,
description="something",
)
# then
op_processor.enqueue_operation.assert_called_once_with(
LogImages(
path,
[
LogImages.ValueType(
ImageValue(base64_encode(file.content), None,
"something"),
3,
self._now(),
)
],
),
wait,
)
def test_upload_single_big_file_in_file_set_attribute(self, upload_raw_data):
# given
exp_uuid = uuid.uuid4()
swagger_mock = self._get_swagger_mock()
upload_id = "placeholder"
set_expected_result(
swagger_mock.api.fileSetFileMultipartUploadStart,
{
"uploadId": upload_id,
"errors": [],
},
)
upload_raw_data.return_value = json.dumps(
{
"errors": [],
}
)
data = self.get_random_bytes(8 * 2 ** 20) # 8 MB
chunk_size = self.multipart_config.min_chunk_size
# when
with create_file(content=data, binary_mode=True) as filename:
upload_file_set_attribute(
swagger_client=swagger_mock,
container_id=str(exp_uuid),
attribute="some/attribute",
file_globs=[filename],
reset=True,
multipart_config=self.multipart_config,
)
# then
swagger_mock.api.fileSetFileMultipartUploadPart.assert_not_called()
swagger_mock.api.fileSetFileUpload.assert_not_called()
swagger_mock.api.fileAtomMultipartUploadStart.assert_not_called()
swagger_mock.api.fileAtomMultipartUploadFinish.assert_not_called()
swagger_mock.api.fileAtomMultipartUploadPart.assert_not_called()
swagger_mock.api.fileAtomUpload.assert_not_called()
swagger_mock.api.fileSetFileMultipartUploadStart.assert_called_once_with(
attribute="some/attribute",
experimentIdentifier=str(exp_uuid),
totalLength=len(data),
subPath=os.path.basename(filename),
)
swagger_mock.api.fileSetFileMultipartUploadFinish.assert_called_once_with(
attribute="some/attribute",
experimentIdentifier=str(exp_uuid),
subPath=os.path.basename(filename),
uploadId=upload_id,
)
upload_raw_data.assert_has_calls(
[
call(
data=data[:chunk_size],
http_client=swagger_mock.swagger_spec.http_client,
url="https://ui.neptune.ai/attributes/storage/fileset/upload/part",
headers={"X-Range": f"bytes=0-{chunk_size - 1}/{len(data)}"},
query_params={
"uploadPartIdx": 0,
"uploadId": upload_id,
"subPath": os.path.basename(filename),
"experimentIdentifier": str(exp_uuid),
"attribute": "some/attribute",
},
),
call(
data=data[chunk_size:],
http_client=swagger_mock.swagger_spec.http_client,
url="https://ui.neptune.ai/attributes/storage/fileset/upload/part",
headers={
"X-Range": f"bytes={chunk_size}-{len(data) - 1}/{len(data)}"
},
query_params={
"uploadPartIdx": 1,
"uploadId": upload_id,
"subPath": os.path.basename(filename),
"experimentIdentifier": str(exp_uuid),
"attribute": "some/attribute",
},
),
]
)