def test_get_content(self):
datalake_file = DatalakeFile(None, uri=self.uri, type=type)
mock_func = create_autospec(datalake_file._get_content_from_remote,
return_value=self.binary_data)
datalake_file._get_content_from_remote = mock_func
content = datalake_file.get_content()
self.assertEqual(content, self.binary_data)
mock_func.assert_called_once_with()
def test_get_content_using_cache(self):
cache_dir = '{}/{}'.format(TEST_MOUNT_DIR, self.channel_id)
os.makedirs(cache_dir, exist_ok=True)
with open('{}/{}'.format(cache_dir, self.file_id), 'wb') as f:
f.write(self.binary_data)
datalake_file = DatalakeFile(None, uri=self.uri, type=type)
mock_func = MagicMock()
datalake_file._get_content_from_remote = mock_func
content = datalake_file.get_content()
self.assertEqual(content, self.binary_data)
mock_func.assert_not_called()
def _get_label(self, i):
item = self.dataset_list[i]
annotation = item.attributes['segmentation']
channel_id = annotation['channel_id']
file_id = annotation['file_id']
uri = 'datalake://{}/{}'.format(channel_id, file_id)
ftype = 'image/png'
source = DatalakeFile(api=self.client, channel_id=channel_id, file_id=file_id, uri=uri, type=ftype)
file_content = source.get_content()
file_like_object = io.BytesIO(file_content)
f = Image.open(file_like_object)
try:
img = f.convert('P')
label = np.asarray(img, dtype=np.int32)
finally:
f.close()
# Label id max_id is for unlabeled pixels.
label[label == self.max_id] = -1
return label
'personal_access_token': ABEJA_PLATFORM_TOKEN
}
client = Client(organization_id=args.organization, credential=credential)
datalake_client = DatalakeClient(credential=credential)
dataset = client.get_dataset(args.dataset)
dataset_list = dataset.dataset_items.list(prefetch=False)
for d in dataset_list:
break
file_content = d.source_data[0].get_content()
file_like_object = io.BytesIO(file_content)
img = Image.open(file_like_object)
img.show()
uri = d.attributes['segmentation'][0]['uri']
ftype = 'image/png'
datalake_file = DatalakeFile(datalake_client,
args.organization,
uri=uri,
type=ftype)
file_content = datalake_file.get_content()
file_like_object = io.BytesIO(file_content)
img = Image.open(file_like_object)
img.show()