def test_upload_file_with_lifetime(self):
mock_api = Mock()
dummy_lifetime = "1day"
mock_api.post_channel_file_upload.return_value = {
"uploaded_at": None,
"metadata": {},
"content_type": "image/jpeg",
"lifetime": dummy_lifetime,
"file_id": "20180515T180605-f4acc798-9afa-40a1-b500-ebce42a4fa3f"
}
channel = Channel(mock_api, ORGANIZATION_ID, CHANNEL_ID)
dummy_file_data = json.dumps({'data': 'dummy'}).encode('utf-8')
with tempfile.NamedTemporaryFile(suffix='.json') as tmp:
tmp.write(dummy_file_data)
tmp.seek(0)
filename = os.path.basename(tmp.name)
file = channel.upload_file(tmp.name, lifetime=dummy_lifetime)
self.assertIsInstance(file, DatalakeFile)
content_type = 'application/json'
metadata = {'x-abeja-meta-filename': filename}
self.assertEqual(mock_api.post_channel_file_upload.call_count, 1)
call_args = mock_api.post_channel_file_upload.call_args[0]
call_kwargs = mock_api.post_channel_file_upload.call_args[1]
self.assertEqual(call_args[0], CHANNEL_ID)
self.assertEqual(call_args[2], content_type)
self.assertDictEqual(
call_kwargs, {
'lifetime': dummy_lifetime,
'conflict_target': None,
'metadata': metadata
})
def test_upload_with_dir(self):
mock_api = Mock()
mock_api.get_channel_file_upload.return_value = {
"url_expires_on": "2018-05-15T19:06:05+00:00",
"upload_url": "http://example.com/dummy_upload_url",
"uploaded_at": None,
"metadata": {},
"content_type": "image/jpeg",
"file_id": "20180515T180605-f4acc798-9afa-40a1-b500-ebce42a4fa3f"
}
channel = Channel(mock_api, ORGANIZATION_ID, CHANNEL_ID)
with tempfile.NamedTemporaryFile() as tmp:
tmp.write(json.dumps({'data': 'dummy'}).encode('utf-8'))
tmp.seek(0)
base_dir = '/'.join(tmp.name.split('/')[:-1])
with self.assertRaises(IsADirectoryError):
channel.upload_file(
base_dir,
metadata={'x-abeja-meta-filename': 'dummy'},
content_type='application/json')
def upload_datum(channel: Channel, dataset_items: DatasetItems, datum: PetData,
is_train: bool, attribute_type: str):
# upload file to the channel
metadata = {
'filename': PurePath(datum.image_path).name,
'type': 'trainval' if is_train else 'test'
}
response = channel.upload_file(datum.image_path, metadata=metadata)
# upload item to the datase
attributes = build_attributes(datum, attribute_type)
source_data = [{
"data_type": response.content_type,
"data_uri": response.uri,
}]
res = dataset_items.create(source_data=source_data, attributes=attributes)
return res
def test_upload_file(self):
mock_api = Mock()
mock_api.post_channel_file_upload.return_value = {
"url_expires_on": "2018-05-15T19:06:05+00:00",
"uploaded_at": None,
"metadata": {},
"content_type": "image/jpeg",
"lifetime": None,
"file_id": "20180515T180605-f4acc798-9afa-40a1-b500-ebce42a4fa3f"
}
channel = Channel(mock_api, ORGANIZATION_ID, CHANNEL_ID)
content_type = 'application/json'
metadata = {'filename': 'dummy', 'label': 'dummy label'}
dummy_file_data = json.dumps({'data': 'dummy'}).encode('utf-8')
with tempfile.NamedTemporaryFile() as tmp:
tmp.write(dummy_file_data)
tmp.seek(0)
file = channel.upload_file(tmp.name,
metadata=metadata,
content_type=content_type)
self.assertIsInstance(file, DatalakeFile)
expected_metadata = {
'x-abeja-meta-filename': 'dummy',
'x-abeja-meta-label': 'dummy label'
}
self.assertEqual(mock_api.post_channel_file_upload.call_count, 1)
call_args = mock_api.post_channel_file_upload.call_args[0]
call_kwargs = mock_api.post_channel_file_upload.call_args[1]
self.assertEqual(call_args[0], CHANNEL_ID)
self.assertDictEqual(
call_kwargs, {
'lifetime': None,
'conflict_target': None,
'metadata': expected_metadata
})
def upload_image_to_datalake(channel: Channel, filepath: Path) -> DatalakeFile:
if filepath.suffix not in ['.jpg', '.jpeg', '.JPG', '.jpeg']:
raise ValueError(f'invalid file format: {filepath}, it is not jpeg file')
content_type = 'image/jpeg'
metadata = {'filename': filepath.name}
return channel.upload_file(str(filepath), metadata=metadata, content_type=content_type)