def test_upload_file_with_lifetime(self): mock_api = Mock() dummy_lifetime = "1day" mock_api.post_channel_file_upload.return_value = { "uploaded_at": None, "metadata": {}, "content_type": "image/jpeg", "lifetime": dummy_lifetime, "file_id": "20180515T180605-f4acc798-9afa-40a1-b500-ebce42a4fa3f" } channel = Channel(mock_api, ORGANIZATION_ID, CHANNEL_ID) dummy_file_data = json.dumps({'data': 'dummy'}).encode('utf-8') with tempfile.NamedTemporaryFile(suffix='.json') as tmp: tmp.write(dummy_file_data) tmp.seek(0) filename = os.path.basename(tmp.name) file = channel.upload_file(tmp.name, lifetime=dummy_lifetime) self.assertIsInstance(file, DatalakeFile) content_type = 'application/json' metadata = {'x-abeja-meta-filename': filename} self.assertEqual(mock_api.post_channel_file_upload.call_count, 1) call_args = mock_api.post_channel_file_upload.call_args[0] call_kwargs = mock_api.post_channel_file_upload.call_args[1] self.assertEqual(call_args[0], CHANNEL_ID) self.assertEqual(call_args[2], content_type) self.assertDictEqual( call_kwargs, { 'lifetime': dummy_lifetime, 'conflict_target': None, 'metadata': metadata })
def test_upload_with_dir(self): mock_api = Mock() mock_api.get_channel_file_upload.return_value = { "url_expires_on": "2018-05-15T19:06:05+00:00", "upload_url": "http://example.com/dummy_upload_url", "uploaded_at": None, "metadata": {}, "content_type": "image/jpeg", "file_id": "20180515T180605-f4acc798-9afa-40a1-b500-ebce42a4fa3f" } channel = Channel(mock_api, ORGANIZATION_ID, CHANNEL_ID) with tempfile.NamedTemporaryFile() as tmp: tmp.write(json.dumps({'data': 'dummy'}).encode('utf-8')) tmp.seek(0) base_dir = '/'.join(tmp.name.split('/')[:-1]) with self.assertRaises(IsADirectoryError): channel.upload_file( base_dir, metadata={'x-abeja-meta-filename': 'dummy'}, content_type='application/json')
def upload_datum(channel: Channel, dataset_items: DatasetItems, datum: PetData, is_train: bool, attribute_type: str): # upload file to the channel metadata = { 'filename': PurePath(datum.image_path).name, 'type': 'trainval' if is_train else 'test' } response = channel.upload_file(datum.image_path, metadata=metadata) # upload item to the datase attributes = build_attributes(datum, attribute_type) source_data = [{ "data_type": response.content_type, "data_uri": response.uri, }] res = dataset_items.create(source_data=source_data, attributes=attributes) return res
def test_upload_file(self): mock_api = Mock() mock_api.post_channel_file_upload.return_value = { "url_expires_on": "2018-05-15T19:06:05+00:00", "uploaded_at": None, "metadata": {}, "content_type": "image/jpeg", "lifetime": None, "file_id": "20180515T180605-f4acc798-9afa-40a1-b500-ebce42a4fa3f" } channel = Channel(mock_api, ORGANIZATION_ID, CHANNEL_ID) content_type = 'application/json' metadata = {'filename': 'dummy', 'label': 'dummy label'} dummy_file_data = json.dumps({'data': 'dummy'}).encode('utf-8') with tempfile.NamedTemporaryFile() as tmp: tmp.write(dummy_file_data) tmp.seek(0) file = channel.upload_file(tmp.name, metadata=metadata, content_type=content_type) self.assertIsInstance(file, DatalakeFile) expected_metadata = { 'x-abeja-meta-filename': 'dummy', 'x-abeja-meta-label': 'dummy label' } self.assertEqual(mock_api.post_channel_file_upload.call_count, 1) call_args = mock_api.post_channel_file_upload.call_args[0] call_kwargs = mock_api.post_channel_file_upload.call_args[1] self.assertEqual(call_args[0], CHANNEL_ID) self.assertDictEqual( call_kwargs, { 'lifetime': None, 'conflict_target': None, 'metadata': expected_metadata })
def upload_image_to_datalake(channel: Channel, filepath: Path) -> DatalakeFile: if filepath.suffix not in ['.jpg', '.jpeg', '.JPG', '.jpeg']: raise ValueError(f'invalid file format: {filepath}, it is not jpeg file') content_type = 'image/jpeg' metadata = {'filename': filepath.name} return channel.upload_file(str(filepath), metadata=metadata, content_type=content_type)