def delete_document(self, spot, document_type):
document = Document.objects.filter(spot=spot, type=document_type).first()
if document:
objstore = ObjectStore(settings.OBJECTSTORE_CONNECTION_CONFIG)
objstore.delete(document)
document.delete()
else:
logger.error(f"Delete document was called for spot {spot.id} and document_type {document_type}, "
f"but it does not exist in our database")
def test_get_connection(self, mocked_get_connection):
"""
Test and assert that objecstore.get_connection is called with the given
config, and that the connection is returned.
"""
mocked_get_connection.return_value = 'this is the connection'
objstore = ObjectStore(config='this is the config')
connection = objstore.get_connection()
mocked_get_connection.assert_called_with('this is the config')
self.assertEqual(connection, 'this is the connection')
def test_get_document(self):
"""
Test and assert that connection.get_object is called with the correct arguments
"""
connection = Mock()
container_name = 'container_name_mock'
object_name = 'object_name_mock'
objstore = ObjectStore(config='this is the config')
objstore.get_document(connection, container_name, object_name)
connection.get_object.assert_called_with(container_name, object_name)
def test_delete(self, mocked_get_container_path, mocked_get_connection):
"""
Test and assert that delete_object is called with the correct arguments
"""
mocked_get_container_path.return_value = 'container/path/'
objstore = ObjectStore(config='this is the config')
connection = Connection()
connection.delete_object = Mock()
mocked_get_connection.return_value = connection
document = Document(filename='doc_file_name.pdf',
type=Document.DocumentType.Ontwerp)
objstore.delete(document)
connection.delete_object.assert_called_with('container/path/',
'doc_file_name.pdf')
def test_get_container_path_rapport(self):
"""
Test and assert that the correct container path is returned for type rapport
"""
self.assertEqual(
ObjectStore.get_container_path(Document.DocumentType.Rapportage),
f"upload_container_name/doc/rapportage")
def test_get_container_path_ontwerp(self):
"""
Test and assert that the correct container path is returned for type ontwerp
"""
self.assertEqual(
ObjectStore.get_container_path(Document.DocumentType.Ontwerp),
f"upload_container_name/doc/ontwerp")
def test_get_file_cache(self, mocked_isfile, mocked_makedirs):
connection = Mock()
container_name = 'container_name_mock'
object_name = 'object_name_mock'
mocked_isfile.return_value = True
objstore = ObjectStore(config='this is the config')
with self.assertLogs(level='INFO') as logs:
return_value = objstore.get_file(connection, container_name,
object_name)
mocked_makedirs.assert_called_with(DOWNLOAD_DIR, exist_ok=True)
self.assertIn(
'INFO:storage.objectstore:Using cached file: object_name_mock',
logs.output)
self.assertEqual(return_value, f"{DOWNLOAD_DIR}{object_name}")
def get_file(self, request, pk=None):
document_model = self.get_object()
container_path = ObjectStore.get_container_path(document_model.type)
filename = document_model.filename
objstore = ObjectStore(settings.OBJECTSTORE_CONNECTION_CONFIG)
connection = objstore.get_connection()
try:
store_object = objstore.get_document(connection, container_path, filename)
except ClientException as e:
return handle_swift_exception(container_path, filename, e)
content_type = store_object[0].get('content-type')
obj_data = store_object[1]
response = HttpResponse(content_type=content_type)
response['Content-Disposition'] = f'attachment; filename="{filename}"'
response.write(obj_data)
return response
def perform_import():
log.info('Clearing models')
clear_models()
objstore = ObjectStore(config=settings.OBJECTSTORE_CONNECTION_CONFIG)
log.info('Opening object store connection')
connection = objstore.get_connection()
log.info('Getting documents list')
document_list = objstore.get_wba_documents_list(connection)
log.info(f'document list size: {len(document_list)}')
log.info('Fetching xls file')
xls_path = objstore.fetch_spots(connection)
log.info('Importing xls file')
process_xls(xls_path, document_list)
log.info(f'Spot count: {Spot.objects.all().count()}')
log.info(f'Document count: {Document.objects.all().count()}')
def test_delete_non_existing_file(self, mocked_get_container_path,
mocked_get_connection):
"""
Test and assert that delete_object is called with the correct arguments
"""
mocked_get_container_path.return_value = 'container/path/'
objstore = ObjectStore(config='this is the config')
connection = Connection()
connection.delete_object = Mock(side_effect=ClientException(''))
mocked_get_connection.return_value = connection
document = Document(id=1,
filename='doc_file_name.pdf',
type=Document.DocumentType.Ontwerp)
with self.assertLogs(level='INFO') as logs:
objstore.delete(document)
connection.delete_object.assert_called_with('container/path/',
'doc_file_name.pdf')
self.assertIn(
'INFO:storage.objectstore:Failed to delete object for document id 1',
logs.output)
def test_get_wba_documents_list(self, mocked_get_full_container_list):
"""
Test and assert that the correct list of documents is returned
"""
mocked_get_full_container_list.return_value = [
{
'hash': 'hash1',
'last_modified': '2019-08-26T09:08:55.150810',
'bytes': 1,
'name': 'ontwerp/filename1.pdf',
'content_type': 'application/pdf'
},
{
'hash': 'hash2',
'last_modified': '2019-08-26T09:08:55.150810',
'bytes': 1,
'name': 'rapport/filename2.pdf',
'content_type': 'application/pdf'
},
{
'hash': 'hash3',
'last_modified': '2019-08-26T09:08:55.150810',
'bytes': 1,
'name': 'ontwerp/filename3.pdf',
'content_type': 'application/pdf'
},
]
objstore = ObjectStore(config='this is the config')
return_value = objstore.get_wba_documents_list(connection=None)
self.assertEqual(return_value, [
('ontwerp', 'filename1.pdf'),
('rapport', 'filename2.pdf'),
('ontwerp', 'filename3.pdf'),
])
def handle_documents(self, spot, rapport_file=None, design_file=None):
objstore = ObjectStore(settings.OBJECTSTORE_CONNECTION_CONFIG)
if rapport_file:
try:
rapport_document, created = Document.objects.get_or_create(
type=Document.DocumentType.Rapportage, spot=spot)
objstore.upload(rapport_file, rapport_document)
except: # noqa E722 - ignore flake8 check, using bare except
raise
if design_file:
try:
design_document, created = Document.objects.get_or_create(
type=Document.DocumentType.Ontwerp, spot=spot)
objstore.upload(design_file, design_document)
except: # noqa E722 - ignore flake8 check, using bare except
raise
def test_fetch_spots(self, mocked_get_file):
objstore = ObjectStore(config='this is the config')
objstore.fetch_spots(connection='test connection')
mocked_get_file.assert_called_with('test connection',
WBA_CONTAINER_NAME, XLS_OBJECT_NAME)