def test_delete_documents_calls_delete_doc_from_kernel(
self, MockZipFile, mk_document_to_delete, MockLogger,
mk_delete_doc_from_kernel):
mk_document_to_delete.side_effect = self.docs_to_delete
delete_documents(**self.kwargs)
for doc_to_delete in self.docs_to_delete:
with self.subTest(doc_to_delete=doc_to_delete):
mk_delete_doc_from_kernel.assert_any_call(doc_to_delete)
def test_delete_documents_opens_zip(self, MockZipFile,
mk_document_to_delete,
mk_delete_doc_from_kernel):
mk_document_to_delete.return_value = (
None,
None,
)
delete_documents(**self.kwargs)
MockZipFile.assert_called_once_with(self.kwargs["sps_package"])
def test_delete_documents_calls_document_to_delete_for_each_xml(
self, MockZipFile, mk_document_to_delete,
mk_delete_doc_from_kernel):
delete_documents(**self.kwargs)
for sps_xml_file in self.kwargs["xmls_filenames"]:
with self.subTest(sps_xml_file=sps_xml_file):
mk_document_to_delete.assert_any_call(
MockZipFile.return_value.__enter__.return_value,
sps_xml_file)
def test_delete_documents_logs_error_if_document_to_delete_error(
self, MockZipFile, mk_document_to_delete, MockLogger,
mk_delete_doc_from_kernel):
mk_document_to_delete.side_effect = DocumentToDeleteException(
"XML Error")
delete_documents(**self.kwargs)
mk_delete_doc_from_kernel.assert_not_called()
for xml_filename in self.kwargs["xmls_filenames"]:
with self.subTest(xml_filename=xml_filename):
MockLogger.error.assert_any_call(
'Error reading document "%s": %s', xml_filename,
"XML Error")
def test_delete_documents_logs_error_if_kernel_connect_error(
self, MockZipFile, mk_document_to_delete, MockLogger,
mk_delete_doc_from_kernel):
mk_document_to_delete.side_effect = self.docs_to_delete
delete_documents(**self.kwargs)
for sps_xml_file, doc_to_delete in zip(self.kwargs["xmls_filenames"],
self.docs_to_delete):
with self.subTest(sps_xml_file=sps_xml_file,
doc_to_delete=doc_to_delete):
MockLogger.info.assert_any_call(
'Document "%s" (scielo_id: "%s") deleted from kernel',
sps_xml_file, doc_to_delete)
def test_delete_documents_logs_error_if_kernel_connect_error(
self, MockZipFile, mk_document_to_delete, MockLogger,
mk_delete_doc_from_kernel):
mk_document_to_delete.side_effect = self.docs_to_delete
mk_delete_doc_from_kernel.side_effect = DeleteDocFromKernelException(
"404 Client Error: Not Found")
delete_documents(**self.kwargs)
for sps_xml_file, doc_to_delete in zip(self.kwargs["xmls_filenames"],
self.docs_to_delete):
with self.subTest(sps_xml_file=sps_xml_file,
doc_to_delete=doc_to_delete):
MockLogger.info.assert_any_call(
'Could not delete "%s" (scielo_id: "%s") from kernel: %s',
sps_xml_file, doc_to_delete, "404 Client Error: Not Found")
def delete_documents(dag_run, **kwargs):
_sps_package = dag_run.conf.get("sps_package")
_xmls_filenames = kwargs["ti"].xcom_pull(key="xmls_filenames",
task_ids="list_docs_task_id")
if _xmls_filenames:
_xmls_to_preserve = sync_documents_to_kernel_operations.delete_documents(
_sps_package, _xmls_filenames)
if _xmls_to_preserve:
kwargs["ti"].xcom_push(key="xmls_to_preserve",
value=_xmls_to_preserve)
def test_delete_documents_logs_error_if_no_scielo_doc_id(
self, MockZipFile, mk_document_to_delete, MockLogger,
mk_delete_doc_from_kernel):
mk_document_to_delete.side_effect = self.docs_to_delete[:2] + [(
True,
None,
)]
result = delete_documents(**self.kwargs)
MockLogger.error.assert_any_call(
'Document "%s" will not be deleted because SciELO PID is None',
self.kwargs["xmls_filenames"][-1])
def test_delete_documents_returns_xmls_to_preserve(
self, MockZipFile, mk_document_to_delete, MockLogger,
mk_delete_doc_from_kernel):
mk_document_to_delete.side_effect = self.docs_to_delete[:2] + [(
False,
None,
)]
result, _ = delete_documents(**self.kwargs)
self.assertEqual(
result,
list(
set(self.kwargs["xmls_filenames"]) -
set(self.kwargs["xmls_filenames"][:-1])))
def test_delete_documents_returns_xmls_to_preserve(
self, MockZipFile, mk_document_to_delete, MockLogger,
mk_delete_doc_from_kernel):
docs_to_delete = [
"FX6F3cbyYmmwvtGmMB7WCgr", "GZ5K2cbyYmmwvtGmMB71243", None
]
mk_document_to_delete.side_effect = docs_to_delete
result = delete_documents(**self.kwargs)
self.assertEqual(
result,
list(
set(self.kwargs["xmls_filenames"]) -
set(self.kwargs["xmls_filenames"][:-1])))
def delete_documents(dag_run, **kwargs):
_sps_package = dag_run.conf.get("sps_package")
_xmls_filenames = kwargs["ti"].xcom_pull(key="xmls_filenames",
task_ids="list_docs_task_id")
if not _xmls_filenames:
return False
_xmls_to_preserve, executions = sync_documents_to_kernel_operations.delete_documents(
_sps_package, _xmls_filenames)
for execution in executions:
execution["dag_run"] = kwargs.get("run_id")
execution["pre_sync_dag_run"] = dag_run.conf.get("pre_syn_dag_run_id")
execution["package_name"] = os.path.basename(_sps_package)
add_execution_in_database(table="xml_documents", data=execution)
if _xmls_to_preserve:
kwargs["ti"].xcom_push(key="xmls_to_preserve", value=_xmls_to_preserve)
return True
else:
return False
