def test_register_update_documents_opens_zip(
self,
MockZipFile,
MockLogger,
mk_put_xml_into_object_store,
mk_put_assets_and_pdfs_in_object_store,
mk_register_update_doc_into_kernel,
):
register_update_documents(**self.kwargs)
MockZipFile.assert_called_once_with(self.kwargs["sps_package"])
def test_register_update_documents_puts_each_doc_in_object_store(
self,
MockZipFile,
mk_put_xml_into_object_store,
mk_put_assets_and_pdfs_in_object_store,
mk_register_update_doc_into_kernel,
):
mk_put_xml_into_object_store.side_effect = self.xmls_data
register_update_documents(**self.kwargs)
for xml_data in self.xmls_data:
with self.subTest(xml_data=xml_data):
mk_put_assets_and_pdfs_in_object_store.assert_any_call(
MockZipFile.return_value.__enter__.return_value, xml_data)
def test_register_update_documents_calls_put_xml_into_object_store(
self,
MockZipFile,
MockLogger,
mk_put_xml_into_object_store,
mk_put_assets_and_pdfs_in_object_store,
mk_register_update_doc_into_kernel,
):
register_update_documents(**self.kwargs)
for xml_filename in self.kwargs["xmls_to_preserve"]:
with self.subTest(xml_filename=xml_filename):
mk_put_xml_into_object_store.assert_any_call(
MockZipFile.return_value.__enter__.return_value,
xml_filename)
def register_update_documents(dag_run, **kwargs):
_xmls_to_preserve = kwargs["ti"].xcom_pull(key="xmls_to_preserve",
task_ids="delete_docs_task_id")
if not _xmls_to_preserve:
return False
_optimized_package = kwargs["ti"].xcom_pull(
key="optimized_package", task_ids="optimize_package_task_id")
package = _optimized_package or dag_run.conf.get("sps_package")
_documents, executions = sync_documents_to_kernel_operations.register_update_documents(
package, _xmls_to_preserve)
for execution in executions:
execution["dag_run"] = kwargs.get("run_id")
execution["pre_sync_dag_run"] = dag_run.conf.get("pre_syn_dag_run_id")
execution["package_name"] = os.path.basename(_optimized_package)
add_execution_in_database(table="xml_documents", data=execution)
if _documents:
kwargs["ti"].xcom_push(key="documents", value=_documents)
return True
else:
return False
def register_update_documents(dag_run, **kwargs):
_sps_package = dag_run.conf.get("sps_package")
_xmls_to_preserve = kwargs["ti"].xcom_pull(key="xmls_to_preserve",
task_ids="delete_docs_task_id")
if _xmls_to_preserve:
_documents = sync_documents_to_kernel_operations.register_update_documents(
_sps_package, _xmls_to_preserve)
if _documents:
kwargs["ti"].xcom_push(key="documents", value=_documents)
def test_register_update_documents_logs_error_if_put_xml_into_object_store_return_pidv3exception(
self,
MockZipFile,
MockLogger,
mk_put_xml_into_object_store,
mk_put_assets_and_pdfs_in_object_store,
mk_register_update_doc_into_kernel,
):
MockZipFile.return_value.__enter__.return_value.read.return_value = b""
mk_put_xml_into_object_store.side_effect = [
{},
Pidv3Exception("Could not get scielo id v3"),
{},
]
register_update_documents(**self.kwargs)
MockLogger.error.assert_any_call(
'Could not put document "%s" in object store: %s',
self.kwargs["xmls_to_preserve"][1],
"Could not get scielo id v3",
)
def test_register_update_documents_logs_error_if_put_xml_into_object_store_error(
self,
MockZipFile,
MockLogger,
mk_put_xml_into_object_store,
mk_put_assets_and_pdfs_in_object_store,
mk_register_update_doc_into_kernel,
):
MockZipFile.return_value.__enter__.return_value.read.return_value = b""
mk_put_xml_into_object_store.side_effect = [
{},
PutXMLInObjectStoreException("Put Doc in Object Store Error"),
{},
]
register_update_documents(**self.kwargs)
MockLogger.info.assert_any_call(
'Could not put document "%s" in object store: %s',
self.kwargs["xmls_to_preserve"][1],
"Put Doc in Object Store Error",
)
def test_register_update_documents_call_register_update_doc_into_kernel_raise_error(
self,
MockZipFile,
MockLogger,
mk_put_xml_into_object_store,
mk_put_assets_and_pdfs_in_object_store,
mk_register_update_doc_into_kernel,
):
mk_put_assets_and_pdfs_in_object_store.side_effect = self.xmls_data
mk_register_update_doc_into_kernel.side_effect = [
None,
RegisterUpdateDocIntoKernelException(
"Register Doc in Kernel Error"),
None,
]
register_update_documents(**self.kwargs)
MockLogger.error.assert_any_call(
'Could not register or update document "%s" in Kernel: %s',
self.kwargs["xmls_to_preserve"][1],
"Register Doc in Kernel Error",
)
def test_register_update_documents_call_register_update_doc_into_kernel(
self,
MockZipFile,
mk_put_xml_into_object_store,
mk_put_assets_and_pdfs_in_object_store,
mk_register_update_doc_into_kernel,
):
expected = copy.deepcopy(self.xmls_data)
mk_put_xml_into_object_store.side_effect = self.xmls_data
mk_assets_and_pdfs = {
"assets": [{
"asset_id": "test"
}],
"pdfs": [{
"filename": "test"
}],
}
mk_put_assets_and_pdfs_in_object_store.return_value = mk_assets_and_pdfs
register_update_documents(**self.kwargs)
for xml_data in expected:
xml_data.update(mk_assets_and_pdfs)
with self.subTest(xml_data=xml_data):
mk_register_update_doc_into_kernel.assert_any_call(xml_data)
def test_register_update_documents_returns_syncronized_documents_metadata_list(
self,
MockZipFile,
mk_put_xml_into_object_store,
mk_put_assets_and_pdfs_in_object_store,
mk_register_update_doc_into_kernel,
):
expected = [self.xmls_data[0], self.xmls_data[2]]
mk_put_xml_into_object_store.side_effect = self.xmls_data
mk_register_update_doc_into_kernel.side_effect = [
None,
RegisterUpdateDocIntoKernelException(
"Register Doc in Kernel Error"),
None,
]
result, _ = register_update_documents(**self.kwargs)
self.assertEqual(result, expected)
