def test_delete_documents_calls_delete_doc_from_kernel( self, MockZipFile, mk_document_to_delete, MockLogger, mk_delete_doc_from_kernel): mk_document_to_delete.side_effect = self.docs_to_delete delete_documents(**self.kwargs) for doc_to_delete in self.docs_to_delete: with self.subTest(doc_to_delete=doc_to_delete): mk_delete_doc_from_kernel.assert_any_call(doc_to_delete)
def test_delete_documents_opens_zip(self, MockZipFile, mk_document_to_delete, mk_delete_doc_from_kernel): mk_document_to_delete.return_value = ( None, None, ) delete_documents(**self.kwargs) MockZipFile.assert_called_once_with(self.kwargs["sps_package"])
def test_delete_documents_calls_document_to_delete_for_each_xml( self, MockZipFile, mk_document_to_delete, mk_delete_doc_from_kernel): delete_documents(**self.kwargs) for sps_xml_file in self.kwargs["xmls_filenames"]: with self.subTest(sps_xml_file=sps_xml_file): mk_document_to_delete.assert_any_call( MockZipFile.return_value.__enter__.return_value, sps_xml_file)
def test_delete_documents_logs_error_if_document_to_delete_error( self, MockZipFile, mk_document_to_delete, MockLogger, mk_delete_doc_from_kernel): mk_document_to_delete.side_effect = DocumentToDeleteException( "XML Error") delete_documents(**self.kwargs) mk_delete_doc_from_kernel.assert_not_called() for xml_filename in self.kwargs["xmls_filenames"]: with self.subTest(xml_filename=xml_filename): MockLogger.error.assert_any_call( 'Error reading document "%s": %s', xml_filename, "XML Error")
def test_delete_documents_logs_error_if_kernel_connect_error( self, MockZipFile, mk_document_to_delete, MockLogger, mk_delete_doc_from_kernel): mk_document_to_delete.side_effect = self.docs_to_delete delete_documents(**self.kwargs) for sps_xml_file, doc_to_delete in zip(self.kwargs["xmls_filenames"], self.docs_to_delete): with self.subTest(sps_xml_file=sps_xml_file, doc_to_delete=doc_to_delete): MockLogger.info.assert_any_call( 'Document "%s" (scielo_id: "%s") deleted from kernel', sps_xml_file, doc_to_delete)
def test_delete_documents_logs_error_if_kernel_connect_error( self, MockZipFile, mk_document_to_delete, MockLogger, mk_delete_doc_from_kernel): mk_document_to_delete.side_effect = self.docs_to_delete mk_delete_doc_from_kernel.side_effect = DeleteDocFromKernelException( "404 Client Error: Not Found") delete_documents(**self.kwargs) for sps_xml_file, doc_to_delete in zip(self.kwargs["xmls_filenames"], self.docs_to_delete): with self.subTest(sps_xml_file=sps_xml_file, doc_to_delete=doc_to_delete): MockLogger.info.assert_any_call( 'Could not delete "%s" (scielo_id: "%s") from kernel: %s', sps_xml_file, doc_to_delete, "404 Client Error: Not Found")
def delete_documents(dag_run, **kwargs): _sps_package = dag_run.conf.get("sps_package") _xmls_filenames = kwargs["ti"].xcom_pull(key="xmls_filenames", task_ids="list_docs_task_id") if _xmls_filenames: _xmls_to_preserve = sync_documents_to_kernel_operations.delete_documents( _sps_package, _xmls_filenames) if _xmls_to_preserve: kwargs["ti"].xcom_push(key="xmls_to_preserve", value=_xmls_to_preserve)
def test_delete_documents_logs_error_if_no_scielo_doc_id( self, MockZipFile, mk_document_to_delete, MockLogger, mk_delete_doc_from_kernel): mk_document_to_delete.side_effect = self.docs_to_delete[:2] + [( True, None, )] result = delete_documents(**self.kwargs) MockLogger.error.assert_any_call( 'Document "%s" will not be deleted because SciELO PID is None', self.kwargs["xmls_filenames"][-1])
def test_delete_documents_returns_xmls_to_preserve( self, MockZipFile, mk_document_to_delete, MockLogger, mk_delete_doc_from_kernel): mk_document_to_delete.side_effect = self.docs_to_delete[:2] + [( False, None, )] result, _ = delete_documents(**self.kwargs) self.assertEqual( result, list( set(self.kwargs["xmls_filenames"]) - set(self.kwargs["xmls_filenames"][:-1])))
def test_delete_documents_returns_xmls_to_preserve( self, MockZipFile, mk_document_to_delete, MockLogger, mk_delete_doc_from_kernel): docs_to_delete = [ "FX6F3cbyYmmwvtGmMB7WCgr", "GZ5K2cbyYmmwvtGmMB71243", None ] mk_document_to_delete.side_effect = docs_to_delete result = delete_documents(**self.kwargs) self.assertEqual( result, list( set(self.kwargs["xmls_filenames"]) - set(self.kwargs["xmls_filenames"][:-1])))
def delete_documents(dag_run, **kwargs): _sps_package = dag_run.conf.get("sps_package") _xmls_filenames = kwargs["ti"].xcom_pull(key="xmls_filenames", task_ids="list_docs_task_id") if not _xmls_filenames: return False _xmls_to_preserve, executions = sync_documents_to_kernel_operations.delete_documents( _sps_package, _xmls_filenames) for execution in executions: execution["dag_run"] = kwargs.get("run_id") execution["pre_sync_dag_run"] = dag_run.conf.get("pre_syn_dag_run_id") execution["package_name"] = os.path.basename(_sps_package) add_execution_in_database(table="xml_documents", data=execution) if _xmls_to_preserve: kwargs["ti"].xcom_push(key="xmls_to_preserve", value=_xmls_to_preserve) return True else: return False