def test_link_documents_to_documentsbundle_should_not_reset_item_list_when_new_documents_arrives(
self, mk_open, mk_update_aop_bundle_items, mk_get_or_create_bundle,
mk_get_bundle_id, mk_regdocument, MockLogger):
new_documents_to_link = self.documents[0:2]
current_bundle_item_list = [{
"id": new_documents_to_link[0]["scielo_id"],
"order": new_documents_to_link[0]["order"],
}]
# journal_issn_map
mk_open.return_value.__enter__.return_value.read.return_value = (
'{"0034-8910": "0101-0101"}')
# Bundle_id traduzido a partir do journal_issn_map
mk_get_bundle_id.return_value = "0034-8910-2014-v48-n2"
mk_get_or_create_bundle.return_value.json.return_value.__getitem__.return_value = (
current_bundle_item_list)
link_documents_to_documentsbundle("path_to_sps_package/package.zip",
new_documents_to_link,
"/some/random/json/path.json")
# Lista produzida a partir dos documentos existes e dos novos
# documentos
new_payload_list = current_bundle_item_list + [{
"id":
new_documents_to_link[1]["scielo_id"],
"order":
new_documents_to_link[1]["order"],
}]
mk_regdocument.assert_called_with("0034-8910-2014-v48-n2",
new_payload_list)
def test_link_documents_to_documentsbundle_should_not_reset_item_list_when_new_documents_arrives(
self, mk_open, mk_issue_id, mk_kernel_connect,
mk_register_document_to_bundle):
new_documents_to_link = self.documents[0:2]
current_bundle_item_list = [{
"id": new_documents_to_link[0]["scielo_id"],
"order": new_documents_to_link[0]["order"],
}]
# journal_issn_map
mk_open.return_value.__enter__.return_value.read.return_value = (
'{"0034-8910": "0101-0101"}')
# Bundle_id traduzido a partir do journal_issn_map
mk_issue_id.return_value = "0034-8910-2014-v48-n2"
mk_kernel_connect.return_value.json.return_value.__getitem__.return_value = (
current_bundle_item_list)
link_documents_to_documentsbundle(new_documents_to_link,
"/some/random/json/path.json")
# Lista produzida a partir dos documentos existes e dos novos
# documentos
new_payload_list = current_bundle_item_list + [{
"id":
new_documents_to_link[1]["scielo_id"],
"order":
new_documents_to_link[1]["order"],
}]
mk_register_document_to_bundle.assert_called_with(
"0034-8910-2014-v48-n2", new_payload_list)
def test_calls_update_aop_bundle_items(self, mk_open,
mk_update_aop_bundle_items,
mk_get_or_create_bundle,
mk_get_bundle_id, mk_regdocument,
MockLogger):
mk_open.return_value.__enter__.return_value.read.return_value = (
self.issn_index_json)
mk_get_bundle_id.return_value = "0034-8910-2014-v48-n2"
bundle_data = {
"id":
"0034-8910-2014-v48-n2",
"items": [{
"id": f"item-{number}",
"order": number
} for number in range(1, 5)],
}
mk_get_or_create_bundle.return_value.json.return_value = bundle_data
link_documents_to_documentsbundle("path_to_sps_package/package.zip",
self.documents, "/json/index.json")
expected = bundle_data["items"]
expected += [{
"id": document["scielo_id"],
"order": document["order"]
} for document in self.documents]
mk_update_aop_bundle_items.assert_called_with("0034-8910", expected)
def test_link_documents_to_documentsbundle_logs_journal_issn_id_error(
self, mk_issue_id, mk_regdocument, MockLogger, mk_open):
mk_open.return_value.__enter__.return_value.read.return_value = "{}"
link_documents_to_documentsbundle(self.documents[:1],
"/json/index.json")
MockLogger.info.assert_any_call(
'Could not get journal ISSN ID: ISSN id "%s" not found',
"0034-8910")
def test_link_documents_to_documentsbundle_logs_journal_issn_id_error(
self, mk_open, mk_update_aop_bundle_items, mk_get_or_create_bundle,
mk_get_bundle_id, mk_regdocument, MockLogger):
mk_open.return_value.__enter__.return_value.read.return_value = '{}'
link_documents_to_documentsbundle("path_to_sps_package/package.zip",
self.documents[:1],
"/json/index.json")
MockLogger.info.assert_any_call(
'Could not get journal ISSN ID: ISSN id "%s" not found',
"0034-8910")
def test_link_documents_to_documentsbundle_calls_issue_id_with_issn_id(
self, mk_issue_id, mk_regdocument, mk_kernel_connect, mk_open):
mk_open.return_value.__enter__.return_value.read.return_value = '{"0034-8910": "0101-0101"}'
link_documents_to_documentsbundle(self.documents[:1],
"/json/index.json")
mk_issue_id.assert_called_once_with(
issn_id="0101-0101",
year=self.documents[0]["year"],
volume=self.documents[0].get("volume", None),
number=self.documents[0].get("number", None),
supplement=self.documents[0].get("supplement", None))
def test_does_not_call_update_aop_bundle_items(self, mk_open,
mk_update_aop_bundle_items,
mk_get_or_create_bundle,
mk_get_bundle_id,
mk_regdocument, MockLogger):
mk_open.return_value.__enter__.return_value.read.return_value = (
'{"0034-8910": "0101-0101"}')
mk_get_bundle_id.return_value = "0101-0101-aop"
link_documents_to_documentsbundle("path_to_sps_package/2019nahead.zip",
self.documents[:1],
"/json/index.json")
mk_update_aop_bundle_items.assert_not_called()
def test_link_documents_to_documentsbundle_calls_get_bundle_id_with_issn_id(
self, mk_open, mk_update_aop_bundle_items, mk_get_or_create_bundle,
mk_get_bundle_id, mk_regdocument, MockLogger):
mk_open.return_value.__enter__.return_value.read.return_value = '{"0034-8910": "0101-0101"}'
link_documents_to_documentsbundle("path_to_sps_package/package.zip",
self.documents[:1],
"/json/index.json")
mk_get_bundle_id.assert_called_once_with(
issn_id="0101-0101",
year=self.documents[0]["year"],
volume=self.documents[0].get("volume", None),
number=self.documents[0].get("number", None),
supplement=self.documents[0].get("supplement", None))
def test_logs_exception_if_update_aop_bundle_items_raise_exception(
self, mk_open, mk_update_aop_bundle_items, mk_get_or_create_bundle,
mk_get_bundle_id, mk_regdocument, MockLogger):
mk_open.return_value.__enter__.return_value.read.return_value = (
'{"0034-8910": "0101-0101"}')
mk_get_bundle_id.return_value = "0034-8910-2014-v48-n2"
error = LinkDocumentToDocumentsBundleException(
"No AOP Bundle in Journal")
error.response = Mock(status_code=404)
mk_update_aop_bundle_items.side_effect = error
link_documents_to_documentsbundle("path_to_sps_package/package.zip",
self.documents, "/json/index.json")
MockLogger.error.assert_called_with("No AOP Bundle in Journal")
def test_if_link_documents_to_documentsbundle_return_none_when_param_document_empty(
self, mk_update_aop_bundle_items, mk_get_or_create_bundle,
mk_get_bundle_id, mk_regdocument, MockLogger):
self.assertIsNone(
link_documents_to_documentsbundle(
"path_to_sps_package/package.zip", [], None), None)
def test_if_some_documents_are_not_register_on_document_store(
self, mk_issue_id, mk_regdocument, mk_kernel_connect):
mk_regdocument.side_effect = [
Mock(status_code=204),
Mock(status_code=422),
Mock(status_code=404)
]
mk_issue_id.side_effect = [
'0034-8910-2014-v48-n2', '0034-8910-2014-v48-n2',
'1518-8787-2014-v2-n2', '1518-8787-2014-v2-n2-s1'
]
with tempfile.TemporaryDirectory() as tmpdirname:
issn_index_json_path = os.path.join(tmpdirname, "issn_index.json")
with open(issn_index_json_path, "w") as index_file:
index_file.write(self.issn_index_json)
self.assertEqual(
link_documents_to_documentsbundle(self.documents,
issn_index_json_path),
[{
'id': '0034-8910-2014-v48-n2',
'status': 204
}, {
'id': '1518-8787-2014-v2-n2',
'status': 422
}, {
'id': '1518-8787-2014-v2-n2-s1',
'status': 404
}])
def link_documents_to_documentsbundle(dag_run, **kwargs):
_sps_package = dag_run.conf.get("sps_package")
documents = kwargs["ti"].xcom_pull(key="documents",
task_ids="register_update_docs_id")
issn_index_json_path = kwargs["ti"].xcom_pull(
task_ids="process_journals_task",
dag_id="sync_isis_to_kernel",
key="issn_index_json_path",
include_prior_dates=True)
if not documents:
return False
linked_bundle, link_executions = sync_documents_to_kernel_operations.link_documents_to_documentsbundle(
_sps_package, documents, issn_index_json_path)
for execution in link_executions:
execution["dag_run"] = kwargs.get("run_id")
execution["pre_sync_dag_run"] = dag_run.conf.get("pre_syn_dag_run_id")
execution["package_name"] = os.path.basename(_sps_package)
add_execution_in_database(table="xml_documentsbundle", data=execution)
if linked_bundle:
kwargs["ti"].xcom_push(key="linked_bundle", value=linked_bundle)
return True
else:
return False
def test_if_some_documents_are_not_register_on_document_store(
self, mk_update_aop_bundle_items, mk_get_or_create_bundle,
mk_get_bundle_id, mk_regdocument, MockLogger):
mk_regdocument.side_effect = [
Mock(status_code=204),
Mock(status_code=422),
Mock(status_code=404)
]
mk_get_bundle_id.side_effect = [
'0034-8910-2014-v48-n2', '0034-8910-2014-v48-n2',
'0034-8910-2014-v2-n2', '0034-8910-2014-v2-n2-s1'
]
with tempfile.TemporaryDirectory() as tmpdirname:
issn_index_json_path = os.path.join(tmpdirname, "issn_index.json")
with open(issn_index_json_path, "w") as index_file:
index_file.write(self.issn_index_json)
result, _ = link_documents_to_documentsbundle(
"path_to_sps_package/package.zip", self.documents,
issn_index_json_path)
self.assertEqual(result, [{
'id': '0034-8910-2014-v48-n2',
'status': 204
}, {
'id': '0034-8910-2014-v2-n2',
'status': 422
}, {
'id': '0034-8910-2014-v2-n2-s1',
'status': 404
}])
def link_documents_to_documentsbundle(dag_run, **kwargs):
documents = kwargs["ti"].xcom_pull(key="documents", task_ids="register_update_docs_id")
if documents:
linked_bundle = sync_documents_to_kernel_operations.link_documents_to_documentsbundle(documents)
if linked_bundle:
kwargs["ti"].xcom_push(key="linked_bundle", value=linked_bundle)
def test_if_some_documents_are_not_register_on_document_store(
self, mk_issue_id, mk_regdocument):
documents = [{
"scielo_id": "S0034-8910.2014048004923",
"issn": "0034-8910",
"year": "2014",
"volume": "48",
"number": "2",
"order": "347",
}, {
"scielo_id": "S0034-8910.2014048004924",
"issn": "0034-8910",
"year": "2014",
"volume": "48",
"number": "2",
"order": "348",
}, {
"scielo_id": "S0034-8910.20140078954641",
"issn": "1518-8787",
"year": "2014",
"volume": "02",
"number": "2",
"order": "978",
}, {
"scielo_id": "S0034-8910.20140078954641",
"issn": "1518-8787",
"year": "2014",
"volume": "02",
"number": "2",
"order": "978",
"supplement": "1",
}]
mk_regdocument.side_effect = [
Mock(status_code=204),
Mock(status_code=422),
Mock(status_code=404)
]
mk_issue_id.side_effect = [
'0034-8910-2014-v48-n2', '0034-8910-2014-v48-n2',
'1518-8787-2014-v2-n2', '1518-8787-2014-v2-n2-s1'
]
self.assertEqual(link_documents_to_documentsbundle(documents),
[{
'id': '0034-8910-2014-v48-n2',
'status': 204
}, {
'id': '1518-8787-2014-v2-n2',
'status': 422
}, {
'id': '1518-8787-2014-v2-n2-s1',
'status': 404
}])
def test_link_documents_to_documentsbundle_should_not_emit_an_update_call_if_the_payload_wont_change(
self, mk_open, mk_issue_id, mk_kernel_connect,
mk_register_document_to_bundle):
new_document_to_link = self.documents[0]
current_bundle_item_list = [{
"id": new_document_to_link["scielo_id"],
"order": new_document_to_link["order"],
}]
# journal_issn_map
mk_open.return_value.__enter__.return_value.read.return_value = (
'{"0034-8910": "0101-0101"}')
# Bundle_id traduzido a partir do journal_issn_map
mk_issue_id.side_effect = ["0034-8910-2014-v48-n2"]
mk_kernel_connect.return_value.json.return_value.__getitem__.return_value = (
current_bundle_item_list)
link_documents_to_documentsbundle([new_document_to_link],
"/some/random/json/path.json")
# Não emita uma atualização do bundle se a nova lista for idêntica a atual
mk_register_document_to_bundle.assert_not_called()
def link_documents_to_documentsbundle(dag_run, **kwargs):
documents = kwargs["ti"].xcom_pull(key="documents",
task_ids="register_update_docs_id")
issn_index_json_path = kwargs["ti"].xcom_pull(
task_ids="process_journals_task",
dag_id="kernel-gate",
key="issn_index_json_path",
include_prior_dates=True)
if documents:
linked_bundle = sync_documents_to_kernel_operations.link_documents_to_documentsbundle(
documents, issn_index_json_path)
if linked_bundle:
kwargs["ti"].xcom_push(key="linked_bundle", value=linked_bundle)
def test_get_or_create_bundle_raises_exception(self, mk_open,
mk_update_aop_bundle_items,
mk_get_or_create_bundle,
mk_get_bundle_id,
mk_regdocument, MockLogger):
def raise_exception(*args, **kwargs):
exc = LinkDocumentToDocumentsBundleException("Bundle not found")
exc.response = Mock(status_code=404)
raise exc
mk_open.return_value.__enter__.return_value.read.return_value = (
'{"0034-8910": "0101-0101"}')
mk_get_bundle_id.return_value = "0101-0101-aop"
mk_get_or_create_bundle.side_effect = raise_exception
result, _ = link_documents_to_documentsbundle(
"path_to_sps_package/2019nahead.zip", self.documents[:1],
"/json/index.json")
self.assertEqual(result, [{'id': '0101-0101-aop', 'status': 404}])
MockLogger.info.assert_called_with(
"Could not get bundle %: Bundle not found", "0101-0101-aop")
def test_if_link_documents_to_documentsbundle_return_none_when_param_document_empty(
self):
self.assertIsNone(link_documents_to_documentsbundle([]), None)