def register_update_documents(dag_run, **kwargs):
_xmls_to_preserve = kwargs["ti"].xcom_pull(key="xmls_to_preserve",
task_ids="delete_docs_task_id")
if not _xmls_to_preserve:
return False
_optimized_package = kwargs["ti"].xcom_pull(
key="optimized_package", task_ids="optimize_package_task_id")
package = _optimized_package or dag_run.conf.get("sps_package")
_documents, executions = sync_documents_to_kernel_operations.register_update_documents(
package, _xmls_to_preserve)
for execution in executions:
execution["dag_run"] = kwargs.get("run_id")
execution["pre_sync_dag_run"] = dag_run.conf.get("pre_syn_dag_run_id")
execution["package_name"] = os.path.basename(_optimized_package)
add_execution_in_database(table="xml_documents", data=execution)
if _documents:
kwargs["ti"].xcom_push(key="documents", value=_documents)
return True
else:
return False
def link_documents_to_documentsbundle(dag_run, **kwargs):
_sps_package = dag_run.conf.get("sps_package")
documents = kwargs["ti"].xcom_pull(key="documents",
task_ids="register_update_docs_id")
issn_index_json_path = kwargs["ti"].xcom_pull(
task_ids="process_journals_task",
dag_id="sync_isis_to_kernel",
key="issn_index_json_path",
include_prior_dates=True)
if not documents:
return False
linked_bundle, link_executions = sync_documents_to_kernel_operations.link_documents_to_documentsbundle(
_sps_package, documents, issn_index_json_path)
for execution in link_executions:
execution["dag_run"] = kwargs.get("run_id")
execution["pre_sync_dag_run"] = dag_run.conf.get("pre_syn_dag_run_id")
execution["package_name"] = os.path.basename(_sps_package)
add_execution_in_database(table="xml_documentsbundle", data=execution)
if linked_bundle:
kwargs["ti"].xcom_push(key="linked_bundle", value=linked_bundle)
return True
else:
return False
def delete_documents(dag_run, **kwargs):
_sps_package = dag_run.conf.get("sps_package")
_xmls_filenames = kwargs["ti"].xcom_pull(key="xmls_filenames",
task_ids="list_docs_task_id")
if not _xmls_filenames:
return False
_xmls_to_preserve, executions = sync_documents_to_kernel_operations.delete_documents(
_sps_package, _xmls_filenames)
for execution in executions:
execution["dag_run"] = kwargs.get("run_id")
execution["pre_sync_dag_run"] = dag_run.conf.get("pre_syn_dag_run_id")
execution["package_name"] = os.path.basename(_sps_package)
add_execution_in_database(table="xml_documents", data=execution)
if _xmls_to_preserve:
kwargs["ti"].xcom_push(key="xmls_to_preserve", value=_xmls_to_preserve)
return True
else:
return False