def register_update_documents(dag_run, **kwargs): _xmls_to_preserve = kwargs["ti"].xcom_pull(key="xmls_to_preserve", task_ids="delete_docs_task_id") if not _xmls_to_preserve: return False _optimized_package = kwargs["ti"].xcom_pull( key="optimized_package", task_ids="optimize_package_task_id") package = _optimized_package or dag_run.conf.get("sps_package") _documents, executions = sync_documents_to_kernel_operations.register_update_documents( package, _xmls_to_preserve) for execution in executions: execution["dag_run"] = kwargs.get("run_id") execution["pre_sync_dag_run"] = dag_run.conf.get("pre_syn_dag_run_id") execution["package_name"] = os.path.basename(_optimized_package) add_execution_in_database(table="xml_documents", data=execution) if _documents: kwargs["ti"].xcom_push(key="documents", value=_documents) return True else: return False
def link_documents_to_documentsbundle(dag_run, **kwargs): _sps_package = dag_run.conf.get("sps_package") documents = kwargs["ti"].xcom_pull(key="documents", task_ids="register_update_docs_id") issn_index_json_path = kwargs["ti"].xcom_pull( task_ids="process_journals_task", dag_id="sync_isis_to_kernel", key="issn_index_json_path", include_prior_dates=True) if not documents: return False linked_bundle, link_executions = sync_documents_to_kernel_operations.link_documents_to_documentsbundle( _sps_package, documents, issn_index_json_path) for execution in link_executions: execution["dag_run"] = kwargs.get("run_id") execution["pre_sync_dag_run"] = dag_run.conf.get("pre_syn_dag_run_id") execution["package_name"] = os.path.basename(_sps_package) add_execution_in_database(table="xml_documentsbundle", data=execution) if linked_bundle: kwargs["ti"].xcom_push(key="linked_bundle", value=linked_bundle) return True else: return False
def delete_documents(dag_run, **kwargs): _sps_package = dag_run.conf.get("sps_package") _xmls_filenames = kwargs["ti"].xcom_pull(key="xmls_filenames", task_ids="list_docs_task_id") if not _xmls_filenames: return False _xmls_to_preserve, executions = sync_documents_to_kernel_operations.delete_documents( _sps_package, _xmls_filenames) for execution in executions: execution["dag_run"] = kwargs.get("run_id") execution["pre_sync_dag_run"] = dag_run.conf.get("pre_syn_dag_run_id") execution["package_name"] = os.path.basename(_sps_package) add_execution_in_database(table="xml_documents", data=execution) if _xmls_to_preserve: kwargs["ti"].xcom_push(key="xmls_to_preserve", value=_xmls_to_preserve) return True else: return False