SAFETY_CHECK = PythonOperator(
task_id="safety_check",
python_callable=helpers.catalog_safety_check,
dag=DAG
)
SET_S3_NAMESPACE = PythonOperator(
task_id="set_s3_namespace",
python_callable=datetime.now().strftime,
op_args=["%Y-%m-%d_%H-%M-%S"],
dag=DAG
)
GET_NUM_SOLR_DOCS_PRE = task_solrgetnumdocs(
DAG,
COLLECTION,
"get_num_solr_docs_pre",
conn_id=SOLR_CONN.conn_id
)
BW_OAI_HARVEST = PythonOperator(
task_id='bw_oai_harvest',
provide_context=True,
python_callable=harvest.oai_to_s3,
op_kwargs={
"access_id": AIRFLOW_S3.login,
"access_secret": AIRFLOW_S3.password,
"bucket_name": AIRFLOW_DATA_BUCKET,
"harvest_from_date": None,
"harvest_until_date": None,
"metadata_prefix": CATALOG_OAI_BW_MD_PREFIX,
"oai_endpoint": CATALOG_OAI_BW_ENDPOINT,
max_active_runs=1,
schedule_interval=SCHEDULE_INTERVAL)
"""
CREATE TASKS
Tasks with all logic contained in a single operator can be declared here.
Tasks with custom logic are relegated to individual Python files.
"""
SET_COLLECTION_NAME = PythonOperator(task_id="set_collection_name",
python_callable=datetime.now().strftime,
op_args=["%Y-%m-%d_%H-%M-%S"],
dag=DAG)
GET_NUM_SOLR_DOCS_PRE = task_solrgetnumdocs(DAG,
ALIAS,
"get_num_solr_docs_pre",
conn_id=SOLR_CONN.conn_id)
CREATE_COLLECTION = tasks.create_sc_collection(
DAG, SOLR_CONN.conn_id,
CONFIGSET + "-{{ ti.xcom_pull(task_ids='set_collection_name') }}",
REPLICATION_FACTOR, CONFIGSET)
INDEX_WEB_CONTENT = BashOperator(
task_id="index_web_content",
bash_command=AIRFLOW_HOME +
"/dags/cob_datapipeline/scripts/ingest_web_content.sh ",
env={
"HOME":
AIRFLOW_USER_HOME,
"SOLR_AUTH_PASSWORD":
dag=DAG)
CREATE_COLLECTION = PythonOperator(
task_id="create_collection",
python_callable=helpers.catalog_create_missing_collection,
provide_context=True,
dag=DAG,
op_kwargs={
"conn": SOLR_CLOUD,
"collection": COLLECTION_NAME,
"replication_factor": REPLICATION_FACTOR,
"configset": CONFIGSET,
})
GET_NUM_SOLR_DOCS_PRE = task_solrgetnumdocs(DAG,
COLLECTION_NAME,
"get_num_solr_docs_pre",
conn_id=SOLR_CLOUD.conn_id)
INDEX_SFTP_MARC = BashOperator(
task_id="index_sftp_marc",
bash_command=AIRFLOW_HOME +
"/dags/cob_datapipeline/scripts/sc_ingest_marc.sh ",
env={
"AWS_ACCESS_KEY_ID": AIRFLOW_S3.login,
"AWS_SECRET_ACCESS_KEY": AIRFLOW_S3.password,
"BUCKET": AIRFLOW_DATA_BUCKET,
"FOLDER": ALMASFTP_S3_PREFIX + "/" + DAG.dag_id +
"/{{ ti.xcom_pull(task_ids='set_s3_namespace') }}/alma_bibs__",
"GIT_BRANCH": COB_INDEX_VERSION,
"HOME": AIRFLOW_USER_HOME,
"LATEST_RELEASE": str(LATEST_RELEASE),