def task_instance_info(dag_id, execution_date, task_id): """ Returns a JSON with a task instance's public instance variables. The format for the exec_date is expected to be "YYYY-mm-DDTHH:MM:SS", for example: "2016-11-16T11:34:15". This will of course need to have been encoded for URL in the request. """ # Convert string datetime into actual datetime try: execution_date = timezone.parse(execution_date) except ValueError: error_message = ( 'Given execution date, {}, could not be identified ' 'as a date. Example date format: 2015-11-16T14:34:15+00:00' .format(execution_date)) log.error(error_message) response = jsonify({'error': error_message}) response.status_code = 400 return response try: ti_info = get_task_instance(dag_id, task_id, execution_date) except AirflowException as err: log.info(err) response = jsonify(error="{}".format(err)) response.status_code = err.status_code return response # JSONify and return. fields = {k: str(v) for k, v in vars(ti_info).items() if not k.startswith('_')} return jsonify(fields)
def task_instance_info(dag_id, execution_date, task_id): """ Returns a JSON with a task instance's public instance variables. The format for the exec_date is expected to be "YYYY-mm-DDTHH:MM:SS", for example: "2016-11-16T11:34:15". This will of course need to have been encoded for URL in the request. """ # Convert string datetime into actual datetime try: execution_date = timezone.parse(execution_date) except ValueError: error_message = ( 'Given execution date, {}, could not be identified ' 'as a date. Example date format: 2015-11-16T14:34:15+00:00'.format( execution_date)) _log.info(error_message) response = jsonify({'error': error_message}) response.status_code = 400 return response try: info = get_task_instance(dag_id, task_id, execution_date) except AirflowException as err: _log.info(err) response = jsonify(error="{}".format(err)) response.status_code = err.status_code return response # JSONify and return. fields = {k: str(v) for k, v in vars(info).items() if not k.startswith('_')} return jsonify(fields)
def _get_leaves_tis(self, execution_date): leaves_tis = [] for leaf in self.subdag.leaves: try: ti = get_task_instance(dag_id=self.subdag.dag_id, task_id=leaf.task_id, execution_date=execution_date) leaves_tis.append(ti) except TaskInstanceNotFound: continue return leaves_tis
def swap_dataset_tables( target_db: str, *tables: sa.Table, use_utc_now_as_source_modified: bool = False, **kwargs, ): """Rename temporary tables to replace current dataset one. Given a one or more dataset tables `tables` this finds the temporary table created for the current DAG run and replaces existing dataset one with it. If a dataset table didn't exist the new table gets renamed, otherwise the existing dataset table is renamed to a temporary "swap" name first. This requires an exclusive lock for the dataset table (similar to TRUNCATE) but doesn't need to copy any data around (reducing the amount of time dataset is unavailable) and will update the table schema at the same time (since it will apply the new schema temporary table was created with). """ engine = sa.create_engine( 'postgresql+psycopg2://', creator=PostgresHook(postgres_conn_id=target_db).get_conn, ) for table in tables: temp_table = get_temp_table(table, kwargs["ts_nodash"]) logger.info("Moving %s to %s", temp_table.name, table.name) with engine.begin() as conn: conn.execute("SET statement_timeout = 600000") grantees = [ grantee[0] for grantee in conn.execute(""" SELECT grantee FROM information_schema.role_table_grants WHERE table_name='{table_name}' AND privilege_type = 'SELECT' AND grantor != grantee """.format(table_name=engine.dialect.identifier_preparer.quote( table.name))).fetchall() ] conn.execute(""" SELECT dataflow.save_and_drop_dependencies('{schema}', '{target_temp_table}'); ALTER TABLE IF EXISTS {schema}.{target_temp_table} RENAME TO {swap_table_name}; ALTER TABLE {schema}.{temp_table} RENAME TO {target_temp_table}; SELECT dataflow.restore_dependencies('{schema}', '{target_temp_table}'); """.format( schema=engine.dialect.identifier_preparer.quote(table.schema), target_temp_table=engine.dialect.identifier_preparer.quote( table.name), swap_table_name=engine.dialect.identifier_preparer.quote( temp_table.name + "_swap"), temp_table=engine.dialect.identifier_preparer.quote( temp_table.name), )) for grantee in grantees + config.DEFAULT_DATABASE_GRANTEES: conn.execute( 'GRANT SELECT ON {schema}.{table_name} TO {grantee}'. format( schema=engine.dialect.identifier_preparer.quote( table.schema), table_name=engine.dialect.identifier_preparer.quote( table.name), grantee=grantee, )) new_modified_utc = kwargs['task_instance'].xcom_pull( key='source-modified-date-utc') if new_modified_utc is None and use_utc_now_as_source_modified: try: new_modified_utc = get_task_instance( kwargs['dag'].safe_dag_id, 'run-fetch', kwargs['execution_date'], ).end_date except TaskNotFound: new_modified_utc = datetime.datetime.utcnow() conn.execute( """ INSERT INTO dataflow.metadata (table_schema, table_name, source_data_modified_utc, dataflow_swapped_tables_utc) VALUES (%s, %s, %s, %s) """, ( table.schema, table.name, new_modified_utc, datetime.datetime.utcnow(), ), )