Python NamedHivePartitionSensor Examples

Example #1

File: test_named_hive_partition_sensor.py Project: AdamUnger/incubator-airflow

 def test_poke_non_existing(self):
     partitions = ["{}.{}/{}={}".format(self.database,
                                        self.table,
                                        self.partition_by,
                                        self.next_day)]
     sensor = NamedHivePartitionSensor(partition_names=partitions,
                                       task_id='test_poke_non_existing',
                                       poke_interval=1,
                                       hook=self.hook,
                                       dag=self.dag)
     self.assertFalse(sensor.poke(None))

Example #2

File: test_named_hive_partition_sensor.py Project: AdamUnger/incubator-airflow

 def test_parse_partition_name_default(self):
     table = 'users'
     partition = 'ds=2016-01-01/state=IT'
     name = '{table}/{partition}'.format(table=table,
                                         partition=partition)
     parsed_schema, parsed_table, parsed_partition = (
         NamedHivePartitionSensor.parse_partition_name(name)
     )
     self.assertEqual('default', parsed_schema)
     self.assertEqual(table, parsed_table)
     self.assertEqual(partition, parsed_partition)

Example #3

 def test_parse_partition_name_correct(self):
     schema = 'default'
     table = 'users'
     partition = 'ds=2016-01-01/state=IT'
     name = '{schema}.{table}/{partition}'.format(schema=schema,
                                                  table=table,
                                                  partition=partition)
     parsed_schema, parsed_table, parsed_partition = (
         NamedHivePartitionSensor.parse_partition_name(name))
     self.assertEqual(schema, parsed_schema)
     self.assertEqual(table, parsed_table)
     self.assertEqual(partition, parsed_partition)

Example #4

File: test_named_hive_partition_sensor.py Project: ybendana/incubator-airflow

 def test_parse_partition_name_incorrect(self):
     name = 'incorrect.name'
     with self.assertRaises(ValueError):
         NamedHivePartitionSensor.parse_partition_name(name)

Example #5

File: product_price_range_pipeline.py Project: noppadollanngain/automating-your-data-pipeline-with-apache-airflow

    default_args=default_args,
    start_date=timezone.datetime(2020, 8, 15),
    catchup=False,
)

start = DummyOperator(
    task_id='start',
    dag=dag,
)

# Define your pipeline here
check_named_partition = NamedHivePartitionSensor(
    task_id='check_named_partition',
    partition_names=[
        'fact_transactions/execution_date={{ macros.ds_add(ds, -1) }}'
    ],
    metastore_conn_id='my_hive_metastore_conn',
    poke_interval=30,
    dag=dag,
)

create_product_transactions_table = HiveOperator(
    task_id='create_product_transactions_table',
    hive_cli_conn_id='my_hive_conn',
    hql='''
        CREATE TABLE IF NOT EXISTS zkan_product_transactions (
            product_description STRING,
            price               DECIMAL(10, 2),
            units               INT,
            visits              INT
        )

Example #6

File: test_named_hive_partition_sensor.py Project: khairulhabibatibm/airflow-python3.8-image

 def test_parses_partitions_with_periods(self):
     name = NamedHivePartitionSensor.parse_partition_name(
         partition="schema.table/part1=this.can.be.an.issue/part2=ok")
     self.assertEqual(name[0], "schema")
     self.assertEqual(name[1], "table")
     self.assertEqual(name[2], "part1=this.can.be.an.issue/part2=ok")

Example #7

File: test_named_hive_partition_sensor.py Project: AdamUnger/incubator-airflow

 def test_parse_partition_name_incorrect(self):
     name = 'incorrect.name'
     with self.assertRaises(ValueError):
         NamedHivePartitionSensor.parse_partition_name(name)