def test_poke_non_existing(self):
partitions = ["{}.{}/{}={}".format(self.database,
self.table,
self.partition_by,
self.next_day)]
sensor = NamedHivePartitionSensor(partition_names=partitions,
task_id='test_poke_non_existing',
poke_interval=1,
hook=self.hook,
dag=self.dag)
self.assertFalse(sensor.poke(None))
def test_parse_partition_name_default(self):
table = 'users'
partition = 'ds=2016-01-01/state=IT'
name = '{table}/{partition}'.format(table=table,
partition=partition)
parsed_schema, parsed_table, parsed_partition = (
NamedHivePartitionSensor.parse_partition_name(name)
)
self.assertEqual('default', parsed_schema)
self.assertEqual(table, parsed_table)
self.assertEqual(partition, parsed_partition)
def test_parse_partition_name_correct(self):
schema = 'default'
table = 'users'
partition = 'ds=2016-01-01/state=IT'
name = '{schema}.{table}/{partition}'.format(schema=schema,
table=table,
partition=partition)
parsed_schema, parsed_table, parsed_partition = (
NamedHivePartitionSensor.parse_partition_name(name))
self.assertEqual(schema, parsed_schema)
self.assertEqual(table, parsed_table)
self.assertEqual(partition, parsed_partition)
def test_parse_partition_name_incorrect(self):
name = 'incorrect.name'
with self.assertRaises(ValueError):
NamedHivePartitionSensor.parse_partition_name(name)
default_args=default_args,
start_date=timezone.datetime(2020, 8, 15),
catchup=False,
)
start = DummyOperator(
task_id='start',
dag=dag,
)
# Define your pipeline here
check_named_partition = NamedHivePartitionSensor(
task_id='check_named_partition',
partition_names=[
'fact_transactions/execution_date={{ macros.ds_add(ds, -1) }}'
],
metastore_conn_id='my_hive_metastore_conn',
poke_interval=30,
dag=dag,
)
create_product_transactions_table = HiveOperator(
task_id='create_product_transactions_table',
hive_cli_conn_id='my_hive_conn',
hql='''
CREATE TABLE IF NOT EXISTS zkan_product_transactions (
product_description STRING,
price DECIMAL(10, 2),
units INT,
visits INT
)
def test_parses_partitions_with_periods(self):
name = NamedHivePartitionSensor.parse_partition_name(
partition="schema.table/part1=this.can.be.an.issue/part2=ok")
self.assertEqual(name[0], "schema")
self.assertEqual(name[1], "table")
self.assertEqual(name[2], "part1=this.can.be.an.issue/part2=ok")
def test_parse_partition_name_incorrect(self):
name = 'incorrect.name'
with self.assertRaises(ValueError):
NamedHivePartitionSensor.parse_partition_name(name)