class HiveOperator(BaseOperator):
"""
Executes hql code in a specific Hive database.
:param hql: the hql to be executed
:type hql: string
:param hive_conn_id: reference to the Hive database
:type hive_conn_id: string
"""
__mapper_args__ = {
'polymorphic_identity': 'HiveOperator'
}
template_fields = ('hql',)
template_ext = ('.hql', '.sql',)
@apply_defaults
def __init__(
self, hql,
hive_conn_id=conf.get('hooks', 'HIVE_DEFAULT_CONN_ID'),
*args, **kwargs):
super(HiveOperator, self).__init__(*args, **kwargs)
self.hive_conn_id = hive_conn_id
self.hook = HiveHook(hive_conn_id=hive_conn_id)
self.hql = hql
def execute(self, execution_date):
logging.info('Executing: ' + self.hql)
self.hook.run_cli(hql=self.hql)
class HiveOperator(BaseOperator):
"""
Executes hql code in a specific Hive database.
:param hql: the hql to be executed
:type hql: string
:param hive_conn_id: reference to the Hive database
:type hive_conn_id: string
"""
__mapper_args__ = {'polymorphic_identity': 'HiveOperator'}
template_fields = ('hql', )
template_ext = (
'.hql',
'.sql',
)
@apply_defaults
def __init__(self,
hql,
hive_conn_id=conf.get('hooks', 'HIVE_DEFAULT_CONN_ID'),
*args,
**kwargs):
super(HiveOperator, self).__init__(*args, **kwargs)
self.hive_conn_id = hive_conn_id
self.hook = HiveHook(hive_conn_id=hive_conn_id)
self.hql = hql
def execute(self, execution_date):
logging.info('Executing: ' + self.hql)
self.hook.run_cli(hql=self.hql)
def __init__(self,
hql,
hive_conn_id=conf.get('hooks', 'HIVE_DEFAULT_CONN_ID'),
*args,
**kwargs):
super(HiveOperator, self).__init__(*args, **kwargs)
self.hive_conn_id = hive_conn_id
self.hook = HiveHook(hive_conn_id=hive_conn_id)
self.hql = hql
def __init__(
self,
table, partition="ds='{{ ds }}'",
hive_conn_id=conf.get('hooks', 'HIVE_DEFAULT_CONN_ID'),
schema='default',
*args, **kwargs):
super(HivePartitionSensor, self).__init__(*args, **kwargs)
if '.' in table:
schema, table = table.split('.')
self.hive_conn_id = hive_conn_id
self.hook = HiveHook(hive_conn_id=hive_conn_id)
self.table = table
self.partition = partition
self.schema = schema
class HivePartitionSensor(BaseSensorOperator):
"""
Waits for the apparation of a partition in Hive
"""
template_fields = ('table', 'partition',)
__mapper_args__ = {
'polymorphic_identity': 'HivePartitionSensor'
}
@apply_defaults
def __init__(
self,
table, partition="ds='{{ ds }}'",
hive_conn_id=conf.get('hooks', 'HIVE_DEFAULT_CONN_ID'),
schema='default',
*args, **kwargs):
super(HivePartitionSensor, self).__init__(*args, **kwargs)
if '.' in table:
schema, table = table.split('.')
self.hive_conn_id = hive_conn_id
self.hook = HiveHook(hive_conn_id=hive_conn_id)
self.table = table
self.partition = partition
self.schema = schema
def poke(self):
logging.info(
'Poking for table {self.schema}.{self.table}, '
'partition {self.partition}'.format(**locals()))
return self.hook.check_for_partition(
self.schema, self.table, self.partition)
def __init__(
self, hql,
hive_conn_id=conf.get('hooks', 'HIVE_DEFAULT_CONN_ID'),
*args, **kwargs):
super(HiveOperator, self).__init__(*args, **kwargs)
self.hive_conn_id = hive_conn_id
self.hook = HiveHook(hive_conn_id=hive_conn_id)
self.hql = hql
