class HiveOperator(BaseOperator): """ Executes hql code in a specific Hive database. :param hql: the hql to be executed :type hql: string :param hive_conn_id: reference to the Hive database :type hive_conn_id: string """ __mapper_args__ = { 'polymorphic_identity': 'HiveOperator' } template_fields = ('hql',) template_ext = ('.hql', '.sql',) @apply_defaults def __init__( self, hql, hive_conn_id=conf.get('hooks', 'HIVE_DEFAULT_CONN_ID'), *args, **kwargs): super(HiveOperator, self).__init__(*args, **kwargs) self.hive_conn_id = hive_conn_id self.hook = HiveHook(hive_conn_id=hive_conn_id) self.hql = hql def execute(self, execution_date): logging.info('Executing: ' + self.hql) self.hook.run_cli(hql=self.hql)
class HiveOperator(BaseOperator): """ Executes hql code in a specific Hive database. :param hql: the hql to be executed :type hql: string :param hive_conn_id: reference to the Hive database :type hive_conn_id: string """ __mapper_args__ = {'polymorphic_identity': 'HiveOperator'} template_fields = ('hql', ) template_ext = ( '.hql', '.sql', ) @apply_defaults def __init__(self, hql, hive_conn_id=conf.get('hooks', 'HIVE_DEFAULT_CONN_ID'), *args, **kwargs): super(HiveOperator, self).__init__(*args, **kwargs) self.hive_conn_id = hive_conn_id self.hook = HiveHook(hive_conn_id=hive_conn_id) self.hql = hql def execute(self, execution_date): logging.info('Executing: ' + self.hql) self.hook.run_cli(hql=self.hql)
def __init__(self, hql, hive_conn_id=conf.get('hooks', 'HIVE_DEFAULT_CONN_ID'), *args, **kwargs): super(HiveOperator, self).__init__(*args, **kwargs) self.hive_conn_id = hive_conn_id self.hook = HiveHook(hive_conn_id=hive_conn_id) self.hql = hql
def __init__( self, table, partition="ds='{{ ds }}'", hive_conn_id=conf.get('hooks', 'HIVE_DEFAULT_CONN_ID'), schema='default', *args, **kwargs): super(HivePartitionSensor, self).__init__(*args, **kwargs) if '.' in table: schema, table = table.split('.') self.hive_conn_id = hive_conn_id self.hook = HiveHook(hive_conn_id=hive_conn_id) self.table = table self.partition = partition self.schema = schema
class HivePartitionSensor(BaseSensorOperator): """ Waits for the apparation of a partition in Hive """ template_fields = ('table', 'partition',) __mapper_args__ = { 'polymorphic_identity': 'HivePartitionSensor' } @apply_defaults def __init__( self, table, partition="ds='{{ ds }}'", hive_conn_id=conf.get('hooks', 'HIVE_DEFAULT_CONN_ID'), schema='default', *args, **kwargs): super(HivePartitionSensor, self).__init__(*args, **kwargs) if '.' in table: schema, table = table.split('.') self.hive_conn_id = hive_conn_id self.hook = HiveHook(hive_conn_id=hive_conn_id) self.table = table self.partition = partition self.schema = schema def poke(self): logging.info( 'Poking for table {self.schema}.{self.table}, ' 'partition {self.partition}'.format(**locals())) return self.hook.check_for_partition( self.schema, self.table, self.partition)
def __init__( self, hql, hive_conn_id=conf.get('hooks', 'HIVE_DEFAULT_CONN_ID'), *args, **kwargs): super(HiveOperator, self).__init__(*args, **kwargs) self.hive_conn_id = hive_conn_id self.hook = HiveHook(hive_conn_id=hive_conn_id) self.hql = hql