class PrestoCheckOperator(BaseOperator):
"""
Performs a simple check using sql code in a specific Presto database.
:param sql: the sql to be executed
:type sql: string
:param presto_dbid: reference to the Presto database
:type presto_dbid: string
"""
__mapper_args__ = {
'polymorphic_identity': 'PrestoCheckOperator'
}
template_fields = ('sql',)
template_ext = ('.hql', '.sql',)
@apply_defaults
def __init__(
self, sql,
presto_conn_id=conf.get('hooks', 'PRESTO_DEFAULT_CONN_ID'),
*args, **kwargs):
super(PrestoCheckOperator, self).__init__(*args, **kwargs)
self.presto_conn_id = presto_conn_id
self.hook = PrestoHook(presto_conn_id=presto_conn_id)
self.sql = sql
def execute(self, execution_date=None):
logging.info('Executing SQL check: ' + self.sql)
records = self.hook.get_records(hql=self.sql)
if not records:
return False
else:
return not any([ bool(r) for r in records[0] ])
def execute(self, context):
presto = PrestoHook(presto_conn_id=self.presto_conn_id)
logging.info("Extracting data from Presto")
logging.info(self.sql)
results = presto.get_records(self.sql)
mysql = MySqlHook(mysql_conn_id=self.mysql_conn_id)
if self.mysql_preoperator:
logging.info("Running MySQL preoperator")
logging.info(self.mysql_preoperator)
mysql.run(self.mysql_preoperator)
logging.info("Inserting rows into MySQL")
mysql.insert_rows(table=self.mysql_table, rows=results)
def execute(self, context):
presto = PrestoHook(presto_conn_id=self.presto_conn_id)
logging.info("Extracting data from Presto")
logging.info(self.sql)
results = presto.get_records(self.sql)
mysql = MySqlHook(mysql_conn_id=self.mysql_conn_id)
if self.mysql_preoperator:
logging.info("Running MySQL preoperator")
logging.info(self.mysql_preoperator)
mysql.run(self.mysql_preoperator)
logging.info("Inserting rows into MySQL")
mysql.insert_rows(table=self.mysql_table, rows=results)
class PrestoCheckOperator(BaseOperator):
"""
Performs a simple check using sql code in a specific Presto database.
:param sql: the sql to be executed
:type sql: string
:param presto_dbid: reference to the Presto database
:type presto_dbid: string
"""
__mapper_args__ = {'polymorphic_identity': 'PrestoCheckOperator'}
template_fields = ('sql', )
template_ext = (
'.hql',
'.sql',
)
@apply_defaults
def __init__(self,
sql,
presto_conn_id=conf.get('hooks', 'PRESTO_DEFAULT_CONN_ID'),
*args,
**kwargs):
super(PrestoCheckOperator, self).__init__(*args, **kwargs)
self.presto_conn_id = presto_conn_id
self.hook = PrestoHook(presto_conn_id=presto_conn_id)
self.sql = sql
def execute(self, execution_date=None):
logging.info('Executing SQL check: ' + self.sql)
records = self.hook.get_records(hql=self.sql)
if not records:
return False
else:
return not any([bool(r) for r in records[0]])
