class PrestoCheckOperator(BaseOperator): """ Performs a simple check using sql code in a specific Presto database. :param sql: the sql to be executed :type sql: string :param presto_dbid: reference to the Presto database :type presto_dbid: string """ __mapper_args__ = { 'polymorphic_identity': 'PrestoCheckOperator' } template_fields = ('sql',) template_ext = ('.hql', '.sql',) @apply_defaults def __init__( self, sql, presto_conn_id=conf.get('hooks', 'PRESTO_DEFAULT_CONN_ID'), *args, **kwargs): super(PrestoCheckOperator, self).__init__(*args, **kwargs) self.presto_conn_id = presto_conn_id self.hook = PrestoHook(presto_conn_id=presto_conn_id) self.sql = sql def execute(self, execution_date=None): logging.info('Executing SQL check: ' + self.sql) records = self.hook.get_records(hql=self.sql) if not records: return False else: return not any([ bool(r) for r in records[0] ])
def execute(self, context): presto = PrestoHook(presto_conn_id=self.presto_conn_id) logging.info("Extracting data from Presto") logging.info(self.sql) results = presto.get_records(self.sql) mysql = MySqlHook(mysql_conn_id=self.mysql_conn_id) if self.mysql_preoperator: logging.info("Running MySQL preoperator") logging.info(self.mysql_preoperator) mysql.run(self.mysql_preoperator) logging.info("Inserting rows into MySQL") mysql.insert_rows(table=self.mysql_table, rows=results)
class PrestoCheckOperator(BaseOperator): """ Performs a simple check using sql code in a specific Presto database. :param sql: the sql to be executed :type sql: string :param presto_dbid: reference to the Presto database :type presto_dbid: string """ __mapper_args__ = {'polymorphic_identity': 'PrestoCheckOperator'} template_fields = ('sql', ) template_ext = ( '.hql', '.sql', ) @apply_defaults def __init__(self, sql, presto_conn_id=conf.get('hooks', 'PRESTO_DEFAULT_CONN_ID'), *args, **kwargs): super(PrestoCheckOperator, self).__init__(*args, **kwargs) self.presto_conn_id = presto_conn_id self.hook = PrestoHook(presto_conn_id=presto_conn_id) self.sql = sql def execute(self, execution_date=None): logging.info('Executing SQL check: ' + self.sql) records = self.hook.get_records(hql=self.sql) if not records: return False else: return not any([bool(r) for r in records[0]])