def send_MIME_email(e_from, e_to, mime_msg, dryrun=False): """发送邮件 .""" log = LoggingMixin().log SMTP_HOST = configuration.conf.get('smtp', 'SMTP_HOST') SMTP_PORT = configuration.conf.getint('smtp', 'SMTP_PORT') SMTP_STARTTLS = configuration.conf.getboolean('smtp', 'SMTP_STARTTLS') SMTP_SSL = configuration.conf.getboolean('smtp', 'SMTP_SSL') SMTP_USER = None SMTP_PASSWORD = None try: SMTP_USER = configuration.conf.get('smtp', 'SMTP_USER') SMTP_PASSWORD = configuration.conf.get('smtp', 'SMTP_PASSWORD') except XToolConfigException: log.debug( "No user/password found for SMTP, so logging in with no authentication." ) if not dryrun: s = smtplib.SMTP_SSL(SMTP_HOST, SMTP_PORT) if SMTP_SSL else smtplib.SMTP( SMTP_HOST, SMTP_PORT) if SMTP_STARTTLS: s.starttls() if SMTP_USER and SMTP_PASSWORD: s.login(SMTP_USER, SMTP_PASSWORD) log.info("Sent an alert email to %s", e_to) s.sendmail(e_from, e_to, mime_msg.as_string()) s.quit()
def send_MIME_email(e_from, e_to, mime_msg, host, port, ssl=False, starttls=False, user=None, password=None, dryrun=False): """发送邮件 .""" log = LoggingMixin().log if not user or not password: log.debug( "No user/password found for SMTP, so logging in with no authentication." ) if not dryrun: s = smtplib.SMTP_SSL(host, port) if ssl else smtplib.SMTP(host, port) if starttls: s.starttls() s.login(user, password) log.info("Sent an alert email to %s", e_to) s.sendmail(e_from, e_to, mime_msg.as_string()) s.quit()
def handle_failure_retry(context): ti = context['ti'] cmd_id = ti.xcom_pull(key='qbol_cmd_id', task_ids=ti.task_id) if cmd_id is not None: cmd = Command.find(cmd_id) if cmd is not None: if cmd.status == 'running': log = LoggingMixin().log log.info('Cancelling the Qubole Command Id: %s', cmd_id) cmd.cancel()
def _post_sendgrid_mail(mail_data): log = LoggingMixin().log sg = sendgrid.SendGridAPIClient(apikey=os.environ.get('SENDGRID_API_KEY')) response = sg.client.mail.send.post(request_body=mail_data) # 2xx status code. if response.status_code >= 200 and response.status_code < 300: log.info('Email with subject %s is successfully sent to recipients: %s' % (mail_data['subject'], mail_data['personalizations'])) else: log.warning('Failed to send out email with subject %s, status code: %s' % (mail_data['subject'], response.status_code))
def get_query_results(self): log = LoggingMixin().log if self.cmd is not None: cmd_id = self.cmd.id log.info("command id: " + str(cmd_id)) query_result_buffer = StringIO() self.cmd.get_results(fp=query_result_buffer, inline=True, delim=COL_DELIM) query_result = query_result_buffer.getvalue() query_result_buffer.close() return query_result else: log.info("Qubole command not found")
def GetDefaultExecutor(): """Creates a new instance of the configured executor if none exists and returns it""" global DEFAULT_EXECUTOR if DEFAULT_EXECUTOR is not None: return DEFAULT_EXECUTOR executor_name = configuration.conf.get('core', 'EXECUTOR') DEFAULT_EXECUTOR = _get_executor(executor_name) log = LoggingMixin().log log.info("Using executor %s", executor_name) return DEFAULT_EXECUTOR
def execute_command(command): """airflow worker 执行shell命令 .""" log = LoggingMixin().log log.info("Executing command in Celery: %s", command) env = os.environ.copy() try: # celery worker 收到消息后,执行消息中的shell命令 subprocess.check_call(command, shell=True, stderr=subprocess.STDOUT, close_fds=True, env=env) except subprocess.CalledProcessError as e: log.exception('execute_command encountered a CalledProcessError') log.error(e.output) raise AirflowException('Celery command failed')
def handle_failure_retry(context): ti = context['ti'] cmd_id = ti.xcom_pull(key='qbol_cmd_id', task_ids=ti.task_id) if cmd_id is not None: cmd = Command.find(cmd_id) if cmd is not None: log = LoggingMixin().log if cmd.status == 'done': log.info( 'Command ID: %s has been succeeded, hence marking this ' 'TI as Success.', cmd_id) ti.state = State.SUCCESS elif cmd.status == 'running': log.info('Cancelling the Qubole Command Id: %s', cmd_id) cmd.cancel()
def _poll_with_exponential_delay(request, max_n, is_done_func, is_error_func): log = LoggingMixin().log for i in range(0, max_n): try: response = request.execute() if is_error_func(response): raise ValueError( 'The response contained an error: {}'.format(response) ) elif is_done_func(response): log.info('Operation is done: %s', response) return response else: time.sleep((2**i) + (random.randint(0, 1000) / 1000)) except errors.HttpError as e: if e.resp.status != 429: log.info('Something went wrong. Not retrying: %s', format(e)) raise else: time.sleep((2**i) + (random.randint(0, 1000) / 1000))
def get_connection(cls, conn_id): conn = random.choice(cls.get_connections(conn_id)) if conn.host: log = LoggingMixin().log log.info("Using connection to: %s", conn.host) return conn
TEST_PLUGINS_FOLDER = os.path.join(AIRFLOW_HOME, 'plugins') # 获得用户自定义的单元测试配置文件 TEST_CONFIG_FILE = AIRFLOW_HOME + '/unittests.cfg' # 如果需要创建一个新的配置文件,则需要产生一个44字节的随机数 if not os.path.isfile(TEST_CONFIG_FILE) or not os.path.isfile(AIRFLOW_CONFIG): FERNET_KEY = generate_fernet_key() else: FERNET_KEY = '' # 如果用户自定义的单元测试配置文件不存在,则从默认配置模版中生成 TEMPLATE_START = ( '# ----------------------- TEMPLATE BEGINS HERE -----------------------') if not os.path.isfile(TEST_CONFIG_FILE): log.info('Creating new Airflow config file for unit tests in: %s', TEST_CONFIG_FILE) with open(TEST_CONFIG_FILE, 'w') as f: cfg = parameterized_config(TEST_CONFIG) f.write(cfg.split(TEMPLATE_START)[-1].strip()) # 如果用户自定义配置文件不存在,则从默认配置模版中生成 if not os.path.isfile(AIRFLOW_CONFIG): log.info('Creating new Airflow config file in: %s', AIRFLOW_CONFIG) with open(AIRFLOW_CONFIG, 'w') as f: cfg = parameterized_config(DEFAULT_CONFIG) cfg = cfg.split(TEMPLATE_START)[-1].strip() if six.PY2: cfg = cfg.encode('utf8') f.write(cfg) log.info("Reading the config from %s", AIRFLOW_CONFIG)