def validate_support_config(cluster_name): """ Validates the provided non-encrypted helm chart values files for the support chart of a specific cluster. """ _prepare_helm_charts_dependencies_and_schemas() config_file_path = find_absolute_path_to_cluster_file(cluster_name) with open(config_file_path) as f: cluster = Cluster(yaml.load(f), config_file_path.parent) if cluster.support: print_colour( f"Validating non-encrypted support values files for {cluster_name}..." ) cmd = [ "helm", "template", str(helm_charts_dir.joinpath("support")), ] for values_file in cluster.support["helm_chart_values_files"]: cmd.append(f"--values={config_file_path.parent.joinpath(values_file)}") try: subprocess.check_output(cmd, text=True) except subprocess.CalledProcessError as e: print(e.stdout) sys.exit(1) else: print_colour(f"No support defined for {cluster_name}. Nothing to validate!")
async def deal_comments_by_id(spider, uid): """ 对应id评论相关 :param spider: :return: """ while True: print_colour('', 'yellow') com2_cmd = input(help_comments2()).lower() com2_cmd = com2_cmd.split(':') if not com2_cmd[0]: print_colour('输入有误!', 'red') continue exit(com2_cmd[0]) if com2_cmd[0] == 'back': break elif com2_cmd[0] == 'up': result = await spider.endorse_comment(uid, False) print_vote_comments(result, 'up') elif com2_cmd[0] == 'neutral': result = await spider.endorse_comment(uid, True) print_colour(result) print_vote_comments(result, 'neutral') elif com2_cmd[0] == 'reply' and len(com2_cmd) == 2: # todo 回复评论 data = { 'content': com2_cmd[1], 'replyToId': uid, } print_colour('功能还在开发中...', 'red') continue else: print_colour('输入有误!', 'red') continue pass
def deploy_support(self, cert_manager_version): cert_manager_url = "https://charts.jetstack.io" print_colour("Provisioning cert-manager...") subprocess.check_call([ "kubectl", "apply", "-f", f"https://github.com/cert-manager/cert-manager/releases/download/{cert_manager_version}/cert-manager.crds.yaml", ]) subprocess.check_call([ "helm", "upgrade", "cert-manager", # given release name (aka. installation name) "cert-manager", # helm chart to install f"--repo={cert_manager_url}", "--install", "--create-namespace", "--namespace=cert-manager", f"--version={cert_manager_version}", ]) print_colour("Done!") print_colour("Provisioning support charts...") support_dir = (Path(__file__).parent.parent).joinpath( "helm-charts", "support") subprocess.check_call(["helm", "dep", "up", support_dir]) # contains both encrypted and unencrypted values files values_file_paths = [ support_dir.joinpath("enc-support.secret.values.yaml") ] + [ self.config_path.joinpath(p) for p in self.support["helm_chart_values_files"] ] with get_decrypted_files(values_file_paths) as values_files: cmd = [ "helm", "upgrade", "--install", "--create-namespace", "--namespace=support", "--wait", "support", str(support_dir), ] for values_file in values_files: cmd.append(f"--values={values_file}") print_colour(f"Running {' '.join([str(c) for c in cmd])}") subprocess.check_call(cmd) print_colour("Done!")
async def main(): try: check_setting() client = await login(USER, PASSWORD) print_logo() await run(client) # except Exception as e: # print_colour(e, 'red') finally: print_colour('欢迎再次使用') await asyncio.sleep(0) await client.close()
def print_logo(): os.system("clear") logo = ''' ;$$; ############# #############;#####o ## o######################### ##### $############################### ## ###$ ######! ########################## ## ### $### ################### ###### ### ### ##o####################### ###### ;### #### ##################### ## ### ###### ######&&################ ## ### ###### ## ############ ####### o## ######## ## ################## ##o ### #### #######o####### ## ###### ###########&##### ## #### #############! ### ######### #####& ## o#### ###### ## ####* ## !## ##### ## ##* ####; ## ##### #####o ##### #### ### ### $###o ### ## ####! $### ## ##### ## ## ;## ### ; ##$ ## ####### ## ##### &## ## ### ### ### ### ### ## ## ;## ## ## ### ## ### ### ## #### ## ### ## ##; ## ##$ ##& ## ## ##; ## ## ##; ### ### ##$ ### ### ## ###################### #####&&&&&&&&&&&&### ### $#####$ ############&$o$&################################ # $&########&o ''' print_colour(logo, 'ultramarine')
def deploy(cluster_name, hub_name, config_path, dask_gateway_version): """ Deploy one or more hubs in a given cluster """ validate_cluster_config(cluster_name) validate_hub_config(cluster_name, hub_name) assert_single_auth_method_enabled(cluster_name, hub_name) with get_decrypted_file(config_path) as decrypted_file_path: with open(decrypted_file_path) as f: config = yaml.load(f) # Most of our hubs use Auth0 for Authentication. This lets us programmatically # determine what auth provider each hub uses - GitHub, Google, etc. Without # this, we'd have to manually generate credentials for each hub - and we # don't want to do that. Auth0 domains are tied to a account, and # this is our auth0 domain for the paid account that 2i2c has. auth0 = config["auth0"] k = KeyProvider(auth0["domain"], auth0["client_id"], auth0["client_secret"]) # Each hub needs a unique proxy.secretToken. However, we don't want # to manually generate & save it. We also don't want it to change with # each deploy - that causes a pod restart with downtime. So instead, # we generate it based on a single secret key (`PROXY_SECRET_KEY`) # combined with the name of each hub. This way, we get unique, # cryptographically secure proxy.secretTokens without having to # keep much state. We can rotate them by changing `PROXY_SECRET_KEY`. # However, if `PROXY_SECRET_KEY` leaks, that means all the hub's # proxy.secretTokens have leaked. So let's be careful with that! SECRET_KEY = bytes.fromhex(config["secret_key"]) config_file_path = find_absolute_path_to_cluster_file(cluster_name) with open(config_file_path) as f: cluster = Cluster(yaml.load(f), config_file_path.parent) with cluster.auth(): hubs = cluster.hubs if hub_name: hub = next((hub for hub in hubs if hub.spec["name"] == hub_name), None) print_colour(f"Deploying hub {hub.spec['name']}...") hub.deploy(k, SECRET_KEY, dask_gateway_version) else: for i, hub in enumerate(hubs): print_colour( f"{i+1} / {len(hubs)}: Deploying hub {hub.spec['name']}..." ) hub.deploy(k, SECRET_KEY, dask_gateway_version)
def assert_single_auth_method_enabled(cluster_name, hub_name): """ For each hub of a specific cluster, it asserts that only a single auth method is enabled. An error is raised when an authenticator other than Auth0 is enabled and `auth0` is not explicitly disabled. """ _prepare_helm_charts_dependencies_and_schemas() config_file_path = find_absolute_path_to_cluster_file(cluster_name) with open(config_file_path) as f: cluster = Cluster(yaml.load(f), config_file_path.parent) hubs = [] if hub_name: hubs = [h for h in cluster.hubs if h.spec["name"] == hub_name] else: hubs = cluster.hubs for i, hub in enumerate(hubs): print_colour( f"{i+1} / {len(hubs)}: Validating authenticator config for {hub.spec['name']}..." ) authenticator_class = "auth0" for values_file_name in hub.spec["helm_chart_values_files"]: if "secret" not in os.path.basename(values_file_name): values_file = config_file_path.parent.joinpath( values_file_name) # Load the hub extra config from its specific values files config = yaml.load(values_file) # Check if there's config that specifies an authenticator class try: if hub.spec["helm_chart"] != "basehub": authenticator_class = config["basehub"]["jupyterhub"][ "hub"]["config"]["JupyterHub"][ "authenticator_class"] else: authenticator_class = config["jupyterhub"]["hub"][ "config"]["JupyterHub"]["authenticator_class"] except KeyError: pass # If the authenticator class is other than auth0, then raise an error # if auth0 is not explicitly disabled from the cluster config if authenticator_class != "auth0" and hub.spec["auth0"].get( "enabled", True): raise ValueError( f"Please disable auth0 for {hub.spec['name']} hub before using another authenticator class!" )
def print_question(question: dict): """ 打印问题及第默认排序下的第一个回答 :param output: :return: """ title = question['title'] # question_id = question['id'] question_content = question['detail'] question_content = html2text.html2text(question_content) print_colour('*' * 50, 'purple') print_colour(f'标题:{title}') print_colour('问题详情:') print_colour(question_content) print_colour('*' * 50, 'purple')
async def login(user, password): """ 登录 :param user: :param password: :return: """ client = ZhihuClient(user, password) load_cookies = False if os.path.exists(client.cookie_file): # 如果cookie缓存存在优先读取缓存 load_cookies = True if not load_cookies and (not USER or not PASSWORD): print_colour('请正确配置USER, PASSWORD', 'red') sys.exit() await client.login(load_cookies=load_cookies) return client
def print_save(article: dict): """ 保存文章到本地 :param article: :return: """ uid = article.get('id') title = article.get('question').get('title') content = article.get('content') save_dir = SAVE_DIR or '/tmp/zhihu_save' file = f'{save_dir}/{title}_{uid}.html' with open(file, 'w') as f: head = '<head> <meta charset="utf-8"><meta http-equiv="Content-Type"' \ ' content="text/html; charset=utf-8" /> </head>' f.write(head) f.write(content) print_colour(f'保存成功!-->{file}')
def validate_hub_config(cluster_name, hub_name): """ Validates the provided non-encrypted helm chart values files for each hub of a specific cluster. """ _prepare_helm_charts_dependencies_and_schemas() config_file_path = find_absolute_path_to_cluster_file(cluster_name) with open(config_file_path) as f: cluster = Cluster(yaml.load(f), config_file_path.parent) hubs = [] if hub_name: hubs = [h for h in cluster.hubs if h.spec["name"] == hub_name] else: hubs = cluster.hubs for i, hub in enumerate(hubs): print_colour( f"{i+1} / {len(hubs)}: Validating non-encrypted hub values files for {hub.spec['name']}..." ) cmd = [ "helm", "template", str(helm_charts_dir.joinpath(hub.spec["helm_chart"])), ] for values_file in hub.spec["helm_chart_values_files"]: if "secret" not in os.path.basename(values_file): cmd.append( f"--values={config_file_path.parent.joinpath(values_file)}" ) # Workaround the current requirement for dask-gateway 0.9.0 to have a # JupyterHub api-token specified, for updates if this workaround can be # removed, see https://github.com/dask/dask-gateway/issues/473. if hub.spec["helm_chart"] in ("daskhub", "binderhub"): cmd.append( "--set=dask-gateway.gateway.auth.jupyterhub.apiToken=dummy") try: subprocess.check_output(cmd, text=True) except subprocess.CalledProcessError as e: print(e.stdout) sys.exit(1)
async def run(client): spider = DataExtractor(client) output = await spider.get_self_info() print_colour(f'hello {output["name"]} 欢迎使用terminal-zhihu!', 'ultramarine') flag = True while flag: print_colour('', 'yellow') cmd = input(help_main()).lower() if not cmd: print_colour('输入有误!', 'red') continue exit(cmd) if cmd == 'remd': await deal_remd(spider) elif cmd == 'aten': # todo 获取关注动态 print_colour('功能还在开发中...', 'red') continue else: print_colour('输入有误!', 'red') continue
async def deal_article(spider, article): """ 处理文章内容命令 :param spider: :param recommend_articles: :param ids: :return: """ while True: print_colour('', 'yellow') arl_cmd = input(help_article()).lower() if not arl_cmd: print_colour('输入有误!', 'red') continue exit(arl_cmd) if arl_cmd == 'back': break elif arl_cmd in ('up', 'down', 'neutral', 'thank', 'unthank'): uid = article.get('id') func = get_com_func(arl_cmd) result = await getattr(spider, func)(uid) print_vote_thank(result, arl_cmd) continue elif arl_cmd == 'comment': typ = article['type'] uid = article.get('id') result, paging = await spider.get_comments(uid, typ) print_comments(result) await deal_comments(spider, result, paging) continue elif arl_cmd == 'save': print_save(article) continue elif arl_cmd == 'enshrine': # todo 收藏回答 print_colour('功能还在开发中...', 'red') continue elif arl_cmd == 'question': await deal_question(spider, article.get('question').get('id'), article.get('id')) continue else: print_colour('输入有误!', 'red') continue
def print_vote_comments(output: dict, typ: str): """ 打印赞同感谢 up', 'down', 'neutral' :param output: :return: """ if output.get('error'): print_colour(output.get('error'), 'red') elif typ == 'up': print_colour(f'点赞评论成功!被赞总数{output["vote_count"]}') elif typ == 'neutral': print_colour(f'保持中立!被赞总数{output["vote_count"]}')
async def deal_remd(spider): """ 处理推荐文章命令 :param spider: :return: """ is_print = True while True: if is_print: recommend_articles = await spider.get_recommend_article() ids = [d.get('id') for d in recommend_articles] print_recommend_article(recommend_articles) is_print = False print_colour('', 'yellow') remd_cmd = input(help_recommend()).lower() remd_cmd = remd_cmd.split(':') if not remd_cmd: print_colour('输入有误!', 'red') continue exit(remd_cmd[0]) if remd_cmd[0] == 'f': is_print = True continue elif remd_cmd[0] == 'r': print_recommend_article(recommend_articles) continue elif remd_cmd[0] == 'read': if len(remd_cmd) != 2: print_colour('输入有误!', 'red') continue if remd_cmd[1] not in ids: print_colour('输入id有误!', 'red') continue output = [d for d in recommend_articles if d['id'] == remd_cmd[1]][0] print_article_content(output) await deal_article(spider, output) continue elif remd_cmd[0] == 'question': question_ids = [ d.get('question').get('id') for d in recommend_articles ] if len(remd_cmd) != 2: print_colour('输入有误!', 'red') continue if remd_cmd[1] not in question_ids: print_colour('输入id有误!', 'red') continue assert len(ids) == len(question_ids) id_map = dict(zip(question_ids, ids)) uid = id_map[remd_cmd[1]] await deal_question(spider, remd_cmd[1], uid) continue elif remd_cmd[0] == 'back': break else: print_colour('输入有误!', 'red') continue
async def deal_question(spider, question_id, uid): """ 处理问题命令 :param spider: :param uid: :param id_map: :return: """ is_print = True while True: if is_print: question_articles, paging = await spider.get_article_by_question( question_id) ids = [d.get('id') for d in question_articles] print_recommend_article(question_articles) is_print = False print_colour('', 'yellow') ques_cmd = input(help_question()).lower() ques_cmd = ques_cmd.split(':') if not ques_cmd: print_colour('输入有误!', 'red') continue exit(ques_cmd[0]) if ques_cmd[0] == 'read': if len(ques_cmd) != 2: print_colour('输入有误!', 'red') continue if ques_cmd[1] not in ids: print_colour('输入id有误!', 'red') continue output = [d for d in question_articles if d['id'] == ques_cmd[1]][0] print_article_content(output) await deal_article(spider, output) continue elif ques_cmd[0] == 'qsdl': question_detail = await spider.get_question_details( question_id, uid) print_question(question_detail) elif ques_cmd[0] == 'n': if paging.get('is_end'): print_colour('已是最后一页!', 'red') continue url = paging['next'] question_articles, paging = await spider.get_article_by_question_url( url) ids = [d.get('id') for d in question_articles] print_recommend_article(question_articles) continue elif ques_cmd[0] == 'p': if paging.get('is_start'): print_colour('已是第一页!', 'red') continue url = paging['previous'] question_articles, paging = await spider.get_article_by_question_url( url) ids = [d.get('id') for d in question_articles] print_recommend_article(question_articles) elif ques_cmd[0] == 'r': print_recommend_article(question_articles) continue elif ques_cmd[0] == 'back': break else: print_colour('输入有误!', 'red') continue
async def deal_comments(spider, result, paging): """ 处理评论命令 :param spider: :return: """ # all_coments = [] while True: comment_ids = [] for d in result: comment_ids.append(d['id']) for clild in d.get('child_comments'): comment_ids.append(clild['id']) comment_ids = list(set(comment_ids)) print_colour('', 'yellow') comm_cmd = input(help_comments()).lower() comm_cmd = comm_cmd.split(':') if not comm_cmd: print_colour('输入有误!', 'red') continue exit(comm_cmd[0]) if comm_cmd[0] == 'back': break elif comm_cmd[0] == 'n': if paging.get('is_end'): print_colour('已是最后一页!', 'red') continue # url = paging['next'].replace('https://www.zhihu.com/', 'https://www.zhihu.com/api/v4/') url = paging['next'] result, paging = await spider.get_comments_by_url(url) print_comments(result) continue elif comm_cmd[0] == 'p': if paging.get('is_start'): print_colour('已是第一页!', 'red') continue # url = paging['previous'].replace('https://www.zhihu.com/', 'https://www.zhihu.com/api/v4/') url = paging['previous'] result, paging = await spider.get_comments_by_url(url) print_comments(result) continue elif comm_cmd[0] == 'com': if len(comm_cmd) != 2: print_colour('输入有误!', 'red') continue if comm_cmd[1] not in comment_ids: print_colour('输入id有误!', 'red') continue await deal_comments_by_id(spider, comm_cmd[1]) continue else: print_colour('输入有误!', 'red') continue
def print_comments(output: list): """ 打印评论 :param output: :return: """ for d in output: author = d.get('author').get('name') reply_to_author = d.get('reply_to_author').get('name') content = d.get('content') vote_count = d.get('vote_count') comment_id = d.get('id') child_comments = d.get('child_comments') print_colour(f'comment_id:{comment_id}', 'purple') if d.get('featured'): print_colour('热评🔥', end='') if reply_to_author: print_colour(f'{author}->{reply_to_author}', end='') else: print_colour(f'{author}', end='') print_colour(f'(赞:{vote_count}):{content}') if child_comments: for clild in child_comments: author = clild.get('author').get('name') reply_to_author = clild.get('reply_to_author').get('name') content = clild.get('content') vote_count = clild.get('vote_count') comment_id = clild.get('id') print_colour(f' comment_id:{comment_id}', 'purple') if d.get('featured'): print_colour(' 热评🔥', end='') if reply_to_author: print_colour(f' {author}->{reply_to_author}', end='') else: print_colour(f' {author}', end='') print_colour(f' (赞:{vote_count}):{content}') print_colour( ' *********************************************************', 'blue') print_colour( '==========================================================', 'blue')
def deploy(self, auth_provider, secret_key, dask_gateway_version): """ Deploy this hub """ # Support overriding domain configuration in the loaded cluster.yaml via # a cluster.yaml specified enc-<something>.secret.yaml file that only # includes the domain configuration of a typical cluster.yaml file. # # Check if this hub has an override file. If yes, apply override. # # FIXME: This could could be generalized so that the cluster.yaml would allow # any of this configuration to be specified in a secret file instead of a # publicly readable file. We should not keep adding specific config overrides # if such need occur but instead make cluster.yaml be able to link to # additional secret configuration. if "domain_override_file" in self.spec.keys(): domain_override_file = self.spec["domain_override_file"] with get_decrypted_file( self.cluster.config_path.joinpath(domain_override_file) ) as decrypted_path: with open(decrypted_path) as f: domain_override_config = yaml.load(f) self.spec["domain"] = domain_override_config["domain"] generated_values = self.get_generated_config(auth_provider, secret_key) if self.spec["helm_chart"] == "daskhub": # Install CRDs for daskhub before deployment manifest_urls = [ f"https://raw.githubusercontent.com/dask/dask-gateway/{dask_gateway_version}/resources/helm/dask-gateway/crds/daskclusters.yaml", f"https://raw.githubusercontent.com/dask/dask-gateway/{dask_gateway_version}/resources/helm/dask-gateway/crds/traefik.yaml", ] for manifest_url in manifest_urls: subprocess.check_call(["kubectl", "apply", "-f", manifest_url]) with tempfile.NamedTemporaryFile( mode="w" ) as generated_values_file, get_decrypted_files( self.cluster.config_path.joinpath(p) for p in self.spec["helm_chart_values_files"] ) as values_files: json.dump(generated_values, generated_values_file) generated_values_file.flush() cmd = [ "helm", "upgrade", "--install", "--create-namespace", "--wait", f"--namespace={self.spec['name']}", self.spec["name"], helm_charts_dir.joinpath(self.spec["helm_chart"]), # Ordering matters here - config explicitly mentioned in cli should take # priority over our generated values. Based on how helm does overrides, this means # we should put the config from cluster.yaml last. f"--values={generated_values_file.name}", ] # Add on the values files for values_file in values_files: cmd.append(f"--values={values_file}") # join method will fail on the PosixPath element if not transformed # into a string first print_colour(f"Running {' '.join([str(c) for c in cmd])}") subprocess.check_call(cmd)
def deploy(self, auth_provider, secret_key, skip_hub_health_test=False): """ Deploy this hub """ # Ensure helm charts are up to date os.chdir("helm-charts") subprocess.check_call(["helm", "dep", "up", "basehub"]) if self.spec["template"] == "daskhub": subprocess.check_call(["helm", "dep", "up", "daskhub"]) os.chdir("..") # Check if this cluster has any secret config. If yes, read it in. secret_config_path = ( Path(os.getcwd()) / "secrets/config/hubs" / f'{self.cluster.spec["name"]}.cluster.yaml' ) secret_hub_config = {} if os.path.exists(secret_config_path): with decrypt_file(secret_config_path) as decrypted_file_path: with open(decrypted_file_path) as f: secret_config = yaml.load(f) if secret_config.get("hubs", {}): hubs = secret_config["hubs"] current_hub = next( (hub for hub in hubs if hub["name"] == self.spec["name"]), {} ) # Support domain name overrides if "domain" in current_hub: self.spec["domain"] = current_hub["domain"] secret_hub_config = current_hub.get("config", {}) generated_values = self.get_generated_config(auth_provider, secret_key) with tempfile.NamedTemporaryFile( mode="w" ) as values_file, tempfile.NamedTemporaryFile( mode="w" ) as generated_values_file, tempfile.NamedTemporaryFile( mode="w" ) as secret_values_file: json.dump(self.spec["config"], values_file) json.dump(generated_values, generated_values_file) json.dump(secret_hub_config, secret_values_file) values_file.flush() generated_values_file.flush() secret_values_file.flush() cmd = [ "helm", "upgrade", "--install", "--create-namespace", "--wait", f"--namespace={self.spec['name']}", self.spec["name"], os.path.join("helm-charts", self.spec["template"]), # Ordering matters here - config explicitly mentioned in clu should take # priority over our generated values. Based on how helm does overrides, this means # we should put the config from config/hubs last. f"--values={generated_values_file.name}", f"--values={values_file.name}", f"--values={secret_values_file.name}", ] print_colour(f"Running {' '.join(cmd)}") # Can't test without deploying, since our service token isn't set by default subprocess.check_call(cmd) if not skip_hub_health_test: # FIXMEL: Clean this up if self.spec["template"] != "basehub": service_api_token = generated_values["basehub"]["jupyterhub"][ "hub" ]["services"]["hub-health"]["apiToken"] else: service_api_token = generated_values["jupyterhub"]["hub"][ "services" ]["hub-health"]["apiToken"] hub_url = f'https://{self.spec["domain"]}' # On failure, pytest prints out params to the test that failed. # This can contain sensitive info - so we hide stderr # FIXME: Don't use pytest - just call a function instead print_colour("Running hub health check...") # Show errors locally but redirect on CI gh_ci = os.environ.get("CI", "false") pytest_args = [ "-q", "deployer/tests", "--hub-url", hub_url, "--api-token", service_api_token, "--hub-type", self.spec["template"], ] if gh_ci == "true": print_colour("Testing on CI, not printing output") with open(os.devnull, "w") as dn, redirect_stderr( dn ), redirect_stdout(dn): exit_code = pytest.main(pytest_args) else: print_colour("Testing locally, do not redirect output") exit_code = pytest.main(pytest_args) if exit_code != 0: print("Health check failed!", file=sys.stderr) sys.exit(exit_code) else: print_colour("Health check succeeded!")
def print_recommend_article(output: list): """ 打印推荐文章简述 :param output: :return: """ for d in output: print_colour('=' * 60, 'white') print_colour(f'article_id:{d["id"]}', 'purple') print_colour(f'question_id:{d["question"]["id"]}', 'purple') print_colour(d['question']['title'], 'purple', end='') print_colour(f"({d['author']['name']})", 'purple') print_colour(d['excerpt']) print_colour( f"*赞同数{d.get('voteup_count')} 感谢数{d.get('thanks_count', 0)} " f"评论数{d.get('comment_count')} 浏览数{d.get('visited_count')}*", 'purple')
def print_article_content(output: dict): """ 打印文章内容 :param output: :return: """ content = output['content'] title = output['question']['title'] question_id = output['question']['id'] content = html2text.html2text(content) print_colour(content) print_colour('-----------------------------------------------------', 'purple') print_colour(f'|article_id:{output["id"]}', 'purple') print_colour(f'|question_id:{question_id}', 'purple') print_colour(f'|title:{title}', 'purple') print_colour('-----------------------------------------------------', 'purple')
def deploy_grafana_dashboards(cluster_name): """ Deploy grafana dashboards to a cluster that provide useful metrics for operating a JupyterHub Grafana dashboards and deployment mechanism in question are maintained in this repo: https://github.com/jupyterhub/grafana-dashboards """ validate_cluster_config(cluster_name) validate_support_config(cluster_name) config_file_path = find_absolute_path_to_cluster_file(cluster_name) with open(config_file_path) as f: cluster = Cluster(yaml.load(f), config_file_path.parent) # If grafana support chart is not deployed, then there's nothing to do if not cluster.support: print_colour( "Support chart has not been deployed. Skipping Grafana dashboards deployment..." ) return grafana_token_file = ( config_file_path.parent).joinpath("enc-grafana-token.secret.yaml") # Read the cluster specific secret grafana token file with get_decrypted_file(grafana_token_file) as decrypted_file_path: with open(decrypted_file_path) as f: config = yaml.load(f) # Check GRAFANA_TOKEN exists in the secret config file before continuing if "grafana_token" not in config.keys(): raise ValueError( f"`grafana_token` not provided in secret file! Please add it and try again: {grafana_token_file}" ) # FIXME: We assume grafana_url and uses_tls config will be defined in the first # file listed under support.helm_chart_values_files. support_values_file = cluster.support.get("helm_chart_values_files", [])[0] with open(config_file_path.parent.joinpath(support_values_file)) as f: support_values_config = yaml.load(f) # Get the url where grafana is running from the support values file grafana_url = (support_values_config.get("grafana", {}).get("ingress", {}).get("hosts", {})) uses_tls = (support_values_config.get("grafana", {}).get("ingress", {}).get("tls", {})) if not grafana_url: print_colour( "Couldn't find `config.grafana.ingress.hosts`. Skipping Grafana dashboards deployment..." ) return grafana_url = (f"https://{grafana_url[0]}" if uses_tls else f"http://{grafana_url[0]}") # Use the jupyterhub/grafana-dashboards deployer to deploy the dashboards to this cluster's grafana print_colour("Cloning jupyterhub/grafana-dashboards...") dashboards_dir = "grafana_dashboards" subprocess.check_call([ "git", "clone", "https://github.com/jupyterhub/grafana-dashboards", dashboards_dir, ]) # We need the existing env too for the deployer to be able to find jssonnet and grafonnet deploy_env = os.environ.copy() deploy_env.update({"GRAFANA_TOKEN": config["grafana_token"]}) try: print_colour(f"Deploying grafana dashboards to {cluster_name}...") subprocess.check_call(["./deploy.py", grafana_url], env=deploy_env, cwd=dashboards_dir) print_colour(f"Done! Dashboards deployed to {grafana_url}.") finally: # Delete the directory where we cloned the repo. # The deployer cannot call jsonnet to deploy the dashboards if using a temp directory here. # Might be because opening more than once of a temp file is tried # (https://docs.python.org/3.8/library/tempfile.html#tempfile.NamedTemporaryFile) shutil.rmtree(dashboards_dir)
def generate_support_matrix_jobs( cluster_file, cluster_config, cluster_info, added_or_modified_files, upgrade_support_on_this_cluster=False, upgrade_support_on_all_clusters=False, ): """Generate a list of dictionaries describing which clusters need to undergo a helm upgrade of their support chart based on whether their associated support chart values files have been modified. To be parsed to GitHub Actions in order to generate jobs in a matrix. Args: cluster_file (path obj): The absolute path to the cluster.yaml file of a given cluster cluster_config (dict): The cluster-wide config for a given cluster in dictionary format cluster_info (dict): A template dictionary for defining matrix jobs prepopulated with some info. "cluster_name": The name of the given cluster; "provider": the cloud provider the given cluster runs on; "reason_for_redeploy": what has changed in the repository to prompt the support chart for this cluster to be redeployed. added_or_modified_files (set[str]): A set of all added or modified files provided in a GitHub Pull Requests upgrade_support_on_this_cluster (bool, optional): If True, generates jobs to update the support chart on the given cluster. This is triggered when the cluster.yaml file itself is modified. Defaults to False. upgrade_support_on_all_clusters (bool, optional): If True, generates jobs to update the support chart on all clusters. This is triggered when common config has been modified in the support helm chart. Defaults to False. Returns: list[dict]: A list of dictionaries. Each dictionary contains: the name of a cluster, the cloud provider that cluster runs on, a Boolean indicating if the support chart should be upgraded, and a reason why the support chart needs upgrading. Example: [ { "cluster_name": 2i2c, "provider": "gcp", "reason_for_support_redeploy": "Support helm chart has been modified", "upgrade_support": True, }, ] """ # Rename dictionary key cluster_info["reason_for_support_redeploy"] = cluster_info.pop( "reason_for_redeploy") # Empty list to store the matrix definitions in matrix_jobs = [] # Double-check that support is defined for this cluster. support_config = cluster_config.get("support", {}) if support_config: if upgrade_support_on_all_clusters or upgrade_support_on_this_cluster: # We know we're upgrading support on all clusters, so just add the cluster # name to the list of matrix jobs and move on matrix_job = cluster_info.copy() matrix_job["upgrade_support"] = True if upgrade_support_on_all_clusters: matrix_job[ "reason_for_support_redeploy"] = "Support helm chart has been modified" matrix_jobs.append(matrix_job) else: # Have the related support values files for this cluster been modified? values_files = [ cluster_file.parent.joinpath(values_file) for values_file in support_config.get( "helm_chart_values_files", {}) ] intersection = added_or_modified_files.intersection(values_files) if intersection: matrix_job = cluster_info.copy() matrix_job["upgrade_support"] = True matrix_job[ "reason_for_support_redeploy"] = "Following helm chart values files were modified: " + ", ".join( [path.name for path in intersection]) matrix_jobs.append(matrix_job) else: print_colour( f"No support defined for cluster: {cluster_info['cluster_name']}") return matrix_jobs
def print_vote_thank(output: dict, typ: str): """ 打印赞同感谢 up', 'down', 'neutral' :param output: :return: """ if output.get('error'): print_colour(output.get('error'), 'red') elif typ == 'thank': print_colour(f'感谢成功!感谢总数{output["thanks_count"]}') elif typ == 'unthank': print_colour(f'取消感谢!感谢总数{output["thanks_count"]}') elif typ == 'up': print_colour(f'赞同成功!赞同总数{output["voteup_count"]}') elif typ == 'down': print_colour(f'反对成功!赞同总数{output["voteup_count"]}') else: print_colour(f'保持中立!赞同总数{output["voteup_count"]}')
def deploy_support(self): cert_manager_url = "https://charts.jetstack.io" cert_manager_version = "v1.3.1" print_colour("Adding cert-manager chart repo...") subprocess.check_call( [ "helm", "repo", "add", "jetstack", cert_manager_url, ] ) print_colour("Updating cert-manager chart repo...") subprocess.check_call( [ "helm", "repo", "update", ] ) print_colour("Provisioning cert-manager...") subprocess.check_call( [ "helm", "upgrade", "--install", "--create-namespace", "--namespace=cert-manager", "cert-manager", "jetstack/cert-manager", f"--version={cert_manager_version}", "--set=installCRDs=true", ] ) print_colour("Done!") print_colour("Provisioning support charts...") subprocess.check_call(["helm", "dep", "up", "support"]) support_dir = Path(__file__).parent.parent / "support" support_secrets_file = support_dir / "secrets.yaml" with tempfile.NamedTemporaryFile(mode="w") as f, decrypt_file( support_secrets_file ) as secret_file: yaml.dump(self.support.get("config", {}), f) f.flush() subprocess.check_call( [ "helm", "upgrade", "--install", "--create-namespace", "--namespace=support", "support", str(support_dir), f"--values={secret_file}", f"--values={f.name}", "--wait", ] ) print_colour("Done!")
def deploy_support(self): cert_manager_url = "https://charts.jetstack.io" cert_manager_version = "v1.3.1" print_colour("Adding cert-manager chart repo...") subprocess.check_call([ "helm", "repo", "add", "jetstack", cert_manager_url, ]) print_colour("Updating cert-manager chart repo...") subprocess.check_call([ "helm", "repo", "update", ]) print_colour("Provisioning cert-manager...") subprocess.check_call([ "helm", "upgrade", "--install", "--create-namespace", "--namespace=cert-manager", "cert-manager", "jetstack/cert-manager", f"--version={cert_manager_version}", "--set=installCRDs=true", ]) print_colour("Done!") print_colour("Provisioning support charts...") support_dir = (Path(__file__).parent.parent).joinpath( "helm-charts", "support") subprocess.check_call(["helm", "dep", "up", support_dir]) support_secrets_file = support_dir.joinpath("enc-support.secret.yaml") # TODO: Update this with statement to handle any number of context managers # containing decrypted support values files. Not critical right now as # no individual cluster has specific support secrets, but it's possible # to support that if we want to in the future. with get_decrypted_file(support_secrets_file) as secret_file: cmd = [ "helm", "upgrade", "--install", "--create-namespace", "--namespace=support", "--wait", "support", str(support_dir), f"--values={secret_file}", ] for values_file in self.support["helm_chart_values_files"]: cmd.append( f"--values={self.config_path.joinpath(values_file)}") print_colour(f"Running {' '.join([str(c) for c in cmd])}") subprocess.check_call(cmd) print_colour("Done!")
async def login(self, load_cookies: bool = False) -> None: """ 登录 :param load_cookies: 是否加载cookie :return: """ if load_cookies: self.cookie_jar.load(self.cookie_file) self.logger.debug(f'加载cookies从:{self.cookie_file}') is_succ = await self.check_login() if is_succ: print_colour('登录成功!', colour='green') return else: print_colour('通过缓存登录失败尝试重新登录', 'red') self.cookie_jar.clear() os.remove(self.cookie_file) login_data = { 'client_id': 'c3cef7c66a1843f8b3a9e6a1e3160e20', 'grant_type': 'password', 'source': 'com.zhihu.web', 'username': self.user, 'password': self.password, 'lang': 'en', # en 4位验证码, cn 中文验证码 'ref_source': 'other_https://www.zhihu.com/signin?next=%2F', 'utm_source': '' } xsrf = await self._get_xsrf() captcha = await self._get_captcha() timestamp = int(time.time() * 1000) login_data.update({ 'captcha': captcha, 'timestamp': timestamp, 'signature': self._get_signature(timestamp, login_data) }) headers = { 'accept-encoding': 'gzip, deflate, br', 'Host': 'www.zhihu.com', 'Referer': 'https://www.zhihu.com/', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' '(KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586', 'content-type': 'application/x-www-form-urlencoded', 'x-zse-83': '3_2.0', 'x-xsrftoken': xsrf } data = self._encrypt(login_data) url = 'https://www.zhihu.com/api/v3/oauth/sign_in' async with self.post(url, data=data, headers=headers) as r: resp = await r.text() if 'error' in resp: print_colour(json.loads(resp)['error'], 'red') self.logger.debug(f"登录失败:{json.loads(resp)['error']}") sys.exit() self.logger.debug(resp) is_succ = await self.check_login() if is_succ: print_colour('登录成功!', colour='green') else: print_colour('登录失败!', colour='red') sys.exit()
def main(): argparser = argparse.ArgumentParser( description="""A command line tool to update Grafana datasources. """) argparser.add_argument( "cluster_name", type=str, nargs="?", help="The name of the cluster where the Grafana lives", default="2i2c", ) args = argparser.parse_args() central_cluster = args.cluster_name grafana_host = get_central_grafana_url(central_cluster) datasource_endpoint = f"https://{grafana_host}/api/datasources" # Get a list of the clusters that already have their prometheus instances used as datasources datasources = get_clusters_used_as_datasources(central_cluster, datasource_endpoint) # Get a list of filepaths to all cluster.yaml files in the repo cluster_files = get_all_cluster_yaml_files() print("Searching for clusters that aren't Grafana datasources...") # Count how many clusters we can't add as datasources for logging exceptions = 0 for cluster_file in cluster_files: # Read in the cluster.yaml file with open(cluster_file) as f: cluster_config = yaml.load(f) # Get the cluster's name cluster_name = cluster_config.get("name", {}) if cluster_name and cluster_name not in datasources: print( f"Found {cluster_name} cluster. Checking if it can be added..." ) # Build the datasource details for the instances that aren't configures as datasources try: datasource_details = build_datasource_details(cluster_name) req_body = json.dumps(datasource_details) # Tell Grafana to create and register a datasource for this cluster headers = build_request_headers(central_cluster) response = requests.post(datasource_endpoint, data=req_body, headers=headers) if response.status_code != 200: print( f"An error occured when creating the datasource. \nError was {response.text}." ) response.raise_for_status() print_colour( f"Successfully created a new datasource for {cluster_name}!" ) except Exception as e: print_colour( f"An error occured for {cluster_name}.\nError was: {e}.\nSkipping...", "yellow", ) exceptions += 1 pass print_colour( f"Failed to add {exceptions} clusters as datasources. See errors above!", "red") print_colour( f"Successfully retrieved {len(datasources)} existing datasources! {datasources}" )
def deploy_jupyterhub_grafana(cluster_name): """ Deploy grafana dashboards for operating a hub """ # Validate our config with JSON Schema first before continuing validate(cluster_name) config_file_path = (Path(os.getcwd()) / "config/hubs" / f"{cluster_name}.cluster.yaml") with open(config_file_path) as f: cluster = Cluster(yaml.load(f)) # If grafana support chart is not deployed, then there's nothing to do if not cluster.support: print_colour( "Support chart has not been deployed. Skipping Grafana dashboards deployment..." ) return secret_config_file = (Path(os.getcwd()) / "secrets/config/hubs" / f"{cluster_name}.cluster.yaml") # Read and set GRAFANA_TOKEN from the cluster specific secret config file with decrypt_file(secret_config_file) as decrypted_file_path: with open(decrypted_file_path) as f: config = yaml.load(f) # Get the url where grafana is running from the cluster config grafana_url = (cluster.support.get("config", {}).get("grafana", {}).get("ingress", {}).get("hosts", {})) uses_tls = (cluster.support.get("config", {}).get("grafana", {}).get("ingress", {}).get("tls", {})) if not grafana_url: print_colour( "Couldn't find `config.grafana.ingress.hosts`. Skipping Grafana dashboards deployment..." ) return grafana_url = ("https://" + grafana_url[0] if uses_tls else "http://" + grafana_url[0]) # Use the jupyterhub/grafana-dashboards deployer to deploy the dashboards to this cluster's grafana print_colour("Cloning jupyterhub/grafana-dashboards...") dashboards_dir = "grafana_dashboards" subprocess.check_call([ "git", "clone", "https://github.com/jupyterhub/grafana-dashboards", dashboards_dir, ]) # We need the existing env too for the deployer to be able to find jssonnet and grafonnet deploy_env = os.environ.copy() deploy_env.update({"GRAFANA_TOKEN": config["grafana_token"]}) try: print_colour(f"Deploying grafana dashboards to {cluster_name}...") subprocess.check_call(["./deploy.py", grafana_url], env=deploy_env, cwd=dashboards_dir) print_colour(f"Done! Dasboards deployed to {grafana_url}.") finally: # Delete the directory where we cloned the repo. # The deployer cannot call jsonnet to deploy the dashboards if using a temp directory here. # Might be because opening more than once of a temp file is tried # (https://docs.python.org/3.8/library/tempfile.html#tempfile.NamedTemporaryFile) shutil.rmtree(dashboards_dir)