def main(): global verbose global logger args = get_args() db_name = args.db_name check_if_exists(db_name) # providers = ['"'+p.strip()+'"' for p in args.providers.split(",")] providers = ['"' + p.strip() + '"' for p in "GitHub,Gist".split(",")] launch_limit = args.launch_limit access_token = args.access_token if not access_token: access_token = None print( "No token for GitHub API, no additional data will be fetched from GitHub API." ) max_workers = args.max_workers verbose = args.verbose _, script_ts_safe = get_utc_ts() logger_name = f'{os.path.basename(__file__)[:-3]}_at_{script_ts_safe}'.replace( "-", "_") logger = get_logger(logger_name) if verbose: print(f"Logs are in {logger_name}.log") create_repo_table(db_name, providers, launch_limit, access_token, max_workers) print(f"""\n Repo data is extracted from `{launch_table}` table and saved into `{repo_table}` table. You can open this database with `sqlite3 {db_name}` command and then run any sqlite3 command, e.g., `select count(*) from {repo_table}` to get number of repos. """)
def deploy_grafana_config(conf_file): '''Deploy Grafana ConfigMap''' secret_name = 'grafana-config' src_file = os.path.join(os.getcwd(), "deploy/monitoring/grafana/" + conf_file) dst_file = os.path.join(os.getcwd(), conf_file) with open(src_file) as fp: data = fp.read() data = data.replace('REPLACE_NAMESPACE', f'"{deploy_options.namespace}"') with open(dst_file, 'w') as fp: fp.write(data) if not utils.check_if_exists(k8s_object='secret', k8s_object_name=secret_name, target=deploy_options.target, namespace=deploy_options.namespace, profile=deploy_options.profile): print("Creating Grafana Configuration") cmd = "{} create secret generic {} --namespace={} --from-file=grafana.ini={}".format( CMD_BIN, secret_name, deploy_options.namespace, dst_file) utils.check_output(cmd) else: print("Updating Grafana Configuration") cmd = "{} delete secret {} --namespace={}".format( CMD_BIN, secret_name, deploy_options.namespace) utils.check_output(cmd) cmd = "{} create secret generic {} --namespace={} --from-file=grafana.ini={}".format( CMD_BIN, secret_name, deploy_options.namespace, dst_file) utils.check_output(cmd)
def main(): utils.set_profile(deploy_options.target, deploy_options.profile) ## Main OLM Manifest for K8s if deploy_options.target != "oc-ingress": # K8s deployed = utils.check_if_exists('namespace', 'olm', namespace='olm') if not deployed: olm_manifests = [ "https://github.com/operator-framework/operator-lifecycle-manager/releases/download/0.15.1/crds.yaml", "https://github.com/operator-framework/operator-lifecycle-manager/releases/download/0.15.1/olm.yaml" ] for manifest_url in olm_manifests: file_name = "build/{}".format( os.path.basename(urlparse(manifest_url).path)) dst_file = os.path.join(os.getcwd(), file_name) print("Deploying {}".format(dst_file)) urlretrieve(manifest_url, dst_file) utils.apply(dst_file) check_deployment() else: # OCP print("OLM Deployment not necessary")
def index(): """ Index page """ count = Address.query.count() form = SpammerForm() # try to validate, and check for AJAX submission if form.validate_on_submit(): if not utils.check_if_exists(form.address.data): flash( u"We've added %s to the database." % form.address.data, "text-success") else: flash( u"We already know about %s, though." % form.address.data, "text-success") if request.is_xhr: # OK to send back a fragment return render_template( 'form.jinja', form=form, ) # GET or no JS, so render a full page return render_template( 'index.jinja', form=form, count=count, recaptcha_public_key=app.config['RECAPTCHA_PUBLIC_KEY'])
def main(): utils.verify_build_directory(deploy_options.namespace) ## Main OLM Manifest for K8s if deploy_options.target != "oc-ingress": # K8s deployed = utils.check_if_exists(k8s_object='namespace', k8s_object_name='olm', target=deploy_options.target, namespace='olm', profile=deploy_options.profile) if not deployed: olm_manifests = [ "https://github.com/operator-framework/operator-lifecycle-manager/releases/download/0.15.1/crds.yaml", "https://github.com/operator-framework/operator-lifecycle-manager/releases/download/0.15.1/olm.yaml" ] for manifest_url in olm_manifests: dst_file = os.path.join( os.getcwd(), 'build', deploy_options.namespace, os.path.basename(urlparse(manifest_url).path)) print("Deploying {}".format(dst_file)) urlretrieve(manifest_url, dst_file) utils.apply(target=deploy_options.target, namespace=None, profile=deploy_options.profile, file=dst_file) check_deployment() else: # OCP print("OLM Deployment not necessary")
def deploy_oauth_reqs(): # Token generation for session_secret session_secret = secrets.token_hex(43) secret_name = 'grafana-proxy' if not utils.check_if_exists('secret', secret_name): cmd = "{} -n assisted-installer create secret generic {} --from-literal=session_secret={}".format( CMD_BIN, secret_name, session_secret) utils.check_output(cmd) ## Create and Annotate Serviceaccount sa_name = 'grafana' if not utils.check_if_exists('sa', sa_name): cmd = "{} -n assisted-installer create serviceaccount {} ".format( CMD_BIN, sa_name) utils.check_output(cmd) json_manifest = '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"grafana"}}' annotation_name = 'serviceaccounts.openshift.io/oauth-redirectreference.grafana' cmd = "{} -n assisted-installer annotate serviceaccount {} --overwrite {}='{}'".format( CMD_BIN, sa_name, annotation_name, json_manifest) utils.check_output(cmd) # Get OCP Certificate if not utils.check_if_exists('secret', 'openshift-custom-ca'): secret_name = 'router-certs-default' ns = 'openshift-ingress' template = '{{index .data "tls.crt"}}' cmd = "{} get secret {} --namespace={} --template '{}'".format( CMD_BIN, secret_name, ns, template) ca_cert = utils.check_output(cmd) # Renderized secret with CA Certificate of the OCP Cluster src_file = os.path.join( os.getcwd(), "deploy/monitoring/prometheus/assisted-installer-ocp-prometheus-custom-ca.yaml" ) dst_file = os.path.join( os.getcwd(), "build/assisted-installer-ocp-prometheus-custom-ca.yaml") topic = 'OCP Custom CA' with open(src_file, "r") as src: with open(dst_file, "w+") as dst: data = src.read() data = data.replace("BASE64_CERT", ca_cert) print("Deploying {}: {}".format(topic, dst_file)) dst.write(data) utils.apply(dst_file)
def deploy_oauth_reqs(): '''oauth Integration in OCP''' ## Token generation for session_secret session_secret = secrets.token_hex(43) secret_name = 'prometheus-k8s-proxy' if not utils.check_if_exists(k8s_object='secret', k8s_object_name=secret_name, target=deploy_options.target, namespace=deploy_options.namespace, profile=deploy_options.profile): cmd = "{} -n {} create secret generic {} --from-literal=session_secret={}" \ .format(CMD_BIN, deploy_options.namespace, secret_name, session_secret) utils.check_output(cmd) ## Annotate Serviceaccount json_manifest = '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"prometheus-assisted"}}' sa_name = 'prometheus-k8s' annotation_name = 'serviceaccounts.openshift.io/oauth-redirectreference.assisted-installer-prometheus' cmd = "{} -n {} annotate serviceaccount {} --overwrite {}='{}'"\ .format(CMD_BIN, deploy_options.namespace, sa_name, annotation_name, json_manifest) utils.check_output(cmd) # Download OCP Certificate as a secret cert_secret_name = 'openshift-custom-ca' cmd = "{} -n {} get secret {} --no-headers".format( CMD_BIN, deploy_options.namespace, cert_secret_name) cert_secret = utils.check_output(cmd) if not cert_secret: # Get OCP Certificate secret_name = 'router-certs-default' namespace = 'openshift-ingress' template = '{{index .data "tls.crt"}}' cmd = "{} get secret {} --namespace={} --template '{}'"\ .format(CMD_BIN, secret_name, namespace, template) ca_cert = utils.check_output(cmd) # Renderized secret with CA Certificate of the OCP Cluster src_file = os.path.join(os.getcwd(), \ 'deploy/monitoring/prometheus/assisted-installer-ocp-prometheus-custom-ca.yaml') dst_file = os.path.join(os.getcwd(), \ 'build', deploy_options.namespace, 'assisted-installer-ocp-prometheus-custom-ca.yaml') topic = 'OCP Custom CA' with open(src_file, "r") as src: with open(dst_file, "w+") as dst: data = src.read() data = data.replace("BASE64_CERT", ca_cert) data = data.replace('REPLACE_NAMESPACE', f'"{deploy_options.namespace}"') print("Deploying {}: {}".format(topic, dst_file)) dst.write(data) utils.apply(target=deploy_options.target, namespace=deploy_options.namespace, profile=deploy_options.profile, file=dst_file) else: print("Secret {} already exists", cert_secret_name)
def deploy_grafana_ds(): ns = 'assisted-installer' secret_name = 'grafana-datasources' src_file = os.path.join(os.getcwd(), "deploy/monitoring/grafana/prometheus.json") if not utils.check_if_exists('secret', secret_name): print("Creating Grafana Datasource") cmd = "{} create secret generic {} --namespace={} --from-file=prometheus.yaml={}".format( CMD_BIN, secret_name, ns, src_file) utils.check_output(cmd)
def deploy_grafana_ds(): '''Deploy grafana daemonSet''' secret_name = 'grafana-datasources' src_file = os.path.join(os.getcwd(), "deploy/monitoring/grafana/prometheus.json") dst_file = os.path.join(os.getcwd(), "build/prometheus.json") with open(src_file) as fp: data = fp.read() data = data.replace('REPLACE_NAMESPACE', deploy_options.namespace) with open(dst_file, 'w') as fp: fp.write(data) if not utils.check_if_exists('secret', secret_name, deploy_options.namespace): print("Creating Grafana Datasource") cmd = "{} create secret generic {} --namespace={} --from-file=prometheus.yaml={}".format(CMD_BIN, secret_name, deploy_options.namespace, dst_file) utils.check_output(cmd)
def deploy_grafana_config(conf_file): ns = 'assisted-installer' secret_name = 'grafana-config' src_file = os.path.join(os.getcwd(), "deploy/monitoring/grafana/" + conf_file) if not utils.check_if_exists('secret', secret_name): print("Creating Grafana Configuration") cmd = "{} create secret generic {} --namespace={} --from-file=grafana.ini={}".format( CMD_BIN, secret_name, ns, src_file) utils.check_output(cmd) else: print("Updating Grafana Configuration") cmd = "{} delete secret {} --namespace={}".format( CMD_BIN, secret_name, ns) utils.check_output(cmd) cmd = "{} create secret generic {} --namespace={} --from-file=grafana.ini={}".format( CMD_BIN, secret_name, ns, src_file) utils.check_output(cmd)
def main(): deploy_options = deployment_options.load_deployment_options() utils.verify_build_directory(deploy_options.namespace) # Render a file without values for the operator as we don't want every deployment to have the same values if not deploy_options.apply_manifest: render_file(deploy_options.namespace, "", "") return secret_name = 'assisted-installer-local-auth-key' exists = utils.check_if_exists("secret", secret_name, target=deploy_options.target, namespace=deploy_options.namespace, profile=deploy_options.profile) if exists: print( f'Secret {secret_name} already exists in namespace {deploy_options.namespace}' ) return output_dir = tempfile.TemporaryDirectory() priv_path = os.path.join(output_dir.name, f'ec-private-key.pem') pub_path = os.path.join(output_dir.name, f'ec-public-key.pem') print( utils.check_output( f'openssl ecparam -name prime256v1 -genkey -noout -out {priv_path}' )) print( utils.check_output( f'openssl ec -in {priv_path} -pubout -out {pub_path}')) secret_file = render_file(deploy_options.namespace, encoded_contents(priv_path), encoded_contents(pub_path)) utils.apply(target=deploy_options.target, namespace=deploy_options.namespace, profile=deploy_options.profile, file=secret_file)
def index(): """ Index page """ count = Address.query.count() form = SpammerForm() if request.method == 'POST' and request.is_xhr: # put request vars into a form and try to validate form.address = request.form.get('address') form.csrf_token = request.form.get('csrf_token') new_form = SpammerForm() if form.validate_on_submit(): if not utils.check_if_exists(form.address): # process the address # send back thank you, and a new form flash(u"Thanks!", "text-success") else: # address exists, send back an error and a new form flash(u"We already know that spammer!", "text-error") else: # Validation error new_form._errors = form.errors return render_template('form.jinja', form=new_form) # GET, just render a page with a blank form return render_template('index.jinja', form=form, count=count)
def index(): """ Index page """ count = Address.query.count() form = SpammerForm() if request.method == "POST" and request.is_xhr: # put request vars into a form and try to validate form.address = request.form.get("address") form.csrf_token = request.form.get("csrf_token") new_form = SpammerForm() if form.validate_on_submit(): if not utils.check_if_exists(form.address): # process the address # send back thank you, and a new form flash(u"Thanks!", "text-success") else: # address exists, send back an error and a new form flash(u"We already know that spammer!", "text-error") else: # Validation error new_form._errors = form.errors return render_template("form.jinja", form=new_form) # GET, just render a page with a blank form return render_template("index.jinja", form=form, count=count)
def main(): args = get_args() # check inputs build_log_folders = [] for f in args.build_log_folders.split(","): abs_f = os.path.abspath(f) check_if_exists(abs_f) build_log_folders.append(abs_f) db_name = args.db_name check_if_exists(db_name) # prepare logger _, script_ts_safe = get_utc_ts() logger_name = f'{os.path.basename(__file__)[:-3]}_at_{script_ts_safe}'.replace( "-", "_") logger = get_logger(logger_name) logger.info("Start") print(f"Logs are in {logger_name}.log") db = Database(db_name) # add new columns if "buildpack" not in db[execution_table].columns_dict: db[execution_table].add_column("buildpack", str) if "build_error" not in db[execution_table].columns_dict: db[execution_table].add_column("build_error", str) # update values of new columns per each row for build_log_folder in build_log_folders: print(build_log_folder) logger.info(build_log_folder) folder_name = os.path.basename(build_log_folder) script_timestamp = folder_name.split("_")[-1] d, t = script_timestamp.split("T") script_timestamp = f"{d}T{t.replace('-', ':')}" log_files = [ i for i in os.listdir(build_log_folder) if i.endswith(".log") ] len_log_files = len(log_files) logger.info(f"{len_log_files} log files") count = 0 for log_file in log_files: repo_id = int(log_file.split("_")[0]) log_file = os.path.join(build_log_folder, log_file) # print(log_file) count += 1 # provider_r2d = "404" buildpack = "404" build_error = "None" with open(log_file, "r") as f: for line in f: line = line.rstrip() # if line.startswith("Picked") and line.endswith("provider."): # provider_r2d = line.split(" ")[1] if buildpack == "404" and line.startswith( "Using") and line.endswith("builder"): buildpack = line.split(" ")[1] # TODO what are other errors, how to detect them? elif build_error == "None" and "ReadTimeoutError" in line: # NOTE: this doesnt catch only docker timeouts, e.g. also from pip # urllib3.exceptions.ReadTimeoutError: UnixHTTPConnectionPool(host='localhost', port=None) -> # this is the timeout error from docker from repo2docker build_error = "ReadTimeoutError" logger.info(f"{repo_id}: ReadTimeoutError: {line}") break # save new data into tables new_data = {"buildpack": buildpack, "build_error": build_error} # print(new_data) if buildpack == "404": # TODO check why? # - docker readtimeout # - repo or ref doesnt exist anymore or another cloning error logger.warning(f"{repo_id}: {new_data}") # update only rows that this log file is related, # a repo can have many builds in different times for different r2d versions db.conn.execute(f"""UPDATE {execution_table} SET buildpack="{buildpack}", build_error="{build_error}" WHERE script_timestamp="{script_timestamp}" AND repo_id={repo_id};""" ) db.conn.commit() # logger.info(f"{script_timestamp} : {repo_id} : {new_data}") print(f'{(count*100)/len_log_files:.3f}%\r', end="") logger.info("Done")
def main(): args = get_args() script_timestamps = [ '"' + st.strip() + '"' for st in args.script_timestamps.split(",") if st ] db_name = args.db_name check_if_exists(db_name) _, script_ts_safe = get_utc_ts() logger_name = f'{os.path.basename(__file__)[:-3]}_at_{script_ts_safe}'.replace( "-", "_") logger = get_logger(logger_name) logger.info("Start") print(f"Logs are in {logger_name}.log") db = Database(db_name) # add new columns if "kernel_name" not in db[execution_table].columns_dict: db[execution_table].add_column("kernel_name", str) if "nb_execution_time" not in db[execution_table].columns_dict: db[execution_table].add_column("nb_execution_time", int) if "nb_error" not in db[execution_table].columns_dict: db[execution_table].add_column("nb_error", str) execution_count = 0 if script_timestamps: executions = db[execution_table].rows_where( f'nb_log_file is not null AND ' f'script_timestamp IN ({", ".join(script_timestamps)})') else: executions = db[execution_table].rows_where('nb_log_file is not null') # add values of new columns per each row for execution in executions: execution_count += 1 nb_log_file = execution["nb_log_file"] # TODO parse also logs of notebooks detection: use repo folder, it is in form of "notebooks_<TS>.log" # repo_folder = os.path.dirname(nb_log_file) # repo_id = int(repo_folder.split("_")[0]) # assert repo_id == execution["repo_id"] # repo_id = execution["repo_id"] # print(nb_log_file) kernel_name = "404" nb_execution_time = 404 nb_error = "None" with open(nb_log_file, "r") as f: for line in f: line = line.rstrip() if kernel_name == "404" and "execute:" in line and "Executing notebook with kernel:" in line: kernel_name = line.split(" ")[-1] elif nb_execution_time == 404 and "inrepo:" in line and "Execution time is" in line: nb_execution_time = int(line.split(" ")[-1]) # TODO what are other errors, how to detect them? elif nb_error == "None" and "execute:" in line and "Timeout waiting for execute reply" in line: nb_error = "TimeoutError" break # save new data into tables # new_data = {"kernel_name": kernel_name, "nb_execution_time": nb_execution_time, "nb_error": nb_error} # if kernel_name == "404": # logger.warning(f"{repo_id} : {nb_log_file} : {new_data}") # if nb_error == "None" and nb_execution_time == 404: # logger.warning(f"undetected error: {repo_id} : {nb_log_file}") # row = list( # db[execution_table].rows_where(f'nb_log_file="{nb_log_file}"') # ) # if not row: # logger.error(f"Error: {repo_id}: nb log file doesnt exist: {nb_log_file} : {new_data}") # elif len(row) > 1: # logger.error(f"Error: {repo_id}: nb log file must be unique: {nb_log_file} : {new_data}") # else: db.conn.execute(f"""UPDATE {execution_table} SET kernel_name="{kernel_name}", nb_execution_time={nb_execution_time}, nb_error="{nb_error}" WHERE nb_log_file="{nb_log_file}";""") db.conn.commit() # logger.info(f"{nb_log_file} : {new_data}") print(f'{execution_count}\r', end="") logger.info(f"{execution_count} executions") logger.info("Done")
def test_exists(self): """ Should pass because we've added the address during setup """ print Address.query.filter_by(address="@test-address.com").first().address assert utils.check_if_exists('test-address.com') print Address.query.filter_by(address="@test-address.com").first().address