def get_input_data(csv_location) -> pd.DataFrame: df = pd.read_csv(csv_location) df.columns = map(str.lower, df.columns) assert "githuburl" in df.columns assert "category" in df.columns duplicated_githuburls = df[df.duplicated(subset=["githuburl"])] duplicated_count = len(duplicated_githuburls) if duplicated_count > 0: logger.warning( f"Duplicate githuburl values found in csv: {duplicated_count}\n{duplicated_githuburls}" ) logger.fatal( f"Fix up {duplicated_count} duplicates from {csv_location} and re-run." ) sys.exit() else: logger.info("No duplicate githuburl values found in csv :)") return df
def init_logging(args): global logger log_level_label, log_level_val = log_level(args.log_level, args.verbose) _, log_file_level = log_level(args.log_file_level, args.verbose) log_trace = log_level_val <= 10 log_file_trace = log_file_level <= 10 colorize = not args.no_log_colors log_fmt_time_c = "[<cyan>{time:HH:mm:ss.SS}</cyan>]" log_fmt_time_fs = "[<cyan>{time:MM-DD HH:mm:ss.SS}</cyan>]" log_fmt_id = "[<cyan>{extra[name]: >17}</cyan>]" log_fmt_mod_c = "[<cyan>{module: >19.19}:{line: <4}</cyan>]" log_fmt_mod_fs = "[<cyan>{module: >19}:{line: <4}</cyan>]" log_fmt_level = "[<lvl>{level: >1.1}</lvl>]" log_fmt_msg = "<level>{message}</level>" log_format_c = [ log_fmt_time_c, log_fmt_id, log_fmt_mod_c, log_fmt_level, log_fmt_msg ] log_format_fs = [ log_fmt_time_fs, log_fmt_id, log_fmt_mod_fs, log_fmt_level, log_fmt_msg ] # Alter the logging capabilities based off the MAD launch settings if not args.no_file_logs: log_format_c[log_format_c.index(log_fmt_time_c)] = log_fmt_time_fs if not log_trace: log_format_c.remove(log_fmt_mod_c) else: log_format_c[log_format_c.index(log_fmt_mod_c)] = log_fmt_mod_fs fs_log_format = ' '.join(log_format_fs) log_format_console = ' '.join(log_format_c) logconfig = { "levels": [{ "name": "DEBUG2", "no": 9, "color": "<blue>" }, { "name": "DEBUG3", "no": 8, "color": "<blue>" }, { "name": "DEBUG4", "no": 7, "color": "<blue>" }, { "name": "DEBUG5", "no": 6, "color": "<blue>" }], "handlers": [{ "sink": sys.stdout, "format": log_format_console, "colorize": colorize, "level": log_level_val, "enqueue": True, "filter": filter_errors }, { "sink": sys.stderr, "format": log_format_console, "colorize": colorize, "level": "ERROR", "diagnose": log_trace, "backtrace": True, "enqueue": True }], "extra": { "name": "Unknown" }, } if not args.no_file_logs: file_logs = { "sink": os.path.join(args.log_path, args.log_filename), "format": fs_log_format, "level": log_file_level, "backtrace": True, "diagnose": log_file_trace, "enqueue": True, "encoding": "UTF-8" } if args.log_file_retention != 0: log_file_retention = str(args.log_file_retention) + " days" file_logs["retention"] = log_file_retention if str(args.log_file_rotation) != "0": file_logs["rotation"] = str(args.log_file_rotation) logconfig["handlers"].append(file_logs) try: logger.configure(**logconfig) init_custom(logger) except ValueError: logger.fatal("Logging parameters/configuration is invalid.") sys.exit(1) logger.info("Setting log level to {} ({}).", str(log_level_val), log_level_label)
def main(parameters: ('resource parameters', 'option', 'p'), region: ('region', 'option', 'r'), noun): """ atlas-cfn-cli.py MongoDB Atlas AWS CloudFormation Custom Resource CLI This tool helps you deploy Atlas resources for your CF templates Set LOGURU_LEVEL to higher for more logs """ TOOL = os.path.realpath(__file__) TOOL_HOME = os.path.dirname(TOOL) logger.info(f"TOOL_HOME listdir: {os.listdir(TOOL_HOME)}") ATLAS_CFN_HOME = os.path.join(TOOL_HOME, "cfn-resources") if TOOL_HOME == "/github/workspace/.github/actions/atlas-cfn-deploy": logger.info( f"Did not detect \"cfn-resources\" folder in {ATLAS_CFN_HOME}, checking running from Github Action." ) ATLAS_CFN_HOME = "/github/workspace/cfn-resources" logger.info(f"TOOL: {TOOL}") logger.info(f"TOOL_HOME: {TOOL_HOME}") logger.info(f"ATLAS_CFN_HOME: {ATLAS_CFN_HOME}") logger.info( f"atlas-cfn-deploy deploying MongoDB Atlas CFN Custom Resources to AWS region {region}." ) logger.info("WARNING: This tool is beta. Swim at your own risk.") check_res = os.listdir(ATLAS_CFN_HOME) if not len(check_res): logger.error(f"os.listdir('{ATLAS_CFN_HOME}'): {check_res}") logger.fatal( "Can't find any cfn-resources to deploy, unable to process.") sys.exit(1) if parameters: params = dict(kvp.strip().split('=') for kvp in parameters.strip().split(',')) else: params = {} noun = noun.lower() logger.info(f"noun:{noun}") logger.info(f"parameters:{parameters}") logger.info(f"params:{params}") res = os.chdir(f'{ATLAS_CFN_HOME}') try_log_error(res, f"Changed to {ATLAS_CFN_HOME}") if noun.startswith("all"): logger.debug("noun was all") res = tier_one_resources if noun == "all+": for r in tier_two_resources: res.append(r) if noun == "all-": res = tier_two_resources for resource in res: command = ["python3", TOOL, f"{resource}", "--region", f"{region}"] if parameters: command.append(["--parameters", f"{parameters}"]) logger.debug(f"resource:{resource}") logger.debug(f"command:{command}") logger.debug(f"res:{res}") res = subprocess.run(command) process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output, error = process.communicate() if error: logger.error(error) logger.info(output) else: submit_cmd = [ "cfn", "submit", "-v", "--region", f"{region}", "--set-default" ] cwd = os.getcwd() try: res_path = os.path.join(f"{ATLAS_CFN_HOME}", noun) res = os.chdir(f'{res_path}') try_log_error(res, f"Changed to resource: {noun} in {res_path}") process = subprocess.Popen(["make"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output, error = process.communicate() if error: logger.error(error) logger.info(output) process = subprocess.Popen(submit_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output, error = process.communicate() if error: logger.error(error) logger.info(output) res = os.chdir(f'{cwd}') try_log_error(res, f"Changed to directory: {cwd}") print(res) except Exception as f: logger.error(f)