def get_input_data(csv_location) -> pd.DataFrame:
df = pd.read_csv(csv_location)
df.columns = map(str.lower, df.columns)
assert "githuburl" in df.columns
assert "category" in df.columns
duplicated_githuburls = df[df.duplicated(subset=["githuburl"])]
duplicated_count = len(duplicated_githuburls)
if duplicated_count > 0:
logger.warning(
f"Duplicate githuburl values found in csv: {duplicated_count}\n{duplicated_githuburls}"
)
logger.fatal(
f"Fix up {duplicated_count} duplicates from {csv_location} and re-run."
)
sys.exit()
else:
logger.info("No duplicate githuburl values found in csv :)")
return df
def init_logging(args):
global logger
log_level_label, log_level_val = log_level(args.log_level, args.verbose)
_, log_file_level = log_level(args.log_file_level, args.verbose)
log_trace = log_level_val <= 10
log_file_trace = log_file_level <= 10
colorize = not args.no_log_colors
log_fmt_time_c = "[<cyan>{time:HH:mm:ss.SS}</cyan>]"
log_fmt_time_fs = "[<cyan>{time:MM-DD HH:mm:ss.SS}</cyan>]"
log_fmt_id = "[<cyan>{extra[name]: >17}</cyan>]"
log_fmt_mod_c = "[<cyan>{module: >19.19}:{line: <4}</cyan>]"
log_fmt_mod_fs = "[<cyan>{module: >19}:{line: <4}</cyan>]"
log_fmt_level = "[<lvl>{level: >1.1}</lvl>]"
log_fmt_msg = "<level>{message}</level>"
log_format_c = [
log_fmt_time_c, log_fmt_id, log_fmt_mod_c, log_fmt_level, log_fmt_msg
]
log_format_fs = [
log_fmt_time_fs, log_fmt_id, log_fmt_mod_fs, log_fmt_level, log_fmt_msg
]
# Alter the logging capabilities based off the MAD launch settings
if not args.no_file_logs:
log_format_c[log_format_c.index(log_fmt_time_c)] = log_fmt_time_fs
if not log_trace:
log_format_c.remove(log_fmt_mod_c)
else:
log_format_c[log_format_c.index(log_fmt_mod_c)] = log_fmt_mod_fs
fs_log_format = ' '.join(log_format_fs)
log_format_console = ' '.join(log_format_c)
logconfig = {
"levels": [{
"name": "DEBUG2",
"no": 9,
"color": "<blue>"
}, {
"name": "DEBUG3",
"no": 8,
"color": "<blue>"
}, {
"name": "DEBUG4",
"no": 7,
"color": "<blue>"
}, {
"name": "DEBUG5",
"no": 6,
"color": "<blue>"
}],
"handlers": [{
"sink": sys.stdout,
"format": log_format_console,
"colorize": colorize,
"level": log_level_val,
"enqueue": True,
"filter": filter_errors
}, {
"sink": sys.stderr,
"format": log_format_console,
"colorize": colorize,
"level": "ERROR",
"diagnose": log_trace,
"backtrace": True,
"enqueue": True
}],
"extra": {
"name": "Unknown"
},
}
if not args.no_file_logs:
file_logs = {
"sink": os.path.join(args.log_path, args.log_filename),
"format": fs_log_format,
"level": log_file_level,
"backtrace": True,
"diagnose": log_file_trace,
"enqueue": True,
"encoding": "UTF-8"
}
if args.log_file_retention != 0:
log_file_retention = str(args.log_file_retention) + " days"
file_logs["retention"] = log_file_retention
if str(args.log_file_rotation) != "0":
file_logs["rotation"] = str(args.log_file_rotation)
logconfig["handlers"].append(file_logs)
try:
logger.configure(**logconfig)
init_custom(logger)
except ValueError:
logger.fatal("Logging parameters/configuration is invalid.")
sys.exit(1)
logger.info("Setting log level to {} ({}).", str(log_level_val),
log_level_label)
def main(parameters: ('resource parameters', 'option', 'p'),
region: ('region', 'option', 'r'), noun):
"""
atlas-cfn-cli.py
MongoDB Atlas AWS CloudFormation Custom Resource CLI
This tool helps you deploy Atlas resources for your CF templates
Set LOGURU_LEVEL to higher for more logs
"""
TOOL = os.path.realpath(__file__)
TOOL_HOME = os.path.dirname(TOOL)
logger.info(f"TOOL_HOME listdir: {os.listdir(TOOL_HOME)}")
ATLAS_CFN_HOME = os.path.join(TOOL_HOME, "cfn-resources")
if TOOL_HOME == "/github/workspace/.github/actions/atlas-cfn-deploy":
logger.info(
f"Did not detect \"cfn-resources\" folder in {ATLAS_CFN_HOME}, checking running from Github Action."
)
ATLAS_CFN_HOME = "/github/workspace/cfn-resources"
logger.info(f"TOOL: {TOOL}")
logger.info(f"TOOL_HOME: {TOOL_HOME}")
logger.info(f"ATLAS_CFN_HOME: {ATLAS_CFN_HOME}")
logger.info(
f"atlas-cfn-deploy deploying MongoDB Atlas CFN Custom Resources to AWS region {region}."
)
logger.info("WARNING: This tool is beta. Swim at your own risk.")
check_res = os.listdir(ATLAS_CFN_HOME)
if not len(check_res):
logger.error(f"os.listdir('{ATLAS_CFN_HOME}'): {check_res}")
logger.fatal(
"Can't find any cfn-resources to deploy, unable to process.")
sys.exit(1)
if parameters:
params = dict(kvp.strip().split('=')
for kvp in parameters.strip().split(','))
else:
params = {}
noun = noun.lower()
logger.info(f"noun:{noun}")
logger.info(f"parameters:{parameters}")
logger.info(f"params:{params}")
res = os.chdir(f'{ATLAS_CFN_HOME}')
try_log_error(res, f"Changed to {ATLAS_CFN_HOME}")
if noun.startswith("all"):
logger.debug("noun was all")
res = tier_one_resources
if noun == "all+":
for r in tier_two_resources:
res.append(r)
if noun == "all-":
res = tier_two_resources
for resource in res:
command = ["python3", TOOL, f"{resource}", "--region", f"{region}"]
if parameters:
command.append(["--parameters", f"{parameters}"])
logger.debug(f"resource:{resource}")
logger.debug(f"command:{command}")
logger.debug(f"res:{res}")
res = subprocess.run(command)
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output, error = process.communicate()
if error:
logger.error(error)
logger.info(output)
else:
submit_cmd = [
"cfn", "submit", "-v", "--region", f"{region}", "--set-default"
]
cwd = os.getcwd()
try:
res_path = os.path.join(f"{ATLAS_CFN_HOME}", noun)
res = os.chdir(f'{res_path}')
try_log_error(res, f"Changed to resource: {noun} in {res_path}")
process = subprocess.Popen(["make"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output, error = process.communicate()
if error:
logger.error(error)
logger.info(output)
process = subprocess.Popen(submit_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output, error = process.communicate()
if error:
logger.error(error)
logger.info(output)
res = os.chdir(f'{cwd}')
try_log_error(res, f"Changed to directory: {cwd}")
print(res)
except Exception as f:
logger.error(f)