def do_slurm_suspend(args):
log.debug(f"reading config file ({args.config_file})")
c = azconfig.ConfigFile()
c.open(args.config_file)
config = c.preprocess()
log.info(f"slurm suspend for {args.nodes}")
_, resource_list = _nodelist_expand(args.nodes)
log.debug("suspend list expanded to: " + ",".join(resource_list))
subscription_id = azutil.get_subscription_id()
resource_group = config["resource_group"]
vm_ids = []
nic_ids = []
disk_ids = []
for resource in resource_list:
vm_ids.append(
f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.Compute/virtualMachines/{resource}"
)
nic_ids.append(
f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.Network/networkInterfaces/{resource}_nic"
)
disk_ids.append(
f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.Compute/disks/{resource}_osdisk"
)
# delete vms
log.debug("deleting vms: " + ",".join(vm_ids))
azutil.delete_resources(vm_ids)
# delete nics and disks
log.debug("deleting nics and disks: " + ",".join(nic_ids + disk_ids))
azutil.delete_resources(nic_ids + disk_ids)
log.debug("exiting do_slurm_suspend")
def do_run_install(args):
c = azconfig.ConfigFile()
c.open(args.config_file)
config = c.preprocess()
tmpdir = "azhpc_install_" + os.path.basename(args.config_file)[:-5]
log.debug(f"tmpdir = {tmpdir}")
if os.path.isdir(tmpdir):
log.debug("removing existing tmp directory")
shutil.rmtree(tmpdir)
adminuser = config["admin_user"]
private_key_file = adminuser + "_id_rsa"
public_key_file = adminuser + "_id_rsa.pub"
start_step = args.step
log.info("building host lists")
azinstall.generate_hostlists(config, tmpdir)
log.info("building install scripts")
azinstall.generate_install(config, tmpdir, adminuser, private_key_file,
public_key_file)
resource_group = c.read_value("resource_group")
fqdn = c.get_install_from_destination()
log.debug(f"running script from : {fqdn}")
azinstall.run(config, tmpdir, adminuser, private_key_file, public_key_file,
fqdn, start_step)
def do_scp(args):
log.debug("reading config file ({})".format(args.config_file))
c = azconfig.ConfigFile()
c.open(args.config_file)
adminuser = c.read_value("admin_user")
sshkey = "{}_id_rsa".format(adminuser)
# TODO: check ssh key exists
fqdn = c.get_install_from_destination()
if not fqdn:
log.error(f"Missing 'install_from' property")
sys.exit(1)
if args.args and args.args[0] == "--":
scp_args = args.args[1:]
else:
scp_args = args.args
scp_exe = "scp"
scp_cmd = [
scp_exe, "-q", "-o", "StrictHostKeyChecking=no", "-o",
"UserKnownHostsFile=/dev/null", "-i", sshkey, "-o",
f"ProxyCommand=ssh -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {sshkey} -W %h:%p {adminuser}@{fqdn}"
] + scp_args
log.debug(" ".join([f"'{a}'" for a in scp_cmd]))
os.execvp(scp_exe, scp_cmd)
def do_scp(args):
log.debug("reading config file ({})".format(args.config_file))
c = azconfig.ConfigFile()
c.open(args.config_file)
adminuser = c.read_value("admin_user")
sshkey = "{}_id_rsa".format(adminuser)
# TODO: check ssh key exists
jumpbox = c.read_value("install_from")
rg = c.read_value("resource_group")
fqdn = azutil.get_fqdn(rg, jumpbox + "pip")
if args.args and args.args[0] == "--":
scp_args = args.args[1:]
else:
scp_args = args.args
scp_exe = "scp"
scp_cmd = [
scp_exe, "-q", "-o", "StrictHostKeyChecking=no", "-o",
"UserKnownHostsFile=/dev/null", "-i", sshkey, "-o",
f"ProxyCommand=ssh -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {sshkey} -W %h:%p {adminuser}@{fqdn}"
] + scp_args
log.debug(" ".join([f"'{a}'" for a in scp_cmd]))
os.execvp(scp_exe, scp_cmd)
def do_build(args):
log.debug(f"reading config file ({args.config_file})")
tmpdir = "azhpc_install_" + os.path.basename(args.config_file)[:-5]
log.debug(f"tmpdir = {tmpdir}")
if os.path.isdir(tmpdir):
log.debug("removing existing tmp directory")
shutil.rmtree(tmpdir)
c = azconfig.ConfigFile()
c.open(args.config_file)
config = c.preprocess()
adminuser = config["admin_user"]
private_key_file = adminuser + "_id_rsa"
public_key_file = adminuser + "_id_rsa.pub"
_create_private_key(private_key_file, public_key_file)
tpl = arm.ArmTemplate()
tpl.read(config, not args.no_vnet)
output_template = "deploy_" + args.config_file
log.info("writing out arm template to " + output_template)
with open(output_template, "w") as f:
f.write(tpl.to_json())
log.info("creating resource group " + config["resource_group"])
resource_tags = config.get("resource_tags", {})
azutil.create_resource_group(
config["resource_group"], config["location"],
[{
"key": "CreatedBy",
"value": os.getenv("USER")
}, {
"key": "CreatedOn",
"value": datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
}] + [{
"key": key,
"value": resource_tags[key]
} for key in resource_tags.keys()])
log.info("deploying arm template")
deployname = azutil.deploy(config["resource_group"], output_template)
log.debug(f"deployment name: {deployname}")
_wait_for_deployment(config["resource_group"], deployname)
log.info("building host lists")
azinstall.generate_hostlists(config, tmpdir)
log.info("building install scripts")
azinstall.generate_install(config, tmpdir, adminuser, private_key_file,
public_key_file)
resource_group = c.read_value("resource_group")
fqdn = c.get_install_from_destination()
log.debug(f"running script from : {fqdn}")
azinstall.run(config, tmpdir, adminuser, private_key_file, public_key_file,
fqdn)
def do_init(args):
if not os.path.exists(args.config_file):
log.error("config file/dir does not exist")
sys.exit(1)
if args.show:
vlist = set()
if os.path.isfile(args.config_file):
__add_unset_vars(vlist, args.config_file)
else:
for root, dirs, files in os.walk(args.config_file):
for name in files:
if os.path.splitext(name)[1] == ".json":
__add_unset_vars(vlist, os.path.join(root, name))
print("Variables to set: " + ",".join(vlist))
print()
print("Example string for '--vars' argument (add values):")
print(" --vars " + ",".join([x + "=" for x in vlist]))
else:
log.debug("creating directory")
os.makedirs(args.dir, exist_ok=True)
if os.path.isfile(args.config_file):
shutil.copy(args.config_file, args.dir)
elif os.path.isdir(args.config_file):
for root, dirs, files in os.walk(args.config_file):
for d in dirs:
newdir = os.path.join(
args.dir,
os.path.relpath(os.path.join(root, d),
args.config_file))
log.debug("creating directory: " + newdir)
os.makedirs(newdir, exist_ok=True)
for f in files:
oldfile = os.path.join(root, f)
newfile = os.path.join(
args.dir,
os.path.relpath(os.path.join(root, f),
args.config_file))
log.debug(f"copying file: {oldfile} -> {newfile}")
shutil.copy(oldfile, newfile, follow_symlinks=False)
# get vars
vset = {}
if args.vars:
for vp in args.vars.split(","):
vk, vv = vp.split("=", 1)
vset[vk] = vv
for root, dirs, files in os.walk(args.dir):
for name in files:
if os.path.splitext(name)[1] == ".json":
config = azconfig.ConfigFile()
config.open(os.path.join(root, name))
config.replace_vars(vset)
config.save(os.path.join(root, name))
def do_slurm_suspend(args):
log.debug(f"reading config file ({args.config_file})")
c = azconfig.ConfigFile()
c.open(args.config_file)
config = c.preprocess()
log.info(f"slurm suspend for {args.nodes}")
# first get the resource name
all_resources = config.get("resources", [])
resource_name, brackets = re.search(r'([^[]*)\[?([\d\-\,]*)\]?',
args.nodes).groups(0)
resource_list = []
if bool(brackets):
for part in brackets.split(","):
if "-" in part:
lo, hi = part.split("-")
assert len(lo) == 4, "expecting number width of 4"
assert len(hi) == 4, "expecting number width of 4"
for i in range(int(lo), int(hi) + 1):
resource_list.append(f"{resource_name}{i:04d}")
else:
assert len(part) == 4, "expecting number width of 4"
resource_list.append(f"{resource_name}{part}")
else:
resource_list.append(resource_name)
resource_name = resource_name[:-4]
subscription_id = azutil.get_subscription_id()
resource_group = config["resource_group"]
vm_ids = []
nic_ids = []
disk_ids = []
for resource in resource_list:
vm_ids.append(
f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.Compute/virtualMachines/{resource}"
)
nic_ids.append(
f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.Network/networkInterfaces/{resource}_nic"
)
disk_ids.append(
f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.Compute/disks/{resource}_osdisk"
)
# delete vms
log.debug("deleting vms: " + ",".join(vm_ids))
azutil.delete_resources(vm_ids)
# delete nics and disks
log.debug("deleting nics and disks: " + ",".join(nic_ids + disk_ids))
azutil.delete_resources(nic_ids + disk_ids)
log.debug("exiting do_slurm_suspend")
def do_destroy(args):
log.info("reading config file ({})".format(args.config_file))
config = azconfig.ConfigFile()
config.open(args.config_file)
log.warning("deleting entire resource group ({})".format(
config.read_value("resource_group")))
if not args.force:
log.info("you have 10s to change your mind and ctrl-c!")
time.sleep(10)
log.info("too late!")
azutil.delete_resource_group(config.read_value("resource_group"),
args.no_wait)
def do_get(args):
config = azconfig.ConfigFile()
config.open(args.config_file)
log.debug(f"azhpc get for {args.path}")
processed_val = config.process_value(args.path)
log.debug(f"processed value is {processed_val}")
read_val = config.read_value(processed_val)
log.debug(f"read value is {read_val}")
if read_val or args.path == processed_val:
# use the read value result if valid or if processed value is the same as the original
val = read_val
else:
val = processed_val
print(f"{args.path} = {val}")
def do_status(args):
log.debug("reading config file ({})".format(args.config_file))
c = azconfig.ConfigFile()
c.open(args.config_file)
adminuser = c.read_value("admin_user")
ssh_private_key="{}_id_rsa".format(adminuser)
fqdn = c.get_install_from_destination()
if not fqdn:
log.error(f"Missing 'install_from' property")
sys.exit(1)
tmpdir = "azhpc_install_" + os.path.basename(args.config_file).strip(".json")
_exec_command(fqdn, adminuser, ssh_private_key, f"pssh -h {tmpdir}/hostlists/linux -i -t 0 'printf \"%-20s%s\n\" \"$(hostname)\" \"$(uptime)\"' | grep -v SUCCESS")
def do_run(args):
log.debug("reading config file ({})".format(args.config_file))
c = azconfig.ConfigFile()
c.open(args.config_file)
adminuser = c.read_value("admin_user")
ssh_private_key = "{}_id_rsa".format(adminuser)
# TODO: check ssh key exists
if args.user == None:
sshuser = adminuser
else:
sshuser = args.user
jumpbox = c.read_value("install_from")
if not jumpbox:
log.error(f"Missing 'install_from' property")
sys.exit(1)
# TODO : Why is this unused ?
resource_group = c.read_value("resource_group")
fqdn = c.get_install_from_destination()
hosts = []
if args.nodes:
for r in args.nodes.split(","):
rtype = c.read_value(f"resources.{r}.type")
if not rtype:
log.debug(f"resource {r} does not exist in config")
hosts.append(r)
if rtype == "vm":
instances = c.read_value(f"resources.{r}.instances", 1)
if instances == 1:
hosts.append(r)
else:
hosts += [f"{r}{n:04}" for n in range(1, instances + 1)]
elif rtype == "vmss":
hosts += azutil.get_vmss_instances(
c.read_value("resource_group"), r)
if not hosts:
hosts.append(jumpbox)
hostlist = " ".join(hosts)
cmd = " ".join(args.args)
_exec_command(fqdn, sshuser, ssh_private_key,
f"pssh -H '{hostlist}' -i -t 0 '{cmd}'")
def do_run(args):
log.debug("reading config file ({})".format(args.config_file))
c = azconfig.ConfigFile()
c.open(args.config_file)
adminuser = c.read_value("admin_user")
ssh_private_key = "{}_id_rsa".format(adminuser)
# TODO: check ssh key exists
if args.user == None:
sshuser = adminuser
else:
sshuser = args.user
jumpbox = c.read_value("install_from")
resource_group = c.read_value("resource_group")
fqdn = azutil.get_fqdn(resource_group, jumpbox + "pip")
if fqdn == "":
log.warning(
"The install node does not have a public IP - trying hostname ({})"
.format(jumpbox))
hosts = []
if args.nodes:
for r in args.nodes.split(" "):
rtype = c.read_value(f"resources.{r}.type", None)
if not rtype:
log.error(f"resource {r} does not exist in config")
sys.exit(1)
if rtype == "vm":
instances = c.read_value(f"resources.{r}.instances", 1)
if instances == 1:
hosts.append(r)
else:
hosts += [f"{r}{n:04}" for n in range(1, instances + 1)]
elif rtype == "vmss":
hosts += azutil.get_vmss_instances(
c.read_value("resource_group"), r)
if not hosts:
hosts.append(jumpbox)
hostlist = " ".join(hosts)
cmd = " ".join(args.args)
_exec_command(fqdn, sshuser, ssh_private_key,
f"pssh -H '{hostlist}' -i -t 0 '{cmd}'")
def do_status(args):
log.debug("reading config file ({})".format(args.config_file))
c = azconfig.ConfigFile()
c.open(args.config_file)
adminuser = c.read_value("admin_user")
ssh_private_key = "{}_id_rsa".format(adminuser)
jumpbox = c.read_value("install_from")
resource_group = c.read_value("resource_group")
fqdn = azutil.get_fqdn(resource_group, jumpbox + "pip")
if fqdn == "":
log.warning(
"The install node does not have a public IP - trying hostname ({})"
.format(jumpbox))
tmpdir = "azhpc_install_" + os.path.basename(
args.config_file).strip(".json")
_exec_command(
fqdn, adminuser, ssh_private_key,
f"pssh -h {tmpdir}/hostlists/linux -i -t 0 'printf \"%-20s%s\n\" \"$(hostname)\" \"$(uptime)\"' | grep -v SUCCESS"
)
def do_slurm_resume(args):
log.debug(f"reading config file ({args.config_file})")
while True:
timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tmpdir = "azhpc_install_" + os.path.basename(
args.config_file)[:-5] + "_" + timestamp
if not os.path.isdir(tmpdir):
break
log.warning(
f"{tmpdir} already exists, sleeping for 5 seconds and retrying")
time.sleep(5)
log.debug(f"tmpdir = {tmpdir}")
c = azconfig.ConfigFile()
c.open(args.config_file)
config_orig = c.preprocess()
adminuser = config_orig["admin_user"]
private_key_file = adminuser + "_id_rsa"
public_key_file = adminuser + "_id_rsa.pub"
log.info(f"slurm resume for {args.nodes}")
# first get the resource name
resource_names, resource_list = _nodelist_expand(args.nodes)
# Create a copy of the configuration to use as template
# for the final deployment configuration
config = copy.deepcopy(config_orig)
config["resources"] = {}
# Loop over all resources
for resource in resource_names:
template_resource = config_orig.get("resources", {}).get(resource)
if not template_resource:
log.error(f"{resource} resource not found in config")
sys.exit(1)
if template_resource.get("type") != "slurm_partition":
log.error(f"invalid resource type for scaling")
template_resource["type"] = "vm"
del template_resource["instances"]
log.info(f"resource= {resource}")
log.info("resource_list= " + ",".join(resource_list))
# Iterate over all nodes which name starts with the resource name
# NOTE: It is assumed that in the nodename the resource name is separated
# by a hyphen from the node index!
for rname in filter(lambda x: x.rsplit('-', 1)[0] == resource,
resource_list):
config["resources"][rname] = template_resource
tpl = arm.ArmTemplate()
tpl.read_resources(config, False)
output_template = f"deploy_{args.config_file}_{timestamp}"
log.info("writing out arm template to " + output_template)
with open(output_template, "w") as f:
f.write(tpl.to_json())
log.info("deploying arm template")
deployname = azutil.deploy(config["resource_group"], output_template)
log.debug(f"deployment name: {deployname}")
_wait_for_deployment(config["resource_group"], deployname)
# remove local scripts
config["install"] = [
step for step in config["install"]
if step.get("type", "jumpbox_script") == "jumpbox_script"
]
log.info("building host lists")
azinstall.generate_hostlists(config, tmpdir)
log.info("building install scripts")
azinstall.generate_install(config, tmpdir, adminuser, private_key_file,
public_key_file)
if socket.gethostname() == config["install_from"]:
fqdn = config["install_from"]
else:
fqdn = c.get_install_from_destination()
log.debug(f"running script from : {fqdn}")
azinstall.run(config, tmpdir, adminuser, private_key_file, public_key_file,
fqdn)
def do_slurm_resume(args):
log.debug(f"reading config file ({args.config_file})")
while True:
timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tmpdir = "azhpc_install_" + os.path.basename(
args.config_file)[:-5] + "_" + timestamp
if not os.path.isdir(tmpdir):
break
log.warning(
f"{tmpdir} already exists, sleeping for 5 seconds and retrying")
time.sleep(5)
log.debug(f"tmpdir = {tmpdir}")
c = azconfig.ConfigFile()
c.open(args.config_file)
config = c.preprocess()
adminuser = config["admin_user"]
private_key_file = adminuser + "_id_rsa"
public_key_file = adminuser + "_id_rsa.pub"
log.info(f"slurm resume for {args.nodes}")
# first get the resource name
all_resources = config.get("resources", [])
resource_name, brackets = re.search(r'([^[]*)\[?([\d\-\,]*)\]?',
args.nodes).groups(0)
resource_list = []
if bool(brackets):
for part in brackets.split(","):
if "-" in part:
lo, hi = part.split("-")
assert len(lo) == 4, "expecting number width of 4"
assert len(hi) == 4, "expecting number width of 4"
for i in range(int(lo), int(hi) + 1):
resource_list.append(f"{resource_name}{i:04d}")
else:
assert len(part) == 4, "expecting number width of 4"
resource_list.append(f"{resource_name}{part}")
else:
resource_list.append(resource_name)
resource_name = resource_name[:-4]
template_resource = config.get("resources", {}).get(resource_name)
if not template_resource:
log.error(f"${res} resource not found in config")
sys.exit(1)
if template_resource.get("type") != "slurm_partition":
log.error(f"invalid resource type for scaling")
template_resource["type"] = "vm"
del template_resource["instances"]
log.info(f"resource_name= {resource_name}")
log.info("resource_list= " + ",".join(resource_list))
config["resources"] = {}
for rname in resource_list:
config["resources"][rname] = template_resource
tpl = arm.ArmTemplate()
tpl.read_resources(config, False)
output_template = f"deploy_{args.config_file}_{timestamp}"
log.info("writing out arm template to " + output_template)
with open(output_template, "w") as f:
f.write(tpl.to_json())
log.info("deploying arm template")
deployname = azutil.deploy(config["resource_group"], output_template)
log.debug(f"deployment name: {deployname}")
_wait_for_deployment(config["resource_group"], deployname)
log.info("building host lists")
azinstall.generate_hostlists(config, tmpdir)
log.info("building install scripts")
azinstall.generate_install(config, tmpdir, adminuser, private_key_file,
public_key_file)
jumpbox = c.read_value("install_from")
resource_group = c.read_value("resource_group")
fqdn = c.get_install_from_destination()
log.debug(f"running script from : {fqdn}")
azinstall.run(config, tmpdir, adminuser, private_key_file, public_key_file,
fqdn)
def do_preprocess(args):
log.debug("reading config file ({})".format(args.config_file))
config = azconfig.ConfigFile()
config.open(args.config_file)
print(json.dumps(config.preprocess(), indent=4))
def do_connect(args):
log.debug("reading config file ({})".format(args.config_file))
c = azconfig.ConfigFile()
c.open(args.config_file)
adminuser = c.read_value("admin_user")
ssh_private_key = "{}_id_rsa".format(adminuser)
# TODO: check ssh key exists
if not args.user:
sshuser = adminuser
else:
sshuser = args.user
jumpbox = c.read_value("install_from")
if not jumpbox:
log.error(f"Missing 'install_from' property")
sys.exit(1)
resource_group = c.read_value("resource_group")
fqdn = c.get_install_from_destination()
log.debug("Getting resource name")
rtype = c.read_value(f"resources.{args.resource}.type", "hostname")
rimage = c.read_value(f"resources.{args.resource}.image", "hostname")
log.debug(f"image is - {rimage}")
target = args.resource
if rtype == "vm":
instances = c.read_value(f"resources.{args.resource}.instances", 1)
if instances > 1:
target = f"{args.resource}{1:04}"
log.info(
f"Multiple instances of {args.resource}, connecting to {target}"
)
elif rtype == "vmss":
vmssnodes = azutil.get_vmss_instances(resource_group, args.resource)
if len(vmssnodes) == 0:
log.error("There are no instances in the vmss")
sys.exit(1)
target = vmssnodes[0]
if len(vmssnodes) > 1:
log.info(
f"Multiple instances of {args.resource}, connecting to {target}"
)
elif rtype == "hostname":
pass
else:
log.debug(f"Unknown resource type - {rtype}")
sys.exit(1)
ros = rimage.split(':')
if ros[0] == "MicrosoftWindowsServer" or ros[
0] == "MicrosoftWindowsDesktop":
log.debug(f"os is - {ros[0]} for node {args.resource}")
fqdn = azutil.get_fqdn(c.read_value("resource_group"),
args.resource + "_pip")
winpassword = c.read_value("variables.win_password")
log.debug(f"fqdn is {fqdn} for node {args.resource}")
cmdkey_exe = "cmdkey.exe"
mstsc_exe = "mstsc.exe"
cmdline = []
if len(args.args) > 0:
cmdline.append(" ".join(args.args))
cmdkey_args = [
"cmdkey.exe", f"/generic:{fqdn}", f"/user:{sshuser}",
f"/password:{winpassword}"
]
mstsc_args = ["mstsc.exe", f"/v:{fqdn}"]
log.debug(" ".join(cmdkey_args + cmdline))
cmdkey_cmdline = " ".join(cmdkey_args)
os.system(cmdkey_cmdline)
log.debug(" ".join(mstsc_args + cmdline))
os.execvp(mstsc_exe, mstsc_args)
else:
ssh_exe = "ssh"
cmdline = []
if len(args.args) > 0:
cmdline.append(" ".join(args.args))
if args.resource == jumpbox:
log.info("logging directly into {}".format(fqdn))
ssh_args = [
"ssh", "-t", "-q", "-o", "StrictHostKeyChecking=no", "-o",
"UserKnownHostsFile=/dev/null", "-i", ssh_private_key,
f"{sshuser}@{fqdn}"
]
log.debug(" ".join(ssh_args + cmdline))
os.execvp(ssh_exe, ssh_args + cmdline)
else:
log.info("logging in to {} (via {})".format(target, fqdn))
ssh_args = [
ssh_exe, "-t", "-q", "-o", "StrictHostKeyChecking=no", "-o",
"UserKnownHostsFile=/dev/null", "-i", ssh_private_key, "-o",
f"ProxyCommand=ssh -i {ssh_private_key} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p {sshuser}@{fqdn}",
f"{sshuser}@{target}"
]
log.debug(" ".join(ssh_args + cmdline))
os.execvp(ssh_exe, ssh_args + cmdline)
def setUp(self):
self.config = azconfig.ConfigFile()
self.config.open("test/test_config_file.json")
def do_build(args):
log.debug(f"reading config file ({args.config_file})")
tmpdir = "azhpc_install_" + os.path.basename(
args.config_file).strip(".json")
log.debug(f"tmpdir = {tmpdir}")
if os.path.isdir(tmpdir):
log.debug("removing existing tmp directory")
shutil.rmtree(tmpdir)
c = azconfig.ConfigFile()
c.open(args.config_file)
config = c.preprocess()
adminuser = config["admin_user"]
private_key_file = adminuser + "_id_rsa"
public_key_file = adminuser + "_id_rsa.pub"
if not (os.path.exists(private_key_file)
and os.path.exists(public_key_file)):
# create ssh keys
key = rsa.generate_private_key(backend=crypto_default_backend(),
public_exponent=65537,
key_size=2048)
private_key = key.private_bytes(
crypto_serialization.Encoding.PEM,
crypto_serialization.PrivateFormat.TraditionalOpenSSL,
crypto_serialization.NoEncryption())
public_key = key.public_key().public_bytes(
crypto_serialization.Encoding.OpenSSH,
crypto_serialization.PublicFormat.OpenSSH)
with open(private_key_file, "wb") as f:
os.chmod(private_key_file, 0o600)
f.write(private_key)
with open(public_key_file, "wb") as f:
os.chmod(public_key_file, 0o644)
f.write(public_key + b'\n')
tpl = arm.ArmTemplate()
tpl.read(config)
log.info("writing out arm template to " + args.output_template)
with open(args.output_template, "w") as f:
f.write(tpl.to_json())
log.info("creating resource group " + config["resource_group"])
resource_tags = config.get("resource_tags", {})
azutil.create_resource_group(
config["resource_group"], config["location"],
[{
"key": "CreatedBy",
"value": os.getenv("USER")
}, {
"key": "CreatedOn",
"value": datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
}] + [{
"key": key,
"value": resource_tags[key]
} for key in resource_tags.keys()])
log.info("deploying arm template")
deployname = azutil.deploy(config["resource_group"], args.output_template)
log.debug(f"deployment name: {deployname}")
building = True
success = True
del_lines = 1
while building:
time.sleep(5)
res = azutil.get_deployment_status(config["resource_group"],
deployname)
log.debug(res)
print("\033[F" * del_lines)
del_lines = 1
for i in res:
props = i["properties"]
status_code = props["statusCode"]
if props.get("targetResource", None):
resource_name = props["targetResource"]["resourceName"]
resource_type = props["targetResource"]["resourceType"]
del_lines += 1
print(
f"{resource_name:15} {resource_type:47} {status_code:15}")
else:
provisioning_state = props["provisioningState"]
del_lines += 1
building = False
if provisioning_state != "Succeeded":
success = False
if success:
log.info("Provising succeeded")
else:
log.error("Provisioning failed")
for i in res:
props = i["properties"]
status_code = props["statusCode"]
if props.get("targetResource", None):
resource_name = props["targetResource"]["resourceName"]
if props.get("statusMessage", None):
if "error" in props["statusMessage"]:
error_code = props["statusMessage"]["error"]["code"]
error_message = textwrap.TextWrapper(width=60).wrap(
text=props["statusMessage"]["error"]["message"])
error_target = props["statusMessage"]["error"].get(
"target", None)
error_target_str = ""
if error_target:
error_target_str = f"({error_target})"
print(
f" Resource : {resource_name} - {error_code} {error_target_str}"
)
print(f" Message : {error_message[0]}")
for line in error_message[1:]:
print(f" {line}")
sys.exit(1)
log.info("building host lists")
azinstall.generate_hostlists(config, tmpdir)
log.info("building install scripts")
azinstall.generate_install(config, tmpdir, adminuser, private_key_file,
public_key_file)
jumpbox = config.get("install_from", None)
fqdn = None
if jumpbox:
fqdn = azutil.get_fqdn(config["resource_group"], jumpbox + "pip")
log.info("running install scripts")
azinstall.run(config, tmpdir, adminuser, private_key_file,
public_key_file, fqdn)
else:
log.info("nothing to install ('install_from' is not set)")
def do_connect(args):
log.debug("reading config file ({})".format(args.config_file))
c = azconfig.ConfigFile()
c.open(args.config_file)
adminuser = c.read_value("admin_user")
ssh_private_key = "{}_id_rsa".format(adminuser)
# TODO: check ssh key exists
if args.user == None:
sshuser = adminuser
else:
sshuser = args.user
jumpbox = c.read_value("install_from")
resource_group = c.read_value("resource_group")
fqdn = azutil.get_fqdn(resource_group, jumpbox + "pip")
if fqdn == "":
log.warning(
f"The install node does not have a public IP - trying hostname ({jumpbox})"
)
log.debug("Getting resource name")
rtype = c.read_value(f"resources.{args.resource}.type", "hostname")
target = args.resource
if rtype == "vm":
instances = c.read_value(f"resources.{args.resource}.instances", 1)
if instances > 1:
target = f"{args.resource}{1:04}"
log.info(
f"Multiple instances of {args.resource}, connecting to {target}"
)
elif rtype == "vmss":
vmssnodes = azutil.get_vmss_instances(resource_group, args.resource)
if len(vmssnodes) == 0:
log.error("There are no instances in the vmss")
sys.exit(1)
target = vmssnodes[0]
if len(vmssnodes) > 1:
log.info(
f"Multiple instances of {args.resource}, connecting to {target}"
)
elif rtype == "hostname":
pass
else:
log.debug(f"Unknown resource type - {rtype}")
sys.exit(1)
ssh_exe = "ssh"
cmdline = []
if len(args.args) > 0:
cmdline.append(" ".join(args.args))
if args.resource == jumpbox:
log.info("logging directly into {}".format(fqdn))
ssh_args = [
"ssh", "-t", "-q", "-o", "StrictHostKeyChecking=no", "-o",
"UserKnownHostsFile=/dev/null", "-i", ssh_private_key,
f"{sshuser}@{fqdn}"
]
log.debug(" ".join(ssh_args + cmdline))
os.execvp(ssh_exe, ssh_args + cmdline)
else:
log.info("logging in to {} (via {})".format(target, fqdn))
ssh_args = [
ssh_exe, "-t", "-q", "-o", "StrictHostKeyChecking=no", "-o",
"UserKnownHostsFile=/dev/null", "-i", ssh_private_key, "-o",
f"ProxyCommand=ssh -i {ssh_private_key} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p {sshuser}@{fqdn}",
f"{sshuser}@{target}"
]
log.debug(" ".join(ssh_args + cmdline))
os.execvp(ssh_exe, ssh_args + cmdline)
def do_get(args):
config = azconfig.ConfigFile()
config.open(args.config_file)
val = config.read_value(args.path)
print(f"{args.path} = {val}")
def __add_unset_vars(vset, config_file):
log.debug(f"looking for vars in {config_file}")
config = azconfig.ConfigFile()
config.open(config_file)
vset.update(config.get_unset_vars())
