def create_nsg():
cmd = """
az network nsg create \
--resource-group %s \
--name %s
""" % (config["azure_cluster"]["resource_group_name"],
config["azure_cluster"]["nsg_name"])
if verbose:
print(cmd)
output = utils.exec_cmd_local(cmd)
print(output)
cmd = """
az network nsg rule create \
--resource-group %s \
--nsg-name %s \
--name allowall \
--protocol tcp \
--priority 1000 \
--destination-port-range 0-65535 \
--access allow
""" % (config["azure_cluster"]["resource_group_name"],
config["azure_cluster"]["nsg_name"])
output = utils.exec_cmd_local(cmd)
print(output)
def create_gc_vm(vmname, addrname, vmsize, location, configCluster, docreate):
print "creating VM %s, size %s ..." % (vmname, vmsize)
uselocation = get_location_string(location)
cmd = """
gcloud compute instances create %s \
--zone %s \
--machine-type %s \
%s """ \
% ( vmname, uselocation, vmsize, configCluster["vm_image"])
if addrname is not None:
cmd += " --address=%s " % addrname
if True: # verbose:
print(cmd)
output = utils.exec_cmd_local(cmd)
print(output)
sshkeyfile = setup_gce_ssh_key(location)
cmd2 = """
gcloud compute instances add-metadata %s \
--zone %s \
--metadata-from-file \
ssh-keys=%s
""" \
% ( vmname, uselocation, sshkeyfile )
output = utils.exec_cmd_local(cmd2)
return output
def create_sql():
cmd = """
az sql server create --resource-group %s \
--location %s \
--name %s \
--administrator-login %s \
--administrator-login-password %s
""" % (config["azure_cluster"]["resource_group_name"],
config["azure_cluster"]["azure_location"],
config["azure_cluster"]["sql_server_name"],
config["azure_cluster"]["sql_admin_name"],
config["azure_cluster"]["sql_admin_password"])
output = utils.exec_cmd_local(cmd)
print(output)
cmd = """
az sql server firewall create --resource-group %s \
--server %s \
--name All \
--start-ip-address 0.0.0.0 \
--end-ip-address 255.255.255.255
""" % (config["azure_cluster"]["resource_group_name"],
config["azure_cluster"]["sql_server_name"])
output = utils.exec_cmd_local(cmd)
print(output)
def create_sql():
cmd = """
az sql server create --resource-group %s \
--location %s \
--name %s \
-u %s \
-p %s
""" % (config["azure_cluster"]["resource_group_name"],
config["azure_cluster"]["azure_location"],
config["azure_cluster"]["sql_server_name"],
config["azure_cluster"]["sql_admin_name"],
config["azure_cluster"]["sql_admin_password"])
if verbose:
print(cmd)
if not no_execution:
output = utils.exec_cmd_local(cmd)
print(output)
cmd = """
az sql server firewall-rule create --resource-group %s \
--server %s \
--name All \
--start-ip-address 0.0.0.0 \
--end-ip-address 255.255.255.255
""" % (config["azure_cluster"]["resource_group_name"],
config["azure_cluster"]["sql_server_name"])
if verbose:
print(cmd)
if not no_execution:
output = utils.exec_cmd_local(cmd)
print(output)
def create_cluster(arm_vm_password=None, parallelism=1):
bSQLOnly = (config["azure_cluster"]["infra_node_num"] <= 0)
assert int(config["azure_cluster"]["nfs_node_num"]) >= len(
config["azure_cluster"]["nfs_vm"])
assert "mysql_password" in config
print "creating resource group..."
create_group()
if not bSQLOnly:
if "file_share" in config["azure_cluster"]:
print "creating storage account..."
create_storage_account()
print "creating file share..."
create_file_share()
print "creating vnet..."
create_vnet()
print "creating network security group..."
create_nsg()
create_nfs_nsg()
if useSqlAzure():
print "creating sql server and database..."
create_sql()
if arm_vm_password is not None:
# dev box, used in extreme condition when there's only one public IP available, then would use dev in cluster to bridge-connect all of them
create_vm_param(0, "dev", config["azure_cluster"]["infra_vm_size"],
True, arm_vm_password)
for i in range(int(config["azure_cluster"]["infra_node_num"])):
create_vm_param(i, "infra", config["azure_cluster"]["infra_vm_size"],
arm_vm_password is not None, arm_vm_password)
if config["priority"] == "regular":
print("entering")
if parallelism > 1:
# TODO: Tolerate faults
from multiprocessing import Pool
args_list = [
(i, "worker", config["azure_cluster"]["worker_vm_size"],
arm_vm_password is not None, arm_vm_password)
for i in range(int(config["azure_cluster"]["worker_node_num"]))
]
pool = Pool(processes=parallelism)
pool.map(create_vm_param_wrapper, args_list)
pool.close()
else:
for i in range(int(config["azure_cluster"]["worker_node_num"])):
create_vm_param(i, "worker",
config["azure_cluster"]["worker_vm_size"],
arm_vm_password is not None, arm_vm_password)
elif config["priority"] == "low":
utils.render_template("./template/vmss/vmss.sh.template",
"scripts/vmss.sh", config)
utils.exec_cmd_local("chmod +x scripts/vmss.sh;./scripts/vmss.sh")
# create nfs server if specified.
for i in range(int(config["azure_cluster"]["nfs_node_num"])):
create_vm_param(
i, "nfs", config["azure_cluster"]["nfs_vm_size"], False,
arm_vm_password, config["azure_cluster"]["nfs_vm"][i]
if i < len(config["azure_cluster"]["nfs_vm"]) else None)
def prepare_vm(docreate=True):
create_firewall(docreate)
cmd1 = "./deploy.py --verbose --sudo runscriptonall ./scripts/platform/gce/configure-vm.sh"
output1 = utils.exec_cmd_local(cmd1)
print output1
cmd2 = "./deploy.py --verbose --sudo runscriptonall ./scripts/prepare_vm_disk.sh"
output2 = utils.exec_cmd_local(cmd2)
print output2
def delete_service_accounts():
projectid = config["gs_cluster"]["project"]["id"]
admin_account = "administrator@%s.iam.gserviceaccount.com" % projectid
output1 = utils.exec_cmd_local("gcloud iam service-accounts delete %s" %
admin_account)
print output1
reader_account = "reader@%s.iam.gserviceaccount.com" % projectid
output2 = utils.exec_cmd_local("gcloud iam service-accounts delete %s" %
reader_account)
print output2
def delete_firewall(docreate=True):
cmd = """
gcloud compute firewall-rules delete tcp80 \
"""
output = utils.exec_cmd_local(cmd)
print output
cmd1 = """
gcloud compute firewall-rules delete allow-all \
"""
output1 = utils.exec_cmd_local(cmd1)
print output1
def create_nfs_nsg():
if "source_addresses_prefixes" in config["cloud_config"]["dev_network"]:
source_addresses_prefixes = config["cloud_config"][
"dev_network"]["source_addresses_prefixes"]
else:
print "Please setup source_addresses_prefixes in config.yaml, otherwise, your cluster cannot be accessed"
exit()
cmd = """
az network nsg create \
--resource-group %s \
--name %s
""" % ( config["azure_cluster"]["resource_group_name"],
config["azure_cluster"]["nfs_nsg_name"])
if verbose:
print(cmd)
if not no_execution:
output = utils.exec_cmd_local(cmd)
print(output)
print type(config["cloud_config"]["nfs_ssh"]["source_ips"]), config["cloud_config"]["nfs_ssh"]["source_ips"],type(source_addresses_prefixes), source_addresses_prefixes
cmd = """
az network nsg rule create \
--resource-group %s \
--nsg-name %s \
--name allow_ssh\
--priority 900 \
--destination-port-ranges %s \
--source-address-prefixes %s \
--access allow
""" % ( config["azure_cluster"]["resource_group_name"],
config["azure_cluster"]["nfs_nsg_name"],
config["cloud_config"]["nfs_ssh"]["port"],
" ".join(config["cloud_config"]["nfs_ssh"]["source_ips"] + source_addresses_prefixes),
)
if not no_execution:
output = utils.exec_cmd_local(cmd)
print(output)
cmd = """
az network nsg rule create \
--resource-group %s \
--nsg-name %s \
--name allow_share \
--priority 1000 \
--source-address-prefixes %s \
--destination-port-ranges \'*\' \
--access allow
""" % ( config["azure_cluster"]["resource_group_name"],
config["azure_cluster"]["nfs_nsg_name"],
" ".join(config["cloud_config"]["nfs_share"]["source_ips"]),
)
if not no_execution:
output = utils.exec_cmd_local(cmd)
print(output)
def check_subscription():
chkcmd = "az account list | grep -A5 -B5 '\"isDefault\": true'"
output = utils.exec_cmd_local(chkcmd).decode()
if not config["azure_cluster"]["subscription"] in output:
setcmd = "az account set --subscription \"{}\"".format(
config["azure_cluster"]["subscription"])
setout = utils.exec_cmd_local(setcmd)
print "Set your subscription to {}, please login.\nIf you want to specify another subscription, please configure azure_cluster.subscription".format(
config["azure_cluster"]["subscription"])
utils.exec_cmd_local("az login")
assert config["azure_cluster"]["subscription"] in utils.exec_cmd_local(
chkcmd).decode()
def set_subscription(config):
if "subscription" not in config["azure_cluster"]:
print("No subscription to set")
return
subscription = config["azure_cluster"]["subscription"]
chkcmd = "az account list | grep -A5 -B5 '\"isDefault\": true'"
output = exec_cmd_local(chkcmd)
if not subscription in output:
setcmd = "az account set --subscription \"{}\"".format(subscription)
setout = exec_cmd_local(setcmd)
assert subscription in exec_cmd_local(chkcmd, True)
def create_firewall(docreate=True):
cmd = """
gcloud compute firewall-rules create tcp80 \
--allow tcp:80\
"""
output = utils.exec_cmd_local(cmd)
print output
cmd1 = """
gcloud compute firewall-rules create allow-all \
--allow tcp:0-65535\
"""
output1 = utils.exec_cmd_local(cmd1)
print output1
def create_security_group(config, cluster_name, location):
group_name = get_security_group_name(cluster_name, location)
cmd = """
aws ec2 create-security-group --group-name %s --description "security group for cluster %s"
aws ec2 authorize-security-group-ingress --group-name %s --protocol tcp --port 0-65535 --cidr 0.0.0.0/0
""" \
% ( group_name, cluster_name, group_name )
output = utils.exec_cmd_local(cmd)
output1 = utils.exec_cmd_local(
"aws ec2 describe-security-groups --group-name %s " % group_name)
if location not in config["aws_cluster"]:
config["aws_cluster"][location] = {}
merge_config(config["aws_cluster"][location], json.loads(output1))
print config["aws_cluster"][location]["SecurityGroups"]
def get_aws_ssh_key_v0(location):
cmd1 = "cd ./deploy/sshkey/; ssh-keygen -f id_rsa.pub -e -m pem"
keyinfo = utils.exec_cmd_local(cmd1)
print keyinfo
keylines = keyinfo.splitlines()
sshkey = "".join(keylines[1:-1])
return sshkey
def nfs_allow_master():
vminfo = list_vm(False)
source_address_prefixes = []
for name, onevm in vminfo.iteritems():
if "-infra" in name:
source_address_prefixes.append(onevm["publicIps"] + "/32")
source_address_prefixes = " ".join(source_address_prefixes)
nsg_names = [config["azure_cluster"]["nfs_nsg_name"]]
if "custom_nfs_nsg_names" in config["azure_cluster"]:
if isinstance(config["azure_cluster"]["custom_nfs_nsg_names"], list):
for nsg_name in config["azure_cluster"]["custom_nfs_nsg_names"]:
nsg_names.append(nsg_name)
for nsg_name in nsg_names:
cmd = """
az network nsg rule create \
--resource-group %s \
--nsg-name %s \
--name nfs_allow_master \
--protocol tcp \
--priority 1400 \
--destination-port-ranges %s \
--source-address-prefixes %s \
--access allow
""" % (config["azure_cluster"]["resource_group_name"],
nsg_name, config["cloud_config"]["nfs_allow_master"]
["tcp_port_ranges"], source_address_prefixes)
if verbose:
print cmd
output = utils.exec_cmd_local(cmd)
print(output)
def create_vm(vmname, vm_ip, bIsWorker):
vm_size = config["azure_cluster"]["worker_vm_size"] if bIsWorker else config["azure_cluster"]["infra_vm_size"]
cmd = """
az vm create --resource-group %s \
--name %s \
--image %s \
--generate-ssh-keys \
--private-ip-address %s \
--public-ip-address-dns-name %s \
--location %s \
--size %s \
--vnet-name %s \
--subnet mySubnet \
--nsg %s \
--admin-username %s \
--storage-sku %s \
--ssh-key-value "%s"
""" % (config["azure_cluster"]["resource_group_name"],
vmname,
config["azure_cluster"]["vm_image"],
vm_ip,
vmname,
config["azure_cluster"]["azure_location"],
vm_size,
config["azure_cluster"]["vnet_name"],
config["azure_cluster"]["nsg_name"],
config["cloud_config"]["default_admin_username"],
config["azure_cluster"]["vm_storage_sku"],
config["azure_cluster"]["sshkey"])
# Try remove static IP
# --public-ip-address-allocation static \
if verbose:
print(cmd)
output = utils.exec_cmd_local(cmd)
print (output)
def create_group():
cmd = """
az group create --name %s --location %s
""" % (config["azure_cluster"]["resource_group_name"],
config["azure_cluster"]["azure_location"])
output = utils.exec_cmd_local(cmd)
print(output)
def vm_interconnects():
vminfo = list_vm(False)
ports = []
for name, onevm in vminfo.iteritems():
ports.append(onevm["publicIps"] + "/32")
portinfo = " ".join(ports)
cmd = """
az network nsg rule create \
--resource-group %s \
--nsg-name %s \
--name tcpinterconnect \
--protocol tcp \
--priority 850 \
--destination-port-range %s \
--source-address-prefixes %s \
--access allow
""" % ( config["azure_cluster"]["resource_group_name"],
config["azure_cluster"]["nsg_name"],
config["cloud_config"]["dev_network"]["tcp_port_ranges"],
portinfo
)
if verbose:
print cmd
output = utils.exec_cmd_local(cmd)
print(output)
def whitelist_source_address_prefixes(config):
resource_group = config["azure_cluster"]["resource_group"]
nsg_name = config["azure_cluster"]["nsg_name"]
cmd = """
az network nsg rule show \
--resource-group %s \
--nsg-name %s \
--name whitelist
""" % (resource_group, nsg_name)
output = exec_cmd_local(cmd)
source_address_prefixes = []
try:
data = json.loads(output)
source = data.get("sourceAddressPrefix")
if source is not None and source != "":
source_address_prefixes.append(source)
sources = data.get("sourceAddressPrefixes")
if sources is not None:
source_address_prefixes += sources
except Exception:
print("Exception when parsing whitelist response. "
"Ignore existing whitelist")
return source_address_prefixes
def create_vm(vmname):
cmd = """
az vm create --resource-group %s \
--name %s \
--image %s \
--generate-ssh-keys \
--public-ip-address-dns-name %s \
--location %s \
--size %s \
--vnet-name %s \
--subnet mySubnet \
--nsg %s \
--public-ip-address-allocation static \
--os-type linux \
--admin-username %s \
--storage-sku %s \
--ssh-key-value "%s"
""" % (config["azure_cluster"]["resource_group_name"], vmname,
config["azure_cluster"]["vm_image"], vmname,
config["azure_cluster"]["azure_location"],
config["azure_cluster"]["vm_size"],
config["azure_cluster"]["vnet_name"],
config["azure_cluster"]["nsg_name"],
config["azure_cluster"]["default_admin_username"],
config["azure_cluster"]["vm_storage_sku"],
config["azure_cluster"]["sshkey"])
output = utils.exec_cmd_local(cmd)
print(output)
def gen_cluster_config(output_file_name):
cmd = """
az storage account show-connection-string \
-n %s \
-g %s \
--query 'connectionString' \
-o tsv
""" % (config["azure_cluster"]["storage_account_name"],
config["azure_cluster"]["resource_group_name"])
output = utils.exec_cmd_local(cmd)
reoutput = re.search('AccountKey\=.*$', output)
file_share_key = None
if reoutput is not None:
file_share_key = reoutput.group(0).replace("AccountKey=", "")
cc = {}
cc["cluster_name"] = config["azure_cluster"]["cluster_name"]
cc["etcd_node_num"] = config["azure_cluster"]["infra_node_num"]
cc["sqlserver-hostname"] = "tcp:%s.database.windows.net" % config[
"azure_cluster"]["sql_server_name"]
cc["sqlserver-username"] = config["azure_cluster"]["sql_admin_name"]
cc["sqlserver-password"] = config["azure_cluster"]["sql_admin_password"]
cc["sqlserver-database"] = config["azure_cluster"]["sql_database_name"]
cc["admin_username"] = config["azure_cluster"]["default_admin_username"]
cc["useclusterfile"] = True
cc["deploydockerETCD"] = False
cc["platform-scripts"] = "ubuntu"
cc["basic_auth"] = "%s,admin,1000" % uuid.uuid4().hex[:7]
cc["network"] = {
"domain":
"%s.cloudapp.azure.com" % config["azure_cluster"]["azure_location"]
}
cc["machines"] = {}
for i in range(int(config["azure_cluster"]["infra_node_num"])):
vmname = "%s-infra%02d" % (config["azure_cluster"]["cluster_name"],
i + 1)
cc["machines"][vmname] = {"role": "infrastructure"}
for i in range(int(config["azure_cluster"]["worker_node_num"])):
vmname = "%s-worker%02d" % (config["azure_cluster"]["cluster_name"],
i + 1)
cc["machines"][vmname] = {"role": "worker"}
cc["WinbindServers"] = []
cc["WebUIauthorizedGroups"] = ['MicrosoftUsers']
cc["mountpoints"] = {"rootshare": {}}
cc["mountpoints"]["rootshare"]["type"] = "azurefileshare"
cc["mountpoints"]["rootshare"]["accountname"] = config["azure_cluster"][
"storage_account_name"]
cc["mountpoints"]["rootshare"]["filesharename"] = config["azure_cluster"][
"file_share_name"]
cc["mountpoints"]["rootshare"]["mountpoints"] = ""
if file_share_key is not None:
cc["mountpoints"]["rootshare"]["accesskey"] = file_share_key
print yaml.dump(cc, default_flow_style=False)
with open(output_file_name, 'w') as outfile:
yaml.dump(cc, outfile, default_flow_style=False)
def delete_group():
cmd = """
az group delete -y --name %s
""" % (config["azure_cluster"]["resource_group_name"])
if verbose:
print(cmd)
output = utils.exec_cmd_local(cmd)
print (output)
def delete_storage_account(name, sku, location):
actual_location = get_region_string(location)
actual_sku = get_sku(sku)
if actual_location is not None:
print "name == %s, sku == %s, location == %s" % (name, actual_sku,
actual_location)
cmd = """
gsutil rm -r gs://%s """ % (name)
if verbose:
print(cmd)
output = utils.exec_cmd_local(cmd)
print(output)
cmd2 = """
gcloud compute backend-buckets delete %s-bucket \
""" % (name)
output2 = utils.exec_cmd_local(cmd2)
print(output2)
def import_key_pair(location):
cmd = """
aws ec2 import-key-pair --key-name publickey-%s \
--public-key-material file://./deploy/sshkey/id_rsa.pub \
""" \
% ( location )
output = utils.exec_cmd_local(cmd)
print output
def delete_security_group(cluster_name, location):
group_name = get_security_group_name(cluster_name, location)
cmd = """
aws ec2 delete-security-group --group-name %s
""" \
% ( group_name )
output = utils.exec_cmd_local(cmd)
print group_name
def delete_address_location(location):
useloc = get_region_string(location)
addrname = get_address_name(location)
infra_node_num = get_num_infra_nodes(location)
for cnt in range(infra_node_num):
addrcur = get_infra_address(location, cnt)
cmd = """
gcloud compute addresses delete %s \
--region %s \
""" \
% ( addrcur, useloc)
utils.exec_cmd_local(cmd)
if "addresses" not in config["gs_cluster"]:
config["gs_cluster"]["addresses"] = {}
config["gs_cluster"]["addresses"][location] = None
# print (config["gs_cluster"])
save_config()
def delete_nic(nicname):
cmd = """
az network nic delete --resource-group %s \
--name %s \
""" % (config["azure_cluster"]["resource_group_name"], nicname)
if verbose:
print(cmd)
output = utils.exec_cmd_local(cmd)
print(output)
def get_vm_private_ip():
cmd = """
az vm list-ip-addresses -g %s --output json --query '[].{name:virtualMachine.name, privateIP:virtualMachine.network.privateIpAddresses}'
""" % (config["azure_cluster"]["resource_group_name"])
if verbose:
print(cmd)
output = utils.exec_cmd_local(cmd)
return utils.json_loads_byteified(output)
def get_disk_from_vm(vmname):
cmd = """
az vm show -g %s -n %s --query "storageProfile.osDisk.managedDisk.id" -o tsv \
""" % (config["azure_cluster"]["resource_group_name"], vmname)
if verbose:
print(cmd)
output = utils.exec_cmd_local(cmd)
return output.split("/")[-1].strip('\n')
def delete_public_ip(ip):
cmd = """
az network public-ip delete --resource-group %s \
--name %s \
""" % (config["azure_cluster"]["resource_group_name"], ip)
if verbose:
print(cmd)
output = utils.exec_cmd_local(cmd)
print(output)
