def install_themis(
config, provider, remote_address, version, github_URL, github_key,
scp_themis, skip_dependencies, skip_configuration):
provider_info = authenticate(provider, config)
private_key = provider_info["private_key"]
remote_user = provider_info["remote_username"]
# Open a connection to the remote VM.
print "Connecting to %s..." % remote_address
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(
remote_address, username=remote_user, key_filename=private_key)
# Upload all install files with scp.
run_remote_command("mkdir -p ~/install/", ssh_client)
print "Uploading install files..."
for install_file in INSTALL_FILES[provider]:
scp["-i"][private_key][install_file]\
["%s@%s:~/install/" % (remote_user, remote_address)]()
# Install dependencies for Themis
if not skip_dependencies:
print "Installing dependencies..."
run_remote_command(
"~/install/install_dependencies.sh", ssh_client, streaming=True)
# Set up static system configuration for Themis
if not skip_configuration:
print "Configuring system..."
run_remote_command(
"~/install/configure_system.sh ~/install/on_startup.sh "
"~/install/motd-banner %s %s" % (version, github_URL), ssh_client,
streaming=True)
if not scp_themis and github_key != None:
# User wants to pre-install Themis.
# Upload key
scp["-i"][private_key][github_key]\
["%s@%s:~/install/" % (remote_user, remote_address)]()
key_basename = os.path.basename(github_key)
# Preinstall themis using this key
run_remote_command(
"~/install/preinstall_themis.sh ~/install/%s" % key_basename,
ssh_client, streaming=True)
if scp_themis != None:
run_remote_command("mkdir -p ~/themis/src", ssh_client, streaming=True)
scp["-i"][private_key]["-r"]["%s/." % os.path.join(scp_themis, "src")]\
["%s@%s:~/themis/src" % (remote_user, remote_address)]()
# Preinstall themis without using the github key
run_remote_command(
"~/install/preinstall_themis.sh",
ssh_client, streaming=True)
ssh_client.close()
def install_themis(config, provider, remote_address, version, github_URL,
github_key, skip_dependencies, skip_configuration):
provider_info = authenticate(provider, config)
private_key = provider_info["private_key"]
remote_user = provider_info["remote_username"]
# Open a connection to the remote VM.
print "Connecting to %s..." % remote_address
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(remote_address,
username=remote_user,
key_filename=private_key)
# Upload all install files with scp.
run_remote_command("mkdir -p ~/install/", ssh_client)
print "Uploading install files..."
for install_file in INSTALL_FILES[provider]:
scp["-i"][private_key][install_file]\
["%s@%s:~/install/" % (remote_user, remote_address)]()
# Install dependencies for Themis
if not skip_dependencies:
print "Installing dependencies..."
run_remote_command("~/install/install_dependencies.sh",
ssh_client,
streaming=True)
# Set up static system configuration for Themis
if not skip_configuration:
print "Configuring system..."
run_remote_command(
"~/install/configure_system.sh ~/install/on_startup.sh "
"~/install/motd-banner %s %s" % (version, github_URL),
ssh_client,
streaming=True)
if github_key != None:
# User wants to pre-install Themis.
# Upload key
scp["-i"][private_key][github_key]\
["%s@%s:~/install/" % (remote_user, remote_address)]()
key_basename = os.path.basename(github_key)
# Preinstall themis using this key
run_remote_command("~/install/preinstall_themis.sh ~/install/%s" %
key_basename,
ssh_client,
streaming=True)
ssh_client.close()
def main():
parser = argparse.ArgumentParser(
description="Launch a Themis cluster on EC2")
parser.add_argument("config", help="Cloud provider config file")
parser.add_argument("cluster_name", help="Unique cluster name")
parser.add_argument("cluster_size",
type=int,
help="The number of worker nodes")
parser.add_argument("instance_type",
help="VM instance type of the worker nodes")
parser.add_argument("AMI_ID", help="Amazon Machine Image")
parser.add_argument("master_instance_type",
help="VM instance type of the master node")
parser.add_argument("subnet_ID", help="Subnet IDS for launch")
parser.add_argument("security_group_ID", help="Security Group ID")
parser.add_argument(
"S3_bucket", help="S3 bucket to use for storing configuration files")
parser.add_argument("private_key", help="Private key file for ssh")
parser.add_argument("public_key", help="Public key file for ssh")
parser.add_argument("themis_config_directory",
help="Local directory containing Themis "
"config files to upload to S3.")
parser.add_argument("--placement_group",
help="The optional placement group to use")
parser.add_argument("--EBS_optimized",
action="store_true",
default=False,
help="Launch VMs with EBS optimization on")
parser.add_argument(
"--username",
default="ec2-user",
help="Username to use for logging into EC2. Default %(default)s")
utils.add_redis_params(parser)
args = parser.parse_args()
provider_info = authenticate("amazon", args.config)
redis_client = redis.StrictRedis(host=args.redis_host,
port=args.redis_port,
db=args.redis_db)
return launch_amazon_cluster(
provider_info, args.cluster_name, args.cluster_size,
args.instance_type, args.AMI_ID, args.master_instance_type,
args.subnet_ID, args.security_group_ID, args.S3_bucket,
args.private_key, args.public_key, args.themis_config_directory,
args.placement_group, args.EBS_optimized, args.username, redis_client)
def main():
parser = argparse.ArgumentParser(
description="Launch a Themis cluster on EC2")
parser.add_argument("config", help="Cloud provider config file")
parser.add_argument("cluster_name", help="Unique cluster name")
parser.add_argument(
"cluster_size", type=int, help="The number of worker nodes")
parser.add_argument(
"instance_type", help="VM instance type of the worker nodes")
parser.add_argument("AMI_ID", help="Amazon Machine Image")
parser.add_argument(
"master_instance_type", help="VM instance type of the master node")
parser.add_argument(
"subnet_ID", help="Subnet IDS for launch")
parser.add_argument("security_group_ID", help="Security Group ID")
parser.add_argument(
"S3_bucket", help="S3 bucket to use for storing configuration files")
parser.add_argument(
"private_key", help="Private key file for ssh")
parser.add_argument(
"public_key", help="Public key file for ssh")
parser.add_argument(
"themis_config_directory", help="Local directory containing Themis "
"config files to upload to S3.")
parser.add_argument(
"--placement_group", help="The optional placement group to use")
parser.add_argument(
"--EBS_optimized", action="store_true", default=False,
help="Launch VMs with EBS optimization on")
parser.add_argument(
"--username", default="ec2-user",
help="Username to use for logging into EC2. Default %(default)s")
utils.add_redis_params(parser)
args = parser.parse_args()
provider_info = authenticate("amazon", args.config)
redis_client = redis.StrictRedis(
host=args.redis_host, port=args.redis_port, db=args.redis_db)
return launch_amazon_cluster(
provider_info, args.cluster_name, args.cluster_size, args.instance_type,
args.AMI_ID, args.master_instance_type, args.subnet_ID,
args.security_group_ID, args.S3_bucket, args.private_key,
args.public_key, args.themis_config_directory, args.placement_group,
args.EBS_optimized, args.username, redis_client)
def main():
parser = argparse.ArgumentParser(
description="Launch a Themis cluster on Google")
parser.add_argument("config", help="Cloud provider config file")
parser.add_argument("cluster_name", help="Unique cluster name")
parser.add_argument("cluster_size",
type=int,
help="The number of worker nodes")
parser.add_argument("instance_type",
help="VM instance type of the worker nodes")
parser.add_argument("local_ssds",
help="Number of local SSDs to add to each node",
type=int)
parser.add_argument("persistent_ssds",
help="Number of persistent SSDs to add to each node",
type=int)
parser.add_argument("image", help="Google Cloud Compute Engine VM Image")
parser.add_argument("master_instance_type",
help="VM instance type of the master node")
parser.add_argument("network", help="Network to run in")
parser.add_argument("zone", help="Compute Engine Zone (eg. us-central1-f)")
parser.add_argument(
"bucket", help="Storage bucket to use for storing configuration files")
parser.add_argument("private_key", help="Private key file for ssh")
parser.add_argument("public_key", help="Public key file for ssh")
parser.add_argument("themis_config_directory",
help="Local directory containing Themis "
"config files to upload to Storage.")
utils.add_redis_params(parser)
args = parser.parse_args()
provider_info = authenticate("google", args.config)
redis_client = redis.StrictRedis(host=args.redis_host,
port=args.redis_port,
db=args.redis_db)
return launch_google_cluster(args.cluster_name, args.cluster_size,
args.instance_type, args.local_ssds,
args.persistent_ssds, args.image,
args.master_instance_type, args.network,
args.zone, args.bucket, args.private_key,
args.public_key, args.themis_config_directory,
provider_info, redis_client)
def main():
parser = argparse.ArgumentParser(description="Launch a Themis cluster on Google")
parser.add_argument("config", help="Cloud provider config file")
parser.add_argument("cluster_name", help="Unique cluster name")
parser.add_argument("cluster_size", type=int, help="The number of worker nodes")
parser.add_argument("instance_type", help="VM instance type of the worker nodes")
parser.add_argument("local_ssds", help="Number of local SSDs to add to each node", type=int)
parser.add_argument("persistent_ssds", help="Number of persistent SSDs to add to each node", type=int)
parser.add_argument("image", help="Google Cloud Compute Engine VM Image")
parser.add_argument("master_instance_type", help="VM instance type of the master node")
parser.add_argument("network", help="Network to run in")
parser.add_argument("zone", help="Compute Engine Zone (eg. us-central1-f)")
parser.add_argument("bucket", help="Storage bucket to use for storing configuration files")
parser.add_argument("private_key", help="Private key file for ssh")
parser.add_argument("public_key", help="Public key file for ssh")
parser.add_argument(
"themis_config_directory", help="Local directory containing Themis " "config files to upload to Storage."
)
utils.add_redis_params(parser)
args = parser.parse_args()
provider_info = authenticate("google", args.config)
redis_client = redis.StrictRedis(host=args.redis_host, port=args.redis_port, db=args.redis_db)
return launch_google_cluster(
args.cluster_name,
args.cluster_size,
args.instance_type,
args.local_ssds,
args.persistent_ssds,
args.image,
args.master_instance_type,
args.network,
args.zone,
args.bucket,
args.private_key,
args.public_key,
args.themis_config_directory,
provider_info,
redis_client,
)
def main():
global provider_info, redis_client
parser = argparse.ArgumentParser(
description="Run a web-interface for provisioning cloud clusters")
parser.add_argument("config", help="Cloud provider config file")
utils.add_redis_params(parser)
parser.add_argument("--port",
"-p",
help="port on which the GUI accepts HTTP connections",
type=int,
default=4281)
args = parser.parse_args()
redis_client = redis.StrictRedis(host=args.redis_host,
port=args.redis_port,
db=args.redis_db)
# Test the connection to redis to fail early if redis isn't running.
clusters = redis_client.smembers("clusters")
# Perform Amazon configuration
amazon_info = authenticate("amazon", args.config)
if amazon_info != None:
print "Fetching Amazon provider information..."
provider_info["amazon"] = amazon_info
aws = plumbum.local["aws"]
# Fetch EC2 configuration information.
# Since these commands take some time we'll run them in the background
subnet_cmd = aws["--profile"]["themis"]["ec2"]["describe-subnets"] & BG
placement_group_cmd =\
aws["--profile"]["themis"]["ec2"]["describe-placement-groups"] & BG
security_group_cmd =\
aws["--profile"]["themis"]["ec2"]["describe-security-groups"] & BG
AMI_cmd =\
aws["--profile"]["themis"]["ec2"]["describe-images"]\
["--owners"]["self"] & BG
S3_cmd = aws["--profile"]["themis"]["s3api"]["list-buckets"] & BG
print "Gathering information for subnets..."
stdout = wait_on_background_command(subnet_cmd)
result = json.loads(stdout)
subnets = result["Subnets"]
subnets = [(x["SubnetId"], x["AvailabilityZone"]) for x in subnets]
provider_info["amazon"]["subnets"] = subnets
print "Gathering information for placement groups..."
stdout = wait_on_background_command(placement_group_cmd)
result = json.loads(stdout)
placement_groups = result["PlacementGroups"]
placement_groups = [x["GroupName"] for x in placement_groups]
provider_info["amazon"]["placement_groups"] = placement_groups
print "Gathering information for security groups..."
stdout = wait_on_background_command(security_group_cmd)
result = json.loads(stdout)
security_groups = result["SecurityGroups"]
security_groups = [(x["GroupName"], x["GroupId"])
for x in security_groups]
provider_info["amazon"]["security_groups"] = security_groups
print "Gathering information for AMIs..."
stdout = wait_on_background_command(AMI_cmd)
result = json.loads(stdout)
images = result["Images"]
HVM_images = [(x["Name"], x["ImageId"]) for x in images \
if x["VirtualizationType"] == "hvm"]
PV_images = [(x["Name"], x["ImageId"]) for x in images \
if x["VirtualizationType"] == "paravirtual"]
provider_info["amazon"]["HVM_images"] = HVM_images
provider_info["amazon"]["PV_images"] = PV_images
print "Gathering information for S3 buckets..."
stdout = wait_on_background_command(S3_cmd)
result = json.loads(stdout)
buckets = result["Buckets"]
buckets = [x["Name"] for x in buckets]
provider_info["amazon"]["buckets"] = buckets
# Load instance type and device information
print "Gathering information for instance types..."
parser = ConfigParser.SafeConfigParser()
parser.read(INSTANCE_TYPE_CONFIG)
device_map = {}
instances = []
for instance_type, num_devices in parser.items("devices"):
device_map[instance_type] = int(num_devices)
instances.append(instance_type)
provider_info["amazon"]["instances"] = instances
provider_info["amazon"]["device_map"] = device_map
vm_type_map = {}
for instance_type, vm_type in parser.items("vm_type"):
vm_type_map[instance_type] = vm_type
provider_info["amazon"]["vm_type_map"] = vm_type_map
placement_groups_map = {}
for instance_type, placement_groups_enabled in parser.items(
"placement_groups"):
placement_groups_map[instance_type] = placement_groups_enabled
provider_info["amazon"]["placement_groups_map"] = placement_groups_map
ebs_optimized_map = {}
for instance_type, ebs_optimized in parser.items("EBS_optimized"):
ebs_optimized_map[instance_type] = ebs_optimized
provider_info["amazon"]["ebs_optimized_map"] = ebs_optimized_map
# Perform Google configuration
google_info = authenticate("google", args.config)
if google_info != None:
print "Fetching Google provider information..."
provider_info["google"] = google_info
gcloud = plumbum.local["gcloud"]
gsutil = plumbum.local["gsutil"]
# Get list of zones
print "Retrieving zone information..."
zones = gcloud["compute"]["zones"]["list"]\
["--format"]["json"]()
zones = json.loads(zones)
zones = [x["name"] for x in zones]
if len(zones) == 0:
print >> sys.stderr, "Found no zones"
sys.exit(1)
provider_info["google"]["zones"] = zones
print "Retrieving network information..."
networks = gcloud["compute"]["networks"]["list"]["--format"]["json"]()
networks = json.loads(networks)
networks = [x["name"] for x in networks]
provider_info["google"]["networks"] = networks
print "Retrieving image information"
images = gcloud["compute"]["images"]["list"]["--no-standard-images"]\
["--format"]["json"]()
images = json.loads(images)
images = [x["name"] for x in images]
provider_info["google"]["images"] = images
print "Retrieving storage bucket information"
buckets = gsutil["ls"]()
buckets = buckets.split("\n")
buckets = [bucket for bucket in buckets if len(bucket) > 0]
buckets = [
bucket.split("gs://")[1].split("/")[0] for bucket in buckets
]
provider_info["google"]["buckets"] = buckets
print "Retrieving instance type information"
instances = gcloud["compute"]["machine-types"]["list"]\
["--format"]["json"]()
instances = json.loads(instances)
instances = [x["name"] for x in instances]
instances = list(set(instances))
instances.sort()
provider_info["google"]["instances"] = instances
try:
bottle.run(host='0.0.0.0', port=args.port)
except socket.error, e:
print e
# Return error 42 to indicate that we can't bind, so that scripts
# calling this one can handle that case specially
return constants.CANNOT_BIND
def main():
global provider_info, redis_client
parser = argparse.ArgumentParser(
description="Run a web-interface for provisioning cloud clusters")
parser.add_argument("config", help="Cloud provider config file")
utils.add_redis_params(parser)
parser.add_argument(
"--port", "-p", help="port on which the GUI accepts HTTP connections",
type=int, default=4281)
args = parser.parse_args()
redis_client = redis.StrictRedis(
host=args.redis_host, port=args.redis_port, db=args.redis_db)
# Test the connection to redis to fail early if redis isn't running.
clusters = redis_client.smembers("clusters")
# Perform Amazon configuration
amazon_info = authenticate("amazon", args.config)
if amazon_info != None:
print "Fetching Amazon provider information..."
provider_info["amazon"] = amazon_info
aws = plumbum.local["aws"]
# Fetch EC2 configuration information.
# Since these commands take some time we'll run them in the background
subnet_cmd = aws["--profile"]["themis"]["ec2"]["describe-subnets"] & BG
placement_group_cmd =\
aws["--profile"]["themis"]["ec2"]["describe-placement-groups"] & BG
security_group_cmd =\
aws["--profile"]["themis"]["ec2"]["describe-security-groups"] & BG
AMI_cmd =\
aws["--profile"]["themis"]["ec2"]["describe-images"]\
["--owners"]["self"] & BG
S3_cmd = aws["--profile"]["themis"]["s3api"]["list-buckets"] & BG
print "Gathering information for subnets..."
stdout = wait_on_background_command(subnet_cmd)
result = json.loads(stdout)
subnets = result["Subnets"]
subnets = [(x["SubnetId"], x["AvailabilityZone"]) for x in subnets]
provider_info["amazon"]["subnets"] = subnets
print "Gathering information for placement groups..."
stdout = wait_on_background_command(placement_group_cmd)
result = json.loads(stdout)
placement_groups = result["PlacementGroups"]
placement_groups = [x["GroupName"] for x in placement_groups]
provider_info["amazon"]["placement_groups"] = placement_groups
print "Gathering information for security groups..."
stdout = wait_on_background_command(security_group_cmd)
result = json.loads(stdout)
security_groups = result["SecurityGroups"]
security_groups = [(x["GroupName"], x["GroupId"]) for x in security_groups]
provider_info["amazon"]["security_groups"] = security_groups
print "Gathering information for AMIs..."
stdout = wait_on_background_command(AMI_cmd)
result = json.loads(stdout)
images = result["Images"]
HVM_images = [(x["Name"], x["ImageId"]) for x in images \
if x["VirtualizationType"] == "hvm"]
PV_images = [(x["Name"], x["ImageId"]) for x in images \
if x["VirtualizationType"] == "paravirtual"]
provider_info["amazon"]["HVM_images"] = HVM_images
provider_info["amazon"]["PV_images"] = PV_images
print "Gathering information for S3 buckets..."
stdout = wait_on_background_command(S3_cmd)
result = json.loads(stdout)
buckets = result["Buckets"]
buckets = [x["Name"] for x in buckets]
provider_info["amazon"]["buckets"] = buckets
# Load instance type and device information
print "Gathering information for instance types..."
parser = ConfigParser.SafeConfigParser()
parser.read(INSTANCE_TYPE_CONFIG)
device_map = {}
instances = []
for instance_type, num_devices in parser.items("devices"):
device_map[instance_type] = int(num_devices)
instances.append(instance_type)
provider_info["amazon"]["instances"] = instances
provider_info["amazon"]["device_map"] = device_map
vm_type_map = {}
for instance_type, vm_type in parser.items("vm_type"):
vm_type_map[instance_type] = vm_type
provider_info["amazon"]["vm_type_map"] = vm_type_map
placement_groups_map = {}
for instance_type, placement_groups_enabled in parser.items("placement_groups"):
placement_groups_map[instance_type] = placement_groups_enabled
provider_info["amazon"]["placement_groups_map"] = placement_groups_map
ebs_optimized_map = {}
for instance_type, ebs_optimized in parser.items("EBS_optimized"):
ebs_optimized_map[instance_type] = ebs_optimized
provider_info["amazon"]["ebs_optimized_map"] = ebs_optimized_map
# Perform Google configuration
google_info = authenticate("google", args.config)
if google_info != None:
print "Fetching Google provider information..."
provider_info["google"] = google_info
gcloud = plumbum.local["gcloud"]
gsutil = plumbum.local["gsutil"]
# Get list of zones
print "Retrieving zone information..."
zones = gcloud["compute"]["zones"]["list"]\
["--format"]["json"]()
zones = json.loads(zones)
zones = [x["name"] for x in zones]
if len(zones) == 0:
print >>sys.stderr, "Found no zones"
sys.exit(1)
provider_info["google"]["zones"] = zones
print "Retrieving network information..."
networks = gcloud["compute"]["networks"]["list"]["--format"]["json"]()
networks = json.loads(networks)
networks = [x["name"] for x in networks]
provider_info["google"]["networks"] = networks
print "Retrieving image information"
images = gcloud["compute"]["images"]["list"]["--no-standard-images"]\
["--format"]["json"]()
images = json.loads(images)
images = [x["name"] for x in images]
provider_info["google"]["images"] = images
print "Retrieving storage bucket information"
buckets = gsutil["ls"]()
buckets = buckets.split("\n")
buckets = [bucket for bucket in buckets if len(bucket) > 0]
buckets = [bucket.split("gs://")[1].split("/")[0] for bucket in buckets]
provider_info["google"]["buckets"] = buckets
print "Retrieving instance type information"
instances = gcloud["compute"]["machine-types"]["list"]\
["--format"]["json"]()
instances = json.loads(instances)
instances = [x["name"] for x in instances]
instances = list(set(instances))
instances.sort()
provider_info["google"]["instances"] = instances
try:
bottle.run(host='0.0.0.0', port=args.port)
except socket.error, e:
print e
# Return error 42 to indicate that we can't bind, so that scripts
# calling this one can handle that case specially
return constants.CANNOT_BIND
