def run_instances(options):
layout_options = {
cli.OPTION_INFRASTRUCTURE : options.infrastructure,
cli.OPTION_DATABASE : options.database,
cli.OPTION_MIN_IMAGES : options.min,
cli.OPTION_MAX_IMAGES : options.max,
cli.OPTION_REPLICATION : options.replication,
cli.OPTION_READ_FACTOR : options.read_q,
cli.OPTION_WRITE_FACTOR : options.write_q
}
node_layout = NodeLayout(options.ips, layout_options)
if options.file is not None:
app_info = commons.get_app_info(options.file, options.database)
else:
app_info = ( 'none' )
appscale_dir = os.path.expanduser(APPSCALE_DIR)
if not os.path.exists(appscale_dir):
os.mkdir(appscale_dir)
if options.infrastructure:
cloud.validate(options.infrastructure, options.machine)
print 'Starting AppScale over', options.infrastructure
else:
print 'Starting AppScale in a non-cloud environment'
secret_key_file = os.path.join(APPSCALE_DIR, options.keyname + '.secret')
secret_key = commons.generate_secret_key(secret_key_file)
if cloud.is_valid_cloud_type(options.infrastructure):
cloud.configure_security(options.infrastructure, options.keyname,
options.group, appscale_dir)
instance_info = cloud.spawn_head_node(options.infrastructure, options.keyname,
options.group, options.machine, options.instance_type)
head_node = instance_info[0][0]
else:
head_node = node_layout.get_head_node()
instance_info = ([head_node.id], [head_node.id], ['virtual_node'])
locations = []
for i in range(len(instance_info[0])):
location = instance_info[0][i] + ':' + instance_info[1][i] + \
':' + instance_info[2][i]
locations.append(location)
named_key_loc = os.path.join(appscale_dir, options.keyname + '.key')
named_backup_key_loc = os.path.join(appscale_dir, options.keyname + '.private')
ssh_key = None
key_exists = False
for key in (named_key_loc, named_backup_key_loc):
if os.path.exists(key):
key_exists = True
if commons.is_ssh_key_valid(key, head_node.id):
ssh_key = key
break
if not key_exists:
msg = 'Unable to find a SSH key to login to AppScale nodes'
raise AppScaleToolsException(msg)
elif ssh_key is None:
msg = 'Unable to login to AppScale nodes with the available SSH keys'
raise AppScaleToolsException(msg)
# TODO: Ensure image is AppScale
if options.scp is not None:
commons.copy_appscale_source(options.scp, head_node.id, ssh_key)
remote_key_file = '/root/.appscale/%s.key' % options.keyname
commons.scp_file(ssh_key, remote_key_file, head_node.id, ssh_key)
credentials = {
'table' : options.database,
'hostname' : head_node.id,
'keyname' : options.keyname,
'keypath' : ssh_key,
'replication' : node_layout.replication,
'appengine' : options.app_engines,
'autoscale' : options.auto_scale,
'group' : options.group
}
if options.database == 'voldemort':
credentials['voldemortr'] = node_layout.read_factor
credentials['voldemortw'] = node_layout.write_factor
elif options.database == 'simpledb':
credentials['SIMPLEDB_ACCESS_KEY'] = os.environ['SIMPLEDB_ACCESS_KEY']
credentials['SIMPLEDB_SECRET_KEY'] = os.environ['SIMPLEDB_SECRET_KEY']
if cloud.is_valid_cloud_type(options.infrastructure):
cloud_credentials = cloud.get_cloud_env_variables(options.infrastructure)
for key, value in cloud_credentials.items():
credentials[key] = value
if options.restore_from_tar:
db_backup = '/root/db-backup.tar.gz'
credentials['restore_from_tar'] = db_backup
commons.scp_file(options.restore_from_tar, db_backup, head_node.id, ssh_key)
if options.restore_neptune_info:
neptune_info = '/etc/appscale/neptune_info.txt'
commons.scp_file(options.restore_neptune_info, neptune_info,
head_node.id, ssh_key)
print 'Head node successfully initialized at', head_node.id
# TODO: copy keys
god_file = '/tmp/controller.god'
commons.scp_file('resources/controller.god', god_file, head_node.id, ssh_key)
commons.run_remote_command('god load ' + god_file, head_node.id, ssh_key)
commons.run_remote_command('god start controller', head_node.id, ssh_key)
client = AppControllerClient(head_node.id, secret_key)
client.set_parameters(locations, credentials, app_info[0])
def __is_simple_format(self):
if self.yaml is None:
return is_valid_cloud_type(self.infrastructure)
else:
return set(self.yaml.keys()).issubset(self.SIMPLE_FORMAT_KEYS)
def __populate_simple_format(self):
cloud = is_valid_cloud_type(self.infrastructure)
if self.yaml is None:
if cloud:
if self.min_images is None or self.max_images is None:
raise AppScaleToolsException(
'Both min and max options must be specified when no '
'input yaml is provided', self.ERROR_MISSING_REQUIRED_OPTIONS)
else:
self.yaml = self.__generate_default_layout()
else:
raise AppScaleToolsException(
'Input yaml must be provided for deployments on virtualized clusters',
self.ERROR_MISSING_INPUT_YAML)
else:
all_ips = commons.flatten(self.yaml.values())
unique_ips = set(all_ips)
if len(all_ips) != len(unique_ips):
msg = 'Duplicate IP addresses found in input yaml'
raise AppScaleToolsException(msg, self.ERROR_DUPLICATE_IPS)
nodes = []
for role,ips in self.yaml.items():
if ips is None:
continue
if isinstance(ips, str):
ips = [ ips ]
for ip in ips:
node = SimpleAppScaleNode(ip, [role])
if node.has_role(ROLE_SHADOW):
node.add_role(ROLE_DATABASE_MASTER)
node.add_role(ROLE_RABBITMQ_MASTER)
else:
node.add_role(ROLE_DATABASE_SLAVE)
node.add_role(ROLE_RABBITMQ_SLAVE)
node.validate()
if cloud and not re.match(self.NODE_ID_REGEX, node.id):
raise AppScaleToolsException('Invalid cloud node ID: %s. '
'Cloud node IDs must be in the '
'format node-{ID}.' % node.id)
elif not cloud and not re.match(self.IP_REGEX, node.id):
raise AppScaleToolsException('Invalid virtualized node ID: %s. '
'Virtualized node IDs must be valid IP '
'addresses.' % node.id)
else:
nodes.append(node)
if len(nodes) == 1:
nodes[0].add_role(ROLE_APPENGINE)
nodes[0].add_role(ROLE_MEMCACHE)
controllers = 0
for node in nodes:
if node.has_role(ROLE_SHADOW):
controllers += 1
if controllers > 1:
raise AppScaleToolsException('Only one controller node is allowed',
self.ERROR_MULTIPLE_CONTROLLERS)
elif not controllers:
raise AppScaleToolsException('No controller node has been assigned',
self.ERROR_NO_CONTROLLER)
if not self.skip_replication:
self.__validate_data_replication(nodes)
self.nodes = nodes
def __populate_advanced_format(self):
nodes = {}
for role, ips in self.yaml.items():
if ips is None:
continue
if isinstance(ips, str):
ips = [ ips ]
index = 0
for ip in ips:
if not nodes.has_key(ip):
node = AdvancedAppScaleNode(ip)
else:
node = nodes[ip]
if role == ROLE_DATABASE:
if not index:
node.add_role(ROLE_DATABASE_MASTER)
else:
node.add_role(ROLE_DATABASE_SLAVE)
elif role == ROLE_DATABASE_MASTER:
node.add_role(ROLE_ZOOKEEPER)
node.add_role(role)
elif role == ROLE_RABBITMQ:
if not index:
node.add_role(ROLE_RABBITMQ_MASTER)
else:
node.add_role(ROLE_RABBITMQ_SLAVE)
else:
node.add_role(role)
nodes[ip] = node
index += 1
nodes = nodes.values()
cloud = is_valid_cloud_type(self.infrastructure)
for node in nodes:
if cloud and not re.match(self.NODE_ID_REGEX, node.id):
raise AppScaleToolsException('Invalid cloud node ID: %s. '
'Cloud node IDs must be in the '
'format node-{ID}.' % node.id)
elif not cloud and not re.match(self.IP_REGEX, node.id):
raise AppScaleToolsException('Invalid virtualized node ID: %s. '
'Virtualized node IDs must be valid IP '
'addresses.' % node.id)
controllers = 0
memcache_nodes = 0
zk_nodes = 0
rabbit_mq_nodes = 0
db_nodes = 0
master_node = None
login_node = None
app_engine_nodes = []
for node in nodes:
if node.has_role(ROLE_SHADOW):
master_node = node
controllers += 1
if node.has_role(ROLE_LOGIN):
login_node = node
if node.has_role(ROLE_APPENGINE):
app_engine_nodes.append(node)
if node.has_role(ROLE_MEMCACHE):
memcache_nodes += 1
if node.has_role(ROLE_ZOOKEEPER):
zk_nodes += 1
if node.has_role(ROLE_RABBITMQ):
rabbit_mq_nodes += 1
if node.has_role(ROLE_DATABASE):
db_nodes += 1
if controllers > 1:
raise AppScaleToolsException('Only one controller node is allowed')
elif not controllers:
raise AppScaleToolsException('No controller node has been assigned')
if not app_engine_nodes:
raise AppScaleToolsException('Not enough appengine nodes were provided')
if not login_node:
master_node.add_role(ROLE_LOGIN)
if not memcache_nodes:
for app_engine in app_engine_nodes:
app_engine.add_role(ROLE_MEMCACHE)
if not zk_nodes:
master_node.add_role(ROLE_ZOOKEEPER)
if not rabbit_mq_nodes:
master_node.add_role(ROLE_RABBITMQ)
master_node.add_role(ROLE_RABBITMQ_MASTER)
for node in nodes:
if node.has_role(ROLE_APPENGINE) and not node.has_role(ROLE_RABBITMQ):
node.add_role(ROLE_RABBITMQ_SLAVE)
if cloud:
if self.min_images is None: self.min_images = len(nodes)
if self.max_images is None: self.max_images = len(nodes)
if len(nodes) < self.min_images:
raise AppScaleToolsException('Too few nodes were provided. %s '
'provided, but %s is the minimum.' %
(len(nodes), self.min_images))
if len(nodes) > self.max_images:
raise AppScaleToolsException('Too many nodes were provided. %s '
'provided, but %s is the maximum.' %
(len(nodes), self.max_images))
if not self.skip_replication:
self.__validate_data_replication(nodes)
self.nodes = nodes
