def refresh(expected=None):
"""Update local cache of IPs for OpenStack instances.
This will write a `farmboy.os.yaml` file of the running instances
on OpenStack tagged with `farmboy` and their associated roles.
"""
conn = client.Client(env.farmboy_os_username,
env.farmboy_os_password,
env.farmboy_os_tenant_name,
env.farmboy_os_auth_url,
service_type='compute')
max_tries = 30
for i in range(max_tries):
util.puts('[os] Fetching running and pending instances')
# check for instances
found_instances = []
running_instances = conn.servers.list()
# TODO(termie): we _probably_ want to just terminate them all and start
# new ones, is there a use case for keeping them?
for instance in running_instances:
if (instance.metadata.get('farmboy')
and instance.status == 'ACTIVE'
and getattr(instance, 'OS-EXT-STS:task_state') == None):
found_instances.append(instance)
if not expected:
if found_instances:
break
util.puts('[os] nothing found, retrying in 5 seconds (%d of %d)'
% (i, max_tries))
else:
if found_instances and len(found_instances) == expected:
break
util.puts('[os] found %d, expecting %d, retrying in 5 seconds'
' (%d of %d)'
% (len(found_instances), expected, i, max_tries))
time.sleep(5)
if not found_instances or (expected and len(found_instances) != expected):
raise Exception('Did not find enough running instances.')
o = {'roledefs': {}}
for inst in found_instances:
role = str(inst.metadata['farmboy'])
ip_address = inst.addresses['private'][0]['addr']
util.puts('[os] found: %s (%s)' % (ip_address, role))
role_l = o['roledefs'].get(role, [])
role_l.append(str('%s@%s' % (env.farmboy_os_image_user, ip_address)))
o['roledefs'][role] = role_l
util.puts('[os] Dumping roledefs to file: %s' % DEFAULT_ROLEDEF_FILE)
util.update(o, DEFAULT_ROLEDEF_FILE)
def refresh(expected=None):
"""Update local cache of IPs for AWS instances.
This will write a `farmboy.aws.yaml` file of the running instances
on AWS tagged with `farmboy` and their associated roles.
"""
conn = ec2.connect_to_region(env.farmboy_aws_region)
max_tries = 15
for i in range(max_tries):
util.puts('[aws] Fetching running and pending instances')
running_instances = conn.get_only_instances(
filters={'tag-key': 'farmboy',
'instance-state-name': 'pending',
'instance-state-name': 'running'})
if not expected:
if running_instances:
break
util.puts('[aws] nothing found, retrying in 5 seconds (%d of %d)'
% (i, max_tries))
else:
if running_instances and len(running_instances) == expected:
break
util.puts('[aws] found %d, expecting %d, retrying in 5 seconds'
' (%d of %d)'
% (len(running_instances), expected, i, max_tries))
time.sleep(5)
if not running_instances or (expected and len(running_instances) != expected):
raise Exception('Did not find enough running instances.')
o = {'roledefs': {}}
for inst in running_instances:
role = str(inst.tags['farmboy'])
util.puts('[aws] found: %s (%s)' % (inst.ip_address, role))
role_l = o['roledefs'].get(role, [])
role_l.append(str('%s@%s' % (env.farmboy_aws_image_user, inst.ip_address)))
o['roledefs'][role] = role_l
util.puts('[aws] Dumping roledefs to file: %s' % DEFAULT_ROLEDEF_FILE)
yaml.dump(o,
stream=open(DEFAULT_ROLEDEF_FILE, 'w'),
default_flow_style=False,
indent=2,
width=72)
def terminate():
"""Terminate running AWS instances tagged with `farmboy`."""
conn = ec2.connect_to_region(env.farmboy_aws_region)
# check for instances
running_instances = conn.get_only_instances(
filters={'tag-key': 'farmboy',
'instance-state-name': 'pending',
'instance-state-name': 'running'})
# TODO(termie): we _probably_ want to just terminate them all and start
# new ones, is there a use case for keeping them?
for instance in running_instances:
util.puts('[aws] Terminating running instance: %s (%s)'
% (instance.id, instance.tags['farmboy']))
instance.terminate()
def terminate():
"""Terminate running OpenStack instances tagged with `farmboy`."""
conn = client.Client(env.farmboy_os_username,
env.farmboy_os_password,
env.farmboy_os_tenant_name,
env.farmboy_os_auth_url,
service_type='compute')
# check for instances
running_instances = conn.servers.list()
# TODO(termie): we _probably_ want to just terminate them all and start
# new ones, is there a use case for keeping them?
for instance in running_instances:
if (instance.metadata.get('farmboy')
and instance.status == 'ACTIVE'
and getattr(instance, 'OS-EXT-STS:task_state') == None):
util.puts('[os] Terminating running instance: %s (%s)'
% (instance.id, instance.metadata['farmboy']))
instance.stop()
instance.delete()
def build():
"""Launch and prepare OpenStack instances to be used by Farm Boy.
This will ensure that a security group and key pair exist.
It will also terminate any running instances tagged with `farmboy`.
"""
conn = client.Client(env.farmboy_os_username,
env.farmboy_os_password,
env.farmboy_os_tenant_name,
env.farmboy_os_auth_url,
service_type='compute')
security_group_name = env.farmboy_os_security_group
# check for security group
security_group = None
security_groups = conn.security_groups.list()
for group in security_groups:
if group.name == security_group_name:
security_group = group
if not security_group:
util.puts('[os] Creating security group: %s' % security_group)
security_group = conn.security_groups.create(
security_group_name, 'default security group for farmboy')
rules = [dict(ip_protocol='tcp',
from_port='22',
to_port='22',
cidr='0.0.0.0/0'),
dict(ip_protocol='tcp',
from_port='80',
to_port='80',
cidr='0.0.0.0/0'),
dict(ip_protocol='tcp',
from_port='8000',
to_port='9000',
cidr='0.0.0.0/0'),
dict(ip_protocol='udp',
from_port='60000',
to_port='61000',
cidr='0.0.0.0/0'),
dict(ip_protocol='tcp',
from_port='3142',
to_port='3142',
cidr='0.0.0.0/0')]
for rule in rules:
util.puts('[os] adding rule: %(ip_protocol)s:%(from_port)s-%(to_port)s'
' (%(cidr)s)' % rule)
conn.security_group_rules.create(parent_group_id=security_group.id,
**rule)
security_groups = [security_group.name]
# check for keypair
keypair = None
keypairs = conn.keypairs.list()
for pair in keypairs:
if pair.name == env.farmboy_os_keypair:
keypair = pair
if not keypair:
execute(build_keypair)
util.puts('[os] Creating keypair: %s' % env.farmboy_os_keypair)
public_key = open(env.farmboy_os_keyfile_public).read()
keypair = conn.keypairs.create(env.farmboy_os_keypair,
public_key=public_key)
execute(terminate)
machines = env.farmboy_os_instances
# launch instances with in security group, with keypair
util.puts('[os] Starting %d instances' % len(machines))
# tag the instances with their roles
for machine in machines:
inst = conn.servers.create(name='farmboy-%s' % machine,
image=env.farmboy_os_image_id,
flavor=env.farmboy_os_flavor_id,
security_groups=security_groups,
key_name=env.farmboy_os_keypair,
meta={'farmboy': machine})
util.puts('[os] started server: %s -> %s' % (inst.id, machine))
execute(refresh, expected=len(machines))
def build():
"""Launch and prepare AWS instances to be used by Farm Boy.
This will ensure that a security group and key pair exist.
It will also terminate any running instances tagged with `farmboy`.
"""
conn = ec2.connect_to_region(env.farmboy_aws_region)
security_group = env.farmboy_aws_security_group
# check for security group
try:
security_groups = conn.get_all_security_groups([security_group])
except boto.exception.EC2ResponseError:
security_groups = []
if not security_groups:
util.puts('[aws] Creating security group: %s' % security_group)
security_group = conn.create_security_group(
security_group, 'default security group for farmboy')
rules = [dict(ip_protocol='tcp',
from_port='22',
to_port='22',
cidr_ip='0.0.0.0/0'),
dict(ip_protocol='tcp',
from_port='80',
to_port='80',
cidr_ip='0.0.0.0/0'),
dict(ip_protocol='tcp',
from_port='8000',
to_port='9000',
cidr_ip='0.0.0.0/0'),
dict(ip_protocol='udp',
from_port='60000',
to_port='61000',
cidr_ip='0.0.0.0/0'),
dict(ip_protocol='tcp',
from_port='3142',
to_port='3142',
cidr_ip='0.0.0.0/0')]
for rule in rules:
util.puts('[aws] adding rule: %(ip_protocol)s:%(from_port)s-%(to_port)s'
' (%(cidr_ip)s)' % rule)
security_group.authorize(**rule)
security_groups = [security_group]
# check for keypair
key_pair = conn.get_key_pair(env.farmboy_aws_key_pair)
if not key_pair:
util.puts('[aws] Creating key pair: %s' % env.farmboy_aws_key_pair)
key_pair = conn.create_key_pair(env.farmboy_aws_key_pair)
key_pair.save('./')
execute(terminate)
machines = env.farmboy_aws_instances
# launch instances with in security group, with keypair
util.puts('[aws] Starting %d instances' % len(machines))
reservation = conn.run_instances(image_id=env.farmboy_aws_image_id,
min_count=len(machines),
max_count=len(machines),
key_name=env.farmboy_aws_key_pair,
security_groups=security_groups)
# get instances for returned reservation
new_instances = reservation.instances[:]
# tag the instances with their roles
for machine in machines:
inst = new_instances.pop()
util.puts('[aws] assigning role: %s -> %s' % (inst.id, machine))
inst.add_tag('farmboy', machine)
execute(refresh, expected=len(machines))