def get_ami_id(self, params):
ami = params.get('ami', None)
if isinstance(ami, str) or isinstance(ami, unicode):
for a in self.ec2.get_all_images():
if a.id == ami:
params['ami'] = a
if not params.get('ami', None):
prop = StringProperty(name='ami',
verbose_name='AMI',
choices=self.get_ami_list)
params['ami'] = propget.get(prop)
def get_key(self, params):
keypair = params.get('keypair', None)
if isinstance(keypair, basestring):
key_list = self.ec2.get_all_key_pairs()
for k in key_list:
if k.name == keypair:
keypair = k.name
params['keypair'] = k.name
if not keypair:
prop = StringProperty(name='keypair', verbose_name='EC2 KeyPair',
choices=self.ec2.get_all_key_pairs)
params['keypair'] = propget.get(prop).name
def get_group(self, params):
group = params.get('group', None)
if isinstance(group, basestring):
group_list = self.ec2.get_all_security_groups()
for g in group_list:
if g.name == group:
group = g
params['group'] = g
if not group:
prop = StringProperty(name='group', verbose_name='EC2 Security Group',
choices=self.ec2.get_all_security_groups)
params['group'] = propget.get(prop)
def get_ami_id(self, params):
valid = False
while not valid:
ami = params.get('ami', None)
if not ami:
prop = StringProperty(name='ami', verbose_name='AMI')
ami = propget.get(prop)
try:
rs = self.ec2.get_all_images([ami])
if len(rs) == 1:
valid = True
params['ami'] = rs[0]
except EC2ResponseError:
pass
def get_zone(self, params):
if not params.get('zone', None):
prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone',
choices=self.ec2.get_all_zones)
params['zone'] = propget.get(prop)
def get_instance_type(self, params):
if not params.get('instance_type', None):
prop = StringProperty(name='instance_type', verbose_name='Instance Type',
choices=InstanceTypes)
params['instance_type'] = propget.get(prop)
class Task(Model):
"""
A scheduled, repeating task that can be executed by any participating servers.
The scheduling is similar to cron jobs. Each task has an hour attribute.
The allowable values for hour are [0-23|*].
To keep the operation reasonably efficient and not cause excessive polling,
the minimum granularity of a Task is hourly. Some examples:
hour='*' - the task would be executed each hour
hour='3' - the task would be executed at 3AM GMT each day.
"""
name = StringProperty()
hour = StringProperty(required=True, validator=check_hour, default='*')
command = StringProperty(required=True)
last_executed = DateTimeProperty()
last_status = IntegerProperty()
last_output = StringProperty()
message_id = StringProperty()
@classmethod
def start_all(cls, queue_name):
for task in cls.all():
task.start(queue_name)
def __init__(self, id=None, **kw):
Model.__init__(self, id, **kw)
self.hourly = self.hour == '*'
self.daily = self.hour != '*'
self.now = datetime.datetime.utcnow()
def check(self):
"""
Determine how long until the next scheduled time for a Task.
Returns the number of seconds until the next scheduled time or zero
if the task needs to be run immediately.
If it's an hourly task and it's never been run, run it now.
If it's a daily task and it's never been run and the hour is right, run it now.
"""
boto.log.info('checking Task[%s]-now=%s, last=%s' %
(self.name, self.now, self.last_executed))
if self.hourly and not self.last_executed:
return 0
if self.daily and not self.last_executed:
if int(self.hour) == self.now.hour:
return 0
else:
return max((int(self.hour) - self.now.hour),
(self.now.hour - int(self.hour))) * 60 * 60
delta = self.now - self.last_executed
if self.hourly:
if delta.seconds >= 60 * 60:
return 0
else:
return 60 * 60 - delta.seconds
else:
if int(self.hour) == self.now.hour:
if delta.days >= 1:
return 0
else:
return 82800 # 23 hours, just to be safe
else:
return max((int(self.hour) - self.now.hour),
(self.now.hour - int(self.hour))) * 60 * 60
def _run(self, msg, vtimeout):
boto.log.info('Task[%s] - running:%s' % (self.name, self.command))
log_fp = StringIO.StringIO()
process = subprocess.Popen(self.command,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
nsecs = 5
current_timeout = vtimeout
while process.poll() == None:
boto.log.info('nsecs=%s, timeout=%s' % (nsecs, current_timeout))
if nsecs >= current_timeout:
current_timeout += vtimeout
boto.log.info('Task[%s] - setting timeout to %d seconds' %
(self.name, current_timeout))
if msg:
msg.change_visibility(current_timeout)
time.sleep(5)
nsecs += 5
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info('Task[%s] - output: %s' % (self.name, log_fp.getvalue()))
self.last_executed = self.now
self.last_status = process.returncode
self.last_output = log_fp.getvalue()[0:1023]
def run(self, msg, vtimeout=60):
delay = self.check()
boto.log.info('Task[%s] - delay=%s seconds' % (self.name, delay))
if delay == 0:
self._run(msg, vtimeout)
queue = msg.queue
new_msg = queue.new_message(self.id)
new_msg = queue.write(new_msg)
self.message_id = new_msg.id
self.put()
boto.log.info('Task[%s] - new message id=%s' %
(self.name, new_msg.id))
msg.delete()
boto.log.info('Task[%s] - deleted message %s' %
(self.name, msg.id))
else:
boto.log.info('new_vtimeout: %d' % delay)
msg.change_visibility(delay)
def start(self, queue_name):
boto.log.info('Task[%s] - starting with queue: %s' %
(self.name, queue_name))
queue = boto.lookup('sqs', queue_name)
msg = queue.new_message(self.id)
msg = queue.write(msg)
self.message_id = msg.id
self.put()
boto.log.info('Task[%s] - start successful' % self.name)
class TestUnique(Model):
name = StringProperty(unique=True)
self.get_instance_type(params)
self.get_zone(params)
self.get_quantity(params)
if __name__ == "__main__":
obj = BuyReservation()
params = {}
obj.get(params)
offerings = obj.ec2.get_all_reserved_instances_offerings(
instance_type=params['instance_type'],
availability_zone=params['zone'].name)
print '\nThe following Reserved Instances Offerings are available:\n'
for offering in offerings:
offering.describe()
prop = StringProperty(name='offering',
verbose_name='Offering',
choices=offerings)
offering = propget.get(prop)
print '\nYou have chosen this offering:'
offering.describe()
unit_price = float(offering.fixed_price)
total_price = unit_price * params['quantity']
print '!!! You are about to purchase %d of these offerings for a total of $%.2f !!!' % (
params['quantity'], total_price)
answer = raw_input('Are you sure you want to do this? If so, enter YES: ')
if answer.strip().lower() == 'yes':
offering.purchase(params['quantity'])
else:
print 'Purchase cancelled'
class TestList(Model):
name = StringProperty()
nums = ListProperty(int)
class TestRequired(Model):
req = StringProperty(required=True, default='foo')
class TestFloat(Model):
name = StringProperty()
value = FloatProperty()
class TestBasic(Model):
name = StringProperty()
size = IntegerProperty()
foo = BooleanProperty()
date = DateTimeProperty()
class Server(Model):
@property
def ec2(self):
if self._ec2 is None:
self._ec2 = boto.connect_ec2()
return self._ec2
@classmethod
def Inventory(cls):
"""
Returns a list of Server instances, one for each Server object
persisted in the db
"""
l = ServerSet()
rs = cls.find()
for server in rs:
l.append(server)
return l
@classmethod
def Register(cls, name, instance_id, description=''):
s = cls()
s.name = name
s.instance_id = instance_id
s.description = description
s.save()
return s
def __init__(self, id=None, **kw):
super(Server, self).__init__(id, **kw)
self._reservation = None
self._instance = None
self._ssh_client = None
self._pkey = None
self._config = None
self._ec2 = None
name = StringProperty(unique=True, verbose_name="Name")
instance_id = StringProperty(verbose_name="Instance ID")
config_uri = StringProperty()
ami_id = StringProperty(verbose_name="AMI ID")
zone = StringProperty(verbose_name="Availability Zone")
security_group = StringProperty(verbose_name="Security Group",
default="default")
key_name = StringProperty(verbose_name="Key Name")
elastic_ip = StringProperty(verbose_name="Elastic IP")
instance_type = StringProperty(verbose_name="Instance Type")
description = StringProperty(verbose_name="Description")
log = StringProperty()
def setReadOnly(self, value):
raise AttributeError
def getInstance(self):
if not self._instance:
if self.instance_id:
try:
rs = self.ec2.get_all_reservations([self.instance_id])
except:
return None
if len(rs) > 0:
self._reservation = rs[0]
self._instance = self._reservation.instances[0]
return self._instance
instance = property(getInstance, setReadOnly, None,
'The Instance for the server')
def getAMI(self):
if self.instance:
return self.instance.image_id
ami = property(getAMI, setReadOnly, None, 'The AMI for the server')
def getStatus(self):
if self.instance:
self.instance.update()
return self.instance.state
status = property(getStatus, setReadOnly, None, 'The status of the server')
def getHostname(self):
if self.instance:
return self.instance.public_dns_name
hostname = property(getHostname, setReadOnly, None,
'The public DNS name of the server')
def getPrivateHostname(self):
if self.instance:
return self.instance.private_dns_name
private_hostname = property(getPrivateHostname, setReadOnly, None,
'The private DNS name of the server')
def getLaunchTime(self):
if self.instance:
return self.instance.launch_time
launch_time = property(getLaunchTime, setReadOnly, None,
'The time the Server was started')
def getConsoleOutput(self):
if self.instance:
return self.instance.get_console_output()
console_output = property(getConsoleOutput, setReadOnly, None,
'Retrieve the console output for server')
def getGroups(self):
if self._reservation:
return self._reservation.groups
else:
return None
groups = property(getGroups, setReadOnly, None,
'The Security Groups controlling access to this server')
def getConfig(self):
if not self._config:
remote_file = BotoConfigPath
local_file = '%s.ini' % self.instance.id
self.get_file(remote_file, local_file)
self._config = Config(local_file)
return self._config
def setConfig(self, config):
local_file = '%s.ini' % self.instance.id
fp = open(local_file)
config.write(fp)
fp.close()
self.put_file(local_file, BotoConfigPath)
self._config = config
config = property(getConfig, setConfig, None,
'The instance data for this server')
def set_config(self, config):
"""
Set SDB based config
"""
self._config = config
self._config.dump_to_sdb("botoConfigs", self.id)
def load_config(self):
self._config = Config(do_load=False)
self._config.load_from_sdb("botoConfigs", self.id)
def stop(self):
if self.instance:
self.instance.stop()
def start(self):
self.stop()
ec2 = boto.connect_ec2()
ami = ec2.get_all_images(image_ids=[str(self.ami_id)])[0]
groups = ec2.get_all_security_groups(
groupnames=[str(self.security_group)])
if not self._config:
self.load_config()
if not self._config.has_section("Credentials"):
self._config.add_section("Credentials")
self._config.set("Credentials", "aws_access_key_id",
ec2.aws_access_key_id)
self._config.set("Credentials", "aws_secret_access_key",
ec2.aws_secret_access_key)
if not self._config.has_section("Pyami"):
self._config.add_section("Pyami")
if self._manager.domain:
self._config.set('Pyami', 'server_sdb_domain',
self._manager.domain.name)
self._config.set("Pyami", 'server_sdb_name', self.name)
cfg = StringIO()
self._config.write(cfg)
cfg = cfg.getvalue()
r = ami.run(min_count=1,
max_count=1,
key_name=self.key_name,
security_groups=groups,
instance_type=self.instance_type,
placement=self.zone,
user_data=cfg)
i = r.instances[0]
self.instance_id = i.id
self.put()
if self.elastic_ip:
ec2.associate_address(self.instance_id, self.elastic_ip)
def reboot(self):
if self.instance:
self.instance.reboot()
def get_ssh_client(self,
key_file=None,
host_key_file='~/.ssh/known_hosts',
uname='root'):
import paramiko
if not self.instance:
print('No instance yet!')
return
if not self._ssh_client:
if not key_file:
iobject = IObject()
key_file = iobject.get_filename('Path to OpenSSH Key file')
self._pkey = paramiko.RSAKey.from_private_key_file(key_file)
self._ssh_client = paramiko.SSHClient()
self._ssh_client.load_system_host_keys()
self._ssh_client.load_host_keys(os.path.expanduser(host_key_file))
self._ssh_client.set_missing_host_key_policy(
paramiko.AutoAddPolicy())
self._ssh_client.connect(self.instance.public_dns_name,
username=uname,
pkey=self._pkey)
return self._ssh_client
def get_file(self, remotepath, localpath):
ssh_client = self.get_ssh_client()
sftp_client = ssh_client.open_sftp()
sftp_client.get(remotepath, localpath)
def put_file(self, localpath, remotepath):
ssh_client = self.get_ssh_client()
sftp_client = ssh_client.open_sftp()
sftp_client.put(localpath, remotepath)
def listdir(self, remotepath):
ssh_client = self.get_ssh_client()
sftp_client = ssh_client.open_sftp()
return sftp_client.listdir(remotepath)
def shell(self, key_file=None):
ssh_client = self.get_ssh_client(key_file)
channel = ssh_client.invoke_shell()
interactive_shell(channel)
def bundle_image(self, prefix, key_file, cert_file, size):
print('bundling image...')
print('\tcopying cert and pk over to /mnt directory on server')
ssh_client = self.get_ssh_client()
sftp_client = ssh_client.open_sftp()
path, name = os.path.split(key_file)
remote_key_file = '/mnt/%s' % name
self.put_file(key_file, remote_key_file)
path, name = os.path.split(cert_file)
remote_cert_file = '/mnt/%s' % name
self.put_file(cert_file, remote_cert_file)
print('\tdeleting %s' % BotoConfigPath)
# delete the metadata.ini file if it exists
try:
sftp_client.remove(BotoConfigPath)
except:
pass
command = 'sudo ec2-bundle-vol '
command += '-c %s -k %s ' % (remote_cert_file, remote_key_file)
command += '-u %s ' % self._reservation.owner_id
command += '-p %s ' % prefix
command += '-s %d ' % size
command += '-d /mnt '
if self.instance.instance_type == 'm1.small' or self.instance_type == 'c1.medium':
command += '-r i386'
else:
command += '-r x86_64'
print('\t%s' % command)
t = ssh_client.exec_command(command)
response = t[1].read()
print('\t%s' % response)
print('\t%s' % t[2].read())
print('...complete!')
def upload_bundle(self, bucket, prefix):
print('uploading bundle...')
command = 'ec2-upload-bundle '
command += '-m /mnt/%s.manifest.xml ' % prefix
command += '-b %s ' % bucket
command += '-a %s ' % self.ec2.aws_access_key_id
command += '-s %s ' % self.ec2.aws_secret_access_key
print('\t%s' % command)
ssh_client = self.get_ssh_client()
t = ssh_client.exec_command(command)
response = t[1].read()
print('\t%s' % response)
print('\t%s' % t[2].read())
print('...complete!')
def create_image(self,
bucket=None,
prefix=None,
key_file=None,
cert_file=None,
size=None):
iobject = IObject()
if not bucket:
bucket = iobject.get_string('Name of S3 bucket')
if not prefix:
prefix = iobject.get_string('Prefix for AMI file')
if not key_file:
key_file = iobject.get_filename('Path to RSA private key file')
if not cert_file:
cert_file = iobject.get_filename('Path to RSA public cert file')
if not size:
size = iobject.get_int('Size (in MB) of bundled image')
self.bundle_image(prefix, key_file, cert_file, size)
self.upload_bundle(bucket, prefix)
print('registering image...')
self.image_id = self.ec2.register_image('%s/%s.manifest.xml' %
(bucket, prefix))
return self.image_id
def attach_volume(self, volume, device="/dev/sdp"):
"""
Attach an EBS volume to this server
:param volume: EBS Volume to attach
:type volume: boto.ec2.volume.Volume
:param device: Device to attach to (default to /dev/sdp)
:type device: string
"""
if hasattr(volume, "id"):
volume_id = volume.id
else:
volume_id = volume
return self.ec2.attach_volume(volume_id=volume_id,
instance_id=self.instance_id,
device=device)
def detach_volume(self, volume):
"""
Detach an EBS volume from this server
:param volume: EBS Volume to detach
:type volume: boto.ec2.volume.Volume
"""
if hasattr(volume, "id"):
volume_id = volume.id
else:
volume_id = volume
return self.ec2.detach_volume(volume_id=volume_id,
instance_id=self.instance_id)
def install_package(self, package_name):
print('installing %s...' % package_name)
command = 'yum -y install %s' % package_name
print('\t%s' % command)
ssh_client = self.get_ssh_client()
t = ssh_client.exec_command(command)
response = t[1].read()
print('\t%s' % response)
print('\t%s' % t[2].read())
print('...complete!')
class Volume(Model):
name = StringProperty(required=True, unique=True, verbose_name='Name')
region_name = StringProperty(required=True, verbose_name='EC2 Region')
zone_name = StringProperty(required=True, verbose_name='EC2 Zone')
mount_point = StringProperty(verbose_name='Mount Point')
device = StringProperty(verbose_name="Device Name", default='/dev/sdp')
volume_id = StringProperty(required=True)
past_volume_ids = ListProperty(item_type=str)
server = ReferenceProperty(Server,
collection_name='volumes',
verbose_name='Server Attached To')
volume_state = CalculatedProperty(verbose_name="Volume State",
calculated_type=str,
use_method=True)
attachment_state = CalculatedProperty(verbose_name="Attachment State",
calculated_type=str,
use_method=True)
size = CalculatedProperty(verbose_name="Size (GB)",
calculated_type=int,
use_method=True)
@classmethod
def create(cls, **params):
getter = CommandLineGetter()
getter.get(cls, params)
region = params.get('region')
ec2 = region.connect()
zone = params.get('zone')
size = params.get('size')
ebs_volume = ec2.create_volume(size, zone.name)
v = cls()
v.ec2 = ec2
v.volume_id = ebs_volume.id
v.name = params.get('name')
v.mount_point = params.get('mount_point')
v.device = params.get('device')
v.region_name = region.name
v.zone_name = zone.name
v.put()
return v
@classmethod
def create_from_volume_id(cls, region_name, volume_id, name):
vol = None
ec2 = boto.ec2.connect_to_region(region_name)
rs = ec2.get_all_volumes([volume_id])
if len(rs) == 1:
v = rs[0]
vol = cls()
vol.volume_id = v.id
vol.name = name
vol.region_name = v.region.name
vol.zone_name = v.zone
vol.put()
return vol
def create_from_latest_snapshot(self, name, size=None):
snapshot = self.get_snapshots()[-1]
return self.create_from_snapshot(name, snapshot, size)
def create_from_snapshot(self, name, snapshot, size=None):
if size < self.size:
size = self.size
ec2 = self.get_ec2_connection()
if self.zone_name == None or self.zone_name == '':
# deal with the migration case where the zone is not set in the logical volume:
current_volume = ec2.get_all_volumes([self.volume_id])[0]
self.zone_name = current_volume.zone
ebs_volume = ec2.create_volume(size, self.zone_name, snapshot)
v = Volume()
v.ec2 = self.ec2
v.volume_id = ebs_volume.id
v.name = name
v.mount_point = self.mount_point
v.device = self.device
v.region_name = self.region_name
v.zone_name = self.zone_name
v.put()
return v
def get_ec2_connection(self):
if self.server:
return self.server.ec2
if not hasattr(self, 'ec2') or self.ec2 == None:
self.ec2 = boto.ec2.connect_to_region(self.region_name)
return self.ec2
def _volume_state(self):
ec2 = self.get_ec2_connection()
rs = ec2.get_all_volumes([self.volume_id])
return rs[0].volume_state()
def _attachment_state(self):
ec2 = self.get_ec2_connection()
rs = ec2.get_all_volumes([self.volume_id])
return rs[0].attachment_state()
def _size(self):
if not hasattr(self, '__size'):
ec2 = self.get_ec2_connection()
rs = ec2.get_all_volumes([self.volume_id])
self.__size = rs[0].size
return self.__size
def install_xfs(self):
if self.server:
self.server.install('xfsprogs xfsdump')
def get_snapshots(self):
"""
Returns a list of all completed snapshots for this volume ID.
"""
ec2 = self.get_ec2_connection()
rs = ec2.get_all_snapshots()
all_vols = [self.volume_id] + self.past_volume_ids
snaps = []
for snapshot in rs:
if snapshot.volume_id in all_vols:
if snapshot.progress == '100%':
snapshot.date = dateutil.parser.parse(snapshot.start_time)
snapshot.keep = True
snaps.append(snapshot)
snaps.sort(cmp=lambda x, y: cmp(x.date, y.date))
return snaps
def attach(self, server=None):
if self.attachment_state == 'attached':
print 'already attached'
return None
if server:
self.server = server
self.put()
ec2 = self.get_ec2_connection()
ec2.attach_volume(self.volume_id, self.server.instance_id, self.device)
def detach(self, force=False):
state = self.attachment_state
if state == 'available' or state == None or state == 'detaching':
print 'already detached'
return None
ec2 = self.get_ec2_connection()
ec2.detach_volume(self.volume_id, self.server.instance_id, self.device,
force)
self.server = None
self.put()
def checkfs(self, use_cmd=None):
if self.server == None:
raise ValueError, 'server attribute must be set to run this command'
# detemine state of file system on volume, only works if attached
if use_cmd:
cmd = use_cmd
else:
cmd = self.server.get_cmdshell()
status = cmd.run('xfs_check %s' % self.device)
if not use_cmd:
cmd.close()
if status[1].startswith('bad superblock magic number 0'):
return False
return True
def wait(self):
if self.server == None:
raise ValueError, 'server attribute must be set to run this command'
with closing(self.server.get_cmdshell()) as cmd:
# wait for the volume device to appear
cmd = self.server.get_cmdshell()
while not cmd.exists(self.device):
boto.log.info('%s still does not exist, waiting 10 seconds' %
self.device)
time.sleep(10)
def format(self):
if self.server == None:
raise ValueError, 'server attribute must be set to run this command'
status = None
with closing(self.server.get_cmdshell()) as cmd:
if not self.checkfs(cmd):
boto.log.info('make_fs...')
status = cmd.run('mkfs -t xfs %s' % self.device)
return status
def mount(self):
if self.server == None:
raise ValueError, 'server attribute must be set to run this command'
boto.log.info('handle_mount_point')
with closing(self.server.get_cmdshell()) as cmd:
cmd = self.server.get_cmdshell()
if not cmd.isdir(self.mount_point):
boto.log.info('making directory')
# mount directory doesn't exist so create it
cmd.run("mkdir %s" % self.mount_point)
else:
boto.log.info('directory exists already')
status = cmd.run('mount -l')
lines = status[1].split('\n')
for line in lines:
t = line.split()
if t and t[2] == self.mount_point:
# something is already mounted at the mount point
# unmount that and mount it as /tmp
if t[0] != self.device:
cmd.run('umount %s' % self.mount_point)
cmd.run('mount %s /tmp' % t[0])
cmd.run('chmod 777 /tmp')
break
# Mount up our new EBS volume onto mount_point
cmd.run("mount %s %s" % (self.device, self.mount_point))
cmd.run('xfs_growfs %s' % self.mount_point)
def make_ready(self, server):
self.server = server
self.put()
self.install_xfs()
self.attach()
self.wait()
self.format()
self.mount()
def freeze(self):
if self.server:
return self.server.run("/usr/sbin/xfs_freeze -f %s" %
self.mount_point)
def unfreeze(self):
if self.server:
return self.server.run("/usr/sbin/xfs_freeze -u %s" %
self.mount_point)
def snapshot(self):
# if this volume is attached to a server
# we need to freeze the XFS file system
try:
self.freeze()
if self.server == None:
snapshot = self.get_ec2_connection().create_snapshot(
self.volume_id)
else:
snapshot = self.server.ec2.create_snapshot(self.volume_id)
boto.log.info('Snapshot of Volume %s created: %s' %
(self.name, snapshot))
except Exception:
boto.log.info('Snapshot error')
boto.log.info(traceback.format_exc())
finally:
status = self.unfreeze()
return status
def get_snapshot_range(self, snaps, start_date=None, end_date=None):
l = []
for snap in snaps:
if start_date and end_date:
if snap.date >= start_date and snap.date <= end_date:
l.append(snap)
elif start_date:
if snap.date >= start_date:
l.append(snap)
elif end_date:
if snap.date <= end_date:
l.append(snap)
else:
l.append(snap)
return l
def trim_snapshots(self, delete=False):
"""
Trim the number of snapshots for this volume. This method always
keeps the oldest snapshot. It then uses the parameters passed in
to determine how many others should be kept.
The algorithm is to keep all snapshots from the current day. Then
it will keep the first snapshot of the day for the previous seven days.
Then, it will keep the first snapshot of the week for the previous
four weeks. After than, it will keep the first snapshot of the month
for as many months as there are.
"""
snaps = self.get_snapshots()
# Always keep the oldest and the newest
if len(snaps) <= 2:
return snaps
snaps = snaps[1:-1]
now = datetime.datetime.now(snaps[0].date.tzinfo)
midnight = datetime.datetime(year=now.year,
month=now.month,
day=now.day,
tzinfo=now.tzinfo)
# Keep the first snapshot from each day of the previous week
one_week = datetime.timedelta(days=7, seconds=60 * 60)
print midnight - one_week, midnight
previous_week = self.get_snapshot_range(snaps, midnight - one_week,
midnight)
print previous_week
if not previous_week:
return snaps
current_day = None
for snap in previous_week:
if current_day and current_day == snap.date.day:
snap.keep = False
else:
current_day = snap.date.day
# Get ourselves onto the next full week boundary
if previous_week:
week_boundary = previous_week[0].date
if week_boundary.weekday() != 0:
delta = datetime.timedelta(days=week_boundary.weekday())
week_boundary = week_boundary - delta
# Keep one within this partial week
partial_week = self.get_snapshot_range(snaps, week_boundary,
previous_week[0].date)
if len(partial_week) > 1:
for snap in partial_week[1:]:
snap.keep = False
# Keep the first snapshot of each week for the previous 4 weeks
for i in range(0, 4):
weeks_worth = self.get_snapshot_range(snaps,
week_boundary - one_week,
week_boundary)
if len(weeks_worth) > 1:
for snap in weeks_worth[1:]:
snap.keep = False
week_boundary = week_boundary - one_week
# Now look through all remaining snaps and keep one per month
remainder = self.get_snapshot_range(snaps, end_date=week_boundary)
current_month = None
for snap in remainder:
if current_month and current_month == snap.date.month:
snap.keep = False
else:
current_month = snap.date.month
if delete:
for snap in snaps:
if not snap.keep:
boto.log.info('Deleting %s(%s) for %s' %
(snap, snap.date, self.name))
snap.delete()
return snaps
def grow(self, size):
pass
def copy(self, snapshot):
pass
def get_snapshot_from_date(self, date):
pass
def delete(self, delete_ebs_volume=False):
if delete_ebs_volume:
self.detach()
ec2 = self.get_ec2_connection()
ec2.delete_volume(self.volume_id)
Model.delete(self)
def archive(self):
# snapshot volume, trim snaps, delete volume-id
pass
def get_region(self, params):
if not params.get('region', None):
prop = StringProperty(name='region',
verbose_name='EC2 Region',
choices=boto.ec2.regions)
params['region'] = propget.get(prop, choices=boto.ec2.regions)
class TestMap(Model):
name = StringProperty()
map = MapProperty()
class TestListReference(Model):
name = StringProperty()
basics = ListProperty(TestBasic)
class Server(Model):
#
# The properties of this object consists of real properties for data that
# is not already stored in EC2 somewhere (e.g. name, description) plus
# calculated properties for all of the properties that are already in
# EC2 (e.g. hostname, security groups, etc.)
#
name = StringProperty(unique=True, verbose_name="Name")
description = StringProperty(verbose_name="Description")
region_name = StringProperty(verbose_name="EC2 Region Name")
instance_id = StringProperty(verbose_name="EC2 Instance ID")
elastic_ip = StringProperty(verbose_name="EC2 Elastic IP Address")
production = BooleanProperty(verbose_name="Is This Server Production", default=False)
ami_id = CalculatedProperty(verbose_name="AMI ID", calculated_type=str, use_method=True)
zone = CalculatedProperty(verbose_name="Availability Zone Name", calculated_type=str, use_method=True)
hostname = CalculatedProperty(verbose_name="Public DNS Name", calculated_type=str, use_method=True)
private_hostname = CalculatedProperty(verbose_name="Private DNS Name", calculated_type=str, use_method=True)
groups = CalculatedProperty(verbose_name="Security Groups", calculated_type=list, use_method=True)
security_group = CalculatedProperty(verbose_name="Primary Security Group Name", calculated_type=str, use_method=True)
key_name = CalculatedProperty(verbose_name="Key Name", calculated_type=str, use_method=True)
instance_type = CalculatedProperty(verbose_name="Instance Type", calculated_type=str, use_method=True)
status = CalculatedProperty(verbose_name="Current Status", calculated_type=str, use_method=True)
launch_time = CalculatedProperty(verbose_name="Server Launch Time", calculated_type=str, use_method=True)
console_output = CalculatedProperty(verbose_name="Console Output", calculated_type=file, use_method=True)
packages = []
plugins = []
@classmethod
def add_credentials(cls, cfg, aws_access_key_id, aws_secret_access_key):
if not cfg.has_section('Credentials'):
cfg.add_section('Credentials')
cfg.set('Credentials', 'aws_access_key_id', aws_access_key_id)
cfg.set('Credentials', 'aws_secret_access_key', aws_secret_access_key)
if not cfg.has_section('DB_Server'):
cfg.add_section('DB_Server')
cfg.set('DB_Server', 'db_type', 'SimpleDB')
cfg.set('DB_Server', 'db_name', cls._manager.domain.name)
@classmethod
def create(cls, config_file=None, logical_volume = None, cfg = None, **params):
"""
Create a new instance based on the specified configuration file or the specified
configuration and the passed in parameters.
If the config_file argument is not None, the configuration is read from there.
Otherwise, the cfg argument is used.
The config file may include other config files with a #import reference. The included
config files must reside in the same directory as the specified file.
The logical_volume argument, if supplied, will be used to get the current physical
volume ID and use that as an override of the value specified in the config file. This
may be useful for debugging purposes when you want to debug with a production config
file but a test Volume.
The dictionary argument may be used to override any EC2 configuration values in the
config file.
"""
if config_file:
cfg = Config(path=config_file)
if cfg.has_section('EC2'):
# include any EC2 configuration values that aren't specified in params:
for option in cfg.options('EC2'):
if option not in params:
params[option] = cfg.get('EC2', option)
getter = CommandLineGetter()
getter.get(cls, params)
region = params.get('region')
ec2 = region.connect()
cls.add_credentials(cfg, ec2.aws_access_key_id, ec2.aws_secret_access_key)
ami = params.get('ami')
kp = params.get('keypair')
group = params.get('group')
zone = params.get('zone')
# deal with possibly passed in logical volume:
if logical_volume != None:
cfg.set('EBS', 'logical_volume_name', logical_volume.name)
cfg_fp = StringIO()
cfg.write(cfg_fp)
# deal with the possibility that zone and/or keypair are strings read from the config file:
if isinstance(zone, Zone):
zone = zone.name
if isinstance(kp, KeyPair):
kp = kp.name
reservation = ami.run(min_count=1,
max_count=params.get('quantity', 1),
key_name=kp,
security_groups=[group],
instance_type=params.get('instance_type'),
placement = zone,
user_data = cfg_fp.getvalue())
l = []
i = 0
elastic_ip = params.get('elastic_ip')
instances = reservation.instances
if elastic_ip is not None and instances.__len__() > 0:
instance = instances[0]
print('Waiting for instance to start so we can set its elastic IP address...')
# Sometimes we get a message from ec2 that says that the instance does not exist.
# Hopefully the following delay will giv eec2 enough time to get to a stable state:
time.sleep(5)
while instance.update() != 'running':
time.sleep(1)
instance.use_ip(elastic_ip)
print('set the elastic IP of the first instance to %s' % elastic_ip)
for instance in instances:
s = cls()
s.ec2 = ec2
s.name = params.get('name') + '' if i==0 else str(i)
s.description = params.get('description')
s.region_name = region.name
s.instance_id = instance.id
if elastic_ip and i == 0:
s.elastic_ip = elastic_ip
s.put()
l.append(s)
i += 1
return l
@classmethod
def create_from_instance_id(cls, instance_id, name, description=''):
regions = boto.ec2.regions()
for region in regions:
ec2 = region.connect()
try:
rs = ec2.get_all_reservations([instance_id])
except:
rs = []
if len(rs) == 1:
s = cls()
s.ec2 = ec2
s.name = name
s.description = description
s.region_name = region.name
s.instance_id = instance_id
s._reservation = rs[0]
for instance in s._reservation.instances:
if instance.id == instance_id:
s._instance = instance
s.put()
return s
return None
@classmethod
def create_from_current_instances(cls):
servers = []
regions = boto.ec2.regions()
for region in regions:
ec2 = region.connect()
rs = ec2.get_all_reservations()
for reservation in rs:
for instance in reservation.instances:
try:
next(Server.find(instance_id=instance.id))
boto.log.info('Server for %s already exists' % instance.id)
except StopIteration:
s = cls()
s.ec2 = ec2
s.name = instance.id
s.region_name = region.name
s.instance_id = instance.id
s._reservation = reservation
s.put()
servers.append(s)
return servers
def __init__(self, id=None, **kw):
super(Server, self).__init__(id, **kw)
self.ssh_key_file = None
self.ec2 = None
self._cmdshell = None
self._reservation = None
self._instance = None
self._setup_ec2()
def _setup_ec2(self):
if self.ec2 and self._instance and self._reservation:
return
if self.id:
if self.region_name:
for region in boto.ec2.regions():
if region.name == self.region_name:
self.ec2 = region.connect()
if self.instance_id and not self._instance:
try:
rs = self.ec2.get_all_reservations([self.instance_id])
if len(rs) >= 1:
for instance in rs[0].instances:
if instance.id == self.instance_id:
self._reservation = rs[0]
self._instance = instance
except EC2ResponseError:
pass
def _status(self):
status = ''
if self._instance:
self._instance.update()
status = self._instance.state
return status
def _hostname(self):
hostname = ''
if self._instance:
hostname = self._instance.public_dns_name
return hostname
def _private_hostname(self):
hostname = ''
if self._instance:
hostname = self._instance.private_dns_name
return hostname
def _instance_type(self):
it = ''
if self._instance:
it = self._instance.instance_type
return it
def _launch_time(self):
lt = ''
if self._instance:
lt = self._instance.launch_time
return lt
def _console_output(self):
co = ''
if self._instance:
co = self._instance.get_console_output()
return co
def _groups(self):
gn = []
if self._reservation:
gn = self._reservation.groups
return gn
def _security_group(self):
groups = self._groups()
if len(groups) >= 1:
return groups[0].id
return ""
def _zone(self):
zone = None
if self._instance:
zone = self._instance.placement
return zone
def _key_name(self):
kn = None
if self._instance:
kn = self._instance.key_name
return kn
def put(self):
super(Server, self).put()
self._setup_ec2()
def delete(self):
if self.production:
raise ValueError("Can't delete a production server")
#self.stop()
super(Server, self).delete()
def stop(self):
if self.production:
raise ValueError("Can't delete a production server")
if self._instance:
self._instance.stop()
def terminate(self):
if self.production:
raise ValueError("Can't delete a production server")
if self._instance:
self._instance.terminate()
def reboot(self):
if self._instance:
self._instance.reboot()
def wait(self):
while self.status != 'running':
time.sleep(5)
def get_ssh_key_file(self):
if not self.ssh_key_file:
ssh_dir = os.path.expanduser('~/.ssh')
if os.path.isdir(ssh_dir):
ssh_file = os.path.join(ssh_dir, '%s.pem' % self.key_name)
if os.path.isfile(ssh_file):
self.ssh_key_file = ssh_file
if not self.ssh_key_file:
iobject = IObject()
self.ssh_key_file = iobject.get_filename('Path to OpenSSH Key file')
return self.ssh_key_file
def get_cmdshell(self):
if not self._cmdshell:
from boto.manage import cmdshell
self.get_ssh_key_file()
self._cmdshell = cmdshell.start(self)
return self._cmdshell
def reset_cmdshell(self):
self._cmdshell = None
def run(self, command):
with closing(self.get_cmdshell()) as cmd:
status = cmd.run(command)
return status
def get_bundler(self, uname='root'):
self.get_ssh_key_file()
return Bundler(self, uname)
def get_ssh_client(self, uname='root', ssh_pwd=None):
from boto.manage.cmdshell import SSHClient
self.get_ssh_key_file()
return SSHClient(self, uname=uname, ssh_pwd=ssh_pwd)
def install(self, pkg):
return self.run('apt-get -y install %s' % pkg)
class SimpleModel(Model):
"""Simple Test Model"""
name = StringProperty()
strs = ListProperty(str)
num = IntegerProperty()
