def _run(self, msg, vtimeout):
boto.log.info('Task[%s] - running:%s' % (self.name, self.command))
log_fp = StringIO()
process = subprocess.Popen(self.command,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
nsecs = 5
current_timeout = vtimeout
while process.poll() is None:
boto.log.info('nsecs=%s, timeout=%s' % (nsecs, current_timeout))
if nsecs >= current_timeout:
current_timeout += vtimeout
boto.log.info('Task[%s] - setting timeout to %d seconds' %
(self.name, current_timeout))
if msg:
msg.change_visibility(current_timeout)
time.sleep(5)
nsecs += 5
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info('Task[%s] - output: %s' % (self.name, log_fp.getvalue()))
self.last_executed = self.now
self.last_status = process.returncode
self.last_output = log_fp.getvalue()[0:1023]
def run(self):
boto.log.info('running:%s' % self.command)
log_fp = StringIO()
process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while process.poll() is None:
time.sleep(1)
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info(log_fp.getvalue())
boto.log.info('output: %s' % log_fp.getvalue())
return (process.returncode, log_fp.getvalue())
def do_start(self):
ami_id = self.sd.get('ami_id')
instance_type = self.sd.get('instance_type', 'm1.small')
security_group = self.sd.get('security_group', 'default')
if not ami_id:
self.parser.error(
'ami_id option is required when starting the service')
ec2 = boto.connect_ec2()
if not self.sd.has_section('Credentials'):
self.sd.add_section('Credentials')
self.sd.set('Credentials', 'aws_access_key_id',
ec2.aws_access_key_id)
self.sd.set('Credentials', 'aws_secret_access_key',
ec2.aws_secret_access_key)
s = StringIO()
self.sd.write(s)
rs = ec2.get_all_images([ami_id])
img = rs[0]
r = img.run(user_data=s.getvalue(),
key_name=self.options.keypair,
max_count=self.options.num_instances,
instance_type=instance_type,
security_groups=[security_group])
print('Starting AMI: %s' % ami_id)
print('Reservation %s contains the following instances:' % r.id)
for i in r.instances:
print('\t%s' % i.id)
def get_contents_as_string(self,
headers=None,
cb=None,
num_cb=10,
torrent=False):
"""
Retrieve file data from the Key, and return contents as a string.
:type headers: dict
:param headers: ignored in this subclass.
:type cb: function
:param cb: ignored in this subclass.
:type cb: int
:param num_cb: ignored in this subclass.
:type cb: int
:param num_cb: ignored in this subclass.
:type torrent: bool
:param torrent: ignored in this subclass.
:rtype: string
:returns: The contents of the file as a string
"""
fp = StringIO()
self.get_contents_to_file(fp)
return fp.getvalue()
def get_contents_as_string(self, headers=None, cb=None, num_cb=10,
torrent=False):
"""
Retrieve file data from the Key, and return contents as a string.
:type headers: dict
:param headers: ignored in this subclass.
:type cb: function
:param cb: ignored in this subclass.
:type cb: int
:param num_cb: ignored in this subclass.
:type cb: int
:param num_cb: ignored in this subclass.
:type torrent: bool
:param torrent: ignored in this subclass.
:rtype: string
:returns: The contents of the file as a string
"""
fp = StringIO()
self.get_contents_to_file(fp)
return fp.getvalue()
def start(self):
self.stop()
ec2 = boto.connect_ec2()
ami = ec2.get_all_images(image_ids = [str(self.ami_id)])[0]
groups = ec2.get_all_security_groups(groupnames=[str(self.security_group)])
if not self._config:
self.load_config()
if not self._config.has_section("Credentials"):
self._config.add_section("Credentials")
self._config.set("Credentials", "aws_access_key_id", ec2.aws_access_key_id)
self._config.set("Credentials", "aws_secret_access_key", ec2.aws_secret_access_key)
if not self._config.has_section("Pyami"):
self._config.add_section("Pyami")
if self._manager.domain:
self._config.set('Pyami', 'server_sdb_domain', self._manager.domain.name)
self._config.set("Pyami", 'server_sdb_name', self.name)
cfg = StringIO()
self._config.write(cfg)
cfg = cfg.getvalue()
r = ami.run(min_count=1,
max_count=1,
key_name=self.key_name,
security_groups = groups,
instance_type = self.instance_type,
placement = self.zone,
user_data = cfg)
i = r.instances[0]
self.instance_id = i.id
self.put()
if self.elastic_ip:
ec2.associate_address(self.instance_id, self.elastic_ip)
class ResponseGroup(xml.sax.ContentHandler):
"""A Generic "Response Group", which can
be anything from the entire list of Items to
specific response elements within an item"""
def __init__(self, connection=None, nodename=None):
"""Initialize this Item"""
self._connection = connection
self._nodename = nodename
self._nodepath = []
self._curobj = None
self._xml = StringIO()
def __repr__(self):
return '<%s: %s>' % (self.__class__.__name__, self.__dict__)
#
# Attribute Functions
#
def get(self, name):
return self.__dict__.get(name)
def set(self, name, value):
self.__dict__[name] = value
def to_xml(self):
return "<%s>%s</%s>" % (self._nodename, self._xml.getvalue(), self._nodename)
#
# XML Parser functions
#
def startElement(self, name, attrs, connection):
self._xml.write("<%s>" % name)
self._nodepath.append(name)
if len(self._nodepath) == 1:
obj = ResponseGroup(self._connection)
self.set(name, obj)
self._curobj = obj
elif self._curobj:
self._curobj.startElement(name, attrs, connection)
return None
def endElement(self, name, value, connection):
self._xml.write("%s</%s>" % (cgi.escape(value).replace("&amp;", "&"), name))
if len(self._nodepath) == 0:
return
obj = None
curval = self.get(name)
if len(self._nodepath) == 1:
if value or not curval:
self.set(name, value)
if self._curobj:
self._curobj = None
#elif len(self._nodepath) == 2:
#self._curobj = None
elif self._curobj:
self._curobj.endElement(name, value, connection)
self._nodepath.pop()
return None
class ShellCommand(object):
def __init__(self, command, wait=True, fail_fast=False, cwd=None):
self.exit_code = 0
self.command = command
self.log_fp = StringIO()
self.wait = wait
self.fail_fast = fail_fast
self.run(cwd=cwd)
def run(self, cwd=None):
boto.log.info('running:%s' % self.command)
self.process = subprocess.Popen(self.command, shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd)
if(self.wait):
while self.process.poll() is None:
time.sleep(1)
t = self.process.communicate()
self.log_fp.write(t[0])
self.log_fp.write(t[1])
boto.log.info(self.log_fp.getvalue())
self.exit_code = self.process.returncode
if self.fail_fast and self.exit_code != 0:
raise Exception("Command " + self.command +
" failed with status " + self.exit_code)
return self.exit_code
def setReadOnly(self, value):
raise AttributeError
def getStatus(self):
return self.exit_code
status = property(getStatus, setReadOnly, None,
'The exit code for the command')
def getOutput(self):
return self.log_fp.getvalue()
output = property(getOutput, setReadOnly, None,
'The STDIN and STDERR output of the command')
def run(self):
"""
Open a subprocess and run a command on the local host.
:rtype: tuple
:return: This function returns a tuple that contains an integer status
and a string with the combined stdout and stderr output.
"""
boto.log.info('running:%s' % self.command)
log_fp = StringIO()
process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while process.poll() is None:
time.sleep(1)
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info(log_fp.getvalue())
boto.log.info('output: %s' % log_fp.getvalue())
return (process.returncode, log_fp.getvalue())
def run(self):
"""
Open a subprocess and run a command on the local host.
:rtype: tuple
:return: This function returns a tuple that contains an integer status
and a string with the combined stdout and stderr output.
"""
boto.log.info('running:%s' % self.command)
log_fp = StringIO()
process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while process.poll() is None:
time.sleep(1)
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info(log_fp.getvalue())
boto.log.info('output: %s' % log_fp.getvalue())
return (process.returncode, log_fp.getvalue())
def _run(self, msg, vtimeout):
boto.log.info('Task[%s] - running:%s' % (self.name, self.command))
log_fp = StringIO()
process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
nsecs = 5
current_timeout = vtimeout
while process.poll() is None:
boto.log.info('nsecs=%s, timeout=%s' % (nsecs, current_timeout))
if nsecs >= current_timeout:
current_timeout += vtimeout
boto.log.info('Task[%s] - setting timeout to %d seconds' % (self.name, current_timeout))
if msg:
msg.change_visibility(current_timeout)
time.sleep(5)
nsecs += 5
t = process.communicate()
log_fp.write(t[0])
log_fp.write(t[1])
boto.log.info('Task[%s] - output: %s' % (self.name, log_fp.getvalue()))
self.last_executed = self.now
self.last_status = process.returncode
self.last_output = log_fp.getvalue()[0:1023]
def bundle(self,
bucket=None,
prefix=None,
key_file=None,
cert_file=None,
size=None,
ssh_key=None,
fp=None,
clear_history=True):
iobject = IObject()
if not bucket:
bucket = iobject.get_string('Name of S3 bucket')
if not prefix:
prefix = iobject.get_string('Prefix for AMI file')
if not key_file:
key_file = iobject.get_filename('Path to RSA private key file')
if not cert_file:
cert_file = iobject.get_filename('Path to RSA public cert file')
if not size:
size = iobject.get_int('Size (in MB) of bundled image')
if not ssh_key:
ssh_key = self.server.get_ssh_key_file()
self.copy_x509(key_file, cert_file)
if not fp:
fp = StringIO()
fp.write('sudo mv %s /mnt/boto.cfg; ' % BotoConfigPath)
fp.write('mv ~/.ssh/authorized_keys /mnt/authorized_keys; ')
if clear_history:
fp.write('history -c; ')
fp.write(self.bundle_image(prefix, size, ssh_key))
fp.write('; ')
fp.write(self.upload_bundle(bucket, prefix, ssh_key))
fp.write('; ')
fp.write('sudo mv /mnt/boto.cfg %s; ' % BotoConfigPath)
fp.write('mv /mnt/authorized_keys ~/.ssh/authorized_keys')
command = fp.getvalue()
print('running the following command on the remote server:')
print(command)
t = self.ssh_client.run(command)
print('\t%s' % t[0])
print('\t%s' % t[1])
print('...complete!')
print('registering image...')
self.image_id = self.server.ec2.register_image(
name=prefix,
image_location='%s/%s.manifest.xml' % (bucket, prefix))
return self.image_id
def write_mime_multipart(content,
compress=False,
deftype='text/plain',
delimiter=':'):
"""Description:
:param content: A list of tuples of name-content pairs. This is used
instead of a dict to ensure that scripts run in order
:type list of tuples:
:param compress: Use gzip to compress the scripts, defaults to no compression
:type bool:
:param deftype: The type that should be assumed if nothing else can be figured out
:type str:
:param delimiter: mime delimiter
:type str:
:return: Final mime multipart
:rtype: str:
"""
wrapper = email.mime.multipart.MIMEMultipart()
for name, con in content:
definite_type = guess_mime_type(con, deftype)
maintype, subtype = definite_type.split('/', 1)
if maintype == 'text':
mime_con = email.mime.text.MIMEText(con, _subtype=subtype)
else:
mime_con = email.mime.base.MIMEBase(maintype, subtype)
mime_con.set_payload(con)
# Encode the payload using Base64
email.encoders.encode_base64(mime_con)
mime_con.add_header('Content-Disposition', 'attachment', filename=name)
wrapper.attach(mime_con)
rcontent = wrapper.as_string()
if compress:
buf = StringIO()
gz = gzip.GzipFile(mode='wb', fileobj=buf)
try:
gz.write(rcontent)
finally:
gz.close()
rcontent = buf.getvalue()
return rcontent
def write_mime_multipart(content, compress=False, deftype='text/plain', delimiter=':'):
"""Description:
:param content: A list of tuples of name-content pairs. This is used
instead of a dict to ensure that scripts run in order
:type list of tuples:
:param compress: Use gzip to compress the scripts, defaults to no compression
:type bool:
:param deftype: The type that should be assumed if nothing else can be figured out
:type str:
:param delimiter: mime delimiter
:type str:
:return: Final mime multipart
:rtype: str:
"""
wrapper = email.mime.multipart.MIMEMultipart()
for name, con in content:
definite_type = guess_mime_type(con, deftype)
maintype, subtype = definite_type.split('/', 1)
if maintype == 'text':
mime_con = email.mime.text.MIMEText(con, _subtype=subtype)
else:
mime_con = email.mime.base.MIMEBase(maintype, subtype)
mime_con.set_payload(con)
# Encode the payload using Base64
email.encoders.encode_base64(mime_con)
mime_con.add_header('Content-Disposition', 'attachment', filename=name)
wrapper.attach(mime_con)
rcontent = wrapper.as_string()
if compress:
buf = StringIO()
gz = gzip.GzipFile(mode='wb', fileobj=buf)
try:
gz.write(rcontent)
finally:
gz.close()
rcontent = buf.getvalue()
return rcontent
def bundle(self, bucket=None, prefix=None, key_file=None, cert_file=None,
size=None, ssh_key=None, fp=None, clear_history=True):
iobject = IObject()
if not bucket:
bucket = iobject.get_string('Name of S3 bucket')
if not prefix:
prefix = iobject.get_string('Prefix for AMI file')
if not key_file:
key_file = iobject.get_filename('Path to RSA private key file')
if not cert_file:
cert_file = iobject.get_filename('Path to RSA public cert file')
if not size:
size = iobject.get_int('Size (in MB) of bundled image')
if not ssh_key:
ssh_key = self.server.get_ssh_key_file()
self.copy_x509(key_file, cert_file)
if not fp:
fp = StringIO()
fp.write('sudo mv %s /mnt/boto.cfg; ' % BotoConfigPath)
fp.write('mv ~/.ssh/authorized_keys /mnt/authorized_keys; ')
if clear_history:
fp.write('history -c; ')
fp.write(self.bundle_image(prefix, size, ssh_key))
fp.write('; ')
fp.write(self.upload_bundle(bucket, prefix, ssh_key))
fp.write('; ')
fp.write('sudo mv /mnt/boto.cfg %s; ' % BotoConfigPath)
fp.write('mv /mnt/authorized_keys ~/.ssh/authorized_keys')
command = fp.getvalue()
print('running the following command on the remote server:')
print(command)
t = self.ssh_client.run(command)
print('\t%s' % t[0])
print('\t%s' % t[1])
print('...complete!')
print('registering image...')
self.image_id = self.server.ec2.register_image(name=prefix, image_location='%s/%s.manifest.xml' % (bucket, prefix))
return self.image_id
def do_start(self):
ami_id = self.sd.get('ami_id')
instance_type = self.sd.get('instance_type', 'm1.small')
security_group = self.sd.get('security_group', 'default')
if not ami_id:
self.parser.error('ami_id option is required when starting the service')
ec2 = boto.connect_ec2()
if not self.sd.has_section('Credentials'):
self.sd.add_section('Credentials')
self.sd.set('Credentials', 'aws_access_key_id', ec2.aws_access_key_id)
self.sd.set('Credentials', 'aws_secret_access_key', ec2.aws_secret_access_key)
s = StringIO()
self.sd.write(s)
rs = ec2.get_all_images([ami_id])
img = rs[0]
r = img.run(user_data=s.getvalue(), key_name=self.options.keypair,
max_count=self.options.num_instances,
instance_type=instance_type,
security_groups=[security_group])
print('Starting AMI: %s' % ami_id)
print('Reservation %s contains the following instances:' % r.id)
for i in r.instances:
print('\t%s' % i.id)
def create(cls, config_file=None, logical_volume = None, cfg = None, **params):
"""
Create a new instance based on the specified configuration file or the specified
configuration and the passed in parameters.
If the config_file argument is not None, the configuration is read from there.
Otherwise, the cfg argument is used.
The config file may include other config files with a #import reference. The included
config files must reside in the same directory as the specified file.
The logical_volume argument, if supplied, will be used to get the current physical
volume ID and use that as an override of the value specified in the config file. This
may be useful for debugging purposes when you want to debug with a production config
file but a test Volume.
The dictionary argument may be used to override any EC2 configuration values in the
config file.
"""
if config_file:
cfg = Config(path=config_file)
if cfg.has_section('EC2'):
# include any EC2 configuration values that aren't specified in params:
for option in cfg.options('EC2'):
if option not in params:
params[option] = cfg.get('EC2', option)
getter = CommandLineGetter()
getter.get(cls, params)
region = params.get('region')
ec2 = region.connect()
cls.add_credentials(cfg, ec2.aws_access_key_id, ec2.aws_secret_access_key)
ami = params.get('ami')
kp = params.get('keypair')
group = params.get('group')
zone = params.get('zone')
# deal with possibly passed in logical volume:
if logical_volume != None:
cfg.set('EBS', 'logical_volume_name', logical_volume.name)
cfg_fp = StringIO()
cfg.write(cfg_fp)
# deal with the possibility that zone and/or keypair are strings read from the config file:
if isinstance(zone, Zone):
zone = zone.name
if isinstance(kp, KeyPair):
kp = kp.name
reservation = ami.run(min_count=1,
max_count=params.get('quantity', 1),
key_name=kp,
security_groups=[group],
instance_type=params.get('instance_type'),
placement = zone,
user_data = cfg_fp.getvalue())
l = []
i = 0
elastic_ip = params.get('elastic_ip')
instances = reservation.instances
if elastic_ip is not None and instances.__len__() > 0:
instance = instances[0]
print('Waiting for instance to start so we can set its elastic IP address...')
# Sometimes we get a message from ec2 that says that the instance does not exist.
# Hopefully the following delay will giv eec2 enough time to get to a stable state:
time.sleep(5)
while instance.update() != 'running':
time.sleep(1)
instance.use_ip(elastic_ip)
print('set the elastic IP of the first instance to %s' % elastic_ip)
for instance in instances:
s = cls()
s.ec2 = ec2
s.name = params.get('name') + '' if i==0 else str(i)
s.description = params.get('description')
s.region_name = region.name
s.instance_id = instance.id
if elastic_ip and i == 0:
s.elastic_ip = elastic_ip
s.put()
l.append(s)
i += 1
return l
def get_userdata_string(self):
s = StringIO()
self.config.write(s)
return s.getvalue()
class ResponseGroup(xml.sax.ContentHandler):
"""A Generic "Response Group", which can
be anything from the entire list of Items to
specific response elements within an item"""
def __init__(self, connection=None, nodename=None):
"""Initialize this Item"""
self._connection = connection
self._nodename = nodename
self._nodepath = []
self._curobj = None
self._xml = StringIO()
def __repr__(self):
return '<%s: %s>' % (self.__class__.__name__, self.__dict__)
#
# Attribute Functions
#
def get(self, name):
return self.__dict__.get(name)
def set(self, name, value):
self.__dict__[name] = value
def to_xml(self):
return "<%s>%s</%s>" % (self._nodename, self._xml.getvalue(),
self._nodename)
#
# XML Parser functions
#
def startElement(self, name, attrs, connection):
self._xml.write("<%s>" % name)
self._nodepath.append(name)
if len(self._nodepath) == 1:
obj = ResponseGroup(self._connection)
self.set(name, obj)
self._curobj = obj
elif self._curobj:
self._curobj.startElement(name, attrs, connection)
return None
def endElement(self, name, value, connection):
self._xml.write(
"%s</%s>" %
(cgi.escape(value).replace("&amp;", "&"), name))
if len(self._nodepath) == 0:
return
obj = None
curval = self.get(name)
if len(self._nodepath) == 1:
if value or not curval:
self.set(name, value)
if self._curobj:
self._curobj = None
#elif len(self._nodepath) == 2:
#self._curobj = None
elif self._curobj:
self._curobj.endElement(name, value, connection)
self._nodepath.pop()
return None
def dump(self):
s = StringIO()
self.write(s)
print(s.getvalue())
def dump(self):
s = StringIO()
self.write(s)
print(s.getvalue())
def get_userdata_string(self):
s = StringIO()
self.config.write(s)
return s.getvalue()