Пример #1
0
 def _run(self, msg, vtimeout):
     boto.log.info('Task[%s] - running:%s' % (self.name, self.command))
     log_fp = StringIO()
     process = subprocess.Popen(self.command,
                                shell=True,
                                stdin=subprocess.PIPE,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
     nsecs = 5
     current_timeout = vtimeout
     while process.poll() is None:
         boto.log.info('nsecs=%s, timeout=%s' % (nsecs, current_timeout))
         if nsecs >= current_timeout:
             current_timeout += vtimeout
             boto.log.info('Task[%s] - setting timeout to %d seconds' %
                           (self.name, current_timeout))
             if msg:
                 msg.change_visibility(current_timeout)
         time.sleep(5)
         nsecs += 5
     t = process.communicate()
     log_fp.write(t[0])
     log_fp.write(t[1])
     boto.log.info('Task[%s] - output: %s' % (self.name, log_fp.getvalue()))
     self.last_executed = self.now
     self.last_status = process.returncode
     self.last_output = log_fp.getvalue()[0:1023]
Пример #2
0
 def run(self):
     boto.log.info('running:%s' % self.command)
     log_fp = StringIO()
     process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     while process.poll() is None:
         time.sleep(1)
         t = process.communicate()
         log_fp.write(t[0])
         log_fp.write(t[1])
     boto.log.info(log_fp.getvalue())
     boto.log.info('output: %s' % log_fp.getvalue())
     return (process.returncode, log_fp.getvalue())
Пример #3
0
 def do_start(self):
     ami_id = self.sd.get('ami_id')
     instance_type = self.sd.get('instance_type', 'm1.small')
     security_group = self.sd.get('security_group', 'default')
     if not ami_id:
         self.parser.error(
             'ami_id option is required when starting the service')
     ec2 = boto.connect_ec2()
     if not self.sd.has_section('Credentials'):
         self.sd.add_section('Credentials')
         self.sd.set('Credentials', 'aws_access_key_id',
                     ec2.aws_access_key_id)
         self.sd.set('Credentials', 'aws_secret_access_key',
                     ec2.aws_secret_access_key)
     s = StringIO()
     self.sd.write(s)
     rs = ec2.get_all_images([ami_id])
     img = rs[0]
     r = img.run(user_data=s.getvalue(),
                 key_name=self.options.keypair,
                 max_count=self.options.num_instances,
                 instance_type=instance_type,
                 security_groups=[security_group])
     print('Starting AMI: %s' % ami_id)
     print('Reservation %s contains the following instances:' % r.id)
     for i in r.instances:
         print('\t%s' % i.id)
Пример #4
0
    def get_contents_as_string(self,
                               headers=None,
                               cb=None,
                               num_cb=10,
                               torrent=False):
        """
        Retrieve file data from the Key, and return contents as a string.

        :type headers: dict
        :param headers: ignored in this subclass.

        :type cb: function
        :param cb: ignored in this subclass.

        :type cb: int
        :param num_cb: ignored in this subclass.

        :type cb: int
        :param num_cb: ignored in this subclass.

        :type torrent: bool
        :param torrent: ignored in this subclass.

        :rtype: string
        :returns: The contents of the file as a string
        """

        fp = StringIO()
        self.get_contents_to_file(fp)
        return fp.getvalue()
Пример #5
0
Файл: key.py Проект: 10sr/hue
    def get_contents_as_string(self, headers=None, cb=None, num_cb=10,
                               torrent=False):
        """
        Retrieve file data from the Key, and return contents as a string.

        :type headers: dict
        :param headers: ignored in this subclass.

        :type cb: function
        :param cb: ignored in this subclass.

        :type cb: int
        :param num_cb: ignored in this subclass.

        :type cb: int
        :param num_cb: ignored in this subclass.

        :type torrent: bool
        :param torrent: ignored in this subclass.

        :rtype: string
        :returns: The contents of the file as a string
        """

        fp = StringIO()
        self.get_contents_to_file(fp)
        return fp.getvalue()
Пример #6
0
    def start(self):
        self.stop()
        ec2 = boto.connect_ec2()
        ami = ec2.get_all_images(image_ids = [str(self.ami_id)])[0]
        groups = ec2.get_all_security_groups(groupnames=[str(self.security_group)])
        if not self._config:
            self.load_config()
        if not self._config.has_section("Credentials"):
            self._config.add_section("Credentials")
            self._config.set("Credentials", "aws_access_key_id", ec2.aws_access_key_id)
            self._config.set("Credentials", "aws_secret_access_key", ec2.aws_secret_access_key)

        if not self._config.has_section("Pyami"):
            self._config.add_section("Pyami")

        if self._manager.domain:
            self._config.set('Pyami', 'server_sdb_domain', self._manager.domain.name)
            self._config.set("Pyami", 'server_sdb_name', self.name)

        cfg = StringIO()
        self._config.write(cfg)
        cfg = cfg.getvalue()
        r = ami.run(min_count=1,
                    max_count=1,
                    key_name=self.key_name,
                    security_groups = groups,
                    instance_type = self.instance_type,
                    placement = self.zone,
                    user_data = cfg)
        i = r.instances[0]
        self.instance_id = i.id
        self.put()
        if self.elastic_ip:
            ec2.associate_address(self.instance_id, self.elastic_ip)
Пример #7
0
Файл: item.py Проект: 10sr/hue
class ResponseGroup(xml.sax.ContentHandler):
    """A Generic "Response Group", which can
    be anything from the entire list of Items to
    specific response elements within an item"""

    def __init__(self, connection=None, nodename=None):
        """Initialize this Item"""
        self._connection = connection
        self._nodename = nodename
        self._nodepath = []
        self._curobj = None
        self._xml = StringIO()

    def __repr__(self):
        return '<%s: %s>' % (self.__class__.__name__, self.__dict__)

    #
    # Attribute Functions
    #
    def get(self, name):
        return self.__dict__.get(name)

    def set(self, name, value):
        self.__dict__[name] = value

    def to_xml(self):
        return "<%s>%s</%s>" % (self._nodename, self._xml.getvalue(), self._nodename)

    #
    # XML Parser functions
    #
    def startElement(self, name, attrs, connection):
        self._xml.write("<%s>" % name)
        self._nodepath.append(name)
        if len(self._nodepath) == 1:
            obj = ResponseGroup(self._connection)
            self.set(name, obj)
            self._curobj = obj
        elif self._curobj:
            self._curobj.startElement(name, attrs, connection)
        return None

    def endElement(self, name, value, connection):
        self._xml.write("%s</%s>" % (cgi.escape(value).replace("&amp;amp;", "&amp;"), name))
        if len(self._nodepath) == 0:
            return
        obj = None
        curval = self.get(name)
        if len(self._nodepath) == 1:
            if value or not curval:
                self.set(name, value)
            if self._curobj:
                self._curobj = None
        #elif len(self._nodepath) == 2:
            #self._curobj = None
        elif self._curobj:
            self._curobj.endElement(name, value, connection)
        self._nodepath.pop()
        return None
Пример #8
0
class ShellCommand(object):

    def __init__(self, command, wait=True, fail_fast=False, cwd=None):
        self.exit_code = 0
        self.command = command
        self.log_fp = StringIO()
        self.wait = wait
        self.fail_fast = fail_fast
        self.run(cwd=cwd)

    def run(self, cwd=None):
        boto.log.info('running:%s' % self.command)
        self.process = subprocess.Popen(self.command, shell=True,
                                        stdin=subprocess.PIPE,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE,
                                        cwd=cwd)
        if(self.wait):
            while self.process.poll() is None:
                time.sleep(1)
                t = self.process.communicate()
                self.log_fp.write(t[0])
                self.log_fp.write(t[1])
            boto.log.info(self.log_fp.getvalue())
            self.exit_code = self.process.returncode

            if self.fail_fast and self.exit_code != 0:
                raise Exception("Command " + self.command +
                                " failed with status " + self.exit_code)

            return self.exit_code

    def setReadOnly(self, value):
        raise AttributeError

    def getStatus(self):
        return self.exit_code

    status = property(getStatus, setReadOnly, None,
                      'The exit code for the command')

    def getOutput(self):
        return self.log_fp.getvalue()

    output = property(getOutput, setReadOnly, None,
                      'The STDIN and STDERR output of the command')
Пример #9
0
 def run(self):
     """
     Open a subprocess and run a command on the local host.
     
     :rtype: tuple
     :return: This function returns a tuple that contains an integer status
             and a string with the combined stdout and stderr output.
     """
     boto.log.info('running:%s' % self.command)
     log_fp = StringIO()
     process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     while process.poll() is None:
         time.sleep(1)
         t = process.communicate()
         log_fp.write(t[0])
         log_fp.write(t[1])
     boto.log.info(log_fp.getvalue())
     boto.log.info('output: %s' % log_fp.getvalue())
     return (process.returncode, log_fp.getvalue())
Пример #10
0
    def run(self):
        """
        Open a subprocess and run a command on the local host.

        :rtype: tuple
        :return: This function returns a tuple that contains an integer status
                and a string with the combined stdout and stderr output.
        """
        boto.log.info('running:%s' % self.command)
        log_fp = StringIO()
        process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        while process.poll() is None:
            time.sleep(1)
            t = process.communicate()
            log_fp.write(t[0])
            log_fp.write(t[1])
        boto.log.info(log_fp.getvalue())
        boto.log.info('output: %s' % log_fp.getvalue())
        return (process.returncode, log_fp.getvalue())
Пример #11
0
Файл: task.py Проект: 10sr/hue
 def _run(self, msg, vtimeout):
     boto.log.info('Task[%s] - running:%s' % (self.name, self.command))
     log_fp = StringIO()
     process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     nsecs = 5
     current_timeout = vtimeout
     while process.poll() is None:
         boto.log.info('nsecs=%s, timeout=%s' % (nsecs, current_timeout))
         if nsecs >= current_timeout:
             current_timeout += vtimeout
             boto.log.info('Task[%s] - setting timeout to %d seconds' % (self.name, current_timeout))
             if msg:
                 msg.change_visibility(current_timeout)
         time.sleep(5)
         nsecs += 5
     t = process.communicate()
     log_fp.write(t[0])
     log_fp.write(t[1])
     boto.log.info('Task[%s] - output: %s' % (self.name, log_fp.getvalue()))
     self.last_executed = self.now
     self.last_status = process.returncode
     self.last_output = log_fp.getvalue()[0:1023]
Пример #12
0
 def bundle(self,
            bucket=None,
            prefix=None,
            key_file=None,
            cert_file=None,
            size=None,
            ssh_key=None,
            fp=None,
            clear_history=True):
     iobject = IObject()
     if not bucket:
         bucket = iobject.get_string('Name of S3 bucket')
     if not prefix:
         prefix = iobject.get_string('Prefix for AMI file')
     if not key_file:
         key_file = iobject.get_filename('Path to RSA private key file')
     if not cert_file:
         cert_file = iobject.get_filename('Path to RSA public cert file')
     if not size:
         size = iobject.get_int('Size (in MB) of bundled image')
     if not ssh_key:
         ssh_key = self.server.get_ssh_key_file()
     self.copy_x509(key_file, cert_file)
     if not fp:
         fp = StringIO()
     fp.write('sudo mv %s /mnt/boto.cfg; ' % BotoConfigPath)
     fp.write('mv ~/.ssh/authorized_keys /mnt/authorized_keys; ')
     if clear_history:
         fp.write('history -c; ')
     fp.write(self.bundle_image(prefix, size, ssh_key))
     fp.write('; ')
     fp.write(self.upload_bundle(bucket, prefix, ssh_key))
     fp.write('; ')
     fp.write('sudo mv /mnt/boto.cfg %s; ' % BotoConfigPath)
     fp.write('mv /mnt/authorized_keys ~/.ssh/authorized_keys')
     command = fp.getvalue()
     print('running the following command on the remote server:')
     print(command)
     t = self.ssh_client.run(command)
     print('\t%s' % t[0])
     print('\t%s' % t[1])
     print('...complete!')
     print('registering image...')
     self.image_id = self.server.ec2.register_image(
         name=prefix,
         image_location='%s/%s.manifest.xml' % (bucket, prefix))
     return self.image_id
Пример #13
0
def write_mime_multipart(content,
                         compress=False,
                         deftype='text/plain',
                         delimiter=':'):
    """Description:
    :param content: A list of tuples of name-content pairs. This is used
    instead of a dict to ensure that scripts run in order
    :type list of tuples:

    :param compress: Use gzip to compress the scripts, defaults to no compression
    :type bool:

    :param deftype: The type that should be assumed if nothing else can be figured out
    :type str:

    :param delimiter: mime delimiter
    :type str:

    :return: Final mime multipart
    :rtype: str:
    """
    wrapper = email.mime.multipart.MIMEMultipart()
    for name, con in content:
        definite_type = guess_mime_type(con, deftype)
        maintype, subtype = definite_type.split('/', 1)
        if maintype == 'text':
            mime_con = email.mime.text.MIMEText(con, _subtype=subtype)
        else:
            mime_con = email.mime.base.MIMEBase(maintype, subtype)
            mime_con.set_payload(con)
            # Encode the payload using Base64
            email.encoders.encode_base64(mime_con)
        mime_con.add_header('Content-Disposition', 'attachment', filename=name)
        wrapper.attach(mime_con)
    rcontent = wrapper.as_string()

    if compress:
        buf = StringIO()
        gz = gzip.GzipFile(mode='wb', fileobj=buf)
        try:
            gz.write(rcontent)
        finally:
            gz.close()
        rcontent = buf.getvalue()

    return rcontent
Пример #14
0
Файл: utils.py Проект: 10sr/hue
def write_mime_multipart(content, compress=False, deftype='text/plain', delimiter=':'):
    """Description:
    :param content: A list of tuples of name-content pairs. This is used
    instead of a dict to ensure that scripts run in order
    :type list of tuples:

    :param compress: Use gzip to compress the scripts, defaults to no compression
    :type bool:

    :param deftype: The type that should be assumed if nothing else can be figured out
    :type str:

    :param delimiter: mime delimiter
    :type str:

    :return: Final mime multipart
    :rtype: str:
    """
    wrapper = email.mime.multipart.MIMEMultipart()
    for name, con in content:
        definite_type = guess_mime_type(con, deftype)
        maintype, subtype = definite_type.split('/', 1)
        if maintype == 'text':
            mime_con = email.mime.text.MIMEText(con, _subtype=subtype)
        else:
            mime_con = email.mime.base.MIMEBase(maintype, subtype)
            mime_con.set_payload(con)
            # Encode the payload using Base64
            email.encoders.encode_base64(mime_con)
        mime_con.add_header('Content-Disposition', 'attachment', filename=name)
        wrapper.attach(mime_con)
    rcontent = wrapper.as_string()

    if compress:
        buf = StringIO()
        gz = gzip.GzipFile(mode='wb', fileobj=buf)
        try:
            gz.write(rcontent)
        finally:
            gz.close()
        rcontent = buf.getvalue()

    return rcontent
Пример #15
0
 def bundle(self, bucket=None, prefix=None, key_file=None, cert_file=None,
            size=None, ssh_key=None, fp=None, clear_history=True):
     iobject = IObject()
     if not bucket:
         bucket = iobject.get_string('Name of S3 bucket')
     if not prefix:
         prefix = iobject.get_string('Prefix for AMI file')
     if not key_file:
         key_file = iobject.get_filename('Path to RSA private key file')
     if not cert_file:
         cert_file = iobject.get_filename('Path to RSA public cert file')
     if not size:
         size = iobject.get_int('Size (in MB) of bundled image')
     if not ssh_key:
         ssh_key = self.server.get_ssh_key_file()
     self.copy_x509(key_file, cert_file)
     if not fp:
         fp = StringIO()
     fp.write('sudo mv %s /mnt/boto.cfg; ' % BotoConfigPath)
     fp.write('mv ~/.ssh/authorized_keys /mnt/authorized_keys; ')
     if clear_history:
         fp.write('history -c; ')
     fp.write(self.bundle_image(prefix, size, ssh_key))
     fp.write('; ')
     fp.write(self.upload_bundle(bucket, prefix, ssh_key))
     fp.write('; ')
     fp.write('sudo mv /mnt/boto.cfg %s; ' % BotoConfigPath)
     fp.write('mv /mnt/authorized_keys ~/.ssh/authorized_keys')
     command = fp.getvalue()
     print('running the following command on the remote server:')
     print(command)
     t = self.ssh_client.run(command)
     print('\t%s' % t[0])
     print('\t%s' % t[1])
     print('...complete!')
     print('registering image...')
     self.image_id = self.server.ec2.register_image(name=prefix, image_location='%s/%s.manifest.xml' % (bucket, prefix))
     return self.image_id
Пример #16
0
Файл: bs.py Проект: 10sr/hue
 def do_start(self):
     ami_id = self.sd.get('ami_id')
     instance_type = self.sd.get('instance_type', 'm1.small')
     security_group = self.sd.get('security_group', 'default')
     if not ami_id:
         self.parser.error('ami_id option is required when starting the service')
     ec2 = boto.connect_ec2()
     if not self.sd.has_section('Credentials'):
         self.sd.add_section('Credentials')
         self.sd.set('Credentials', 'aws_access_key_id', ec2.aws_access_key_id)
         self.sd.set('Credentials', 'aws_secret_access_key', ec2.aws_secret_access_key)
     s = StringIO()
     self.sd.write(s)
     rs = ec2.get_all_images([ami_id])
     img = rs[0]
     r = img.run(user_data=s.getvalue(), key_name=self.options.keypair,
                 max_count=self.options.num_instances,
                 instance_type=instance_type,
                 security_groups=[security_group])
     print('Starting AMI: %s' % ami_id)
     print('Reservation %s contains the following instances:' % r.id)
     for i in r.instances:
         print('\t%s' % i.id)
Пример #17
0
    def create(cls, config_file=None, logical_volume = None, cfg = None, **params):
        """
        Create a new instance based on the specified configuration file or the specified
        configuration and the passed in parameters.

        If the config_file argument is not None, the configuration is read from there.
        Otherwise, the cfg argument is used.

        The config file may include other config files with a #import reference. The included
        config files must reside in the same directory as the specified file.

        The logical_volume argument, if supplied, will be used to get the current physical
        volume ID and use that as an override of the value specified in the config file. This
        may be useful for debugging purposes when you want to debug with a production config
        file but a test Volume.

        The dictionary argument may be used to override any EC2 configuration values in the
        config file.
        """
        if config_file:
            cfg = Config(path=config_file)
        if cfg.has_section('EC2'):
            # include any EC2 configuration values that aren't specified in params:
            for option in cfg.options('EC2'):
                if option not in params:
                    params[option] = cfg.get('EC2', option)
        getter = CommandLineGetter()
        getter.get(cls, params)
        region = params.get('region')
        ec2 = region.connect()
        cls.add_credentials(cfg, ec2.aws_access_key_id, ec2.aws_secret_access_key)
        ami = params.get('ami')
        kp = params.get('keypair')
        group = params.get('group')
        zone = params.get('zone')
        # deal with possibly passed in logical volume:
        if logical_volume != None:
           cfg.set('EBS', 'logical_volume_name', logical_volume.name)
        cfg_fp = StringIO()
        cfg.write(cfg_fp)
        # deal with the possibility that zone and/or keypair are strings read from the config file:
        if isinstance(zone, Zone):
            zone = zone.name
        if isinstance(kp, KeyPair):
            kp = kp.name
        reservation = ami.run(min_count=1,
                              max_count=params.get('quantity', 1),
                              key_name=kp,
                              security_groups=[group],
                              instance_type=params.get('instance_type'),
                              placement = zone,
                              user_data = cfg_fp.getvalue())
        l = []
        i = 0
        elastic_ip = params.get('elastic_ip')
        instances = reservation.instances
        if elastic_ip is not None and instances.__len__() > 0:
            instance = instances[0]
            print('Waiting for instance to start so we can set its elastic IP address...')
            # Sometimes we get a message from ec2 that says that the instance does not exist.
            # Hopefully the following delay will giv eec2 enough time to get to a stable state:
            time.sleep(5)
            while instance.update() != 'running':
                time.sleep(1)
            instance.use_ip(elastic_ip)
            print('set the elastic IP of the first instance to %s' % elastic_ip)
        for instance in instances:
            s = cls()
            s.ec2 = ec2
            s.name = params.get('name') + '' if i==0 else str(i)
            s.description = params.get('description')
            s.region_name = region.name
            s.instance_id = instance.id
            if elastic_ip and i == 0:
                s.elastic_ip = elastic_ip
            s.put()
            l.append(s)
            i += 1
        return l
Пример #18
0
 def get_userdata_string(self):
     s = StringIO()
     self.config.write(s)
     return s.getvalue()
Пример #19
0
class ResponseGroup(xml.sax.ContentHandler):
    """A Generic "Response Group", which can
    be anything from the entire list of Items to
    specific response elements within an item"""
    def __init__(self, connection=None, nodename=None):
        """Initialize this Item"""
        self._connection = connection
        self._nodename = nodename
        self._nodepath = []
        self._curobj = None
        self._xml = StringIO()

    def __repr__(self):
        return '<%s: %s>' % (self.__class__.__name__, self.__dict__)

    #
    # Attribute Functions
    #
    def get(self, name):
        return self.__dict__.get(name)

    def set(self, name, value):
        self.__dict__[name] = value

    def to_xml(self):
        return "<%s>%s</%s>" % (self._nodename, self._xml.getvalue(),
                                self._nodename)

    #
    # XML Parser functions
    #
    def startElement(self, name, attrs, connection):
        self._xml.write("<%s>" % name)
        self._nodepath.append(name)
        if len(self._nodepath) == 1:
            obj = ResponseGroup(self._connection)
            self.set(name, obj)
            self._curobj = obj
        elif self._curobj:
            self._curobj.startElement(name, attrs, connection)
        return None

    def endElement(self, name, value, connection):
        self._xml.write(
            "%s</%s>" %
            (cgi.escape(value).replace("&amp;amp;", "&amp;"), name))
        if len(self._nodepath) == 0:
            return
        obj = None
        curval = self.get(name)
        if len(self._nodepath) == 1:
            if value or not curval:
                self.set(name, value)
            if self._curobj:
                self._curobj = None
        #elif len(self._nodepath) == 2:
        #self._curobj = None
        elif self._curobj:
            self._curobj.endElement(name, value, connection)
        self._nodepath.pop()
        return None
Пример #20
0
 def dump(self):
     s = StringIO()
     self.write(s)
     print(s.getvalue())
Пример #21
0
 def dump(self):
     s = StringIO()
     self.write(s)
     print(s.getvalue())
Пример #22
0
 def get_userdata_string(self):
     s = StringIO()
     self.config.write(s)
     return s.getvalue()