def create_service_dns(self, elb, zone, vpc_id): """ """ records = self.r53.get_all_rrsets(zone) old_names = [r.name for r in records] HOST_TEMPLATE = "{service}.{vpc_id}" + self.DNS_SUFFIX service = self.get_elb_service(elb) dns_name = HOST_TEMPLATE.format(service=service, vpc_id=vpc_id) change_set = ResourceRecordSets() if dns_name + '.' in old_names: print "adding delete" change = change_set.add_change('DELETE', dns_name, 'CNAME', 600) change.add_value(elb.dns_name) change = change_set.add_change('CREATE', dns_name, 'CNAME', 600) change.add_value(elb.dns_name) print change_set.to_xml() self.r53.change_rrsets(zone, change_set.to_xml())
def create_service_dns(self, elb, zone, vpc_id): """ """ records = self.r53.get_all_rrsets(zone) old_names = [r.name for r in records] HOST_TEMPLATE = "{service}.{vpc_id}" + self.DNS_SUFFIX service = self.get_elb_service(elb) dns_name = HOST_TEMPLATE.format(service=service, vpc_id=vpc_id) change_set = ResourceRecordSets() if dns_name + "." in old_names: print "adding delete" change = change_set.add_change("DELETE", dns_name, "CNAME", 600) change.add_value(elb.dns_name) change = change_set.add_change("CREATE", dns_name, "CNAME", 600) change.add_value(elb.dns_name) print change_set.to_xml() self.r53.change_rrsets(zone, change_set.to_xml())
def _uninstall(self): logger = self.get_logger('uninstall') self.terminate_instances() service_map = dict((s.name, s) for s in self.services) if self.route53_hosted_zone_id and self.route53_records: changeset = ResourceRecordSets( self.app.route53_connection, self.route53_hosted_zone_id, 'Changed by Asuka: {0}, {1} [clean]'.format(self.app.name, self.branch.label) ) from .service import DomainService for service_name, domain_format in self.route53_records.items(): service = service_map[service_name] if not isinstance(service, DomainService): raise TypeError(repr(service) + 'is not an instance of ' 'crosspop.service.DomainService') domain = domain_format.format(branch=self.branch) service.remove_domain(domain, changeset) if changeset.changes: logger.info('Route 53 changeset:\n%s', changeset.to_xml()) changeset.commit() for name, service in service_map.iteritems(): logger.info('Uninstall %s...', name) service.uninstall() logger.info('Uninstalled %s', name)
def do_delete(self, line): if not self.shell.amazon_access_key_id or not self.shell.amazon_secret_key: print "Authentication variables are not set." return parts = line.split(' ') count = len(parts) if count == 1 and not self.zone: print "Zone not selected. Either pass a zone ID or set it with the zone command" return (record_name, record_type, record_ttl, record_value) = self.parse_change_command(parts) print "Delete:\n\tName:\t%(name)s\n\tType:\t%(type)s\n\tTTL:\t%(ttl)d\n\tValue:\t%(value)s" % dict( name=record_name, type=record_type, ttl=record_ttl, value=record_value) conn = Route53Connection(self.shell.amazon_access_key_id, self.shell.amazon_secret_key) r = ResourceRecordSets(conn, self.zone) r.add_change('DELETE', record_name, record_type, record_ttl) if record_value: r.changes[0][1].add_value(record_value) change = conn.change_rrsets(self.zone, r.to_xml()) print change
def do_add(self, line): """ Add a new record """ parts = line.split(' ') count = len(parts) if count == 0 and not self.zone: print "Zone not selected. Either pass a zone ID or set it with the zone command" return (record_name, record_type, record_ttl, record_value) = self.parse_change_command(parts) print "Create:\n\tName:\t%(name)s\n\tType:\t%(type)s\n\tTTL:\t%(ttl)d\n\tValue:\t%(value)s" % dict( name=record_name, type=record_type, ttl=record_ttl, value=record_value) conn = Route53Connection(self.shell.amazon_access_key_id, self.shell.amazon_secret_key) r = ResourceRecordSets(conn, self.zone) r.add_change('CREATE', record_name, record_type, record_ttl) if record_value: r.changes[0][1].add_value(record_value) change = conn.change_rrsets(self.zone, r.to_xml()) print change
def do_change(self, line): if not self.shell.amazon_access_key_id or not self.shell.amazon_secret_key: print "Authentication variables are not set." return parts = line.split(' ') count = len(parts) if count == 1 and not self.zone: print "Zone not selected. Either pass a zone ID or set it with the zone command" return (record_name, record_type, record_ttl, record_value) = self.parse_change_command(parts) conn = Route53Connection(self.shell.amazon_access_key_id, self.shell.amazon_secret_key) records = conn.get_all_rrsets(self.zone) this_record = None for record in records: if record.name == record_name or record.name == "%s." % record_name: this_record = record break print "Current:\n\tName:\t%(name)s\n\tType:\t%(type)s\n\tTTL:\t%(ttl)s\n\tValue:\t%(value)s" % dict( name=this_record.name, type=this_record.type, ttl=this_record.ttl, value=this_record.resource_records) print "New:\n\tName:\t%(name)s\n\tType:\t%(type)s\n\tTTL:\t%(ttl)d\n\tValue:\t%(value)s" % dict( name=record_name, type=record_type, ttl=record_ttl, value=record_value) r = ResourceRecordSets(conn, self.zone) r.add_change('DELETE', record_name, record_type, record_ttl) for value in this_record.resource_records: r.changes[0][1].add_value(value) r.add_change('CREATE', record_name, record_type, record_ttl) if record_value: r.changes[1][1].add_value(record_value) # print r.to_xml() change = conn.change_rrsets(self.zone, r.to_xml()) print change
def test_record_commit(self): rrsets = ResourceRecordSets(self.service_connection) rrsets.add_change_record( 'CREATE', Record('vanilla.example.com', 'A', 60, ['1.2.3.4'])) rrsets.add_change_record( 'CREATE', Record('alias.example.com', 'AAAA', alias_hosted_zone_id='Z123OTHER', alias_dns_name='target.other', alias_evaluate_target_health=True)) rrsets.add_change_record( 'CREATE', Record('wrr.example.com', 'CNAME', 60, ['cname.target'], weight=10, identifier='weight-1')) rrsets.add_change_record( 'CREATE', Record('lbr.example.com', 'TXT', 60, ['text record'], region='us-west-2', identifier='region-1')) rrsets.add_change_record( 'CREATE', Record('failover.example.com', 'A', 60, ['2.2.2.2'], health_check='hc-1234', failover='PRIMARY', identifier='primary')) changes_xml = rrsets.to_xml() # the whitespacing doesn't match exactly, so we'll pretty print and drop all new lines # not the best, but actual_xml = re.sub( r"\s*[\r\n]+", "\n", xml.dom.minidom.parseString(changes_xml).toprettyxml()) expected_xml = re.sub( r"\s*[\r\n]+", "\n", xml.dom.minidom.parseString(b""" <ChangeResourceRecordSetsRequest xmlns="https://route53.amazonaws.com/doc/2013-04-01/"> <ChangeBatch> <Comment>None</Comment> <Changes> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>vanilla.example.com</Name> <Type>A</Type> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>1.2.3.4</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> </Change> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>alias.example.com</Name> <Type>AAAA</Type> <AliasTarget> <HostedZoneId>Z123OTHER</HostedZoneId> <DNSName>target.other</DNSName> <EvaluateTargetHealth>true</EvaluateTargetHealth> </AliasTarget> </ResourceRecordSet> </Change> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>wrr.example.com</Name> <Type>CNAME</Type> <SetIdentifier>weight-1</SetIdentifier> <Weight>10</Weight> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>cname.target</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> </Change> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>lbr.example.com</Name> <Type>TXT</Type> <SetIdentifier>region-1</SetIdentifier> <Region>us-west-2</Region> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>text record</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> </Change> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>failover.example.com</Name> <Type>A</Type> <SetIdentifier>primary</SetIdentifier> <Failover>PRIMARY</Failover> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>2.2.2.2</Value> </ResourceRecord> </ResourceRecords> <HealthCheckId>hc-1234</HealthCheckId> </ResourceRecordSet> </Change> </Changes> </ChangeBatch> </ChangeResourceRecordSetsRequest> """).toprettyxml()) # Note: the alias XML should not include the TTL, even if it's specified in the object model self.assertEqual(actual_xml, expected_xml)
def test_record_commit(self): rrsets = ResourceRecordSets(self.service_connection) rrsets.add_change_record('CREATE', Record('vanilla.example.com', 'A', 60, ['1.2.3.4'])) rrsets.add_change_record('CREATE', Record('alias.example.com', 'AAAA', alias_hosted_zone_id='Z123OTHER', alias_dns_name='target.other', alias_evaluate_target_health=True)) rrsets.add_change_record('CREATE', Record('wrr.example.com', 'CNAME', 60, ['cname.target'], weight=10, identifier='weight-1')) rrsets.add_change_record('CREATE', Record('lbr.example.com', 'TXT', 60, ['text record'], region='us-west-2', identifier='region-1')) rrsets.add_change_record('CREATE', Record('failover.example.com', 'A', 60, ['2.2.2.2'], health_check='hc-1234', failover='PRIMARY', identifier='primary')) changes_xml = rrsets.to_xml() # the whitespacing doesn't match exactly, so we'll pretty print and drop all new lines # not the best, but actual_xml = re.sub(r"\s*[\r\n]+", "\n", xml.dom.minidom.parseString(changes_xml).toprettyxml()) expected_xml = re.sub(r"\s*[\r\n]+", "\n", xml.dom.minidom.parseString(b""" <ChangeResourceRecordSetsRequest xmlns="https://route53.amazonaws.com/doc/2013-04-01/"> <ChangeBatch> <Comment>None</Comment> <Changes> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>vanilla.example.com</Name> <Type>A</Type> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>1.2.3.4</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> </Change> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>alias.example.com</Name> <Type>AAAA</Type> <AliasTarget> <HostedZoneId>Z123OTHER</HostedZoneId> <DNSName>target.other</DNSName> <EvaluateTargetHealth>true</EvaluateTargetHealth> </AliasTarget> </ResourceRecordSet> </Change> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>wrr.example.com</Name> <Type>CNAME</Type> <SetIdentifier>weight-1</SetIdentifier> <Weight>10</Weight> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>cname.target</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> </Change> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>lbr.example.com</Name> <Type>TXT</Type> <SetIdentifier>region-1</SetIdentifier> <Region>us-west-2</Region> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>text record</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> </Change> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>failover.example.com</Name> <Type>A</Type> <SetIdentifier>primary</SetIdentifier> <Failover>PRIMARY</Failover> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>2.2.2.2</Value> </ResourceRecord> </ResourceRecords> <HealthCheckId>hc-1234</HealthCheckId> </ResourceRecordSet> </Change> </Changes> </ChangeBatch> </ChangeResourceRecordSetsRequest> """).toprettyxml()) # Note: the alias XML should not include the TTL, even if it's specified in the object model self.assertEqual(actual_xml, expected_xml)
def _install(self): logger = self.get_logger('install') sudo = self.instance.sudo logger.info( 'START TO INSTALL: branch = %r, commit = %r, instance = %r', self.branch, self.commit, self.instance ) def setup_instance(service_manifests, service_manifests_available): logger = self.get_logger('install.setup_instance') with self.instance: def aptitude(*commands): sudo(['aptitude', '-y'] + list(commands), environ={'DEBIAN_FRONTEND': 'noninteractive'}) # create user for app sudo(['useradd', '-U', '-G', 'users,www-data', '-Mr', self.app.name]) # assume instance uses Ubuntu >= 12.04 apt_sources = re.sub( r'\n#\s*(deb(?:-src)?\s+' r'http://[^.]\.ec2\.archive\.ubuntu\.com/' r'ubuntu/\s+[^-]+multiverse\n)', lambda m: '\n' + m.group(1), self.instance.read_file('/etc/apt/sources.list', sudo=True) ) self.instance.write_file('/etc/apt/sources.list', apt_sources, sudo=True) apt_repos = set() apt_packages = set([ 'build-essential', 'python-dev', 'python-setuptools', 'python-pip' ]) with service_manifests_available: while not service_manifests[0]: service_manifests_available.wait() for service in service_manifests[1:]: apt_repos.update(service.required_apt_repositories) apt_packages.update(service.required_apt_packages) if apt_repos: for repo in apt_repos: sudo(['apt-add-repository', '-y', repo]) aptitude('update') with self.instance.sftp(): self.instance.write_file( '/usr/bin/apt-fast', resource_string(__name__, 'apt-fast'), sudo=True ) self.instance.write_file('/etc/apt-fast.conf', ''' _APTMGR=aptitude DOWNLOADBEFORE=true _MAXNUM=20 DLLIST='/tmp/apt-fast.list' _DOWNLOADER='aria2c -c -j ${_MAXNUM} -i ${DLLIST} --connect-timeout=10 \ --timeout=600 -m0' DLDIR='/var/cache/apt/archives/apt-fast' APTCACHE='/var/cache/apt/archives/' ''', sudo=True) sudo(['chmod', '+x', '/usr/bin/apt-fast']) aptitude('install', 'aria2') sudo(['apt-fast', '-q', '-y', 'install'] + list(apt_packages), environ={'DEBIAN_FRONTEND': 'noninteractive'}) service_manifests_available = threading.Condition() service_manifests = [False] instance_setup_worker = threading.Thread( target=setup_instance, kwargs={ 'service_manifests_available': service_manifests_available, 'service_manifests': service_manifests } ) instance_setup_worker.start() # setup metadata of the instance self.update_instance_metadata() self.instance.status = 'started' # making package (pybundle) fd, package_path = tempfile.mkstemp() os.close(fd) with self.fetch() as download_path: service_manifests.extend(self.services) service_manifests[0] = True with service_manifests_available: service_manifests_available.notify() config_temp_path = tempfile.mkdtemp() shutil.copytree( os.path.join(download_path, self.app.config_dir), os.path.join(config_temp_path, self.app.name) ) with self.dist.bundle_package() as (package, filename, temp_path): shutil.copyfile(temp_path, package_path) remote_path = os.path.join('/tmp', filename) with self.instance.sftp(): # upload config files self.instance.put_directory( os.path.join(config_temp_path, self.app.name), '/etc/' + self.app.name, sudo=True ) shutil.rmtree(config_temp_path) python_packages = set() for service in service_manifests[1:]: python_packages.update(service.required_python_packages) # uploads package self.instance.put_file(package_path, remote_path) # join instance_setup_worker instance_setup_worker.join() self.instance.status = 'apt-installed' pip_cmd = ['pip', 'install', '-i', PYPI_INDEX_URLS[0]] for idx in PYPI_INDEX_URLS[1:]: pip_cmd.append('--extra-index-url=' + idx) sudo(pip_cmd + [remote_path], environ={'CI': '1'}) sudo(pip_cmd + ['-I'] + list(python_packages), environ={'CI': '1'}) self.instance.status = 'installed' for service in service_manifests[1:]: for cmd in service.pre_install: sudo(cmd, environ={'DEBIAN_FRONTEND': 'noninteractive'}) values_path = '/etc/{0}/values.json'.format(self.app.name) service_values = { '.build': dict( commit=self.commit.ref, branch=self.branch.label ) } refresh_values = lambda: self.instance.write_file( values_path, json.dumps(service_values), sudo=True ) refresh_values() for service in service_manifests[1:]: service_value = service.install(self.instance) service_values[service.name] = service_value refresh_values() for service in service_manifests[1:]: for cmd in service.post_install: sudo(cmd, environ={'DEBIAN_FRONTEND': 'noninteractive'}) service_map = dict((service.name, service) for service in service_manifests[1:]) deployed_domains = {} if self.route53_hosted_zone_id and self.route53_records: self.instance.status = 'run' changeset = ResourceRecordSets( self.app.route53_connection, self.route53_hosted_zone_id, 'Changed by Asuka: {0}, {1} [{2}]'.format(self.app.name, self.branch.label, self.commit.ref) ) from .service import DomainService for service_name, domain_format in self.route53_records.items(): service = service_map[service_name] if not isinstance(service, DomainService): raise TypeError(repr(service) + 'is not an instance of ' 'crosspop.service.DomainService') domain = domain_format.format(branch=self.branch) deployed_domains[service_name] = domain service.route_domain(domain, changeset) self.instance.tags['Domain-' + service_name] = domain if changeset.changes: logger.info('Route 53 changeset:\n%s', changeset.to_xml()) changeset.commit() self.instance.status = 'done' self.terminate_instances() return deployed_domains