def create_service_dns(self, elb, zone, vpc_id):
"""
"""
records = self.r53.get_all_rrsets(zone)
old_names = [r.name for r in records]
HOST_TEMPLATE = "{service}.{vpc_id}" + self.DNS_SUFFIX
service = self.get_elb_service(elb)
dns_name = HOST_TEMPLATE.format(service=service, vpc_id=vpc_id)
change_set = ResourceRecordSets()
if dns_name + '.' in old_names:
print "adding delete"
change = change_set.add_change('DELETE', dns_name, 'CNAME', 600)
change.add_value(elb.dns_name)
change = change_set.add_change('CREATE', dns_name, 'CNAME', 600)
change.add_value(elb.dns_name)
print change_set.to_xml()
self.r53.change_rrsets(zone, change_set.to_xml())
def create_service_dns(self, elb, zone, vpc_id):
"""
"""
records = self.r53.get_all_rrsets(zone)
old_names = [r.name for r in records]
HOST_TEMPLATE = "{service}.{vpc_id}" + self.DNS_SUFFIX
service = self.get_elb_service(elb)
dns_name = HOST_TEMPLATE.format(service=service, vpc_id=vpc_id)
change_set = ResourceRecordSets()
if dns_name + "." in old_names:
print "adding delete"
change = change_set.add_change("DELETE", dns_name, "CNAME", 600)
change.add_value(elb.dns_name)
change = change_set.add_change("CREATE", dns_name, "CNAME", 600)
change.add_value(elb.dns_name)
print change_set.to_xml()
self.r53.change_rrsets(zone, change_set.to_xml())
def _uninstall(self):
logger = self.get_logger('uninstall')
self.terminate_instances()
service_map = dict((s.name, s) for s in self.services)
if self.route53_hosted_zone_id and self.route53_records:
changeset = ResourceRecordSets(
self.app.route53_connection,
self.route53_hosted_zone_id,
'Changed by Asuka: {0}, {1} [clean]'.format(self.app.name,
self.branch.label)
)
from .service import DomainService
for service_name, domain_format in self.route53_records.items():
service = service_map[service_name]
if not isinstance(service, DomainService):
raise TypeError(repr(service) + 'is not an instance of '
'crosspop.service.DomainService')
domain = domain_format.format(branch=self.branch)
service.remove_domain(domain, changeset)
if changeset.changes:
logger.info('Route 53 changeset:\n%s', changeset.to_xml())
changeset.commit()
for name, service in service_map.iteritems():
logger.info('Uninstall %s...', name)
service.uninstall()
logger.info('Uninstalled %s', name)
def do_delete(self, line):
if not self.shell.amazon_access_key_id or not self.shell.amazon_secret_key:
print "Authentication variables are not set."
return
parts = line.split(' ')
count = len(parts)
if count == 1 and not self.zone:
print "Zone not selected. Either pass a zone ID or set it with the zone command"
return
(record_name, record_type, record_ttl, record_value) = self.parse_change_command(parts)
print "Delete:\n\tName:\t%(name)s\n\tType:\t%(type)s\n\tTTL:\t%(ttl)d\n\tValue:\t%(value)s" % dict(
name=record_name, type=record_type, ttl=record_ttl, value=record_value)
conn = Route53Connection(self.shell.amazon_access_key_id, self.shell.amazon_secret_key)
r = ResourceRecordSets(conn, self.zone)
r.add_change('DELETE', record_name, record_type, record_ttl)
if record_value:
r.changes[0][1].add_value(record_value)
change = conn.change_rrsets(self.zone, r.to_xml())
print change
def do_add(self, line):
"""
Add a new record
"""
parts = line.split(' ')
count = len(parts)
if count == 0 and not self.zone:
print "Zone not selected. Either pass a zone ID or set it with the zone command"
return
(record_name, record_type, record_ttl, record_value) = self.parse_change_command(parts)
print "Create:\n\tName:\t%(name)s\n\tType:\t%(type)s\n\tTTL:\t%(ttl)d\n\tValue:\t%(value)s" % dict(
name=record_name, type=record_type, ttl=record_ttl, value=record_value)
conn = Route53Connection(self.shell.amazon_access_key_id, self.shell.amazon_secret_key)
r = ResourceRecordSets(conn, self.zone)
r.add_change('CREATE', record_name, record_type, record_ttl)
if record_value:
r.changes[0][1].add_value(record_value)
change = conn.change_rrsets(self.zone, r.to_xml())
print change
def do_change(self, line):
if not self.shell.amazon_access_key_id or not self.shell.amazon_secret_key:
print "Authentication variables are not set."
return
parts = line.split(' ')
count = len(parts)
if count == 1 and not self.zone:
print "Zone not selected. Either pass a zone ID or set it with the zone command"
return
(record_name, record_type, record_ttl, record_value) = self.parse_change_command(parts)
conn = Route53Connection(self.shell.amazon_access_key_id, self.shell.amazon_secret_key)
records = conn.get_all_rrsets(self.zone)
this_record = None
for record in records:
if record.name == record_name or record.name == "%s." % record_name:
this_record = record
break
print "Current:\n\tName:\t%(name)s\n\tType:\t%(type)s\n\tTTL:\t%(ttl)s\n\tValue:\t%(value)s" % dict(
name=this_record.name, type=this_record.type, ttl=this_record.ttl, value=this_record.resource_records)
print "New:\n\tName:\t%(name)s\n\tType:\t%(type)s\n\tTTL:\t%(ttl)d\n\tValue:\t%(value)s" % dict(
name=record_name, type=record_type, ttl=record_ttl, value=record_value)
r = ResourceRecordSets(conn, self.zone)
r.add_change('DELETE', record_name, record_type, record_ttl)
for value in this_record.resource_records:
r.changes[0][1].add_value(value)
r.add_change('CREATE', record_name, record_type, record_ttl)
if record_value:
r.changes[1][1].add_value(record_value)
# print r.to_xml()
change = conn.change_rrsets(self.zone, r.to_xml())
print change
def test_record_commit(self):
rrsets = ResourceRecordSets(self.service_connection)
rrsets.add_change_record(
'CREATE', Record('vanilla.example.com', 'A', 60, ['1.2.3.4']))
rrsets.add_change_record(
'CREATE',
Record('alias.example.com',
'AAAA',
alias_hosted_zone_id='Z123OTHER',
alias_dns_name='target.other',
alias_evaluate_target_health=True))
rrsets.add_change_record(
'CREATE',
Record('wrr.example.com',
'CNAME',
60, ['cname.target'],
weight=10,
identifier='weight-1'))
rrsets.add_change_record(
'CREATE',
Record('lbr.example.com',
'TXT',
60, ['text record'],
region='us-west-2',
identifier='region-1'))
rrsets.add_change_record(
'CREATE',
Record('failover.example.com',
'A',
60, ['2.2.2.2'],
health_check='hc-1234',
failover='PRIMARY',
identifier='primary'))
changes_xml = rrsets.to_xml()
# the whitespacing doesn't match exactly, so we'll pretty print and drop all new lines
# not the best, but
actual_xml = re.sub(
r"\s*[\r\n]+", "\n",
xml.dom.minidom.parseString(changes_xml).toprettyxml())
expected_xml = re.sub(
r"\s*[\r\n]+", "\n",
xml.dom.minidom.parseString(b"""
<ChangeResourceRecordSetsRequest xmlns="https://route53.amazonaws.com/doc/2013-04-01/">
<ChangeBatch>
<Comment>None</Comment>
<Changes>
<Change>
<Action>CREATE</Action>
<ResourceRecordSet>
<Name>vanilla.example.com</Name>
<Type>A</Type>
<TTL>60</TTL>
<ResourceRecords>
<ResourceRecord>
<Value>1.2.3.4</Value>
</ResourceRecord>
</ResourceRecords>
</ResourceRecordSet>
</Change>
<Change>
<Action>CREATE</Action>
<ResourceRecordSet>
<Name>alias.example.com</Name>
<Type>AAAA</Type>
<AliasTarget>
<HostedZoneId>Z123OTHER</HostedZoneId>
<DNSName>target.other</DNSName>
<EvaluateTargetHealth>true</EvaluateTargetHealth>
</AliasTarget>
</ResourceRecordSet>
</Change>
<Change>
<Action>CREATE</Action>
<ResourceRecordSet>
<Name>wrr.example.com</Name>
<Type>CNAME</Type>
<SetIdentifier>weight-1</SetIdentifier>
<Weight>10</Weight>
<TTL>60</TTL>
<ResourceRecords>
<ResourceRecord>
<Value>cname.target</Value>
</ResourceRecord>
</ResourceRecords>
</ResourceRecordSet>
</Change>
<Change>
<Action>CREATE</Action>
<ResourceRecordSet>
<Name>lbr.example.com</Name>
<Type>TXT</Type>
<SetIdentifier>region-1</SetIdentifier>
<Region>us-west-2</Region>
<TTL>60</TTL>
<ResourceRecords>
<ResourceRecord>
<Value>text record</Value>
</ResourceRecord>
</ResourceRecords>
</ResourceRecordSet>
</Change>
<Change>
<Action>CREATE</Action>
<ResourceRecordSet>
<Name>failover.example.com</Name>
<Type>A</Type>
<SetIdentifier>primary</SetIdentifier>
<Failover>PRIMARY</Failover>
<TTL>60</TTL>
<ResourceRecords>
<ResourceRecord>
<Value>2.2.2.2</Value>
</ResourceRecord>
</ResourceRecords>
<HealthCheckId>hc-1234</HealthCheckId>
</ResourceRecordSet>
</Change>
</Changes>
</ChangeBatch>
</ChangeResourceRecordSetsRequest>
""").toprettyxml())
# Note: the alias XML should not include the TTL, even if it's specified in the object model
self.assertEqual(actual_xml, expected_xml)
def test_record_commit(self):
rrsets = ResourceRecordSets(self.service_connection)
rrsets.add_change_record('CREATE', Record('vanilla.example.com', 'A', 60, ['1.2.3.4']))
rrsets.add_change_record('CREATE', Record('alias.example.com', 'AAAA', alias_hosted_zone_id='Z123OTHER', alias_dns_name='target.other', alias_evaluate_target_health=True))
rrsets.add_change_record('CREATE', Record('wrr.example.com', 'CNAME', 60, ['cname.target'], weight=10, identifier='weight-1'))
rrsets.add_change_record('CREATE', Record('lbr.example.com', 'TXT', 60, ['text record'], region='us-west-2', identifier='region-1'))
rrsets.add_change_record('CREATE', Record('failover.example.com', 'A', 60, ['2.2.2.2'], health_check='hc-1234', failover='PRIMARY', identifier='primary'))
changes_xml = rrsets.to_xml()
# the whitespacing doesn't match exactly, so we'll pretty print and drop all new lines
# not the best, but
actual_xml = re.sub(r"\s*[\r\n]+", "\n", xml.dom.minidom.parseString(changes_xml).toprettyxml())
expected_xml = re.sub(r"\s*[\r\n]+", "\n", xml.dom.minidom.parseString(b"""
<ChangeResourceRecordSetsRequest xmlns="https://route53.amazonaws.com/doc/2013-04-01/">
<ChangeBatch>
<Comment>None</Comment>
<Changes>
<Change>
<Action>CREATE</Action>
<ResourceRecordSet>
<Name>vanilla.example.com</Name>
<Type>A</Type>
<TTL>60</TTL>
<ResourceRecords>
<ResourceRecord>
<Value>1.2.3.4</Value>
</ResourceRecord>
</ResourceRecords>
</ResourceRecordSet>
</Change>
<Change>
<Action>CREATE</Action>
<ResourceRecordSet>
<Name>alias.example.com</Name>
<Type>AAAA</Type>
<AliasTarget>
<HostedZoneId>Z123OTHER</HostedZoneId>
<DNSName>target.other</DNSName>
<EvaluateTargetHealth>true</EvaluateTargetHealth>
</AliasTarget>
</ResourceRecordSet>
</Change>
<Change>
<Action>CREATE</Action>
<ResourceRecordSet>
<Name>wrr.example.com</Name>
<Type>CNAME</Type>
<SetIdentifier>weight-1</SetIdentifier>
<Weight>10</Weight>
<TTL>60</TTL>
<ResourceRecords>
<ResourceRecord>
<Value>cname.target</Value>
</ResourceRecord>
</ResourceRecords>
</ResourceRecordSet>
</Change>
<Change>
<Action>CREATE</Action>
<ResourceRecordSet>
<Name>lbr.example.com</Name>
<Type>TXT</Type>
<SetIdentifier>region-1</SetIdentifier>
<Region>us-west-2</Region>
<TTL>60</TTL>
<ResourceRecords>
<ResourceRecord>
<Value>text record</Value>
</ResourceRecord>
</ResourceRecords>
</ResourceRecordSet>
</Change>
<Change>
<Action>CREATE</Action>
<ResourceRecordSet>
<Name>failover.example.com</Name>
<Type>A</Type>
<SetIdentifier>primary</SetIdentifier>
<Failover>PRIMARY</Failover>
<TTL>60</TTL>
<ResourceRecords>
<ResourceRecord>
<Value>2.2.2.2</Value>
</ResourceRecord>
</ResourceRecords>
<HealthCheckId>hc-1234</HealthCheckId>
</ResourceRecordSet>
</Change>
</Changes>
</ChangeBatch>
</ChangeResourceRecordSetsRequest>
""").toprettyxml())
# Note: the alias XML should not include the TTL, even if it's specified in the object model
self.assertEqual(actual_xml, expected_xml)
def _install(self):
logger = self.get_logger('install')
sudo = self.instance.sudo
logger.info(
'START TO INSTALL: branch = %r, commit = %r, instance = %r',
self.branch, self.commit, self.instance
)
def setup_instance(service_manifests, service_manifests_available):
logger = self.get_logger('install.setup_instance')
with self.instance:
def aptitude(*commands):
sudo(['aptitude', '-y'] + list(commands),
environ={'DEBIAN_FRONTEND': 'noninteractive'})
# create user for app
sudo(['useradd', '-U', '-G', 'users,www-data', '-Mr',
self.app.name])
# assume instance uses Ubuntu >= 12.04
apt_sources = re.sub(
r'\n#\s*(deb(?:-src)?\s+'
r'http://[^.]\.ec2\.archive\.ubuntu\.com/'
r'ubuntu/\s+[^-]+multiverse\n)',
lambda m: '\n' + m.group(1),
self.instance.read_file('/etc/apt/sources.list', sudo=True)
)
self.instance.write_file('/etc/apt/sources.list', apt_sources,
sudo=True)
apt_repos = set()
apt_packages = set([
'build-essential', 'python-dev', 'python-setuptools',
'python-pip'
])
with service_manifests_available:
while not service_manifests[0]:
service_manifests_available.wait()
for service in service_manifests[1:]:
apt_repos.update(service.required_apt_repositories)
apt_packages.update(service.required_apt_packages)
if apt_repos:
for repo in apt_repos:
sudo(['apt-add-repository', '-y', repo])
aptitude('update')
with self.instance.sftp():
self.instance.write_file(
'/usr/bin/apt-fast',
resource_string(__name__, 'apt-fast'),
sudo=True
)
self.instance.write_file('/etc/apt-fast.conf', '''
_APTMGR=aptitude
DOWNLOADBEFORE=true
_MAXNUM=20
DLLIST='/tmp/apt-fast.list'
_DOWNLOADER='aria2c -c -j ${_MAXNUM} -i ${DLLIST} --connect-timeout=10 \
--timeout=600 -m0'
DLDIR='/var/cache/apt/archives/apt-fast'
APTCACHE='/var/cache/apt/archives/'
''', sudo=True)
sudo(['chmod', '+x', '/usr/bin/apt-fast'])
aptitude('install', 'aria2')
sudo(['apt-fast', '-q', '-y', 'install'] + list(apt_packages),
environ={'DEBIAN_FRONTEND': 'noninteractive'})
service_manifests_available = threading.Condition()
service_manifests = [False]
instance_setup_worker = threading.Thread(
target=setup_instance,
kwargs={
'service_manifests_available': service_manifests_available,
'service_manifests': service_manifests
}
)
instance_setup_worker.start()
# setup metadata of the instance
self.update_instance_metadata()
self.instance.status = 'started'
# making package (pybundle)
fd, package_path = tempfile.mkstemp()
os.close(fd)
with self.fetch() as download_path:
service_manifests.extend(self.services)
service_manifests[0] = True
with service_manifests_available:
service_manifests_available.notify()
config_temp_path = tempfile.mkdtemp()
shutil.copytree(
os.path.join(download_path, self.app.config_dir),
os.path.join(config_temp_path, self.app.name)
)
with self.dist.bundle_package() as (package, filename, temp_path):
shutil.copyfile(temp_path, package_path)
remote_path = os.path.join('/tmp', filename)
with self.instance.sftp():
# upload config files
self.instance.put_directory(
os.path.join(config_temp_path, self.app.name),
'/etc/' + self.app.name,
sudo=True
)
shutil.rmtree(config_temp_path)
python_packages = set()
for service in service_manifests[1:]:
python_packages.update(service.required_python_packages)
# uploads package
self.instance.put_file(package_path, remote_path)
# join instance_setup_worker
instance_setup_worker.join()
self.instance.status = 'apt-installed'
pip_cmd = ['pip', 'install', '-i', PYPI_INDEX_URLS[0]]
for idx in PYPI_INDEX_URLS[1:]:
pip_cmd.append('--extra-index-url=' + idx)
sudo(pip_cmd + [remote_path], environ={'CI': '1'})
sudo(pip_cmd + ['-I'] + list(python_packages),
environ={'CI': '1'})
self.instance.status = 'installed'
for service in service_manifests[1:]:
for cmd in service.pre_install:
sudo(cmd, environ={'DEBIAN_FRONTEND': 'noninteractive'})
values_path = '/etc/{0}/values.json'.format(self.app.name)
service_values = {
'.build': dict(
commit=self.commit.ref,
branch=self.branch.label
)
}
refresh_values = lambda: self.instance.write_file(
values_path,
json.dumps(service_values),
sudo=True
)
refresh_values()
for service in service_manifests[1:]:
service_value = service.install(self.instance)
service_values[service.name] = service_value
refresh_values()
for service in service_manifests[1:]:
for cmd in service.post_install:
sudo(cmd, environ={'DEBIAN_FRONTEND': 'noninteractive'})
service_map = dict((service.name, service)
for service in service_manifests[1:])
deployed_domains = {}
if self.route53_hosted_zone_id and self.route53_records:
self.instance.status = 'run'
changeset = ResourceRecordSets(
self.app.route53_connection,
self.route53_hosted_zone_id,
'Changed by Asuka: {0}, {1} [{2}]'.format(self.app.name,
self.branch.label,
self.commit.ref)
)
from .service import DomainService
for service_name, domain_format in self.route53_records.items():
service = service_map[service_name]
if not isinstance(service, DomainService):
raise TypeError(repr(service) + 'is not an instance of '
'crosspop.service.DomainService')
domain = domain_format.format(branch=self.branch)
deployed_domains[service_name] = domain
service.route_domain(domain, changeset)
self.instance.tags['Domain-' + service_name] = domain
if changeset.changes:
logger.info('Route 53 changeset:\n%s', changeset.to_xml())
changeset.commit()
self.instance.status = 'done'
self.terminate_instances()
return deployed_domains
