def parse_loc_list(loc_list):
"wrangle the list of paths the user gave us. expand if they specify a directory, etc"
# give the convenient user-form some structure
p_loc_list = lmap(_parse_loc, loc_list)
# do some post processing
def expand_dirs(triple):
protocol, host, path = triple
if protocol in ['dir', 'file'] and not os.path.exists(path):
LOG.warn("could not resolve %r, skipping", path)
return [None]
if protocol == 'dir':
yaml_files = utils.listfiles(path, ['.yaml'])
return [('file', host, ppath) for ppath in yaml_files]
return [triple]
# we don't want dirs, we want files
p_loc_list = utils.shallow_flatten(map(expand_dirs, p_loc_list))
# remove any bogus values
p_loc_list = lfilter(None, p_loc_list)
# remove any duplicates. can happen when we expand dir => files
p_loc_list = utils.unique(p_loc_list)
return p_loc_list
def parse_loc_list(loc_list):
"wrangle the list of paths the user gave us. expand if they specify a directory, etc"
# give the convenient user-form some structure
p_loc_list = map(_parse_loc, loc_list)
# do some post processing
def expand_dirs(triple):
protocol, host, path = triple
if protocol in ['dir', 'file'] and not os.path.exists(path):
LOG.warn("could not resolve %r, skipping", path)
return [None]
if protocol == 'dir':
yaml_files = utils.listfiles(path, ['.yaml'])
return [('file', host, ppath) for ppath in yaml_files]
return [triple]
# we don't want dirs, we want files
p_loc_list = utils.shallow_flatten(map(expand_dirs, p_loc_list))
# remove any bogus values
p_loc_list = filter(None, p_loc_list)
# remove any duplicates. can happen when we expand dir => files
p_loc_list = utils.unique(p_loc_list)
return p_loc_list
def test_shallow_flatten(self):
case_list = [
([], []),
([[1], [2], [3]], [1, 2, 3]),
([[[1]], [2], [3]], [[1], 2, 3]),
([[None]], [None]),
]
for given, expected in case_list:
self.assertEqual(utils.shallow_flatten(given), expected)
def test_shallow_flatten(self):
case_list = [
([], []),
([[1], [2], [3]], [1, 2, 3]),
([[[1]], [2], [3]], [[1], 2, 3]),
([[None]], [None]),
]
for given, expected in case_list:
self.assertEqual(utils.shallow_flatten(given), expected)
def get_records(dns):
"""fetches all of the DNS entries for the Hosted Zone the given `dns` entry belongs to.
results are cached in `record_map` to prevent multiple lookups."""
zone_id = hosted_zone_id(dns)
if zone_id in record_map:
return record_map[zone_id]
paginator = conn.get_paginator('list_resource_record_sets')
results = shallow_flatten([page['ResourceRecordSets'] for page in paginator.paginate(HostedZoneId=zone_id)])
record_map[zone_id] = results
return results
def all_formulas():
"returns a list of all known formulas"
# `project` will give us a map of `project-name: formula-list`
formula_list = project.project_formulas().values()
# flatten the list of lists into a single unique list
formula_list = set(core_utils.shallow_flatten(formula_list))
# remove any empty values, probably from reading the `defaults` section
formula_list = filter(None, formula_list)
# extract just the name from the formula url
formula_list = map(os.path.basename, formula_list)
return formula_list
def all_adhoc_ec2_instances(state='running'):
"returns a list of all ec2 instance names whose instance ID doesn't match a known environment"
# extract a list of environment names from the project data
def known_environments(pdata):
"returns the names of all alt-config names for given project data"
return list(pdata.get('aws-alt', {}).keys())
env_list = core_utils.shallow_flatten(
map(known_environments,
project.project_map().values()))
# append known good environments and ensure there are no dupes
fixed_env_list = [
'prod', 'end2end', 'continuumtest', 'ci', 'preview',
'continuumtestpreview'
]
env_list += fixed_env_list
env_list = list(set(filter(None, env_list)))
# extract the names of ec2 instances that are not part of any known environment
def adhoc_instance(stackname):
"predicate, returns True if stackname is *not* in a known environment"
try:
iid = core.parse_stackname(stackname, all_bits=True,
idx=True)['instance_id']
return iid not in env_list
except (ValueError, AssertionError):
# thrown by `parse_stackname` when given value isn't a string or
# delimiter not found in string.
return True
instance_list = [
ec2['TagsDict']['Name'] for ec2 in core.ec2_instance_list(state=state)
]
return filter(adhoc_instance, instance_list)
def known_formulas():
"a simple list of all known project formulas"
return utils.lfilter(None, utils.unique(utils.shallow_flatten(project_formulas().values())))
def _dns_check(stackname, context):
"looks for all of the possible DNS entries for the given route and returns `True` if all are found"
title('ec2 DNS')
keys = [
'ext_node_hostname', # "prod--lax--%s.elifesciences.org"
'full_hostname', # "prod--lax.elifesciences.org", "prod--journal.elifesciences.org"
'project_hostname', # "lax.elifesciences.org", "journal.elifesciences.org"
]
domain_map = subdict(context, keys)
# these don't appear in Route53
# domain_map['cloudfront'] = (context.get('cloudfront') or {}).get('subdomains', []) # todo
domain_map['fastly'] = (context.get('fastly') or {}).get('subdomains', [])
# `project_hostname` is only exactly what it is for prod stacks.
if context['instance_id'] != 'prod':
del domain_map['project_hostname']
conn = core.boto_conn(stackname, 'route53', client=True)
paginator = conn.get_paginator('list_hosted_zones')
hosted_zone_list = shallow_flatten([page['HostedZones'] for page in paginator.paginate()])
hosted_zone_map = {z['Name']: z['Id'] for z in hosted_zone_list}
num_nodes = lu(context, 'project.aws.ec2.cluster-size', default=1)
if 'ext_node_hostname' in domain_map:
if num_nodes > 1:
domain_map.update({"node-%s" % i: domain_map['ext_node_hostname'] % i for i in range(1, num_nodes + 1)})
del domain_map['ext_node_hostname']
def hosted_zone_id(dns):
"given a DNS entry, figures out which AWS Hosted Zone it belongs to"
bits = dns.split('.')
if len(bits) == 3:
name = '.'.join(bits[1:]) + '.'
else:
name = dns + '.'
return hosted_zone_map[name]
record_map = {}
def get_records(dns):
"""fetches all of the DNS entries for the Hosted Zone the given `dns` entry belongs to.
results are cached in `record_map` to prevent multiple lookups."""
zone_id = hosted_zone_id(dns)
if zone_id in record_map:
return record_map[zone_id]
paginator = conn.get_paginator('list_resource_record_sets')
results = shallow_flatten([page['ResourceRecordSets'] for page in paginator.paginate(HostedZoneId=zone_id)])
record_map[zone_id] = results
return results
def find_records(dns):
"fetches all of the DNS records for the given `dns` record's Hosted Zone and returns the A or CNAME ones, if any"
dnsd = dns + '.'
records = get_records(dns)
type_list = ['A', 'CNAME']
return [r for r in records if r['Name'] == dnsd and r['Type'] in type_list]
def check_record(label, dns):
"looks for the given `dns` entry and returns a `problem` if one is not found."
print('* checking %r' % dns)
records = find_records(dns)
if not records:
description = 'record %r (%r) not found in Route53' % (dns, label)
details = "records found: %s" % (pformat(records),)
solution = './bldr update_dns:%s' % (stackname,)
return problem(description, solution, details)
results = []
for label, dns in domain_map.items():
if isinstance(dns, str):
_results = check_record(label, dns)
if _results:
results.append(_results)
elif isinstance(dns, list):
for subdns in dns:
_results = check_record(label, subdns)
if _results:
results.append(_results)
return success() if not results else results
def known_formulas():
"a simple list of all known project formulas (excluding the private-repo)"
lst = utils.unique(utils.shallow_flatten(project_formulas().values()))
if None in lst:
lst.remove(None)
return lst