def parse_loc_list(loc_list): "wrangle the list of paths the user gave us. expand if they specify a directory, etc" # give the convenient user-form some structure p_loc_list = lmap(_parse_loc, loc_list) # do some post processing def expand_dirs(triple): protocol, host, path = triple if protocol in ['dir', 'file'] and not os.path.exists(path): LOG.warn("could not resolve %r, skipping", path) return [None] if protocol == 'dir': yaml_files = utils.listfiles(path, ['.yaml']) return [('file', host, ppath) for ppath in yaml_files] return [triple] # we don't want dirs, we want files p_loc_list = utils.shallow_flatten(map(expand_dirs, p_loc_list)) # remove any bogus values p_loc_list = lfilter(None, p_loc_list) # remove any duplicates. can happen when we expand dir => files p_loc_list = utils.unique(p_loc_list) return p_loc_list
def parse_loc_list(loc_list): "wrangle the list of paths the user gave us. expand if they specify a directory, etc" # give the convenient user-form some structure p_loc_list = map(_parse_loc, loc_list) # do some post processing def expand_dirs(triple): protocol, host, path = triple if protocol in ['dir', 'file'] and not os.path.exists(path): LOG.warn("could not resolve %r, skipping", path) return [None] if protocol == 'dir': yaml_files = utils.listfiles(path, ['.yaml']) return [('file', host, ppath) for ppath in yaml_files] return [triple] # we don't want dirs, we want files p_loc_list = utils.shallow_flatten(map(expand_dirs, p_loc_list)) # remove any bogus values p_loc_list = filter(None, p_loc_list) # remove any duplicates. can happen when we expand dir => files p_loc_list = utils.unique(p_loc_list) return p_loc_list
def test_shallow_flatten(self): case_list = [ ([], []), ([[1], [2], [3]], [1, 2, 3]), ([[[1]], [2], [3]], [[1], 2, 3]), ([[None]], [None]), ] for given, expected in case_list: self.assertEqual(utils.shallow_flatten(given), expected)
def get_records(dns): """fetches all of the DNS entries for the Hosted Zone the given `dns` entry belongs to. results are cached in `record_map` to prevent multiple lookups.""" zone_id = hosted_zone_id(dns) if zone_id in record_map: return record_map[zone_id] paginator = conn.get_paginator('list_resource_record_sets') results = shallow_flatten([page['ResourceRecordSets'] for page in paginator.paginate(HostedZoneId=zone_id)]) record_map[zone_id] = results return results
def all_formulas(): "returns a list of all known formulas" # `project` will give us a map of `project-name: formula-list` formula_list = project.project_formulas().values() # flatten the list of lists into a single unique list formula_list = set(core_utils.shallow_flatten(formula_list)) # remove any empty values, probably from reading the `defaults` section formula_list = filter(None, formula_list) # extract just the name from the formula url formula_list = map(os.path.basename, formula_list) return formula_list
def all_adhoc_ec2_instances(state='running'): "returns a list of all ec2 instance names whose instance ID doesn't match a known environment" # extract a list of environment names from the project data def known_environments(pdata): "returns the names of all alt-config names for given project data" return list(pdata.get('aws-alt', {}).keys()) env_list = core_utils.shallow_flatten( map(known_environments, project.project_map().values())) # append known good environments and ensure there are no dupes fixed_env_list = [ 'prod', 'end2end', 'continuumtest', 'ci', 'preview', 'continuumtestpreview' ] env_list += fixed_env_list env_list = list(set(filter(None, env_list))) # extract the names of ec2 instances that are not part of any known environment def adhoc_instance(stackname): "predicate, returns True if stackname is *not* in a known environment" try: iid = core.parse_stackname(stackname, all_bits=True, idx=True)['instance_id'] return iid not in env_list except (ValueError, AssertionError): # thrown by `parse_stackname` when given value isn't a string or # delimiter not found in string. return True instance_list = [ ec2['TagsDict']['Name'] for ec2 in core.ec2_instance_list(state=state) ] return filter(adhoc_instance, instance_list)
def known_formulas(): "a simple list of all known project formulas" return utils.lfilter(None, utils.unique(utils.shallow_flatten(project_formulas().values())))
def _dns_check(stackname, context): "looks for all of the possible DNS entries for the given route and returns `True` if all are found" title('ec2 DNS') keys = [ 'ext_node_hostname', # "prod--lax--%s.elifesciences.org" 'full_hostname', # "prod--lax.elifesciences.org", "prod--journal.elifesciences.org" 'project_hostname', # "lax.elifesciences.org", "journal.elifesciences.org" ] domain_map = subdict(context, keys) # these don't appear in Route53 # domain_map['cloudfront'] = (context.get('cloudfront') or {}).get('subdomains', []) # todo domain_map['fastly'] = (context.get('fastly') or {}).get('subdomains', []) # `project_hostname` is only exactly what it is for prod stacks. if context['instance_id'] != 'prod': del domain_map['project_hostname'] conn = core.boto_conn(stackname, 'route53', client=True) paginator = conn.get_paginator('list_hosted_zones') hosted_zone_list = shallow_flatten([page['HostedZones'] for page in paginator.paginate()]) hosted_zone_map = {z['Name']: z['Id'] for z in hosted_zone_list} num_nodes = lu(context, 'project.aws.ec2.cluster-size', default=1) if 'ext_node_hostname' in domain_map: if num_nodes > 1: domain_map.update({"node-%s" % i: domain_map['ext_node_hostname'] % i for i in range(1, num_nodes + 1)}) del domain_map['ext_node_hostname'] def hosted_zone_id(dns): "given a DNS entry, figures out which AWS Hosted Zone it belongs to" bits = dns.split('.') if len(bits) == 3: name = '.'.join(bits[1:]) + '.' else: name = dns + '.' return hosted_zone_map[name] record_map = {} def get_records(dns): """fetches all of the DNS entries for the Hosted Zone the given `dns` entry belongs to. results are cached in `record_map` to prevent multiple lookups.""" zone_id = hosted_zone_id(dns) if zone_id in record_map: return record_map[zone_id] paginator = conn.get_paginator('list_resource_record_sets') results = shallow_flatten([page['ResourceRecordSets'] for page in paginator.paginate(HostedZoneId=zone_id)]) record_map[zone_id] = results return results def find_records(dns): "fetches all of the DNS records for the given `dns` record's Hosted Zone and returns the A or CNAME ones, if any" dnsd = dns + '.' records = get_records(dns) type_list = ['A', 'CNAME'] return [r for r in records if r['Name'] == dnsd and r['Type'] in type_list] def check_record(label, dns): "looks for the given `dns` entry and returns a `problem` if one is not found." print('* checking %r' % dns) records = find_records(dns) if not records: description = 'record %r (%r) not found in Route53' % (dns, label) details = "records found: %s" % (pformat(records),) solution = './bldr update_dns:%s' % (stackname,) return problem(description, solution, details) results = [] for label, dns in domain_map.items(): if isinstance(dns, str): _results = check_record(label, dns) if _results: results.append(_results) elif isinstance(dns, list): for subdns in dns: _results = check_record(label, subdns) if _results: results.append(_results) return success() if not results else results
def known_formulas(): "a simple list of all known project formulas (excluding the private-repo)" lst = utils.unique(utils.shallow_flatten(project_formulas().values())) if None in lst: lst.remove(None) return lst