def getCRConfigs(A:TOSession, B:TOSession) -> typing.Generator[str, None, None]:
"""
Generates a list of routes to CRConfig files for all CDNs present in both A and B
:param A: The first Traffic Ops instance
:param B: The second Traffic Ops instance
:returns: A list of routes to CRConfig files
"""
try:
Acdns = A.get_cdns()[0]
Bcdns = B.get_cdns()[0]
except (UnicodeError, IndexError, KeyError, InvalidJSONError, OperationError) as e:
logging.debug("%r", e, exc_info=True, stack_info=True)
logging.critical("Unable to get CDN lists: %s", e)
return
cdns = {c.name for c in Acdns}.intersection({c.name for c in Bcdns})
if not cdns:
logging.error("The two instances have NO CDNs in common! This almost certainly means that "\
"you're not doing what you want to do")
return
for cdn in cdns:
yield "/CRConfig-Snapshots/%s/CRConfig.json" % cdn
yield "/api/1.3/cdns/%s/snapshot" % cdn
yield "/api/1.3/cdns/%s/snapshot/new" % cdn
def main(kwargs: argparse.Namespace) -> int:
"""
Runs the commandline specified by ``kwargs``.
:param kwargs: An object that provides the attribute namespace representing this script's
options. See ``genConfigRoutes.py --help`` for more information.
:returns: an exit code for the program
:raises KeyError: when ``kwargs`` does not faithfully represent a valid command line
"""
global LOG_FMT
if kwargs.quiet:
level = logging.CRITICAL + 1
else:
level = logging.getLevelName(kwargs.log_level)
try:
logging.basicConfig(level=level, format=LOG_FMT)
logging.getLogger().setLevel(level)
except ValueError:
print("Unrecognized log level:", kwargs.log_level, file=sys.stderr)
return 1
try:
instanceA, instanceB, loginA, loginB = consolidateVariables(kwargs)
except ValueError as e:
logging.debug("%s", e, exc_info=True, stack_info=True)
logging.critical("(hint: try '-h'/'--help')")
return 1
verify = not kwargs.insecure
# Instantiate connections and login
with TOSession(host_ip=instanceA["host"], host_port=instanceA["port"], verify_cert=verify) as A,\
TOSession(host_ip=instanceB["host"], host_port=instanceB["port"], verify_cert=verify) as B:
try:
A.login(loginA[0], loginA[1])
B.login(loginB[0], loginB[1])
except (OSError, LoginError) as e:
logging.debug("%s", e, exc_info=True, stack_info=True)
logging.critical("Failed to connect to Traffic Ops")
return 2
except (OperationError, InvalidJSONError) as e:
logging.debug("%s", e, exc_info=True, stack_info=True)
logging.critical("Failed to log in to Traffic Ops")
logging.error(
"Error was '%s' - are you sure your URLs and credentials are correct?",
e)
return 2
for route in genRoutes(A, B, kwargs.snapshot,
kwargs.no_server_configs):
print(route)
return 0
def getConfigRoutesForServers(servers:typing.List[dict], inst:TOSession) \
-> typing.Generator[str, None, None]:
"""
Generates a list of routes to the config files for a given set of servers and a given traffic
ops instance
:param servers: a list of server objects
:param inst: A valid, authenticated, and connected Traffic Ops instance
:returns: A list of routes to config files for the ``servers``. These will be relative to the
url of the ``inst``
"""
for server in servers:
try:
yield "/api/1.3/servers/%s/configfiles/ats" % server.hostName
for file in inst.get_server_config_files(
host_name=server.hostName)[0].configFiles:
if "apiUri" in file:
yield file.apiUri
else:
logging.info(
"config file %s for server %s has non-API URI - skipping",
file.location, server.hostName)
except (AttributeError, UnicodeError, IndexError, KeyError,
InvalidJSONError, OperationError) as e:
logging.debug("%r", e, exc_info=True, stack_info=True)
logging.error(
"Invalid API response for server %s config files: %s",
server.hostName, e)
def getCRConfigs(A: TOSession,
B: TOSession) -> typing.Generator[str, None, None]:
"""
Generates a list of routes to CRConfig files for all CDNs present in both A and B
:param A: The first Traffic Ops instance
:param B: The second Traffic Ops instance
:returns: A list of routes to CRConfig files
"""
cdns = {c.name
for c in A.get_cdns()[0]
}.intersection({c.name
for c in B.get_cdns()[0]})
if not cdns:
logging.error("The two instances have NO CDNs in common! This almost certainly means that "\
"you're not doing what you want to do")
yield from ["CRConfig-Snapshots/%s/CRConfig.json" % cdn for cdn in cdns]
def getConfigRoutesForServers(servers:typing.List[dict], inst:TOSession) \
-> typing.Generator[str, None, None]:
"""
Generates a list of routes to the config files for a given set of servers and a given traffic
ops instance
:param servers: a list of server objects
:param inst: A valid, authenticated, and connected Traffic Ops instance
:returns: A list of routes to config files for the ``servers``. These will be relative to the
url of the ``inst``
"""
for server in servers:
for file in inst.getServerConfigFiles(
servername=server.hostName)[0].configFiles:
if "apiUri" in file:
yield file.apiUri
else:
logging.info(
"config file %s for server %s has non-API URI - skipping",
file.location, server.hostName)
def genRoutes(A:TOSession, B:TOSession, snapshots:bool, skip_servers:bool) ->\
typing.Generator[str, None, None]:
"""
Generates routes to check for ATS config files from two valid Traffic Ops sessions
:param A: The first Traffic Ops instance
:param B: The second Traffic Ops instance
:param snapshots: If ``true``, generate CDN snapshot routes, otherwise don't
:param skip_servers: If ``true``, generation of server config files will be skipped
:returns: A list of routes representative of the configuration files for a bunch of servers
"""
generatedRoutes = set()
if not skip_servers:
try:
profiles = ({p.id: p for p in A.get_profiles()[0]}, {p.id: p for p in B.get_profiles()[0]})
except (UnicodeError, InvalidJSONError, OperationError) as e:
logging.critical("Could not fetch server profiles: %s", e)
logging.debug("%r", e, exc_info=True, stack_info=True)
return
profileIds = (set(profiles[0].keys()), set(profiles[1].keys()))
# Differences and intersections:
for key in profileIds[0].difference(profileIds[1]):
del profiles[0][key]
logging.warning("profile %s found in %s but not in %s!", key, A.to_url, B.to_url)
for key in profileIds[1].difference(profileIds[0]):
del profiles[1][key]
logging.warning("profile %s found in %s but not in %s!", key, B.to_url, A.to_url)
# Now only check for identical profiles - we wouldn't expect the config files generated from
# different profiles to be the same.
commonProfiles = set()
for profileId, profile in profiles[0].items():
if profiles[1][profileId].name == profile.name:
commonProfiles.add((profileId, profile.name, profile.type))
else:
logging.error("profile %s is not the same profile in both instances!", profileId)
sampleServers = []
for profile in commonProfiles:
if profile[2] == "ATS_PROFILE":
try:
servers = A.get_servers(query_params={"profileId": profile[0]})[0]
serverIndex = random.randint(0, len(servers)-1)
sampleServer = servers[serverIndex]
del servers[serverIndex]
while not B.get_servers(query_params={"id": sampleServer.id})[0]:
logging.warning("Server %s found in %s but not in %s!", sampleServer.id,
A.to_url, B.to_url)
serverIndex = random.randint(0, len(servers)-1)
sampleServer = servers[serverIndex]
del servers[serverIndex]
except (IndexError, ValueError):
logging.error("Server list for profile %s exhausted without finding a sample!",
profile[1])
except (UnicodeError, InvalidJSONError, OperationError) as e:
logging.error("Invalid JSON response fetching server list for %s: %s", profile[2],e)
logging.debug("%r", e, exc_info=True, stack_info=True)
else:
sampleServers.append(sampleServer)
for route in getConfigRoutesForServers(sampleServers, A):
if route not in generatedRoutes:
yield route
generatedRoutes.add(route)
if snapshots:
for route in getCRConfigs(A, B):
if route not in generatedRoutes:
yield route
generatedRoutes.add(route)
def generate_inventory_list(self, target_to):
"""Generate the inventory list for the specified TrafficOps instance"""
with TOSession(self.to_url, verify_cert=self.verify_cert) as traffic_ops_api:
traffic_ops_api.login(self.to_user, self.to_pass)
servers = traffic_ops_api.get_servers()[0]
out = {}
out['_meta'] = {}
out['_meta']['hostvars'] = {}
out[target_to] = {}
out[target_to]['hosts'] = []
out["ungrouped"] = {}
out['ungrouped']['hosts'] = []
out['cachegroup'] = {}
out['cachegroup']['children'] = []
out['server_type'] = {}
out['server_type']['children'] = []
out['server_cdnName'] = {}
out['server_cdnName']['children'] = []
out['server_profile'] = {}
out['server_profile']['children'] = []
out['server_status'] = {}
out['server_status']['children'] = []
for server in servers:
fqdn = server['hostName'] + '.' + server['domainName']
out["ungrouped"]['hosts'].append(fqdn)
out[target_to]['hosts'].append(fqdn)
out['_meta']['hostvars'][fqdn] = {}
out['_meta']['hostvars'][fqdn]['server_toFQDN'] = target_to
out['_meta']['hostvars'][fqdn]['server_cachegroup'] = server['cachegroup']
out['_meta']['hostvars'][fqdn]['server_cdnName'] = server['cdnName']
out['_meta']['hostvars'][fqdn]['server_id'] = server['id']
out['_meta']['hostvars'][fqdn]['server_ipAddress'] = server['ipAddress']
out['_meta']['hostvars'][fqdn]['server_ip6Address'] = server['ip6Address']
out['_meta']['hostvars'][fqdn]['server_offlineReason'] = server['offlineReason']
out['_meta']['hostvars'][fqdn]['server_physLocation'] = server['physLocation']
out['_meta']['hostvars'][fqdn]['server_profile'] = server['profile']
out['_meta']['hostvars'][fqdn]['server_profileDesc'] = server['profileDesc']
out['_meta']['hostvars'][fqdn]['server_status'] = server['status']
out['_meta']['hostvars'][fqdn]['server_type'] = server['type']
flat_server_profile = "server_profile|" + server['profile']
flat_cachegroup = "cachegroup|" + server['cachegroup']
flat_server_type = "server_type|" + server['type']
flat_server_cdn_name = "server_cdnName|" + server['cdnName']
flat_server_status = "server_status|" + server['status']
if flat_server_profile not in out:
out['server_profile']['children'].append(
flat_server_profile)
out[flat_server_profile] = self.populate_server_profile_vars(
traffic_ops_api, server['profileId'])
out[flat_server_profile]['hosts'].append(fqdn)
if flat_cachegroup not in out:
out['cachegroup']['children'].append(flat_cachegroup)
cgdata = self.populate_cachegroups(
traffic_ops_api,
server['cachegroupId'])
out[flat_cachegroup] = cgdata.cgvars
flat_parent_cg = cgdata.primary_parent_group_name
flat_second_parent_cg = cgdata.secondary_parent_group_name
if flat_parent_cg not in out:
out[flat_parent_cg] = {}
out[flat_parent_cg]['children'] = []
if flat_second_parent_cg not in out:
out[flat_second_parent_cg] = {}
out[flat_second_parent_cg]['children'] = []
out[flat_parent_cg]['children'].append(flat_cachegroup)
out[flat_second_parent_cg]['children'].append(
flat_cachegroup)
out[flat_cachegroup]['hosts'].append(fqdn)
if flat_server_type not in out:
out['server_type']['children'].append(flat_server_type)
out[flat_server_type] = {}
out[flat_server_type]['hosts'] = []
out[flat_server_type]['hosts'].append(fqdn)
if flat_server_cdn_name not in out:
out['server_cdnName']['children'].append(
flat_server_cdn_name)
out[flat_server_cdn_name] = {}
out[flat_server_cdn_name]['hosts'] = []
out[flat_server_cdn_name]['hosts'].append(fqdn)
if flat_server_status not in out:
out['server_status']['children'].append(flat_server_status)
out[flat_server_status] = {}
out[flat_server_status]['hosts'] = []
out[flat_server_status]['hosts'].append(fqdn)
return out
def parse_arguments(program):
"""
A common-use function that parses the command line arguments.
:param program: The name of the program being run - used for usage informational output
:returns: The Traffic Ops HTTP session object, the requested path, any data to be sent, an output
format specification, whether or not the path is raw, and whether or not output should
be prettified
"""
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
parser = ArgumentParser(
prog=program,
formatter_class=ArgumentDefaultsHelpFormatter,
description="A helper program for interfacing with the Traffic Ops API",
epilog=(
"Typically, one will want to connect and authenticate by defining "
"the 'TO_URL', 'TO_USER' and 'TO_PASSWORD' environment variables "
"rather than (respectively) the '--to-url', '--to-user' and "
"'--to-password' command-line flags. Those flags are only "
"required when said environment variables are not defined.\n"
"%(prog)s will exit with a success provided a response was "
"received and the status code of said response was less than 400. "
"The exit code will be 1 if command line arguments cannot be "
"parsed or authentication with the Traffic Ops server fails. "
"In the event of some unknown error occurring when waiting for a "
"response, the exit code will be 2. If the server responds with "
"a status code indicating a client or server error, that status "
"code will be used as the exit code."))
parser.add_argument(
"--to-url",
type=str,
help=
("The fully qualified domain name of the Traffic Ops server. Overrides "
"'$TO_URL'. The format for both the environment variable and the flag "
"is '[scheme]hostname[:port]'. That is, ports should be specified here, "
"and they need not start with 'http://' or 'https://'. HTTPS is the "
"assumed protocol unless the scheme _is_ provided and is 'http://'."))
parser.add_argument(
"--to-user",
type=str,
help=
"The username to use when connecting to Traffic Ops. Overrides '$TO_USER"
)
parser.add_argument(
"--to-password",
type=str,
help=
"The password to use when authenticating to Traffic Ops. Overrides '$TO_PASSWORD'"
)
parser.add_argument("-k",
"--insecure",
action="store_true",
help="Do not verify SSL certificates")
parser.add_argument(
"-f",
"--full",
action="store_true",
help=
("Also output HTTP request/response lines and headers, and request payload. "
"This is equivalent to using '--request-headers', '--response-headers' "
"and '--request-payload' at the same time."))
parser.add_argument("--request-headers",
action="store_true",
help="Output request method line and headers")
parser.add_argument("--response-headers",
action="store_true",
help="Output response status line and headers")
parser.add_argument(
"--request-payload",
action="store_true",
help=
"Output request payload (will try to pretty-print if '--pretty' is given)"
)
parser.add_argument(
"-r",
"--raw-path",
action="store_true",
help=
"Request exactly PATH; it won't be prefaced with '/api/{{api-version}}/"
)
parser.add_argument("-a",
"--api-version",
type=float,
default=2.0,
help="Specify the API version to request against")
parser.add_argument(
"-p",
"--pretty",
action="store_true",
help=
("Pretty-print payloads as JSON. "
"Note that this will make Content-Type headers \"wrong\", in general"
))
parser.add_argument("-v",
"--version",
action="version",
help="Print version information and exit",
version="%(prog)s v" + __version__)
parser.add_argument(
"PATH",
help="The path to the resource being requested - omit '/api/2.x'")
parser.add_argument(
"DATA",
help=("An optional data string to pass with the request. If this is a "
"filename, the contents of the file will be sent instead."),
nargs='?')
args = parser.parse_args()
try:
to_host = args.to_url if args.to_url else os.environ["TO_URL"]
except KeyError as e:
raise KeyError("Traffic Ops hostname not set! Set the TO_URL environment variable or use "\
"'--to-url'.") from e
original_to_host = to_host
to_host = urlparse(to_host, scheme="https")
useSSL = to_host.scheme.lower() == "https"
to_port = to_host.port
if to_port is None:
if useSSL:
to_port = 443
else:
to_port = 80
to_host = to_host.hostname
if not to_host:
raise KeyError(
f"Invalid URL/host for Traffic Ops: '{original_to_host}'")
s = TOSession(to_host,
host_port=to_port,
ssl=useSSL,
api_version=f"{args.api_version:.1f}",
verify_cert=not args.insecure)
data = args.DATA
if data and os.path.isfile(data):
with open(data) as f:
data = f.read()
if isinstance(data, str):
data = data.encode()
try:
to_user = args.to_user if args.to_user else os.environ["TO_USER"]
except KeyError as e:
raise KeyError("Traffic Ops user not set! Set the TO_USER environment variable or use "\
"'--to-user'.") from e
try:
to_passwd = args.to_password if args.to_password else os.environ[
"TO_PASSWORD"]
except KeyError as e:
raise KeyError("Traffic Ops password not set! Set the TO_PASSWORD environment variable or "\
"use '--to-password'") from e
try:
s.login(to_user, to_passwd)
except (OperationError, InvalidJSONError, LoginError) as e:
raise PermissionError from e
return (s, args.PATH, data, (args.request_headers
or args.full, args.response_headers
or args.full, args.request_payload
or args.full), args.raw_path, args.pretty)
def genRoutes(A: TOSession, B: TOSession) -> typing.Generator[str, None, None]:
"""
Generates routes to check for ATS config files from two valid Traffic Ops sessions
:param A: The first Traffic Ops instance
:param B: The second Traffic Ops instance
:returns: A list of routes representative of the configuration files for a bunch of servers
"""
profiles = ({p.id: p
for p in A.get_profiles()[0]},
{p.id: p
for p in B.get_profiles()[0]})
profileIds = (set(profiles[0].keys()), set(profiles[1].keys()))
# Differences and intersections:
for key in profileIds[0].difference(profileIds[1]):
del profiles[0][key]
logging.warning("profile %s found in %s but not in %s!", key, A.to_url,
B.to_url)
for key in profileIds[1].difference(profileIds[0]):
del profiles[1][key]
logging.warning("profile %s found in %s but not in %s!", key, B.to_url,
A.to_url)
# Now only check for identical profiles - we wouldn't expect the config files generated from
# different profiles to be the same.
commonProfiles = set()
for profileId, profile in profiles[0].items():
if profiles[1][profileId].name == profile.name:
commonProfiles.add((profileId, profile.name, profile.type))
else:
logging.error(
"profile %s is not the same profile in both instances!",
profileId)
sampleServers = []
for profile in commonProfiles:
if profile[2] == "ATS_PROFILE":
servers = A.get_servers(query_params={"profileId": profile[0]})[0]
try:
serverIndex = random.randint(0, len(servers) - 1)
sampleServer = servers[serverIndex]
del servers[serverIndex]
while not B.get_servers(
query_params={"id": sampleServer.id})[0]:
logging.warning("Server %s found in %s but not in %s!",
sampleServer.id, A.to_url, B.to_url)
serverIndex = random.randint(0, len(servers) - 1)
sampleServer = servers[serverIndex]
del servers[serverIndex]
except (IndexError, ValueError):
logging.error(
"Server list for profile %s exhausted without finding a sample!",
profile[1])
else:
sampleServers.append(sampleServer)
generatedRoutes = set()
for route in getConfigRoutesForServers(sampleServers, A):
if route not in generatedRoutes:
yield route
generatedRoutes.add(route)
for route in getCRConfigs(A, B):
if route not in generatedRoutes:
yield route
generatedRoutes.add(route)
def main(kwargs: argparse.Namespace) -> int:
"""
Runs the commandline specified by ``kwargs``.
:param kwargs: An object that provides the attribute namespace representing this script's
options. See ``genConfigRoutes.py --help`` for more information.
:returns: an exit code for the program
:raises KeyError: when ``kwargs`` does not faithfully represent a valid command line
"""
global LOG_FMT
if kwargs.quiet:
level = logging.CRITICAL + 1
else:
level = logging.getLevelName(kwargs.log_level)
try:
logging.basicConfig(level=level, format=LOG_FMT)
logging.getLogger().setLevel(level)
except ValueError:
print("Unrecognized log level:", kwargs.log_level, file=sys.stderr)
return 1
instanceA = kwargs.InstanceA
instanceB = kwargs.InstanceB
try:
loginA = kwargs.LoginA.split(':')
loginA = (loginA[0], ':'.join(loginA[1:]))
except (KeyError, IndexError) as e:
logging.critical(
"Bad username/password pair: '%s' (hint: try -h/--help)",
kwargs.LoginA)
return 1
loginB = loginA
try:
if kwargs.LoginB:
loginB = kwargs.LoginB.split(':')
loginB = (loginB[0], ':'.join(loginB[1:]))
loginB = (loginB[0] if loginB[0] else loginA[0],
loginB[1] if loginB[1] else loginA[1])
except (KeyError, IndexError) as e:
logging.critical(
"Bad username/password pair: '%s' (hint: try -h/--help)",
kwargs.LoginB)
return 1
verify = not kwargs.insecure
# Peel off all schemas
if instanceA.startswith("https://"):
instanceA = instanceA[8:]
elif instanceA.startswith("http://"):
instanceA = instanceA[7:]
if instanceB.startswith("https://"):
instanceB = instanceB[8:]
elif instanceB.startswith("http://"):
instanceB = instanceB[7:]
# Parse out port numbers, if specified
try:
if ':' in instanceA:
instanceA = instanceA.split(':')
if len(instanceA) != 2:
logging.critical("'%s' is not a valid Traffic Ops URL!",
kwargs.InstanceA)
return 1
instanceA = {"host": instanceA[0], "port": int(instanceA[1])}
else:
instanceA = {"host": instanceA, "port": 443}
except TypeError as e:
logging.critical("'%s' is not a valid port number!", instanceA[1])
logging.debug("%s", e, exc_info=True, stack_info=True)
return 1
try:
if ':' in instanceB:
instanceB = instanceB.split(':')
if len(instanceB) != 2:
logging.critical("'%s' is not a valid Traffic Ops URL!",
kwargs.InstanceB)
instanceB = {"host": instanceB[0], "port": int(instanceB[1])}
else:
instanceB = {"host": instanceB, "port": 443}
except TypeError as e:
logging.critical("'%s' is not a valid port number!", instanceB[1])
logging.debug("%s", e, exc_info=True, stack_info=True)
return 1
# Instantiate connections and login
with TOSession(host_ip=instanceA["host"], host_port=instanceA["port"], verify_cert=verify) as A,\
TOSession(host_ip=instanceB["host"], host_port=instanceB["port"], verify_cert=verify) as B:
try:
A.login(loginA[0], loginA[1])
B.login(loginB[0], loginB[1])
except OSError as e:
logging.debug("%s", e, exc_info=True, stack_info=True)
logging.critical("Failed to connect to Traffic Ops")
return 2
except (OperationError, LoginError) as e:
logging.debug("%s", e, exc_info=True, stack_info=True)
logging.critical("Failed to log in to Traffic Ops")
logging.error(
"Error was '%s' - are you sure your URLs and credentials are correct?",
e)
for route in genRoutes(A, B):
print(route)
return 0
