def clean_products(client: ApigeeClient, env: str, dry_run: bool = False):
products = client.list_products()
pr_products = [
product for product in products if PRODUCT_MATCHER.match(product)
]
for product in pr_products:
print(f"DELETE PRODUCT {product}")
if not dry_run:
client.delete_product(product)
def clean_specs(client: ApigeeClient, env: str, dry_run: bool = False):
specs = client.list_specs()
spec_names = [spec["name"] for spec in specs["contents"]]
# Use the spec matcher to find specs that look like something we'd want to clean
pr_specs = [spec for spec in spec_names if SPEC_MATCHER.match(spec)]
for spec in pr_specs:
print(f"DELETE SPEC {spec}")
if not dry_run:
client.delete_spec(spec)
print(f"DELETED SPEC {spec}")
def clean_proxies(
client: ApigeeClient,
github_client: GithubClient,
env: str,
dry_run: bool = False,
sandboxes_only: bool = False,
min_age: Optional[int] = None,
undeploy_only: bool = False,
respect_prs: bool = False,
):
open_prs = set()
if respect_prs:
# Get open PRs, if there are any
open_prs = github_client.get_open_prs(env)
proxies = client.list_proxies()
pr_proxies = [proxy for proxy in proxies if PROXY_MATCHER.match(proxy)]
proxy_deployments = client.list_env_proxy_deployments(
env
)["aPIProxy"] # This is not a typo, it's actually spelled like that in the response
deployed_proxy_revisions = set()
for deployment in proxy_deployments:
for revision in deployment["revision"]:
if revision["state"] == "deployed":
deployed_proxy_revisions.add(
(deployment["name"], revision["name"]))
if sandboxes_only:
pr_proxies = [
proxy for proxy in pr_proxies if SANDBOX_MATCHER.match(proxy)
and not SANDBOX_ANTIMATCHER.match(proxy)
]
if respect_prs:
pr_proxies = [proxy for proxy in pr_proxies if proxy not in open_prs]
for proxy in pr_proxies:
proxy_info = client.get_proxy(proxy)
for revision in proxy_info["revision"]:
if (proxy, revision) in deployed_proxy_revisions:
print(f"UNDEPLOY {proxy} REVISION {revision}")
if not dry_run:
try:
client.undeploy_proxy_revision(env, proxy, revision)
except requests.exceptions.HTTPError:
print(
f"ERROR UNDEPLOYING {proxy} REVISION {revision}: may already be undeployed"
)
if not undeploy_only:
print(f"DELETE PROXY {proxy}")
if not dry_run:
client.delete_proxy(proxy)
with open(os.path.join(specs_dir, spec), 'r') as f:
response = client.update_spec(spec_id, f.read())
print(f'{spec} updated')
# For this, sometimes the product refs change between deploys: instead of updating, delete the old one and recreate.
for env in envs:
if 'sandbox' in env: # we don't want to publish stuff for sandbox
continue
print(f'checking if this spec is on the portal in {env}')
ns_spec_name = f'{spec_name}-{env}'
if ns_spec_name in portal_specs:
print(f'{ns_spec_name} is on the portal, updating')
apidoc_id = portal_specs[ns_spec_name]['id']
client.update_portal_api(apidoc_id,
to_friendly_name(spec_name, env),
ns_spec_name, spec_id, portal_id)
client.update_spec_snapshot(portal_id, apidoc_id)
else:
print(f'{ns_spec_name} is not on the portal, adding it')
client.create_portal_api(to_friendly_name(spec_name, env),
ns_spec_name, spec_id, portal_id)
print('done.')
if __name__ == "__main__":
args = docopt(__doc__)
client = ApigeeClient(args['<apigee_org>'], args['<username>'],
args['<password>'], args['<apigee_token>'])
upload_specs(ENV_NAMES[args['<apigee_org>']], args['<specs_dir>'], client)
github_client,
env,
dry_run,
sandboxes_only,
min_age,
undeploy_only,
respect_prs,
)
if should_clean_products:
clean_products(client, env, dry_run)
if __name__ == "__main__":
args = docopt(__doc__)
client = ApigeeClient(args["<apigee_org>"],
access_token=args["--access-token"])
github_client = GithubClient()
clean_env(
client,
github_client,
args["<apigee_env>"],
should_clean_specs=args["--specs"],
should_clean_proxies=args["--proxies"],
should_clean_products=args["--products"],
sandboxes_only=args["--sandbox-only"],
dry_run=args["--dry-run"],
min_age=args["--min-age"],
undeploy_only=args["--undeploy-only"],
respect_prs=args["--respect-prs"],
)
def upload_specs(
envs: List[str],
spec_path: str,
client: ApigeeClient,
friendly_name: str = None,
is_pr: bool = False,
):
if "*" in spec_path:
spec_files = glob(spec_path, recursive=True)
if not spec_files:
raise RuntimeError(
f"Could not find any files with glob {spec_path}")
if len(spec_files) > 1:
raise RuntimeError(
f"Found too many spec files for {spec_path}: {spec_files}")
spec_path = spec_files[0]
# Grab a list of remote specs
folder = client.list_specs()
folder_id = folder["id"]
existing_specs = {v["name"]: v["id"] for v in folder["contents"]}
# Figure out where the portal is
portal_id = client.get_portals().json()["data"][0]["id"]
print(f"portal is {portal_id}")
portal_specs = {
i["specId"]: i
for i in client.get_apidocs(portal_id).json()["data"]
}
print(f"grabbed apidocs")
spec_name = os.path.splitext(os.path.basename(spec_path))[0]
if spec_name in existing_specs:
print(f"{spec_name} exists")
spec_id = existing_specs[spec_name]
else:
print(f"{spec_name} does not exist, creating")
response = client.create_spec(spec_name, folder_id)
spec_id = response.json()["id"]
print(f"{spec_name} id is {spec_id}")
with open(spec_path, "r") as f:
response = client.update_spec(spec_id, f.read())
print(f"{spec_name} updated")
# Skip blacklisted services, e.g. identity service that do not need a portal entry
if spec_name not in SERVICE_BLACKLIST:
# For this, sometimes the product refs change between deploys: instead of updating, delete the old one and recreate.
for env in envs:
if env in PORTAL_BLACKLIST: # we don't want to publish in blacklisted envs
continue
if "sandbox" in env: # we don't want to publish stuff for sandbox
continue
print(f"checking if this spec is on the portal in {env}")
ns_spec_name = spec_name if is_pr else f"{spec_name}-{env}"
if ns_spec_name in portal_specs:
print(f"{ns_spec_name} is on the portal, updating")
apidoc_id = portal_specs[ns_spec_name]["id"]
client.update_portal_api(
apidoc_id,
to_friendly_name(spec_name, env, friendly_name),
ns_spec_name,
spec_id,
portal_id,
True,
CALLBACK_REQUIRED.get(spec_name, False),
)
client.update_spec_snapshot(portal_id, apidoc_id)
else:
print(f"{ns_spec_name} is not on the portal, adding it")
client.create_portal_api(
to_friendly_name(spec_name, env, friendly_name),
ns_spec_name,
spec_id,
portal_id,
True,
CALLBACK_REQUIRED.get(spec_name, False),
)
print("done.")