def test_get_deployments(mock_zone): mock_zone.return_value = "us-central1-a" test_dir = os.path.join(os.path.dirname(__file__), "test_data") with open(os.path.join(test_dir, "deployments.yaml")) as hf: deployments = yaml.load(hf) dm_reconciler = reconciler.Reconciler() dm_reconciler._get_deployments(deployments=deployments) # pylint: disable=protected-access vmaster = [ auto_deploy_util.AutoDeployment( deployment_name="kf-vmaster-0126-1c1", manifests_branch="master", create_time="2020-01-26 04:04:19.267000-08:00"), auto_deploy_util.AutoDeployment( deployment_name="kf-vmaster-0127-502", manifests_branch="master", create_time="2020-01-26 16:04:13.855000-08:00"), auto_deploy_util.AutoDeployment( deployment_name="kf-vmaster-0127-082", manifests_branch="master", create_time="2020-01-27 04:15:52.111000-08:00"), ] expected = { "vmaster": vmaster, } assertions.assert_dicts_equal(dm_reconciler._deployments, expected) # pylint: disable=protected-access
def _get_deployments(self, deployments=None): """Build a map of all deployments Args: deployments: (Optional) Iterator over GCP deployments. """ logging.info("Building map of auto deployments") self._deployments = collections.defaultdict(lambda: []) if not deployments: deployments = gcp_util.deployments_iterator(self.config["project"]) for d in deployments: is_auto_deploy = False # Use labels to identify auto-deployed instances labels = {} for label_pair in d.get("labels", []): # Newer clusters if label_pair["key"] == "auto-deploy": is_auto_deploy = True # Older clusters if (label_pair["key"] == "purpose" and label_pair["value"] == "kf-test-cluster"): is_auto_deploy = True labels[label_pair["key"]] = label_pair["value"] if not is_auto_deploy: logging.info( "Skipping deployment %s; its missing the label " "auto-deploy", d["name"]) continue if d.get("operation", {}).get("operationType") == "delete": logging.info( f"Skipping deployment {d['name']} it is being deleted.") if auto_deploy_util.is_storage_deployment(d["name"]): logging.info(f"Skipping deployment {d['name']}; it is storage") continue version_name = labels.get(auto_deploy_util.AUTO_NAME_LABEL, "unknown") if not "manifest" in d: # Since we don't know the manifest we can't get the zone. # It looks like the manfiest might also be stored in the operation. # However, it looks like the reason the manifest isn't there # is because the deployment failed. So we can just set zone to the # empty string. I think zone only matters for getting cluster # credentials but since the deployment failed that shouldn't matter. logging.error(f"Deployment {d['name']} doesn't " "have a manifest. This typically indicates the " "deployment failed") zone = "" else: dm_manifest_name = d["manifest"].split("/")[-1] zone = self._get_deployment_zone(d["name"], dm_manifest_name) context = { "deployment_name": d['name'], "version_name": version_name, } manifests_branch = labels.get(auto_deploy_util.BRANCH_LABEL, "unknown") create_time = date_parser.parse(d.get("insertTime")) deployment = auto_deploy_util.AutoDeployment( manifests_branch=manifests_branch, create_time=create_time, deployment_name=d["name"], labels=labels) deployment.zone = zone logging.info( f"Found auto deployment={d['name']} for version={version_name}", extra=context) self._deployments[version_name] = ( self._deployments[version_name] + [deployment]) # Sort the values by timestamp branches = self._deployments.keys() for b in branches: self._deployments[b] = sorted(self._deployments[b], key=lambda x: x.create_time) self._save_deployments()
def _get_deployments(self, deployments=None): """Build a map of all deployments Args: deployments: (Optional) Iterator over GCP deployments. """ logging.info("Building map of auto deployments") self._deployments = collections.defaultdict(lambda: []) if not deployments: deployments = gcp_util.deployments_iterator(self.config["project"]) for d in deployments: is_auto_deploy = False # Use labels to identify auto-deployed instances labels = {} for label_pair in d.get("labels", []): # Newer clusters if label_pair["key"] == "auto-deploy": is_auto_deploy = True # Older clusters if (label_pair["key"] == "purpose" and label_pair["value"] == "kf-test-cluster"): is_auto_deploy = True labels[label_pair["key"]] = label_pair["value"] if not is_auto_deploy: logging.info( "Skipping deployment %s; its missing the label " "auto-deploy", d["name"]) continue if d.get("operation", {}).get("operationType") == "delete": logging.info( f"Skipping deployment {d['name']} it is being deleted.") if auto_deploy_util.is_storage_deployment(d["name"]): logging.info(f"Skipping deployment {d['name']}; it is storage") continue version_name = labels.get(auto_deploy_util.AUTO_NAME_LABEL, "unknown") dm_manifest_name = d["manifest"].split("/")[-1] zone = self._get_deployment_zone(d["name"], dm_manifest_name) context = { "deployment_name": d['name'], "version_name": version_name, } manifests_branch = labels.get(auto_deploy_util.BRANCH_LABEL, "unknown") create_time = date_parser.parse(d.get("insertTime")) deployment = auto_deploy_util.AutoDeployment( manifests_branch=manifests_branch, create_time=create_time, deployment_name=d["name"], labels=labels) deployment.zone = zone logging.info( f"Found auto deployment={d['name']} for version={version_name}", extra=context) self._deployments[version_name] = ( self._deployments[version_name] + [deployment]) # Sort the values by timestamp branches = self._deployments.keys() for b in branches: self._deployments[b] = sorted(self._deployments[b], key=lambda x: x.create_time) self._save_deployments()