def get_gcr_auth_info(layer: Layer) -> Tuple[str, str]:
if GCP.using_service_account():
service_account_key = GCP.get_service_account_raw_credentials()
return "_json_key", service_account_key
credentials, _ = GCP.get_credentials()
return "oauth2accesstoken", credentials.token
def _gcp_delete_state_storage(cls, layer: "Layer") -> None:
providers = layer.gen_providers(0)
if "gcs" not in providers.get("terraform", {}).get("backend", {}):
return
bucket_name = providers["terraform"]["backend"]["gcs"]["bucket"]
credentials, project_id = GCP.get_credentials()
gcs_client = storage.Client(project=project_id,
credentials=credentials)
try:
bucket_obj = gcs_client.get_bucket(bucket_name)
bucket_obj.delete(force=True)
logger.info("Successfully deleted GCP state storage")
except NotFound:
logger.warning("State bucket was already deleted")
def _gcp_get_configs(layer: "Layer") -> List[str]:
bucket_name = layer.state_storage()
gcs_config_dir = "opta_config/"
credentials, project_id = GCP.get_credentials()
gcs_client = storage.Client(project=project_id, credentials=credentials)
try:
bucket_object = gcs_client.get_bucket(bucket_name)
except NotFound:
logger.warning(
"Couldn't find the state bucket, must have already been destroyed in a previous destroy run"
)
return []
blobs: List[storage.Blob] = list(
gcs_client.list_blobs(bucket_object, prefix=gcs_config_dir)
)
configs = [blob.name[len(gcs_config_dir) :] for blob in blobs]
if layer.name in configs:
configs.remove(layer.name)
return configs
def _create_gcp_state_storage(cls, providers: dict) -> None:
bucket_name = providers["terraform"]["backend"]["gcs"]["bucket"]
region = providers["provider"]["google"]["region"]
project_name = providers["provider"]["google"]["project"]
credentials, project_id = GCP.get_credentials()
if project_id != project_name:
raise UserErrors(
f"We got {project_name} as the project name in opta, but {project_id} in the google credentials"
)
gcs_client = storage.Client(project=project_id,
credentials=credentials)
try:
bucket = gcs_client.get_bucket(bucket_name)
bucket_project_number = bucket.project_number
except GoogleClientError as e:
if e.code == 403:
raise UserErrors(
f"The Bucket Name: {bucket_name} (Opta needs to store state here) already exists.\n"
"Possible Failures:\n"
" - Bucket is present in some other project and User does not have access to the Project.\n"
"Please change the name in the Opta Configuration file or please change the User Permissions.\n"
"Please fix it and try again.")
elif e.code != 404:
raise UserErrors(
"When trying to determine the status of the state bucket, we got an "
f"{e.code} error with the message "
f"{e.message}")
logger.debug(
"GCS bucket for terraform state not found, creating a new one")
try:
bucket = gcs_client.create_bucket(bucket_name, location=region)
bucket_project_number = bucket.project_number
except Conflict:
raise UserErrors(
f"It looks like a gcs bucket with the name {bucket_name} was created recently, but then deleted "
"and Google keeps hold of gcs bucket names for 30 days after deletion-- pls wait until the end of "
"that time or change your environment name slightly.")
# Enable the APIs
credentials = GoogleCredentials.get_application_default()
service = discovery.build("serviceusage",
"v1",
credentials=credentials,
static_discovery=False)
new_api_enabled = False
for service_name in [
"container.googleapis.com",
"iam.googleapis.com",
"containerregistry.googleapis.com",
"cloudkms.googleapis.com",
"dns.googleapis.com",
"servicenetworking.googleapis.com",
"redis.googleapis.com",
"compute.googleapis.com",
"secretmanager.googleapis.com",
"cloudresourcemanager.googleapis.com",
]:
request = service.services().enable(
name=f"projects/{project_name}/services/{service_name}")
try:
response = request.execute()
new_api_enabled = new_api_enabled or (
response.get("name") != "operations/noop.DONE_OPERATION")
except HttpError as e:
if e.resp.status == 400:
raise UserErrors(
f"Got a 400 response when trying to enable the google {service_name} service with the following error reason: {e._get_reason()}"
)
logger.debug(f"Google service {service_name} activated")
if new_api_enabled:
logger.info(
"New api has been enabled, waiting 120 seconds before progressing"
)
time.sleep(120)
service = discovery.build(
"cloudresourcemanager",
"v1",
credentials=credentials,
static_discovery=False,
)
request = service.projects().get(projectId=project_id)
response = request.execute()
if response["projectNumber"] != str(bucket_project_number):
raise UserErrors(
f"State storage bucket {bucket_name}, has already been created, but it was created in another project. "
f"Current project's number {response['projectNumber']}. Bucket's project number: {bucket_project_number}. "
"You do, however, have access to view that bucket, so it sounds like you already run this opta apply in "
"your org, but on a different project."
"Note: project number is NOT project id. It is yet another globally unique identifier for your project "
"I kid you not, go ahead and look it up.")
def download_state(cls, layer: "Layer") -> bool:
if layer.is_stateless_mode() is True:
# no remote state for stateless mode
return False
if not cls.verify_storage(layer):
logger.debug(
fmt_msg("""
We store state in S3/GCP buckets/Azure Storage. Since the state bucket was not found,
~this probably means that you either haven't created your opta resources yet,
~or you previously successfully destroyed your opta resources.
"""))
return False
state_file: str = "./tmp.tfstate"
providers = layer.gen_providers(0)
terraform_backends = providers.get("terraform", {}).get("backend", {})
if "s3" in terraform_backends:
bucket = providers["terraform"]["backend"]["s3"]["bucket"]
region = providers["terraform"]["backend"]["s3"]["region"]
key = providers["terraform"]["backend"]["s3"]["key"]
logger.debug(
f"Found an s3 backend in bucket {bucket} and key {key}, "
"gonna try to download the statefile from there")
s3 = boto3.client("s3", config=Config(region_name=region))
try:
s3.download_file(Bucket=bucket, Key=key, Filename=state_file)
except ClientError as e:
if e.response["Error"]["Code"] == "404":
# The object does not exist.
logger.debug("Did not find terraform state file")
return False
raise
elif "gcs" in terraform_backends:
bucket = providers["terraform"]["backend"]["gcs"]["bucket"]
prefix = providers["terraform"]["backend"]["gcs"]["prefix"]
credentials, project_id = GCP.get_credentials()
gcs_client = storage.Client(project=project_id,
credentials=credentials)
bucket_object = gcs_client.get_bucket(bucket)
blob = storage.Blob(f"{prefix}/default.tfstate", bucket_object)
try:
with open(state_file, "wb") as file_obj:
gcs_client.download_blob_to_file(blob, file_obj)
except GoogleClientError as e:
if e.code == 404:
# The object does not exist.
os.remove(state_file)
return False
raise
elif "azurerm" in terraform_backends:
storage_account_name = providers["terraform"]["backend"][
"azurerm"]["storage_account_name"]
container_name = providers["terraform"]["backend"]["azurerm"][
"container_name"]
key = providers["terraform"]["backend"]["azurerm"]["key"]
credentials = Azure.get_credentials()
try:
blob = (BlobServiceClient(
f"https://{storage_account_name}.blob.core.windows.net/",
credential=credentials,
).get_container_client(container_name).get_blob_client(key))
with open(state_file, "wb") as file_obj:
blob_data = blob.download_blob()
blob_data.readinto(file_obj)
except ResourceNotFoundError:
return False
elif layer.cloud == "local":
try:
tf_file = os.path.join(
cls.get_local_opta_dir(),
"tfstate",
f"{layer.name}",
)
if os.path.exists(tf_file):
copyfile(tf_file, state_file)
else:
return False
except Exception:
UserErrors(f"Could copy local state file to {state_file}")
elif layer.cloud == "helm":
set_kube_config(layer)
load_opta_kube_config()
v1 = CoreV1Api()
secret_name = f"tfstate-default-{layer.state_storage()}"
secrets: V1SecretList = v1.list_namespaced_secret(
"default", field_selector=f"metadata.name={secret_name}")
if len(secrets.items) == 0:
return False
secret: V1Secret = secrets.items[0]
decoded_secret = gzip.decompress(
base64.b64decode(secret.data["tfstate"]))
with open(state_file, "wb") as file_obj:
file_obj.write(decoded_secret)
else:
raise UserErrors(
"Need to get state from S3 or GCS or Azure storage")
with open(state_file, "r") as file:
raw_state = file.read().strip()
os.remove(state_file)
if raw_state != "":
cls.downloaded_state[layer.name] = json.loads(raw_state)
return True
return False
def metadata_hydration(self) -> Dict[Any, Any]:
parent_name = self.parent.name if self.parent is not None else "nil"
parent = None
if self.parent is not None:
parent = SimpleNamespace(
**{
k: f"${{data.terraform_remote_state.parent.outputs.{k}}}"
for k in self.parent.outputs()
}
)
providers = self.providers
if self.parent is not None:
providers = deep_merge(providers, self.parent.providers)
provider_hydration = {}
for name, values in providers.items():
provider_hydration[name] = SimpleNamespace(**values)
region: Optional[str] = None
k8s_access_token = None
if self.cloud == "google":
gcp = GCP(self)
region = gcp.region
credentials = gcp.get_credentials()[0]
if isinstance(credentials, service_account.Credentials):
service_account_credentials: service_account.Credentials = (
credentials.with_scopes(
[
"https://www.googleapis.com/auth/userinfo.email",
"https://www.googleapis.com/auth/cloud-platform",
]
)
)
service_account_credentials.refresh(
google.auth.transport.requests.Request()
)
k8s_access_token = service_account_credentials.token
else:
k8s_access_token = credentials.token
if k8s_access_token is None:
raise Exception("Was unable to get GCP access token")
elif self.cloud == "aws":
aws = AWS(self)
region = aws.region
elif self.cloud == "azurerm":
region = self.root().providers["azurerm"]["location"]
elif self.cloud == "local":
pass
return {
"parent": parent,
"vars": SimpleNamespace(**self.variables),
"variables": SimpleNamespace(**self.variables),
"parent_name": parent_name,
"layer_name": self.name,
"state_storage": self.state_storage(),
"env": self.get_env(),
"kubeconfig": KUBE_CONFIG_DEFAULT_LOCATION,
"k8s_access_token": k8s_access_token,
"region": region,
**provider_hydration,
}