def get_azure_credentials(config):
credentials = None
cloud_environment = get_azure_cloud_environment(config)
if config.UseMSI and cloud_environment:
from msrestazure.azure_active_directory import MSIAuthentication
credentials = MSIAuthentication(cloud_environment=cloud_environment)
elif config.UseMSI:
from msrestazure.azure_active_directory import MSIAuthentication
credentials = MSIAuthentication()
elif cloud_environment:
from azure.common.credentials import ServicePrincipalCredentials
credentials = ServicePrincipalCredentials(
client_id = config.ApplicationId,
secret = config.ApplicationKey,
tenant = config.Tenantid,
cloud_environment=cloud_environment
)
else:
from azure.common.credentials import ServicePrincipalCredentials
credentials = ServicePrincipalCredentials(
client_id = config.ApplicationId,
secret = config.ApplicationKey,
tenant = config.Tenantid
)
return credentials
def _get_client(self):
if os.getenv("USE_MSI", "false").lower() == "true":
_logger.info('Using MSI')
if "MSI_CLIENT_ID" in os.environ:
msi_client_id = os.environ["MSI_CLIENT_ID"]
_logger.info('Using client_id: %s', msi_client_id)
credentials = MSIAuthentication(resource=VAULT_RESOURCE_NAME, client_id=msi_client_id)
elif "MSI_OBJECT_ID" in os.environ:
msi_object_id = os.environ["MSI_OBJECT_ID"]
_logger.info('Using object_id: %s', msi_object_id)
credentials = MSIAuthentication(resource=VAULT_RESOURCE_NAME, object_id=msi_object_id)
elif "MSI_RESOURCE_ID" in os.environ:
msi_resource_id = os.environ["MSI_RESOURCE_ID"]
_logger.info('Using resource_id: %s', msi_resource_id)
credentials = MSIAuthentication(resource=VAULT_RESOURCE_NAME, msi_res_id=msi_resource_id)
else:
credentials = MSIAuthentication(resource=VAULT_RESOURCE_NAME)
else:
self._parse_sp_file()
authority = '/'.join([AZURE_AUTHORITY_SERVER.rstrip('/'), self.tenant_id])
_logger.info('Using authority: %s', authority)
context = AuthenticationContext(authority)
_logger.info('Using vault resource name: %s and client id: %s', VAULT_RESOURCE_NAME, self.client_id)
credentials = AdalAuthentication(context.acquire_token_with_client_credentials, VAULT_RESOURCE_NAME,
self.client_id, self.client_secret)
return KeyVaultClient(credentials)
def get_azure_credentials(config):
credentials = None
cloud_environment = get_azure_cloud_environment(config)
if config.UseMSI and cloud_environment:
from msrestazure.azure_active_directory import MSIAuthentication
credentials = MSIAuthentication(cloud_environment=cloud_environment)
elif config.UseMSI:
from msrestazure.azure_active_directory import MSIAuthentication
credentials = MSIAuthentication()
elif cloud_environment:
from azure.identity import ClientSecretCredential
credentials = ClientSecretCredential(
client_id = config.ApplicationId,
client_secret = config.ApplicationKey,
tenant_id = config.Tenantid,
cloud_environment=cloud_environment
)
else:
from azure.identity import ClientSecretCredential
credentials = ClientSecretCredential(
client_id = config.ApplicationId,
client_secret = config.ApplicationKey,
tenant_id = config.Tenantid
)
return credentials
def _get_token_from_cloud_shell(self, resource): # pylint: disable=no-self-use
from msrestazure.azure_active_directory import MSIAuthentication
auth = MSIAuthentication(resource=resource)
auth.set_token()
token_entry = auth.token
return (token_entry['token_type'], token_entry['access_token'],
token_entry)
def test_msi_vm_imds_no_retry_on_bad_error(self):
httpretty.register_uri(httpretty.GET,
'http://169.254.169.254/metadata/identity/oauth2/token',
status=499)
credentials = MSIAuthentication()
with self.assertRaises(HTTPError) as cm:
credentials.set_token()
def _authenticate(self):
keyvault_client_id = self._auth_params.get('keyvault_client_id')
keyvault_secret_id = self._auth_params.get('keyvault_secret_id')
# If user provided KeyVault secret, we will pull auth params information from it
if keyvault_secret_id:
self._auth_params.update(
json.loads(
get_keyvault_secret(keyvault_client_id,
keyvault_secret_id)))
client_id = self._auth_params.get('client_id')
client_secret = self._auth_params.get('client_secret')
access_token = self._auth_params.get('access_token')
tenant_id = self._auth_params.get('tenant_id')
use_msi = self._auth_params.get('use_msi')
subscription_id = self._auth_params.get('subscription_id')
if access_token and subscription_id:
self.log.info("Creating session with Token Authentication")
self.subscription_id = subscription_id
self.credentials = BasicTokenAuthentication(
token={'access_token': access_token})
self._is_token_auth = True
elif client_id and client_secret and tenant_id and subscription_id:
self.log.info(
"Creating session with Service Principal Authentication")
self.subscription_id = subscription_id
self.credentials = ServicePrincipalCredentials(
client_id=client_id,
secret=client_secret,
tenant=tenant_id,
resource=self.resource_namespace)
self.tenant_id = tenant_id
elif use_msi and subscription_id:
self.log.info("Creating session with MSI Authentication")
self.subscription_id = subscription_id
if client_id:
self.credentials = MSIAuthentication(
client_id=client_id, resource=self.resource_namespace)
else:
self.credentials = MSIAuthentication(
resource=self.resource_namespace)
elif self._auth_params.get('enable_cli_auth'):
self.log.info("Creating session with Azure CLI Authentication")
self._is_cli_auth = True
try:
(self.credentials, self.subscription_id,
self.tenant_id) = Profile().get_login_credentials(
resource=self.resource_namespace)
except Exception:
self.log.error('Unable to authenticate with Azure')
self.log.info("Session using Subscription ID: %s" %
self.subscription_id)
def msi_auth_factory(cli_account_name, identity, resource):
from msrestazure.azure_active_directory import MSIAuthentication
if cli_account_name == MsiAccountTypes.system_assigned:
return MSIAuthentication(resource=resource)
elif cli_account_name == MsiAccountTypes.user_assigned_client_id:
return MSIAuthentication(resource=resource, client_id=identity)
elif cli_account_name == MsiAccountTypes.user_assigned_object_id:
return MSIAuthentication(resource=resource, object_id=identity)
elif cli_account_name == MsiAccountTypes.user_assigned_resource_id:
return MSIAuthentication(resource=resource, msi_res_id=identity)
else:
raise ValueError("unrecognized msi account name '{}'".format(cli_account_name))
def get_azure_credentials(config):
credentials = None
cloud_environment = get_azure_cloud_environment(config)
if config.UseMSI and cloud_environment:
try:
from azure.identity import ManagedIdentityCredential
credentials = ManagedIdentityCredential(
cloud_environment=cloud_environment)
except ImportError:
from msrestazure.azure_active_directory import MSIAuthentication
credentials = MSIAuthentication(
cloud_environment=cloud_environment)
elif config.UseMSI:
try:
from azure.identity import ManagedIdentityCredential
credentials = ManagedIdentityCredential()
except ImportError:
from msrestazure.azure_active_directory import MSIAuthentication
credentials = MSIAuthentication()
elif cloud_environment:
try:
# try to use new libraries ClientSecretCredential (azure.identity, based on azure.core)
from azure.identity import ClientSecretCredential
credentials = ClientSecretCredential(
client_id=config.ApplicationId,
client_secret=config.ApplicationKey,
tenant_id=config.Tenantid,
cloud_environment=cloud_environment)
except ImportError:
# use old libraries ServicePrincipalCredentials (azure.common) if new one is not available
from azure.common.credentials import ServicePrincipalCredentials
credentials = ServicePrincipalCredentials(
client_id=config.ApplicationId,
secret=config.ApplicationKey,
tenant=config.Tenantid,
cloud_environment=cloud_environment)
else:
try:
# try to use new libraries ClientSecretCredential (azure.identity, based on azure.core)
from azure.identity import ClientSecretCredential
credentials = ClientSecretCredential(
client_id=config.ApplicationId,
client_secret=config.ApplicationKey,
tenant_id=config.Tenantid)
except ImportError:
# use old libraries ServicePrincipalCredentials (azure.common) if new one is not available
from azure.common.credentials import ServicePrincipalCredentials
credentials = ServicePrincipalCredentials(
client_id=config.ApplicationId,
secret=config.ApplicationKey,
tenant=config.Tenantid)
return credentials
def _get_azure_credentials(client_id=None,
client_secret=None,
tenant=None,
msi_client_id=None):
has_sp = all((client_id, client_secret, tenant))
if has_sp:
return ServicePrincipalCredentials(client_id=client_id,
secret=client_secret,
tenant=tenant)
elif msi_client_id:
return MSIAuthentication(client_id=msi_client_id)
else:
return MSIAuthentication()
def get_token_from_msi(self) -> dict:
try:
if self.msi_auth_context is None:
# Create the MSI Authentication object, the first token is acquired implicitly
self.msi_auth_context = MSIAuthentication(**self.msi_params)
else:
# Acquire a fresh token
self.msi_auth_context.set_token()
except Exception as e:
raise KustoClientError("Failed to obtain MSI context for [" + str(self.msi_params) + "]\n" + str(e))
return self.msi_auth_context.token
def run_example():
"""MSI Authentication example."""
#
# Create System Assigned MSI Authentication
#
credentials = MSIAuthentication()
#
# Create User Assigned MSI Authentication, using client_id
#
credentials = MSIAuthentication(
client_id='00000000-0000-0000-0000-000000000000')
#
# Create User Assigned MSI Authentication, using object_id
#
credentials = MSIAuthentication(
object_id='00000000-0000-0000-0000-000000000000')
#
# Create User Assigned MSI Authentication, using msi_res_id
#
credentials = MSIAuthentication(
msi_res_id=
'/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/msiname'
)
#
# Create a Subscription client, and get the subscription ID attached to that credentials.
# This assumes there is only ONE subscription attached to this MSI token (most likely scenario)
#
subscription_client = SubscriptionClient(credentials)
subscription = next(subscription_client.subscriptions.list())
subscription_id = subscription.subscription_id
#
# Create a Resource Management client
#
resource_client = ResourceManagementClient(credentials, subscription_id)
#
# List resource groups as an example. The only limit is what role and policy
# are assigned to this MSI token.
#
for resource_group in resource_client.resource_groups.list():
print(resource_group.name)
class ProductionConfig(Config):
CACHE_TYPE = "filesystem"
CACHE_DIR = os.environ.get("TRACKER_CACHE_DIR", "./.cache")
CACHE_DEFAULT_TIMEOUT = int(os.environ.get("TRACKER_CACHE_TIMEOUT", A_DAY))
if os.environ.get('TRACKER_ENV', None) == "production":
if os.environ.get("TRACKER_KEYVAULT_URI",
None) is None or os.environ.get(
"SECRET_NAME_RO", None) is None:
# Error and crash hard: Production should be configured as expected.
LOGGER.error(
"KeyVault uri or secret name missing from local environment.")
sys.exit(4)
kv_uri = os.environ.get("TRACKER_KEYVAULT_URI")
kv_secret = os.environ.get("SECRET_NAME_RO")
kv_creds = MSIAuthentication(resource='https://vault.azure.net')
kv_client = KeyVaultClient(kv_creds)
MONGO_URI = kv_client.get_secret(kv_uri, kv_secret, "").value
@staticmethod
def init_app(app):
Config.init_app(app)
import logging
from logging.handlers import SysLogHandler
handler = SysLogHandler(
address=os.environ.get("TRACKER_SYSLOG", "/dev/log"))
handler.setLevel(logging.ERROR)
app.logger.addHandler(handler)
def _get_database_uri():
if _is_app_service():
logger.info("Getting Production DB credentials from Keyvault")
credentials = MSIAuthentication()
key_vault_client = KeyVaultClient(credentials)
key_vault_uri = os.environ.get("KEY_VAULT_URI",
"PUT YOUR KV BASE URL HERE")
logger.info("Using KEY_VAULT_URI: {}".format(key_vault_uri))
result = key_vault_client.get_secret(
vault_base_url=key_vault_uri,
secret_name="database-url",
secret_version="" # empty string -> latest version
).value
logger.info("Secret info: {} {}".format(type(result),
len(result))) # XXX TEMP DEBUG
return result
else:
# TODO: this is just to allow the create_db script to be run locally on initial setup
# Really, the KV code above should be updated to work with other non-MSI credentials
if "DATABASE_URL" in os.environ:
logger.info("Using DATABASE_URL credentials")
return os.environ['DATABASE_URL']
else:
logger.info("Using local Testing Database")
return TestingConfig.SQLALCHEMY_DATABASE_URI
def sign(key_hash):
p2sig = ''
response = None
try:
data = request.get_json(force=True)
if key_hash in config['keys']:
info('Found key_hash {} in config'.format(key_hash))
key = config['keys'][key_hash]
kvclient = KeyVaultClient(
MSIAuthentication(resource='https://vault.azure.net'))
info('Calling remote-signer method {}'.format(data))
p2sig = RemoteSigner(kvclient, key['kv_keyname'], config,
request.environ['REMOTE_ADDR'], data).sign()
response = jsonify({'signature': p2sig})
info('Response is {}'.format(response))
else:
warning("Couldn't find key {}".format(key_hash))
response = Response('Key not found', status=404)
except Exception as e:
data = {'error': str(e)}
error('Exception thrown during request: {}'.format(str(e)))
response = app.response_class(response=json.dumps(data),
status=500,
mimetype='application/json')
info('Returning flask response {}'.format(response))
return response
def get_blob_with_system_MSI():
"""
"""
storage_credential = MSIAuthentication(
resource='https://storage.azure.com/')
jwt_token = storage_credential.token["access_token"]
jwt_json = jwt.decode(jwt_token, verify=False)
json_t = json.dumps(jwt_json)
json_dict = json.loads(json_t)
print("MSI Token:")
print("aud: {}\niss: {}\nxms_mirid: {}\n".format(json_dict['aud'],
json_dict['iss'],
json_dict['xms_mirid']))
service = BlockBlobService('azuredocuments',
token_credential=storage_credential)
print("\nListing blobs in the 'azuredocs' container:\n")
generator = service.list_blobs('azuredocs')
for blob in generator:
print("\tBlob name: " + blob.name)
def get_blob_with_user_MSI():
storage_credential = MSIAuthentication(
client_id='3d5368bc-4bfd-4ccf-aa6a-649a237504ec',
resource='https://storage.azure.com/')
#print("Storage Credential {}".format(storage_credential))
#print("Storage Token {}".format(storage_credential.token["access_token"]))
base_64_token = storage_credential.token["access_token"] + '=='
jwt_token = storage_credential.token["access_token"]
jwt_json = jwt.decode(jwt_token, verify=False)
json_t = json.dumps(jwt_json)
json_dict = json.loads(json_t)
print("MSI Token:".format(json_t))
print("aud: {}\niss: {}\nxms_mirid: {}\n".format(json_dict['aud'],
json_dict['iss'],
json_dict['xms_mirid']))
service = BlockBlobService('azuredocuments',
token_credential=storage_credential)
print("\nListing blobs in the 'azuredocs' container:\n")
generator = service.list_blobs('azuredocs')
for blob in generator:
print("\tBlob name: " + blob.name)
def get_keyvault_client(self):
# Don't use MSI credentials if the auth_source isn't set to MSI. The below will Always result in credentials when running on an Azure VM.
if self.module.params['auth_source'] == 'msi':
try:
self.log("Get KeyVaultClient from MSI")
resource = self.azure_auth._cloud_environment.suffixes.keyvault_dns.split('.', 1).pop()
credentials = MSIAuthentication(resource="https://{0}".format(resource))
return KeyVaultClient(credentials)
except Exception:
self.log("Get KeyVaultClient from service principal")
# Create KeyVault Client using KeyVault auth class and auth_callback
def auth_callback(server, resource, scope):
if self.credentials['client_id'] is None or self.credentials['secret'] is None:
self.fail('Please specify client_id, secret and tenant to access azure Key Vault.')
tenant = self.credentials.get('tenant')
if not self.credentials['tenant']:
tenant = "common"
authcredential = ServicePrincipalCredentials(
client_id=self.credentials['client_id'],
secret=self.credentials['secret'],
tenant=tenant,
cloud_environment=self._cloud_environment,
resource="https://vault.azure.net")
token = authcredential.token
return token['token_type'], token['access_token']
return KeyVaultClient(KeyVaultAuthentication(auth_callback))
def _get_access_token_msi():
"""
get_access_token_msi()
get an Azure access token using the MSI library
"""
return MSIAuthentication().token.get('access_token')
def authenticate(self):
# type: () -> TokenProvider.AuthenticationResult
try:
if self.client_id:
credential = MSIAuthentication(client_id=self.client_id,
resource=self.resource_endpoint)
else:
credential = MSIAuthentication(resource=self.resource_endpoint)
except HTTPError as e:
e.message = 'Failed to authenticate with MSI'
raise
return TokenProvider.AuthenticationResult(
credential=credential,
subscription_id=self.subscription_id,
tenant_id=None)
def main(mytimer: func.TimerRequest) -> None:
utc_timestamp = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc).isoformat()
if mytimer.past_due:
logging.info('The timer is past due!')
logging.info('Python timer trigger function ran at %s', utc_timestamp)
#credentials = DefaultAzureCredential()
credentials = MSIAuthentication()
compute_client = ComputeManagementClient(credentials, subscription_id)
# List VM in resource group
print('\nList VMs in resource group')
for group in groups:
for vm in compute_client.virtual_machines.list(group):
logging.info("\tVM: {}".format(vm.name))
jsonStr = json.dumps(vm.tags)
logging.info(jsonStr)
logging.info(vm.id)
temp_id_list = vm.id.split('/')
resource_group = temp_id_list[4]
statuses = compute_client.virtual_machines.instance_view(
group, vm.name).statuses
status = len(statuses) >= 2 and statuses[1]
if vm.tags is not None:
if key in vm.tags:
if key_value in vm.tags[key]:
if status and status.code == 'PowerState/running':
logging.info('This VM is running %s', vm.name)
async_vm_stop = compute_client.virtual_machines.power_off(
group, vm.name)
async_vm_stop.wait()
logging.info('Stop command executed for %s',
vm.name)
def _determine_auth(**kwargs):
'''
Acquire Azure ARM Credentials
'''
if 'profile' in kwargs:
azure_credentials = __salt__['config.option'](kwargs['profile'])
kwargs.update(azure_credentials)
service_principal_creds_kwargs = ['client_id', 'secret', 'tenant']
user_pass_creds_kwargs = ['username', 'password']
try:
if kwargs.get('cloud_environment') and kwargs.get('cloud_environment').startswith('http'):
cloud_env = get_cloud_from_metadata_endpoint(kwargs['cloud_environment'])
else:
cloud_env_module = importlib.import_module('msrestazure.azure_cloud')
cloud_env = getattr(cloud_env_module, kwargs.get('cloud_environment', 'AZURE_PUBLIC_CLOUD'))
except (AttributeError, ImportError, MetadataEndpointError):
raise sys.exit('The Azure cloud environment {0} is not available.'.format(kwargs['cloud_environment']))
if set(service_principal_creds_kwargs).issubset(kwargs):
if not (kwargs['client_id'] and kwargs['secret'] and kwargs['tenant']):
raise SaltInvocationError(
'The client_id, secret, and tenant parameters must all be '
'populated if using service principals.'
)
else:
credentials = ServicePrincipalCredentials(kwargs['client_id'],
kwargs['secret'],
tenant=kwargs['tenant'],
cloud_environment=cloud_env)
elif set(user_pass_creds_kwargs).issubset(kwargs):
if not (kwargs['username'] and kwargs['password']):
raise SaltInvocationError(
'The username and password parameters must both be '
'populated if using username/password authentication.'
)
else:
credentials = UserPassCredentials(kwargs['username'],
kwargs['password'],
cloud_environment=cloud_env)
elif 'subscription_id' in kwargs:
credentials = MSIAuthentication(cloud_environment=cloud_env)
else:
raise SaltInvocationError(
'Unable to determine credentials. '
'A subscription_id with username and password, '
'or client_id, secret, and tenant or a profile with the '
'required parameters populated'
)
if 'subscription_id' not in kwargs:
raise SaltInvocationError(
'A subscription_id must be specified'
)
subscription_id = salt.utils.stringutils.to_str(kwargs['subscription_id'])
return credentials, subscription_id, cloud_env
def handleHello():
try:
# get credentials
sys.stdout.write(keyVaultURL + "\n")
credentials = MSIAuthentication(resource="https://vault.azure.net")
# create a KeyVault client
sys.stdout.write("before kv client\n")
key_vault_client = KeyVaultClient(credentials)
# get the secret
sys.stdout.write("before secret\n")
secret = key_vault_client.get_secret(
keyVaultURL, # Your KeyVault URL
"hellomessage", # Name of your secret. If you followed the README 'secret' should exists
"" # The version of the secret. Empty string for latest
)
sys.stdout.write("after secret\n")
message = secret.value
res = jsonify({"Message:": message})
except Exception as e:
template = "An exception of type {0} occurred. Arguments:\n{1!r}"
message = template.format(type(e).__name__, e.args)
res = jsonify({"Message:": message})
res.status_code = 500
sys.stdout.flush()
return res
def main(req: func.HttpRequest) -> func.HttpResponse:
req_body = req.get_json()
source_container = req_body.get("source_container")
source_blob_name = req_body.get("source_blob")
dest_folder = req_body.get("dest_folder")
dest_container = req_body.get("dest_container")
credentials = MSIAuthentication(resource='https://vault.azure.net')
kvclient = KeyVaultClient(credentials)
key = kvclient.get_secret("https://zhenzh-databricks-kv.vault.azure.net/",
"zhenzh-python-func-account-key", "").value
converter = Excel2Csv(
BlockBlobService(account_name='zhenzhadfblobsource', account_key=key))
if source_blob_name is None:
converter.convert_and_upload(source_container, dest_container, "raw",
"curated")
else:
converter.convert_and_upload_blob("apachelog-analysis",
"apachelog-analysis",
source_blob_name, dest_folder)
return json.dumps({"result": "Conversion Finished!"})
def get_keyvault_client(self):
try:
self.log("Get KeyVaultClient from MSI")
credentials = MSIAuthentication(resource='https://vault.azure.net')
return KeyVaultClient(credentials)
except Exception:
self.log("Get KeyVaultClient from service principal")
# Create KeyVault Client using KeyVault auth class and auth_callback
def auth_callback(server, resource, scope):
if self.credentials['client_id'] is None or self.credentials[
'secret'] is None:
self.fail(
'Please specify client_id, secret and tenant to access azure Key Vault.'
)
tenant = self.credentials.get('tenant')
if not self.credentials['tenant']:
tenant = "common"
authcredential = ServicePrincipalCredentials(
client_id=self.credentials['client_id'],
secret=self.credentials['secret'],
tenant=tenant,
cloud_environment=self._cloud_environment,
resource="https://vault.azure.net")
token = authcredential.token
return token['token_type'], token['access_token']
return KeyVaultClient(KeyVaultAuthentication(auth_callback))
def main(msg: func.QueueMessage) -> None:
logging.info('Python queue trigger function processed a queue item: %s',
msg.get_body().decode('utf-8'))
# get bearer token and authenticate to ADLSgen2 using Managed Identity of Azure Function
credentials = MSIAuthentication(resource='https://storage.azure.com/')
blob_service = BlockBlobService("testedlstorgen",
token_credential=credentials)
# get timestamp
now = datetime.now()
nowstr = datetime.strftime(datetime.now(), "%Y%m%dT%H%M%S%Z")
key = round((now - datetime(2019, 1, 1, 0, 0, 0)).total_seconds())
logging.info("key: " + str(key))
# Add record to csv file. Notice that AppendBlob is not yet supported on ADLSgen2, see https://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-known-issues
records = blob_service.get_blob_to_text(
"raw",
"testprivcmddataflow/WideWorldImporters-Sales/address/SalesLTAddress.txt"
).content
records += "\n" + str(
key
) + ",8713 Yosemite Ct.,,Bothell,Washington,United States,98011,268af621-76d7-4c78-9441-144fd139821a,2006-07-01 00:00:00.0000000"
blob_service.create_blob_from_text(
"raw",
"testprivcmddataflow/WideWorldImporters-Sales/address/SalesLTAddress.txt",
records)
# Create event such that ADFv2 is triggered
blob_service = BlockBlobService("testedlstorgen",
token_credential=credentials)
blob_service.create_blob_from_text("adftrigger",
"adftrigger" + nowstr + ".txt", "")
def __init__(self, provider_config, cluster_name):
NodeProvider.__init__(self, provider_config, cluster_name)
kwargs = {}
if "subscription_id" in provider_config:
kwargs["subscription_id"] = provider_config["subscription_id"]
try:
self.compute_client = get_client_from_cli_profile(
client_class=ComputeManagementClient, **kwargs)
self.network_client = get_client_from_cli_profile(
client_class=NetworkManagementClient, **kwargs)
self.resource_client = get_client_from_cli_profile(
client_class=ResourceManagementClient, **kwargs)
except CLIError as e:
if str(e) != "Please run 'az login' to setup account.":
raise
else:
logger.info("CLI profile authentication failed. Trying MSI")
credentials = MSIAuthentication()
self.compute_client = ComputeManagementClient(
credentials=credentials, **kwargs)
self.network_client = NetworkManagementClient(
credentials=credentials, **kwargs)
self.resource_client = ResourceManagementClient(
credentials=credentials, **kwargs)
self.lock = RLock()
# cache node objects
self.cached_nodes = {}
def get_secret_from_keyvault(resource_packet: str) -> str:
"""
Passes the information necessary to access the secret value as URI:ID:Version
Args:
resource_packet: Informnation packet with contents used for retrieving secret for KeyVault.
Returns: extracted secret value for the specified version of the secret
"""
# This method will raise an exception on failure. It has not been hardened in any way.
logger.info('INFO: Attempting to retrieve secret from KeyVault')
# Create MSI Authentication
credentials = MSIAuthentication()
# Obtain hooks into the KeyVault - establish a client
client = KeyVaultClient(credentials)
resource_uri, secret_id, secret_version = resource_packet.split("|")
# Retrieve the secret
secret_bundle = client.get_secret(resource_uri, secret_id, secret_version)
logger.info('INFO: Secret successfully retrieved from KeyVault')
# The secret is stored in a value field. Retrieve this and return to the caller
return secret_bundle.value
def storage_client(self):
"""Create or return BlockBlobService client."""
if not self._block_blob_service:
account_name = self.get_azure_storage_account_name()
if os.environ.has_key('STORAGE_KEY'):
# We got the storage key through an environment variable
# (mostly for testing purposes)
self._block_blob_service = BlockBlobService(
account_name=account_name,
account_key=os.environ['STORAGE_KEY'])
else:
#
# Use the Azure Managed Service Identity ('MSI') to fetch an
# Azure AD token to talk to Azure Storage (PREVIEW!!!)
#
token_credential = MSIAuthentication(
resource='https://{account_name}.blob.core.windows.net'.format(
account_name=account_name))
self._block_blob_service = BlockBlobService(
account_name=account_name,
token_credential=token_credential)
logging.debug('Changing block sizes')
self._block_blob_service.MAX_SINGLE_PUT_SIZE = 256 * 1024 *1024
self._block_blob_service.MAX_BLOCK_SIZE = 100 * 1024 * 1024
self._block_blob_service.MIN_LARGE_BLOCK_UPLOAD_THRESHOLD = 100 * 1024 * 1024 + 1
return self._block_blob_service
def get_secret_value(secret):
managed_identity_client_id = os.environ.get("MANAGED_IDENTITY_CLIENT_ID")
key_vault_url = os.environ.get("KEY_VAULT_URL")
credentials = MSIAuthentication(client_id = managed_identity_client_id)
key_vault_client = KeyVaultClient(credentials)
secret_value = key_vault_client.get_secret(key_vault_url,secret,"")
return secret_value.value
def _get_credentials(self):
"""
Try to get a token via MSI, or fallback to service principal auth
"""
credentials = None
if os.getenv("MSI_ENDPOINT") and os.getenv("MSI_SECRET"):
credentials = MSIAuthentication(
resource=self.AUTH_RESOURCE
)
else:
try:
credentials = ServicePrincipalCredentials(
client_id=self.client_id,
secret=self.secret,
tenant=self.tenant,
resource=self.AUTH_RESOURCE
)
except AuthenticationError:
message = f'Error authenticating with client_id ' \
f'{self.client_id[0]}***{self.client_id[-1]}, ' \
f'secret {self.secret[0]}***{self.secret[-1]}, ' \
f'tenant {self.tenant[0]}***{self.tenant[-1]}.'
raise KeyVaultAuthenticationError(message)
return credentials
def find_subscriptions_in_vm_with_msi(self, identity_id=None):
import jwt
from requests import HTTPError
from msrestazure.azure_active_directory import MSIAuthentication
from msrestazure.tools import is_valid_resource_id
resource = self.cli_ctx.cloud.endpoints.active_directory_resource_id
msi_creds = MSIAuthentication()
token_entry = None
if identity_id:
if is_valid_resource_id(identity_id):
msi_creds = MSIAuthentication(resource=resource, msi_res_id=identity_id)
identity_type = MsiAccountTypes.user_assigned_resource_id
else:
msi_creds = MSIAuthentication(resource=resource, client_id=identity_id)
try:
msi_creds.set_token()
token_entry = msi_creds.token
identity_type = MsiAccountTypes.user_assigned_client_id
except HTTPError as ex:
if ex.response.reason == 'Bad Request' and ex.response.status == 400:
identity_type = MsiAccountTypes.user_assigned_object_id
msi_creds = MSIAuthentication(resource=resource, object_id=identity_id)
else:
raise
else:
identity_type = MsiAccountTypes.system_assigned
msi_creds = MSIAuthentication(resource=resource)
if not token_entry:
msi_creds.set_token()
token_entry = msi_creds.token
token = token_entry['access_token']
logger.info('MSI: token was retrieved. Now trying to initialize local accounts...')
decode = jwt.decode(token, verify=False, algorithms=['RS256'])
tenant = decode['tid']
subscription_finder = SubscriptionFinder(self.cli_ctx, self.auth_ctx_factory, None)
subscriptions = subscription_finder.find_from_raw_token(tenant, token)
if not subscriptions:
raise CLIError('No access was configured for the VM, hence no subscriptions were found')
base_name = ('{}-{}'.format(identity_type, identity_id) if identity_id else identity_type)
user = _USER_ASSIGNED_IDENTITY if identity_id else _SYSTEM_ASSIGNED_IDENTITY
consolidated = self._normalize_properties(user, subscriptions, is_service_principal=True)
for s in consolidated:
s[_SUBSCRIPTION_NAME] = base_name
# key-off subscription name to allow accounts with same id(but under different identities)
self._set_subscriptions(consolidated, secondary_key_name=_SUBSCRIPTION_NAME)
return deepcopy(consolidated)
def _get_token_from_cloud_shell(self, resource): # pylint: disable=no-self-use
from msrestazure.azure_active_directory import MSIAuthentication
auth = MSIAuthentication(resource=resource)
auth.set_token()
token_entry = auth.token
return (token_entry['token_type'], token_entry['access_token'], token_entry)
class Session(object):
def __init__(self, subscription_id=None, authorization_file=None,
resource=constants.RESOURCE_ACTIVE_DIRECTORY):
"""
:param subscription_id: If provided overrides environment variables.
:param authorization_file: Path to file populated from 'get_functions_auth_string'
:param resource: Resource endpoint for OAuth token.
"""
self.log = logging.getLogger('custodian.azure.session')
self._provider_cache = {}
self.subscription_id_override = subscription_id
self.credentials = None
self.subscription_id = None
self.tenant_id = None
self.resource_namespace = resource
self._is_token_auth = False
self._is_cli_auth = False
self.authorization_file = authorization_file
def _initialize_session(self):
"""
Creates a session using available authentication type.
Auth priority:
1. Token Auth
2. Tenant Auth
3. Azure CLI Auth
"""
# Only run once
if self.credentials is not None:
return
tenant_auth_variables = [
constants.ENV_TENANT_ID, constants.ENV_SUB_ID,
constants.ENV_CLIENT_ID, constants.ENV_CLIENT_SECRET
]
token_auth_variables = [
constants.ENV_ACCESS_TOKEN, constants.ENV_SUB_ID
]
msi_auth_variables = [
constants.ENV_USE_MSI, constants.ENV_SUB_ID
]
if self.authorization_file:
self.credentials, self.subscription_id = self.load_auth_file(self.authorization_file)
self.log.info("Creating session with authorization file")
elif all(k in os.environ for k in token_auth_variables):
# Token authentication
self.credentials = BasicTokenAuthentication(
token={
'access_token': os.environ[constants.ENV_ACCESS_TOKEN]
})
self.subscription_id = os.environ[constants.ENV_SUB_ID]
self.log.info("Creating session with Token Authentication")
self._is_token_auth = True
elif all(k in os.environ for k in tenant_auth_variables):
# Tenant (service principal) authentication
self.credentials = ServicePrincipalCredentials(
client_id=os.environ[constants.ENV_CLIENT_ID],
secret=os.environ[constants.ENV_CLIENT_SECRET],
tenant=os.environ[constants.ENV_TENANT_ID],
resource=self.resource_namespace)
self.subscription_id = os.environ[constants.ENV_SUB_ID]
self.tenant_id = os.environ[constants.ENV_TENANT_ID]
self.log.info("Creating session with Service Principal Authentication")
elif all(k in os.environ for k in msi_auth_variables):
# MSI authentication
if constants.ENV_CLIENT_ID in os.environ:
self.credentials = MSIAuthentication(
client_id=os.environ[constants.ENV_CLIENT_ID],
resource=self.resource_namespace)
else:
self.credentials = MSIAuthentication(
resource=self.resource_namespace)
self.subscription_id = os.environ[constants.ENV_SUB_ID]
self.log.info("Creating session with MSI Authentication")
else:
# Azure CLI authentication
self._is_cli_auth = True
(self.credentials,
self.subscription_id,
self.tenant_id) = Profile().get_login_credentials(
resource=self.resource_namespace)
self.log.info("Creating session with Azure CLI Authentication")
# Let provided id parameter override everything else
if self.subscription_id_override is not None:
self.subscription_id = self.subscription_id_override
self.log.info("Session using Subscription ID: %s" % self.subscription_id)
if self.credentials is None:
self.log.error('Unable to locate credentials for Azure session.')
def get_session_for_resource(self, resource):
return Session(
subscription_id=self.subscription_id_override,
authorization_file=self.authorization_file,
resource=resource)
def client(self, client):
self._initialize_session()
service_name, client_name = client.rsplit('.', 1)
svc_module = importlib.import_module(service_name)
klass = getattr(svc_module, client_name)
client = klass(self.credentials, self.subscription_id)
# Override send() method to log request limits & custom retries
service_client = client._client
service_client.orig_send = service_client.send
service_client.send = types.MethodType(custodian_azure_send_override, service_client)
# Don't respect retry_after_header to implement custom retries
service_client.config.retry_policy.policy.respect_retry_after_header = False
return client
def get_credentials(self):
self._initialize_session()
return self.credentials
def get_subscription_id(self):
self._initialize_session()
return self.subscription_id
def get_function_target_subscription_name(self):
self._initialize_session()
if constants.ENV_FUNCTION_MANAGED_GROUP_NAME in os.environ:
return os.environ[constants.ENV_FUNCTION_MANAGED_GROUP_NAME]
return os.environ.get(constants.ENV_FUNCTION_SUB_ID, self.subscription_id)
def get_function_target_subscription_ids(self):
self._initialize_session()
if constants.ENV_FUNCTION_MANAGED_GROUP_NAME in os.environ:
return ManagedGroupHelper.get_subscriptions_list(
os.environ[constants.ENV_FUNCTION_MANAGED_GROUP_NAME], self.get_credentials())
return [os.environ.get(constants.ENV_FUNCTION_SUB_ID, self.subscription_id)]
def resource_api_version(self, resource_id):
""" latest non-preview api version for resource """
namespace = ResourceIdParser.get_namespace(resource_id)
resource_type = ResourceIdParser.get_resource_type(resource_id)
cache_id = namespace + resource_type
if cache_id in self._provider_cache:
return self._provider_cache[cache_id]
resource_client = self.client('azure.mgmt.resource.ResourceManagementClient')
provider = resource_client.providers.get(namespace)
rt = next((t for t in provider.resource_types
if StringUtils.equal(t.resource_type, resource_type)), None)
if rt and rt.api_versions:
versions = [v for v in rt.api_versions if 'preview' not in v.lower()]
api_version = versions[0] if versions else rt.api_versions[0]
self._provider_cache[cache_id] = api_version
return api_version
def get_tenant_id(self):
self._initialize_session()
if self._is_token_auth:
decoded = jwt.decode(self.credentials['token']['access_token'], verify=False)
return decoded['tid']
return self.tenant_id
def get_bearer_token(self):
self._initialize_session()
if self._is_cli_auth:
return self.credentials._token_retriever()[1]
return self.credentials.token['access_token']
def load_auth_file(self, path):
with open(path) as json_file:
data = json.load(json_file)
self.tenant_id = data['credentials']['tenant']
return (ServicePrincipalCredentials(
client_id=data['credentials']['client_id'],
secret=data['credentials']['secret'],
tenant=self.tenant_id,
resource=self.resource_namespace
), data.get('subscription', None))
def get_functions_auth_string(self, target_subscription_id):
"""
Build auth json string for deploying
Azure Functions. Look for dedicated
Functions environment variables or
fall back to normal Service Principal
variables.
"""
self._initialize_session()
function_auth_variables = [
constants.ENV_FUNCTION_TENANT_ID,
constants.ENV_FUNCTION_CLIENT_ID,
constants.ENV_FUNCTION_CLIENT_SECRET
]
# Use dedicated function env vars if available
if all(k in os.environ for k in function_auth_variables):
auth = {
'credentials':
{
'client_id': os.environ[constants.ENV_FUNCTION_CLIENT_ID],
'secret': os.environ[constants.ENV_FUNCTION_CLIENT_SECRET],
'tenant': os.environ[constants.ENV_FUNCTION_TENANT_ID]
},
'subscription': target_subscription_id
}
elif type(self.credentials) is ServicePrincipalCredentials:
auth = {
'credentials':
{
'client_id': os.environ[constants.ENV_CLIENT_ID],
'secret': os.environ[constants.ENV_CLIENT_SECRET],
'tenant': os.environ[constants.ENV_TENANT_ID]
},
'subscription': target_subscription_id
}
else:
raise NotImplementedError(
"Service Principal credentials are the only "
"supported auth mechanism for deploying functions.")
return json.dumps(auth, indent=2)
def _initialize_session(self):
"""
Creates a session using available authentication type.
Auth priority:
1. Token Auth
2. Tenant Auth
3. Azure CLI Auth
"""
# Only run once
if self.credentials is not None:
return
tenant_auth_variables = [
constants.ENV_TENANT_ID, constants.ENV_SUB_ID,
constants.ENV_CLIENT_ID, constants.ENV_CLIENT_SECRET
]
token_auth_variables = [
constants.ENV_ACCESS_TOKEN, constants.ENV_SUB_ID
]
msi_auth_variables = [
constants.ENV_USE_MSI, constants.ENV_SUB_ID
]
if self.authorization_file:
self.credentials, self.subscription_id = self.load_auth_file(self.authorization_file)
self.log.info("Creating session with authorization file")
elif all(k in os.environ for k in token_auth_variables):
# Token authentication
self.credentials = BasicTokenAuthentication(
token={
'access_token': os.environ[constants.ENV_ACCESS_TOKEN]
})
self.subscription_id = os.environ[constants.ENV_SUB_ID]
self.log.info("Creating session with Token Authentication")
self._is_token_auth = True
elif all(k in os.environ for k in tenant_auth_variables):
# Tenant (service principal) authentication
self.credentials = ServicePrincipalCredentials(
client_id=os.environ[constants.ENV_CLIENT_ID],
secret=os.environ[constants.ENV_CLIENT_SECRET],
tenant=os.environ[constants.ENV_TENANT_ID],
resource=self.resource_namespace)
self.subscription_id = os.environ[constants.ENV_SUB_ID]
self.tenant_id = os.environ[constants.ENV_TENANT_ID]
self.log.info("Creating session with Service Principal Authentication")
elif all(k in os.environ for k in msi_auth_variables):
# MSI authentication
if constants.ENV_CLIENT_ID in os.environ:
self.credentials = MSIAuthentication(
client_id=os.environ[constants.ENV_CLIENT_ID],
resource=self.resource_namespace)
else:
self.credentials = MSIAuthentication(
resource=self.resource_namespace)
self.subscription_id = os.environ[constants.ENV_SUB_ID]
self.log.info("Creating session with MSI Authentication")
else:
# Azure CLI authentication
self._is_cli_auth = True
(self.credentials,
self.subscription_id,
self.tenant_id) = Profile().get_login_credentials(
resource=self.resource_namespace)
self.log.info("Creating session with Azure CLI Authentication")
# Let provided id parameter override everything else
if self.subscription_id_override is not None:
self.subscription_id = self.subscription_id_override
self.log.info("Session using Subscription ID: %s" % self.subscription_id)
if self.credentials is None:
self.log.error('Unable to locate credentials for Azure session.')