def authenticate_cli(self) -> Credentials:
"""
Implements authentication for the Azure provider
"""
try:
# Set logging level to error for libraries as otherwise generates a lot of warnings
logging.getLogger('adal-python').setLevel(logging.ERROR)
logging.getLogger('msrest').setLevel(logging.ERROR)
logging.getLogger('msrestazure.azure_active_directory').setLevel(logging.ERROR)
logging.getLogger('urllib3').setLevel(logging.ERROR)
arm_credentials, subscription_id, tenant_id = get_azure_cli_credentials(with_tenant=True)
aad_graph_credentials, placeholder_1, placeholder_2 = get_azure_cli_credentials(
with_tenant=True, resource='https://graph.windows.net',
)
profile = get_cli_profile()
return Credentials(
arm_credentials, aad_graph_credentials, tenant_id=tenant_id,
current_user=profile.get_current_account_user(), subscription_id=subscription_id,
)
except HttpResponseError as e:
if ', AdalError: Unsupported wstrust endpoint version. ' \
'Current supported version is wstrust2005 or wstrust13.' in e.args:
logger.error(
f'You are likely authenticating with a Microsoft Account. \
This authentication mode only supports Azure Active Directory principal authentication.\
{e}',
)
raise e
def skip_without_credentials(func):
try:
get_azure_cli_credentials()
except FileNotFoundError:
return pytest.mark.skip(
reason="""
You must configure your Azure credentials to run this test.
$ az login
"""
)(func)
rg = dask.config.get("cloudprovider.azure.azurevm.resource_group", None)
vnet = dask.config.get("cloudprovider.azure.azurevm.vnet", None)
security_group = dask.config.get("cloudprovider.azure.azurevm.security_group", None)
location = dask.config.get("cloudprovider.azure.location", None)
if rg is None or vnet is None or security_group is None or location is None:
return pytest.mark.skip(
reason="""
You must configure your Azure resource group and vnet to run this test.
$ export DASK_CLOUDPROVIDER__AZURE__LOCATION="<LOCATION>"
$ export DASK_CLOUDPROVIDER__AZURE__AZUREVM__RESOURCE_GROUP="<RESOURCE GROUP>"
$ export DASK_CLOUDPROVIDER__AZURE__AZUREVM__VNET="<VNET>"
$ export DASK_CLOUDPROVIDER__AZURE__AZUREVM__SECURITY_GROUP="<SECUROTY GROUP>"
"""
)(func)
return func
def main(n: int = None) -> None:
"""
Show all Azure Subscriptions in current profile using the `az` command-line utility.
Ask user input for switching to another subscription.
"""
try:
subscriptions = json.loads(
subprocess.getoutput('az account list --output json'))
current_nr = _print_options(subscriptions)
if not n:
n = click.prompt('Switch', type=int, default=current_nr)
if n not in range(1, len(subscriptions) + 1):
raise ValueError("Value not in range! Not changing subscription.")
_select_subscription(n, subscriptions)
_, subscription_id = get_azure_cli_credentials()
active = next(
filter(lambda x: x['id'] == subscription_id, subscriptions))
click.echo("Active: " + click.style(
active['id'] + ": " + active['name'], fg='green', bold=True))
except subprocess.CalledProcessError:
# Issue is already printed to stderr.
pass
except click.exceptions.Abort:
# No need to raise exception when CTRL-C out of the cli
pass
except ValueError as e:
print(e)
def auth_azuread(self,
workspace_resource_id=None,
token_callback=None,
subscription_id=None,
resource_group=None,
workspace_name=None):
if token_callback is None:
from azure.common.credentials import get_azure_cli_credentials
credentials, subscription_id = get_azure_cli_credentials()
def token_callback(resource):
return credentials.get_token(resource).token
if workspace_resource_id is None:
if resource_group is None or workspace_name is None:
raise ValueError(
"Either workspace_resource_id or both of "
"resource_group and workspace_name must be provided")
if subscription_id is None:
raise ValueError("subscription_id must be provided")
workspace_resource_id = (
'/subscriptions/%s/resourceGroups/%s/providers/'
'Microsoft.Databricks/workspaces/%s' %
(subscription_id, resource_group, workspace_name))
adb_token = token_callback('2ff814a6-3304-4ab8-85cb-cd0e6f879c1d')
arm_token = token_callback('https://management.core.windows.net/')
self.dbricks_auth = {
'Authorization': 'Bearer %s' % adb_token,
'X-Databricks-Azure-SP-Management-Token': arm_token,
'X-Databricks-Azure-Workspace-Resource-Id': workspace_resource_id,
}
def get_azure_client():
client_id = environ.get("CLIENT_ID", "")
client_secret = environ.get("CLIENT_SECRET", "")
tenant_id = environ.get("TENANT_ID", "")
if client_id and client_secret and tenant_id:
logger.info(
"Found credentials in environment variables, using service principle for auth"
)
credentials = ServicePrincipalCredentials(
tenant=tenant_id,
client_id=client_id,
secret=client_secret,
resource="https://graph.windows.net",
)
return GraphRbacManagementClient(credentials, tenant_id)
try:
credentials, _, tenant_id = get_azure_cli_credentials(
resource="https://graph.windows.net", with_tenant=True)
except FileNotFoundError:
logger.fatal(
"Failed to find Azure credentials. Please configure environment variables or an az login JSON."
)
raise
return GraphRbacManagementClient(credentials, tenant_id)
def _configure_resource_group(config):
# TODO: look at availability sets
# https://docs.microsoft.com/en-us/azure/virtual-machines/windows/tutorial-availability-sets
resource_client = _get_client(ResourceManagementClient, config)
_, cli_subscription_id = get_azure_cli_credentials(
resource=ResourceManagementClient)
subscription_id = config["provider"].get("subscription_id",
cli_subscription_id)
logger.info("Using subscription id: %s", subscription_id)
config["provider"]["subscription_id"] = subscription_id
assert "resource_group" in config["provider"], (
"Provider config must include resource_group field")
resource_group = config["provider"]["resource_group"]
assert "location" in config["provider"], (
"Provider config must include location field")
params = {"location": config["provider"]["location"]}
if "tags" in config["provider"]:
params["tags"] = config["provider"]["tags"]
logger.info("Creating/Updating Resource Group: %s", resource_group)
resource_client.resource_groups.create_or_update(
resource_group_name=resource_group, parameters=params)
# load the template file
current_path = Path(__file__).parent
template_path = current_path.joinpath("azure-config-template.json")
with open(template_path, "r") as template_fp:
template = json.load(template_fp)
# choose a random subnet, skipping most common value of 0
random.seed(resource_group)
subnet_mask = "10.{}.0.0/16".format(random.randint(1, 254))
parameters = {
"properties": {
"mode": DeploymentMode.incremental,
"template": template,
"parameters": {
"subnet": {
"value": subnet_mask
}
}
}
}
if hasattr(resource_client.deployments, "create_or_update"):
create_or_update = resource_client.deployments.create_or_update
else:
create_or_update = resource_client.deployments.begin_create_or_update
create_or_update(resource_group_name=resource_group,
deployment_name="ray-config",
parameters=parameters).wait()
return config
def _get_azure_cli_credentials(self):
credentials, subscription_id = get_azure_cli_credentials()
cloud_environment = get_cli_active_cloud()
cli_credentials = {
'credentials': credentials,
'subscription_id': subscription_id,
'cloud_environment': cloud_environment
}
return cli_credentials
def _get_azure_cli_credentials(self):
credentials, subscription_id = get_azure_cli_credentials()
cloud_environment = get_cli_active_cloud()
cli_credentials = {
'credentials': credentials,
'subscription_id': subscription_id,
'cloud_environment': cloud_environment
}
return cli_credentials
def get_subscription(self):
try:
return get_azure_cli_credentials()[1]
except Exception as error:
logging.warning("Couldn't get subscription id from Azure CLI.")
try:
client, secret, tenant, sub = self.get_config(self.config_path)
return sub
except Exception as error:
logging.error("Failed to get a subscription id.")
def _get_azure_cli_profile(self):
if not HAS_AZURE_CLI_CORE:
self.fail("Do you have azure-cli-core installed? Try `pip install 'azure-cli-core' --upgrade`")
try:
credentials, subscription_id = get_azure_cli_credentials()
self._cloud_environment = get_cli_active_cloud()
return {
'credentials': credentials,
'subscription_id': subscription_id
}
except CLIError as err:
self.fail("AzureCLI profile cannot be loaded - {0}".format(err))
def _get_azure_cli_profile(self):
if not HAS_AZURE_CLI_CORE:
self.fail("Do you have azure-cli-core installed? Try `pip install 'azure-cli-core' --upgrade`")
try:
credentials, subscription_id = get_azure_cli_credentials()
self._cloud_environment = get_cli_active_cloud()
return {
'credentials': credentials,
'subscription_id': subscription_id
}
except CLIError as err:
self.fail("AzureCLI profile cannot be loaded - {0}".format(err))
def _get_azure_cli_profile(self):
if not HAS_AZURE_CLI_CORE:
self.fail("Do you have azure-cli-core installed? Try `pip install 'azure-cli-core' --upgrade`")
try:
credentials, subscription_id = get_azure_cli_credentials()
base_url = get_cli_active_cloud().endpoints.resource_manager
return {
'credentials': credentials,
'subscription_id': subscription_id,
'base_url': base_url
}
except CLIError as err:
self.fail("AzureCLI profile cannot be loaded - {0}".format(err))
def main(req: func.HttpRequest) -> func.HttpResponse:
if "MSI_ENDPOINT" in os.environ:
credentials = MSIAuthentication()
else:
credentials, *_ = get_azure_cli_credentials()
subscription_client = SubscriptionClient(credentials)
subscription = next(subscription_client.subscriptions.list())
subscription_id = subscription.subscription_id
client = ResourceManagementClient(credentials, subscription_id)
resource_groups = [g.name for g in client.resource_groups.list()]
return func.HttpResponse(json.dumps(resource_groups),
mimetype="application/json")
async def main(req: func.HttpRequest) -> func.HttpResponse:
"""
The main entry point to the function.
"""
if "MSI_ENDPOINT" in os.environ:
credentials = MSIAuthentication()
else:
credentials, *_ = get_azure_cli_credentials()
subscription_id = os.environ.get('AZURE_SUBSCRIPTION_ID',
'11111111-1111-1111-1111-111111111111')
list_of_rgs = await list_rgs(credentials, subscription_id)
return func.HttpResponse(list_of_rgs, mimetype="application/json")
def main(n: int = None, v: bool = False) -> None:
"""
Show all Azure Subscriptions in current profile using the `az` command-line utility.
Ask user input for switching to another subscription.
"""
try:
# Using --query to map subset of fields and sort by name (ascending)
list_cmd = 'az account list --all --output json ' \
'--query "sort_by([].{name:name, isDefault:isDefault, id:id, state:state}, &name)"'
if v:
click.echo(f'Issuing AZ CLI command: {list_cmd}')
subscriptions = json.loads(subprocess.getoutput(list_cmd))
current_nr = _print_options(subscriptions)
if not n:
n = click.prompt('Switch', type=int, default=current_nr)
if n not in range(1, len(subscriptions) + 1):
raise ValueError("Value not in range! Not changing subscription.")
if n == current_nr:
click.echo(
"Selection is same as current. Not changing subscription.")
else:
_select_subscription(n, v, subscriptions)
_, subscription_id = get_azure_cli_credentials()
active = next(
filter(lambda x: x['id'] == subscription_id, subscriptions))
click.echo("Active: " + click.style(
active['id'] + ": " + active['name'], fg='green', bold=True))
if active['state'].lower() == 'disabled':
click.echo(
click.style(
"Subscription state is Disabled, requires: az login!",
fg='yellow'))
except subprocess.CalledProcessError:
# Issue is already printed to stderr.
pass
except click.exceptions.Abort:
# No need to raise exception when CTRL-C out of the cli
pass
except ValueError as e:
print(e)
def resource_apply_tags(id, subscription, provider, tags):
# NOTE: I've observed that the app serice needs to be restarted once
# you add the web app managed identity to the subscription. Otherwise it_Owner ends up blank.
if "LOCAL_DEBUG" in os.environ and os.environ["LOCAL_DEBUG"] == '1':
# When running locally for debug/development
sys.stderr.write("****** USING CLI AUTHENTICATION ***********")
creds, _ = credentials.get_azure_cli_credentials(resource=None,
with_tenant=False)
else:
# When running on Azure, and managed identity is used to grant tag priviledge.
sys.stderr.write("****** USING MSI AUTHENTICATION ***********")
creds = MSIAuthentication()
resource_client = ResourceManagementClient(creds, subscription)
try:
r = resource_client.resources.get_by_id(id,
api_version_lookup[provider])
except:
print("Lookup Failed: Skipped...", id)
return False
# The tag operation is not additive. Preserve tags already there and add new ones.
try:
current_tags = r.tags
if 'it_Owner' in current_tags and len(current_tags['it_Owner']) > 0:
sys.stderr.write(
"**************it_Owner exists. Do Nothing. ***************")
return False
except:
assert False, "Invalid tags provided."
merged_tags = {}
if current_tags:
merged_tags.update(current_tags)
merged_tags.update(tags)
r.tags = merged_tags
resource_client.resources.create_or_update_by_id(
id, api_version_lookup[provider], r)
return True
def resizeDisksToStandardSSD(diskIDs):
#credentials = get_credentials()
cred = get_azure_cli_credentials()
newSKU = DiskSku(name="StandardSSD_LRS")
for sub in subscriptionIds:
compute_client = ComputeManagementClient(cred[0], sub)
logging.warning("Changing level in subscription {}".format(sub))
logging.warning("Resource group \t diskname")
for disk in diskIDs:
if (disk['id'].split('/')[2] == sub):
rgName = disk['id'].split('/')[4]
diskName = disk['id'].split('/')[8]
logging.warning("{}\t{}".format(rgName, diskName))
disk = compute_client.disks.get(rgName, diskName)
disk.sku = newSKU
async_disk_update = compute_client.disks.begin_create_or_update(
rgName, diskName, disk)
async_disk_update.wait()
return True
def authenticate_from_cli(self, resource):
"""
Create the azure resource manager client from the user logged into azure cli
If a user is not logged in the run the az login command so they can authenticate
"""
print(f"Attempting to get cli credentials for resource {resource}")
from azure.common.credentials import get_azure_cli_credentials
try:
credential, subscription_id, self.tenant_id = get_azure_cli_credentials(
resource=resource, with_tenant=True)
except:
print("We were not able to find a cli profile attempting to login")
from subprocess import run
import platform
shell = platform.system() == "Windows"
run(["az", "login", "--use-device-code"], shell=shell)
return self.authenticate_from_cli(resource)
return credential
def main(mytimer: func.TimerRequest, template: func.InputStream) -> None:
"""main is run from the timer function
Args:
mytimer (func.TimerRequest): [description]
"""
logging.info('RandomDeployer started at %s', formatted_time())
subscription_id = None
if "MSI_ENDPOINT" in os.environ:
credentials = MSIAuthentication()
else:
credentials, subscription_id = get_azure_cli_credentials()
subscription_id = os.environ.get('RANDOM_DEPLOY_SUBSCRIPTION_ID', subscription_id)
logging.info('Using subscription %s', subscription_id)
deployer = RandomDeployer(subscription_id, credentials, json.load(template))
deployer.deploy(int(os.environ.get('RANDOM_DEPLOY_LIFETIME', 60*60*24)))
logging.info('RandomDeployer done at %s', formatted_time())
try:
tenant = os.environ["AZ_AD_TENANT"]
client_id = os.environ["AZ_AD_ID"]
secret = os.environ["AZ_AD_PASS"]
except KeyError as ke:
print("set the following env variables:")
print("AZ_AD_ID")
print("AZ_AD_TENANT")
print("AZ_AD_PASS")
sys.exit(1)
credentials = ServicePrincipalCredentials(client_id=client_id,
secret=secret,
tenant=tenant)
subscription = get_azure_cli_credentials()[1]
new_resource_groups = ["wewewe", "ererere", "fgfgfgfg"]
res_client = azure.mgmt.resource.ResourceManagementClient(
credentials, subscription)
def print_resource_group_info(res_client):
for res_group in res_client.resource_groups.list():
for thing in ["name", "id", "location", "tags"]:
print("{:.<10}{}{}{}".format(thing, "{0.", thing,
"}").format(res_group))
print("." * 20)
print("===============Before")
def main():
"""
Builds the Azure ML pipeline for data engineering and model training.
"""
databricks_workspace_name = os.environ['DATABRICKS_WORKSPACE_NAME']
training_data_account_name = os.environ['TRAINING_DATA_ACCOUNT_NAME']
build_id = os.getenv('BUILD_BUILDID', 0)
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=os.environ['AML_WORKSPACE_NAME'],
subscription_id=os.environ['SUBSCRIPTION_ID'],
resource_group=os.environ['RESOURCE_GROUP'],
)
print(aml_workspace)
# Generate Databricks credentials, see https://aka.ms/databricks-aad
dbricks_region = aml_workspace.location
dbricks_api = f"https://{dbricks_region}.azuredatabricks.net/api/2.0"
dbricks_client = databricks_client.create(dbricks_api)
dbricks_client.auth_azuread(resource_group=aml_workspace.resource_group,
workspace_name=databricks_workspace_name)
dbricks_client.ensure_available()
# Attach Databricks as Azure ML training compute
dbricks_compute_name = "databricks"
dbricks_compute = get_databricks_compute(
aml_workspace,
dbricks_compute_name,
)
if dbricks_compute is None:
pat_token = dbricks_client.post(
'token/create',
json={"comment": "Azure ML Token generated by Build " + build_id
})['token_value']
dbricks_compute = create_databricks_compute(
aml_workspace,
databricks_workspace_name,
dbricks_compute_name,
pat_token,
)
print("dbricks_compute:")
print(dbricks_compute)
# Create Databricks instance pool
pool_name = "azureml_training"
instance_pool_id = get_instance_pool(dbricks_client, pool_name)
if not instance_pool_id:
dbricks_client.post('instance-pools/create',
json={
"instance_pool_name":
pool_name,
"node_type_id":
"Standard_D3_v2",
"idle_instance_autotermination_minutes":
10,
"preloaded_spark_versions":
[DATABRICKS_RUNTIME_VERSION],
})
instance_pool_id = get_instance_pool(dbricks_client, pool_name)
notebook_folder = f"/Shared/AzureMLDeployed"
workspace_datastore = Datastore(aml_workspace, "workspaceblobstore")
# Create a datastore for the training data container
credentials, subscription = get_azure_cli_credentials()
storage_client = StorageManagementClient(credentials, subscription)
training_storage_keys = storage_client.storage_accounts.list_keys(
aml_workspace.resource_group, training_data_account_name)
training_datastore = Datastore.register_azure_blob_container(
workspace=aml_workspace,
datastore_name="trainingdata",
container_name="trainingdata",
account_name=training_data_account_name,
account_key=training_storage_keys.keys[0].value,
)
# FEATURE ENGINEERING STEP (DATABRICKS)
# Create feature engineering pipeline step
training_data_input = DataReference(datastore=training_datastore,
path_on_datastore="/",
data_reference_name="training")
feature_eng_output = PipelineData("feature_engineered",
datastore=workspace_datastore)
notebook_path = upload_notebook(dbricks_client, notebook_folder,
"code/prepare", "feature_engineering")
training_dataprep_step = DatabricksStep(
name="FeatureEngineering",
inputs=[training_data_input],
outputs=[feature_eng_output],
spark_version=DATABRICKS_RUNTIME_VERSION,
instance_pool_id=instance_pool_id,
num_workers=3,
notebook_path=notebook_path,
run_name="FeatureEngineering",
compute_target=dbricks_compute,
allow_reuse=True,
)
# You can add Azure ML model training tasks using
# feature_eng_output as input.
# ...
# Create Azure ML Pipeline
steps = [training_dataprep_step]
ml_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
ml_pipeline.validate()
published_pipeline = ml_pipeline.publish(
name="Feature Engineering",
description="Feature engineering pipeline",
version=build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
# When running in Azure DevOps, set AMLPIPELINE_ID variable
# for AML Pipeline task in next job
print("Setting Azure DevOps variable")
print(f"##vso[task.setvariable variable=AMLPIPELINE_ID;isOutput=true]"
f"{published_pipeline.id}")
def main():
print('Azure Import Tags Util v1.4')
# Check that we have args
if (len(sys.argv) < 10):
print('YOU MUST SPECIFY THE CORRECT COMMAND LINE PARAMETERS:')
print('Usage:')
print(
'azureimporttags.py [vm_input_file] [input_file] [migrated_from] [migrate_project] [azure_region] [subscriptiom] [client_id] [secret] [tenant]'
)
print(
'NOTE: the migrated_from field should contain one of these values:'
)
print(' On Premise')
print(' AWS')
print(' Azure')
print(' Equinix')
print(' CGI')
print('')
sys.exit(1)
# Get command line args
if (sys.argv[0] == 'python'):
vm_input_file = sys.argv[2]
inputfile = sys.argv[3]
migrated_from = sys.argv[4]
migrate_project = sys.argv[5]
region = sys.argv[6]
sub_id = sys.argv[7]
client_id = sys.argv[8]
secret = sys.argv[9]
tenant = sys.argv[10]
else:
vm_input_file = sys.argv[1]
inputfile = sys.argv[2]
migrated_from = sys.argv[3]
migrate_project = sys.argv[4]
region = sys.argv[5]
sub_id = sys.argv[6]
client_id = sys.argv[7]
secret = sys.argv[8]
tenant = sys.argv[9]
print('VM List file: ', vm_input_file)
print('Inputfile: ', inputfile)
print('Region: ', region)
print('MigrateProject: ', migrate_project)
print('sub_id: ', sub_id)
print('client_id: ', client_id)
print('secret: ', secret)
print('tenant: ', tenant)
# Start logging
updatelog('------ Starting run ------')
updatelog('Region: ', region)
# If not a valid migrated_from field, abort with error
migrated_from_cases = {
'On Premise': 'On Premise',
'AWS': 'AWS',
'Azure': 'Azure',
'Equinix': 'Equinix',
'CGI': 'CGI',
'Native': 'Native'
}
migrated_from = migrated_from_cases.get(migrated_from, 'INVALID')
if (migrated_from == 'INVALID'):
print('')
print('ERROR: invalid migrated_from arg was passed')
updatelog('ERROR: invalid migrated_from arg was passed: ',
migrated_from)
print('')
exit(2)
print('migrated_from: ', migrated_from)
updatelog('migrated_from: ', migrated_from)
# Get creds
updatelog('Calling ServicePrincipalCredentials()')
subscription_id = sub_id
#credentials = ServicePrincipalCredentials(client_id = client_id, secret = secret, tenant = tenant)
# Testing this code
credentials = creds.get_azure_cli_credentials(resource=None,
with_tenant=False)[0]
sub_client = SubscriptionClient(credentials)
# Create object for compute-related interactions
updatelog('Calling ComputeManagementClient()')
compute_client = ComputeManagementClient(credentials, subscription_id)
# Read all tag data into taglist var
taglist = []
updatelog('Calling loadrecordsfromfile()')
loadrecordsfromfile(inputfile, taglist)
# Get VM input list file
target_vm_list = get_target_vm_list(vm_input_file)
# Get list of all Azure VMs and add to list
#az_vm_list = []
updatelog('Calling getallazvms()')
#getallazvms(compute_client, az_vm_list, target_vm_list)
az_vm_list = getallazvms(compute_client, target_vm_list)
print('VMs: ', az_vm_list)
# Tag'em
updatelog('Calling tageachvm()')
tageachvmfromlist(credentials, subscription_id, compute_client,
migrated_from, migrate_project, region, taglist,
az_vm_list)
updatelog('------ Exiting ------')
def __init__(
self,
location: str = None,
resource_group: str = None,
vnet: str = None,
security_group: str = None,
public_ingress: bool = None,
vm_size: str = None,
scheduler_vm_size: str = None,
vm_image: dict = {},
disk_size: int = None,
bootstrap: bool = None,
auto_shutdown: bool = None,
docker_image=None,
debug: bool = False,
marketplace_plan: dict = {},
**kwargs,
):
self.config = dask.config.get("cloudprovider.azure.azurevm", {})
self.scheduler_class = AzureVMScheduler
self.worker_class = AzureVMWorker
self.location = (location if location is not None else
dask.config.get("cloudprovider.azure.location"))
if self.location is None:
raise ConfigError("You must configure a location")
self.resource_group = (resource_group if resource_group is not None
else self.config.get("resource_group"))
if self.resource_group is None:
raise ConfigError("You must configure a resource_group")
self.public_ingress = (public_ingress if public_ingress is not None
else self.config.get("public_ingress"))
self.credentials, self.subscription_id = get_azure_cli_credentials()
self.compute_client = ComputeManagementClient(self.credentials,
self.subscription_id)
self.network_client = NetworkManagementClient(self.credentials,
self.subscription_id)
self.vnet = vnet if vnet is not None else self.config.get("vnet")
if self.vnet is None:
raise ConfigError("You must configure a vnet")
self.security_group = (security_group if security_group is not None
else self.config.get("security_group"))
if self.security_group is None:
raise ConfigError(
"You must configure a security group which allows traffic on 8786 and 8787"
)
self.vm_size = vm_size if vm_size is not None else self.config.get(
"vm_size")
self.disk_size = (disk_size if disk_size is not None else
self.config.get("disk_size"))
if self.disk_size > 1023:
raise ValueError(
"VM OS disk canot be larger than 1023. Please change the ``disk_size`` config option."
)
self.scheduler_vm_size = (scheduler_vm_size
if scheduler_vm_size is not None else
self.config.get("scheduler_vm_size"))
if self.scheduler_vm_size is None:
self.scheduler_vm_size = self.vm_size
self.gpu_instance = ("_NC" in self.vm_size.upper()
or "_ND" in self.vm_size.upper())
self.vm_image = self.config.get("vm_image")
for key in vm_image:
self.vm_image[key] = vm_image[key]
self.bootstrap = (bootstrap if bootstrap is not None else
self.config.get("bootstrap"))
self.auto_shutdown = (auto_shutdown if auto_shutdown is not None else
self.config.get("auto_shutdown"))
self.docker_image = docker_image or self.config.get("docker_image")
self.debug = debug
self.marketplace_plan = marketplace_plan or self.config.get(
"marketplace_plan")
if self.marketplace_plan:
# Check that self.marketplace_plan contains the right options with values
if not all(
self.marketplace_plan.get(item, "") != ""
for item in ["name", "publisher", "product"]):
raise ConfigError(
"""To create a virtual machine from Marketplace image or a custom image sourced
from a Marketplace image with a plan, all 3 fields 'name', 'publisher' and 'product' must be passed."""
)
self.options = {
"cluster": self,
"config": self.config,
"security_group": self.security_group,
"location": self.location,
"vm_image": self.vm_image,
"disk_size": self.disk_size,
"gpu_instance": self.gpu_instance,
"bootstrap": self.bootstrap,
"auto_shutdown": self.auto_shutdown,
"docker_image": self.docker_image,
"marketplace_plan": self.marketplace_plan,
}
self.scheduler_options = {
"vm_size": self.scheduler_vm_size,
"public_ingress": self.public_ingress,
**self.options,
}
self.worker_options = {"vm_size": self.vm_size, **self.options}
super().__init__(debug=debug, **kwargs)
#------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for # license information. #-------------------------------------------------------------------------- import os from azure.mgmt.peering import PeeringManagementClient from azure.mgmt.resource import ResourceManagementClient # from azure.common.credentials import ServicePrincipalCredentials from azure.common.credentials import get_azure_cli_credentials cred, sub, tenant = get_azure_cli_credentials(with_tenant=True) AZURE_LOCATION = 'eastus' RESOURCE_GROUP = "myResourceGroup" # credentials from environment #SUBSCRIPTION_ID = os.environ['AZURE_SUBSCRIPTION_ID'] SUBSCRIPTION_ID = 'xxx' #TENANT_ID = os.environ['AZURE_TENANT'] #CLIENT_ID = os.environ['AZURE_CLIENT_ID'] #CLIENT_SECRET = os.environ['AZURE_SECRET'] PEER_ASN_NAME = "myPeerAsn" PEERING_NAME = "myPeering" + "B" REGISTERED_ASN_NAME = "myRegisteredAsn" REGISTERED_PREFIX_NAME = "myRegisteredPrefix" PEERING_SERVICE_NAME = "myPeeringService" PREFIX_NAME = "myPrefix" # management client #credentials = ServicePrincipalCredentials( # client_id=CLIENT_ID,
def delete_spn(dbr_tmp_pat, spn_id):
# Delete SPN only works with PAT token
response = requests.delete(
f"{dbricks_api}/preview/scim/v2/ServicePrincipals/{spn_id}",
headers={
"Accept": "application/scim+json",
"Authorization": "Bearer " + dbr_tmp_pat,
})
print(response.status_code)
if __name__ == "__main__":
# 1. get admin credentials, subscription and tenant
credentials, subscription_id, tenant_id = get_azure_cli_credentials(
with_tenant=True)
# 2.1 Deploy new Databricks workspace
create_databricks_workspace()
# 2.2 Wait 5 minutes, since it takes some time before Workspace is initialized
time.sleep(300)
# 3. Get token to authenticate to DataBricks, user needs to be admin in Databricks
admin_adb_token = get_admin_token(credentials,
"2ff814a6-3304-4ab8-85cb-cd0e6f879c1d")
admin_az_token = get_admin_token(credentials,
"https://management.core.windows.net/")
dbricks_admin_auth = get_dbr_auth(admin_adb_token, admin_az_token)
# 4. Create tmp dbr pat token to authenticate create/delete SPN in SCIM interface (works only with PAT)
dbr_tmp_pat = create_tmp_dbrpat(dbricks_admin_auth)
# 3. Add spn to Databricks and provide rights to SPN to run manage clusters
spn_dbr_id = check_spn_exists(dbricks_admin_auth)
if spn_dbr_id == "":
def __init__(
self,
location: str = None,
resource_group: str = None,
vnet: str = None,
security_group: str = None,
public_ingress: bool = None,
vm_size: str = None,
scheduler_vm_size: str = None,
vm_image: dict = {},
disk_size: int = None,
bootstrap: bool = None,
auto_shutdown: bool = None,
docker_image=None,
debug: bool = False,
**kwargs,
):
self.config = dask.config.get("cloudprovider.azure.azurevm", {})
self.scheduler_class = AzureVMScheduler
self.worker_class = AzureVMWorker
self.location = (location if location is not None else
dask.config.get("cloudprovider.azure.location"))
if self.location is None:
raise ConfigError("You must configure a location")
self.resource_group = (resource_group if resource_group is not None
else self.config.get("resource_group"))
if self.resource_group is None:
raise ConfigError("You must configure a resource_group")
self.public_ingress = (public_ingress if public_ingress is not None
else self.config.get("public_ingress"))
self.credentials, self.subscription_id = get_azure_cli_credentials()
self.compute_client = ComputeManagementClient(self.credentials,
self.subscription_id)
self.network_client = NetworkManagementClient(self.credentials,
self.subscription_id)
self.vnet = vnet if vnet is not None else self.config.get("vnet")
if self.vnet is None:
raise ConfigError("You must configure a vnet")
self.security_group = (security_group if security_group is not None
else self.config.get("security_group"))
if self.security_group is None:
raise ConfigError(
"You must configure a security group which allows traffic on 8786 and 8787"
)
self.vm_size = vm_size if vm_size is not None else self.config.get(
"vm_size")
self.disk_size = (disk_size if disk_size is not None else
self.config.get("disk_size"))
if self.disk_size > 1023:
raise ValueError(
"VM OS disk canot be larger than 1023. Please change the ``disk_size`` config option."
)
self.scheduler_vm_size = (scheduler_vm_size
if scheduler_vm_size is not None else
self.config.get("scheduler_vm_size"))
if self.scheduler_vm_size is None:
self.scheduler_vm_size = self.vm_size
self.gpu_instance = ("_NC" in self.vm_size.upper()
or "_ND" in self.vm_size.upper())
self.vm_image = self.config.get("vm_image")
for key in vm_image:
self.vm_image[key] = vm_image[key]
self.bootstrap = (bootstrap if bootstrap is not None else
self.config.get("bootstrap"))
self.auto_shutdown = (auto_shutdown if auto_shutdown is not None else
self.config.get("auto_shutdown"))
self.docker_image = docker_image or self.config.get("docker_image")
self.debug = debug
self.options = {
"cluster": self,
"config": self.config,
"security_group": self.security_group,
"location": self.location,
"vm_image": self.vm_image,
"disk_size": self.disk_size,
"gpu_instance": self.gpu_instance,
"bootstrap": self.bootstrap,
"auto_shutdown": self.auto_shutdown,
"docker_image": self.docker_image,
}
self.scheduler_options = {
"vm_size": self.scheduler_vm_size,
"public_ingress": self.public_ingress,
**self.options,
}
self.worker_options = {"vm_size": self.vm_size, **self.options}
super().__init__(debug=debug, **kwargs)
import databricks_client
import os
import pytest
import requests
import requests_mock
import adal
from azure.common.credentials import get_azure_cli_credentials
credentials, subscription_id = get_azure_cli_credentials()
dbricks_api = os.environ["DATABRICKS_HOST"]
resource_group = os.environ["DATABRICKS_RG"]
workspace_name = os.environ["DATABRICKS_WORKSPACE"]
resource_id = ('/subscriptions/%s/resourceGroups/%s/providers/'
'Microsoft.Databricks/workspaces/%s' %
(subscription_id, resource_group, workspace_name))
non_provisioned_response = {
"error_code": "INVALID_PARAMETER_VALUE",
"message":
"Unknown worker environment WorkerEnvId(workerenv-63435k2940201085)"
}
client_id = os.environ["CLIENT_ID"]
client_secret = os.environ["CLIENT_SECRET"]
tenant_id = os.environ["TENANT_ID"]
authority_host_uri = 'https://login.microsoftonline.com'
authority_uri = authority_host_uri + '/' + tenant_id
context = adal.AuthenticationContext(authority_uri)
def get_clusters_list(client):
def authenticate(self,
cli=None, user_account=None, user_account_browser=None,
service_principal=None, file_auth=None, msi=None,
tenant_id=None,
subscription_id=None,
client_id=None, client_secret=None,
username=None, password=None,
programmatic_execution=False,
**kargs):
"""
Implements authentication for the Azure provider
"""
try:
# Set logging level to error for libraries as otherwise generates a lot of warnings
logging.getLogger('adal-python').setLevel(logging.ERROR)
logging.getLogger('msrest').setLevel(logging.ERROR)
logging.getLogger('urllib3').setLevel(logging.ERROR)
if cli:
arm_credentials, subscription_id, tenant_id = get_azure_cli_credentials(with_tenant=True)
aad_graph_credentials, placeholder_1, placeholder_2 = \
get_azure_cli_credentials(with_tenant=True, resource='https://graph.windows.net')
elif user_account:
if not (username and password):
if not programmatic_execution:
username = username if username else input("Username: "******"Password: "******"Tenant ID: ")
else:
raise AuthenticationException('No Tenant ID set')
if not client_id:
if not programmatic_execution:
client_id = input("Client ID: ")
else:
raise AuthenticationException('No Client ID set')
if not client_secret:
if not programmatic_execution:
client_secret = getpass("Client secret: ")
else:
raise AuthenticationException('No Client Secret set')
arm_credentials = ServicePrincipalCredentials(
client_id=client_id,
secret=client_secret,
tenant=tenant_id
)
aad_graph_credentials = ServicePrincipalCredentials(
client_id=client_id,
secret=client_secret,
tenant=tenant_id,
resource='https://graph.windows.net'
)
elif file_auth:
data = json.loads(file_auth.read())
tenant_id = data.get('tenantId')
client_id = data.get('clientId')
client_secret = data.get('clientSecret')
arm_credentials = ServicePrincipalCredentials(
client_id=client_id,
secret=client_secret,
tenant=tenant_id
)
aad_graph_credentials = ServicePrincipalCredentials(
client_id=client_id,
secret=client_secret,
tenant=tenant_id,
resource='https://graph.windows.net'
)
elif msi:
arm_credentials = MSIAuthentication()
aad_graph_credentials = MSIAuthentication(resource='https://graph.windows.net')
else:
raise AuthenticationException('Unknown authentication method')
return AzureCredentials(arm_credentials, aad_graph_credentials,
tenant_id, subscription_id)
except Exception as e:
raise AuthenticationException(e)
def get_az_credentials():
credentials, _, tenant = get_azure_cli_credentials(
resource='https://graph.windows.net', with_tenant=True)
return credentials, tenant
import os, uuid, sys from azure.mgmt.iotcentral import IotCentralClient from azure.mgmt.iotcentral.models import App, AppSkuInfo, AppPatch from msrestazure.azure_active_directory import MSIAuthentication from azure.common.credentials import UserPassCredentials, get_azure_cli_credentials # login with az login creds = get_azure_cli_credentials() subId = "FILL IN SUB ID" appName = "iot-central-app-tocreate" resourceGroup = "myResourceGroup" print(creds[0]) print(creds[1]) client = IotCentralClient(creds[0], subId) result = client.apps.check_name_availability(appName) print(result) app = App(location="unitedstates", sku=AppSkuInfo(name="ST2")) app.subdomain = appName app.display_name = appName createResult = client.apps.create_or_update(resourceGroup, appName, app) print(createResult) getResult = client.apps.get(resourceGroup, appName) print(getResult)
def main():
parser = argparse.ArgumentParser(description="Library path in ADF")
parser.add_argument("-r",
"--resource_group",
help="Resource group",
required=True)
parser.add_argument("-a", "--adf_name", help="ADF NAME", required=True)
parser.add_argument("-p",
"--adf_pipeline_name",
help="ADF pipeline name",
required=True)
parser.add_argument("-o",
"--output_file_path",
help="Output file path",
required=True)
parser.add_argument("-pa",
"--parameters",
help="Parameters",
required=False)
args = parser.parse_args()
resource_group = args.resource_group
adf_name = args.adf_name
adf_pipeline_name = args.adf_pipeline_name
output_file_path = args.output_file_path
parameters = args.parameters
print(f"-resource_group is {resource_group}")
print(f"-adf_name is {adf_name}")
print(f"-adf_pipeline_name is {adf_pipeline_name}")
print(f"-output_file_path is {output_file_path}")
print(f"-parameters is {parameters}")
credentials, subscription_id = get_azure_cli_credentials()
# The data factory name. It must be globally unique.
get_azure_cli_credentials()
adf_client = DataFactoryManagementClient(credentials, subscription_id)
# Create a pipeline run
run_response = adf_client.pipelines.create_run(resource_group,
adf_name,
adf_pipeline_name,
parameters=parameters)
# Monitor the pipeline run
time.sleep(5)
pipeline_run = adf_client.pipeline_runs.get(resource_group, adf_name,
run_response.run_id)
print("\n\tPipeline run status: {}".format(pipeline_run.status))
filter_params = RunFilterParameters(
last_updated_after=datetime.utcnow() - timedelta(1),
last_updated_before=datetime.utcnow() + timedelta(1))
query_response = adf_client.activity_runs.query_by_pipeline_run(
resource_group, adf_name, pipeline_run.run_id, filter_params)
while query_response.value[0].status in ['InProgress']:
print_activity_run_details(query_response.value[0])
time.sleep(3)
query_response = adf_client.activity_runs.query_by_pipeline_run(
resource_group, adf_name, pipeline_run.run_id, filter_params)
print_activity_run_details(query_response.value[0])
def authenticate(self, key_file=None, user_account=None, service_account=None, azure_cli=None, azure_msi=None,
azure_service_principal=None, azure_file_auth=None, azure_user_credentials=None, **kargs):
"""
Implements authentication for the Azure provider using azure-cli.
Refer to https://docs.microsoft.com/en-us/python/azure/python-sdk-azure-authenticate?view=azure-python.
:return:
"""
try:
if azure_cli:
cli_credentials, self.aws_account_id = get_azure_cli_credentials() # TODO: Remove aws_account_id
self.credentials = AzureCredentials(cli_credentials, self.aws_account_id)
return True
elif azure_msi:
credentials = MSIAuthentication()
# Get the subscription ID
subscription_client = SubscriptionClient(credentials)
try:
# Tries to read the subscription list
subscription = next(subscription_client.subscriptions.list())
self.aws_account_id = subscription.subscription_id
except StopIteration:
# If the VM cannot read subscription list, ask Subscription ID:
self.aws_account_id = input('Subscription ID: ')
self.credentials = AzureCredentials(credentials, self.aws_account_id)
return True
elif azure_file_auth:
with open(azure_file_auth) as f:
data = json.loads(f.read())
subscription_id = data.get('subscriptionId')
tenant_id = data.get('tenantId')
client_id = data.get('clientId')
client_secret = data.get('clientSecret')
self.aws_account_id = tenant_id # TODO this is for AWS
credentials = ServicePrincipalCredentials(
client_id=client_id,
secret=client_secret,
tenant=tenant_id
)
self.credentials = AzureCredentials(credentials, subscription_id)
return True
elif azure_service_principal:
subscription_id = input("Subscription ID: ")
tenant_id = input("Tenant ID: ")
client_id = input("Client ID: ")
client_secret = getpass("Client secret: ")
self.aws_account_id = tenant_id # TODO this is for AWS
credentials = ServicePrincipalCredentials(
client_id=client_id,
secret=client_secret,
tenant=tenant_id
)
self.credentials = AzureCredentials(credentials, subscription_id)
return True
elif azure_user_credentials:
username = input("Username: "******"Password: "******"" # TODO this is for AWS
self.credentials = AzureCredentials(credentials, self.aws_account_id)
return True
except Exception as e:
printError('Failed to authenticate to Azure')
printException(e)
return False
