def login(username, password, tenant, subscriptionId):
from azure.common.client_factory import get_client_from_json_dict
global resource_management_client
global compute_management_client
global storage_management_client
config_dict = {
"clientId": username,
"clientSecret": password,
"subscriptionId": subscriptionId,
"tenantId": tenant,
"activeDirectoryEndpointUrl": "https://login.microsoftonline.com",
"resourceManagerEndpointUrl": "https://management.azure.com/",
"activeDirectoryGraphResourceId": "https://graph.windows.net/",
"sqlManagementEndpointUrl":
"https://management.core.windows.net:8443/",
"galleryEndpointUrl": "https://gallery.azure.com/",
"managementEndpointUrl": "https://management.core.windows.net/"
}
resource_management_client = get_client_from_json_dict(
ResourceManagementClient, config_dict)
compute_management_client = get_client_from_json_dict(
ComputeManagementClient, config_dict)
storage_management_client = get_client_from_json_dict(
StorageManagementClient, config_dict)
def get_conn(self) -> Any:
"""
Authenticates the resource using the connection id passed during init.
:return: the authenticated client.
"""
conn = self.get_connection(self.conn_id)
tenant = conn.extra_dejson.get('extra__azure__tenantId') or conn.extra_dejson.get('tenantId')
subscription_id = conn.extra_dejson.get('extra__azure__subscriptionId') or conn.extra_dejson.get(
'subscriptionId'
)
key_path = conn.extra_dejson.get('key_path')
if key_path:
if not key_path.endswith('.json'):
raise AirflowException('Unrecognised extension for key file.')
self.log.info('Getting connection using a JSON key file.')
return get_client_from_auth_file(client_class=self.sdk_client, auth_path=key_path)
key_json = conn.extra_dejson.get('key_json')
if key_json:
self.log.info('Getting connection using a JSON config.')
return get_client_from_json_dict(client_class=self.sdk_client, config_dict=key_json)
self.log.info('Getting connection using specific credentials and subscription_id.')
return self.sdk_client(
credentials=ServicePrincipalCredentials(
client_id=conn.login, secret=conn.password, tenant=tenant
),
subscription_id=subscription_id,
)
def download_disk(self):
client_class = compute.ComputeManagementClient
compute_client = get_client_from_json_dict(
client_class=client_class, config_dict=self.credentials_compute)
snapshot = compute_client.snapshots.grant_access(
resource_group_name=self.cloud_resource_group_name,
snapshot_name=self.server_name,
access="read",
duration_in_seconds=18000)
snapshot.wait()
access_sas = snapshot.result().access_sas
blob_name = "{0}.{1}".format(self.server_name, "vhd")
path = "/home/ubuntu/volume"
path = os.path.join(path, "{0}".format(blob_name))
response = requests.get(access_sas, stream=True)
handle = open(path, "wb")
for chunk in response.iter_content(chunk_size=512):
if chunk:
handle.write(chunk)
disk = "/home/ubuntu/volume/{0}".format(blob_name)
print("disk size (bytes): {}".format(os.path.getsize(disk)))
def export_disk(self):
client_class = compute.ComputeManagementClient
compute_client = get_client_from_json_dict(
client_class=client_class, config_dict=self.credentials_compute)
virtual_machine = compute_client.virtual_machines.get(
resource_group_name=self.cloud_resource_group_name,
vm_name=self.server_name)
storage_profile = virtual_machine.storage_profile
operating_system_disk = storage_profile.os_disk
managed_disk = operating_system_disk.managed_disk
source_uri = managed_disk.id
disk_size_gb = operating_system_disk.disk_size_gb
async_snapshot_creation = compute_client.snapshots.create_or_update(
resource_group_name=self.cloud_resource_group_name,
snapshot_name=self.server_name,
snapshot={
"location": self.cloud_location,
"creation_data": {
"create_option": "Copy",
"source_uri": source_uri,
"disk_size_gb": disk_size_gb
}
})
async_snapshot_creation.wait()
def import_disk(self):
client_class = compute.ComputeManagementClient
compute_client = get_client_from_json_dict(
client_class=client_class, config_dict=self.credentials_compute)
page_blob_service = blob.PageBlobService(
account_name=self.account_name, account_key=self.account_key)
blob_name = "{0}.{1}".format(self.server_name, "vhd")
blob_uri = page_blob_service.make_blob_url(
container_name=self.bucket_name, blob_name=blob_name)
image = {
"location": self.cloud_location,
"storage_profile": {
"os_disk": {
"os_type": "Linux",
"os_state": "Generalized",
"blob_uri": blob_uri,
"caching": "ReadWrite",
}
}
}
async_image_creation = compute_client.images.create_or_update(
resource_group_name=self.cloud_resource_group_name,
image_name=self.server_name,
parameters=image)
async_image_creation.wait()
def delete_server(self):
client_class = compute.ComputeManagementClient
compute_client = get_client_from_json_dict(
client_class=client_class, config_dict=self.credentials_compute)
virtual_machine = compute_client.virtual_machines.delete(
resource_group_name=self.cloud_resource_group_name,
vm_name=self.server_name)
virtual_machine.wait()
def get_client(resource, config_dict):
"""Get azure client based on the give resource.
This method will first verify if we can get the client
by using the information provided on the login account
of the user machine. If the user is not logged into Azure,
we will try to get the client from the ids given by the
user to this class.
Args:
resource: Azure Resource, An Azure resource that we want to get
a client for.
config_dict: dict, Id parameters passed by the user to this class.
Returns:
The client for the resource passed as parameter.
"""
try:
return get_client_from_cli_profile(resource)
except CLIError:
logger.debug(
"No valid azure-cli config found. Trying explicit config params"
)
required_keys = frozenset(
{"clientId", "clientSecret", "tenantId", "subscriptionId"}
)
missing_keys = required_keys.difference(set(config_dict.keys()))
if missing_keys:
raise RuntimeError(
"No AZ cli config found, missing required keys: {}".format(
", ".join(missing_keys)
)
)
parameters = {
"activeDirectoryEndpointUrl":
"https://login.microsoftonline.com",
"resourceManagerEndpointUrl": "https://management.azure.com/",
"activeDirectoryGraphResourceId":
"https://graph.windows.net/",
"sqlManagementEndpointUrl":
"https://management.core.windows.net:8443/",
"galleryEndpointUrl": "https://gallery.azure.com/",
"managementEndpointUrl":
"https://management.core.windows.net/"
}
parameters.update(config_dict)
client = get_client_from_json_dict(
resource,
parameters
)
return client
def get_management_client(azure_auth, client_class):
"""Returns SDK Client for the given client_class."""
client = None
if azure_auth is not None:
_LOGGER.info("Getting SDK Client for %s", client_class)
auth_config_dict = json.loads(azure_auth)
client = get_client_from_json_dict(client_class, auth_config_dict)
else:
raise AuthException()
return client
def get_container_client(azure_auth):
"""Get Container Client."""
if azure_auth is not None:
_LOGGER.info("Authenticating Azure using credentials")
client = get_client_from_json_dict(ContainerInstanceManagementClient,
azure_auth)
else:
_LOGGER.error("\nFailed to authenticate to Azure. Have you set the"
" AZURE_AUTH_LOCATION environment variable?\n")
return client
def delete_image(self):
client_class = compute.ComputeManagementClient
compute_client = get_client_from_json_dict(
client_class=client_class, config_dict=self.credentials_compute)
snapshot = compute_client.snapshots.revoke_access(
resource_group_name=self.cloud_resource_group_name,
snapshot_name=self.server_name)
snapshot.wait()
snapshot = compute_client.snapshots.delete(
resource_group_name=self.cloud_resource_group_name,
snapshot_name=self.server_name)
snapshot.wait()
def get_container_client(azure_auth):
"""Get Container Client."""
if azure_auth is not None:
logging.info("Authenticating Azure using credentials")
auth_config_dict = json.loads(azure_auth)
client = get_client_from_json_dict(
ContainerInstanceManagementClient, auth_config_dict
)
else:
logging.error(
"\nFailed to authenticate to Azure. Have you set the"
" AZURE_AUTH environment variable?\n"
)
return client
def get_client(tenant_id, subscription_id, client_id, client_secret):
config_dict = {
'clientId': client_id,
'clientSecret': client_secret,
'subscriptionId': subscription_id,
'tenantId': tenant_id,
'activeDirectoryEndpointUrl': 'https://login.microsoftonline.com',
'resourceManagerEndpointUrl': 'https://management.azure.com/',
'activeDirectoryGraphResourceId': 'https://graph.windows.net/',
'sqlManagementEndpointUrl': 'https://management.core.windows.net:8443/',
'galleryEndpointUrl': 'https://gallery.azure.com/',
'managementEndpointUrl': 'https://management.core.windows.net/'
}
client = get_client_from_json_dict(ResourceManagementClient, config_dict)
return client
def ingest(table_name, options):
tenant_id = options['tenant_id']
client_id = options['client_id']
client_secret = options['client_secret']
subscriptions_service = get_client_from_json_dict(
SubscriptionClient, {
"tenantId": tenant_id,
"clientId": client_id,
"clientSecret": client_secret,
"activeDirectoryEndpointUrl": "https://login.microsoftonline.com",
"resourceManagerEndpointUrl": "https://management.azure.com/",
"managementEndpointUrl": "https://management.core.windows.net/",
}).subscriptions
subscription_list = subscriptions_service.list()
subscriptions = [s.as_dict() for s in subscription_list]
db.insert(
f'data.{table_name}',
values=[(
parse(subscription_list.raw.response.headers['Date']).isoformat(),
tenant_id,
row,
row['id'],
row['subscription_id'],
row['display_name'],
row['state'],
row['subscription_policies'],
row['authorization_source'],
) for row in subscriptions],
select=(
'column1',
'column2',
'PARSE_JSON(column3)',
'column4',
'column5',
'column6',
'column7',
'PARSE_JSON(column8)',
'column9',
))
yield len(subscriptions)
def get_conn(self) -> HDInsightManagementClient:
"""
Return a HDInsight Management client from the Azure Python SDK for HDInsight
This hook requires a service principal in order to work.
You can create a service principal from the az CLI like so::
az ad sp create-for-rbac --name localtest-sp-rbac --skip-assignment \\
--sdk-auth > local-sp.json
.. seealso::
* `Create an Azure AD app & service principal in the portal - Microsoft identity platform <https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal>`_
* `Azure HDInsight SDK for Python <https://docs.microsoft.com/en-us/python/api/overview/azure/hdinsight?view=azure-python#authentication-example-using-a-service-principal>`_
* `How to authenticate Python applications with Azure services <https://docs.microsoft.com/en-us/azure/developer/python/azure-sdk-authenticate?view=azure-python&tabs=cmd#authenticate-with-a-json-dictionary>`_
"""
conn = self.get_connection(self.conn_id)
servicePrincipal = conn.extra_dejson.get('servicePrincipal', False)
if servicePrincipal:
return get_client_from_json_dict(HDInsightManagementClient,
servicePrincipal)
# Read the file.
config.read('config/config.ini')
# Grab the Azure Credentials needed.
subscription_id = config.get('azure_credentials', 'azure_subscription_id')
tenant_id = config.get('azure_credentials', 'azure_tenant_id')
client_id = config.get('azure_credentials', 'azure_client_id')
client_secret = config.get('azure_credentials', 'azure_client_secret')
# Define the configuration dictionary.
config_dict = {
"subscriptionId": subscription_id,
"tenantId": tenant_id,
"clientId": client_id,
"clientSecret": client_secret,
"activeDirectoryEndpointUrl": "https://login.microsoftonline.com",
"resourceManagerEndpointUrl": "https://management.azure.com/",
"activeDirectoryGraphResourceId": "https://graph.windows.net/",
"sqlManagementEndpointUrl": "https://management.core.windows.net:8443/",
"galleryEndpointUrl": "https://gallery.azure.com/",
"managementEndpointUrl": "https://management.core.windows.net/"
}
# Grab the subscription client.
sql_management_client: SqlManagementClient = get_client_from_json_dict(
client_class=SqlManagementClient, config_dict=config_dict)
# Print out all the servers associated with my Subscription.
server = next(sql_management_client.servers.list())
print(server)
def test_policy_client_with_spn(self):
policy_client = get_client_from_json_dict(PolicyClient,
self.config_dict)
print(policy_client)
def __init__(self, filePath):
with open(filePath) as f:
credData = json.load(f)
self.scope = "subscriptions/" + credData['subscriptionId']
self.consumption_client = get_client_from_json_dict(
ConsumptionManagementClient, credData)
def create_server(self):
client_class = network.NetworkManagementClient
network_client = get_client_from_json_dict(
client_class=client_class, config_dict=self.credentials_compute)
subnet = network_client.subnets.get(
resource_group_name=self.cloud_resource_group_name,
virtual_network_name=self.network_name,
subnet_name=self.subnet_name)
network_interface_name = "{}{}".format(self.server_name, "-interface")
ip_name = "{}{}".format(self.server_name, "-ip")
async_nic_creation = network_client.network_interfaces.create_or_update(
resource_group_name=self.cloud_resource_group_name,
network_interface_name=network_interface_name,
parameters={
"location":
self.cloud_location,
"ip_configurations": [{
"name": ip_name,
"subnet": {
"id": subnet.id
}
}]
})
async_nic_creation.wait()
result = async_nic_creation.result()
nic_id = result.id
url_parts = [
"/subscriptions/", self.subscription_id, "/resourceGroups/",
self.cloud_resource_group_name, "/providers/", "Microsoft.Compute",
"/images/", self.server_name
]
image_reference_id = "".join(url_parts)
parameters = {
"location": self.cloud_location,
"os_profile": {
"computer_name": self.server_name,
"admin_username": "******",
"admin_password": "******"
},
"hardware_profile": {
"vm_size": self.server_capacity
},
"storage_profile": {
"image_reference": {
"id": image_reference_id
}
},
"network_profile": {
"network_interfaces": [{
"id": nic_id,
}]
}
}
if (self.cloud_location != "westus"):
parameters["zones"] = [self.cloud_zones]
client_class = compute.ComputeManagementClient
compute_client = get_client_from_json_dict(
client_class=client_class, config_dict=self.credentials_compute)
async_vm_creation = compute_client.virtual_machines.create_or_update(
resource_group_name=self.cloud_resource_group_name,
vm_name=self.server_name,
parameters=parameters)
async_vm_creation.wait()
def get_subscription_service(options, cloud_type='reg'):
options.update(API_ENDPOINTS[cloud_type])
return get_client_from_json_dict(SubscriptionClient, options).subscriptions
def get_vms(options):
cli = get_client_from_json_dict(ComputeManagementClient, options)
vms = [vm.as_dict() for vm in cli.virtual_machines.list_all()]
for vm in vms:
vm['subscription_id'] = options['subscriptionId']
return vms
def get_nics(options):
cli = get_client_from_json_dict(NetworkManagementClient, options)
return [nic.as_dict() for nic in cli.network_interfaces.list_all()]
