def _get_profiles(self, attribute_type, sub_index, sub):
"""Return an Azure monitor record.
Arguments:
attribute_type (str): Attribute type name.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
Yields:
dict: An Azure monitor record.
"""
_log.info('Working on %s', util.outline_az_sub(sub_index, sub,
self._tenant))
try:
monitor_client = \
MonitorManagementClient(self._credentials,
sub.get('subscription_id'))
iterator = \
_get_attribute_iterator(attribute_type, monitor_client,
sub, sub_index, self._tenant)
yield from _get_record(iterator, attribute_type, self._max_recs,
sub_index, sub, self._tenant)
except Exception as e:
_log.error('Failed to fetch details for %s; %s; error: %s: %s',
attribute_type,
util.outline_az_sub(sub_index, sub, self._tenant),
type(e).__name__, e)
def _get_web_app_configs(self, app_index, app, sub_index, sub):
"""Get web app records with config details.
Arguments:
app_index (int): Web app index (for logging only).
app (dict): Raw web app record.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
Yields:
dict: An Azure web app record with config details.
"""
app_name = app.get('name')
_log.info('Working on web app #%d: %s; %s', app_index, app_name,
util.outline_az_sub(sub_index, sub, self._tenant))
try:
creds = self._credentials
sub_id = sub.get('subscription_id')
web_client = WebSiteManagementClient(creds, sub_id)
app_id = app.get('id')
rg_name = tools.parse_resource_id(app_id)['resource_group']
app_config = web_client.web_apps.get_configuration(
rg_name, app_name)
app_config = app_config.as_dict()
yield _process_app_config(app_index, app, app_config, sub_index,
sub, self._tenant)
except Exception as e:
_log.error(
'Failed to fetch app_config for web app #%d: '
'%s; %s; error: %s: %s', app_index, app_name,
util.outline_az_sub(sub_index, sub, self._tenant),
type(e).__name__, e)
def _get_disk_details(self, disk_index, disk_name, rg_name, sub_index,
sub):
"""Get details of disk.
Arguments:
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
rg_name (str): Resource group name.
disk_index (int): Disk index (for logging only).
disk_name (str): Name of the disk.
Yields:
dict: An Azure disk record.
"""
_log.info('Working on disk #%d: %s; %s', disk_index, disk_name,
util.outline_az_sub(disk_index, sub, self._tenant))
try:
sub_id = sub.get('subscription_id')
creds = self._credentials
compute_client = ComputeManagementClient(creds, sub_id)
disk = compute_client.disks.get(rg_name, disk_name)
disk = disk.as_dict()
yield _process_disk_details(sub, disk)
except Exception as e:
_log.error(
'Failed to fetch disk details #%d: '
'%s; %s; error: %s: %s', disk_index, disk_name,
util.outline_az_sub(sub_index, sub, self._tenant),
type(e).__name__, e)
def _get_vm_instance_views(self, vm_index, vm, sub_index, sub):
"""Get virtual machine records with instance view details.
Arguments:
vm_index (int): Virtual machine index (for logging only).
vm (dict): Raw virtual machine record.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
Yields:
dict: An Azure virtual machine record with instance view details.
"""
vm_name = vm.get('name')
_log.info('Working on VM #%d: %s; %s', vm_index, vm_name,
util.outline_az_sub(sub_index, sub, self._tenant))
try:
creds = self._credentials
sub_id = sub.get('subscription_id')
compute_client = ComputeManagementClient(creds, sub_id)
vm_id = vm.get('id')
rg_name = tools.parse_resource_id(vm_id)['resource_group']
vm_iv = compute_client.virtual_machines.instance_view(
rg_name, vm_name)
vm_iv = vm_iv.as_dict()
yield _process_vm_instance_view(vm_index, vm, vm_iv, sub_index,
sub, self._tenant)
except Exception as e:
_log.error(
'Failed to fetch vm_instance_view for VM #%d: '
'%s; %s; error: %s: %s', vm_index, vm_name,
util.outline_az_sub(sub_index, sub, self._tenant),
type(e).__name__, e)
def _get_subscription_storage_accounts(self, sub_index, sub):
"""Get storage accounts from a single subscrption.
Yields:
tuple: A tuple which when unpacked forms valid arguments for
:meth:` _get_storage_account_properties`.
"""
try:
tenant = self._tenant
creds = self._credentials
sub_id = sub.get('subscription_id')
client = StorageManagementClient(creds, sub_id)
storage_account_list = client.storage_accounts.list()
for t in enumerate(storage_account_list):
(storage_account_index, storage_account) = t
storage_account = storage_account.as_dict()
_log.info('Found storage account #%d: %s; %s',
storage_account_index, storage_account.get('name'),
util.outline_az_sub(sub_index, sub, tenant))
yield (storage_account_index, storage_account, sub_index, sub)
if storage_account_index + 1 == self._max_recs:
_log.info(
'Stopping storage accounts fetch due '
'to _max_recs: %d; %s', self._max_recs,
util.outline_az_sub(sub_index, sub, tenant))
break
except Exception as e:
_log.error('Failed to fetch storage accounts; %s; error: %s: %s',
util.outline_az_sub(sub_index, sub, tenant),
type(e).__name__, e)
def _process_key_vault(self, key_vault_index, key_vault_name, rg_name,
sub_index, sub):
"""Get details of Key Vault in management and data plane.
Arguments:
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
rg_name (str): Resource group name.
key_vault_index (int): Key Vault index (for logging only).
key_vault_name (str): Name of the Key Vault.
Yields:
dict: An Azure Key Vault server record.
"""
def _auth_callback(server, resource, scope):
credentials = self._key_vault_credentials
token = credentials.token
return token['token_type'], token['access_token']
_log.info('Working on key_vault #%d: %s; %s', key_vault_index,
key_vault_name,
util.outline_az_sub(sub_index, sub, self._tenant))
subscription_id = sub.get('subscription_id')
creds = self._credentials
key_vault_mgmt_client = KeyVaultManagementClient(
creds, subscription_id)
key_vault_details = key_vault_mgmt_client.vaults.get(
rg_name, key_vault_name)
yield from _get_normalized_key_vault_record(key_vault_details, sub)
try:
kv_client = \
KeyVaultClient(KeyVaultAuthentication(_auth_callback))
secrets = \
kv_client.get_secrets(key_vault_details.properties.vault_uri)
keys = \
kv_client.get_keys(key_vault_details.properties.vault_uri)
# Retrieve data using each iterator.
for record in itertools.chain(
_get_data_record(secrets, 'key_vault_secret', sub_index,
sub, self._tenant),
_get_data_record(keys, 'key_vault_key', sub_index, sub,
self._tenant),
):
yield record
except CloudError as e:
_log.error('Failed to fetch key vault details; %s; error: %s: %s',
util.outline_az_sub(sub_index, sub, self._tenant),
type(e).__name__, e)
def _get_data_record(iterator, azure_record_type, sub_index, sub, tenant):
"""Normalize the Key Vault data plane record.
Arguments:
iterator: An iterator like instance of
:class:`msrest.serialization.Model` objects.
azure_record_type (str): Record type name.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
tenant (str): Azure tenant ID (for logging only).
Returns:
dict: Normalized Key Vault data plane record.
"""
try:
for i, v in enumerate(iterator):
raw_record = v.as_dict()
reference = raw_record.get('id')
if azure_record_type == 'key_vault_key':
reference = raw_record.get('kid')
expiry_set = raw_record['attributes'].get('expires') is not None
enabled = raw_record['attributes'].get('enabled')
record = {
'raw': raw_record,
'ext': {
'cloud_type': 'azure',
'expiry_set': expiry_set,
'enabled': enabled,
'record_type': azure_record_type,
'reference': reference,
'subscription_id': sub.get('subscription_id'),
'subscription_name': sub.get('display_name'),
'subscription_state': sub.get('state'),
},
'com': {
'cloud_type': 'azure',
'record_type': azure_record_type,
'reference': reference,
}
}
_log.info('Found %s #%d: %s; %s', azure_record_type, i, reference,
util.outline_az_sub(sub_index, sub, tenant))
yield record
except KeyVaultErrorException as e:
_log.error('Failed to fetch details for %s; %s; error: %s: %s',
azure_record_type,
util.outline_az_sub(sub_index, sub, tenant),
type(e).__name__, e)
def _get_subscriptions(self):
"""Generate tuples of record types and subscriptions.
The yielded tuples when unpacked would become arguments for
:meth:`_get_resources`. Each such tuple represents a single unit
of work that :meth:`_get_resources` can work on independently in
its own worker thread.
Yields:
tuple: A tuple which when unpacked forms valid arguments for
:meth:`_get_resources`.
"""
try:
sub_client = SubscriptionClient(self._credentials)
sub_list = sub_client.subscriptions.list()
monitor_attributes = ('log_profile',)
tenant = self._tenant
for sub_index, sub in enumerate(sub_list):
sub = sub.as_dict()
_log.info('Found %s', util.outline_az_sub(sub_index,
sub, tenant))
# Each record type for each subscription is a unit of
# work that would be fed to _get_resources().
for attribute_type in monitor_attributes:
if attribute_type == 'log_profile':
sub['locations'] = []
locations = sub_client.subscriptions. \
list_locations(sub.get('subscription_id'))
for location in locations:
sub['locations'].append(location.as_dict()
.get('name'))
yield (attribute_type, sub_index, sub)
# Break after pulling data for self._max_subs number of
# subscriptions. Note that if self._max_subs is 0 or less,
# then the following condition never evaluates to True.
if sub_index + 1 == self._max_subs:
_log.info('Stopping subscriptions fetch due to '
'_max_subs: %d; tenant: %s', self._max_subs,
self._tenant)
break
except Exception as e:
_log.error('Failed to fetch subscriptions; %s; error: %s: %s',
util.outline_az_sub(sub_index, sub, tenant),
type(e).__name__, e)
def _get_attribute_iterator(attribute_type, monitor_client,
sub, sub_index, tenant):
"""Return an appropriate iterator for ``attribute_type``.
Arguments:
attribute_type (str): Attribute type.
monitor_client(MonitorManagementClient): Monitor client.
credentials (ServicePrincipalCredentials): Credentials.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Subscription object.
tenant (str): Tenant ID (for logging only).
Returns:
msrest.paging.Paged: An Azure paging container for iterating
over a list of Azure resource objects.
"""
if attribute_type == 'log_profile':
return monitor_client.log_profiles.list()
# If control reaches here, there is a bug in this plugin. It means
# there is a value in attributes variable in _get_subscriptions
# that is not handled in the above if-statements.
_log.warning('Unrecognized profile_type: %s; %s', attribute_type,
util.outline_az_sub(sub_index, sub, tenant))
return None
def _process_app_config(app_index, app, app_config, sub_index, sub, tenant):
"""Process web app record and yield them.
Arguments:
app_index (int): Web app index (for logging only).
app (dict): Raw web app record.
app_config (dict): Raw web app config record.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
tenant (str): Azure tenant ID.
Yields:
dict: An Azure record of type ``web_app_config``.
"""
app['config'] = app_config
record = {
'raw': app,
'ext': {
'cloud_type': 'azure',
'record_type': 'web_app_config',
'min_tls_version': app_config.get('min_tls_version'),
'subscription_id': sub.get('subscription_id'),
'subscription_name': sub.get('display_name'),
'subscription_state': sub.get('state'),
},
'com': {
'cloud_type': 'azure',
'reference': app.get('id')
}
}
_log.info('Found web_app_config #%d: %s; %s', app_index, app.get('name'),
util.outline_az_sub(sub_index, sub, tenant))
return record
def _get_postgres_server_details(self, server_index, server_name, rg_name,
sub_index, sub):
"""Get details Postgres server.
Arguments:
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
rg_name (str): Resource group name.
server_index (int): Server index (for logging only).
server_name (str): Name of the Postgres server.
Yields:
dict: An Azure Postgres server record with configuration.
"""
_log.info('Working on Postgres server #%d: %s; %s',
server_index, server_name,
util.outline_az_sub(sub_index, sub, self._tenant))
sub_id = sub.get('subscription_id')
creds = self._credentials
postgres_client = PostgreSQLManagementClient(creds, sub_id)
server_details = postgres_client.servers.get(rg_name, server_name)
server_details = server_details.as_dict()
server_configuration_list = \
postgres_client.configurations.list_by_server(rg_name, server_name)
configurations, derived_configs = \
self._get_postgres_server_configuration(server_configuration_list)
yield from self._process_postgres_server_details(
sub, server_details, configurations, derived_configs)
def _get_resources(self, record_type, sub_index, sub):
"""Return an Azure cloud infrastructure configuration record.
Arguments:
record_type (str): Record type name.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
Yields:
dict: An Azure cloud infrastructure configuration record.
"""
_log.info('Working on %s list; %s', record_type,
util.outline_az_sub(sub_index, sub, self._tenant))
if record_type == 'subscription':
record = {
'raw': sub,
'ext': {
'cloud_type': 'azure',
'record_type': record_type,
'subscription_id': sub.get('subscription_id'),
'tenant_id': self._tenant,
'subscription_name': sub.get('display_name'),
'subscription_state': sub.get('state'),
},
'com': {
'cloud_type': 'azure',
'record_type': 'subscription'
}
}
yield record
return
try:
iterator = \
_get_resource_iterator(record_type, self._credentials,
sub_index, sub, self._tenant)
yield from _get_record(iterator, record_type, self._max_recs,
sub_index, sub, self._tenant)
except Exception as e:
_log.error('Failed to fetch details for %s; %s; error: %s: %s',
record_type,
util.outline_az_sub(sub_index, sub, self._tenant),
type(e).__name__, e)
def _get_subscription_kvs(self, sub_index, sub):
"""Get Key Vaults from a single subscrption.
Arguments:
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
Yields:
tuple: A tuple which when unpacked forms valid arguments for
:meth:`_process_key_vault`.
"""
try:
tenant = self._tenant
creds = self._credentials
subscription_id = sub.get('subscription_id')
key_vault_mgmt_client = KeyVaultManagementClient(
creds, subscription_id)
key_vault_list = key_vault_mgmt_client.vaults.list()
for key_vault_index, key_vault in enumerate(key_vault_list):
key_vault = key_vault.as_dict()
key_vault_name = key_vault.get('name')
key_vault_id = key_vault.get('id')
_log.info('Found key_vault #%d: %s; %s', key_vault_index,
key_vault_name,
util.outline_az_sub(sub_index, sub, tenant))
rg_name = \
tools.parse_resource_id(key_vault_id)['resource_group']
yield (key_vault_index, key_vault_name, rg_name, sub_index,
sub)
# Break after pulling data for self._max_recs number
# of Key Vault for a subscriber. Note that if
# self._max_recs is 0 or less, then the following
# condition never evaluates to True.
if key_vault_index + 1 == self._max_recs:
_log.info(
'Stopping Key Vault fetch due '
'to _max_recs: %d; %s', self._max_recs,
util.outline_az_sub(sub_index, sub, self._tenant))
break
except CloudError as e:
_log.error('Failed to fetch Key Vault; %s; error: %s: %s',
util.outline_az_sub(sub_index, sub, tenant),
type(e).__name__, e)
def _get_subscriptions(self):
"""Generate tuples of record types and subscriptions.
The yielded tuples when unpacked would become arguments for
:meth:`_get_resources`. Each such tuple represents a single unit
of work that :meth:`_get_resources` can work on independently in
its own worker thread.
Yields:
tuple: A tuple which when unpacked forms valid arguments for
:meth:`_get_resources`.
"""
try:
sub_client = SubscriptionClient(self._credentials)
sub_list = sub_client.subscriptions.list()
record_types = ('virtual_machine', 'app_gateway', 'lb', 'nic',
'nsg', 'public_ip', 'storage_account',
'resource_group', 'mysql_server',
'web_apps', 'subscription')
tenant = self._tenant
for sub_index, sub in enumerate(sub_list):
sub = sub.as_dict()
_log.info('Found %s', util.outline_az_sub(sub_index,
sub, tenant))
# Each record type for each subscription is a unit of
# work that would be fed to _get_resources().
for record_type in record_types:
yield (record_type, sub_index, sub)
# Break after pulling data for self._max_subs number of
# subscriptions. Note that if self._max_subs is 0 or less,
# then the following condition never evaluates to True.
if sub_index + 1 == self._max_subs:
_log.info('Stopping subscriptions fetch due to '
'_max_subs: %d; tenant: %s', self._max_subs,
self._tenant)
break
except Exception as e:
_log.error('Failed to fetch subscriptions; %s; error: %s: %s',
util.outline_az_sub(sub_index, sub, tenant),
type(e).__name__, e)
def _process_storage_account_properties(storage_account_index, storage_account,
storage_account_properties, sub_index,
sub, tenant):
"""Get storage account records with property details.
Arguments:
storage_account_index (int): Storage account index (logging only).
storage_account (dict): Raw storage account record.
storage_account_properties (dict): Storage account properties record.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
tenant (str): Azure tenant ID.
Yields:
dict: An Azure record of type ``storage_account_properties``.
"""
storage_account['properties'] = storage_account_properties
default_network_access_allowed = True
if storage_account['network_rule_set'].get('default_action') != 'Allow':
default_network_access_allowed = False
# Azure services is an umbrella term for trusted Microsoft Azure services
# including Azure Backup, Azure Site Recovery, Azure DevTest Labs,
# Azure Event Grid, Azure Event Hubs, Azure Networking, Azure Monitor and
# Azure SQL Data Warehouse
bypass_trusted_services = True
if storage_account['network_rule_set'].get('bypass') != 'AzureServices':
bypass_trusted_services = False
record = {
'raw': storage_account,
'ext': {
'cloud_type':
'azure',
'record_type':
'storage_account_properties',
'secure_transfer_required':
storage_account_properties.get('enable_https_traffic_only'),
'default_network_access_allowed':
default_network_access_allowed,
'trusted_services_allowed':
bypass_trusted_services,
'subscription_id':
sub.get('subscription_id'),
'subscription_name':
sub.get('display_name'),
'subscription_state':
sub.get('state'),
},
'com': {
'cloud_type': 'azure',
'reference': storage_account.get('id')
}
}
_log.info('Found storage_account_properties #%d: %s; %s',
storage_account_index, storage_account.get('name'),
util.outline_az_sub(sub_index, sub, tenant))
return record
def _get_subscription_postgres_servers(self, sub_index, sub):
"""Get Postgres servers from a single subscrption.
Arguments:
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
Yields:
tuple: A tuple which when unpacked forms valid arguments for
:meth:`_get_postgres_details`.
"""
try:
tenant = self._tenant
creds = self._credentials
sub_id = sub.get('subscription_id')
postgres_client = PostgreSQLManagementClient(creds, sub_id)
db_server_list = postgres_client.servers.list()
for server_index, postgres_server in enumerate(db_server_list):
postgres_server = postgres_server.as_dict()
server_id = postgres_server.get('id')
server_name = postgres_server.get('name')
_log.info('Found Postgres Server #%d: %s; %s', server_index,
server_name,
util.outline_az_sub(sub_index, sub, tenant))
rg_name = \
tools.parse_resource_id(server_id)['resource_group']
yield (server_index, server_name, rg_name, sub_index, sub)
# Break after pulling data for self._max_recs number
# of Postgres servers for a subscriber. Note that if
# self._max_recs is 0 or less, then the following
# condition never evaluates to True.
if server_index + 1 == self._max_recs:
_log.info(
'Stopping Postgres server fetch due '
'to _max_recs: %d; %s', self._max_recs,
util.outline_az_sub(sub_index, sub, self._tenant))
break
except CloudError as e:
_log.error('Failed to fetch Postgres servers; %s; error: %s: %s',
util.outline_az_sub(sub_index, sub, tenant),
type(e).__name__, e)
def _get_subscription_disks(self, sub_index, sub):
"""Get disks from a single subscrption.
Arguments:
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
Yields:
tuple: A tuple which when unpacked forms valid arguments for
:meth:`_get_disk_details`.
"""
try:
tenant = self._tenant
creds = self._credentials
sub_id = sub.get('subscription_id')
compute_client = ComputeManagementClient(creds, sub_id)
disk_list = compute_client.disks.list()
for disk_index, disk in enumerate(disk_list):
disk = disk.as_dict()
disk_id = disk.get('id')
disk_name = disk.get('name')
_log.info('Found disk #%d: %s; %s', disk_index, disk_name,
util.outline_az_sub(sub_index, sub, tenant))
rg_name = \
tools.parse_resource_id(disk_id)['resource_group']
yield (disk_index, disk_name, rg_name, sub_index, sub)
# Break after pulling data for self._max_recs number
# of disks for a subscriber. Note that if
# self._max_recs is 0 or less, then the following
# condition never evaluates to True.
if disk_index + 1 == self._max_recs:
_log.info(
'Stopping disk fetch due '
'to _max_recs: %d; %s', self._max_recs,
util.outline_az_sub(sub_index, sub, self._tenant))
break
except Exception as e:
_log.error('Failed to fetch disks; %s; error: %s: %s',
util.outline_az_sub(sub_index, sub, tenant),
type(e).__name__, e)
def _get_subscription_apps(self, sub_index, sub):
"""Get web apps from a single subscrption.
Yields:
tuple: A tuple which when unpacked forms valid arguments for
:meth:` _get_web_app_configs`.
"""
try:
tenant = self._tenant
creds = self._credentials
sub_id = sub.get('subscription_id')
web_client = WebSiteManagementClient(creds, sub_id)
web_list = web_client.web_apps.list()
for app_index, app in enumerate(web_list):
app = app.as_dict()
_log.info('Found web app #%d: %s; %s', app_index,
app.get('name'),
util.outline_az_sub(sub_index, sub, tenant))
# Each app is a unit of work.
yield (app_index, app, sub_index, sub)
# Break after pulling data for self._max_recs number
# of web apps for a subscriber. Note that if
# self._max_recs is 0 or less, then the following
# condition never evaluates to True.
if app_index + 1 == self._max_recs:
_log.info(
'Stopping web app fetch due '
'to _max_recs: %d; %s', self._max_recs,
util.outline_az_sub(sub_index, sub, tenant))
break
except Exception as e:
_log.error('Failed to fetch web apps; %s; error: %s: %s',
util.outline_az_sub(sub_index, sub, tenant),
type(e).__name__, e)
def _get_subscription_vms(self, sub_index, sub):
"""Get VMs from a single subscrption.
Yields:
tuple: A tuple which when unpacked forms valid arguments for
:meth:`_get_vm_instance_views`.
"""
try:
tenant = self._tenant
creds = self._credentials
sub_id = sub.get('subscription_id')
compute_client = ComputeManagementClient(creds, sub_id)
vm_list = compute_client.virtual_machines.list_all()
for vm_index, vm in enumerate(vm_list):
vm = vm.as_dict()
_log.info('Found VM #%d: %s; %s', vm_index, vm.get('name'),
util.outline_az_sub(sub_index, sub, tenant))
# Each VM is a unit of work.
yield (vm_index, vm, sub_index, sub)
# Break after pulling data for self._max_recs number
# of VMs for a subscriber. Note that if
# self._max_recs is 0 or less, then the following
# condition never evaluates to True.
if vm_index + 1 == self._max_recs:
_log.info(
'Stopping vm_instance_view fetch due '
'to _max_recs: %d; %s', self._max_recs,
util.outline_az_sub(sub_index, sub, tenant))
break
except Exception as e:
_log.error('Failed to fetch VMs; %s; error: %s: %s',
util.outline_az_sub(sub_index, sub, tenant),
type(e).__name__, e)
def _get_tenant_postgres(self):
"""Get Postgres details from all subscriptions in a tenant.
The yielded tuples when unpacked would become arguments for
:meth:`_get_postgres_server_details`. Each such tuple represents
a single unit of work that :meth:`_get_postgres_details` can
work on independently in its own worker thread.
Yields:
tuple: A tuple which when unpacked forms valid arguments for
:meth:`_get_postgres_server_details`.
"""
try:
tenant = self._tenant
creds = self._credentials
sub_client = SubscriptionClient(creds)
sub_list = sub_client.subscriptions.list()
for sub_index, sub in enumerate(sub_list):
sub = sub.as_dict()
_log.info('Found %s',
util.outline_az_sub(sub_index, sub, tenant))
yield from self._get_subscription_postgres_servers(
sub_index, sub)
# Break after pulling data for self._max_subs number of
# subscriptions. Note that if self._max_subs is 0 or less,
# then the following condition never evaluates to True.
if sub_index + 1 == self._max_subs:
_log.info(
'Stopping subscriptions fetch due to '
'_max_subs: %d; tenant: %s', self._max_subs, tenant)
break
except CloudError as e:
_log.error('Failed to fetch subscriptions; %s; error: %s: %s',
util.outline_az_sub(sub_index, sub, tenant),
type(e).__name__, e)
def _get_storage_account_properties(self, storage_account_index,
storage_account, sub_index, sub):
"""Get storage account records with property details.
Arguments:
storage_account_index (int): Storage account index (logging only).
storage_account (dict): Raw storage account record.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
Yields:
dict: An Azure storage account record with property details.
"""
act_name = storage_account.get('name')
_log.info('Working on storage account #%d: %s; %s',
storage_account_index, act_name,
util.outline_az_sub(sub_index, sub, self._tenant))
try:
creds = self._credentials
sub_id = sub.get('subscription_id')
client = StorageManagementClient(creds, sub_id)
account_id = storage_account.get('id')
rg_name = tools.parse_resource_id(account_id)['resource_group']
properties = client.storage_accounts.get_properties(
rg_name, act_name)
properties = properties.as_dict()
yield _process_storage_account_properties(storage_account_index,
storage_account,
properties, sub_index,
sub, self._tenant)
except Exception as e:
_log.error(
'Failed to fetch properties for storage accounts'
'#%d:%s; %s; error: %s: %s', storage_account_index, act_name,
util.outline_az_sub(sub_index, sub, self._tenant),
type(e).__name__, e)
def _process_storage_account_properties(storage_account_index, storage_account,
storage_account_properties, sub_index,
sub, tenant):
"""Get storage account records with property details.
Arguments:
storage_account_index (int): Storage account index (logging only).
storage_account (dict): Raw storage account record.
storage_account_properties (dict): Storage account properties record.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
tenant (str): Azure tenant ID.
Yields:
dict: An Azure record of type ``storage_account_properties``.
"""
storage_account['properties'] = storage_account_properties
record = {
'raw': storage_account,
'ext': {
'cloud_type':
'azure',
'record_type':
'storage_account_properties',
'secure_transfer_required':
storage_account_properties.get('enable_https_traffic_only'),
'subscription_id':
sub.get('subscription_id'),
'subscription_name':
sub.get('display_name'),
'subscription_state':
sub.get('state'),
},
'com': {
'cloud_type': 'azure',
'reference': storage_account.get('id')
}
}
_log.info('Found storage_account_properties #%d: %s; %s',
storage_account_index, storage_account.get('name'),
util.outline_az_sub(sub_index, sub, tenant))
return record
def _process_vm_instance_view(vm_index, vm, vm_iv, sub_index, sub, tenant):
"""Process virtual machine record and yeild them.
Arguments:
vm_index (int): Virtual machine index (for logging only).
vm (dict): Raw virtual machine record.
vm_iv (dict): Raw virtual machine instance view record.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription object.
tenant (str): Azure tenant ID.
Yields:
dict: An Azure record of type ``vm_instance_view``.
"""
vm['instance_view'] = vm_iv
record = {
'raw': vm,
'ext': {
'cloud_type': 'azure',
'record_type': 'vm_instance_view',
'subscription_id': sub.get('subscription_id'),
'subscription_name': sub.get('display_name'),
'subscription_state': sub.get('state'),
},
'com': {
'cloud_type': 'azure',
'record_type': 'compute',
'reference': vm.get('id')
}
}
record['ext'] = util.merge_dicts(
record['ext'], _get_normalized_vm_statuses(vm_iv),
_get_normalized_vm_disk_encryption_status(vm, vm_iv),
_get_vm_extension_list(vm_iv))
_log.info('Found vm_instance_view #%d: %s; %s', vm_index, vm.get('name'),
util.outline_az_sub(sub_index, sub, tenant))
return record
def _get_record(iterator, attribute_type, max_recs,
sub_index, sub, tenant):
"""Process a list of :class:`msrest.serialization.Model` objects.
Arguments:
iterator: An iterator like instance of
:class:`msrest.serialization.Model` objects.
attribute_type (str): Type of record as per Azure vocabulary.
max_recs (int): Maximum number of records to fetch.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription model object.
tenant (str): Azure tenant ID (for logging only).
Yields:
dict: An Azure record of type ``attribute_type``.
"""
base_record = {
'ext': {
'cloud_type': 'azure',
'record_type': attribute_type,
'subscription_id': sub.get('subscription_id'),
'subscription_name': sub.get('display_name'),
'subscription_state': sub.get('state'),
},
'com': {
'cloud_type': 'azure',
'record_type': attribute_type,
}
}
records_missing = True
for i, v in enumerate(iterator):
raw_record = v.as_dict()
_log.info('Found %s #%d: %s; %s', attribute_type, i,
raw_record.get('name'),
util.outline_az_sub(sub_index, sub, tenant))
retention_policy = raw_record.get('retention_policy')
record = util.merge_dicts(base_record, {
'raw': raw_record,
'ext': {
'cloud_type': 'azure',
'record_type': attribute_type,
'subscription_id': sub.get('subscription_id'),
'subscription_name': sub.get('display_name'),
'subscription_state': sub.get('state'),
'retention_enabled': retention_policy.get('enabled'),
'retention_days': retention_policy.get('days'),
},
'com': {
'reference': raw_record.get('id'),
}
})
if 'locations' in sub:
# Record the locations in which the subscription exists.
record['ext']['subscription_locations'] = sub.get('locations')
if attribute_type == 'log_profile':
# Record the locations in which the log profile is enabled.
record['ext']['locations'] = raw_record.get('locations')
# We have found at least one record, so we set this flag to False.
records_missing = False
yield record
if i + 1 == max_recs:
_log.info('Stopping %s fetch due to _max_recs: %d; %s',
attribute_type, max_recs,
util.outline_az_sub(sub_index, sub, tenant))
break
if records_missing:
_log.info('Missing %s; %s', attribute_type,
util.outline_az_sub(sub_index, sub, tenant))
record = util.merge_dicts(base_record, {
'raw': None,
'ext': {
'record_type': attribute_type + '_missing',
},
'com': {
'record_type': attribute_type + '_missing',
'reference': sub.get('id'),
}
})
yield record
def _get_resource_iterator(record_type, credentials,
sub_index, sub, tenant):
"""Return an appropriate iterator for ``record_type``.
Arguments:
record_type (str): Record type.
credentials (ServicePrincipalCredentials): Credentials.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Subscription object.
tenant (str): Tenant ID (for logging only).
Returns:
msrest.paging.Paged: An Azure paging container for iterating
over a list of Azure resource objects.
"""
sub_id = sub.get('subscription_id')
if record_type == 'virtual_machine':
client = ComputeManagementClient(credentials, sub_id)
return client.virtual_machines.list_all()
if record_type == 'app_gateway':
client = NetworkManagementClient(credentials, sub_id)
return client.application_gateways.list_all()
if record_type == 'lb':
client = NetworkManagementClient(credentials, sub_id)
return client.load_balancers.list_all()
if record_type == 'nic':
client = NetworkManagementClient(credentials, sub_id)
return client.network_interfaces.list_all()
if record_type == 'nsg':
client = NetworkManagementClient(credentials, sub_id)
return client.network_security_groups.list_all()
if record_type == 'public_ip':
client = NetworkManagementClient(credentials, sub_id)
return client.public_ip_addresses.list_all()
if record_type == 'storage_account':
client = StorageManagementClient(credentials, sub_id)
return client.storage_accounts.list()
if record_type == 'resource_group':
client = ResourceManagementClient(credentials, sub_id)
return client.resource_groups.list()
if record_type == 'mysql_server':
client = MySQLManagementClient(credentials, sub_id)
return client.servers.list()
if record_type == 'web_apps':
client = WebSiteManagementClient(credentials, sub_id)
return client.web_apps.list()
# If control reaches here, there is a bug in this plugin. It means
# there is a value in record_types variable in _get_subscriptions
# that is not handled in the above if-statements.
_log.warning('Unrecognized record_type: %s; %s', record_type,
util.outline_az_sub(sub_index, sub, tenant))
return None
def _get_record(iterator, azure_record_type, max_recs,
sub_index, sub, tenant):
"""Process a list of :class:`msrest.serialization.Model` objects.
Arguments:
iterator: An iterator like instance of
:class:`msrest.serialization.Model` objects.
azure_record_type (str): Type of record as per Azure vocabulary.
max_recs (int): Maximum number of records to fetch.
sub_index (int): Subscription index (for logging only).
sub (Subscription): Azure subscription model object.
tenant (str): Azure tenant ID (for logging only).
Yields:
dict: An Azure record of type ``record_type``.
"""
# Dictionary to map Azure record types to common record types.
record_type_map = {
'virtual_machine': 'compute',
'mysql_server': 'rdbms',
}
for i, v in enumerate(iterator):
raw_record = v.as_dict()
record = {
'raw': raw_record,
'ext': {
'cloud_type': 'azure',
'record_type': azure_record_type,
'subscription_id': sub.get('subscription_id'),
'tenant_id': tenant,
'subscription_name': sub.get('display_name'),
'subscription_state': sub.get('state'),
},
'com': {
'cloud_type': 'azure',
'record_type': record_type_map.get(azure_record_type)
}
}
_log.info('Found %s #%d: %s; %s', azure_record_type, i,
raw_record.get('name'),
util.outline_az_sub(sub_index, sub, tenant))
# For every security rule found in an NSG, generate a
# separate security rule (firewall rule) record to maintain
# parity with separate records for separate firewall rules
# in GCP.
if azure_record_type == 'nsg':
yield from _get_normalized_firewall_rules(
record, sub_index, sub, tenant)
if azure_record_type in ['mysql_server']:
yield from _get_normalized_rdbms_record(record)
return
yield record
if i + 1 == max_recs:
_log.info('Stopping %s fetch due to _max_recs: %d; %s',
azure_record_type, max_recs,
util.outline_az_sub(sub_index, sub, tenant))
break
def _get_normalized_firewall_rules(nsg_record, sub_index, sub, tenant):
"""Split a network security group (NSG) into multiple firewall rules.
An Azure NSG record contains a top-level key named
``security_rules`` whose value is a list of security rules.
In order to make it easier to write event plugins to detect security
issues in an NSG, we generate a new firewall rule record for each
security rule found in the NSG.
Arguments:
nsg_record (dict): NSG record generated by this plugin.
sub_index (int): Subscription index (for logging only)
sub (Subscription): Azure subscription object (for logging only)
tenant (str): Azure tenant ID (for logging only)
Yields:
dict: A normalized firewall rule record with ``com`` bucket
populated with firewall rule properties in common notation.
"""
security_rules = nsg_record.get('raw', {}).get('security_rules')
nsg_name = nsg_record.get('raw', {}).get('name')
if security_rules is None:
_log.warning('Found NSG without security_rules; name: %s; %s',
nsg_name, util.outline_az_sub(sub_index, sub, tenant))
return
for i, security_rule in enumerate(security_rules):
record = {
'raw': security_rule,
# Preserve the extended properties from NSG record.
'ext': util.merge_dicts(nsg_record.get('ext'), {
# Set extended properties specific to a security rule.
'record_type': 'security_rule',
'nsg_id': nsg_record.get('raw', {}).get('id'),
'security_rule_id': security_rule.get('id'),
}),
'com': {
'cloud_type': 'azure',
'record_type': 'firewall_rule',
'reference': security_rule.get('id'),
'enabled':
_get_normalized_firewall_state(security_rule),
'direction':
_get_normalized_firewall_direction(security_rule),
'access':
_get_normalized_firewall_access(security_rule),
'source_addresses':
_get_normalized_firewall_source_addresses(security_rule),
'protocol':
_get_normalized_firewall_protocol(security_rule),
'destination_ports':
_get_normalized_firewall_destination_ports(security_rule),
}
}
_log.info('Found security_rule #%d: %s; %s',
i, security_rule.get('name'),
util.outline_az_sub(sub_index, sub, tenant))
yield record
