def _get_profiles(self, attribute_type, sub_index, sub): """Return an Azure monitor record. Arguments: attribute_type (str): Attribute type name. sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. Yields: dict: An Azure monitor record. """ _log.info('Working on %s', util.outline_az_sub(sub_index, sub, self._tenant)) try: monitor_client = \ MonitorManagementClient(self._credentials, sub.get('subscription_id')) iterator = \ _get_attribute_iterator(attribute_type, monitor_client, sub, sub_index, self._tenant) yield from _get_record(iterator, attribute_type, self._max_recs, sub_index, sub, self._tenant) except Exception as e: _log.error('Failed to fetch details for %s; %s; error: %s: %s', attribute_type, util.outline_az_sub(sub_index, sub, self._tenant), type(e).__name__, e)
def _get_web_app_configs(self, app_index, app, sub_index, sub): """Get web app records with config details. Arguments: app_index (int): Web app index (for logging only). app (dict): Raw web app record. sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. Yields: dict: An Azure web app record with config details. """ app_name = app.get('name') _log.info('Working on web app #%d: %s; %s', app_index, app_name, util.outline_az_sub(sub_index, sub, self._tenant)) try: creds = self._credentials sub_id = sub.get('subscription_id') web_client = WebSiteManagementClient(creds, sub_id) app_id = app.get('id') rg_name = tools.parse_resource_id(app_id)['resource_group'] app_config = web_client.web_apps.get_configuration( rg_name, app_name) app_config = app_config.as_dict() yield _process_app_config(app_index, app, app_config, sub_index, sub, self._tenant) except Exception as e: _log.error( 'Failed to fetch app_config for web app #%d: ' '%s; %s; error: %s: %s', app_index, app_name, util.outline_az_sub(sub_index, sub, self._tenant), type(e).__name__, e)
def _get_disk_details(self, disk_index, disk_name, rg_name, sub_index, sub): """Get details of disk. Arguments: sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. rg_name (str): Resource group name. disk_index (int): Disk index (for logging only). disk_name (str): Name of the disk. Yields: dict: An Azure disk record. """ _log.info('Working on disk #%d: %s; %s', disk_index, disk_name, util.outline_az_sub(disk_index, sub, self._tenant)) try: sub_id = sub.get('subscription_id') creds = self._credentials compute_client = ComputeManagementClient(creds, sub_id) disk = compute_client.disks.get(rg_name, disk_name) disk = disk.as_dict() yield _process_disk_details(sub, disk) except Exception as e: _log.error( 'Failed to fetch disk details #%d: ' '%s; %s; error: %s: %s', disk_index, disk_name, util.outline_az_sub(sub_index, sub, self._tenant), type(e).__name__, e)
def _get_vm_instance_views(self, vm_index, vm, sub_index, sub): """Get virtual machine records with instance view details. Arguments: vm_index (int): Virtual machine index (for logging only). vm (dict): Raw virtual machine record. sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. Yields: dict: An Azure virtual machine record with instance view details. """ vm_name = vm.get('name') _log.info('Working on VM #%d: %s; %s', vm_index, vm_name, util.outline_az_sub(sub_index, sub, self._tenant)) try: creds = self._credentials sub_id = sub.get('subscription_id') compute_client = ComputeManagementClient(creds, sub_id) vm_id = vm.get('id') rg_name = tools.parse_resource_id(vm_id)['resource_group'] vm_iv = compute_client.virtual_machines.instance_view( rg_name, vm_name) vm_iv = vm_iv.as_dict() yield _process_vm_instance_view(vm_index, vm, vm_iv, sub_index, sub, self._tenant) except Exception as e: _log.error( 'Failed to fetch vm_instance_view for VM #%d: ' '%s; %s; error: %s: %s', vm_index, vm_name, util.outline_az_sub(sub_index, sub, self._tenant), type(e).__name__, e)
def _get_subscription_storage_accounts(self, sub_index, sub): """Get storage accounts from a single subscrption. Yields: tuple: A tuple which when unpacked forms valid arguments for :meth:` _get_storage_account_properties`. """ try: tenant = self._tenant creds = self._credentials sub_id = sub.get('subscription_id') client = StorageManagementClient(creds, sub_id) storage_account_list = client.storage_accounts.list() for t in enumerate(storage_account_list): (storage_account_index, storage_account) = t storage_account = storage_account.as_dict() _log.info('Found storage account #%d: %s; %s', storage_account_index, storage_account.get('name'), util.outline_az_sub(sub_index, sub, tenant)) yield (storage_account_index, storage_account, sub_index, sub) if storage_account_index + 1 == self._max_recs: _log.info( 'Stopping storage accounts fetch due ' 'to _max_recs: %d; %s', self._max_recs, util.outline_az_sub(sub_index, sub, tenant)) break except Exception as e: _log.error('Failed to fetch storage accounts; %s; error: %s: %s', util.outline_az_sub(sub_index, sub, tenant), type(e).__name__, e)
def _process_key_vault(self, key_vault_index, key_vault_name, rg_name, sub_index, sub): """Get details of Key Vault in management and data plane. Arguments: sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. rg_name (str): Resource group name. key_vault_index (int): Key Vault index (for logging only). key_vault_name (str): Name of the Key Vault. Yields: dict: An Azure Key Vault server record. """ def _auth_callback(server, resource, scope): credentials = self._key_vault_credentials token = credentials.token return token['token_type'], token['access_token'] _log.info('Working on key_vault #%d: %s; %s', key_vault_index, key_vault_name, util.outline_az_sub(sub_index, sub, self._tenant)) subscription_id = sub.get('subscription_id') creds = self._credentials key_vault_mgmt_client = KeyVaultManagementClient( creds, subscription_id) key_vault_details = key_vault_mgmt_client.vaults.get( rg_name, key_vault_name) yield from _get_normalized_key_vault_record(key_vault_details, sub) try: kv_client = \ KeyVaultClient(KeyVaultAuthentication(_auth_callback)) secrets = \ kv_client.get_secrets(key_vault_details.properties.vault_uri) keys = \ kv_client.get_keys(key_vault_details.properties.vault_uri) # Retrieve data using each iterator. for record in itertools.chain( _get_data_record(secrets, 'key_vault_secret', sub_index, sub, self._tenant), _get_data_record(keys, 'key_vault_key', sub_index, sub, self._tenant), ): yield record except CloudError as e: _log.error('Failed to fetch key vault details; %s; error: %s: %s', util.outline_az_sub(sub_index, sub, self._tenant), type(e).__name__, e)
def _get_data_record(iterator, azure_record_type, sub_index, sub, tenant): """Normalize the Key Vault data plane record. Arguments: iterator: An iterator like instance of :class:`msrest.serialization.Model` objects. azure_record_type (str): Record type name. sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. tenant (str): Azure tenant ID (for logging only). Returns: dict: Normalized Key Vault data plane record. """ try: for i, v in enumerate(iterator): raw_record = v.as_dict() reference = raw_record.get('id') if azure_record_type == 'key_vault_key': reference = raw_record.get('kid') expiry_set = raw_record['attributes'].get('expires') is not None enabled = raw_record['attributes'].get('enabled') record = { 'raw': raw_record, 'ext': { 'cloud_type': 'azure', 'expiry_set': expiry_set, 'enabled': enabled, 'record_type': azure_record_type, 'reference': reference, 'subscription_id': sub.get('subscription_id'), 'subscription_name': sub.get('display_name'), 'subscription_state': sub.get('state'), }, 'com': { 'cloud_type': 'azure', 'record_type': azure_record_type, 'reference': reference, } } _log.info('Found %s #%d: %s; %s', azure_record_type, i, reference, util.outline_az_sub(sub_index, sub, tenant)) yield record except KeyVaultErrorException as e: _log.error('Failed to fetch details for %s; %s; error: %s: %s', azure_record_type, util.outline_az_sub(sub_index, sub, tenant), type(e).__name__, e)
def _get_subscriptions(self): """Generate tuples of record types and subscriptions. The yielded tuples when unpacked would become arguments for :meth:`_get_resources`. Each such tuple represents a single unit of work that :meth:`_get_resources` can work on independently in its own worker thread. Yields: tuple: A tuple which when unpacked forms valid arguments for :meth:`_get_resources`. """ try: sub_client = SubscriptionClient(self._credentials) sub_list = sub_client.subscriptions.list() monitor_attributes = ('log_profile',) tenant = self._tenant for sub_index, sub in enumerate(sub_list): sub = sub.as_dict() _log.info('Found %s', util.outline_az_sub(sub_index, sub, tenant)) # Each record type for each subscription is a unit of # work that would be fed to _get_resources(). for attribute_type in monitor_attributes: if attribute_type == 'log_profile': sub['locations'] = [] locations = sub_client.subscriptions. \ list_locations(sub.get('subscription_id')) for location in locations: sub['locations'].append(location.as_dict() .get('name')) yield (attribute_type, sub_index, sub) # Break after pulling data for self._max_subs number of # subscriptions. Note that if self._max_subs is 0 or less, # then the following condition never evaluates to True. if sub_index + 1 == self._max_subs: _log.info('Stopping subscriptions fetch due to ' '_max_subs: %d; tenant: %s', self._max_subs, self._tenant) break except Exception as e: _log.error('Failed to fetch subscriptions; %s; error: %s: %s', util.outline_az_sub(sub_index, sub, tenant), type(e).__name__, e)
def _get_attribute_iterator(attribute_type, monitor_client, sub, sub_index, tenant): """Return an appropriate iterator for ``attribute_type``. Arguments: attribute_type (str): Attribute type. monitor_client(MonitorManagementClient): Monitor client. credentials (ServicePrincipalCredentials): Credentials. sub_index (int): Subscription index (for logging only). sub (Subscription): Subscription object. tenant (str): Tenant ID (for logging only). Returns: msrest.paging.Paged: An Azure paging container for iterating over a list of Azure resource objects. """ if attribute_type == 'log_profile': return monitor_client.log_profiles.list() # If control reaches here, there is a bug in this plugin. It means # there is a value in attributes variable in _get_subscriptions # that is not handled in the above if-statements. _log.warning('Unrecognized profile_type: %s; %s', attribute_type, util.outline_az_sub(sub_index, sub, tenant)) return None
def _process_app_config(app_index, app, app_config, sub_index, sub, tenant): """Process web app record and yield them. Arguments: app_index (int): Web app index (for logging only). app (dict): Raw web app record. app_config (dict): Raw web app config record. sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. tenant (str): Azure tenant ID. Yields: dict: An Azure record of type ``web_app_config``. """ app['config'] = app_config record = { 'raw': app, 'ext': { 'cloud_type': 'azure', 'record_type': 'web_app_config', 'min_tls_version': app_config.get('min_tls_version'), 'subscription_id': sub.get('subscription_id'), 'subscription_name': sub.get('display_name'), 'subscription_state': sub.get('state'), }, 'com': { 'cloud_type': 'azure', 'reference': app.get('id') } } _log.info('Found web_app_config #%d: %s; %s', app_index, app.get('name'), util.outline_az_sub(sub_index, sub, tenant)) return record
def _get_postgres_server_details(self, server_index, server_name, rg_name, sub_index, sub): """Get details Postgres server. Arguments: sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. rg_name (str): Resource group name. server_index (int): Server index (for logging only). server_name (str): Name of the Postgres server. Yields: dict: An Azure Postgres server record with configuration. """ _log.info('Working on Postgres server #%d: %s; %s', server_index, server_name, util.outline_az_sub(sub_index, sub, self._tenant)) sub_id = sub.get('subscription_id') creds = self._credentials postgres_client = PostgreSQLManagementClient(creds, sub_id) server_details = postgres_client.servers.get(rg_name, server_name) server_details = server_details.as_dict() server_configuration_list = \ postgres_client.configurations.list_by_server(rg_name, server_name) configurations, derived_configs = \ self._get_postgres_server_configuration(server_configuration_list) yield from self._process_postgres_server_details( sub, server_details, configurations, derived_configs)
def _get_resources(self, record_type, sub_index, sub): """Return an Azure cloud infrastructure configuration record. Arguments: record_type (str): Record type name. sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. Yields: dict: An Azure cloud infrastructure configuration record. """ _log.info('Working on %s list; %s', record_type, util.outline_az_sub(sub_index, sub, self._tenant)) if record_type == 'subscription': record = { 'raw': sub, 'ext': { 'cloud_type': 'azure', 'record_type': record_type, 'subscription_id': sub.get('subscription_id'), 'tenant_id': self._tenant, 'subscription_name': sub.get('display_name'), 'subscription_state': sub.get('state'), }, 'com': { 'cloud_type': 'azure', 'record_type': 'subscription' } } yield record return try: iterator = \ _get_resource_iterator(record_type, self._credentials, sub_index, sub, self._tenant) yield from _get_record(iterator, record_type, self._max_recs, sub_index, sub, self._tenant) except Exception as e: _log.error('Failed to fetch details for %s; %s; error: %s: %s', record_type, util.outline_az_sub(sub_index, sub, self._tenant), type(e).__name__, e)
def _get_subscription_kvs(self, sub_index, sub): """Get Key Vaults from a single subscrption. Arguments: sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. Yields: tuple: A tuple which when unpacked forms valid arguments for :meth:`_process_key_vault`. """ try: tenant = self._tenant creds = self._credentials subscription_id = sub.get('subscription_id') key_vault_mgmt_client = KeyVaultManagementClient( creds, subscription_id) key_vault_list = key_vault_mgmt_client.vaults.list() for key_vault_index, key_vault in enumerate(key_vault_list): key_vault = key_vault.as_dict() key_vault_name = key_vault.get('name') key_vault_id = key_vault.get('id') _log.info('Found key_vault #%d: %s; %s', key_vault_index, key_vault_name, util.outline_az_sub(sub_index, sub, tenant)) rg_name = \ tools.parse_resource_id(key_vault_id)['resource_group'] yield (key_vault_index, key_vault_name, rg_name, sub_index, sub) # Break after pulling data for self._max_recs number # of Key Vault for a subscriber. Note that if # self._max_recs is 0 or less, then the following # condition never evaluates to True. if key_vault_index + 1 == self._max_recs: _log.info( 'Stopping Key Vault fetch due ' 'to _max_recs: %d; %s', self._max_recs, util.outline_az_sub(sub_index, sub, self._tenant)) break except CloudError as e: _log.error('Failed to fetch Key Vault; %s; error: %s: %s', util.outline_az_sub(sub_index, sub, tenant), type(e).__name__, e)
def _get_subscriptions(self): """Generate tuples of record types and subscriptions. The yielded tuples when unpacked would become arguments for :meth:`_get_resources`. Each such tuple represents a single unit of work that :meth:`_get_resources` can work on independently in its own worker thread. Yields: tuple: A tuple which when unpacked forms valid arguments for :meth:`_get_resources`. """ try: sub_client = SubscriptionClient(self._credentials) sub_list = sub_client.subscriptions.list() record_types = ('virtual_machine', 'app_gateway', 'lb', 'nic', 'nsg', 'public_ip', 'storage_account', 'resource_group', 'mysql_server', 'web_apps', 'subscription') tenant = self._tenant for sub_index, sub in enumerate(sub_list): sub = sub.as_dict() _log.info('Found %s', util.outline_az_sub(sub_index, sub, tenant)) # Each record type for each subscription is a unit of # work that would be fed to _get_resources(). for record_type in record_types: yield (record_type, sub_index, sub) # Break after pulling data for self._max_subs number of # subscriptions. Note that if self._max_subs is 0 or less, # then the following condition never evaluates to True. if sub_index + 1 == self._max_subs: _log.info('Stopping subscriptions fetch due to ' '_max_subs: %d; tenant: %s', self._max_subs, self._tenant) break except Exception as e: _log.error('Failed to fetch subscriptions; %s; error: %s: %s', util.outline_az_sub(sub_index, sub, tenant), type(e).__name__, e)
def _process_storage_account_properties(storage_account_index, storage_account, storage_account_properties, sub_index, sub, tenant): """Get storage account records with property details. Arguments: storage_account_index (int): Storage account index (logging only). storage_account (dict): Raw storage account record. storage_account_properties (dict): Storage account properties record. sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. tenant (str): Azure tenant ID. Yields: dict: An Azure record of type ``storage_account_properties``. """ storage_account['properties'] = storage_account_properties default_network_access_allowed = True if storage_account['network_rule_set'].get('default_action') != 'Allow': default_network_access_allowed = False # Azure services is an umbrella term for trusted Microsoft Azure services # including Azure Backup, Azure Site Recovery, Azure DevTest Labs, # Azure Event Grid, Azure Event Hubs, Azure Networking, Azure Monitor and # Azure SQL Data Warehouse bypass_trusted_services = True if storage_account['network_rule_set'].get('bypass') != 'AzureServices': bypass_trusted_services = False record = { 'raw': storage_account, 'ext': { 'cloud_type': 'azure', 'record_type': 'storage_account_properties', 'secure_transfer_required': storage_account_properties.get('enable_https_traffic_only'), 'default_network_access_allowed': default_network_access_allowed, 'trusted_services_allowed': bypass_trusted_services, 'subscription_id': sub.get('subscription_id'), 'subscription_name': sub.get('display_name'), 'subscription_state': sub.get('state'), }, 'com': { 'cloud_type': 'azure', 'reference': storage_account.get('id') } } _log.info('Found storage_account_properties #%d: %s; %s', storage_account_index, storage_account.get('name'), util.outline_az_sub(sub_index, sub, tenant)) return record
def _get_subscription_postgres_servers(self, sub_index, sub): """Get Postgres servers from a single subscrption. Arguments: sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. Yields: tuple: A tuple which when unpacked forms valid arguments for :meth:`_get_postgres_details`. """ try: tenant = self._tenant creds = self._credentials sub_id = sub.get('subscription_id') postgres_client = PostgreSQLManagementClient(creds, sub_id) db_server_list = postgres_client.servers.list() for server_index, postgres_server in enumerate(db_server_list): postgres_server = postgres_server.as_dict() server_id = postgres_server.get('id') server_name = postgres_server.get('name') _log.info('Found Postgres Server #%d: %s; %s', server_index, server_name, util.outline_az_sub(sub_index, sub, tenant)) rg_name = \ tools.parse_resource_id(server_id)['resource_group'] yield (server_index, server_name, rg_name, sub_index, sub) # Break after pulling data for self._max_recs number # of Postgres servers for a subscriber. Note that if # self._max_recs is 0 or less, then the following # condition never evaluates to True. if server_index + 1 == self._max_recs: _log.info( 'Stopping Postgres server fetch due ' 'to _max_recs: %d; %s', self._max_recs, util.outline_az_sub(sub_index, sub, self._tenant)) break except CloudError as e: _log.error('Failed to fetch Postgres servers; %s; error: %s: %s', util.outline_az_sub(sub_index, sub, tenant), type(e).__name__, e)
def _get_subscription_disks(self, sub_index, sub): """Get disks from a single subscrption. Arguments: sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. Yields: tuple: A tuple which when unpacked forms valid arguments for :meth:`_get_disk_details`. """ try: tenant = self._tenant creds = self._credentials sub_id = sub.get('subscription_id') compute_client = ComputeManagementClient(creds, sub_id) disk_list = compute_client.disks.list() for disk_index, disk in enumerate(disk_list): disk = disk.as_dict() disk_id = disk.get('id') disk_name = disk.get('name') _log.info('Found disk #%d: %s; %s', disk_index, disk_name, util.outline_az_sub(sub_index, sub, tenant)) rg_name = \ tools.parse_resource_id(disk_id)['resource_group'] yield (disk_index, disk_name, rg_name, sub_index, sub) # Break after pulling data for self._max_recs number # of disks for a subscriber. Note that if # self._max_recs is 0 or less, then the following # condition never evaluates to True. if disk_index + 1 == self._max_recs: _log.info( 'Stopping disk fetch due ' 'to _max_recs: %d; %s', self._max_recs, util.outline_az_sub(sub_index, sub, self._tenant)) break except Exception as e: _log.error('Failed to fetch disks; %s; error: %s: %s', util.outline_az_sub(sub_index, sub, tenant), type(e).__name__, e)
def _get_subscription_apps(self, sub_index, sub): """Get web apps from a single subscrption. Yields: tuple: A tuple which when unpacked forms valid arguments for :meth:` _get_web_app_configs`. """ try: tenant = self._tenant creds = self._credentials sub_id = sub.get('subscription_id') web_client = WebSiteManagementClient(creds, sub_id) web_list = web_client.web_apps.list() for app_index, app in enumerate(web_list): app = app.as_dict() _log.info('Found web app #%d: %s; %s', app_index, app.get('name'), util.outline_az_sub(sub_index, sub, tenant)) # Each app is a unit of work. yield (app_index, app, sub_index, sub) # Break after pulling data for self._max_recs number # of web apps for a subscriber. Note that if # self._max_recs is 0 or less, then the following # condition never evaluates to True. if app_index + 1 == self._max_recs: _log.info( 'Stopping web app fetch due ' 'to _max_recs: %d; %s', self._max_recs, util.outline_az_sub(sub_index, sub, tenant)) break except Exception as e: _log.error('Failed to fetch web apps; %s; error: %s: %s', util.outline_az_sub(sub_index, sub, tenant), type(e).__name__, e)
def _get_subscription_vms(self, sub_index, sub): """Get VMs from a single subscrption. Yields: tuple: A tuple which when unpacked forms valid arguments for :meth:`_get_vm_instance_views`. """ try: tenant = self._tenant creds = self._credentials sub_id = sub.get('subscription_id') compute_client = ComputeManagementClient(creds, sub_id) vm_list = compute_client.virtual_machines.list_all() for vm_index, vm in enumerate(vm_list): vm = vm.as_dict() _log.info('Found VM #%d: %s; %s', vm_index, vm.get('name'), util.outline_az_sub(sub_index, sub, tenant)) # Each VM is a unit of work. yield (vm_index, vm, sub_index, sub) # Break after pulling data for self._max_recs number # of VMs for a subscriber. Note that if # self._max_recs is 0 or less, then the following # condition never evaluates to True. if vm_index + 1 == self._max_recs: _log.info( 'Stopping vm_instance_view fetch due ' 'to _max_recs: %d; %s', self._max_recs, util.outline_az_sub(sub_index, sub, tenant)) break except Exception as e: _log.error('Failed to fetch VMs; %s; error: %s: %s', util.outline_az_sub(sub_index, sub, tenant), type(e).__name__, e)
def _get_tenant_postgres(self): """Get Postgres details from all subscriptions in a tenant. The yielded tuples when unpacked would become arguments for :meth:`_get_postgres_server_details`. Each such tuple represents a single unit of work that :meth:`_get_postgres_details` can work on independently in its own worker thread. Yields: tuple: A tuple which when unpacked forms valid arguments for :meth:`_get_postgres_server_details`. """ try: tenant = self._tenant creds = self._credentials sub_client = SubscriptionClient(creds) sub_list = sub_client.subscriptions.list() for sub_index, sub in enumerate(sub_list): sub = sub.as_dict() _log.info('Found %s', util.outline_az_sub(sub_index, sub, tenant)) yield from self._get_subscription_postgres_servers( sub_index, sub) # Break after pulling data for self._max_subs number of # subscriptions. Note that if self._max_subs is 0 or less, # then the following condition never evaluates to True. if sub_index + 1 == self._max_subs: _log.info( 'Stopping subscriptions fetch due to ' '_max_subs: %d; tenant: %s', self._max_subs, tenant) break except CloudError as e: _log.error('Failed to fetch subscriptions; %s; error: %s: %s', util.outline_az_sub(sub_index, sub, tenant), type(e).__name__, e)
def _get_storage_account_properties(self, storage_account_index, storage_account, sub_index, sub): """Get storage account records with property details. Arguments: storage_account_index (int): Storage account index (logging only). storage_account (dict): Raw storage account record. sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. Yields: dict: An Azure storage account record with property details. """ act_name = storage_account.get('name') _log.info('Working on storage account #%d: %s; %s', storage_account_index, act_name, util.outline_az_sub(sub_index, sub, self._tenant)) try: creds = self._credentials sub_id = sub.get('subscription_id') client = StorageManagementClient(creds, sub_id) account_id = storage_account.get('id') rg_name = tools.parse_resource_id(account_id)['resource_group'] properties = client.storage_accounts.get_properties( rg_name, act_name) properties = properties.as_dict() yield _process_storage_account_properties(storage_account_index, storage_account, properties, sub_index, sub, self._tenant) except Exception as e: _log.error( 'Failed to fetch properties for storage accounts' '#%d:%s; %s; error: %s: %s', storage_account_index, act_name, util.outline_az_sub(sub_index, sub, self._tenant), type(e).__name__, e)
def _process_storage_account_properties(storage_account_index, storage_account, storage_account_properties, sub_index, sub, tenant): """Get storage account records with property details. Arguments: storage_account_index (int): Storage account index (logging only). storage_account (dict): Raw storage account record. storage_account_properties (dict): Storage account properties record. sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. tenant (str): Azure tenant ID. Yields: dict: An Azure record of type ``storage_account_properties``. """ storage_account['properties'] = storage_account_properties record = { 'raw': storage_account, 'ext': { 'cloud_type': 'azure', 'record_type': 'storage_account_properties', 'secure_transfer_required': storage_account_properties.get('enable_https_traffic_only'), 'subscription_id': sub.get('subscription_id'), 'subscription_name': sub.get('display_name'), 'subscription_state': sub.get('state'), }, 'com': { 'cloud_type': 'azure', 'reference': storage_account.get('id') } } _log.info('Found storage_account_properties #%d: %s; %s', storage_account_index, storage_account.get('name'), util.outline_az_sub(sub_index, sub, tenant)) return record
def _process_vm_instance_view(vm_index, vm, vm_iv, sub_index, sub, tenant): """Process virtual machine record and yeild them. Arguments: vm_index (int): Virtual machine index (for logging only). vm (dict): Raw virtual machine record. vm_iv (dict): Raw virtual machine instance view record. sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription object. tenant (str): Azure tenant ID. Yields: dict: An Azure record of type ``vm_instance_view``. """ vm['instance_view'] = vm_iv record = { 'raw': vm, 'ext': { 'cloud_type': 'azure', 'record_type': 'vm_instance_view', 'subscription_id': sub.get('subscription_id'), 'subscription_name': sub.get('display_name'), 'subscription_state': sub.get('state'), }, 'com': { 'cloud_type': 'azure', 'record_type': 'compute', 'reference': vm.get('id') } } record['ext'] = util.merge_dicts( record['ext'], _get_normalized_vm_statuses(vm_iv), _get_normalized_vm_disk_encryption_status(vm, vm_iv), _get_vm_extension_list(vm_iv)) _log.info('Found vm_instance_view #%d: %s; %s', vm_index, vm.get('name'), util.outline_az_sub(sub_index, sub, tenant)) return record
def _get_record(iterator, attribute_type, max_recs, sub_index, sub, tenant): """Process a list of :class:`msrest.serialization.Model` objects. Arguments: iterator: An iterator like instance of :class:`msrest.serialization.Model` objects. attribute_type (str): Type of record as per Azure vocabulary. max_recs (int): Maximum number of records to fetch. sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription model object. tenant (str): Azure tenant ID (for logging only). Yields: dict: An Azure record of type ``attribute_type``. """ base_record = { 'ext': { 'cloud_type': 'azure', 'record_type': attribute_type, 'subscription_id': sub.get('subscription_id'), 'subscription_name': sub.get('display_name'), 'subscription_state': sub.get('state'), }, 'com': { 'cloud_type': 'azure', 'record_type': attribute_type, } } records_missing = True for i, v in enumerate(iterator): raw_record = v.as_dict() _log.info('Found %s #%d: %s; %s', attribute_type, i, raw_record.get('name'), util.outline_az_sub(sub_index, sub, tenant)) retention_policy = raw_record.get('retention_policy') record = util.merge_dicts(base_record, { 'raw': raw_record, 'ext': { 'cloud_type': 'azure', 'record_type': attribute_type, 'subscription_id': sub.get('subscription_id'), 'subscription_name': sub.get('display_name'), 'subscription_state': sub.get('state'), 'retention_enabled': retention_policy.get('enabled'), 'retention_days': retention_policy.get('days'), }, 'com': { 'reference': raw_record.get('id'), } }) if 'locations' in sub: # Record the locations in which the subscription exists. record['ext']['subscription_locations'] = sub.get('locations') if attribute_type == 'log_profile': # Record the locations in which the log profile is enabled. record['ext']['locations'] = raw_record.get('locations') # We have found at least one record, so we set this flag to False. records_missing = False yield record if i + 1 == max_recs: _log.info('Stopping %s fetch due to _max_recs: %d; %s', attribute_type, max_recs, util.outline_az_sub(sub_index, sub, tenant)) break if records_missing: _log.info('Missing %s; %s', attribute_type, util.outline_az_sub(sub_index, sub, tenant)) record = util.merge_dicts(base_record, { 'raw': None, 'ext': { 'record_type': attribute_type + '_missing', }, 'com': { 'record_type': attribute_type + '_missing', 'reference': sub.get('id'), } }) yield record
def _get_resource_iterator(record_type, credentials, sub_index, sub, tenant): """Return an appropriate iterator for ``record_type``. Arguments: record_type (str): Record type. credentials (ServicePrincipalCredentials): Credentials. sub_index (int): Subscription index (for logging only). sub (Subscription): Subscription object. tenant (str): Tenant ID (for logging only). Returns: msrest.paging.Paged: An Azure paging container for iterating over a list of Azure resource objects. """ sub_id = sub.get('subscription_id') if record_type == 'virtual_machine': client = ComputeManagementClient(credentials, sub_id) return client.virtual_machines.list_all() if record_type == 'app_gateway': client = NetworkManagementClient(credentials, sub_id) return client.application_gateways.list_all() if record_type == 'lb': client = NetworkManagementClient(credentials, sub_id) return client.load_balancers.list_all() if record_type == 'nic': client = NetworkManagementClient(credentials, sub_id) return client.network_interfaces.list_all() if record_type == 'nsg': client = NetworkManagementClient(credentials, sub_id) return client.network_security_groups.list_all() if record_type == 'public_ip': client = NetworkManagementClient(credentials, sub_id) return client.public_ip_addresses.list_all() if record_type == 'storage_account': client = StorageManagementClient(credentials, sub_id) return client.storage_accounts.list() if record_type == 'resource_group': client = ResourceManagementClient(credentials, sub_id) return client.resource_groups.list() if record_type == 'mysql_server': client = MySQLManagementClient(credentials, sub_id) return client.servers.list() if record_type == 'web_apps': client = WebSiteManagementClient(credentials, sub_id) return client.web_apps.list() # If control reaches here, there is a bug in this plugin. It means # there is a value in record_types variable in _get_subscriptions # that is not handled in the above if-statements. _log.warning('Unrecognized record_type: %s; %s', record_type, util.outline_az_sub(sub_index, sub, tenant)) return None
def _get_record(iterator, azure_record_type, max_recs, sub_index, sub, tenant): """Process a list of :class:`msrest.serialization.Model` objects. Arguments: iterator: An iterator like instance of :class:`msrest.serialization.Model` objects. azure_record_type (str): Type of record as per Azure vocabulary. max_recs (int): Maximum number of records to fetch. sub_index (int): Subscription index (for logging only). sub (Subscription): Azure subscription model object. tenant (str): Azure tenant ID (for logging only). Yields: dict: An Azure record of type ``record_type``. """ # Dictionary to map Azure record types to common record types. record_type_map = { 'virtual_machine': 'compute', 'mysql_server': 'rdbms', } for i, v in enumerate(iterator): raw_record = v.as_dict() record = { 'raw': raw_record, 'ext': { 'cloud_type': 'azure', 'record_type': azure_record_type, 'subscription_id': sub.get('subscription_id'), 'tenant_id': tenant, 'subscription_name': sub.get('display_name'), 'subscription_state': sub.get('state'), }, 'com': { 'cloud_type': 'azure', 'record_type': record_type_map.get(azure_record_type) } } _log.info('Found %s #%d: %s; %s', azure_record_type, i, raw_record.get('name'), util.outline_az_sub(sub_index, sub, tenant)) # For every security rule found in an NSG, generate a # separate security rule (firewall rule) record to maintain # parity with separate records for separate firewall rules # in GCP. if azure_record_type == 'nsg': yield from _get_normalized_firewall_rules( record, sub_index, sub, tenant) if azure_record_type in ['mysql_server']: yield from _get_normalized_rdbms_record(record) return yield record if i + 1 == max_recs: _log.info('Stopping %s fetch due to _max_recs: %d; %s', azure_record_type, max_recs, util.outline_az_sub(sub_index, sub, tenant)) break
def _get_normalized_firewall_rules(nsg_record, sub_index, sub, tenant): """Split a network security group (NSG) into multiple firewall rules. An Azure NSG record contains a top-level key named ``security_rules`` whose value is a list of security rules. In order to make it easier to write event plugins to detect security issues in an NSG, we generate a new firewall rule record for each security rule found in the NSG. Arguments: nsg_record (dict): NSG record generated by this plugin. sub_index (int): Subscription index (for logging only) sub (Subscription): Azure subscription object (for logging only) tenant (str): Azure tenant ID (for logging only) Yields: dict: A normalized firewall rule record with ``com`` bucket populated with firewall rule properties in common notation. """ security_rules = nsg_record.get('raw', {}).get('security_rules') nsg_name = nsg_record.get('raw', {}).get('name') if security_rules is None: _log.warning('Found NSG without security_rules; name: %s; %s', nsg_name, util.outline_az_sub(sub_index, sub, tenant)) return for i, security_rule in enumerate(security_rules): record = { 'raw': security_rule, # Preserve the extended properties from NSG record. 'ext': util.merge_dicts(nsg_record.get('ext'), { # Set extended properties specific to a security rule. 'record_type': 'security_rule', 'nsg_id': nsg_record.get('raw', {}).get('id'), 'security_rule_id': security_rule.get('id'), }), 'com': { 'cloud_type': 'azure', 'record_type': 'firewall_rule', 'reference': security_rule.get('id'), 'enabled': _get_normalized_firewall_state(security_rule), 'direction': _get_normalized_firewall_direction(security_rule), 'access': _get_normalized_firewall_access(security_rule), 'source_addresses': _get_normalized_firewall_source_addresses(security_rule), 'protocol': _get_normalized_firewall_protocol(security_rule), 'destination_ports': _get_normalized_firewall_destination_ports(security_rule), } } _log.info('Found security_rule #%d: %s; %s', i, security_rule.get('name'), util.outline_az_sub(sub_index, sub, tenant)) yield record