def set_product(self, pricing_mgr, service_code): # print(f'--- START @@@ {service_code} @@@') for product in pricing_mgr.list_products(service_code): # print(product.get('product', {}).get('sku')) try: product_dic = { 'service_code': product.get('serviceCode'), 'region_name': self.get_region_name_from_location( product.get('product', {}).get('attributes', {}).get('location', '')), 'product_family': product.get('product', {}).get('productFamily'), 'sku': product.get('product', {}).get('sku'), # 'attributes': product.get('product', {}).get('attributes'), 'attributes': self.set_product_attributes(product.get('product', {}).get('attributes')), 'publication_date': product.get('publicationDate'), 'version': product.get('version'), 'terms': self.get_terms(product.get('terms', {})) } product_data = Product(product_dic, strict=False) product_resource = ProductResource({ 'data': product_data, 'region_code': product_dic['region_name'], 'reference': ReferenceModel(product_data.reference()) }) # print(f'##### {product_data.get("service_code")}: {product_data.get("sku")}') yield ProductResponse({ 'resource': product_resource }) except Exception as e: print(f"[[ ERROR ]] {e}")
def set_database_resource(vm_cluster_list, region): result = [] backup_list = [] for vm_cluster in vm_cluster_list: db_list = vm_cluster.get('list_database', []) db_backup = vm_cluster.get('list_backup', []) for db in db_list: db_data = Database(db, strict=False) db_resource = ExadataDatabaseResource({ 'data': db_data, 'region_code': region, 'reference': ReferenceModel(db_data.reference()), 'tags': db.get('_freeform_tags', []), 'name': db_data.db_name }) result.append( ExadataDatabaseResponse({'resource': db_resource})) backup_list.extend(db_backup) return result, backup_list
def collect_power_state(self, params): print("** Auto Scaling Start **") start_time = time.time() auto_scaling_conn: AutoScalingConnector = self.locator.get_connector( self.connector_name, **params) ec2_conn: EC2Connector = self.locator.get_connector( 'EC2Connector', **params) auto_scaling_resources = [] for region_name in params.get('regions', []): # print(f'[ AutoScaling {region_name} ]') auto_scaling_conn.set_client(region_name) ec2_conn.set_client(region_name) for asg in auto_scaling_conn.describe_auto_scaling_groups(): asg.update({ 'instances': self.get_asg_instances(asg.get('Instances', []), ec2_conn), }) auto_scaling_data = AutoScalingGroup(asg, strict=False) auto_scaling_resource = AutoScalingResource({ 'data': auto_scaling_data, 'reference': ReferenceModel(auto_scaling_data.reference()) }) auto_scaling_resources.append( AutoScalingResponse({'resource': auto_scaling_resource})) print(f' Auto Scaling Finished {time.time() - start_time} Seconds') return auto_scaling_resources
def set_image_resource(self, image_list, region, compartment): result = [] for image in image_list: image = self.convert_dictionary(image) image.update({ 'region': region, 'compartment_name': compartment.name, '_freeform_tags': self.convert_tags(image.get('_freeform_tags')) }) image_data = DatabaseSoftwareImage(image, strict=False) image_resource = ExadataDatabaseSoftwareImageResource({ 'data': image_data, 'region_code': region, 'reference': ReferenceModel(image_data.reference()), 'tags': image.get('_freeform_tags', []), 'name': image_data.display_name }) result.append( ExadataDatabaseSoftwareImageResponse( {'resource': image_resource})) return result
def collect_cloud_services(self, params): print("** Trusted Advisor Start **") start_time = time.time() ta_conn: TrustedAdvisorConnector = self.locator.get_connector(self.connector_name, **params) ta_conn.set_client() language = params.get('options', {}).get('language', DEFAULT_LANGUAGE) need_refresh = params.get('options', {}).get('refresh', DEFAULT_REFRESH) ta_resources = [] for check in ta_conn.describe_trusted_advisor_checks(language): check_id_data = CheckId(check, strict=False) check_id = check['id'] if need_refresh: ta_conn.refresh_trusted_advisor_check(check_id) check_result = ta_conn.describe_trusted_advisor_check_result(check_id, language) if not check_result: # Nothing to do continue # if status != "ok", there exists flagged resources # Processing flagged resource for easy viewing(a.k.a. TableDynamic) # if raw['status'] != "ok": # print(checkId.metadata) # print(raw) # import time # time.sleep(5) # raw.update({'headers': ['a','b','c'], 'flagged_resources': [[1,2,3], [4,5,6]]}) # Change 1.1 # { 'flagged_resources': [{'a': 1, 'b': 2, 'c': 3}, {'a': 4, 'b': 5, 'c': 6}] } flagged_resources = self._merge_flagged_resources(check_id_data, check_result) check_result.update({'flagged_resources': flagged_resources}) check_result.update({ 'name': check_id_data.name, 'category': check_id_data.category, 'description': check_id_data.description, 'arn': self.generate_arn(service='support', region=DEFAULT_REGION, account_id=params['account_id'], resource_type="trusted_advisor", resource_id=check_id), 'account_id': params['account_id'] }) check_data = Check(check_result, strict=False) check_resource = CheckResource({ 'name': check_data.name, 'data': check_data, 'region_code': 'global', 'reference': ReferenceModel(check_data.reference()) }) ta_resources.append(CheckResponse({'resource': check_resource})) print(f' Trusted Advisor Finished {time.time() - start_time} Seconds') return ta_resources
def collect_cloud_service(self, params): print("** Route START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones Response: CloudServiceResponse """ collected_cloud_services = [] secret_data = params['secret_data'] route_conn: RouteConnector = self.locator.get_connector( self.connector_name, **params) # Get lists that relate with snapshots through Google Cloud API routes = route_conn.list_routes() compute_vms = route_conn.list_instance() region = 'global' for route in routes: display = { 'network_display': self._get_matched_last_target('network', route), 'next_hop': self.get_next_hop(route), 'instance_tags_on_list': self._get_tags_display(route, 'list'), 'instance_tags': self._get_tags_display(route, 'not list'), } route.update({ 'display': display, 'project': secret_data['project_id'], 'applicable_instance': self.get_matched_instace(route, secret_data['project_id'], compute_vms), }) # No Labels route_data = Route(route, strict=False) route_resource = RouteResource({ 'region_code': region, 'data': route_data, 'reference': ReferenceModel(route_data.reference()) }) self.set_region_code(region) collected_cloud_services.append( RouteResponse({'resource': route_resource})) print(f'** Route Finished {time.time() - start_time} Seconds **') return collected_cloud_services
def get_rds_databases(self, rds_conn): databases = [] for cluster in rds_conn.describe_db_clusters( Filters=self.get_rds_filter()): rds_cluster = { 'arn': cluster['DBClusterArn'], 'db_identifier': cluster['DBClusterIdentifier'], 'status': cluster['Status'], 'role': 'cluster' } rds_data = RDS(rds_cluster, strict=False) rds_resource = RDSDatabaseResource({ 'data': rds_data, 'reference': ReferenceModel(rds_data.reference()) }) databases.append(RDSDatabaseResponse({'resource': rds_resource})) for instance in rds_conn.describe_db_instances( Filters=self.get_rds_filter()): if not instance.get('DBClusterIdentifier'): rds_instance = { 'arn': instance['DBInstanceArn'], 'db_identifier': instance['DBInstanceIdentifier'], 'status': instance['DBInstanceStatus'], 'role': 'instance' } rds_data = RDS(rds_instance, strict=False) rds_resource = RDSDatabaseResource({ 'data': rds_data, 'reference': ReferenceModel(rds_data.reference()) }) databases.append( RDSDatabaseResponse({'resource': rds_resource})) return databases
def collect_cloud_service(self, params): print("** Cloud SQL START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones Response: CloudServiceResponse """ cloud_sql_conn: CloudSQLConnector = self.locator.get_connector( self.connector_name, **params) collected_cloud_services = [] for instance in cloud_sql_conn.list_instances(): instance_name = instance['name'] project = instance.get('project', '') # Get Databases databases = cloud_sql_conn.list_databases(instance_name) # Get Users users = cloud_sql_conn.list_users(instance_name) stackdriver = self.get_stackdriver(project, instance_name) instance.update({ 'stackdriver': stackdriver, 'display_state': self._get_display_state(instance), 'databases': [Database(database, strict=False) for database in databases], 'users': [User(user, strict=False) for user in users], }) # No labels!! instance_data = Instance(instance, strict=False) instance_resource = InstanceResource({ 'data': instance_data, 'region_code': instance['region'], 'name': instance_name, 'reference': ReferenceModel(instance_data.reference()) }) self.set_region_code(instance['region']) collected_cloud_services.append( InstanceResponse({'resource': instance_resource})) print(f'** Cloud SQL Finished {time.time() - start_time} Seconds **') return collected_cloud_services
def collect_cloud_service(self, params): print("** External IP Address START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones Response: CloudServiceResponse """ collected_cloud_services = [] secret_data = params['secret_data'] exp_conn: ExternalIPAddressConnector = self.locator.get_connector(self.connector_name, **params) regional_global_addresses = exp_conn.list_regional_addresses() compute_engine_vm_address = exp_conn.list_instance_for_networks() forwarding_rule_address = exp_conn.list_forwarding_rule() # Get lists that relate with snapshots through Google Cloud API all_external_ip_addresses = self.get_external_ip_addresses(regional_global_addresses, compute_engine_vm_address, forwarding_rule_address) for external_ip_juso in all_external_ip_addresses: region = external_ip_juso.get('region') if external_ip_juso.get('region') else 'global' external_ip_juso.update({'project': secret_data['project_id'], 'status_display': external_ip_juso.get('status').replace('_', ' ').title() }) if external_ip_juso.get('selfLink') is None: external_ip_juso.update({'self_link': self._get_external_self_link_when_its_empty(external_ip_juso)}) # No Labels (exists on console but No option on APIs) external_ip_juso_data = ExternalIpAddress(external_ip_juso, strict=False) external_ip_juso_resource = ExternalIpAddressResource({ 'region_code': region, 'data': external_ip_juso_data, 'reference': ReferenceModel(external_ip_juso_data.reference()) }) self.set_region_code(region) collected_cloud_services.append(ExternalIpAddressResponse({'resource': external_ip_juso_resource})) print(f'** External IP Address Finished {time.time() - start_time} Seconds **') return collected_cloud_services
def set_image_resources(images, region): result = [] for image in images: image_data = DatabaseSoftwareImage(image, strict=False) image_resource = DatabaseSoftwareImagesResource({ 'data': image_data, 'region_code': region, 'reference': ReferenceModel(image_data.reference()), 'tags': image.get('_freeform_tags', []), 'name': image_data.display_name }) result.append( DatabaseSoftwareImagesResponse({'resource': image_resource})) return result
def set_database_resources(databases, region): result = [] for database in databases: database.update({'region': region}) database_data = Database(database, strict=False) database_resource = DatabaseResource({ 'data': database_data, 'region_code': region, 'reference': ReferenceModel(database_data.reference()), 'tags': database.get('_freeform_tags', []), 'name': database_data.db_name }) result.append(DatabaseResponse({'resource': database_resource})) return result
def _set_resources(self, raw_data): result = [] for raw in raw_data: autonomous_db_data = Database(raw, strict=False) autonomous_db_resource = DatabaseResource({ 'data': autonomous_db_data, 'region_code': autonomous_db_data.region, 'reference': ReferenceModel(autonomous_db_data.reference()), 'tags': raw['_freeform_tags'] }) # self.set_region_code(autonomous_db_data.region) result.append( DatabaseResponse({'resource': autonomous_db_resource})) return result
def get_rds_instances(self, region_name, rds_conn): instances = [] for instance in rds_conn.describe_db_instances(Filters=self.get_rds_filter(region_name)): rds = { 'arn': instance['DBInstanceArn'], 'db_identifier': instance['DBInstanceIdentifier'], 'status': instance['DBInstanceStatus'], 'role': 'instance' } rds_data = RDS(rds, strict=False) rds_resource = RDSInstanceResource({ 'data': rds_data, 'reference': ReferenceModel(rds_data.reference()) }) instances.append(RDSInstanceResponse({'resource': rds_resource})) return instances
def collect_power_state(self, params): """ Args: params: - secret_data - resource_group Response: [VirtualMachineResponse, ...] """ vm_conn: VirtualMachineConnector = self.locator.get_connector(self.connector_name, **params) vm_conn.set_connect(params['secret_data']) # print("** VM Start **") start_time = time.time() vms = [] for vm in vm_conn.list_vms(): vm_info = vm_conn.get_vm(vm.id.split('/')[4], vm.name) reference_id = self.get_reference_id(vm.id) power_state, instance_state = self._get_status_map(vm_info.instance_view.statuses) compute_vm = { 'compute': Compute({'instance_state': instance_state, 'instance_id': reference_id}, strict=False), 'power_state': PowerState({'status': power_state}, strict=False) } compute_vm_data = Server(compute_vm, strict=False) # print("vm_data") # print(compute_vm_data.to_primitive()) compute_vm_resource = VirtualMachineResource({ 'data': compute_vm_data, 'reference': ReferenceModel(compute_vm_data.reference()) }) vms.append(VirtualMachineResponse({'resource': compute_vm_resource})) print(f' Virtual Machine Finished {time.time() - start_time} Seconds') return vms
def set_backup_resources(backups, region, compartment): result = [] for backup in backups: backup.update({ 'region': region, 'compartment_name': compartment.name }) backup_data = Backup(backup, strict=False) backup_resource = BackupResource({ 'data': backup_data, 'region_code': region, 'reference': ReferenceModel(backup_data.reference()), 'tags': backup.get('_freeform_tags', []), 'name': backup_data.display_name }) result.append(BackupResponse({'resource': backup_resource})) return result
def collect_cloud_service(self, params): start_time = time.time() """ Args: params: - options - schema - secret_data - filter - region - compartment Response: CloudServiceResponse """ secret_data = params['secret_data'] region = params['region'] compartment = params['compartment'] secret_data.update({'region': region}) bmvm_conn: BareMetalVMDatabaseConnector = self.locator.get_connector( self.connector_name, **params) bmvm_conn.set_connect(secret_data) # Get list of BareMetal,VM Database Resources basic_bmvm_dbsystem_list = bmvm_conn.list_database_dbsystems( compartment) bmvm_images_list = bmvm_conn.list_database_images(compartment) bmvm_backup_list = bmvm_conn.list_database_backups(compartment) bmvm_database = [] for db_system in basic_bmvm_dbsystem_list: ''' if db_system.lifecycle_state == DbSystemSummary.LIFECYCLE_STATE_TERMINATED: continue ''' db_system_raw = self.convert_nested_dictionary(self, db_system) db_homes = bmvm_conn.list_database_home(compartment, db_system_raw['_id']) db_nodes, node_conn = self._collect_db_nodes( bmvm_conn, compartment, db_system_raw.get('_id')) db_system_raw.update({ 'region': region, 'compartment_name': compartment.name, '_maintenance_window': bmvm_conn.load_database_maintenance_windows( db_system_raw['_maintenance_window']), '_freeform_tags': self.convert_tags(db_system_raw['_freeform_tags']), 'last_maintenance_run': bmvm_conn.load_maintenance_run( db_system_raw['_last_maintenance_run_id'], db_system_raw['_display_name'] + ' - ' + db_system_raw['_shape']), 'next_maintenance_run': bmvm_conn.load_maintenance_run( db_system_raw['_next_maintenance_run_id'], db_system_raw['_display_name'] + ' - ' + db_system_raw['_shape']), 'list_db_Home': self._convert_database_homes(db_homes), 'list_database': self._collect_matched_database(bmvm_conn, compartment, db_homes), 'list_db_node': db_nodes, 'console_connections': node_conn, 'list_patch_history': self._collect_db_system_patch_history( bmvm_conn, db_system_raw.get('list_db_Home')), 'list_patches': self._collect_db_system_patch(bmvm_conn, db_system_raw.get('_id'), compartment), 'list_backups': self._convert_object_to_list(bmvm_backup_list), 'list_software_images': self._update_software_images(bmvm_images_list, region, compartment) }) db_system_data = DbSystem(db_system_raw, strict=False) db_system_resource = DBSystemsResource({ 'data': db_system_data, 'region_code': region, 'reference': ReferenceModel(db_system_data.reference()), 'tags': db_system_raw.get('_freeform_tags', []), 'name': db_system_data.display_name }) bmvm_database.extend( self.set_database_resources( db_system_raw.get('list_database', []), region)) bmvm_database.append( DBSystemResponse({'resource': db_system_resource})) bmvm_database.extend( self.set_image_resources( db_system_raw.get('list_software_images', []), region)) bmvm_database.extend( self.set_backup_resources( db_system_raw.get('list_backups', []), region, compartment)) if bmvm_database: print( f"SET REGION CODE FROM BareMetal,VM DB... {params.get('region')} // {params.get('compartment').name}" ) self.set_region_code(region) return bmvm_database
def collect_cloud_service(self, params): print("** Snapshot START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones - subscription_info Response: CloudServiceResponse """ secret_data = params['secret_data'] # subscription_info = params['subscription_info'] subscription_info = { 'subscription_id': '3ec64e1e-1ce8-4f2c-82a0-a7f6db0899ca', 'subscription_name': 'Azure subscription 1', 'tenant_id': '35f43e22-0c0b-4ff3-90aa-b2c04ef1054c' } snapshot_conn: SnapshotConnector = self.locator.get_connector(self.connector_name, **params) snapshots = [] for snapshot in snapshot_conn.list_snapshots(): snapshot_dict = self.convert_dictionary(snapshot) sku_dict = self.convert_dictionary(snapshot.sku) creation_data_dict = self.convert_dictionary(snapshot.creation_data) encryption_dict = self.convert_dictionary(snapshot.encryption) # update sku_dict # switch SnapshotStorageAccountType to snapshot_sku_name for user-friendly words. # (ex.Premium_LRS -> Premium SSD, Standard HDD..) sku_dict.update({ 'name': self.get_disk_sku_name(sku_dict['name']) }) # update creation_data dict # update creation_data_dict >> image_reference_dict if snapshot.creation_data.image_reference in snapshot_dict: image_reference_dict = self.convert_dictionary(snapshot.creation_data.image_reference) creation_data_dict.update({ 'image_reference': image_reference_dict }) # update creation_data_dict >> gallery_image_reference_dict if snapshot.creation_data.gallery_image_reference in snapshot_dict: gallery_image_dict = self.convert_dictionary(snapshot.creation_data.gallery_image_reference) creation_data_dict.update({ 'gallery_image_reference': gallery_image_dict }) # update encryption_dict type to user-friendly words # (ex.EncryptionAtRestWithPlatformKey -> Platform-managed key...) if snapshot.encryption.type is not None: if snapshot.encryption.type == 'EncryptionAtRestWithPlatformKey': encryption_type = 'Platform-managed key' elif snapshot.encryption.type == 'EncryptionAtRestWithPlatformAndCustomerKeys': encryption_type = 'Platform and customer managed key' elif snapshot.encryption.type == 'EncryptionAtRestWithCustomerKey': encryption_type = 'Customer-managed key' encryption_dict.update({ 'type_display': encryption_type }) # update snapshot_dict snapshot_dict.update({ 'resource_group': self.get_resource_group_from_id(snapshot_dict['id']), # parse resource_group from ID 'subscription_id': subscription_info['subscription_id'], 'subscription_name': subscription_info['subscription_name'], 'size': snapshot_dict['disk_size_bytes'], 'sku': sku_dict, 'creation_data': creation_data_dict, 'encryption': encryption_dict, 'incremental_display': self.get_incremental_display(snapshot_dict['incremental']) }) if 'network_access_policy' in snapshot_dict: snapshot_dict.update({ 'network_access_policy_display': self.get_network_access_policy( snapshot_dict['network_access_policy']) }) # get attached vm's name managed_by = snapshot_dict['managed_by'] if managed_by: snapshot_dict.update({ 'managed_by': self.get_attached_vm_name_from_managed_by(snapshot_dict['managed_by']) }) # get source_disk_name from source_resource_id source_disk_name = creation_data_dict['source_resource_id'] if source_disk_name: snapshot_dict.update({ 'source_disk_name': self.get_source_disk_name(creation_data_dict['source_resource_id']) }) # switch tags form tags = snapshot_dict.get('tags', {}) _tags = self.convert_tag_format(tags) snapshot_dict.update({ 'tags': _tags }) snapshot_data = Snapshot(snapshot_dict, strict=False) snapshot_resource = SnapshotResource({ 'data': snapshot_data, 'region_code': snapshot_data.location, 'reference': ReferenceModel(snapshot_data.reference()), 'tags': _tags }) # print("snapshot_dict") # print(snapshot_dict) # Must set_region_code method for region collection self.set_region_code(snapshot_data['location']) snapshots.append(SnapshotResponse({'resource': snapshot_resource})) print(f'** Snapshot Finished {time.time() - start_time} Seconds **') return snapshots
def collect_cloud_service(self, params): print("** LoadBalancer START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones - subscription_info Response: CloudServiceResponse """ secret_data = params['secret_data'] subscription_info = params['subscription_info'] load_balancer_conn: LoadBalancerConnector = self.locator.get_connector( self.connector_name, **params) load_balancers = [] for load_balancer in load_balancer_conn.list_load_balancers(): load_balancer_dict = self.convert_nested_dictionary( self, load_balancer) # update vm_scale_set_dict load_balancer_dict.update({ 'resource_group': self.get_resource_group_from_id(load_balancer_dict['id']), # parse resource_group from ID 'subscription_id': subscription_info['subscription_id'], 'subscription_name': subscription_info['subscription_name'], }) # Get Network Interfaces attached in this load balancer load_balancer_dict.update({ 'network_interfaces': self.get_network_interfaces( self, load_balancer_conn, load_balancer_dict['resource_group'], load_balancer_dict['name']) }) # Get Frontend IP Configurations information if load_balancer_dict.get( 'frontend_ip_configurations') is not None: private_ip_address_list = list() used_by_list = list() for fic in load_balancer_dict['frontend_ip_configurations']: if fic.get( 'subnet' ): # If the 'public' type, Skip this part because there isn't subnet information for them. fic['subnet'][ 'address_prefix'] = self.get_frontend_address_prefix( self, load_balancer_conn, fic['subnet']) fic['subnet'][ 'name'] = self.get_frontend_ip_subnet_name( fic['subnet']['id']) # Get used inbound NAT rules if fic.get('inbound_nat_rules') is not None: load_balancer_dict.update({ 'frontend_ip_configurations_used_by_display': self. get_frontend_ip_configurations_used_by_display( used_by_list, fic['inbound_nat_rules']) }) # Get used load balancing NAT rules if fic.get('load_balancing_rules') is not None: load_balancer_dict.update({ 'frontend_ip_configurations_used_by_display': self. get_frontend_ip_configurations_used_by_display( used_by_list, fic['load_balancing_rules']), }) # Get all of private ip addresses private_ip_address_list.append(fic['private_ip_address']) load_balancer_dict.update({ 'private_ip_address_display': private_ip_address_list }) # Since Azure python sdk returns only one backend pool, delete the backend pool list first, and then use the new API connection if load_balancer_dict.get('backend_address_pools') is not None: load_balancer_dict['backend_address_pools'].clear() load_balancer_dict.update({ 'backend_address_pools': self.list_load_balancer_backend_address_pools( self, load_balancer_conn, load_balancer_dict['resource_group'], load_balancer_dict['name'], load_balancer_dict['network_interfaces']) }) # get backend address pool's count load_balancer_dict.update({ 'backend_address_pools_count_display': self.get_backend_address_pools_count( self, load_balancer_dict['backend_address_pools']) }) # Get load balancing Rules for display if load_balancer_dict.get('load_balancing_rules') is not None: load_balancer_dict.update({ 'load_balancing_rules_display': self.get_load_balancing_rules_display( self, load_balancer_dict['load_balancing_rules']), }) for lbr in load_balancer_dict['load_balancing_rules']: if lbr.get('backend_address_pool') is not None: lbr.update({ 'backend_address_pool_display': self.get_backend_address_pool_name( lbr['backend_address_pool']), }) if lbr.get('load_distribution') is not None: lbr.update({ 'load_distribution_display': self.get_load_distribution_display( lbr['load_distribution']) }) if lbr.get('frontend_ip_configuration') is not None: lbr.update({ 'frontend_ip_configuration_display': self.get_frontend_ip_configuration_display( lbr['frontend_ip_configuration']) }) # Get Inbound NAT Rules for display if load_balancer_dict.get('inbound_nat_rules') is not None: load_balancer_dict.update({ 'inbound_nat_rules_display': self.get_nat_rules_display( self, load_balancer_dict['inbound_nat_rules']) }) for inr in load_balancer_dict['inbound_nat_rules']: inr.update({ 'frontend_ip_configuration_display': self.get_frontend_ip_configuration_display( inr['frontend_ip_configuration']), 'port_mapping_display': self.get_port_mapping_display(inr['frontend_port'], inr['backend_port']), 'target_virtual_machine': self.get_matched_vm_info( self, inr['backend_ip_configuration']['id'], load_balancer_dict['network_interfaces']) }) # Get Health Probes for display if load_balancer_dict.get('probes') is not None: load_balancer_dict.update({ 'probes_display': self.get_probe_display_list(self, load_balancer_dict['probes']) }) # switch tags form tags = load_balancer_dict.get('tags', {}) _tags = self.convert_tag_format(tags) load_balancer_dict.update({'tags': _tags}) # print("load_balancer_dict") # print(load_balancer_dict) load_balancer_data = LoadBalancer(load_balancer_dict, strict=False) load_balancer_resource = LoadBalancerResource({ 'data': load_balancer_data, 'region_code': load_balancer_data.location, 'reference': ReferenceModel(load_balancer_data.reference()), 'tags': _tags }) # Must set_region_code method for region collection self.set_region_code(load_balancer_data['location']) load_balancers.append( LoadBalancerResponse({'resource': load_balancer_resource})) print( f'** LoadBalancer Finished {time.time() - start_time} Seconds **') return load_balancers
def collect_cloud_service(self, params): print("** Disk START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones - subscription_info Response: CloudServiceResponse """ secret_data = params['secret_data'] subscription_info = params['subscription_info'] disk_conn: DiskConnector = self.locator.get_connector( self.connector_name, **params) disks = [] for disk in disk_conn.list_disks(): disk_dict = self.convert_dictionary(disk) sku_dict = self.convert_dictionary(disk.sku) creation_data_dict = self.convert_dictionary(disk.creation_data) encryption_dict = self.convert_dictionary(disk.encryption) # update sku_dict # switch DiskStorageAccountType to disk_sku_name for user-friendly words. # (ex.Premium SSD, Standard HDD..) sku_dict.update({'name': self.get_disk_sku_name(sku_dict['name'])}) # update creation_data dict if disk.creation_data.image_reference is not None: image_reference_dict = self.convert_dictionary( disk.creation_data.image_reference) creation_data_dict.update( {'image_reference': image_reference_dict}) # update disk_data dict disk_dict.update({ 'resource_group': self.get_resource_group_from_id( disk_dict['id']), # parse resource_group from ID 'subscription_id': subscription_info['subscription_id'], 'subscription_name': subscription_info['subscription_name'], 'size': disk_dict['disk_size_bytes'], 'sku': sku_dict, 'creation_data': creation_data_dict, 'encryption': encryption_dict, 'tier_display': self.get_tier_display(disk_dict['disk_iops_read_write'], disk_dict['disk_m_bps_read_write']), }) if 'network_access_policy' in disk_dict: disk_dict.update({ 'network_access_policy_display': self.get_network_access_policy( disk_dict['network_access_policy']) }) # get attached vm's name managed_by = disk_dict['managed_by'] if managed_by: disk_dict.update({ 'managed_by': self.get_attached_vm_name_from_managed_by( disk_dict['managed_by']) }) # switch tags form tags = disk_dict.get('tags', {}) _tags = self.convert_tag_format(tags) disk_dict.update({'tags': _tags}) disk_data = Disk(disk_dict, strict=False) disk_resource = DiskResource({ 'data': disk_data, 'region_code': disk_data.location, 'reference': ReferenceModel(disk_data.reference()), 'tags': _tags, 'name': disk_data.name }) # Must set_region_code method for region collection self.set_region_code(disk_data['location']) disks.append(DiskResponse({'resource': disk_resource})) print(f'** Disk Finished {time.time() - start_time} Seconds **') return disks
def collect_cloud_service(self, params): print("** Snapshot START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones Response: CloudServiceResponse """ collected_cloud_services = [] secret_data = params['secret_data'] snapshot_conn: SnapshotConnector = self.locator.get_connector( self.connector_name, **params) # Get lists that relate with snapshots through Google Cloud API snapshots = snapshot_conn.list_snapshot() all_region_resource_policies = snapshot_conn.list_resource_policies() disk_list_info = snapshot_conn.list_all_disks_for_snapshots() for snapshot in snapshots: region = self.get_matching_region(snapshot.get('storageLocations')) snapshot_schedule = [] snapshot_schedule_display = [] disk_name_key = self._get_disk_name_key(snapshot.get('name')) for resource_policy in disk_list_info.get(disk_name_key, []): snapshot_schedule_display.append( self._get_last_target(resource_policy)) matched_po = self.get_matched_snapshot_schedule( all_region_resource_policies.get(resource_policy)) snapshot_schedule.append( SnapShotSchedule(matched_po, strict=False)) labels = self.convert_labels_format(snapshot.get('labels', {})) snapshot.update({ 'project': secret_data['project_id'], 'disk': self.get_disk_info(snapshot), 'snapshot_schedule': snapshot_schedule, 'snapshot_schedule_display': snapshot_schedule_display, 'creation_type': 'Scheduled' if snapshot.get('autoCreated') else 'Manual', 'encryption': self._get_encryption_info(snapshot), 'labels': labels }) _name = snapshot_data.get('name', '') # labels -> tags snapshot_data = Snapshot(snapshot, strict=False) snapshots_resource = SnapshotResource({ 'name': _name, 'region_code': region.get('region_code'), 'data': snapshot_data, 'tags': labels, 'reference': ReferenceModel(snapshot_data.reference()) }) self.set_region_code(region.get('region_code')) collected_cloud_services.append( SnapshotResponse({'resource': snapshots_resource})) print(f'** SnapShot Finished {time.time() - start_time} Seconds **') return collected_cloud_services
def collect_cloud_service(self, params): print("** VPC Network START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones Response: CloudServiceResponse """ collected_cloud_services = [] secret_data = params['secret_data'] vpc_conn: VPCNetworkConnector = self.locator.get_connector( self.connector_name, **params) # Get lists that relate with snapshots through Google Cloud API networks = vpc_conn.list_networks() firewalls = vpc_conn.list_firewall() subnets = vpc_conn.list_subnetworks() routes = vpc_conn.list_routes() regional_address = vpc_conn.list_regional_addresses() for network in networks: network_identifier = network.get('selfLink') matched_firewall = self._get_matched_firewalls( network_identifier, firewalls) matched_route = self.get_matched_route(network_identifier, routes) matched_subnets = self._get_matched_subnets( network_identifier, subnets) region = self.match_region_info('global') peerings = self.get_peering(network) network.update({ 'mode': 'Auto' if network.get('autoCreateSubnetworks') else 'Custom', 'project': secret_data['project_id'], 'global_dynamic_route': self._get_global_dynamic_route(network, 'not_mode'), 'dynamic_routing_mode': self._get_global_dynamic_route(network, 'mode'), 'subnet_creation_mode': 'Auto' if network.get('autoCreateSubnetworks') else 'Custom', 'ip_address_data': self.get_internal_ip_address_in_use(network, regional_address), 'peerings': peerings, 'route_data': { 'total_number': len(matched_route), 'route': matched_route }, 'firewall_data': { 'total_number': len(matched_firewall), 'firewall': matched_firewall }, 'subnetwork_data': { 'total_number': len(matched_subnets), 'subnets': matched_subnets }, }) # No labels _name = network.get('name', '') vpc_data = VPCNetwork(network, strict=False) vpc_resource = VPCNetworkResource({ 'name': _name, 'region_code': region.get('region_code'), 'data': vpc_data, 'reference': ReferenceModel(vpc_data.reference()) }) self.set_region_code('global') collected_cloud_services.append( VPCNetworkResponse({'resource': vpc_resource})) print(f'** VPC Network Finished {time.time() - start_time} Seconds **') return collected_cloud_services
def collect_cloud_service(self, params): print("** Storage START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones Response: CloudServiceResponse """ collected_cloud_services = [] secret_data = params['secret_data'] storage_conn: StorageConnector = self.locator.get_connector( self.connector_name, **params) # Get lists that relate with snapshots through Google Cloud API buckets = storage_conn.list_buckets() for bucket in buckets: bucket_name = bucket.get('name') objects = storage_conn.list_objects(bucket_name) obj_count, size = self._get_number_of_obj_and_size(objects) iam_policy = storage_conn.list_iam_policy(bucket_name) st_class = bucket.get('storageClass').lower() region = self.get_matching_region(bucket) labels = self.convert_labels_format(bucket.get('labels', {})) stackdriver = self.get_stackdriver(bucket_name) bucket.update({ 'project': secret_data['project_id'], 'encryption': self._get_encryption(bucket), 'requester_pays': self._get_requester_pays(bucket), 'retention_policy_display': self._get_retention_policy_display(bucket), 'links': self._get_config_link(bucket), 'size': size, 'stackdriver': stackdriver, 'default_event_based_hold': 'Enabled' if bucket.get('defaultEventBasedHold') else 'Disabled', 'iam_policy': iam_policy, 'iam_policy_binding': self._get_iam_policy_binding(iam_policy), 'object_count': obj_count, 'object_total_size': size, 'lifecycle_rule': self._get_lifecycle_rule(bucket), 'location': self.get_location(bucket), 'default_storage_class': st_class.capitalize(), 'access_control': self._get_access_control(bucket), 'public_access': self._get_public_access(bucket, iam_policy), 'labels': labels }) _name = bucket.get('name', '') bucket_data = Storage(bucket, strict=False) # labels -> tags bucket_resource = StorageResource({ 'name': _name, 'tags': labels, 'region_code': region.get('region_code'), 'data': bucket_data, 'reference': ReferenceModel(bucket_data.reference()) }) self.set_region_code(region.get('region_code')) collected_cloud_services.append( StorageResponse({'resource': bucket_resource})) print(f'** Storage Finished {time.time() - start_time} Seconds **') return collected_cloud_services
def collect_cloud_service(self, params): print("** Big Query START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones Response: CloudServiceResponse """ collected_cloud_services = [] try: secret_data = params['secret_data'] project_id = secret_data['project_id'] big_query_conn: BigQueryConnector = self.locator.get_connector( self.connector_name, **params) data_sets = big_query_conn.list_dataset() projects = big_query_conn.list_projects() jobs = big_query_conn.list_job() for data_set in data_sets: data_refer = data_set.get('datasetReference', {}) data_set_id = data_refer.get('datasetId') dataset_project_id = data_refer.get('projectId') bq_dataset = big_query_conn.get_dataset(data_set_id) bq_dt_tables = big_query_conn.list_tables(data_set_id) update_bq_dt_tables, table_schemas = self._get_table_list_with_schema( big_query_conn, bq_dt_tables) matched_projects = self._get_matching_project( dataset_project_id, projects) matched_jobs = self._get_matching_jobs(data_set, update_bq_dt_tables, jobs) creation_time = bq_dataset.get('creationTime') if creation_time: bq_dataset.update({ 'creationTime': datetime.fromtimestamp(int(creation_time) / 1000) }) last_modified_time = bq_dataset.get('lastModifiedTime') if last_modified_time: bq_dataset.update({ 'lastModifiedTime': datetime.fromtimestamp(int(last_modified_time) / 1000) }) region = self.get_region(bq_dataset.get('location', '')) exp_partition_ms = bq_dataset.get( 'defaultPartitionExpirationMs') exp_table_ms = bq_dataset.get('defaultTableExpirationMs') if exp_partition_ms: bq_dataset.update({ 'default_partition_expiration_ms_display': self.get_ms_display(exp_partition_ms) }) if exp_table_ms: bq_dataset.update({ 'default_table_expiration_ms_display': self.get_ms_display(exp_table_ms) }) labels = self.convert_labels_format( bq_dataset.get('labels', {})) bq_dataset.update({ 'name': data_set_id, 'project': project_id, 'tables': update_bq_dt_tables, 'table_schemas': table_schemas, 'region': region, 'visible_on_console': self.get_visible_on_console(data_set_id), 'jobs': matched_jobs, 'matching_projects': matched_projects, 'labels': labels }) big_query_data = BigQueryWorkSpace(bq_dataset, strict=False) big_query_work_space_resource = SQLWorkSpaceResource({ 'tags': labels, 'data': big_query_data, 'region_code': region, 'reference': ReferenceModel(big_query_data.reference()) }) self.set_region_code(region) collected_cloud_services.append( SQLWorkSpaceResponse( {'resource': big_query_work_space_resource})) print( f'** Big Query Finished {time.time() - start_time} Seconds **') except Exception as e: print(e) pass return collected_cloud_services
def collect_cloud_service(self, params): print("** Instance Template START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones Response: CloudServiceResponse """ secret_data = params['secret_data'] instance_template_conn: InstanceTemplateConnector = self.locator.get_connector( self.connector_name, **params) # Get Instance Templates instance_templates = instance_template_conn.list_instance_templates() instance_groups = instance_template_conn.list_instance_group_managers() machine_types = instance_template_conn.list_machine_types() collected_cloud_services = [] for inst_template in instance_templates: properties = inst_template.get('properties', {}) tags = properties.get('tags', {}) in_used_by, matched_instance_group = self.match_instance_group( inst_template, instance_groups) disks = self.get_disks(properties) labels = self.convert_labels_format(properties.get('labels', {})) inst_template.update({ 'project': secret_data['project_id'], 'in_used_by': in_used_by, 'ip_forward': properties.get('canIpForward', False), 'machine': MachineType(self._get_machine_type(properties, machine_types), strict=False), 'network_tags': tags.get('items', []), 'scheduling': self._get_scheduling(properties), 'disk_display': self._get_disk_type_display(disks, 'disk_type'), 'image': self._get_disk_type_display(disks, 'source_image_display'), 'instance_groups': matched_instance_group, 'network_interfaces': self.get_network_interface(properties), 'fingerprint': self._get_properties_item(properties, 'metadata', 'fingerprint'), 'labels': labels, 'disks': disks }) svc_account = properties.get('serviceAccounts', []) if len(svc_account) > 0: inst_template.update({ 'service_account': self._get_service_account(svc_account) }) instance_template_data = InstanceTemplate(inst_template, strict=False) # labels -> tags default_region = 'global' instance_template_resource = InstanceTemplateResource({ 'tags': labels, 'data': instance_template_data, 'reference': ReferenceModel(instance_template_data.reference()), 'region_code': default_region }) self.set_region_code(default_region) collected_cloud_services.append( InstanceTemplateResponse( {'resource': instance_template_resource})) print( f'** Instance Template Finished {time.time() - start_time} Seconds **' ) return collected_cloud_services
def collect_cloud_service(self, params): print("** VmScaleSet START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones - subscription_info Response: CloudServiceResponse """ secret_data = params['secret_data'] subscription_info = params['subscription_info'] vm_scale_set_conn: VmScaleSetConnector = self.locator.get_connector( self.connector_name, **params) vm_scale_sets = [] for vm_scale_set in vm_scale_set_conn.list_vm_scale_sets(): vm_scale_set_dict = self.convert_nested_dictionary( self, vm_scale_set) # update vm_scale_set_dict vm_scale_set_dict.update({ 'resource_group': self.get_resource_group_from_id( vm_scale_set_dict['id']), # parse resource_group from ID 'subscription_id': subscription_info['subscription_id'], 'subscription_name': subscription_info['subscription_name'], }) if vm_scale_set_dict.get( 'proximity_placement_group' ): # if it has a key -> get value -> check if it isn't None / if no 'Key' -> return None vm_scale_set_dict.update({ 'proximity_placement_group_display': self.get_proximity_placement_group_name( vm_scale_set_dict['proximity_placement_group']['id']) }) # Get Instance termination notification display if vm_scale_set_dict.get('virtual_machine_profile') is not None: if vm_scale_set_dict['virtual_machine_profile'].get( 'scheduled_events_profile'): if vm_scale_set.virtual_machine_profile.scheduled_events_profile.terminate_notification_profile.enable: terminate_notification_display = 'On' else: terminate_notification_display = 'Off' vm_scale_set_dict.update({ 'terminate_notification_display': terminate_notification_display }) # Convert disks' sku-dict to string display if vm_scale_set_dict['virtual_machine_profile'].get( 'storage_profile') is not None: if vm_scale_set_dict['virtual_machine_profile'][ 'storage_profile'].get('image_reference'): image_reference_dict = vm_scale_set_dict[ 'virtual_machine_profile']['storage_profile'][ 'image_reference'] image_reference_str = \ str(image_reference_dict['publisher']) + " / " + str(image_reference_dict['offer']) + " / " + str(image_reference_dict['sku']) + " / " + str(image_reference_dict['version']) vm_scale_set_dict['virtual_machine_profile'][ 'storage_profile'].update({ 'image_reference_display': image_reference_str }) # switch storage_account_type to storage_account_type for user-friendly words. # (ex.Premium LRS -> Premium SSD, Standard HDD..) if vm_scale_set_dict['virtual_machine_profile'][ 'storage_profile'].get('data_disks'): for data_disk in vm_scale_set_dict[ 'virtual_machine_profile']['storage_profile'][ 'data_disks']: data_disk['managed_disk'].update({ 'storage_type': self.get_disk_storage_type( data_disk['managed_disk'] ['storage_account_type']) }) # Get VM Profile's operating_system type (Linux or Windows) if vm_scale_set_dict['virtual_machine_profile'].get( 'os_profile') is not None: vm_scale_set_dict['virtual_machine_profile'][ 'os_profile'].update({ 'operating_system': self.get_operating_system( vm_scale_set_dict['virtual_machine_profile'] ['os_profile']) }) # Get VM Profile's primary Vnet\ if vm_scale_set_dict['virtual_machine_profile'].get( 'network_profile') is not None: vmss_vm_network_profile_dict = vm_scale_set_dict[ 'virtual_machine_profile']['network_profile'] vmss_vm_network_profile_dict.update({ 'primary_vnet': self.get_primary_vnet(vmss_vm_network_profile_dict[ 'network_interface_configurations']) }) # Add vm instances list attached to VMSS vm_instances_list = list() instance_count = 0 for vm_instance in vm_scale_set_conn.list_vm_scale_set_vms( vm_scale_set_dict['resource_group'], vm_scale_set_dict['name']): instance_count += 1 vm_scale_set_dict.update({'instance_count': instance_count}) vm_instance_dict = self.get_vm_instance_dict( self, vm_instance, vm_scale_set_conn, vm_scale_set_dict['resource_group'], vm_scale_set_dict['name']) vm_instances_list.append(vm_instance_dict) vm_scale_set_dict['vm_instances'] = vm_instances_list # Get auto scale settings by resource group and vm id vm_scale_set_dict.update({ 'autoscale_settings': self.list_auto_scale_settings_obj( self, vm_scale_set_conn, vm_scale_set_dict['resource_group'], vm_scale_set_dict['id']) }) # Set virtual_machine_scale_set_power_state information if vm_scale_set_dict.get('autoscale_settings') is not None: vm_scale_set_dict.update({ 'virtual_machine_scale_set_power_state': self.list_virtual_machine_scale_set_power_state( self, vm_scale_set_dict['autoscale_settings']), }) # update auto_scale_settings to autoscale_setting_resource_collection auto_scale_setting_resource_col_dict = dict() auto_scale_setting_resource_col_dict.update({ 'value': self.list_auto_scale_settings( self, vm_scale_set_conn, vm_scale_set_dict['resource_group'], vm_scale_set_dict['id']) }) vm_scale_set_dict.update({ 'autoscale_setting_resource_collection': auto_scale_setting_resource_col_dict }) # switch tags form tags = vm_scale_set_dict.get('tags', {}) _tags = self.convert_tag_format(tags) vm_scale_set_dict.update({'tags': _tags}) # print("vm_scale_set_dict") # print(vm_scale_set_dict) vm_scale_set_data = VirtualMachineScaleSet(vm_scale_set_dict, strict=False) vm_scale_set_resource = VmScaleSetResource({ 'data': vm_scale_set_data, 'region_code': vm_scale_set_data.location, 'reference': ReferenceModel(vm_scale_set_data.reference()), 'tags': _tags, 'name': vm_scale_set_data.name }) # Must set_region_code method for region collection self.set_region_code(vm_scale_set_data['location']) vm_scale_sets.append( VmScaleSetResponse({'resource': vm_scale_set_resource})) print(f'** VmScaleSet Finished {time.time() - start_time} Seconds **') return vm_scale_sets
def collect_cloud_service(self, params): start_time = time.time() """ Args: params: - options - schema - secret_data - filter - region - compartment Response: CloudServiceResponse """ secret_data = params['secret_data'] region = params['region'] compartment = params['compartment'] secret_data.update({'region': region}) adb_conn: AutonomousDatabaseConnector = self.locator.get_connector( self.connector_name, **params) autonomous_database_list = [] adb_conn.set_connect(secret_data) basic_adb_list = adb_conn.list_of_autonomous_databases( params['compartment']) for basic_adb in basic_adb_list: adb_raw = self.convert_nested_dictionary(self, basic_adb) adb_raw.update({ 'region': region, 'compartment_name': compartment.name, '_freeform_tags': self.convert_tags(adb_raw['_freeform_tags']), 'db_workload_display': self._set_workload_type(adb_raw['_db_workload']), 'size': OCIManager.gigabyte_to_byte( adb_raw['_data_storage_size_in_gbs'] ), # 바이트 단위로 정규화 후 넣어주기 '_data_storage_size_in_tbs': self.gbs_to_tbs(adb_raw['_data_storage_size_in_gbs']), '_license_model': self.define_license_type(adb_raw['_license_model']), '_permission_level': self.define_permission_level(adb_raw['_permission_level']), 'list_autonomous_backup': self._set_backup_list( adb_conn.list_autonomous_database_backup(adb_raw['_id'])), 'list_autonomous_database_clones': self._set_clone_list( adb_conn.list_autonomous_database_clones( adb_raw['_compartment_id'], adb_raw['_id'])) }) autonomous_db_data = Database(adb_raw, strict=False) autonomous_db_resource = DatabaseResource({ 'data': autonomous_db_data, 'region_code': autonomous_db_data.region, 'reference': ReferenceModel(autonomous_db_data.reference()), 'tags': adb_raw['_freeform_tags'], 'name': autonomous_db_data.display_name }) # self.set_region_code(autonomous_db_data.region) autonomous_database_list.append( DatabaseResponse({'resource': autonomous_db_resource})) # Must set_region_code method for region collection if basic_adb_list: print( f"SET REGION CODE FROM AUTONOMOUS DB... {params.get('region')} // {params.get('compartment').name}" ) self.set_region_code(region) # raw_data = self._set_mandatory_param(adb_conn, basic_adb_list, # params['region'], params['compartment'].name) # autonomous_database_list.extend(self._set_resources(raw_data)) ''' for region in regions: secret_data['region'] = region adb_conn.set_connect(secret_data) for compartment in compartments: basic_adb_list = adb_conn.list_of_autonomous_databases(compartment) if basic_adb_list: raw_data = self._set_mandatory_param(adb_conn, basic_adb_list, region, compartment.name) # Must set_region_code method for region collection self.set_region_code(region) autonomous_database_list.extend(self._set_resources(raw_data)) ''' # print(f'** Autonomous Database Finished {time.time() - start_time} Seconds **') return autonomous_database_list
display_loc = { 'region': location, 'zone': ''} if loc_type == 'region' \ else {'region': location[:-2], 'zone': location} instance_group.update({'display_location': display_loc}) instance_group.update({ 'power_scheduler': scheduler, 'instances': self.get_instances(instances), 'instance_counts': len(instances) }) # No labels instance_group_data = InstanceGroup(instance_group, strict=False) instance_group_resource = InstanceGroupResource({ 'data': instance_group_data, 'region_code': region, 'reference': ReferenceModel(instance_group_data.reference()) }) self.set_region_code(region) collected_cloud_services.append(InstanceGroupResponse({'resource': instance_group_resource})) print(f'** Instance Group Finished {time.time() - start_time} Seconds **') return collected_cloud_services def get_instance_group_loc(self, instance_group): inst_type = 'zone' if 'zone' in instance_group else 'region' loc = self._get_last_target(instance_group, inst_type) return inst_type, loc def get_instances(self, instances): _instances = []
def collect_cloud_service(self, params): start_time = time.time() """ Args: params: - options - schema - secret_data - filter - region - compartment Response: CloudServiceResponse """ secret_data = params['secret_data'] region = params['region'] compartment = params['compartment'] secret_data.update({'region': region}) exa_conn: ExadataCloudDatabaseConnector = self.locator.get_connector( self.connector_name, **params) exa_conn.set_connect(secret_data) result = [] basic_exadata_infra_list = exa_conn.list_cloud_exadata_infra( compartment) exadata_image_list = exa_conn.list_database_images(compartment) exadata_software_image_response = self.set_image_resource( exadata_image_list, region, compartment) result.extend(exadata_software_image_response) for exadata_infra in basic_exadata_infra_list: exadata_infra_raw = self.convert_nested_dictionary( self, exadata_infra) list_vm_cluster = exa_conn.list_cloud_vm_cluster( region, compartment, exa_conn, exadata_infra_raw['_id']) exadata_infra_raw.update({ 'region': region, 'compartment_name': compartment.name, '_maintenance_window': exa_conn.load_database_maintenance_windows( exadata_infra_raw['_maintenance_window']), '_freeform_tags': self.convert_tags(exadata_infra_raw['_freeform_tags']), 'last_maintenance_run': exa_conn.load_maintenance_run( exadata_infra_raw['_last_maintenance_run_id'], exadata_infra_raw['_display_name'] + ' - ' + exadata_infra_raw['_shape']), 'next_maintenance_run': exa_conn.load_maintenance_run( exadata_infra_raw['_next_maintenance_run_id'], exadata_infra_raw['_display_name'] + ' - ' + exadata_infra_raw['_shape']), 'list_cloud_vm_cluster': self.set_vm_cluster_data(region, compartment, exa_conn, list_vm_cluster) }) exadata_vm_cluster_list = exadata_infra_raw.get[ 'list_cloud_vm_cluster'] exadata_vm_database_list, exadata_vm_database_backup_list = \ self.set_database_resource(exadata_vm_cluster_list, region) exadata_backup_response_list = self.set_backup_resource( exadata_vm_database_backup_list, region, compartment) exadata_infra_data = CloudExadataInfra(exadata_infra_raw, strict=False) exadata_infra_resource = ExadataInfrastructureResource({ 'data': exadata_infra_data, 'region_code': region, 'reference': ReferenceModel(exadata_infra_data.reference()), 'tags': exadata_infra_raw.get('_freeform_tags', []), 'name': exadata_infra_data.display_name }) result.append( ExadataInfrastructureResponse( {'resource': exadata_infra_resource})) result.extend(exadata_vm_database_list) result.extend(exadata_backup_response_list) return result
def collect_cloud_service(self, params): print("** Big Query START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones Response: CloudServiceResponse """ collected_cloud_services = [] secret_data = params['secret_data'] project_id = secret_data['project_id'] big_query_conn: BigQueryConnector = self.locator.get_connector( self.connector_name, **params) data_sets = big_query_conn.list_dataset() projects = big_query_conn.list_projects() jobs = big_query_conn.list_job() ''' class BigQuery(Model): # id = StringType() # name = StringType() #project = StringType() #region = StringType() #dataset_reference = ModelType(DatasetReference, deserialize_from='datasetReference', serialize_when_none=False) #friendly_name = StringType(deserialize_from='friendlyName', serialize_when_none=False) #tables = ListType(ModelType(Table), default=[]) #table_schemas = ListType(ModelType(TableSchemaRef), default=[]) jobs = ListType(ModelType(Job), default=[]) #projects = ListType(ModelType(Project), default=[]) #access = ListType(ModelType(Access), default=[]) #labels = ListType(ModelType(Labels), default=[]) #self_link = StringType() #etags = StringType() #location = StringType() #visible_on_console = BooleanType() #default_partition_expiration_ms = DateTimeType(deserialize_from='defaultPartitionExpirationMs') #default_table_expiration_ms = DateTimeType(deserialize_from='defaultTableExpirationMs') #creation_time = DateTimeType(deserialize_from='creationTime') #last_modified_time = DateTimeType(deserialize_from='lastModifiedTime') ''' for data_set in data_sets: data_refer = data_set.get('datasetReference', {}) data_set_id = data_refer.get('datasetId') dataset_project_id = data_refer.get('projectId') bq_dataset = big_query_conn.get_dataset(data_set_id) bq_dt_tables = big_query_conn.list_tables(data_set_id) update_bq_dt_tables, table_schemas = self._get_table_list_with_schema( big_query_conn, bq_dt_tables) matched_projects = self._get_matching_project( dataset_project_id, projects) matched_jobs = self._get_matching_jobs(update_bq_dt_tables, jobs) creation_time = bq_dataset.get('creationTime') if creation_time: bq_dataset.update({ 'creationTime': datetime.fromtimestamp(int(creation_time) / 1000) }) last_modified_time = bq_dataset.get('lastModifiedTime') if last_modified_time: bq_dataset.update({ 'lastModifiedTime': datetime.fromtimestamp(int(last_modified_time) / 1000) }) region = self.get_region(bq_dataset.get('location', '')) exp_partition_ms = bq_dataset.get('defaultPartitionExpirationMs') exp_table_ms = bq_dataset.get('defaultTableExpirationMs') if exp_partition_ms: bq_dataset.update({ 'default_partition_expiration_ms_display': self.get_ms_display(exp_partition_ms) }) if exp_table_ms: bq_dataset.update({ 'default_table_expiration_ms_display': self.get_ms_display(exp_table_ms) }) labels = self.convert_labels_format(bq_dataset.get('labels', {})) bq_dataset.update({ 'name': data_set_id, 'project': project_id, 'tables': update_bq_dt_tables, 'table_schemas': table_schemas, 'region': region, 'visible_on_console': self.get_visible_on_console(data_set_id), 'jobs': matched_jobs, 'matching_projects': matched_projects, 'labels': labels }) big_query_data = BigQueryWorkSpace(bq_dataset, strict=False) big_query_work_space_resource = SQLWorkSpaceResource({ 'tags': labels, 'data': big_query_data, 'region_code': region, 'reference': ReferenceModel(big_query_data.reference()) }) self.set_region_code(region) collected_cloud_services.append( SQLWorkSpaceResponse( {'resource': big_query_work_space_resource})) print(f'** Big Query Finished {time.time() - start_time} Seconds **') return collected_cloud_services
def collect_cloud_service(self, params): print("** Load Balancing START **") start_time = time.time() """ Args: params: - options - schema - secret_data - filter - zones Response: CloudServiceResponse """ collected_cloud_services = [] secret_data = params['secret_data'] load_bal_conn: LoadBalancingConnector = self.locator.get_connector( self.connector_name, **params) project_id = secret_data.get('project_id') load_balancers = [] instance_groups = load_bal_conn.list_instance_groups() target_pools = load_bal_conn.list_target_pools() url_maps = load_bal_conn.list_url_maps() forwarding_rules = load_bal_conn.list_forwarding_rules() backend_services = load_bal_conn.list_back_end_services() backend_buckets = load_bal_conn.list_back_end_buckets() ssl_certificates = load_bal_conn.list_ssl_certificates() auto_scalers = load_bal_conn.list_auto_scalers() health_checks = load_bal_conn.list_health_checks() legacy_health_checks = [] http_health_checks = load_bal_conn.list_http_health_checks() https_health_checks = load_bal_conn.list_https_health_checks() legacy_health_checks.extend(http_health_checks) legacy_health_checks.extend(https_health_checks) # proxies grpc_proxies = load_bal_conn.list_grpc_proxies() http_proxies = load_bal_conn.list_target_http_proxies() https_proxies = load_bal_conn.list_target_https_proxies() ssl_proxies = load_bal_conn.list_ssl_proxies() tcp_proxies = load_bal_conn.list_tcp_proxies() target_proxies, selective_proxies = self.get_all_proxy_list( grpc_proxies, http_proxies, https_proxies, ssl_proxies, tcp_proxies, forwarding_rules) lbs_from_proxy = self.get_load_balacer_from_target_proxy( backend_services, selective_proxies, project_id) lbs_from_url_map = self.get_load_balancer_from_url_maps( url_maps, backend_services, backend_buckets, project_id) lbs_from_target_pool = self.get_load_balancer_from_target_pools( target_pools, project_id) load_balancers.extend(lbs_from_proxy) load_balancers.extend(lbs_from_url_map) load_balancers.extend(lbs_from_target_pool) for load_balancer in load_balancers: lb_type = load_balancer.get('lb_type') health_checks_vo = load_balancer.get('heath_check_vos', {}) health_self_links = health_checks_vo.get( 'health_check_self_link_list', []) ################################## # Set Target Proxies ################################## if lb_type != 'target_proxy': matched_target_proxies, matched_certificates = self.get_matched_target_proxies( load_balancer, target_proxies, ssl_certificates) load_balancer.update({ 'target_proxies': matched_target_proxies, 'certificates': matched_certificates }) ################################## # Set forwarding Rules to Load Balancer ################################## matched_forwarding_rules = self.get_matched_forwarding_rules( load_balancer, forwarding_rules) load_balancer.update( {'forwarding_rules': matched_forwarding_rules}) ################################## # Set Health Check to Load Balancer ################################## if len(health_self_links) > 0: filter_check_list = list( set(health_checks_vo.get('health_check_list', []))) filter_check_self_link_list = list( set(health_checks_vo.get('health_check_self_link_list', []))) matched_health_list = self._get_matched_health_checks( filter_check_self_link_list, health_checks) if len(matched_health_list) == len(filter_check_list): load_balancer['heath_check_vos'].update({ 'health_check_list': filter_check_list, 'health_check_self_link_list': filter_check_self_link_list, 'health_checks': matched_health_list }) else: matched_health_legacy_list = self._get_matched_health_checks( filter_check_self_link_list, legacy_health_checks) matched_health_list.extend(matched_health_legacy_list) load_balancer['heath_check_vos'].update({ 'health_check_list': filter_check_list, 'health_check_self_link_list': filter_check_self_link_list, 'health_checks': matched_health_list }) ############################ # Set Front to Load Balancer ############################ frontends = self.get_front_from_loadbalancer(load_balancer) frontend_display = self._get_frontend_display(frontends) if len(frontends) > 0: load_balancer.update({ 'frontends': frontends, 'frontend_display': frontend_display }) ############################# # Set Backend to Load Balancer ############################# backend_vo = {} if lb_type in ['target_pool']: backend_vo.update({ 'type': 'target_pool', 'target_pool_backend': self.get_backend_from_target_pools(load_balancer, instance_groups) }) elif lb_type in ['url_map', 'target_proxy']: key = 'proxy_backend' if lb_type == 'target_proxy' else 'url_map_backend' backends = self.get_backend_from_url_map_and_proxy( load_balancer, instance_groups, auto_scalers) backend_vo.update({ 'type': 'proxy' if lb_type == 'target_proxy' else 'url_map', key: backends }) load_balancer.update({'backends': backend_vo}) ######################################## # Set Backend Tab to LoadBlancer ######################################## backends_tab = self._get_backend_tabs(load_balancer) load_balancer.update({'backend_tabs': backends_tab}) ######################################## # Set Backend Display ######################################## backend_display = self._get_backend_display(load_balancer) load_balancer.update({'backends_display': backend_display}) ''' Get Appropriate Region & Protocols Protocols - 1. Frontend's forwarding Maps 2. Backend's end protocol Region - backend-svc's backend ''' lead_protocol = self._get_lead_protocol(load_balancer) region = self._get_proper_region(load_balancer) load_balancer.update({ 'lead_protocol': lead_protocol, 'region': region }) refer_link = self._get_refer_link(load_balancer, project_id) load_balance_data = LoadBalancing(load_balancer, strict=False) lb_resource = LoadBalancingResource({ 'region_code': region, 'data': load_balance_data, 'reference': ReferenceModel(load_balance_data.reference(refer_link)) }) self.set_region_code(region) collected_cloud_services.append( LoadBalancingResponse({'resource': lb_resource})) print( f'** Load Balancing Finished {time.time() - start_time} Seconds **' ) return collected_cloud_services