def setUp(self) -> None: load_dotenv() with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(TESTVARIABLES) logging_level = int(self.config['LOGGING']['level']) logging.basicConfig(level=logging_level) logging.debug("Current working directory {}".format(os.getcwd())) with change_dir(OPERATIONSPATH): logging.debug("Current change_dir working directory {}".format( os.getcwd())) self.config = ConfigParser() self.config.read(TESTVARIABLES)
def __init__(self, personal_access_token): load_dotenv() with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(CONFVARIABLES) self.personal_access_token = personal_access_token
def test_correlate_management_grp_data(self): parquet_file = "TestingCorrelation.parquet" xlsx_file = "TestingCorrelation.xlsx" with change_dir(OPERATIONSPATH): if os.path.isfile(parquet_file): os.remove(parquet_file) subscription = os.environ["SUB"] management_grp = os.environ["MANGRP"] credentials = AzureConnections().get_authenticated_client() df = EvaluatePolicies(credentials).correlate_management_grp_data( management_grp=management_grp, subscription=subscription) df = df.drop(columns=['serialize', 'timestamp']) df.to_excel(xlsx_file) pm = pluggy.PluginManager("Analysis") pm.add_hookspecs(DatbasePlugins) pm.register(SQLServerDatabase()) engine_list = pm.hook.get_db_engine( driver=self.driver, server=self.server, database=self.database, user=self.user, password=self.password, ) engine = engine_list[0] df.to_sql('test_noncompliant_002', con=engine, if_exists='append', chunksize=1000)
def __init__(self, db_server_instance="instance01"): load_dotenv() with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(CONFVARIABLES) sql_instance = self.config['SQLSERVER'][db_server_instance] sql_instance = json.loads(sql_instance.replace("\'", "\"")) self.server = sql_instance['server'] self.database = sql_instance['database'] self.username = sql_instance['username'] self.db_driver = sql_instance['db_driver'] self.dialect = sql_instance['dialect'] password = os.environ['DATABASEPWD'] self.baseDb = BaseDb(server=self.server, database=self.database, username=self.username, driver=self.db_driver, dialect=self.dialect, password=password) self.engine = self.baseDb.get_db_engine() self.Session = sessionmaker(bind=self.engine)
def __init__(self, credentials=None): load_dotenv() with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(CONFVARIABLES) self.credentials = AzureConnections().get_authenticated_client()
def test_directory_context_manager(self): """Test the creation of the app.config.ini file and the test.app.config.ini""" subscription_id = os.environ["SUB"] tenant_id = os.environ["TENANT"] with change_dir(OPERATIONSPATH): create_baseline_configuration() self.assertEqual(os.path.isfile("app.config.ini"), True) self.assertEqual(os.path.isfile("test.app.config.ini"), True)
def __init__(self): load_dotenv() self.credentials = AzureConnections().get_authenticated_client() with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(CONFVARIABLES) self.subscriptions = Subscriptions().list_displayname_and_id() self.resource_cls = Resource()
def setUp(self) -> None: load_dotenv() logging.basicConfig(level=logging.DEBUG) logging.debug("Current working directory {}".format(os.getcwd())) with change_dir(OPERATIONSPATH): logging.debug( "Current change_dir working directory {}".format(os.getcwd()) ) self.config = ConfigParser() self.config.read(CONFVARIABLES)
def __init__(self, credentials): load_dotenv() self.credentials = credentials self.subscriptions = None self.policy_insights = None with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(CONFVARIABLES)
def __init__(self): load_dotenv() self.credentials = AzureConnections().get_authenticated_client() self.subscription_client = SubscriptionClient(self.credentials, base_url=None) with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(CONFVARIABLES) logging_level = int(self.config['LOGGING']['level']) logging.basicConfig(level=logging_level)
def run(self, plugin_name): plugins = self.config['PLUGINS'] root_path = self.config["DEFAULT"]["plugin_root_path"] with change_dir(OPERATIONSPATH): path = '{}/{}'.format(os.getcwd(), root_path) for plugin in plugins: if re.match('plugin_', plugin, re.IGNORECASE): self.plugins[plugin] = self.config['PLUGINS'][plugin] return path
def __init__(self, client_id, key, tenant_id, url, cosmos_key): load_dotenv() self.url = url self.__comos_key = cosmos_key self.cosmos_client = cosmos_client.CosmosClient(url, cosmos_key) self.credentials = AzureConnections().authenticate_device_code( CLIENT=client_id, KEY=key, TENANT_ID=tenant_id) with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(CONFVARIABLES)
def setUp(self) -> None: load_dotenv() with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(TESTVARIABLES) load_dotenv() sql_instance = self.config['SQLSERVER']['instance01'] sql_instance = json.loads(sql_instance.replace("\'", "\"")) self.server = sql_instance['server'] self.database = sql_instance['database'] self.username = sql_instance['username'] self.db_driver = sql_instance['db_driver'] self.dialect = sql_instance['dialect'] password = os.environ['DATABASEPWD']
def setUp(self) -> None: load_dotenv() with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(TESTVARIABLES) # The driver often needs to be obtained from the database publisher self.driver = "{ODBC Driver 17 for SQL Server}" # Server is the IP address or DNS of the database server self.server = "172.17.0.1" # Can be any database name, as long as you are consistent self.database = "TestDB2" # Never place passwords in code. Your just asking for trouble otherwise self.password = os.environ["DATABASEPWD"] # Do not use SA in production self.user = "******"
def __init__(self): load_dotenv() self.credentials = AzureConnections().get_authenticated_client() # Not the default path is debug. This exists in order to fail safe. Production must be explicit. if 'mop_debug' is os.environ and os.environ['mop_debug'].lower() in 'false': config_file = CONFVARIABLES else: config_file = TESTVARIABLES with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(config_file) logging_level = int(self.config['LOGGING']['level']) logging.basicConfig(level=logging_level)
def __init__(self, driver, database, dialect, password, server, username, configuration_file="CONFVARIABLES"): with change_dir(OPERATIONSPATH): self.config = ConfigParser() if "CONFVARIABLES" in configuration_file: self.config.read(CONFVARIABLES) else: self.config.read(TESTVARIABLES) self.driver = driver self.dialect = dialect self.server = server self.database = database self.username = username self.password = password self.get_db_engine()
def get_json_policy_definitions(self): paths = list() if os.getcwd().endswith("tests"): search_path = "../comprehension/resource_management/policydefinitions" else: search_path = "policydefinitions" with change_dir(search_path): for file in glob.glob("*.json"): policy_definition_path = os.path.abspath(file) base_name = os.path.basename(policy_definition_path) policy_defintion_name = os.path.splitext(base_name)[0] if not os.path.isfile(policy_definition_path): raise FileNotFoundError paths.append({ "policy_defintion_name": policy_defintion_name, "policy_definition_path": policy_definition_path }) return paths
def __init__(self, filter_name_regex=None, filter_category=None, semaphore=10): load_dotenv() with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(CONFVARIABLES) self.policy_assignments_create_api = self.config["AZURESDK"][ "policy_assignments_create"] self.queue = asyncio.Queue() self.loop = asyncio.get_event_loop() self.semaphore = asyncio.BoundedSemaphore(semaphore) self.filter_name_regex = filter_name_regex self.filter_category_regex = filter_category self.queue_url = 'default' self.credential = 'default'
def test_create_config_file_sections(self): tmpTESTVARIABLES = "tmp_" + TESTVARIABLES config = ConfigParser() # An active subscription is generally needed for this application # However, subscription ids should be guarded. Hence, we use an .env value config["DEFAULT"] = {"subscription": self.subscription} config["AZURESDK"] = { "PolicyStatesSummarizeForPolicyDefinition": "https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.Authorization/policyDefinitions/{policyDefinitionName}/providers/Microsoft.PolicyInsights/policyStates/latest/summarize?api-version=2019-10-01", "PolicyStatesSummarizeForSubscription": "https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.PolicyInsights/policyStates/latest/summarize?api-version=2019-10-01", "PolicyStatesSummarizeForSubscriptionFiltered": "https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.PolicyInsights/policyStates/latest/summarize?api-version=2019-10-01&$filter={filter}", } with change_dir(TESTINGPATH): with atomic_write(tmpTESTVARIABLES, "w") as configfile: config.write(configfile) self.assertEqual(os.path.isfile(tmpTESTVARIABLES), True) # This is a temporary local test file because the working directory for the test has not been altered if os.path.isfile(tmpTESTVARIABLES): os.remove(tmpTESTVARIABLES)
def __init__(self, db_server_instance="instance01"): load_dotenv() with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(CONFVARIABLES) sql_instance = self.config['SQLSERVER'][db_server_instance] sql_instance = json.loads(sql_instance.replace("\'", "\"")) self.server = sql_instance['server'] self.database = sql_instance['database'] self.username = sql_instance['username'] self.driver = sql_instance['db_driver'] self.dialect = sql_instance['dialect'] self.password = os.environ['DATABASEPWD'] self.get_db_engine() self.Session = sessionmaker(bind=self.engine) url = self.config['COSMOSDB']['uri_01'] key = os.environ['COSMODB_KEY'] self.CosmosDBclient = CosmosClient(url, credential=key)
def setUp(self) -> None: load_dotenv() with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(TESTVARIABLES)
def __init__(self): load_dotenv() with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(CONFVARIABLES)
def subscription_policy_compliance(subscriptionId): with change_dir(OPERATIONSPATH): config = ConfigParser() config.read(CONFVARIABLES) jmespath_expression = jmespath.compile( "value[*].policyAssignments[*].policyDefinitions[*]") jmespath_results = jmespath.compile("[0].results.resourceDetails") scour_policy = PolicyStates() response = scour_policy.policy_states_summarize_for_subscription( subscriptionId) if response is None: response = scour_policy.policy_states_summarize_for_subscription( subscriptionId) # Execute returns a method the can be executed anywhere more than once result = response.json() df = pd.DataFrame(columns=[ 'subscription_id', 'policy_definition_name', 'policy_definition_id', 'compliant', 'noncompliant', 'total_resources_measured', 'percent_compliant' ]) jmes_result = jmespath_expression.search(result) if jmespath_expression is None or jmes_result is None or jmes_result[ 0] is None: return else: # flatten results try: policyresults = jmes_result[0][0] for policyresult in policyresults: policy_definition_name = str( policyresult['policyDefinitionId']).split('/')[-1] resourceDetails = jmespath.search('results.resourceDetails[*]', policyresult) compliance_ratio = determine_compliance_ratio(resourceDetails) new_row = { 'subscription_id': subscriptionId, 'policy_definition_name': policy_definition_name, 'policy_definition_id': policyresult['policyDefinitionId'], 'compliant': compliance_ratio['compliant'], 'noncompliant': compliance_ratio['noncompliant'], 'total_resources_measured': compliance_ratio['total_resources_measured'], 'percent_compliant': compliance_ratio['percent_compliant'], } df = df.append(new_row, ignore_index=True) except IndexError: return None return df
def test_read_testvariables_ini(self): with change_dir(TESTINGPATH): config = ConfigParser() print(config.read(TESTVARIABLES)) print(config["DEFAULT"]["subscription_id"])
def create_management_grp_policy_assignments(self, management_grp, subscription_id): """ This method create a policies within a management group per subscription The subscription id in required to make a query against the Python SDK when querying a management group. Looking for alternatives. :param management_grp: :param subscription_id: :return: """ with change_dir(OPERATIONSPATH): config = ConfigParser() config.read(CONFVARIABLES) management_client = ManagementGroupsAPI(self.credentials) mngrp_subscriptions = management_client.entities.list(group_name=management_grp) for subscription in mngrp_subscriptions: if ( "/providers/Microsoft.Management/managementGroups" not in subscription.type ): policy_client = PolicyClient( credentials=self.credentials, subscription_id=subscription_id, base_url=None, ) policies = policy_client.policy_definitions.list_by_management_group( management_grp ) scope = "/subscriptions/{subscriptionId}".format( subscriptionId=subscription.name ) for policy in policies: if ( not policy.metadata.get("category") is None and policy.metadata.get("category") in config["FILTERS"]["policy_defition_category"] ): assignment = PolicyAssignment( display_name=policy.display_name, policy_definition_id=policy.id, scope=scope, parameters=None, description=policy.description, ) policy_assignment_name = "{}-assignment".format( assignment.display_name ) try: result = policy_client.policy_assignments.create( scope=scope, policy_assignment_name=policy_assignment_name, parameters=assignment, ) if result: print( "Subscription deployed: {}".format( subscription.name, policy_assignment_name, policy.display_name, ) ) except ErrorResponseException: print( "Subcription failed: {}".format( subscription.name, policy_assignment_name, policy.display_name, ) )