def setUp(self) -> None:
load_dotenv()
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(TESTVARIABLES)
logging_level = int(self.config['LOGGING']['level'])
logging.basicConfig(level=logging_level)
logging.debug("Current working directory {}".format(os.getcwd()))
with change_dir(OPERATIONSPATH):
logging.debug("Current change_dir working directory {}".format(
os.getcwd()))
self.config = ConfigParser()
self.config.read(TESTVARIABLES)
def __init__(self, personal_access_token):
load_dotenv()
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(CONFVARIABLES)
self.personal_access_token = personal_access_token
def test_correlate_management_grp_data(self):
parquet_file = "TestingCorrelation.parquet"
xlsx_file = "TestingCorrelation.xlsx"
with change_dir(OPERATIONSPATH):
if os.path.isfile(parquet_file):
os.remove(parquet_file)
subscription = os.environ["SUB"]
management_grp = os.environ["MANGRP"]
credentials = AzureConnections().get_authenticated_client()
df = EvaluatePolicies(credentials).correlate_management_grp_data(
management_grp=management_grp, subscription=subscription)
df = df.drop(columns=['serialize', 'timestamp'])
df.to_excel(xlsx_file)
pm = pluggy.PluginManager("Analysis")
pm.add_hookspecs(DatbasePlugins)
pm.register(SQLServerDatabase())
engine_list = pm.hook.get_db_engine(
driver=self.driver,
server=self.server,
database=self.database,
user=self.user,
password=self.password,
)
engine = engine_list[0]
df.to_sql('test_noncompliant_002',
con=engine,
if_exists='append',
chunksize=1000)
def __init__(self, db_server_instance="instance01"):
load_dotenv()
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(CONFVARIABLES)
sql_instance = self.config['SQLSERVER'][db_server_instance]
sql_instance = json.loads(sql_instance.replace("\'", "\""))
self.server = sql_instance['server']
self.database = sql_instance['database']
self.username = sql_instance['username']
self.db_driver = sql_instance['db_driver']
self.dialect = sql_instance['dialect']
password = os.environ['DATABASEPWD']
self.baseDb = BaseDb(server=self.server,
database=self.database,
username=self.username,
driver=self.db_driver,
dialect=self.dialect,
password=password)
self.engine = self.baseDb.get_db_engine()
self.Session = sessionmaker(bind=self.engine)
def __init__(self, credentials=None):
load_dotenv()
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(CONFVARIABLES)
self.credentials = AzureConnections().get_authenticated_client()
def test_directory_context_manager(self):
"""Test the creation of the app.config.ini file and the test.app.config.ini"""
subscription_id = os.environ["SUB"]
tenant_id = os.environ["TENANT"]
with change_dir(OPERATIONSPATH):
create_baseline_configuration()
self.assertEqual(os.path.isfile("app.config.ini"), True)
self.assertEqual(os.path.isfile("test.app.config.ini"), True)
def __init__(self):
load_dotenv()
self.credentials = AzureConnections().get_authenticated_client()
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(CONFVARIABLES)
self.subscriptions = Subscriptions().list_displayname_and_id()
self.resource_cls = Resource()
def setUp(self) -> None:
load_dotenv()
logging.basicConfig(level=logging.DEBUG)
logging.debug("Current working directory {}".format(os.getcwd()))
with change_dir(OPERATIONSPATH):
logging.debug(
"Current change_dir working directory {}".format(os.getcwd())
)
self.config = ConfigParser()
self.config.read(CONFVARIABLES)
def __init__(self, credentials):
load_dotenv()
self.credentials = credentials
self.subscriptions = None
self.policy_insights = None
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(CONFVARIABLES)
def __init__(self):
load_dotenv()
self.credentials = AzureConnections().get_authenticated_client()
self.subscription_client = SubscriptionClient(self.credentials, base_url=None)
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(CONFVARIABLES)
logging_level = int(self.config['LOGGING']['level'])
logging.basicConfig(level=logging_level)
def run(self, plugin_name):
plugins = self.config['PLUGINS']
root_path = self.config["DEFAULT"]["plugin_root_path"]
with change_dir(OPERATIONSPATH):
path = '{}/{}'.format(os.getcwd(), root_path)
for plugin in plugins:
if re.match('plugin_', plugin, re.IGNORECASE):
self.plugins[plugin] = self.config['PLUGINS'][plugin]
return path
def __init__(self, client_id, key, tenant_id, url, cosmos_key):
load_dotenv()
self.url = url
self.__comos_key = cosmos_key
self.cosmos_client = cosmos_client.CosmosClient(url, cosmos_key)
self.credentials = AzureConnections().authenticate_device_code(
CLIENT=client_id, KEY=key, TENANT_ID=tenant_id)
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(CONFVARIABLES)
def setUp(self) -> None:
load_dotenv()
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(TESTVARIABLES)
load_dotenv()
sql_instance = self.config['SQLSERVER']['instance01']
sql_instance = json.loads(sql_instance.replace("\'", "\""))
self.server = sql_instance['server']
self.database = sql_instance['database']
self.username = sql_instance['username']
self.db_driver = sql_instance['db_driver']
self.dialect = sql_instance['dialect']
password = os.environ['DATABASEPWD']
def setUp(self) -> None:
load_dotenv()
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(TESTVARIABLES)
# The driver often needs to be obtained from the database publisher
self.driver = "{ODBC Driver 17 for SQL Server}"
# Server is the IP address or DNS of the database server
self.server = "172.17.0.1"
# Can be any database name, as long as you are consistent
self.database = "TestDB2"
# Never place passwords in code. Your just asking for trouble otherwise
self.password = os.environ["DATABASEPWD"]
# Do not use SA in production
self.user = "******"
def __init__(self):
load_dotenv()
self.credentials = AzureConnections().get_authenticated_client()
# Not the default path is debug. This exists in order to fail safe. Production must be explicit.
if 'mop_debug' is os.environ and os.environ['mop_debug'].lower() in 'false':
config_file = CONFVARIABLES
else:
config_file = TESTVARIABLES
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(config_file)
logging_level = int(self.config['LOGGING']['level'])
logging.basicConfig(level=logging_level)
def __init__(self, driver, database, dialect, password, server, username, configuration_file="CONFVARIABLES"):
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
if "CONFVARIABLES" in configuration_file:
self.config.read(CONFVARIABLES)
else:
self.config.read(TESTVARIABLES)
self.driver = driver
self.dialect = dialect
self.server = server
self.database = database
self.username = username
self.password = password
self.get_db_engine()
def get_json_policy_definitions(self):
paths = list()
if os.getcwd().endswith("tests"):
search_path = "../comprehension/resource_management/policydefinitions"
else:
search_path = "policydefinitions"
with change_dir(search_path):
for file in glob.glob("*.json"):
policy_definition_path = os.path.abspath(file)
base_name = os.path.basename(policy_definition_path)
policy_defintion_name = os.path.splitext(base_name)[0]
if not os.path.isfile(policy_definition_path):
raise FileNotFoundError
paths.append({
"policy_defintion_name": policy_defintion_name,
"policy_definition_path": policy_definition_path
})
return paths
def __init__(self,
filter_name_regex=None,
filter_category=None,
semaphore=10):
load_dotenv()
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(CONFVARIABLES)
self.policy_assignments_create_api = self.config["AZURESDK"][
"policy_assignments_create"]
self.queue = asyncio.Queue()
self.loop = asyncio.get_event_loop()
self.semaphore = asyncio.BoundedSemaphore(semaphore)
self.filter_name_regex = filter_name_regex
self.filter_category_regex = filter_category
self.queue_url = 'default'
self.credential = 'default'
def test_create_config_file_sections(self):
tmpTESTVARIABLES = "tmp_" + TESTVARIABLES
config = ConfigParser()
# An active subscription is generally needed for this application
# However, subscription ids should be guarded. Hence, we use an .env value
config["DEFAULT"] = {"subscription": self.subscription}
config["AZURESDK"] = {
"PolicyStatesSummarizeForPolicyDefinition":
"https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.Authorization/policyDefinitions/{policyDefinitionName}/providers/Microsoft.PolicyInsights/policyStates/latest/summarize?api-version=2019-10-01",
"PolicyStatesSummarizeForSubscription":
"https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.PolicyInsights/policyStates/latest/summarize?api-version=2019-10-01",
"PolicyStatesSummarizeForSubscriptionFiltered":
"https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.PolicyInsights/policyStates/latest/summarize?api-version=2019-10-01&$filter={filter}",
}
with change_dir(TESTINGPATH):
with atomic_write(tmpTESTVARIABLES, "w") as configfile:
config.write(configfile)
self.assertEqual(os.path.isfile(tmpTESTVARIABLES), True)
# This is a temporary local test file because the working directory for the test has not been altered
if os.path.isfile(tmpTESTVARIABLES):
os.remove(tmpTESTVARIABLES)
def __init__(self, db_server_instance="instance01"):
load_dotenv()
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(CONFVARIABLES)
sql_instance = self.config['SQLSERVER'][db_server_instance]
sql_instance = json.loads(sql_instance.replace("\'", "\""))
self.server = sql_instance['server']
self.database = sql_instance['database']
self.username = sql_instance['username']
self.driver = sql_instance['db_driver']
self.dialect = sql_instance['dialect']
self.password = os.environ['DATABASEPWD']
self.get_db_engine()
self.Session = sessionmaker(bind=self.engine)
url = self.config['COSMOSDB']['uri_01']
key = os.environ['COSMODB_KEY']
self.CosmosDBclient = CosmosClient(url, credential=key)
def setUp(self) -> None:
load_dotenv()
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(TESTVARIABLES)
def __init__(self):
load_dotenv()
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(CONFVARIABLES)
def subscription_policy_compliance(subscriptionId):
with change_dir(OPERATIONSPATH):
config = ConfigParser()
config.read(CONFVARIABLES)
jmespath_expression = jmespath.compile(
"value[*].policyAssignments[*].policyDefinitions[*]")
jmespath_results = jmespath.compile("[0].results.resourceDetails")
scour_policy = PolicyStates()
response = scour_policy.policy_states_summarize_for_subscription(
subscriptionId)
if response is None:
response = scour_policy.policy_states_summarize_for_subscription(
subscriptionId)
# Execute returns a method the can be executed anywhere more than once
result = response.json()
df = pd.DataFrame(columns=[
'subscription_id', 'policy_definition_name', 'policy_definition_id',
'compliant', 'noncompliant', 'total_resources_measured',
'percent_compliant'
])
jmes_result = jmespath_expression.search(result)
if jmespath_expression is None or jmes_result is None or jmes_result[
0] is None:
return
else:
# flatten results
try:
policyresults = jmes_result[0][0]
for policyresult in policyresults:
policy_definition_name = str(
policyresult['policyDefinitionId']).split('/')[-1]
resourceDetails = jmespath.search('results.resourceDetails[*]',
policyresult)
compliance_ratio = determine_compliance_ratio(resourceDetails)
new_row = {
'subscription_id':
subscriptionId,
'policy_definition_name':
policy_definition_name,
'policy_definition_id':
policyresult['policyDefinitionId'],
'compliant':
compliance_ratio['compliant'],
'noncompliant':
compliance_ratio['noncompliant'],
'total_resources_measured':
compliance_ratio['total_resources_measured'],
'percent_compliant':
compliance_ratio['percent_compliant'],
}
df = df.append(new_row, ignore_index=True)
except IndexError:
return None
return df
def test_read_testvariables_ini(self):
with change_dir(TESTINGPATH):
config = ConfigParser()
print(config.read(TESTVARIABLES))
print(config["DEFAULT"]["subscription_id"])
def create_management_grp_policy_assignments(self, management_grp, subscription_id):
"""
This method create a policies within a management group per subscription
The subscription id in required to make a query against the Python SDK when querying a management group.
Looking for alternatives.
:param management_grp:
:param subscription_id:
:return:
"""
with change_dir(OPERATIONSPATH):
config = ConfigParser()
config.read(CONFVARIABLES)
management_client = ManagementGroupsAPI(self.credentials)
mngrp_subscriptions = management_client.entities.list(group_name=management_grp)
for subscription in mngrp_subscriptions:
if (
"/providers/Microsoft.Management/managementGroups"
not in subscription.type
):
policy_client = PolicyClient(
credentials=self.credentials,
subscription_id=subscription_id,
base_url=None,
)
policies = policy_client.policy_definitions.list_by_management_group(
management_grp
)
scope = "/subscriptions/{subscriptionId}".format(
subscriptionId=subscription.name
)
for policy in policies:
if (
not policy.metadata.get("category") is None
and policy.metadata.get("category") in config["FILTERS"]["policy_defition_category"]
):
assignment = PolicyAssignment(
display_name=policy.display_name,
policy_definition_id=policy.id,
scope=scope,
parameters=None,
description=policy.description,
)
policy_assignment_name = "{}-assignment".format(
assignment.display_name
)
try:
result = policy_client.policy_assignments.create(
scope=scope,
policy_assignment_name=policy_assignment_name,
parameters=assignment,
)
if result:
print(
"Subscription deployed: {}".format(
subscription.name,
policy_assignment_name,
policy.display_name,
)
)
except ErrorResponseException:
print(
"Subcription failed: {}".format(
subscription.name,
policy_assignment_name,
policy.display_name,
)
)
