def run_sample():
AUTH_URI = os.environ.get("ACCOUNT_URI")
AUTH_KEY = os.environ.get("ACCOUNT_KEY")
DATABASE_ID = "testdocumentmanagementdb"
CONTAINER_ID = "testdocumentmanagementcollection"
client = CosmosClient(AUTH_URI, AUTH_KEY)
database = client.create_database(id=DATABASE_ID)
partition_key = PartitionKey(path="/purchase_order_number")
try:
container = database.create_container(id=CONTAINER_ID,
partition_key=partition_key)
print(f'Container with id "{CONTAINER_ID}"" created')
except HTTPFailure as e:
if e.status_code == 409:
print(f"Container with id {CONTAINER_ID} already exists")
container = database.get_container(container=CONTAINER_ID)
else:
raise
DocumentManagement.create_documents(container)
DocumentManagement.read_document(container, "SalesOrder1")
DocumentManagement.read_documents(container)
client.delete_database(database=DATABASE_ID)
print("\nrun_sample done")
def main(req: func.HttpRequest) -> func.HttpResponse:
URL = "https://junctionx.documents.azure.com:443/"
KEY = "BCw9nhNiTSl9ndUokCw6uHhzZSDgsSGoMjPUa0Ech6E9yiuGKCrsswFIXRatcguQYcNLnYdhFPWPhuaRtLfoxg=="
DATABASE_NAME = "MitoSoftCorp"
CONTAINER_NAME = "workType"
client = CosmosClient(URL, credential=KEY)
work_type = req.params.get("work_type")
try:
database = client.create_database(DATABASE_NAME)
except exceptions.CosmosResourceExistsError:
database = client.get_database_client(DATABASE_NAME)
try:
container = database.create_container(
id=CONTAINER_NAME, partition_key=PartitionKey(path="/name"))
except exceptions.CosmosResourceExistsError:
container = database.get_container_client(CONTAINER_NAME)
user_list = []
for item in container.query_items(
query='SELECT * FROM {} w WHERE w.work_type = "{}"'.format(
CONTAINER_NAME, work_type),
enable_cross_partition_query=True):
user_list.append(item.get("user_id"))
return_data = dict(id=user_list)
return func.HttpResponse(json.dumps(return_data),
headers={"Content-Type": "application/json"},
status_code=200)
def UploadCosmos(guid, jsondata):
client = CosmosClient(cosmosuri, credential=cosmoskey)
database_name = 'cosmosnetatmo'
container_name = 'mesures'
try: # create database if not exists
database = client.create_database(database_name)
except exceptions.CosmosResourceExistsError:
database = client.get_database_client(database_name)
try: # create collection if not exists
container = database.create_container(
id=container_name, partition_key=PartitionKey(path="/id"))
except exceptions.CosmosResourceExistsError:
container = database.get_container_client(container_name)
except exceptions.CosmosHttpResponseError:
raise
#Upsert after adding a guid as key to the document
jsondata["id"] = guid
container.upsert_item(jsondata)
def run_sample():
AUTH_URL = os.environ.get("ACCOUNT_HOST")
AUTH_KEY = os.environ.get("ACCOUNT_KEY")
DATABASE_ID = 'testdocumentmanagementdb'
CONTAINER_ID = 'testdocumentmanagementcollection'
client = CosmosClient(AUTH_URL, AUTH_KEY)
database = client.create_database(id=DATABASE_ID)
try:
container = database.create_container(id=CONTAINER_ID)
print(f'Container with id "{CONTAINER_ID}"" created')
except HTTPFailure as e:
if e.status_code == 409:
print(f'Container with id {CONTAINER_ID} already exists')
container = database.get_container(container=CONTAINER_ID)
else:
raise
DocumentManagement.create_documents(container)
DocumentManagement.read_document(container, 'SalesOrder1')
DocumentManagement.read_documents(container)
client.delete_database(database=DATABASE_ID)
print("\nrun_sample done")
from azure.cosmos import HTTPFailure, CosmosClient, Container, Database
# It all starts with a client instance:
import os
url = os.environ['ACCOUNT_HOST']
key = os.environ['ACCOUNT_KEY']
client = CosmosClient(url, key)
test_database_name = 'testDatabase'
test_container_name = 'testContainer'
#### Create environnment ###
db = client.create_database(id=test_database_name, fail_if_exists=False)
try:
db.create_container(id=test_container_name)
except HTTPFailure as e:
if e.status_code != 409:
raise
db.get_container(test_container_name)
####
# In order to retreive a container where you know the database name and container name:
container = Container(client.client_context,
database=test_database_name,
id=test_container_name)
container.upsert_item({'id': 'something', 'value': 'else'})
# If you want to walk down the hierarchy, you can also get it from the client->databases->containers
database = client.get_database(test_database_name)
container = database.get_container(test_container_name)
container.upsert_item({'id': 'something', 'value': 'new'})
from azure.cosmos import CosmosClient, PartitionKey, exceptions
# Create the cosmos client
config = {"endpoint": "", "primarykey": ""}
client = CosmosClient(config["endpoint"], config["primarykey"])
# Create database
database_name = 'testDatabase'
try:
database = client.create_database(database_name)
except exceptions.CosmosResourceExistsError:
database = client.get_database_client(database_name)
# Create Container
container_name = 'testContainer'
try:
container = database.create_container(
id=container_name, partition_key=PartitionKey(path="/country"))
except exceptions.CosmosResourceExistsError:
container = database.get_container_client(container_name)
except exceptions.CosmosHttpResponseError:
raise
database_client = client.get_database_client(database_name)
container_client = database.get_container_client(container_name)
from azure.cosmos import CosmosClient, PartitionKey, exceptions
import os
from dotenv import load_dotenv
load_dotenv()
DATABASE_NAME = 'testDb'
CONTAINER_NAME = 'products'
COSMOS_URL = os.environ['COSMOS_URL']
COSMOS_KEY = os.environ['COSMOS_KEY']
client = CosmosClient(COSMOS_URL, credential=COSMOS_KEY)
try:
database = client.create_database(DATABASE_NAME)
except exceptions.CosmosResourceExistsError:
database = client.get_database_client(DATABASE_NAME)
print(
f"### Connected to {client.client_connection.url_connection}/{database.id}"
)
try:
container = database.create_container(
id=CONTAINER_NAME, partition_key=PartitionKey(path="/productName"))
except exceptions.CosmosResourceExistsError:
container = database.get_container_client(CONTAINER_NAME)
except exceptions.CosmosHttpResponseError:
raise
container_client = database.get_container_client(CONTAINER_NAME)
class CosmosDB:
def __init__(self):
URL = os.environ["COSMOS_ENDPOINT"]
KEY = os.environ["COSMOS_KEY"]
self.client = CosmosClient(URL, {"masterKey": KEY})
self.dbName = "pySolarSystem-" + uuid.uuid1().hex
def create_database(self):
print("Creating '{0}' database...".format(self.dbName))
return self.client.create_database(self.dbName)
def create_container(self, db):
collectionName = "Planets"
print("Creating '{0}' collection...".format(collectionName))
partition_key = PartitionKey(path="/id", kind="Hash")
return db.create_container(id="Planets", partition_key=partition_key)
def create_documents(self, container):
# Cosmos will look for an 'id' field in the items, if the 'id' is not specify Cosmos is going to assing a random key.
planets = [
{
"id": "Earth",
"HasRings": False,
"Radius": 3959,
"Moons": [{"Name": "Moon"}],
},
{
"id": "Mars",
"HasRings": False,
"Radius": 2106,
"Moons": [{"Name": "Phobos"}, {"Name": "Deimos"}],
},
]
print("Inserting items in the collection...")
for planet in planets:
container.create_item(planet)
print("\t'{0}' created".format(planet["id"]))
print("\tdone")
def simple_query(self, container):
print("Quering the container...")
items = list(
container.query_items(
query="SELECT c.id FROM c", enable_cross_partition_query=True
)
)
print("\tdone: {0}".format(items))
def delete_database(self):
print("Cleaning up the resource...")
self.client.delete_database(self.dbName)
print("\tdone")
def run(self):
print("")
print("------------------------")
print("Cosmos DB")
print("------------------------")
print("1) Create a Database")
print("2) Create a Container in the database")
print("3) Insert Documents (items) into the Container")
print("4) Delete Database (Clean up the resource)")
print("")
# Ensure that the database does not exists
try:
self.delete_database()
except:
pass
try:
db = self.create_database()
container = self.create_container(db=db)
self.create_documents(container=container)
self.simple_query(container=container)
finally:
# if something goes wrong, the resource should be cleaned anyway
self.delete_database()
cred = DefaultAzureCredential()
secret_client = SecretClient(vault_url=kv_endpoint, credential=cred)
retrieved_secret = secret_client.get_secret(secret_name)
# Connect to Azure Cosmos DB
cosmos_account = os.environ['COSMOS_ACCOUNT_NAME']
url = f"https://{cosmos_account}.documents.azure.com:443/"
client = CosmosClient(url,
retrieved_secret.value,
consistency_level=cons_level)
print(f"[{cosmos_account}] Connected successfully to Azure Cosmos DB account.")
# Create/Get Database
try:
database = client.create_database(id=database_name,
offer_throughput=throuput_val)
print(
f"[{cosmos_account}] Created successfully a database. name[{database_name}]"
)
except exceptions.CosmosResourceExistsError:
database = client.get_database_client(database=database_name)
print(
f"[{cosmos_account}] Get successfully a database. name[{database_name}]"
)
# Create/Get Container
try:
container = database.create_container(
id=container_name, partition_key=PartitionKey(path=part_key))
print(
f"[{cosmos_account}] Created successfully a container. name[{container_name}]"
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
import time
import json
import geocoder
from azure.cosmos import CosmosClient, exceptions
url = "a"
key = "test"
client = CosmosClient(url, credential=key)
database = "hacktx_addresses"
try:
database = client.create_database(database)
except exceptions.CosmosResourceExistError:
database = client.get_database_client(database)
container_name = 'addresses'
container = database.get_container_client(container_name)
def get_coord(address):
#g = geocoder.google(address, key ='AIzaSyCogh5_2lxRy8SPG8rHaXaY0-umKhl1UyA')
return [30.285560, -97.752790]
#return g.latlong
def parse_text(text, photo_url, address):
txt = text.split("\n")
line1 = txt[0]
AUTH_URI = os.environ["ACCOUNT_URI"]
AUTH_KEY = os.environ["ACCOUNT_KEY"]
client = CosmosClient(url=AUTH_URI, key=AUTH_KEY)
def do_basic_stuff():
for database in client.list_databases():
print(database)
for container in database.list_containers():
print(container)
for item in container.list_items():
print(item)
try:
database = client.create_database("swaggers")
except HTTPFailure:
database = client.get_database("swaggers")
try:
database.delete_container('publicmaster')
except:
pass
try:
container = database.get_container(
"publicmaster") # TODO: Why not have a fail_if_exists on every create?
except HTTPFailure: # TODO: What is the appropriate exception here?
container = database.create_container(
"publicmaster", partition_key=PartitionKey('/info/version'))
class SummaryForPolicyDefinition(BaseDb):
def __init__(self, db_server_instance="instance01"):
load_dotenv()
with change_dir(OPERATIONSPATH):
self.config = ConfigParser()
self.config.read(CONFVARIABLES)
sql_instance = self.config['SQLSERVER'][db_server_instance]
sql_instance = json.loads(sql_instance.replace("\'", "\""))
self.server = sql_instance['server']
self.database = sql_instance['database']
self.username = sql_instance['username']
self.driver = sql_instance['db_driver']
self.dialect = sql_instance['dialect']
self.password = os.environ['DATABASEPWD']
self.get_db_engine()
self.Session = sessionmaker(bind=self.engine)
url = self.config['COSMOSDB']['uri_01']
key = os.environ['COSMODB_KEY']
self.CosmosDBclient = CosmosClient(url, credential=key)
def summarize_fact_compliance_for_definition(self, subscription_id,
policy_definition_name):
jmespath_expression = jmespath.compile(
"value[*].policyAssignments[*].policyDefinitions[*]")
policy_states = PolicyStates()
batch_uuid = uuid.uuid4()
created = datetime.datetime.utcnow()
# create a configured "Session" class
# create a Session
# Execute returns a method the can be executed anywhere more than once
models = self.get_db_model(self.engine)
FactCompliance = models.classes.factcompliance
subscriptions = models.classes.subscriptions
session = self.Session()
if subscription_id is not None:
subscriptions_list = session.query(subscriptions).filter_by(
subscription_id=subscription_id)
else:
subscriptions_list = session.query(subscriptions).all()
for subscription in subscriptions_list:
tenant_id = subscription.tenant_id
print("Subscription: {}, Policy Name {}".format(
subscription.subscription_id, policy_definition_name))
policy_states_of_definition = policy_states.policy_states_summarize_for_policy_definition(
subscriptionId=subscription.subscription_id,
policyDefinitionName=policy_definition_name).json()
jmes_result = jmespath_expression.search(
policy_states_of_definition)
if jmespath_expression is None or jmes_result is None or jmes_result[
0] is None or len(jmes_result[0]) == 0:
continue
else:
# flatten results
policyresults = jmes_result[0][0]
bulk_insert = list()
for policyresult in policyresults:
policy_definition_name = str(
policyresult['policyDefinitionId']).split('/')[-1]
resourceDetails = jmespath.search(
'results.resourceDetails[*]', policyresult)
compliance_ratio = self.determine_compliance_ratio(
resourceDetails)
fact = FactCompliance(
tenant_id=tenant_id,
subscription_id=subscription.subscription_id,
policy_definition_name=policy_definition_name,
compliant=compliance_ratio['compliant'],
noncompliant=compliance_ratio['noncompliant'],
total_resources_measured=compliance_ratio[
'total_resources_measured'],
percent_compliant=compliance_ratio[
'percent_compliant'],
batch_uuid=batch_uuid,
created=created,
modified=created)
bulk_insert.append(fact)
session.bulk_save_objects(bulk_insert)
session.commit()
def summarize_policy_defintions_for_all_management_grp(
self, management_group_id, policy_definition_name_like,
metadata_category):
"""
This method take the policy definition and find the statistics and saves it to the fact table
Submit a policy definition name without a wild card to pull back a single policy definition name
:param policy_definition_name:
"""
batch_uuid = uuid.uuid4()
created = datetime.datetime.utcnow()
database_name = 'policyinsights'
comsos_database = self.get_cosmodb(database_name)
container_name = 'policydefintionresults'
partition_key = "/{}".format(metadata_category.strip().lower().replace(
' ', ''))
self.cosmos_container_client = self.get_cosmosdb_container(
container_name, comsos_database, partition_key)
# create a configured "Session" class
# create a Session
# Execute returns a method the can be executed anywhere more than once
policy_definition_cls = PolicyDefinition()
models = self.get_db_model(self.engine)
subscriptions_model = models.classes.subscriptions
policy_definition_model = models.classes.policydefinitions
session = self.Session()
subscriptions = session.query(subscriptions_model).all()
policies = session.query(policy_definition_model).filter(
policy_definition_model.metadata_category == metadata_category,
policy_definition_model.policy_definition_name.like(
'security-pol-%')).all()
policy_compliance = PolicyCompliance()
for subscription in subscriptions:
for policy_definition in policies:
self.summarize_fact_compliance_for_definition(
category=metadata_category,
policy_definition_name=policy_definition.
policy_definition_name,
tenant_id=subscription.tenant_id,
subscription_id=subscription.subscription_id,
batch_uuid=batch_uuid)
def summarize_fact_compliance_for_definition(self, category,
policy_definition_name,
tenant_id, subscription_id,
batch_uuid):
"""
This method captures the results for a policy compliance within a single subscription
publishes results to non-SQL Cosmos DB
:param category:
:param policy_definition_name:
:param subscription_id:
"""
policy_states = PolicyStates()
created = datetime.datetime.utcnow()
# create a configured "Session" class
print("Subscription: {}, Policy Name {}".format(
subscription_id, policy_definition_name))
'''
Retreive the states for a policy and subscription
'''
policy_states_of_definition_response = policy_states.policy_states_summarize_for_policy_definition(
subscriptionId=subscription_id,
policyDefinitionName=policy_definition_name)
'''
The Rest API may fail to pull back states for many reasons.
Failed states are skipped for various reasons but mostly, some results are better than none
You are free to act differently.
'''
if not policy_states_of_definition_response.status_code in range(
200, 299):
print("Skipping with Error {} Subscription: {}, Policy Name {}".
format(policy_states_of_definition_response.status_code,
subscription_id, policy_definition_name))
return
policy_states_of_definitions_json = policy_states_of_definition_response.json(
)
if 'value' in policy_states_of_definitions_json:
policy_states_of_definitions = policy_states_of_definitions_json[
'value']
for policy_states_of_definition in policy_states_of_definitions:
policy_states_of_definition['category'] = category
policy_states_of_definition[
'policy_definition_name'] = policy_definition_name
policy_states_of_definition['tenant_id'] = tenant_id
policy_states_of_definition[
'subscription_id'] = subscription_id
policy_states_of_definition['batch_uuid'] = str(batch_uuid)
self.cosmos_container_client.upsert_item(
policy_states_of_definition)
else:
return 0
def get_cosmosdb_container(self, container_name, database, partition_key):
container = None
try:
container = database.create_container(
id=container_name,
partition_key=PartitionKey(path=partition_key))
except exceptions.CosmosResourceExistsError:
container = database.get_container_client(container_name)
except exceptions.CosmosHttpResponseError:
raise
finally:
if container is not None:
return container
def get_cosmodb(self, database_name):
try:
database = self.CosmosDBclient.create_database(database_name)
except exceptions.CosmosResourceExistsError:
database = self.CosmosDBclient.get_database_client(database_name)
return database
def determine_compliance_ratio(self, resourceDetails):
compliance_dict = dict()
if len(resourceDetails) <= 2:
complianceState = resourceDetails[0]
if complianceState['complianceState'] in 'compliant':
compliance_dict["compliant"] = int(complianceState["count"])
else:
compliance_dict["noncompliant"] = int(complianceState["count"])
if len(resourceDetails) == 2:
complianceState = resourceDetails[1]
if complianceState['complianceState'] in 'compliant':
compliance_dict["compliant"] = int(complianceState["count"])
else:
compliance_dict["noncompliant"] = int(complianceState["count"])
if not 'compliant' in compliance_dict:
compliance_dict["compliant"] = 0
if not 'noncompliant' in compliance_dict:
compliance_dict['noncompliant'] = 0
compliance_dict['total_resources_measured'] = compliance_dict[
"compliant"] + compliance_dict['noncompliant']
compliance_dict['percent_compliant'] = compliance_dict[
"compliant"] / compliance_dict['total_resources_measured'] * 100
return compliance_dict
from azure.cosmos import CosmosClient, HTTPFailure
AUTH_URL = os.environ["ACCOUNT_HOST"]
AUTH_KEY = os.environ["ACCOUNT_KEY"]
client = CosmosClient(url=AUTH_URL, key=AUTH_KEY)
def do_basic_stuff():
for database in client.list_databases():
print(database)
for container in database.list_containers():
print(container)
for item in container.list_items():
print(item)
database = client.create_database("swaggers", fail_if_exists=False)
try:
container = database.get_container(
"publicmaster"
) # TODO: Why not have a fail_if_exists on every create?
except ValueError: # TODO: What is the appropriate exception here?
container = database.create_container("publicmaster")
import time
database.set_container_properties(container, default_ttl=time.time() + 60 * 60)
try:
database.delete_container('containerwithspecificsettings')
except HTTPFailure:
class CRUD:
'''
Class:
CRUD
Description:
Class to perform Basic CRUD Operations on Cosmos DB SQL API
Functions:
dbConnection()
create_database()
create_container()
insert_record()
delete_record()
read_record()
drop_database()
Variable:
None
'''
def __init__(self):
self.url = config('ACCOUNT_URI')
self.key = config('ACCOUNT_KEY')
self.clientConnection()
self.create_database()
self.create_container()
def clientConnection(self):
'''
Description:
Connection to Azure Cosmos DB
Parameter:
None
Return:
None
'''
try:
self.client = CosmosClient(self.url, credential=self.key)
logger.info("Connection Successfull")
except Exception:
logger.exception("Connection Unsuccessfull")
def create_database(self):
'''
Description:
Create a Database
Parameter:
None
Return:
None
'''
try:
database_name = 'testDatabase'
self.mydb = self.client.create_database(database_name)
logger.info("Database Creation Successfull")
except exceptions.CosmosResourceExistsError:
self.mydb = self.client.get_database_client(database_name)
logger.warning("Database Exists")
except Exception:
logger.exception("Database Creation Unsuccessfull")
def create_container(self):
'''
Description:
Create a Container
Parameter:
None
Return:
None
'''
try:
container_name = 'products'
self.myContainer = self.mydb.create_container(
id=container_name,
partition_key=PartitionKey(path="/productName"))
logger.info("Container Creation Successfull")
except exceptions.CosmosResourceExistsError:
self.myContainer = self.mydb.get_container_client(container_name)
logger.warning("Container Exists")
except Exception:
logger.exception("Container Creation Unsuccessfull")
def insert_record(self):
'''
Description:
Insert Records to the container
Parameter:
None
Return:
None
'''
try:
for i in range(1, 10):
self.myContainer.upsert_item({
'id':
'item{0}'.format(i),
'productName':
'Widget',
'productModel':
'Model {0}'.format(i)
})
logger.info("Insert Record Successfull")
except Exception:
logger.exception("Insert Record Unsuccessfull")
def delete_record(self):
'''
Description:
Delete a record in the container
Parameter:
None
Return:
None
'''
try:
myQuery = "SELECT * FROM products p WHERE p.id = 'item3'"
for item in self.myContainer.query_items(
query=myQuery, enable_cross_partition_query=True):
self.myContainer.delete_item(item, partition_key='Widget')
logger.info("Delete Record Successfull")
except Exception:
logger.exception("Delete Record Unsuccessfull")
def read_record(self):
'''
Description:
Read a record from the container
Parameter:
None
Return:
None
'''
try:
myQuery = "SELECT * FROM products p WHERE p.productModel = 'Model 2'"
for item in self.myContainer.query_items(
query=myQuery, enable_cross_partition_query=True):
print(json.dumps(item, indent=True))
logger.info("Read Record Successfull")
except Exception:
logger.exception("Read Record Unsuccessfull")
def drop_database(self):
'''
Description:
Delete the created Database
Parameter:
None
Return:
None
'''
try:
self.client.delete_database("testDatabase")
logger.info("Database Drop Successfull")
except Exception:
logger.exception("Drop Database Unsuccessfull")
