def run_sample(): AUTH_URI = os.environ.get("ACCOUNT_URI") AUTH_KEY = os.environ.get("ACCOUNT_KEY") DATABASE_ID = "testdocumentmanagementdb" CONTAINER_ID = "testdocumentmanagementcollection" client = CosmosClient(AUTH_URI, AUTH_KEY) database = client.create_database(id=DATABASE_ID) partition_key = PartitionKey(path="/purchase_order_number") try: container = database.create_container(id=CONTAINER_ID, partition_key=partition_key) print(f'Container with id "{CONTAINER_ID}"" created') except HTTPFailure as e: if e.status_code == 409: print(f"Container with id {CONTAINER_ID} already exists") container = database.get_container(container=CONTAINER_ID) else: raise DocumentManagement.create_documents(container) DocumentManagement.read_document(container, "SalesOrder1") DocumentManagement.read_documents(container) client.delete_database(database=DATABASE_ID) print("\nrun_sample done")
def main(req: func.HttpRequest) -> func.HttpResponse: URL = "https://junctionx.documents.azure.com:443/" KEY = "BCw9nhNiTSl9ndUokCw6uHhzZSDgsSGoMjPUa0Ech6E9yiuGKCrsswFIXRatcguQYcNLnYdhFPWPhuaRtLfoxg==" DATABASE_NAME = "MitoSoftCorp" CONTAINER_NAME = "workType" client = CosmosClient(URL, credential=KEY) work_type = req.params.get("work_type") try: database = client.create_database(DATABASE_NAME) except exceptions.CosmosResourceExistsError: database = client.get_database_client(DATABASE_NAME) try: container = database.create_container( id=CONTAINER_NAME, partition_key=PartitionKey(path="/name")) except exceptions.CosmosResourceExistsError: container = database.get_container_client(CONTAINER_NAME) user_list = [] for item in container.query_items( query='SELECT * FROM {} w WHERE w.work_type = "{}"'.format( CONTAINER_NAME, work_type), enable_cross_partition_query=True): user_list.append(item.get("user_id")) return_data = dict(id=user_list) return func.HttpResponse(json.dumps(return_data), headers={"Content-Type": "application/json"}, status_code=200)
def UploadCosmos(guid, jsondata): client = CosmosClient(cosmosuri, credential=cosmoskey) database_name = 'cosmosnetatmo' container_name = 'mesures' try: # create database if not exists database = client.create_database(database_name) except exceptions.CosmosResourceExistsError: database = client.get_database_client(database_name) try: # create collection if not exists container = database.create_container( id=container_name, partition_key=PartitionKey(path="/id")) except exceptions.CosmosResourceExistsError: container = database.get_container_client(container_name) except exceptions.CosmosHttpResponseError: raise #Upsert after adding a guid as key to the document jsondata["id"] = guid container.upsert_item(jsondata)
def run_sample(): AUTH_URL = os.environ.get("ACCOUNT_HOST") AUTH_KEY = os.environ.get("ACCOUNT_KEY") DATABASE_ID = 'testdocumentmanagementdb' CONTAINER_ID = 'testdocumentmanagementcollection' client = CosmosClient(AUTH_URL, AUTH_KEY) database = client.create_database(id=DATABASE_ID) try: container = database.create_container(id=CONTAINER_ID) print(f'Container with id "{CONTAINER_ID}"" created') except HTTPFailure as e: if e.status_code == 409: print(f'Container with id {CONTAINER_ID} already exists') container = database.get_container(container=CONTAINER_ID) else: raise DocumentManagement.create_documents(container) DocumentManagement.read_document(container, 'SalesOrder1') DocumentManagement.read_documents(container) client.delete_database(database=DATABASE_ID) print("\nrun_sample done")
from azure.cosmos import HTTPFailure, CosmosClient, Container, Database # It all starts with a client instance: import os url = os.environ['ACCOUNT_HOST'] key = os.environ['ACCOUNT_KEY'] client = CosmosClient(url, key) test_database_name = 'testDatabase' test_container_name = 'testContainer' #### Create environnment ### db = client.create_database(id=test_database_name, fail_if_exists=False) try: db.create_container(id=test_container_name) except HTTPFailure as e: if e.status_code != 409: raise db.get_container(test_container_name) #### # In order to retreive a container where you know the database name and container name: container = Container(client.client_context, database=test_database_name, id=test_container_name) container.upsert_item({'id': 'something', 'value': 'else'}) # If you want to walk down the hierarchy, you can also get it from the client->databases->containers database = client.get_database(test_database_name) container = database.get_container(test_container_name) container.upsert_item({'id': 'something', 'value': 'new'})
from azure.cosmos import CosmosClient, PartitionKey, exceptions # Create the cosmos client config = {"endpoint": "", "primarykey": ""} client = CosmosClient(config["endpoint"], config["primarykey"]) # Create database database_name = 'testDatabase' try: database = client.create_database(database_name) except exceptions.CosmosResourceExistsError: database = client.get_database_client(database_name) # Create Container container_name = 'testContainer' try: container = database.create_container( id=container_name, partition_key=PartitionKey(path="/country")) except exceptions.CosmosResourceExistsError: container = database.get_container_client(container_name) except exceptions.CosmosHttpResponseError: raise database_client = client.get_database_client(database_name) container_client = database.get_container_client(container_name)
from azure.cosmos import CosmosClient, PartitionKey, exceptions import os from dotenv import load_dotenv load_dotenv() DATABASE_NAME = 'testDb' CONTAINER_NAME = 'products' COSMOS_URL = os.environ['COSMOS_URL'] COSMOS_KEY = os.environ['COSMOS_KEY'] client = CosmosClient(COSMOS_URL, credential=COSMOS_KEY) try: database = client.create_database(DATABASE_NAME) except exceptions.CosmosResourceExistsError: database = client.get_database_client(DATABASE_NAME) print( f"### Connected to {client.client_connection.url_connection}/{database.id}" ) try: container = database.create_container( id=CONTAINER_NAME, partition_key=PartitionKey(path="/productName")) except exceptions.CosmosResourceExistsError: container = database.get_container_client(CONTAINER_NAME) except exceptions.CosmosHttpResponseError: raise container_client = database.get_container_client(CONTAINER_NAME)
class CosmosDB: def __init__(self): URL = os.environ["COSMOS_ENDPOINT"] KEY = os.environ["COSMOS_KEY"] self.client = CosmosClient(URL, {"masterKey": KEY}) self.dbName = "pySolarSystem-" + uuid.uuid1().hex def create_database(self): print("Creating '{0}' database...".format(self.dbName)) return self.client.create_database(self.dbName) def create_container(self, db): collectionName = "Planets" print("Creating '{0}' collection...".format(collectionName)) partition_key = PartitionKey(path="/id", kind="Hash") return db.create_container(id="Planets", partition_key=partition_key) def create_documents(self, container): # Cosmos will look for an 'id' field in the items, if the 'id' is not specify Cosmos is going to assing a random key. planets = [ { "id": "Earth", "HasRings": False, "Radius": 3959, "Moons": [{"Name": "Moon"}], }, { "id": "Mars", "HasRings": False, "Radius": 2106, "Moons": [{"Name": "Phobos"}, {"Name": "Deimos"}], }, ] print("Inserting items in the collection...") for planet in planets: container.create_item(planet) print("\t'{0}' created".format(planet["id"])) print("\tdone") def simple_query(self, container): print("Quering the container...") items = list( container.query_items( query="SELECT c.id FROM c", enable_cross_partition_query=True ) ) print("\tdone: {0}".format(items)) def delete_database(self): print("Cleaning up the resource...") self.client.delete_database(self.dbName) print("\tdone") def run(self): print("") print("------------------------") print("Cosmos DB") print("------------------------") print("1) Create a Database") print("2) Create a Container in the database") print("3) Insert Documents (items) into the Container") print("4) Delete Database (Clean up the resource)") print("") # Ensure that the database does not exists try: self.delete_database() except: pass try: db = self.create_database() container = self.create_container(db=db) self.create_documents(container=container) self.simple_query(container=container) finally: # if something goes wrong, the resource should be cleaned anyway self.delete_database()
cred = DefaultAzureCredential() secret_client = SecretClient(vault_url=kv_endpoint, credential=cred) retrieved_secret = secret_client.get_secret(secret_name) # Connect to Azure Cosmos DB cosmos_account = os.environ['COSMOS_ACCOUNT_NAME'] url = f"https://{cosmos_account}.documents.azure.com:443/" client = CosmosClient(url, retrieved_secret.value, consistency_level=cons_level) print(f"[{cosmos_account}] Connected successfully to Azure Cosmos DB account.") # Create/Get Database try: database = client.create_database(id=database_name, offer_throughput=throuput_val) print( f"[{cosmos_account}] Created successfully a database. name[{database_name}]" ) except exceptions.CosmosResourceExistsError: database = client.get_database_client(database=database_name) print( f"[{cosmos_account}] Get successfully a database. name[{database_name}]" ) # Create/Get Container try: container = database.create_container( id=container_name, partition_key=PartitionKey(path=part_key)) print( f"[{cosmos_account}] Created successfully a container. name[{container_name}]"
from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.keys import Keys import time import json import geocoder from azure.cosmos import CosmosClient, exceptions url = "a" key = "test" client = CosmosClient(url, credential=key) database = "hacktx_addresses" try: database = client.create_database(database) except exceptions.CosmosResourceExistError: database = client.get_database_client(database) container_name = 'addresses' container = database.get_container_client(container_name) def get_coord(address): #g = geocoder.google(address, key ='AIzaSyCogh5_2lxRy8SPG8rHaXaY0-umKhl1UyA') return [30.285560, -97.752790] #return g.latlong def parse_text(text, photo_url, address): txt = text.split("\n") line1 = txt[0]
AUTH_URI = os.environ["ACCOUNT_URI"] AUTH_KEY = os.environ["ACCOUNT_KEY"] client = CosmosClient(url=AUTH_URI, key=AUTH_KEY) def do_basic_stuff(): for database in client.list_databases(): print(database) for container in database.list_containers(): print(container) for item in container.list_items(): print(item) try: database = client.create_database("swaggers") except HTTPFailure: database = client.get_database("swaggers") try: database.delete_container('publicmaster') except: pass try: container = database.get_container( "publicmaster") # TODO: Why not have a fail_if_exists on every create? except HTTPFailure: # TODO: What is the appropriate exception here? container = database.create_container( "publicmaster", partition_key=PartitionKey('/info/version'))
class SummaryForPolicyDefinition(BaseDb): def __init__(self, db_server_instance="instance01"): load_dotenv() with change_dir(OPERATIONSPATH): self.config = ConfigParser() self.config.read(CONFVARIABLES) sql_instance = self.config['SQLSERVER'][db_server_instance] sql_instance = json.loads(sql_instance.replace("\'", "\"")) self.server = sql_instance['server'] self.database = sql_instance['database'] self.username = sql_instance['username'] self.driver = sql_instance['db_driver'] self.dialect = sql_instance['dialect'] self.password = os.environ['DATABASEPWD'] self.get_db_engine() self.Session = sessionmaker(bind=self.engine) url = self.config['COSMOSDB']['uri_01'] key = os.environ['COSMODB_KEY'] self.CosmosDBclient = CosmosClient(url, credential=key) def summarize_fact_compliance_for_definition(self, subscription_id, policy_definition_name): jmespath_expression = jmespath.compile( "value[*].policyAssignments[*].policyDefinitions[*]") policy_states = PolicyStates() batch_uuid = uuid.uuid4() created = datetime.datetime.utcnow() # create a configured "Session" class # create a Session # Execute returns a method the can be executed anywhere more than once models = self.get_db_model(self.engine) FactCompliance = models.classes.factcompliance subscriptions = models.classes.subscriptions session = self.Session() if subscription_id is not None: subscriptions_list = session.query(subscriptions).filter_by( subscription_id=subscription_id) else: subscriptions_list = session.query(subscriptions).all() for subscription in subscriptions_list: tenant_id = subscription.tenant_id print("Subscription: {}, Policy Name {}".format( subscription.subscription_id, policy_definition_name)) policy_states_of_definition = policy_states.policy_states_summarize_for_policy_definition( subscriptionId=subscription.subscription_id, policyDefinitionName=policy_definition_name).json() jmes_result = jmespath_expression.search( policy_states_of_definition) if jmespath_expression is None or jmes_result is None or jmes_result[ 0] is None or len(jmes_result[0]) == 0: continue else: # flatten results policyresults = jmes_result[0][0] bulk_insert = list() for policyresult in policyresults: policy_definition_name = str( policyresult['policyDefinitionId']).split('/')[-1] resourceDetails = jmespath.search( 'results.resourceDetails[*]', policyresult) compliance_ratio = self.determine_compliance_ratio( resourceDetails) fact = FactCompliance( tenant_id=tenant_id, subscription_id=subscription.subscription_id, policy_definition_name=policy_definition_name, compliant=compliance_ratio['compliant'], noncompliant=compliance_ratio['noncompliant'], total_resources_measured=compliance_ratio[ 'total_resources_measured'], percent_compliant=compliance_ratio[ 'percent_compliant'], batch_uuid=batch_uuid, created=created, modified=created) bulk_insert.append(fact) session.bulk_save_objects(bulk_insert) session.commit() def summarize_policy_defintions_for_all_management_grp( self, management_group_id, policy_definition_name_like, metadata_category): """ This method take the policy definition and find the statistics and saves it to the fact table Submit a policy definition name without a wild card to pull back a single policy definition name :param policy_definition_name: """ batch_uuid = uuid.uuid4() created = datetime.datetime.utcnow() database_name = 'policyinsights' comsos_database = self.get_cosmodb(database_name) container_name = 'policydefintionresults' partition_key = "/{}".format(metadata_category.strip().lower().replace( ' ', '')) self.cosmos_container_client = self.get_cosmosdb_container( container_name, comsos_database, partition_key) # create a configured "Session" class # create a Session # Execute returns a method the can be executed anywhere more than once policy_definition_cls = PolicyDefinition() models = self.get_db_model(self.engine) subscriptions_model = models.classes.subscriptions policy_definition_model = models.classes.policydefinitions session = self.Session() subscriptions = session.query(subscriptions_model).all() policies = session.query(policy_definition_model).filter( policy_definition_model.metadata_category == metadata_category, policy_definition_model.policy_definition_name.like( 'security-pol-%')).all() policy_compliance = PolicyCompliance() for subscription in subscriptions: for policy_definition in policies: self.summarize_fact_compliance_for_definition( category=metadata_category, policy_definition_name=policy_definition. policy_definition_name, tenant_id=subscription.tenant_id, subscription_id=subscription.subscription_id, batch_uuid=batch_uuid) def summarize_fact_compliance_for_definition(self, category, policy_definition_name, tenant_id, subscription_id, batch_uuid): """ This method captures the results for a policy compliance within a single subscription publishes results to non-SQL Cosmos DB :param category: :param policy_definition_name: :param subscription_id: """ policy_states = PolicyStates() created = datetime.datetime.utcnow() # create a configured "Session" class print("Subscription: {}, Policy Name {}".format( subscription_id, policy_definition_name)) ''' Retreive the states for a policy and subscription ''' policy_states_of_definition_response = policy_states.policy_states_summarize_for_policy_definition( subscriptionId=subscription_id, policyDefinitionName=policy_definition_name) ''' The Rest API may fail to pull back states for many reasons. Failed states are skipped for various reasons but mostly, some results are better than none You are free to act differently. ''' if not policy_states_of_definition_response.status_code in range( 200, 299): print("Skipping with Error {} Subscription: {}, Policy Name {}". format(policy_states_of_definition_response.status_code, subscription_id, policy_definition_name)) return policy_states_of_definitions_json = policy_states_of_definition_response.json( ) if 'value' in policy_states_of_definitions_json: policy_states_of_definitions = policy_states_of_definitions_json[ 'value'] for policy_states_of_definition in policy_states_of_definitions: policy_states_of_definition['category'] = category policy_states_of_definition[ 'policy_definition_name'] = policy_definition_name policy_states_of_definition['tenant_id'] = tenant_id policy_states_of_definition[ 'subscription_id'] = subscription_id policy_states_of_definition['batch_uuid'] = str(batch_uuid) self.cosmos_container_client.upsert_item( policy_states_of_definition) else: return 0 def get_cosmosdb_container(self, container_name, database, partition_key): container = None try: container = database.create_container( id=container_name, partition_key=PartitionKey(path=partition_key)) except exceptions.CosmosResourceExistsError: container = database.get_container_client(container_name) except exceptions.CosmosHttpResponseError: raise finally: if container is not None: return container def get_cosmodb(self, database_name): try: database = self.CosmosDBclient.create_database(database_name) except exceptions.CosmosResourceExistsError: database = self.CosmosDBclient.get_database_client(database_name) return database def determine_compliance_ratio(self, resourceDetails): compliance_dict = dict() if len(resourceDetails) <= 2: complianceState = resourceDetails[0] if complianceState['complianceState'] in 'compliant': compliance_dict["compliant"] = int(complianceState["count"]) else: compliance_dict["noncompliant"] = int(complianceState["count"]) if len(resourceDetails) == 2: complianceState = resourceDetails[1] if complianceState['complianceState'] in 'compliant': compliance_dict["compliant"] = int(complianceState["count"]) else: compliance_dict["noncompliant"] = int(complianceState["count"]) if not 'compliant' in compliance_dict: compliance_dict["compliant"] = 0 if not 'noncompliant' in compliance_dict: compliance_dict['noncompliant'] = 0 compliance_dict['total_resources_measured'] = compliance_dict[ "compliant"] + compliance_dict['noncompliant'] compliance_dict['percent_compliant'] = compliance_dict[ "compliant"] / compliance_dict['total_resources_measured'] * 100 return compliance_dict
from azure.cosmos import CosmosClient, HTTPFailure AUTH_URL = os.environ["ACCOUNT_HOST"] AUTH_KEY = os.environ["ACCOUNT_KEY"] client = CosmosClient(url=AUTH_URL, key=AUTH_KEY) def do_basic_stuff(): for database in client.list_databases(): print(database) for container in database.list_containers(): print(container) for item in container.list_items(): print(item) database = client.create_database("swaggers", fail_if_exists=False) try: container = database.get_container( "publicmaster" ) # TODO: Why not have a fail_if_exists on every create? except ValueError: # TODO: What is the appropriate exception here? container = database.create_container("publicmaster") import time database.set_container_properties(container, default_ttl=time.time() + 60 * 60) try: database.delete_container('containerwithspecificsettings') except HTTPFailure:
class CRUD: ''' Class: CRUD Description: Class to perform Basic CRUD Operations on Cosmos DB SQL API Functions: dbConnection() create_database() create_container() insert_record() delete_record() read_record() drop_database() Variable: None ''' def __init__(self): self.url = config('ACCOUNT_URI') self.key = config('ACCOUNT_KEY') self.clientConnection() self.create_database() self.create_container() def clientConnection(self): ''' Description: Connection to Azure Cosmos DB Parameter: None Return: None ''' try: self.client = CosmosClient(self.url, credential=self.key) logger.info("Connection Successfull") except Exception: logger.exception("Connection Unsuccessfull") def create_database(self): ''' Description: Create a Database Parameter: None Return: None ''' try: database_name = 'testDatabase' self.mydb = self.client.create_database(database_name) logger.info("Database Creation Successfull") except exceptions.CosmosResourceExistsError: self.mydb = self.client.get_database_client(database_name) logger.warning("Database Exists") except Exception: logger.exception("Database Creation Unsuccessfull") def create_container(self): ''' Description: Create a Container Parameter: None Return: None ''' try: container_name = 'products' self.myContainer = self.mydb.create_container( id=container_name, partition_key=PartitionKey(path="/productName")) logger.info("Container Creation Successfull") except exceptions.CosmosResourceExistsError: self.myContainer = self.mydb.get_container_client(container_name) logger.warning("Container Exists") except Exception: logger.exception("Container Creation Unsuccessfull") def insert_record(self): ''' Description: Insert Records to the container Parameter: None Return: None ''' try: for i in range(1, 10): self.myContainer.upsert_item({ 'id': 'item{0}'.format(i), 'productName': 'Widget', 'productModel': 'Model {0}'.format(i) }) logger.info("Insert Record Successfull") except Exception: logger.exception("Insert Record Unsuccessfull") def delete_record(self): ''' Description: Delete a record in the container Parameter: None Return: None ''' try: myQuery = "SELECT * FROM products p WHERE p.id = 'item3'" for item in self.myContainer.query_items( query=myQuery, enable_cross_partition_query=True): self.myContainer.delete_item(item, partition_key='Widget') logger.info("Delete Record Successfull") except Exception: logger.exception("Delete Record Unsuccessfull") def read_record(self): ''' Description: Read a record from the container Parameter: None Return: None ''' try: myQuery = "SELECT * FROM products p WHERE p.productModel = 'Model 2'" for item in self.myContainer.query_items( query=myQuery, enable_cross_partition_query=True): print(json.dumps(item, indent=True)) logger.info("Read Record Successfull") except Exception: logger.exception("Read Record Unsuccessfull") def drop_database(self): ''' Description: Delete the created Database Parameter: None Return: None ''' try: self.client.delete_database("testDatabase") logger.info("Database Drop Successfull") except Exception: logger.exception("Drop Database Unsuccessfull")