def run_sample(): with IDisposable(document_client.DocumentClient(HOST, {'masterKey': MASTER_KEY} )) as client: try: # setup database for this sample try: client.CreateDatabase({"id": DATABASE_ID}) except errors.DocumentDBError as e: if e.status_code == 409: pass else: raise errors.HTTPFailure(e.status_code) # setup collection for this sample try: client.CreateCollection(database_link, {"id": COLLECTION_ID}) print('Collection with id \'{0}\' created'.format(COLLECTION_ID)) except errors.DocumentDBError as e: if e.status_code == 409: print('Collection with id \'{0}\' was found'.format(COLLECTION_ID)) else: raise errors.HTTPFailure(e.status_code) DocumentManagement.CreateDocuments(client) DocumentManagement.ReadDocument(client,'SalesOrder1') DocumentManagement.ReadDocuments(client) except errors.HTTPFailure as e: print('\nrun_sample has caught an error. {0}'.format(e.message)) finally: print("\nrun_sample done")
def create_collection(client, id): """ Execute the most basic Create of collection. This will create a collection with 400 RUs throughput and default indexing policy """ print("\n2.1 Create Collection - Basic") try: client.CreateCollection(database_link, {"id": id}) print('Collection with id \'{0}\' created'.format(id)) except errors.DocumentDBError as e: if e.status_code == 409: print('A collection with id \'{0}\' already exists'.format(id)) else: raise errors.HTTPFailure(e.status_code) print("\n2.2 Create Collection - With custom index policy") try: coll = { "id": "collection_custom_index_policy", "indexingPolicy": { "indexingMode": "lazy", "automatic": False } } collection = client.CreateCollection(database_link, coll) print('Collection with id \'{0}\' created'.format( collection['id'])) print('IndexPolicy Mode - \'{0}\''.format( collection['indexingPolicy']['indexingMode'])) print('IndexPolicy Automatic - \'{0}\''.format( collection['indexingPolicy']['automatic'])) except errors.DocumentDBError as e: if e.status_code == 409: print('A collection with id \'{0}\' already exists'.format( collection['id'])) else: raise errors.HTTPFailure(e.status_code) print("\n2.3 Create Collection - With custom offer throughput") try: coll = {"id": "collection_custom_throughput"} collection_options = {'offerThroughput': 400} collection = client.CreateCollection(database_link, coll, collection_options) print('Collection with id \'{0}\' created'.format( collection['id'])) except errors.DocumentDBError as e: if e.status_code == 409: print('A collection with id \'{0}\' already exists'.format( collection['id'])) else: raise errors.HTTPFailure(e.status_code)
def run_sample(): with IDisposable( document_client.DocumentClient( HOST, {'masterKey': MASTER_KEY})) as client: try: # setup database for this sample try: client.CreateDatabase({"id": DATABASE_ID}) except errors.DocumentDBError as e: if e.status_code == 409: pass else: raise errors.HTTPFailure(e.status_code) # query for a collection CollectionManagement.find_collection(client, COLLECTION_ID) # create a collection CollectionManagement.create_collection(client, COLLECTION_ID) # get & change Offer Throughput of collection CollectionManagement.manage_offer_throughput(client, COLLECTION_ID) # get a collection using its id CollectionManagement.read_collection(client, COLLECTION_ID) # list all collection on an account CollectionManagement.list_collections(client) # delete collection by id CollectionManagement.delete_collection(client, COLLECTION_ID) # cleanup database after sample try: client.DeleteDatabase(database_link) except errors.DocumentDBError as e: if e.status_code == 404: pass else: raise errors.HTTPFailure(e.status_code) except errors.HTTPFailure as e: print('\nrun_sample has caught an error. {0}'.format(e.message)) finally: print("\nrun_sample done")
def setDatabaseRU(client, databaseName, RUs): """Set the provisioned capacity of a Cosmos DB database """ if RUs < 10000 or RUs > 260000 or RUs%1000 != 0: raise ValueError( 'RUs must be between 10000 and 260000 and a multiple of 1000') else: try: vdb = list(client.QueryDatabases ("SELECT * FROM d WHERE d.id = '{0}'".format(databaseName)) ) if len(vdb) == 1: dbres = vdb[0]['_self'] dboffr = list(client.QueryOffers ("SELECT * FROM o WHERE o.resource = '{0}'".format(dbres) )) if len(dboffr) == 1: dboffr[0]['content']['offerThroughput'] = RUs offer = client.ReplaceOffer(dboffr[0]['_self'], dboffr[0]) return offer['content']['offerThroughput'] elif len(dboffr) == 0: # No offers for the database - collection provisioned? return None else: # got more than one offer ?error condition raise RuntimeError("More than 1 offer found for resource") elif len(vdb) == 0 : raise RuntimeError("Database {0} not found ".format(databaseName)) else: raise RuntimeError("More than 1 Database {0} found".format(databaseName)) # this really shouldn't ever happen except errors.HTTPFailure as e: if e.status_code == 404: print('A database with id \'{0}\' does not exist'.format(databaseName)) else: raise errors.HTTPFailure(e.status_code)
def getDatabaseRU(client, databaseName): """Retrieve the provisioned capacity of a Cosmos DB database """ try: vdb = list(client.QueryDatabases ("SELECT * FROM d WHERE d.id = '{0}'".format(databaseName)) ) if len(vdb) == 1: dbres = vdb[0]['_self'] dboffr = list(client.QueryOffers ("SELECT * FROM o WHERE o.resource = '{0}'".format(dbres) )) if len(dboffr) == 1: return dboffr[0]['content']['offerThroughput'] elif len(dboffr) == 0: # we didn't get any offers for the database return None else: # got more than one offer ?error condition raise RuntimeError("More than 1 offer found for resource") elif len(vdb) == 0 : raise RuntimeError("Database {0} not found ".format(databaseName)) else: raise RuntimeError("More than 1 Database {0} found".format(databaseName)) # this really shouldn't ever happen except errors.HTTPFailure as e: if e.status_code == 404: print('A database with id \'{0}\' does not exist'.format(databaseName)) else: raise errors.HTTPFailure(e.status_code)
def _MockExecuteFunctionConnectionReset(self, function, *args, **kwargs): self.counter += 1; if self.counter % 3 == 0: return self.OriginalExecuteFunction(function, *args, **kwargs) else: raise errors.HTTPFailure(10054, "Connection was reset", {})
def createDatabasePT(client, id, ruThroughput): """Create a CosmosDB Database with Provisioned Storage""" try: client.CreateDatabase({"id": id}, {"offerThroughput": ruThroughput}) print('Database with id \'{0}\' created'.format(id)) except errors.DocumentDBError as e: if e.status_code == 409: print('A database with id \'{0}\' already exists'.format(id)) raise errors.HTTPFailure(e.status_code) elif ( e.status_code == 400 and isinstance(e,pydocumentdb.errors.HTTPFailure) ): if 'throughput values between' in e.args[0]: print('Invalid throughput value {0}'.format(ruThroughput)) raise errors.HTTPFailure(e.status_code) else: raise errors.HTTPFailure(e.status_code)
def _InternalRequest(connection_policy, request_options, request_body): """Makes one http request. :Parameters: - `connection_policy`: documents.ConnectionPolicy - `request_options`: dict - `request_body`: str, unicode or Non :Returns: tuple of (result, headers), where both result and headers are dicts. """ is_media = request_options['path'].find('media') > -1 connection_timeout = (connection_policy.MediaRequestTimeout if is_media else connection_policy.RequestTimeout) if connection_policy.ProxyConfiguration and connection_policy.ProxyConfiguration.Host: connection = https_connection.HTTPSConnection(connection_policy.ProxyConfiguration.Host, port=int(connection_policy.ProxyConfiguration.Port), ssl_configuration=connection_policy.SSLConfiguration, timeout=connection_timeout / 1000.0) connection.set_tunnel(request_options['host'], request_options['port'], None) else: connection = https_connection.HTTPSConnection(request_options['host'], port=request_options['port'], ssl_configuration=connection_policy.SSLConfiguration, timeout=connection_timeout / 1000.0) connection.request(request_options['method'], request_options['path'], request_body, request_options['headers']) response = connection.getresponse() headers = response.getheaders() # In case of media response, return the response to the user and the user # will need to handle reading the response. if (is_media and connection_policy.MediaReadMode == documents.MediaReadMode.Streamed): return (response, dict(headers)) data = response.read() if response.status >= 400: raise errors.HTTPFailure(response.status, data, headers) result = None if is_media: result = data else: if len(data) > 0: try: result = json.loads(data) except: raise errors.JSONParseFailure(data) return (result, dict(headers))
def create_database(self, id): try: database = self.client.CreateDatabase({"id": id}) # print('Database with id \'{0}\' created'.format(id)) return database except errors.DocumentDBError as e: raise errors.HTTPFailure(e.status_code)
def main(): # Twitter application key client = KeyVaultClient(KeyVaultAuthentication(auth_callback)) _appkey = client.get_secret("https://adfbookkeyvault.vault.azure.net/", "Twitter-appkey", "19bd289d86f449cbb98fd6a51cc63156") _appsecret= client.get_secret("https://adfbookkeyvault.vault.azure.net/", "Twitter-appsecret", "510ceec80ef14af28dd961382be78e66") _appaccesstoken = client.get_secret("https://adfbookkeyvault.vault.azure.net/", "Twitter-appaccesstoken", "e24ad1dfef6b4392bd6277e9ac51e4b8") _appaccesstokensecret = client.get_secret("https://adfbookkeyvault.vault.azure.net/", "Twitter-appaccesstokensecret", "e46bb6b253584ea59d1941bd7f9f1905") _tweetTag= sys.argv[1] # like Azure _tweetReadSince= sys.argv[2] #date from when you want to read tweets like '2018/07/28' _PipelineRunId= sys.argv[3] #Azure Data Factory Pipeline ID 'testrun' # Azure Storage Credential _accountname=client.get_secret("https://adfbookkeyvault.vault.azure.net/", "Storage-accountname", "0f9adaf2abb545b38757b37a0d63fc68") _accountkey=client.get_secret("https://adfbookkeyvault.vault.azure.net/", "Storage-accountkey", "141200d435694c949a7c011bbf55d40a") _InputContainerName='tweetcontainer' # CosmosDB Credential _cdbhost = client.get_secret("https://adfbookkeyvault.vault.azure.net/", "cosmosdbURI", "7c885660bce64bd6ae7b44f1c925486c") _cdbmasterkey = client.get_secret("https://adfbookkeyvault.vault.azure.net/", "cosmosdbPK", "f220ab6df8d240759435953af5d01e43") #hashtag, tweetreadsince, filename includes pipeline id, auth = tweepy.OAuthHandler(_appkey.value, _appsecret.value) auth.set_access_token(_appaccesstoken.value, _appaccesstokensecret.value) tweetapi = tweepy.API(auth,wait_on_rate_limit=True) #local_path=os.path.expanduser("~/Documents") local_file_name ="Tweets_" + _tweetTag + _PipelineRunId + ".csv" full_path_to_file =os.path.join(os.getcwd(), local_file_name) outFile = open(local_file_name,'a') fieldnames = ['Tweet_Time', 'Tweet_Desc'] filewriter = csv.writer(outFile) filewriter.writerow(fieldnames) for tweet in tweepy.Cursor(tweetapi.search,q=_tweetTag,lang="en", since=_tweetReadSince).items(15): try: if tweet.text.encode('utf-8') != '' : filewriter.writerow([tweet.created_at,tweet.text.encode('utf-8')]) insertintoCosmosDB (_cdbhost.value, _cdbmasterkey.value, tweet.created_at,tweet.text.encode('utf-8')) except errors.DocumentDBError as e: if e.status_code == 409: pass else: raise errors.HTTPFailure(e.status_code) print("Error while fetching and storing tweets!!!") outFile.close() break try: print full_path_to_file print local_file_name _blob_service = BlockBlobService(account_name=_accountname.value, account_key=_accountkey.value) _blob_service.create_blob_from_path(_InputContainerName, local_file_name, full_path_to_file) #print(local_file_name) except: print("Error while uploading file to Azure Blob Storage !!!")
def delete_collection(self, database_id, id): database_link = 'dbs/' + database_id try: collection_link = database_link + '/colls/{0}'.format(id) self.client.DeleteCollection(collection_link) # print('Collection with id \'{0}\' was deleted'.format(id)) except errors.DocumentDBError as e: raise errors.HTTPFailure(e.status_code)
def delete_database(self, id): try: database_link = 'dbs/' + id self.client.DeleteDatabase(database_link) # print('Database with id \'{0}\' was deleted'.format(id)) except errors.DocumentDBError as e: raise errors.HTTPFailure(e.status_code)
def MockExecuteFunction(self, function, *args, **kwargs): global location_changed if self.endpoint_discovery_retry_count == 2: retry_utility._ExecuteFunction = self.OriginalExecuteFunction return (json.dumps([{'id': 'mock database'}]), None) else: self.endpoint_discovery_retry_count += 1 location_changed = True raise errors.HTTPFailure(403, "Forbidden", {'x-ms-substatus': 3})
def CreateDatabaseIfNotExists(client, database_id): try: database = Query_Entities(client, 'database', id=database_id) if database == None: database = client.CreateDatabase({"id": database_id}) return database except errors.DocumentDBError as e: if e.status_code == 409: # Move these constants to an enum pass else: raise errors.HTTPFailure(e.status_code)
def read_collection(self, database_id, id): database_link = 'dbs/' + database_id try: collection_link = database_link + '/colls/{0}'.format(id) collection = self.client.ReadCollection(collection_link) # print('Collection with id \'{0}\' was found, it\'s _self is {1}'.format(collection['id'], # collection['_self'])) return collection except errors.DocumentDBError as e: raise errors.HTTPFailure(e.status_code)
def read_database(self, id): try: database_link = 'dbs/' + id database = self.client.ReadDatabase(database_link) # print('Database with id \'{0}\' was found, it\'s _self is {1}'.format(id, database['_self'])) return database except errors.DocumentDBError as e: raise errors.HTTPFailure(e.status_code)
def insertintoCosmosDB(cdbhost, cdbmasterkey, tweetDate, tweetText): tweetmessage = {'tweetDate': str(tweetDate),'id' : str(tweetDate), 'tweetText': tweetText} _database_link = 'dbs/tweetdb' _collection_link = _database_link + '/colls/tweetcollec' with IDisposable(document_client.DocumentClient(cdbhost, {'masterKey': cdbmasterkey} )) as client: try: client.CreateDocument(_collection_link, tweetmessage, options=False) except errors.DocumentDBError as e: if e.status_code == 409: pass else: raise errors.HTTPFailure(e.status_code)
def deleteDatabase(client, id): try: database_link = 'dbs/' + id client.DeleteDatabase(database_link) print('Database with id \'{0}\' was deleted'.format(id)) except errors.DocumentDBError as e: if e.status_code == 404: print('A database with id \'{0}\' does not exist'.format(id)) else: raise errors.HTTPFailure(e.status_code)
def create_doc(self, database_id, collection_id, doc): docs = self.find_doc_by_id(database_id, collection_id, doc['id']) if len(docs) != 0: return self.replace_doc(docs[0], doc) try: collection_link = self.get_collection_link(database_id, collection_id) doc = self.client.CreateDocument(collection_link, doc) return doc except errors.DocumentDBError as e: raise errors.HTTPFailure(e.status_code)
def create_database(client, id): print("\n2. Create Database") try: client.CreateDatabase({"id": id}) print('Database with id \'{0}\' created'.format(id)) except errors.DocumentDBError as e: if e.status_code == 409: print('A database with id \'{0}\' already exists'.format(id)) else: raise errors.HTTPFailure(e.status_code)
def read_collection(self): # self.collection = next((coll for coll in self.client.ReadCollections(self.db['_self']) if coll['id'] == DOCUMENTDB_COLLECTION)) # print(self.collection) try: self.collection_link = self.database_link + '/colls/{0}'.format(DOCUMENTDB_COLLECTION) self.collection = self.client.ReadCollection(self.collection_link) print('Collection with id \'{0}\' was found, it\'s _self is {1}'.format(DOCUMENTDB_COLLECTION, self.collection['_self'])) except errors.DocumentDBError as e: if e.status_code == 404: print('A collection with id \'{0}\' does not exist'.format(DOCUMENTDB_COLLECTION)) else: raise errors.HTTPFailure(e.status_code)
def read_database(self): # if self.client: # self.db = next((data for data in self.client.ReadDatabases() if data['id'] == DOCUMENTDB_DATABASE)) try: self.database_link = 'dbs/' + DOCUMENTDB_DATABASE self.db = self.client.ReadDatabase(self.database_link) print('Database with id \'{0}\' was found, it\'s _self is {1}'.format(DOCUMENTDB_DATABASE, self.db['_self'])) except errors.DocumentDBError as e: if e.status_code == 404: print('A database with id \'{0}\' does not exist'.format(DOCUMENTDB_DATABASE)) else: raise errors.HTTPFailure(e.status_code)
def create_doc(self, database_id, collection_id, doc): docs = self.find_doc_by_id(database_id, collection_id, doc['id']) if len(docs) != 0: return self.replace_doc(docs[0]) try: collection_link = self.get_collection_link(database_id, collection_id) doc = self.client.CreateDocument(collection_link, doc) return doc except errors.DocumentDBError as e: if e.status_code == 404: print('A collection with id \'{0}\' does not exist'.format(collection_id)) else: raise errors.HTTPFailure(e.status_code)
def delete_collection(client, id): print("\n6. Delete Collection") try: collection_link = database_link + '/colls/{0}'.format(id) client.DeleteCollection(collection_link) print('Collection with id \'{0}\' was deleted'.format(id)) except errors.DocumentDBError as e: if e.status_code == 404: print('A collection with id \'{0}\' does not exist'.format(id)) else: raise errors.HTTPFailure(e.status_code)
def initiate_database(my_cosmos_client): cosmos_db_handle = None try: cosmos_db_handle = my_cosmos_client.CreateDatabase( {"id": config['db']}) print('Created a new database with id \'{0}\'.'.format(config['db'])) except errors.DocumentDBError as e: if e.status_code == 409: print('Initializing the database with id \'{0}\' ...'.format( config['db'])) cosmos_db_handle = my_cosmos_client.ReadDatabase('dbs/' + config['db']) else: raise errors.HTTPFailure(e.status_code) return cosmos_db_handle
def read_database(client, id): """Read a CosmosDB database. Args: client (obj): A pydocumentdb client object. id (str): Database ID. Returns: obj: A database. """ try: database_link = "dbs/" + id database = client.ReadDatabase(database_link) return database except errors.DocumentDBError as e: if e.status_code == 404: print("A database with id '{0}' does not exist".format(id)) else: raise errors.HTTPFailure(e.status_code)
def create_collection(self, database_id, id, offer_enable_ru_per_min_throughput, offer_version, offer_throughput): database_link = 'dbs/' + database_id options = { 'offerEnableRUPerMinuteThroughput': offer_enable_ru_per_min_throughput, 'offerVersion': offer_version, 'offerThroughput': offer_throughput } try: collection = self.client.CreateCollection(database_link, {"id": id}, options) print('Collection with id \'{0}\' created'.format(id)) return collection except errors.DocumentDBError as e: if e.status_code == 409: print('A collection with id \'{0}\' already exists'.format(id)) else: raise errors.HTTPFailure(e.status_code)
def read_collection(client, id): print("\n4. Get a Collection by id") try: # All DocumentDB resources are addressable via a link # This link is constructed from a combination of resource hierachy and # the resource id. # Eg. The link for collection with an id of Bar in database Foo would be dbs/Foo/colls/Bar collection_link = database_link + '/colls/{0}'.format(id) collection = client.ReadCollection(collection_link) print('Collection with id \'{0}\' was found, it\'s _self is {1}'.format(collection['id'], collection['_self'])) except errors.DocumentDBError as e: if e.status_code == 404: print('A collection with id \'{0}\' does not exist'.format(id)) else: raise errors.HTTPFailure(e.status_code)
def read_database(client, id): print("\n3. Get a Database by id") try: # All DocumentDB resources are addressable via a link # This link is constructed from a combination of resource hierachy and # the resource id. # Eg. The link for database with an id of Foo would be dbs/Foo database_link = 'dbs/' + id database = client.ReadDatabase(database_link) print('Database with id \'{0}\' was found, it\'s _self is {1}'.format(id, database['_self'])) except errors.DocumentDBError as e: if e.status_code == 404: print('A database with id \'{0}\' does not exist'.format(id)) else: raise errors.HTTPFailure(e.status_code)
def read_collection(client, dbid, id): """Read a CosmosDB collection. Args: client (obj): A pydocumentdb client object. dbid (str): Database ID. id (str): Collection ID. Returns: obj: A collection. """ try: database_link = "dbs/" + dbid collection_link = database_link + "/colls/{0}".format(id) collection = client.ReadCollection(collection_link) return collection except errors.DocumentDBError as e: if e.status_code == 404: print("A collection with id '{0}' does not exist".format(id)) else: raise errors.HTTPFailure(e.status_code)