def run_sample():
with IDisposable(document_client.DocumentClient(HOST, {'masterKey': MASTER_KEY} )) as client:
try:
# setup database for this sample
try:
client.CreateDatabase({"id": DATABASE_ID})
except errors.DocumentDBError as e:
if e.status_code == 409:
pass
else:
raise errors.HTTPFailure(e.status_code)
# setup collection for this sample
try:
client.CreateCollection(database_link, {"id": COLLECTION_ID})
print('Collection with id \'{0}\' created'.format(COLLECTION_ID))
except errors.DocumentDBError as e:
if e.status_code == 409:
print('Collection with id \'{0}\' was found'.format(COLLECTION_ID))
else:
raise errors.HTTPFailure(e.status_code)
DocumentManagement.CreateDocuments(client)
DocumentManagement.ReadDocument(client,'SalesOrder1')
DocumentManagement.ReadDocuments(client)
except errors.HTTPFailure as e:
print('\nrun_sample has caught an error. {0}'.format(e.message))
finally:
print("\nrun_sample done")
def create_collection(client, id):
""" Execute the most basic Create of collection.
This will create a collection with 400 RUs throughput and default indexing policy """
print("\n2.1 Create Collection - Basic")
try:
client.CreateCollection(database_link, {"id": id})
print('Collection with id \'{0}\' created'.format(id))
except errors.DocumentDBError as e:
if e.status_code == 409:
print('A collection with id \'{0}\' already exists'.format(id))
else:
raise errors.HTTPFailure(e.status_code)
print("\n2.2 Create Collection - With custom index policy")
try:
coll = {
"id": "collection_custom_index_policy",
"indexingPolicy": {
"indexingMode": "lazy",
"automatic": False
}
}
collection = client.CreateCollection(database_link, coll)
print('Collection with id \'{0}\' created'.format(
collection['id']))
print('IndexPolicy Mode - \'{0}\''.format(
collection['indexingPolicy']['indexingMode']))
print('IndexPolicy Automatic - \'{0}\''.format(
collection['indexingPolicy']['automatic']))
except errors.DocumentDBError as e:
if e.status_code == 409:
print('A collection with id \'{0}\' already exists'.format(
collection['id']))
else:
raise errors.HTTPFailure(e.status_code)
print("\n2.3 Create Collection - With custom offer throughput")
try:
coll = {"id": "collection_custom_throughput"}
collection_options = {'offerThroughput': 400}
collection = client.CreateCollection(database_link, coll,
collection_options)
print('Collection with id \'{0}\' created'.format(
collection['id']))
except errors.DocumentDBError as e:
if e.status_code == 409:
print('A collection with id \'{0}\' already exists'.format(
collection['id']))
else:
raise errors.HTTPFailure(e.status_code)
def run_sample():
with IDisposable(
document_client.DocumentClient(
HOST, {'masterKey': MASTER_KEY})) as client:
try:
# setup database for this sample
try:
client.CreateDatabase({"id": DATABASE_ID})
except errors.DocumentDBError as e:
if e.status_code == 409:
pass
else:
raise errors.HTTPFailure(e.status_code)
# query for a collection
CollectionManagement.find_collection(client, COLLECTION_ID)
# create a collection
CollectionManagement.create_collection(client, COLLECTION_ID)
# get & change Offer Throughput of collection
CollectionManagement.manage_offer_throughput(client, COLLECTION_ID)
# get a collection using its id
CollectionManagement.read_collection(client, COLLECTION_ID)
# list all collection on an account
CollectionManagement.list_collections(client)
# delete collection by id
CollectionManagement.delete_collection(client, COLLECTION_ID)
# cleanup database after sample
try:
client.DeleteDatabase(database_link)
except errors.DocumentDBError as e:
if e.status_code == 404:
pass
else:
raise errors.HTTPFailure(e.status_code)
except errors.HTTPFailure as e:
print('\nrun_sample has caught an error. {0}'.format(e.message))
finally:
print("\nrun_sample done")
def setDatabaseRU(client, databaseName, RUs):
"""Set the provisioned capacity of a Cosmos DB database """
if RUs < 10000 or RUs > 260000 or RUs%1000 != 0:
raise ValueError(
'RUs must be between 10000 and 260000 and a multiple of 1000')
else:
try:
vdb = list(client.QueryDatabases
("SELECT * FROM d WHERE d.id = '{0}'".format(databaseName)) )
if len(vdb) == 1:
dbres = vdb[0]['_self']
dboffr = list(client.QueryOffers
("SELECT * FROM o WHERE o.resource = '{0}'".format(dbres) ))
if len(dboffr) == 1:
dboffr[0]['content']['offerThroughput'] = RUs
offer = client.ReplaceOffer(dboffr[0]['_self'], dboffr[0])
return offer['content']['offerThroughput']
elif len(dboffr) == 0:
# No offers for the database - collection provisioned?
return None
else:
# got more than one offer ?error condition
raise RuntimeError("More than 1 offer found for resource")
elif len(vdb) == 0 :
raise RuntimeError("Database {0} not found ".format(databaseName))
else:
raise RuntimeError("More than 1 Database {0} found".format(databaseName))
# this really shouldn't ever happen
except errors.HTTPFailure as e:
if e.status_code == 404:
print('A database with id \'{0}\' does not exist'.format(databaseName))
else:
raise errors.HTTPFailure(e.status_code)
def getDatabaseRU(client, databaseName):
"""Retrieve the provisioned capacity of a Cosmos DB database """
try:
vdb = list(client.QueryDatabases
("SELECT * FROM d WHERE d.id = '{0}'".format(databaseName)) )
if len(vdb) == 1:
dbres = vdb[0]['_self']
dboffr = list(client.QueryOffers
("SELECT * FROM o WHERE o.resource = '{0}'".format(dbres) ))
if len(dboffr) == 1:
return dboffr[0]['content']['offerThroughput']
elif len(dboffr) == 0:
# we didn't get any offers for the database
return None
else:
# got more than one offer ?error condition
raise RuntimeError("More than 1 offer found for resource")
elif len(vdb) == 0 :
raise RuntimeError("Database {0} not found ".format(databaseName))
else:
raise RuntimeError("More than 1 Database {0} found".format(databaseName))
# this really shouldn't ever happen
except errors.HTTPFailure as e:
if e.status_code == 404:
print('A database with id \'{0}\' does not exist'.format(databaseName))
else:
raise errors.HTTPFailure(e.status_code)
def _MockExecuteFunctionConnectionReset(self, function, *args, **kwargs):
self.counter += 1;
if self.counter % 3 == 0:
return self.OriginalExecuteFunction(function, *args, **kwargs)
else:
raise errors.HTTPFailure(10054, "Connection was reset", {})
def createDatabasePT(client, id, ruThroughput):
"""Create a CosmosDB Database with Provisioned Storage"""
try:
client.CreateDatabase({"id": id}, {"offerThroughput": ruThroughput})
print('Database with id \'{0}\' created'.format(id))
except errors.DocumentDBError as e:
if e.status_code == 409:
print('A database with id \'{0}\' already exists'.format(id))
raise errors.HTTPFailure(e.status_code)
elif ( e.status_code == 400 and
isinstance(e,pydocumentdb.errors.HTTPFailure) ):
if 'throughput values between' in e.args[0]:
print('Invalid throughput value {0}'.format(ruThroughput))
raise errors.HTTPFailure(e.status_code)
else:
raise errors.HTTPFailure(e.status_code)
def _InternalRequest(connection_policy, request_options, request_body):
"""Makes one http request.
:Parameters:
- `connection_policy`: documents.ConnectionPolicy
- `request_options`: dict
- `request_body`: str, unicode or Non
:Returns:
tuple of (result, headers), where both result and headers
are dicts.
"""
is_media = request_options['path'].find('media') > -1
connection_timeout = (connection_policy.MediaRequestTimeout
if is_media
else connection_policy.RequestTimeout)
if connection_policy.ProxyConfiguration and connection_policy.ProxyConfiguration.Host:
connection = https_connection.HTTPSConnection(connection_policy.ProxyConfiguration.Host,
port=int(connection_policy.ProxyConfiguration.Port),
ssl_configuration=connection_policy.SSLConfiguration,
timeout=connection_timeout / 1000.0)
connection.set_tunnel(request_options['host'], request_options['port'], None)
else:
connection = https_connection.HTTPSConnection(request_options['host'],
port=request_options['port'],
ssl_configuration=connection_policy.SSLConfiguration,
timeout=connection_timeout / 1000.0)
connection.request(request_options['method'],
request_options['path'],
request_body,
request_options['headers'])
response = connection.getresponse()
headers = response.getheaders()
# In case of media response, return the response to the user and the user
# will need to handle reading the response.
if (is_media and
connection_policy.MediaReadMode == documents.MediaReadMode.Streamed):
return (response, dict(headers))
data = response.read()
if response.status >= 400:
raise errors.HTTPFailure(response.status, data, headers)
result = None
if is_media:
result = data
else:
if len(data) > 0:
try:
result = json.loads(data)
except:
raise errors.JSONParseFailure(data)
return (result, dict(headers))
def create_database(self, id):
try:
database = self.client.CreateDatabase({"id": id})
# print('Database with id \'{0}\' created'.format(id))
return database
except errors.DocumentDBError as e:
raise errors.HTTPFailure(e.status_code)
def main():
# Twitter application key
client = KeyVaultClient(KeyVaultAuthentication(auth_callback))
_appkey = client.get_secret("https://adfbookkeyvault.vault.azure.net/", "Twitter-appkey", "19bd289d86f449cbb98fd6a51cc63156")
_appsecret= client.get_secret("https://adfbookkeyvault.vault.azure.net/", "Twitter-appsecret", "510ceec80ef14af28dd961382be78e66")
_appaccesstoken = client.get_secret("https://adfbookkeyvault.vault.azure.net/", "Twitter-appaccesstoken", "e24ad1dfef6b4392bd6277e9ac51e4b8")
_appaccesstokensecret = client.get_secret("https://adfbookkeyvault.vault.azure.net/", "Twitter-appaccesstokensecret", "e46bb6b253584ea59d1941bd7f9f1905")
_tweetTag= sys.argv[1] # like Azure
_tweetReadSince= sys.argv[2] #date from when you want to read tweets like '2018/07/28'
_PipelineRunId= sys.argv[3] #Azure Data Factory Pipeline ID 'testrun'
# Azure Storage Credential
_accountname=client.get_secret("https://adfbookkeyvault.vault.azure.net/", "Storage-accountname", "0f9adaf2abb545b38757b37a0d63fc68")
_accountkey=client.get_secret("https://adfbookkeyvault.vault.azure.net/", "Storage-accountkey", "141200d435694c949a7c011bbf55d40a")
_InputContainerName='tweetcontainer'
# CosmosDB Credential
_cdbhost = client.get_secret("https://adfbookkeyvault.vault.azure.net/", "cosmosdbURI", "7c885660bce64bd6ae7b44f1c925486c")
_cdbmasterkey = client.get_secret("https://adfbookkeyvault.vault.azure.net/", "cosmosdbPK", "f220ab6df8d240759435953af5d01e43")
#hashtag, tweetreadsince, filename includes pipeline id,
auth = tweepy.OAuthHandler(_appkey.value, _appsecret.value)
auth.set_access_token(_appaccesstoken.value, _appaccesstokensecret.value)
tweetapi = tweepy.API(auth,wait_on_rate_limit=True)
#local_path=os.path.expanduser("~/Documents")
local_file_name ="Tweets_" + _tweetTag + _PipelineRunId + ".csv"
full_path_to_file =os.path.join(os.getcwd(), local_file_name)
outFile = open(local_file_name,'a')
fieldnames = ['Tweet_Time', 'Tweet_Desc']
filewriter = csv.writer(outFile)
filewriter.writerow(fieldnames)
for tweet in tweepy.Cursor(tweetapi.search,q=_tweetTag,lang="en", since=_tweetReadSince).items(15):
try:
if tweet.text.encode('utf-8') != '' :
filewriter.writerow([tweet.created_at,tweet.text.encode('utf-8')])
insertintoCosmosDB (_cdbhost.value, _cdbmasterkey.value, tweet.created_at,tweet.text.encode('utf-8'))
except errors.DocumentDBError as e:
if e.status_code == 409:
pass
else:
raise errors.HTTPFailure(e.status_code)
print("Error while fetching and storing tweets!!!")
outFile.close()
break
try:
print full_path_to_file
print local_file_name
_blob_service = BlockBlobService(account_name=_accountname.value, account_key=_accountkey.value)
_blob_service.create_blob_from_path(_InputContainerName, local_file_name, full_path_to_file)
#print(local_file_name)
except:
print("Error while uploading file to Azure Blob Storage !!!")
def delete_collection(self, database_id, id):
database_link = 'dbs/' + database_id
try:
collection_link = database_link + '/colls/{0}'.format(id)
self.client.DeleteCollection(collection_link)
# print('Collection with id \'{0}\' was deleted'.format(id))
except errors.DocumentDBError as e:
raise errors.HTTPFailure(e.status_code)
def delete_database(self, id):
try:
database_link = 'dbs/' + id
self.client.DeleteDatabase(database_link)
# print('Database with id \'{0}\' was deleted'.format(id))
except errors.DocumentDBError as e:
raise errors.HTTPFailure(e.status_code)
def MockExecuteFunction(self, function, *args, **kwargs):
global location_changed
if self.endpoint_discovery_retry_count == 2:
retry_utility._ExecuteFunction = self.OriginalExecuteFunction
return (json.dumps([{'id': 'mock database'}]), None)
else:
self.endpoint_discovery_retry_count += 1
location_changed = True
raise errors.HTTPFailure(403, "Forbidden", {'x-ms-substatus': 3})
def CreateDatabaseIfNotExists(client, database_id):
try:
database = Query_Entities(client, 'database', id=database_id)
if database == None:
database = client.CreateDatabase({"id": database_id})
return database
except errors.DocumentDBError as e:
if e.status_code == 409: # Move these constants to an enum
pass
else:
raise errors.HTTPFailure(e.status_code)
def read_collection(self, database_id, id):
database_link = 'dbs/' + database_id
try:
collection_link = database_link + '/colls/{0}'.format(id)
collection = self.client.ReadCollection(collection_link)
# print('Collection with id \'{0}\' was found, it\'s _self is {1}'.format(collection['id'],
# collection['_self']))
return collection
except errors.DocumentDBError as e:
raise errors.HTTPFailure(e.status_code)
def read_database(self, id):
try:
database_link = 'dbs/' + id
database = self.client.ReadDatabase(database_link)
# print('Database with id \'{0}\' was found, it\'s _self is {1}'.format(id, database['_self']))
return database
except errors.DocumentDBError as e:
raise errors.HTTPFailure(e.status_code)
def insertintoCosmosDB(cdbhost, cdbmasterkey, tweetDate, tweetText):
tweetmessage = {'tweetDate': str(tweetDate),'id' : str(tweetDate), 'tweetText': tweetText}
_database_link = 'dbs/tweetdb'
_collection_link = _database_link + '/colls/tweetcollec'
with IDisposable(document_client.DocumentClient(cdbhost, {'masterKey': cdbmasterkey} )) as client:
try:
client.CreateDocument(_collection_link, tweetmessage, options=False)
except errors.DocumentDBError as e:
if e.status_code == 409:
pass
else:
raise errors.HTTPFailure(e.status_code)
def deleteDatabase(client, id):
try:
database_link = 'dbs/' + id
client.DeleteDatabase(database_link)
print('Database with id \'{0}\' was deleted'.format(id))
except errors.DocumentDBError as e:
if e.status_code == 404:
print('A database with id \'{0}\' does not exist'.format(id))
else:
raise errors.HTTPFailure(e.status_code)
def create_doc(self, database_id, collection_id, doc):
docs = self.find_doc_by_id(database_id, collection_id, doc['id'])
if len(docs) != 0:
return self.replace_doc(docs[0], doc)
try:
collection_link = self.get_collection_link(database_id,
collection_id)
doc = self.client.CreateDocument(collection_link, doc)
return doc
except errors.DocumentDBError as e:
raise errors.HTTPFailure(e.status_code)
def create_database(client, id):
print("\n2. Create Database")
try:
client.CreateDatabase({"id": id})
print('Database with id \'{0}\' created'.format(id))
except errors.DocumentDBError as e:
if e.status_code == 409:
print('A database with id \'{0}\' already exists'.format(id))
else:
raise errors.HTTPFailure(e.status_code)
def read_collection(self):
# self.collection = next((coll for coll in self.client.ReadCollections(self.db['_self']) if coll['id'] == DOCUMENTDB_COLLECTION))
# print(self.collection)
try:
self.collection_link = self.database_link + '/colls/{0}'.format(DOCUMENTDB_COLLECTION)
self.collection = self.client.ReadCollection(self.collection_link)
print('Collection with id \'{0}\' was found, it\'s _self is {1}'.format(DOCUMENTDB_COLLECTION, self.collection['_self']))
except errors.DocumentDBError as e:
if e.status_code == 404:
print('A collection with id \'{0}\' does not exist'.format(DOCUMENTDB_COLLECTION))
else:
raise errors.HTTPFailure(e.status_code)
def read_database(self):
# if self.client:
# self.db = next((data for data in self.client.ReadDatabases() if data['id'] == DOCUMENTDB_DATABASE))
try:
self.database_link = 'dbs/' + DOCUMENTDB_DATABASE
self.db = self.client.ReadDatabase(self.database_link)
print('Database with id \'{0}\' was found, it\'s _self is {1}'.format(DOCUMENTDB_DATABASE, self.db['_self']))
except errors.DocumentDBError as e:
if e.status_code == 404:
print('A database with id \'{0}\' does not exist'.format(DOCUMENTDB_DATABASE))
else:
raise errors.HTTPFailure(e.status_code)
def create_doc(self, database_id, collection_id, doc):
docs = self.find_doc_by_id(database_id, collection_id, doc['id'])
if len(docs) != 0:
return self.replace_doc(docs[0])
try:
collection_link = self.get_collection_link(database_id, collection_id)
doc = self.client.CreateDocument(collection_link, doc)
return doc
except errors.DocumentDBError as e:
if e.status_code == 404:
print('A collection with id \'{0}\' does not exist'.format(collection_id))
else:
raise errors.HTTPFailure(e.status_code)
def delete_collection(client, id):
print("\n6. Delete Collection")
try:
collection_link = database_link + '/colls/{0}'.format(id)
client.DeleteCollection(collection_link)
print('Collection with id \'{0}\' was deleted'.format(id))
except errors.DocumentDBError as e:
if e.status_code == 404:
print('A collection with id \'{0}\' does not exist'.format(id))
else:
raise errors.HTTPFailure(e.status_code)
def initiate_database(my_cosmos_client):
cosmos_db_handle = None
try:
cosmos_db_handle = my_cosmos_client.CreateDatabase(
{"id": config['db']})
print('Created a new database with id \'{0}\'.'.format(config['db']))
except errors.DocumentDBError as e:
if e.status_code == 409:
print('Initializing the database with id \'{0}\' ...'.format(
config['db']))
cosmos_db_handle = my_cosmos_client.ReadDatabase('dbs/' +
config['db'])
else:
raise errors.HTTPFailure(e.status_code)
return cosmos_db_handle
def read_database(client, id):
"""Read a CosmosDB database.
Args:
client (obj): A pydocumentdb client object.
id (str): Database ID.
Returns:
obj: A database.
"""
try:
database_link = "dbs/" + id
database = client.ReadDatabase(database_link)
return database
except errors.DocumentDBError as e:
if e.status_code == 404:
print("A database with id '{0}' does not exist".format(id))
else:
raise errors.HTTPFailure(e.status_code)
def create_collection(self, database_id, id, offer_enable_ru_per_min_throughput, offer_version, offer_throughput):
database_link = 'dbs/' + database_id
options = {
'offerEnableRUPerMinuteThroughput': offer_enable_ru_per_min_throughput,
'offerVersion': offer_version,
'offerThroughput': offer_throughput
}
try:
collection = self.client.CreateCollection(database_link, {"id": id}, options)
print('Collection with id \'{0}\' created'.format(id))
return collection
except errors.DocumentDBError as e:
if e.status_code == 409:
print('A collection with id \'{0}\' already exists'.format(id))
else:
raise errors.HTTPFailure(e.status_code)
def read_collection(client, id):
print("\n4. Get a Collection by id")
try:
# All DocumentDB resources are addressable via a link
# This link is constructed from a combination of resource hierachy and
# the resource id.
# Eg. The link for collection with an id of Bar in database Foo would be dbs/Foo/colls/Bar
collection_link = database_link + '/colls/{0}'.format(id)
collection = client.ReadCollection(collection_link)
print('Collection with id \'{0}\' was found, it\'s _self is {1}'.format(collection['id'], collection['_self']))
except errors.DocumentDBError as e:
if e.status_code == 404:
print('A collection with id \'{0}\' does not exist'.format(id))
else:
raise errors.HTTPFailure(e.status_code)
def read_database(client, id):
print("\n3. Get a Database by id")
try:
# All DocumentDB resources are addressable via a link
# This link is constructed from a combination of resource hierachy and
# the resource id.
# Eg. The link for database with an id of Foo would be dbs/Foo
database_link = 'dbs/' + id
database = client.ReadDatabase(database_link)
print('Database with id \'{0}\' was found, it\'s _self is {1}'.format(id, database['_self']))
except errors.DocumentDBError as e:
if e.status_code == 404:
print('A database with id \'{0}\' does not exist'.format(id))
else:
raise errors.HTTPFailure(e.status_code)
def read_collection(client, dbid, id):
"""Read a CosmosDB collection.
Args:
client (obj): A pydocumentdb client object.
dbid (str): Database ID.
id (str): Collection ID.
Returns:
obj: A collection.
"""
try:
database_link = "dbs/" + dbid
collection_link = database_link + "/colls/{0}".format(id)
collection = client.ReadCollection(collection_link)
return collection
except errors.DocumentDBError as e:
if e.status_code == 404:
print("A collection with id '{0}' does not exist".format(id))
else:
raise errors.HTTPFailure(e.status_code)
