class SummaryTable:
def __init__(self, account_name, account_key, table_name="summary"):
"""Initialiaze a table to store summary data. Values must be provided
for 'account_name' and 'account_key' which are values
associated with the Azure Storage account. 'table_name' is
optional and is the name of the table used (and created if
necessary) in the storage account.
"""
self.log = Log()
self.account_name = account_name
self.account_key = account_key
self.table_name = table_name
self.createAzureTable()
def createAzureTable(self):
"""
Create an Azure Table in which to store the summary results.
"""
self.table_service = TableService(self.account_name, self.account_key)
self.table_service.create_table(self.table_name)
def deleteTable(self, name):
"""
Delete a table in which summary results have been stored.
"""
self.table_service.delete_table(name, False)
def writeCount(self, count_type, count):
entry = {'PartitionKey': "count", "RowKey": count_type, 'total_count' : count}
self.table_service.insert_entity(self.table_name, entry)
def updateCount(self, count_type, count):
entry = {'total_count' : count}
self.table_service.update_entity(self.table_name, "count", count_type, entry)
def getCount(self, event_type):
"""
Get the total number of events of a given type.
"""
count = 0
entries = self.table_service.query_entities(self.table_name, "PartitionKey eq 'count' and RowKey eq '" + event_type + "'")
if len(entries) == 0:
self.writeCount(event_type, 0)
elif len(entries) > 1:
raise Exception('We have more than one summary entry for ' + event_type)
else:
count = entries[0].total_count
return count
for item in items:
print('Name: ' + item.description)
print('Cost: ' + str(item.cost) + '\n')
items = table_service.query_entities('itemstable', filter="PartitionKey eq 'coffeeshop'", select='description,price')
for item in items:
print('Name: ' + item.description)
print('Price: ' + str(item.price) + '\n')
time.sleep(1)
###
# This was a quick demo to see Tables in action.
# Although the actual cost is minimal (fractions of a cent per month) for the three entities we created, it's good to clean up resources when you're done
###
print('\nThis is a basic example of how Azure Storage Tables behave like a database.\nTo keep things tidy, let\'s clean up the Azure Storage resources we created.')
raw_input('Press Enter to continue...')
response = table_service.delete_table('itemstable')
if response == True:
print('Storage table: itemstable deleted successfully.')
else:
print('Error deleting Storage Table')
response = azurerm.delete_resource_group(auth_token, subscription_id, resourcegroup_name)
if response.status_code == 202:
print('Resource group: ' + resourcegroup_name + ' deleted successfully.')
else:
print('Error deleting resource group.')
break
#**********************Now update final df into excel*******************
final_df.to_excel('result' + meetingID +'.xlsx')
#delete container & full table/intermittent chunks from Azure table storage.
import azure
from azure.storage.table import TableService, Entity
from azure.storage.table.models import EntityProperty
from azure.storage.blob import BlockBlobService, PublicAccess
key ='WY3062K+hMvWGuZG2WlLVGtJNj1bi9************************************************/Xg8j1ITmmGg=='
accountname ='intelligentappstorage01'
block_blob_service = BlockBlobService(account_name=accountname,account_key=key)
block_blob_service.delete_container('audiofiles')
table_service = TableService(account_name=accountname, account_key=key)
table_service.delete_table('transcriptiontable')
for i in range(len(df)):
chunk=df['ChunkNo'][i]
starttime=df['Starttime'][i]
endtime=df['Endtime'][i]
filename = meetingID +"_" + "chunk" + str(chunk) + "_" + str(starttime) + "_" + str(endtime) + ".wav"
os.remove(meetingID +"_" + "chunk" + str(chunk) + "_" + str(starttime) + "_" + str(endtime) + ".wav")
print("Local Audio Files Removed!")
########******************Sentiment Score*******************
import urllib.request
import json
# Configure API access
apiKey = 'f*****************d0'
# Structuring queries like this improves performance as your application scales up and keeps the queries efficient
pizzas = table_service.query_entities('pizzatable',
filter="PartitionKey eq 'pizzamenu'",
select='description,cost')
for pizza in pizzas:
print('Name: ' + pizza.description)
print('Cost: ' + str(pizza.cost) + '\n')
time.sleep(1)
###
# This was a quick demo to see Tables in action.
# Although the actual cost is minimal (fractions of a cent per month) for the three entities we created, it's good to clean up resources when you're done
###
print(
'\nThis is a basic example of how Azure Storage Tables behave like a database.\nTo keep things tidy, let\'s clean up the Azure Storage resources we created.'
)
raw_input('Press Enter to continue...')
response = table_service.delete_table('pizzatable')
if response == True:
print('Storage table: pizzatable deleted successfully.')
else:
print('Error deleting Storage Table')
response = azurerm.delete_resource_group(auth_token, subscription_id,
resourcegroup_name)
if response.status_code == 202:
print('Resource group: ' + resourcegroup_name + ' deleted successfully.')
else:
print('Error deleting resource group.')
class TableStorage():
def __init__(self, CONNECTION_STRING):
"""
Constructor. Espera el Connection String del Azure Storage Account.
Se obtiene ingresando al recurso de Storage -> Access Keys
Parametros:
CONNECTION_STRING = El string que incluye el AccountName,
AccountKey y el EndPointSuffix
"""
self.CONNECTION_STRING = CONNECTION_STRING
# Separa por partes el string de conexión
Config = dict(
s.split('=', 1) for s in CONNECTION_STRING.split(';') if s)
# Obtiene el nombre de la cuenta de storage y en EndpointSuffix
self.AccountName = Config.get('AccountName')
self.EndPointSuffix = Config.get('EndpointSuffix')
def CreateTableServices(self):
"""
Inicializa una instancia del Table Services para poder comunicarse con
el storage en Azure
"""
self.TableService = TableService(
account_name=self.AccountName,
connection_string=self.CONNECTION_STRING,
endpoint_suffix=self.EndPointSuffix)
def createTable(self, TableName):
"""
Revisa si la tabla no exista ya y la crea. De lo contrario, avisa que ya existe.
Paramentros:
TableName = Nombre de la tabla que se quiere crear
"""
print('\nCreate a table with name - ' + TableName)
if (self.TableService.exists(TableName) != True):
self.TableService.create_table(TableName)
print("Table created succesfully!")
else:
print('Error creating table, ' + TableName +
' check if it already exists')
def insertEntity(self, TableName, Entity):
"""
Se inserta una entidad a la tabla especificada.
Paramentros:
TableName = Nombre de la tabla que se quiere crear
Entity = El objecto con la entidad que se quiere agregar
"""
print('\nInserting a new entity into table - ' + TableName)
self.TableService.insert_or_merge_entity(TableName, Entity)
print('Successfully inserted the new entity')
def getEntity(self, TableName, PartitionKey, RowKey):
"""
Traerse la entidad completa en base a la Partition Key y Row Key.
Regresa un objeto como tal, no hay que hacer json.loads()
Paramentros:
TableName = Nombre de la tabla que se quiere crear
PartitionKey = String con la partition key de la entidad deseada
RowKey = String con la row key de la entidad deseada
"""
print('\nGetting entity.')
Entity = self.TableService.get_entity(TableName, PartitionKey, RowKey)
return Entity
def updateEntity(self, TableName, NewEntity):
"""
Toma el objeto con los datos actualizados y hace update en la table storage.
Paramentros:
TableName = Nombre de la tabla que se quiere crear
NewEntity = El objecto con la entidad que se quiere hacer update
"""
print('\nUpdating entity. PK: ' + NewEntity.PartitionKey + ' RK: ' +
NewEntity.RowKey)
self.TableService.update_entity(TableName, NewEntity)
def deleteEntity(self, TableName, PartitionKey, RowKey):
"""
Borrar la entidad que coincida en Partition Key y Row Key
Paramentros:
TableName = Nombre de la tabla que se quiere crear
PartitionKey = String con la partition key de la entidad
RowKey = String con la row key de la entidad
"""
print('\nDeleting entity')
self.TableService.delete_entity(TableName, PartitionKey, RowKey)
def deleteTable(self, TableName):
"""
Revisa si la tabla existe y la borra, en caso contrario solo avisa que no existe.
Paramentros:
TableName = Nombre de la tabla que se quiere borrar
"""
print('\nDeleting the table.')
if (self.TableService.exists(TableName)):
self.TableService.delete_table(TableName)
print('Successfully deleted the table')
else:
print('The table does not exists')
}
task11 = {
'PartitionKey': 'tasksSeattle',
'RowKey': '11',
'description': 'Clean the bathroom',
'priority': 100
}
table_service.begin_batch()
table_service.insert_entity('tasktable', task10)
table_service.insert_entity('tasktable', task11)
table_service.commit_batch()
task = table_service.get_entity('tasktable', 'tasksSeattle', '1')
print(task.description)
print(task.priority)
tasks = table_service.query_entities('tasktable',
"PartitionKey eq 'tasksSeattle'")
for task in tasks:
print(task.description)
print(task.priority)
tasks = table_service.query_entities('tasktable',
"PartitionKey eq 'tasksSeattle'",
'description')
for task in tasks:
print(task.description)
table_service.delete_entity('tasktable', 'tasksSeattle', '1')
table_service.delete_table('tasktable')
table_service.update_entity('tasktable', 'tasksSeattle', '1', task)
#
task = {'description' : 'Take out the garbage again', 'priority' : 250}
table_service.insert_or_replace_entity('tasktable', 'tasksSeattle', '1', task)
task = {'description' : 'Buy detergent', 'priority' : 300}
table_service.insert_or_replace_entity('tasktable', 'tasksSeattle', '3', task)
task10 = {'PartitionKey': 'tasksSeattle', 'RowKey': '10', 'description' : 'Go grocery shopping', 'priority' : 400}
task11 = {'PartitionKey': 'tasksSeattle', 'RowKey': '11', 'description' : 'Clean the bathroom', 'priority' : 100}
table_service.begin_batch()
table_service.insert_entity('tasktable', task10)
table_service.insert_entity('tasktable', task11)
table_service.commit_batch()
task = table_service.get_entity('tasktable', 'tasksSeattle', '1')
print(task.description)
print(task.priority)
tasks = table_service.query_entities('tasktable', "PartitionKey eq 'tasksSeattle'")
for task in tasks:
print(task.description)
print(task.priority)
tasks = table_service.query_entities('tasktable', "PartitionKey eq 'tasksSeattle'", 'description')
for task in tasks:
print(task.description)
table_service.delete_entity('tasktable', 'tasksSeattle', '1')
table_service.delete_table('tasktable')
items = table_service.query_entities('randomitemswk6',
filter="PartitionKey eq 'coffeestore'",
select='brand,flavor')
for item in items:
print('Brand: ' + item.brand)
print('Flavor: ' + str(item.flavor) + '\n')
time.sleep(1)
###
# This was a quick demo to see Tables in action.
# Although the actual cost is minimal (fractions of a cent per month) for the three entities we created, it's good to clean up resources when you're done
###
print(
'\nThis is a basic example of how Azure Storage Tables behave like a database.\nTo keep things tidy, let\'s clean up the Azure Storage resources we created.'
)
raw_input('Press Enter to continue...')
response = table_service.delete_table('randomitemswk6')
if response == True:
print('Storage table: randomitemswk6 deleted successfully.')
else:
print('Error deleting Storage Table')
response = azurerm.delete_resource_group(auth_token, subscription_id,
resourcegroup_name)
if response.status_code == 202:
print('Resource group: ' + resourcegroup_name + ' deleted successfully.')
else:
print('Error deleting resource group.')
class SummaryTable:
def __init__(self, account_name, account_key, table_name="summary"):
"""Initialiaze a table to store summary data. Values must be provided
for 'account_name' and 'account_key' which are values
associated with the Azure Storage account. 'table_name' is
optional and is the name of the table used (and created if
necessary) in the storage account.
"""
self.log = Log()
self.account_name = account_name
self.account_key = account_key
self.table_name = table_name
self.createAzureTable()
def createAzureTable(self):
"""
Create an Azure Table in which to store the summary results.
"""
self.table_service = TableService(self.account_name, self.account_key)
self.table_service.create_table(self.table_name)
def deleteTable(self, name):
"""
Delete a table in which summary results have been stored.
"""
self.table_service.delete_table(name, False)
def writeCount(self, count_type, count):
entry = {
'PartitionKey': "count",
"RowKey": count_type,
'total_count': count
}
self.table_service.insert_entity(self.table_name, entry)
def updateCount(self, count_type, count):
entry = {'total_count': count}
self.table_service.update_entity(self.table_name, "count", count_type,
entry)
def getCount(self, event_type):
"""
Get the total number of events of a given type.
"""
count = 0
entries = self.table_service.query_entities(
self.table_name,
"PartitionKey eq 'count' and RowKey eq '" + event_type + "'")
if len(entries) == 0:
self.writeCount(event_type, 0)
elif len(entries) > 1:
raise Exception('We have more than one summary entry for ' +
event_type)
else:
count = entries[0].total_count
return count
def writeLastProcessingTime(self, processing_time):
entry = {
'PartitionKey': "processing",
"RowKey": "last",
'duration': str(processing_time)
}
self.table_service.insert_entity(self.table_name, entry)
def updateLastProcessingTime(self, processing_time):
last_time = self.table_service.query_entities(
self.table_name,
"PartitionKey eq 'processing' and RowKey eq 'last'")
entry = {"duration": str(processing_time)}
self.table_service.update_entity(self.table_name, "processing", "last",
entry)
def getLastProcessingTime(self):
"""Gete the processing time for the last processed event."""
last = self.table_service.query_entities(
self.table_name,
"PartitionKey eq 'processing' and RowKey eq 'last'")
if len(last) == 0:
print("No last processing duration, creating record")
self.writeLastProcessingTime(0)
return 0
elif len(last) > 1:
raise Exception(
'We have more than one summary entry for last processing duration'
)
else:
duration = last[0].duration
print("Last processing duration: " + duration)
return duration
class az(object):
def __init__(self,
default_table_name=DEFAULT_TABLE,
partitionKey='default'):
self.TABLE_STORAGE_KEY = os.getenv('AZURE_STORAGE_KEY')
self.STORAGE_NAME = os.getenv('STORAGE_NAME')
self.default_table_name = default_table_name
self.default_partition = partitionKey
if self.TABLE_STORAGE_KEY == None:
from tokens import TABLE_STORAGE_ACCESS_KEY, STORAGE_ACCOUNT_NAME
self.TABLE_STORAGE_KEY = TABLE_STORAGE_ACCESS_KEY
self.STORAGE_NAME = STORAGE_ACCOUNT_NAME
self.table_service = TableService(account_name=self.STORAGE_NAME,
account_key=self.TABLE_STORAGE_KEY)
#create_table_if_does_not_exists(self.default_table_name)
def insert_or_replace_entity_to_azure(self,
rowKey,
entry,
t_name=DEFAULT_TABLE):
'''
takes table service
Takes a list
Uploads to azure table storage
'''
segment = Entity()
segment.PartitionKey = self.default_partition
segment.RowKey = str(rowKey).zfill(8)
segment.latA = str(entry['latA'])
segment.longA = str(entry['longA'])
segment.latB = str(entry['latB'])
segment.longB = str(entry['longB'])
segment.colorKey = str(entry['color'])
#print segment.colorKey
if os.name == 'nt':
self.table_service.insert_or_replace_entity(
t_name, self.default_partition,
str(rowKey).zfill(8), segment)
else:
self.table_service.insert_or_replace_entity(t_name, segment)
def create_table(self, name):
return self.table_service.create_table(name)
def delete_table(self, name):
return self.table_service.delete_table(name)
def delete_entity_by_rowKey(self, rowKey, table_name=DEFAULT_TABLE):
return self.table_service.delete_entity(table_name,
self.default_partition, rowKey)
def does_table_exist(self, table_name):
if os.name == 'nt':
for i in self.table_service.query_tables():
if i.name == table_name:
return True
else:
for i in self.table_service.list_tables():
if i.name == table_name:
return True
return False
def list_tables(self):
if os.name == 'nt':
for j in self.table_service.query_tables():
print j.name
else:
for j in self.table_service.list_tables():
print j.name
def create_table_if_does_not_exist(self, table_name=DEFAULT_TABLE):
if self.does_table_exist(table_name):
return 'already exists'
else:
self.table_service.create_table(table_name)
def create_entry(self, latA, lonA, latB, lonB, bumpiness):
x = {
'latA': latA,
'longA': lonA,
'latB': latB,
'longB': lonB,
'color': bumpiness
}
return x
def create_random_entry(self):
x = {
'latA': random.uniform(37, 38),
'longA': random.uniform(-122, -123),
'latB': random.uniform(37, 38),
'longB': random.uniform(-122, -123),
'color': random.randint(0, 7)
}
return x
def create_and_insert_or_replace_entity_azure(self,
latA,
lonA,
latB,
lonB,
bumpiness,
rowKey,
table_name=DEFAULT_TABLE):
return self.insert_or_replace_entity_to_azure(
rowKey, create_entry(latA, lonA, latB, lonB, bumpiness),
table_name)
class az(object):
def __init__(self, default_table_name=DEFAULT_TABLE, partitionKey='default'):
self.TABLE_STORAGE_KEY = os.getenv('AZURE_STORAGE_KEY')
self.STORAGE_NAME = os.getenv('STORAGE_NAME')
self.default_table_name = default_table_name
self.default_partition = partitionKey
if self.TABLE_STORAGE_KEY == None:
from tokens import TABLE_STORAGE_ACCESS_KEY, STORAGE_ACCOUNT_NAME
self.TABLE_STORAGE_KEY = TABLE_STORAGE_ACCESS_KEY
self.STORAGE_NAME = STORAGE_ACCOUNT_NAME
self.table_service = TableService(account_name=self.STORAGE_NAME, account_key=self.TABLE_STORAGE_KEY)
#create_table_if_does_not_exists(self.default_table_name)
def insert_or_replace_entity_to_azure(self, rowKey, entry, t_name=DEFAULT_TABLE):
'''
takes table service
Takes a list
Uploads to azure table storage
'''
segment = Entity()
segment.PartitionKey = self.default_partition
segment.RowKey = str(rowKey).zfill(8)
segment.latA = str(entry['latA'])
segment.longA = str(entry['longA'])
segment.latB = str(entry['latB'])
segment.longB = str(entry['longB'])
segment.colorKey = str(entry['color'])
#print segment.colorKey
if os.name == 'nt':
self.table_service.insert_or_replace_entity(t_name, self.default_partition, str(rowKey).zfill(8), segment)
else:
self.table_service.insert_or_replace_entity(t_name, segment)
def create_table(self, name):
return self.table_service.create_table(name)
def delete_table(self, name):
return self.table_service.delete_table(name)
def delete_entity_by_rowKey(self, rowKey, table_name=DEFAULT_TABLE):
return self.table_service.delete_entity(table_name, self.default_partition, rowKey)
def does_table_exist(self, table_name):
if os.name == 'nt':
for i in self.table_service.query_tables():
if i.name == table_name:
return True
else:
for i in self.table_service.list_tables():
if i.name == table_name:
return True
return False
def list_tables(self):
if os.name == 'nt':
for j in self.table_service.query_tables():
print j.name
else:
for j in self.table_service.list_tables():
print j.name
def create_table_if_does_not_exist(self, table_name=DEFAULT_TABLE):
if self.does_table_exist(table_name):
return 'already exists'
else:
self.table_service.create_table(table_name)
def create_entry(self, latA, lonA, latB, lonB, bumpiness):
x = {
'latA':latA,
'longA':lonA,
'latB':latB,
'longB':lonB,
'color': bumpiness
}
return x
def create_random_entry(self):
x = {
'latA':random.uniform(37,38),
'longA':random.uniform(-122,-123),
'latB':random.uniform(37,38),
'longB':random.uniform(-122,-123),
'color': random.randint(0,7)
}
return x
def create_and_insert_or_replace_entity_azure(self, latA, lonA, latB, lonB, bumpiness, rowKey, table_name=DEFAULT_TABLE ):
return self.insert_or_replace_entity_to_azure(rowKey, create_entry(latA, lonA, latB, lonB, bumpiness), table_name)
