def test_sas_add_inside_range(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
token = self.ts.generate_table_shared_access_signature(
self.table_name,
TablePermissions.ADD,
datetime.utcnow() + timedelta(hours=1),
start_pk='test', start_rk='test1',
end_pk='test', end_rk='test1',
)
# Act
service = TableService(
account_name=self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
)
self._set_service_options(service, self.settings)
entity = self._create_random_entity_dict('test', 'test1')
service.insert_entity(self.table_name, entity)
# Assert
resp = self.ts.get_entity(self.table_name, 'test', 'test1')
self._assert_default_entity(resp)
def main():
blob_container = request.json["container"]
blob_id = request.json["id"]
# Load up the .json from blob service
blob_service = BlockBlobService(account_name=_storage_account,
account_key=_storage_key)
blob = blob_service.get_blob_to_text(blob_container, blob_id)
# verbatims is a list of strings
verbatims = json.loads(blob.content)
# Generate a UUID for this job, since it's going to be a long running task
# we're going to return the id to the caller and track job status is the table 'ldajobs'
jobid = str(uuid.uuid4())
# Create the table row for this job, initially status is 'started'
table_service = TableService(account_name=_storage_account,
account_key=_storage_key)
table_service.create_table("ldajobs")
task = {'PartitionKey': 'lda_jobs', 'RowKey': jobid, 'status': 'started'}
table_service.insert_entity('ldajobs', task)
# Actually start the job
threading.Thread(target=lda, args=(
jobid,
verbatims,
)).start()
# .. and immediately return the jobid to the caller
return Response("%s verbatims now processing" % len(verbatims),
status=200,
mimetype='plain/text')
def test_sas_add_outside_range(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recordingfile(self.test_mode):
return
# Arrange
token = self.ts.generate_table_shared_access_signature(
self.table_name,
TablePermissions.ADD,
datetime.utcnow() + timedelta(hours=1),
start_pk='test',
start_rk='test1',
end_pk='test',
end_rk='test1',
)
# Act
service = TableService(
account_name=self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
)
self._set_service_options(service, self.settings)
with self.assertRaises(AzureHttpError):
entity = self._create_random_entity_dict()
service.insert_entity(self.table_name, entity)
def test_sas_add(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recordingfile(self.test_mode):
return
# Arrange
token = self.ts.generate_table_shared_access_signature(
self.table_name,
TablePermissions.ADD,
datetime.utcnow() + timedelta(hours=1),
datetime.utcnow() - timedelta(minutes=1),
)
# Act
service = TableService(
account_name=self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
)
self._set_service_options(service, self.settings)
entity = self._create_random_entity_dict()
service.insert_entity(self.table_name, entity)
# Assert
resp = self.ts.get_entity(self.table_name, entity['PartitionKey'],
entity['RowKey'])
self._assert_default_entity(resp)
def receiver():
bus_service = ServiceBusService(
service_namespace='comp6905',
shared_access_key_name='RootManageSharedAccessKey',
shared_access_key_value='rK2FMzVKvCjpad7xVSj1AB3hDimhxZq3WtEE4y28yaM=')
table_service = TableService(
account_name='comp6905kirk',
account_key=
'H1YuP8hBxJ2PKw2hoW4Dr+DMAMvKZ/nGhstHw+87mE+OSBTb23cBxhkUvILgKOHWHA3hi3oaoohwVkp6lOXOlA=='
)
while (True):
msg = bus_service.receive_queue_message('queue1', peek_lock=False)
msg1 = msg.body.decode("utf-8")
print(msg1)
parsed_json = json.loads(msg1)
#print(parsed_json['UserId'
task = {
'PartitionKey': 'Zanko',
'RowKey': parsed_json['TransactionID'],
'UserId': parsed_json['UserId'],
'SellerId': parsed_json['SellerID'],
'ProductName': parsed_json['Product Name'],
'SalePrice': parsed_json['Sale Price'],
'TransactionDate': parsed_json['Transaction Date']
}
table_service.insert_entity('Requests', task)
class SummaryTable:
def __init__(self, account_name, account_key, table_name="summary"):
"""Initialiaze a table to store summary data. Values must be provided
for 'account_name' and 'account_key' which are values
associated with the Azure Storage account. 'table_name' is
optional and is the name of the table used (and created if
necessary) in the storage account.
"""
self.log = Log()
self.account_name = account_name
self.account_key = account_key
self.table_name = table_name
self.createAzureTable()
def createAzureTable(self):
"""
Create an Azure Table in which to store the summary results.
"""
self.table_service = TableService(self.account_name, self.account_key)
self.table_service.create_table(self.table_name)
def deleteTable(self, name):
"""
Delete a table in which summary results have been stored.
"""
self.table_service.delete_table(name, False)
def writeCount(self, count_type, count):
entry = {'PartitionKey': "count", "RowKey": count_type, 'total_count' : count}
self.table_service.insert_entity(self.table_name, entry)
def updateCount(self, count_type, count):
entry = {'total_count' : count}
self.table_service.update_entity(self.table_name, "count", count_type, entry)
def getCount(self, event_type):
"""
Get the total number of events of a given type.
"""
count = 0
entries = self.table_service.query_entities(self.table_name, "PartitionKey eq 'count' and RowKey eq '" + event_type + "'")
if len(entries) == 0:
self.writeCount(event_type, 0)
elif len(entries) > 1:
raise Exception('We have more than one summary entry for ' + event_type)
else:
count = entries[0].total_count
return count
class MonitorManager(object):
def __init__(self):
#Please first create Azure Storage account and obtain your account name and key
self._tableService = TableService(account_name='YOUR_ACCOUNT_NAME', account_key='YOUR_ACCOUNT_KEY')
self._tableService.create_table('sensordata')
def Insert(self, distance, currentTime):
distanceData = Entity()
distanceData.PartitionKey = 'sensorKey'
distanceData.RowKey = str(uuid.uuid1())
distanceData.distance = str(distance)
distanceData.time = str(currentTime)
self._tableService.insert_entity('sensordata', distanceData)
def write_scores(predictions_queue, storage_account_name, storage_account_key):
table_service = TableService(account_name=storage_account_name,
account_key=storage_account_key)
table_service.create_table('predictions')
aggregate_predictions = {}
while True:
prediction_row = predictions_queue.get()
stripped_datetime = str(prediction_row[0].replace(second=0,
microsecond=0))
device_id = prediction_row[1]
prediction = prediction_row[2]
if stripped_datetime not in aggregate_predictions:
aggregate_predictions[stripped_datetime] = {}
if device_id not in aggregate_predictions[stripped_datetime]:
aggregate_predictions[stripped_datetime][device_id] = {}
if prediction not in aggregate_predictions[stripped_datetime][
device_id]:
aggregate_predictions[stripped_datetime][device_id][prediction] = 0
aggregate_predictions[stripped_datetime][device_id][prediction] += 1
if len(aggregate_predictions) < 2:
continue
timestamps = list(aggregate_predictions.keys())
timestamps.sort()
oldest_timestamp = timestamps[0]
device_ids = aggregate_predictions[oldest_timestamp].keys()
predictions = aggregate_predictions[oldest_timestamp].values()
for partition_key, row_key, data in zip(
[oldest_timestamp] * len(device_ids), device_ids, predictions):
entity = {
'PartitionKey': partition_key,
'RowKey': row_key,
'Prediction': max(data, key=data.get),
'Debug': json.dumps(data)
}
# TODO: implement optimistic concurrency and merge to run this truly at scale.
# For now, simply inserting the record...
table_service.insert_entity('predictions', entity)
del aggregate_predictions[oldest_timestamp]
def connection(tasks, query, types):
table_service = TableService(account_name=acc_name, account_key=acc_key)
table_service.create_table('customer')
if types == "insert":
print(tasks)
table_service.insert_entity('customer', tasks)
return "added Successfully!!!!"
elif types == "reterive":
#print(query)
tasks = table_service.query_entities('customer', filter=query)
#print(tasks)
df = pd.DataFrame(df_con(tasks))
data1 = df.to_json(orient='records')
#print(data1)
return data1
def load_from_csv(relative_path: str, storage_account: TableService):
filenames = os.listdir(relative_path)
for filename in filenames:
if filename.endswith(".csv"):
table_name = filename[:-4]
storage_account.create_table(table_name)
df = pd.read_csv(os.path.join(relative_path, filename))
for _, row in df.iterrows():
d = pd.Series.to_dict(row)
entity = {}
for k, v in d.items():
if k == 'Timestamp' or k.endswith('@type'):
pass
else:
entity[k] = v
storage_account.insert_entity(table_name, entity)
class Repository(object):
"""Azure Table storage repository for UOTD."""
def __init__(self, settings):
"""Initialise UOTD repository with the given settings dict.
Required settings:
STORAGE_NAME -- the Azure Storage account name
STORAGE_KEY -- an access key for the Storage account
STORAGE_TABLE_UOTD -- the name of the table
"""
self.service = TableService(settings["STORAGE_NAME"],
settings["STORAGE_KEY"])
self.uotd_table = settings["STORAGE_TABLE_UOTD"]
self.service.create_table(self.uotd_table)
self.partition_key_format = "%Y%m"
self.row_key_format = "%d"
def get_uotd(self):
"""Get the UUID for the current day.
If the UUID does not yet exist then it will be created.
"""
partition_key = date.today().strftime(self.partition_key_format)
row_key = date.today().strftime(self.row_key_format)
try:
uotd_entity = self.service.get_entity(self.uotd_table,
partition_key,
row_key)
uuid = uotd_entity.uuid
except AzureMissingResourceHttpError:
uuid = str(uuid4())
uotd_entity = {
"PartitionKey": partition_key,
"RowKey": row_key,
"uuid": uuid
}
self.service.insert_entity(self.uotd_table, uotd_entity)
return uuid
def get_uuids(self, partition_key):
"""Get all the UUIDs in a given partition."""
filter = "PartitionKey eq '{0}'".format(partition_key)
entities = self.service.query_entities(self.uotd_table, filter=filter)
return entities
def post_table_log(json_str):
from azure.storage.table import TableService, Entity
table_service = TableService(account_name=pkey.azure_storage_id, account_key=pkey.azure_storage_key)
table_service.create_table("facebooklog")
def get_table_timestamp_key():
import time
current_time = time.gmtime()
start = time.mktime(current_time)
last = time.mktime(time.struct_time((2070,1,1,0,0,0,3,100,-1)))
return str(int(last - start))
task = Entity()
task.PartitionKey = 'feedlog'
task.RowKey = get_table_timestamp_key()
task.json = json_str
table_service.insert_entity('facebooklog', task)
def SendAzure():
table_service = TableService(account_name='[NAMEHERE]', account_key='[KEYHERE]')
table_name = 'tempData'
partition_key = 'central'
table_service.create_table(table_name, False)
date = datetime.datetime.now()
iso_date = date.isoformat()
tempRecord = ReadTemp()
result = ""
if(tempRecord < 70):
entry = {'PartitionKey': partition_key, 'RowKey': iso_date, 'Temperature': tempRecord}
table_service.insert_entity(table_name, entry)
result = "SENT " + str(tempRecord)
else:
result = "ERROR " + str(tempRecord)
return result
def SendAzure():
table_service = TableService(account_name='[NAMEHERE]',
account_key='[KEYHERE]')
table_name = 'tempData'
partition_key = 'central'
table_service.create_table(table_name, False)
date = datetime.datetime.now()
iso_date = date.isoformat()
tempRecord = ReadTemp()
result = ""
if (tempRecord < 70):
entry = {
'PartitionKey': partition_key,
'RowKey': iso_date,
'Temperature': tempRecord
}
table_service.insert_entity(table_name, entry)
result = "SENT " + str(tempRecord)
else:
result = "ERROR " + str(tempRecord)
return result
class AzureStorage(BaseStorage):
def __init__(self, env):
super(AzureStorage, self).__init__(env)
self._ts = \
TableService(
account_name=Config.azure_storage_account_name,
account_key=Config.azure_storage_account_key)
self._benchmark = self._env.benchmark.name
# Make sure our table exists
Debug.info << "Creating tableservice for benchmark : " << \
self.table_name() << "\n"
self._ts.create_table(self.table_name())
def table_name(self):
return self._env.table_name
def save(self, dic, partition=None, key=''):
dic['RowKey'] = str(self.reverse_timestamp())
dic['Cloud'] = str(self._env.cloud_name)
if key:
dic['RowKey'] = dic['RowKey'] + '_' + str(key)
# Don't really need the partition key right now
if partition is None:
dic['PartitionKey'] = self._env.benchmark.name
else:
dic['PartitionKey'] = partition
try:
self._ts.insert_entity(self.table_name(), dic)
except:
print >> sys.stderr, "Error saving: %s" % dic
import struct
def get_ip_address(ifname):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
return socket.inet_ntoa(
fcntl.ioctl(
s.fileno(),
0x8915, # SIOCGIFADDR
struct.pack('256s', ifname[:15]))[20:24])
ip = get_ip_address('eth0')
else:
# http://stackoverflow.com/questions/166506/finding-local-ip-addresses-using-pythons-stdlib
ip = socket.gethostbyname(socket.getfqdn())
# http://stackoverflow.com/questions/799767/getting-name-of-windows-computer-running-python-script
name = socket.gethostname()
# https://azure.microsoft.com/en-us/documentation/articles/storage-python-how-to-use-table-storage/
account_name = 'xxx'
account_key = 'xxx'
table_service = TableService(account_name, account_key)
table_service.create_table('deviceip')
task = {
'PartitionKey': 'localIp',
'RowKey': str(uuid.uuid4()),
'ip': '' + ip,
'hostName': '' + name
}
table_service.insert_entity('deviceip', task)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
initialHash = None
# Get url from header
urlHeader = req.headers.get('url')
# Get account_name and account_key from body
body = req.get_body().decode('utf8').replace("'", '"')
bodyJSON = json.loads(body)
account_name = bodyJSON.get('account_name')
account_key = bodyJSON.get('account_key')
# Connection to table and read initial Hash
table_service = TableService(account_name, account_key)
# Get inital Hash from table
# Search URL in table
filterContent = "URL eq '{}'".format(urlHeader)
initialHashRows = list(
table_service.query_entities('Hashes', filter=filterContent))
# Catch Request Exceptions
try:
requ = requests.get(url=urlHeader)
except requests.exceptions.RequestException:
return func.HttpResponse("Problem with URL or connection",
status_code=209)
# Test if URL is still active
statusCode = requ.status_code
# For status code 204, it is not possible to write something into the response body, therefor only the status code is returned
if str(statusCode) == '404':
return func.HttpResponse(status_code=204)
# Sort Hashes, which fit to URL
if len(initialHashRows) != 0:
sortedHashes = sorted(initialHashRows,
key=lambda i: i['Timestamp'],
reverse=True)
initialHashLine = sortedHashes[0]
initialHash = initialHashLine['Hash']
if urlHeader and account_name and account_key:
# Get Bytes behind URL
try:
fb = urllib.request.urlopen(urlHeader)
except requests.exceptions.RequestException:
return func.HttpResponse("Problem with URL or connection",
status_code=209)
htmlInBytes = fb.read()
fb.close()
# Apply hash function
htmlHash = hashlib.md5(htmlInBytes)
htmlHashString = htmlHash.hexdigest()
# Save Hash in Table Storage
newHash = {
'PartitionKey': 'HashAGB',
'RowKey': str(uuid.uuid1()),
'Hash': htmlHashString,
'URL': urlHeader
}
table_service.insert_entity('Hashes', newHash)
# Compare the newly calculated Hash with inital Hash
# First time hash is applied on this url
if initialHash is None:
return func.HttpResponse("First time URL has been tested",
status_code=202)
# Hash has changed
elif initialHash != htmlHashString:
return func.HttpResponse("URL has changed", status_code=201)
# Hash has not changed
else:
return func.HttpResponse("URL has not changed", status_code=200)
else:
return func.HttpResponse(
"Please pass the required parameters in body and header: URL in header, Account Key and Account Name for storage in body",
status_code=400)
# Here, we add an entry for first pizza with two pieces of data - the name, and the cost
#
# A partition key tracks how like-minded entries in the Table are created and queried.
# A row key is a unique ID for each entity in the partition
# These two properties are used as a primary key to index the Table. This makes queries much quicker.
cardealership = Entity()
cardealership.PartitionKey = 'cardealership'
cardealership.RowKey = '001'
cardealership.make = 'Tesla'
cardealership.model = 'Model X'
cardealership.year = '2017'
cardealership.color = 'Black'
cardealership.price = 110000
table_service.insert_entity('itemstable', cardealership)
print('Created entry for Tesla...')
cardealership = Entity()
cardealership.PartitionKey = 'cardealership'
cardealership.RowKey = '002'
cardealership.make = 'Lexus'
cardealership.model = 'IS250'
cardealership.year = '2017'
cardealership.color = 'Grey'
cardealership.price = 45000
table_service.insert_entity('itemstable', cardealership)
print('Created entry for Lexus...')
cardealership = Entity()
cardealership.PartitionKey = 'cardealership'
)
raw_input('Press Enter to continue...')
# Each entry in a Table is called an 'entity'.
# Here, we add an entry for first pizza with two pieces of data - the name, and the cost
#
# A partition key tracks how like-minded entries in the Table are created and queried.
# A row key is a unique ID for each entity in the partition
# These two properties are used as a primary key to index the Table. This makes queries much quicker.
pizza = Entity()
pizza.PartitionKey = 'pizzamenu'
pizza.RowKey = '001'
pizza.description = 'Pepperoni'
pizza.cost = 18
table_service.insert_entity('pizzatable', pizza)
print('Created entry for pepperoni...')
pizza = Entity()
pizza.PartitionKey = 'pizzamenu'
pizza.RowKey = '002'
pizza.description = 'Veggie'
pizza.cost = 15
table_service.insert_entity('pizzatable', pizza)
print('Created entry for veggie...')
pizza = Entity()
pizza.PartitionKey = 'pizzamenu'
pizza.RowKey = '003'
pizza.description = 'Hawaiian'
pizza.cost = 12
class SummaryTable:
def __init__(self, account_name, account_key, table_name="summary"):
"""Initialiaze a table to store summary data. Values must be provided
for 'account_name' and 'account_key' which are values
associated with the Azure Storage account. 'table_name' is
optional and is the name of the table used (and created if
necessary) in the storage account.
"""
self.log = Log()
self.account_name = account_name
self.account_key = account_key
self.table_name = table_name
self.createAzureTable()
def createAzureTable(self):
"""
Create an Azure Table in which to store the summary results.
"""
self.table_service = TableService(self.account_name, self.account_key)
self.table_service.create_table(self.table_name)
def deleteTable(self, name):
"""
Delete a table in which summary results have been stored.
"""
self.table_service.delete_table(name, False)
def writeCount(self, count_type, count):
entry = {
'PartitionKey': "count",
"RowKey": count_type,
'total_count': count
}
self.table_service.insert_entity(self.table_name, entry)
def updateCount(self, count_type, count):
entry = {'total_count': count}
self.table_service.update_entity(self.table_name, "count", count_type,
entry)
def getCount(self, event_type):
"""
Get the total number of events of a given type.
"""
count = 0
entries = self.table_service.query_entities(
self.table_name,
"PartitionKey eq 'count' and RowKey eq '" + event_type + "'")
if len(entries) == 0:
self.writeCount(event_type, 0)
elif len(entries) > 1:
raise Exception('We have more than one summary entry for ' +
event_type)
else:
count = entries[0].total_count
return count
def writeLastProcessingTime(self, processing_time):
entry = {
'PartitionKey': "processing",
"RowKey": "last",
'duration': str(processing_time)
}
self.table_service.insert_entity(self.table_name, entry)
def updateLastProcessingTime(self, processing_time):
last_time = self.table_service.query_entities(
self.table_name,
"PartitionKey eq 'processing' and RowKey eq 'last'")
entry = {"duration": str(processing_time)}
self.table_service.update_entity(self.table_name, "processing", "last",
entry)
def getLastProcessingTime(self):
"""Gete the processing time for the last processed event."""
last = self.table_service.query_entities(
self.table_name,
"PartitionKey eq 'processing' and RowKey eq 'last'")
if len(last) == 0:
print("No last processing duration, creating record")
self.writeLastProcessingTime(0)
return 0
elif len(last) > 1:
raise Exception(
'We have more than one summary entry for last processing duration'
)
else:
duration = last[0].duration
print("Last processing duration: " + duration)
return duration
# Each entry in a Table is called an 'Entity'.
# Here, we add five entries for cars with five pieces of data: 1) make, 2) model, 3) year, 4) color, 5) price
#
# A partition key tracks how like-minded entries in the Table are created and queried.
# A row key is a unique ID for each entity in the partition
# These two properties are used as a primary key to index the Table. This makes queries much quicker.
cars = Entity()
cars.PartitionKey = 'car_selections'
cars.RowKey = '001'
cars.make = 'Volkswagen'
cars.model = 'Jetta'
cars.year = 2010
cars.color = 'silver'
cars.price = 13300
table_service.insert_entity('htangitemstable', cars)
print('Created entry for Volkswagen/Jetta...')
cars = Entity()
cars.PartitionKey = 'car_selections'
cars.RowKey = '002'
cars.make = 'Ford'
cars.model = 'Focus'
cars.year = 2011
cars.color = 'white'
cars.price = 13600
table_service.insert_entity('htangitemstable', cars)
print('Created entry for Ford/Focus...')
cars = Entity()
cars.PartitionKey = 'car_selections'
# Each entry in a Table is called an 'Entity'.
# Here, we add an entry for first car with two pieces of data - the name, and the cost
#
# A partition key tracks how like-minded entries in the Table are created and queried.
# A row key is a unique ID for each entity in the partition
# These two properties are used as a primary key to index the Table. This makes queries much quicker.
car = Entity()
car.PartitionKey = 'carmenu'
car.RowKey = '001'
car.make = 'Dodge'
car.model = 'Charger'
car.year = 2017
car.color = 'Black'
car.price = '55000'
table_service.insert_entity('itemstable', car)
print('Created entry for charger...')
car = Entity()
car.PartitionKey = 'carmenu'
car.make = 'Plymouth'
car.model = 'Duster'
car.year = 1976
car.color = 'Blue'
car.price = '5000'
table_service.insert_entity('itemstable', car)
print('Created entry for duster...')
car.PartitionKey = 'carmenu'
car.RowKey = '003'
car.make = 'Ford'
class AzureTableDatabase(object):
def __init__(self):
self.connection = TableService(account_name=storage_account,
account_key=table_connection_string)
self.table_name = table_name
def _update_entity(self, record):
record.LastModified = datetime.now()
self.connection.update_entity(self.table_name, record)
def create_table(self):
self.connection.create_table(self.table_name)
def raw_table(self, limit=100):
"""
Retrieve a list of rows in the table.
"""
calls = self.connection.query_entities(self.table_name,
num_results=limit)
return calls
def list_calls(self, limit=100, select='PartitionKey'):
"""
Retrieve a set of records that need a phone call
"""
calls = self.connection.query_entities(self.table_name,
num_results=limit,
select=select)
return [c.PartitionKey for c in calls]
def reset_stale_calls(self, time_limit):
"""
Retrieve calls that are not done and whose last modified time was older than the limit.
"""
records = self.connection.query_entities(
self.table_name,
filter="LastModified lt datetime'{0}' and Status ne '{1}'".format(
time_limit.date(), Statuses.extracting_done))
if not records.items:
raise NoRecordsToProcessError()
num_records = len(records.items)
for record in records:
if 'LastErrorStep' in record:
record.Status = record.LastErrorStep
del record.LastErrorStep
record.Status = Statuses.reset_map.get(record.Status,
record.Status)
self._update_entity(record)
return num_records
def retrieve_next_record_for_call(self):
"""
Retrieve a set of records that need a phone call
"""
records = self.connection.query_entities(
self.table_name,
num_results=1,
filter="Status eq '{0}'".format(Statuses.new))
if len(records.items) == 0:
raise NoRecordsToProcessError()
record = records.items[0]
record.Status = Statuses.calling
self._update_entity(record)
return record.PartitionKey
def set_error(self, partition_key, step):
""" Reset a row from error state
"""
record = self.connection.get_entity(self.table_name, partition_key,
partition_key)
record.Status = Statuses.error
record['LastErrorStep'] = step
self._update_entity(record)
def retrieve_next_record_for_transcribing(self):
records = self.connection.query_entities(
self.table_name,
num_results=1,
filter="Status eq '{0}'".format(Statuses.recording_ready),
)
if not records.items:
raise NoRecordsToProcessError()
record = records.items[0]
record.Status = Statuses.transcribing
self._update_entity(record)
return record.CallUploadUrl, record.PartitionKey
def update_transcript(self, partition_key, transcript, status):
record = self.connection.get_entity(
self.table_name,
partition_key,
partition_key,
)
if status == TranscriptionStatus.success:
record.CallTranscript = transcript
record.Status = Statuses.transcribing_done
record.TranscribeTimestamp = datetime.now()
self._update_entity(record)
elif status == TranscriptionStatus.request_error:
self.set_error(partition_key, Statuses.transcribing)
else:
record.Status = Statuses.transcribing_failed
self._update_entity(record)
def change_status(self, original_status, new_status):
records = self.connection.query_entities(
self.table_name,
filter="Status eq '{0}'".format(original_status),
)
if not records.items:
return
for record in records.items:
record.Status = new_status
self.connection.update_entity(self.table_name, record)
def query(self, column, value, limit=1):
records = self.connection.query_entities(self.table_name,
num_results=limit,
filter="{0} eq '{1}'".format(
column, value))
return records
def retrieve_next_record_for_extraction(self):
records = self.connection.query_entities(
self.table_name,
num_results=1,
filter="Status eq '{0}'".format(Statuses.transcribing_done))
if not records.items:
raise NoRecordsToProcessError()
record = records.items[0]
record.Status = Statuses.extracting
self._update_entity(record)
return record.CallTranscript, record.PartitionKey
def update_location_date(self, partition_key, location_dict, date_dict):
record = self.connection.get_entity(self.table_name, partition_key,
partition_key)
if location_dict and date_dict:
record.update(**location_dict)
record.update(**date_dict)
record.Status = Statuses.extracting_done
else:
record.Status = Statuses.failed_to_return_info
self._update_entity(record)
def upload_new_requests(self, request_ids):
"""
Upload new request ids to the database
"""
for request_id in request_ids:
record = {
'PartitionKey': request_id,
'RowKey': request_id,
'Status': Statuses.new,
'LastModified': datetime.now()
}
try:
self.connection.insert_entity(self.table_name, record)
except AzureConflictHttpError:
pass # already exists. silently ignore.
def update_call_id(self, alien_registration_id, call_id):
record = self.connection.get_entity(self.table_name,
alien_registration_id,
alien_registration_id)
record.CallID = call_id
record.Status = Statuses.calling
record.CallTimestamp = datetime.now()
self._update_entity(record)
def update_azure_path(self, alien_registration_id, azure_path):
record = self.connection.get_entity(self.table_name,
alien_registration_id,
alien_registration_id)
record.Status = Statuses.recording_ready
record.CallUploadUrl = azure_path
self._update_entity(record)
def delete_ain(self, ain):
return self.connection.delete_entity(self.table_name, ain, ain)
def get_ain(self, ain):
return self.connection.get_entity(self.table_name, ain, ain)
class Repository(object):
"""Azure Table Storage repository."""
def __init__(self, settings):
"""Initializes the repository with the specified settings dict.
Required settings are:
- STORAGE_NAME
- STORAGE_KEY
- STORAGE_TABLE_POLL
- STORAGE_TABLE_CHOICE
"""
self.name = 'Azure Table Storage'
self.storage_name = settings['STORAGE_NAME']
self.storage_key = settings['STORAGE_KEY']
self.poll_table = settings['STORAGE_TABLE_POLL']
self.choice_table = settings['STORAGE_TABLE_CHOICE']
self.svc = TableService(self.storage_name, self.storage_key)
self.svc.create_table(self.poll_table)
self.svc.create_table(self.choice_table)
def get_polls(self):
"""Returns all the polls from the repository."""
poll_entities = self.svc.query_entities(self.poll_table)
polls = [_poll_from_entity(entity) for entity in poll_entities]
return polls
def get_poll(self, poll_key):
"""Returns a poll from the repository."""
try:
partition, row = _key_to_partition_and_row(poll_key)
poll_entity = self.svc.get_entity(self.poll_table, partition, row)
choice_entities = self.svc.query_entities(
self.choice_table,
"PollPartitionKey eq '{0}' and PollRowKey eq '{1}'" \
.format(partition, row)
)
poll = _poll_from_entity(poll_entity)
poll.choices = [_choice_from_entity(choice_entity)
for choice_entity in choice_entities]
return poll
except AzureMissingResourceHttpError:
raise PollNotFound()
def increment_vote(self, poll_key, choice_key):
"""Increment the choice vote count for the specified poll."""
try:
partition, row = _key_to_partition_and_row(choice_key)
entity = self.svc.get_entity(self.choice_table, partition, row)
entity.Votes += 1
self.svc.update_entity(self.choice_table, entity)
except AzureMissingResourceHttpError:
raise PollNotFound()
def add_sample_polls(self):
"""Adds a set of polls from data stored in a samples.json file."""
poll_partition = '2014'
poll_row = 0
choice_partition = '2014'
choice_row = 0
for sample_poll in _load_samples_json():
poll_entity = {
'PartitionKey': poll_partition,
'RowKey': str(poll_row),
'Text': sample_poll['text'],
}
self.svc.insert_entity(self.poll_table, poll_entity)
for sample_choice in sample_poll['choices']:
choice_entity = {
'PartitionKey': choice_partition,
'RowKey': str(choice_row),
'Text': sample_choice,
'Votes': 0,
'PollPartitionKey': poll_partition,
'PollRowKey': str(poll_row),
}
self.svc.insert_entity(self.choice_table, choice_entity)
choice_row += 1
poll_row += 1
# Each entry in a Table is called an 'Entity'.
# Here, we add an entry for first automobile with two pieces of data - the name, and the cost
#
# A partition key tracks how like-minded entries in the Table are created and queried.
# A row key is a unique ID for each entity in the partition
# These two properties are used as a primary key to index the Table. This makes queries much quicker.
automobile = Entity()
automobile.PartitionKey = 'automobilelisting'
automobile.RowKey = '001'
automobile.make = 'Volkswagen'
automobile.model = 'GTI'
automobile.year = '2014'
automobile.color = 'White'
automobile.price = 16499
table_service.insert_entity('itemstable', automobile)
print('Created entry for Volkswagen GTI...')
automobile = Entity()
automobile.PartitionKey = 'automobilelisting'
automobile.RowKey = '002'
automobile.make = 'Nissan'
automobile.model = 'Pathfinder LE'
automobile.year = '2008'
automobile.color = 'Gray'
automobile.price = 8498
table_service.insert_entity('itemstable', automobile)
print('Created entry for Nissan Pathfinder...')
automobile = Entity()
automobile.PartitionKey = 'automobilelisting'
# sleep(2)
z = 0
# #
companies = [
dict(name="Skanska", lat=52.235704, lon=20.978249, price="7.48"),
dict(name="Microsoft", lat=52.200437, lon=20.936103, price="7.52"),
dict(name="Sage", lat=52.221016, lon=20.973410, price="7.53"),
dict(name="Ilmet", lat=52.232243, lon=20.998048, price="4.20")
]
for i, c in enumerate(companies):
lat = c.get('lat')
lon = c.get('lon')
name = c.get('name')
for j in range(24):
data = dict(PartitionKey='poland_warsaw',
RowKey=str(z + 1),
lat=lat,
lon=lon,
description=name,
id=str(i),
time_from=j,
time_to=j + 1,
free_spots=chet.random.randint(0, 90))
print(data)
ts.insert_entity('parking', data)
z += 1
# Here, we add an entry for first pizza with two pieces of data - the name, and the cost
#
# A partition key tracks how like-minded entries in the Table are created and queried.
# A row key is a unique ID for each entity in the partition
# These two properties are used as a primary key to index the Table. This makes queries much quicker.
# cars
cars = Entity()
cars.PartitionKey = 'CarDealer'
cars.RowKey = '001'
cars.make = 'Toyota'
cars.model = 'Camary'
cars.year = 2017
cars.color = 'Silver'
cars.price = '$20,000'
table_service.insert_entity('itemstable', cars)
print('Created entry for Toyata Cars')
cars = Entity()
cars.PartitionKey = 'CarDealer'
cars.RowKey = '002'
cars.make = 'Toyota'
cars.model = 'Carola'
cars.year = 2016
cars.color = 'Black'
cars.price = '$17,000'
table_service.insert_entity('itemstable', cars)
print('Created entry for Toyota Carola')
cars = Entity()
cars.PartitionKey = 'CarDealer'
# -*- coding: utf-8 -*-
"""
Created on Fri Oct 09 16:47:26 2015
@author: justin.malinchak
"""
from azure.storage.table import TableService, Entity
table_service = TableService(account_name='portalvhdss5m831rhl98hj', account_key='Z1MliCYE7p9Ks9kYQoGeM4V99hODtiJL82BVi/zIm06jLYh7n0tV8YaZHzITKixMwUUmjJ1Vp05XrgHG+gXFlg==')
table_service.create_table('tasktable')
task = {'PartitionKey': 'tasksSeattle', 'RowKey': '1', 'description' : 'Take out the trash', 'priority' : 200}
table_service.insert_entity('tasktable', task)
task = Entity()
task.PartitionKey = 'tasksSeattle'
task.RowKey = '2'
task.description = 'Wash the car'
task.priority = 100
table_service.insert_entity('tasktable', task)
#
task = {'description' : 'Take out the garbage', 'priority' : 250}
table_service.update_entity('tasktable', 'tasksSeattle', '1', task)
#
task = {'description' : 'Take out the garbage again', 'priority' : 250}
table_service.insert_or_replace_entity('tasktable', 'tasksSeattle', '1', task)
task = {'description' : 'Buy detergent', 'priority' : 300}
table_service.insert_or_replace_entity('tasktable', 'tasksSeattle', '3', task)
class PersistentSubscriber(subscriber.AzureSubscriber):
__metaclass__ = ABCMeta
def __init__(self, tableName, topicName, subscriptionName,
ruleName = None, rule = None, table_cred = None):
# Call super class constructor
subscriber.AzureSubscriber.__init__(self, topicName, subscriptionName, ruleName, rule)
# Table Service and operations
self.tableName = tableName
if table_cred is None:
table_cred = azurehook.table_cred
self.table = TableService(account_name=table_cred['account_name'],
account_key=table_cred['mykey'])
if not self.table.exists(tableName):
self.table.create_table(tableName)
self.dump = False
# Specify behavior on message received (from subscription).
# Default is insert entity.
def onNewMessage(self, dic):
entity = self.dictToEntity(dic)
# print("INSERT CAMERA IN TABLE")
self.table.insert_entity(self.tableName, entity)
# Wrapper function for querying the table
# Azure limitation: only a maximum of 1000 entities can be retrieved per query
#
# Reference:
# http://stackoverflow.com/questions/28019437/python-querying-all-rows-of-azure-table
def queryTable(self, query_string):
if not self.table.exists(self.TABLE):
raise ValueError('Table %s does not exist', self.TABLE)
# hasRows = True
marker = None
results = []
entities = self.table.query_entities(
self.TABLE,
query_string,
marker = marker,
num_results=1000)
for entity in entities:
results.append(entity)
return results
# Retrieve all entities from a given partition (i.e. that match a given partitionkey)
def retrievePartition(self, partitionKey):
return self.queryTable("PartitionKey eq '%s'" % partitionKey);
# Flush all entities from a given partition (i.e. that match a given partitionkey)
def flushPartition(self, partitionKey):
if not self.table.exists(self.tableName):
raise ValueError("Given table does not exist")
entities = self.retrievePartition(partitionKey)
for entity in entities:
self.table.delete_entity(self.tableName,
entity.PartitionKey,
entity.RowKey)
# To be implemented by child classes: return an entity given the body
# of the message (as a dictionary).
@abstractmethod
def dictToEntity(self, dic):
pass
if len(word) == 0 or len(word) < min_length:
continue
if word not in word_count:
word_count[word] = 1
else:
word_count[word] += 1
sorted_list = sorted(word_count.items(),
key=lambda item: item[1],
reverse=True)
top_words = sorted_list[:num_words]
pprint(top_words)
file.close()
if useStorage == True:
table_service = TableService(storage_name, storage_key)
table_service.create_table('tasktable')
partition_key = uuid.uuid4().hex
for result in top_words:
task = Entity()
task.PartitionKey = partition_key
task.RowKey = uuid.uuid4().hex
task.url = url_to_scan
task.word = result[0]
task.count = result[1]
table_service.insert_entity('tasktable', task)
import time
from sense_hat import SenseHat
from datetime import datetime
from azure.storage.table import TableService
__author__ = "Anatoly Mironov @mirontoli"
sense = SenseHat()
table_service = TableService(account_name='tolle', account_key='ho2zakf/8rmDckS3pGOTPWwIwCzNwVJxd5hDb3R15wms2fZJG/aX53PDsTWBYsuTPwF7802IKk2QcrJ5FO7i6w==')
table_name = 'climateData'
partition_key = 'climate2'
table_service.create_table(table_name, False)
while True:
sense.show_letter('S', text_colour=[0, 114, 198])
date = datetime.now()
iso_date = date.isoformat()
raw_temp = sense.temp
#calculate temperature https://www.raspberrypi.org/forums/viewtopic.php?t=111457&p=769672
calctemp = 0.0071 * raw_temp * raw_temp + 0.86 * raw_temp - 10.0
temp = "{0:.2f}".format(calctemp)
humidity = "{0:.2f}".format(sense.humidity)
pressure = "{0:.2f}".format(sense.pressure)
entry = {'PartitionKey': partition_key, 'RowKey': iso_date, 'Temperature': temp, 'Humidity':humidity, 'Pressure':pressure}
table_service.insert_entity(table_name, entry)
time.sleep(2)
sense.clear()
time.sleep(58) # wait one minute
'date': str(timestamp),
'answer': dic['answer'],
'predicted': dic['predicted'],
'title': dic['title']
}
#print "metadata_item ="+str(metadata_item)
try:
table.insert_entity('BookTable', metadata_item)
#print "got the arg:"+st
except:
return "item may already be there or another failure"
return "table loaded from " + hostnm
run(host='0.0.0.0', port=8055)
#goes in the next call
metadata_item = {
'PartitionKey': hostnm,
'RowKey': str(i),
'date': str(timestamp),
'answer': source,
'predicted': str(predicted),
'title': title
}
try:
table.insert_entity('BookTable', metadata_item)
except:
print("item may already be there or another failure")
# Each entry in a Table is called an 'Entity'.
# Here, we add an entry for first pizza with two pieces of data - the name, and the cost
#
# A partition key tracks how like-minded entries in the Table are created and queried.
# A row key is a unique ID for each entity in the partition
# These two properties are used as a primary key to index the Table. This makes queries much quicker.
car = Entity()
car.PartitionKey = 'dealerinventory'
car.RowKey = '001'
car.make = 'Toyota'
car.model = 'Camry'
car.year = '1998'
car.color = 'gray'
table_service.insert_entity('randomitemswk6', car)
print('Created entry for Camry...')
car = Entity()
car.PartitionKey = 'dealerinventory'
car.RowKey = '002'
car.make = 'Toyota'
car.model = 'Sienna'
car.year = '2012'
car.color = 'peacock green'
table_service.insert_entity('randomitemswk6', car)
print('Created entry for Sienna...')
car = Entity()
car.PartitionKey = 'dealerinventory'
car.RowKey = '003'
)
raw_input('Press Enter to continue...')
# Each entry in a Table is called an 'Entity'.
# Here, we add an entry for first pizza with two pieces of data - the name, and the cost
#
# A partition key tracks how like-minded entries in the Table are created and queried.
# A row key is a unique ID for each entity in the partition
# These two properties are used as a primary key to index the Table. This makes queries much quicker.
pizza = Entity()
pizza.PartitionKey = 'pizzamenu'
pizza.RowKey = '001'
pizza.description = 'Pepperoni'
pizza.cost = 18
table_service.insert_entity('itemstable', pizza)
print('Created entry for pepperoni...')
pizza = Entity()
pizza.PartitionKey = 'pizzamenu'
pizza.RowKey = '002'
pizza.description = 'Veggie'
pizza.cost = 15
table_service.insert_entity('itemstable', pizza)
print('Created entry for veggie...')
pizza = Entity()
pizza.PartitionKey = 'pizzamenu'
pizza.RowKey = '003'
pizza.description = 'Hawaiian'
pizza.cost = 12
print('Now let\'s add some entries to our Table.\nRemember, Azure Storage Tables is a NoSQL datastore, so this is similar to adding records to a database.')
raw_input('Press Enter to continue...')
# Each entry in a Table is called an 'Entity'.
# Here, we add an entry for first pizza with two pieces of data - the name, and the cost
#
# A partition key tracks how like-minded entries in the Table are created and queried.
# A row key is a unique ID for each entity in the partition
# These two properties are used as a primary key to index the Table. This makes queries much quicker.
pizza = Entity()
pizza.PartitionKey = 'pizzamenu'
pizza.RowKey = '001'
pizza.description = 'Pepperoni'
pizza.cost = 18
table_service.insert_entity('itemstable', pizza)
print('Created entry for pepperoni...')
pizza = Entity()
pizza.PartitionKey = 'pizzamenu'
pizza.RowKey = '002'
pizza.description = 'Veggie'
pizza.cost = 15
table_service.insert_entity('itemstable', pizza)
print('Created entry for veggie...')
pizza = Entity()
pizza.PartitionKey = 'pizzamenu'
pizza.RowKey = '003'
pizza.description = 'Hawaiian'
pizza.cost = 12
class Repository(object):
"""Azure Table Storage repository."""
def __init__(self, settings):
"""Initializes the repository with the specified settings dict.
Required settings are:
- STORAGE_NAME
- STORAGE_KEY
- STORAGE_TABLE_POLL
- STORAGE_TABLE_CHOICE
"""
self.name = 'Azure Table Storage'
self.storage_name = settings['STORAGE_NAME']
self.storage_key = settings['STORAGE_KEY']
self.poll_table = settings['STORAGE_TABLE_POLL']
self.choice_table = settings['STORAGE_TABLE_CHOICE']
self.svc = TableService(self.storage_name, self.storage_key)
self.svc.create_table(self.poll_table)
self.svc.create_table(self.choice_table)
def get_polls(self):
"""Returns all the polls from the repository."""
poll_entities = self.svc.query_entities(self.poll_table)
polls = [_poll_from_entity(entity) for entity in poll_entities]
return polls
def get_poll(self, poll_key):
"""Returns a poll from the repository."""
try:
partition, row = _key_to_partition_and_row(poll_key)
poll_entity = self.svc.get_entity(self.poll_table, partition, row)
choice_entities = self.svc.query_entities(
self.choice_table,
"PollPartitionKey eq '{0}' and PollRowKey eq '{1}'" \
.format(partition, row)
)
poll = _poll_from_entity(poll_entity)
poll.choices = [
_choice_from_entity(choice_entity)
for choice_entity in choice_entities
]
return poll
except AzureMissingResourceHttpError:
raise PollNotFound()
def increment_vote(self, poll_key, choice_key):
"""Increment the choice vote count for the specified poll."""
try:
partition, row = _key_to_partition_and_row(choice_key)
entity = self.svc.get_entity(self.choice_table, partition, row)
entity.Votes += 1
self.svc.update_entity(self.choice_table, entity)
except AzureMissingResourceHttpError:
raise PollNotFound()
def add_sample_polls(self):
"""Adds a set of polls from data stored in a samples.json file."""
poll_partition = '2014'
poll_row = 0
choice_partition = '2014'
choice_row = 0
for sample_poll in _load_samples_json():
poll_entity = {
'PartitionKey': poll_partition,
'RowKey': str(poll_row),
'Text': sample_poll['text'],
}
self.svc.insert_entity(self.poll_table, poll_entity)
for sample_choice in sample_poll['choices']:
choice_entity = {
'PartitionKey': choice_partition,
'RowKey': str(choice_row),
'Text': sample_choice,
'Votes': 0,
'PollPartitionKey': poll_partition,
'PollRowKey': str(poll_row),
}
self.svc.insert_entity(self.choice_table, choice_entity)
choice_row += 1
poll_row += 1
# Each entry in a Table is called an 'Entity'.
# Here, we add an entry for first car with two pieces of data - the name, and the cost
#
# A partition key tracks how like-minded entries in the Table are created and queried.
# A row key is a unique ID for each entity in the partition
# These two properties are used as a primary key to index the Table. This makes queries much quicker.
car = Entity()
car.PartitionKey = 'cars'
car.RowKey = '001'
car.make = 'Ford'
car.model = 'Edge'
car.year = '2007'
car.color = 'Gray'
car.price = 32000
table_service.insert_entity('itemstable', car)
print('Created entry for 2007 Ford Edge...')
car = Entity()
car.PartitionKey = 'cars'
car.RowKey = '002'
car.make = 'GMC'
car.model = 'Acadia'
car.year = '2016'
car.color = 'Silver'
car.price = 55000
table_service.insert_entity('itemstable', car)
print('Created entry for 2016 GMC Acadia...')
car = Entity()
car.PartitionKey = 'cars'
# Each entry in a Table is called an 'Entity'.
# Here, we add an entry for first cars with two pieces of data - the name, and the cost
#
# A partition key tracks how like-minded entries in the Table are created and queried.
# A row key is a unique ID for each entity in the partition
# These two properties are used as a primary key to index the Table. This makes queries much quicker.
car = Entity()
car.PartitionKey = 'carInventory'
car.RowKey = '001'
car.make = 'Acura'
car.model = 'MDX'
car.year = 2008
car.color = 'gray'
car.price= 11594.00
table_service.insert_entity('itemstable', car)
print('Created entry for Acura MDX...')
time.sleep(1)
car = Entity()
car.PartitionKey = 'carInventory'
car.RowKey = '002'
car.make = 'BMW'
car.model = '4 Series'
car.year = 2015
car.color = 'black'
car.price= 28281.00
table_service.insert_entity('itemstable', car)
print('Created entry for BMW 4 Series...')
time.sleep(1)
