def main(req: func.HttpRequest) -> func.HttpResponse:
try:
logging.info("Trigger started")
ret = {}
if "code" not in req.params:
logging.info("Invalid code")
ret["message"] = "The parameter code is no present in the request."
ret["status"] = False
return func.HttpResponse(json.dumps(ret), headers=headers)
else:
code = req.params.get('code')
logging.info("Processing " + str(code) + "...")
table_service = TableService(account_name=ACCOUNT_NAME,
account_key=ACCOUNT_KEY)
records = table_service.query_entities(
TABLE_NAME_TRACKING,
filter="PartitionKey eq 'tracking-analysis' and RowKey eq '" +
code + "'")
if len(records.items) == 0:
ret["message"] = "Meeting coding not found"
ret["status"] = False
logging.info("Code not found.")
return func.HttpResponse(json.dumps(ret), headers=headers)
else:
additional_stop_words = table_service.get_entity(
TABLE_NAME_PARAMETERS, "stopwords", "general").Value
record = records.items[0]
freq_dist = json.loads(record["FreqDist"])
words = []
for word in freq_dist:
if freq_dist[word] > 1 and len(
word) > 2 and word not in additional_stop_words:
words.append({"name": word, "weight": freq_dist[word]})
ret["message"] = "Code found at the database"
ret["status"] = True
ret["words"] = words
logging.info("Code successfully processed.")
return func.HttpResponse(json.dumps(ret), headers=headers)
except Exception as error:
logging.error(error)
return func.HttpResponse(error, status_code=400, headers=headers)
class Repository(object):
"""Azure Table storage repository for UOTD."""
def __init__(self, settings):
"""Initialise UOTD repository with the given settings dict.
Required settings:
STORAGE_NAME -- the Azure Storage account name
STORAGE_KEY -- an access key for the Storage account
STORAGE_TABLE_UOTD -- the name of the table
"""
self.service = TableService(settings["STORAGE_NAME"],
settings["STORAGE_KEY"])
self.uotd_table = settings["STORAGE_TABLE_UOTD"]
self.service.create_table(self.uotd_table)
self.partition_key_format = "%Y%m"
self.row_key_format = "%d"
def get_uotd(self):
"""Get the UUID for the current day.
If the UUID does not yet exist then it will be created.
"""
partition_key = date.today().strftime(self.partition_key_format)
row_key = date.today().strftime(self.row_key_format)
try:
uotd_entity = self.service.get_entity(self.uotd_table,
partition_key,
row_key)
uuid = uotd_entity.uuid
except AzureMissingResourceHttpError:
uuid = str(uuid4())
uotd_entity = {
"PartitionKey": partition_key,
"RowKey": row_key,
"uuid": uuid
}
self.service.insert_entity(self.uotd_table, uotd_entity)
return uuid
def get_uuids(self, partition_key):
"""Get all the UUIDs in a given partition."""
filter = "PartitionKey eq '{0}'".format(partition_key)
entities = self.service.query_entities(self.uotd_table, filter=filter)
return entities
class AztkCluster:
def __init__(self,
vm_count=0,
sku_type='standard_d2_v2',
username='******',
password='******'):
self.vm_count = int(vm_count)
self.sku_type = sku_type
self.username = username
self.password = password
self.BATCH_ACCOUNT_NAME = os.environ['BATCH_ACCOUNT_NAME']
BATCH_ACCOUNT_KEY = os.environ['BATCH_ACCOUNT_KEY']
BATCH_SERVICE_URL = os.environ['BATCH_ACCOUNT_URL']
STORAGE_ACCOUNT_SUFFIX = 'core.windows.net'
self.STORAGE_ACCOUNT_NAME = os.environ['STORAGE_ACCOUNT_NAME']
self.STORAGE_ACCOUNT_KEY = os.environ['STORAGE_ACCOUNT_KEY']
self.secrets_config = aztk.spark.models.SecretsConfiguration(
shared_key=aztk.models.SharedKeyConfiguration(
batch_account_name=self.BATCH_ACCOUNT_NAME,
batch_account_key=BATCH_ACCOUNT_KEY,
batch_service_url=BATCH_SERVICE_URL,
storage_account_name=self.STORAGE_ACCOUNT_NAME,
storage_account_key=self.STORAGE_ACCOUNT_KEY,
storage_account_suffix=STORAGE_ACCOUNT_SUFFIX),
ssh_pub_key="")
self.table_service = TableService(
account_name=self.STORAGE_ACCOUNT_NAME,
account_key=self.STORAGE_ACCOUNT_KEY)
def createCluster(self):
# create a client
client = aztk.spark.Client(self.secrets_config)
# list available clusters
clusters = client.list_clusters()
SPARK_CONFIG_PATH = os.path.normpath(
os.path.join(os.path.dirname(__file__), 'spark', 'spark',
'.config'))
SPARK_JARS_PATH = os.path.normpath(
os.path.join(os.path.dirname(__file__), 'spark', 'spark', 'jars'))
SPARK_CORE_SITE = os.path.join(SPARK_CONFIG_PATH, 'core-site.xml')
jars = glob.glob(os.path.join(SPARK_JARS_PATH, '*.jar'))
# define spark configuration
spark_conf = aztk.spark.models.SparkConfiguration(
spark_defaults_conf=os.path.join(SPARK_CONFIG_PATH,
'spark-defaults.conf'),
spark_env_sh=os.path.join(SPARK_CONFIG_PATH, 'spark-env.sh'),
core_site_xml=SPARK_CORE_SITE,
jars=jars)
clusterDetails = self.table_service.get_entity(
'cluster', 'predictivemaintenance', 'predictivemaintenance')
cluster_number = int(clusterDetails.ClusterNumber) + 1
cluster_id = clusterDetails.PartitionKey + str(cluster_number)
jupyterCustomScript = aztk.models.CustomScript(
"jupyter",
"D:/home/site/wwwroot/flask/spark/customScripts/jupyter.sh",
"all-nodes")
azuremlProjectFileShare = aztk.models.FileShare(
self.STORAGE_ACCOUNT_NAME, self.STORAGE_ACCOUNT_KEY,
'azureml-project', '/mnt/azureml-project')
azuremlFileShare = aztk.models.FileShare(self.STORAGE_ACCOUNT_NAME,
self.STORAGE_ACCOUNT_KEY,
'azureml-share',
'/mnt/azureml-share')
# configure my cluster
cluster_config = aztk.spark.models.ClusterConfiguration(
docker_repo='aztk/python:spark2.2.0-python3.6.2-base',
cluster_id=
cluster_id, # Warning: this name must be a valid Azure Blob Storage container name
vm_count=self.vm_count,
# vm_low_pri_count=2, #this and vm_count are mutually exclusive
vm_size=self.sku_type,
custom_scripts=[jupyterCustomScript],
spark_configuration=spark_conf,
file_shares=[azuremlProjectFileShare, azuremlFileShare],
user_configuration=UserConfiguration(
username=self.username,
password=self.password,
))
try:
cluster = client.create_cluster(cluster_config)
except Exception as e:
clusterDetails = {
'PartitionKey': 'predictivemaintenance',
'RowKey': 'predictivemaintenance',
'Status': ClusterStatus.Failed,
'UserName': self.username,
'ClusterNumber': cluster_number,
'Message': str(e)
}
self.table_service.insert_or_merge_entity('cluster',
clusterDetails)
return
clusterDetails = {
'PartitionKey': 'predictivemaintenance',
'RowKey': 'predictivemaintenance',
'Status': ClusterStatus.Provisioning,
'UserName': self.username,
'ClusterNumber': cluster_number
}
self.table_service.insert_or_merge_entity('cluster', clusterDetails)
def getCluster(self):
# create a client
client = aztk.spark.Client(self.secrets_config)
try:
clusterDetails = self.table_service.get_entity(
'cluster', 'predictivemaintenance', 'predictivemaintenance')
except Exception as e:
clusterDetails = {
'PartitionKey': 'predictivemaintenance',
'RowKey': 'predictivemaintenance',
'Status': ClusterStatus.NotCreated,
'ClusterNumber': '0'
}
self.table_service.insert_or_merge_entity('cluster',
clusterDetails)
return clusterDetails
cluster_id = clusterDetails.PartitionKey + str(
clusterDetails.ClusterNumber)
if clusterDetails.Status == ClusterStatus.Deleted or clusterDetails.Status == ClusterStatus.NotCreated:
return clusterDetails
try:
cluster = client.get_cluster(cluster_id=cluster_id)
for node in cluster.nodes:
remote_login_settings = client.get_remote_login_settings(
cluster.id, node.id)
if node.state in [
batch_models.ComputeNodeState.unknown,
batch_models.ComputeNodeState.unusable,
batch_models.ComputeNodeState.start_task_failed
]:
errorMsg = "An error occured while starting the Nodes in the batch account " + self.BATCH_ACCOUNT_NAME + ". Details: "
if node.start_task_info.failure_info != None:
errorMsg += node.start_task_info.failure_info.message
clusterDetails = {
'PartitionKey': 'predictivemaintenance',
'RowKey': 'predictivemaintenance',
'Status': ClusterStatus.Failed,
'UserName': self.username,
'ClusterNumber': clusterDetails.ClusterNumber,
'Message': errorMsg
}
self.table_service.insert_or_merge_entity(
'cluster', clusterDetails)
return clusterDetails
if node.id == cluster.master_node_id:
master_ipaddress = remote_login_settings.ip_address
master_Port = remote_login_settings.port
clusterDetails = {
'PartitionKey': 'predictivemaintenance',
'RowKey': 'predictivemaintenance',
'Status': ClusterStatus.Provisioned,
'Master_Ip_Address': master_ipaddress,
'Master_Port': master_Port,
'UserName': clusterDetails.UserName,
'ClusterNumber': clusterDetails.ClusterNumber
}
self.table_service.insert_or_merge_entity(
'cluster', clusterDetails)
except (AztkError, BatchErrorException):
clusterDetails = self.table_service.get_entity(
'cluster', 'predictivemaintenance', 'predictivemaintenance')
return clusterDetails
def deleteCluster(self):
# create a client
client = aztk.spark.Client(self.secrets_config)
clusterDetails = self.table_service.get_entity(
'cluster', 'predictivemaintenance', 'predictivemaintenance')
cluster_id = clusterDetails.PartitionKey + str(
clusterDetails.ClusterNumber)
try:
client.delete_cluster(cluster_id=cluster_id)
except Exception as e:
clusterDetails = {
'PartitionKey': 'predictivemaintenance',
'RowKey': 'predictivemaintenance',
'Status': ClusterStatus.DeletionFailed,
'UserName': self.username,
'ClusterNumber': clusterDetails.ClusterNumber,
'Message': str(e)
}
self.table_service.insert_or_merge_entity('cluster',
clusterDetails)
return
clusterDetails = {
'PartitionKey': 'predictivemaintenance',
'RowKey': 'predictivemaintenance',
'Status': ClusterStatus.Deleted,
'ClusterNumber': clusterDetails.ClusterNumber
}
self.table_service.insert_or_merge_entity('cluster', clusterDetails)
from azure.storage.table import TableService, Entity
table_service = TableService(
account_name='seva',
account_key=
'SgbxLwWkBH4XuGebxECoXfNVG3mVM5YjOs+SWTDUSacc+3YgUmcafYXrXdz5k0HtlZQ3AuEJ1IcFtZYeGVR9Hw=='
)
#table_service.delete_table('tasktable')
#table_service.create_table('Table1')
#test1 = {'PartitionKey': 'tasksSeattle', 'RowKey': '001', 'description' : 'This is the first table for testing', 'priority' : 200}
#table_service.insert_entity('Table1', test1)
test1 = table_service.get_entity('Table1', 'tasksSeattle', '001')
print(test1.description)
print(test1.priority)
import vk, urllib.request, csv, json, pprint, time
from pprint import pprint
from azure.storage.table import TableService, Entity, TableBatch
table_service = TableService(account_name='seva', account_key='SgbxLwWkBH4XuGebxECoXfNVG3mVM5YjOs+SWTDUSacc+3YgUmcafYXrXdz5k0HtlZQ3AuEJ1IcFtZYeGVR9Hw==')
batch =TableBatch()
#table_service.delete_table('MyVkApp')
table_service.create_table('MyVkApp')
login = input("Введите имя пользователя: ")
password = input("Введите пароль: ")
session = vk.AuthSession(app_id='5889724', user_login=login, user_password=password)
api = vk.API(session)
print('Вставте access token, полученный от вконтакте')
accesstoken =input()
user_data ={'PartitionKey': 'my_user_data', 'RowKey': '001', 'description': accesstoken}
table_service.insert_or_replace_entity('MyVkApp', user_data)
user_data =table_service.get_entity('MyVkApp', 'my_user_data', '001')
print(user_data.description)
class TableStorage():
def __init__(self, CONNECTION_STRING):
"""
Constructor. Espera el Connection String del Azure Storage Account.
Se obtiene ingresando al recurso de Storage -> Access Keys
Parametros:
CONNECTION_STRING = El string que incluye el AccountName,
AccountKey y el EndPointSuffix
"""
self.CONNECTION_STRING = CONNECTION_STRING
# Separa por partes el string de conexión
Config = dict(
s.split('=', 1) for s in CONNECTION_STRING.split(';') if s)
# Obtiene el nombre de la cuenta de storage y en EndpointSuffix
self.AccountName = Config.get('AccountName')
self.EndPointSuffix = Config.get('EndpointSuffix')
def CreateTableServices(self):
"""
Inicializa una instancia del Table Services para poder comunicarse con
el storage en Azure
"""
self.TableService = TableService(
account_name=self.AccountName,
connection_string=self.CONNECTION_STRING,
endpoint_suffix=self.EndPointSuffix)
def createTable(self, TableName):
"""
Revisa si la tabla no exista ya y la crea. De lo contrario, avisa que ya existe.
Paramentros:
TableName = Nombre de la tabla que se quiere crear
"""
print('\nCreate a table with name - ' + TableName)
if (self.TableService.exists(TableName) != True):
self.TableService.create_table(TableName)
print("Table created succesfully!")
else:
print('Error creating table, ' + TableName +
' check if it already exists')
def insertEntity(self, TableName, Entity):
"""
Se inserta una entidad a la tabla especificada.
Paramentros:
TableName = Nombre de la tabla que se quiere crear
Entity = El objecto con la entidad que se quiere agregar
"""
print('\nInserting a new entity into table - ' + TableName)
self.TableService.insert_or_merge_entity(TableName, Entity)
print('Successfully inserted the new entity')
def getEntity(self, TableName, PartitionKey, RowKey):
"""
Traerse la entidad completa en base a la Partition Key y Row Key.
Regresa un objeto como tal, no hay que hacer json.loads()
Paramentros:
TableName = Nombre de la tabla que se quiere crear
PartitionKey = String con la partition key de la entidad deseada
RowKey = String con la row key de la entidad deseada
"""
print('\nGetting entity.')
Entity = self.TableService.get_entity(TableName, PartitionKey, RowKey)
return Entity
def updateEntity(self, TableName, NewEntity):
"""
Toma el objeto con los datos actualizados y hace update en la table storage.
Paramentros:
TableName = Nombre de la tabla que se quiere crear
NewEntity = El objecto con la entidad que se quiere hacer update
"""
print('\nUpdating entity. PK: ' + NewEntity.PartitionKey + ' RK: ' +
NewEntity.RowKey)
self.TableService.update_entity(TableName, NewEntity)
def deleteEntity(self, TableName, PartitionKey, RowKey):
"""
Borrar la entidad que coincida en Partition Key y Row Key
Paramentros:
TableName = Nombre de la tabla que se quiere crear
PartitionKey = String con la partition key de la entidad
RowKey = String con la row key de la entidad
"""
print('\nDeleting entity')
self.TableService.delete_entity(TableName, PartitionKey, RowKey)
def deleteTable(self, TableName):
"""
Revisa si la tabla existe y la borra, en caso contrario solo avisa que no existe.
Paramentros:
TableName = Nombre de la tabla que se quiere borrar
"""
print('\nDeleting the table.')
if (self.TableService.exists(TableName)):
self.TableService.delete_table(TableName)
print('Successfully deleted the table')
else:
print('The table does not exists')
class StorageTableContext():
"""Initializes the repository with the specified settings dict.
Required settings in config dict are:
- AZURE_STORAGE_NAME
- STORAGE_KEY
"""
_models = []
_encryptproperties = False
_encrypted_properties = []
_tableservice = None
_storage_key = ''
_storage_name = ''
def __init__(self, **kwargs):
self._storage_name = kwargs.get('AZURE_STORAGE_NAME', '')
self._storage_key = kwargs.get('AZURE_STORAGE_KEY', '')
""" service init """
self._models = []
if self._storage_key != '' and self._storage_name != '':
self._tableservice = TableService(account_name = self._storage_name, account_key = self._storage_key, protocol='https')
""" encrypt queue service """
if kwargs.get('AZURE_REQUIRE_ENCRYPTION', False):
# Create the KEK used for encryption.
# KeyWrapper is the provided sample implementation, but the user may use their own object as long as it implements the interface above.
kek = KeyWrapper(kwargs.get('AZURE_KEY_IDENTIFIER', 'otrrentapi'), kwargs.get('SECRET_KEY', 'super-duper-secret')) # Key identifier
# Create the key resolver used for decryption.
# KeyResolver is the provided sample implementation, but the user may use whatever implementation they choose so long as the function set on the service object behaves appropriately.
key_resolver = KeyResolver()
key_resolver.put_key(kek)
# Set the require Encryption, KEK and key resolver on the service object.
self._encryptproperties = True
self._tableservice.key_encryption_key = kek
self._tableservice.key_resolver_funcion = key_resolver.resolve_key
self._tableservice.encryption_resolver_function = self.__encryptionresolver__
pass
def __createtable__(self, tablename) -> bool:
if (not self._tableservice is None):
try:
self._tableservice.create_table(tablename)
return True
except AzureException as e:
log.error('failed to create {} with error {}'.format(tablename, e))
return False
else:
return True
pass
# Define the encryption resolver_function.
def __encryptionresolver__(self, pk, rk, property_name):
if property_name in self._encrypted_properties:
return True
#log.debug('encrypt field {}'.format(property_name))
#log.debug('dont encrypt field {}'.format(property_name))
return False
def register_model(self, storagemodel:object):
modelname = storagemodel.__class__.__name__
if isinstance(storagemodel, StorageTableModel):
if (not modelname in self._models):
self.__createtable__(storagemodel._tablename)
self._models.append(modelname)
""" set properties to be encrypted client side """
if self._encryptproperties:
self._encrypted_properties += storagemodel._encryptedproperties
log.info('model {} registered successfully. Models are {!s}. Encrypted fields are {!s} '.format(modelname, self._models, self._encrypted_properties))
pass
def table_isempty(self, tablename, PartitionKey='', RowKey = '') -> bool:
if (not self._tableservice is None):
filter = "PartitionKey eq '{}'".format(PartitionKey) if PartitionKey != '' else ''
if filter == '':
filter = "RowKey eq '{}'".format(RowKey) if RowKey != '' else ''
else:
filter = filter + ("and RowKey eq '{}'".format(RowKey) if RowKey != '' else '')
try:
entities = list(self._tableservice.query_entities(tablename, filter = filter, select='PartitionKey', num_results=1))
if len(entities) == 1:
return False
else:
return True
except AzureMissingResourceHttpError as e:
log.debug('failed to query {} with error {}'.format(tablename, e))
return True
else:
return True
pass
def exists(self, storagemodel) -> bool:
exists = False
if isinstance(storagemodel, StorageTableModel):
modelname = storagemodel.__class__.__name__
if (modelname in self._models):
if storagemodel._exists is None:
try:
entity = self._tableservice.get_entity(storagemodel._tablename, storagemodel.PartitionKey, storagemodel.RowKey)
storagemodel._exists = True
exists = True
except AzureMissingResourceHttpError:
storagemodel._exists = False
else:
exists = storagemodel._exists
else:
log.debug('please register model {} first'.format(modelname))
return exists
def get(self, storagemodel) -> StorageTableModel:
""" load entity data from storage to vars in self """
if isinstance(storagemodel, StorageTableModel):
modelname = storagemodel.__class__.__name__
if (modelname in self._models):
try:
entity = self._tableservice.get_entity(storagemodel._tablename, storagemodel.PartitionKey, storagemodel.RowKey)
storagemodel._exists = True
""" sync with entity values """
for key, default in vars(storagemodel).items():
if not key.startswith('_') and key not in ['','PartitionKey','RowKey']:
value = getattr(entity, key, None)
if not value is None:
setattr(storagemodel, key, value)
except AzureMissingResourceHttpError as e:
log.debug('can not get table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'.format(storagemodel._tablename, storagemodel.PartitionKey, storagemodel.RowKey, e))
storagemodel._exists = False
except Exception as e:
log.debug('can not get table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'.format(storagemodel._tablename, storagemodel.PartitionKey, storagemodel.RowKey, e))
storagemodel._exists = False
else:
log.debug('please register model {} first to {!s}'.format(modelname, self._models))
return storagemodel
else:
return None
def insert(self, storagemodel) -> StorageTableModel:
""" insert model into storage """
if isinstance(storagemodel, StorageTableModel):
modelname = storagemodel.__class__.__name__
if (modelname in self._models):
try:
self._tableservice.insert_or_replace_entity(storagemodel._tablename, storagemodel.entity())
storagemodel._exists = True
except AzureMissingResourceHttpError as e:
log.debug('can not insert or replace table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'.format(storagemodel._tablename, storagemodel.PartitionKey, storagemodel.RowKey, e))
else:
log.debug('please register model {} first'.format(modelname))
return storagemodel
else:
return None
def merge(self, storagemodel) -> StorageTableModel:
""" try to merge entry """
if isinstance(storagemodel, StorageTableModel):
modelname = storagemodel.__class__.__name__
if (modelname in self._models):
try:
self._tableservice.insert_or_merge_entity(storagemodel._tablename, storagemodel.entity())
storagemodel._exists = True
except AzureMissingResourceHttpError as e:
log.debug('can not insert or merge table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'.format(storagemodel._tablename, storagemodel.PartitionKey, storagemodel.RowKey, e))
else:
log.debug('please register model {} first'.format(modelname))
return storagemodel
else:
return None
def delete(self,storagemodel):
""" delete existing Entity """
if isinstance(storagemodel, StorageTableModel):
modelname = storagemodel.__class__.__name__
if (modelname in self._models):
try:
self._tableservice.delete_entity(storagemodel._tablename, storagemodel.PartitionKey, storagemodel.RowKey)
storagemodel._exists = False
except AzureMissingResourceHttpError as e:
log.debug('can not delete table entity: Table {}, PartitionKey {}, RowKey {} because {!s}'.format(storagemodel._tablename, storagemodel.PartitionKey, storagemodel.RowKey, e))
else:
log.debug('please register model {} first'.format(modelname))
return storagemodel
else:
return None
def __changeprimarykeys__(self, PartitionKey = '', RowKey = ''):
""" Change Entity Primary Keys into new instance:
- PartitionKey and/or
- RowKey
"""
PartitionKey = PartitionKey if PartitionKey != '' else self._PartitionKey
RowKey = RowKey if RowKey != '' else self._RowKey
""" change Primary Keys if different to existing ones """
if (PartitionKey != self._PartitionKey) or (RowKey != self._RowKey):
return True, PartitionKey, RowKey
else:
return False, PartitionKey, RowKey
pass
def moveto(self, PartitionKey = '', RowKey = ''):
""" Change Entity Primary Keys and move in Storage:
- PartitionKey and/or
- RowKey
"""
changed, PartitionKey, RowKey = self.__changeprimarykeys__(PartitionKey, RowKey)
if changed:
""" sync self """
new = self.copyto(PartitionKey, RowKey)
new.save()
""" delete Entity if exists in Storage """
self.delete()
def copyto(self, PartitionKey = '', RowKey = '') -> object:
""" Change Entity Primary Keys and copy to new Instance:
- PartitionKey and/or
- RowKey
"""
changed, PartitionKey, RowKey = self.__changeprimarykeys__(PartitionKey, RowKey)
self.load()
new = self
new._PartitionKey = PartitionKey
new._RowKey = RowKey
new.load()
return new
def query(self, storagecollection) -> StorageTableCollection:
if isinstance(storagecollection, StorageTableCollection):
try:
storagecollection.extend(self._tableservice.query_entities(storagecollection._tablename,storagecollection._filter))
except AzureMissingResourceHttpError as e:
log.debug('can not query table {} with filters {} because {!s}'.format(storagecollection._tablename, storagecollection._filter, e))
return storagecollection
else:
return None
sequence_number=0)
i = 0
t = 0
start = time()
last_speed = 0
while True:
# data = (np.sin(2*np.pi*np.arange(8000)*440/8000)).astype(np.float32)
# scaled = np.int16(data/np.max(np.abs(data)) * 32767)
# scaled.resize(16*512)
# d = scaled.tobytes()
entity = table_service.get_entity(table_name='devices',
partition_key='engine',
row_key='1')
signal, last_speed = generate(t,
wk,
Ak,
last_speed=last_speed,
target_speed=entity.speed)
signal.resize(16 * 512)
signal[fs] = last_speed
d = signal.tobytes()
page_start = i * 32 * 512
page_end = page_start + 32 * 512 - 1
import sys
from azure.storage.table import TableService
if __name__ == '__main__':
storage_account = sys.argv[1]
storage_key = sys.argv[2]
entity_pk = sys.argv[3]
entity_rk = sys.argv[4]
state = sys.argv[5]
error = None
if len(sys.argv) == 7:
error = sys.argv[6]
table_service = TableService(account_name=storage_account,
account_key=storage_key)
# entity = table_service.get_entity('SearchEntity', entity_pk, entity_rk)
entity = table_service.get_entity('AnalysisEntity', entity_pk, entity_rk)
try:
if entity._State == 'WaitingForResources':
entity._State = state
if error:
entity.Errors = error
# table_service.update_entity('SearchEntity', entity, if_match=entity.etag)
table_service.update_entity('AnalysisEntity',
entity,
if_match=entity.etag)
except Exception as e:
print('Error updating entityt {}'.format(e))
print('\nFinished :) ')
else:
print('Azure Table \n')
data_out = Entity()
data_out.PartitionKey = userID
data_out.RowKey = timestamp
data_out.anger = resultAPI[0]['scores']['anger']
data_out.contempt = resultAPI[0]['scores']['contempt']
data_out.disgust = resultAPI[0]['scores']['disgust']
data_out.fear = resultAPI[0]['scores']['fear']
data_out.happiness = resultAPI[0]['scores']['happiness']
data_out.neutral = resultAPI[0]['scores']['neutral']
data_out.sadness = resultAPI[0]['scores']['sadness']
data_out.surprise = resultAPI[0]['scores']['surprise']
data_out.timeSpent = timeSpent
data_out.click = clicks
table_service = TableService(account_name=STORAGE_ACCOUNT_NAME,
account_key=STORAGE_ACCOUNT_KEY)
table_service.create_table('Results')
table_service.insert_or_replace_entity('Results', data_out)
output = table_service.get_entity('Results', userID, timestamp)
print(output)
print('\nFinished :) ')
print "TRY # " + messages[0].dequeue_count
msgs = messages[0].content.split('(^_^)')
url = msgs[1]
RowKey = msgs[0]
filename, file_extension = os.path.splitext(
urlparse.urlsplit(url).path)
local_filename = dir_path + '/data/' + RowKey + file_extension
cnt = 0
# Thumbnail should go prior to those filters who need SMALLER images due to memory limitation
filters = ['thumbnail', 'facedetect', 'classify']
try:
rt = downloadImage(url, local_filename)
if rt:
local_filename = rt
filename, file_extension = os.path.splitext(local_filename)
entity = tablesvc.get_entity(TASK_TABLE, RowKey[:3],
RowKey)
for f in filters:
try:
if entity[f] <> "":
cnt = cnt + 1
print f + " Done"
except:
applyFilter(local_filename, f, RowKey,
file_extension)
if cnt < len(filters):
cnt = 0
entity = tablesvc.get_entity(TASK_TABLE, RowKey[:3],
RowKey)
for f in filters:
try:
if entity[f] <> "":
import sys
from azure.storage.table import TableService
if __name__ == '__main__':
storage_account = sys.argv[1]
storage_key = sys.argv[2]
entity_pk = sys.argv[3]
entity_rk = sys.argv[4]
state = sys.argv[5]
error = None
if len(sys.argv) == 7:
error = sys.argv[6]
table_service = TableService(account_name=storage_account,
account_key=storage_key)
entity = table_service.get_entity('SearchEntity', entity_pk, entity_rk)
try:
if entity._State == 'WaitingForResources':
entity._State = state
if error:
entity.Errors = error
table_service.update_entity('SearchEntity',
entity,
if_match=entity.etag)
except Exception as e:
print('Error updating entityt {}'.format(e))
table_service.update_entity('tasktable', 'tasksSeattle', '1', task)
#
task = {'description' : 'Take out the garbage again', 'priority' : 250}
table_service.insert_or_replace_entity('tasktable', 'tasksSeattle', '1', task)
task = {'description' : 'Buy detergent', 'priority' : 300}
table_service.insert_or_replace_entity('tasktable', 'tasksSeattle', '3', task)
task10 = {'PartitionKey': 'tasksSeattle', 'RowKey': '10', 'description' : 'Go grocery shopping', 'priority' : 400}
task11 = {'PartitionKey': 'tasksSeattle', 'RowKey': '11', 'description' : 'Clean the bathroom', 'priority' : 100}
table_service.begin_batch()
table_service.insert_entity('tasktable', task10)
table_service.insert_entity('tasktable', task11)
table_service.commit_batch()
task = table_service.get_entity('tasktable', 'tasksSeattle', '1')
print(task.description)
print(task.priority)
tasks = table_service.query_entities('tasktable', "PartitionKey eq 'tasksSeattle'")
for task in tasks:
print(task.description)
print(task.priority)
tasks = table_service.query_entities('tasktable', "PartitionKey eq 'tasksSeattle'", 'description')
for task in tasks:
print(task.description)
table_service.delete_entity('tasktable', 'tasksSeattle', '1')
table_service.delete_table('tasktable')
class Repository(object):
"""Azure Table Storage repository."""
def __init__(self, settings):
"""Initializes the repository with the specified settings dict.
Required settings are:
- STORAGE_NAME
- STORAGE_KEY
- STORAGE_TABLE_POLL
- STORAGE_TABLE_CHOICE
"""
self.name = 'Azure Table Storage'
self.storage_name = settings['STORAGE_NAME']
self.storage_key = settings['STORAGE_KEY']
self.poll_table = settings['STORAGE_TABLE_POLL']
self.choice_table = settings['STORAGE_TABLE_CHOICE']
self.svc = TableService(self.storage_name, self.storage_key)
self.svc.create_table(self.poll_table)
self.svc.create_table(self.choice_table)
def get_polls(self):
"""Returns all the polls from the repository."""
poll_entities = self.svc.query_entities(self.poll_table)
polls = [_poll_from_entity(entity) for entity in poll_entities]
return polls
def get_poll(self, poll_key):
"""Returns a poll from the repository."""
try:
partition, row = _key_to_partition_and_row(poll_key)
poll_entity = self.svc.get_entity(self.poll_table, partition, row)
choice_entities = self.svc.query_entities(
self.choice_table,
"PollPartitionKey eq '{0}' and PollRowKey eq '{1}'" \
.format(partition, row)
)
poll = _poll_from_entity(poll_entity)
poll.choices = [
_choice_from_entity(choice_entity)
for choice_entity in choice_entities
]
return poll
except AzureMissingResourceHttpError:
raise PollNotFound()
def increment_vote(self, poll_key, choice_key):
"""Increment the choice vote count for the specified poll."""
try:
partition, row = _key_to_partition_and_row(choice_key)
entity = self.svc.get_entity(self.choice_table, partition, row)
entity.Votes += 1
self.svc.update_entity(self.choice_table, entity)
except AzureMissingResourceHttpError:
raise PollNotFound()
def add_sample_polls(self):
"""Adds a set of polls from data stored in a samples.json file."""
poll_partition = '2014'
poll_row = 0
choice_partition = '2014'
choice_row = 0
for sample_poll in _load_samples_json():
poll_entity = {
'PartitionKey': poll_partition,
'RowKey': str(poll_row),
'Text': sample_poll['text'],
}
self.svc.insert_entity(self.poll_table, poll_entity)
for sample_choice in sample_poll['choices']:
choice_entity = {
'PartitionKey': choice_partition,
'RowKey': str(choice_row),
'Text': sample_choice,
'Votes': 0,
'PollPartitionKey': poll_partition,
'PollRowKey': str(poll_row),
}
self.svc.insert_entity(self.choice_table, choice_entity)
choice_row += 1
poll_row += 1
from azure.storage.table import TableService, Entity
table_service = TableService(
account_name='seva',
account_key=
'SgbxLwWkBH4XuGebxECoXfNVG3mVM5YjOs+SWTDUSacc+3YgUmcafYXrXdz5k0HtlZQ3AuEJ1IcFtZYeGVR9Hw=='
)
#table_service.delete_table('tasktable')
#table_service.create_table('tasktable')
#task = {'PartitionKey': 'tasksSeattle', 'RowKey': '001', 'description' : 'Take out the trash', 'priority' : 200}
#table_service.insert_entity('tasktable', task)
task = table_service.get_entity('tasktable', 'tasksSeattle', '001')
print(task.description)
print(task.priority)
import vk, urllib.request, csv, json, pprint, time
from pprint import pprint
from azure.storage.table import TableService, Entity, TableBatch
table_service = TableService(
account_name='seva',
account_key=
'SgbxLwWkBH4XuGebxECoXfNVG3mVM5YjOs+SWTDUSacc+3YgUmcafYXrXdz5k0HtlZQ3AuEJ1IcFtZYeGVR9Hw=='
)
batch = TableBatch()
#table_service.delete_table('MyVkApp')
#table_service.create_table('MyVkApp')
login = input("Введите имя пользователя: ")
password = input("Введите пароль: ")
session = vk.AuthSession(app_id='5889724',
user_login=login,
user_password=password)
api = vk.API(session)
#Вычисление количества друзей текущего пользователя
friends = api.friends.get(count='')
print('количество друзей:')
#pprint(len(friends))
user_info = {
'PartitionKey': 'user_info',
'RowKey': '001',
'description': len(friends)
}
user_info = table_service.insert_or_replace_entity('MyVkApp', user_info)
user_info = table_service.get_entity('MyVkApp', 'user_info', '001')
print(user_info.description)
import vk, urllib.request, csv, json, pprint, time
from pprint import pprint
from azure.storage.table import TableService, Entity, TableBatch
table_service = TableService(account_name='seva', account_key='SgbxLwWkBH4XuGebxECoXfNVG3mVM5YjOs+SWTDUSacc+3YgUmcafYXrXdz5k0HtlZQ3AuEJ1IcFtZYeGVR9Hw==')
batch =TableBatch()
#table_service.delete_table('MyVkApp')
#table_service.create_table('MyVkApp')
login = input("Введите имя пользователя: ")
password = input("Введите пароль: ")
session = vk.AuthSession(app_id='5889724', user_login=login, user_password=password)
api = vk.API(session)
friends =api.friends.get(count='')
#table_service.create_table('MyVkApp')
k =0
friends_info =table_service.get_entity('MyVkApp', 'my_friends1', str('1'))
print(friends_info.user_id)
print(friends_info.first_name)
print(friends_info.last_name)
storage_key = sys.argv[2]
batch_account = sys.argv[3]
batch_key = sys.argv[4]
batch_url = sys.argv[5]
table_name = sys.argv[6]
job_id = sys.argv[7]
entity_pk = sys.argv[8]
entity_rk = sys.argv[9]
table_service = TableService(account_name=storage_account,
account_key=storage_key)
blob_service = BlockBlobService(account_name=storage_account,
account_key=storage_key)
credentials = batchauth.SharedKeyCredentials(batch_account, batch_key)
batch_client = batch.BatchServiceClient(credentials, base_url=batch_url)
entity = table_service.get_entity(table_name, entity_pk, entity_rk)
wait_for_tasks_to_complete(table_service, batch_client, table_name, entity,
job_id)
if table_name == 'DatabaseEntity':
container_name = sys.argv[10]
files = 0
total_size = 0
db_type = 'Nucleotide'
generator = blob_service.list_blobs(container_name,
prefix=entity_rk + '.')
for blob in generator:
files += 1
total_size += blob.properties.content_length
extension = blob.name.split(".")[-1]
class Repository(object):
"""Azure Table Storage repository."""
def __init__(self, settings):
"""Initializes the repository with the specified settings dict.
Required settings are:
- STORAGE_NAME
- STORAGE_KEY
- STORAGE_TABLE_POLL
- STORAGE_TABLE_CHOICE
"""
self.name = 'Azure Table Storage'
self.storage_name = settings['STORAGE_NAME']
self.storage_key = settings['STORAGE_KEY']
self.poll_table = settings['STORAGE_TABLE_POLL']
self.choice_table = settings['STORAGE_TABLE_CHOICE']
self.svc = TableService(self.storage_name, self.storage_key)
self.svc.create_table(self.poll_table)
self.svc.create_table(self.choice_table)
def get_polls(self):
"""Returns all the polls from the repository."""
poll_entities = self.svc.query_entities(self.poll_table)
polls = [_poll_from_entity(entity) for entity in poll_entities]
return polls
def get_poll(self, poll_key):
"""Returns a poll from the repository."""
try:
partition, row = _key_to_partition_and_row(poll_key)
poll_entity = self.svc.get_entity(self.poll_table, partition, row)
choice_entities = self.svc.query_entities(
self.choice_table,
"PollPartitionKey eq '{0}' and PollRowKey eq '{1}'" \
.format(partition, row)
)
poll = _poll_from_entity(poll_entity)
poll.choices = [_choice_from_entity(choice_entity)
for choice_entity in choice_entities]
return poll
except AzureMissingResourceHttpError:
raise PollNotFound()
def increment_vote(self, poll_key, choice_key):
"""Increment the choice vote count for the specified poll."""
try:
partition, row = _key_to_partition_and_row(choice_key)
entity = self.svc.get_entity(self.choice_table, partition, row)
entity.Votes += 1
self.svc.update_entity(self.choice_table, entity)
except AzureMissingResourceHttpError:
raise PollNotFound()
def add_sample_polls(self):
"""Adds a set of polls from data stored in a samples.json file."""
poll_partition = '2014'
poll_row = 0
choice_partition = '2014'
choice_row = 0
for sample_poll in _load_samples_json():
poll_entity = {
'PartitionKey': poll_partition,
'RowKey': str(poll_row),
'Text': sample_poll['text'],
}
self.svc.insert_entity(self.poll_table, poll_entity)
for sample_choice in sample_poll['choices']:
choice_entity = {
'PartitionKey': choice_partition,
'RowKey': str(choice_row),
'Text': sample_choice,
'Votes': 0,
'PollPartitionKey': poll_partition,
'PollRowKey': str(poll_row),
}
self.svc.insert_entity(self.choice_table, choice_entity)
choice_row += 1
poll_row += 1
import vk, urllib.request, csv, json, pprint, time
from pprint import pprint
from azure.storage.table import TableService, Entity, TableBatch
table_service = TableService(account_name='seva', account_key='SgbxLwWkBH4XuGebxECoXfNVG3mVM5YjOs+SWTDUSacc+3YgUmcafYXrXdz5k0HtlZQ3AuEJ1IcFtZYeGVR9Hw==')
batch =TableBatch()
#table_service.delete_table('MyVkApp')
#table_service.create_table('MyVkApp')
login = input("Введите имя пользователя: ")
password = input("Введите пароль: ")
session = vk.AuthSession(app_id='5889724', user_login=login, user_password=password)
api = vk.API(session)
posts =api.wall.get(count='')
#table_service.create_table('MyVkApp')
all_posts_info ={'PartitionKey': 'posts', 'RowKey': '001', 'all_posts': posts[0]}
all_posts_info =table_service.insert_or_replace_entity('MyVkApp', all_posts_info)
all_posts_info =table_service.get_entity('MyVkApp', 'posts', '001')
print(all_posts_info.all_posts)
class AzureTableDatabase(object):
def __init__(self):
self.connection = TableService(account_name=storage_account,
account_key=table_connection_string)
self.table_name = table_name
def _update_entity(self, record):
record.LastModified = datetime.now()
self.connection.update_entity(self.table_name, record)
def create_table(self):
self.connection.create_table(self.table_name)
def raw_table(self, limit=100):
"""
Retrieve a list of rows in the table.
"""
calls = self.connection.query_entities(self.table_name,
num_results=limit)
return calls
def list_calls(self, limit=100, select='PartitionKey'):
"""
Retrieve a set of records that need a phone call
"""
calls = self.connection.query_entities(self.table_name,
num_results=limit,
select=select)
return [c.PartitionKey for c in calls]
def reset_stale_calls(self, time_limit):
"""
Retrieve calls that are not done and whose last modified time was older than the limit.
"""
records = self.connection.query_entities(
self.table_name,
filter="LastModified lt datetime'{0}' and Status ne '{1}'".format(
time_limit.date(), Statuses.extracting_done))
if not records.items:
raise NoRecordsToProcessError()
num_records = len(records.items)
for record in records:
if 'LastErrorStep' in record:
record.Status = record.LastErrorStep
del record.LastErrorStep
record.Status = Statuses.reset_map.get(record.Status,
record.Status)
self._update_entity(record)
return num_records
def retrieve_next_record_for_call(self):
"""
Retrieve a set of records that need a phone call
"""
records = self.connection.query_entities(
self.table_name,
num_results=1,
filter="Status eq '{0}'".format(Statuses.new))
if len(records.items) == 0:
raise NoRecordsToProcessError()
record = records.items[0]
record.Status = Statuses.calling
self._update_entity(record)
return record.PartitionKey
def set_error(self, partition_key, step):
""" Reset a row from error state
"""
record = self.connection.get_entity(self.table_name, partition_key,
partition_key)
record.Status = Statuses.error
record['LastErrorStep'] = step
self._update_entity(record)
def retrieve_next_record_for_transcribing(self):
records = self.connection.query_entities(
self.table_name,
num_results=1,
filter="Status eq '{0}'".format(Statuses.recording_ready),
)
if not records.items:
raise NoRecordsToProcessError()
record = records.items[0]
record.Status = Statuses.transcribing
self._update_entity(record)
return record.CallUploadUrl, record.PartitionKey
def update_transcript(self, partition_key, transcript, status):
record = self.connection.get_entity(
self.table_name,
partition_key,
partition_key,
)
if status == TranscriptionStatus.success:
record.CallTranscript = transcript
record.Status = Statuses.transcribing_done
record.TranscribeTimestamp = datetime.now()
self._update_entity(record)
elif status == TranscriptionStatus.request_error:
self.set_error(partition_key, Statuses.transcribing)
else:
record.Status = Statuses.transcribing_failed
self._update_entity(record)
def change_status(self, original_status, new_status):
records = self.connection.query_entities(
self.table_name,
filter="Status eq '{0}'".format(original_status),
)
if not records.items:
return
for record in records.items:
record.Status = new_status
self.connection.update_entity(self.table_name, record)
def query(self, column, value, limit=1):
records = self.connection.query_entities(self.table_name,
num_results=limit,
filter="{0} eq '{1}'".format(
column, value))
return records
def retrieve_next_record_for_extraction(self):
records = self.connection.query_entities(
self.table_name,
num_results=1,
filter="Status eq '{0}'".format(Statuses.transcribing_done))
if not records.items:
raise NoRecordsToProcessError()
record = records.items[0]
record.Status = Statuses.extracting
self._update_entity(record)
return record.CallTranscript, record.PartitionKey
def update_location_date(self, partition_key, location_dict, date_dict):
record = self.connection.get_entity(self.table_name, partition_key,
partition_key)
if location_dict and date_dict:
record.update(**location_dict)
record.update(**date_dict)
record.Status = Statuses.extracting_done
else:
record.Status = Statuses.failed_to_return_info
self._update_entity(record)
def upload_new_requests(self, request_ids):
"""
Upload new request ids to the database
"""
for request_id in request_ids:
record = {
'PartitionKey': request_id,
'RowKey': request_id,
'Status': Statuses.new,
'LastModified': datetime.now()
}
try:
self.connection.insert_entity(self.table_name, record)
except AzureConflictHttpError:
pass # already exists. silently ignore.
def update_call_id(self, alien_registration_id, call_id):
record = self.connection.get_entity(self.table_name,
alien_registration_id,
alien_registration_id)
record.CallID = call_id
record.Status = Statuses.calling
record.CallTimestamp = datetime.now()
self._update_entity(record)
def update_azure_path(self, alien_registration_id, azure_path):
record = self.connection.get_entity(self.table_name,
alien_registration_id,
alien_registration_id)
record.Status = Statuses.recording_ready
record.CallUploadUrl = azure_path
self._update_entity(record)
def delete_ain(self, ain):
return self.connection.delete_entity(self.table_name, ain, ain)
def get_ain(self, ain):
return self.connection.get_entity(self.table_name, ain, ain)
