def uploadToTable():
connection_string, container_name = get_credentials()
blob = BlobClient.from_connection_string(conn_str=connection_string,
container_name=container_name,
blob_name=filename)
blob_data = blob.download_blob().readall()
json_file_content = blob_data.decode('utf8')
jlist = json.loads(json_file_content)
table_service = TableService(connection_string=connection_string)
table_service.create_table(table_name=fixed_tablename)
partition_key = 'Fishlanding'
row = 1
batch_list = []
for jdict in jlist:
task = {'PartitionKey': partition_key, 'RowKey': str(row)}
for key in jdict:
keyVal = key.replace(' ', '')
if keyVal == '':
pass
task[keyVal] = jdict[key]
batch_list.append(task)
row += 1
seperated_batch_list = list_of_groups(init_list=batch_list,
childern_list_len=50)
for children_list in seperated_batch_list:
batch = TableBatch()
for task in children_list:
batch.insert_or_replace_entity(task)
table_service.commit_batch(table_name=tablename, batch=batch)
return batch_list
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
# Connect to Azure Table Storage
table_service = TableService(
connection_string=os.environ['AzureWebJobsStorage'])
table_service.create_table(
'intents') if not table_service.exists('intents') else None
req_body = req.get_json()
if req_body:
# Create row to be saved on Azure Table Storage
print(req_body.get('ConversationId'))
data = req_body
data["PartitionKey"] = req_body.get('ConversationId')
data["RowKey"] = req_body.get('MessageId')
# Save row on Azure Table Storage
table_service.insert_or_replace_entity('intents', data)
return func.HttpResponse(
f"Row {req_body.get('MessageId')} for {req_body.get('ConversationId')} added"
)
else:
return func.HttpResponse("Please pass valid request body",
status_code=400)
def throttle_ip_requests(ip_entry):
max_from_single_ip = 5
table_service = TableService(account_name=os.environ['STORAGE_ACCOUNT'],
account_key=os.environ['STORAGE_KEY'])
table_service.create_table(
table_name=os.environ['BILLING_TABLE']) #create if it doesn't exist
ip_row = None
try:
ip_row = table_service.get_entity(os.environ['BILLING_TABLE'],
ip_entry['PartitionKey'],
ip_entry['RowKey'])
except:
pass
if not ip_row:
ip_entry['count'] = 1
table_service.insert_entity(table_name=os.environ['BILLING_TABLE'],
entity=ip_entry)
ip_row = ip_entry
else:
lastdatetime = datetime.strptime(ip_row['time'], "%d/%m/%Y %H:%M:%S")
currdatetime = datetime.strptime(ip_entry['time'], "%d/%m/%Y %H:%M:%S")
tdelta = currdatetime - lastdatetime
if tdelta.days < 1 and ip_row['count'] > max_from_single_ip:
return True # throttle this entry..
elif tdelta.days > 0: #over 1 day has passed, update the count to 1 and reset time
ip_row['count'] = 1
ip_row['time'] = currdatetime.strftime("%d/%m/%Y %H:%M:%S")
table_service.update_entity(os.environ['BILLING_TABLE'], ip_row)
else: # less than 1 day but count is < max_from_single_ip, update the count
ip_row['count'] = ip_row['count'] + 1
table_service.update_entity(os.environ['BILLING_TABLE'], ip_row)
# However we got here, do not throttle
return False
class AzureTableProvider:
def __init__(self, account_name, key, table_name):
self.target_table = table_name
if not account_name or not key:
raise Exception('Account or key not specified')
self.table_service = TableService(account_name=account_name, account_key=key)
self.table_service.create_table(self.target_table)
def get_all(self):
return self.table_service.query_entities(self.target_table)
def remove(self, item):
query_str = "Link eq '%s'" % item
tasks = self.table_service.query_entities(self.target_table, filter=query_str)
if any(tasks):
for task in tasks:
self.table_service.delete_entity(self.target_table, task.PartitionKey, task.RowKey)
return True
return False
def add(self, item):
track = {
'PartitionKey': 'MusicBotEntry',
'RowKey': str(uuid.uuid4()),
'Link': item
}
self.table_service.insert_entity(self.target_table, track)
class StorageManager:
def __init__(self, table_name=None):
self.azure_storage_name = cnf.get('credentials', 'azure_storage_name')
self.azure_storage_key = cnf.get('credentials', 'azure_storage_key')
self.table_service = TableService(account_name=self.azure_storage_name,
account_key=self.azure_storage_key)
self.table_name = table_name if table_name is not None else cnf.get(
'resources', 'table_name')
def create_table(self):
self.table_service.create_table(self.table_name)
def upload_data(self, entities):
# Count records to upload
num_entities = len(entities)
# Upload record by record and print info
time_start = time.time()
for i, entity in enumerate(entities):
self.table_service.insert_or_replace_entity(
self.table_name, entity)
print_uploading_state(i + 1, num_entities, self.table_name)
print_successful_upload_state(num_entities, self.table_name,
time.time() - time_start)
def query_entities(self, query_filter=None, query_selector=None):
return self.table_service.query_entities(self.table_name,
filter=query_filter,
select=query_selector)
def create_if_missing(self, table_service: TableService) -> None:
try:
self.disable_storage_client_logging()
if not table_service.exists(TABLE_NAME):
table_service.create_table(TABLE_NAME)
finally:
self.enable_storage_client_logging()
def table_service():
table_name = "apmagentpythonci" + str(uuid.uuid4().hex)
table_service = TableService(connection_string=CONNECTION_STRING)
table_service.create_table(table_name)
table_service.table_name = table_name
yield table_service
table_service.delete_table(table_name)
def tableStorage(table_name, partition_key, row_key, hins_processed, timesaved,
time_by_system, time_by_user, requests):
try:
table_service = TableService(
account_name=config.AZURE['STORAGE_ACCOUNT_NAME'],
account_key=config.AZURE['STORAGE_ACCOUNT_KEY'])
entity = {
'PartitionKey': partition_key,
'RowKey': row_key,
'HinsProcessed': hins_processed,
'TimeSaved': timesaved,
'TimeBySystem': time_by_system,
'TimeByUser': time_by_user,
'Requests': requests
}
if not table_service.exists(table_name, timeout=None):
table_service.create_table(table_name, fail_on_exist=False)
try:
table_service.insert_entity(table_name, entity)
print("Entity Doesn't Exist")
print("Creating Entity\n")
except Exception as e:
print("Entity Exists")
print("Updating entity\n")
currentEntity = table_service.get_entity(table_name, partition_key,
row_key)
tempHinProcessed = currentEntity.HinsProcessed + hins_processed
tempTimeSaved = currentEntity.TimeSaved + timesaved
tempTimeBySystem = currentEntity.TimeBySystem + time_by_system
tempTimeByUser = currentEntity.TimeByUser + time_by_user
tempRequest = currentEntity.Requests + requests
entity = {
'PartitionKey': partition_key,
'RowKey': row_key,
'HinsProcessed': tempHinProcessed,
'TimeSaved': tempTimeSaved,
'TimeBySystem': tempTimeBySystem,
'TimeByUser': tempTimeByUser,
'Requests': tempRequest
}
table_service.update_entity(table_name,
entity,
if_match='*',
timeout=None)
except Exception as e:
print(e)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP Submit trigger received a request')
logging.debug('Creating blob service')
table_service = TableService(
account_name=os.getenv('AZURE_STORAGE_ACCOUNT'),
account_key=os.getenv('AZURE_STORAGE_ACCESS_KEY'))
headers_dict = {
"Access-Control-Allow-Credentials": "true",
"Access-Control-Allow-Origin": "*",
"Access-Control-Allow-Methods": "Post"
}
schema = submit_schema.SubmitMessageSchema()
try:
job_dict = schema.loads(req.get_body())
except ValidationError:
error = f'Failed to validate the submit message'
return func.HttpResponse(error, headers=headers_dict, status_code=400)
table_name = os.getenv('AZURE_TABLE_NAME')
table_service.create_table(table_name)
guid = uuid.uuid4()
try:
job_dict = schema.dump(job_dict)
except ValidationError:
error = f'Failed to submit job'
return func.HttpResponse(error, headers=headers_dict, status_code=400)
entity = Entity()
entity.PartitionKey = 'await'
entity.RowKey = str(guid)
entity.Error = ""
entity.area_name = job_dict['area_name']
entity.crop = job_dict['crop']
entity.planting_date = job_dict['planting_date']
entity.irrigated = job_dict['irrigated']
entity.fraction = job_dict['fraction']
entity.geometry = json.dumps(job_dict['geometry'])
try:
table_service.insert_entity(table_name, entity)
except TypeError:
error = f'Failed to insert to table'
return func.HttpResponse(error, headers=headers_dict, status_code=400)
response_dict = {}
response_dict['guid'] = guid
schema = submit_schema.SubmitResponseSchema()
response_message = schema.dumps(response_dict)
return func.HttpResponse(response_message,
headers=headers_dict,
mimetype='application/json')
def putTableEntity(data):
'''
'''
try:
# Get Azure Storage Table connection
logging.info('Creating Azure Storage Table')
table_service = TableService(
connection_string=os.environ['SA_CONNX_STRING'])
try:
logging.info('Checking for existing Azure Storage Table')
# Check for table, if none then create new table
table_service.create_table(os.environ['SA_TABLE_INSIGHTS'],
fail_on_exist=True)
logging.info('Success: New table created')
except Exception as e:
logging.info('Failed: Table already exists {0}'.format(e))
for entity in data:
try:
# Create unique row key
row_key = '{0}_{1}_{2}'.format(entity['vi_file_name'],
entity['vi_feature_type'],
entity['vi_feature'])
task = {
'PartitionKey': 'examplekey',
'RowKey': row_key,
'FileName': entity['vi_file_name'],
'SourceLanguage': entity['vi_source_language'],
'FeatureType': entity['vi_feature_type'],
'Feature': entity['vi_feature'],
'ConfidenceScore': entity['vi_confidence_score']
}
# Write entry to Azure Storage Table
table_service.insert_or_merge_entity(
os.environ['SA_TABLE_INSIGHTS'], task)
except Exception as e:
logging.info(
'Failed: Put entity to Azure Storage Table {0}'.format(e))
logging.info('Success: Put entity to Azure Storage Table')
except Exception as e:
logging.info(
'Failed: Put entities to Azure Storage Table {0}'.format(e))
def test_table_create(instrument, elasticapm_client):
table_name = "apmagentpythonci" + str(uuid.uuid4().hex)
table_service = TableService(connection_string=CONNECTION_STRING)
elasticapm_client.begin_transaction("transaction.test")
table_service.create_table(table_name)
table_service.delete_table(table_name)
elasticapm_client.end_transaction("MyView")
span = elasticapm_client.events[constants.SPAN][0]
assert span["name"] == "AzureTable Create {}".format(table_name)
assert span["type"] == "storage"
assert span["subtype"] == "azuretable"
assert span["action"] == "Create"
def main(event: func.EventGridEvent):
event_type = event.event_type
event_subject = event.subject
containername = event_subject.split("/")[-3]
# credentials needed
connection_string = os.getenv('AZURE_CONNECTION_STRING')
container_name = os.getenv('ContainerName')
# Only blob creations at container would trigger this function
if event_type == "Microsoft.Storage.BlobCreated" and containername == container_name:
filename = event_subject.split("/")[-1]
tablename = gettablename(
filename=filename) + datetime.now().strftime("%H%M%S")
table_service = TableService(connection_string=connection_string)
table_service.create_table(table_name=tablename)
# Download the blob data
blob = BlobClient.from_connection_string(conn_str=connection_string,
container_name=container_name,
blob_name=filename)
blob_data = blob.download_blob().readall()
json_file_content = blob_data.decode('utf8').replace("'", '"')
jlist = json.loads(json_file_content)
# The partition key might be changed later. This is only for DEVELOPMENT purpose
partition_key = tablename
row = 1
batch_list = []
for jdict in jlist:
if 'Cruise_ID' in jdict:
partition_key = jdict['Cruise_ID']
task = {'PartitionKey': partition_key, 'RowKey': str(row)}
for key in jdict:
keyVal = key.replace(' ', '')
if keyVal == '':
continue
task[keyVal] = jdict[key]
batch_list.append(task)
row += 1
seperated_batch_list = list_of_groups(init_list=batch_list,
childern_list_len=50)
for children_list in seperated_batch_list:
batch = TableBatch()
for task in children_list:
batch.insert_or_replace_entity(task)
table_service.commit_batch(table_name=tablename, batch=batch)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
with open('config.json', 'r') as config_file:
config_data = json.load(config_file)
connectionstring = config_data["connectionstring"]
table_service = TableService(connection_string=connectionstring)
table = config_data["table"]
tableExists = table_service.create_table(table)
courses = readcsv()
for item in courses:
# print(item)
task = Entity()
task.PartitionKey = item.subject
task.RowKey = item.instructor
task.lectures = item.lectures
task.labs = item.labs
task.points = item.points
task.isWeekend = item.isWeekend
table_service.insert_entity(table, task)
return func.HttpResponse(
"Cosmos DB - Table API example database is created.")
def put_object(obj,
table_name='default',
account_name='cloudmaticafunc9b4c',
account_key='YOUR_KEY',
partition_key='default',
row_key=None):
table_service = TableService(account_name=account_name,
account_key=account_key)
if not table_service.exists(table_name):
table_service.create_table(table_name)
if not row_key:
row_key = str(uuid.uuid4())
obj['PartitionKey'] = partition_key
obj['RowKey'] = row_key
table_service.insert_or_replace_entity(table_name, obj)
return obj
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Starting clean table.')
ret = dict()
name = req.headers.get('name')
if not name: #If name wasnt added as header, search for it in the parameters
name = req.params.get('name')
if name:
retrieved_secret = getConnectionString()
table_service = TableService(connection_string=retrieved_secret.value)
table_service.delete_table(name)
time.sleep(1)
existe = False
while (not existe):
logging.info("Intentando crearla...")
time.sleep(5)
existe = table_service.create_table(name)
logging.info("Done!!")
ret['result'] = 'Done!'
return func.HttpResponse(json.dumps(ret), status_code=200)
else:
ret['result'] = 'Please pass a name on the query string or in the request body!'
return func.HttpResponse(json.dumps(ret), status_code=400)
def _wait_for_table_to_be_created(table_service: TableService,
table_name: str) -> bool:
timer = datetime.now()
is_created = False
while not is_created:
try:
table_service.create_table(table_name, fail_on_exist=True)
is_created = True
print(f"Created table '{table_name}'")
except AzureConflictHttpError as e:
if datetime.now() - timer > timedelta(minutes=5):
print(e)
print(f"Could not crate {table_name} in 5 minutes")
print("Giving up")
print("Could not create Azure Table!")
break
print(f"Retrying in 5 seconds...")
sleep(5)
return is_created
class TodoDatabase:
instance = None
def __init__(self):
if not TodoDatabase.instance:
self.table_name = 'todostable'
self.table_service = TableService(account_name='maheshsdcard',
account_key=KEY)
self.table_service.create_table(self.table_name)
TodoDatabase.instance = self
else:
raise Exception("Singleton")
@staticmethod
def getInstance():
if TodoDatabase.instance == None:
TodoDatabase()
return TodoDatabase.instance
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Starting create table.')
ret = dict()
name = req.headers.get('name')
if not name: #If name wasnt added as header, search for it in the parameters
name = req.params.get('name')
if name:
retrieved_secret = getConnectionString()
table_service = TableService(connection_string=retrieved_secret.value)
table_service.create_table(name)
ret['result'] = 'Success'
return func.HttpResponse(json.dumps(ret), status_code=200)
else:
ret['result'] = 'Please pass a name on the query string or in the request body!'
return func.HttpResponse(json.dumps(ret), status_code=400)
class CreateTableInstance:
def __init__(self, identity, **options):
self.identity = identity
self.account_key = options.get('account_key', "")
self.storage_account_name = options.get('storage_account_name', "")
self.table_service = TableService(account_name=self.storage_account_name, account_key=self.account_key)\
if self.storage_account_name and self.account_key else None
if options.get('connection_string'):
self.connection_string = options.get('connection_string', "")
self.table_service = TableService(connection_string=self.connection_string)
self.table_name = options.get('table_name', "defaulttablename")
def create_storage_account(self, **options):
"""
Create storage account
:return:
"""
meta_data = create_storage_account(id_=self.identity, **options)
self.account_key = meta_data.keys.keys[0].value
self.storage_account_name = meta_data.storage_account_name
def create_table(self, table_name = None, **options):
"""
Create a table
:param table_name:
:return:
"""
self.table_name = table_name if table_name else self.table_name
try:
self.table_service.create_table(table_name=self.table_name)
except AttributeError:
try:
assert self.storage_account_name and self.account_key
self.table_service = TableService(account_name=self.storage_account_name, account_key=self.account_key)
except AssertionError:
assert self.connection_string, "Provide at least connection_string, or" \
" storage_account_name with account_key"
self.table_service = TableService(connection_string=self.connection_string)
self.table_service.create_table(table_name=self.table_name)
except Exception as e:
print(e.args[0])
def __connect(table_name: str,
account_name: str = None,
account_key: str = None,
connection_string: str = None) -> TableService:
"""Helper function connecting to Azure Table service"""
# Set up connection to table service
if account_name is not None and account_key is not None:
table_service = TableService(account_name=account_name,
account_key=account_key)
elif connection_string is not None:
table_service = TableService(connection_string=connection_string)
else:
raise ValueError(
"Specify either the account name & key or the connection string")
# Create a new table only if it does not exists
table_service.create_table(table_name)
return table_service
def create_table(account_name: str, account_key: str, table_name: str) -> bool:
""" Create an Azure Table.
:param account_name: Azure Storage account name.
:param account_key: Azure Storage account key.
:param table_name: name of the table to create.
:return: whether the table was created or not.
"""
service = TableService(account_name=account_name, account_key=account_key)
return service.create_table(table_name)
def putTableEntity(video_id, video_name, video_path, insights_path, video_url):
'''
'''
try:
logging.info('Putting new entity to Azure Storage Table')
# Get Azure Storage Table connection
table_service = TableService(
connection_string=os.environ['SA_CONNX_STRING'])
try:
logging.info('Checking for existing Azure Storage Table')
# Check for table, if none then create new table
table_service.create_table(os.environ['SA_TABLE_TRACKER'],
fail_on_exist=True)
logging.info('Success: New table created')
except Exception as e:
logging.info('Failed: Table already exists {0}'.format(e))
task = {
'PartitionKey': 'examplekey',
'RowKey': video_id,
'VideoIndexerId': video_id,
'VideoName': video_name,
'VideoPath': video_path,
'VideoUrl': video_url,
'InsightsPath': insights_path,
'State': 'Processed'
}
# Write entry to Azure Storage Table
table_service.insert_or_merge_entity(os.environ['SA_TABLE_TRACKER'],
task)
logging.info('Success: Put entity to Azure Storage Table')
except Exception as e:
logging.info('Failed: Put entity to Azure Storage Table {0}'.format(e))
class AzureTableStorageHandler(logging.Handler):
"""
A handler class which writes formatted logging records to Azure Table Storage.
"""
def __init__(self, account_name, account_key, table_name, *, level=logging.NOTSET):
"""
Setup TableService and the specified table for logging.
"""
super().__init__(level=level)
self.table_service = TableService(account_name=account_name, account_key=account_key)
self.table_name = table_name
if not self.table_service.exists(self.table_name):
self.table_service.create_table(self.table_name)
self.formatter = logging.Formatter("%(message)s")
self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="AzHndlr")
self.epoch_max = datetime(MAXYEAR, 12, 31, 23, 59, 59, 999999, timezone.utc).timestamp()
def insert_log(self, record):
"""
Insert log to Azure Table Storage.
"""
entity = {
"PartitionKey": record.name,
"RowKey": str(self.epoch_max - time()),
"LocalTimestamp": self.formatter.formatTime(record),
"LevelName": record.levelname,
"Level": record.levelno,
"Message": self.format(record)
}
self.table_service.insert_entity(self.table_name, entity)
def emit(self, record):
"""
Emit a record.
This method just submit the logging task to worker thread and return immediately.
"""
self.executor.submit(self.insert_log, record)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
# Connect to Azure Table Storage
table_service = TableService(
connection_string=os.environ['AzureWebJobsStorage'])
table_service.create_table(
'intents') if not table_service.exists('intents') else None
# Retrieve entities from Azure Table Storage
entries = list(table_service.query_entities('intents'))
# Convert to pandas dataframe and output as csv
dataframe = pd.DataFrame(entries)
list_of_dictionaries = dataframe.to_dict('records')
# Set HTTP Response header and mimetype
mimetype = 'text/csv'
headers = {'Content-Disposition': 'attachment; filename="export.csv"'}
# Set response body
body = dataframe.to_csv(index=False)
return func.HttpResponse(body=body, mimetype=mimetype, headers=headers)
class AzureTable():
def __init__(self, account_name, account_key):
self.table_service = TableService(account_name=account_name,
account_key=account_key)
def create_table(self, table_name):
return self.table_service.create_table(table_name)
def exists_table(self, table_name):
return self.table_service.exists(table_name)
def insert_or_replace_entity(self, table_name, partition_key, row_key,
**kwargs):
try:
entity = self.table_service.get_entity(table_name, partition_key,
row_key)
except Exception:
# Insert a new entity
entity = {'PartitionKey': partition_key, 'RowKey': row_key}
for (k, v) in kwargs.items():
entity[k] = v
return self.table_service.insert_or_replace_entity(table_name, entity)
def insert_or_replace_entity2(self, table_name, entity):
return self.table_service.insert_or_replace_entity(table_name, entity)
def insert_entity(self, table_name, entity):
return self.table_service.insert_entity(table_name, entity)
def update_entity(self, table_name, entity):
return self.table_service.update_entity(table_name, entity)
def get_entity(self, table_name, partition_key, row_key):
return self.table_service.get_entity(table_name, partition_key,
row_key)
def delete_entity(self, table_name, partition_key, row_key):
self.table_service.delete_entity(table_name, partition_key, row_key)
def delete_table(self, table_name):
return self.table_service.delete_table(table_name)
def get_entities(self, table_name, partition_key):
filter = "PartitionKey eq '{0}'".format(partition_key)
return self.table_service.query_entities(table_name, filter)
raw_input('Press Enter to continue...')
# Each storage account has a primary and secondary access key.
# These keys are used by aplications to access data in your storage account, such as Tables.
# Obtain the primary storage access key for use with the rest of the demo
response = azurerm.get_storage_account_keys(auth_token, subscription_id,
resourcegroup_name,
storageaccount_name)
storageaccount_keys = json.loads(response.text)
storageaccount_primarykey = storageaccount_keys['keys'][0]['value']
# Create the Table with the Azure Storage SDK and the access key obtained in the previous step
table_service = TableService(account_name=storageaccount_name,
account_key=storageaccount_primarykey)
response = table_service.create_table('pizzatable')
if response == True:
print('Storage Table: pizzatable created successfully.\n')
else:
print('Error creating Storage Table.\n')
time.sleep(1)
###
# Use the Azure Storage Storage SDK for Python to create some entries in the Table
###
print(
'Now let\'s add some entries to our Table.\nRemember, Azure Storage Tables is a NoSQL datastore, so this is similar to adding records to a database.'
)
raw_input('Press Enter to continue...')
class AzureTable(object):
def __init__(self, account_name: str, account_key: str, table_name: str,
partition_key_field: str, clustering_key_field: str):
self.table = TableService(account_name=account_name,
account_key=account_key)
self.table_name = self.table_name
self.partition_key_field = partition_key_field
self.clustering_key_field = clustering_key_field
@property
def partition_key_name(self) -> str:
return 'PartitionKey'
@property
def clustering_key_name(self) -> str:
return 'RowKey'
def get_payload(self, payload: dict):
item = deepcopy(payload)
partition_key = payload.get(self.partition_key_field)
clustering_key = payload.get(self.clustering_key_field)
if partition_key is None:
raise PartitionKeyNotFoundError(
'payload={} does not have a partition key')
if clustering_key is None:
raise ClusteringKeyNotFoundError(
'payload={} does not have a clustering key')
item.update({
self.partition_key_name: partition_key,
self.clustering_key_name: clustering_key
})
return item
def create(self):
return self.table.create_table(self.table_name)
def insert(self, item: dict):
return self.table.insert_entity(self.table_name,
self.get_payload(item))
def update(self, item: dict):
pass
def upsert(self, item: dict):
pass
def delete(self, key: str):
pass
def read(self, key: str):
pass
def insert_batch(self, items: list):
batch = TableBatch()
for item in items:
batch.insert_entity(self.get_payload(item))
return self.table.commit_batch(self.table_name, batch)
def get(self, partition_key: str, clustering_key: str):
return self.table.get_entity(self.table_name, partition_key,
clustering_key)
def get_by_partition(self, partition_key: str) -> list:
return self.table.query_entities(self.table_name,
filter="{} eq '{}'".format(
self.partition_key_name,
partition_key))
the_connection_string = os.getenv('TABLE_CONNECTION_STRING','not_set')
# timeout in seconds
#timeout = 5
#socket.setdefaulttimeout(timeout)
# TableService
# table_service = TableService(account_name='myaccount', account_key='mykey')
# CosmosDB
table_service = TableService(endpoint_suffix="table.cosmos.azure.com", connection_string=the_connection_string)
#table_service.set_proxy("myproxy", 8888)
print("create table...")
table_service.create_table('tasktable')
# dictionary (read from JSON)
print("creating task 001...")
task = {'PartitionKey': 'tasksSeattle', 'RowKey': '001',
'description': 'Take out the trash', 'priority': 200}
table_service.insert_entity('tasktable', task)
print("Create task through an Entity object...")
task = Entity()
task.PartitionKey = 'tasksSeattle'
task.RowKey = '002'
task.description = 'Wash the car'
task.priority = 100
table_service.insert_entity('tasktable', task)
# Databricks notebook source
# See https://docs.microsoft.com/en-us/azure/cosmos-db/table-storage-how-to-use-python
# Make sure that azure-cosmosdb-table is installed on the cluster. This can be done by adding azure-cosmosdb-table as PyPi library on the cluster
AccountName="<<storage account>>"
AccountKey="<<access key>>"
table_name="<<your table name>>"
# COMMAND ----------
from azure.cosmosdb.table.tableservice import TableService
from azure.cosmosdb.table.models import Entity
table_service = TableService(account_name=AccountName, account_key=AccountKey)
table_service.create_table(table_name)
# COMMAND ----------
entity_example = {
'PartitionKey': 'Car',
'RowKey': 'Tesla',
'text': 'Musk',
'color': 'Purple',
'price': '5'
}
table_service.insert_entity(table_name, entity_example)
entity_example = {
'PartitionKey': 'Car',
'RowKey': 'Audi',
'text': 'Germany',
'color': 'Green',
class AzureClient(object):
"""
Azure client is the wrapper on top of azure python sdk
"""
def __init__(self, config):
self._config = config
self.subscription_id = str(config.get('azure_subscription_id'))
self._credentials = ServicePrincipalCredentials(
client_id=config.get('azure_client_id'),
secret=config.get('azure_secret'),
tenant=config.get('azure_tenant')
)
self._access_token = config.get('azure_access_token')
self._resource_client = None
self._storage_client = None
self._network_management_client = None
self._subscription_client = None
self._compute_client = None
self._access_key_result = None
self._block_blob_service = None
self._table_service = None
self._storage_account = None
log.debug("azure subscription : %s", self.subscription_id)
@property
@tenacity.retry(stop=tenacity.stop_after_attempt(5), reraise=True)
def access_key_result(self):
if not self._access_key_result:
storage_account = self.storage_account
if self.get_storage_account(storage_account).\
provisioning_state.value != 'Succeeded':
log.debug(
"Storage account %s is not in Succeeded state yet. ",
storage_account)
raise WaitStateException(
"Waited too long for storage account: {0} to "
"become ready.".format(
storage_account,
self.get_storage_account(storage_account).
provisioning_state))
self._access_key_result = self.storage_client.storage_accounts. \
list_keys(self.resource_group, storage_account)
return self._access_key_result
@property
def resource_group(self):
return self._config.get('azure_resource_group')
@property
def storage_account(self):
return self._config.get('azure_storage_account')
@property
def region_name(self):
return self._config.get('azure_region_name')
@property
def public_key_storage_table_name(self):
return self._config.get('azure_public_key_storage_table_name')
@property
def storage_client(self):
if not self._storage_client:
self._storage_client = \
StorageManagementClient(self._credentials,
self.subscription_id)
return self._storage_client
@property
def subscription_client(self):
if not self._subscription_client:
self._subscription_client = SubscriptionClient(self._credentials)
return self._subscription_client
@property
def resource_client(self):
if not self._resource_client:
self._resource_client = \
ResourceManagementClient(self._credentials,
self.subscription_id)
return self._resource_client
@property
def compute_client(self):
if not self._compute_client:
self._compute_client = \
ComputeManagementClient(self._credentials,
self.subscription_id)
return self._compute_client
@property
def network_management_client(self):
if not self._network_management_client:
self._network_management_client = NetworkManagementClient(
self._credentials, self.subscription_id)
return self._network_management_client
@property
def blob_service(self):
self._get_or_create_storage_account()
if not self._block_blob_service:
if self._access_token:
token_credential = TokenCredential(self._access_token)
self._block_blob_service = BlockBlobService(
account_name=self.storage_account,
token_credential=token_credential)
else:
self._block_blob_service = BlockBlobService(
account_name=self.storage_account,
account_key=self.access_key_result.keys[0].value)
return self._block_blob_service
@property
def table_service(self):
self._get_or_create_storage_account()
if not self._table_service:
self._table_service = TableService(
self.storage_account,
self.access_key_result.keys[0].value)
if not self._table_service. \
exists(table_name=self.public_key_storage_table_name):
self._table_service.create_table(
self.public_key_storage_table_name)
return self._table_service
def get_resource_group(self, name):
return self.resource_client.resource_groups.get(name)
def create_resource_group(self, name, parameters):
return self.resource_client.resource_groups. \
create_or_update(name, parameters)
def get_storage_account(self, storage_account):
return self.storage_client.storage_accounts. \
get_properties(self.resource_group, storage_account)
def create_storage_account(self, name, params):
return self.storage_client.storage_accounts. \
create(self.resource_group, name.lower(), params).result()
# Create a storage account. To prevent a race condition, try
# to get or create at least twice
@tenacity.retry(stop=tenacity.stop_after_attempt(2),
retry=tenacity.retry_if_exception_type(CloudError),
reraise=True)
def _get_or_create_storage_account(self):
if self._storage_account:
return self._storage_account
else:
try:
self._storage_account = \
self.get_storage_account(self.storage_account)
except CloudError as cloud_error:
if cloud_error.error.error == "ResourceNotFound":
storage_account_params = {
'sku': {
'name': 'Standard_LRS'
},
'kind': 'storage',
'location': self.region_name,
}
try:
self._storage_account = \
self.create_storage_account(self.storage_account,
storage_account_params)
except CloudError as cloud_error2: # pragma: no cover
if cloud_error2.error.error == "AuthorizationFailed":
mess = 'The following error was returned by ' \
'Azure:\n%s\n\nThis is likely because the' \
' Role associated with the provided ' \
'credentials does not allow for Storage ' \
'Account creation.\nA Storage Account is ' \
'necessary in order to perform the ' \
'desired operation. You must either ' \
'provide an existing Storage Account name' \
' as part of the configuration, or ' \
'elevate the associated Role.\nFor more ' \
'information on roles, see: https://docs.' \
'microsoft.com/en-us/azure/role-based-' \
'access-control/overview\n' % cloud_error2
raise ProviderConnectionException(mess)
elif cloud_error2.error.error == \
"StorageAccountAlreadyTaken":
mess = 'The following error was ' \
'returned by Azure:\n%s\n\n' \
'Note that Storage Account names must be ' \
'unique across Azure (not just in your ' \
'subscription).\nFor more information ' \
'see https://docs.microsoft.com/en-us/' \
'azure/azure-resource-manager/resource-' \
'manager-storage-account-name-errors\n' \
% cloud_error2
raise InvalidLabelException(mess)
else:
raise cloud_error2
else:
raise cloud_error
def list_locations(self):
return self.subscription_client.subscriptions. \
list_locations(self.subscription_id)
def list_vm_firewall(self):
return self.network_management_client.network_security_groups. \
list(self.resource_group)
def create_vm_firewall(self, name, parameters):
return self.network_management_client.network_security_groups. \
create_or_update(self.resource_group, name,
parameters).result()
def update_vm_firewall_tags(self, fw_id, tags):
url_params = azure_helpers.parse_url(VM_FIREWALL_RESOURCE_ID,
fw_id)
name = url_params.get(VM_FIREWALL_NAME)
return self.network_management_client.network_security_groups. \
create_or_update(self.resource_group, name,
{'tags': tags,
'location': self.region_name}).result()
def get_vm_firewall(self, fw_id):
url_params = azure_helpers.parse_url(VM_FIREWALL_RESOURCE_ID,
fw_id)
fw_name = url_params.get(VM_FIREWALL_NAME)
return self.network_management_client.network_security_groups. \
get(self.resource_group, fw_name)
def delete_vm_firewall(self, fw_id):
url_params = azure_helpers.parse_url(VM_FIREWALL_RESOURCE_ID,
fw_id)
name = url_params.get(VM_FIREWALL_NAME)
self.network_management_client \
.network_security_groups.delete(self.resource_group, name).wait()
def create_vm_firewall_rule(self, fw_id,
rule_name, parameters):
url_params = azure_helpers.parse_url(VM_FIREWALL_RESOURCE_ID,
fw_id)
vm_firewall_name = url_params.get(VM_FIREWALL_NAME)
return self.network_management_client.security_rules. \
create_or_update(self.resource_group, vm_firewall_name,
rule_name, parameters).result()
def delete_vm_firewall_rule(self, fw_rule_id, vm_firewall):
url_params = azure_helpers.parse_url(VM_FIREWALL_RULE_RESOURCE_ID,
fw_rule_id)
name = url_params.get(VM_FIREWALL_RULE_NAME)
return self.network_management_client.security_rules. \
delete(self.resource_group, vm_firewall, name).result()
def list_containers(self, prefix=None, limit=None, marker=None):
results = self.blob_service.list_containers(prefix=prefix,
num_results=limit,
marker=marker)
return (results.items, results.next_marker)
def create_container(self, container_name):
try:
self.blob_service.create_container(container_name,
fail_on_exist=True)
except AzureConflictHttpError as cloud_error:
if cloud_error.error_code == "ContainerAlreadyExists":
msg = "The given Bucket name '%s' already exists. Please " \
"use the `get` or `find` method to get a reference to " \
"an existing Bucket, or specify a new Bucket name to " \
"create.\nNote that in Azure, Buckets are contained " \
"in Storage Accounts." % container_name
raise DuplicateResourceException(msg)
return self.blob_service.get_container_properties(container_name)
def get_container(self, container_name):
return self.blob_service.get_container_properties(container_name)
def delete_container(self, container_name):
self.blob_service.delete_container(container_name)
def list_blobs(self, container_name, prefix=None):
return self.blob_service.list_blobs(container_name, prefix=prefix)
def get_blob(self, container_name, blob_name):
return self.blob_service.get_blob_properties(container_name, blob_name)
def create_blob_from_text(self, container_name, blob_name, text):
self.blob_service.create_blob_from_text(container_name,
blob_name, text)
def create_blob_from_file(self, container_name, blob_name, file_path):
self.blob_service.create_blob_from_path(container_name,
blob_name, file_path)
def delete_blob(self, container_name, blob_name):
self.blob_service.delete_blob(container_name, blob_name)
def get_blob_url(self, container_name, blob_name, expiry_time):
expiry_date = datetime.datetime.utcnow() + datetime.timedelta(
seconds=expiry_time)
sas = self.blob_service.generate_blob_shared_access_signature(
container_name, blob_name, permission=BlobPermissions.READ,
expiry=expiry_date)
return self.blob_service.make_blob_url(container_name, blob_name,
sas_token=sas)
def get_blob_content(self, container_name, blob_name):
out_stream = BytesIO()
self.blob_service.get_blob_to_stream(container_name,
blob_name, out_stream)
return out_stream
def create_empty_disk(self, disk_name, params):
return self.compute_client.disks.create_or_update(
self.resource_group,
disk_name,
params
).result()
def create_snapshot_disk(self, disk_name, params):
return self.compute_client.disks.create_or_update(
self.resource_group,
disk_name,
params
).result()
def get_disk(self, disk_id):
url_params = azure_helpers.parse_url(VOLUME_RESOURCE_ID,
disk_id)
disk_name = url_params.get(VOLUME_NAME)
return self.compute_client.disks.get(self.resource_group, disk_name)
def list_disks(self):
return self.compute_client.disks. \
list_by_resource_group(self.resource_group)
def delete_disk(self, disk_id):
url_params = azure_helpers.parse_url(VOLUME_RESOURCE_ID,
disk_id)
disk_name = url_params.get(VOLUME_NAME)
self.compute_client.disks.delete(self.resource_group, disk_name).wait()
def update_disk_tags(self, disk_id, tags):
url_params = azure_helpers.parse_url(VOLUME_RESOURCE_ID,
disk_id)
disk_name = url_params.get(VOLUME_NAME)
return self.compute_client.disks.update(
self.resource_group,
disk_name,
{'tags': tags},
raw=True
)
def list_snapshots(self):
return self.compute_client.snapshots. \
list_by_resource_group(self.resource_group)
def get_snapshot(self, snapshot_id):
url_params = azure_helpers.parse_url(SNAPSHOT_RESOURCE_ID,
snapshot_id)
snapshot_name = url_params.get(SNAPSHOT_NAME)
return self.compute_client.snapshots.get(self.resource_group,
snapshot_name)
def create_snapshot(self, snapshot_name, params):
return self.compute_client.snapshots.create_or_update(
self.resource_group,
snapshot_name,
params
).result()
def delete_snapshot(self, snapshot_id):
url_params = azure_helpers.parse_url(SNAPSHOT_RESOURCE_ID,
snapshot_id)
snapshot_name = url_params.get(SNAPSHOT_NAME)
self.compute_client.snapshots.delete(self.resource_group,
snapshot_name).wait()
def update_snapshot_tags(self, snapshot_id, tags):
url_params = azure_helpers.parse_url(SNAPSHOT_RESOURCE_ID,
snapshot_id)
snapshot_name = url_params.get(SNAPSHOT_NAME)
return self.compute_client.snapshots.update(
self.resource_group,
snapshot_name,
{'tags': tags},
raw=True
)
def is_gallery_image(self, image_id):
url_params = azure_helpers.parse_url(IMAGE_RESOURCE_ID,
image_id)
# If it is a gallery image, it will always have an offer
return 'offer' in url_params
def create_image(self, name, params):
return self.compute_client.images. \
create_or_update(self.resource_group, name,
params).result()
def delete_image(self, image_id):
url_params = azure_helpers.parse_url(IMAGE_RESOURCE_ID,
image_id)
if not self.is_gallery_image(image_id):
name = url_params.get(IMAGE_NAME)
self.compute_client.images.delete(self.resource_group, name).wait()
def list_images(self):
azure_images = list(self.compute_client.images.
list_by_resource_group(self.resource_group))
return azure_images
def list_gallery_refs(self):
return gallery_image_references
def get_image(self, image_id):
url_params = azure_helpers.parse_url(IMAGE_RESOURCE_ID,
image_id)
if self.is_gallery_image(image_id):
return GalleryImageReference(publisher=url_params['publisher'],
offer=url_params['offer'],
sku=url_params['sku'],
version=url_params['version'])
else:
name = url_params.get(IMAGE_NAME)
return self.compute_client.images.get(self.resource_group, name)
def update_image_tags(self, image_id, tags):
url_params = azure_helpers.parse_url(IMAGE_RESOURCE_ID,
image_id)
if self.is_gallery_image(image_id):
return True
else:
name = url_params.get(IMAGE_NAME)
return self.compute_client.images. \
create_or_update(self.resource_group, name,
{
'tags': tags,
'location': self.region_name
}).result()
def list_vm_types(self):
return self.compute_client.virtual_machine_sizes. \
list(self.region_name)
def list_networks(self):
return self.network_management_client.virtual_networks.list(
self.resource_group)
def get_network(self, network_id):
url_params = azure_helpers.parse_url(NETWORK_RESOURCE_ID,
network_id)
network_name = url_params.get(NETWORK_NAME)
return self.network_management_client.virtual_networks.get(
self.resource_group, network_name)
def create_network(self, name, params):
return self.network_management_client.virtual_networks. \
create_or_update(self.resource_group,
name,
parameters=params).result()
def delete_network(self, network_id):
url_params = azure_helpers.parse_url(NETWORK_RESOURCE_ID, network_id)
network_name = url_params.get(NETWORK_NAME)
return self.network_management_client.virtual_networks. \
delete(self.resource_group, network_name).wait()
def update_network_tags(self, network_id, tags):
url_params = azure_helpers.parse_url(NETWORK_RESOURCE_ID, network_id)
network_name = url_params.get(NETWORK_NAME)
return self.network_management_client.virtual_networks. \
create_or_update(self.resource_group,
network_name, tags).result()
def get_network_id_for_subnet(self, subnet_id):
url_params = azure_helpers.parse_url(SUBNET_RESOURCE_ID, subnet_id)
network_id = NETWORK_RESOURCE_ID[0]
for key, val in url_params.items():
network_id = network_id.replace("{" + key + "}", val)
return network_id
def list_subnets(self, network_id):
url_params = azure_helpers.parse_url(NETWORK_RESOURCE_ID, network_id)
network_name = url_params.get(NETWORK_NAME)
return self.network_management_client.subnets. \
list(self.resource_group, network_name)
def get_subnet(self, subnet_id):
url_params = azure_helpers.parse_url(SUBNET_RESOURCE_ID,
subnet_id)
network_name = url_params.get(NETWORK_NAME)
subnet_name = url_params.get(SUBNET_NAME)
return self.network_management_client.subnets. \
get(self.resource_group, network_name, subnet_name)
def create_subnet(self, network_id, subnet_name, params):
url_params = azure_helpers.parse_url(NETWORK_RESOURCE_ID, network_id)
network_name = url_params.get(NETWORK_NAME)
result_create = self.network_management_client \
.subnets.create_or_update(
self.resource_group,
network_name,
subnet_name,
params
)
subnet_info = result_create.result()
return subnet_info
def __if_subnet_in_use(e):
# return True if the CloudError exception is due to subnet being in use
if isinstance(e, CloudError):
if e.error.error == "InUseSubnetCannotBeDeleted":
return True
return False
@tenacity.retry(stop=tenacity.stop_after_attempt(5),
retry=tenacity.retry_if_exception(__if_subnet_in_use),
wait=tenacity.wait_fixed(5),
reraise=True)
def delete_subnet(self, subnet_id):
url_params = azure_helpers.parse_url(SUBNET_RESOURCE_ID,
subnet_id)
network_name = url_params.get(NETWORK_NAME)
subnet_name = url_params.get(SUBNET_NAME)
try:
result_delete = self.network_management_client \
.subnets.delete(
self.resource_group,
network_name,
subnet_name
)
result_delete.wait()
except CloudError as cloud_error:
log.exception(cloud_error.message)
raise cloud_error
def create_floating_ip(self, public_ip_name, public_ip_parameters):
return self.network_management_client.public_ip_addresses. \
create_or_update(self.resource_group,
public_ip_name,
public_ip_parameters).result()
def get_floating_ip(self, public_ip_id):
url_params = azure_helpers.parse_url(PUBLIC_IP_RESOURCE_ID,
public_ip_id)
public_ip_name = url_params.get(PUBLIC_IP_NAME)
return self.network_management_client. \
public_ip_addresses.get(self.resource_group, public_ip_name)
def delete_floating_ip(self, public_ip_id):
url_params = azure_helpers.parse_url(PUBLIC_IP_RESOURCE_ID,
public_ip_id)
public_ip_name = url_params.get(PUBLIC_IP_NAME)
self.network_management_client. \
public_ip_addresses.delete(self.resource_group,
public_ip_name).wait()
def update_fip_tags(self, fip_id, tags):
url_params = azure_helpers.parse_url(PUBLIC_IP_RESOURCE_ID,
fip_id)
fip_name = url_params.get(PUBLIC_IP_NAME)
self.network_management_client.public_ip_addresses. \
create_or_update(self.resource_group,
fip_name, tags).result()
def list_floating_ips(self):
return self.network_management_client.public_ip_addresses.list(
self.resource_group)
def list_vm(self):
return self.compute_client.virtual_machines.list(
self.resource_group
)
def restart_vm(self, vm_id):
url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
vm_id)
vm_name = url_params.get(VM_NAME)
return self.compute_client.virtual_machines.restart(
self.resource_group, vm_name).wait()
def delete_vm(self, vm_id):
url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
vm_id)
vm_name = url_params.get(VM_NAME)
return self.compute_client.virtual_machines.delete(
self.resource_group, vm_name).wait()
def get_vm(self, vm_id):
url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
vm_id)
vm_name = url_params.get(VM_NAME)
return self.compute_client.virtual_machines.get(
self.resource_group,
vm_name,
expand='instanceView'
)
def create_vm(self, vm_name, params):
return self.compute_client.virtual_machines. \
create_or_update(self.resource_group,
vm_name, params).result()
def update_vm(self, vm_id, params):
url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
vm_id)
vm_name = url_params.get(VM_NAME)
return self.compute_client.virtual_machines. \
create_or_update(self.resource_group,
vm_name, params, raw=True)
def deallocate_vm(self, vm_id):
url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
vm_id)
vm_name = url_params.get(VM_NAME)
self.compute_client. \
virtual_machines.deallocate(self.resource_group,
vm_name).wait()
def generalize_vm(self, vm_id):
url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
vm_id)
vm_name = url_params.get(VM_NAME)
self.compute_client.virtual_machines. \
generalize(self.resource_group, vm_name)
def start_vm(self, vm_id):
url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
vm_id)
vm_name = url_params.get(VM_NAME)
self.compute_client.virtual_machines. \
start(self.resource_group,
vm_name).wait()
def update_vm_tags(self, vm_id, tags):
url_params = azure_helpers.parse_url(VM_RESOURCE_ID,
vm_id)
vm_name = url_params.get(VM_NAME)
self.compute_client.virtual_machines. \
create_or_update(self.resource_group,
vm_name, tags).result()
def delete_nic(self, nic_id):
nic_params = azure_helpers.\
parse_url(NETWORK_INTERFACE_RESOURCE_ID, nic_id)
nic_name = nic_params.get(NETWORK_INTERFACE_NAME)
self.network_management_client. \
network_interfaces.delete(self.resource_group,
nic_name).wait()
def get_nic(self, nic_id):
nic_params = azure_helpers.\
parse_url(NETWORK_INTERFACE_RESOURCE_ID, nic_id)
nic_name = nic_params.get(NETWORK_INTERFACE_NAME)
return self.network_management_client. \
network_interfaces.get(self.resource_group, nic_name)
def update_nic(self, nic_id, params):
nic_params = azure_helpers.\
parse_url(NETWORK_INTERFACE_RESOURCE_ID, nic_id)
nic_name = nic_params.get(NETWORK_INTERFACE_NAME)
async_nic_creation = self.network_management_client. \
network_interfaces.create_or_update(
self.resource_group,
nic_name,
params
)
nic_info = async_nic_creation.result()
return nic_info
def create_nic(self, nic_name, params):
return self.network_management_client. \
network_interfaces.create_or_update(
self.resource_group,
nic_name,
params
).result()
def create_public_key(self, entity):
return self.table_service. \
insert_or_replace_entity(self.public_key_storage_table_name,
entity)
def get_public_key(self, name):
entities = self.table_service. \
query_entities(self.public_key_storage_table_name,
"Name eq '{0}'".format(name), num_results=1)
return entities.items[0] if len(entities.items) > 0 else None
def delete_public_key(self, entity):
self.table_service.delete_entity(self.public_key_storage_table_name,
entity.PartitionKey, entity.RowKey)
def list_public_keys(self, partition_key, limit=None, marker=None):
entities = self.table_service. \
query_entities(self.public_key_storage_table_name,
"PartitionKey eq '{0}'".format(partition_key),
marker=marker, num_results=limit)
return (entities.items, entities.next_marker)
def delete_route_table(self, route_table_name):
self.network_management_client. \
route_tables.delete(self.resource_group, route_table_name
).wait()
def attach_subnet_to_route_table(self, subnet_id, route_table_id):
url_params = azure_helpers.parse_url(SUBNET_RESOURCE_ID,
subnet_id)
network_name = url_params.get(NETWORK_NAME)
subnet_name = url_params.get(SUBNET_NAME)
subnet_info = self.network_management_client.subnets.get(
self.resource_group,
network_name,
subnet_name
)
if subnet_info:
subnet_info.route_table = {
'id': route_table_id
}
result_create = self.network_management_client. \
subnets.create_or_update(
self.resource_group,
network_name,
subnet_name,
subnet_info)
subnet_info = result_create.result()
return subnet_info
def detach_subnet_to_route_table(self, subnet_id, route_table_id):
url_params = azure_helpers.parse_url(SUBNET_RESOURCE_ID,
subnet_id)
network_name = url_params.get(NETWORK_NAME)
subnet_name = url_params.get(SUBNET_NAME)
subnet_info = self.network_management_client.subnets.get(
self.resource_group,
network_name,
subnet_name
)
if subnet_info and subnet_info.route_table.id == route_table_id:
subnet_info.route_table = None
result_create = self.network_management_client. \
subnets.create_or_update(
self.resource_group,
network_name,
subnet_name,
subnet_info)
subnet_info = result_create.result()
return subnet_info
def list_route_tables(self):
return self.network_management_client. \
route_tables.list(self.resource_group)
def get_route_table(self, router_id):
url_params = azure_helpers.parse_url(ROUTER_RESOURCE_ID,
router_id)
router_name = url_params.get(ROUTER_NAME)
return self.network_management_client. \
route_tables.get(self.resource_group, router_name)
def create_route_table(self, route_table_name, params):
return self.network_management_client. \
route_tables.create_or_update(
self.resource_group,
route_table_name, params).result()
def update_route_table_tags(self, route_table_name, tags):
self.network_management_client.route_tables. \
create_or_update(self.resource_group,
route_table_name, tags).result()
class DataControl:
def __init__(self):
self.table_service = TableService(account_name='toonestorage01', account_key='sIo3TKwG40eH2a9MpjZdGgWwetkGDV3NcgFhwZN2sFerhrj3kWLTiKQO3wGO7bd9sjmGoBnPl2CbqrIJcsnG8g==')
#table_service = TableService(connection_string='DefaultEndpointsProtocol=https;AccountName=toonestorage01;AccountKey=sIo3TKwG40eH2a9MpjZdGgWwetkGDV3NcgFhwZN2sFerhrj3kWLTiKQO3wGO7bd9sjmGoBnPl2CbqrIJcsnG8g==;EndpointSuffix=core.windows.net')
#SOURCE_TABLE = "stockday"
###########################################################################################################################
def set_table(self, tabletype, tooneday):
if tabletype == 'D':
table_name = 'stock' + tabletype + tooneday
#오늘 날짜 분석 테이블이 존재 하면 삭제 하고 새로 생성 / Azure Table은 삭제 후 재 생성 불가 (delay 발생)
print(table_name)
if self.table_service.exists(table_name) == False:
self.table_service.create_table(table_name)
print(">> table created : " + table_name )
else :
print(">> table exists " + table_name)
elif tabletype =='5M':
print(">>> 5M table")
return table_name
def set_target(self, targetdf):
prep_df = preprocess(targetdf)
print ( prep_df )
###########################################################################################################################
def set_stock_day(self, targettable, stockdf):
# print('>> insert data to azure table')
'''
for data in stockdata:
task = {'PartitionKey': 'tasksSeattle', 'RowKey': '001','description': 'Take out the trash', 'priority': 200}
self.table_service.insert_entity('stockday', task)
'''
# dataframe에서 partitionkey 및 row_id 로 칼럼명 변경 필요
# key 값을 두개만 가질 수 있음
# particionkey = code / date = row_id
#print (stockdf.head())
stockdf_table = stockdf.rename(columns={"code": "PartitionKey", "date": "RowKey"})
#print (stockdf_table)
for index, row in stockdf_table.iterrows():
#print(row)
#print(row['PartitionKey'])
#print(">> start row")
#print(row)
task = Entity()
task.PartitionKey = row.to_dict()['PartitionKey']
task.RowKey = str(row.to_dict()['RowKey'])
task.open = row.to_dict()['open']
task.high = row.to_dict()['high']
task.low = row.to_dict()['low']
task.close = row.to_dict()['close']
task.volume = row.to_dict()['volume']
self.table_service.insert_or_merge_entity(targettable, task, timeout=None)
#print('>> end set stockday')
def get_stock_day(self, code):
print('>> start get data ')
# 여러 데이타 추출
target_table = 'stockD' + time.strftime('%Y%m%d')
filter_target = "PartitionKey eq '" + code+ "'"
rows = self.table_service.query_entities( target_table , filter=filter_target, select='PartitionKey,RowKey,open,high,low,volume,close')
df_stock_day = pd.DataFrame(rows)
#print(df_stock_day.head())
return df_stock_day
def set_stock_min(self, stockdf):
# print('>> insert data to azure table')
#print (stockdf.head())
stockdf_table = stockdf.rename(columns={"code": "PartitionKey", "date": "RowKey"})
#print (stockdf_table)
stockdf_table = stockdf_table.astype({"time":str, "RowKey": str})
# date column을 time 컬럼과 합성
if len(str(stockdf_table["time"])) <= 3:
stockdf_table["time"] = '0' + str(stockdf_table["time"])
print(stockdf_table.head())
stockdf_table["RowKey"] = stockdf_table["RowKey"] + stockdf_table["time"]
print(stockdf_table.head())
stockdf_last = pd.DataFrame()
stockdf_last = stockdf_table[ ['PartitionKey', 'RowKey', 'time', 'open', 'high', 'low', 'close', 'volume'] ]
print(stockdf_last)
for index, row in stockdf_last.iterrows():
task = Entity()
task.PartitionKey = row.to_dict()['PartitionKey']
task.RowKey = str(row.to_dict()['RowKey'])
task.time = row.to_dict()['time']
task.open = row.to_dict()['open']
task.high = row.to_dict()['high']
task.low = row.to_dict()['low']
task.close = row.to_dict()['close']
task.volume = row.to_dict()['volume']
#print(task)
self.table_service.insert_or_merge_entity('stockM', task, timeout=None)
###########################################################################################################################
def get_max_date(self, stockcode):
filter_str = "PartitionKey eq '" + stockcode +"'"
rows = self.table_service.query_entities( 'stockday', filter=filter_str, select='open,close')
#print (rows.RowKey)
for row in rows:
print (row)
return row
def get_max_time(self, stockcode):
filter_str = "RowKey eq '" + stockcode +"'"
rows = self.table_service.query_entities( 'stockM', filter=filter_str, select='open,close')
#print (rows.RowKey)
for row in rows:
print (row)
return row
def set_target_stock(self,df_target):
# ['price','volume', 'per','eps']
df_target["date"]=time.strftime('%Y%m%d')
stockdf_table = df_target.rename(columns={"date": "PartitionKey", "code": "RowKey"})
for index, row in stockdf_table.iterrows():
#print(row)
#print(row['PartitionKey'])
#print(">> start row")
#print(row)
task = Entity()
task.PartitionKey = row.to_dict()['PartitionKey']
task.RowKey = str(row.to_dict()['RowKey'])
task.price = row.to_dict()['price']
task.volume = row.to_dict()['volume']
task.per = row.to_dict()['per']
task.eps = row.to_dict()['eps']
self.table_service.insert_or_merge_entity('stocktarget', task)
print(">> set target stock..." + str(row.to_dict()['RowKey']) )
def get_target_stock(self):
filter_target = "PartitionKey eq '" + time.strftime('%Y%m%d')+ "'"
rows = self.table_service.query_entities( 'stocktarget', filter= filter_target, select='RowKey, status')
#print (rows.RowKey)
df_target = pd.DataFrame(rows)
return df_target
