def test_sas_add(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
token = self.ts.generate_table_shared_access_signature(
self.table_name,
TablePermissions.ADD,
datetime.utcnow() + timedelta(hours=1),
datetime.utcnow() - timedelta(minutes=1),
)
# Act
service = TableService(
account_name=self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
)
self._set_test_proxy(service, self.settings)
entity = self._create_random_entity_dict()
service.insert_entity(self.table_name, entity)
# Assert
resp = self.ts.get_entity(self.table_name, entity['PartitionKey'],
entity['RowKey'])
self._assert_default_entity(resp)
def test_sas_add_outside_range(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
token = self.ts.generate_table_shared_access_signature(
self.table_name,
TablePermissions.ADD,
datetime.utcnow() + timedelta(hours=1),
start_pk='test',
start_rk='test1',
end_pk='test',
end_rk='test1',
)
# Act
service = TableService(
account_name=self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
)
self._set_test_proxy(service, self.settings)
with self.assertRaises(AzureHttpError):
entity = self._create_random_entity_dict()
service.insert_entity(self.table_name, entity)
def process_event(table_client: azuretable.TableService, table_name: str,
source: str, event: str, ts: float, message: str) -> None:
"""Process event
:param azure.cosmosdb.table.TableService table_client: table client
:param str table_name: table name
:param str source: source
:param str event: event
:param float ts: time stamp
:param str message: message
"""
entity = {
'PartitionKey': _PARTITION_KEY,
'RowKey': str(ts),
'Event': '{}:{}'.format(source, event),
'NodeId': _NODEID,
'Message': message,
}
while True:
try:
table_client.insert_entity(table_name, entity)
break
except azure.common.AzureConflictHttpError:
if not isinstance(ts, float):
ts = float(ts)
ts += 0.000001
entity['RowKey'] = str(ts)
def _merge_resource(table_client: azuretable.TableService, resource: str,
nglobalresources: int) -> None:
"""Merge resource to the image table
:param azuretable.TableService table_client: table client
:param str resource: resource to add to the image table
:param int nglobalresources: number of global resources
"""
# merge resource to the image table
entity = {
'PartitionKey': _PARTITION_KEY,
'RowKey': compute_resource_hash(resource),
'Resource': resource,
'VmList0': _NODEID,
}
logger.debug('merging entity {} to the image table'.format(entity))
try:
table_client.insert_entity(_STORAGE_CONTAINERS['table_images'],
entity=entity)
except azure.common.AzureConflictHttpError:
while True:
entity = table_client.get_entity(
_STORAGE_CONTAINERS['table_images'], entity['PartitionKey'],
entity['RowKey'])
# merge VmList into entity
evms = []
for i in range(0, _MAX_VMLIST_PROPERTIES):
prop = 'VmList{}'.format(i)
if prop in entity:
evms.extend(entity[prop].split(','))
if _NODEID in evms:
break
evms.append(_NODEID)
for i in range(0, _MAX_VMLIST_PROPERTIES):
prop = 'VmList{}'.format(i)
start = i * _MAX_VMLIST_IDS_PER_PROPERTY
end = start + _MAX_VMLIST_IDS_PER_PROPERTY
if end > len(evms):
end = len(evms)
if start < end:
entity[prop] = ','.join(evms[start:end])
else:
entity[prop] = None
etag = entity['etag']
entity.pop('etag')
try:
table_client.merge_entity(_STORAGE_CONTAINERS['table_images'],
entity=entity,
if_match=etag)
break
except azure.common.AzureHttpError as ex:
if ex.status_code != 412:
raise
logger.info('entity {} merged to the image table'.format(entity))
global _GR_DONE
if not _GR_DONE:
try:
entities = table_client.query_entities(
_STORAGE_CONTAINERS['table_images'],
filter='PartitionKey eq \'{}\''.format(_PARTITION_KEY))
except azure.common.AzureMissingResourceHttpError:
entities = []
count = 0
for entity in entities:
for i in range(0, _MAX_VMLIST_PROPERTIES):
prop = 'VmList{}'.format(i)
mode_prefix = _CONTAINER_MODE.name.lower() + ':'
if (prop in entity and _NODEID in entity[prop]
and entity['Resource'].startswith(mode_prefix)):
count += 1
if count == nglobalresources:
_record_perf('gr-done',
'nglobalresources={}'.format(nglobalresources))
_GR_DONE = True
logger.info(
'all {} global resources of container mode "{}" loaded'.format(
nglobalresources, _CONTAINER_MODE.name.lower()))
else:
logger.info(
'{}/{} global resources of container mode "{}" loaded'.format(
count, nglobalresources, _CONTAINER_MODE.name.lower()))
class Azure:
# Tags used
RG_RULE_PROGRAMMED_TAG = 'PANORAMA_PROGRAMMED'
HUB_MANAGED_TAG = 'PanoramaManaged'
# Resource types
VMSS_TYPE = 'Microsoft.Compute/virtualMachineScaleSets'
ILB_TYPE = 'Microsoft.Network/loadBalancers'
APPINSIGHTS_TYPE = 'Microsoft.Insights/components'
# Hardcoded names used for internal Azure resources
ILB_NAME = 'myPrivateLB'
ALPHANUM = r'[^A-Za-z0-9]+'
def __init__(self,
cred,
subs_id,
hub,
vmss_rg_name,
vmss_name,
storage,
pan_handle,
logger=None):
self.credentials = cred
self.subscription_id = subs_id
self.logger = logger
self.hub_name = hub
self.storage_name = storage
self.panorama_handler = pan_handle
self.vmss_table_name = re.sub(self.ALPHANUM, '',
vmss_name + 'vmsstable')
self.vmss_rg_name = vmss_rg_name
try:
self.resource_client = ResourceManagementClient(cred, subs_id)
self.compute_client = ComputeManagementClient(cred, subs_id)
self.network_client = NetworkManagementClient(cred, subs_id)
self.store_client = StorageManagementClient(cred, subs_id)
store_keys = self.store_client.storage_accounts.list_keys(
hub, storage).keys[0].value
self.table_service = TableService(account_name=storage,
account_key=store_keys)
except Exception as e:
self.logger.error("Getting Azure Infra handlers failed %s" %
str(e))
raise e
# Start -> List out all RGs and identify new spokes to mark them with tags.
# Look for Resource Groups (RGs) which do not have tags or does not have a
# a tag named "PANORAMA_PROGRAMMED".
# potential_new_spokes = [x.name for x in self.resource_client.resource_groups.list()\
# if not x.tags or not x.tags.get(self.RG_RULE_PROGRAMMED_TAG, None)]
# If the RG has a VMSS which has a tag named "PanoramaManaged" with a value
# as Hub Resource Group name then we know that this is a new spoke that is
# launched managed by the Hub and not yet programmed for NAT/Azure Instrumentation
# key.
# for rg in potential_new_spokes:
# fw_vm_list = [x for x in self.resource_client.resources.list_by_resource_group(rg)
# if x.type == self.VMSS_TYPE and self.filter_vmss(rg, x.name)]
# if fw_vm_list:
# rg_params = {'location': self.resource_client.resource_groups.get(rg).location}
# rg_params.update(tags={
# self.RG_RULE_PROGRAMMED_TAG : 'No',
# self.HUB_MANAGED_TAG : self.hub_name
# })
# self.resource_client.resource_groups.create_or_update(rg, rg_params)
# self.logger.info("RG %s marked as a spoke managed by this hub %s" % (rg, self.hub_name))
# End -> List out all RGs and identify new spokes to mark them with tags.
# Populate the list of spokes managed by this Azure hub.
rg_list = self.resource_client.resource_groups.list()
self.managed_spokes = []
self.managed_spokes.append(vmss_rg_name)
self.new_spokes = []
# for rg in rg_list:
# if rg.tags and rg.tags.get(self.HUB_MANAGED_TAG, None) == self.hub_name:
# self.managed_spokes.append(rg.name)
# if rg.tags.get(self.RG_RULE_PROGRAMMED_TAG, 'Yes') == 'No':
# self.new_spokes.append(rg.name)
# self.logger.debug('%s identified as spokes managed by %s' % (self.managed_spokes, self.hub_name))
# if self.new_spokes:
# self.logger.info('%s identified as new spokes to be programmed by %s' % (self.new_spokes, self.hub_name))
#
#
def filter_vmss(self, spoke, vmss_name):
vmss = self.compute_client.virtual_machine_scale_sets.get(
spoke, vmss_name)
if vmss.tags and vmss.tags.get(self.HUB_MANAGED_TAG,
None) == self.hub_name:
return True
return False
def get_ilb_ip(self, spoke):
for resource in self.resource_client.resources.list_by_resource_group(
spoke):
# Get the ILB IP Address from the spoke. The ILB address is always
# hardcoded to be myPrivateILB.
if resource.name == self.ILB_NAME and resource.type == self.ILB_TYPE:
ilb_obj = self.network_client.load_balancers.get(
spoke, resource.name)
ilb_frontend_cfg = ilb_obj.frontend_ip_configurations
try:
ilb_private_ip = ilb_frontend_cfg[0].private_ip_address
except IndexError as e:
self.logger.info("ILB is not setup yet in RG %s." % spoke)
return None
return ilb_private_ip
return None
def get_appinsights_instr_key(self, spoke):
for resource in self.resource_client.resources.list_by_resource_group(
spoke):
# Get the Appinsights instance where the custom metrics are being
# published.
if resource.type == self.APPINSIGHTS_TYPE and 'appinsights' in resource.name:
appinsights_obj = self.resource_client.resources.get_by_id(
resource.id, '2014-04-01')
instr_key = appinsights_obj.properties.get(
'InstrumentationKey', '')
if not instr_key:
self.logger.info("InstrKey is not setup yet in %s." %
spoke)
return None
return instr_key
return None
def set_spoke_as_programmed(self, spoke):
spoke_params = {
'location':
self.resource_client.resource_groups.get(spoke).location
}
spoke_tags = self.resource_client.resource_groups.get(spoke).tags
spoke_tags[self.RG_RULE_PROGRAMMED_TAG] = 'Yes'
spoke_params.update(tags=spoke_tags)
self.resource_client.resource_groups.create_or_update(
spoke, spoke_params)
self.logger.info(
"RG %s marked as programmed and spoke managed by this hub %s" %
(spoke, self.hub_name))
def create_worker_ready_tag(self, worker_name):
self.compute_client.virtual_machines.create_or_update(
self.vmss_rg, worker_name, {
'location':
self.resource_client.resource_groups.get(
self.vmss_rg).location,
'tags': {
'WORKER_READY': 'Yes'
}
})
def create_new_cosmos_table(self, table_name):
# Create the Cosmos DB if it does not exist already
if not self.table_service.exists(table_name):
try:
ok = self.table_service.create_table(table_name)
if not ok:
self.logger.error('Creating VMSS table failed')
return False
self.logger.info('VMSS Table %s created succesfully' %
table_name)
except Exception as e:
self.logger.error('Creating VMSS table failed ' + str(e))
return False
return True
def clear_cosmos_table(self, table_name):
self.table_service.delete_table(table_name)
def get_vmss_by_name(self, spoke, vmss_name):
vmss_list = [
x.name for x in
self.resource_client.resources.list_by_resource_group(spoke)
if x.type == self.VMSS_TYPE and x.name == vmss_name
]
if vmss_list:
return vmss_list[0]
else:
self.logger.error("No VMSS found in Resource Group %s" % spoke)
return None
def get_vmss_in_spoke(self, spoke):
vmss_list = [
x.name for x in
self.resource_client.resources.list_by_resource_group(spoke)
if x.type == self.VMSS_TYPE and self.filter_vmss(spoke, x.name)
]
if vmss_list:
return vmss_list[0]
else:
self.logger.error("No VMSS found in Resource Group %s" % spoke)
return None
def get_vms_in_vmss(self, spoke, vmss_name):
return self.compute_client.virtual_machine_scale_set_vms.list(
spoke, vmss_name)
def get_vm_in_cosmos_db(self, spoke, vm_hostname):
try:
db_vm_info = self.table_service.get_entity(self.vmss_table_name,
spoke, vm_hostname)
except AzureMissingResourceHttpError:
self.logger.info("New VM %s found in spoke %s" %
(vm_hostname, spoke))
return None
except Exception as e:
self.logger.error("Querying for %s failed" % vm_hostname)
return None
else:
# IF possible update status TODO
self.logger.debug("VM %s is available in VMSS, Pan and DB" %
(vm_hostname))
return db_vm_info
# 'name' : global_device['@name'],
# 'hostname' : global_device['hostname'],
# 'serial' : global_device['serial'],
# 'ip-address' : global_device['ip-address'],
# 'connected' : global_device['connected'],
# 'deactivated': global_device['deactivated']
def create_db_entity(self, spoke, vm_details):
vm = Entity()
# PartitionKey is nothing but the spoke name
vm.PartitionKey = spoke
# RowKey is nothing but the VM name itself.
vm.RowKey = vm_details['hostname']
vm.name = vm_details['name']
vm.serial_no = vm_details['serial']
vm.ip_addr = vm_details['ip-address']
vm.connected = vm_details['connected']
vm.deactivated = vm_details['deactivated']
vm.subs_id = self.subscription_id
vm.delicensed_on = 'not applicable'
vm.is_delicensed = 'No'
try:
self.table_service.insert_entity(self.vmss_table_name, vm)
self.logger.info("VM %s with serial no. %s in db" %
(vm_details['hostname'], vm_details['serial']))
except Exception as e:
self.logger.info("Insert entry to db for %s failed with error %s" %
(vm_details['hostname'], e))
return False
return True
def get_fw_vms_in_cosmos_db(self, spoke=None):
if spoke:
filter_str = "PartitionKey eq '%s'" % spoke
else:
filter_str = None
db_vms_list = self.table_service.query_entities(self.vmss_table_name,
filter=filter_str)
if spoke:
db_hostname_list = [{'hostname': x.RowKey, 'serial': x.serial_no, 'name': x.name} \
for x in db_vms_list if x.PartitionKey == spoke]
return db_hostname_list
else:
return db_vms_list
def delete_vm_from_cosmos_db(self, spoke, vm_name):
self.table_service.delete_entity(self.vmss_table_name, spoke, vm_name)
class Azure:
# Tags used
RG_RULE_PROGRAMMED_TAG = 'PANORAMA_PROGRAMMED'
HUB_MANAGED_TAG = 'PanoramaManaged'
# Resource types
VMSS_TYPE = 'Microsoft.Compute/virtualMachineScaleSets'
ILB_TYPE = 'Microsoft.Network/loadBalancers'
APPINSIGHTS_TYPE = 'Microsoft.Insights/components'
# Hardcoded names used for internal Azure resources
ILB_NAME = 'myPrivateLB'
ALPHANUM = r'[^A-Za-z0-9]+'
def __init__(self,
cred,
subs_id,
my_storage_rg,
vmss_rg_name,
vmss_name,
storage,
pan_handle,
logger=None):
self.credentials = cred
self.subscription_id = subs_id
self.logger = logger
self.hub_name = vmss_rg_name
self.storage_name = storage
self.panorama_handler = pan_handle
self.vmss_table_name = re.sub(self.ALPHANUM, '',
vmss_name + 'vmsstable')
self.vmss_rg_name = vmss_rg_name
try:
self.resource_client = ResourceManagementClient(cred, subs_id)
self.compute_client = ComputeManagementClient(cred, subs_id)
self.network_client = NetworkManagementClient(cred, subs_id)
self.store_client = StorageManagementClient(cred, subs_id)
store_keys = self.store_client.storage_accounts.list_keys(
my_storage_rg, storage).keys[0].value
self.table_service = TableService(account_name=storage,
account_key=store_keys)
except Exception as e:
self.logger.error("Getting Azure Infra handlers failed %s" %
str(e))
raise e
rg_list = self.resource_client.resource_groups.list()
self.managed_spokes = []
self.managed_spokes.append(vmss_rg_name)
self.new_spokes = []
def filter_vmss(self, spoke, vmss_name):
vmss = self.compute_client.virtual_machine_scale_sets.get(
spoke, vmss_name)
if vmss.tags and vmss.tags.get(self.HUB_MANAGED_TAG,
None) == self.hub_name:
return True
return False
def create_worker_ready_tag(self, worker_name):
self.compute_client.virtual_machines.create_or_update(
self.vmss_rg, worker_name, {
'location':
self.resource_client.resource_groups.get(
self.vmss_rg).location,
'tags': {
'WORKER_READY': 'Yes'
}
})
def create_new_cosmos_table(self, table_name):
# Create the Cosmos DB if it does not exist already
if not self.table_service.exists(table_name):
try:
ok = self.table_service.create_table(table_name)
if not ok:
self.logger.error('Creating VMSS table failed')
return False
self.logger.info('VMSS Table %s created succesfully' %
table_name)
except Exception as e:
self.logger.error('Creating VMSS table failed ' + str(e))
return False
return True
def clear_cosmos_table(self, table_name):
self.table_service.delete_table(table_name)
def get_vmss_by_name(self, spoke, vmss_name):
vmss_list = [
x.name for x in
self.resource_client.resources.list_by_resource_group(spoke)
if x.type == self.VMSS_TYPE and x.name == vmss_name
]
if vmss_list:
return vmss_list[0]
else:
self.logger.error("No VMSS found in Resource Group %s" % spoke)
return None
def get_vmss_in_spoke(self, spoke):
vmss_list = [
x.name for x in
self.resource_client.resources.list_by_resource_group(spoke)
if x.type == self.VMSS_TYPE and self.filter_vmss(spoke, x.name)
]
if vmss_list:
return vmss_list[0]
else:
self.logger.error("No VMSS found in Resource Group %s" % spoke)
return None
def get_vms_in_vmss(self, spoke, vmss_name):
return self.compute_client.virtual_machine_scale_set_vms.list(
spoke, vmss_name)
def get_vm_in_cosmos_db(self, spoke, vm_hostname):
try:
db_vm_info = self.table_service.get_entity(self.vmss_table_name,
spoke, vm_hostname)
except AzureMissingResourceHttpError:
self.logger.info("New VM %s found in spoke %s" %
(vm_hostname, spoke))
return None
except Exception as e:
self.logger.error("Querying for %s failed" % vm_hostname)
return None
else:
# IF possible update status TODO
self.logger.debug("VM %s is available in VMSS, Pan and DB" %
(vm_hostname))
return db_vm_info
# 'name' : global_device['@name'],
# 'hostname' : global_device['hostname'],
# 'serial' : global_device['serial'],
# 'ip-address' : global_device['ip-address'],
# 'connected' : global_device['connected'],
# 'deactivated': global_device['deactivated']
def create_db_entity(self, spoke, vm_details):
vm = Entity()
# PartitionKey is nothing but the spoke name
vm.PartitionKey = spoke
# RowKey is nothing but the VM name itself.
vm.RowKey = vm_details['hostname']
vm.name = vm_details['name']
vm.serial_no = vm_details['serial']
vm.ip_addr = vm_details['ip-address']
vm.connected = vm_details['connected']
vm.deactivated = vm_details['deactivated']
vm.subs_id = self.subscription_id
vm.delicensed_on = 'not applicable'
vm.is_delicensed = 'No'
try:
self.table_service.insert_entity(self.vmss_table_name, vm)
self.logger.info("VM %s with serial no. %s in db" %
(vm_details['hostname'], vm_details['serial']))
except Exception as e:
self.logger.info("Insert entry to db for %s failed with error %s" %
(vm_details['hostname'], e))
return False
return True
def get_fw_vms_in_cosmos_db(self, spoke=None):
if spoke:
filter_str = "PartitionKey eq '%s'" % spoke
else:
filter_str = None
db_vms_list = self.table_service.query_entities(self.vmss_table_name,
filter=filter_str)
if spoke:
db_hostname_list = [{'hostname': x.RowKey, 'serial': x.serial_no, 'name': x.name} \
for x in db_vms_list if x.PartitionKey == spoke]
return db_hostname_list
else:
return db_vms_list
def delete_vm_from_cosmos_db(self, spoke, vm_name):
self.table_service.delete_entity(self.vmss_table_name, spoke, vm_name)
from azure.cosmosdb.table.models import Entity
tablename = 'tasktable'
task = {
'PartitionKey': 'tasksSeattle',
'RowKey': '001',
'description': 'Take out the trash',
'priority': 200
}
#Blob Storage - Table Service
print("Demo / Blob Storage - Table Service ... Start")
Blob_table_service = TableService(account_name='BlobStorageAccountName',
account_key='BlobStorageAccountKey')
Blob_table_service.create_table(tablename)
Blob_table_service.insert_entity(tablename, task)
# get_table_service_stats()
# Retrieves statistics related to replication for the Table service.
# IMPORTANT NOTE:
# It is only available when read-access geo-redundant replication is enabled for the storage account.
# ref: https://docs.microsoft.com/en-us/python/api/azure-cosmosdb-table/azure.cosmosdb.table.tableservice.tableservice?view=azure-python#get-table-service-stats-timeout-none-
Blob_table_service.get_table_service_stats()
print("Demo / Blob Storage - Table Service ... End")
#Cosmos DB - Table API
print("Demo / Cosmos DB - Table API ... Start")
connstring = 'DefaultEndpointsProtocol=https;AccountName=CosmosDBAccountName;AccountKey=CosmosDBAccountKey;TableEndpoint=https://CosmosDBAccountName.table.cosmos.azure.com:443/;'
Cosmos_table_service = TableService(endpoint_suffix="table.cosmos.azure.com",
connection_string=connstring)
Cosmos_table_service.create_table(tablename)
Cosmos_table_service.insert_entity(tablename, task)
