def table_sas(self): table_name = self._create_table() entity = { 'PartitionKey': 'test', 'RowKey': 'test1', 'text': 'hello world', } self.service.insert_entity(table_name, entity) # Access only to the entities in the given table # Query permissions to access entities # Expires in an hour token = self.service.generate_table_shared_access_signature( table_name, TablePermissions.QUERY, datetime.utcnow() + timedelta(hours=1), ) # Create a service and use the SAS sas_service = TableService( account_name=self.account.account_name, sas_token=token, ) entities = sas_service.query_entities(table_name) for entity in entities: print(entity.text) # hello world self.service.delete_table(table_name)
def account_sas(self): table_name = self._create_table() entity = { 'PartitionKey': 'test', 'RowKey': 'test1', 'text': 'hello world', } self.service.insert_entity(table_name, entity) # Access to all entities in all the tables # Expires in an hour token = self.service.generate_account_shared_access_signature( ResourceTypes.OBJECT, AccountPermissions.READ, datetime.utcnow() + timedelta(hours=1), ) # Create a service and use the SAS sas_service = TableService( account_name=self.account.account_name, sas_token=token, ) entities = list(sas_service.query_entities(table_name)) for entity in entities: print(entity.text) # hello world self.service.delete_table(table_name)
def test_logging(self): # key to identify this session test_id = str(uuid4()) # setup logger logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) # setup handler and add to logger handler = AzureTableStorageHandler( account_name=account_name, account_key=account_key, table_name=table_name ) logger.addHandler(handler) # write logs logger.debug(f"DEBUG: {test_id}") logger.info(f"INFO: {test_id}") logger.warning(f"WARNING: {test_id}") logger.error(f"ERROR: {test_id}") logger.critical(f"CRITICAL: {test_id}") # get log messages ts = TableService(account_name=account_name, account_key=account_key) for ent in ts.query_entities(table_name=table_name, filter="PartitionKey eq '__main__'"): self.assertEqual(ent["LevelName"] + ": " + test_id, ent["Message"])
def sas_with_signed_identifiers(self): table_name = self._create_table() entity = { 'PartitionKey': 'test', 'RowKey': 'test1', 'text': 'hello world', } self.service.insert_entity(table_name, entity) # Set access policy on table access_policy = AccessPolicy(permission=TablePermissions.QUERY, expiry=datetime.utcnow() + timedelta(hours=1)) identifiers = {'id': access_policy} acl = self.service.set_table_acl(table_name, identifiers) # Wait 30 seconds for acl to propagate time.sleep(30) # Indicates to use the access policy set on the table token = self.service.generate_table_shared_access_signature(table_name, id='id') # Create a service and use the SAS sas_service = TableService( account_name=self.account.account_name, sas_token=token, ) entities = list(sas_service.query_entities(table_name)) for entity in entities: print(entity.text) # hello world self.service.delete_table(table_name)
def test_sas_upper_case_table_name(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange entity = self._insert_random_entity() # Table names are case insensitive, so simply upper case our existing table name to test token = self.ts.generate_table_shared_access_signature( self.table_name.upper(), TablePermissions.QUERY, datetime.utcnow() + timedelta(hours=1), datetime.utcnow() - timedelta(minutes=1), ) # Act service = TableService( account_name=self.settings.STORAGE_ACCOUNT_NAME, sas_token=token, ) self._set_test_proxy(service, self.settings) entities = list( service.query_entities(self.table_name, filter="PartitionKey eq '{}'".format( entity['PartitionKey']))) # Assert self.assertEqual(len(entities), 1) self._assert_default_entity(entities[0])
class AzureTableConnection: def __init__(self, tableName): self.tableName = tableName self.tableService = TableService( account_name=os.environ['STORAGEACCOUNTNAME'], account_key=os.environ['STORAGEACCOUNTKEY']) def commitBatch(self, batch): self.tableService.commit_batch(self.tableName, batch) def getData(self, partitionKey, rowKey): startRowKey = '{0}_0'.format(rowKey) endRowKey = '{0}_9999'.format(rowKey) filterExpression = "PartitionKey eq '{0}' and \ RowKey gt '{1}' and \ RowKey lt '{2}'" \ .format(partitionKey, startRowKey, endRowKey) return self.tableService.query_entities(self.tableName, filter=filterExpression)
def _unmerge_resources(table_client: azuretable.TableService) -> None: """Remove node from the image table :param azuretable.TableService table_client: table client """ logger.debug( 'removing node {} from the image table for container mode {}'.format( _NODEID, _CONTAINER_MODE.name.lower())) try: entities = table_client.query_entities( _STORAGE_CONTAINERS['table_images'], filter='PartitionKey eq \'{}\''.format(_PARTITION_KEY)) except azure.common.AzureMissingResourceHttpError: entities = [] mode_prefix = _CONTAINER_MODE.name.lower() + ':' for entity in entities: if entity['Resource'].startswith(mode_prefix): _unmerge_resource(table_client, entity) logger.info( 'node {} removed from the image table for container mode {}'.format( _NODEID, _CONTAINER_MODE.name.lower()))
def distribute_global_resources(loop: asyncio.BaseEventLoop, blob_client: azureblob.BlockBlobService, table_client: azuretable.TableService) -> None: """Distribute global services/resources :param asyncio.BaseEventLoop loop: event loop :param azureblob.BlockBlobService blob_client: blob client :param azuretable.TableService table_client: table client """ # remove node from the image table because cascade relies on it to know # when its work is done _unmerge_resources(table_client) # get globalresources from table try: entities = table_client.query_entities( _STORAGE_CONTAINERS['table_globalresources'], filter='PartitionKey eq \'{}\''.format(_PARTITION_KEY)) except azure.common.AzureMissingResourceHttpError: entities = [] nentities = 0 for ent in entities: resource = ent['Resource'] grtype, image = get_container_image_name_from_resource(resource) if grtype == _CONTAINER_MODE.name.lower(): nentities += 1 _DIRECTDL_QUEUE.put(resource) key_fingerprint = ent.get('KeyFingerprint', None) if key_fingerprint is not None: _DIRECTDL_KEY_FINGERPRINT_DICT[image] = key_fingerprint else: logger.info('skipping resource {}:'.format(resource) + 'not matching container mode "{}"'.format( _CONTAINER_MODE.name.lower())) if nentities == 0: logger.info('no global resources specified') return logger.info('{} global resources matching container mode "{}"'.format( nentities, _CONTAINER_MODE.name.lower())) # run async func in loop loop.run_until_complete( download_monitor_async(loop, blob_client, table_client, nentities))
class AzureStorage(): def __init__(self, container=None): self.AZURE_STORAGE_ACCOUNT = 'logodetectionstorage' self.AZURE_STORAGE_KEY = 'jPJyzct+8WD1lKU5M+ZwDflWUGRu+YBpH8n/3Z6qR7WD7uc3HV2U1rtiQKesLRq2tU3jtXIe26RklAYdKzoydA==' self.table_service = TableService(account_name=self.AZURE_STORAGE_ACCOUNT, account_key=self.AZURE_STORAGE_KEY) self.blob_service = BlockBlobService(account_name=self. AZURE_STORAGE_ACCOUNT, account_key=self.AZURE_STORAGE_KEY) self.container = "input" self.table_list = [] #everything in the table for this logo self.logo = "" def query(self, tableName, partitionKey, rowKey): task = self.table_service.get_entity(tableName, partitionKey, rowKey) return task def retrieve_table(self, tableName): #tasks = table_service.query_entities(tableName, filter="PartitionKey eq 'tasksSeattle'", select='description') try: tasks = self.table_service.query_entities(tableName) except: return None self.logo = tableName for task in tasks: self.table_list.append(task) self.table_list = sorted(self.table_list, key=lambda k: k['has_logo'], reverse=True) return self.table_list def download_blob(self, path, logoName): #download pic into logoName file path = "images/" + logoName self.blob_service.get_blob_to_path(self.container, path, "test.jpeg") def exists(self, name): try: self.blob_service.get_blob_properties(self.container, name) return True except: return False
def test_account_sas(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange table_name = self._create_table() entity = { 'PartitionKey': 'test', 'RowKey': 'test1', 'text': 'hello', } self.ts.insert_entity(table_name, entity) entity['RowKey'] = 'test2' self.ts.insert_entity(table_name, entity) token = self.ts.generate_account_shared_access_signature( ResourceTypes.OBJECT, AccountPermissions.READ, datetime.utcnow() + timedelta(hours=1), datetime.utcnow() - timedelta(minutes=1), ) # Act service = TableService( account_name=self.settings.STORAGE_ACCOUNT_NAME, sas_token=token, ) self._set_test_proxy(service, self.settings) entities = list(service.query_entities(table_name)) # Assert self.assertEqual(len(entities), 2) self.assertEqual(entities[0].text, 'hello') self.assertEqual(entities[1].text, 'hello')
def coalesce_data(table_client: azuretable.TableService) -> tuple: """Coalesce perf data from table :param azure.cosmosdb.table.TableService table_client: table client :rtype: tuple :return: (timing, sizes, offer, sku) """ print('graphing data from {} with pk={}'.format(_TABLE_NAME, _PARTITION_KEY)) entities = table_client.query_entities( _TABLE_NAME, filter='PartitionKey eq \'{}\''.format(_PARTITION_KEY)) data = {} # process events for ent in entities: nodeid = ent['NodeId'] event = ent['Event'] if nodeid not in data: data[nodeid] = {} if event not in data[nodeid]: data[nodeid][event] = [] ev = { 'timestamp': datetime.datetime.fromtimestamp(float(ent['RowKey'])), } try: ev['message'] = _parse_message(event, ent['Message']) except KeyError: ev['message'] = None data[nodeid][event].append(ev) del entities sizes = {} offer = None sku = None for nodeid in data: if offer is None: offer = data[nodeid]['nodeprep:start'][0]['message']['offer'] sku = data[nodeid]['nodeprep:start'][0]['message']['sku'] # calculate dt timings timing = { 'nodeprep': _compute_delta_t(data, nodeid, 'nodeprep:start', 0, 'nodeprep:end', 0), 'global_resources_loaded': _compute_delta_t(data, nodeid, 'cascade:start', 0, 'cascade:gr-done', 0), } try: timing['docker_install'] = _compute_delta_t( data, nodeid, 'nodeprep:start', 0, 'privateregistry:start', 0) except KeyError: # when no private registry setup exists, install time is # equivalent to nodeprep time timing['docker_install'] = timing['nodeprep'] try: timing['private_registry_setup'] = _compute_delta_t( data, nodeid, 'privateregistry:start', 0, 'privateregistry:end', 0) except KeyError: timing['private_registry_setup'] = 0 try: timing['docker_shipyard_container_pull'] = _compute_delta_t( data, nodeid, 'shipyard:pull-start', 0, 'shipyard:pull-end', 0) except KeyError: timing['docker_shipyard_container_pull'] = 0 data[nodeid]['start'] = data[nodeid]['nodeprep:start'][0][ 'timestamp'].timestamp() data[nodeid].pop('nodeprep:start') data[nodeid].pop('nodeprep:end') data[nodeid].pop('privateregistry:start', None) data[nodeid].pop('privateregistry:end', None) data[nodeid].pop('shipyard:pull-start', None) data[nodeid].pop('shipyard:pull-end', None) data[nodeid].pop('cascade:start') data[nodeid].pop('cascade:gr-done') for event in data[nodeid]: # print(event, data[nodeid][event]) if event == 'cascade:pull-start': _diff_events(data, nodeid, event, 'cascade:pull-end', timing, 'pull:', sizes) elif event == 'cascade:save-start': _diff_events(data, nodeid, event, 'cascade:save-end', timing, 'save:', sizes) elif event == 'cascade:torrent-start': _diff_events(data, nodeid, event, 'cascade:load-start', timing, 'torrent:') elif event == 'cascade:load-start': _diff_events(data, nodeid, event, 'cascade:load-end', timing, 'load:', sizes) data[nodeid].pop('cascade:pull-start', None) data[nodeid].pop('cascade:pull-end', None) data[nodeid].pop('cascade:save-start', None) data[nodeid].pop('cascade:save-end', None) data[nodeid].pop('cascade:torrent-start', None) data[nodeid].pop('cascade:load-start', None) data[nodeid].pop('cascade:load-end', None) data[nodeid]['timing'] = timing return data, sizes, offer, sku
class EventRepository: events_by_date_table = "eventsByDate" event_duplicates_table = "eventDuplicates" def __init__(self, connection_string=None): if not connection_string: connection_string = "AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;" self.table_client = TableService(connection_string=connection_string, is_emulated=True) def list_events_by_date(self, dt: datetime.date) -> List[dict]: pk = self._date_to_pk(dt) for event in self.table_client.query_entities( self.events_by_date_table, filter="PartitionKey eq '%s'" % (pk, )): if 'place' in event: event['place'] = json.loads(event['place']) if 'dates' in event: event['dates'] = json.loads(event['dates']) if 'raw_dates' in event: event['raw_dates'] = event['raw_dates'].split('\n') if 'tags' in event: event['tags'] = event['tags'].split(',') if 'type' in event: event['type'] = event['type'].split(',') if 'cost' in event: event['cost'] = event['cost'].split(',') yield event def remove_rows(self, dt, row_keys): pk = self._date_to_pk(dt) for key in row_keys: self.table_client.delete_entity(self.events_by_date_table, pk, key) def save_events_by_date(self, events: List[dict], dt: datetime.date, table_name=events_by_date_table): partition_keys = set() for event in events: if 'PartitionKey' not in event: if dt: event['PartitionKey'] = self._date_to_pk(dt) else: event['PartitionKey'] = str(datetime.date.today().year) if 'RowKey' not in event: full_text = event['title'] + "\n" + event[ 'short_description'] + "\n" + event['description'] event['RowKey'] = str(hash(full_text)) event['place'] = json.dumps(event['place'], ensure_ascii=False) event['dates'] = json.dumps(event['dates']) event['tags'] = ",".join(event['tags']) if 'type' in event: event['type'] = ",".join(event['type']) if "raw_dates" in event: event['raw_dates'] = "\n".join(event['raw_dates']) if 'cost' in event and event['cost']: event['cost'] = ",".join(str(c) for c in event['cost']) else: event['cost'] = None self.table_client.insert_or_replace_entity(table_name, event) partition_keys.add(event['PartitionKey']) for pk in partition_keys: self.table_client.insert_or_replace_entity(table_name, { "PartitionKey": "PARTITIONS", "RowKey": pk }) def save_events_json(self, events: List[dict]): grouped_events = group_by_dates(events) for dt, events in grouped_events.items(): self.save_events_by_date(events, dt) def save_verified_events(self, events: List[Event]): pk = datetime.datetime.now().timestamp() % 255 for event in events: event_description = event.to_str() event_hash = hash(event_description) self.table_client.insert_or_replace_entity( "verifiedEvents", { "PartitionKey": str(pk), "RowKey": str(event_hash), "Text": event_description, "Labels": ",".join(event.event_tags) }) @staticmethod def _date_to_pk(dt: datetime.date): return "%d_%d_%d" % (dt.year, dt.month, dt.day)
def _merge_resource(table_client: azuretable.TableService, resource: str, nglobalresources: int) -> None: """Merge resource to the image table :param azuretable.TableService table_client: table client :param str resource: resource to add to the image table :param int nglobalresources: number of global resources """ # merge resource to the image table entity = { 'PartitionKey': _PARTITION_KEY, 'RowKey': compute_resource_hash(resource), 'Resource': resource, 'VmList0': _NODEID, } logger.debug('merging entity {} to the image table'.format(entity)) try: table_client.insert_entity(_STORAGE_CONTAINERS['table_images'], entity=entity) except azure.common.AzureConflictHttpError: while True: entity = table_client.get_entity( _STORAGE_CONTAINERS['table_images'], entity['PartitionKey'], entity['RowKey']) # merge VmList into entity evms = [] for i in range(0, _MAX_VMLIST_PROPERTIES): prop = 'VmList{}'.format(i) if prop in entity: evms.extend(entity[prop].split(',')) if _NODEID in evms: break evms.append(_NODEID) for i in range(0, _MAX_VMLIST_PROPERTIES): prop = 'VmList{}'.format(i) start = i * _MAX_VMLIST_IDS_PER_PROPERTY end = start + _MAX_VMLIST_IDS_PER_PROPERTY if end > len(evms): end = len(evms) if start < end: entity[prop] = ','.join(evms[start:end]) else: entity[prop] = None etag = entity['etag'] entity.pop('etag') try: table_client.merge_entity(_STORAGE_CONTAINERS['table_images'], entity=entity, if_match=etag) break except azure.common.AzureHttpError as ex: if ex.status_code != 412: raise logger.info('entity {} merged to the image table'.format(entity)) global _GR_DONE if not _GR_DONE: try: entities = table_client.query_entities( _STORAGE_CONTAINERS['table_images'], filter='PartitionKey eq \'{}\''.format(_PARTITION_KEY)) except azure.common.AzureMissingResourceHttpError: entities = [] count = 0 for entity in entities: for i in range(0, _MAX_VMLIST_PROPERTIES): prop = 'VmList{}'.format(i) mode_prefix = _CONTAINER_MODE.name.lower() + ':' if (prop in entity and _NODEID in entity[prop] and entity['Resource'].startswith(mode_prefix)): count += 1 if count == nglobalresources: _record_perf('gr-done', 'nglobalresources={}'.format(nglobalresources)) _GR_DONE = True logger.info( 'all {} global resources of container mode "{}" loaded'.format( nglobalresources, _CONTAINER_MODE.name.lower())) else: logger.info( '{}/{} global resources of container mode "{}" loaded'.format( count, nglobalresources, _CONTAINER_MODE.name.lower()))
class Azure: # Tags used RG_RULE_PROGRAMMED_TAG = 'PANORAMA_PROGRAMMED' HUB_MANAGED_TAG = 'PanoramaManaged' # Resource types VMSS_TYPE = 'Microsoft.Compute/virtualMachineScaleSets' ILB_TYPE = 'Microsoft.Network/loadBalancers' APPINSIGHTS_TYPE = 'Microsoft.Insights/components' # Hardcoded names used for internal Azure resources ILB_NAME = 'myPrivateLB' ALPHANUM = r'[^A-Za-z0-9]+' def __init__(self, cred, subs_id, hub, vmss_rg_name, vmss_name, storage, pan_handle, logger=None): self.credentials = cred self.subscription_id = subs_id self.logger = logger self.hub_name = hub self.storage_name = storage self.panorama_handler = pan_handle self.vmss_table_name = re.sub(self.ALPHANUM, '', vmss_name + 'vmsstable') self.vmss_rg_name = vmss_rg_name try: self.resource_client = ResourceManagementClient(cred, subs_id) self.compute_client = ComputeManagementClient(cred, subs_id) self.network_client = NetworkManagementClient(cred, subs_id) self.store_client = StorageManagementClient(cred, subs_id) store_keys = self.store_client.storage_accounts.list_keys( hub, storage).keys[0].value self.table_service = TableService(account_name=storage, account_key=store_keys) except Exception as e: self.logger.error("Getting Azure Infra handlers failed %s" % str(e)) raise e # Start -> List out all RGs and identify new spokes to mark them with tags. # Look for Resource Groups (RGs) which do not have tags or does not have a # a tag named "PANORAMA_PROGRAMMED". # potential_new_spokes = [x.name for x in self.resource_client.resource_groups.list()\ # if not x.tags or not x.tags.get(self.RG_RULE_PROGRAMMED_TAG, None)] # If the RG has a VMSS which has a tag named "PanoramaManaged" with a value # as Hub Resource Group name then we know that this is a new spoke that is # launched managed by the Hub and not yet programmed for NAT/Azure Instrumentation # key. # for rg in potential_new_spokes: # fw_vm_list = [x for x in self.resource_client.resources.list_by_resource_group(rg) # if x.type == self.VMSS_TYPE and self.filter_vmss(rg, x.name)] # if fw_vm_list: # rg_params = {'location': self.resource_client.resource_groups.get(rg).location} # rg_params.update(tags={ # self.RG_RULE_PROGRAMMED_TAG : 'No', # self.HUB_MANAGED_TAG : self.hub_name # }) # self.resource_client.resource_groups.create_or_update(rg, rg_params) # self.logger.info("RG %s marked as a spoke managed by this hub %s" % (rg, self.hub_name)) # End -> List out all RGs and identify new spokes to mark them with tags. # Populate the list of spokes managed by this Azure hub. rg_list = self.resource_client.resource_groups.list() self.managed_spokes = [] self.managed_spokes.append(vmss_rg_name) self.new_spokes = [] # for rg in rg_list: # if rg.tags and rg.tags.get(self.HUB_MANAGED_TAG, None) == self.hub_name: # self.managed_spokes.append(rg.name) # if rg.tags.get(self.RG_RULE_PROGRAMMED_TAG, 'Yes') == 'No': # self.new_spokes.append(rg.name) # self.logger.debug('%s identified as spokes managed by %s' % (self.managed_spokes, self.hub_name)) # if self.new_spokes: # self.logger.info('%s identified as new spokes to be programmed by %s' % (self.new_spokes, self.hub_name)) # # def filter_vmss(self, spoke, vmss_name): vmss = self.compute_client.virtual_machine_scale_sets.get( spoke, vmss_name) if vmss.tags and vmss.tags.get(self.HUB_MANAGED_TAG, None) == self.hub_name: return True return False def get_ilb_ip(self, spoke): for resource in self.resource_client.resources.list_by_resource_group( spoke): # Get the ILB IP Address from the spoke. The ILB address is always # hardcoded to be myPrivateILB. if resource.name == self.ILB_NAME and resource.type == self.ILB_TYPE: ilb_obj = self.network_client.load_balancers.get( spoke, resource.name) ilb_frontend_cfg = ilb_obj.frontend_ip_configurations try: ilb_private_ip = ilb_frontend_cfg[0].private_ip_address except IndexError as e: self.logger.info("ILB is not setup yet in RG %s." % spoke) return None return ilb_private_ip return None def get_appinsights_instr_key(self, spoke): for resource in self.resource_client.resources.list_by_resource_group( spoke): # Get the Appinsights instance where the custom metrics are being # published. if resource.type == self.APPINSIGHTS_TYPE and 'appinsights' in resource.name: appinsights_obj = self.resource_client.resources.get_by_id( resource.id, '2014-04-01') instr_key = appinsights_obj.properties.get( 'InstrumentationKey', '') if not instr_key: self.logger.info("InstrKey is not setup yet in %s." % spoke) return None return instr_key return None def set_spoke_as_programmed(self, spoke): spoke_params = { 'location': self.resource_client.resource_groups.get(spoke).location } spoke_tags = self.resource_client.resource_groups.get(spoke).tags spoke_tags[self.RG_RULE_PROGRAMMED_TAG] = 'Yes' spoke_params.update(tags=spoke_tags) self.resource_client.resource_groups.create_or_update( spoke, spoke_params) self.logger.info( "RG %s marked as programmed and spoke managed by this hub %s" % (spoke, self.hub_name)) def create_worker_ready_tag(self, worker_name): self.compute_client.virtual_machines.create_or_update( self.vmss_rg, worker_name, { 'location': self.resource_client.resource_groups.get( self.vmss_rg).location, 'tags': { 'WORKER_READY': 'Yes' } }) def create_new_cosmos_table(self, table_name): # Create the Cosmos DB if it does not exist already if not self.table_service.exists(table_name): try: ok = self.table_service.create_table(table_name) if not ok: self.logger.error('Creating VMSS table failed') return False self.logger.info('VMSS Table %s created succesfully' % table_name) except Exception as e: self.logger.error('Creating VMSS table failed ' + str(e)) return False return True def clear_cosmos_table(self, table_name): self.table_service.delete_table(table_name) def get_vmss_by_name(self, spoke, vmss_name): vmss_list = [ x.name for x in self.resource_client.resources.list_by_resource_group(spoke) if x.type == self.VMSS_TYPE and x.name == vmss_name ] if vmss_list: return vmss_list[0] else: self.logger.error("No VMSS found in Resource Group %s" % spoke) return None def get_vmss_in_spoke(self, spoke): vmss_list = [ x.name for x in self.resource_client.resources.list_by_resource_group(spoke) if x.type == self.VMSS_TYPE and self.filter_vmss(spoke, x.name) ] if vmss_list: return vmss_list[0] else: self.logger.error("No VMSS found in Resource Group %s" % spoke) return None def get_vms_in_vmss(self, spoke, vmss_name): return self.compute_client.virtual_machine_scale_set_vms.list( spoke, vmss_name) def get_vm_in_cosmos_db(self, spoke, vm_hostname): try: db_vm_info = self.table_service.get_entity(self.vmss_table_name, spoke, vm_hostname) except AzureMissingResourceHttpError: self.logger.info("New VM %s found in spoke %s" % (vm_hostname, spoke)) return None except Exception as e: self.logger.error("Querying for %s failed" % vm_hostname) return None else: # IF possible update status TODO self.logger.debug("VM %s is available in VMSS, Pan and DB" % (vm_hostname)) return db_vm_info # 'name' : global_device['@name'], # 'hostname' : global_device['hostname'], # 'serial' : global_device['serial'], # 'ip-address' : global_device['ip-address'], # 'connected' : global_device['connected'], # 'deactivated': global_device['deactivated'] def create_db_entity(self, spoke, vm_details): vm = Entity() # PartitionKey is nothing but the spoke name vm.PartitionKey = spoke # RowKey is nothing but the VM name itself. vm.RowKey = vm_details['hostname'] vm.name = vm_details['name'] vm.serial_no = vm_details['serial'] vm.ip_addr = vm_details['ip-address'] vm.connected = vm_details['connected'] vm.deactivated = vm_details['deactivated'] vm.subs_id = self.subscription_id vm.delicensed_on = 'not applicable' vm.is_delicensed = 'No' try: self.table_service.insert_entity(self.vmss_table_name, vm) self.logger.info("VM %s with serial no. %s in db" % (vm_details['hostname'], vm_details['serial'])) except Exception as e: self.logger.info("Insert entry to db for %s failed with error %s" % (vm_details['hostname'], e)) return False return True def get_fw_vms_in_cosmos_db(self, spoke=None): if spoke: filter_str = "PartitionKey eq '%s'" % spoke else: filter_str = None db_vms_list = self.table_service.query_entities(self.vmss_table_name, filter=filter_str) if spoke: db_hostname_list = [{'hostname': x.RowKey, 'serial': x.serial_no, 'name': x.name} \ for x in db_vms_list if x.PartitionKey == spoke] return db_hostname_list else: return db_vms_list def delete_vm_from_cosmos_db(self, spoke, vm_name): self.table_service.delete_entity(self.vmss_table_name, spoke, vm_name)
class Azure: # Tags used RG_RULE_PROGRAMMED_TAG = 'PANORAMA_PROGRAMMED' HUB_MANAGED_TAG = 'PanoramaManaged' # Resource types VMSS_TYPE = 'Microsoft.Compute/virtualMachineScaleSets' ILB_TYPE = 'Microsoft.Network/loadBalancers' APPINSIGHTS_TYPE = 'Microsoft.Insights/components' # Hardcoded names used for internal Azure resources ILB_NAME = 'myPrivateLB' ALPHANUM = r'[^A-Za-z0-9]+' def __init__(self, cred, subs_id, my_storage_rg, vmss_rg_name, vmss_name, storage, pan_handle, logger=None): self.credentials = cred self.subscription_id = subs_id self.logger = logger self.hub_name = vmss_rg_name self.storage_name = storage self.panorama_handler = pan_handle self.vmss_table_name = re.sub(self.ALPHANUM, '', vmss_name + 'vmsstable') self.vmss_rg_name = vmss_rg_name try: self.resource_client = ResourceManagementClient(cred, subs_id) self.compute_client = ComputeManagementClient(cred, subs_id) self.network_client = NetworkManagementClient(cred, subs_id) self.store_client = StorageManagementClient(cred, subs_id) store_keys = self.store_client.storage_accounts.list_keys( my_storage_rg, storage).keys[0].value self.table_service = TableService(account_name=storage, account_key=store_keys) except Exception as e: self.logger.error("Getting Azure Infra handlers failed %s" % str(e)) raise e rg_list = self.resource_client.resource_groups.list() self.managed_spokes = [] self.managed_spokes.append(vmss_rg_name) self.new_spokes = [] def filter_vmss(self, spoke, vmss_name): vmss = self.compute_client.virtual_machine_scale_sets.get( spoke, vmss_name) if vmss.tags and vmss.tags.get(self.HUB_MANAGED_TAG, None) == self.hub_name: return True return False def create_worker_ready_tag(self, worker_name): self.compute_client.virtual_machines.create_or_update( self.vmss_rg, worker_name, { 'location': self.resource_client.resource_groups.get( self.vmss_rg).location, 'tags': { 'WORKER_READY': 'Yes' } }) def create_new_cosmos_table(self, table_name): # Create the Cosmos DB if it does not exist already if not self.table_service.exists(table_name): try: ok = self.table_service.create_table(table_name) if not ok: self.logger.error('Creating VMSS table failed') return False self.logger.info('VMSS Table %s created succesfully' % table_name) except Exception as e: self.logger.error('Creating VMSS table failed ' + str(e)) return False return True def clear_cosmos_table(self, table_name): self.table_service.delete_table(table_name) def get_vmss_by_name(self, spoke, vmss_name): vmss_list = [ x.name for x in self.resource_client.resources.list_by_resource_group(spoke) if x.type == self.VMSS_TYPE and x.name == vmss_name ] if vmss_list: return vmss_list[0] else: self.logger.error("No VMSS found in Resource Group %s" % spoke) return None def get_vmss_in_spoke(self, spoke): vmss_list = [ x.name for x in self.resource_client.resources.list_by_resource_group(spoke) if x.type == self.VMSS_TYPE and self.filter_vmss(spoke, x.name) ] if vmss_list: return vmss_list[0] else: self.logger.error("No VMSS found in Resource Group %s" % spoke) return None def get_vms_in_vmss(self, spoke, vmss_name): return self.compute_client.virtual_machine_scale_set_vms.list( spoke, vmss_name) def get_vm_in_cosmos_db(self, spoke, vm_hostname): try: db_vm_info = self.table_service.get_entity(self.vmss_table_name, spoke, vm_hostname) except AzureMissingResourceHttpError: self.logger.info("New VM %s found in spoke %s" % (vm_hostname, spoke)) return None except Exception as e: self.logger.error("Querying for %s failed" % vm_hostname) return None else: # IF possible update status TODO self.logger.debug("VM %s is available in VMSS, Pan and DB" % (vm_hostname)) return db_vm_info # 'name' : global_device['@name'], # 'hostname' : global_device['hostname'], # 'serial' : global_device['serial'], # 'ip-address' : global_device['ip-address'], # 'connected' : global_device['connected'], # 'deactivated': global_device['deactivated'] def create_db_entity(self, spoke, vm_details): vm = Entity() # PartitionKey is nothing but the spoke name vm.PartitionKey = spoke # RowKey is nothing but the VM name itself. vm.RowKey = vm_details['hostname'] vm.name = vm_details['name'] vm.serial_no = vm_details['serial'] vm.ip_addr = vm_details['ip-address'] vm.connected = vm_details['connected'] vm.deactivated = vm_details['deactivated'] vm.subs_id = self.subscription_id vm.delicensed_on = 'not applicable' vm.is_delicensed = 'No' try: self.table_service.insert_entity(self.vmss_table_name, vm) self.logger.info("VM %s with serial no. %s in db" % (vm_details['hostname'], vm_details['serial'])) except Exception as e: self.logger.info("Insert entry to db for %s failed with error %s" % (vm_details['hostname'], e)) return False return True def get_fw_vms_in_cosmos_db(self, spoke=None): if spoke: filter_str = "PartitionKey eq '%s'" % spoke else: filter_str = None db_vms_list = self.table_service.query_entities(self.vmss_table_name, filter=filter_str) if spoke: db_hostname_list = [{'hostname': x.RowKey, 'serial': x.serial_no, 'name': x.name} \ for x in db_vms_list if x.PartitionKey == spoke] return db_hostname_list else: return db_vms_list def delete_vm_from_cosmos_db(self, spoke, vm_name): self.table_service.delete_entity(self.vmss_table_name, spoke, vm_name)