def get_multiple_vm_details(conn, uuids, count):
vm_list = []
try:
multi_vm_results = conn.get_supplychains(
uuids,
types=['VirtualMachine'],
detail='aspects',
aspects=['virtualMachineAspect'])[0]
for vm in multi_vm_results['seMap']['VirtualMachine'][
'instances'].values():
try:
vm_details = get_vm_info(vm)
vm_list.append({
'uuid': vm_details[0],
'name': vm_details[1],
'ip_address': vm_details[2]
})
except KeyError:
continue
except vc.HTTP500Error as e:
umsg.log(
'Problem retrieving bulk VM information, trying individual VMs',
level='error')
umsg.log(e)
vm_list.extend(get_individual_vm_details(conn, uuids, count))
return vm_list
def get_turbo_vms(conn, start=None, end=None, step=100):
vm_list = []
uuids = [
x['uuid']
for x in conn.search(types=['VirtualMachine'], detail_type='compact')
]
if not start:
start = 0
if not end:
end = len(uuids)
while end < len(uuids):
umsg.log(
f'Getting VMs between {start} and {end}, out of a total {len(uuids)}',
level='debug')
uuid_subset = uuids[start:end]
vm_details = get_multiple_vm_details(conn, uuid_subset, start)
vm_list.extend(vm_details)
start += step
end += step
else:
if start < len(uuids):
umsg.log(
f'Getting VMs between {start} and {len(uuids)}, out of a total {len(uuids)}',
level='debug')
uuid_subset = uuids[start:len(uuids)]
vm_details = get_multiple_vm_details(conn, uuid_subset, start)
vm_list.extend(vm_details)
return vm_list
def get_individual_vm_details(conn, uuids, count):
vm_list = []
for vm_uuid in uuids:
umsg.log(f'Processing VM {count}')
count += 1
try:
single_vm_results = conn.get_entities(uuid=vm_uuid, detail=True)[0]
except vc.HTTP500Error:
umsg.log(
f'Error getting details for VM with UUID: {vm_uuid}, skipping',
level='error')
continue
try:
vm_details = get_vm_info(single_vm_results)
except KeyError:
continue
vm_list.append({
'uuid': vm_details[0],
'name': vm_details[1],
'ip_address': vm_details[2]
})
return vm_list
def parse_csv_into_apps(csv_data, prefix=''):
"""Parse input CSV into dictionary of UserDefinedApps
Parameters:
csv_data - list - List of dicts from read_csv
prefix - str - Optional prefix for user-defined app name
"""
app_dict = {}
row_count = 1
umsg.log('Looking for apps and associated VMs...')
for row in csv_data:
row_count += 1
if not row['app_name']:
umsg.log(
f'No application defined on row {row_count} of input CSV, skipping',
level='warn')
continue
app_name = f"{prefix}{row['app_name']}"
if app_name in app_dict.keys():
app = app_dict[app_name]
else:
app = UserDefinedApp(app_name)
app_dict[app_name] = app
app.add_member(member_name=row['entity_name'],
member_ip=row.get('entity_ip'))
return app_dict
def read_csv(self, csv_str_io):
"""Parse CSV StringIO to dict
Parameters:
filename - StringIO - IO data from CSV file
Returns:
List of dicts, where each dict is a row in the input CSV
"""
data = []
csv_data = csv.DictReader(csv_str_io)
row_count = 1
for row in csv_data:
row_count += 1
if not row[self.entity_headers['app_name']]:
umsg.log(
f'No application defined on row {row_count} of input CSV, skipping',
level='warn')
continue
try:
data.append((self._process_entity_headers(row),
self._process_metric_headers(row)))
except KeyError:
umsg.log(
f'Something went wrong on line {row_count} while processing CSV'
)
raise
return data
def create_topology(self):
data = self.get_csv_data()
topology = Topology()
apps = parse_data_into_apps(data, topology, self.app_prefix)
umsg.log('Building DIF topology...')
build_dif_topology(apps)
return topology
def test_prefix_inline(self, caplog):
prefix = 'default'
msg = 'txt'
umsg.log(msg, prefix=prefix)
expected = '[{}] {}'.format(prefix, msg)
assert_that(caplog.record_tuples[0][2]).is_equal_to(expected)
def read_config_file(config_file):
"""Read JSON config file"""
with open(config_file, 'r') as fp:
try:
return json.loads(fp.read())
except TypeError:
umsg.log(f'{config_file} must be JSON format', level='error')
def test_verbose_end(self, capsys):
msg = 'print this'
end = '.'
umsg.log(msg, 'verbose', end=end)
expected = '{}{}'.format(msg, end)
captured = capsys.readouterr()
assert_that(captured.out).is_equal_to(expected)
def test_prefix_setting(self, caplog):
prefix = 'default'
msg = 'text'
umsg.set_attr('msg_prefix', prefix)
umsg.log(msg)
expected = '[{}] {}'.format(prefix, msg)
assert_that(caplog.record_tuples[0][2]).is_equal_to(expected)
def test_prefix_formatting(self, caplog):
prefix = 'inline'
msg = 'text'
umsg.set_attr('msg_prefix_format', '<{prefix}> ')
umsg.log(msg, prefix=prefix)
expected = '<{}> {}'.format(prefix, msg)
assert_that(caplog.record_tuples[0][2]).is_equal_to(expected)
def _msg(self, *args, **kwargs):
"""Inteface for module level :py:func:`umsg._msg` function."""
if 'prefix' not in kwargs:
kwargs['prefix'] = self._umsg_log_prefix
if 'logger' not in kwargs:
kwargs['logger'] = self._umsg_logger
umsg.log(*args, **kwargs)
def create_dif_entities(self):
for member_info in self.members.values():
try:
self.add_member_to_dif_topo(member_info)
except ParentEntityNotFoundError:
msg = f"No parent found for entity named: {member_info['name']} with type: {member_info['type']}, skipping"
umsg.log(msg, level='error')
return self.dif_topology
def test_verbose_prefix_ignored(self, capsys):
prefix = 'inline'
msg = 'print this'
end = '\n'
umsg.log(msg, 'verbose', prefix=prefix)
expected = '{}{}'.format(msg, end)
captured = capsys.readouterr()
assert_that(captured.out).is_equal_to(expected)
def _make_dummy_levels(self):
umsg.log(f'Creating dummy application and service for {self.name}',
level='debug')
service_name = f'{self.name}_service'
app_name = f'{self.name}_application'
dummy_service = self._make_dummy_info('service', self.name,
'businessApplication')
dummy_app = self._make_dummy_info('application', service_name,
'service')
self.members[(service_name, 'service')] = dummy_service
self.members[(app_name, 'application')] = dummy_app
def get_csv_data(filename, csv_location, entity_headers, match_ip):
reader = DifCsvReader(filename, csv_location, entity_headers, match_ip)
while not reader.file_downloaded:
try:
data = reader.download_csv_data()
except CsvFileNotFoundError:
umsg.log('No CSV found, waiting and then retrying')
time.sleep(60)
return reader.read_csv(data)
def actions(source, config, logger):
umsg.log(f"Retrieving data from {config['resource']}", logger=logger)
# source.connect() gives us the vmt-connect.Connection instance returned by
# vmt-report; here we use the get_actions() method inline.
# Another use would be the standard vmt-connect idiom:
# vmt = source.connection()
# res = vmt.get_action()
res = source.connect().get_actions()
fields = ['createTime', 'actionType', 'details']
return [{x: res[x]} for x in res if x in fields]
def test_formatter(self, capsys):
prefix = 'fmt'
msg = 'stream it'
umsg.add_handler(logging.StreamHandler)
umsg.log(msg, prefix=prefix)
date = datetime.date.today().strftime('%Y-%m-%d')
expected = '{} ([\\d]{{2}}:?){{3}} - INFO - <{}> {}'.format(
date, prefix, msg)
captured = capsys.readouterr()
assert (re.match(expected, captured.err))
def _process_entity_headers(self, row):
entities = {}
for k, v in self.entity_headers.items():
try:
entities[k] = row[v]
except KeyError:
umsg.log(
f'Incorrect entity field map entry: key: {k}, value: {v}',
level='error')
raise
return entities
def read_config_file(config_file):
"""Read JSON config file
Parameters:
config_file - str - Name of JSON config file
Output:
Dict - Dict representation of JSON config file
"""
with open(config_file, 'r') as fp:
try:
return json.loads(fp.read())
except TypeError:
umsg.log(f'{config_file} must be JSON format', level='error')
def _process_metric_headers(self, row):
metrics = {}
for k, v in self.metric_headers.items():
try:
metrics[k] = row[v]
except KeyError:
if v:
umsg.log(
f'Incorrect metric field map entry: key: {k}, value: {v}',
level='error')
raise
return metrics
def main(config_file):
args = read_config_file(config_file)
requests.packages.urllib3.disable_warnings(
requests.packages.urllib3.exceptions.InsecureRequestWarning)
umsg.init(level=args.get('LOG_LEVEL', 'info'))
log_file = os.path.join(args.get('LOG_DIR', ''), args.get('LOG_FILE', ''))
if log_file:
handler = logging.handlers.RotatingFileHandler(log_file,
mode='a',
maxBytes=10 * 1024 *
1024,
backupCount=1,
encoding=None,
delay=0)
umsg.add_handler(handler)
else:
umsg.add_handler(logging.StreamHandler())
umsg.log('Starting script')
csv_data = get_csv_data(filename=args['INPUT_CSV_NAME'],
csv_location=args['CSV_LOCATION'],
entity_headers=args.get('ENTITY_FIELD_MAP'),
match_ip=args.get('MATCH_IP', False))
apps = parse_csv_into_apps(csv_data, args.get('APP_PREFIX', ''))
if args.get('IGNORE_TURBO_VERSION'):
spec = vc.VersionSpec(versions=[], required=False)
vmt_conn = vc.Connection(os.environ['TURBO_ADDRESS'],
os.environ['TURBO_USERNAME'],
os.environ['TURBO_PASSWORD'],
req_versions=spec)
else:
vmt_conn = vc.Connection(os.environ['TURBO_ADDRESS'],
os.environ['TURBO_USERNAME'],
os.environ['TURBO_PASSWORD'])
turbo_vms = get_turbo_vms(vmt_conn, start=0, end=500, step=500)
apps = match_apps_to_turbo_vms(apps, turbo_vms,
args.get('MATCH_IP', False))
make_apps_thru_atm(vmt_conn, apps)
umsg.log('Finished script')
def do_GET(self):
if self.path == '/dif_metrics':
try:
topology = self.create_topology()
umsg.log('Serving DIF topology JSON')
self.respond({'status': 200, 'content': topology.ToJSON()})
except CsvDownloadError as e:
self.respond({
'status': e.status_code,
'content': e.message
})
except Exception as e:
self.respond({
'status': 500,
'content': f'Something went wrong: {e}'
})
def parse_data_into_apps(csv_data, dif_topology, prefix):
"""Parse input CSV into dictionary of UserDefinedApps
Parameters:
csv_data - list - List of dicts from read_csv
dif_topology - dif.Topology - DIF Topology object
entity_headers - dict - Optional mapping for CSV entity column names
metric_headers - dict - Optional mapping for CSV metric column names
"""
app_dict = {}
row_count = 1
umsg.log('Looking for apps and associated VMs...')
for entity, metric in csv_data:
row_count += 1
app_name = entity['app_name']
if app_name in app_dict.keys():
app = app_dict[app_name]
else:
app = UserDefinedApp(app_name, dif_topology, prefix)
app_dict[app_name] = app
try:
app.add_member(member_name=entity['entity_name'],
member_type=entity['entity_type'],
parent_name=entity['parent_name'],
parent_type=entity['parent_type'],
member_ip=entity['entity_ip'])
except InvalidParentTypeError:
msg = f'Invalid parent_type found on row {row_count} of input CSV'
raise InvalidParentTypeError(msg)
try:
app.add_metrics(member_name=entity['entity_name'],
member_type=entity['entity_type'],
metrics=metric)
except Exception as e:
raise e
return app_dict
def add_metrics(self, member_name, member_type, metrics):
for metric_type, metric_val in metrics.items():
metric_type, cap_or_used = metric_type.split('_')
if metric_val:
try:
self.is_valid_metric(member_type, metric_type)
except InvalidMetricTypeError:
msg = f'{member_name} has an invalid metric type, skipping metric named: {metric_type}'
dbug_msg = (
f'Member type: [ {member_type} ], provided metric type: [ {metric_type} ], '
+
f'valid metric types: [ {", ".join(self.acceptable_metric_types[member_type])} ]'
)
umsg.log(msg, level='warning')
umsg.log(dbug_msg, level='debug')
self.member_metrics[(member_name, member_type)][(
metric_type, cap_or_used)] = float(metric_val)
def make_apps_thru_atm(conn, apps):
current_apps = {
app['displayName']: app['uuid']
for app in conn.request('topologydefinitions')
}
for app in apps.values():
if not app.member_uuids:
umsg.log(
f'No matching VMs found for app named: {app.name}, skipping')
continue
app.remove_members_without_matches()
try:
app.update_appl_topo(conn, current_apps[app.name])
except KeyError:
app.create_appl_topo(conn)
return True
def process_csv_location(self, provider):
if provider not in {'AZURE', 'AWS', 'FTP'}:
umsg.log(
'Value for CSV_LOCATION is invalid. It must be one of: [ AZURE, AWS, FTP ]',
level='error')
raise InvalidConfigError()
if provider == 'AZURE':
self.provider = 'AZURE'
self.connect_str = os.environ['AZURE_CONNECTION_STRING']
self.container_name = os.environ['AZURE_CONTAINER_NAME']
if provider == 'AWS':
self.provider = 'AWS'
self.access_key_id = os.environ['AWS_ACCESS_KEY_ID']
self.secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY']
self.region_name = os.environ['AWS_REGION_NAME']
self.bucket_name = os.environ['AWS_BUCKET_NAME']
if provider == 'FTP':
self.provider = 'FTP'
self.path = '/opt/turbonomic/data'
def add_member(self, member_name, member_ip):
member_info = {
'name': member_name,
'ip_address': '',
'turbo_oid': None
}
try:
member_info['ip_address'] = self._process_ips(member_ip)
except IpAddressNotFoundError:
umsg.log(f'No IP address defined for VM {member_name}',
level='debug')
pass
if member_info in self.members:
umsg.log(
f'Member {member_info["name"]} already exists in {self.name} application group',
level='warn')
pass
else:
self.members.append(member_info)
def main(config_file):
args = read_config_file(config_file)
host_name = '0.0.0.0'
port_number = 8081
umsg.init(level=args.get('LOG_LEVEL', 'INFO'))
log_file = os.path.join(args.get('LOG_DIR'), args.get('LOG_FILE'))
if log_file:
handler = logging.handlers.RotatingFileHandler(log_file,
mode='a',
maxBytes=10 * 1024 *
1024,
backupCount=1,
encoding=None,
delay=0)
umsg.add_handler(handler)
else:
umsg.add_handler(logging.StreamHandler())
topology_handler = MakeHandlerClassFromArgs(args['INPUT_CSV_NAME'],
args['CSV_LOCATION'],
args.get('ENTITY_FIELD_MAP'),
args.get('METRIC_FIELD_MAP'),
args.get('APP_PREFIX'))
httpd = HTTPServer((host_name, port_number), topology_handler)
umsg.log(f'Server Starts - {host_name}:{port_number}')
try:
httpd.serve_forever()
except KeyboardInterrupt:
pass
httpd.server_close()
umsg.log(f'Server Stops - {host_name}:{port_number}')
def create_appl_topo(self, conn):
umsg.log(
f'Creating application for BusinessApplication named: {self.name}')
dto = self._prep_app_topo_dto()
res = conn.request('topologydefinitions', method='POST', dto=dto)
umsg.log(f'Successfully created app {self.name}.')
umsg.log(f'Response Details: {res}', level='debug')
return True
