def _generate_ansible_cfg_file(self):
project_path = self.config['project_path']
ansible_cfg_filename = 'ansible.cfg'
ssh_config_filename = 'ssh.config'
tpl_file = self.config['ansible']['ansible_cfg_template']
ansible_cfg_template = self.j2_env.get_template(tpl_file)
ssh_key_name = self.project_name + '_ssh'
self.logger.info("Generating ansible.cfg file from template '%s'...",
tpl_file)
template_vars = {
'ansible_dir':
os.path.abspath(self.config['ansible']['ansible_dir']),
'ssh_config_path':
os.path.abspath(os.path.join(project_path, ssh_config_filename)),
'ssh_private_key_path':
os.path.abspath(os.path.join(project_path, ssh_key_name)),
'ssh_control_path':
'~/.ssh/ansible-%r@%h:%p'
}
ansible_cfg_file_content = ansible_cfg_template.render(template_vars)
target = os.path.join(project_path, ansible_cfg_filename)
self.logger.info("Saving ansible.cfg file to '%s'...", target)
utils.save_string_to_file(ansible_cfg_file_content, target)
def benchmark(parameter_name, values, config=CONFIG):
save_folder = generate_folder_name()
print("\n\nBenchmark for parameter '{}' with values {}".format(parameter_name, values))
results = np.empty((len(values), 2), dtype=float)
for index, v in enumerate(values):
print("Run {} with value {} on GPU".format(index+1, v))
results[index, 0] = run((parameter_name, v), ('gpu', True))
utils.clear_current_line()
print("Run {} with value {} on CPU".format(index+1, v))
results[index, 1] = run((parameter_name, v), ('gpu', False))
utils.clear_current_line()
result_str = "Results: \n{}".format(results)
print(result_str)
utils.save_dict(config, save_folder, 'config_' + parameter_name + '.txt')
utils.save_string_to_file(result_str, save_folder, 'results.txt')
def benchmark(parameter_name, values, config=CONFIG):
save_folder = generate_folder_name()
print("\n\nBenchmark for parameter '{}' with values {}".format(
parameter_name, values))
results = np.empty((len(values), 2), dtype=float)
for index, v in enumerate(values):
print("Run {} with value {} on GPU".format(index + 1, v))
results[index, 0] = run((parameter_name, v), ('gpu', True))
utils.clear_current_line()
print("Run {} with value {} on CPU".format(index + 1, v))
results[index, 1] = run((parameter_name, v), ('gpu', False))
utils.clear_current_line()
result_str = "Results: \n{}".format(results)
print(result_str)
utils.save_dict(config, save_folder, 'config_' + parameter_name + '.txt')
utils.save_string_to_file(result_str, save_folder, 'results.txt')
def parameter_tester(parameter_name, values, save_folder=None):
if save_folder is None:
save_folder = utils.generate_folder_name()
print("\n\nExperiment for parameter '{}' with values {}".format(
parameter_name, values))
plt.clf()
all_scores = []
for index, v in enumerate(values):
print("Run {} with value {}".format(index + 1, v))
scores = run((parameter_name, v))
all_scores.append(scores)
utils.print_on_current_line('')
header = ','.join(str(s) for s in values)
result_str = header + '\n' + ','.join(["{:.4f}".format(sum(s) / CLASSIFIER_CONFIG.runs) for s in all_scores]) + \
'\n\n' + 'all scores:\n'
for a in all_scores:
result_str += ','.join('{:.4f}'.format(s) for s in a) + '\n'
utils.save_string_to_file(result_str,
parent_folder=save_folder,
file_name='result_' + parameter_name + '.txt')
print("\nRun {}/{}\n".format(i + 1, CLASSIFIER_CONFIG['runs']))
for j, c in enumerate(configurations):
results[j].append(run(**c))
mean_results = np.array(results).mean(1).tolist()
def np_list_to_csv_string(npl):
return ",".join(list(map(lambda f: "{:.4f}".format(f), npl)))
csv = []
for arr in mean_results:
csv.append(np_list_to_csv_string(arr))
utils.save_object(mean_results, SAVE_FOLDER, 'results')
utils.save_string_to_file("\n".join(csv), SAVE_FOLDER, 'results.csv')
utils.save_dict(CLASSIFIER_CONFIG, SAVE_FOLDER, 'config.json')
data = np.array(mean_results)
x = range(data.shape[1])
fig, ax = plt.subplots()
plotter.hide_top_and_right_axis(ax)
ax.yaxis.grid(color='gray')
ax.set_xlabel('Time (seconds)')
ax.set_ylabel('Best polygon solution')
ax.set_prop_cycle(cycler('color', ['c', 'm', 'y', 'k', 'r', 'g', 'b']))
lines = []
for i in range(len(configurations)):
data_set = data_manager.load_data_set(conf.data_set)
X = data_set.data
y = data_set.target
class_indices = list(set(y))
# Split data into training and testing set
if conf.training_test_split:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33)
else:
X_train = X_test = X
y_train = y_test = y
clf = acoc.PolyACO(X.shape[1], class_indices, save_folder=SAVE_FOLDER)
clf.train(X_train, y_train, start_time=conf.start_time)
predictions = clf.evaluate(X_test)
return acoc.compute_score(predictions, y_test)
if __name__ == "__main__":
utils.save_dict(CLASSIFIER_CONFIG, parent_folder=SAVE_FOLDER, file_name='config.json')
scores = []
runs = 1
result_str = ''
start_time = time()
for i in range(runs):
scores.append(run(start_time=start_time))
print("\nRun {}/{} score: {:.4f}".format(i + 1, runs, scores[-1]))
result_str = ','.join([str(s) for s in scores]) + "\nAverage score with {}-fold cross validation: {:.5f}".format(runs, sum(scores) / runs)
utils.save_string_to_file(result_str, parent_folder=SAVE_FOLDER, file_name='result.txt')
print("\n" + result_str)
for j, c in enumerate(configurations):
results[j].append(run(**c))
mean_results = np.array(results).mean(1).tolist()
def np_list_to_csv_string(npl):
return ",".join(list(map(lambda f: "{:.4f}".format(f), npl)))
csv = []
for arr in mean_results:
csv.append(np_list_to_csv_string(arr))
utils.save_object(mean_results, SAVE_FOLDER, 'results')
utils.save_string_to_file("\n".join(csv), SAVE_FOLDER, 'results.csv')
utils.save_dict(CLASSIFIER_CONFIG, SAVE_FOLDER, 'config.json')
data = np.array(mean_results)
x = range(data.shape[1])
fig, ax = plt.subplots()
plotter.hide_top_and_right_axis(ax)
ax.yaxis.grid(color='gray')
ax.set_xlabel('Time (seconds)')
ax.set_ylabel('Best polygon solution')
ax.set_prop_cycle(cycler('color', ['c', 'm', 'y', 'k', 'r', 'g', 'b']))
lines = []
for i in range(len(configurations)):
lines.append(ax.plot(x, data[i], label=labels[i]))
def _generate_inventory_file(self, cloud_nodes):
project_path = self.config['project_path']
tpl_file = self.config['ansible']['inventory_template']
inventory_template = self.j2_env.get_template(tpl_file)
self.logger.info(
"Generating ansible inventory file from template '%s'...",
tpl_file)
hosts = self.config['hosts']
template_vars = {}
# iterate through all the the hosts defined
for host in hosts:
host_ansible_settings = host['ansible_settings']
cnt = host['count']
# each host 'count' times
for i in range(0, cnt):
host_name = self.project_name + "-" + host['name']
inventory_host_name = host['name']
if cnt > 1:
host_name += "_" + str(i + 1)
inventory_host_name += "_" + str(i + 1)
# iterate through 'ansible_settings' array
for ans_setting in host_ansible_settings:
# get the array of the hosts for the inventory group so
# that we can add more
inventory_group = template_vars.setdefault(
ans_setting['ansible_group'], [])
# we'll generate the string based on this dict
inventory_item = {}
# check item_vars -- all 'count' entries will have these
# properties
item_vars = ans_setting.get('item_vars', [])
for item_var in item_vars:
for item_var_key in item_var:
sub_fn_name = self.substitution_rules[
'item_vars'].get(item_var_key)
if sub_fn_name:
sub_fn = getattr(self, sub_fn_name)
if not sub_fn:
self.logger.warn(
"%s method not found in the implementation as a substitution rule. Skipping...",
sub_fn_name)
continue
# make a copy and remove "additional" 'self'
# key in order to pass the dict
_locals = dict(locals())
del _locals['self']
# call function for substitution rule
inventory_item[item_var_key] = sub_fn(
**_locals)
else:
inventory_item[item_var_key] = item_var[
item_var_key]
# check group_vars -- entries with the proper index will be
# added
group_vars = ans_setting.get('group_vars', []) or []
for group_var in group_vars:
# 'index' mandatory
index = group_var['index']
if index == i or index == 'all':
for group_var_key in group_var:
if group_var_key != 'index':
inventory_item[group_var_key] = group_var[
group_var_key]
if index == 'counter':
for group_var_key in group_var:
if group_var_key != 'index':
inventory_item[group_var_key] = str(i + 1)
inventory_line = inventory_host_name + _SPACES + _SPACES.join(
("%s=%s" % (k, v)
for (k, v) in inventory_item.items()))
inventory_group.append(inventory_line)
inventory_file_content = inventory_template.render(template_vars)
project_path = self.config['project_path']
target = os.path.join(project_path, 'ansible_inventory')
self.logger.info("Saving ansible inventory file to '%s'", target)
utils.save_string_to_file(inventory_file_content, target)
def _generate_ssh_config_file(self, cloud_nodes):
project_path = self.config['project_path']
ssh_config_filename = 'ssh.config'
tpl_file = self.config['ansible']['ssh_config_template']
ssh_config_template = self.j2_env.get_template(tpl_file)
ssh_key_name = self.project_name + '_ssh'
self.logger.info("Generating ssh.config file from template '%s'...",
tpl_file)
template_vars = {
'project_name':
self.project_name,
'ssh_config_path':
os.path.abspath(os.path.join(project_path, ssh_config_filename)),
'ssh_private_key_path':
os.path.abspath(os.path.join(project_path, ssh_key_name)),
'ssh_control_path':
'~/.ssh/ansible-%r@%h:%p'
}
hosts = self.config['hosts']
main_host = next(
(host for host in hosts if host.get('type') == 'bastion'), None)
if not main_host:
self.logger.error(
"No bastion host found in the default config. Picking first host in the list..."
)
# return
main_host = hosts[0]
host_name = self.project_name + "-" + main_host['name']
node = next((node for node in cloud_nodes if node.name == host_name),
None)
if not node:
self.logger.error(
"Couldn't find host '%s'. Skipping generating ssh.config file...",
host_name)
return
private_ip = self.get_ip(node, 'private')
public_ip = self.get_ip(node, 'public')
if public_ip:
template_vars[
'bastion_public_ip'] = public_ip # remove in next version
template_vars['main_public_ip'] = public_ip
template_vars['public_ip_address_space'] = re.sub(
r'(\d+\.\d+\.\d+\.)\d+', r'\1*', public_ip)
else:
self.logger.error(
"Bastion/main host doesn't have public_ips. Skipping public ip related vars..."
)
# return
if private_ip:
template_vars['private_ip_address_space'] = re.sub(
r'(\d+\.\d+\.\d+\.)\d+', r'\1*', private_ip)
else:
self.logger.error(
"Bastion/main host doesn't have private_ips. Skipping private ip related vars..."
)
# return
ssh_config_file_content = ssh_config_template.render(template_vars)
target = os.path.join(project_path, ssh_config_filename)
self.logger.info("Saving ssh.config file to '%s'", target)
utils.save_string_to_file(ssh_config_file_content, target)
