def fit(self,
stable_antecedents: List[str],
flexible_antecedents: List[str],
consequent: str,
conf: float,
supp: float,
desired_classes: List[str] = None,
desired_changes: List[list] = None,
is_nan: bool = False,
is_reduction: bool = True,
min_stable_antecedents: int = 1,
min_flexible_antecedents: int = 1):
"""
Get action rules.
Define antecedent and consequent.
- stable_antecedents - List of column names.
- flexible_antecedents - List of column names.
- consequent - Name of column.
Confidence and support.
- conf - value in % for confidence of classification rules.
- supp - value in % for support of classification rules.
Desired classes or desired changes must be entered.
- desired_classes - List of decision states. For example ["1"]. DEFAULT: None
- desired_changes - List of desired changes. For example [["0", "1"]]. DEFAULT: None
Should nan values be used.
- is_nan - True means nan values are used, False means nan values are not used. DEFAULT: FALSE
Should the reduction table be used.
- is_reduction - is reduction table used DEFAULT: TRUE
Minimal number of stable and flexible couples
- min_stable_antecedents - min. stable couples. DEFAULT: 1
- min_flexible_antecedents - min. flexible couples. DEFAULT: 1
"""
if bool(desired_classes) != bool(desired_changes):
desired_state = DesiredState(desired_classes=desired_classes,
desired_changes=desired_changes)
else:
raise Exception(
"Desired classes or desired changes must be entered")
antecedents = stable_antecedents + flexible_antecedents
self.decisions.prepare_data_fim(antecedents, consequent)
self.decisions.fit_fim_apriori(conf=conf, support=supp)
self.decisions.generate_decision_table()
stable = self.decisions.decision_table[stable_antecedents]
flex = self.decisions.decision_table[flexible_antecedents]
target = self.decisions.decision_table[[consequent]]
supp = self.decisions.support
conf = self.decisions.confidence
reduced_tables = Reduction(stable, flex, target, desired_state, supp,
conf, is_nan)
if is_reduction:
reduced_tables.reduce()
action_rules = ActionRules(
reduced_tables.stable_tables, reduced_tables.flexible_tables,
reduced_tables.decision_tables, desired_state, reduced_tables.supp,
reduced_tables.conf, is_nan, min_stable_antecedents,
min_flexible_antecedents)
action_rules.fit()
self.action_rules = action_rules.action_rules
def main(argv):
inputfile = ''
outputfile = ''
debug = False
try:
opts, args = getopt.getopt(argv,"hi:o:d",["ifile=","ofile="])
except getopt.GetoptError:
print 'single.py -i <inputfile> -o <outputfile>'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print 'single.py -i <inputfile> -o <outputfile> -d for debug'
sys.exit()
elif opt in ("-d", "--debug"):
debug = True
elif opt in ("-i", "--ifile"):
inputfile = arg
elif opt in ("-o", "--ofile"):
outputfile = arg
try:
fi = open(inputfile,"r")
fl = open(outputfile,"a")
except IOError:
print 'main.py -i <inputfile> -o <outputfile>'
sys.exit(2)
l = []
pols = []
files = [inputfile]
for fileName in files:
save = outputfile
f = open(fileName,'r')
#read, pols = recoverfile(save, f)
if True:
for line in f:
try:
pol = Polynomial(line)
pols.append(pol)
except Exception as e:
print line
sys.exit(2)
result = defaultdict(list)
print len(pols)
for pol in pols:
if len(pol.coefs()) > 1:
red = Reduction(debug)
count = red.reduction(pol.coefs())
result = str(pol.coefs()) + ":" + str(count)
print result
fl.write(result + "\n")
def SNF_reduction(self):
while True:
self._computedSNF = False
self.computeSNF()
all_diag = True
for p in self.dimensions:
T = np.zeros_like(self.D[p])
T[:min(self.D[p].shape), :min(self.D[p].shape)] = np.diag(np.diag(self.D[p]))
all_diag = all_diag and np.all(T == self.D[p])
if not all_diag:
continue
if all_diag:
break
f_A = ChainMap(self, matrices={q: self.A[q].T for q in self.dimensions})
f_G = ChainMap(self, matrices={q: self.G[q].T for q in self.dimensions})
f_D = ChainMap(self, matrices={q: self.D[q].T for q in self.dimensions}, degree=self._degree)
M = ChainComplex(cell_class=self.cell_class)
M._degree = +1
for q in self.dimensions:
M[q] = self[q]
M.d = f_D
return Reduction(
src=self, dst=M,
projection=f_G,
inclusion=f_A,
integral=ChainMap(self, degree=-self._degree)
)
def run(self):
print "Starting thread: " + str(self.threadID) + '\n'
for i in self.polynomials:
reduc = Reduction()
self.lockscreen.acquire()
print 'Thread: '+ str(self.threadID) + ' Doing: ' + str(i.coefs())
self.lockscreen.release()
count = reduc.reduction(i.coefs())
self.locker.acquire()
r = str(i.coefs()) + ":" + str(count)
f = open(self.save, "a")
f.write(r + '\n')
f.close()
self.locker.release()
self.lockscreen.acquire()
#print r
self.lockscreen.release()
del reduc
def __init__(self, config_path='openstack_dashboard/api/telemetry_api/environment.conf'):
self.__config = ConfigParser.ConfigParser()
self.__config.read(config_path)
#self.__ceilometer = CeilometerClient(self.__config)
self.__ceilometer = CeilometerClient()
self.__keystone = KeystoneClient(self.__config)
#self.__nova = NovaClient(self.__config)
self.__nova = NovaClient()
server = self.__config.get('Misc', 'dbserver')
user = self.__config.get('Misc', 'dbuser')
passwd = self.__config.get('Misc', 'dbpass')
database = self.__config.get('Misc', 'hostsdbname')
table = self.__config.get('Misc', 'hostsdbtable')
self.__hosts = ast.literal_eval(self.get_config().get('Openstack', 'Hosts'))
self.__hosts_db = HostDataHandler()
self.__benchmark_db = BenchmarkDataHandler(server, user, passwd)
self.__reduction = Reduction()
def reduction(self):
cplx = self.decomposition()
d = self.src.d
pi_t, pi_s, pi_c, iota_t, iota_s, iota_c = self._projections_inclusions(cplx)
d_33 = pi_c * d * iota_c
d_31 = pi_c * d * iota_t
d_21 = pi_s * d * iota_t
d_21_inv = ChainMap(d_21.dst, d_21.src, degree=+1)
for p in self.dimensions:
d_21_inv[p] = np.linalg.inv(d_21[p + 1]).astype(np.int32)
d_23 = pi_s * d * iota_c
cplx['c'].d = d_33 - d_31 * d_21_inv * d_23
f = pi_c - d_31 * d_21_inv * pi_s
g = iota_c - iota_t * d_21_inv * d_23
h = iota_t * d_21_inv * pi_s
return Reduction(self.src, cplx['c'], f, g, h)
def __reduction(self):
reductor = Reduction(self.__population, self.population_size,
self.x_min, self.x_max)
reductor.perform()
print "I am here" # threads = [] # i = 0 # j = 2 # for temp in range(0, len(pols)/2): # if (j > len(pols)): # j = len(pols) # thread = ThreadCount(temp,lockScreen, lock, pols[i:j], save) # i = j+1 # j += 2 # threads.append(thread) # for thread in threads: # thread.start() # for current in threads: # current.join() for i in pols: print "doing pol: " + str(i.coefs()) reduc = Reduction() count = reduc.reduction(i.coefs()) r = str(i.coefs()) + ":" + str(count) f_save.write(r + '\n') del reduc
class DataHandler:
def __init__(self, config_path='openstack_dashboard/api/telemetry_api/environment.conf'):
self.__config = ConfigParser.ConfigParser()
self.__config.read(config_path)
#self.__ceilometer = CeilometerClient(self.__config)
self.__ceilometer = CeilometerClient()
self.__keystone = KeystoneClient(self.__config)
#self.__nova = NovaClient(self.__config)
self.__nova = NovaClient()
server = self.__config.get('Misc', 'dbserver')
user = self.__config.get('Misc', 'dbuser')
passwd = self.__config.get('Misc', 'dbpass')
database = self.__config.get('Misc', 'hostsdbname')
table = self.__config.get('Misc', 'hostsdbtable')
self.__hosts = ast.literal_eval(self.get_config().get('Openstack', 'Hosts'))
self.__hosts_db = HostDataHandler()
self.__benchmark_db = BenchmarkDataHandler(server, user, passwd)
self.__reduction = Reduction()
def get_config(self):
return self.__config
def projects(self):
return json.dumps(self.__keystone.projects)
def get_critical_hosts(self,instances_critical, information):
critical_hosts = []
for cpn in information:
cpn_name = cpn.keys()[0]
cpn_server_list = cpn[cpn_name]['vms'].keys()
for server in cpn_server_list:
if server in instances_critical:
critical_hosts.append(cpn_name)
return critical_hosts
def suggestion(self, list_not_ignore=[]):
project_list = [ project['name'] for project in json.loads(self.projects())] #returns the list of existing projects
compute_nodes_info_list = self.__nova.vm_info(project_list) #list of jsons - json correspond to a compute node information
instances_id_project = {} #dict instances_id : project_name
for compute_node_aux in compute_nodes_info_list:
try:
compute_name = compute_node_aux.keys()[0] #get the name of compute node to access information
for key in compute_node_aux[compute_name]['Info_project'].keys(): #key represent a project
#id represent de id of any instance present in a given project(key) and in the compute_name
for id in compute_node_aux[compute_name]['Info_project'][key]:
instances_id_project[id] = key
except Exception as excp:
return {"error": excp.message}
#get the list of critical instances (can't migrate them)
critical_instances = self.__nova.critical_instances(project_list)
#list of critical hosts
critical_cpn = self.get_critical_hosts(critical_instances,compute_nodes_info_list)
shutdown = {} #dict compute_node : True/False for shutdown
migrations = {} #dict with all migrations
compute_nodes_copy = compute_nodes_info_list[:] #copy to aux with the algorithm
owner_cpn_instance = {}
#for cp in compute_nodes_info_list:
# owner_cpn = cp.keys()[0]
# vms_cp = cp[owner_cpn]['vms'].keys()
# for v in vms_cp:
# owner_cpn_instance[v] = owner_cpn
#begin of algorithm
#cpn_data - compute node data
try:
for cpn_data in compute_nodes_info_list:
data = cpn_data.copy() #contains all information about a compute node
actual_cpn = cpn_data.keys()[0] #actual compute node (try migration for all instances)
#verify is the compute node has critical instances
if actual_cpn not in critical_cpn:
if( len( data[actual_cpn]['vms'].keys()) > 0 ):
instances_data = data[actual_cpn]['vms'].copy() #copy the list of vms
compute_nodes_copy.remove(cpn_data) #remove all data from compute node in the copy
migration_flag = False #flag to say if the instace can migrate to other compute node
migrations[actual_cpn] = {} #dict with all migrations for compute node
for instance_id in instances_data:
for other_cpn in compute_nodes_copy:
#verification - instance is not critical
if (other_cpn not in critical_cpn and instance_id not in critical_instances):
migration_flag = False
other_cpn_name = other_cpn.keys()[0]
if list_not_ignore == [] or actual_cpn in list_not_ignore:
if(other_cpn[other_cpn_name]['Livre'][0] >= instances_data[instance_id][0] and
other_cpn[other_cpn_name]['Livre'][1] >= instances_data[instance_id][1] and
other_cpn[other_cpn_name]['Livre'][2] >= instances_data[instance_id][2]):
from_cpn = None
migrate_to_cpn = None
if instance_id in owner_cpn_instance:
from_cpn = owner_cpn_instance[instance_id].keys()[0]
migrate_to_cpn = owner_cpn_instance[instance_id][from_cpn]
else:
from_cpn = actual_cpn
migrate_to_cpn = other_cpn_name
if actual_cpn == from_cpn or actual_cpn == migrate_to_cpn:
#update values of free resources
new_values = [other_cpn[other_cpn_name]['Livre'][0] - instances_data[instance_id][0],
other_cpn[other_cpn_name]['Livre'][1] - instances_data[instance_id][1],
other_cpn[other_cpn_name]['Livre'][2] - instances_data[instance_id][2]]
other_cpn[other_cpn_name]['Livre'] = new_values
#sending instance to the other host with all information
instances_other_cpn = other_cpn[other_cpn_name]['vms']
instances_other_cpn[instance_id] = instances_data[instance_id]
other_cpn[other_cpn_name]['vms'] = instances_other_cpn
other_cpn[other_cpn_name]['nomes'][instance_id] = data[actual_cpn]['nomes'][instance_id]
migration_flag = True
migrations[actual_cpn][instance_id] = [ other_cpn_name , cpn_data[actual_cpn]['nomes'].get(instance_id) ,instances_id_project[instance_id]]
#update future owner of instance
owner_cpn_instance[instance_id] = {actual_cpn : other_cpn_name}
else:
continue
else:
break
else:
migration_flag = False
else:
continue
#update owner
if migration_flag == False:
if instance_id in owner_cpn_instance:
if( owner_cpn_instance[instance_id].has_key(actual_cpn) or actual_cpn in owner_cpn_instance[instance_id].values()):
continue
else:
migrations[actual_cpn][instance_id] = None
shutdown[actual_cpn] = False
if not actual_cpn in shutdown:
shutdown[actual_cpn] = True
else:
compute_nodes_copy.remove(cpn_data)
shutdown[actual_cpn] = True
continue
else:
compute_nodes_copy.remove(cpn_data)
shutdown[actual_cpn] = False
except Exception as excp2:
return {"error in algorithm suggestion":excp2.message}
for host_key in migrations.keys():
if None in migrations[host_key].values():
migrations[host_key] = {}
output = {} #json output with all data
output['Hosts'] = shutdown
output['Migracoes'] = migrations
#recomendation = self.remove_duplicated_migrations(output)
return output
#Not using anymore
#def remove_duplicated_migrations(self, output):
# result = output
# for compute_node in result['Migracoes'].keys():
# for server in result['Migracoes'][compute_node].keys():
# if not self.__nova.verify_host_has_server(compute_node,server):
# result['Migracoes'][compute_node].pop(server)
# return result
def cpu_util_from(self, timestamp_begin=None, timestamp_end=None, resource_id=None):
return json.dumps(self.__ceilometer.get_cpu_util(timestamp_begin, timestamp_end, resource_id))
def cpu_util_flavors(self, timestamp_begin=None, timestamp_end=None):
data = self.__ceilometer.get_cpu_util_flavors(timestamp_begin, timestamp_end)
ret = analytics.recommendations.recomenda_flavor(data)
return json.dumps(ret)
def network_incoming_bytes_rate_from(self, timestamp_begin=None, timestamp_end=None, resource_id=None):
return json.dumps(self.__ceilometer.get_network_incoming_bytes_rate(timestamp_begin, timestamp_end, resource_id))
def network_outgoing_bytes_rate_from(self, timestamp_begin=None, timestamp_end=None, resource_id=None):
return json.dumps(self.__ceilometer.get_network_outgoing_bytes_rate(timestamp_begin, timestamp_end, resource_id))
def projects_with_instances_and_cpu_util(self):
projects = self.__keystone.tenants
ret = { 'name' : 'cloud', 'children' : [] }
for p in projects:
proj = { 'name' : p.name, 'children' : [] }
instances = self.__nova.instances(p.name)
for i in instances:
proj['children'].append({ 'resource_id' : i.id, 'instance_name' : i.name })
ret['children'].append(proj)
return ret
def alarms_history(self, timestamp_begin=None, timestamp_end=None):
return self.__ceilometer.get_alarms_history(timestamp_begin, timestamp_end)
def add_alarm(self, name, resource, threshold, operator, period, ev_period, send_mail, email_admin, instance=""):
return self.__ceilometer.set_alarm(name, resource, threshold, operator, period, ev_period, send_mail, email_admin, instance)
def alarm_email(self, data_requested):
alarm_id = ast.literal_eval(data_requested)['alarm_id']
userId = self.__ceilometer.get_alarm_userid(alarm_id)
projectId = self.__ceilometer.get_alarm_projectid(alarm_id)
userEmail = self.__keystone.get_user_email(userId, projectId)
copy_admin = self.__ceilometer.get_alarm_email_status(alarm_id)
adminEmail = self.__keystone.get_user_email(self.__keystone.get_user(projectId,'admin'),projectId)
if 'True' in copy_admin[0] and 'True' in copy_admin[1] :
send_email('*****@*****.**',
[adminEmail],
[],
'Alert Telemetry Cloud',
'Email disparado pelo alarme!!!',
'*****@*****.**',
'4n4lyt1cs')
send_email('*****@*****.**',
[userEmail],
[],
'Alert Telemetry Cloud',
'Email disparado pelo alarme!!!',
'*****@*****.**',
'4n4lyt1cs')
elif 'True' in copy_admin[0] and 'False' in copy_admin[1]:
send_email('*****@*****.**',
[userEmail],
[],
'Alert Telemetry Cloud',
'Email disparado pelo alarme!!!',
'*****@*****.**',
'4n4lyt1cs')
elif 'False' in copy_admin[0] and 'True' in copy_admin[1]:
send_email('*****@*****.**',
[adminEmail],
[],
'Alert Telemetry Cloud',
'Email disparado pelo alarme!!!',
'*****@*****.**',
'4n4lyt1cs')
def alarm_description(self):
return self.__ceilometer.get_alarm_parameters()
def delete_alarm(self, alarm_id):
return json.dumps(self.__ceilometer.delete_alarms(alarm_id))
def hosts_cpu(self, timestamp_begin, timestamp_end):
return self.__hosts_db.get_data_db('Cpu_Util', timestamp_begin, timestamp_end)
def hosts_memory(self, timestamp_begin, timestamp_end):
return self.__hosts_db.get_data_db('Memory', timestamp_begin, timestamp_end)
def hosts_disk(self, timestamp_begin, timestamp_end):
return self.__hosts_db.get_data_db('Disk', timestamp_begin, timestamp_end)
def hosts_network(self, timestamp_begin, timestamp_end):
return self.__hosts_db.get_data_db('Network', timestamp_begin, timestamp_end)
def host_metrics(self, project):
return self.__nova.metrics(project)
def host_aggregates(self, project):
return self.__nova.host_aggregates(project)
def resource_host(self, host):
return self.__nova.resource_host(host)
def hosts_recommendation(self, r_cpu, r_memory , r_disk):
resource = []
ret = {}
r_cpu = json.loads(r_cpu)
r_memory = json.loads(r_memory)
r_disk = json.loads(r_disk)
for host in r_cpu:
host_http = host["host_address"]
if host["data"] is None:
continue
for data in host["data"]:
resource.append(data["data"])
resource = sorted(resource)
if(len(resource)%2 == 0):
index = len(resource)/2
mediana = (resource[index-1] + resource[index+1])/2
else:
mediana = resource[int(math.ceil(len(resource)/2))]
if mediana >= 95:
ret[host_http] ="sobrecarregado"
else:
resource = []
for host_mem in r_memory:
if(host["host_address"] == host_mem["host_address"]):
for data in host_mem["data"]:
for value in json.loads(data["data"]):
resource.append(value["percent"])
resource = sorted(resource)
if(len(resource)%2 == 0):
index = len(resource)/2
mediana = (resource[index-1] + resource[index+1])/2
else:
mediana = resource[int(math.ceil(len(resource)/2))]
if mediana >= 95:
ret[host_http] ="sobrecarregado"
else:
ret[host_http] ="normal"
return json.dumps(ret)
#return json.dumps(cpu)
def instances_from_host(self, host_name):
attr_host = 'OS-EXT-SRV-ATTR:host'
ret = []
projects = self.__keystone.projects
for project in projects:
instances = self.__nova.instances(project['name'])
for instance in instances:
print instance._info
if instance._info[attr_host] == host_name:
ret.append({'instance_name' : instance.name, 'instance_id' : instance.id})
return ret
def migrate_to_host(self, project_name, host_name, instance_id):
#host_vm = self.__nova.vm_hostname(project_name,instance_id)
#attr_host = 'OS-EXT-SRV-ATTR:host'
#if host_vm._info[attr_host] == host_name:
# raise MigrateException(400,"Migracao para o mesmo destino")
#elif host_vm._info[attr_host] == 'truta' and host_name != 'truta':
# raise MigrateException(500,"Migracao de host para compute node")
#else:
try:
retorno = self.__nova.vm_migration(project_name,host_name,instance_id)
except Exception as a:
return {"erro":a.message}
return {"status":"success"}
def get_benchmark_bd(self):
ret = self.__benchmark_db.get_data_db()
return ret
def start_instance_bench(self, project, host):
return self.__nova.start_instance_bench(project, host)
def get_benchmark(self, project, host):
benchmark_ip = self.__nova.get_benchmark_ip(project, host)
data = requests.get('http://'+benchmark_ip+':5151/get_benchmarking')
return data.json()
def get_benchmark_status(self, project, host):
benchmark_ip = self.__nova.get_benchmark_ip(project, host)
print benchmark_ip
data = requests.get('http://'+benchmark_ip+':5151/get_status')
return data.text
def repeat_benchmark(self, project):
benchmark_ip = self.__nova.get_benchmark_ip(project)
data = requests.get('http://'+benchmark_ip+':5151/start_benchmarking')
return data.text
def remove_benchmark_instance(self, host):
id = self.__nova.benchmark_id(host)
if id == None:
return "sem instancia benchmark"
else:
remove = self.__nova.remove_instance(id)
return remove
def hosts_aggregation_cpu(self, timestamp_begin=None, timestamp_end=None):
ret = []
cpu_data = self.hosts_cpu(timestamp_begin, timestamp_end)
aggregates = self.__nova.host_aggregates('admin')
for aggregate in aggregates:
result = []
host_address = aggregate["host_address"]
for host in host_address:
host_name = self.__nova.server_name_by_ip(host)
host_cpu = self.__nova.resource_host(host_name)["cpu"]
for data in cpu_data:
if(data["host_address"]==host):
convert = []
for cpu_percent in data["data"]:
cpu_percent["data"] = (1 - cpu_percent["data"]/100.0)* host_cpu
convert.append(cpu_percent)
if(len(result)==0):
result = convert
else:
if(len(result) > len(convert)):
result = result[0:len(convert)]
for i in range(len(result)):
value = result[i]
value["data"] = value["data"] + (convert[i])["data"]
result[i] = value
break
ret.append({"Aggregate":aggregate["name"], "data":result})
return json.dumps(ret)
def hosts_aggregation_memory(self, timestamp_begin=None, timestamp_end=None):
ret = []
memory_data = self.hosts_memory(timestamp_begin, timestamp_end)
aggregates = self.__nova.host_aggregates('admin')
for aggregate in aggregates:
result = []
host_address = aggregate["host_address"]
aggregate_memory = self.__nova.resource_aggregates(aggregate['name'])['memory_mb']
for host in host_address:
host_name = self.__nova.server_name_by_ip(host)
host_memory = self.__nova.resource_host(host_name)["memory_mb"]
for data in memory_data:
if(data["host_address"]==host):
convert = []
for memory_percent in data["data"]:
memory_percent['data'] = ((json.loads(memory_percent['data'])[0]['percent']/100.0 )*host_memory)/aggregate_memory
convert.append(memory_percent)
if(len(result)==0):
result = convert
else:
if(len(result) > len(convert)):
result = result[0:len(convert)]
for i in range(len(result)):
value = result[i]
value["data"] = (value["data"] + (convert[i])["data"])
result[i] = value
break
ret.append({"Aggregate":aggregate["name"], "data":result})
return json.dumps(ret)
def hosts_aggregation_disk(self, timestamp_begin=None, timestamp_end=None):
ret = []
disk_data = self.hosts_disk(timestamp_begin, timestamp_end)
aggregates = self.__nova.host_aggregates('admin')
for aggregate in aggregates:
result = []
host_address = aggregate["host_address"]
aggregate_disk = self.__nova.resource_aggregates(aggregate['name'])['disk']
for host in host_address:
host_name = self.__nova.server_name_by_ip(host)
host_disk = self.__nova.resource_host(host_name)["disk_gb"]
for data in disk_data:
if(data["host_address"]==host):
convert = []
for disk_percent in data["data"]:
disk_percent['data'] = ((json.loads(disk_percent['data'])[0]['percent']/100)*host_disk)/aggregate_disk
convert.append(disk_percent)
if(len(result)==0):
result = convert
else:
if(len(result) > len(convert)):
result = result[0:len(convert)]
for i in range(len(result)):
value = result[i]
value["data"] = value["data"] + (convert[i])["data"]
result[i] = value
break
ret.append({"Aggregate":aggregate["name"], "data":result})
return json.dumps(ret)
def hosts_aggregation_network(self, timestamp_begin=None, timestamp_end=None):
ret = []
hosts = self.get_compute_nodes_ips()
network_data = self.points_reduction_by_server_network(timestamp_begin, timestamp_end, hosts)
aggregates = self.__nova.host_aggregates('admin')
for aggregate in aggregates:
result_incoming = []
result_outgoing = []
host_address = aggregate["host_address"]
for host in host_address:
for data in network_data:
if(data["host_address"]==host):
convert_incoming = []
for network_incoming_rate in data["incoming_rate"]:
convert_incoming.append(network_incoming_rate)
if(len(result_incoming)==0):
result_incoming = convert_incoming
else:
if(len(result_incoming) > len(convert_incoming)):
result_incoming = result_incoming[0:len(convert_incoming)]
for i in range(len(result_incoming)):
value = result_incoming[i]
value["net_bytes_recv"] = value["net_bytes_recv"] + (convert_incoming[i])["net_bytes_recv"]
result_incoming[i] = value
convert_outgoing = []
for network_outgoing_rate in data["outgoing_rate"]:
convert_outgoing.append(network_outgoing_rate)
if(len(result_outgoing)==0):
result_outgoing = convert_outgoing
else:
if(len(result_outgoing) > len(convert_outgoing)):
result_outgoing = result_outgoing[0:len(convert_outgoing)]
for i in range(len(result_outgoing)):
value = result_outgoing[i]
value["net_bytes_sent"] = value["net_bytes_sent"] + (convert_outgoing[i])["net_bytes_sent"]
result_outgoing[i] = value
break
ret.append({"Aggregate":aggregate["name"], 'incoming_rate': result_incoming, 'outgoing_rate': result_outgoing})
return json.dumps(ret)
def points_reduction_by_server_cpu(self, timestamp_begin, timestamp_end, hosts):
data = []
old_data = self.hosts_cpu(timestamp_begin, timestamp_end)
key = 'data'
if len(old_data)==0 or len(old_data[0][key]) <= 3:
result = old_data
else:
for host in range(len(hosts)):
dict_host = {}
dict_host["host_address"] = hosts[host]
dict_host['data'] = self.__reduction.points_reduction(old_data[host]['data'],key)
data.append(dict_host)
result = data
return result
def points_reduction_by_server_memory(self, timestamp_begin, timestamp_end, hosts):
data = []
old_data = self.hosts_memory(timestamp_begin, timestamp_end)
key = 'data'
if len(old_data)==0 or len(old_data[0][key]) <= 3:
result = old_data
else:
for host in range(len(hosts)):
dict_host = {}
dict_host["host_address"] = hosts[host]
dict_host['data'] = self.__reduction.points_reduction_for_percent(old_data[host]['data'],key)
data.append(dict_host)
result = data
return result
def points_reduction_by_server_disk(self, timestamp_begin, timestamp_end, hosts):
data = []
old_data = self.hosts_disk(timestamp_begin, timestamp_end)
if len(old_data)==0 or len(old_data[0]['data']) <= 3:
result = old_data
else:
for host in range(len(hosts)):
dict_host = {}
dict_host["host_address"] = hosts[host]
dict_host['data'] = self.__reduction.points_reduction_disk(old_data[host]['data'])
data.append(dict_host)
result = data
return result
def points_reduction_by_server_network(self, timestamp_begin, timestamp_end, hosts):
data = []
old_data = self.hosts_network(timestamp_begin, timestamp_end)
for host in old_data:
single_host_data = {'host_address': host['host_address'], 'incoming_rate': [], 'outgoing_rate': []}
if len(host['data']) > 1:
sample = host['data'][0]
for sample_index in range(len(host['data']) - 1):
sample_index += 1
network_data = json.loads(sample['data'])[0]
before_timestamp = datetime.strptime(sample['timestamp'], '%Y-%m-%dT%H:%M:%S')
before_net_bytes_recv = network_data['net_bytes_recv']
before_net_bytes_sent = network_data['net_bytes_sent']
next_sample = host['data'][sample_index]
network_data = json.loads(next_sample['data'])[0]
after_timestamp = datetime.strptime(next_sample['timestamp'], '%Y-%m-%dT%H:%M:%S')
after_net_bytes_recv = network_data['net_bytes_recv']
after_net_bytes_sent = network_data['net_bytes_sent']
timestamp_delta = (after_timestamp - before_timestamp).total_seconds()
net_bytes_recv_delta = after_net_bytes_recv - before_net_bytes_recv
net_bytes_sent_delta = after_net_bytes_sent - before_net_bytes_sent
if (net_bytes_recv_delta < 0) or (net_bytes_sent_delta < 0):
continue
else:
sample = next_sample
net_bytes_recv_delta = net_bytes_recv_delta / timestamp_delta
net_bytes_sent_delta = net_bytes_sent_delta / timestamp_delta
single_host_data['incoming_rate'].append({'timestamp': sample['timestamp'], 'net_bytes_recv': net_bytes_recv_delta})
single_host_data['outgoing_rate'].append({'timestamp': sample['timestamp'], 'net_bytes_sent': net_bytes_sent_delta})
single_host_data['incoming_rate'] = self.__reduction.points_reduction(single_host_data['incoming_rate'], 'net_bytes_recv')
if math.isnan(single_host_data['incoming_rate'][-1]['net_bytes_recv']):
single_host_data['incoming_rate'].pop()
single_host_data['outgoing_rate'] = self.__reduction.points_reduction(single_host_data['outgoing_rate'], 'net_bytes_sent')
if math.isnan(single_host_data['outgoing_rate'][-1]['net_bytes_sent']):
single_host_data['outgoing_rate'].pop()
data.append(single_host_data)
return data
def points_reduction_vm(self, timestamp_begin,timestamp_end,resource_id):
old_data = json.loads(self.cpu_util_from(timestamp_begin,timestamp_end,resource_id))
key2 = "cpu_util_percent"
data = self.__reduction.points_reduction(old_data,key2)
return data
def points_reduction_vm_network_incoming(self, timestamp_begin,timestamp_end,resource_id):
old_data = json.loads(self.network_incoming_bytes_rate_from(timestamp_begin,timestamp_end,resource_id))
key2 = "network_incoming_bytes_rate"
data = self.__reduction.points_reduction(old_data,key2)
return data
def points_reduction_vm_network_outgoing(self, timestamp_begin,timestamp_end,resource_id):
old_data = json.loads(self.network_outgoing_bytes_rate_from(timestamp_begin,timestamp_end,resource_id))
key2 = "network_outgoing_bytes_rate"
data = self.__reduction.points_reduction(old_data,key2)
return data
def vm_info(self):
ret = []
project = []
for project_data in json.loads(self.projects()):
project.append(project_data["name"])
for project_name in project:
informations = self.__nova.vm_info(project)
vms_data = {}
for node in informations:
for node_name in node.keys():
vms_name = (node[node_name])['nomes']
for key in vms_name.keys():
vms_data[key] = vms_name[key]
ret.append({project_name:vms_data})
return ret
def instances_by_project(self):
project = []
for project_data in json.loads(self.projects()):
project.append(project_data["name"])
informations = self.__nova.vm_info(project)
vms_data = {}
for node in informations:
for node_name in node.keys():
project_name = (node[node_name])['Info_project']
vms_name = (node[node_name])['nomes']
for key in project_name.keys():
for instance_id in project_name[key]:
if not (key in vms_data.keys()):
vms_data[key] = {}
vms_data[key][instance_id] = vms_name[instance_id]
else:
vms_data[key][instance_id] = vms_name[instance_id]
return vms_data
def vcpus_for_aggregate(self, project):
return json.dumps(self.__nova.vcpus_for_aggregate(project))
def create_snapshot(self, instance_id):
self.__nova.create_snapshot(instance_id)
def suspend_instance(self, instance_id):
self.__nova.suspend_instance(instance_id)
def get_computenodes_names(self):
return self.__nova.list_compute_nodes()
def get_compute_nodes_ips(self):
ips = []
all_hosts = self.get_hosts('compute_node')
for host in all_hosts:
ips.append(host.get_ip())
return ips
def get_host_availability_metrics(self, timestamp_begin, timestamp_end):
calculator = HostMetricsCalculator()
return calculator.get_host_availability_metrics(timestamp_begin, timestamp_end)
def get_services_status(self, host, timestamp_begin, timestamp_end):
return self.__hosts_db.get_service_status_db(host, timestamp_begin, timestamp_end)
def points_reduction_services_status(self, host, timestamp_begin, timestamp_end):
data = self.get_services_status(host, timestamp_begin, timestamp_end)
output = {}
for key in data.keys():
output[key] = self.__reduction.points_reduction_for_step(data[key], 'status')
return output
def get_host_status(self, host, timestamp_begin, timestamp_end):
return self.__hosts_db.get_host_status_db(host, timestamp_begin, timestamp_end)
def points_reduction_host_status(self, host, timestamp_begin, timestamp_end):
data = self.get_host_status(host, timestamp_begin, timestamp_end)
return {'data': self.__reduction.points_reduction_for_step(data['data'], 'status')}
def get_hosts(self, type=None):
return hosts_from_dict_list(self.__hosts, type)
def am_model(self, do_SNF=True, verbose=False):
st = time()
pt = time()
reduction = self.reduction()
if verbose:
print('Vector field reduction calculated in {:.3f}s'.format(time() - pt))
pt = time()
V = create_vector_field(reduction.dst)
if verbose:
print('Reduced vector field built in {:.3f}s'.format(time() - pt))
while V != 0:
pt = time()
reduction = V.reduction() * reduction
if verbose:
print('Vector field reduction calculated in {:.3f}s'.format(time() - pt))
pt =time()
V = create_vector_field(reduction.dst)
if verbose:
print('Reduced vector field built in {:.3f}s'.format(time() - pt))
if do_SNF:
# Compute SNF if necessary
if reduction.dst.d != 0:
pt = time()
snf_reduction = reduction.dst.SNF_reduction()
if verbose:
print('SNF reduction calculated in {:.3f}s'.format(time() - pt))
pt = time()
reduction = snf_reduction * reduction
if verbose:
print('Final reduction built in {:.3f}s'.format(time() - pt))
# Make all differentials have positive coefficient
pt = time()
f_sign = ChainMap(reduction.dst)
h_sign = ChainMap(reduction.dst, degree=1)
for sigma in reduction.dst:
co_d = reduction.dst.d.T
c = co_d(sigma)
if c:
tau = next(iter(c))
if c[tau] < 0:
f_sign.set_image(sigma, -sigma)
else:
f_sign.set_image(sigma, sigma)
else:
f_sign.set_image(sigma, sigma)
new_dst = ChainComplex(cell_class=reduction.dst.cell_class)
for p in reduction.dst.dimensions:
new_dst[p] = reduction.dst[p]
new_d = ChainMap(new_dst, degree=-1)
for p, M in reduction.dst.d.matrices.items():
new_d[p] = np.abs(M)
new_dst.d = new_d
reduction = Reduction(reduction.dst, new_dst, f_sign, f_sign, h_sign) * reduction
if verbose:
print('Signs adjusted in {:.3f}s'.format(time() - pt))
print('Full time {:.3f}s'.format(time() - pt))
return reduction
def getSummary(): #actually calculate summary
opLabel.pack()
opEntry.pack()
opEntry.config(state='normal' , bg="white" , fg="black")
opEntry.delete(1.0,END)
opEntry.insert(INSERT,"Generating Summary...")
global endTime1,endTime2
startTime = time.time()
text = ipEntry.get("1.0", END)
ps = PorterStemmer()
red_ra=redRatio.get()/100.00
if var1.get():
words = text.split()
for w in words:
text += ps.stem(w)
text += " "
if var6.get()==1:
r = Reduction() #object of class Red..
reduced_text = r.reduce(text, red_ra)
op=' '.join(reduced_text)
global opTA
opTA=op
elif var6.get()==2:
m = modelVC.summary()
op = m.summarize(text,red_ra)
global opPA
opPA=op
if var2.get()==1:
op=shortForm.shortF(op)
opEntry.delete(1.0,END)
opEntry.insert(INSERT, op)
if var6.get()==1:
endTime1 = time.time() - startTime
print("Time taken: %f secs" %endTime1)
if var4.get():
tkMessageBox.showinfo("Statistics", "Time taken to complete: %f secs" %endTime)
if var6.get()==2:
endTime2 = time.time() - startTime
print("Time taken: %f secs" %endTime2)
if var4.get():
tkMessageBox.showinfo("Statistics", "Time taken to complete: %f secs" %endTime)
if var3.get() == 1:
file_f = open("log.txt", "a")
lenText=len(text)
file_f.write("\nInput length: %d" %lenText)
lenText = len(op)
file_f.write("\tSummary length: %d" % lenText)
file_f.write('\nStemming: %d' %var1.get())
file_f.write('\tShort Forms: %d' %var2.get())
file_f.write('\tReduction Ratio: %d percent' % redRatio.get())
file_f.write('\tModel: %d ' % var6.get())
if var6.get()==1:
file_f.write("\nTime taken: %f secs\n" %endTime1)
if var6.get()==2:
file_f.write("\nTime taken: %f secs\n" %endTime2)
file_f.close()
