def deleteevents(self, ids=None, outputfile=None):
status = 0
table_name = 'ipmievents'
# delet all event records if no ids provided
sqlcommand = "DELETE from {0}".format(table_name)
# delete selected event records
if ids is not None:
sqlcommand += " WHERE id IN ({0})".format(ids)
if outputfile is not None:
reply = "{'" + table_name + "':["
else:
reply = ''
try:
ipmidata = DataModel().ExecuteRawQueryStatement(sqlcommand)
except sqlalchemy.exc.OperationalError as ex:
self.logger.error(traceback.format_exc())
if outputfile is not None:
reply = reply + '{"exception":' + str(ex) + '}'
status = 1
except Exception as ex:
self.logger.error(traceback.format_exc())
if outputfile is not None:
reply = reply + '{"exception":' + str(ex) + '}'
status = 1
if outputfile is not None:
reply = reply + "\n]}"
self.logger.info("status=0 json={0}".format(reply))
return status, reply
def train_lstm(model_filename, weights_filename, l1_d=128, l2_d=128, b_s=128, bi=False):
"""
Trains a 2 layer lstm and saves the model in the files specified.
Args:
model_filename: The filename to save the model
weights_filename: The filename to save the weights
l1_d: Layer one dimensions
l2_d: Layer two dimensions
b_s: Batch size
Returns:
model: The lstm model fit on the training data
"""
dm=DataModel()
texts, labels=dm.get_train_data()
word_index, data=utils.get_word_index(texts)
x_train, y_train, x_val, y_val, _, _=utils.get_train_val_test_data(
data, labels)
word_embeddings=utils.get_glove_embeddings()
embedding_matrix=utils.get_embedding_matrix(word_embeddings, word_index)
if bi:
model=_bi_lstm(embedding_matrix, x_train, y_train, x_val,
y_val, 3, word_index, l1_d, l2_d, b_s)
else:
model=_lstm(embedding_matrix, x_train, y_train, x_val,
y_val, 3, word_index, l1_d, l2_d, b_s)
save_model(model, model_filename, weights_filename)
return model
def run_lstm_on_test_data(model_filename, weights_filename, kaggle_filename):
dm = DataModel()
test_x, test_id = dm.get_test_data()
print(test_x.shape)
_, data = utils.get_word_index(test_x)
# word_embeddings = utils.get_glove_embeddings()
# _ = utils.get_embedding_matrix(word_embeddings, word_index)
y = utils.load_model_and_evaluate(model_filename, weights_filename, data)
utils.write_kaggle_file(y, test_id, kaggle_filename)
def main(args):
dm = DataModel(args.gig_file, args.chat_file)
dm.read_data()
exp = Experimenter(dm)
if args.classify is True:
scores = exp.classify_gigs()
if args.feature_values is True:
scores = exp.evaluate_feature_values()
return dm
def main(argv):
train_count = -1
if (len(argv) > 0):
train_count = int(argv[0])
dm = DataModel()
dm.get_data(train_count)
# (training, feature_names) = get_rich_featured_training(dm,lines)
print(len(dm.data.keys()))
print(len(dm.train))
print(len(dm.test))
def main(args):
dm = DataModel(args.train_file)
dm.read_train_data()
exp = Experimenter(dm)
distances = [x.get_distance() for x in dm.data]
print(max(distances))
print(min(distances))
print(stats.mean(distances))
t1 = time.time()
t2 = time.time()
timeused = t2 - t1
logging.getLogger(LOGGER).info('Time used in experiment (hour:min:sec): %d:%d:%d' % \
(timeused/3600, timeused/60, timeused%60))
return exp
def accuracy_test(model_filename, weights_filename):
dm = DataModel()
x_train, y_train = dm.get_train_data()
train_samples = int(0.8 * len(x_train))
x_test = x_train[train_samples:]
y_test = y_train[train_samples:]
_, data = utils.get_word_index(x_test)
labels = to_categorical(np.asarray(y_test))
# word_embeddings = utils.get_glove_embeddings()
# _ = utils.get_embedding_matrix(word_embeddings, word_index)
md = utils.load_model(model_filename, weights_filename)
result = md.evaluate(data, labels)
print('\nTest loss:', result[0])
print('Test accuracy:', result[1])
def listevents(self, outputfile):
"""
Get common DB entries for table ipmievents, stored by captureevents.
:param: outputfile - JSON output file, if None - special format to stdout.
:returns: status, reply
:raises: None
"""
table_name = 'ipmievents'
if outputfile is not None:
reply = "{'" + table_name + "':["
else:
reply = ''
sqlcommand = "SELECT id, datetime, event, details, status from {0}".format(table_name)
try:
ipmidata = DataModel().ExecuteRawQueryStatement(sqlcommand)
except sqlalchemy.exc.OperationalError as ex:
self.logger.error(traceback.format_exc())
if outputfile is not None:
reply = reply + '{"exception":' + str(ex) + '}'
except Exception as ex:
self.logger.error(traceback.format_exc())
if outputfile is not None:
reply = reply + '{"exception":' + str(ex) + '}'
# Print out the ipmievents data in the common database.
first = True
ipmidata_keys = ipmidata.keys()
for entry in ipmidata:
if outputfile is not None:
reply = reply + ("\n {" if first else "},\n {")
first = second = False
for v in ipmidata_keys:
reply = ((reply + ",") if second else reply) + "'" + str(v) + "':'" + str(entry[v]) + "'"
second = True
self.logger.info(entry)
else:
print(entry[0], '|', entry[1], '|', entry[2], '|', entry[3], '|', entry[4])
if outputfile is not None:
if ipmidata is not None:
reply = reply + '}'
if outputfile is not None:
reply = reply + "\n]}"
self.logger.info("status=0 json={0}".format(reply))
return 0, reply
def __init__(self, parent=None):
super().__init__(parent)
self.setupUi(self)
self.mplWidget = MplWidget.warp_a_widget(self.matplotlibWidget)
self.model = DataModel(self.mplWidget.canvas)
self.tableView.setModel(self.model)
validator = QDoubleValidator(0, 1000, 5)
self.leastIntLineEdit.setValidator(validator)
self.maxZDiffLineEdit.setValidator(validator)
self.leastIntLineEdit.setText('200')
self.maxZDiffLineEdit.setText('1')
# Signals-Slots
self.actionImportData.triggered.connect(self.setSampleFile)
self.startFindButton.clicked.connect(self.startSearch)
def sel_timeupdated(self):
# Update the timeupdated table with domain ipmievents for datetime now.
domain_name = 'ipmievents'
table_name = 'timeupdated'
sqlcommand = ("UPDATE {0} SET datetime=DATETIME('now') WHERE domain='{1}'"
.format(table_name, domain_name))
try:
ipmidata = DataModel().ExecuteRawQueryStatement(sqlcommand)
except sqlalchemy.exc.OperationalError as ex:
self.logger.error(traceback.format_exc())
print('{"exception":' + str(ex) + '}')
return 1
except Exception as ex:
self.logger.error(traceback.format_exc())
print('{"exception":' + str(ex) + '}')
return 1
return 0
def generate_w2c_word_embeddings():
"""
Generates word 2 vector embeddings.
"""
dm = DataModel()
train_x, _ = dm.get_train_data()
w2c = Word2Vec(method="skipgram",
corpus=list(train_x),
window_size=5,
n_hidden=128,
n_epochs=3,
learning_rate=0.08)
W1, W2, loss_vs_epoch = w2c.run()
pkl_dump = [W1, W2, loss_vs_epoch]
with open('embeddings.pickle', 'wb') as handle:
pickle.dump(pkl_dump, handle)
def updateTargetHBAs(self):
self.logger.info("ENTERED updateTargetHBAs")
try:
db = DataModel()
table = 'target_hbas'
self.logger.info("clearing table: {0}".format(table))
cmd = 'DELETE FROM {0} WHERE nodeId={1}'.format(table, self.nodeID)
db.ExecuteRawQueryStatement(cmd)
for wwn in self.__get_target_hba_wwns():
cmd = ("INSERT INTO {0} (wwn, created_at, updated_at, nodeId) VALUES ('{1}',"
"(select datetime('now')), (select datetime('now')), {2});".format(table, wwn, self.nodeID))
self.logger.info("Executing the following command: {0}".format(cmd))
db.ExecuteRawQueryStatement(cmd)
self.logger.info("Exiting updateTargetHBAs")
return True
except Exception as ex:
self.logger.error("An exception occurred while adding new wwn to TargetHBA table: {0}".format(ex))
return False
def main(args):
dm = DataModel(args.data_file)
dm.read_data(to_read_count=10000)
exp = Experimenter(dm, \
process_datamodel=True, \
serialise=False)
t1 = time.time()
exp.perform_multiclass_experiment(
pred_mode=INDEPENDENT,
use_exclusion=True,
need_to_extract_features=True,
prediction_file=
'../results/predictions_multiclass_independent_englishonly_legibleonly_wordunibigram_chartrigram_10000.csv',
result_file=
'../results/results_multiclass_independent_englishonly_legibleonly_wordunibigram_chartrigram_10000.txt',
english_only=True,
legible_only=True)
t2 = time.time()
timeused = t2 - t1
logging.getLogger(LOGGER).info('Time used in experiment (hour:min:sec): %d:%d:%d' % \
(timeused/3600, timeused/60, timeused%60))
return exp
def listhbas():
"""
\b
NAME
listhbas - will show a list of hbas
SYNOPSIS
listhbas [OPTIONS]
DESCRIPTION
Will list out available hbas seen by the LightSpeed
No options are available for this command
"""
targetHBAStable = DataModel()
cmd = "SELECT id,wwn from target_hbas"
q = targetHBAStable.GetListOfQuery(cmd)
cli_logger.info('List of available target HBAs: ')
cli_logger.info(
'---------------------------------------------------------------------------'
)
for row in q:
cli_logger.info('ID: {0} | HBA: {1}'.format(row[0], row[1]))
cli_logger.info(
'---------------------------------------------------------------------------'
)
def getconfiguration(self):
"""
Fetch IPMI configuration from database
:param: none
:returns: status, reply
:raises: None
"""
status = 1
obj = PxJSON("Unable to obtain IPMI configuration")
try:
table_name = 'systemsetups'
res = DataModel().ExecuteRawQueryStatement("SELECT ipmi_connection_type, ipmi_address, ipmi_netmask, ipmi_gateway, ipmi_vlan from {0}".format(table_name))
reply = {}
for row in res:
self.logger.info(row)
reply[PxJSON.CONNECTIONTYPE] = row['ipmi_connection_type']
reply[PxJSON.IPV4] = row['ipmi_address']
reply[PxJSON.NETMASK] = row['ipmi_netmask']
reply[PxJSON.GATEWAY] = row['ipmi_gateway']
reply[PxJSON.VLAN] = row['ipmi_vlan'] if row['ipmi_vlan'] != 0 else 'undefined'
obj.setroute(PxJSON.IPMI_INFO, reply)
obj.setsuccess()
status = 0
except sqlalchemy.exc.OperationalError as ex:
self.logger.error(traceback.format_exc())
obj.internal({"exception": str(ex)})
except Exception as ex:
self.logger.error(traceback.format_exc())
obj.internal({"exception": str(ex)})
self.logger.info("status={0} json={1}".format(status, obj.getjsonpretty()))
return status, obj.getjson()
def updateInitiatorHBAs(self):
# This function will update all HBA related information. There are two types present:
# a) Target HBA's
# QLE which will operate in Target mode. Target HBA's will need to be configured in database
# and LIO config.
# b) Initiator HBA's database entries only. (Some could be QLE.)
self.logger.info("ENTERED updateInitiatorHBAs")
try:
table = 'initiator_hbas'
db = DataModel()
self.logger.info("clearing table: {0}".format(table))
cmd = 'DELETE FROM {0} WHERE nodeId={1}'.format(table, self.nodeID)
db.ExecuteRawQueryStatement(cmd)
self.logger.info("Now adding the provided wwns.")
for wwn in self.__get_initiator_hba_wwns():
self.logger.info("Adding the wwn {0} to the table".format(wwn))
cmd = ("INSERT INTO {0} (wwn, created_at, updated_at, nodeId) VALUES ('{1}', (select datetime('now')), "
"(select datetime('now')), {2});".format(table, wwn, self.nodeID))
self.logger.info("Executing the command: {0}".format(cmd))
db.ExecuteRawQueryStatement(cmd)
return True
except Exception as ex:
self.logger.error("An exception occurred while updating the initiator table: {0}".format(ex))
return False
def setconfiguration2(self, obj):
"""
Set IPMI configuration in database
:param obj - PxJSON object
:returns: status, PxJSON
:raises: None
Any attribute set to None, is set to null in the database.
"""
status = 1
tmpobj = PxJSON("Unable to set IPMI configuration")
tmpstatus, tmpjson = self.getconfiguration()
tmpobj.setjson(json.loads(tmpjson))
if tmpstatus != 0 or not tmpobj.issuccess():
obj.internal(tmpobj.getinternal())
return status, obj
old_connectiontype = tmpobj.getroute(PxJSON.IPMI_INFO)[PxJSON.CONNECTIONTYPE]
old_ipaddress = tmpobj.getroute(PxJSON.IPMI_INFO)[PxJSON.IPV4]
old_netmask = tmpobj.getroute(PxJSON.IPMI_INFO)[PxJSON.NETMASK]
old_gateway = tmpobj.getroute(PxJSON.IPMI_INFO)[PxJSON.GATEWAY]
old_vlan = tmpobj.getroute(PxJSON.IPMI_INFO)[PxJSON.VLAN]
new_connectiontype = obj.getroute(PxJSON.IPMI_INFO)[PxJSON.CONNECTIONTYPE][PxJSON.VALUE]
new_ipaddress = obj.getroute(PxJSON.IPMI_INFO)[PxJSON.IPV4][PxJSON.VALUE]
new_netmask = obj.getroute(PxJSON.IPMI_INFO)[PxJSON.NETMASK][PxJSON.VALUE]
new_gateway = obj.getroute(PxJSON.IPMI_INFO)[PxJSON.GATEWAY][PxJSON.VALUE]
new_vlan = obj.getroute(PxJSON.IPMI_INFO)[PxJSON.VLAN][PxJSON.VALUE]
if old_connectiontype != new_connectiontype:
self.logger.info("Changing {0}: {1} to {2}".format(PxJSON.CONNECTIONTYPE, old_connectiontype, new_connectiontype))
if old_ipaddress != new_ipaddress:
stat, tmpobj = self.testip(new_ipaddress)
if stat:
self.logger.error("{0} already exists".format(new_ipaddress))
return 1, tmpobj
self.logger.info("Changing {0}: {1} to {2}".format(PxJSON.IPV4, old_ipaddress, new_ipaddress))
if old_netmask != new_netmask:
self.logger.info("Changing {0}: {1} to {2}".format(PxJSON.NETMASK, old_netmask, new_netmask))
if old_gateway != new_gateway:
self.logger.info("Changing {0}: {1} to {2}".format(PxJSON.GATEWAY, old_gateway, new_gateway))
if old_vlan != new_vlan:
self.logger.info("Changing {0}: {1} to {2}".format(PxJSON.VLAN, old_vlan, new_vlan))
try:
table_name = 'systemsetups'
DataModel().ExecuteRawQueryStatement2("UPDATE {0} SET ipmi_connection_type=?, ipmi_address=?, ipmi_netmask=?, ipmi_gateway=?, ipmi_vlan=?".format(table_name), (new_connectiontype, new_ipaddress, new_netmask, new_gateway, new_vlan))
obj.setsuccess()
status = 0
except sqlalchemy.exc.OperationalError as ex:
self.logger.error(traceback.format_exc())
obj.internal({"exception": str(ex)})
except Exception as ex:
self.logger.error(traceback.format_exc())
obj.internal({"exception": str(ex)})
self.logger.info("status={0} json={1}".format(status, obj.getjsonpretty()))
return status, obj
# -*- coding: utf-8 -*-
from PyQt4.QtGui import QApplication
import os
import sys
from mainWindow import MainWindow
from datamodel import DataModel
from moses import Moses
if __name__ == "__main__":
app = QApplication(sys.argv)
workdir = os.path.join(os.path.join(os.path.expanduser('~'), 'mosesgui'))
if not os.path.exists(workdir):
os.makedirs(workdir)
dm = DataModel(filename=os.path.join(workdir, "models.sqlite"))
moses = Moses()
if not moses.detect():
sys.exit(1)
MainWindow = MainWindow(dm=dm, moses=moses, workdir=workdir)
MainWindow.setupUi(MainWindow)
MainWindow.show()
sys.exit(app.exec_())
def captureevents(self):
"""
Capture BMC/IPMI System Event Log entries (SEL) and put into common DB.
:param: none
:returns: status, reply
:raises: None
Make sure IPMI is present.
Get the IPMI sel logs.
There may be more than 1 line (usually come in doubles).
Insert into the common database. That routine eliminates duplicates.
When finished, clear all SEL logs.
"""
# Only do this if ipmi is in the kernel and active. (Ignore VM's)
if not self.ipmisupported:
return 0, ''
reply = "Unable to capture IPMI/BMC System Event Logs (SEL)"
# Get the BMC/IPMI System Event Logs (SEL).
try:
# stderr=subprocess.PIPE and ignoring sel_lines.stderr -- tosses the output.
sel_lines = subprocess.check_output(['/usr/bin/ipmitool', 'sel', 'list'], stderr=subprocess.PIPE)
except subprocess.CalledProcessError as ex:
self.logger.error(traceback.format_exc())
reply = reply + '{"exception":' + str(ex) + '}'
return 1, reply
# If None, leave.
if sel_lines is None:
return 0, ''
# Convert output to usable format.
sel_lines = sel_lines.decode('utf-8').splitlines()
# If nothing in it, leave -- nothing new to enter into common database.
if sel_lines == []:
return 0, ''
# Decode the SEL logs, place into format for SQL table, and put there.
for a in sel_lines:
# Split line into list on space pipe space.
selarr = a.split(' | ')
# Date and time are separated by ' | ', combine and separate with space.
dt = selarr[1] + ' ' + selarr[2]
# Get 'date time' in SQL format datetime.
selarr[2] = parse(dt)
# List slicing to remove elements 0 and 1.
selarr = selarr[2:]
done = False
# Get events stored in common database.
table_name = 'ipmievents'
sqlcommand = ("SELECT * FROM {0} WHERE (datetime='{1}' and event='{2}' and details='{3}' and status='{4}')"
.format(table_name, selarr[0], selarr[1], selarr[2], selarr[3]))
try:
ipmidata = DataModel().ExecuteRawQueryStatement(sqlcommand)
except sqlalchemy.exc.OperationalError as ex:
self.logger.error(traceback.format_exc())
reply = reply + '{"exception":' + str(ex) + '}'
return 1, reply
except Exception as ex:
self.logger.error(traceback.format_exc())
reply = reply + '{"exception":' + str(ex) + '}'
return 1, reply
for l in ipmidata:
done = True
break
# New entry, insert into database.
if not done:
query = "INSERT INTO ipmievents (datetime, event, details, status) " \
"VALUES ( '{0}', '{1}', '{2}', '{3}');" \
.format(selarr[0], selarr[1], selarr[2], selarr[3])
try:
DataModel().ExecuteRawQueryStatement(query)
except sqlalchemy.exc.OperationalError as ex:
self.logger.error(traceback.format_exc())
reply = reply + '{"exception":' + str(ex) + '}'
return 1, reply
except Exception as ex:
self.logger.error(traceback.format_exc())
reply = reply + '{"exception":' + str(ex) + '}'
return 1, reply
# Clear all SEL logs. Note: minor race condition from read to clear is possible.
try:
output = subprocess.check_output(['/usr/bin/ipmitool', 'sel', 'clear'], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
print("error in ipmitool sel clear") # DEBUG
self.logger.error(traceback.format_exc())
reply = reply + '{"exception":' + str(ex) + '}'
return 1, reply
# Everything is good at this point.
return 0, ''
