def _output_performance(self):
'''
输出所有Performance
'''
print("Creating equity curve...")
df = pf.create_equity_curve_dataframe(self.portfolio.all_holdings)
print(df.head(40))
bench = pf.create_benchmark_dataframe(self.benchmark, self.start_date,
self.end_date)
print("Creating summary stats...")
stats, strategy_drawdown, bench_drawdown = pf.output_summary_stats(
df, bench)
print(stats)
print("Signals: %s" % self.signals)
print("Orders: %s" % self.orders)
print("Fills: %s" % self.fills)
output_df = pd.DataFrame({
'strategy_equity': df['equity_curve'],
'bench_equity': bench['bench_curve'],
'strategy_drawdown': strategy_drawdown,
'bench_drawdown': bench_drawdown
})
output_df.fillna(method='ffill', inplace=True)
Plot.plot_equity_performance(output_df)
print("Creating trade statistics...")
trade_log = self.portfolio.Tradelog
trade_log.create_trade_summary()
trade_plot_data = pf.create_trade_plot_dataframe(
self.symbol_list, self.data_handler)
Plot.plot_trade(trade_log.get_Tradelog(), trade_plot_data)
def perforamnce(self):
max_drawdown = []
sharpe_ratio = []
cumulative_return = []
volatility = []
performance_indicator = [
'Cumulative Return', 'Max Drawdown', 'Sharpe Ratio', 'Volatility'
]
for i, model in enumerate(self.model):
max_drawdown.append(Performance.max_drawdown(self.model_return[i]))
sharpe_ratio.append(Performance.sharpe_ratio(self.model_return[i]))
cumulative_return.append(
Performance.cummulative_return(self.model_return[i])[-1])
volatility.append(Performance.volatility(self.model_return[i]))
table = pandas.DataFrame(
np.array(
[cumulative_return, max_drawdown, sharpe_ratio, volatility]),
performance_indicator, self.model)
print("")
print("Trading Period: {} - {}".format(
self.date[0].strftime('%Y-%m-%d'),
self.date[-1].strftime('%Y-%m-%d')))
print(table)
print("")
print("Highest Cumulative Return: {}".format(
self.model[np.argmax(cumulative_return).item()]))
print("Lowest Maximum Drawdown: {}".format(
self.model[np.argmin(max_drawdown).item()]))
print("Highest Sharpe Ratio: {}".format(
self.model[np.argmax(sharpe_ratio).item()]))
print("Lowest Volatility: {}".format(
self.model[np.argmin(volatility).item()]))
def finastro_sim():
# Add stock data
ticker_group = 'FinAstro'
tickers = ['LON:FTSE100']
df_tickers = da.create_df_from_tickers(tickers)
# Add Ephemeris data
df_eph = create_ephemeris_df()
df_eph = df_eph[(df_eph.index > '2001-01-01')
& (df_eph.index < '2019-09-30')]
df = df_eph.join(df_tickers, how='inner')
# Generate long/short tickers i.e. signals
df = Strategy.create_long_short_tickers(tickers, df, ticker_group)
# Run backtest
portfolio = Portfolio(df)
Backtest.run(df, portfolio)
Performance.metrics(portfolio, ticker_group)
# Outputs
pd.options.display.float_format = '{:20,.2f}'.format
portfolio.orders_df.to_csv(cp.files['orders'])
portfolio.trades_df.to_csv(cp.files['trades'])
portfolio.df.to_csv(cp.files['backtest'])
portfolio.metrics_df.to_csv(cp.files['metrics'])
print(portfolio.metrics_df)
Graphs.equity_curve(df=portfolio.df)
def main():
try:
params = UserInput.getUserInput('test')
ExampleLoader.loadExamples(params)
w = CacheObj.loadObject(params.modelFile)
Performance.writePerformance(params, w, params.resultFile)
Performance.printStrongAndWeakTrainError(params, w)
utils.dumpCurrentLatentVariables(params, params.latentVariableFile)
except Exception, e :
import traceback
traceback.print_exc(file=sys.stdout)
def testPerformance(period=100):
data = np.random.rand(period) - 0.5
t = time()
print "performance:"
print "Sharpe:", Performance.Sharpe(data)
print "SortinoFull:", Performance.SortinoFull(data)
print "SortinoPartial:", Performance.SortinoPartial(data)
print "%.3f secs" % (time() - t)
t = time()
def plot_porfolio_return(self):
plt.title('Portfolio Return')
plt.ylabel('Cumulative Return')
plt.xlabel('Date')
# plt.ylim((0.75,2.75))
for i in range(len(self.model)):
if i < 4:
plt.plot(self.date,
Performance.cummulative_return(self.model_return[i]),
linestyle='-')
else:
plt.plot(self.date,
Performance.cummulative_return(self.model_return[i]),
linestyle='--')
plt.legend(self.model, loc=2)
def driver(q, ds: str, data_package: list, regression: bool, perf: Performance,
hidden_layers: list, hyper_params: dict, count: int,
total_counter: int, total: int):
print("Job ", ds, count, "started")
try:
# init all test data values
test_data, test_labels, training_data, training_labels, output_size, input_size = data_package
layers = [input_size] + hidden_layers + [output_size]
# init neural network
nn = NeuralNetwork(input_size, hidden_layers, regression, output_size)
nn.set_input_data(training_data, training_labels)
total_weights = 0
for i in range(len(layers) - 1):
total_weights += layers[i] * layers[i + 1]
#self, hyperparameters:dict , Total_Weight:int ,NN
ga = GA(hyper_params, total_weights, nn)
# plt.ion
for gen in range(ga.maxGen):
ga.fitness()
ga.selection()
ga.crossover()
# get best overall solution and set the NN weights
bestSolution = ga.bestChromie.getChromie()
bestWeights = ga.nn.weight_transform(bestSolution)
ga.nn.weights = bestWeights
# pass the test data through the trained NN
results = classify(test_data, test_labels, regression, ga, perf)
# headers = ["Data set", "layers", "pop", "Beta", "CR", "generations", "loss1", "loss2"]
Meta = [
ds,
len(hidden_layers), hyper_params["maxGen"],
hyper_params["pop_size"], hyper_params["mutation_rate"],
hyper_params["mutation_range"], hyper_params["crossover_rate"]
]
results_performance = perf.LossFunctionPerformance(regression, results)
data_point = Meta + results_performance
data_point_string = ','.join([str(x) for x in data_point])
# put the result on the multiprocessing queue
q.put(data_point_string)
print(f"{ds} {count}/{int(total/6)}. {total_counter}/{total}")
except Exception as e:
print('Caught exception in worker thread')
# This prints the type, value, and stack trace of the
# current exception being handled.
traceback.print_exc()
print()
raise e
def plot_return_and_weights(self):
ax1 = plt.subplot(211)
plt.title('Portfolio Return')
ax2 = plt.subplot(212, sharex=ax1)
plt.title('Weights of assets')
ax1.plot(self.date,
Performance.cummulative_return(self.model_return[0]))
ax1.tick_params(labelbottom='off')
ax2.plot(self.date, self.weights)
ax2.set_xlabel('Date')
ax2.legend(np.concatenate([self.tickers, ['Cash']]))
def classify(test_data: np.ndarray, test_labels: np.ndarray, regression: bool, pso: PSO, perf: Performance):
estimates = pso.NN.classify(test_data, test_labels)
if regression == False:
#Decode the One Hot encoding Value
estimates = pso.NN.PickLargest(estimates)
ground_truth = pso.NN.PickLargest(test_labels)
else:
estimates = estimates.tolist()
ground_truth = test_labels.tolist()[0]
estimates = estimates[0]
results = perf.ConvertResultsDataStructure(ground_truth, estimates)
return results
def main():
try:
params = UserInput.getUserInput('train')
ExampleLoader.loadExamples(params)
CommonApp.setExampleCosts(params)
w = None
if params.initialModelFile:
w = CacheObj.loadObject(params.initialModelFile)
else:
w = CommonApp.PsiObject(params,False)
globalSPLVars = SPLSelector.SPLVar()
if params.splParams.splMode != 'CCCP':
SPLSelector.setupSPL(params)
w = LSSVM.optimize(w, globalSPLVars, params)
CacheObj.cacheObject(params.modelFile,w)
Performance.printStrongAndWeakTrainError(params, w)
except Exception, e :
import traceback
traceback.print_exc(file=sys.stdout)
def rolling_stats(df, y_column, x_column, window=20):
# dataframe to hold the results
res = pd.DataFrame(index=df.index)
for i in range(0, len(df.index)):
if len(df) - i >= window:
# break the df into smaller chunks
chunk = df.iloc[i:window + i, :]
# calc_stats is a function created from the code above,
# refer to the Gist at the end of the article.
beta, r2 = Pf.calc_stats(chunk, y_column, x_column)
res.set_value(chunk.tail(1).index[0], "beta", beta)
res.set_value(chunk.tail(1).index[0], "r2", r2)
res = res.dropna()
return res
def __init__(self):
self.sepwin = []
super().__init__()
self.setupUi(self)
self.tabWidget.addTab(Process(), "进程")
self.tabWidget.addTab(Performance(), "性能")
self.tabWidget.addTab(StartItem(), "启动项")
self.tabWidget.customContextMenuRequested.connect(
self.tabWidget_showrightmenu)
if (os.path.exists('.computerhousekeeper\\img\\taskmanager.jpg')):
self.setWindowIcon(
QIcon('.computerhousekeeper\\img\\taskmanager.jpg'))
else:
self.setWindowIcon(
QIcon('..\\.computerhousekeeper\\img\\taskmanager.jpg'))
def __init__(self, learningBonesList, referenceBonesList, option):
self.option = option
self.perfEval = perf.PerformanceEvaluator()
self.util = utils.Utility()
self.computeUtil = utils.ComputingUtility()
self.math1 = math1.SimpleMath()
self.vertUtil = utils.VertexUtility()
self.helperBonesList = learningBonesList
self.helperBoneThetaMatList = []
# to start simple,
self.referenceBonesList = referenceBonesList
# Learner's learning parameters
self.motionLearnerParms = config.MotionLearnerParms()
self.finalError = 0
self.finalIteration = 0
def __init__(self, regularMesh, referenceMesh, primarySkinMod, option):
self.option = option
self.perfEval = perf.PerformanceEvaluator()
self.util = utils.Utility()
self.skinModUtil = utils.SkinModUtility()
self.computeUtil = utils.ComputingUtility()
self.vertUtil = utils.VertexUtility()
self.regularMeshData = dt.RegularMeshData(regularMesh, primarySkinMod,
option.primaryBoneNameList,
self.option)
self.referenceMeshData = dt.ReferenceMeshData(referenceMesh,
self.option)
# Learner's learning parameters
self.skinningParms = option.skinningParms
self.translateParms = option.translateParms
self.rotateParms = option.rotateParms
self.totalError = 0
def is_pair_cointegrated_(df, ticker1, ticker2):
is_cointegrated = False
df = df.tail(Cp.lookback * 100)
df = df.rename(columns={'Close_{}'.format(ticker2):
's_close'}) # strong (y)
df = df.rename(columns={'Close_{}'.format(ticker1): 'w_close'}) # weak (x)
# First check if cointegrated over long time period
beta_hr, alpha, r2 = Pf.calc_stats(df, 's_close', 'w_close')
df['residuals'] = df['s_close'] - (beta_hr * df['w_close'])
ts_res = ts.adfuller(df['residuals'])
critical_values = ts_res[4]
# if test statistic is less than 10% level, then co-integrated at that confidence level
if ts_res[0] < critical_values['10%'] and 0.7 < beta_hr < 1.3:
is_cointegrated = True
return is_cointegrated, beta_hr, df['residuals']
Specificities = []
kf = KFold(n_splits=10)
for train_i, test_i in kf.split(X):
x_train = X[train_i].reshape(X[train_i].shape[0],
X[train_i].shape[1] * X[train_i].shape[2])
x_test = X[test_i].reshape(X[test_i].shape[0],
X[test_i].shape[1] * X[test_i].shape[2])
y_train, y_test = Y[train_i], Y[test_i]
svm_model = SVC(C=100, kernel='linear', gamma='scale')
svm_model.fit(x_train, y_train)
y_pred = svm_model.predict(x_test)
print("-------------------------------")
ACC, TPR, TNR, PPV, NPV, FPR = per.GetPerformanceMetrics(y_test,
y_pred,
weighted=True)
Accuracies.append(ACC)
Recalls.append(TPR)
Specificities.append(TNR)
Precisions.append(PPV)
NPVs.append(NPV)
FPRs.append(FPR)
print("Accuracy: ", ACC)
print("Recall: ", TPR)
print("Specificity: ", TNR)
print("Precision: ", PPV)
print("Negative Predictive Value: ", NPV)
print("FP rate(fall-out): ", FPR)
print(confusion_matrix(y_test, y_pred))
# jqdatasdk.auth('15026415693', 'quiet2520')
# dataapi_beta = DataApi()
ax, factor, data = DV_init()
fx = FA_init(ax)
# x = list(data.index.get_level_values('date'))
# y = list(factor.index.get_level_values('date'))
# y = [ type(datetime_) for datetime_ in x]
# print(min(x)-datetime.timedelta(days = 3))
# print(fx._cleaned_factor_data.columns)
# print(fx._cleaned_factor_data.index)
# print(fx._cleaned_factor_data['period_5'])
Protfilio1 = Performance.factor_returns(fx.cleaned_factor_data, 'beta')
Protfilio2 = Performance.factor_returns(fx.cleaned_factor_data, 'beta',
False)
booksize = 20000000
print((booksize * Protfilio1['period_5']).sum())
print((booksize * Protfilio2['period_1']).sum())
"""
采用了2015-04-01的沪深三百的前一百只股票并进行了基于因子权重的收益分析，
因子值为beta、book_to_price_ratio, gross_profit_ttm。
"""
# print((booksize*Protfilio2['period_5']).sum())
# print(factor.loc[data.index[0:5]])
# factor_copy = factor
# index_group = data.index[0:5]
def run_simulation(simulation_name,
universe_of_tickers_df,
column_to_group_by='Exchange_Sector',
generate_outputs=False,
run_backtest=True):
# Get the Universe of tickers as a single DataFrame
universe_of_tickers = universe_of_tickers_df.index.tolist()
# universe_of_tickers = ['NASDAQ:AAPL', 'NASDAQ:GOOG', 'NASDAQ:MSFT', 'NASDAQ:AMZN', 'NASDAQ:AMAT', 'NASDAQ:INTC']
# universe_of_tickers = ['LON:CBG', 'LON:BARC', 'LON:HSBA', 'LON:LLOY', 'LON:RSA', 'LON:LGEN']
# universe_of_tickers = ['NYSE:AFL', 'NYSE:AON', 'NYSE:BAC', 'NYSE:BSAC', 'NYSE:JPM', 'NYSE:RE', 'NYSE:JEF']
# Only keep tickers that enough data
tickers = Data.get_tickers_with_good_data(universe_of_tickers)
tickers_df = universe_of_tickers_df[universe_of_tickers_df.index.isin(
tickers)]
df = Data.create_df_from_tickers(tickers)
pd.options.display.float_format = '{:20,.2f}'.format
ticker_groups = tickers_df[column_to_group_by].unique().tolist()
# Determine tickers to long and short, by using the strategy to score/rank within ticker_group
for ticker_group in ticker_groups:
try:
# Get all the tickers in the ticker_group, but only keep those with enough price data
tickers = tickers_df[tickers_df[column_to_group_by] ==
ticker_group].index.tolist()
# At least 2 tickers are required for a long-short strategy
if len(tickers) > 4:
df = Strategy.create_long_short_tickers(
tickers, df, ticker_group)
print('Done: {}'.format(ticker_group))
else:
print('Not enough tickers for: ' + ticker_group)
except:
print('Failed: {}'.format(ticker_group))
if run_backtest is True:
# Initialise Portfolio and run a backtest
portfolio = Portfolio(df)
Backtest.run(df, portfolio)
Performance.metrics(portfolio, simulation_name)
# Outputs
if generate_outputs:
portfolio.orders_df.to_csv(cp.files['orders'])
portfolio.trades_df.to_csv(cp.files['trades'])
if len(ticker_groups) == 1:
portfolio.df.to_csv(cp.files['backtest'])
portfolio.metrics_df.to_csv(cp.files['metrics'])
print(portfolio.metrics_df)
Graphs.equity_curve(df=portfolio.df)
return portfolio
else:
print('\n\nTrading Day: {}'.format(df.index[-1].strftime('%d-%b-%Y')))
df = df[-1:].T
df = df[df.index.str.contains('long_ticker')
| df.index.str.contains('short_ticker')]
df_long_tickers = df[df.index.str.contains('long_ticker')]
df_long_tickers = df_long_tickers.rename(
columns={df_long_tickers.columns[0]: "long"})
df_long_tickers.index = df_long_tickers.index.str.replace(
'_long_ticker', '')
df_short_tickers = df[df.index.str.contains('short_ticker')]
df_short_tickers = df_short_tickers.rename(
columns={df_short_tickers.columns[0]: "short"})
df_short_tickers.index = df_short_tickers.index.str.replace(
'_short_ticker', '')
df = df_long_tickers.join(df_short_tickers)
print(df)
df.to_csv(cp.files['latest_signals'])
# -*- coding: UTF-8 -*- import Performance import Division Performance.MainPerformance() Division.MainDivision()
data_string = q.get()
if data_string == 'kill':
f.write('\n')
break
f.write(data_string + '\n')
if __name__ == '__main__':
headers = [
"Data set", "layers", "pop", "Beta", "CR", "generations", "loss1",
"loss2"
]
filename = 'DE_results.csv'
Per = Performance.Results()
Per.PipeToFile([], headers, filename)
data_sets = [
"soybean", "glass", "Cancer", "forestfires", "machine", "abalone"
]
regression_data_set = {
"soybean": False,
"Cancer": False,
"glass": False,
"forestfires": True,
"machine": True,
"abalone": True
}
categorical_attribute_indices = {
# show current holdings and orders record
print("\nCurrent Holdings:")
#print("symbol \t\t num \t\t price \t\t avg_cost \t\t date")
print("{:10}\t{:10}\t{:10}\t{:10}\t{:10}".format('symbol', 'num', 'price', 'avg_cost', 'date'))
cur.execute('select * from current_holdings')
temp_records = cur.fetchall()
for stock in temp_records:
#print(stock[0], '\t\t', stock[1], '\t\t', str.format('{0:.2f}', stock[2]), '\t\t', str.format('{0:.2f}', stock[3]), '\t\t', stock[4])
print("{:10}\t{:10}\t{:10}\t{:10}\t{:10}".format(stock[0], stock[1], str.format('{0:.2f}', stock[2]), str.format('{0:.2f}', stock[3]), stock[4]))
# update asset values and holdings record
holdings.update_tables()
# simulate another day
opt.date = available_dates[1+i]
cur.execute('select * from asset_values_record')
print("\n-----------Asset Values Record-----------")
#print("date \t\t asset_value \t orders \t\t diversity")
print("{:10}\t{:10}\t\t{:10}\t{:10}".format('date', 'asset_value', 'orders', 'diversity'))
temp_records = cur.fetchall()
for av_record in temp_records:
#print(av_record[0], '\t', str.format('{0:.2f}', av_record[1]), '\t', av_record[2], '\t\t', av_record[3])
print("{:10}\t{:10}\t{:10}\t{:10}".format(av_record[0], str.format('{0:.2f}', av_record[1]), av_record[2], av_record[3]))
# performance
print("\n-----------Performance Indicators-----------")
perf = Performance.Performance(opt, holdings, stock_price.stock_mat, available_dates[:test_days], 2.6, 0.05)
perf.get_performance()
# close database connection
holdings.conn.close()
def startAnalysis():
arr_performance_obj = []
# str_path = input('Insira o path para o arquivo .asm: ')
# str_path = "./simpleExpression/simple_expression.asm"
str_path = input('Informe o arquivo .asm (informe o path):')
data(str_path)
labels_dict, total_execution_lines = getLabelsDict(str_path)
total_text_lines = getTotalTextLine(str_path)
arr_bin_machine = []
line_pos = 0
fake_line = 0
#print('--------------')
#print(labels_dict)
#print('--------------')
for i in range(total_text_lines):
line = progamCounter(str_path, i)
fake_line += 1
if line.isspace():
continue
#simplifica a instrução removendo partes inutilizadas segundo nossa lógica
line_regex = re.sub("[$,() ]"," ", line)
line_regex = re.sub(" "," ",line_regex) #debug
line_regex = re.sub(" "," ",line_regex) #debug
instruction = line_regex.split()
if instruction[0][-1] == ':' or instruction[0][0] == '.':
continue
if instruction[0] == 'lw' or instruction[0] == 'sw':
## instrução para conseguir performance
performance = Performance(instruction[0], 'type R', datetime.datetime.now())
opcode = TypeI.getOpcode(instruction[0])
rs, rt = TypeI.getIRegisters(instruction)
address = TypeI.getAddress(instruction[2])
## instrução para conseguir performance
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineI(line, str(line_pos), opcode, rs, rt, address, fake_line)
arr_bin_machine.append(bm)
elif instruction[0] in ('add','sub','and','or','nor','xor', 'slt'):
performance = Performance(instruction[0], 'type R', datetime.datetime.now())
rs, rt, rd = TypeR.getRRegisters(instruction)
funct = TypeR.getFunct(instruction[0])
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineR(line, str(line_pos), '000000', rs, rt, rd, '00000', funct, fake_line)
arr_bin_machine.append(bm)
elif instruction[0] == 'sll' or instruction[0] == 'srl' or instruction[0] == 'sra':
performance = Performance(instruction[0], 'type R', datetime.datetime.now())
rs, rt, rd = TypeR.getRRegisters(instruction)
funct = TypeR.getFunct(instruction[0])
shamt = '{0:05b}'.format(int(instruction[3]))
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineR(line, str(line_pos), '000000', rs, rt, rd, shamt, funct, fake_line)
arr_bin_machine.append(bm)
elif instruction[0] == 'mult' or instruction[0] == 'div':
performance = Performance(instruction[0], 'type R', datetime.datetime.now())
rs, rt, rd = TypeR.getRRegisters(instruction)
funct = TypeR.getFunct(instruction[0])
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineR(line, str(line_pos), '000000', rs, rt, rd, '00000', funct, fake_line)
arr_bin_machine.append(bm)
elif instruction[0] == 'mfhi' or instruction[0] == 'mflo':
performance = Performance(instruction[0], 'type R', datetime.datetime.now())
rs = '00000'
rt = '00000'
rd = Registers.getReg(instruction[1][0],instruction[1][1])
funct = TypeR.getFunct(instruction[0])
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineR(line, str(line_pos), '000000', rs, rt, rd, '00000', funct, fake_line)
arr_bin_machine.append(bm)
elif instruction[0] == 'srav':
performance = Performance(instruction[0], 'type R', datetime.datetime.now())
rt, rs, rd = TypeR.getRRegisters(instruction)
funct = TypeR.getFunct(instruction[0])
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineR(line, str(line_pos), '000000', rs, rt, rd, '00000', funct, fake_line)
arr_bin_machine.append(bm)
elif instruction[0] == 'madd' or instruction[0] == 'msubu':
performance = Performance(instruction[0], 'type R', datetime.datetime.now())
rs, rt, rd = TypeR.getRRegisters(instruction)
funct = TypeR.getFunct(instruction[0])
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineR(line, str(line_pos), '011100', rs, rt, rd, '00000', funct)
arr_bin_machine.append(bm)
elif instruction[0] == 'beq' or instruction[0] == 'bne' or\
instruction[0] == 'bgez' or instruction[0] == 'bgezal':
performance = Performance(instruction[0], 'type I', datetime.datetime.now())
opcode = TypeI.getOpcode(instruction[0])
rs, rt = TypeI.getIRegisters(instruction)
address = TypeI.getAddress(instruction, line_pos, labels_dict)
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineI(line, str(line_pos), opcode, rs, rt, address, fake_line)
arr_bin_machine.append(bm)
elif instruction[0] == 'j' or instruction[0] == 'jal':
performance = Performance(instruction[0], 'type J', datetime.datetime.now())
opcode = TypeJ.getOpcode(instruction[0])
address = TypeJ.getAddress(instruction, line_pos, labels_dict)
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineJ(line, str(line_pos), opcode, address, fake_line)
arr_bin_machine.append(bm)
elif instruction[0] == 'jr':
performance = Performance(instruction[0], 'type J', datetime.datetime.now())
opcode = TypeJ.getOpcode(instruction[0])
address = TypeJ.getAddress(instruction, line_pos, labels_dict, instruction[0])
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineJ(line, str(line_pos), opcode, address, fake_line)
arr_bin_machine.append(bm)
elif instruction[0] == 'jalr':
performance = Performance(instruction[0], 'type R', datetime.datetime.now())
rs, rt, rd = TypeR.getRRegisters(instruction)
funct = TypeR.getFunct(instruction[0])
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineR(line, str(line_pos), '000000', rs, rt, rd, '00000', funct, fake_line)
arr_bin_machine.append(bm)
elif instruction[0] == 'lui':
performance = Performance(instruction[0], 'type I', datetime.datetime.now())
opcode = TypeI.getOpcode(instruction[0])
rs, rt = TypeI.getIRegisters(instruction)
address = TypeI.getAddress(instruction)
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineI(line, str(line_pos), opcode, rs, rt, address, fake_line)
arr_bin_machine.append(bm)
elif instruction[0] == 'li':
performance = Performance(instruction[0], 'type I', datetime.datetime.now())
num = int(instruction[2], 0)
if num/65536.0 > 1:
# PSEUDO INSTRUÇÃO
num_lui = '0x'
num_hex = hex(num)[6:]
for i in range(8 - len(num_hex)):
num_lui += '0'
num_lui += num_hex
first_inst = ['lui', '1', num_lui]
opcode = TypeI.getOpcode(first_inst[0])
rs, rt = TypeI.getIRegisters(first_inst)
address = TypeI.getAddress(first_inst)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineI(line, str(line_pos), opcode, rs, rt, address, fake_line)
arr_bin_machine.append(bm)
#### incrementar linha pseudo instrução
line_pos += 1
num_lui = '0x'
num_hex = hex(num)[:6]
for i in range(8 - len(num_hex)):
num_lui += '0'
num_lui += num_hex[2:]
second_inst = ['ori', instruction[1], '1', num_lui]
opcode = TypeI.getOpcode(second_inst[0])
rs, rt = TypeI.getIRegisters(second_inst)
address = TypeI.getAddress(second_inst)
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineI(line, str(line_pos), opcode, rs, rt, address, fake_line)
bm.setIsPseudo(True)
arr_bin_machine.append(bm)
else:
analogo_inst = ['addiu', instruction[1], '0', instruction[2]]
opcode = TypeI.getOpcode(analogo_inst[0])
rs, rt = TypeI.getIRegisters(analogo_inst)
address = TypeI.getAddress(analogo_inst)
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineI(line, str(line_pos), opcode, rs, rt, address, fake_line)
arr_bin_machine.append(bm)
elif instruction[0] == 'addi' or instruction[0] == 'andi' or\
instruction[0] == 'ori' or instruction[0] == 'xori':
performance = Performance(instruction[0], 'type I', datetime.datetime.now())
imidiate_int = int(instruction[3], 0)
if imidiate_int < 0 and (instruction[0] == 'andi' or\
instruction[0] == 'ori' or instruction[0] == 'xori'):
first_pseudo_inst = ['lui', '1', '0xffff']
opcode = TypeI.getOpcode(first_pseudo_inst[0])
rs, rt = TypeI.getIRegisters(first_pseudo_inst)
address = TypeI.getAddress(first_pseudo_inst)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineI(line, str(line_pos), opcode, rs, rt, address, fake_line)
bm.setIsPseudo(True)
arr_bin_machine.append(bm)
#### incrementar linha pseudo instrução
line_pos += 1
### Como temos uma pseudo intrução, será necessário salvar mais objetos
second_pseudo_inst = ['ori', '1', '1', instruction[3]]
opcode = TypeI.getOpcode(second_pseudo_inst[0])
rs, rt = TypeI.getIRegisters(second_pseudo_inst)
address = TypeI.getAddress(second_pseudo_inst)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineI(line, str(line_pos), opcode, rs, rt, address, fake_line)
bm.setIsPseudo(True)
arr_bin_machine.append(bm)
#### incrementar linha pseudo instrução
line_pos += 1
if(instruction[0] == 'andi'):
third_pseudo_inst = ['and', instruction[1], instruction[2], '1']
rs, rt, rd = TypeR.getRRegisters(third_pseudo_inst)
funct = TypeR.getFunct(third_pseudo_inst[0])
elif(instruction[0] == 'ori'):
third_pseudo_inst = ['or', instruction[1], instruction[2], '1']
rs, rt, rd = TypeR.getRRegisters(third_pseudo_inst)
funct = TypeR.getFunct(third_pseudo_inst[0])
elif(instruction[0] == 'xori'):
third_pseudo_inst = ['xor', instruction[1], instruction[2], '1']
rs, rt, rd = TypeR.getRRegisters(third_pseudo_inst)
funct = TypeR.getFunct(third_pseudo_inst[0])
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineR(line, str(line_pos), '000000', rs, rt, rd, '00000', funct, fake_line)
arr_bin_machine.append(bm)
else:
opcode = TypeI.getOpcode(instruction[0])
rs, rt = TypeI.getIRegisters(instruction)
address = TypeI.getAddress(instruction)
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineI(line, str(line_pos), opcode, rs, rt, address, fake_line)
arr_bin_machine.append(bm)
elif instruction[0] == 'clo':
performance = Performance(instruction[0], 'type I', datetime.datetime.now())
rs, rt, rd = TypeR.getRRegisters(instruction)
funct = TypeR.getFunct(instruction[0])
performance.setTime2(datetime.datetime.now())
arr_performance_obj.append(performance)
# Criar o objeto e salva-lo em uma list para acesso posteriormente
bm = BinaryMachineR(line, str(line_pos), '011100', rs, rt, rd, '00000', funct, fake_line)
arr_bin_machine.append(bm)
else:
print("ERROR - INSTRUCTION NOT RECOGNIZED")
print('------------------')
print(instruction)
print('------------------')
line_pos += 1
## Salvar arquivo .mif
Saida.saveFileText(arr_bin_machine, 'saida_text')
return arr_bin_machine, arr_performance_obj
def main():
print("Program Start")
headers = [
"Data set", "layers", "pop", "Beta", "CR", "generations", "loss1",
"loss2"
]
filename = 'VIDEORESULTS.csv'
Per = Performance.Results()
Per.PipeToFile([], headers, filename)
data_sets = [
"soybean", "glass", "abalone", "Cancer", "forestfires", "machine"
]
regression_data_set = {
"soybean": False,
"Cancer": False,
"glass": False,
"forestfires": True,
"machine": True,
"abalone": True
}
categorical_attribute_indices = {
"soybean": [],
"Cancer": [],
"glass": [],
"forestfires": [],
"machine": [],
"abalone": []
}
tuned_0_hl = {
"soybean": {
"omega": .5,
"c1": .1,
"c2": 5,
"hidden_layer": []
},
"Cancer": {
"omega": .5,
"c1": .5,
"c2": 5,
"hidden_layer": []
},
"glass": {
"omega": .2,
"c1": .9,
"c2": 5,
"hidden_layer": []
},
"forestfires": {
"omega": .2,
"c1": 5,
"c2": .5,
"hidden_layer": []
},
"machine": {
"omega": .5,
"c1": .9,
"c2": 5,
"hidden_layer": []
},
"abalone": {
"omega": .2,
"c1": 5,
"c2": .9,
"hidden_layer": []
}
}
tuned_1_hl = {
"soybean": {
"omega": .5,
"c1": .5,
"c2": 1,
"hidden_layer": [7]
},
"Cancer": {
"omega": .2,
"c1": .5,
"c2": 5,
"hidden_layer": [4]
},
"glass": {
"omega": .2,
"c1": .9,
"c2": 5,
"hidden_layer": [8]
},
"forestfires": {
"omega": .2,
"c1": 5,
"c2": 5,
"hidden_layer": [8]
},
"machine": {
"omega": .5,
"c1": 5,
"c2": .5,
"hidden_layer": [4]
},
"abalone": {
"omega": .2,
"c1": .1,
"c2": 5,
"hidden_layer": [8]
}
}
tuned_2_hl = {
"soybean": {
"omega": .5,
"c1": .9,
"c2": .1,
"hidden_layer": [7, 12]
},
"Cancer": {
"omega": .2,
"c1": .5,
"c2": 5,
"hidden_layer": [4, 4]
},
"glass": {
"omega": .2,
"c1": .9,
"c2": 5,
"hidden_layer": [8, 6]
},
"forestfires": {
"omega": .2,
"c1": .9,
"c2": 5,
"hidden_layer": [8, 8]
},
"machine": {
"omega": .2,
"c1": .9,
"c2": .1,
"hidden_layer": [7, 2]
},
"abalone": {
"omega": .2,
"c1": 5,
"c2": 5,
"hidden_layer": [6, 8]
}
}
du = DataUtility.DataUtility(categorical_attribute_indices,
regression_data_set)
total_counter = 0
for data_set in data_sets:
if data_set != 'Cancer':
continue
data_set_counter = 0
# ten fold data and labels is a list of [data, labels] pairs, where
# data and labels are numpy arrays:
tenfold_data_and_labels = du.Dataset_and_Labels(data_set)
for j in range(10):
test_data, test_labels = copy.deepcopy(tenfold_data_and_labels[j])
#Append all data folds to the training data set
remaining_data = [
x[0] for i, x in enumerate(tenfold_data_and_labels) if i != j
]
remaining_labels = [
y[1] for i, y in enumerate(tenfold_data_and_labels) if i != j
]
#Store off a set of the remaining dataset
X = np.concatenate(remaining_data, axis=1)
#Store the remaining data set labels
labels = np.concatenate(remaining_labels, axis=1)
print(data_set, "training data prepared")
regression = regression_data_set[data_set]
#If the data set is a regression dataset
if regression == True:
#The number of output nodes is 1
output_size = 1
#else it is a classification data set
else:
#Count the number of classes in the label data set
output_size = du.CountClasses(labels)
#Get the test data labels in one hot encoding
test_labels = du.ConvertLabels(test_labels, output_size)
#Get the Labels into a One hot encoding
labels = du.ConvertLabels(labels, output_size)
input_size = X.shape[0]
data_set_size = X.shape[1] + test_data.shape[1]
tuned_parameters = [
tuned_0_hl[data_set], tuned_1_hl[data_set],
tuned_2_hl[data_set]
]
for z in range(1):
hidden_layers = tuned_parameters[z]["hidden_layer"]
layers = [input_size] + hidden_layers + [output_size]
nn = NeuralNetwork(input_size, hidden_layers, regression,
output_size)
nn.set_input_data(X, labels)
nn1 = NeuralNetwork(input_size, hidden_layers, regression,
output_size)
nn1.set_input_data(X, labels)
nn2 = NeuralNetwork(input_size, hidden_layers, regression,
output_size)
nn2.set_input_data(X, labels)
total_weights = 0
for i in range(len(layers) - 1):
total_weights += layers[i] * layers[i + 1]
hyperparameters = {
"population_size": 10 * total_weights,
"beta": .5,
"crossover_rate": .6,
"max_gen": 100
}
hyperparameterss = {
"maxGen": 100,
"pop_size": 100,
"mutation_rate": .5,
"mutation_range": 10,
"crossover_rate": .5
}
hyperparametersss = {
"position_range": 10,
"velocity_range": 1,
"omega": .1,
# tuned_parameters[z]["omega"],
"c1": .9,
# tuned_parameters[z]["c1"],
"c2": .1,
# tuned_parameters[z]["c2"],
"vmax": 1,
"pop_size": 1000,
"max_t": 50
}
de = DE.DE(hyperparameters, total_weights, nn)
ga = GA.GA(hyperparameterss, total_weights, nn1)
pso = PSO.PSO(layers, hyperparametersss, nn2)
learning_rate = 3
momentum = 0
VNN = VideoNN.NeuralNetworks(input_size, hidden_layers,
regression, output_size,
learning_rate, momentum)
VNN.set_input_data(X, labels)
for gen in range(de.maxgens):
de.mutate_and_crossover()
for gen in range(ga.maxGen):
ga.fitness()
ga.selection()
ga.crossover()
counter = 0
for epoch in range(pso.max_t):
pso.update_fitness()
pso.update_position_and_velocity()
for epoch in range(100):
VNN.forward_pass()
VNN.backpropagation_pass()
bestSolution = de.bestChromie.getchromie()
bestWeights = de.nn.weight_transform(bestSolution)
de.nn.weights = bestWeights
Estimation_Values = de.nn.classify(test_data, test_labels)
Estimation_Values1 = ga.nn.classify(test_data, test_labels)
Estimation_Values2 = pso.NN.classify(test_data, test_labels)
Estimation_Values3 = VNN.classify(test_data, test_labels)
if regression == False:
#Decode the One Hot encoding Value
Estimation_Values = de.nn.PickLargest(Estimation_Values)
test_labels_list = de.nn.PickLargest(test_labels)
Estimation_Values1 = ga.nn.PickLargest(Estimation_Values1)
Tll = ga.nn.PickLargest(test_labels)
Estimation_Values2 = pso.NN.PickLargest(Estimation_Values2)
tll1 = pso.NN.PickLargest(test_labels)
Estimation_Values3 = VNN.PickLargest(Estimation_Values3)
tll = VNN.PickLargest(test_labels)
# print("ESTiMATION VALUES BY GIVEN INDEX (CLASS GUESS) ")
# print(Estimation_Values)
else:
Estimation_Values = Estimation_Values.tolist()
test_labels_list = test_labels.tolist()[0]
Estimation_Values = Estimation_Values[0]
Estimat = Estimation_Values
groun = test_labels_list
meta = list()
Nice = Per.ConvertResultsDataStructure(groun, Estimat)
Nice1 = Per.ConvertResultsDataStructure(
Tll, Estimation_Values1)
Nice2 = Per.ConvertResultsDataStructure(
tll1, Estimation_Values2)
Nice3 = Per.ConvertResultsDataStructure(
tll, Estimation_Values3)
DEss = Per.StartLossFunction(regression, Nice, meta)
GAss = Per.StartLossFunction(regression, Nice1, meta)
PSOSS = Per.StartLossFunction(regression, Nice2, meta)
VNNS = Per.StartLossFunction(regression, Nice3, meta)
print("DE")
print(DEss)
print("GA")
print(GAss)
print("PSO")
print(PSOSS)
print("NN Back prop.")
print(VNNS)
# print("THE GROUND VERSUS ESTIMATION:")
# print(Nice)
# headers = ["Data set", "layers", "pop", "Beta", "CR", "generations", "loss1", "loss2"]
Meta = [
data_set,
len(hidden_layers), hyperparameters["population_size"],
hyperparameters["beta"], hyperparameters["crossover_rate"],
hyperparameters["max_gen"]
]
Per.StartLossFunction(regression, Nice, Meta, filename)
data_set_counter += 1
total_counter += 1
print("Program End ")
def driver(q, maxIter: int, ds: str, data_package: list, regression: bool, perf: Performance, hidden_layers: list, hyper_params: dict, count: int, total_counter:int, total: int):
'''
q: a multiprocessing manager Queue object to pass finished data to
maxIter: int. number of epochs to run PSO for
ds: string. name of the data set
data_package: list. see the data_package method below. Provides all the data needed to run the experiment
regression: boolean. Is this a regression task?
perf: Performance type object, used to process estimates vs ground truth
hidden_layers: list. tells the algorithm how many layers there are in the NN architecture
hyper_params: dictionary of hyperparameters. useful for tuning.
count: int. the job number for this data set
total_counter: int. the job number wrt the total run
total: int. the total number of jobs across all data sets. used for progress printing.
'''
print("Job ", ds, count, "started")
# multiprocessor in python supresses exceptions when workers fail. This try catch block forces it to raise an exception and stack trace for debugging.
try:
# init all test data values
test_data, test_labels, training_data, training_labels, output_size, input_size = data_package
layers = [input_size] + hidden_layers + [output_size]
# init neural network
nn = NeuralNetwork(input_size, hidden_layers, regression, output_size)
nn.set_input_data(training_data, training_labels)
# initi PSO and train it
pso = PSO(layers, hyper_params, nn, maxIter)
for epoch in range(pso.max_t):
print("job", count, "generation", epoch)
pso.update_fitness()
pso.update_position_and_velocity()
# get best overall solution from the PSO and set the NN weights
bestSolution = pso.gbest_position
bestWeights = pso.NN.weight_transform(bestSolution)
pso.NN.weights = bestWeights
# pass the test data through the trained NN
results = classify(test_data, test_labels, regression, pso, perf)
# headers = ["Data set", "layers", "omega", "c1", "c2", "vmax", "pop_size", "maxIter", "loss1", "loss2"]
Meta = [
ds,
len(hidden_layers),
hyper_params["omega"],
hyper_params["c1"],
hyper_params["c2"],
hyper_params["vmax"],
hyper_params["pop_size"],
hyper_params["max_iter"]
]
# get the performance of the network w.r.t. the ground truth
results_performance = perf.LossFunctionPerformance(regression, results)
# construct the data point to be written to disk via csv file
data_point = Meta + results_performance
data_point_string = ','.join([str(x) for x in data_point])
# put the result on the multiprocessing queue
q.put(data_point_string)
# status update
print(f"{ds} {count}/{int(total/6)}. {total_counter}/{total}")
# if something goes wrong raise an exception
except Exception as e:
print('Caught exception in worker thread')
# This prints the type, value, and stack trace of the
# current exception being handled.
traceback.print_exc()
print()
raise e
def Support_Vector_Machine(train_tf_idf_dict, test_tf_idf_dict):
'''
#Train data
'''
train_1_tf_idf = train_tf_idf_dict[1]
train_2_tf_idf = train_tf_idf_dict[2]
train_3_tf_idf = train_tf_idf_dict[3]
train_4_tf_idf = train_tf_idf_dict[4]
train_5_tf_idf = train_tf_idf_dict[5]
training_data_X = []
training_data_Y = []
for list_each in train_1_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(1)
for list_each in train_2_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(2)
for list_each in train_3_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(3)
for list_each in train_4_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(4)
for list_each in train_5_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(5)
vec = DictVectorizer()
#print(vec.fit_transform(training_data_X).toarray())
clf = svm.SVC()
clf.fit(vec.fit_transform(training_data_X), training_data_Y)
print('Training end')
'''
#Test data
'''
classified_as_1 = 0
correctly_classified_as_1 = 0
belongs_to_1 = 0
classified_as_2 = 0
correctly_classified_as_2 = 0
belongs_to_2 = 0
classified_as_3 = 0
correctly_classified_as_3 = 0
belongs_to_3 = 0
classified_as_4 = 0
correctly_classified_as_4 = 0
belongs_to_4 = 0
classified_as_5 = 0
correctly_classified_as_5 = 0
belongs_to_5 = 0
test_1_tf_idf = test_tf_idf_dict[1]
test_2_tf_idf = test_tf_idf_dict[2]
test_3_tf_idf = test_tf_idf_dict[3]
test_4_tf_idf = test_tf_idf_dict[4]
test_5_tf_idf = test_tf_idf_dict[5]
training_data_X = []
for list_each in test_1_tf_idf:
strdata = clf.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
correctly_classified_as_1 = correctly_classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_1 = belongs_to_1 + 1
for list_each in test_2_tf_idf:
strdata = clf.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
correctly_classified_as_2 = correctly_classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_2 = belongs_to_2 + 1
for list_each in test_3_tf_idf:
strdata = clf.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
correctly_classified_as_3 = correctly_classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_3 = belongs_to_3 + 1
for list_each in test_4_tf_idf:
strdata = clf.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
correctly_classified_as_4 = correctly_classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_4 = belongs_to_4 + 1
for list_each in test_5_tf_idf:
strdata = clf.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
correctly_classified_as_5 = correctly_classified_as_5 + 1
belongs_to_5 = belongs_to_5 + 1
dict1 = {}
dict1['classified_as_1'] = classified_as_1
dict1['correctly_classified_as_1'] = correctly_classified_as_1
dict1['belongs_to_1'] = belongs_to_1
dict1['classified_as_2'] = classified_as_2
dict1['correctly_classified_as_2'] = correctly_classified_as_2
dict1['belongs_to_2'] = belongs_to_2
dict1['classified_as_3'] = classified_as_3
dict1['correctly_classified_as_3'] = correctly_classified_as_3
dict1['belongs_to_3'] = belongs_to_3
dict1['classified_as_4'] = classified_as_4
dict1['correctly_classified_as_4'] = correctly_classified_as_4
dict1['belongs_to_4'] = belongs_to_4
dict1['classified_as_5'] = classified_as_5
dict1['correctly_classified_as_5'] = correctly_classified_as_5
dict1['belongs_to_5'] = belongs_to_5
performance.calculate_accuracy('SVM', dict1)
def Multinomial_Naive_Bayes(train_tf_idf_dict, test_tf_idf_dict):
'''
#Train data
'''
# pos_list=train_input_list['pos']
# neg_list=train_input_list['neg']
train_1_tf_idf = train_tf_idf_dict[1]
train_2_tf_idf = train_tf_idf_dict[2]
train_3_tf_idf = train_tf_idf_dict[3]
train_4_tf_idf = train_tf_idf_dict[4]
train_5_tf_idf = train_tf_idf_dict[5]
training_data_X = []
training_data_Y = []
for list_each in train_1_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(1)
for list_each in train_2_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(2)
for list_each in train_3_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(3)
for list_each in train_4_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(4)
for list_each in train_5_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(5)
vec = DictVectorizer()
from sklearn.naive_bayes import MultinomialNB
clf = MultinomialNB()
clf.fit(vec.fit_transform(training_data_X), training_data_Y)
'''
#Test data
'''
print('Training end')
classified_as_1 = 0
correctly_classified_as_1 = 0
belongs_to_1 = 0
classified_as_2 = 0
correctly_classified_as_2 = 0
belongs_to_2 = 0
classified_as_3 = 0
correctly_classified_as_3 = 0
belongs_to_3 = 0
classified_as_4 = 0
correctly_classified_as_4 = 0
belongs_to_4 = 0
classified_as_5 = 0
correctly_classified_as_5 = 0
belongs_to_5 = 0
test_1_tf_idf = test_tf_idf_dict[1]
test_2_tf_idf = test_tf_idf_dict[2]
test_3_tf_idf = test_tf_idf_dict[3]
test_4_tf_idf = test_tf_idf_dict[4]
test_5_tf_idf = test_tf_idf_dict[5]
for list_each in test_1_tf_idf:
strdata = clf.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
correctly_classified_as_1 = correctly_classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_1 = belongs_to_1 + 1
for list_each in test_2_tf_idf:
strdata = clf.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
correctly_classified_as_2 = correctly_classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_2 = belongs_to_2 + 1
for list_each in test_3_tf_idf:
strdata = clf.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
correctly_classified_as_3 = correctly_classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_3 = belongs_to_3 + 1
for list_each in test_4_tf_idf:
strdata = clf.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
correctly_classified_as_4 = correctly_classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_4 = belongs_to_4 + 1
for list_each in test_5_tf_idf:
strdata = clf.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
correctly_classified_as_5 = correctly_classified_as_5 + 1
belongs_to_5 = belongs_to_5 + 1
# print('Classification end')
# print('classified_as_pos-->', classified_as_pos)
# print('correctly_classified_as_pos->', correctly_classified_as_pos)
# print('belongs_to_pos-->', belongs_to_pos)
# print('classified_as_neg->', classified_as_neg)
# print('correctly_classified_as_neg-->', correctly_classified_as_neg)
# print('belongs_to_neg-->', belongs_to_neg)
dict1 = {}
dict1['classified_as_1'] = classified_as_1
dict1['correctly_classified_as_1'] = correctly_classified_as_1
dict1['belongs_to_1'] = belongs_to_1
dict1['classified_as_2'] = classified_as_2
dict1['correctly_classified_as_2'] = correctly_classified_as_2
dict1['belongs_to_2'] = belongs_to_2
dict1['classified_as_3'] = classified_as_3
dict1['correctly_classified_as_3'] = correctly_classified_as_3
dict1['belongs_to_3'] = belongs_to_3
dict1['classified_as_4'] = classified_as_4
dict1['correctly_classified_as_4'] = correctly_classified_as_4
dict1['belongs_to_4'] = belongs_to_4
dict1['classified_as_5'] = classified_as_5
dict1['correctly_classified_as_5'] = correctly_classified_as_5
dict1['belongs_to_5'] = belongs_to_5
performance.calculate_accuracy('Multinomial_NaiveBayes', dict1)
def main():
print("Program Start")
headers = [
"Data set", "layers", "pop", "Beta", "CR", "generations", "loss1",
"loss2"
]
filename = 'DE_experimental_resultsFINAL.csv'
Per = Performance.Results()
Per.PipeToFile([], headers, filename)
data_sets = [
"soybean", "glass", "abalone", "Cancer", "forestfires", "machine"
]
regression_data_set = {
"soybean": False,
"Cancer": False,
"glass": False,
"forestfires": True,
"machine": True,
"abalone": True
}
categorical_attribute_indices = {
"soybean": [],
"Cancer": [],
"glass": [],
"forestfires": [],
"machine": [],
"abalone": []
}
tuned_0_hl = {
"soybean": {
"omega": .5,
"c1": .1,
"c2": 5,
"hidden_layer": []
},
"Cancer": {
"omega": .5,
"c1": .5,
"c2": 5,
"hidden_layer": []
},
"glass": {
"omega": .2,
"c1": .9,
"c2": 5,
"hidden_layer": []
},
"forestfires": {
"omega": .2,
"c1": 5,
"c2": .5,
"hidden_layer": []
},
"machine": {
"omega": .5,
"c1": .9,
"c2": 5,
"hidden_layer": []
},
"abalone": {
"omega": .2,
"c1": 5,
"c2": .9,
"hidden_layer": []
}
}
tuned_1_hl = {
"soybean": {
"omega": .5,
"c1": .5,
"c2": 1,
"hidden_layer": [7]
},
"Cancer": {
"omega": .2,
"c1": .5,
"c2": 5,
"hidden_layer": [4]
},
"glass": {
"omega": .2,
"c1": .9,
"c2": 5,
"hidden_layer": [8]
},
"forestfires": {
"omega": .2,
"c1": 5,
"c2": 5,
"hidden_layer": [8]
},
"machine": {
"omega": .5,
"c1": 5,
"c2": .5,
"hidden_layer": [4]
},
"abalone": {
"omega": .2,
"c1": .1,
"c2": 5,
"hidden_layer": [8]
}
}
tuned_2_hl = {
"soybean": {
"omega": .5,
"c1": .9,
"c2": .1,
"hidden_layer": [7, 12]
},
"Cancer": {
"omega": .2,
"c1": .5,
"c2": 5,
"hidden_layer": [4, 4]
},
"glass": {
"omega": .2,
"c1": .9,
"c2": 5,
"hidden_layer": [8, 6]
},
"forestfires": {
"omega": .2,
"c1": .9,
"c2": 5,
"hidden_layer": [8, 8]
},
"machine": {
"omega": .2,
"c1": .9,
"c2": .1,
"hidden_layer": [7, 2]
},
"abalone": {
"omega": .2,
"c1": 5,
"c2": 5,
"hidden_layer": [6, 8]
}
}
du = DataUtility.DataUtility(categorical_attribute_indices,
regression_data_set)
total_counter = 0
for data_set in data_sets:
data_set_counter = 0
# ten fold data and labels is a list of [data, labels] pairs, where
# data and labels are numpy arrays:
tenfold_data_and_labels = du.Dataset_and_Labels(data_set)
for j in range(10):
test_data, test_labels = copy.deepcopy(tenfold_data_and_labels[j])
#Append all data folds to the training data set
remaining_data = [
x[0] for i, x in enumerate(tenfold_data_and_labels) if i != j
]
remaining_labels = [
y[1] for i, y in enumerate(tenfold_data_and_labels) if i != j
]
#Store off a set of the remaining dataset
X = np.concatenate(remaining_data, axis=1)
#Store the remaining data set labels
labels = np.concatenate(remaining_labels, axis=1)
print(data_set, "training data prepared")
regression = regression_data_set[data_set]
#If the data set is a regression dataset
if regression == True:
#The number of output nodes is 1
output_size = 1
#else it is a classification data set
else:
#Count the number of classes in the label data set
output_size = du.CountClasses(labels)
#Get the test data labels in one hot encoding
test_labels = du.ConvertLabels(test_labels, output_size)
#Get the Labels into a One hot encoding
labels = du.ConvertLabels(labels, output_size)
input_size = X.shape[0]
data_set_size = X.shape[1] + test_data.shape[1]
tuned_parameters = [
tuned_0_hl[data_set], tuned_1_hl[data_set],
tuned_2_hl[data_set]
]
for z in range(3):
hidden_layers = tuned_parameters[z]["hidden_layer"]
layers = [input_size] + hidden_layers + [output_size]
nn = NeuralNetwork(input_size, hidden_layers, regression,
output_size)
nn.set_input_data(X, labels)
total_weights = 0
for i in range(len(layers) - 1):
total_weights += layers[i] * layers[i + 1]
hyperparameters = {
"population_size": 10 * total_weights,
"beta": .5,
"crossover_rate": .6,
"max_gen": 100
}
hyperparameterss = {
"maxGen": 100,
"pop_size": 100,
"mutation_rate": .5,
"mutation_range": 10,
"crossover_rate": .5
}
hyperparametersss = {
"position_range": 10,
"velocity_range": 1,
"omega": .1,
# tuned_parameters[z]["omega"],
"c1": .9,
# tuned_parameters[z]["c1"],
"c2": .1,
# tuned_parameters[z]["c2"],
"vmax": 1,
"pop_size": 1000,
"max_t": 50
}
de = DE.DE(hyperparameters, total_weights, nn)
ga = GA.GA(hyperparameterss, total_weights, nn)
pso = PSO.PSO(layers, hyperparametersss, nn)
learning_rate = 3
momentum = 0
VNN = VideoNN.NeuralNetworks(input_size, hidden_layers,
regression, output_size,
learning_rate, momentum)
counter = 0
print("DE OPERATIONS ")
for gen in range(de.maxgens):
if counter == 1:
break
print("MUTATE AND CROSS OVER ")
de.Pmutate_and_crossover()
counter = counter + 1
time.sleep(200)
counter = 0
print("GA OPERATIONS")
for gen in range(ga.maxGen):
if counter == 1:
break
print()
ga.pfitness()
ga.Pselection()
ga.Pcrossover()
counter = counter + 1
time.sleep(200)
counter = 0
print("PSO OPERATIONS")
for epoch in range(pso.max_t):
if counter == 1:
break
pso.Pupdate_fitness()
pso.Pupdate_position_and_velocity()
counter = counter + 1
time.sleep(200)
# plt.plot(list(range(len(de.globalbest))), de.globalbest)
# plt.draw()
# plt.pause(0.00001)
#plt.clf()
# get the best overall solution and set the NN to those weights
#DE
bestSolution = de.bestChromie.getchromie()
bestWeights = de.nn.weight_transform(bestSolution)
de.nn.weights = bestWeights
#GA
#PS
# ################################ new code for de end ###################################
# plt.ioff()
# plt.plot(list(range(len(de.globalbest))), de.globalbest)
# plt.show()
# img_name = data_set + '_l' + str(len(hidden_layers)) + '_pr' + str(a) + '_vr' + str(b) + '_w' + str(c) + '_c' + str(d) + '_cc' + str(e) + '_v' + str(f) + '_ps' + str(g) + '.png'
# plt.savefig('tuning_plots/' + img_name)
# plt.clf()
Estimation_Values = de.nn.classify(test_data, test_labels)
if regression == False:
#Decode the One Hot encoding Value
Estimation_Values = de.nn.PickLargest(Estimation_Values)
test_labels_list = de.nn.PickLargest(test_labels)
# print("ESTiMATION VALUES BY GIVEN INDEX (CLASS GUESS) ")
# print(Estimation_Values)
else:
Estimation_Values = Estimation_Values.tolist()
test_labels_list = test_labels.tolist()[0]
Estimation_Values = Estimation_Values[0]
Estimat = Estimation_Values
groun = test_labels_list
Nice = Per.ConvertResultsDataStructure(groun, Estimat)
# print("THE GROUND VERSUS ESTIMATION:")
# print(Nice)
# headers = ["Data set", "layers", "pop", "Beta", "CR", "generations", "loss1", "loss2"]
Meta = [
data_set,
len(hidden_layers), hyperparameters["population_size"],
hyperparameters["beta"], hyperparameters["crossover_rate"],
hyperparameters["max_gen"]
]
Per.StartLossFunction(regression, Nice, Meta, filename)
print(f"{data_set_counter}/30 {data_set}. {total_counter}/180")
data_set_counter += 1
total_counter += 1
print("DEMO FINISHED")
time.sleep(10000)
print("Program End ")
def Naive_Bayes(train_tf_idf_dict, test_tf_idf_dict):
'''
#Train data
'''
#pos_list=train_input_list['pos']
#neg_list=train_input_list['neg']
training_data = []
train_1_tf_idf = train_tf_idf_dict[1]
train_2_tf_idf = train_tf_idf_dict[2]
train_3_tf_idf = train_tf_idf_dict[3]
train_4_tf_idf = train_tf_idf_dict[4]
train_5_tf_idf = train_tf_idf_dict[5]
print('Naive_Bayes')
for list_each in train_1_tf_idf:
training_data.append((list_each, 1))
for list_each in train_2_tf_idf:
training_data.append((list_each, 2))
for list_each in train_3_tf_idf:
training_data.append((list_each, 3))
for list_each in train_4_tf_idf:
training_data.append((list_each, 4))
for list_each in train_5_tf_idf:
training_data.append((list_each, 5))
classifier = nltk.NaiveBayesClassifier.train(training_data)
'''
#Test data
'''
print('Training end')
classified_as_1 = 0
correctly_classified_as_1 = 0
belongs_to_1 = 0
classified_as_2 = 0
correctly_classified_as_2 = 0
belongs_to_2 = 0
classified_as_3 = 0
correctly_classified_as_3 = 0
belongs_to_3 = 0
classified_as_4 = 0
correctly_classified_as_4 = 0
belongs_to_4 = 0
classified_as_5 = 0
correctly_classified_as_5 = 0
belongs_to_5 = 0
test_1_tf_idf = test_tf_idf_dict[1]
test_2_tf_idf = test_tf_idf_dict[2]
test_3_tf_idf = test_tf_idf_dict[3]
test_4_tf_idf = test_tf_idf_dict[4]
test_5_tf_idf = test_tf_idf_dict[5]
for list_each in test_1_tf_idf:
strdata = classifier.classify(list_each)
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
correctly_classified_as_1 = correctly_classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_1 = belongs_to_1 + 1
for list_each in test_2_tf_idf:
strdata = classifier.classify(list_each)
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
correctly_classified_as_2 = correctly_classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_2 = belongs_to_2 + 1
for list_each in test_3_tf_idf:
strdata = classifier.classify(list_each)
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
correctly_classified_as_3 = correctly_classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_3 = belongs_to_3 + 1
for list_each in test_4_tf_idf:
strdata = classifier.classify(list_each)
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
correctly_classified_as_4 = correctly_classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_4 = belongs_to_4 + 1
for list_each in test_5_tf_idf:
strdata = classifier.classify(list_each)
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
correctly_classified_as_5 = correctly_classified_as_5 + 1
belongs_to_5 = belongs_to_5 + 1
# print('Classification end')
# print('classified_as_pos-->', classified_as_pos)
# print('correctly_classified_as_pos->', correctly_classified_as_pos)
# print('belongs_to_pos-->', belongs_to_pos)
# print('classified_as_neg->', classified_as_neg)
# print('correctly_classified_as_neg-->', correctly_classified_as_neg)
# print('belongs_to_neg-->', belongs_to_neg)
dict1 = {}
dict1['classified_as_1'] = classified_as_1
dict1['correctly_classified_as_1'] = correctly_classified_as_1
dict1['belongs_to_1'] = belongs_to_1
dict1['classified_as_2'] = classified_as_2
dict1['correctly_classified_as_2'] = correctly_classified_as_2
dict1['belongs_to_2'] = belongs_to_2
dict1['classified_as_3'] = classified_as_3
dict1['correctly_classified_as_3'] = correctly_classified_as_3
dict1['belongs_to_3'] = belongs_to_3
dict1['classified_as_4'] = classified_as_4
dict1['correctly_classified_as_4'] = correctly_classified_as_4
dict1['belongs_to_4'] = belongs_to_4
dict1['classified_as_5'] = classified_as_5
dict1['correctly_classified_as_5'] = correctly_classified_as_5
dict1['belongs_to_5'] = belongs_to_5
performance.calculate_accuracy('NaiveBayes', dict1)
])
model.compile(optimizer="adam",
loss="sparse_categorical_crossentropy",
metrics=['accuracy'])
model.summary()
model.fit(train_x, train_y, verbose=2, batch_size=2, epochs=50)
model.evaluate(test_x, test_y, batch_size=2, verbose=2)
ypred = np.argmax(model.predict(test_x, batch_size=2), axis=-1)
print("-------------------------------")
ACC, TPR, TNR, PPV, NPV, FPR = per.GetPerformanceMetrics(test_y,
ypred,
weighted=True)
print("Accuracy: ", ACC)
print("Recall: ", TPR)
print("Specificity: ", TNR)
print("Precision: ", PPV)
print("Negative Predictive Value: ", NPV)
print("FP rate(fall-out): ", FPR)
print(confusion_matrix(test_y, ypred))
print("-------------------------------")
print(
"0: fighting\n 1: front\n 2: ready\n 3: cat\n 4: horse \n 5: hicho \n 6: seiza"
)
import Performance as Pfce
name_Workwook = 'FLOW Operation and support report ASAP 7.0.2 JAMU66%2c WASS133 2018-09-10.xlsx'
name_Sheet = 'WASS WEEK '
'''instancia'''
object1 = Pfce.Performance(name_Workwook, name_Sheet)
'''obtencion de datos, function Text to Columns'''
wos_host_data = object1.text_To_Columns('A2009', 'A2072')
completed_data = object1.text_To_Columns('A1961', 'A2003')
failed_data = object1.text_To_Columns('A1908', 'A1955')
timeout_data = object1.text_To_Columns('A28', 'A43')
'''conversion a diccionarios'''
wos_host_dic = object1.convert_to_directory(wos_host_data[:,1], wos_host_data[:,0])
completed_dic = object1.convert_to_directory(completed_data[:,1], completed_data[:,0])
failed_dic = object1.convert_to_directory(failed_data[:,1], failed_data[:,0])
timout_dic = object1.convert_to_directory(timeout_data[:,1], timeout_data[:,0])
'''--------generate the failed and timeout details section -------'''
object1.write_xlsx_Document(object1.dictionary_to_pyMatrix(failed_dic), 'WASS-FailedDetails.xlsx')
object1.write_xlsx_Document(object1.dictionary_to_pyMatrix(timout_dic), 'WASS-TimeoutDetails.xlsx')
'''------------------ALINEAMIENTO------------------------'''
wos_host_listA, completed_listA = object1.alignment(wos_host_data[:,1], completed_data[:,1])
def SVM_Multiclass(train_tf_idf_dict, test_tf_idf_dict):
'''
#Train data
'''
# pos_list=train_input_list['pos']
# neg_list=train_input_list['neg']
train_1_tf_idf = train_tf_idf_dict[1]
train_2_tf_idf = train_tf_idf_dict[2]
train_3_tf_idf = train_tf_idf_dict[3]
train_4_tf_idf = train_tf_idf_dict[4]
train_5_tf_idf = train_tf_idf_dict[5]
training_data_X = []
training_data_Y = []
for list_each in train_1_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(1)
for list_each in train_2_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(2)
for list_each in train_3_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(3)
for list_each in train_4_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(4)
for list_each in train_5_tf_idf:
training_data_X.append(list_each)
training_data_Y.append(5)
vec = DictVectorizer()
#from sklearn.naive_bayes import MultinomialNB
#clf = MultinomialNB()
#clf.fit(vec.fit_transform(training_data_X), training_data_Y)
'''
param_grid = [
{'C': [1, 10, 100, 1000], 'kernel': ['linear']},
{'C': [1, 10, 100, 1000], 'gamma': [0.001, 0.0001], 'kernel': ['rbf']},
]
'''
#svm_multi = OneVsRestClassifier(LinearSVC(random_state=0))
#cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)
#grid = GridSearchCV(svm_multi, param_grid=param_grid, cv=cv)
#grid.fit(vec.fit_transform(training_data_X), training_data_Y)
#print("The best parameters are %s with a score of %0.2f->" % (grid.best_params_, grid.best_score_))
'''
C_2d_range = [1e-2, 1, 1e2]
gamma_2d_range = [1e-1, 1, 1e1]
classifiers = []
for C in C_2d_range:
for gamma in gamma_2d_range:
clf = svm_multi(C=C, gamma=gamma)
clf.fit(X_2d, y_2d)
classifiers.append((C, gamma, clf))
'''
svm_multi = OneVsRestClassifier(LinearSVC(random_state=0))
svm_multi.fit(vec.fit_transform(training_data_X), training_data_Y)
'''
#Test data
'''
print('Training end')
classified_as_1 = 0
correctly_classified_as_1 = 0
belongs_to_1 = 0
classified_as_2 = 0
correctly_classified_as_2 = 0
belongs_to_2 = 0
classified_as_3 = 0
correctly_classified_as_3 = 0
belongs_to_3 = 0
classified_as_4 = 0
correctly_classified_as_4 = 0
belongs_to_4 = 0
classified_as_5 = 0
correctly_classified_as_5 = 0
belongs_to_5 = 0
test_1_tf_idf = test_tf_idf_dict[1]
test_2_tf_idf = test_tf_idf_dict[2]
test_3_tf_idf = test_tf_idf_dict[3]
test_4_tf_idf = test_tf_idf_dict[4]
test_5_tf_idf = test_tf_idf_dict[5]
for list_each in test_1_tf_idf:
strdata = svm_multi.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
correctly_classified_as_1 = correctly_classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_1 = belongs_to_1 + 1
for list_each in test_2_tf_idf:
strdata = svm_multi.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
correctly_classified_as_2 = correctly_classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_2 = belongs_to_2 + 1
for list_each in test_3_tf_idf:
strdata = svm_multi.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
correctly_classified_as_3 = correctly_classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_3 = belongs_to_3 + 1
for list_each in test_4_tf_idf:
strdata = svm_multi.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
correctly_classified_as_4 = correctly_classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
belongs_to_4 = belongs_to_4 + 1
for list_each in test_5_tf_idf:
strdata = svm_multi.predict(vec.transform(list_each))[0]
if (strdata == 1):
classified_as_1 = classified_as_1 + 1
elif (strdata == 2):
classified_as_2 = classified_as_2 + 1
elif (strdata == 3):
classified_as_3 = classified_as_3 + 1
elif (strdata == 4):
classified_as_4 = classified_as_4 + 1
elif (strdata == 5):
classified_as_5 = classified_as_5 + 1
correctly_classified_as_5 = correctly_classified_as_5 + 1
belongs_to_5 = belongs_to_5 + 1
# print('Classification end')
# print('classified_as_pos-->', classified_as_pos)
# print('correctly_classified_as_pos->', correctly_classified_as_pos)
# print('belongs_to_pos-->', belongs_to_pos)
# print('classified_as_neg->', classified_as_neg)
# print('correctly_classified_as_neg-->', correctly_classified_as_neg)
# print('belongs_to_neg-->', belongs_to_neg)
dict1 = {}
dict1['classified_as_1'] = classified_as_1
dict1['correctly_classified_as_1'] = correctly_classified_as_1
dict1['belongs_to_1'] = belongs_to_1
dict1['classified_as_2'] = classified_as_2
dict1['correctly_classified_as_2'] = correctly_classified_as_2
dict1['belongs_to_2'] = belongs_to_2
dict1['classified_as_3'] = classified_as_3
dict1['correctly_classified_as_3'] = correctly_classified_as_3
dict1['belongs_to_3'] = belongs_to_3
dict1['classified_as_4'] = classified_as_4
dict1['correctly_classified_as_4'] = correctly_classified_as_4
dict1['belongs_to_4'] = belongs_to_4
dict1['classified_as_5'] = classified_as_5
dict1['correctly_classified_as_5'] = correctly_classified_as_5
dict1['belongs_to_5'] = belongs_to_5
performance.calculate_accuracy('SVM_Multiclass', dict1)
inboundlist_path = Library.getSetting('source', 'inboundlist', 0, 0, 0)
isexist = 0
while (not isexist):
inputString = '0'
itmes = ['1', '2', '3', '4', '9']
while (inputString not in itmes):
inputString = input(
'1. Execute the Performance Report\n2. Execute the Performance Report with UT rate markup\n3. Execute the 分時表 Report\n4. Execute the Inboundlist Report\n9. Exit!!!\nWhat is your choise (input 1 or 2 or 3 or 4 or 9): '
)
if inputString == '1':
Performance.MainPerformance(rawdata_path,
rawdata_date,
0,
utrate,
wordtimemarkup,
debug=1)
elif inputString == '2':
Performance.MainPerformance(rawdata_path,
rawdata_date,
1,
utrate,
wordtimemarkup,
debug=1)
elif inputString == '3':
Division.MainDivision(rawdata_path, rawdata_date)
elif inputString == '4':
Inbound.MainInbound(inboundlist_path)
elif inputString == '9':
isexist = 1