def getCorrelationOfReturns(country):
dir = r"C:\\Users\\michael\\Desktop\\Test_output\\"
factors = ['Book Value', 'Div Pay Ratio (N)', 'Div Yld', 'Engs Yld', 'Cfl Yld',
'Sales to Pr', 'EBIT to EV (N)', 'EBITDA to Pr', 'RoA (N)', 'RoE', 'ROIC (N)',
'Sustainable GR', 'Asset Turnover (N)', 'Gross Prof to Assts (N)', 'Sales Gr',
'Inc to Sales', 'Op Prof Marg (N)', 'Gross Prof Marg (N)', 'Ass to Eq (N)']
nfactors = len(factors)
arr = []
for j in range(nfactors):
factor = factors[j].replace(" (N)","")
file = dir + "Michael Style 50-100 " + factor + " " + country + " MC M1 200611 to 201610 Dec.xlsx"
book=xlrd.open_workbook(file)
sheet=book.sheet_by_name('Return Data')
list = []
for i in range(5,123):
value = sheet.cell_value(12,i)
list.append(value)
arr.append([])
arr[j].append(list)
results = []
for k in range(nfactors):
results.append([])
for l in range(nfactors):
corr = np.corrcoef(arr[k],arr[l])
temp = corr[0,1]
temp2 = round(temp,2)
results[k].append(temp2)
fig = FF.create_annotated_heatmap(x = factors, y = factors, z=results)
fig['layout'].update(
title=country+' (Correlation)',
xaxis=dict(ticks='', ticksuffix='', side='bottom'),
width=450,
height=300,
margin=go.Margin(
l=180,
r=80,
b=100,
t=100,
pad=4
),
autosize=False
)
py.iplot(fig, filename=country+'-Correlation')
def makeCorrelationByFactorVal(engine):
df = pd.read_sql_query(string2, engine)
corr = df.corr(method='spearman')
fig = FF.create_annotated_heatmap(x=list(corr.columns.values),
y=list(corr.columns.values),
z=np.round(corr.values, 2))
fig['layout'].update(title='Correlations by factor val. (Dec 2015)',
xaxis=dict(ticks='', ticksuffix='', side='bottom'),
width=450,
height=300,
margin=go.Margin(l=180, r=80, b=100, t=100, pad=4),
autosize=False)
py.iplot(fig, filename='Correlations')
def correlation(df):
correl = np.fliplr(df.corr(method='pearson').as_matrix())
x = df.columns.format()
y = x[::-1]
z_text = np.around(correl, 2)
colorscale = [[-1, '#0000FF'], [1, '#FF0000']] # custom colorscale
fig = FF.create_annotated_heatmap(correl, x=x, y=y, annotation_text=z_text)
fig.layout.update(
dict(title="Pearson Correlation Matrix",
height=800,
width=800,
autosize=False))
fig.data.update(
dict(colorscale=colorscale,
showscale=True,
colorbar=dict(title="correlation coefficient")))
return fig
def getMarketsAnalyzerOutput(s):
dir = r"C:\\Users\\michael\\Desktop\\Test_output\\"
factors = ['Book Value', 'Div Pay Ratio (N)', 'Div Yld', 'Engs Yld', 'Cfl Yld',
'Sales to Pr', 'EBIT to EV (N)', 'EBITDA to Pr', 'RoA (N)', 'RoE', 'ROIC (N)',
'Sustainable GR', 'Asset Turnover (N)', 'Gross Prof to Assts (N)', 'Sales Gr',
'Inc to Sales', 'Op Prof Marg (N)', 'Gross Prof Marg (N)', 'Ass to Eq (N)']
countries = ['UNITED STATES', 'CHINA', 'JAPAN', 'UNITED KINGDOM', 'SWITZERLAND', 'GERMANY', 'HONG KONG',
'FRANCE', 'CANADA', 'AUSTRALIA', 'NETHERLANDS', 'SOUTH AFRICA', 'SINGAPORE']
if (s == 'Return_10Year'):
plottitle = '10 Year Return (%)'
y=21
x=4
rounddp=0
percent=100
if (s == 'Return_1Year'):
plottitle = '12 Month Return (%)'
y=22
x=4
rounddp=0
percent=100
if (s == 'TrackingError'):
plottitle = 'Tracking Error (%)'
y=21
x=6
rounddp=0
percent=100
if (s == 'TrackingError_2Year'):
plottitle = '2 Year Tracking Error (%)'
y=22
x=6
rounddp=0
percent=100
if (s == 'StdDev'):
plottitle = 'Standard Deviation'
y=23
x=6
rounddp=3
percent=1
if (s == 'StdDev_2Year'):
plottitle = '2 Year Standard Deviation'
y=24
x=6
rounddp=3
percent=1
if (s == 'StyleBeta'):
plottitle = 'Beta'
y=25
x=6
rounddp=2
percent=1
if (s == 'Regularity_3Month'):
plottitle = '3 Month Regularity'
y=21
x=8
rounddp=3
percent=1
if (s == 'Regularity_6Month'):
plottitle = '6 Month Regularity'
y=22
x=8
rounddp=3
percent=1
if (s == 'Regularity_12Month'):
plottitle = '12 Month Regularity'
y=23
x=8
rounddp=3
percent=1
if (s == 'Identity'):
plottitle = 'Identity (%)'
y=25
x=2
rounddp=0
percent=100
if (s == 'Attrib'):
plottitle = 'Attribution'
y=23
x=2
rounddp=2
percent=1
corr = []
for i in range(len(factors)):
corr.append([])
for coun in countries:
factor = factors[i].replace(" (N)","")
file = dir + "Michael Style 50-100 " + factor + " " + coun + " MC M1 200611 to 201610 Dec.xlsx"
book=xlrd.open_workbook(file)
sheet=book.sheet_by_name('Style Graph')
value = sheet.cell_value(y,x)
corr[i].append(value*percent)
fig = FF.create_annotated_heatmap(x = countries, y = factors, z=np.round(corr,rounddp))
fig['layout'].update(
title=plottitle,
xaxis=dict(ticks='', ticksuffix='', side='bottom'),
width=450,
height=300,
margin=go.Margin(
l=180,
r=50,
b=100,
t=100,
pad=4
),
autosize=False
)
py.plot(fig, filename=s)
test_result = []
""" convert from coo matrix to csr matrix """
X_new_test = X_new_test.tocsr()
""" make prediction """
for i in xrange(0, len(test_ind)):
test_result.extend(
gbm_final.predict(X_new_test[test_ind[i], :].toarray()).tolist())
result = map(int, test_result)
""" create confusion matrix on the new test set """
from sklearn.metrics import confusion_matrix
confusion_matrix(Y_new_test, result)
"""
-----------------------------------------------------------------------------------------------
Visualize Confusion Matrix by A Heatmap
-----------------------------------------------------------------------------------------------
"""
import plotly.plotly as py
py.sign_in('wenbo5565', 'HwoyFhKNNytDqVjYSR3q')
import plotly.graph_objs as go
from plotly.graph_objs import *
from plotly.tools import FigureFactory as FF
z = [[1752, 193348], [224267, 20092]]
z_text = [['False Negative:1752', 'True Negative:193348'],
['True Positive:224267', 'False Positive:20092']]
x = ['True: 1', 'True: 0']
y = ['Predicted: 0', 'Predicted: 1']
fig = FF.create_annotated_heatmap(z, x=x, y=y, annotation_text=z_text)
plot_url = py.plot(fig)
def main():
parser = argparse.ArgumentParser(description='MalConv adversarial examples')
parser.add_argument('filename', type=str)
args = parser.parse_args()
model = load_model('malconv.h5')
maxlen = model.input_shape[1]
padding_char = 256
inp = np.asarray([get_file(args.filename, maxlen, padding_char)], dtype=np.uint16)
whole_saliency_grad = visualize_saliency(model, -1, filter_indices=None, seed_input=inp, wrt_tensor=model.layers[1].output)
whole_saliency_grad = whole_saliency_grad.reshape(-1, int(sqrt(whole_saliency_grad.shape[0])))
layout= go.Layout(
autosize=True,
#width = 1200,
height = 800,
margin=dict(
l=40,
r=40,
b=40,
t=5
),
hovermode="closest",
xaxis = {'constrain': 'domain'},
yaxis = { 'autorange': 'reversed',
'showgrid':False,
'showline':False,
'zeroline':False,
'scaleanchor': "x",
'scaleratio':1},
)
# Display element index on hover
hover=list(range(whole_saliency_grad.shape[0]))
for j in range(whole_saliency_grad.shape[0]):
hover[j] = ["0x{0:02X}".format(i) for i in range(j*whole_saliency_grad.shape[1],(j+1)*whole_saliency_grad.shape[1])]
data = [
go.Heatmap(
z=whole_saliency_grad,
colorscale='jet',
hoverinfo='text',
text=hover
)
]
plot({"data": data, "layout": layout}, filename='{}.html'.format(args.filename.replace('.exe','')))
LENGTH = 1024 #1024 * 1024
LINE_LEN = 32
saliency_grad = visualize_saliency(model, -1, filter_indices=None, seed_input=inp, wrt_tensor=model.layers[1].output)
saliency_grad = saliency_grad[:LENGTH].reshape(-1, LINE_LEN)
bytez = np.array(numpy_to_bytez(inp[0], padding_char)[:LENGTH]).reshape(-1, LINE_LEN).tolist()
bytez_str = [[chr(x) if chr(x) in printable[:-5] else "{0:02X}".format(x) for x in chunk] for chunk in bytez]
x = ["0x{0:02X}".format(i) for i in range(0, LINE_LEN)]#list(range(0,LINE_LEN))
y = ["0x{0:02X}".format(i) for i in range(0, LENGTH, LINE_LEN)]
x_text = ["{0:02X}".format(i) for i in range(0, LINE_LEN)]#list(range(0,LINE_LEN))
y_text = ["{0:02X}".format(i) for i in range(0, LENGTH, LINE_LEN)]
# Display element index on hover
hover=list(range(len(bytez_str)))
for j in range(len(bytez_str)):
hover[j] = ["0x{0:02X}".format(i) for i in range(j*LINE_LEN,(j+1)*LINE_LEN)]
fig = ff.create_annotated_heatmap(saliency_grad, x=x, y=y, annotation_text=bytez_str, hoverinfo='text', text=hover, colorscale='jet')
fig.layout.yaxis.autorange = 'reversed'
fig.layout.xaxis.tickvals = x
fig.layout.yaxis.tickvals = y
fig.layout.xaxis.ticktext = x_text
fig.layout.yaxis.ticktext = y_text
fig.layout.height = 800
plot(fig, filename='{}_HEAD.html'.format(args.filename.replace('.exe','')))
import plotly.plotly as py
import plotly.graph_objs as go
from plotly.tools import FigureFactory as FF
#plotly.tools.set_credentials_file(username='******', api_key='dpVqkr9iHl7kiQKea9mG')
import numpy as np
import pandas as pd
import scipy
matrix1 = np.matrix(
[[1, 5],
[6, 2]]
)
matrix2 = np.matrix(
[[-1, 4],
[3, -6]]
)
matrix_sum = matrix1 + matrix2
print(matrix_sum)
colorscale = [[0, '#EAEFC4'], [1, '#9BDF46']]
font=['#000000', '#000000']
table = FF.create_annotated_heatmap(matrix_sum.tolist(), colorscale=colorscale, font_colors=font)
py.iplot(table, filename='matrix-sum')
def heatmap(probabilities, piece="all"): if piece == "all": z = np.argmax(probabilities, axis=0) data = [ go.Heatmap( z=z, colorscale=custom_colorscale ) ] # annotations = [] # for n, row in enumerate(z): # for m, val in enumerate(row): # var = z[n][m] # annotations.append( # dict( # text=str(val), # x=x[m], y=y[n], # xref='x1', yref='y1', # font=dict(color='white' if val > 0 else 'black'), # showarrow=False) # ) layout = go.Layout( title = "Board", # annotations = annotations, xaxis = dict(ticks=''), yaxis = dict(ticks='', ticksuffix=' '), width = 800, height = 800 ) fig = go.Figure(data=data, layout=layout) plot_url = py.plot(fig, filename="heatmap-all-1") else: piece_class = piece_classes[piece] z = probabilities[piece_class,:,:] # data = [ # go.Heatmap( # x=x, # y=y, # z=z # ) # ] # annotations = [] # for n, row in enumerate(z): # for m, val in enumerate(row): # var = z[n][m] # annotations.append( # dict( # text=str(val), # x=x[m], y=y[n], # xref='x1', yref='y1', # font=dict(color='white' if val > 0 else 'black'), # showarrow=False) # ) # layout = go.Layout( # title = piece, # annotations = probabilities[piece_class,:,:], # xaxis = dict(ticks=''), # yaxis = dict(ticks='', ticksuffix=' '), # width = 800, # height = 800 # ) # fig = go.Figure(data=data, layout=layout) z_text = np.around(z, decimals=3) fig = FF.create_annotated_heatmap(z, annotation_text=z_text) plot_url = py.plot(fig, filename="heatmap-" + piece + "-1")