def corr_plot(data):
'''
Get correlation matrix of numeric columns in the data and make a correlation plot
Parameter description:
(1) data - Dataframe for which correlation needs to be found out
'''
corr = data.corr()
corr_unpivot = pd.melt(
corr.reset_index(), id_vars='index'
) # Unpivot the dataframe, so we can get pair of arrays for x and y
corr_unpivot.columns = ['x', 'y', 'value']
n_colors = 256 # Use 256 colors for the diverging color palette
palette = sns.diverging_palette(20, 220, n=n_colors) # Create the palette
heatmap(x=corr_unpivot['x'],
y=corr_unpivot['y'],
size=corr_unpivot['value'].abs(),
color=corr_unpivot['value'],
color_range=(-1, 1),
palette=palette)
return corr
def create(self, file_path: str) -> None:
plt.figure(figsize=(12, 11.0))
plt.gcf().subplots_adjust(bottom=0.28)
small_size = 16
medium_size = 16
bigger_size = 16
plt.rc('font', size=small_size) # controls default text sizes
plt.rc('axes', titlesize=small_size) # fontsize of the axes title
plt.rc('axes', labelsize=medium_size) # fontsize of the x and y labels
plt.rc('xtick', labelsize=small_size) # fontsize of the tick labels
plt.rc('ytick', labelsize=small_size) # fontsize of the tick labels
plt.rc('legend', fontsize=small_size) # legend fontsize
plt.rc('figure', titlesize=bigger_size) # fontsize of the figure title
heatmap(
x=self._data["pattern"],
y=self._data["metric"],
x_order=DESIGN_PATTERN_NAMES.values(),
y_order=reversed(list(MISC_METRICS_NAMES.values())),
size=self._data["correlation"].abs(),
size_range=[0, 1],
size_scale=1500,
color=self._data["correlation"],
color_range=[-1, 1],
#palette=sns.diverging_palette(220, 20, n=256),
palette=sns.color_palette("coolwarm", 256),
xlabel="Design Pattern",
ylabel="C&K+ Metric",
)
plt.savefig(file_path)
def display_results(all_results, bac, total_mat, mats, name):
"""Creates a heatmap confusion matrix and swarmplot for prediction results
Arguments:
all_results -- results dictionary from train_all
bac -- balanced accuracy score
total_mat -- confusion matrix with values from all folds
mats -- list of individual confusion matrices from each fold
name -- name of the trial for file naming
"""
hm.heatmap(total_mat, name, './trials/{}/total.png'.format(name))
with open('./trials/{}/matrix.txt'.format(name), 'w') as f:
f.write('Balanced Accuracy: {}\n'.format(youden_j))
f.write(str(total_mat))
for i, mat in enumerate(mats):
hm.heatmap(mat, name + ' {}'.format(i + 1),
'./trials/{}/{}.png'.format(name, i + 1))
f_2, ax_2 = plt.subplots(1, 1, figsize=(6, 10), dpi=300)
sns.swarmplot(x='Actual',
y='Prediction',
hue='Person',
data=all_results,
ax=ax_2)
f_2.tight_layout()
plt.savefig('./trials/{}/swarmplot.png'.format(name))
def football():
path = "../data/football/football.txt"
gnc_path = "../data/football/footballTSEinputConference.clu"
name = 'American College Football Network'
# load network and ground-truth communities
G = loadG(path)
G.graph['name'] = name
gnc = loadGNC(gnc_path)
print(nx.info(G))
# community detection
# Either Choice 1
comms = list(
multiscale_community_detection(G,
resolution=0.5,
verbose=True,
force_pvalue_sampling=True))
# Or Choice 2
#comms = greedy_modularity_communities(G)
# check NMI
map_comm = {v: i for i, c in enumerate(comms) for v in c}
a = [map_comm[k] for k in G.nodes()]
b = [gnc[k] for k in G.nodes()]
print("NMI=", metrics.adjusted_mutual_info_score(a, b))
print("#Comm=", len(comms))
print(comms)
# draw topology
#draw(G, map_comm)
# draw heatmap
heatmap(G, comms)
def generate_heat_new(self,index=1):
circs=[entry for entry in self.entries if entry.check_WT_KO_circadian(index)!=0];
noncircs=[entry for entry in self.entries if entry.check_WT_KO_circadian(index)==0];
mean_wt=[entry.mean[0] for entry in circs];
mean_ko=[entry.mean[1] for entry in circs];
hp.heatmap(np.array(mean_wt),"WT");
hp.heatmap(np.array(mean_ko),"KO");
return
def edit_heatmap (request, dataset, filename):
infodir=helpers.get_infodir(dataset)
if not os.path.exists(infodir+'/heatmaps'):
os.makedirs(infodir+'/heatmaps')
args=helpers.read_csv_dict('%s/heatmaps/%s_meta.csv' % (infodir,filename))
print '%s/heatmaps/%s_meta.csv' % (infodir, filename)
h=heatmap.heatmap()
col_info, coltypes_bycol=helpers.get_col_types(infodir)
colnames,groupcolnames=get_colnames_for_heatmap (infodir, args.get('displaymode','heatmap'), col_info)
template = loader.get_template('makemap.html')
args_json=cjson.encode(args)
context = RequestContext(request, {'colnames':colnames,
'add_new_heatmap':True,
'groupcolnames':groupcolnames,
'dataset':dataset,
'defaults_json':args_json,
'simple_vars':h.simple_vars,
'no_rebuild':h.no_rebuild,
'booleans':h.booleans,
'colormapnames':colormapnames})
return HttpResponse(template.render(context))
def corrheatmap(data, method, linkage):
corr = data.corr(method=method)
heat = heatmap(corr, method=linkage, metric='correlation')
# Put optional secondary index in _class column.
if len(data.index.names) > 1:
data['_class'] = data.index.get_level_values(data.index.names[1])
data.index = data.index.get_level_values(data.index.names[0])
return pd.concat([heat, data])
def add_tensorboard(inputs, targets, outputs, global_step, name='Train'):
# Make targets and output slices
trgt_slice = targets.sum(dim=1, keepdim=True)
otpt_slice = outputs.sum(dim=1, keepdim=True)
trgt_htmp = heatmap(trgt_slice).to(args.device)
otpt_htmp = heatmap(otpt_slice).to(args.device)
# Make grids
image_grid = make_grid(inputs, nrow=4, padding=2, pad_value=1)
trgt_slice_grid = make_grid(trgt_slice, nrow=4, padding=2, pad_value=1)
otpt_slice_grid = make_grid(otpt_slice, nrow=4, padding=2, pad_value=1)
trgt_htmp_grid = make_grid(trgt_htmp, nrow=4, padding=2, pad_value=1)
otpt_htmp_grid = make_grid(otpt_htmp, nrow=4, padding=2, pad_value=1)
# Create Heatmaps grid
args.writer.add_image('{}/gt'.format(name), trgt_htmp_grid, global_step)
args.writer.add_image('{}/gt_image'.format(name),
image_grid + trgt_slice_grid, global_step)
args.writer.add_image('{}/pred'.format(name), otpt_htmp_grid, global_step)
args.writer.add_image('{}/pred_image'.format(name),
image_grid + otpt_slice_grid, global_step)
def create(self, file_path: str) -> None:
plt.figure(figsize=(12, 11.5))
plt.gcf().subplots_adjust(bottom=0.25, left=0.25)
small_size = 16
medium_size = 16
bigger_size = 16
plt.rc('font', size=small_size) # controls default text sizes
plt.rc('axes', titlesize=small_size) # fontsize of the axes title
plt.rc('axes', labelsize=medium_size) # fontsize of the x and y labels
plt.rc('xtick', labelsize=small_size) # fontsize of the tick labels
plt.rc('ytick', labelsize=small_size) # fontsize of the tick labels
plt.rc('legend', fontsize=small_size) # legend fontsize
plt.rc('figure', titlesize=bigger_size) # fontsize of the figure title
heatmap(
x=self._data["pattern_1"],
y=self._data["pattern_2"],
x_order=DESIGN_PATTERN_NAMES.values(),
y_order=reversed(list(DESIGN_PATTERN_NAMES.values())),
size=self._data["correlation"].abs(),
size_range=[0, 1],
size_scale=900,
color=self._data["correlation"],
color_range=[-1, 1],
#palette=sns.diverging_palette(220, 20, n=256),
palette=sns.color_palette("coolwarm", 256),
#xlabel="Metric 1",
#ylabel="Metric 2",
)
#sns.heatmap(self._data, annot=True, square=True)
plt.savefig(file_path)
def testBlock(_tdate = None, _twin = None): if (_tdate == None) or (_twin == None): print "Error: No target date or time window input" return try: #Get list of sensors sensorsInfo = qs.GetSensorDF() columns = sensorsInfo["name"] for column in columns: print """\n\nCurrent Sensor Column: %s, Target Date: %s, Time Window: %s \n\n""" % (column, _tdate, _twin) hmap_data = hmap.heatmap(col=column, t_timestamp=_tdate) print hmap_data except IndexError: print '>> Error in writing extracting database data to files..'
def display_heatmap(self, img_path, img_high, img_width, layer_name,
load_data):
IMG = image.load_img(img_path, target_size=(img_high, img_width))
plt.show()
img = image.img_to_array(IMG)
img = np.expand_dims(img, axis=0)
img = preprocess_input(img)
hmap = heatmap()
img_heatmap = hmap.create_heat_map(self.model, layer_name, img,
img_high, img_width, 3, load_data)
print('heatmap')
plt.imshow(img_heatmap)
plt.show()
img_heatmap = hmap.super_impose_heatmap(img_path, img_heatmap)
plt.imshow(img_heatmap)
plt.show()
def testBlock(_tdate=None, _twin=None):
if (_tdate == None) or (_twin == None):
print "Error: No target date or time window input"
return
try:
#Get list of sensors
sensorsInfo = qs.GetSensorDF()
columns = sensorsInfo["name"]
for column in columns:
print """\n\nCurrent Sensor Column: %s, Target Date: %s, Time Window: %s \n\n""" % (
column, _tdate, _twin)
hmap_data = hmap.heatmap(col=column, t_timestamp=_tdate)
print hmap_data
except IndexError:
print '>> Error in writing extracting database data to files..'
def plotComparison(aDict, struct, rows, cols):
fig, axes = plt.subplots(nrows=3, ncols=2, figsize=(10, 6))
mini = []
maxi = []
for inm, name_model in enumerate(aDict):
temp = aDict[name_model][struct]
mini.append(
[np.min(temp['AC']),
np.min(temp['AC|B']),
np.min(temp['ABC'])])
maxi.append(
[np.max(temp['AC']),
np.max(temp['AC|B']),
np.max(temp['ABC'])])
mini = np.min(mini, axis=0)
maxi = np.max(maxi, axis=0)
for inm, name_model in enumerate(aDict):
temp = aDict[name_model][struct]
#title = "I_" + name_info
norm = mcolors.DivergingNorm(0., vmin=mini[0], vmax=maxi[0])
hm.heatmap(np.array(temp['AC']),
np.round(cols, decimals=1),
rows,
norm=norm,
cmap='seismic',
ax=axes[0][inm],
cbarlabel='$I(A;C)$')
norm = mcolors.DivergingNorm(0., vmin=mini[1], vmax=maxi[1])
hm.heatmap(np.array(temp['AC|B']),
np.round(cols, decimals=1),
rows,
norm=norm,
cmap='seismic',
ax=axes[1][inm],
cbarlabel='$I(A;C|B)$')
norm = mcolors.DivergingNorm(0., vmin=mini[2], vmax=maxi[2])
hm.heatmap(np.array(temp['ABC']),
np.round(cols, decimals=1),
rows,
norm=norm,
cmap='seismic',
ax=axes[2][inm],
cbarlabel='$I(A;B;C)$')
#texts = annotate_heatmap(im, valfmt="{x:.3f}")
axes[0][inm].set_title(name_model.capitalize())
plt.savefig('comparison_' + '_'.join([nm for nm in aDict]) + '_' + struct +
'.pdf',
transparent=True)
def optimal_play_chart(player):
data = np.genfromtxt('../HU_push_fold/nash_' + player + '.csv',
delimiter=',',
dtype=float)
data = np.reshape(data, (13, 13))
fig, ax = plt.subplots()
title = 'Small blind optimal play' if player == 'sb' else 'Big blind optimal play'
ax.set_title(title)
axis_labels = [
"A", "K", "Q", "J", "T", "9", "8", "7", "6", "5", "4", "3", "2"
]
im, cbar = heatmap.heatmap(data,
axis_labels,
axis_labels,
ax=ax,
cmap="YlGn",
cbarlabel="Big Blinds")
texts = heatmap.annotate_heatmap(im, valfmt="{x:.1f}")
fig.tight_layout()
fig.savefig('./' + player + '_range.svg')
def make_score_heatmap(parameters, param1, param2, grid_model, dataset):
"""
Produce heatmap of mean scores for each combination of the parameters
Parameters
----------
parameters : dictionary of lists
Grid values for the Grid Search algorithm
param1 : string
The parameter C
param2 : string
The parameter gamma
grid_model : GridSearchCV object
Trained model
dataset : string
Type of dataset: 'All' or 'Reduced'
Returns
-------
Produce chart files in directory ../../target/visualization
"""
scores = [score for score in grid_model.cv_results_['mean_test_score']]
scores = np.array(scores).reshape(6, 6)
fig, ax = plt.subplots()
im, cbar = heatmap(scores,
parameters[param1],
parameters[param2],
ax=ax,
cmap="YlGn",
cbarlabel="Mean Scores")
annotate_heatmap(im, valfmt="{x:.2f}")
ax.set_xlabel(param2)
ax.set_ylabel(param1)
title = dataset + ' Datapoints GridSearchCV Mean Scores'
ax.set_title(title)
fig.tight_layout()
outfile = '../../target/visualization/' + title + '.png'
plt.savefig(outfile, format="png")
def show_heatmap_form (request, dataset, x_var=None, y_var=None):
infodir=helpers.get_infodir(dataset)
if not os.path.exists(infodir):
os.makedirs(infodir)
if not os.path.exists(infodir+'/heatmaps'):
os.makedirs(infodir+'/heatmaps')
col_info, coltypes_bycol=helpers.get_col_types(infodir)
# args['displaymode'] uit post peuteren.
colnames, groupcolnames=get_colnames_for_heatmap (infodir, 'heatmap', col_info)
colnames, groupcolnames=get_all_colnames (infodir, 'heatmap', col_info)
#print 'colnames:', len(colnames), colnames
if len(colnames)<2:
msg='Geen heatmaps te maken van deze dataset'
template = loader.get_template('makemap.html')
context = RequestContext(request, {'msg':'msg','colnames':colnames, 'dataset':dataset})
return HttpResponse(template.render(context))
h=heatmap.heatmap()
args={}
for defaultval in h.defaults:
args[defaultval[0]]=defaultval[1]
if x_var is None:
args['x_var']=colnames[0]
else:
args['x_var']=str(x_var)
if y_var is None:
args['y_var']=colnames[1]
else:
args['y_var']=str(y_var)
x_types=coltypes_bycol[args['x_var']] # info van variabelenaam x-kolom ophalen
y_types=coltypes_bycol[args['y_var']]
args['x_min']=x_types.get(args['x_min'],args['x_min'])
args['y_min']=y_types.get(args['y_min'],args['y_min'])
args['x_max']=x_types.get(args['x_max'],args['x_max'])
args['y_max']=y_types.get(args['y_max'],args['y_max'])
print 'xmin/xmax:',args['x_min'],args['x_max']
x_min=float(args['x_min'])
x_max=float(args['x_max'])
y_min=float(args['y_min'])
y_max=float(args['y_max'])
dx=x_max-x_min
dy=y_max-y_min
steps_set=False
if dy>20 and dy<150:
steps_set=True
args['y_steps']=dy
args['x_steps']=dy
if dy<70:
args['imgheight']=8*dy
args['imgwidth']=8*dy
else:
args['imgheight']=4*dy
args['imgwidth']=4*dy
if (steps_set==False) and (dx>20 and dx<150):
steps_set=True
args['y_steps']=dx
args['x_steps']=dx
if dx<70:
args['imgheight']=8*dx
args['imgwidth']=8*dx
else:
args['imgheight']=4*dx
args['imgwidth']=4*dx
if steps_set==False:
xkeys=args['x_steps'] # autorange steps op aantal keys
setkeys=False
x_types=coltypes_bycol[args['x_var']] # info van variabelenaam x-kolom ophalen
y_types=coltypes_bycol[args['y_var']]
if x_types['num_keys']<args['x_steps']:
xkeys=x_types['num_keys']
setkeys=True
ykeys=args['y_steps']
if y_types['num_keys']<args['y_steps']:
ykeys=y_types['num_keys']
setkeys=True
if setkeys:
keys=xkeys
if ykeys<xkeys:
keys=ykeys
width=args['imgwidth']
stepsizes=[]
for i in range(50,0,-1):
if (width/(i*1.0))==int(width/i):
stepsizes.append(int(width/i))
prevk=stepsizes[0]
for k in stepsizes[1:]:
if keys<k:
keys=prevk
break
prevk=k
args['x_steps']=keys
args['y_steps']=keys
labels=helpers.read_csv_dict (infodir+'/labels.csv')
col_x=x_types['colname']
col_y=y_types['colname']
args['x_label']=labels.get(col_x,col_x)
args['y_label']=labels.get(col_y,col_y)
args['title']=labels.get(col_x,col_x)+' vs '+labels.get(col_y,col_y)
filename='%(infodir)s\heatmaps\%(x_var)s_%(y_var)s_*meta.js' % locals()
heatmaps=glob.glob(filename)
add_new_heatmap=False
if len(heatmaps)>0:
add_new_map=True
heatmap_indexnr=len(heatmaps)
args['heatmap_indexnr']=heatmap_indexnr
args_json=cjson.encode(args)
defaults_json=args_json.replace('False','false').replace('True','true')
template = loader.get_template('makemap.html')
context = RequestContext(request, {'colnames':colnames,
'add_new_heatmap':add_new_heatmap,
'groupcolnames':groupcolnames,
'dataset':dataset,
'defaults_json':args_json,
'simple_vars':h.simple_vars,
'no_rebuild':h.no_rebuild,
'booleans':h.booleans,
'colormapnames':colormapnames})
return HttpResponse(template.render(context))
# create vector to label as mutated or non-mutated
mut_status = []
for x in xrange(0,len(endpoint)):
if endpoint[x] == 0:
mut_status.append("-")
else:
mut_status.append("MUTATION")
# create vector of biomarker names
col_names = [train_header[z] for z in s]
"""heatmap(x, row_header, column_header, row_method,
column_method, row_metric, column_metric,
color_gradient, filename)
"""
fname = 'gene-heatmaps/' + endpoint_header[i] + '.png'
heatmap.heatmap(x = dat.transpose(), row_header = col_names, column_header = mut_status, \
row_method = 'ward', column_method = 'ward', row_metric = 'euclidean', column_metric = 'euclidean', \
color_gradient = 'red_white_blue', filename = fname)
# Survival analysis time
# Calculate survival scores the significant genes
time = robjects.IntVector(surv_data[:, 0])
event = robjects.IntVector(surv_data[:, 1])
for x in xrange(0, len(s)):
y = [getGrps(dat[l, x], endpoint[l]) for l in range(len(endpoint))]
y = robjects.StrVector(y)
fit1 = surv.Surv(time, event)
robjects.globalenv["fit1"] = fit1
robjects.globalenv["y"] = y
fmla = robjects.Formula("fit1 ~ y")
fit2 = surv.survfit(fmla)
r.plot(fit2, lty=1:4, col=1:4, conf_int = FALSE, xlab="Survival in Days", ylab = "Proportion Survived")
import misssion as ms
import heatmap as ht
import histogram as hg
import scatter_plot as sp
import range as rg
a=ms.statisticsGet('sh600418')
b=ms.timeGet(a)
ms.plotTrend(a,b)
ht.heatmap(a)
hg.histogram(a,1)
sp.scatter_plot(a,400)
rg.Range_of_decline_and_rise_plot(a,b)
dep[i] = 'Não especificado'
dep_options = sorted(list(set(dep)))
chap_set = list(set([x for x in chap if x != 19]))
prov_chap = np.zeros((len(dep_options), len(chap_set)))
for aux in range(len(dep)):
i = dep_options.index(dep[aux])
for j in icd9_chap([x[:3] for x in labels_cid[aux]]):
if j < 19: prov_chap[i][j - 1] += 1
fig, ax = plt.subplots(figsize=(20, 25))
im = heatmap(prov_chap,
dep_options,
chap_set,
ax=ax,
cmap="Reds",
cbarlabel="Prevalence")
text = annotate_heatmap(im, valfmt="{x:.0f}")
plt.ylabel('Department', fontsize='large')
ax.set_xlabel('ICD-9 chapters', fontsize='large')
ax.xaxis.set_label_position('top')
fig.tight_layout()
fig.savefig('heatmap_dep.png', dpi=250)
#%%
print('- Age Groups:')
idade = [int(line[2]) for line in texts]
def make_heatmap (request):
print 'make_heatmap'
if request.is_ajax()==True:
args={}
cmd=request.POST['cmd']
dataset=request.POST['dataset']
infodir=helpers.get_infodir(dataset)
# overzetten van waardes uit POST naar args-dict.
for key, val in request.POST.iteritems():
if key=='cmd' or key=='dataset':
continue
try:
v=float(val)
if v.is_integer():
args[key]=int(v)
else:
args[key]=v
except:
args[key]=val
print 'x x y:',args['imgheight'], args['imgwidth']
if cmd=='update':
col_info, coltypes_bycol=helpers.get_col_types(infodir)
colnames,groupcolnames=get_colnames_for_heatmap (infodir, args['displaymode'], col_info)
data={'msg':'','colnames':colnames,'groupcolnames':groupcolnames}
if cmd=='makemap':
msg='ok'
col_info, coltypes_bycol=helpers.get_col_types(infodir)
new_heatmap=args.get('add_new_heatmap',u'true')
if new_heatmap==u'true':
add_new_heatmap=True
else:
add_new_heatmap=False
print 'add new heatmap:', new_heatmap, add_new_heatmap
indexnr=args.get('heatmap_indexnr','0')
print 'INDEXNR',indexnr, type(indexnr)
args['infodir']=infodir
if add_new_heatmap:
print 'add new heatmap'
filename='%(infodir)s\heatmaps\%(x_var)s_%(y_var)s_meta.js' % args
print filename
heatmaps=glob.glob(filename)
print heatmaps
indexnr=len(heatmaps)
print 'indexnr:', indexnr
args['heatmap_indexnr']=indexnr
outfile='%(x_var)s_%(y_var)s_%(heatmap_indexnr)s' % args
args['outfile']=outfile
print coltypes_bycol.keys()[:10]
x_types=coltypes_bycol[args['x_var']] # info van variabelenaam x-kolom ophalen
y_types=coltypes_bycol[args['y_var']]
# args['x_min']=x_types.get(args['x_min'],args['x_min'])
# args['y_min']=y_types.get(args['y_min'],args['y_min'])
# args['x_max']=x_types.get(args['x_max'],args['x_max'])
# args['y_max']=y_types.get(args['y_max'],args['y_max'])
for key,value in args.items():
if value=='true':
value=True
args[key]=value
continue
if value=='false':
value=False
args[key]=value
continue
try:
value=ast.literal_eval(value)
except:
pass
args[key]=value
# kijk of we een nieuwe heatmap moeten uitrekenen
data={'msg':msg, 'heatmap_index':indexnr,
'x_var':args['x_var'],'y_var':args['y_var']}
do_recalc=True
h=heatmap.heatmap()
h.check_args(args)
print add_new_heatmap
if add_new_heatmap==False:
csvfile='%(infodir)s\heatmaps\%(x_var)s_%(y_var)s_%(heatmap_indexnr)s_meta' % args
old_args=h.load_options_from_csv(csvfile+'.csv')
do_not_recalc_args=['x_label', 'y_label',
'gradmin','gradmax','gradsteps',
'colormap','title', 'do_recalc',
'plot_mean', 'plot_mean_pixelsize', 'plot_mean_color',
'plot_median', 'plot_median_pixelsize', 'plot_median_color']
changed=False
do_recalc=False
print args['plot_mean'], old_args['plot_mean']
old_args['do_recalc']=False
new_args=old_args.copy()
for k,v in args.items():
new_args[k]=v
if old_args[k]!=args[k]:
new_args[k]=args[k]
print 'veranderd:%s, %s->%s' % ( k,old_args[k],args[k])
changed=True
if not(k in do_not_recalc_args):
do_recalc=True
if not changed:
return HttpResponse(cjson.encode(data))
if do_recalc==False:
optiejs=h.opties_to_js(new_args)
f=open(csvfile+'.js','w')
f.write ('var opties=[];\n');
f.write(optiejs)
f.close()
h.save_options_to_csv(new_args, csvfile+'.csv')
return HttpResponse(cjson.encode(data))
args['do_recalc']=do_recalc
h.make_heatmap(args)
return HttpResponse(cjson.encode(data))
# shouldn't get here
return HttpResponse('')
df = pd.read_csv("../data/Complete_Data_With_Targets.csv")
df = df.replace(
{'rel': {
0: 'Not-relevant',
1: 'Relevant',
2: 'Very-Relevant'
}})
df = df.replace(
{'imp': {
0: 'Not-important',
1: 'Important',
2: 'Very-Important'
}})
df['cnt'] = np.ones(len(df))
g = df.groupby(['rel', 'imp']).count()[['cnt']].replace(np.nan,
0).reset_index()
print(g)
plt.rcParams["figure.figsize"] = (5, 5)
heatmap(
x=g['rel'], # Column to use as horizontal dimension
y=g['imp'], # Column to use as vertical dimension
size_scale=7900, # Change this to see how it affects the plot
# x_order=['0', '1', '2'], # Sort order for x labels
# y_order=['Not important','Important','Very Important'], # Sort order for y labels
color=
g['cnt'], # Values to map to color, here we use number of items in each bucket
palette=sns.cubehelix_palette(125) # We'll use black->red palette
)
# input()
plt.show()
ax.plot(indices, values, label=bike_ids[i])
ax.legend(title="Bike IDs")
fig_name = fig_name + '-' + str(bike_ids[i])
if (count+1)%5 == 0:
ax.set_xlabel("Month")
ax.set_ylabel("Average duration (sec) per month")
ax.set_title("Average Duration of Rides Each Month")
fig_name += '.png'
fig.savefig(fig_name)
fig, ax = plt.subplots(figsize=(12,6))
fig_name = 'data/line-graphs/bike-ids'
count += 1
#b. Where are walkups most frequent
walkup_counts = df.groupby(['Passholder Type']).get_group('Walk-up')['Starting Station ID'].dropna().astype(np.int32).value_counts()
heatmap(df,walkup_counts,'data/heatmaps/walkup_start_heatmap_url','blue','Starting')
#c. popularity of each station versus month: 8 * 2 total heatmaps
df_grouped = df.groupby(['Time Elapsed'])
months = ["July '16","August '16","September '16","October '16","November '16","December '16",
"January '17","February '17","March '17"]
m = 0
for time, group in df_grouped:
start_ids_grouped = df_grouped.get_group(time).groupby(['Starting Station ID'])
month = months[m]
heatmap(df, group['Starting Station ID'].value_counts(), 'data/heatmaps/months/start/' + month[0:3] + month[-2:] + 'start_heatmap_url','blue','Starting')
heatmap(df, group['Ending Station ID'].value_counts(), 'data/heatmaps/months/end/' + month[0:3] + month[-2:] + 'end_heatmap_url','blue','Ending')
m += 1
for j in range(0, n, 1):
aux.append(0)
matriz.append(aux)
for i in range(0, len(valores), 1):
matriz[vetor1[i]][vetor2[i]] = rel.query('hours_per_week == "' +
str(vetor1[i]) +
'" & relationship == "' +
str(vetor2[i]) +
'"')['age'].astype(int).mean()
fig, ax = plt.subplots()
im, cbar = heatmap.heatmap(np.array(matriz),
hours_per_week.inverse_transform(np.arange(m)),
relationship.inverse_transform(np.arange(n)),
ax=ax,
cmap="YlGn",
cbarlabel="average age")
texts = heatmap.annotate_heatmap(im, valfmt="{x:.1f}")
ax.set_xlabel('relationship')
ax.set_ylabel('hours_per_week')
fig.tight_layout()
plt.show(
) #Quem trabalha mais horas em média: o marido, a esposa, pessoas com filhos, etc? Este comportamento varia dependendo da idade?
import pandas as pd
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn import preprocessing
Normalize.__init__(self, vmin, vmax, clip)
def __call__(self, value, clip=None):
# I'm ignoring masked values and all kinds of edge cases to make a
# simple example...
x, y = [self.vmin, self.midpoint, self.vmax], [0, 0.5, 1]
return np.ma.masked_array(np.interp(value, x, y))
# create a matplotlib figure for the Q table
fig = plt.figure('Q-table')
ax = fig.add_subplot(111)
im, cbar = heatmap(q_table[0],
list(range(numDistSensorStates)),
action_names,
ax=ax,
cmap="RdBu",
cbarlabel="Q table",
norm=DivergingNorm(vcenter=0.0))
#norm=MidpointNormalize(midpoint=0))
texts = annotate_heatmap(im, valfmt="{x:.1f}")
plt.draw()
#fig, ax = plt.subplots()
total_reward = 0
start_time = datetime.datetime.now() # log start of episode
rewards = np.zeros(200)
for step in range(0, steps - 1):
# Use epsilon greedy policy based on Q table
if N0 == 0:
epsilon = 0
# Conteggio pedoni e conversione degli indici numerici degli incroci in testo
pedoni_feriti = istat_utils.get_people(incidenti_pedoni, fields)
incidenti_pedoni = pd.DataFrame(
[incidenti_pedoni['intersezione_o_non_interse3'], pedoni_feriti],
['tipo_incrocio', 'pedoni_feriti']).transpose()
incidenti_pedoni = incidenti_pedoni[incidenti_pedoni['pedoni_feriti'] != 0]
incidenti_labels = label_utils.join_labels(
incidenti_pedoni['tipo_incrocio'],
'dataset/incidenti/istat/Classificazioni/intersezione_o_non_interse3.csv')
incidenti_pedoni = pd.DataFrame(
[incidenti_labels, incidenti_pedoni['pedoni_feriti']],
['tipo_incrocio', 'pedoni_feriti']).transpose()
pedoni_contati = pd.DataFrame(
pd.crosstab(incidenti_pedoni['tipo_incrocio'],
incidenti_pedoni['pedoni_feriti'])).transpose()
pedoni_contati.index = pedoni_contati.index.astype(int)
fig, ax = plt.subplots()
im = H.heatmap(pedoni_contati,
pedoni_contati.index,
pedoni_contati.columns,
ax=ax,
xticks_rotated=True,
cmap="YlGn",
cbar_visible=False)
texts = H.annotate_heatmap(im, valfmt="{x}")
plt.ylabel("Numero di pedoni coinvolti")
fig.tight_layout()
plt.show()
es_score_n = []
#others_n = []
for N in test_sizes:
select = df.loc[(df["P"] == P) & (df["N"] == N)]
if selected_dataset:
select = select.loc[select["Dataset"] == datasets[selected_dataset-1]]
if P in test_sizes[2:] and N in test_sizes[2:]:
overall = select.loc[:, "ROC(auc)"].mean()
else:
overall = select.loc[:, "Balanced accuracy"].mean()
es_score_n.append(overall)
score_dict["es_score"].append(es_score_n)
test_sizes_p = [x+"P" for x in test_sizes]
test_sizes_n = [x+"N" for x in test_sizes]
ext = "hepmark_es_svm_"+str(selected_dataset-1)+".pdf" if selected_dataset else "hepmark_es_svm.pdf"
# Uncomment to Latexify the heatmap
#latexify(columns=2)
# Generate a Heatmap
scores = np.array(score_dict["es_score"])
fig, ax = plt.subplots()
im, cbar = heatmap.heatmap(scores, test_sizes_p, test_sizes_n, ax=ax,
vmin = 0.0, vmax = 1.0, cmap=cm.Reds, cbarlabel="score [AUC / Acc]")
texts = heatmap.annotate_heatmap(im, valfmt="{x:.2f}")
fig.tight_layout()
plt.show()
fig.savefig("C:/Users/Vegard/Desktop/Master/Mastersproject/Plots/analyze/"+ext)
def to_heatmap(a):
a = (255 * a).astype(np.uint8)
return 255 * heatmap(a).astype(np.float32)
if SAMPLE == "B1":
temp_gene[159] = "Homsap IGHJ3"
temp_gene[396] = "Homsap IGHJ4"
for ge in range(len(temp_gene)):
temp_gene[ge] = str(
temp_gene[ge]).split("*")[0].split(" ")[1]
genes_data[gene] = temp_gene
all_genes = natsorted(genes_data[gene].unique())
pathToSave = OUTPUT_FOLDER_HEATMAP + '/' + "heatmap_" + threshold_name + "_" + str(
gene.split("-")[0]) + '_.jpg'
heatmap(gene, genes_data, labels_fastgreedy, all_genes,
pathToSave)
# step 1 : map all genes to one number
# step 2 : produce the labels array according to the dictionary d
# step 3 : make it a function
# - input : genes_column , all_genes ( not necessary - maybe with unique )
# - output : returns the array with the labels_genes
# step 4 : plot the graph with these labels
# ------ plot the same graph but now labels are from genes ------
d = dict([(y, x + 1)
for x, y in enumerate(sorted(set(all_genes)))])
labels_genes = np.array([], dtype=int)
for my_g in genes_data[gene]:
def analyze_confused5 (result_dic_path=args.json_result_dict):
#############################
# Prediction Result
#############################
result = read_result_dict(result_dic_path, pickle_file=True)
# Numpy array
gt = result['video_label'] # (b, )
pred = result['prediction'] # (b, 5) top 5 prediction
unique, counts = np.unique(gt, return_counts=True)
label_count = dict(zip(unique, counts))
#############################
# Prediction Result
#############################
"""
pred_err_rows: wrong prediction (i.e., corrent label does not equal to prediction)
confused_rows: confused prediction (i.e., correct label is in top 5 prediction)
"""
gt = np.expand_dims(gt, -1)
gt = np.repeat(gt, 5, 1)
# Err Rows
pred_err_rows = (gt[:, 0] != pred[:, 0]).sum()
# Confused Rows
confused_rows = (gt[:, 1:] == pred[:, 1:]).sum(axis=1).astype(bool)
confused_pred = pred[confused_rows, 0]
confused_gt = gt[confused_rows, 0]
confusion_dict = list(zip(confused_gt, confused_pred))
confusion_dict = collections.Counter(confusion_dict)
print('Total prediction err: {0}\t'
'Total confusion err: {1}\t'
.format(pred_err_rows, confused_rows.sum()))
# Establish heatmap matrix to plot
confusion_matrix = np.zeros((args.num_classes, args.num_classes))
for gt in range(args.num_classes):
num_confused_per_gt = sum([val for key, val in confusion_dict.items() if key[0]==gt])
for pred in range(args.num_classes):
if (gt,pred) in confusion_dict:
confusion_matrix[gt, pred] = confusion_dict[(gt,pred)] / num_confused_per_gt * 100
with open(args.json_file_labels, 'rb') as fr:
catg_dict = json.load(fr)
category_labels = [l[0] for l in sorted(catg_dict.items(), key=lambda item: item[1])]
# Plot
grid = 3
fig, ax = plt.subplots()
fig.set_size_inches(18.5, 10.5)
for i in range(grid):
for j in range(grid):
section_size = int(args.num_classes/grid)
x_range = [i * section_size, (i+1) * section_size]
y_range = [j * section_size, (j+1) * section_size]
grid_matrix = confusion_matrix[y_range[0] : y_range[1], x_range[0] : x_range[1]]
x_category = category_labels[x_range[0]: x_range[1]]
y_category = category_labels[y_range[0]: y_range[1]]
im, cbar = plt_hmp.heatmap(grid_matrix, y_category, x_category, ax=ax,
cmap="YlGn",
vmin=0, vmax=100,
cbarlabel="Top 5 Confusion Percentage Per Category")
fig.tight_layout()
plt.savefig(f'heatmap_{i}_{j}.png')
cbar.remove()
strade_extraurbane = data[
(data['localizzazione_incidente'] == 4) |
(data['localizzazione_incidente'] == 5) |
(data['localizzazione_incidente'] == 6)]['tipo_veicolo_a']
strade_extraurbane = label_utils.join_labels(
strade_extraurbane,
"dataset/incidenti/istat/Classificazioni/tipo_veicoli__b_.csv"
).value_counts(normalize=True).sort_index()
# Rimozione delle auto private
autostrade_veicoli = autostrade_veicoli[
autostrade_veicoli.index != 'Auto privata']
strade_urbane = strade_urbane[strade_urbane.index != 'Auto privata']
strade_extraurbane = strade_extraurbane[
strade_extraurbane.index != 'Auto privata']
df = pd.DataFrame([autostrade_veicoli, strade_urbane, strade_extraurbane],
index=['Autostrade', 'Strade urbane', 'Strade Extra-Urbane'
]).transpose().dropna().transpose()
fig, ax = plt.subplots()
im, cbar = H.heatmap(df,
df.index,
df.columns,
ax=ax,
cmap="YlGn",
cbarlabel="Percentuale del tipo di veicolo (2018)",
xticks_rotated=True)
texts = H.annotate_heatmap(im, valfmt="")
fig.tight_layout()
plt.show()
def make_subsel(request, dataset):
post=request.POST
print post
xvar=post['xvar']
yvar=post['yvar']
indexnr=int(post['heatmap_index'])
num_annotaties=int(post['num_annotaties'])
annotaties={}
for i in range(0,num_annotaties):
a={}
a['selectie_id']=post['selectie_id_%d' % i]
a['xvar']=xvar
a['xmin']=float(post['xmin_%d' % i])
a['xmax']=float(post['xmax_%d' % i])
a['yvar']=yvar
a['ymin']=float(post['ymin_%d' % i])
a['ymax']=float(post['ymax_%d' % i])
a['areatype']=post['areatype_%d' % i]
a['indexnr']=int(post['heatmap_index'])
a['connector_direction']=post['connector_direction_%d' % i]
a['text_xpos']=float(post['text_xpos_%d' % i])
a['text_ypos']=float(post['text_ypos_%d' % i])
a['filename']=post['filename_%d' % i]
a['text']=post['text_%d' % i]
a['label']=post['label_%d' % i]
annotaties[str(i)]=a
#print xvar,xmin,xmax
#print yvar,ymin,ymax
infodir=settings.datadir+'/'+dataset+'_info'
print 'get_col_types'
col_info, coltypes_bycol=helpers.get_col_types(infodir)
selectiedir='%(infodir)s/selections/%(xvar)s_%(yvar)s_%(indexnr)s' % locals()
print 'prepare_subsel'
# duurt lang , aparte optie voor maken om onderscheid tussen dataselectie en ui-selectie te maken
# subsel=makehist.prepare_subsel (infodir, coltypes_bycol, xvar,xmin,xmax, yvar,ymin,ymax)
subsel=[]
print 'save_subsel'
save_subsel (selectiedir, subsel, annotaties)
# bijwerken heatmap-javascript
print 'bijwerken js'
infile='%(xvar)s_%(yvar)s_%(indexnr)s'% locals()
csvfile='%(infodir)s/heatmaps/%(infile)s_meta.csv' % (locals())
h=heatmap.heatmap()
h.infodir=infodir
args=h.load_options_from_csv(csvfile)
print args
args['annotate']=annotaties
for k,v in args.items():
setattr(h,k,v)
newjs=h.opties_to_js(args)
f=open ('%s/heatmaps/%s_meta.js' % (infodir, infile), 'w')
newjs='var opties=[];\n'+newjs
f.write(newjs)
f.close()
msg='ok'
data={'msg':msg,'annotaties':annotaties}
return HttpResponse(cjson.encode(data))
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
from heatmap import heatmap, corrplot
data = pd.read_csv('met-data.csv')
corr = data.corr()
corr = pd.melt(
corr.reset_index(), id_vars='index'
) # Unpivot the dataframe, so we can get pair of arrays for x and y
corr.columns = ['x', 'y', 'value']
heatmap(x=corr['x'], y=corr['y'], size=corr['value'].abs())
plt.figure(figsize=(10, 10))
corrplot(data.corr())
plt.show()
# Selezione dati per Adriatica (SS1), Aurelia (SS16),
# A1 (del Sole), Torino-Trieste (A4) e Raccordo anulare di Roma
adriatica = data[data['CODICE'] == 'SS01601'][mesi]
a1 = data[data['CODICE'] == 'AA00101'][mesi]
aurelia = data[data['CODICE'] == 'SS00101'][mesi]
a4 = data[data['CODICE'] == 'AA00401'][mesi]
a90 = data[data['CODICE'] == 'AA09001'][mesi]
adriatica = aci_utils.sum_columns(adriatica)
a1 = aci_utils.sum_columns(a1)
aurelia = aci_utils.sum_columns(aurelia)
a4 = aci_utils.sum_columns(a4)
a90 = aci_utils.sum_columns(a90)
df = pd.DataFrame([a1, adriatica, aurelia, a4, a90], [
'A1 Milano-Roma-Napoli', 'SS16 Adriatica', 'SS1 Aurelia',
'A4 Torino Trieste', 'A90 Raccordo Anulare'
])
fig, ax = plt.subplots()
im = H.heatmap(df,
df.index,
df.columns,
ax=ax,
cmap="OrRd",
xticks_rotated=True,
cbar_visible=False)
texts = H.annotate_heatmap(im, valfmt="{x}")
fig.tight_layout()
plt.show()
import heatmap
import annotation
import volcano_plot
import scatter_and_violin_plots
# The outputs are already in the output_file. Running this client file will
# give you the same thing as the outputs in that file
# Will output a heatmap
print(heatmap.heatmap())
# Will output a new csv file: results_L-vs-NL_full_annotated
# It will show up within the Final Project folder
# This will take some time as it needs to query
print(annotation.annotation("results_L-vs-NL_full.csv"))
# Will output a volcano plot
print(volcano_plot.volcano_plot())
# Will output a scatter plot
print(scatter_and_violin_plots.scatterplot())
# Will output a violin plot
print(scatter_and_violin_plots.violin_plot())
import os
import sys
path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../Analysis/Soms/'))
if not path in sys.path:
sys.path.insert(1, path)
del path
import heatmap as htmap
site = sys.argv[1]
tdate = sys.argv[2]
days = sys.argv[3]
#site = 'laysa'
#tdate = '2016-03-03'
#days = '1d'
data = htmap.heatmap(site, tdate, t_win=days)
data = pd.read_csv(path + str(year) + ".txt",
sep="\t",
error_bad_lines=False,
engine='python')
else:
data = pd.read_csv(path + str(year) + ".txt",
sep="\t",
encoding="latin1")
natura_incidente = data['natura_incidente']
natura_incidente_labels = label_utils.join_labels(
natura_incidente,
"dataset/incidenti/istat/Classificazioni/natura_incidente.csv"
).value_counts(normalize=True)
natura_incidente_labels = natura_incidente_labels[tipo_incidenti]
inc_per_anno[str(year)] = pd.Series(natura_incidente_labels)
fig, ax = plt.subplots()
im = H.heatmap(inc_per_anno,
inc_per_anno.index,
inc_per_anno.columns,
ax=ax,
cmap="OrRd",
cbar_visible=False)
texts = H.annotate_heatmap(im)
fig.tight_layout()
plt.show()