def plot_chain_hbond(iso, ratio, fig, subplot_num):
ax=fig.add_subplot(3, 4, subplot_num)
fig.subplots_adjust(bottom=0.05,top=0.95,left=0.05,right=0.95,wspace=0.4, hspace=0.4)
# pylab.rcParams['xtick.labelsize']='10'
# pylab.rcParams['ytick.labelsize']='12'
pylab.rcParams['legend.fontsize']='6'
# pylab.rcParams['figure.figsize'] = [2.5,2.5]
# pylab.rcParams["axes.titlesize"]='small'
A=utils.smooth(numpy.genfromtxt('%(iso)s_sys%(subplot_num)d_chain_0_1_hbonds.dat' % vars(), comments="#"), 1000, timestep=2)
B=utils.smooth(numpy.genfromtxt('%(iso)s_sys%(subplot_num)d_chain_1_2_hbonds.dat' % vars(), comments="#"), 1000, timestep=2)
C=utils.smooth(numpy.genfromtxt('%(iso)s_sys%(subplot_num)d_chain_2_3_hbonds.dat' % vars(), comments="#"), 1000, timestep=2)
D=utils.smooth(numpy.genfromtxt('%(iso)s_sys%(subplot_num)d_chain_3_4_hbonds.dat' % vars(), comments="#"), 1000, timestep=2)
if A == []:
print "WARNING: data file empty ... quitting ..."
return
output_data = numpy.hstack((A, B, C, D))
numpy.savetxt('%(iso)s_sys%(subplot_num)d_chain_hbonds_1000.dat' % vars(), output_data)
time = A[:,0]/1000.0
ax.plot(time, A[:,1], label="chain 1-2")
ax.plot(time, B[:,1], label="chain 2-3")
ax.plot(time, C[:,1], label="chain 3-4")
ax.plot(time, D[:,1], label="chain 4-5")
pylab.ylim(0, 30)
pylab.xlim(0, 200)
pylab.grid(True)
# xlabel('Time (ns)')
# ylabel('Number of interchain hydrogen bonds')
pylab.legend(loc='lower right')
pylab.savefig('test_%(iso)s_%(ratio)s_chain_hbond.png' % vars())
def handle_data2():
import time
import re
from utils import smooth, interpl, chunk_decode
len_e = 3000
register = 0x0B
last_ts = None
plt.ion()
ax = plt.gca()
result_ts = time.time()
while True:
response, timestamp = q2.get()
if not response:
return
# Fix raw value
val = int.from_bytes(response, 'big')
val_fixed = val
raw_frames_m.push(np.array([
[timestamp, val_fixed],
]))
if last_ts is None:
last_ts = raw_frames_m.window[0][0]
if raw_frames_m.window[-1][0] - last_ts < 0.6:
continue
M = raw_frames_m.window
M = M[np.logical_and(M[:, 0] > last_ts - 0.5, M[:, 0] < last_ts + 0.5)]
Mtime = M[:, 0]
value = M[:, 1]
last_ts = Mtime[-1]
sample_time = np.arange(Mtime[0], Mtime[-1], 1 / 2400)
sample_time = sample_time[sample_time < Mtime[-1]]
sample_value = interpl(Mtime, value, sample_time, 'nearest')
sample_value_smooth = smooth(sample_value, 41)
sample_value_DCremove = smooth(sample_value - sample_value_smooth, 5)
value = np.zeros((10, 1))
for i in range(10):
temp_sample = sample_value_DCremove[i:len_e:10]
value[i] = max(temp_sample) + min(temp_sample)
std_min = max(value)
shift_index = np.where(value == std_min)[0][0]
sample_wave = sample_value_DCremove[shift_index:len_e:10]
temp_sample = sample_value_DCremove[shift_index:len_e:10]
# bit_stream = sample_wave <= (max(temp_sample) + min(temp_sample)) / 2
bit_stream = sample_wave <= np.mean(temp_sample)
bit_stream = bit_stream.astype(int)
result = chunk_decode(bit_stream)
if result is None:
result = chunk_decode(bit_stream, flip=True)
if result is not None:
result_ts = time.time()
for i in result:
print(i)
def __getitem__(self, index):
paths = self.data[index]
rt_path = os.path.join(self.opt.dataroot, self.video_bag, paths[0],
paths[1], paths[2] + "_aligned_rt.npy")
audio_path = os.path.join(
self.opt.dataroot, 'unzip/dev_audio' if self.mode == 'train' else
'unzip/test_audio', paths[0], paths[1], paths[2]) + '.wav'
if not os.path.exists(audio_path):
return self.__getitem__(random.choice(range(len(self.data))))
ani_id = paths[3]
rt = np.load(rt_path)
fs, audio = self.parse_audio(audio_path, rt.shape[0])
rt_mean = np.array([0.07, -0.008, -0.016, 11.4, 5.69, -4.37])
rt_std = np.array([0.066, 0.132, 0.071, 9.99, 9.24, 16.95])
rt = (rt - rt_mean) / rt_std
for i in range(rt.shape[1]):
rt[:, i] = utils.smooth(rt[:, i])
for i in range(audio.shape[1]):
for j in range(audio.shape[2]):
audio[:, i, j] = utils.smooth(audio[:, i, j], 21)
# audio = audio[:,3,:].mean(-1)
# audio = utils.smooth(audio, 21)
# pdb.set_trace()
# plt.plot(range(rt.shape[0]), rt[:,0], label='rt_0')
# plt.plot(range(audio.shape[0]), audio, label='audio_s')
# plt.legend()
# plt.savefig('/u/zkou2/Code/Trans/vs.png')
# plt.close()
return torch.from_numpy(rt), torch.from_numpy(audio)
def updateDatasets(self, fields, helper):
""" This function should *update* the dataset(s) returned by getDatasets
"""
# get the input dataset - helper provides methods for getting other
# datasets from Veusz
ds_y = np.array(helper.getDataset(fields['ds_y']).data)
# get the value to add
order = fields['order']
method = fields['method']
smooth_num = fields['smooth']
ds_x = None
if smooth_num > 0:
ds_y = smooth(ds_y, smooth_num, 'hanning')
if fields['ds_x'] != '':
# Derive with respect to X
ds_x = np.array(helper.getDataset(fields['ds_x']).data)
if smooth_num > 0:
ds_x = smooth(ds_x, smooth_num, 'hanning')
out = xyderive(ds_x, ds_y, order, method)
else:
# Derive with respect to point indexes
out = derive(ds_y, method, order)
self.ds_out.update(data=out)
return [self.ds_out]
def makeSegmentation(d,
s,
nbins=256,
verbose=True,
sigma=0.75,
min_distance=1):
fn = os.path.join(datadir, 'data_stage%d_%s.npy' % (s, features[0]))
dists = np.load(fn)
fn = os.path.join(datadir, 'data_stage%d_%s.npy' % (s, features[1]))
rots = np.load(fn)
ddata = np.vstack([np.log(dists[:, d]), (rots[:, d])]).T
#gmmdata = np.vstack([dists[:,j], (rots[:,j])]).T
#ddata.shape
nanids = np.logical_or(np.any(np.isnan(ddata), axis=1),
np.any(np.isinf(ddata), axis=1))
ddata = ddata[~nanids, :]
#ddata.shape
imgbin = None
img2 = smooth(ddata, nbins=[nbins, nbins], sigma=(sigma, sigma))
#img = smooth(ddata, nbins = [nbins, nbins], sigma = (1,1))
local_maxi = peak_local_max(img2, indices=False, min_distance=min_distance)
imgm2 = img2.copy()
imgm2[local_maxi] = 3 * imgm2.max()
if verbose:
imgbin = smooth(ddata, nbins=[nbins, nbins], sigma=None)
plt.figure(220)
plt.clf()
plt.subplot(2, 2, 1)
plt.imshow(imgbin)
plt.subplot(2, 2, 2)
plt.imshow(img2)
plt.subplot(2, 2, 3)
plt.imshow(imgm2, cmap=plt.cm.jet, interpolation='nearest')
markers = ndi.label(local_maxi)[0]
labels = watershed(-img2, markers, mask=None)
print "max labels: %d" % labels.max()
if verbose:
fig, axes = plt.subplots(ncols=3,
sharex=True,
sharey=True,
subplot_kw={'adjustable': 'box-forced'})
ax0, ax1, ax2 = axes
ax0.imshow(img2, cmap=plt.cm.jet, interpolation='nearest')
ax0.set_title('PDF')
labels[imgbin == 0] = 0
labels[0, 0] = -1
ax1.imshow(labels, cmap=plt.cm.rainbow, interpolation='nearest')
ax1.set_title('Segmentation on Data')
#labelsws[0,0] = -1;
#ax2.imshow(labelsws, cmap=plt.cm.rainbow, interpolation='nearest')
#ax1.set_title('Segmentation Full')
return labels
def plot_stats_single(statistics,
ylog=False,
view=False,
filename='avg_fitness.svg',
momentum=0.99):
""" Plots the population's average and best fitness. """
if plt is None:
warnings.warn(
"This display is not available due to a missing optional dependency (matplotlib)"
)
return
generation = range(len(statistics.best_fitnesses))
best_fitness = [c[1] for c in statistics.best_fitnesses]
avg_fitness = np.array(statistics.get_fitness_mean())
stdev_fitness = np.array(statistics.get_fitness_stdev())
print(avg_fitness.shape)
print(stdev_fitness.shape)
print(len(best_fitness))
plt.plot(generation,
smooth(avg_fitness, momentum=momentum),
'b-',
label="average")
plt.plot(generation,
smooth(best_fitness, momentum=momentum),
'r-',
label="best")
plt.plot(generation,
smooth(avg_fitness - stdev_fitness, momentum=momentum),
'g-.',
label="-1 sd")
plt.plot(generation,
smooth(avg_fitness + stdev_fitness, momentum=momentum),
'g-.',
label="+1 sd")
plt.title("Population's average and best fitness")
plt.xlabel("Generations")
plt.ylabel("Fitness")
plt.grid()
plt.legend(loc="best")
if ylog:
plt.gca().set_yscale('symlog')
plt.savefig(filename)
if view:
plt.show()
plt.close()
def process(h5file, ratio):
isomerlist = ["scyllo", "chiro", "water"]
plot_data = []
mean_contact_list = []
std_contact_list = []
#read in files for each system and aggregate
format="pp_nonpolar_vs_t.xvg"
for iso in isomerlist:
print "processing", iso
pattern = re.compile(r"%(iso)s.*%(ratio)s.*%(format)s" % vars())
if iso == "water":
pattern = re.compile(r"%(iso)s.*%(format)s" % vars())
datalist=[]
for table in h5file.listNodes(where='/pp_nonpolar'):
table_path = os.path.join('/pp_nonpolar', table.name)
if pattern.search(table.name):
data = myh5.getTableAsMatrix(h5file, table_path)
if data is not None:
data = data.astype('float')
datalist.append(data[0:config.LASTFRAME, 1])
else:
print "no data was read in"
print "datalist", datalist
data_matrix = numpy.transpose(numpy.vstack(datalist))
print "data_matrix", data_matrix, data_matrix.shape
avg, std = utils.summary_statistics(data_matrix, sum_across="columns")
avg_contacts = numpy.average(data_matrix[config.STARTFRAME:config.LASTFRAME], axis=0)
mean_contact = numpy.average(avg_contacts)
std_contact = numpy.std(avg_contacts)
print mean_contact
print std_contact
mean_contact_list.append(mean_contact)
std_contact_list.append(std_contact)
avg_smoothed = utils.smooth(avg/config.NMOLECULES, 500, time_present=False, timestep=2)
std_smoothed = utils.smooth(std/config.NMOLECULES, 500, time_present=True, timestep=2)
plot_data.append(avg_smoothed)
plot_data.append(std_smoothed)
timeseries_matrix = numpy.hstack(plot_data)
print "timeseries_matrix", timeseries_matrix, timeseries_matrix.shape
print "time", timeseries_matrix[:,0]
numpy.savetxt(ratio + "_pp_nonpolar_smoothed.txt.gz", timeseries_matrix, fmt='%0.3f')
utils.savetxt(ratio + "_avg_pp_nonpolar_contact.txt", "#scyllo chiro water", numpy.vstack([mean_contact_list, std_contact_list]), fmt='%0.3f')
return timeseries_matrix
def plot_both_games_curves(
num_epochs=5000,
window_size=200,
log_path='./log/',
):
recon_matrix = np.asarray(
read_topo_sim_dir(log_path + 'recon_topsim/'))[:, :num_epochs]
refer_matrix = np.asarray(
read_topo_sim_dir(log_path + 'refer_topsim/'))[:, :num_epochs]
recon_mean, recon_upper, recon_lower = get_plot_components(recon_matrix)
refer_mean, refer_upper, refer_lower = get_plot_components(refer_matrix)
print('topo-sim:', ttest_ind(recon_mean[-29:], refer_mean[-29:]))
recon_mean, recon_upper, recon_lower = tuple([
smooth(x, window_size)[:num_epochs]
for x in [recon_mean, recon_upper, recon_lower]
])
refer_mean, refer_upper, refer_lower = tuple([
smooth(x, window_size)[:num_epochs]
for x in [refer_mean, refer_upper, refer_lower]
])
# Start plotting the two lines with variance
x_axis = np.arange(recon_mean.size) + 1
plt.plot(x_axis,
recon_mean,
label='reconstruction language',
linewidth=0.5)
plt.fill_between(x_axis, recon_upper, recon_lower, color='blue', alpha=0.2)
plt.plot(x_axis, refer_mean, label='referential language', linewidth=0.5)
plt.fill_between(x_axis,
refer_upper,
refer_lower,
color='orange',
alpha=0.2)
plt.xlabel('Epochs')
plt.ylabel('Topological Similarity')
plt.legend()
plt.grid()
plt.ylim([0.28, 0.36])
fig_file = './result/compare_topo_sim.pdf'
create_dir_for_file(fig_file)
plt.savefig(fig_file, format='pdf', bbox_inches='tight')
def makeSegmentation(d, s, nbins = 256, verbose = True, sigma = 0.75, min_distance = 1):
fn = os.path.join(datadir, 'data_stage%d_%s.npy' % (s,features[0]));
dists = np.load(fn);
fn = os.path.join(datadir, 'data_stage%d_%s.npy' % (s,features[1]));
rots = np.load(fn);
ddata = np.vstack([np.log(dists[:,d]), (rots[:,d])]).T
#gmmdata = np.vstack([dists[:,j], (rots[:,j])]).T
#ddata.shape
nanids = np.logical_or(np.any(np.isnan(ddata), axis=1), np.any(np.isinf(ddata), axis=1));
ddata = ddata[~nanids,:];
#ddata.shape
imgbin = None;
img2 = smooth(ddata, nbins = [nbins, nbins], sigma = (sigma,sigma))
#img = smooth(ddata, nbins = [nbins, nbins], sigma = (1,1))
local_maxi = peak_local_max(img2, indices=False, min_distance = min_distance)
imgm2 = img2.copy();
imgm2[local_maxi] = 3 * imgm2.max();
if verbose:
imgbin = smooth(ddata, nbins = [nbins, nbins], sigma = None)
plt.figure(220); plt.clf()
plt.subplot(2,2,1)
plt.imshow(imgbin)
plt.subplot(2,2,2)
plt.imshow(img2)
plt.subplot(2,2,3);
plt.imshow(imgm2, cmap=plt.cm.jet, interpolation='nearest')
markers = ndi.label(local_maxi)[0]
labels = watershed(-img2, markers, mask = None);
print "max labels: %d" % labels.max()
if verbose:
fig, axes = plt.subplots(ncols=3, sharex=True, sharey=True, subplot_kw={'adjustable':'box-forced'})
ax0, ax1, ax2 = axes
ax0.imshow(img2, cmap=plt.cm.jet, interpolation='nearest')
ax0.set_title('PDF')
labels[imgbin==0] = 0;
labels[0,0] = -1;
ax1.imshow(labels, cmap=plt.cm.rainbow, interpolation='nearest')
ax1.set_title('Segmentation on Data')
#labelsws[0,0] = -1;
#ax2.imshow(labelsws, cmap=plt.cm.rainbow, interpolation='nearest')
#ax1.set_title('Segmentation Full')
return labels;
def initialize(self, sp):
sp.dt = 300#120#240#18#45#15
sp.nsub = 10
sp.dx = 4000
sp.KBi = 2e8#1e9
self.E_floating = False
self.U_floating = True
self.V_floating = True
self.os = sp.getOcean(220, 200, 5)
self.os.dz[:] = np.array([10, 10, 20, 20, 40])
self.os.depth[:] = 100
## Shallow area:
self.os.depth[int(0.4*self.os.imax):int(0.6*self.os.imax), \
int(0.4*self.os.jmax):int(0.6*self.os.jmax)] = self.os.depth[0,0]*0.45
self.os.depth[...] = utils.smooth(self.os.depth, D=0.1, repeats=5)
#self.os.depth[:] = netcdfStorage.getDepthMatrix('../matlab/openArea.nc')
#self.os.depth[:] = netcdfStorage.getDepthMatrix('../matlab/test.nc')
self.os.calcKmmDzz()
tval = [12, 11, 10, 9, 8]
sval = [25, 27, 28, 29, 30]
for i in range(0,self.os.imax):
for j in range(0,self.os.jmax):
#self.os.E[i,j] = 0.2 - 0.4*i/self.os.imax
for k in range(0,self.os.kmm[i,j]):
if i<self.os.imax-1:
self.os.U[i,j,k] = -0*0.05
self.os.T[i,j,k] = tval[k]
self.os.S[i,j,k] = sval[k] #+ 3.5*(i/self.os.imax)*math.exp(-(self.os.midLayerDepths[k]-5)/10)
self.initBounds(self.os, self.os.imax, self.os.jmax, self.os.kmax, tval, sval)
def get_spectrogram(y, padding=0):
"""Extract spectrogram from audio data.
This implementation is identical to what's described in the original paper.
Parameters
----------
y: np.ndarray, the input singal(audio time series).
padding(optional): int, pad the input signal.
Returns
-------
D: np.ndarray, STFT matrix.
"""
y = rosa.stft(y, n_fft=const_args['n_fft'],
hop_length=const_args['hop_length'],
window=const_args['window'])
y = np.abs(y)
y = np.flipud(y)
y = np.log(1e-5+y) - np.log(1e-5)
y = y - np.mean(y)
y = y / np.sqrt(np.mean(np.power(y, 2)))
r, c = y.shape
for c_ in range(c):
y[:, c_] = utils.smooth(y[:, c_], 19)
y = np.concatenate((np.zeros((r, padding)), y, np.zeros((r, padding))), axis=1)
return y
def computeRCurv(self, XY):
vx = (sigut.smooth(XY[:, 0], XY[:, 1], 5, 2, 1))
# vx = abs(dxy[:,0])
vy = (sigut.smooth(XY[:, 0], XY[:, 2], 5, 2, 1))
# vy = abs(dxy[:,1])
ax = sigut.smooth(XY[:, 0], vx, 5, 2, 1)
ay = sigut.smooth(XY[:, 0], vy, 5, 2, 1)
curv = np.power(vx * vx + vy * vy, 1.5) / abs(vx * ay - vy * ax)
# curv = sigut.smooth(XY[:,0], curv, 7, 5, 0)
curv_int = np.log(curv)
curv_int = curv_int.astype(int)
curv_int[curv_int < 0] = 0
curv_int[curv_int > CURV_TRONCON[-1]] = CURV_TRONCON[-1]
return curv, curv_int
def geneggfilter(data):
data = data / np.max(np.abs(data))
# data[data > 0] = 0
# data = medfilt(data, 9)
# data = savgol_filter(data, 21, 2)
data = smooth(data, 49)
return data
def task22(method = 'point_feature_matching'):
video_path = 'travi.mp4'
vCapture = cv2.VideoCapture(video_path)
n_frames = int(vCapture.get(cv2.CAP_PROP_FRAME_COUNT))
w = int(vCapture.get(cv2.CAP_PROP_FRAME_WIDTH))
h = int(vCapture.get(cv2.CAP_PROP_FRAME_HEIGHT))
if method == 'point_feature_matching':
start_time = time.time()
rad = 50 # define radius
# Step 1
# Define the codec for output video
fourcc = cv2.VideoWriter_fourcc(*'MJPG')
# Set up output video
fps = 30
outVideo = cv2.VideoWriter('video_out.mp4', fourcc, fps, (w, h))
#Step 2
_,prev= vCapture.read()
prev_gray = cv2.cvtColor(prev,cv2.COLOR_BGR2GRAY)
# Step 3
transformations = np.zeros(((n_frames-1),3))
for i in trange(n_frames-2):
dx,dy,a,current_gray = opticalFlow.extractRotation_translation(prev_gray,vCapture)
transformations[i] = [dx,dy,a]
# Update frame
prev_gray = current_gray
# Calc smooth motion btw frames
motion = np.cumsum(transformations,axis=0)
smooth_motion = utils.smooth(motion,rad)
diff = smooth_motion - motion
transformations_smooth = transformations + diff
# Apply smoothed camera motion to frames
vCapture.set(cv2.CAP_PROP_POS_FRAMES, 0)
for i in range(n_frames-2):
available,f = vCapture.read()
if not available:
break
stabilised_f,frame_out = opticalFlow.stabilise_frame(transformations_smooth, i, f, w, h)
if True:
if (frame_out.shape[1] >= 1920):
frame_out = cv2.resize(frame_out, (frame_out.shape[1] / 2, frame_out.shape[0] / 2))
cv2.imshow('BEFORE & AFTER',frame_out)
cv2.waitKey(10)
outVideo.write(stabilised_f)
end_time = time.time()
print('Duration:',end_time-start_time)
elif method == 'mesh':
start_time = time.time()
x_motion_meshes, y_motion_meshes, x_paths, y_paths = Stabilization.read_video(vCapture)
sx_paths, sy_paths = Stabilization.stabilize(x_paths, y_paths)
x_motion_meshes, y_motion_meshes, new_x_motion_meshes, new_y_motion_meshes = Stabilization.get_frame_warp(x_motion_meshes,y_motion_meshes,x_paths, y_paths,sx_paths, sy_paths)
Stabilization.generate_stabilized_video(vCapture, x_motion_meshes, y_motion_meshes, new_x_motion_meshes, new_y_motion_meshes)
end_time = time.time()
print('Duration:',end_time-start_time)
def isCopa(time, throughput, RTT):
RTT += 0.01
samples_per_RTT = int(len(time) / (time[-1] - time[0])) * RTT
throughput = smooth(throughput, int(samples_per_RTT))[5:-5]
time = time[5:-5]
peaks, properties = find_peaks(throughput,
distance=3 * samples_per_RTT,
width=samples_per_RTT,
prominence=0.1)
diffs = np.diff(time[peaks])
# # print(np.var(diffs))
# plt.plot(time, throughput)
# plt.plot(time[peaks], throughput[peaks], "x")
# plt.show()
if (len(diffs) == 0 or np.var(diffs) > 0.085):
# print(np.var(diffs))
# plt.plot(time, throughput)
# plt.plot(time[peaks], throughput[peaks], "x")
# plt.show()
return False
freq = np.average(diffs)
# if isClose(freq, 5 * RTT, threshold=RTT*1.1):
# print(freq, 5*RTT)
# plt.plot(time, throughput)
# plt.plot(time[peaks], throughput[peaks], "x")
# plt.show()
return True if isClose(freq, 5 * RTT, threshold=RTT * 1.1) else False
def isBBR(time, throughput, RTT):
#RTT+=0.01
samples_per_RTT = int(len(time) / (time[-1] - time[0])) * RTT
throughput = smooth(throughput, 10)[3:-3]
time = time[3:-3]
peaks, properties = find_peaks(throughput,
width=samples_per_RTT,
prominence=0.15)
diffs = np.diff(time[peaks])
if (len(diffs) == 0 or np.var(diffs) > 0.025):
# print(np.var(diffs))
# plt.plot(time, throughput)
# plt.plot(time[peaks], throughput[peaks], "x")
# plt.show()
return False
freq = np.average(diffs)
# if not isClose(freq, 8 * RTT, RTT*1.3):
# print(freq, 8*RTT)
# plt.plot(time, throughput)
# plt.plot(time[peaks], throughput[peaks], "x")
# plt.show()
#print(freq)
return True if isClose(freq, 8 * RTT, threshold=RTT * 1.3) else False
def display(self, momentum=0.0):
plt.figure()
plt.plot(self.period * np.arange(len(self.eer)),
smooth(self.eer, momentum=0.9),
label="dev EER every {} generations".format(self.period))
plt.xlabel("generations")
plt.ylabel("EER")
plt.show()
plt.figure()
plt.plot(self.period * np.arange(len(self.acc)),
smooth(self.acc, momentum=0.9),
label="dev acc every {} generations".format(self.period))
plt.xlabel("generations")
plt.ylabel("accuracy")
plt.show()
def model(data,
ix_to_char,
char_to_ix,
num_iterations=200000,
n_a=50,
dino_names=7,
vocab_size=27):
"""
Trains the model and generates dinosaur names.
Arguments:
data -- text corpus
ix_to_char -- dictionary that maps the index to a character
char_to_ix -- dictionary that maps a character to an index
num_iterations -- number of iterations to train the model for
n_a -- number of units of the RNN cell
dino_names -- number of dinosaur names you want to sample at each iteration.
vocab_size -- number of unique characters found in the text, size of the vocabulary
Returns:
parameters -- learned parameters
"""
n_x, n_y = vocab_size, vocab_size
parameters = initialize_parameters(n_a, n_x, n_y)
loss = get_initial_loss(vocab_size, dino_names)
with open("Name.txt") as f:
examples = f.readlines()
examples = [x.lower().strip() for x in examples]
np.random.seed(0)
np.random.shuffle(examples)
a_prev = np.zeros((n_a, 1))
for j in range(num_iterations):
index = j % len(examples)
X = [None] + [char_to_ix[ch] for ch in examples[index]]
Y = X[1:] + [char_to_ix["\n"]]
curr_loss, gradients, a_prev = optimize(X, Y, a_prev, parameters, 0.01)
loss = smooth(loss, curr_loss)
if j % 2000 == 0:
print('Iteration: %d, Loss: %f' % (j, loss) + '\n')
seed = 0
for name in range(dino_names):
sampled_indices = sample(parameters, char_to_ix, seed)
print_sample(sampled_indices, ix_to_char)
seed += 1
print('\n')
return parameters
def process_map(self, x_in):
"""Process data."""
x = np.array(x_in, copy=True)
if self.ng: # add noise if ng is not None
x = add_shape_noise(x, self.A_pix, self.ng, self.rng)
if self.smoothing: # smooth if smoothing is not None
x = smooth(x, self.smoothing, self.scale)
return x
def get_kappa_rms(self, x, omega_m_list, sigma_8_list):
"""Return the mean kappa r.m.s. of fiducial maps."""
kappa_rms = []
for xi, o, s in zip(x, omega_m_list, sigma_8_list): # loop maps
if o == self.fiducial_params[0] and s == self.fiducial_params[1]:
im = smooth(xi, self.smoothing, self.scale) # apply smoothing
kappa_rms.append(im.std()) # get kappa rms for a map
return np.mean(kappa_rms) # return mean kappa rms
def compute_foreground_scores(group_A_segmentations,
group_B_segmentations,
random_chunks,
states,
metric_A,
metric_B,
to_smooth=False,
max_min_window=0,
compute_per_state_scores=False):
foreground_scores = dict((k, {}) for k in [OVERALL_SCORE] + (states if compute_per_state_scores else []))
datasets_A = sorted(group_A_segmentations)
datasets_B = sorted(group_B_segmentations)
for chrom in random_chunks:
chrom_segmentations_A = [chrom_segmentation_to_list(group_A_segmentations[d][chrom], BIN_SIZE) for d in datasets_A]
chrom_segmentations_B = [chrom_segmentation_to_list(group_B_segmentations[d][chrom], BIN_SIZE) for d in datasets_B]
for chunk_start, chunk_end in random_chunks[chrom]:
chunk_segmentations_A = dict((d, seg[chunk_start:chunk_end]) for d, seg in izip(datasets_A, chrom_segmentations_A))
chunk_segmentations_B = dict((d, seg[chunk_start:chunk_end]) for d, seg in izip(datasets_B, chrom_segmentations_B))
chunk_scores = get_overall_and_per_state_diff_score(chrom,
chunk_segmentations_A,
metric_A,
chunk_segmentations_B,
metric_B,
BIN_SIZE,
states,
None,
compute_per_state_scores,
max_min_window,
background_chunk=True)
if to_smooth:
chunk_scores = dict((score_type, smooth(chunk_scores[score_type])) for score_type in chunk_scores)
for score_type in foreground_scores:
for s in chunk_scores[score_type]:
abs_score = abs(s)
if abs_score not in foreground_scores[score_type]:
foreground_scores[score_type][abs_score] = 0
foreground_scores[score_type][abs_score] += 1
for score_type in foreground_scores:
total_regions = sum(foreground_scores[score_type].itervalues())
total_so_far = 0
for s in sorted(foreground_scores[score_type], reverse=True):
total_so_far += foreground_scores[score_type][s]
foreground_scores[score_type][s] = total_so_far / float(total_regions)
return dict((score_type, sorted(foreground_scores[score_type].items())) for score_type in foreground_scores)
def caculate_instance_prior_confidence_score(train_set, k, num_level = 2):
""" 在holdout dataset(或者训练集)中计算每个instance在每个dynamic factor的先验信心分数
"""
import random
num_factor = len(train_set[0][1][1]) # get the number of factors
topic_popularity = dict() # topic_id ==> (level, target_comment_count, prediction_comment_count, ratio)
for train_topic_id, train_ins, level in train_set:
target_comment_count = train_ins[0][0]
prediction_comment_count = train_ins[0][4]
ratio = target_comment_count * 1.0 / prediction_comment_count
topic_popularity[train_topic_id] = (level, target_comment_count, prediction_comment_count, ratio)
# topic_id ==> [i, j] for factor i on class j
prior_score = dict()
total = len(train_set)
index = 0
factor_correct_count = np.zeros((num_factor,), float)
for topic_id, ins, true_level in train_set:
print 'Topic id: %s, Iteration: %d' % (topic_id, index)
# 记录评分矩阵
score_matrix = np.zeros((num_factor, num_level))
for findex in range(num_factor):
level_confidence_score = factor_score_knn(findex, ins, train_set, topic_popularity, k, num_level)
level_confidence_score = smooth(level_confidence_score)
score_matrix[findex, :] = level_confidence_score
# 计算先验，满足两个要求
level_prior = np.ones((num_factor, num_level)) # prior for classes(levels)
pred_level_list = [0] * num_factor
num_correct = 0 # 得出正确结果的factor的个数
print 'Topic %s true level: %d' % (topic_id, true_level)
for findex in range(num_factor):
# predict based on confidence
pred_level_list[findex] = pred_level = np.argmax(score_matrix[findex, :])
print 'Factor %d prediction: %d' % (findex, pred_level)
score = score_matrix[findex, true_level]
if pred_level != true_level:
# 如果当前factor分类错误，则此factor对分类的贡献忽略
level_prior[findex, :] = 1.0 / num_level
else:
# 如果预测正确，则在此类中，添加权重
beta = score / (1 - score)
level_prior[findex, true_level] *= beta
num_correct += 1
factor_correct_count[findex] += 1
Z = np.sum(level_prior[findex, :])
level_prior[findex, :] /= Z
prior_score[topic_id] = level_prior
print 'Instance level prior for %s: %r' % (topic_id, level_prior)
index += 1
#print 'Training acc of single factors:', factor_correct_count / total
print 'Training acc of single factors:', factor_correct_count / total
return prior_score
def run_NAIE(config):
graph = load_graph(config['dataset'], labels_is_onehot=False)
if config['task'] == 'lp':
graph.G = remove_edges(
graph.G,
config['lp_test_path'] + config['dataset'] + "_lp_test.edgelist")
print("Left edges in G: {}".format(graph.G.number_of_edges()))
test_pairs, test_labels = read_test_links(config['lp_test_path'] +
config['dataset'] +
"_lp_test.edgelist")
config['link_test_pairs'] = [
(edges[0], edges[1], label)
for edges, label in zip(test_pairs, test_labels)
]
y = graph.labels
X = graph.features
A = graph.adjcency_matrix(is_sparse=False)
C = np.concatenate([A, config['lambda'] * X], axis=1)
smooth_X = smooth(A, X, 1.0)
smooth_A = smooth(A, A, 1.0)
if config['strategy'] == 'nc':
gamma_adj = 1 - get_balance_coefficient(graph.G, smooth_A)
gamma_attr = 1 - get_balance_coefficient(graph.G, smooth_X)
elif config['strategy'] == 'sw':
omega = get_omega(graph.G)
omega = abs(omega)
if omega > 1:
omega = 1.0
gamma_adj = omega
gamma_attr = omega
print("gamma_adj={:4f}, gamma_attr={:.4f}".format(gamma_adj, gamma_attr))
ada_smooth_A = smooth(A, A, gamma_adj)
ada_smooth_X = smooth(A, X, gamma_attr)
target = np.concatenate([ada_smooth_A, config['lambda'] * ada_smooth_X],
axis=1)
config['struct'][0] = C.shape[1]
data = {'C': C, 'target': target, 'adj': A, 'y': y}
model = NAIE(config)
model.train(data)
def main():
args = parse()
file = args.file
speech_path = os.path.join(args.speechfolder, file)
true_egg_path = os.path.join(args.eggfolder, file)
estimated_egg_path = os.path.join(args.geneggfolder, file)
speech = np.load(speech_path)
speech = minmaxnormalize(speech)
speech = smooth(speech, 21)
true_egg = np.load(true_egg_path)
true_egg = minmaxnormalize(true_egg)
if args.detrend:
_, true_egg = detrend(None, true_egg)
estimated_egg = np.load(estimated_egg_path)
estimated_egg = minmaxnormalize(estimated_egg)
# srange = slice(0, speech.shape[0])
srange = slice(10700, 15400)
speech = speech[srange]
plt.figure()
fig = plt.gcf()
plt.subplot(211)
plt.title("Speech")
plt.plot(speech, "k", label="Speech Waveform")
plt.xlabel("Sample")
plt.ylabel("Amplitude")
dft = np.fft.rfft(speech)
freqs = np.fft.rfftfreq(np.size(speech, 0), 1 / 16e3)
assert freqs.shape == dft.shape
plt.subplot(212)
plt.title("Fourier Spectra")
plt.gca().semilogx(freqs, np.abs(dft)**2, "b", label="DFT")
# plt.plot(freqs, np.abs(dft), "b", label="DFT")
plt.xlabel("Frequency")
plt.ylabel("PSD")
plt.subplots_adjust(top=0.926,
bottom=0.117,
left=0.078,
right=0.981,
hspace=0.476,
wspace=0.2)
mng = plt.get_current_fig_manager()
mng.window.showMaximized()
plt.show()
fig.savefig("images/fft.png")
def filter_by_dbscan(self, df, bm):
"""
Do filter by dbscan name
"""
du, sb = df[df.bmnum==bm], [np.nan, np.nan, np.nan]
sb[0] = len(du.groupby("time"))
self.gen_summary[bm] = {"bmnum": bm, "v": np.median(du.v), "p": np.median(du.p_l), "w": np.median(du.w_l),
"sound":sb[0], "echo":len(du), "v_mad": stats.median_absolute_deviation(du.v),
"p_mad": stats.median_absolute_deviation(du.p_l), "w_mad": stats.median_absolute_deviation(du.w_l)}
xpt = 100./sb[0]
if bm == "all":
sb[1] = "[" + str(int(np.min(df.bmnum))) + "-" + str(int(np.max(df.bmnum))) + "]"
sb[2] = len(self.scans) * int((np.max(df.bmnum)-np.min(df.bmnum)+1))
else:
sb[1] = "[" + str(int(bm)) + "]"
sb[2] = len(self.scans)
l = du.groupby("time")
logger.info(f" Beam Analysis: {bm}, {len(l)}")
rng, eco = np.array(du.groupby(["slist"]).count().reset_index()["slist"]),\
np.array(du.groupby(["slist"]).count().reset_index()["p_l"])
Rng, Eco = np.arange(np.max(du.slist)+1), np.zeros((np.max(du.slist)+1))
for e, r in zip(eco, rng):
Eco[Rng.tolist().index(r)] = e
eco, Eco = utils.smooth(eco, self.window), utils.smooth(Eco, self.window)
glims, labels = {}, []
ds = DBSCAN(eps=self.eps, min_samples=self.min_samples).fit(du[["slist"]].values)
du["labels"] = ds.labels_
names = {}
for j, r in enumerate(set(ds.labels_)):
x = du[du.labels==r]
glims[r] = [np.min(x.slist), np.max(x.slist)]
names[r] = "C"+str(j)
if r >= 0: self.boundaries[bm].append({"peak": Rng[np.min(x.slist) + Eco[np.min(x.slist):np.max(x.slist)].argmax()],
"ub": np.max(x.slist), "lb": np.min(x.slist), "value": np.max(eco)*xpt, "bmnum": bm, "echo": len(x), "sound": sb[0]})
logger.info(f" Individual clster detected: {set(du.labels)}")
loc_folder = self.fig_folder + "gate_summary/"
if not os.path.exists(loc_folder): os.system("mkdir -p " + loc_folder)
fig_file = loc_folder + "gate_{bm}_summary.png".format(bm="%02d"%bm)
to_midlatitude_gate_summary(self.rad, du, glims, names, (rng,eco), fig_file, sb)
return
def find_bc(profile, mc_input):
s = read_mesa(profile)
r = np.flipud(s.radius)
m = np.flipud(s.mass)
rho = np.flipud(s.rho)
p = np.flipud(s.pressure)
drhodr = np.gradient(rho,r)
drhodr_smooth = smooth(drhodr)
dpdr = np.gradient(p,r)
dpdr_smooth = smooth(dpdr)
ic = find_nearest(m, mc_input)
bc = {'r' : r[ic],
'm' : m[ic],
'rho' : rho[ic],
'p' : p[ic],
'drhodr' : drhodr[ic],
'dpdr' : dpdr[ic]}
return bc
def generate_candidate_default(self, original,
unnormalized_source_direction,
source_direction, source_norm):
spherical_step = self.spherical_step
source_step = self.source_step
perturbation = np.random.randn(*original.shape)
perturbation = perturbation.astype(np.float32)
shape = perturbation.shape
# apply hanning filter
if self.args.smooth:
if self.args.win % 2 == 1:
# Verifying Odd Window Size
perturbation = smooth(perturbation.squeeze(),
window_len=self.args.win)
perturbation = perturbation.reshape(shape)
# ===========================================================
# calculate candidate on sphere
# ===========================================================
dot = np.vdot(perturbation, source_direction)
perturbation -= dot * source_direction
perturbation *= spherical_step * source_norm / norm(perturbation)
D = 1 / np.sqrt(spherical_step**2 + 1)
direction = perturbation - unnormalized_source_direction
spherical_candidate = original + D * direction
# ===========================================================
# add perturbation in direction of source
# ===========================================================
new_source_direction = original - spherical_candidate
new_source_direction_norm = norm(new_source_direction)
# length if spherical_candidate would be exactly on the sphere
length = source_step * source_norm
# length including correction for deviation from sphere
deviation = new_source_direction_norm - source_norm
length += deviation
# make sure the step size is positive
length = max(0, length)
# normalize the length
length = length / new_source_direction_norm
candidate = spherical_candidate + length * new_source_direction
return (candidate, spherical_candidate)
def main(): fname = '../images/original/chibombo1.png' original = cv.imread(fname) utils.show_image(original, 'original') img = utils.smooth(original, 'bilateral') utils.show_image(img, 'bilateral filter') # get image dimensions xdim, ydim, nchannels = img.shape veg_to_background = detectvegetation.detect(img, xdim, ydim) segmented = segmentcolor.mask(veg_to_background, xdim, ydim) detect = detectpolygon.detect(segmented, original, xdim, ydim)
def main():
fname = '../images/original/chibombo1.png'
original = cv.imread(fname)
utils.show_image(original, 'original')
img = utils.smooth(original, 'bilateral')
utils.show_image(img, 'bilateral filter')
# get image dimensions
xdim, ydim, nchannels = img.shape
veg_to_background = detectvegetation.detect(img, xdim, ydim)
segmented = segmentcolor.mask(veg_to_background, xdim, ydim)
detect = detectpolygon.detect(segmented, original, xdim, ydim)
def __init__(self, path):
t = path
if type and path.__class__ == str:
t = Image.open(path)
phash = imagehash.phash(t, 8)
histogram = np.array(t.convert('L').histogram())
self.md5 = md5(t.tostring()).hexdigest()
self.phash = str(phash)
self.histogram = utils.smooth(histogram, 100)
self.mins = argrelextrema(self.histogram, np.less)[0]
self.maxs = argrelextrema(self.histogram, np.greater)[0]
self.histogram = np.array(map(lambda x: int(x), self.histogram))
if len(self.mins) < 2: self.mins = np.append(self.mins, [1000] * (2-len(self.mins)) )
if len(self.maxs) < 2: self.maxs = np.append(self.maxs, [1000] * (2-len(self.maxs)) )
def updateDatasets(self, fields, helper):
"""Do shifting of dataset.
This function should *update* the dataset(s) returned by getDatasets
"""
# get the input dataset - helper provides methods for getting other
# datasets from Veusz
ds_in = helper.getDataset(fields['ds_in'])
# get the value to add
window = fields['window']
method = fields['method']
start_index = fields['start_index']
end_index = fields['end_index']
if end_index == 0:
end_index = len(ds_in.data)
x = numpy.array(ds_in.data)
x = x[start_index:end_index]
if ds_in == helper.getDataset(fields['ds_out']):
raise plugins.DatasetPluginException(
"Input and output datasets should differ.")
if x.ndim != 1:
raise plugins.DatasetPluginException(
"smooth only accepts 1 dimension arrays.")
if x.size < window:
raise plugins.DatasetPluginException(
"Input vector needs to be bigger than window size.")
if window < 3:
raise plugins.DatasetPluginException("Window is too small.")
if not method in [
'flat', 'hanning', 'hamming', 'bartlett', 'blackman'
]:
raise plugins.DatasetPluginException(
"Mehtod is one of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'"
)
y = smooth(x, window, method)
y = numpy.concatenate(
(ds_in.data[0:start_index], y, ds_in.data[end_index:]))
# update output dataset with input dataset (plus value) and errorbars
self.ds_out.update(data=y,
serr=ds_in.serr,
perr=ds_in.perr,
nerr=ds_in.nerr)
return [self.ds_out]
def run(c, num_gen=100, params=None, display=False):
"""
Runs NEAT for num_gen generations
and with custom config parameters params.
"""
# Manually sets some parameters with custom values
if params is None:
params = {}
for p in params:
setattr(c.genome_config, p, params[p])
pop = neat.Population(c)
stats = neat.StatisticsReporter()
pop.add_reporter(stats)
reporter = neat.StdOutReporter(True)
pop.add_reporter(reporter)
evaluator = MNISTEvaluator(eval_genome, load_MNIST(train=True))
# evaluator = MNISTParallelEvaluator(multiprocessing.cpu_count(), eval_genome, load_MNIST(train=True))
acc_reporter = AccuracyReporter(evaluator)
pop.add_reporter(acc_reporter)
start = time.time()
winner = pop.run(evaluator.evaluate, num_gen)
end = time.time()
print("execution time:", end - start)
if display:
visualize.draw_net(c, winner, True, filename="graph_hyperneat_test")
visualize.plot_stats(stats,
ylog=False,
view=True,
filename="stats_hyperneat_test")
plt.figure()
plt.plot(smooth(acc_reporter.accuracy, momentum=0.995))
plt.show()
visualize.plot_species(stats,
view=True,
filename="species_hyperneat_test1")
return winner
def updateDatasets(self, fields, helper):
"""Do shifting of dataset.
This function should *update* the dataset(s) returned by getDatasets
"""
# get the input dataset - helper provides methods for getting other
# datasets from Veusz
ds_in = helper.getDataset(fields['ds_in'])
# get the value to add
window = fields['window']
method = fields['method']
start_index = fields['start_index']
end_index = fields['end_index']
if end_index == 0:
end_index = len(ds_in.data)
x = numpy.array(ds_in.data)
x = x[start_index:end_index]
if ds_in == helper.getDataset(fields['ds_out']):
raise plugins.DatasetPluginException(
"Input and output datasets should differ.")
if x.ndim != 1:
raise plugins.DatasetPluginException(
"smooth only accepts 1 dimension arrays.")
if x.size < window:
raise plugins.DatasetPluginException(
"Input vector needs to be bigger than window size.")
if window < 3:
raise plugins.DatasetPluginException("Window is too small.")
if not method in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
raise plugins.DatasetPluginException(
"Mehtod is one of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'")
y = smooth(x, window, method)
y = numpy.concatenate((ds_in.data[0:start_index], y, ds_in.data[end_index:]))
# update output dataset with input dataset (plus value) and errorbars
self.ds_out.update(
data=y, serr=ds_in.serr, perr=ds_in.perr, nerr=ds_in.nerr)
return [self.ds_out]
def plot_dssp(iso, ratio, fig, subplot_num):
ax=fig.add_subplot(3, 4, subplot_num)
fig.subplots_adjust(bottom=0.05,top=0.95,left=0.05,right=0.95,wspace=0.4, hspace=0.4)
# pylab.rcParams['xtick.labelsize']='10'
# pylab.rcParams['ytick.labelsize']='12'
pylab.rcParams['legend.fontsize']='6'
# pylab.rcParams['figure.figsize'] = [2.5,2.5]
# pylab.rcParams["axes.titlesize"]='small'
# TODO: refactor this to a configuration file
#filename = '%(subplot_num)d/sys%(subplot_num)d_dssp.dat' % vars()
filename = 'ab_%(iso)s_%(ratio)s_%(subplot_num)d_sc.xvg' % vars()
A=[]
if os.path.exists(filename):
A=utils.smooth(numpy.genfromtxt(filename, comments="#"), 1000, timestep=2)
else:
print "WARNING: %(filename)s not found" % vars()
if A == []:
print "WARNING: data file empty ... quitting ..."
return
time = A[:,0]/1000.0
ax.plot(time, A[:,2], label="Coil")
ax.plot(time, A[:,3], label="B-sheet")
ax.plot(time, A[:,4], label="B-bridge")
ax.plot(time, A[:,5], label="Bend")
ax.plot(time, A[:,6], label="Turn")
pylab.ylim(0, 80)
pylab.xlim(0, 100)
pylab.grid(True)
# xlabel('Time (ns)')
# ylabel('Number of interchain hydrogen bonds')
pylab.legend(loc='lower right')
for i in range(nbins):
ii = (qv == i)
h[i, 0] = np.nanmean(m[ii]) # mean mortality in this bin
if m[ii].size > 0:
h[i, 4] = np.nanmean(
a[ii]
) #record the mean of Q values in the bin (to make sure it matches what I expect)
h[i, 1] = np.std(m[ii]) / np.sqrt(
m[ii].size) # SEM of mortality in this bin
h[i, 2] = m[ii].size # nb of data points in this bin
h[:, 3] = h[:, 0] * h[:, 2] / qv.size # weighted average!!
# [nansum(h(:,4)) mean(prog(:,2))] %check that both are close!
yy1 = smooth(np.array(range(nbins)), h[:, 0], 0.1)
fig, ax = plt.subplots()
plt.xlim(0, nbins)
plt.ylim(0, 1)
plt.plot([0, nbins], [0.5, 0.5], linestyle=':', color='black')
plt.plot([nbins / 2, nbins / 2], [0, 1], linestyle=':', color='black')
plt.plot(h[:, 0], linewidth=1, color='blue')
plt.plot(h[:, 0] + h[:, 1], linewidth=0.5, color='blue')
plt.plot(h[:, 0] - h[:, 1], linewidth=0.5, color='blue')
plt.plot(yy1, linewidth=1, color='red')
ax.set_ylabel('Mortality risk')
ax.set_xlabel('Return of actions')
plt.xticks(np.arange(0, nbins + 10, step=nbins / 10),
np.arange(-100, 120, step=20))
plt.show()
def process(h5file, ratio, format="p2p_vs_t.dat"):
# given a h5file return a list of data to be plotted as line plots
# and a corresponding list of labels
header = "# time average_inter std_inter average_intra std_intra"
datalist = []
labellist = []
isomerlist = ["scyllo", "chiro", "water"]
mean_contact_list = []
std_contact_list = []
for iso in isomerlist:
print "processing", iso
pattern = re.compile(r"%(iso)s.*%(ratio)s.*%(format)s" % vars())
if iso == "water":
pattern = re.compile(r"%(iso)s.*%(format)s" % vars())
data_inter = []
data_intra = []
for table in h5file.listNodes(where='/polar'):
table_path = os.path.join('/polar', table.name)
if pattern.search(table.name):
print "processing", table.name
data = myh5.getTableAsMatrix(h5file, table_path, dtype=numpy.int32)
data = data.astype('float')
print "converted to float32", data
nrows, ncols = data.shape
assert nrows > ncols
print "Test data read in dimensions", data.shape, data.dtype
data_inter.append(data[0:config.LASTFRAME,1])
data_intra.append(data[0:config.LASTFRAME,2])
# compute summary statistics
print "summarizing statistics ... "
inter_matrix = utils.array_list_to_matrix(data_inter)
intra_matrix = utils.array_list_to_matrix(data_intra)
average_inter, std_inter = utils.summary_statistics(inter_matrix)
average_intra, std_intra = utils.summary_statistics(intra_matrix)
# compute the time average number of contacts and its std error
avg_contacts = numpy.average(inter_matrix, axis=0)
mean_contact = numpy.average(avg_contacts)
std_contact = numpy.std(avg_contacts)
mean_contact_list.append(mean_contact)
std_contact_list.append(std_contact)
print mean_contact, std_contact
time = data[0:config.LASTFRAME,0]
# print "Test: dimensions of average_inter", average_inter.shape
plotdata = utils.array_list_to_matrix([ time, average_inter, std_inter, average_intra, std_intra ])
print "plotdata", plotdata
print "Test: dimensions of plotdata for", iso, ratio, plotdata.shape
plotdata_smoothed = utils.smooth(plotdata, 500, time_present=True, timestep=2)
print plotdata_smoothed
datalist.append(plotdata_smoothed)
print "smoothed data", plotdata_smoothed, plotdata_smoothed.shape
ratiolabel = config.RATIO[ratio]
if iso == "water":
labellist.append("water" % vars())
else:
labellist.append("%(iso)s (%(ratiolabel)s)" % vars())
utils.savetxt('%(ratio)s_p2p_vs_t.txt' % vars(), header, plotdata, fmt='%0.2f')
utils.savetxt('%(ratio)s_p2p_vs_t_smoothed.txt' % vars(), header, plotdata_smoothed, fmt='%0.2f')
utils.savetxt('%(ratio)s_avg_contacts_w_err.txt' % vars(), "#scyllo chiro water", numpy.vstack([mean_contact_list, std_contact_list]), fmt='%0.2f')
return (datalist, labellist)
def do(fields, helper):
"""Actually do the CurveOperationPlugin calculation."""
op = fields['operation'].lower()
xtol = fields['tolerance']
# get the input dataset - helper provides methods for getting other
# datasets from Veusz
ay = np.array(helper.getDataset(fields['ay']).data)
ax = np.array(helper.getDataset(fields['ax']).data)
N = len(ax)
if len(ay) != N:
raise plugins.DatasetPluginException(
'Curve A X,Y datasets must have same length')
by = np.array(helper.getDataset(fields['by']).data)
bx = np.array(helper.getDataset(fields['bx']).data)
Nb = len(bx)
if len(by) != Nb:
raise plugins.DatasetPluginException(
'Curve B X,Y datasets must have same length')
error = 0
# Relativize
if fields['relative']:
d = by[0] - ay[0]
by -= d
logging.debug('relative correction', d)
# If the two curves share the same X dataset, directly operate
if fields['bx'] == fields['ax']:
out = numexpr.evaluate(op, local_dict={'a': ay, 'b': by})
return out, 0
# Smooth x data
if fields['smooth']:
ax = utils.smooth(ax)
bx = utils.smooth(bx)
# Rectify x datas so they can be used for interpolation
if xtol > 0:
rax, dax, erra = utils.rectify(ax)
rbx, dbx, errb = utils.rectify(bx)
logging.debug('rectification errors', erra, errb)
if erra > xtol or errb > xtol:
raise plugins.DatasetPluginException(
'X Datasets are not comparable in the required tolerance.')
# TODO: manage extrapolation!
# Get rectified B(x) spline for B(y)
logging.debug('rbx', rbx[-1] - bx[-1], rbx)
logging.debug('by', by)
N = len(rbx)
margin = 1 + int(N / 10)
step = 2 + int((N - 2 * margin) / 100)
logging.debug( 'interpolating', len(rbx), len(by), margin, step)
bsp = interpolate.LSQUnivariateSpline(rbx, by, rbx[margin:-margin:step]) #ext='const' scipy>=0.15
error = bsp.get_residual()
# Evaluate B(y) spline with rectified A(x) array
b = bsp(rax)
logging.debug('rax', rax[-1] - ax[-1], rax)
logging.debug('a', ay)
logging.debug('b', b, b[1000:1010])
# np.save('/tmp/misura/rbx',rbx)
# np.save('/tmp/misura/by',by)
# np.save('/tmp/misura/rax',rax)
# np.save('/tmp/misura/ay',ay)
# Perform the operation using numexpr
out = numexpr.evaluate(op, local_dict={'a': ay, 'b': b})
logging.debug('out', out)
return out, error
def caculate_instance_prior_confidence_score(train_set, test_set, num_neigh, num_factor, num_level = 2):
""" 在holdout dataset(或者训练集)中计算每个instance在每个dynamic factor的先验信心分数
"""
# merge the train and test set
dataset = list(train_set)
dataset.extend(test_set)
topic_popularity = dict() # topic_id ==> (level, comment_count)
for topic_id, ins, level in train_set:
target_comment_count = ins[0][0]
prediction_comment_count = ins[0][4]
# ratio的值不小于1
ratio = target_comment_count * 1.0 / prediction_comment_count
# 最后一个值表示该topic是否是train topic
topic_popularity[topic_id] = (level, target_comment_count, prediction_comment_count, ratio, ins, True)
# 对于测试样本, None值表示未知
for topic_id, ins, level in test_set:
target_comment_count = ins[0][0]
prediction_comment_count = ins[0][4]
# ratio的值不小于1
ratio = target_comment_count * 1.0 / prediction_comment_count
topic_popularity[topic_id] = (level, target_comment_count, prediction_comment_count, ratio, ins, False)
print 'Creating mutual knn graphs...'
# Note: 一些帖子可能包含比num_neigh少的mutual knn neighbour
mutual_knn_graph_list = create_mutual_knn_graph(dataset, num_neigh, num_factor, topic_popularity)
#import ipdb; ipdb.set_trace()
# train_topic_id ==> [i, j] for factor i on class j
prior_score = dict()
total = len(train_set)
index = 0
factor_correct_count = np.zeros((num_factor,), float)
for train_topic_id, train_ins, true_level in train_set:
print 'Topic id: %s, Iteration: %d, true level: %d' % (train_topic_id, index, true_level)
# 记录评分矩阵
score_matrix = np.zeros((num_factor, num_level))
for findex in range(num_factor):
# 使用原来的近邻挑选方法
level_confidence_score, level_prior_score = factor_score_knn(findex, mutual_knn_graph_list[findex], train_topic_id, topic_popularity, num_level)
level_confidence_score = smooth(level_confidence_score)
score_matrix[findex, :] = level_confidence_score
pred_level_list = [0] * num_factor
num_correct = 0 # 得出正确结果的factor的个数
for findex in range(num_factor):
# predict based on confidence
pred_level_list[findex] = pred_level = np.argmax(score_matrix[findex, :])
print 'Factor %d: confidence = %r, prediction = %d' % (findex, score_matrix[findex, :], pred_level)
if pred_level != true_level:
pass
else:
num_correct += 1
factor_correct_count[findex] += 1 # 每个factor预测正确的样本个数
# 计算先验，满足两个要求
# 每个instance都保存有某个factor对其分类结果的信息，如果分类正确，则权重大于1，如果分类错误，则小于1
level_prior = np.ones((num_factor, )) # prior for classes(levels)
# 在factor之间之间进行区别：例如如果只有一个factor预测正确，那么奖励会更多
delta = 1.0
if num_correct == 0:
correct_reward = 1.0
else:
correct_reward = delta / num_correct
rho = 3
for findex in range(num_factor):
# 预测两类的信心值之差
diff_score = abs(score_matrix[findex, 0] - score_matrix[findex, 1])
if pred_level_list[findex] != true_level:
tmp = math.exp(-1 * rho * diff_score)
else:
tmp = math.exp(+1 * rho * diff_score)
# 另外一层考虑：例如如果只有一个factor预测正确，则奖励会更多
if pred_level_list[findex] == true_level:
tmp *= math.exp(correct_reward)
level_prior[findex] *= tmp
#level_prior /= np.sum(level_prior)
prior_score[train_topic_id] = level_prior
print 'Instance level prior for %s: %r\n' % (train_topic_id, level_prior)
index += 1
#print 'Training acc of single factors:', factor_correct_count / total
print 'Training acc of single factors:', factor_correct_count / total
return topic_popularity, prior_score, mutual_knn_graph_list
def handle_data(pos_data):
from utils import smooth, interpl, chunk_decode
from numpy import genfromtxt
correct_pck_num = 0
diff_num = []
first_detected = False
# plt.ion()
# f = plt.figure()
# ax = f.add_subplot(311)
# ax2 = f.add_subplot(312)
# ax3 = f.add_subplot(313)
# pos_data = genfromtxt(FILE_NAME, delimiter=',')
real_time = pos_data[:,1]
real_val = pos_data[:,0]
even_time = np.arange(real_time[0], real_time[-1], 1/2400)
even_time = even_time[even_time < real_time[-1]]
even_val = interpl(real_time, real_val, even_time, 'nearest')
even_val_smooth = smooth(even_val, 21)
even_val_DCremove = smooth(even_val - even_val_smooth, 5)
# np.savetxt('even_val_DCremove.csv', even_val_DCremove, delimiter=',')
total_points = len(even_time)
# maintain slide window within 0.3s
raw_frames_m = deque(maxlen=int(2400*0.3))
for i in range(total_points):
val = even_val_DCremove[i]
ts = even_time[i]
raw_frames_m.append([ts, val])
if len(raw_frames_m) != raw_frames_m.maxlen:
continue
sample_value_DCremove = np.array(raw_frames_m)[:,1]
value = np.zeros((10, 1))
for i in range(10):
temp_sample = sample_value_DCremove[i:raw_frames_m.maxlen:10]
value[i] = max(temp_sample) - min(temp_sample)
std_min = max(value)
shift_index = np.where(value==std_min)[0][0]
sample_wave = sample_value_DCremove[shift_index:raw_frames_m.maxlen:10]
temp_sample = sample_value_DCremove[shift_index:raw_frames_m.maxlen:10]
# bit_stream = sample_wave <= (max(temp_sample) + min(temp_sample)) / 2
bit_stream = sample_wave <= np.mean(temp_sample)
bit_stream = bit_stream.astype(int)
result, diff_count = chunk_decode(bit_stream)
if result is None:
result, diff_count = chunk_decode(bit_stream, flip=True)
if result is not None:
if not first_detected: first_detected = True
for _ in range(int(raw_frames_m.maxlen/2)):
raw_frames_m.popleft()
# print(result)
start_ts = np.array(raw_frames_m)[0,0]
end_ts = np.array(raw_frames_m)[-1,0]
to_plot = pos_data[np.logical_and(pos_data[:,1] > start_ts, pos_data[:,1] < end_ts)][:,0]
to_plot_FFT = fft(to_plot[:250])
l = 30
to_plot_FFT[1+l:to_plot_FFT.size-l] = 0
# ax.plot(to_plot[:250])
# ax2.plot(ifft(to_plot_FFT))
# ax3.plot(to_plot_FFT[1:])
# plt.pause(10)
# plt.show()
correct_pck_num += 1
diff_num.append(diff_count[0])
elif first_detected and diff_count != []:
diff_num.append(diff_count[0])
print('correct_pck_num:', correct_pck_num)
# print('diff_num:', diff_num)
diff_num = sorted(diff_num)
diff_num = diff_num[:100]
# print(diff_num)
total_error_num = sum(diff_num) + (100 - len(diff_num)) * 62
# smooth the pdf + watershed
from utils import smooth
nbins = (400,400);
j = 400;
gmmdata = np.vstack([np.log(dists[:,j]), (rots[:,j])]).T
#gmmdata = np.vstack([dists[:,j], (rots[:,j])]).T
gmmdata.shape
nanids = np.any(np.isnan(gmmdata), axis=1);
gmmdata = gmmdata[~nanids,:];
gmmdata.shape
imgbin = smooth(gmmdata, nbins = nbins, sigma = None)
img = smooth(gmmdata, nbins = nbins, sigma = (7,7))
#plt.matshow(img)
from scipy import ndimage as ndi
from skimage.morphology import watershed
from skimage.feature import peak_local_max
local_maxi = peak_local_max(img, indices=False, footprint=np.ones((3, 3)))
markers = ndi.label(local_maxi)[0]
labels = watershed(-img, markers, mask = None)
fig, axes = plt.subplots(ncols=2, sharex=True, sharey=True, subplot_kw={'adjustable':'box-forced'})
ax0, ax1 = axes
ax0.imshow(img, cmap=plt.cm.jet, interpolation='nearest')
ax0.set_title('PDF')
mean_fitness.append(mean(fitness))
best_fitness.append(max(fitness))
solutions = [solution for solution in solutions if solution]
if solutions:
best_solutions.append(min(solutions, key=lambda x: len(x)))
fig, ax1 = pylab.subplots()
#ax1.plot(mean_fitness)
#ax1.plot(best_fitness)
ax1.plot(smooth(mean_fitness, generations // 10))
ax1.plot(smooth(best_fitness, generations // 10))
ax1.axhline(1.0 / len(path), ls='dashed')
ax1.set_xlabel('Generations')
ax1.set_ylabel('Fitness')
ax1.set_ylim(0, top=max(best_fitness) * 0.05 + ax1.get_ylim()[1])
ax1.set_xlim(0, right=len(mean_fitness) - 1)
ax1.legend(('Mean', 'Best', 'Max'), 'best')
fig.savefig('fitness_{0}.pdf'.format(agents))
fig = pylab.figure()
def caculate_class_prior_confidence_score(train_set, k, num_level = 2):
""" 在holdout dataset(或者训练集)中计算每个dynamic factor的先验信心分数
"""
import random
num_factor = len(train_set[0][1][1]) # get the number of factors
topic_popularity = dict() # topic_id ==> (level, comment_count)
for train_topic_id, train_ins, level in train_set:
target_comment_count = train_ins[0][0]
prediction_comment_count = train_ins[0][4]
ratio = target_comment_count * 1.0 / prediction_comment_count
topic_popularity[train_topic_id] = (level, target_comment_count, prediction_comment_count, ratio)
# prior_score[i, j]: for factor i, P(true=j | pred=j)
prior_score = np.ones((num_factor, num_level))
total = len(train_set)
# 检查是否收敛
score_history = np.zeros((num_factor * num_level, total), float)
index = 0
score_matrix = np.zeros((num_factor, num_level))
for topic_id, ins, true_level in train_set:
#if random.random() < 0:
# continue
print 'Iteration: ', index
for findex in range(num_factor):
level_confidence_score = factor_score_knn(findex, ins, train_set, topic_popularity, k, num_level)
level_confidence_score = smooth(level_confidence_score)
score_matrix[findex, :] = level_confidence_score
#import ipdb; ipdb.set_trace()
for findex in range(num_factor):
# predict based on confidence
pred_level = np.argmax(score_matrix[findex, :])
if pred_level != true_level:
# 如果预测错误，则将各个信心值调的尽量接近
score = score_matrix[findex, true_level]
beta = (1 - score) / score
prior_score[findex, true_level] *= beta
else:
# 如果预测正确，则在此类中，添加权重
score = score_matrix[findex, true_level]
beta = score / (1 - score)
prior_score[findex, true_level] *= beta
# normalize
Z = np.sum(prior_score[findex, :])
prior_score[findex, :] /= Z
prior_score[findex, :] = smooth(prior_score[findex, :])
print 'Current prior info: \n', prior_score
#import ipdb; ipdb.set_trace()
# 记录历史prior数据
for i in range(num_factor):
for j in range(num_level):
row_index = i * num_level + j
score_history[row_index, index] = prior_score[i, j]
index += 1
#score_history = score_history[:, index]
#check_convergence_plot(score_history)
return prior_score
ts_context_generator = TSContextGenerator(probabilities_of_users,
arms_samples_for_each_user, arms)
for i in range(number_of_days):
ts_context_generator.update_regret_after_day_passed(
number_of_clicks_per_day)
if i % number_of_days_for_splitting == 0 and i != 0:
print(i + 1)
ts_context_generator.split()
regret.append(ts_context_generator.regrets)
reward.append(ts_context_generator.rewards)
clair.append(ts_context_generator.clairs)
plt.figure(0)
plt.plot(np.cumsum(np.mean(regret, axis=0)))
plt.legend(["6 arms - TS"])
plt.xlabel("number of days")
plt.ylabel("cumulative regret [€]")
plt.xticks(np.linspace(0, number_of_clicks_per_day * number_of_days, 8),
np.linspace(0, number_of_days, 8, dtype=np.int32))
plt.show()
plt.figure(1)
plt.plot(u.smooth(np.mean(reward, axis=0), 800))
plt.plot(u.smooth(np.mean(clair, axis=0), 800))
plt.legend(["reward", "clairvoyant"])
plt.xlabel("number of days")
plt.ylabel("mean reward for click [€]")
plt.xticks(np.linspace(0, number_of_clicks_per_day * number_of_days, 8),
np.linspace(0, number_of_days, 8, dtype=np.int32))
plt.show()
m1 = graph['means'][n]
m2 = graph['means'][n+1]
dm = int(t2-t1)
for tp in range(dm):
av = ( (m1*(dm-tp)/dm) + (m2*tp)/dm )
ynew.append( av )
window_len = 11
xnew = np.linspace( min(graph['timepoints']), max(graph['timepoints']), len(ynew) )
ynew = np.array( ynew )
# Iteratively smooth with a small window to minimise distortion
# smooth the xaxis alongside to prevent x-distortion
for x in range( 20 ):
ynew = utils.smooth(ynew, window_len=window_len, window='blackman')
xnew = utils.smooth(xnew, window_len=window_len, window='blackman')
plt.plot( xnew, ynew, color=graph['color'])
else:
if graph['control']:
plt.fill_between( graph['timepoints'], [a+b for a, b in zip( graph['means'], graph['stddev'] )], y2=[a-b for a, b in zip( graph['means'], graph['stddev'] )], alpha=0.05, color=graph['style'][0])
plt.plot( graph['timepoints'], graph['means'], color=graph['style'][0], linestyle=graph['style'][2], label=classl, alpha=0.2)
else:
plt.errorbar( graph['timepoints'], graph['means'], yerr=graph['stddev'], label=classl, fmt=graph['style'][0], marker=graph['style'][1], linestyle=graph['style'][2])
if options.control:
pass
#for classl in baselines:
# p = mpatches.Rectangle(xy, width, height, facecolor="orange", edgecolor="red")
fn = os.path.join(datadir, 'data_stage%d_%s.npy' % (s,features[0])); dists = np.load(fn); fn = os.path.join(datadir, 'data_stage%d_%s.npy' % (s,features[1])); rots = np.load(fn); ddata = np.vstack([np.log(dists[:,d]), (rots[:,d])]).T #gmmdata = np.vstack([dists[:,j], (rots[:,j])]).T ddata.shape nanids = np.logical_or(np.any(np.isnan(ddata), axis=1), np.any(np.isinf(ddata), axis=1)); ddata = ddata[~nanids,:]; ddata.shape imgbin = smooth(ddata, nbins = [nbins, nbins], sigma = None) img2 = smooth(ddata, nbins = [nbins, nbins], sigma = (0.75,0.75)) img = smooth(ddata, nbins = [nbins, nbins], sigma = (1,1)) plt.figure(220); plt.clf() plt.subplot(2,2,1) plt.imshow(imgbin) plt.subplot(2,2,2) plt.imshow(img) plt.subplot(2,2,3) plt.imshow(img2) local_maxi = peak_local_max(img2, indices=False, min_distance = 1) imgm2 = img2.copy(); imgm2[local_maxi] = 3 * imgm2.max(); plt.subplot(2,2,3);
def caculate_instance_prior_confidence_score(train_set, k, num_level = 2):
""" 在holdout dataset(或者训练集)中计算每个instance在每个dynamic factor的先验信心分数
"""
import random
num_factor = len(train_set[0][1][1]) # get the number of factors
topic_popularity = dict() # topic_id ==> (level, comment_count)
for train_topic_id, train_ins, level in train_set:
target_comment_count = train_ins[0][0]
prediction_comment_count = train_ins[0][4]
ratio = target_comment_count * 1.0 / prediction_comment_count
topic_popularity[train_topic_id] = (level, target_comment_count, prediction_comment_count, ratio)
# topic_id ==> [i, j] for factor i on class j
prior_score = dict()
total = len(train_set)
index = 0
factor_correct_count = np.zeros((num_factor,), float)
for topic_id, ins, true_level in train_set:
print 'Topic id: %s, Iteration: %d' % (topic_id, index)
# 记录评分矩阵
score_matrix = np.zeros((num_factor, num_level))
for findex in range(num_factor):
level_confidence_score = factor_score_knn(findex, ins, train_set, topic_popularity, k, num_level)
level_confidence_score = smooth(level_confidence_score)
score_matrix[findex, :] = level_confidence_score
# 计算先验，满足两个要求
level_prior = np.ones((num_factor, num_level)) # prior for classes(levels)
pred_level_list = [0] * num_factor
num_correct = 0 # 得出正确结果的factor的个数
print 'Topic %s, true level: %d' % (topic_id, true_level)
for findex in range(num_factor):
# predict based on confidence
pred_level_list[findex] = pred_level = np.argmax(score_matrix[findex, :])
print 'Factor %d prediction: %d' % (findex, pred_level)
if pred_level != true_level:
pass
else:
num_correct += 1
factor_correct_count[findex] += 1
# 最简单的情形：直接使用score_matrix来作为factor信心值
level_prior = score_matrix
# 在factor之间之间进行区别：例如如果只有一个factor预测正确，那么奖励会更多
rho = 2.0
for findex in range(num_factor):
if pred_level_list[findex] != true_level:
the_other_level = 1 - true_level
level_prior[findex, the_other_level] *= math.exp(-1 * 1.0 * rho / (num_factor-num_correct))
else:
level_prior[findex, true_level] *= math.exp(+1 * 1.0 * rho / num_correct)
# normalize
level_prior[findex, :] /= np.sum(level_prior[findex, :])
prior_score[topic_id] = level_prior
print 'Instance level prior for %s: %r' % (topic_id, level_prior)
index += 1
#print 'Training acc of single factors:', factor_correct_count / total
print 'Training acc of single factors:', factor_correct_count / total
return prior_score