def function3():
print('This is function 2 calling function 1 and saving the result')
import numpy as np
result = submodule1.function1()
np.save('results/some_result',result)
def save(dat, filename):
"""Save a ``Data`` object into a NumPy .npy file.
Parameters
----------
dat : Data
`Data` object
filename : str
Filename of the file to save to. If the filename does not end
with ``.npy``, the ``.npy`` extension will be automatically
appended.
See Also
--------
:func:`load`
Examples
--------
>>> io.save(dat, 'foo.npy')
>>> dat2 = io.load('foo.npy')
"""
np.save(filename, dat)
def export_mod_mask_file(Louvain_mod_file,Louvain_node_file,coords_file,mask_file):
import numpy as np
import nibabel as nib
import os
from dmgraphanalysis.utils_cor import return_mod_mask_corres,read_Louvain_corres_nodes,read_mod_file
print 'Loading node_corres'
node_corres = read_Louvain_corres_nodes(Louvain_node_file)
print node_corres
print node_corres.shape
print 'Loading coords'
coords = np.array(np.loadtxt(coords_file),dtype = int)
print coords.shape
print 'Loading mask parameters'
mask = nib.load(mask_file)
data_mask_shape = mask.get_data().shape
mask_header = mask.get_header().copy()
mask_affine = np.copy(mask.get_affine())
print "Loading community belonging file" + Louvain_mod_file
community_vect = read_mod_file(Louvain_mod_file)
#print community_vect
print community_vect.shape
print "transforming to nii file"
mod_mask_data = return_mod_mask_corres(community_vect,node_corres,coords,data_mask_shape)
#print mod_mask_data
print mod_mask_data.shape
print "saving npy file"
mod_mask_file = os.path.abspath("mod_mask_data.npy")
np.save(mod_mask_file ,np.array(mod_mask_data,dtype = int))
print "saving nii file"
mod_mask_file = os.path.abspath("mod_mask_data.nii")
nib.save(nib.Nifti1Image(np.array(mod_mask_data,dtype = int),mask_affine,mask_header),mod_mask_file)
print "returning"
return mod_mask_file
def segment_request(request):
max_iteration=int(request[:,0].max()//optimal_interval)
for i in range(max_iteration+1):
temp=request[np.logical_and(request[:,0]<=optimal_interval*(i+1),request[:,0]>=(optimal_interval*i+1))]
temp[:,0]=temp[:,0]-optimal_interval*i
np.save('experiment_%d/new_passenger_%d.npy'% (experiment,i),temp)
return max_iteration
def compute_signif_conf_Z_list(cor_mat_file,conf_cor_mat_file,coords_file):
import rpy,os
import nibabel as nib
import numpy as np
from dmgraphanalysis.utils_cor import export_List_net_from_list,export_Louvain_net_from_list
from dmgraphanalysis.utils_cor import return_signif_conf_net_list
from dmgraphanalysis.utils_plot import plot_cormat
print "loading cor_mat_file"
cor_mat = np.load(cor_mat_file)
print "loading conf_cor_mat_file"
conf_cor_mat = np.load(conf_cor_mat_file)
print 'load coords'
coords = np.array(np.loadtxt(coords_file),dtype = int)
print "computing net_list by thresholding conf_cor_mat based on distance and net_threshold"
net_list,binary_signif_matrix = return_signif_conf_net_list(cor_mat,conf_cor_mat)
print binary_signif_matrix.shape
print "saving binary_signif_matrix"
binary_signif_matrix_file = os.path.abspath('binary_signif_matrix.npy')
np.save(binary_signif_matrix_file,binary_signif_matrix)
print "plotting binary_signif_matrix"
plot_binary_signif_matrix_file = os.path.abspath('binary_signif_matrix.eps')
plot_cormat(plot_binary_signif_matrix_file,binary_signif_matrix,list_labels = [])
## Z correl_mat as list of edges
print "saving net_list as list of edges"
net_List_file = os.path.abspath('net_List_signif_conf.txt')
export_List_net_from_list(net_List_file,net_list)
### Z correl_mat as Louvain format
print "saving net_list as Louvain format"
net_Louvain_file = os.path.abspath('net_Louvain_signif_conf.txt')
export_Louvain_net_from_list(net_Louvain_file,net_list,coords)
#net_List_file = ''
#net_Louvain_file = ''
return net_List_file, net_Louvain_file
def split_data(min_timbre=100, timbre_width=12, min_songs=4, data_file='mfcc'):
ArtistMapping, ArtistIdMapping, data = generate_data(
min_timbre=min_timbre, timbre_width=timbre_width, min_songs=min_songs)
train = numpy.zeros((0, min_timbre * timbre_width + 1))
validation = numpy.zeros((0, min_timbre * timbre_width + 1))
test = numpy.zeros((0, min_timbre * timbre_width + 1))
print 'Splitting data...'
for artist_id in ArtistIdMapping:
indices = ArtistIdMapping[artist_id][1]
valid_data = data[indices[0]].reshape(1, -1)
validation = numpy.vstack((validation, valid_data))
test_indx = 1 + max(int((len(indices) - 1) * .3), 1)
for i in range(1, test_indx):
test_data = data[indices[i]].reshape(1, -1)
test = numpy.vstack((test, test_data))
for i in range(test_indx, len(indices)):
train_data = data[indices[i]].reshape(1, -1)
train = numpy.vstack((train, train_data))
print 'Saving Data...'
numpy.save('data/' + data_file + '_songs_{}'.format(min_songs) + '_test', test, allow_pickle=True)
numpy.save('data/' + data_file + '_songs_{}'.format(min_songs) + '_train', train, allow_pickle=True)
numpy.save('data/' + data_file + '_songs_{}'.format(min_songs) + '_valid', validation, allow_pickle=True)
numpy.save('data/' + data_file + '_dict_songs_{}'.format(min_songs), ArtistMapping, allow_pickle=True)
numpy.save('data/' + data_file + '_dictId_songs_{}'.format(min_songs), ArtistIdMapping, allow_pickle=True)
return ArtistMapping, ArtistIdMapping, train, validation, test
def main():
for i in list(range(4))[::-1]:
print(i+1)
time.sleep(1)
paused = False
while(True):
if not paused:
# 800x600 windowed mode
screen = grab_screen(region=(0,40,800,640))
last_time = time.time()
screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY)
screen = cv2.resize(screen, (160,120))
# resize to something a bit more acceptable for a CNN
keys = key_check()
output = keys_to_output(keys)
training_data.append([screen,output])
if len(training_data) % 1000 == 0:
print(len(training_data))
np.save(file_name,training_data)
keys = key_check()
if 'T' in keys:
if paused:
paused = False
print('unpaused!')
time.sleep(1)
else:
print('Pausing!')
paused = True
time.sleep(1)
def pcaNIPALS(K=5,tol=1e-4,verbose=False):
''' Reference:
Section 2.2 in Andrecut, M. (2009).
Parallel GPU implementation of iterative PCA algorithms.
Journal of Computational Biology, 16(11), 1593-1599.
TODO - replace custom linear algebra (e.g. XmeanCenter) with
numpy algebra
'''
if verbose: print 'Mean centering columns'
XmeanCenter(1)
latent=[]
for k in range(K):
lam0=0;lam1=np.inf
T=np.matrix(XgetColumn(k))
if verbose: print 'Computing PC ',k
h=0
while abs(lam1-lam0)>tol and h<100:
P=Xleftmult(T,True)
P=P/np.linalg.norm(P)
T=Xleftmult(P)
lam0=lam1
lam1=np.linalg.norm(T)
if verbose: print '\t Iteration '+str(h)+', Convergence =', abs(lam1-lam0)
h+=1
latent.append(lam1)
XminusOuterProduct(T,P)
#np.save(inpath+'T%02d'%k,T)
np.save(inpath+'coeffT%d'%k,P.T)
np.save(inpath+'latent',latent)
def setUp(self):
"""Generate files"""
self.ary = numpy.random.randint(0, 6500, size=99).reshape(11, 9).astype("uint16")
self.fn = os.path.join(UtilsTest.tempdir, "numpy.npy")
self.fn2 = os.path.join(UtilsTest.tempdir, "numpy2.npy")
numpy.save(self.fn, self.ary)
def relax_system(mesh):
sim = Sim(mesh, chi=1e-3, name='relax', driver='llbar_full')
sim.driver.set_tols(rtol=1e-7, atol=1e-7)
sim.Ms = 8.0e5
sim.driver.alpha = 0.1
sim.beta = 0
sim.driver.gamma = 2.211e5
sim.set_m((1, 0.25, 0.1))
# sim.set_m(np.load('m0.npy'))
A = 1.3e-11
exch = UniformExchange(A=A)
sim.add(exch)
mT = 795.7747154594767
zeeman = Zeeman([-100 * mT, 4.3 * mT, 0], name='H')
sim.add(zeeman, save_field=True)
demag = Demag()
sim.add(demag)
ONE_DEGREE_PER_NS = 17453292.52
sim.relax(dt=1e-12, stopping_dmdt=0.01,
max_steps=5000, save_m_steps=100, save_vtk_steps=50)
np.save('m0.npy', sim.spin)
def plotForce():
figure(size=3,aspect=0.5)
subplot(1,2,1)
from EvalTraj import plotFF
plotFF(vp=351,t=28,f=900,cm=0.6,foffset=8)
subplot_annotate()
subplot(1,2,2)
for i in [1,2,3,4]:
R=np.squeeze(np.load('Rdpse%d.npy'%i))
R=stats.nanmedian(R,axis=2)[:,1:,:]
dps=np.linspace(-1,1,201)[1:]
plt.plot(dps,R[:,:,2].mean(0));
plt.legend([0,0.1,0.2,0.3],loc=3)
i=2
R=np.squeeze(np.load('Rdpse%d.npy'%i))
R=stats.nanmedian(R,axis=2)[:,1:,:]
mn=np.argmin(R,axis=1)
y=np.random.randn(mn.shape[0])*0.00002+0.0438
plt.plot(np.sort(dps[mn[:,2]]),y,'+',mew=1,ms=6,mec=[ 0.39 , 0.76, 0.64])
plt.xlabel('Displacement of Force Origin')
plt.ylabel('Average Net Force Magnitude')
hh=dps[mn[:,2]]
err=np.std(hh)/np.sqrt(hh.shape[0])*stats.t.ppf(0.975,hh.shape[0])
err2=np.std(hh)/np.sqrt(hh.shape[0])*stats.t.ppf(0.75,hh.shape[0])
m=np.mean(hh)
print m, m-err,m+err
np.save('force',[m, m-err,m+err,m-err2,m+err2])
plt.xlim([-0.5,0.5])
plt.ylim([0.0435,0.046])
plt.grid(b=True,axis='x')
subplot_annotate()
def generate( self, out_path, aux, idx_in, idx_out ):
scheme_high = amico.lut.create_high_resolution_scheme( self.scheme, b_scale = 1 )
protocolHR = self.scheme2noddi( scheme_high )
nATOMS = len(self.IC_ODs)*len(self.IC_VFs) + 1
progress = ProgressBar( n=nATOMS, prefix=" ", erase=True )
# Coupled contributions
IC_KAPPAs = 1 / np.tan(self.IC_ODs*np.pi/2)
for kappa in IC_KAPPAs:
signal_ic = self.synth_meas_watson_SH_cyl_neuman_PGSE( np.array([self.dPar*1E-6, 0, kappa]), protocolHR['grad_dirs'], np.squeeze(protocolHR['gradient_strength']), np.squeeze(protocolHR['delta']), np.squeeze(protocolHR['smalldel']), np.array([0,0,1]), 0 )
for v_ic in self.IC_VFs:
dPerp = self.dPar*1E-6 * (1 - v_ic)
signal_ec = self.synth_meas_watson_hindered_diffusion_PGSE( np.array([self.dPar*1E-6, dPerp, kappa]), protocolHR['grad_dirs'], np.squeeze(protocolHR['gradient_strength']), np.squeeze(protocolHR['delta']), np.squeeze(protocolHR['smalldel']), np.array([0,0,1]) )
signal = v_ic*signal_ic + (1-v_ic)*signal_ec
lm = amico.lut.rotate_kernel( signal, aux, idx_in, idx_out, False )
np.save( pjoin( out_path, 'A_%03d.npy'%progress.i) , lm )
progress.update()
# Isotropic
signal = self.synth_meas_iso_GPD( self.dIso*1E-6, protocolHR)
lm = amico.lut.rotate_kernel( signal, aux, idx_in, idx_out, True )
np.save( pjoin( out_path, 'A_%03d.npy'%progress.i) , lm )
progress.update()
def vectorize_and_aggregate(in_data_file_list, mask_file, matrix_name, parcellation_path, fwhm, use_diagonal,
use_fishers_z, df_file, df_col_names):
import os, pickle
import numpy as np
from LeiCA_LIFE.learning.prepare_data_utils import vectorize_ss
# get an example of the data:
#save_template: template file; for behav: col names
vectorized_data, data_type, masker, save_template = vectorize_ss(in_data_file_list[0], mask_file, matrix_name,
parcellation_path, fwhm, use_diagonal,
use_fishers_z, df_file,
df_col_names)
vectorized_data = np.zeros((len(in_data_file_list), vectorized_data.shape[1]))
vectorized_data.fill(np.nan)
for i, in_data_file_ss in enumerate(in_data_file_list):
vectorized_data[i, :], _, _, _ = vectorize_ss(in_data_file_ss, mask_file, matrix_name, parcellation_path, fwhm,
use_diagonal, use_fishers_z, df_file, df_col_names)
vectorized_aggregated_file = os.path.abspath('vectorized_aggregated_data.npy')
np.save(vectorized_aggregated_file, vectorized_data)
unimodal_backprojection_info = {'data_type': data_type,
'masker': masker,
'save_template': save_template
}
unimodal_backprojection_info_file = os.path.abspath('unimodal_backprojection_info.pkl')
pickle.dump(unimodal_backprojection_info, open(unimodal_backprojection_info_file, 'w'))
return vectorized_aggregated_file, unimodal_backprojection_info_file
def labels(self):
'''
Decide the labels
2 for unsecure Jos, 1 for secure Jos, 0 for others
'''
#TODO consider labeling different authors for multiclass assignment
fname = 'features/labels'
if self.use_saved_features and path.isfile(fname):
return np.load(fname)
with open(self._get_path('work_list/Josquin_secure.txt')) as f:
secure_jos = set(f.read().splitlines())
labels = []
for work in self._works:
label = 0
if 'Ock' in work:
label = -1
if 'Jos' in work:
if work in secure_jos:
label = 1
else:
label = 2
labels.append(label)
labels = np.array(labels, dtype=int)
if self.save_data:
np.save(fname, labels)
return labels
def __init__(
self, save_data=True,
use_saved_features=True, use_saved_npz=True):
'Init by getting all the works.'
self._self_dir = path.abspath(path.dirname(__file__))
self.use_saved_features = use_saved_features
self.use_saved_npz = use_saved_npz
self.save_data = save_data
self.npz_data = None
self._vectorizer = DictVectorizer()
if save_data and not path.isdir('features'):
os.mkdir('features')
work_fname = 'features/all_works.npy'
if use_saved_features and path.isfile(work_fname):
self._works = np.load(work_fname)
else:
works = []
with open(self._get_path('work_list/AllWorks.txt')) as f:
for line in f:
works.append(line.split('-')[0])
self._works = np.array(works)
if self.save_data:
np.save('features/all_works', self.works)
def save_medians(b_inst,save_path):
medians = b_inst.medians
poiss_comp_numbers = b_inst.poiss_comp_numbers
save_vec = [ [key, medians[ poiss_comp_numbers[key] ] ] for key in poiss_comp_numbers.keys() ]
np.save(save_path,save_vec)
def sample_lnprob(weight_index):
import emcee
ndim = 4
nwalkers = 8 * ndim
print("using {} walkers".format(nwalkers))
p0 = np.vstack((np.random.uniform(-0.5, 2, size=(1, nwalkers)),
np.random.uniform(50, 300, size=(1, nwalkers)),
np.random.uniform(0.2, 1.5, size=(1, nwalkers)),
np.random.uniform(0.2, 1.5, size=(1, nwalkers)))).T
sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, args=(weight_index,), threads=cfg['threads'])
print("Running Sampler")
pos, prob, state = sampler.run_mcmc(p0, cfg['burn_in'])
print("Burn-in complete")
sampler.reset()
sampler.run_mcmc(pos, cfg['samples'])
samples = sampler.flatchain
np.save(cfg['outdir'] + "samples_w{}.npy".format(weight_index), samples)
import triangle
fig = triangle.corner(samples)
fig.savefig(cfg['outdir'] + "triangle_w{}.png".format(weight_index))
def run(self, y, x, initial_value, num_its, altered):
"""Run the model with the arguments: y_length, x_length, initial_value, number_of_iterations, altered"""
self.initialise_grid(y, x, initial_value)
self.write_file()
self.initialise_shadow_map()
self.num_iterations = num_its
# Standard parameter values
self.jump_length = 1
self.pd_s = 0.6
self.pd_ns = 0.4
self.altered = altered
if self.altered == True:
# Create the depth grid
self.depth = np.load("Gradual_Stepped_Full.npy")
self.avcount = np.zeros(num_its + 1)
# Run the model
self.main_loop()
print self.avcount
io.savemat("Counts.mat", { "count":self.avcount})
np.save("Counts.npy", self.avcount)
def run_altered(self, depth):
x = 500
y = 200
initial_value = 3
num_its = 2000
self.initialise_grid(y, x, initial_value)
self.write_file()
self.initialise_shadow_map()
self.num_iterations = num_its
# Standard parameter values
self.jump_length = 1
self.pd_s = 0.6
self.pd_ns = 0.4
self.create_gradual_grid(depth)
self.altered = True
self.avcount = np.zeros(num_its + 1)
# Run the model
self.main_loop()
print self.avcount
io.savemat("Counts.mat", { "count":self.avcount})
np.save("Counts.npy", self.avcount)
def calculate_and_save():
energies = np.empty((B_list.size, 2**number_of_spins))
for i,B in enumerate(B_list):
print i
calc = ising_calculator_AFM(number_of_spins, alpha, B)
energies[i,:] = calc.find_energies()
np.save('energy_array', energies)
def relax_system(mesh):
sim = Sim(mesh, name='relax')
# sim.set_options(rtol=1e-10,atol=1e-14)
sim.alpha = 1.0
sim.gamma = 1.0
sim.mu_s = 1.0
sim.set_m(init_m)
# sim.set_m(random_m)
# sim.set_m(np.load('m_10000.npy'))
J = 1.0
exch = UniformExchange(J)
sim.add(exch)
D = 0.09
dmi = DMI(D)
sim.add(dmi)
zeeman = Zeeman([0, 0, 3.75e-3])
sim.add(zeeman)
sim.relax(dt=2.0, stopping_dmdt=1e-6, max_steps=1000,
save_m_steps=100, save_vtk_steps=50)
np.save('m0.npy', sim.spin)
def assertCubeDataAlmostEqual(self, cube, reference_filename, *args, **kwargs):
reference_path = self.get_result_path(reference_filename)
if self._check_reference_file(reference_path):
kwargs.setdefault("err_msg", "Reference file %s" % reference_path)
result = np.load(reference_path)
if isinstance(result, np.lib.npyio.NpzFile):
self.assertIsInstance(cube.data, ma.MaskedArray, "Cube data was not a masked array.")
# Avoid comparing any non-initialised array data.
data = cube.data.filled()
np.testing.assert_array_almost_equal(data, result["data"], *args, **kwargs)
np.testing.assert_array_equal(cube.data.mask, result["mask"])
else:
np.testing.assert_array_almost_equal(cube.data, result, *args, **kwargs)
else:
self._ensure_folder(reference_path)
logger.warning("Creating result file: %s", reference_path)
if isinstance(cube.data, ma.MaskedArray):
# Avoid recording any non-initialised array data.
data = cube.data.filled()
with open(reference_path, "wb") as reference_file:
np.savez(reference_file, data=data, mask=cube.data.mask)
else:
with open(reference_path, "wb") as reference_file:
np.save(reference_file, cube.data)
def train_word2id():
"""把训练集的所有词转成对应的id。"""
time0 = time.time()
print('Processing train data.')
df_train = pd.read_csv('../raw_data/question_train_set.txt', sep='\t', usecols=[0, 2, 4],
names=['question_id', 'word_title', 'word_content'], dtype={'question_id': object})
print('training question number %d ' % len(df_train))
# 没有 content 的问题用 title 来替换
na_content_indexs = list()
for i in tqdm(xrange(len(df_train))):
word_content = df_train.word_content.values[i]
if type(word_content) is float:
na_content_indexs.append(i)
print('There are %d train questions without content.' % len(na_content_indexs))
for na_index in tqdm(na_content_indexs):
df_train.at[na_index, 'word_content'] = df_train.at[na_index, 'word_title']
# 没有 title 的问题， 丢弃
na_title_indexs = list()
for i in xrange(len(df_train)):
word_title = df_train.word_title.values[i]
if type(word_title) is float:
na_title_indexs.append(i)
print('There are %d train questions without title.' % len(na_title_indexs))
df_train = df_train.drop(na_title_indexs)
print('After dropping, training question number(should be 2999952) = %d' % len(df_train))
# 转为 id 形式
p = Pool()
train_title = np.asarray(p.map(get_id4words, df_train.word_title.values))
np.save('../data/wd_train_title.npy', train_title)
train_content = np.asarray(p.map(get_id4words, df_train.word_content.values))
np.save('../data/wd_train_content.npy', train_content)
p.close()
p.join()
print('Finished changing the training words to ids. Costed time %g s' % (time.time() - time0))
def concat_ts(all_ts_files):
import numpy as np
import os
print all_ts_files
for i,ts_file in enumerate(all_ts_files):
## loading ROI coordinates
ts = np.load(ts_file)
#print "all_ts: "
print ts.shape
if i == 0:
concat_ts = ts.copy()
#print concat_ts.shape
else:
concat_ts = np.concatenate((concat_ts,ts),axis = 0)
#print concat_ts.shape
print concat_ts.shape
### saving time series
concat_ts_file = os.path.abspath("concat_ts.npy")
np.save(concat_ts_file,concat_ts)
return concat_ts_file
def test_word2id():
"""把测试集的所有词转成对应的id。"""
time0 = time.time()
print('Processing eval data.')
df_eval = pd.read_csv('../raw_data/question_eval_set.txt', sep='\t', usecols=[0, 2, 4],
names=['question_id', 'word_title', 'word_content'], dtype={'question_id': object})
print('test question number %d' % len(df_eval))
# 没有 title 的问题用 content 来替换
na_title_indexs = list()
for i in xrange(len(df_eval)):
word_title = df_eval.word_title.values[i]
if type(word_title) is float:
na_title_indexs.append(i)
print('There are %d test questions without title.' % len(na_title_indexs))
for na_index in na_title_indexs:
df_eval.at[na_index, 'word_title'] = df_eval.at[na_index, 'word_content']
# 没有 content 的问题用 title 来替换
na_content_indexs = list()
for i in tqdm(xrange(len(df_eval))):
word_content = df_eval.word_content.values[i]
if type(word_content) is float:
na_content_indexs.append(i)
print('There are %d test questions without content.' % len(na_content_indexs))
for na_index in tqdm(na_content_indexs):
df_eval.at[na_index, 'word_content'] = df_eval.at[na_index, 'word_title']
# 转为 id 形式
p = Pool()
eval_title = np.asarray(p.map(get_id4words, df_eval.word_title.values))
np.save('../data/wd_eval_title.npy', eval_title)
eval_content = np.asarray(p.map(get_id4words, df_eval.word_content.values))
np.save('../data/wd_eval_content.npy', eval_content)
p.close()
p.join()
print('Finished changing the eval words to ids. Costed time %g s' % (time.time() - time0))
def main(root='/tmp/measurements', output=None):
data = []
for s in os.listdir(root):
subject = []
for b in os.listdir(os.path.join(root, s)):
block = []
bweight, bspeed, bhand, bpaths = b.split('-')[1:]
for t in os.listdir(os.path.join(root, s, b)):
thand, tspeed = re.search(r'(left|right)-speed_(\d\.\d+)', t).groups()
config = np.tile([
C[bweight], C[bspeed], C[bhand], C[bpaths],
C[thand], float(tspeed)], (120, 1))
block.append(
np.hstack([
config,
np.loadtxt(os.path.join(root, s, b, t),
skiprows=1, delimiter=',')]))
subject.append(block)
if len(subject) == 3:
data.append(subject)
else:
print('incorrect block count! discarding {}'.format(s))
data = np.array(data)
logging.info('loaded data %s', data.shape)
if output:
np.save(output, data.astype('f'))
def get_buffer_callback(overviewBuffers,overflow,triggeredAt,triggered,auto_stop,nValues):
#print('Callback for saving to disk')
#create filename based on actual timestamp
#filename = time.strftime("%Y%m%d_%H_%M_%S_%f.csv")
filename=datetime.datetime.now()
filename= filename.strftime("%Y%m%d_%H_%M_%S_%f")
CH1='CH1_' + filename
#CH2='CH2_' + filename
#cast 2d-pointer from c- callback into python pointer
ob = ctypes.cast(overviewBuffers,ctypes.POINTER(ctypes.POINTER(ctypes.c_short)))
#create array from pointer data ob[0]-> CH1 ob[1]-> CH2
streamed_data_CH1=np.fromiter(ob[0], dtype=np.short, count=nValues)
#streamed_data_CH2=np.fromiter(ob[1], dtype=np.short, count=nValues)
#save array data into numpy fileformat
path1 = os.path.normpath('C:\\Users\ckattmann\Documents\GitHub\pqpico\Data')+'/'+CH1
#path2 = os.path.normpath('C:\\Users\ckattmann\Documents\GitHub\pqpico\Data')+'/'+CH2
np.save(path1,streamed_data_CH1)
#np.save(path2,streamed_data_CH2)
#print('File saved:',CH1,CH2)
return 0
def consolidate_games(self, name, samples):
print('>>> Creating consolidated numpy arrays')
if self.use_generator:
print('>>> Return generator')
generator = DataGenerator(self.data_dir, samples)
return generator
files_needed = set(file_name for file_name, index in samples)
print('>>> Total number of files: ' + str(len(files_needed)))
file_names = []
for zip_file_name in files_needed:
file_name = zip_file_name.replace('.zip', '') + name
file_names.append(file_name)
feature_list = []
label_list = []
for file_name in file_names:
X = np.load(self.data_dir + '/' + file_name + '_features.npy')
y = np.load(self.data_dir + '/' + file_name + '_labels.npy')
feature_list.append(X)
label_list.append(y)
print('>>> Done')
features = np.concatenate(feature_list, axis=0)
labels = np.concatenate(label_list, axis=0)
feature_file = self.data_dir + '/' + str(self.num_planes) + '_plane_features_' + name
label_file = self.data_dir + '/' + str(self.num_planes) + '_plane_labels_' + name
np.save(feature_file, features)
np.save(label_file, labels)
return features, labels
def Cluster(param, DATA_FOLDER):
ts = ListaSet(param)
Data = scipy.io.loadmat('./data/filter_template3.mat')
Filter2 = np.rot90( Data['Filter2'], 2)
# corList = []
# TVList= []
corArr = np.empty(ts.get_num_images())
TVArr = np.empty(ts.get_num_images())
for i in range( ts.get_num_images()):
# for i in range( 100):
tmp = ts.get_input(i)
tmp2 = tmp - np.mean( tmp)
tmp3 = tmp2 / np.linalg.norm(tmp2, 'fro')
# Cor = scipy.signal.convolve2d(tmp3, Filter2, 'valid')
# corList.append( Cor)
corArr[i] = scipy.signal.convolve2d(tmp3, Filter2, 'valid')
dx = scipy.ndimage.sobel(tmp, 0)
dy = scipy.ndimage.sobel(tmp, 1)
mag = np.hypot(dx, dy)
# TVList.append( np.sum(mag))
TVArr[i] = np.sum(mag)
if i % 10000 == 0:
print i
np.save( DATA_FOLDER+'/trainCorrelation.npy', corArr)
np.save( DATA_FOLDER+'/trainTotalVariation.npy', TVArr)
return
def convert_single_propagators(files):
"Construct pion correlators from individual overlap propagators."
# Some basic checks on the input.
head0, config0, middle0, mass0 = files[0].split('.')
for f in files:
check_length(f)
head, config, middle, mass = f.split('.')
if (head != head0) or (middle != middle0) or (mass != mass0):
print "You might not want to combine these!"
return 1
# Construct the block of correlators.
correlators = []
for f in files:
print f
correlators.append(pion_correlator(f))
correlators = np.array(correlators)
# Basic checks on the output.
print correlators.shape
assert (len(files), nt) == correlators.shape
# Write output.
m = float('0.'+mass0)
np.save(correlator_name(m), correlators)
print correlators[0]
def create_test_data(patient):
testing_data = []
img, origin, spacing = load_itk("./MHA/" + patient + ".mha")
RESIZE_SPACING = [1, 1, 1]
resize_factor = spacing / RESIZE_SPACING
new_real_shape = img.shape * resize_factor
new_shape = np.round(new_real_shape)
real_resize = new_shape / img.shape
new_spacing = spacing / real_resize
lung_img = np.load("./images1/" + patient + ".npy")
world_pos, world_neg = generator(patient)
for i in range(world_pos.shape[0]):
voxel = world_2_voxel(world_pos[i, :][::-1], origin, spacing)
voxel = np.round(voxel).astype('int')
voxel = np.round(voxel * resize_factor).astype('int')
z = voxel[0]
y = voxel[1]
x = voxel[2]
t1_slice = lung_img[z - 16:z + 16, y - 16:y + 16, x - 16:x + 16].copy()
t1_slice = resize_image_with_crop_or_pad(t1_slice, [32, 32, 32],
mode='symmetric')
print("Processing Positive ", patient)
print(t1_slice.shape, z, y, x)
# Add a feature dimension and normalise
t1_norm = np.expand_dims(normalise_one_one(t1_slice), axis=-1)
# Randomly flip the image along axis 1
t1_flipped1 = flip1(t1_norm.copy(), axis=2)
# Add a Gaussian offset (independently for each channel)
t1_offset1 = add_gaussian_offset(t1_norm.copy(), sigma=0.1)
t1_offset2 = add_gaussian_offset(t1_norm.copy(), sigma=0.5)
# Add Gaussian noise
t1_noise1 = add_gaussian_noise(t1_norm.copy(), sigma=0.02)
t1_noise2 = add_gaussian_noise(t1_norm.copy(), sigma=0.05)
# Elastic transforms according to:
# [1] Simard, Steinkraus and Platt, "Best Practices for Convolutional
# Neural Networks applied to Visual Document Analysis", in Proc. of the
# International Conference on Document Analysis and Recognition, 2003.
t1_trans_low_s = elastic_transform(t1_norm.copy(),
alpha=[1, 1e4, 1e4],
sigma=[1, 8, 8])
t1_trans_high_s = elastic_transform(t1_norm.copy(),
alpha=[1, 6e4, 6e4],
sigma=[1, 16, 16])
testing_data.append([np.array(t1_norm), np.array([0, 1])])
testing_data.append([np.array(t1_flipped1), np.array([0, 1])])
testing_data.append([np.array(t1_offset1), np.array([0, 1])])
testing_data.append([np.array(t1_offset2), np.array([0, 1])])
testing_data.append([np.array(t1_noise1), np.array([0, 1])])
testing_data.append([np.array(t1_noise2), np.array([0, 1])])
testing_data.append([np.array(t1_trans_low_s), np.array([0, 1])])
testing_data.append([np.array(t1_trans_high_s), np.array([0, 1])])
util = world_neg.shape[0]
if (world_neg.shape[0] > (2 * world_pos.shape[0])):
util = 2 * world_pos.shape[0]
for i in range(util):
voxel = world_2_voxel(world_neg[i, :][::-1], origin, spacing)
voxel = np.round(voxel).astype('int')
voxel = np.round(voxel * resize_factor).astype('int')
z = voxel[0]
y = voxel[1]
x = voxel[2]
t1_slice = lung_img[z - 16:z + 16, y - 16:y + 16, x - 16:x + 16].copy()
t1_slice = resize_image_with_crop_or_pad(t1_slice, [32, 32, 32],
mode='symmetric')
print("Processing Negative", patient)
print(t1_slice.shape, z, y, x)
# Add a feature dimension and normalise
t1_norm = np.expand_dims(normalise_one_one(t1_slice), axis=-1)
# Randomly flip the image along axis 1
t1_flipped1 = flip1(t1_norm.copy(), axis=2)
# Add a Gaussian offset (independently for each channel)
t1_offset1 = add_gaussian_offset(t1_norm.copy(), sigma=0.1)
# Add Gaussian noise
t1_noise1 = add_gaussian_noise(t1_norm.copy(), sigma=0.02)
# Elastic transforms according to:
# [1] Simard, Steinkraus and Platt, "Best Practices for Convolutional
# Neural Networks applied to Visual Document Analysis", in Proc. of the
# International Conference on Document Analysis and Recognition, 2003.
t1_trans_low_s = elastic_transform(t1_norm.copy(),
alpha=[1, 1e4, 1e4],
sigma=[1, 8, 8])
testing_data.append([np.array(t1_norm), np.array([1, 0])])
testing_data.append([np.array(t1_flipped1), np.array([1, 0])])
testing_data.append([np.array(t1_offset1), np.array([1, 0])])
testing_data.append([np.array(t1_noise1), np.array([1, 0])])
testing_data.append([np.array(t1_trans_low_s), np.array([1, 0])])
print("#############################")
print()
shuffle(testing_data)
np.save(testing_data_output_path + patient + ".npy", testing_data)
X = np.array([i[0] for i in testing_data]).reshape(-1, 32, 32, 32, 1)
Y = np.array([i[1] for i in testing_data])
print(X.shape, Y.shape)
def cal_cosod_matrics(
data_type: str = "rgb_sod",
txt_path: str = "",
to_append: bool = True,
xlsx_path: str = "",
drawing_info: dict = None,
dataset_info: dict = None,
save_npy: bool = True,
curves_npy_path: str = "./curves.npy",
metrics_npy_path: str = "./metrics.npy",
num_bits: int = 3,
):
"""
Save the results of all models on different datasets in a `npy` file in the form of a
dictionary.
::
{
dataset1:{
method1:[fm, em, p, r],
method2:[fm, em, p, r],
.....
},
dataset2:{
method1:[fm, em, p, r],
method2:[fm, em, p, r],
.....
},
....
}
:param data_type: the type of data
:param txt_path: the path of the txt for saving results
:param to_append: whether to append results to the original record
:param xlsx_path: the path of the xlsx file for saving results
:param drawing_info: the method information for plotting figures
:param dataset_info: the dataset information
:param save_npy: whether to save results into npy files
:param curves_npy_path: the npy file path for saving curve data
:param metrics_npy_path: the npy file path for saving metric values
:param num_bits: the number of bits used to format results
"""
curves = defaultdict(dict) # Two curve metrics
metrics = defaultdict(dict) # Six numerical metrics
txt_recoder = TxtRecorder(
txt_path=txt_path,
to_append=to_append,
max_method_name_width=max([len(x) for x in drawing_info.keys()]), # 显示完整名字
)
excel_recorder = MetricExcelRecorder(
xlsx_path=xlsx_path,
sheet_name=data_type,
row_header=["methods"],
dataset_names=sorted(list(dataset_info.keys())),
metric_names=["sm", "wfm", "mae", "adpf", "avgf", "maxf", "adpe", "avge", "maxe"],
)
for dataset_name, dataset_path in dataset_info.items():
txt_recoder.add_row(row_name="Dataset", row_data=dataset_name, row_start_str="\n")
# 获取真值图片信息
gt_info = dataset_path["mask"]
gt_root = gt_info["path"]
gt_ext = gt_info["suffix"]
# 真值名字列表
gt_index_file = dataset_path.get("index_file")
if gt_index_file:
gt_name_list = get_name_with_group_list(data_path=gt_index_file, file_ext=gt_ext)
else:
gt_name_list = get_name_with_group_list(data_path=gt_root, file_ext=gt_ext)
assert len(gt_name_list) > 0, "there is not ground truth."
# ==>> test the intersection between pre and gt for each method <<==
for method_name, method_info in drawing_info.items():
method_root = method_info["path_dict"]
method_dataset_info = method_root.get(dataset_name, None)
if method_dataset_info is None:
colored_print(
msg=f"{method_name} does not have results on {dataset_name}", mode="warning"
)
continue
# 预测结果存放路径下的图片文件名字列表和扩展名称
pre_ext = method_dataset_info["suffix"]
pre_root = method_dataset_info["path"]
pre_name_list = get_name_with_group_list(data_path=pre_root, file_ext=pre_ext)
# get the intersection
eval_name_list = sorted(list(set(gt_name_list).intersection(set(pre_name_list))))
num_names = len(eval_name_list)
if num_names == 0:
colored_print(
msg=f"{method_name} does not have results on {dataset_name}", mode="warning"
)
continue
grouped_data = group_names(names=eval_name_list)
num_groups = len(grouped_data)
colored_print(
f"Evaluating {method_name} with {num_names} images and {num_groups} groups"
f" (G:{len(gt_name_list)},P:{len(pre_name_list)}) images on dataset {dataset_name}"
)
group_metric_recorder = GroupedMetricRecorder()
inter_group_bar = tqdm(
grouped_data.items(),
total=num_groups,
leave=False,
ncols=79,
desc=f"[{dataset_name}]",
)
for group_name, names_in_group in inter_group_bar:
intra_group_bar = tqdm(
names_in_group,
total=len(names_in_group),
leave=False,
ncols=79,
desc=f"({group_name})",
)
for img_name in intra_group_bar:
img_name_with_group = os.path.join(group_name, img_name)
gt, pre = get_gt_pre_with_name(
gt_root=gt_root,
pre_root=pre_root,
img_name=img_name_with_group,
pre_ext=pre_ext,
gt_ext=gt_ext,
to_normalize=False,
)
group_metric_recorder.update(group_name=group_name, pre=pre, gt=gt)
method_results = group_metric_recorder.show(num_bits=num_bits, return_ndarray=False)
method_curves = method_results["sequential"]
method_metrics = method_results["numerical"]
curves[dataset_name][method_name] = method_curves
metrics[dataset_name][method_name] = method_metrics
excel_recorder(
row_data=method_metrics, dataset_name=dataset_name, method_name=method_name
)
txt_recoder(method_results=method_metrics, method_name=method_name)
if save_npy:
make_dir(os.path.basename(curves_npy_path))
np.save(curves_npy_path, curves)
np.save(metrics_npy_path, metrics)
colored_print(f"all methods have been saved in {curves_npy_path} and {metrics_npy_path}")
formatted_string = formatter_for_tabulate(metrics)
colored_print(f"all methods have been tested:\n{formatted_string}")
def add_labels(self, labels):
if self.print_progress:
print('%-40s\r' % 'Saving labels...', end='', flush=True)
assert labels.shape[0] == self.cur_images
with open(self.tfr_prefix + '-rxx.labels', 'wb') as f:
np.save(f, labels.astype(np.float32))
# vectorize
cross_corr = np.corrcoef(sub_FS.T)
tril_inds = np.tril_indices_from(cross_corr, k=-1)
cc_ravel = cross_corr[tril_inds]
# cross_corrs.append(cc_ravel)
cur_FS[i_rs] = cc_ravel
# plt.matshow(cross_corr, cmap=plt.cm.RdBu)
# plt.title(rs_base)
# plt.yticks(np.arange(len(roi_names)), roi_names)
# plt.show()
# plt.colorbar()
if i_rs % 10 == 0:
np.save('dump_FS_corrs', arr=cur_FS)
# dump the engineered feature space and corresponding meta-info
cur_FS = np.nan_to_num(cur_FS)
joblib.dump((cur_FS, niipaths_all, labels_all, sub_ids_all),
'all_FS_paths_labels_subids',
compress=9)
stuff = joblib.load('all_FS_paths_labels_subids')
FS, paths, labels, sub_ids = stuff
# trim cohorts by age and by gender (only males)
import pandas
meta_autism = pandas.read_excel('Finalinput_ABIDE_ASD_TD_MALE_BELOW21.xlsx')
meta_adhd = pandas.read_excel('Finalinput_ADHD200_ADHD_TD_MALE_BELOW21.xlsx')
def main(args):
if len(args) != 1:
sys.exit("Usage: python origianl.py <graphml file>")
C=3
L=6 #损失值
select_strength=1 #选择强度
fname = args[0]
G = networkx.read_graphml(fname)
G = networkx.convert_node_labels_to_integers(G)
'''
给每个参与者初始化投资额，利用之前initial_investment.npy文件
'''
a=[]
defender = [Defender(i) for i in range(0, len(G))]
for i in range(len(G)):
defender[i].set_degree(G.degree()[i])
for i in range(len(G)):
if defender[i].get_degree()<2:
defender[i].set_investment(random.uniform(0,0.2))
elif defender[i].get_degree()<4 and defender[i].get_degree()>=2:
defender[i].set_investment(random.uniform(0.2,0.4))
elif defender[i].get_degree()<6 and defender[i].get_degree()>=4:
defender[i].set_investment(random.uniform(0.4,0.6))
elif defender[i].get_degree()<8 and defender[i].get_degree()>=6:
defender[i].set_investment(random.uniform(0.6,0.8))
elif defender[i].get_degree()>=8:
defender[i].set_investment(random.uniform(0.8,1))
a.append(defender[i].get_investment())
numpy.save('initial_investment.npy',a)
for i in range(len(G)):
sum_weight=0.0+defender[i].get_degree()
for j in neighbors(G,i):
sum_weight+=defender[j].get_degree()
defender[i].set_weight((defender[i].get_degree())/sum_weight)
#pdb.set_trace()
'''开始演化博弈'''
for i in range(300):
#计算每个参与者的风险和收益
for j in range(0,len(G)):
sum=0
for k in neighbors(G,j):
sum = sum+defender[k].get_weight()*defender[k].get_investment()
defender[j].set_risk(math.exp(-sum-defender[j].get_weight()*defender[j].get_investment()))
defender[j].set_payoff(-C*defender[j].get_investment()-L*defender[j].get_risk())
if i==0:
pp=0
for jo in range(0,len(G)):
pp=pp+defender[jo].get_payoff()
print pp
'''更新策略'''
for jj in range(0,len(G)):
jjj=random_neighbor(G, jj)
if defender[jj].get_payoff() < defender[jjj].get_payoff():
imitation_probabily=1.0/(1+math.exp((-select_strength)*(defender[jjj].get_payoff()-defender[jj].get_payoff())))
if random.random() <= imitation_probabily:
defender[jj].set_investment_update(defender[jjj].get_investment())
#defender[jj].set_investment_update(((defender[jj].get_degree()+0.1)/defender[jjj].get_degree()+0.1)*defender[jjj].get_investment())
else:
defender[jj].set_investment_update((defender[jj]).get_investment())
else:
(defender[jj]).set_investment_update((defender[jj]).get_investment())
for jjjj in range(0,len(G)):
defender[jjjj].set_investment(defender[jjjj].get_investment_update())
'''
for j in range(0,len(G)):
print defender[j].get_investment()
'''
p=0
for j in range(0,len(G)):
p=p+defender[j].get_payoff()
print p
ph_vol = ph_vol[:, offset:-offset, offset:-offset]
# psnr and pearsonr correlation
mse_score = np.mean(np.square(pr_vol - gt_vol))
psnr_score = calculate_psnr(pr_vol, gt_vol)
cor_score = calculate_pearsonr(pr_vol, gt_vol)
mse_scores.append(mse_score)
psnr_scores.append(psnr_score)
cor_scores.append(cor_score)
# save prediction
pred_save = True
if pred_save:
pr_vol_dir = model_folder + '/pred_vols'
generate_folder(pr_vol_dir)
np.save(os.path.join(pr_vol_dir, 'Pr_{}.npy'.format(vol_fn)),
pr_vol)
np.save(os.path.join(pr_vol_dir, 'GT_{}.npy'.format(vol_fn)),
gt_vol)
print(pr_vol.shape)
print('{}: psnr {:.4f}, cor {:.4f}, mse {:.4f}\n'.format(
vol_fn, psnr_score, cor_score, mse_score))
# save prediction examples
prediction_dir = model_folder + '/pred_examples'
generate_folder(prediction_dir)
plot_fig_file = prediction_dir + '/{}.png'.format(vol_fn)
z_index = 158
x_index = 250
plot_prediction_zx(plot_fig_file, ph_vol, gt_vol, pr_vol, z_index,
x_index)
def save_weight(self, dir, name):
#print 'weight saved: ' + name
np.save(dir + name + '.npy', self.val.get_value())
count=int(width) * int(height),
offset=len(header)).reshape((int(height), int(width)))
# fullFile = '/home/nick/data/1pdata/imag003/20181217_noiseburst/fc2_save_2018-12-17-121550-0000.pgm'
dataDir = '/home/nick/data/1pdata/imag003/20181217_noiseburst/'
dataFiles = sorted(os.listdir(dataDir))
#Only load every 4th frame
skipBy = 4
im = read_pgm(os.path.join(dataDir, dataFiles[0]))
indsToRead = range(0, len(dataFiles), skipBy)
imgArr = np.empty((len(indsToRead), im.shape[0], im.shape[1]))
for indFrame, indFile in enumerate(indsToRead):
im = read_pgm(os.path.join(dataDir, dataFiles[indFile]))
imgArr[indFrame, :, :] = im
#Save out intermediate data
saveDir = '/home/nick/data/1pdata/tmpData'
outputFn = 'imag003_noiseburst_skipby{}.npy'.format(skipBy)
np.save(os.path.join(saveDir, outputFn), imgArr.astype('uint8'))
# im = read_pgm(fullFile)
# plt.clf()
# plt.imshow(im)
# plt.show()
sess = tf.InteractiveSession()
caption_generator = Caption_Generator(
n_words=n_words,
dim_embed=dim_embed,
dim_ctx=dim_ctx,
dim_hidden=dim_hidden,
n_lstm_steps=maxlen,
batch_size=batch_size,
ctx_shape=ctx_shape)
context, generated_words, logit_list, alpha_list = caption_generator.build_generator(maxlen=maxlen)
saver = tf.train.Saver()
saver.restore(sess, model_path)
generated_word_index = sess.run(generated_words, feed_dict={context: feat})
alpha_list_val = sess.run(alpha_list, feed_dict={context: feat})
generated_words = [ixtoword[x[0]] for x in generated_word_index]
punctuation = np.argmax(np.array(generated_words) == '.') + 1
generated_words = generated_words[:punctuation]
alpha_list_val = alpha_list_val[:punctuation]
return generated_words, alpha_list_val
#train()
a,b = test()
f = open('test.txt','w')
sep = '\t'
f.write(sep.join(a))
np.save('attention.npy',b)
# Need to modify the word list as well
wordList.append('<pad>')
wordList.append('<EOS>')
vocabSize = vocabSize + 2
if (os.path.isfile('Seq2SeqXTrain.npy')
and os.path.isfile('Seq2SeqYTrain.npy')):
xTrain = np.load('Seq2SeqXTrain.npy')
yTrain = np.load('Seq2SeqYTrain.npy')
print 'Finished loading training matrices'
numTrainingExamples = xTrain.shape[0]
else:
numTrainingExamples, xTrain, yTrain = createTrainingMatrices(
'conversationDictionary.npy', wordList, maxEncoderLength)
np.save('Seq2SeqXTrain.npy', xTrain)
np.save('Seq2SeqYTrain.npy', yTrain)
print 'Finished creating training matrices'
tf.reset_default_graph()
# Create the placeholders
encoderInputs = [
tf.placeholder(tf.int32, shape=(None, )) for i in range(maxEncoderLength)
]
decoderLabels = [
tf.placeholder(tf.int32, shape=(None, )) for i in range(maxDecoderLength)
]
decoderInputs = [
tf.placeholder(tf.int32, shape=(None, )) for i in range(maxDecoderLength)
]
all_data = open("data/all_data.txt", "r", encoding="utf-8").read()
bucket_structure, data_count = prepare_parameters(hParams, all_data)
# Load Embedding model.
if hParams.embedding_use_pretrained:
model_embedding, embedding_matrix, keyed_vector = embedding_load("model/EmbeddingModel", hParams, all_data)
else:
model_embedding, embedding_matrix, keyed_vector = embedding_train("model/EmbeddingModel", hParams, all_data)
VOCAB = list(keyed_vector.vocab.keys())
vocab_length = len(VOCAB)
# Create data matrices.
encX, decX, decy = create_matrix(all_data, VOCAB, bucket_structure, hParams, model_embedding)
# Save to disk.
vocabF = open("data/vocab.txt", "w", encoding="utf-8")
for word in VOCAB:
vocabF.write(word + "\n")
vocabF.close()
np.save("data/encX", encX)
np.save("data/decX", decX)
np.save("data/decy", decy)
print("Example count:", len(apply_filter(all_data.split("\n"))))
print("Vocabulary length:", vocab_length)
print("Matrix sizes:")
print("encX", encX.shape)
print("decX", decX.shape)
print("decy", decy.shape)
c_p = 10000
for i in range(iteration):
_,cost_val = sess.run([optimizer,cost],feed_dict = {x:s,f:f_ref})
# print('fs={}'.format(fs))
if abs(cost_val-c_p) <=0.0000000000001:
break
c_p = cost_val
print('iteration {}: {}'.format(i,cost_val))
w_v,c_v,h_v = sess.run([w,c,h])
# In[ ]:
np.save('Weights/Joint{}/w.npy'.format(joint_select),w_v)
np.save('Weights/Joint{}/c.npy'.format(joint_select),c_v)
np.save('Weights/Joint{}/h.npy'.format(joint_select),h_v)
# In[ ]:
# In[ ]:
_loss = keras.losses.BinaryCrossentropy()
return _loss(y_true, tf.nn.sigmoid(y_pred))
elif loss_func == 'xent':
_loss = keras.losses.SparseCategoricalCrossentropy()
return _loss(y_true, tf.nn.softmax(y_pred))
elif loss_func == 'balance':
y_true[y_true == 0] = -1
return -1 * np.sum(y_true * (y_pred - .5))
return loss
print(model_dir)
model = keras.models.load_model(model_dir + '.h5',
custom_objects={
'custom_loss': custom_loss(),
'loss': custom_loss()
},
compile=False)
output = model.predict(x_test, batch_size=eval_batch_size)
nat_labels = np.zeros(output.shape).astype(np.float32)
nat_labels[output >= 0.5] = 1.
nat_dists = np.sum(np.absolute(nat_labels - y_test), axis=-1)
nat_acc = np.mean(nat_dists == 0)
print('natural: {:.2f}%'.format(100 * nat_acc))
np.save(
'preds/pred_{}_{}'.format(model_dir.split('/')[1],
dataset.split('/')[-1]), output)
def create_dataset_for_one_song(
song_name,
wav_path,
y,
sr,
k,
idx,
crop_size=6000,
beat_dir_path="/n/work1/ooyama/dataset/rwc4-4/annotation/fixed_popular_beat/",
):
for shift in tqdm(range(-12, 13), desc=song_name):
shifted_y = y if shift == 0 else pitch_shift(y, sr, n_steps=shift)
save_name1 = song_name if shift == 0 else f"{song_name}shift{shift}"
for stretch_i in range(6): # stretch 5 times randomly
if stretch_i == 0:
save_name2 = f"{save_name1}original" if shift == 0 else save_name1
beat_path = f"{beat_dir_path}{song_name}.BEAT.TXT"
melspec = convertAudio2MelSpec(wav_path) # 100Hz
activation, downbeats = convertBeatText2Activation(
beat_path, song_length=len(melspec), units="ms"
)
bpm = convertBeatText2Bpm(beat_path, len(melspec))
max_bpm = np.max(bpm)
else:
max_rate = 300 / (max_bpm + 10)
stretch_rate = random.choice(np.arange(0.5, max_rate, 0.05))
rounded_rate = np.round(stretch_rate, decimals=2)
save_name2 = f"{save_name1}stretch{stretch_i}x{rounded_rate}"
stretched_y = time_stretch(shifted_y, stretch_rate)
melspec = convertAudio2MelSpec(None, True, stretched_y, sr) # 100Hz
stretched_activation = stretch_beat(activation, stretch_rate)
stretched_bpm = stretch_bpm(bpm, stretch_rate)
stretched_downbeats = (
(np.rint(downbeats / stretch_rate)).astype(np.int64)
)
beattheta = activation2beattheta(
activation if stretch_i == 0 else stretched_activation
)
bartheta = activation2bartheta(
activation if stretch_i == 0 else stretched_activation,
downbeats if stretch_i == 0 else stretched_downbeats,
)
assert np.max(bartheta) > 0.5
assert np.max(beattheta) > 0.5
features = [
["melspec", melspec],
["activation", activation if stretch_i == 0 else stretched_activation],
["bpm", bpm if stretch_i == 0 else stretched_bpm],
["beattheta", beattheta],
["bartheta", bartheta],
]
for feature in features:
if feature[0] == "bpm":
feature[1][feature[1] > 300] = 300
feature[1] = np.ascontiguousarray(feature[1])
feature_length = len(feature[1])
cropped_features = (
[feature[1]]
if feature_length < crop_size
else librosa.util.frame(
x=feature[1],
frame_length=crop_size,
hop_length=crop_size,
axis=0,
)
)
for index, cropped_feature in enumerate(cropped_features):
cropped_feature[cropped_feature < 0] = 0
fname = feature[0]
if fname == "activation":
fname = "beat"
elif fname == "melspec":
fname = f"melspec_sr{sr}_nfft1024"
path = gen_path(k, "all", fname)
np.save(
f"{path}{save_name2}-{str(index)}.{feature[0]}",
cropped_feature,
)
if idx > 25 and (idx - 1) % 25 < 5:
path = gen_path(k, "valid", fname)
np.save(
f"{path}{save_name2}-{str(index)}.{feature[0]}",
cropped_feature,
)
else:
path = gen_path(k, "train", fname)
np.save(
f"{path}{save_name2}-{str(index)}.{feature[0]}",
cropped_feature,
)
def generate_masks(model, train_iter):
total_epoch_loss = 0
total_epoch_acc = 0
model.eval()
to_write_all=[]
max_length=0
print("length of training set if",len(val_iter))
with torch.no_grad():
for idx, batch in enumerate(tqdm(val_iter)):
text = batch.text[0]
q=text[0]
text_numpy=text.numpy()
n=batch.text[1][0].item()
mq=[]
q_len=text[0].size()[0]
mq.append(q.clone())
mask = torch.ones(n)
mask = mask.long()
for i in range(n):
mask[i] = 0
for j in range(i+1,n):
mask[j] = 0
m = torch.mul(mask,q[0:n])
temp = []
temp.append(m.clone())
temp2 = torch.zeros(q_len-n)
temp.append(temp2.long())
temp = torch.cat(temp)
mq.append(temp.clone())
mask[j] = 1
mask[i] = 1
mq=torch.stack(mq)
target = batch.label
target = torch.autograd.Variable(target).long()
if torch.cuda.is_available():
text = text.cuda()
target = target.cuda()
attMax = torch.ones(1,mq.size()[1])
max_prediction=-100000
for txt in mq:
txt=txt.view(1,-1)
prediction ,q_att= model(txt)
if(prediction[0][target[0].item()].item()>max_prediction):
max_prediction=prediction[0][target[0].item()].item()
attMax=q_att
attMax_numpy=attMax.numpy()
to_write=[]
to_write.append(text_numpy)
to_write.append(attMax_numpy)
to_write = np.asarray(to_write)
loss = loss_fn(prediction, target)
num_corrects = (torch.max(prediction, 1)[1].view(target.size()).data == target.data).sum()
acc = 100.0 * num_corrects/len(batch)
total_epoch_loss += loss.item()
total_epoch_acc += acc.item()
to_write_all.append(to_write)
to_write_all=np.array(to_write_all)
np.save("question_attention_masks",to_write_all)
return total_epoch_loss/len(val_iter), total_epoch_acc/len(val_iter)
def run_analysis(X,
Y,
T_pi_vals,
dim_vals,
offset_vals,
res_name,
num_cv_folds,
decoding_window,
args,
n_init=1,
verbose=False,
index=1):
# X: 1363 * 30
# Y: 1363 * 30
use_gpu = True
device = torch.device("cuda:{}".format(args.gpuid))
results_size = (num_cv_folds, len(dim_vals), len(offset_vals),
len(T_pi_vals) + 2)
results = np.zeros(results_size) # 5*4*4*12
min_std = 1e-6
good_cols = (X.std(axis=0) > min_std)
X = X[:, good_cols]
# loop over CV folds
cv = CrossValidate(X, Y, num_cv_folds, stack=False)
for X_train, X_test, Y_train, Y_test, fold_idx in cv:
if fold_idx: break
if verbose:
print("fold", fold_idx + 1, "of", num_cv_folds)
# mean-center X and Y
X_mean = np.concatenate(X_train).mean(axis=0, keepdims=True)
X_train_ctd = [Xi - X_mean for Xi in X_train]
X_test_ctd = X_test - X_mean
Y_mean = np.concatenate(Y_train).mean(axis=0, keepdims=True)
Y_train_ctd = [Yi - Y_mean for Yi in Y_train]
Y_test_ctd = Y_test - Y_mean
# chunk
chunk_size = 500
X_train_seqs, L_train = chunk_long_seq(X_train_ctd[0], 30, chunk_size)
X_test_seqs, L_test = [X_test_ctd], [X_test_ctd.shape[0]]
# compute cross-cov mats for DCA
T_max = 2 * np.max(T_pi_vals)
# loop over dimensionalities
for dim_idx in range(len(dim_vals)):
dim = dim_vals[dim_idx]
if verbose:
print("dim", dim_idx + 1, "of", len(dim_vals))
# loop over T_pi vals
for T_pi_idx in range(len(T_pi_vals)):
T_pi = T_pi_vals[T_pi_idx]
idim = X_test_ctd.shape[-1]
fdim = dim
T = T_pi
params = 'obj={}_encoder={}_fdim={}_context={}_T={}_lr={}_bs={}_dropout={}_rate-lambda={}_ortho-lambda={}_recon-lambda={}_seed={}'.format(
args.obj, args.encoder_type, args.fdim, args.input_context,
args.T, args.lr, args.batchsize, args.dropout,
args.rate_lambda, args.ortho_lambda, args.recon_lambda,
args.seed)
dapc_model = DAPC(args.obj,
idim,
fdim,
T,
encoder_type=args.encoder_type,
ortho_lambda=args.ortho_lambda,
recon_lambda=args.recon_lambda,
dropout=args.dropout,
masked_recon=args.masked_recon,
args=args,
device=device)
dapc_model = fit_dapc(dapc_model,
X_train_seqs,
L_train,
X_test_seqs,
L_test,
None,
args.lr,
use_gpu,
batch_size=args.batchsize,
max_epochs=args.epochs,
device=device,
snapshot=params + ".cpt",
use_writer=False,
pred_data=[
X_train_ctd[0], Y_train_ctd[0],
X_test_ctd, Y_test_ctd,
decoding_window, offset_vals[-1],
args
])
# compute DCA R2 over offsets
X_train_dapc = dapc_model.encode(
torch.from_numpy(X_train_ctd[0]).to(
device, dtype=dapc_model.dtype)).cpu().numpy()
X_test_dapc = dapc_model.encode(
torch.from_numpy(X_test_ctd).to(
device, dtype=dapc_model.dtype)).cpu().numpy()
for offset_idx in range(len(offset_vals)):
offset = offset_vals[offset_idx]
r2_dapc = linear_decode_r2(X_train_dapc,
Y_train_ctd,
X_test_dapc,
Y_test_ctd,
decoding_window=decoding_window,
offset=offset)
results[fold_idx, dim_idx, offset_idx, T_pi_idx] = r2_dapc
np.save(res_name, results)
return results
ad_list = []
for name, group in tqdm(grouped):
t = group[feat].to_list()
if len(t) > 1:
ad_list.append([str(i) for i in t])
model = Word2Vec(
ad_list, sg=1, size=128, window=8, seed=2020, min_count=1,
workers=int(multiprocessing.cpu_count()) / 2, iter=10)
embeddings = []
for i in range(ad_df[feat].max() + 1):
try:
embeddings.append(model[str(i)])
except:
embeddings.append(np.zeros(128))
embeddings = np.stack(embeddings)
np.save("data/w2v_%s.npy" % feat, embeddings)
# user_id
grouped = click_log_df.groupby("creative_id")
ad_list = []
for name, group in tqdm(grouped):
t = group["user_id"].to_list()
if len(t) > 1:
ad_list.append([str(i) for i in t])
model = Word2Vec(
ad_list, sg=1, size=128, window=8, seed=2020, min_count=1,
workers=int(multiprocessing.cpu_count()) / 2, iter=10)
embeddings = []
for i in range(click_log_df["user_id"].max() + 1):
try:
embeddings.append(model[str(i)])
context = zmq.Context()
req = context.socket(zmq.REQ) #open a req port to talk to pupil
req.connect("tcp://%s:%s" %(addr,req_port))
req.send(b'SUB_PORT') # ask for the sub port
sub_port = req.recv()
# open a sub port to listen to pupil in eye_1_3d
sub_1_3d = context.socket(zmq.SUB)
sub_1_3d.connect(b"tcp://%s:%s" %(addr.encode('utf-8'),sub_port))
sub_1_3d.setsockopt(zmq.SUBSCRIBE, b'pupil.1.3d')
need_calculate = input("Start Calculating convert matrix?(Y/n) : ")
if (need_calculate.upper() == "Y") :
convert_matrix = make_convert_matrix(sub_1_3d)
np.save('./convert_matrix',convert_matrix)
else:
convert_matrix = np.load('convert_matrix.npy')
print("convert matrix : ", convert_matrix)
input("Start convert blurred image(press enter)")
time_0 = time.time()
# Start convert blurred image
try:
while True:
current_time = time.time()
# Collect Data from pupil_tracker & Wait for a coherent pair of frames: depth and color
sub_1_3d.connect(b"tcp://%s:%s" %(addr.encode('utf-8'),sub_port))
#check words of first document
# are the words of the doc present at the unique variable
check=[docsobj.unique[i] for i in range(0,len(docsobj.unique)) if docsobj.unique[i] in docsobj.tokens[0]]
# extract the counts from the actual doc tokens
check2=[docsobj.doc_term[0][i] for i in range(0,len(docsobj.unique)) if docsobj.doc_term[0][i]>0]
# extract the counted words from .unique
counts_doc_term=[docsobj.unique[i] for i in range(0,len(docsobj.unique)) if docsobj.doc_term[0][i]>0]
docsobj.docs[0]
=======
docsobj = RawDocs(data.speech[0:100],'stopwords.txt')
dic=['Mister','president']
docsobj.doc_term()
docsobj.count(dic)
>>>>>>> Stashed changes
np.save('sample_doc_term.npy',docsobj.doc_term)
# Exercise 2
data = pd.read_table("../HW1/output_hw1ex4_fdez_verdu_kreienkamp.CSV",encoding="utf-8")
#convert text to lowercase
for i in range(0, len(data.Text)):
data.Text[i] = data.Text[i].lower()
#with one dictionary
docsobj = RawDocs(data.Text[0:100],'stopwords.txt')
docsobj.token_clean(2)
docsobj.stopword_remove()
# If running first time
docsobj.doc_term()
def main():
path = "/home/yren2/cs66/labs/FinalProject-yren2-hhuang2/code/Dota2"
allFiles = glob.glob(os.path.join(path,"*.csv"))
np_array_list = []
for file_ in allFiles:
df = pd.read_csv(file_,index_col=None, header=0)
np_array_list.append(df.as_matrix())
comb_np_array = np.vstack(np_array_list)
big_frame = pd.DataFrame(comb_np_array)
big_frame.columns = ["Match ID", "League", "Start Date", "Duration (s)", "Duration (mm:ss)", \
"Radiant Team", "Team A", "Team A Heroes", "Team B", "Team B Heroes", "Winner"]
rawData = big_frame.drop_duplicates(["Match ID"], keep='last')
# get data from only 2014
rawData = rawData[rawData["Start Date"].str.contains("2013")]
with open('heroes.json') as json_data:
d = json.load(json_data)
heroes = {}
for i in range(len(d["heroes"])):
name = d["heroes"][i]["localized_name"]
hid = d["heroes"][i]["id"]
heroes[name] = hid-1
sorted_heores = OrderedDict(sorted(heroes.items(), key=lambda t: t[1]))
data = np.zeros((1,115), dtype=float) #initialize so it's easier to stack
label = np.zeros((1,1), dtype=float) #initialize so it's easier to stack
for index, row in rawData.iterrows():
if row["Radiant Team"] == "Team A":
team1 = 1
team2 = -1
if row["Winner"] == "Team A":
label = np.vstack((label, [1]))
else:
label = np.vstack((label, [-1]))
else:
team1 = -1
team2 = 1
if row["Winner"] == "Team A":
label = np.vstack((label, [-1]))
else:
label = np.vstack((label, [1]))
hero1 = row["Team A Heroes"]
hero2 = row["Team B Heroes"]
data = np.vstack((data, createRow(heroes, hero1, team1, hero2, team2)))
data = np.delete(data, (0), axis=0) #delete first row of 0
label = np.delete(label, (0), axis=0) #delete first row of 0
# sdata = sparse.csr_matrix(data.astype(float))
# sparse.save_npz("data.npz", sdata)
X_train, X_test, y_train, y_test = train_test_split(data, label, test_size = 0.2, random_state = 42)
sdata = sparse.csr_matrix(X_train.astype(float))
sparse.save_npz("trainData2014.npz", sdata)
sdata = sparse.csr_matrix(X_test.astype(float))
sparse.save_npz("testData2014.npz", sdata)
np.save("trainLabel2014.npy", y_train)
np.save("testLabel2014.npy", y_test)
def main(args):
if args.gpu and torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.n_workers == -1:
args.n_workers = args.gpu * 4 * torch.cuda.device_count()
device = torch.device("cuda" if args.gpu else "cpu")
# Load trained ANN from disk.
if args.arch == 'vgg15ab':
ann = vgg_15_avg_before_relu(dataset=args.dataset)
# add other architectures here#
else:
raise ValueError('Unknown architecture')
ann.features = torch.nn.DataParallel(ann.features)
ann.cuda()
if not os.path.isdir(args.job_dir):
os.mkdir(args.job_dir)
f = os.path.join('.', args.model)
try:
dictionary = torch.load(f=f)['state_dict']
except KeyError:
dictionary = torch.load(f=f)
ann.load_state_dict(state_dict=dictionary, strict=True)
if args.dataset == 'imagenet':
input_shape = (3, 224, 224)
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
# the actual data to be evaluated
val_loader = ImageNet(image_encoder=RepeatEncoder(time=args.time,
dt=1.0),
label_encoder=None,
root=args.data,
download=False,
transform=transforms.Compose([
transforms.Resize((256, 256)),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]),
split='val')
# a wrapper class
dataloader = DataLoader(
val_loader,
batch_size=args.batch_size,
shuffle=True,
num_workers=4,
pin_memory=args.gpu,
)
# A loader of samples for normalization of the SNN from the training set
norm_loader = ImageNet(
image_encoder=RepeatEncoder(time=args.time, dt=1.0),
label_encoder=None,
root=args.data,
download=False,
split='train',
transform=transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]),
)
elif args.dataset == 'cifar100':
input_shape = (3, 32, 32)
print('==> Using Pytorch CIFAR-100 Dataset')
normalize = transforms.Normalize(mean=[0.507, 0.487, 0.441],
std=[0.267, 0.256, 0.276])
val_loader = CIFAR100(image_encoder=RepeatEncoder(time=args.time,
dt=1.0),
label_encoder=None,
root=args.data,
download=True,
train=False,
transform=transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor(),
normalize,
]))
dataloader = DataLoader(
val_loader,
batch_size=args.batch_size,
shuffle=True,
num_workers=0,
pin_memory=args.gpu,
)
norm_loader = CIFAR100(image_encoder=RepeatEncoder(time=args.time,
dt=1.0),
label_encoder=None,
root=args.data,
download=True,
train=True,
transform=transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor(),
normalize,
]))
else:
raise ValueError('Unsupported dataset.')
if args.eval_size == -1:
args.eval_size = len(val_loader)
for step, batch in enumerate(
torch.utils.data.DataLoader(norm_loader, batch_size=args.norm)):
data = batch['image']
break
snn = ann_to_snn(ann,
input_shape=input_shape,
data=data,
percentile=args.percentile)
torch.cuda.empty_cache()
snn = snn.to(device)
correct = 0
t0 = time()
accuracies = np.zeros((args.time, (args.eval_size // args.batch_size) + 1),
dtype=np.float32)
for step, batch in enumerate(tqdm(dataloader)):
if (step + 1) * args.batch_size > args.eval_size:
break
# Prep next input batch.
inputs = batch["encoded_image"]
labels = batch["label"]
inpts = {"Input": inputs}
if args.gpu:
inpts = {k: v.cuda() for k, v in inpts.items()}
snn.run(inpts=inpts,
time=args.time,
step=step,
acc=accuracies,
labels=labels,
one_step=args.one_step)
last_layer = list(snn.layers.keys())[-1]
output_voltages = snn.layers[last_layer].summed
prediction = torch.softmax(output_voltages, dim=1).argmax(dim=1)
correct += (prediction.cpu() == labels).sum().item()
snn.reset_()
t1 = time() - t0
final = accuracies.sum(axis=1) / args.eval_size
plt.plot(final)
plt.suptitle('{} {} ANN-SNN@{} percentile'.format(args.dataset, args.arch,
args.percentile),
fontsize=20)
plt.xlabel('Timestep', fontsize=19)
plt.ylabel('Accuracy', fontsize=19)
plt.grid()
plt.show()
plt.savefig('{}/{}_{}.png'.format(args.job_dir, args.arch,
args.percentile))
np.save(
'{}/voltage_accuracy_{}_{}.npy'.format(args.job_dir, args.arch,
args.percentile), final)
accuracy = 100 * correct / args.eval_size
print(f"SNN accuracy: {accuracy:.2f}")
print(f"Clock time used: {t1:.4f} ms.")
path = os.path.join(args.job_dir, "results", args.results_file)
os.makedirs(os.path.dirname(path), exist_ok=True)
if not os.path.isfile(path):
with open(path, "w") as f:
f.write(
"seed,simulation time,batch size,inference time,accuracy\n")
to_write = [args.seed, args.time, args.batch_size, t1, accuracy]
to_write = ",".join(map(str, to_write)) + "\n"
with open(path, "a") as f:
f.write(to_write)
return t1
# Load the roi-stats for leaf rois (from the cluster job)
print("Loading leaf roi-stats")
df = pd.DataFrame(np.load('roi-stats.npy', allow_pickle=True))
# Pivot so roi names are in the columns
print("Pivoting stats")
pdf = df[['body', 'roi', 'voxels_s1']].pivot('body', 'roi')
pdf = pdf.fillna(0).astype(np.int64)
pdf.columns = pdf.columns.droplevel()
# Compute non-leaf roi stats by summing the proper leaf rois for each
print("Computing non-leaf stats")
for roi in tqdm(leaf_descendants.keys()):
if roi not in pdf.columns:
ld = leaf_descendants[roi]
pdf[roi] = pdf.loc[:, ld].sum(axis=1)
# Sort columns by name
pdf = pdf.sort_index(axis=1)
# Save as npy and csv
p = 'roi-stats-pivoted-plus-nonleaf-rois.npy'
print(f"Saving {p}")
np.save(p, pdf.to_records(index=True))
p = 'roi-stats-pivoted-plus-nonleaf-rois.csv'
print(f"Saving {p}")
pdf.to_csv('roi-stats-pivoted-plus-nonleaf-rois.csv', index=True, header=True)
print("DONE")
features = model.predict(x)
print(features.shape)
features = [features]
for i in range(1, 98):
img_path = 'old_%d.jpg' % i
print(img_path)
img = image.load_img(img_path, target_size=(224, 224))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
temp = model.predict(x)
features.append(temp)
features = np.array(features)
print(features.shape) #(99, 1, 7, 7, 512)
np.save('old_features.npy', features)
img_path = 'ad_0.jpg'
img = image.load_img(img_path, target_size=(224, 224))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
features = model.predict(x)
print(features.shape)
features = [features]
for i in range(1, 100):
img_path = 'ad_%d.jpg' % i
print(img_path)
img = image.load_img(img_path, target_size=(224, 224))
x = image.img_to_array(img)
if energy_diff > 0:
flip_prob = flip_probs[i]
flip = np.random.choice([1, flip_state], size=1, p=[1-flip_prob, flip_prob])[0]
flips[a] = flip
else:
flip_prob = flip_others_probs[i]
flip = np.random.choice([1, flip_state], size=1, p=[1-flip_prob, flip_prob])[0]
flips[a] = flip
flipped_s = s * flips
new_mus.append(mu + flipped_s*sigma*zoom_factor)
sigma *= zoom_factor
mus = new_mus
ground_energies[f,i]=total_hamiltonian(mus[0],C_i,C_ij)/(train_size-1)
ground_energies_test[f,i] = total_hamiltonian(mus[0],C_i_test,C_ij_test)/(test_size-1)
np.save('./mus/' +'mus' + str(train_size) + "_fold" + str(f) + '_iter' + str(i) + '.npy', np.array(mus))
final_predictions_train=[]
final_predictions_test=[]
strong_classifier_train = strong_classifier(predictions_train , mus[0])
strong_classifier_test = strong_classifier(predictions_test , mus[0])
for i in range(len(tag_train)) :
final_predictions_train.append([strong_classifier_train[i] , tag_train[i]])
for i in range(len(tag_test)) :
final_predictions_test.append([strong_classifier_test[i], tag_test[i]])
np.save("./strong_train_predictions/prediction_lables_train_f"+str(f)+".npy", final_predictions_train)
np.save("./strong_train_predictions/prediction_lables_test_f"+str(f)+".npy", final_predictions_test)
def main():
###make sure to change these when running in a new enviorment!###
#location of data directory
filepath_cat1 = cu.get_output_path() + 'processed_data/Berland_groupcat/'
filepath_cat2 = cu.get_output_path() + 'processed_data/NYU_VAGC/'
#save data to directory...
savepath1 = filepath_cat1 + 'nyu_vagc_match/'
savepath2 = filepath_cat2 + 'berland_groupcat_match/'
#################################################################
catalogues_1 = ['mr19_groups', 'smthresh10.2.groups', 'smthresh9.8.groups']
catalogues_2 = ['nyu_vagc_dr7']
catalogue1 = catalogues_1[0]
catalogue2 = catalogues_2[0]
print catalogue1, 'match into', catalogue2
f1 = h5py.File(filepath_cat1 + catalogue1 + '.hdf5',
'r') #open catalogue file
GC = f1.get(catalogue1)
f2 = h5py.File(filepath_cat2 + catalogue2 + '.hdf5',
'r') #open catalogue file
W = f2.get(catalogue2)
da = 2.0 * 1.0 / 3600.0 #matching length
result = cu.spherematch(GC['RA'],
GC['DEC'],
W['RA'],
W['DEC'],
tol=da,
nnearest=1)
#check to see if anything was matched to the same object
repeats = [
item for item, count in Counter(result[1]).iteritems() if count > 1
]
if len(repeats) > 0:
print 'number of double matched objects:', len(repeats)
#check to see if every object has a match
if len(result[0]) == len(GC):
print 'a match was found for every object.'
else:
print 'some objects do not have matches.'
filename1 = catalogue2 + '_' + catalogue1 + '_match'
filename2 = catalogue1 + '_' + catalogue2 + '_match'
np.save(savepath1 + filename1, result[1])
np.save(savepath2 + filename2, result[0])
catalogue1 = catalogues_1[1]
catalogue2 = catalogues_2[0]
print catalogue1, 'match into', catalogue2
f1 = h5py.File(filepath_cat1 + catalogue1 + '.hdf5',
'r') #open catalogue file
GC = f1.get(catalogue1)
f2 = h5py.File(filepath_cat2 + catalogue2 + '.hdf5',
'r') #open catalogue file
W = f2.get(catalogue2)
da = 2.0 * 1.0 / 3600.0 #matching length
result = cu.spherematch(GC['ra'],
GC['dec'],
W['RA'],
W['DEC'],
tol=da,
nnearest=1)
#check to see if anything was matched to the same object
repeats = [
item for item, count in Counter(result[1]).iteritems() if count > 1
]
if len(repeats) > 0:
print 'number of double matched objects:', len(repeats)
#check to see if every object has a match
if len(result[0]) == len(GC):
print 'a match was found for every object.'
else:
print 'some objects do not have matches.'
filename1 = catalogue2 + '_' + catalogue1 + '_match'
filename2 = catalogue1 + '_' + catalogue2 + '_match'
np.save(savepath1 + filename1, result[1])
np.save(savepath2 + filename2, result[0])
catalogue1 = catalogues_1[2]
catalogue2 = catalogues_2[0]
print catalogue1, 'match into', catalogue2
f1 = h5py.File(filepath_cat1 + catalogue1 + '.hdf5',
'r') #open catalogue file
GC = f1.get(catalogue1)
f2 = h5py.File(filepath_cat2 + catalogue2 + '.hdf5',
'r') #open catalogue file
W = f2.get(catalogue2)
da = 2.0 * 1.0 / 3600.0 #matching length
result = cu.spherematch(GC['ra'],
GC['dec'],
W['RA'],
W['DEC'],
tol=da,
nnearest=1)
#check to see if anything was matched to the same object
repeats = [
item for item, count in Counter(result[1]).iteritems() if count > 1
]
if len(repeats) > 0:
print 'number of double matched objects:', len(repeats)
#check to see if every object has a match
if len(result[0]) == len(GC):
print 'a match was found for every object.'
else:
print 'some objects do not have matches.'
filename1 = catalogue2 + '_' + catalogue1 + '_match'
filename2 = catalogue1 + '_' + catalogue2 + '_match'
np.save(savepath1 + filename1, result[1])
np.save(savepath2 + filename2, result[0])
def main():
parser = argparse.ArgumentParser()
############## environment parameters ##############
parser.add_argument('--edge_capability', default = 3*1e2*GHZ, metavar='G', help = "total edge CPU capability", type=int)
parser.add_argument('--cloud_capability', default = 2.1*1e3*GHZ, metavar='G', help = "total cloud CPU capability", type=int) # clock per tick
parser.add_argument('--task_rate', default = 10, metavar='G', help = "application arrival task rate")
parser.add_argument('--channel', default = WIRED, metavar='G')
parser.add_argument('--applications', default = (SPEECH_RECOGNITION, NLP, FACE_RECOGNITION), metavar='G')#, SEARCH_REQ, LANGUAGE_TRANSLATION, PROC_3D_GAME, VR, AR
parser.add_argument('--use_beta', action = 'store_true', help = "use 'offload' to cloud")
parser.add_argument('--silence', action = 'store_true', help= "shush environment messages")
parser.add_argument('--comment', default=None)
############## Hyperparameters ##############
parser.add_argument('--log_interval', default = 20 , metavar='N', help="print avg reward in the interval", type=int)
parser.add_argument('--max_episodes', default = 1000, metavar='N', help="max training episodes", type=int)
parser.add_argument('--max_timesteps', default = 2000, metavar='N', help="max timesteps in one episode", type=int)
parser.add_argument('--update_timestep', default = 1000, metavar='N', help="update policy every n timesteps", type=int)
parser.add_argument('--action_std', default = 0.5 , metavar='N', help="constant std for action distribution (Multivariate Normal)", type=float)
parser.add_argument('--K_epochs', default = 80 , metavar='N', help="update policy for K epochs")
parser.add_argument('--eps_clip', default = 0.2 , metavar='N', help="clip parameter for PPO", type=float)
parser.add_argument('--gamma', default = 0.9 , metavar='N', help="discount factor", type=float)
parser.add_argument('--lr', default = 0.0003 , metavar='N', help="parameters for Adam optimizer", type=float)
parser.add_argument('--betas', default = (0.9, 0.999), metavar='N')
parser.add_argument('--random_seed', default = None, metavar='N')
#############################################
############## save parameters ##############
file_name = 'ppo_fixed_len'+str(datetime.now())
eval_dir = "./results/{}/eval_results".format(file_name)
model_dir = "./results/{}/pytorch_models".format(file_name)
if not os.path.exists(eval_dir):
os.makedirs(eval_dir)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
args = parser.parse_args()
args_dict = vars(args)
############## parser arguments to plain variabales ##############
edge_capability = args.edge_capability
cloud_capability = args.cloud_capability
task_rate = args.task_rate
channel = args.channel
applications = args.applications
use_beta = args.use_beta
silence = args.silence
number_of_apps = len(applications)
cloud_policy = [1/number_of_apps]*number_of_apps
args_dict['number_of_apps'] = number_of_apps
args_dict['cloud_policy'] = cloud_policy
log_interval = args.log_interval
max_episodes = args.max_episodes
max_timesteps = args.max_timesteps
update_timestep = args.update_timestep
action_std = args.action_std
K_epochs = args.K_epochs
eps_clip = args.eps_clip
gamma = args.gamma
lr = args.lr
betas = args.betas
random_seed = args.random_seed
##################################################################
with open("./results/{}/args.json".format(file_name), 'w') as f:
json.dump(args_dict, f, indent='\t')
# import pdb; pdb.set_trace()
# creating environment
env = environment.Environment_sosam(task_rate, *applications, use_beta=use_beta)
state = env.init_for_sosam(edge_capability, cloud_capability, channel)
state_dim = env.state_dim
action_dim = env.action_dim
if random_seed:
print("Random Seed: {}".format(random_seed))
torch.manual_seed(random_seed)
# env.seed(random_seed)
np.random.seed(random_seed)
memory = Memory()
ppo = PPO(state_dim, action_dim, action_std, lr, betas, gamma, K_epochs, eps_clip)
# logging variables
running_reward = 0
avg_length = 0
time_step = 0
evaluations_empty_reward = []
evaluations = []
# evaluations_empty_reward_1000 = []
# evaluations_1000 = []
# training loop
for i_episode in range(1, max_episodes+1):
state = env.reset()
for t in range(max_timesteps):
time_step +=1
# Running policy_old:
action = ppo.select_action(state, memory)
if t%250==0:
print("---------------------------------------")
print("estimated arrival: {}".format(state[:8]))
print("just arrived: {}".format(state[8:16]))
print("queue length: {}".format(state[16:24]))
print("queue explosion: {}".format(state[24:32]))
if use_beta:
print("c_estimated arrival: {}".format(state[32:40]))
print("c_just arrived: {}".format(state[40:48]))
print("c_queue length: {}".format(state[48:56]))
print("c_queue explosion: {}".format(state[56:]))
print("---------------------------------------")
print("------action\t{}".format(action))
print("---------------------------------------")
state, cost, done = env.step_together(t, action, cloud_policy, silence=silence)
if t%250==0:
print("new_estimated arrival: {}".format(state[:8]))
print("new_just arrived: {}".format(state[8:16]))
print("new_queue length: {}".format(state[16:24]))
print("new_queue explosion: {}".format(state[24:32]))
if use_beta:
print("new_c_estimated arrival: {}".format(state[32:40]))
print("new_c_just arrived: {}".format(state[40:48]))
print("new_c_queue length: {}".format(state[48:56]))
print("new_c_queue explosion: {}".format(state[56:]))
reward = -cost
# Saving reward:
memory.rewards.append(reward)
# update if its time
if time_step % update_timestep == 0:
ppo.update(memory)
memory.clear_memory()
time_step = 0
running_reward += reward
if t%250==0:
print("---------------------------------------")
print("cost:{}, episode reward{}".format(cost, running_reward))
print("---------------------------------------")
if done:
break
# if t%200==0:
# print("episode {}, average length {}, running_reward{}".format(i_episode, avg_length, running_reward))
avg_length += t
evaluations_empty_reward.append(evaluate_policy(env, ppo, cloud_policy, memory, epsd_length=max_timesteps*2))
evaluations.append(evaluate_policy(env, ppo, cloud_policy, memory, epsd_length=max_timesteps*2, empty_reward=False))
# evaluations_empty_reward_1000.append(evaluate_policy(env, ppo, cloud_policy, memory, epsd_length=1000))
# evaluations_1000.append(evaluate_policy(env, ppo, cloud_policy, memory, epsd_length=1000, empty_reward=False))
np.save("{}/eval_empty_reward".format(eval_dir), evaluations_empty_reward)
np.save("{}/eval".format(eval_dir), evaluations)
# np.save("{}/eval_empty_reward_1000".format(eval_dir), evaluations_empty_reward_1000)
# np.save("{}/eval_1000".format(eval_dir), evaluations_1000)
# stop training if avg_reward > solved_reward
# if running_reward > (log_interval*solved_reward):
# print("########## Solved! ##########")
# torch.save(ppo.policy.state_dict(), './PPO_continuous_solved_{}.pth'.format(env_name))
# break
# save every 500 episodes
if i_episode % 50 == 0:
ppo.save('env3_{}_{}'.format(i_episode, t), directory=model_dir)
# logging
if i_episode % log_interval == 0:
avg_length = int(avg_length/log_interval)
running_reward = int((running_reward/log_interval))
print('Episode {} \t Avg length: {} \t Avg reward: {}'.format(i_episode, avg_length, running_reward))
running_reward = 0
avg_length = 0
def set_area(self):
gids_volumes = self.mb.tag_get_data(self.tags['GIDV'],
self.all_volumes,
flat=True)
map_volumes = dict(zip(self.all_volumes, gids_volumes))
areas = np.zeros(len(self.all_faces))
normais = np.zeros((len(self.all_faces), 3))
Adjs = [self.mb.get_adjacencies(f, 3) for f in self.all_faces]
vertss = [
self.mtu.get_bridge_adjacencies(f, 2, 0) for f in self.all_faces
]
lim = 1e-9
bound_faces = self.mb.create_meshset()
bfs = []
for i, f in enumerate(self.all_faces):
elems = Adjs[i]
if len(elems) < 2:
bfs.append(f)
verts = vertss[i]
coords = self.mb.get_coords(verts).reshape([len(verts), 3])
mins = coords.min(axis=0)
maxs = coords.max(axis=0)
dx, dy, dz = np.absolute(maxs - mins)
if dx < lim:
dx = 1.0
if dy < lim:
dy = 1.0
if dz < lim:
dz = 1.0
area = dx * dy * dz
areas[i] = area
if len(elems) > 1:
id0 = map_volumes[elems[0]]
id1 = map_volumes[elems[1]]
cent0 = self.all_centroids[id0]
cent1 = self.all_centroids[id1]
direction = cent1 - cent0
norma = np.linalg.norm(direction)
uni = np.absolute(direction / norma)
normais[i] = uni
else:
p0 = coords[0]
p1 = coords[1]
p2 = coords[2]
direction = np.cross(p0 - p1, p0 - p2)
norma = np.linalg.norm(direction)
uni = np.absolute(direction / norma)
normais[i] = uni
self.mb.tag_set_data(self.tags['AREA'], self.all_faces, areas)
self.mb.tag_set_data(self.tags['NORMAL'], self.all_faces, normais)
bfs = rng.Range(bfs)
self.mb.add_entities(bound_faces, bfs)
self.mb.tag_set_data(self.tags['BOUND_FACES'], 0, bound_faces)
self.bound_faces = bfs
self.faces_in = rng.subtract(self.all_faces, bfs)
self.Adjs_in = np.array(
[np.array(self.mb.get_adjacencies(f, 3)) for f in self.faces_in])
Adjs_d = self.Adjs_in[:, 1]
Adjs_e = self.Adjs_in[:, 0]
os.chdir(flying_dir)
np.save('Adjs_d', Adjs_d)
np.save('Adjs_e', Adjs_e)
G.load_state_dict(torch.load('./WGAN_for_DOS/generator_MP2020.pt'))
G.eval()
def sample_generator(G, num_samples, feature):
generated_data_all = 0
num_sam = 100
for i in range(num_sam):
latent_samples = Variable(G.sample_latent(num_samples))
latent_samples = latent_samples
generated_data = G(torch.cat((feature, latent_samples), dim=1))
generated_data_all += generated_data
generated_data = generated_data_all/num_sam
return generated_data
print('Testing...')
gen_data = sample_generator(G, batch_size, test_feat).detach()
print('Done!')
#gen_data = gen_data*std+mean
test_label = (test_label-mean)/std
MAE = torch.mean(torch.abs(gen_data-test_label))
print('MAE:', MAE.numpy())
# save testing results
np.save('./WGAN_for_DOS/test_label.npy', test_label)
np.save('./WGAN_for_DOS/test_pred.npy', gen_data)
data_size = [100, 500, 1000, 5000, 10000]
if args.all:
data_size.append(70000)
results = []
basename = os.path.basename(os.path.splitext(__file__)[0])
log_filename = os.path.join(LOG_DIR, basename + ".json")
for n in data_size:
X_train = X[:n]
y_train = y[:n]
n = X_train.shape[0]
for name, method in methods:
print("Fitting {} on {} samples...".format(name, n))
t0 = time()
np.save(
os.path.join(LOG_DIR, "mnist_{}_{}.npy".format("original", n)), X_train
)
np.save(
os.path.join(LOG_DIR, "mnist_{}_{}.npy".format("original_labels", n)),
y_train,
)
X_embedded, n_iter = method(X_train)
duration = time() - t0
precision_5 = nn_accuracy(X_train, X_embedded)
print(
"Fitting {} on {} samples took {:.3f}s in {:d} iterations, "
"nn accuracy: {:0.3f}".format(name, n, duration, n_iter, precision_5)
)
results.append(dict(method=name, duration=duration, n_samples=n))
with open(log_filename, "w", encoding="utf-8") as f:
json.dump(results, f)
BoxCoords[yVAL,xVAL,(idBoxMask*4)+3] = h
# iterate through every list element
for i in range(0, len(ListElements)):
currentField = ListElements[i]
cx, cy, cx2, cy2 = getBounds(currentField)
x, y, w, h = getCoords(currentField)
w = width / 2
# fill segment boxes
for xVAL in range(cx, cx2):
for yVAL in range(cy2, cy):
SegmentImage[yVAL,xVAL] = segMapper(currentField.id)
# Fill box masks and box coords for list items once per row
if currentField.id == "ItemName":
for xVal in range(0,width-1):
for yVal in range(cy2, cy):
BoxMasks[yVal,xVal,(num_anchors-1)*2] = 1
BoxMasks[yVal,xVal,((num_anchors-1)*2)+1] = 0
BoxCoords[yVal,xVal,(num_anchors-1)*4] = w # center x is for list items also w
BoxCoords[yVal,xVal,((num_anchors-1)*4)+1] = y
BoxCoords[yVal,xVal,((num_anchors-1)*4)+2] = w
BoxCoords[yVal,xVal,((num_anchors-1)*4)+3] = h
# save target data
np.save(seg_path + str(id) +".npy", SegmentImage)
np.save(box_path + str(id) +".npy", BoxMasks)
np.save(coord_path + str(id) +".npy", BoxCoords)
print("--- %s seconds ---" % (time.time() - start_time))
