def calc_defence(diff_ids, int_ids, model_id, model_w, model_handler):
defence = 0
tolerance = 0
for diff_id in diff_ids:
neighs = model_handler.get_neighbours_ids(diff_id, model_id)
neighs_list = list(neighs)
neighs_in_int = np.intersect1d(neighs_list, int_ids, True)
w_diff = model_handler.get_node_weight(diff_id, model_id)
if len(neighs_in_int):
neighs_in_diff = np.intersect1d(neighs_list, diff_ids)
sum_rel_diff = sum([model_handler.get_relation_weight(diff_id, i, model_id) for i in neighs_in_diff])
sum_rel_int = sum([model_handler.get_relation_weight(diff_id, i, model_id) for i in neighs_in_int])
if sum_rel_diff == 0:
sum_rel_diff = 1
if sum_rel_int == 0:
sum_rel_int = 1
for neigh in neighs:
w_neigh = model_handler.get_node_weight(neigh, model_id)
w_r_neigh = model_handler.get_relation_weight(neigh, diff_id, model_id)
if neigh in neighs_in_int:
ch = float(math.fabs(float(w_neigh - w_diff)) * w_r_neigh)
tolerance += ch / sum_rel_diff
else:
ch = (float(w_neigh) + w_diff) * w_r_neigh
defence += ch / sum_rel_int
else:
defence += float(w_diff) / model_w
return tolerance, defence
def evaluate_result(data, target, result):
assert(data.shape[0] == target.shape[0])
assert(target.shape[0] == result.shape[0])
correct = np.where( result == target )
miss = np.where( result != target )
class_rate = float(correct[0].shape[0]) / target.shape[0]
print "Correct classification rate:", class_rate
#get the 3s
mask = np.where(target == wanted[0])
data_3_correct = data[np.intersect1d(mask[0],correct[0])]
data_3_miss = data[np.intersect1d(mask[0],miss[0])]
#get the 8s
mask = np.where(target == wanted[1])
data_8_correct = data[np.intersect1d(mask[0],correct[0])]
data_8_miss = data[np.intersect1d(mask[0],miss[0])]
#plot
plot.title("Scatter")
plot.xlabel("x_0")
plot.ylabel("x_1")
size = 20
plot.scatter(data_3_correct[:,0], data_3_correct[:,1], marker = "x", c = "r", s = size )
plot.scatter( data_3_miss[:,0], data_3_miss[:,1], marker = "x", c = "b", s = size )
plot.scatter(data_8_correct[:,0], data_8_correct[:,1], marker = "o", c = "r", s = size )
plot.scatter( data_8_miss[:,0], data_8_miss[:,1], marker = "o", c = "b", s = size )
plot.show()
def inFootprint(config, pixels, nside=None):
"""
Open each valid filename for the set of pixels and determine the set
of subpixels with valid data.
"""
config = Config(config)
nside_catalog = config['coords']['nside_catalog']
nside_likelihood = config['coords']['nside_likelihood']
nside_pixel = config['coords']['nside_pixel']
if np.isscalar(pixels): pixels = np.array([pixels])
if nside is None: nside = nside_likelihood
filenames = config.getFilenames()
catalog_pixels = filenames['pix'].compressed()
inside = np.zeros(len(pixels), dtype=bool)
if not nside_catalog:
catalog_pix = [0]
else:
catalog_pix = superpixel(pixels,nside,nside_catalog)
catalog_pix = np.intersect1d(catalog_pix,catalog_pixels)
for fnames in filenames[catalog_pix]:
logger.debug("Loading %s"%filenames['mask_1'])
#subpix_1,val_1 = ugali.utils.skymap.readSparseHealpixMap(fnames['mask_1'],'MAGLIM',construct_map=False)
_nside,subpix_1,val_1 = ugali.utils.healpix.read_partial_map(fnames['mask_1'],'MAGLIM',fullsky=False)
logger.debug("Loading %s"%fnames['mask_2'])
#subpix_2,val_2 = ugali.utils.skymap.readSparseHealpixMap(fnames['mask_2'],'MAGLIM',construct_map=False)
_nside,subpix_2,val_2 = ugali.utils.healpix.read_partial_map(fnames['mask_2'],'MAGLIM',fullsky=False)
subpix = np.intersect1d(subpix_1,subpix_2)
superpix = np.unique(superpixel(subpix,nside_pixel,nside))
inside |= np.in1d(pixels, superpix)
return inside
def afterObjectCompute(self, feedforwardInput, lateralInputs=(),
feedforwardGrowthCandidates=None, learn=True):
activeCells = self.objectLayer.getActiveCells()
cells = dict((cell, [])
for cell in activeCells.tolist())
for cell in activeCells:
connectedSynapses = np.where(
self.objectLayer.proximalPermanences.getRow(cell)
>= self.objectLayer.connectedPermanenceProximal)[0]
activeSynapses = np.intersect1d(connectedSynapses, feedforwardInput)
segmentData = [
["input", activeSynapses.tolist()]
]
cells[cell].append(segmentData)
self.csvOut.writerow(("layer", "object"))
self.csvOut.writerow([json.dumps(cells.items())])
decodings = [k
for k, sdr in self.objectRepresentations.iteritems()
if np.intersect1d(activeCells, sdr).size == sdr.size]
self.csvOut.writerow([json.dumps(decodings)])
def get_movers_from_overfilled_locations(self, agent_set, agents_index, config=None):
"""Returns an index (relative to agents_index) of agents that should be removed from their locations.
"""
id_name = self.choice_set.get_id_name()[0]
agents_locations = agent_set.get_attribute_by_index(id_name, agents_index)
# check if there was an overfilling of locations
movers = array([], dtype='int32')
if self.compute_capacity_flag:
overfilled_string = config.get("is_choice_overfilled_string", None)
if overfilled_string:
tmp_agent_set = copy.copy(agent_set)
overfilled_locations = where(self.choice_set.compute_variables(overfilled_string, self.dataset_pool))[0]
current_agents_in_overfilled_locations = intersect1d(agents_locations, overfilled_locations)
while current_agents_in_overfilled_locations.size > 0:
for location in current_agents_in_overfilled_locations:
agents_of_this_location = where(agents_locations == location)[0]
if agents_of_this_location.size > 1:
sampled_agents = probsample_noreplace(agents_of_this_location, 1)
else:
sampled_agents = agents_of_this_location
movers = concatenate((movers, sampled_agents))
tmp_agent_set.set_values_of_one_attribute(id_name, -1, agents_index[movers])
agents_locations = tmp_agent_set.get_attribute_by_index(id_name, agents_index)
self.dataset_pool.replace_dataset(tmp_agent_set.get_dataset_name(), tmp_agent_set)
overfilled_locations = where(self.choice_set.compute_variables(overfilled_string, self.dataset_pool))[0]
current_agents_in_overfilled_locations = intersect1d(agents_locations, overfilled_locations)
self.dataset_pool.replace_dataset(agent_set.get_dataset_name(), agent_set)
else:
new_locations_vacancy = self.get_locations_vacancy(agent_set)
movers = self.choose_agents_to_move_from_overfilled_locations(new_locations_vacancy,
agent_set, agents_index, agents_locations)
return concatenate((movers, where(agents_locations <= 0)[0]))
def select_source_in_label(src, label, random_state=None):
"""Select source positions using a label
Parameters
----------
src : list of dict
The source space
label : Label
the label (read with mne.read_label)
random_state : None | int | np.random.RandomState
To specify the random generator state.
Returns
-------
lh_vertno : list
selected source coefficients on the left hemisphere
rh_vertno : list
selected source coefficients on the right hemisphere
"""
lh_vertno = list()
rh_vertno = list()
rng = check_random_state(random_state)
if label.hemi == 'lh':
src_sel_lh = np.intersect1d(src[0]['vertno'], label.vertices)
idx_select = rng.randint(0, len(src_sel_lh), 1)
lh_vertno.append(src_sel_lh[idx_select][0])
else:
src_sel_rh = np.intersect1d(src[1]['vertno'], label.vertices)
idx_select = rng.randint(0, len(src_sel_rh), 1)
rh_vertno.append(src_sel_rh[idx_select][0])
return lh_vertno, rh_vertno
def _lf_acc(self, subset, lf_idx):
gt = self.gt._gt_vec
pred = np.ravel(self.lf_matrix.tocsc()[:,lf_idx].todense())
has_label = np.where(pred != 0)
has_gt = np.where(gt != 0)
# Get labels/gt for candidates in dev set, with label, with gt
gd_idxs = np.intersect1d(has_label, subset)
gd_idxs = np.intersect1d(has_gt, gd_idxs)
gt = np.ravel(gt[gd_idxs])
pred_sub = np.ravel(pred[gd_idxs])
n_neg = np.sum(pred_sub == -1)
n_pos = np.sum(pred_sub == 1)
if np.sum(pred == -1) == 0:
neg_acc = -1
elif n_neg == 0:
neg_acc = 0
else:
neg_acc = float(np.sum((pred_sub == -1) * (gt == -1))) / n_neg
if np.sum(pred == 1) == 0:
pos_acc = -1
elif n_pos == 0:
pos_acc = 0
else:
pos_acc = float(np.sum((pred_sub == 1) * (gt == 1))) / n_pos
return (pos_acc, n_pos, neg_acc, n_neg)
def test_intersection(self):
# intersect with Int64Index
other = Index(np.arange(1, 6))
result = self.index.intersection(other)
expected = Index(np.sort(np.intersect1d(self.index.values,
other.values)))
self.assert_index_equal(result, expected)
result = other.intersection(self.index)
expected = Index(np.sort(np.asarray(np.intersect1d(self.index.values,
other.values))))
self.assert_index_equal(result, expected)
# intersect with increasing RangeIndex
other = RangeIndex(1, 6)
result = self.index.intersection(other)
expected = Index(np.sort(np.intersect1d(self.index.values,
other.values)))
self.assert_index_equal(result, expected)
# intersect with decreasing RangeIndex
other = RangeIndex(5, 0, -1)
result = self.index.intersection(other)
expected = Index(np.sort(np.intersect1d(self.index.values,
other.values)))
self.assert_index_equal(result, expected)
def slice_xyz(self, xslice, yslice, zslice):
"""TODO: doesn't remove unused nodes/renumber elements"""
x = self.xyz[:, 0]
y = self.xyz[:, 1]
z = self.xyz[:, 2]
inodes = []
if xslice is not None:
xslice = float(xslice)
inodes.append(where(x < xslice)[0])
if yslice is not None:
yslice = float(yslice)
inodes.append(where(y < yslice)[0])
if zslice is not None:
zslice = float(zslice)
inodes.append(where(z < zslice)[0])
if len(inodes) == 1:
nodes = inodes[0]
elif len(inodes) == 2:
nodes = intersect1d(inodes[0], inodes[1], assume_unique=True)
elif len(inodes) == 3:
nodes = intersect1d(
intersect1d(inodes[0], inodes[1], assume_unique=True),
inodes[2], assume_unique=True)
inodes = arange(self.nodes.shape[0])
# nodes = unique(hstack(inodes))
self._slice_plane_inodes(nodes)
def _remove_useless_states(self):
"""Check for states that don't do anything, and remove them.
Scan the A, B, and C matrices for rows or columns of zeros. If the
zeros are such that a particular state has no effect on the input-output
dynamics, then remove that state from the A, B, and C matrices.
"""
# Search for useless states and get the indices of these states
# as an array.
ax1_A = np.where(~self.A.any(axis=1))[0]
ax1_B = np.where(~self.B.any(axis=1))[0]
ax0_A = np.where(~self.A.any(axis=0))[1]
ax0_C = np.where(~self.C.any(axis=0))[1]
useless_1 = np.intersect1d(ax1_A, ax1_B, assume_unique=True)
useless_2 = np.intersect1d(ax0_A, ax0_C, assume_unique=True)
useless = np.union1d(useless_1, useless_2)
# Remove the useless states.
self.A = delete(self.A, useless, 0)
self.A = delete(self.A, useless, 1)
self.B = delete(self.B, useless, 0)
self.C = delete(self.C, useless, 1)
self.states = self.A.shape[0]
self.inputs = self.B.shape[1]
self.outputs = self.C.shape[0]
def rh_by_runtime(RTFc, RTFh, RHi, **hourly):
heads = ['Hours above 50% RH; days with no cooling or heating',
'Number of days; no cooling or heating',
'Mean RH, days with no cooling or heating',
'Hours above 50% RH; days with cooling, no heating',
'Number of days; cooling, no heating',
'Mean RH, days with cooling, no heating',
'Hours above 50% RH; days with heating, no cooling',
'Number of days; heating, no cooling',
'Mean RH, days with heating, no cooling',
'Hours above 50% RH; days with heating and cooling',
'Number of days; heating and cooling',
'Mean RH, days with heating and cooling']
vals = []
conditions = [np.intersect1d(np.where(daily_total(RTFc)==0)[0], np.where(daily_total(RTFh)==0)[0]),
np.intersect1d(np.where(daily_total(RTFc)>0)[0], np.where(daily_total(RTFh)==0)[0]),
np.intersect1d(np.where(daily_total(RTFc)==0)[0], np.where(daily_total(RTFh)>0)[0]),
np.intersect1d(np.where(daily_total(RTFc)>0)[0], np.where(daily_total(RTFh)>0)[0])]
for condition in conditions:
vals.append(daily_total(np.where(RHi > 50, 1, 0))[condition].sum())
vals.append(len(condition))
vals.append(daily_mean(RHi)[condition].mean())
if len(heads) != len(vals):
print("only {0} values for {1}".format(len(vals), hourly['name']))
return (heads, vals)
def calc_avg_B(self):
'''Calculate the average B around the center from existing data.'''
center_box_size = 0.02
print "[calc] Finding data points around desired location..."
x_small = find_indices(np.abs(self.x.array) <= center_box_size)
y_small = find_indices(np.abs(self.y.array) <= center_box_size)
z_small = find_indices(np.abs(self.z.array) <= center_box_size)
close_to_center = np.intersect1d(np.intersect1d(x_small,
y_small), z_small)
if len(close_to_center) == 0:
print "[calc] No suitable points to calculate average B."
self.avg_Bx = raw_input("[calc] Enter center Bx [mG]: ") or 600.0
self.avg_Bz = raw_input("[calc] Enter center Bz [mG]: ") or 0.0
else:
print "[calc] Sampling from %i points." % len(close_to_center)
Bx_sample = np.array([])
Bz_sample = np.array([])
for index in close_to_center:
Bx_sample = np.append(Bx_sample, self.Bx[index])
Bz_sample = np.append(Bz_sample, self.Bz[index])
self.avg_Bx = np.average(Bx_sample)
self.avg_Bz = np.average(Bz_sample)
print "[calc] Calculated average Bx as %f mG." % self.avg_Bx
print "[calc] Calculated average Bz as %f mG." % self.avg_Bz
def intersect_coords(coords1,coords2):
"""For two sets of coordinates, find the coordinates that are common to
both, where the dimensionality is the coords1.shape[0]"""
#find the longer one
if coords1.shape[-1]>coords2.shape[-1]:
coords_long = coords1
coords_short = coords2
else:
coords_long = coords2
coords_short = coords1
ans = np.array([[],[],[]],dtype='int') #Initialize as a 3 row variable
#Loop over the longer of the coordinate sets
for i in xrange(coords_long.shape[-1]):
#For each coordinate:
this_coords = coords_long[:,i]
#Find the matches in the other set of coordinates:
x = np.where(coords_short[0,:] == this_coords[0])[0]
y = np.where(coords_short[1,:] == this_coords[1])[0]
z = np.where(coords_short[2,:] == this_coords[2])[0]
#Use intersect1d, such that there can be more than one match (and the
#size of idx will reflect how many such matches exist):
idx = np.intersect1d(np.intersect1d(x,y),z)
#append the places where there are matches in all three dimensions:
if len(idx):
ans = np.hstack([ans,coords_short[:,idx]])
return ans
def pre_processing_impl(self, data):
transcripts, cells = data.shape
# 1. cell filter
remain_cell_inds = np.arange(0, cells)
for c in self.cell_filter_list:
res = c(data)
remain_cell_inds = np.intersect1d(remain_cell_inds, res)
print('1. Remaining number of cells after filtering: {0}/{1}'.format(remain_cell_inds.size, cells))
A = data[:, remain_cell_inds]
# 2. gene filter
remain_gene_inds = np.arange(0, transcripts)
for g in self.gene_filter_list:
res = g(data)
remain_gene_inds = np.intersect1d(remain_gene_inds, res)
print('2. Remaining number of transcripts after filtering: {0}/{1}'.format(remain_gene_inds.size, transcripts))
# 3. data transformation
B = A[remain_gene_inds, :]
print '3. Data transformation'
print 'Before data transformation: '
print '- Mean\median\max values: ', np.mean(B), np.median(B), np.max(B)
print '- Percentiles: ', np.percentile(B, [50, 75, 90, 99])
X = self.data_transf(B)
print 'After data transformation: '
print '- Mean\median\max values: ', np.mean(X), np.median(X), np.max(X)
print '- Percentiles: ', np.percentile(X, [50, 75, 90, 99])
return X, remain_gene_inds, remain_cell_inds
def find_largest_coalition(previous, current):
"""
Returns the largest coalition given a current set of
coalitions and the previous largest coalition.
:param previous: List containing previous largest coalition.
:param current: Numpy array containing the set of current coalitions.
:return: List containing current largest coalition.
"""
# Record largest coalition.
largest_coalition = []
# Iterate through all coalitions to find the largest.
for coalition in current:
c = coalition[0]
if len(c) > len(largest_coalition):
largest_coalition = c
# If two coalitions have the same length, choose
# the one that resembles the previous one the most.
elif len(c) == len(largest_coalition):
score_new = np.intersect1d(c, previous, True).size
score_old = np.intersect1d(largest_coalition, previous, True).size
if score_new > score_old:
largest_coalition = c
# If both resemble the previous one equally, choose a random one.
elif score_new == score_old:
flip = np.random.randint(2)
if flip == 0:
largest_coalition = c
return largest_coalition
def test_tabulate_dofs(self):
L0 = self.W.sub(0)
L1 = self.W.sub(1)
L01 = L1.sub(0)
L11 = L1.sub(1)
for i, cell in enumerate(cells(self.mesh)):
dofs0 = L0.dofmap().cell_dofs(cell.index())
dofs1 = L01.dofmap().cell_dofs(cell.index())
dofs2 = L11.dofmap().cell_dofs(cell.index())
dofs3 = L1.dofmap().cell_dofs(cell.index())
self.assertTrue(np.array_equal(dofs0, \
L0.dofmap().cell_dofs(i)))
self.assertTrue(np.array_equal(dofs1,
L01.dofmap().cell_dofs(i)))
self.assertTrue(np.array_equal(dofs2,
L11.dofmap().cell_dofs(i)))
self.assertTrue(np.array_equal(dofs3,
L1.dofmap().cell_dofs(i)))
self.assertEqual(len(np.intersect1d(dofs0, dofs1)), 0)
self.assertEqual(len(np.intersect1d(dofs0, dofs2)), 0)
self.assertEqual(len(np.intersect1d(dofs1, dofs2)), 0)
self.assertTrue(np.array_equal(np.append(dofs1, dofs2), dofs3))
def linearCouplingCoeff2(dataH, dataX, timeH, timeX, transFnXtoH, segStartTime,
segEndTime, timeShift, samplFreq, logFid, debugLevel):
# LINEARCOUPLINGCOEFF - calculate the cross correlation coeff b/w the gravitational
# ave channel H and the "projected" instrumental channel X. The noise in the
# instrumental channel X is projected to the domain of the H using a linear coupling
# function Txh
rXH = np.asarray([])
rMaxXH = np.asarray([])
if((len(dataH)==0) | (len(dataX)==0)):
logFid.write('Error: One or more data vectors are empty..\n')
logFid.write('Error: len(dataH) = %d len(dataX) = %d..\n' %(len(dataH), len(dataX[0])))
elif(len(dataH)!=len(dataX[0])):
logFid.write('Error: Different lengths. len(dataH) = %d len(dataX) = %d..\n'%(len(dataH), len(dataX[0])))
else:
dataH = dataH #- np.mean(dataH)
dataX = dataX[0] #- np.mean(dataX[0])
segIdxH = np.intersect1d(np.where(timeH>=segStartTime)[0], np.where(timeH<segEndTime)[0])
dataH = dataH[segIdxH]
segIdxX = np.intersect1d(np.where(timeX + timeShift >= segStartTime)[0], np.where(timeX + timeShift < segEndTime)[0])
dataX = dataX[segIdxX]
a = np.correlate(dataH, dataX)/(np.sqrt(np.correlate(dataH, dataH)*np.correlate(dataX, dataX)))
rXH = np.append(rXH, a)
rMaxXH = np.append(rMaxXH, a)
return [rXH, rMaxXH]
def had_bounds(strmfunc, return_max=False):
"""Hadley cell poleward extent and center location."""
lat = strmfunc[LAT_STR]
# Get data max and min values and indices, such that min is north of max.
z_max_ind, y_max_ind = np.where(strmfunc == strmfunc.max())
z_max_ind = z_max_ind[0]; y_max_ind=y_max_ind[0]
z_min_ind, y_min_ind = np.where(strmfunc[:,y_max_ind:] ==
strmfunc[:,y_max_ind:].min())
z_min_ind = z_min_ind[0]; y_min_ind = y_min_ind[0] + y_max_ind
min_north = strmfunc[z_min_ind, y_min_ind]
# Return locations and values of Hadley cells' strengths.
if return_max:
return [z_min_ind, y_min_ind, strmfunc[z_min_ind, y_min_ind],
z_max_ind, y_max_ind, strmfunc[z_max_ind, y_max_ind]]
# Find latitude where streamfunction at its level of maximum decreases
# to 10% of that maximum value.
had_max = np.where(np.diff(np.sign(strmfunc[z_max_ind] -
0.1*strmfunc.max())))[0]
had_min = np.where(np.diff(np.sign(strmfunc[z_min_ind] -
0.1*strmfunc.min())))[0]
had_lims = (np.intersect1d(range(y_max_ind), had_max)[-1],
np.intersect1d(range(y_min_ind, lat.size), had_min)[0])
# Center is streamfunction zero crossing between the two cells at level
# halfway between the levels of the two maxima.
p_ind = 0.5*(z_min_ind + z_max_ind)
zero_cross = np.where(np.diff(np.sign(np.squeeze(
strmfunc[p_ind,y_max_ind:y_min_ind]))))[0]
had_center = zero_cross[0] + y_max_ind
return np.array([lat[had_lims[0]], lat[had_center], lat[had_lims[1]]])
def plot(self,reports,itime,ftime,fname,outdir,Nlim=False,
Elim=False,Slim=False,Wlim=False,
annotate=True,fig=False,ax=False,ss=50,color='blue'):
reportidx = N.array([n for n,t in zip(range(len(self.r['EVENT_TYPE'])),self.r['EVENT_TYPE']) if reports in t])
lateidx = N.where(self.datetimes > itime)
earlyidx = N.where(self.datetimes < ftime)
timeidx = N.intersect1d(earlyidx,lateidx,)#assume_unique=True)
plotidx = N.intersect1d(reportidx,timeidx)
from mpl_toolkits.basemap import Basemap
if fig==False:
fig,ax = plt.subplots(1,figsize=(6,6))
m = Basemap(projection='merc',
llcrnrlat=Slim,
llcrnrlon=Wlim,
urcrnrlat=Nlim,
urcrnrlon=Elim,
lat_ts=(Nlim-Slim)/2.0,
resolution='i',
ax=ax)
m.drawcoastlines()
m.drawstates()
m.drawcountries()
m.scatter(self.r['BEGIN_LON'][plotidx],self.r['BEGIN_LAT'][plotidx],latlon=True,
marker='D',facecolors=color,edgecolors='black',s=ss)
fig.tight_layout()
plt.savefig(os.path.join(outdir,fname))
def get_obstList(self,X,Y,Z):
"""
Define areas external to pipe.
"""
#Pipe in - find all points exterior of large pipe
pipe_in = np.array(np.where((X - 1)**2 + (Y - 1)**2 > (self.diam_in/2)**2)).flatten()
pipe_in_stop = np.array(np.where(Z <= 4)).flatten()
pipe_in = np.intersect1d(pipe_in[:],pipe_in_stop[:])
#Contraction - find all points exterior of contraction
r_cone = self.diam_out
h_cone = self.diam_out
contraction = np.array(np.where((X - 1)**2 + (Y - 1)**2 > (r_cone/h_cone)**2*(Z - (4 + h_cone))**2)).flatten()
contraction_start = np.array(np.where(Z >= 4)).flatten()
contraction_stop = np.array(np.where(Z <= 4 + .5*self.diam_out)).flatten()
contraction = np.intersect1d(contraction[:],contraction_start[:])
contraction = np.intersect1d(contraction[:],contraction_stop[:])
#Pipe out - final all points exterior of smaller pipe
pipe_out = np.array(np.where((X - 1)**2 + (Y - 1)**2 > (self.diam_out/2)**2)).flatten()
pipe_out_start = np.array(np.where(Z >= 4 + .5*self.diam_out)).flatten()
pipe_out = np.intersect1d(pipe_out[:],pipe_out_start[:])
#Put the pieces together
#pipe = pipe_in[:]
pipe = np.union1d(contraction[:],pipe_in[:])
pipe = np.union1d(pipe[:],pipe_out[:])
obst_list = pipe[:]
return list(obst_list[:])
def get_obstList(self,X,Y,Z):
"""
Define areas external to pipe.
"""
#Pipe in - find all points exterior of small
pipe_in = np.array(np.where((X - 1)**2 + (Y - 1)**2 > (self.diam_in/2)**2)).flatten()
pipe_in_stop = np.array(np.where(Z <= 3 + 0.5*(self.diam_out - self.diam_in))).flatten()
pipe_in = np.intersect1d(pipe_in[:],pipe_in_stop[:])
#Expansion - find all points exterior of expansion
r_cone = self.diam_in
h_cone = self.diam_in
expansion = np.array(np.where((X - 1)**2 + (Y - 1)**2 > (r_cone/h_cone)**2*(Z - 3)**2)).flatten()
expansion_start = np.array(np.where(Z >= 3 + 0.5*(self.diam_out - self.diam_in)))
#expansion_stop = np.array(np.where(Z <= 4)).flatten()
expansion = np.intersect1d(expansion[:],expansion_start[:])
#expansion = np.intersect1d(expansion[:],expansion_stop[:])
#Pipe out - final all points exterior of smaller pipe
pipe_out = np.array(np.where((X - 1)**2 + (Y - 1)**2 > (self.diam_out/2)**2)).flatten()
pipe_out_start = np.array(np.where(Z >= 3 + 0.5*(self.diam_in - self.diam_out))).flatten()
pipe_out = np.intersect1d(pipe_out[:],pipe_out_start[:])
#Put the pieces together
pipe = expansion[:]
pipe = np.union1d(expansion[:],pipe_in[:])
pipe = np.union1d(pipe[:],pipe_out[:])
obst_list = pipe[:]
return list(obst_list[:])
def onpick(self, event):
ind = event.ind[0]
if len(self.ordered_list_of_objectives) == 2: #2D
print '2D picker not implemented'
elif len(self.ordered_list_of_objectives) >= 3: #3D or 4D plot
x, y, z = event.artist._offsets3d
idx = np.where(self.objective_values_matrix[0] == x[ind])
idy = np.where(self.objective_values_matrix[1] == y[ind])
idz = np.where(self.objective_values_matrix[2] == z[ind])
SolutionIndex = np.intersect1d(idx[0], np.intersect1d(idy[0], idz[0]))[0]
ThisSolution = self.legal_solutions[SolutionIndex]
print ThisSolution.description
for objIndex in range(0, len(self.objective_column_headers)):
if self.objective_column_headers[objIndex] == 'Flight time (days)' and self.TimeUnit == 0:
print 'Flight time (years)', ': ', ThisSolution.objective_values[objIndex] / 365.25
elif self.objective_column_headers[objIndex] == 'Launch epoch (MJD)' and self.EpochUnit == 0:
dt = datetime.datetime.fromtimestamp(wx.DateTimeFromJDN(ThisSolution.objective_values[objIndex] + 2400000.5).GetTicks())
print 'Launch Epoch (TDB Gregorian):', dt.strftime('%m/%d/%Y')
elif self.objective_column_headers[objIndex] == 'Thruster preference':
print 'Thruster type: ', ThisSolution.thruster
elif self.objective_column_headers[objIndex] == 'Launch vehicle preference':
print 'Launch vehicle: ', ThisSolution.launch_vehicle
else:
print self.objective_column_headers[objIndex], ': ', ThisSolution.objective_values[objIndex]
print '---------------------------------------------------------------------------------------------'
def _flags_fill(flags,N,M):
'''Expand a cube of True around singleton True values in flags array'''
idx = np.argwhere(flags == True)
max_idx = N**3 - 1
last = flags[max_idx]
# find where there is room for expansion in each direction
i = np.where(idx % N < N-1)[0]
j = np.where(idx % (N**2) < (N-1)*N)[0]
k = np.where(idx % (N**3) < (N-1)*N**2)[0]
# find room for expansion in multiple directions
ij = np.intersect1d(i,j)
ik = np.intersect1d(i,k)
jk = np.intersect1d(j,k)
ijk = np.intersect1d(ij,k)
# this is hardcoded for M=2 right now
flags[np.clip(idx[i]+1,0,max_idx)] = True # i+1
flags[np.clip(idx[j]+N,0,max_idx)] = True # j+1
flags[np.clip(idx[k]+N**2,0,max_idx)] = True # k+1
flags[np.clip(idx[ij]+1+N,0,max_idx)] = True # i+1, j+1
flags[np.clip(idx[ik]+1+N**2,0,max_idx)] = True # i+1, k+1
flags[np.clip(idx[jk]+N*(N+1),0,max_idx)] = True # j+1, k+1
flags[np.clip(idx[ijk]+1+N*(N+1),0,max_idx)] = True # i+1, j+1, k+1
# needed?
flags[max_idx] = last
return flags
def Find_AI(self,time, Gen,PL):
#Need to supply Tau and PL for a fast speed
#This is done by assuming at one PL value there should only be one tau value, then this occurs though changing Ai
#We also restrict the data to a 'Good Region'
if self.Width==None:
print 'Need to input Width in cm'
return False
self.Gen = Gen/self.Width
self.PL = PL
self.time = time
Max_gen = np.amax(self.Gen)
Max_Time = np.amax(self.time)
self.Limited_Index = np.where((self.Gen>=Max_gen*self.LowerLimit))[0]
Max_Index = np.where((self.Gen==Max_gen))[0]
Analysis_Upper = np.where((self.time>=self.time[Max_Index]))[0]
Analysis_Lower = np.where((self.time<=self.time[Max_Index]))[0]
self.Analysis_Upper = np.intersect1d(Analysis_Upper,self.Limited_Index)
self.Analysis_Lower = np.intersect1d(Analysis_Lower,self.Limited_Index)
res = minimize(self.Minimisation_Function,self.Ai,tol=1e-0)
self.Ai = abs(res.x)
self.Tau = self.Generalised_Lifetime(self.time,self.Gen,self.PL,self.Ai)
def split_set_by_indices(dataset, train_fold, valid_fold, test_fold):
n_trials = dataset.get_topological_view().shape[0]
# Make sure there are no overlaps and we have all possible trials
# assigned
assert np.intersect1d(valid_fold, test_fold).size == 0
assert np.intersect1d(train_fold, test_fold).size == 0
assert np.intersect1d(train_fold, valid_fold).size == 0
assert (set(np.concatenate((train_fold, valid_fold, test_fold))) ==
set(range(n_trials)))
train_set = DenseDesignMatrixWrapper(
topo_view=dataset.get_topological_view()[train_fold],
y=dataset.y[train_fold],
axes=dataset.view_converter.axes)
valid_set = DenseDesignMatrixWrapper(
topo_view=dataset.get_topological_view()[valid_fold],
y=dataset.y[valid_fold],
axes=dataset.view_converter.axes)
test_set = DenseDesignMatrixWrapper(
topo_view=dataset.get_topological_view()[test_fold],
y=dataset.y[test_fold],
axes=dataset.view_converter.axes)
# make ordered dict to make it easier to iterate, i.e. for logging
datasets = OrderedDict([('train', train_set), ('valid', valid_set),
('test', test_set)])
return datasets
def filter_lines(gr, max_gradient, min_flow_area):
"""Filter lines from the gridresultadmin and return 3 filtered sets:
:param gr: GridH5ResultAdmin
:param max_gradient: GridH5ResultAdmin.nodes to get waterlevels (s1) from
:param min_flow_area: the minimum flow area, in square meters
:returns: tuple of 3 filtered GridH5ResultAdmin.lines objects:
- 2D-2D lines that have flow_area >= min_flow_area and
gradient <= max_gradient
- 1D-2D lines that have flow_area >= min_flow_area
- 1D-2D lines that have flow_area >= min_flow_area and
gradient <= max_gradient
"""
lines_active = filter_min_flow_area(gr.lines, min_flow_area)
lines_valid = filter_max_gradient(gr.lines, gr.nodes, max_gradient)
lines2d2d_valid = gr.lines.subset('2D_ALL').filter(
id__in=np.intersect1d(lines_valid, lines_active)
)
lines1d2d_active = gr.lines.subset('1D2D').filter(
id__in=lines_active
)
lines1d2d_valid = gr.lines.subset('1D2D').filter(
id__in=np.intersect1d(lines_valid, lines_active)
)
return lines2d2d_valid, lines1d2d_active, lines1d2d_valid
def reverse_interpolate_two_array(value1, array1, value2, array2, delta1=0.1, delta2=0.1):
"""
Tries to reverse interpolate two vales from two arrays with the same dimensions, and finds a common index
for value1 and value2 in their respective arrays. the deltas define the search radius for a close value match
to the arrays.
:return: index1, index2
"""
tth_ind = np.argwhere(np.abs(array1 - value1) < delta1)
azi_ind = np.argwhere(np.abs(array2 - value2) < delta2)
tth_ind_ravel = np.ravel_multi_index((tth_ind[:, 0], tth_ind[:, 1]), dims=array1.shape)
azi_ind_ravel = np.ravel_multi_index((azi_ind[:, 0], azi_ind[:, 1]), dims=array2.shape)
common_ind_ravel = np.intersect1d(tth_ind_ravel, azi_ind_ravel)
result_ind = np.unravel_index(common_ind_ravel, dims=array1.shape)
while len(result_ind[0]) > 1:
if np.max(np.diff(array1)) > 0:
delta1 = np.max(np.diff(array1[result_ind]))
if np.max(np.diff(array2)) > 0:
delta2 = np.max(np.diff(array2[result_ind]))
tth_ind = np.argwhere(np.abs(array1[result_ind] - value1) < delta1)
azi_ind = np.argwhere(np.abs(array2[result_ind] - value2) < delta2)
print(result_ind)
common_ind = np.intersect1d(tth_ind, azi_ind)
result_ind = (result_ind[0][common_ind], result_ind[1][common_ind])
return result_ind[0], result_ind[1]
def check(lattice, chains):
for chain in chains:
chain = chain[chain[:,0]>=0]
# for monomer in chain:
# for j in xrange(0,chain.shape[0]-1):
# index = np.intersect1d(np.where((chain[:,0] == monomer[0]))[0], np.where((chain[:,1] == monomer[1]))[0])
# intersect = np.array([x for x in set(tuple(x) for x in chain) & set(tuple(x) for x in chain)])
intersect = [tuple(x) for x in chain]
dups = [item for item,count in collections.Counter(intersect).items() if count > 1]
# if index == j:
# print "\n"
if len(dups) > 0:
print "ERROR: Duplicate in chain"
print dups
index = np.intersect1d(np.where((chain[:,0] == dups[0][0])), np.where((chain[:,1] == dups[0][1])))
print index
#print "Monomer is: " + str(monomer) + "at index " + str(index)
for chain1, chain2 in itertools.combinations(chains,2):
chain1 = chain1[chain1[:,0]>=0]
chain2 = chain2[chain2[:,0]>=0]
array = np.concatenate((chain1, chain2), 0)
intersect2 = [tuple(x) for x in array]
dups2 = [item for item,count in collections.Counter(intersect2).items() if count > 1]
if len(dups2) > 0:
print "overlap"
print dups2
index2 = np.intersect1d(np.where((chain[:,0] == dups2[0][0])), np.where((chain[:,1] == dups2[0][1])))
print index2
return 0
def beforeTimestep(self, locationSDR, transitionSDR, featureSDR,
egocentricLocation, learn):
self.csvOut.writerow(("t",))
self.csvOut.writerow(("input", "newLocation"))
self.csvOut.writerow([json.dumps(locationSDR.tolist())])
self.csvOut.writerow([json.dumps(
[decoding
for decoding, sdr in self.exp.locations.iteritems()
if np.intersect1d(locationSDR, sdr).size == sdr.size])])
self.csvOut.writerow(("input", "deltaLocation"))
self.csvOut.writerow([json.dumps(transitionSDR.tolist())])
self.csvOut.writerow([json.dumps(
[decoding
for decoding, sdr in self.exp.transitions.iteritems()
if np.intersect1d(transitionSDR, sdr).size == sdr.size])])
self.csvOut.writerow(("input", "feature"))
self.csvOut.writerow([json.dumps(featureSDR.tolist())])
self.csvOut.writerow([json.dumps(
[k
for k, sdr in self.exp.features.iteritems()
if np.intersect1d(featureSDR, sdr).size == sdr.size])])
self.csvOut.writerow(("egocentricLocation",))
self.csvOut.writerow([json.dumps(egocentricLocation)])
def get_extrema(self, limits=(0.0, float('inf')), order=5):
"""Computes masks (i.e. arrays of indices) of the extrema of the force.
Parameters
----------
order: integer, optional
Number of neighboring points used to define an extremum;
default: 5.
Returns
-------
minima: 1D array of integers
Index of all minima.
maxima: 1D array of integers
Index of all maxima.
"""
minima = signal.argrelextrema(self.values, numpy.less_equal,
order=order)[0][:-1]
maxima = signal.argrelextrema(self.values, numpy.greater_equal,
order=order)[0][:-1]
mask = numpy.where(numpy.logical_and(self.times >= limits[0],
self.times <= limits[1]))[0]
minima = numpy.intersect1d(minima, mask, assume_unique=True)
maxima = numpy.intersect1d(maxima, mask, assume_unique=True)
# remove indices that are too close
minima = minima[numpy.append(True, minima[1:]-minima[:-1] > order)]
maxima = maxima[numpy.append(True, maxima[1:]-maxima[:-1] > order)]
return minima, maxima
def calc_number_density(gro_file, trj_file, top_file, area,
dim, box_range, n_bins, frame_range=None,maxs=None, mins=None):
"""
Calculate a 1-dimensional number density profile for each residue
Parameters
----------
gro_file: str
GROMACS '.gro' file to load
trj_file: str
Trajectory to load
top_file: str
GROMACS '.top' file to load
bin_width: int
Width (nm) of numpy histogram bins
dim: int
Dimension to calculate number density profile (0,1 or 2)
box_range: array
Range of coordinates in 'dim' to evaluate
frame_range: Python range() (optional)
Range of frames to calculate number density function over
maxs: array (optional)
Maximum coordinate to evaluate
mins: array (optional)
Minimum coordinate to evalute
Attributes
----------
"""
trj = md.load(trj_file, top=gro_file)
com_trj = make_comtrj(trj)
resnames = np.unique([x.name for x in
com_trj.topology.residues])
rho_list = list()
for resname in resnames:
sliced = com_trj.topology.select('resname {}'.format(resname))
trj_slice = com_trj.atom_slice(sliced)
if frame_range:
trj_slice = trj_slice[frame_range]
for i,frame in enumerate(trj_slice):
if maxs is None:
indices = [[atom.index for atom in compound.atoms]
for compound in
list(frame.topology.residues)]
else:
indices = np.intersect1d(
np.intersect1d(np.where(frame.xyz[-1, :, 0]
> mins[0]),
np.where(frame.xyz[-1, :, 0] < maxs[0])),
np.intersect1d(np.where(frame.xyz[-1, :, 1]
> box_range[0]),
np.where(frame.xyz[-1, :, 1] < box_range[1])))
if frame_range:
if i == 0:
x = np.histogram(frame.xyz[0,indices,dim].flatten(),
bins=n_bins, range=(box_range[0], box_range[1]))
rho = x[0]
bins = x[1]
else:
rho += np.histogram(frame.xyz[0, indices, dim].
flatten(),bins=n_bins, range=(box_range[0],
box_range[1]))[0]
else:
if i == 0:
x = np.histogram(frame.xyz[0,indices,dim].flatten(),
bins=n_bins, range=(box_range[0], box_range[1]))
rho = x[0]
bins = x[1]
else:
rho += np.histogram(frame.xyz[0, indices, dim].
flatten(),bins=n_bins, range=(box_range[0],
box_range[1]))[0]
rho = np.divide(rho, trj_slice.n_frames * area *
2 / n_bins)
rho_list.append(rho)
bin_list = bins[:-1]
return(rho_list, bin_list)
def test_make_dics(tmpdir, _load_forward, idx, mat_tol, vol_tol):
"""Test making DICS beamformer filters."""
# We only test proper handling of parameters here. Testing the results is
# done in test_apply_dics_timeseries and test_apply_dics_csd.
fwd_free, fwd_surf, fwd_fixed, fwd_vol = _load_forward
epochs, _, csd, _, label, vertices, source_ind = \
_simulate_data(fwd_fixed, idx)
with pytest.raises(RuntimeError, match='several sensor types'):
make_dics(epochs.info, fwd_surf, csd, label=label, pick_ori=None)
epochs.pick_types(meg='grad')
with pytest.raises(ValueError, match="Invalid value for the 'pick_ori'"):
make_dics(epochs.info, fwd_fixed, csd, pick_ori="notexistent")
with pytest.raises(ValueError, match='rank, if str'):
make_dics(epochs.info, fwd_fixed, csd, rank='foo')
with pytest.raises(TypeError, match='rank must be'):
make_dics(epochs.info, fwd_fixed, csd, rank=1.)
# Test if fixed forward operator is detected when picking normal
# orientation
with pytest.raises(ValueError, match='forward operator with free ori'):
make_dics(epochs.info, fwd_fixed, csd, pick_ori="normal")
# Test if non-surface oriented forward operator is detected when picking
# normal orientation
with pytest.raises(ValueError, match='oriented in surface coordinates'):
make_dics(epochs.info, fwd_free, csd, pick_ori="normal")
# Test if volume forward operator is detected when picking normal
# orientation
with pytest.raises(ValueError, match='oriented in surface coordinates'):
make_dics(epochs.info, fwd_vol, csd, pick_ori="normal")
# Test invalid combinations of parameters
with pytest.raises(ValueError, match='reduce_rank cannot be used with'):
make_dics(epochs.info, fwd_free, csd, inversion='single',
reduce_rank=True)
with pytest.raises(ValueError, match='not stable with depth'):
make_dics(epochs.info, fwd_free, csd, weight_norm='unit-noise-gain',
inversion='single', normalize_fwd=True)
# Sanity checks on the returned filters
n_freq = len(csd.frequencies)
vertices = np.intersect1d(label.vertices, fwd_free['src'][0]['vertno'])
n_verts = len(vertices)
n_orient = 3
n_channels = len(epochs.ch_names)
# Test return values
filters = make_dics(epochs.info, fwd_surf, csd, label=label, pick_ori=None,
weight_norm='unit-noise-gain', normalize_fwd=False)
assert filters['weights'].shape == (n_freq, n_verts * n_orient, n_channels)
assert np.iscomplexobj(filters['weights'])
assert filters['csd'] == csd
assert filters['ch_names'] == epochs.ch_names
assert_array_equal(filters['proj'], np.eye(n_channels))
assert_array_equal(filters['vertices'][0], vertices)
assert_array_equal(filters['vertices'][1], []) # Label was on the LH
assert filters['subject'] == fwd_free['src']._subject
assert filters['pick_ori'] is None
assert filters['n_orient'] == n_orient
assert filters['inversion'] == 'single'
assert not filters['normalize_fwd']
assert filters['weight_norm'] == 'unit-noise-gain'
assert 'DICS' in repr(filters)
assert 'subject "sample"' in repr(filters)
assert str(len(vertices)) in repr(filters)
assert str(n_channels) in repr(filters)
assert 'rank' not in repr(filters)
_test_weight_norm(filters)
# Test picking orientations. Also test weight norming under these different
# conditions.
filters = make_dics(epochs.info, fwd_surf, csd, label=label,
pick_ori='normal', weight_norm='unit-noise-gain',
normalize_fwd=False)
n_orient = 1
assert filters['weights'].shape == (n_freq, n_verts * n_orient, n_channels)
assert filters['n_orient'] == n_orient
_test_weight_norm(filters)
filters = make_dics(epochs.info, fwd_surf, csd, label=label,
pick_ori='max-power', weight_norm='unit-noise-gain',
normalize_fwd=False)
n_orient = 1
assert filters['weights'].shape == (n_freq, n_verts * n_orient, n_channels)
assert filters['n_orient'] == n_orient
_test_weight_norm(filters)
# From here on, only work on a single frequency
csd = csd[0]
# Test using a real-valued filter
filters = make_dics(epochs.info, fwd_surf, csd, label=label,
pick_ori='normal', real_filter=True)
assert not np.iscomplexobj(filters['weights'])
# Test forward normalization. When inversion='single', the power of a
# unit-noise CSD should be 1, even without weight normalization.
csd_noise = csd.copy()
inds = np.triu_indices(csd.n_channels)
# Using [:, :] syntax for in-place broadcasting
csd_noise._data[:, :] = np.eye(csd.n_channels)[inds][:, np.newaxis]
filters = make_dics(epochs.info, fwd_surf, csd_noise, label=label,
weight_norm=None, normalize_fwd=True)
w = filters['weights'][0][:3]
assert_allclose(np.diag(w.dot(w.conjugate().T)), 1.0, rtol=1e-6, atol=0)
# Test turning off both forward and weight normalization
filters = make_dics(epochs.info, fwd_surf, csd, label=label,
weight_norm=None, normalize_fwd=False)
w = filters['weights'][0][:3]
assert not np.allclose(np.diag(w.dot(w.conjugate().T)), 1.0,
rtol=1e-2, atol=0)
# Test neural-activity-index weight normalization. It should be a scaled
# version of the unit-noise-gain beamformer.
filters_nai = make_dics(
epochs.info, fwd_surf, csd, label=label, pick_ori='max-power',
weight_norm='nai', normalize_fwd=False)
w_nai = filters_nai['weights'][0]
filters_ung = make_dics(
epochs.info, fwd_surf, csd, label=label, pick_ori='max-power',
weight_norm='unit-noise-gain', normalize_fwd=False)
w_ung = filters_ung['weights'][0]
assert_allclose(np.corrcoef(np.abs(w_nai).ravel(),
np.abs(w_ung).ravel()), 1, atol=1e-7)
# Test whether spatial filter contains src_type
assert 'src_type' in filters
fname = op.join(str(tmpdir), 'filters-dics.h5')
filters.save(fname)
filters_read = read_beamformer(fname)
assert isinstance(filters, Beamformer)
assert isinstance(filters_read, Beamformer)
for key in ['tmin', 'tmax']: # deal with strictness of object_diff
setattr(filters['csd'], key, np.float(getattr(filters['csd'], key)))
assert object_diff(filters, filters_read) == ''
def histplot(xname=None, yname=None, snrcut=0, dolog2d=False, dolog1d=False,
xlbl=None, ylbl=None, shift=None, shiftlbl=None,
nbins=100, outname = '', extrema = [], cd = ''):
# xname and yname are the two necessary inputs
if cd != '':
if os.path.exists(cd) == 1:
print('Found {}, changing directory...'.format(cd))
os.chdir(cd)
else:
print('Directory {} doesn\'t exist, creating and changing...\n'.format(cd))
os.mkdir(cd)
os.chdir(cd)
####### Data aquisition and error modification #######
# Checks if file names are provided
if xname == None:
xname = input('No file specified, enter name of file for x-axis data: ')
if yname == None:
yname = input('No file specified, enter name of file for y-axis data: ')
xfile = fits.open(xname)
yfile = fits.open(yname)
print('\nOpening {0} as xfile and {1} as yfile\n'.format(xname,yname))
# Data extraction
xread = xfile[0].data.flatten()
yread = yfile[0].data.flatten()
# Reject NaN values
good = np.intersect1d(np.where(~np.isnan(xread))[0], np.where(~np.isnan(yread))[0])
xdata = xread[good]
ydata = yread[good]
print('Raw xdata ranges from {0} to {1}'.format(min(xdata),max(xdata)))
print('Raw ydata ranges from {0} to {1}\n'.format(min(ydata),max(ydata)))
# When snrcut > 0, error data is required, and file names are inputted next
if snrcut > 0:
# Redefine snrcut as a float variable for ease later
scut = float(snrcut)
# Automatically assigns error data file names following a pattern
xerrname = xname.replace('mom','emom')
yerrname = yname.replace('mom','emom')
# Checks and if necessary corrects error file names
if os.path.exists(xerrname) == False:
xerrname = input('Enter name of file for x-axis error data: ')
if os.path.exists(yerrname) == False:
yerrname = input('Enter name of file for y-axis error data: ')
xerrfile = fits.open(xerrname)
yerrfile = fits.open(yerrname)
print('Opening {0} as xerrfile and {1} as yerrfile\n'.format(xerrname,yerrname))
# Error data extraction
xerrdata = xerrfile[0].data.flatten()[good]
yerrdata = yerrfile[0].data.flatten()[good]
print('Raw xerrdata ranges from {0} to {1}'.format(min(xerrdata),max(xerrdata)))
print('Raw yerrdata ranges from {0} to {1}\n'.format(min(yerrdata),max(yerrdata)))
# Sets up arrays for detections and non-detections in each axis and the four posibilites for each point
xydet = np.intersect1d(np.where(xdata >= scut*xerrdata)[0],
np.where(ydata >= scut*yerrdata)[0])
xndet = np.intersect1d(np.where(xdata < scut*xerrdata)[0],
np.where(ydata >= scut*yerrdata)[0])
yndet = np.intersect1d(np.where(xdata >= scut*xerrdata)[0],
np.where(ydata < scut*yerrdata)[0])
xyndet = np.intersect1d(np.where(xdata < scut*xerrdata)[0],
np.where(ydata < scut*yerrdata)[0])
# Replace non-detections with upper limit values
for i in np.concatenate([xndet,xyndet]):
xdata[i] = scut*xerrdata[i]
for i in np.concatenate([yndet,xyndet]):
ydata[i] = scut*yerrdata[i]
print('Number of detections: {0}'.format(len(xydet)))
print('Non-detections only in xdata: {0}'.format(len(xndet)))
print('Non-detections only in ydata: {0}'.format(len(yndet)))
print('Non-detections in both xdata and ydata: {0}\n'.format(len(xyndet)))
print('With upper limits xdata ranges from {0} to {1}'.format(min(xdata),max(xdata)))
print('With upper limits ydata ranges from {0} to {1}'.format(min(ydata),max(ydata)))
# Modify raw/error-corrected data into log scale if desired
if dolog2d == True:
pos = np.intersect1d(np.where(xdata>0)[0], np.where(ydata>0)[0])
x = np.log10(xdata[pos])
y = np.log10(ydata[pos])
else:
x = xdata
y = ydata
# Max and min boundaries for the graphs
if extrema == []:
xmin = math.floor(min(x))
ymin = math.floor(min(y))
xmax = math.ceil(max(x))
ymax = math.ceil(max(y))
else:
xmin = extrema[0]
ymin = extrema[1]
xmax = extrema[2]
ymax = extrema[3]
# finitex = np.isfinite(x)
# finitey = np.isfinite(y)
#
# fordeletion = []
# for i in range(len(x)):
# if (finitex[i] == 0 or finitey[i] == 0):
# fordeletion.append(i)
#
# x = np.delete(x, fordeletion)
# y = np.delete(y, fordeletion)
####### Set up geometry of three plots #######
# Values pulled directly from histomom.py
left, width = 0.12, 0.55
bottom, height = 0.12, 0.55
bottom_h = left_h = left+width+0.02
img = [left, bottom, width, height] # Dimensions of temperature plot
rect_histx = [left, bottom_h, width, 0.2] # Dimensions of x-histogram
rect_histy = [left_h, bottom, 0.2, height] # Dimensions of y-histogram
fig = plt.figure(1, figsize=(9.5,9))
ax = plt.axes(img) # Temperature plot
axHistx = plt.axes(rect_histx) # x-histogram
axHisty = plt.axes(rect_histy) # y-histogram
# Remove the inner axes numbers of this histograms to save space
nullfmt = tck.NullFormatter()
axHistx.xaxis.set_major_formatter(nullfmt)
axHisty.yaxis.set_major_formatter(nullfmt)
####### Plot 2-D histogram #######
xbins = np.linspace(xmin, xmax, nbins)
ybins = np.linspace(ymin, ymax, nbins)
# Tuple[s] setup for data plotting
### Transposing counts after definition seems to be important to data processing, not yet understood
if snrcut > 0:
# 0 -> xydet, 1 -> xndet, 2 -> yndet, 3 -> xyndet
counts0, xedge0, yedge0 = np.histogram2d(x[xydet], y[xydet], bins=(xbins,ybins))
counts1, xedge1, yedge1 = np.histogram2d(x[xndet], y[xndet], bins=(xbins,ybins))
counts2, xedge2, yedge2 = np.histogram2d(x[yndet], y[yndet], bins=(xbins,ybins))
counts3, xedge3, yedge3 = np.histogram2d(x[xyndet], y[xyndet], bins=(xbins,ybins))
counts0 = np.transpose(counts0)
counts1 = np.transpose(counts1)
counts2 = np.transpose(counts2)
counts3 = np.transpose(counts3)
else:
# Also using 0 to streamline later plotting commands
counts0, xedge0, yedge0 = np.histogram2d(x, y, bins=(xbins,ybins))
counts0 = np.transpose(counts0)
# Mesh edges to create coupled matricies
xmesh, ymesh = np.meshgrid(xedge0, yedge0)
### Contour levels, need to figure out how to customize
### Change to np.logspace, add new input variable
### May need to compare the three ndet; whicever one has the largest amount of values, set that
levels = [25, 100, 400, 1600, 6400, 25600]
# Set limits on axes
ax.set_xlim(xedge0[0], xedge0[-1])
ax.set_ylim(yedge0[0], yedge0[-1])
# Colorizes detections
img = ax.pcolormesh(xmesh, ymesh, counts0, norm=LogNorm(), cmap=plt.cm.gist_ncar_r)
### If logscale on colorbar is not desired, change norm=None
# Keeps contour boundaries in range of x and y edges
extent = [xedge0[0], xedge0[-1], yedge0[0], yedge0[-1]]
# Build contours
#cset0 = ax.contour(counts0, levels, colors='green', extent=extent)
if snrcut > 0:
cset1 = ax.contour(counts1, levels, colors='black', extent=extent)
cset2 = ax.contour(counts2, levels, colors='blue', extent=extent)
cset3 = ax.contour(counts3, levels, colors='red', extent=extent)
### Section of histomom.py that is commented out
### Unknown reason why
# cset1.collections[0].set_label('x non-det')
# cset2.collections[0].set_label('y non-det')
# cset3.collections[0].set_label('x and y non-det')
# ax.clabel(cset1, inline=True, colors='k', fontsize=8)
# legend = plt.legend(loc='upper left', fontsize='medium')
### End of commented section
####### Plot 2-D labels/ticks #######
if xlbl is None:
xlbl = xname
if ylbl is None:
ylbl = yname
# Name labels
if dolog2d == True:
xlabel = 'log ({0})'.format(xlbl)
ylabel = 'log ({0})'.format(ylbl)
else:
xlabel = xlbl
ylabel = ylbl
ax.set_xlabel(xlabel, fontsize=14, labelpad=10)
ax.set_ylabel(ylabel, fontsize=14)
# Format ticks
ticklabels = ax.get_xticklabels()
for label in ticklabels:
label.set_fontsize(10)
label.set_family('serif')
ticklabels = ax.get_yticklabels()
for label in ticklabels:
label.set_fontsize(10)
label.set_family('serif')
####### Plot 1-D histogram #######
### May need to add another log scale somewhere
# Set limits
axHistx.set_xlim(xmin, xmax)
axHisty.set_ylim(ymin, ymax)
# Set bins
nxbins = np.arange(xmin, xmax, (xmax-xmin)/nbins)
nybins = np.arange(ymin, ymax, (ymax-ymin)/nbins)
# Plot histograms
if dolog1d == True:
if snrcut > 0:
axHistx.hist(x[xydet], bins=nxbins, color='blue')
axHisty.hist(y[xydet], bins=nybins, orientation='horizontal', color='red')
axHistx.set_yscale('log')
axHisty.set_xscale('log')
else:
axHistx.hist(x, bins=nxbins, color='blue')
axHisty.hist(y, bins=nybins, orientation='horizontal', color='red')
axHistx.set_yscale('log')
axHisty.set_xscale('log')
else:
if snrcut > 0:
axHistx.hist(x[xydet], bins=nxbins, color='blue')
axHisty.hist(y[xydet], bins=nybins, orientation='horizontal', color='red')
else:
axHistx.hist(x, bins=nxbins, color='blue')
axHisty.hist(y, bins=nybins, orientation='horizontal', color='red')
####### Plot 1-D ticks #######
# Format ticks
ticklabels = axHistx.get_yticklabels()
for label in ticklabels:
label.set_fontsize(10)
label.set_family('serif')
ticklabels = axHisty.get_xticklabels()
for label in ticklabels:
label.set_fontsize(10)
label.set_family('serif')
# Set location
axHistx.yaxis.set_major_locator(tck.MaxNLocator(4))
axHisty.xaxis.set_major_locator(tck.MaxNLocator(4))
####### Miscellaneous #######
### Arbitrary line plotted for limit I think?
z = np.linspace(-10, 300)
ax.plot(z, z)
if shift is not None:
ax.plot(z, z-shift, color='red', linewidth=4, alpha=0.5)
ax.text((xmax-0.0*(xmax-xmin)), (ymax-shift-0.08*(ymax-ymin)), shiftlbl,
horizontalalignment='right', color='r', rotation=45, size=14)
####### Output file name #######
if outname == '':
figname = input('Enter output file name: ')
else:
figname = outname
figname += '.pdf'
plt.savefig(figname, bbox_inches='tight')
plt.close()
def fit_transform(self, X, y):
"""Create meta-features for the training dataset.
Parameters
----------
X : DataFrame, shape = [n_samples, n_features]
The training dataset.
y : pandas series of shape = [n_samples, ]
The target
Returns
-------
X_transform : DataFrame, shape = [n_samples, n_features*int(copy)+n_metafeatures]
Returns the transformed training dataset.
"""
### sanity checks
if ((type(X) != pd.SparseDataFrame) & (type(X) != pd.DataFrame)):
raise ValueError("X must be a DataFrame")
if (type(y) != pd.core.series.Series):
raise ValueError("y must be a Series")
cv = KFold(n_splits=self.n_folds,
shuffle=True,
random_state=self.random_state)
preds = pd.DataFrame([], index=y.index)
if (self.verbose):
print("")
print(
"[=============================================================================] LAYER [===================================================================================]"
)
print("")
for c, reg in enumerate(self.base_estimators):
if (self.verbose):
print("> fitting estimator n°" + str(c + 1) + " : " +
str(reg.get_params()) + " ...")
print("")
y_pred = cross_val_predict(
estimator=reg, X=X, y=y, cv=cv
) #for each base estimator, we create the meta feature on train set
preds["est" + str(c + 1)] = y_pred
reg.fit(X,
y) # and we refit the base estimator on entire train set
layer = 1
while (len(
np.intersect1d(
X.columns,
["layer" + str(layer) + "_" + s
for s in preds.columns])) > 0):
layer = layer + 1
preds.columns = ["layer" + str(layer) + "_" + s for s in preds.columns]
self.__fittransformOK = True
if (self.copy == True):
return pd.concat([X, preds],
axis=1) #we keep also the initial features
else:
return preds #we keep only the meta features
# convert to binary
seq1 = np.sign(seq1 - 1.5)
w = np.loadtxt('w.txt')
cols_active = np.loadtxt('cols_selected.txt').astype(int)
cols_conserved = np.setdiff1d(np.arange(28 * 28), cols_active)
#=========================================================================================
# select hidden
#hidden = np.loadtxt('cols_hidden.dat').astype(int)
#n_hidden = len(hidden)
# select hidden as random
n_hidden = 100
hidden = np.random.choice(np.arange(28 * 28), n_hidden, replace=False)
hidden_active = np.intersect1d(hidden, cols_active)
hidden_conserved = np.intersect1d(hidden, cols_conserved)
n_hidden_active = len(hidden_active)
n_hidden_conserved = len(hidden_conserved)
print('n_hidden_active:', len(hidden_active))
#n_hidden_active = 16
#n_hidden_conserved = 184
# hidden from active cols
#cols_active = np.loadtxt('cols_selected.txt').astype(int)
#hidden_active = np.random.choice(cols_active,n_hidden_active,replace=False)
#print(len(hidden_active))
# hidden from conserved cols
#cols_conserved = np.setdiff1d(np.arange(28*28),cols_active)
def similarity_SPLIF(reference, query, rmsd_cutoff=1.):
"""Calculates similarity between structural interaction fingerprints,
based on doi:http://pubs.acs.org/doi/abs/10.1021/ci500319f.
Parameters
----------
reference, query: numpy.array
SPLIFs, which are compared in order to determine similarity.
rmsd_cutoff : int (default = 1)
Specific treshold for which, bits are considered as fully matching.
Returns
-------
SimilarityScore : float
Similarity between given fingerprints.
"""
# intersection of reference and query hashed atoms
index = np.intersect1d(reference['hash'], query['hash'])
ref_intersection = reference[np.where(np.in1d(reference['hash'], index))]
ref_group_intersection = np.split(ref_intersection, np.searchsorted(
ref_intersection['hash'], index[1:])) # reference
query_intersection = query[np.where(np.in1d(query['hash'], index))]
query_group_intersection = np.split(query_intersection, np.searchsorted(
query_intersection['hash'], index[1:])) # query
numla = 0 # number of unique matching ligand atoms
nula = 0 # number of unique ligand atoms
numpa = 0 # number of unique matching protein atoms
nupa = 0 # number of unique protein atoms
def combinatorial_rmsd(reference, query):
"""Calculates root mean square deviation between groups of points. It
takes two matrices of shapes e.g (2, 5, 3) and (4, 5, 3) -> (2, 4)."""
return np.sqrt(np.nansum(np.mean(
(reference[:, np.newaxis, ...] - query)**2, axis=-1), axis=-1))
for pair in range(len(ref_group_intersection)):
# reference protein-ligand pair
ref_pair = ref_group_intersection[pair]
# query protein-ligand pair
query_pair = query_group_intersection[pair]
ref_ligand = ref_pair['ligand_coords']
ref_protein = ref_pair['protein_coords']
query_ligand = query_pair['ligand_coords']
query_protein = query_pair['protein_coords']
rmsd_ligand = combinatorial_rmsd(ref_ligand, query_ligand)
rmsd_protein = combinatorial_rmsd(ref_protein, query_protein)
n_matching = ((rmsd_ligand < rmsd_cutoff) &
(rmsd_protein < rmsd_cutoff)).sum()
numla += n_matching
numpa += n_matching
nula += (rmsd_ligand < rmsd_cutoff).sum()
nupa += (rmsd_protein < rmsd_cutoff).sum()
if nula == 0 or nupa == 0:
return 0.
else:
return np.sqrt((numla / nula) * (numpa / nupa))
]
megapat = bottlenecks + embayments + depressions + bumps
paths = ['embayments/', 'depressions/', 'bottlenecks/', 'bumps/']
loadpath = './Models/'
fs = 15
ls = 12
fig, ax = plt.subplots(3, 1)
ax[0].tick_params(axis='both', which='major', labelsize=ls)
ax[1].tick_params(axis='both', which='major', labelsize=ls)
ax[2].tick_params(axis='both', which='major', labelsize=ls)
for p in megapat:
z = np.nonzero(megapat == np.intersect1d(megapat, p))[0]
for r in paths:
try:
mods = glob.glob(loadpath + r + p)
mods.sort(key=os.path.getmtime)
md = loadmodel(mods[0])
except:
continue
if z == 0:
palette = 'Gold'
marker = '-'
if z == 1:
palette = 'Orange'
def test_tf_dics(_load_forward):
"""Test 5D time-frequency beamforming based on DICS."""
fwd_free, fwd_surf, fwd_fixed, _, label = _load_forward
epochs, _, _, source_vertno = _simulate_data(fwd_fixed)
vertices = np.intersect1d(label.vertices, fwd_free['src'][0]['vertno'])
source_ind = vertices.tolist().index(source_vertno)
reg = 1 # Lots of regularization for our toy dataset
tmin = 0
tmax = 9
tstep = 4
win_lengths = [5, 5]
frequencies = [10, 20]
freq_bins = [(8, 12), (18, 22)]
with pytest.raises(RuntimeError, match='several sensor types'):
stcs = tf_dics(epochs,
fwd_surf,
None,
tmin,
tmax,
tstep,
win_lengths,
freq_bins=freq_bins,
frequencies=frequencies,
decim=10,
reg=reg,
label=label)
epochs.pick_types(meg='grad')
# Compute DICS for two time windows and two frequencies
for mode in ['fourier', 'multitaper', 'cwt_morlet']:
stcs = tf_dics(epochs,
fwd_surf,
None,
tmin,
tmax,
tstep,
win_lengths,
mode=mode,
freq_bins=freq_bins,
frequencies=frequencies,
decim=10,
reg=reg,
label=label)
# Did we find the true source at 20 Hz?
assert np.argmax(stcs[1].data[:, 0]) == source_ind
assert np.argmax(stcs[1].data[:, 1]) == source_ind
# 20 Hz power should decrease over time
assert stcs[1].data[source_ind, 0] > stcs[1].data[source_ind, 1]
# 20 Hz power should be more than 10 Hz power at the true source
assert stcs[1].data[source_ind, 0] > stcs[0].data[source_ind, 0]
# Manually compute source power and compare with the last tf_dics result.
source_power = []
time_windows = [(0, 5), (4, 9)]
for time_window in time_windows:
csd = csd_morlet(epochs,
frequencies=[frequencies[1]],
tmin=time_window[0],
tmax=time_window[1],
decim=10)
csd = csd.sum()
csd._data /= csd.n_fft
filters = make_dics(epochs.info, fwd_surf, csd, reg=reg, label=label)
stc_source_power, _ = apply_dics_csd(csd, filters)
source_power.append(stc_source_power.data)
# Comparing tf_dics results with dics_source_power results
assert_allclose(stcs[1].data, np.array(source_power).squeeze().T, atol=0)
# Test using noise csds. We're going to use identity matrices. That way,
# since we're using unit-noise-gain weight normalization, there should be
# no effect.
stcs = tf_dics(epochs,
fwd_surf,
None,
tmin,
tmax,
tstep,
win_lengths,
mode='cwt_morlet',
frequencies=frequencies,
decim=10,
reg=reg,
label=label,
normalize_fwd=False,
weight_norm='unit-noise-gain')
noise_csd = csd.copy()
inds = np.triu_indices(csd.n_channels)
# Using [:, :] syntax for in-place broadcasting
noise_csd._data[:, :] = 2 * np.eye(csd.n_channels)[inds][:, np.newaxis]
noise_csd.n_fft = 2 # Dividing by n_fft should yield an identity CSD
noise_csds = [noise_csd, noise_csd] # Two frequency bins
stcs_norm = tf_dics(epochs,
fwd_surf,
noise_csds,
tmin,
tmax,
tstep,
win_lengths,
mode='cwt_morlet',
frequencies=frequencies,
decim=10,
reg=reg,
label=label,
normalize_fwd=False,
weight_norm='unit-noise-gain')
assert_allclose(stcs_norm[0].data, stcs[0].data, atol=0)
assert_allclose(stcs_norm[1].data, stcs[1].data, atol=0)
# Test invalid parameter combinations
with pytest.raises(ValueError, match='fourier.*freq_bins" parameter'):
tf_dics(epochs,
fwd_surf,
None,
tmin,
tmax,
tstep,
win_lengths,
mode='fourier',
freq_bins=None)
with pytest.raises(ValueError, match='cwt_morlet.*frequencies" param'):
tf_dics(epochs,
fwd_surf,
None,
tmin,
tmax,
tstep,
win_lengths,
mode='cwt_morlet',
frequencies=None)
# Test if incorrect number of noise CSDs is detected
with pytest.raises(ValueError, match='One noise CSD object expected per'):
tf_dics(epochs,
fwd_surf, [noise_csds[0]],
tmin,
tmax,
tstep,
win_lengths,
freq_bins=freq_bins)
# Test if freq_bins and win_lengths incompatibility is detected
with pytest.raises(ValueError, match='One time window length expected'):
tf_dics(epochs,
fwd_surf,
None,
tmin,
tmax,
tstep,
win_lengths=[0, 1, 2],
freq_bins=freq_bins)
# Test if time step exceeding window lengths is detected
with pytest.raises(ValueError, match='Time step should not be larger'):
tf_dics(epochs,
fwd_surf,
None,
tmin,
tmax,
tstep=0.15,
win_lengths=[0.2, 0.1],
freq_bins=freq_bins)
# Test if incorrect number of n_ffts is detected
with pytest.raises(ValueError, match='When specifying number of FFT'):
tf_dics(epochs,
fwd_surf,
None,
tmin,
tmax,
tstep,
win_lengths,
freq_bins=freq_bins,
n_ffts=[1])
# Test if incorrect number of mt_bandwidths is detected
with pytest.raises(ValueError, match='When using multitaper mode and'):
tf_dics(epochs,
fwd_surf,
None,
tmin,
tmax,
tstep,
win_lengths=win_lengths,
freq_bins=freq_bins,
mode='multitaper',
mt_bandwidths=[20])
# Test if subtracting evoked responses yields NaN's, since we only have one
# epoch. Suppress division warnings.
assert len(epochs) == 1, len(epochs)
with np.errstate(invalid='ignore'):
stcs = tf_dics(epochs,
fwd_surf,
None,
tmin,
tmax,
tstep,
win_lengths,
mode='cwt_morlet',
frequencies=frequencies,
subtract_evoked=True,
reg=reg,
label=label,
decim=20)
assert np.all(np.isnan(stcs[0].data))
print("Prob is {0}".format(
np.array(difference).shape[0] / transactionsData.shape[0]))
print("Prob is {0}".format((transactionsData.shape[0] - intersect.shape[0]) /
transactionsData.shape[0]))
friendDegree = [[x, len(friendDegreeSet[x])] for x in friendDegreeSet.keys()]
friendDegree = sorted(friendDegree, key=lambda d: d[1], reverse=True)
transDegree = [[x, transDegreeSet[x]] for x in transDegreeSet.keys()]
transDegree = sorted(transDegree, key=lambda d: d[1], reverse=True)
f = [0.05 * x for x in range(1, 11)]
r_TFf = [
float(
len(
np.intersect1d(friendDegree[:int((len(friendDegree) * i))],
transDegree[:int((len(transDegree) * i))])) /
float(len(friendDegree[:int((len(friendDegree) * i))]))) for i in f
]
r_FTf = [
float(
len(
np.intersect1d(transDegree[:int((len(transDegree) * i))],
friendDegree[:int((len(friendDegree) * i))])) /
float(len(transDegree[:int((len(transDegree) * i))]))) for i in f
]
plt.figure(1)
plt.plot(f, r_TFf, 'red', label="$r_{TF}(f)$")
plt.plot(f, r_FTf, 'blue', label="$r_{FT}(f)$")
plt.legend()
def treat_missing_joints(video):
## OPENPOSE DATASET
normal_vectors_dict = {
"head" : (0, 1),
"body" : (1, 8),
"legs" : (8, 1),
"feet" : (8, 1)
}
joints_pairs = {
0 : (17, 18),
17 : 18,
18 : 17,
15 : 16,
16 : 15,
2 : 5,
5 : 2,
3 : 6,
6 : 3,
4 : 7,
7 : 4,
9 : 12,
12 : 9,
10 : 13,
13 : 10,
11 : 14,
14 : 11,
22 : 19,
19 : 22,
23 : 20,
20 : 23,
21 : 24,
24 : 21
}
nvec_to_joints = {
17 : "head",
15 : "head",
16 : "head",
18 : "head",
4 : "body",
3 : "body",
2 : "body",
5 : "body",
6 : "body",
7 : "body",
11 : "legs",
10 : "legs",
9 : "legs",
12 : "legs",
13 : "legs",
14 : "legs",
23 : "feet" ,
22 : "feet" ,
24 : "feet",
21 : "feet",
19 : "feet",
20 : "feet"
}
for joint, sibling in joints_pairs.items():
good_frames = np.where(video[:, joint, :].any(axis=1))[0]
if len(good_frames) == 0:
if joint == 0: ## Case when cant find joint 0.
sibling0, sibling1 = sibling
sibling0_good_frames = np.where(video[:, sibling0, :].any(axis=1))[0]
sibling1_good_frames = np.where(video[:, sibling1, :].any(axis=1))[0]
frame_idx = np.intersect1d(sibling0_good_frames, sibling1_good_frames)[0]
video[frame_idx, 0, :] = (video[frame_idx, sibling0, :] + video[frame_idx, sibling1, :]) / 2
else:
sibling_good_frames = np.where(video[:, sibling, :].any(axis=1))[0]
if len(sibling_good_frames) > 0: ## No treatment for missing joints with missing siblings.
## Point to mirror
sibling_frame = sibling_good_frames[0]
sibling_joint = video[sibling_frame, sibling, :]
skeleton_group = nvec_to_joints[joint]
p0_idx , p1_idx = normal_vectors_dict[skeleton_group]
## Normal vec(N)
p0 = video[sibling_frame, p0_idx, :]
p1 = video[sibling_frame, p1_idx, :]
n_vec = p1 - p0
n_vec = n_vec/np.linalg.norm(n_vec) ## Norm 1 normal vector
## Incidence vector(D)
d_vec = p0 - sibling_joint
## Reflection vector(R) | R = D - 2(D.N)N
r_vec = d_vec - 2*np.dot(d_vec, n_vec)*n_vec
## Reflected JOINT
reflected_joint = p0 + r_vec
## Case when body is perpendicular to camera frame.
if np.allclose(reflected_joint, sibling_joint):
reflected_joint[0] = reflected_joint[0] + 1
video[sibling_frame, joint, :] = reflected_joint
else:
if joint == 15:
joint0, joint1 = (17, 0)
joint0_good_frames = np.where(video[:, joint0, :].any(axis=1))[0]
joint1_good_frames = np.where(video[:, joint1, :].any(axis=1))[0]
frame_idx = np.intersect1d(joint0_good_frames, joint1_good_frames)[0]
mean_horizontal = (video[frame_idx, joint0, 0] + video[frame_idx, joint1, 0])/2
half_dist = np.abs(mean_horizontal - video[frame_idx, joint0, 0])
video[frame_idx, joint, 0] = mean_horizontal
video[frame_idx, joint, 1] = video[frame_idx, joint0, 1] + half_dist
if joint == 16:
joint0, joint1 = (18, 0)
joint0_good_frames = np.where(video[:, joint0, :].any(axis=1))[0]
joint1_good_frames = np.where(video[:, joint1, :].any(axis=1))[0]
frame_idx = np.intersect1d(joint0_good_frames, joint1_good_frames)[0]
mean_horizontal = (video[frame_idx, joint0, 0] + video[frame_idx, joint1, 0])/2
half_dist = np.abs(mean_horizontal - video[frame_idx, joint0, 0])
video[frame_idx, joint, 0] = mean_horizontal
video[frame_idx, joint, 1] = video[frame_idx, joint0, 1] + half_dist
return video
if posx not in ('TIME', 'FSC-A'):
ax.set_xscale('log')
if posy not in ('TIME', 'FSC-A'):
ax.set_yscale('log')
ax.grid(True)
if 'antibodies' in config[tissue]:
if posx in config[tissue]['antibodies']:
ax.set_xlabel(config[tissue]['antibodies'][posx])
if posy in config[tissue]['antibodies']:
ax.set_ylabel(config[tissue]['antibodies'][posy])
if args.with_comparison:
for ipl, (chname, abname, gname) in enumerate(zip(ch_comp, ab_comp, gene_comp)):
ind = np.intersect1d(
facs_datum['index_data'].index,
ds.counts.columns,
)
x = facs_datum['index_data'].loc[ind, chname]
y = ds.counts.loc[gname, ind] + ds.counts.pseudocount
ctype = facs_datum['index_data'].loc[ind, 'cell type call']
df = pd.concat([x, y, ctype], axis=1)
rho = spearmanr(x, y)[0]
if ('xlim' in config[tissue]) and (chname in config[tissue]['xlim']):
xlim = config[tissue]['xlim'][chname]
elif ('ylim' in config[tissue]) and (chname in config[tissue]['ylim']):
xlim = config[tissue]['ylim'][chname]
else:
xlim = (1e1, 1e6)
ax = axs[ipl + int(has_dead) + 2 + len(config[tissue]['plots'])]
for qual, datum in df.groupby('cell type call'):
datum.plot(
def test_apply_dics_timeseries(_load_forward):
"""Test DICS applied to timeseries data."""
fwd_free, fwd_surf, fwd_fixed, fwd_vol, label = _load_forward
epochs, evoked, csd, source_vertno = _simulate_data(fwd_fixed)
vertices = np.intersect1d(label.vertices, fwd_free['src'][0]['vertno'])
source_ind = vertices.tolist().index(source_vertno)
reg = 5 # Lots of regularization for our toy dataset
with pytest.raises(RuntimeError, match='several sensor types'):
make_dics(evoked.info, fwd_surf, csd)
evoked.pick_types(meg='grad')
multiple_filters = make_dics(evoked.info,
fwd_surf,
csd,
label=label,
reg=reg)
# Sanity checks on the resulting STC after applying DICS on evoked
stcs = apply_dics(evoked, multiple_filters)
assert isinstance(stcs, list)
assert len(stcs) == len(multiple_filters['weights'])
assert_array_equal(stcs[0].vertices[0], multiple_filters['vertices'][0])
assert_array_equal(stcs[0].vertices[1], multiple_filters['vertices'][1])
assert_allclose(stcs[0].times, evoked.times)
# Applying filters for multiple frequencies on epoch data should fail
with pytest.raises(ValueError, match='computed for a single frequency'):
apply_dics_epochs(epochs, multiple_filters)
# From now on, only apply filters with a single frequency (20 Hz).
csd20 = csd.pick_frequency(20)
filters = make_dics(evoked.info, fwd_surf, csd20, label=label, reg=reg)
# Sanity checks on the resulting STC after applying DICS on epochs.
# Also test here that no warnings are thrown - implemented to check whether
# src should not be None warning occurs
with pytest.warns(None) as w:
stcs = apply_dics_epochs(epochs, filters)
assert len(w) == 0
assert isinstance(stcs, list)
assert len(stcs) == 1
assert_array_equal(stcs[0].vertices[0], filters['vertices'][0])
assert_array_equal(stcs[0].vertices[1], filters['vertices'][1])
assert_allclose(stcs[0].times, epochs.times)
# Did we find the source?
stc = (stcs[0]**2).mean()
assert np.argmax(stc.data) == source_ind
# Apply filters to evoked
stc = apply_dics(evoked, filters)
stc = (stc**2).mean()
assert np.argmax(stc.data) == source_ind
# Test if wrong channel selection is detected in application of filter
evoked_ch = cp.deepcopy(evoked)
evoked_ch.pick_channels(evoked_ch.ch_names[:-1])
with pytest.raises(ValueError, match='MEG 2633 which is not present'):
apply_dics(evoked_ch, filters)
# Test whether projections are applied, by adding a custom projection
filters_noproj = make_dics(evoked.info, fwd_surf, csd20, label=label)
stc_noproj = apply_dics(evoked, filters_noproj)
evoked_proj = evoked.copy()
p = compute_proj_evoked(evoked_proj, n_grad=1, n_mag=0, n_eeg=0)
proj_matrix = make_projector(p, evoked_proj.ch_names)[0]
evoked_proj.info['projs'] += p
filters_proj = make_dics(evoked_proj.info, fwd_surf, csd20, label=label)
assert_array_equal(filters_proj['proj'], proj_matrix)
stc_proj = apply_dics(evoked_proj, filters_proj)
assert np.any(np.not_equal(stc_noproj.data, stc_proj.data))
# Test detecting incompatible projections
filters_proj['proj'] = filters_proj['proj'][:-1, :-1]
with pytest.raises(ValueError, match='operands could not be broadcast'):
apply_dics(evoked_proj, filters_proj)
# Test returning a generator
stcs = apply_dics_epochs(epochs, filters, return_generator=False)
stcs_gen = apply_dics_epochs(epochs, filters, return_generator=True)
assert_array_equal(stcs[0].data, next(stcs_gen).data)
# Test computing timecourses on a volume source space
filters_vol = make_dics(evoked.info, fwd_vol, csd20, reg=reg)
stc = apply_dics(evoked, filters_vol)
stc = (stc**2).mean()
assert np.argmax(stc.data) == 3851 # TODO: don't make this hard coded
# check whether a filters object without src_type throws expected warning
del filters_vol['src_type'] # emulate 0.16 behaviour to cause warning
with pytest.warns(RuntimeWarning, match='filter does not contain src_typ'):
apply_dics_epochs(epochs, filters_vol)
def unify_label(matfiles, savename):
atmp = []
flab = [] #final labels
ftrjID = [] #final trjID
for matidx in range(len(matfiles) - 1):
if matidx == 0:
L1 = loadmat(matfiles[matidx])['label'][0]
L1 = L1 + 1 # class label starts from 1 instead of 0
M1 = loadmat(matfiles[matidx])['trjID'][0]
else:
L1 = L2
M1 = M2
L2 = loadmat(matfiles[matidx + 1])['label'][0]
L2 = L2 + 1
M2 = loadmat(matfiles[matidx + 1])['trjID'][0]
L1max = max(L1) ## not duplicate
L2[:] = L2 + L1max + 1 # to make sure there're no duplicate labels
commonidx = np.intersect1d(
M1, M2) #trajectories existing in both 2 trucations
print('L1 : {0}, L2 : {1} ,common term : {2}').format(
len(np.unique(L1)), len(np.unique(L2)), len(commonidx))
for i in commonidx:
if i not in atmp:
# label1 = L1[np.where((M1 == i)!=0)[0][0]] # use np.arange(len(M1))[M1 == i]
# label2 = L2[np.where((M2 == i)!=0)[0][0]]
label1 = L1[M1 == i][0]
label2 = L2[M2 == i][0]
if label2 <= L1max: # already been united with a previous chunk label1
# print "previous split, now as one."
L1[M1 == i] = label2
continue
idx1 = np.where(L1 == label1)
idx2 = np.where(L2 == label2)
tmp1 = np.intersect1d(
M1[idx1],
commonidx) # appear in both trunks and label is label1
tmp2 = np.intersect1d(M2[idx2], commonidx)
# pdb.set_trace()
L1idx = list(idx1[0])
L2idx = list(idx2[0])
# diff1 = np.setdiff1d(tmp1,tmp2) # this difference only accounts for elements in tmp1
# diff = np.union1d(np.setdiff1d(tmp1,np.intersect1d(tmp1,tmp2)) , np.setdiff1d(tmp2,np.intersect1d(tmp1,tmp2) ))
atmp = np.unique(np.hstack((tmp1, tmp2)))
L1[L1idx] = label1 ## keep the first appearing label
L2[L2idx] = label1
if matidx == 0:
flab[:] = flab + list(L1)
ftrjID[:] = ftrjID + list(M1)
flab[:] = flab + list(L2)
ftrjID[:] = ftrjID + list(M2)
#== eliminate duplicate part ==
data = np.vstack((ftrjID, flab))
result = data[:, data[0].argsort()]
labels = list(result[1])
savetrjID = list(result[0])
dpidx = np.where(
np.diff(sorted(savetrjID)) == 0)[0] #find duplicate ID in trjID
for k in dpidx[::-1]:
labels.pop(k)
savetrjID.pop(k)
result = {}
result['label'] = labels
result['trjID'] = savetrjID
savemat(savename, result)
def csv_to_pickle(mainfilename, metadatafilename):
mainfilename = mainfilename.split(".csv")[0]
metadatafilename = metadatafilename.split(".csv")[0]
if (os.path.isfile(
os.path.join(pickle_location, f"{mainfilename}_3d_pickle"))
or os.path.isfile(
os.path.join(pickle_location, f"{mainfilename}_2d_pickle"))
or os.path.isfile(
os.path.join(pickle_location,
f"{mainfilename}_label_pickle"))):
boolchoice = query_yes_no(
f"{mainfilename} pickles found! Do you want to rebuild?")
if boolchoice == False:
return
print(f"Preprocessing {mainfilename}!")
t1 = process_time()
pbar = tqdm(total=30)
df = pd.read_csv(os.path.join(data_location, f"{mainfilename}.csv"))
pbar.update(1)
df2d = pd.read_csv(os.path.join(data_location, f"{metadatafilename}.csv"))
pbar.update(1)
current_objs = np.unique(df.object_id.values)
all_objs = np.unique(df2d.object_id.values)
indices_to_consider = np.intersect1d(current_objs,
all_objs,
return_indices=True)[2]
df2d = df2d.iloc[indices_to_consider].reset_index(drop=True)
pbar.update(1)
df['flux'] = np.random.normal(df['flux'], df['flux_err'] / 1.5)
if "train" in mainfilename:
flux_max = np.max(df['flux'])
flux_min = np.min(df['flux'])
else:
#Lines added after training
flux_max = 2751626.3243459687
flux_min = -1596742.6487144697
flux_pow = np.log2(flux_max - flux_min)
df['flux'] /= flux_pow
df['flux_err'] /= flux_pow
pbar.update(1)
# df['flux']/=pow(2,flux_pow)
# df['flux_err']/=pow(2,flux_pow)
train = df.copy()
train['flux_ratio_sq'] = np.power(train['flux'] / train['flux_err'], 2.0)
pbar.update(1)
train['flux_by_flux_ratio_sq'] = train['flux'] * train['flux_ratio_sq']
pbar.update(1)
train['flux_ratio_sq'] = np.power(train['flux'] / train['flux_err'], 2.0)
train['flux_by_flux_ratio_sq'] = train['flux'] * train['flux_ratio_sq']
pbar.update(1)
aggs = {
'flux': ['min', 'max', 'mean', 'median', 'std', 'sum', 'skew'],
'flux_err': ['min', 'max', 'mean', 'skew'],
'detected': ['mean', 'std', 'sum'],
'flux_ratio_sq': ['mean', 'sum', 'skew'],
'flux_by_flux_ratio_sq': ['mean', 'sum', 'skew'],
}
aggs_global = {
'mjd': ['size'],
'flux': ['min', 'max', 'mean', 'median', 'std', 'sum', 'skew'],
'flux_err': ['min', 'max', 'mean', 'median', 'std', 'sum', 'skew'],
'detected': ['mean', 'skew', 'median', 'sum'],
'flux_ratio_sq': ['min', 'max', 'mean', 'sum', 'skew'],
'flux_by_flux_ratio_sq': ['min', 'max', 'mean', 'sum', 'skew'],
}
pbar.update(1)
agg_train_global_feat = train.groupby('object_id').agg(aggs_global)
new_columns = [k + '_' + agg for k in aggs.keys() for agg in aggs[k]]
new_columns_global = [
k + '_' + agg for k in aggs_global.keys() for agg in aggs_global[k]
]
agg_train_global_feat.columns = new_columns_global
agg_train = train.groupby(['object_id', 'passband']).agg(aggs)
agg_train = agg_train.unstack()
pbar.update(1)
col_names = []
for col in new_columns:
for i in range(6):
col_names.append(col + '_' + str(i))
agg_train.columns = col_names
agg_train_global_feat['flux_diff'] = agg_train_global_feat[
'flux_max'] - agg_train_global_feat['flux_min']
agg_train_global_feat['flux_dif2'] = (
agg_train_global_feat['flux_max'] -
agg_train_global_feat['flux_min']) / agg_train_global_feat['flux_mean']
agg_train_global_feat['flux_w_mean'] = agg_train_global_feat[
'flux_by_flux_ratio_sq_sum'] / agg_train_global_feat[
'flux_ratio_sq_sum']
agg_train_global_feat['flux_dif3'] = (
agg_train_global_feat['flux_max'] - agg_train_global_feat['flux_min']
) / agg_train_global_feat['flux_w_mean']
pbar.update(1)
# Legacy code. There are much better ways to compute this but for train set this suffices so
# i got too lazy to change https://www.kaggle.com/c/PLAsTiCC-2018/discussion/71398
def detected_max(mjd, detected):
try:
return max(mjd[detected == 1]) - min(mjd[detected == 1])
except:
return 0
temp = train.groupby('object_id').apply(
lambda x: detected_max(x['mjd'], x['detected']))
temp1 = train.groupby([
'object_id', 'passband'
]).apply(lambda x: detected_max(x['mjd'], x['detected'])).unstack()
temp.columns = ['mjd_global_diff']
temp1.columns = [
'mjd_pb0', 'mjd_pb1', 'mjd_pb2', 'mjd_pb3', 'mjd_pb4', 'mjd_pb5'
]
temp = temp.reset_index()
temp1 = temp1.reset_index()
pbar.update(1)
aggs_det = {
'flux': ['min', 'mean', 'max', 'skew'],
'flux_ratio_sq': ['min', 'mean', 'max', 'skew'],
'flux_by_flux_ratio_sq': ['min', 'max', 'mean', 'skew'],
}
train_detected = train[train.detected == 1]
temp2 = train_detected.groupby(['object_id']).agg(aggs_det)
del (train_detected)
del (train)
new_columns_det = [
k + '_det_' + agg for k in aggs_det.keys() for agg in aggs_det[k]
]
temp2.columns = new_columns_det
temp2['flux_diff_det'] = temp2['flux_det_max'] - temp2['flux_det_min']
temp2['flux_ratio_sq_diff_det'] = temp2['flux_ratio_sq_det_max'] - temp2[
'flux_ratio_sq_det_min']
temp2['flux_by_flux_ratio_sq_diff_det'] = temp2[
'flux_by_flux_ratio_sq_det_max'] - temp2[
'flux_by_flux_ratio_sq_det_min']
pbar.update(1)
del temp2['flux_by_flux_ratio_sq_det_max'], temp2[
'flux_by_flux_ratio_sq_det_min']
del temp2['flux_ratio_sq_det_max'], temp2['flux_ratio_sq_det_min']
del temp2['flux_det_max'], temp2['flux_det_min']
meta_train = df2d.copy()
del (df2d)
target_dict = {
6: 0,
15: 1,
16: 2,
42: 3,
52: 4,
53: 5,
62: 6,
64: 7,
65: 8,
67: 9,
88: 10,
90: 11,
92: 12,
95: 13,
99: 14,
991: 14,
992: 14,
993: 14,
994: 14
}
meta_train["target"] = meta_train.loc[:, ["target"]].replace(target_dict)
full_train = agg_train.reset_index().merge(right=meta_train,
how='outer',
on='object_id')
del (agg_train)
del (meta_train)
full_train = full_train.merge(right=agg_train_global_feat,
how='outer',
on='object_id')
del (agg_train_global_feat)
full_train = full_train.merge(right=temp, how='outer', on='object_id')
del (temp)
full_train = full_train.merge(right=temp1, how='outer', on='object_id')
del (temp1)
full_train = full_train.merge(right=temp2, how='outer', on='object_id')
del (temp2)
labels = full_train['target']
full_train.drop("target", axis=1, inplace=True)
pbar.update(1)
if 'object_id' in full_train:
oof_df = full_train[['object_id']]
del full_train['object_id'], full_train['hostgal_specz']
del full_train['ra'], full_train['decl'], full_train[
'gal_l'], full_train['gal_b'], full_train['ddf']
useless_cols = [
'flux_max_2',
'flux_median_0',
'flux_median_4',
'flux_err_skew_1',
'flux_err_skew_3',
'detected_mean_4',
'detected_std_3',
'detected_std_4',
'detected_sum_4',
'flux_ratio_sq_mean_4',
'flux_ratio_sq_sum_3',
'flux_ratio_sq_sum_4',
'flux_median',
'flux_err_skew',
'flux_ratio_sq_sum',
'mjd_pb5',
'flux_ratio_sq_det_skew',
]
full_train_new = full_train.drop(useless_cols, axis=1)
del (full_train)
del (oof_df)
from sklearn.preprocessing import PowerTransformer
ss = PowerTransformer()
twod_data = ss.fit_transform(np.nan_to_num(full_train_new))
del (full_train_new)
with open(os.path.join(pickle_location, f"{mainfilename}_2d_pickle"),
"wb") as fp: #Pickling
pickle.dump(twod_data, fp)
pbar.update(1)
del (twod_data)
#Calculate time diff between all observations in mjd
df["mjd_diff"] = df['mjd'].diff()
df["mjd_diff"] = df["mjd_diff"].fillna(0)
#Find indexes where new objects appear, and set the mjd_diff for this to 0
obj_change_index = np.where(
df["object_id"].values[:-1] != df["object_id"].values[1:])[0] + 1
df.loc[obj_change_index, ['mjd_diff']] = 0
# Use groupby method to find seperate cumsums for all objects
df["cumulative_mjd_diff"] = df.loc[:, ["object_id", "mjd_diff"]].groupby(
"object_id").cumsum()
pbar.update(1)
mjd_arr = df["mjd"].values
time_diff_arr = df["mjd_diff"].values
grouped_mjd_arr = np.zeros_like(mjd_arr)
pbar.update(1)
prev_time = 0
for i in range(len(mjd_arr)):
current_time = mjd_arr[i]
time_diff = time_diff_arr[i]
if time_diff == 0 or current_time - prev_time > 0.33:
grouped_mjd_arr[i] = current_time
prev_time = current_time
else:
grouped_mjd_arr[i] = prev_time
pbar.update(1)
df["grouped_mjd"] = grouped_mjd_arr
del (grouped_mjd_arr)
del (time_diff_arr)
del (mjd_arr)
df = df.sort_values("flux_err").groupby(
["object_id", "grouped_mjd", "passband"]).first()
df = df.reset_index()
pbar.update(1)
#Drop all unnecessary columns. Note : mjd_diff and cumulative_mjd_diff are dropped as cause problems when pivoting. Will recalculate later
pbar.update(1)
df = df.drop(
[
"mjd",
"detected",
"mjd_diff",
"cumulative_mjd_diff",
],
axis=1,
)
mini_df = df[["object_id"]].groupby("object_id").first()
df = df.drop(mini_df, axis=1)
df = pd.pivot_table(df,
index=["object_id", "grouped_mjd"],
columns=["passband"])
df.columns = [f"{tup[0]}_passband_{tup[1]}" for tup in df.columns.values]
df = df.reset_index(["grouped_mjd"])
df = df.join(mini_df, how="left")
pbar.update(1)
del (mini_df)
df = df.rename(columns={"grouped_mjd": "mjd"})
df = df.reset_index()
#Calculate time diff between all observations in mjd
df["mjd_diff"] = df['mjd'].diff()
df["mjd_diff"] = df["mjd_diff"].fillna(0)
#Find indexes where new objects appear, and set the mjd_diff for this to 0
obj_change_index = np.where(
df["object_id"].values[:-1] != df["object_id"].values[1:])[0] + 1
df.loc[obj_change_index, ['mjd_diff']] = 0
pbar.update(1)
# Use groupby method to find seperate cumsums for all objects
df["cumulative_mjd_diff"] = df.loc[:, ["object_id", "mjd_diff"]].groupby(
"object_id").cumsum()
pbar.update(1)
df = df.set_index(["object_id"])
df = df.drop(
[
"mjd",
],
axis=1,
)
pbar.update(1)
colstointer = [
"object_id", "flux_passband_0", "flux_passband_1", "flux_passband_2",
"flux_passband_3", "flux_passband_4", "flux_passband_5",
"flux_err_passband_0", "flux_err_passband_1", "flux_err_passband_2",
"flux_err_passband_3", "flux_err_passband_4", "flux_err_passband_5",
"cumulative_mjd_diff"
]
df = df.reset_index()
temp_df = df.loc[:, colstointer].groupby('object_id').apply(
lambda group: group.set_index("cumulative_mjd_diff").interpolate(
method="values").ffill().bfill())
temp_df.reset_index(drop=True, inplace=True)
df.loc[:, temp_df.columns] = temp_df
del (temp_df)
df.drop(["cumulative_mjd_diff"], 1, inplace=True)
df.set_index("object_id", inplace=True, drop=True)
pbar.update(1)
df = df.fillna(0)
#Save some memory by converting float64 to float32 as 32 enough
for col in df.columns:
if df[col].dtype == np.float64:
df[col] = df[col].astype(np.float32)
pbar.update(1)
#Recalculate time diff between all rows and set new objs to zero as before
all_obj_ids = np.unique(df.index.get_level_values(0).values)
dfarray = df.reset_index().to_numpy()
all_obj_ids_long = dfarray[0:, 0]
all_labels_long = dfarray[0:, 1]
obj_change_index = np.where(
all_obj_ids_long[:-1] != all_obj_ids_long[1:])[0] + 1
pbar.update(1)
tuplist = list(zip(np.insert(obj_change_index, 0, 0), obj_change_index))
list_of_data_arrays = []
for tup in tuplist:
list_of_data_arrays.append(dfarray[tup[0]:tup[1], 1:])
list_of_data_arrays.append(dfarray[obj_change_index[-1]:, 1:])
pbar.update(1)
pbar.update(1)
with open(os.path.join(pickle_location, f"{mainfilename}_3d_pickle"),
"wb") as fp: #Pickling
pickle.dump(list_of_data_arrays, fp)
pbar.update(1)
with open(os.path.join(pickle_location, f"{mainfilename}_label_pickle"),
"wb") as fp: #Pickling
pickle.dump(labels, fp)
pbar.update(1)
gc.collect()
pbar.close()
t2 = process_time()
print(f"Preprocessing took {t2-t1} seconds.")
def fit(self, W, C, vocab, AD, author_list, timestamp_list, verbose=True):
# {{{ run gibbs sampling
import sys
import numpy as np
from utils import cartesian
from scipy.stats import poisson
'''
W[:,0] -> word index
W[:,1] -> document index
W[:,2] -> timestamp index
C: document-citation sparse matrix
C[i, t] -> citation count for the ith document at timestamp t
AD: author-document sparse matrix
'''
# number of authors
A = author_list.size
# number of unique tokens
V = vocab.size
# total number of tokens
nnz = W.shape[0]
# number of timestamps
T = timestamp_list.size
# save meta-info to this object
self.vocabulary = vocab
self.authors = author_list
self.timestamps = timestamp_list
# init to one above the max val
z_states = np.zeros((self.n_iter + 1, nnz), dtype=np.uint32) + self.K
a_states = np.zeros((self.n_iter + 1, nnz), dtype=np.uint32) + A
t_states = np.zeros((self.n_iter + 1, nnz), dtype=np.uint32) + T
# create all needed sequences
k_range = np.arange(self.K)
t_range = np.arange(T)
a_range = np.arange(A)
v_range = np.arange(V)
# 1.1 initialize topic assignment randomly
z_states[0, :] = np.random.choice(self.K, nnz, True)
# 1.2 initialize author and timestamp assignment
for i in np.arange(nnz):
t = W[i, 2]
di = W[i, 1]
ad = AD[:, di].nonzero()[0]
a_states[0, i] = np.random.choice(ad)
t_states[0, i] = np.random.choice(np.arange(t, T))
# 2. initialize lambda matrix
lambda_ = np.zeros((self.K, T), dtype=np.uint32)
for k in k_range:
k_indices = np.where(z_states[0, :] == k)[0]
for t in np.arange(T):
t_indices = np.where(t_states[0, :] == t)[0]
kt_indices = np.intersect1d(t_indices, k_indices)
d_indices = W[kt_indices, 1]
lambda_[k, t] = C[d_indices, t].mean()
# zeros set to overall mean
lam_x, lam_y = np.where(lambda_ == 0)
if lam_x.size:
lambda_[lam_x, lam_y] = C.mean() / float((lam_x.size))
# 3. sample
# {{{
for iter_ in np.arange(1, self.n_iter + 1):
if verbose:
print 'Iter %i...... (Total %i)' % (iter_, self.n_iter)
sys.stdout.flush()
else:
if iter_ % 100:
print 'Iter %i...... (Total %i)' % (iter_, self.n_iter)
sys.stdout.flush()
for i in np.arange(nnz):
#{{{ sample each token sequentially
# {{{ denominators
den_author = np.zeros(A, dtype=np.float_)
for a in a_range:
for k in k_range:
# words that are assigned to topic k, excluding the current one
k_indices = np.append(np.where(z_states[iter_-1, i+1:]==k)[0] + (i+1),\
np.where(z_states[iter_, :i]==k)[0])
n_a_k_i = (a_states[iter_-1, k_indices[k_indices > i]] == a).sum() + \
(a_states[iter_, k_indices[k_indices < i]] == a).sum()
den_author[a] += n_a_k_i
den_author[a] += self.K * self.alpha
den_timestamp = np.zeros(self.K, dtype=np.float_)
den_token = np.zeros(self.K, dtype=np.float_)
for k in k_range:
for t in t_range:
# words that are assigned to timestamp t, excluding the current one
t_indices = np.append(np.where(t_states[iter_-1, i+1:]==t)[0] + (i+1),\
np.where(t_states[iter_, :i]==t)[0])
n_k_t_i = (z_states[iter_-1, t_indices[t_indices > i]] == k).sum() + \
(z_states[iter_, t_indices[t_indices < i]] == k).sum()
den_timestamp[k] += n_k_t_i
den_timestamp[k] += T * self.pi
for v in v_range:
# words that are tokens v, excluding the current one
v_indices = np.append(
np.where(W[i + 1:, 0] == v)[0] + (i + 1),
np.where(W[:i, 0] == v)[0])
n_k_v_i = (z_states[iter_-1, v_indices[v_indices > i]] == k).sum() + \
(z_states[iter_, v_indices[v_indices < i]] == k).sum()
den_token[k] += n_k_v_i
den_token[k] += V * self.beta
# }}}
v = W[i, 0]
t = W[i, 2]
di = W[i, 1]
ci = C[di, :]
# find its authors
ad = AD[:, di].nonzero()[0]
comb_list = cartesian((np.arange(t, T), k_range, ad))
comb_p_list = np.zeros(comb_list.shape[0], dtype=np.float_)
# excluding the current one
v_indices = np.append(
np.where(W[i + 1:, 0] == v)[0] + (i + 1),
np.where(W[:i, 0] == v)[0])
# {{{ for each combination, obtain full conditional probability
for comb_index in np.arange(comb_p_list.size):
comb = comb_list[comb_index]
t, k, a = comb
# 1
t_indices = np.append(np.where(t_states[iter_-1, i+1:]==t)[0] + (i+1),\
np.where(t_states[iter_, :i]==t)[0])
n_k_t_i = (z_states[iter_-1, t_indices[t_indices > i]] == k).sum() + \
(z_states[iter_, t_indices[t_indices < i]] == k).sum()
p1 = (n_k_t_i + self.pi) / den_timestamp[k]
# 2
n_k_v_i = (z_states[iter_-1, v_indices[v_indices > i]] == k).sum() + \
(z_states[iter_, v_indices[v_indices < i]] == k).sum()
p2 = (n_k_v_i + self.beta) / den_token[k]
# 3
# excluding the current one
k_indices = np.append(np.where(z_states[iter_-1, i+1:] == k)[0] + (i+1),\
np.where(z_states[iter_, :i] == k)[0])
n_a_k_i = (a_states[iter_-1, k_indices[k_indices > i]] == a).sum() + \
(a_states[iter_, k_indices[k_indices < i]] == a).sum()
p3 = (n_a_k_i + self.alpha) / den_author[a]
# poisson pmf
p4 = poisson.pmf(ci[t], mu=lambda_[k, t])
#print p1, p2, p3, p4, lambda_[k,t], ci[t]
comb_p_list[comb_index] = p1 * p2 * p3 * p4
# }}}
# rescale to [0,1]
comb_p_list = comb_p_list / comb_p_list.sum()
# sample for i-th word
comb_index = np.random.choice(np.arange(comb_p_list.size),
p=comb_p_list)
t, k, a = comb_list[comb_index]
t_states[iter_, i] = t
z_states[iter_, i] = k
a_states[iter_, i] = a
#}}} END for i-th TOKEN
# update lambda after each iteration
for k in k_range:
k_indices = np.where(z_states[iter_, :] == k)[0]
for t in t_range:
t_indices = np.where(t_states[iter_, :] == t)[0]
kt_indices = np.intersect1d(k_indices, t_indices)
d_indices = W[kt_indices, 1]
# if no word is assigned to topic k and timestamp t, keep it as before
if d_indices.size > 0:
lambda_[k, t] = C[d_indices, t].mean()
# }}}
# 4. obtain \theta, \phi, and \psi
# burn-in: first half
z_samples = z_states[1:, :][self.n_iter / 2:, :]
a_samples = a_states[1:, :][self.n_iter / 2:, :]
t_samples = t_states[1:, :][self.n_iter / 2:, :]
# author-topic
theta = np.zeros((A, self.K), dtype=np.float_)
# topic-word
phi = np.zeros((self.K, V), dtype=np.float_)
# topic-timestamp
psi = np.zeros((self.K, T), dtype=np.float_)
for a in a_range:
den = self.K * self.alpha + (a_samples == a).sum()
a_x, a_y = np.where(a_samples == a)
for k in k_range:
n_a_k = (z_samples[a_x, a_y] == k).sum()
theta[a, k] = float(n_a_k + self.alpha) / (den)
for k in k_range:
k_count = (z_samples == k).sum()
den_v = V * self.beta + k_count
den_t = T * self.pi + k_count
# x is iteration number, y is word index
k_x, k_y = np.where(z_samples == k)
for v in v_range:
n_k_v = (W[k_y, 0] == v).sum()
phi[k, v] = float(n_k_v + self.beta) / den_v
for t in t_range:
n_k_t = (t_samples[k_x, k_y] == t).sum()
psi[k, t] = float(n_k_t + self.pi) / den_t
# update lambda
t_x, t_y = np.where(t_samples == t)
kt_indices = np.intersect1d(k_y, t_y)
d_indices = W[kt_indices, 1]
# if no word is assigned to topic k and timestamp t, keep it as before
if d_indices.size > 0:
lambda_[k, t] = C[d_indices, t].mean()
self.theta = theta
self.phi = phi
self.psi = psi
self.lambda_ = lambda_
self.z_samples = z_samples
self.a_samples = a_samples
self.t_samples = t_samples
# }}}
return theta, phi, psi, lambda_
def fit_nuisance_iivm(Y,
X,
D,
Z,
learner_m,
learner_g,
learner_r,
smpls,
g0_params=None,
g1_params=None,
m_params=None,
r0_params=None,
r1_params=None,
trimming_threshold=1e-12):
ml_g0 = clone(learner_g)
g_hat0 = []
for idx, (train_index, test_index) in enumerate(smpls):
if g0_params is not None:
ml_g0.set_params(**g0_params[idx])
train_index0 = np.intersect1d(np.where(Z == 0)[0], train_index)
g_hat0.append(
ml_g0.fit(X[train_index0], Y[train_index0]).predict(X[test_index]))
ml_g1 = clone(learner_g)
g_hat1 = []
for idx, (train_index, test_index) in enumerate(smpls):
if g1_params is not None:
ml_g1.set_params(**g1_params[idx])
train_index1 = np.intersect1d(np.where(Z == 1)[0], train_index)
g_hat1.append(
ml_g1.fit(X[train_index1], Y[train_index1]).predict(X[test_index]))
ml_m = clone(learner_m)
m_hat = []
for idx, (train_index, test_index) in enumerate(smpls):
if m_params is not None:
ml_m.set_params(**m_params[idx])
xx = ml_m.fit(X[train_index],
Z[train_index]).predict_proba(X[test_index])[:, 1]
if trimming_threshold > 0:
xx[xx < trimming_threshold] = trimming_threshold
xx[xx > 1 - trimming_threshold] = 1 - trimming_threshold
m_hat.append(xx)
ml_r0 = clone(learner_r)
r_hat0 = []
for idx, (train_index, test_index) in enumerate(smpls):
if r0_params is not None:
ml_r0.set_params(**r0_params[idx])
train_index0 = np.intersect1d(np.where(Z == 0)[0], train_index)
r_hat0.append(
ml_r0.fit(X[train_index0],
D[train_index0]).predict_proba(X[test_index])[:, 1])
ml_r1 = clone(learner_r)
r_hat1 = []
for idx, (train_index, test_index) in enumerate(smpls):
if r1_params is not None:
ml_r1.set_params(**r1_params[idx])
train_index1 = np.intersect1d(np.where(Z == 1)[0], train_index)
r_hat1.append(
ml_r1.fit(X[train_index1],
D[train_index1]).predict_proba(X[test_index])[:, 1])
return g_hat0, g_hat1, m_hat, r_hat0, r_hat1
import numpy as np
directory = '/nfs/maciej/mcdoi/'
models = [
'louvain', 'correlated-linear-threshold', 'dynamic-linear-threshold',
'linear-threshold', 'linear-threshold-model',
'linear-threshold-model-14all', 'linear-threshold-random-dynamic',
'linear-threshold-random-dynamic-14all'
]
list_of_predicted = []
for model in models:
model_directory = directory + model + '/'
with open(model_directory + 'predicted_7days', 'r',
encoding='utf-8') as predicted:
predicted = predicted.readlines()
predicted = [x.strip() for x in predicted]
list_of_predicted.append(predicted)
predicted_4all = list_of_predicted[0]
for predicted in list_of_predicted[1::]:
predicted_4all = np.intersect1d(predicted_4all, predicted)
print(len(predicted_4all))
open(directory + 'predicted_4all', 'w', encoding='utf-8').close()
for predicted in predicted_4all:
with open(directory + 'predicted_4all', 'a', encoding='utf-8') as file:
file.write(predicted + '\n')
def expose(states, t, idx, N_c):
idx = np.intersect1d(idx, patients_in_state(states, t, 0))
if len(idx) > 0:
states[idx, t + 1] = if_else(idx, constants.beta0 / N_c, 1, 0)
return states
def paul15_raw():
"""Development of Myeloid Progenitors [Paul15]_.
Non-logarithmized raw data.
The data has been sent out by Email from the Amit Lab. An R version for
loading the data can be found here
https://github.com/theislab/scAnalysisTutorial
Returns
-------
adata : :class:`~scanpy.api.AnnData`
Annotated data matrix.
"""
import h5py
filename = 'data/paul15/paul15.h5'
backup_url = 'http://falexwolf.de/data/paul15.h5'
sc.utils.check_presence_download(filename, backup_url)
with h5py.File(filename, 'r') as f:
X = f['data.debatched'][()]
gene_names = f['data.debatched_rownames'][()].astype(str)
cell_names = f['data.debatched_colnames'][()].astype(str)
clusters = f['cluster.id'][()].flatten()
infogenes_names = f['info.genes_strings'][()].astype(str)
# each row has to correspond to a sample, therefore transpose
adata = sc.AnnData(X.transpose())
adata.var_names = gene_names
adata.row_names = cell_names
# names reflecting the cell type identifications from the paper
cell_type = {
7: 'MEP',
8: 'Mk',
9: 'GMP',
10: 'GMP',
11: 'DC',
12: 'Baso',
13: 'Baso',
14: 'Mo',
15: 'Mo',
16: 'Neu',
17: 'Neu',
18: 'Eos',
19: 'Lymph'
}
cell_type.update({i: 'Ery' for i in range(1, 7)})
adata.obs['paul15_clusters'] = [
str(i) + cell_type[i] for i in clusters.astype(int)
]
# make string annotations categorical (optional)
sc.utils.sanitize_anndata(adata)
# just keep the first of the two equivalent names per gene
adata.var_names = [gn.split(';')[0] for gn in adata.var_names]
# remove 10 corrupted gene names
infogenes_names = np.intersect1d(infogenes_names, adata.var_names)
# restrict data array to the 3461 informative genes
adata = adata[:, infogenes_names]
# usually we'd set the root cell to an arbitrary cell in the MEP cluster
# adata.uns['iroot': np.flatnonzero(adata.obs['paul15_clusters'] == '7MEP')[0]
# here, set the root cell as in Haghverdi et al. (2016)
adata.uns[
'iroot'] = 840 # note that other than in Matlab/R, counting starts at 1
return adata
Repeat = 1000
SigCases = 0
inSigCases = 0
CI_1 = np.zeros(2000).reshape(1000, 2)
CI_2 = np.zeros(2000).reshape(1000, 2)
CI_3 = np.zeros(2000).reshape(1000, 2)
np.random.seed(0)
for i in range(Repeat):
# generate Y1,Y2
Y1, Y2 = np.random.multivariate_normal(mean, cov, N).T
tmp = np.random.randint(1, 100, N)
# generate M
M = np.zeros(N)
M[np.intersect1d(np.where(Y1 < 0), np.where(tmp <= 20))] = 1
M[np.intersect1d(np.where(Y1 >= 0), np.where(tmp <= 80))] = 1
Y1obs = Y1[np.where(M == 0)]
Y1mis = Y1[np.where(M == 1)]
Y2obs = Y2[np.where(M == 0)]
# carry out a t-test between Y1obs and Y1mis
test = stats.ttest_ind(Y1obs, Y1mis)
if (test[1] < 0.05):
SigCases = SigCases + 1
else:
inSigCases = inSigCases + 1
# 95% confidence interval
r = len(Y2obs)
# (1) the data before values were deleted
mu2 = np.mean(Y2)
[[s11, s12], [s21, s22]] = np.cov([Y1, Y2], bias=1)
THIS TAKES A LONG TIME
PiBotFinal = dictionary of bot probabilities.
"""
start_time = time.time()
print("Calculate bot probability for each labeled node in retweet graph")
PiBotFinal = {}
for counter, node in enumerate(Gretweet.nodes()):
if counter % 1000 == 0: print("Node %s" % counter)
if node in Gretweet.nodes():
neighbors = list(
np.unique(
[i for i in nx.all_neighbors(H, node) if i not in [0, 1]]))
ebots = list(np.unique(np.intersect1d(neighbors, BotsIsing)))
ehumans = list(set(neighbors) - set(ebots))
psi_l = sum([H[node][j]['capacity'] for j in ehumans]) - sum(
[H[node][i]['capacity'] for i in ebots])
psi_l_bis = psi_l + H[node][0]['capacity'] - H[1][node][
'capacity'] ##probability to be in 1 = notPL
if (psi_l_bis) > 12:
PiBotFinal[node] = 0
else:
PiBotFinal[node] = 1. / (1 + np.exp(psi_l_bis)
) #Probability in the target (0) class
print("--- %s seconds ---" % (time.time() - start_time))
"""## Save probabilities to file
import numpy as np
if __name__ == "__main__":
# compute intersection of 2 arrays on the worker
# numpy dependency and this code have been shipped by cluster-pack to the worker
a = np.random.random_integers(0, 100, 100)
b = np.random.random_integers(0, 100, 100)
print("Computed intersection of two arrays:")
print(np.intersect1d(a, b))
def vehilce_ditribution_pic_new():
c07_dict = read_from_json(ditribution_json_path07)
c04_dict = read_from_json(ditribution_json_path04)
c02_dict = read_from_json(ditribution_json_path02)
d07_dict = read_from_json(distance_json_path07)
d04_dict = read_from_json(distance_json_path04)
d02_dict = read_from_json(distance_json_path02)
# 车的请求，求事务
x7_d_08, y7_d_08 = sort_key_dict_scle(d07_dict['8'])
x4_d_08, y4_d_08 = sort_key_dict_scle(d04_dict['8'])
x2_d_08, y2_d_08 = sort_key_dict_scle(d02_dict['8'])
x7_d_18, y7_d_18 = sort_key_dict_scle(d07_dict['18'])
x4_d_18, y4_d_18 = sort_key_dict_scle(d04_dict['18'])
x2_d_18, y2_d_18 = sort_key_dict_scle(d02_dict['18'])
# 车的数量，求分布
x7_08, y7_08 = sort_value_dict(c07_dict['8'])
x4_08, y4_08 = sort_value_dict(c04_dict['8'])
x2_08, y2_08 = sort_value_dict(c02_dict['8'])
x7_18, y7_18 = sort_value_dict(c07_dict['18'])
x4_18, y4_18 = sort_value_dict(c04_dict['18'])
x2_18, y2_18 = sort_value_dict(c02_dict['18'])
x7_08 = np.array(x7_08)
x4_08 = np.array(x4_08)
x2_08 = np.array(x2_08)
x7_18 = np.array(x7_18)
x4_18 = np.array(x4_18)
x2_18 = np.array(x2_18)
a08_ = np.intersect1d(x4_08, x2_08)
a08 = np.intersect1d(x7_08, a08_)
a18_ = np.intersect1d(x4_18, x2_18)
a18 = np.intersect1d(x7_18, a18_)
#
x08 = a08.tolist()
x18 = a18.tolist()
print("7_08:", count_txn_number(y7_d_08))
print("4_08:", count_txn_number(y4_d_08))
print("2_08:", count_txn_number(y2_d_08))
print("7_18:", count_txn_number(y7_d_18))
print("4_18:", count_txn_number(y4_d_18))
print("2_18:", count_txn_number(y2_d_18))
plt.rcdefaults()
fig, ax = plt.subplots()
# Example data
people = x7_08[:19]
y_pos = np.arange(len(people))
error = np.random.rand(len(people))
ax.barh(y_pos, y7_08[:19], xerr=error, align='center')
ax.set_yticks(y_pos)
ax.set_yticklabels(people)
ax.invert_yaxis() # labels read top-to-bottom
ax.set_xlabel('Number of vehicles')
ax.set_ylabel('zone')
fig.tight_layout()
# plt.grid(linestyle='-.')
# plt.savefig('(2)transaction distribution.pdf')
plt.show()
plt.imshow(softmax_res_func(all_prediction), interpolation='none')
plt.subplot(1, 4, 3)
plt.imshow(softmax_res_func(
np.mean(all_prediction.reshape((-1, 10, 30)),
axis=1)).astype(np.int),
interpolation='none')
plt.subplot(1, 4, 4)
all_res = np.array([
target_per_char[x][train_mode_per_block != 1] for x in range(30)
]).T
plt.imshow(np.mean(all_res.reshape((-1, 10, 30)), axis=1),
interpolation='none')
# plt.show()
actual_untrained = \
np.where(np.round(softmax_res_func(np.mean(all_prediction_untrained.reshape((-1, 10, 30)), axis=1))) == 1)[0]
actual = np.where(
np.round(
softmax_res_func(
np.mean(all_prediction.reshape((-1, 10,
30)), axis=1))) == 1)[0]
gt = np.where(np.mean(all_res.reshape((-1, 10, 30)), axis=1) == 1)[0]
np.intersect1d(actual, gt)
accuracy = len(np.intersect1d(actual, gt)) / float(len(gt))
accuracy_untrained = len(np.intersect1d(actual_untrained, gt)) / float(
len(gt))
print "subject:{0} accu:{1} acc_untrained{2}".format(
subject, accuracy, accuracy_untrained)
# target_per_char_as_matrix[0:10,:]
def __call__(self, roi):
frame = roi.mask[0].todense().copy()
frame[frame > 0] = 1
check = frame[:-2, :-2] + frame[1:-1, :-2] + frame[2:, :-2] + \
frame[:-2, 1:-1] + frame[2:, 1:-1] + frame[:-2:, 2:] + \
frame[1:-1, 2:] + frame[2:, 2:]
z = np.zeros(frame.shape)
z[1:-1, 1:-1] = check
# initialize and array to hold the new polygon and find the first point
b = []
rows, cols = np.where(z > 0)
p = [cols[0], rows[0]]
base = p
# establish an iteration limit to ensue the loop terminates if
# smoothing is unsuccessful
limit = 1500
radius = self.radius
# store wether the radius of search is increased aboved the initial
# value
tmp_rad = False
for _ in range(limit - 1):
b.append(p)
# find the ist of all points at the given radius and adjust to be
# lined up for clockwise traversal
x = np.roll(
np.array(
list(p[0] + list(range(-radius, radius))) +
[p[0] + radius] * (2 * radius + 1) +
list(p[0] + list(range(-radius, radius))[::-1]) +
[p[0] - (radius + 1)] * (2 * radius + 1)), -2)
y = np.roll(
np.array([p[1] - radius] * (2 * radius) +
list(p[1] + list(range(-radius, radius))) +
[p[1] + radius] * (2 * radius + 1) +
list(p[1] +
list(range(-radius, (radius + 1)))[::-1])),
-radius)
# insure that the x and y points are within the image
x[x < 0] = 0
y[y < 0] = 0
x[x >= z.shape[1]] = z.shape[1] - 1
y[y >= z.shape[0]] = z.shape[0] - 1
vals = z[y, x]
# ensure that the vals array has a valid transition from 0 to 1
# otherwise the algorithm has failed
if len(np.where(np.roll(vals, 1) == 0)[0]) == 0 or \
len(np.where(vals > 0)[0]) == 0:
return roi, False
idx = np.intersect1d(
np.where(vals > 0)[0],
np.where(np.roll(vals, 1) == 0)[0])[0]
p = [x[idx], y[idx]]
# check if the traveral is near to the starting point indicating
# that the algirthm has completed. If less then 3 points are found
# this is not yet a valid ROI
if ((p[0] - base[0]) ** 2 + (p[1] - base[1]) ** 2) ** 0.5 < \
1.5 * radius and len(b) > 3:
new_roi = ROI(polygons=[b], im_shape=roi.im_shape)
if new_roi.mask[0].size != 0:
# "well formed ROI"
return new_roi, True
# if p is already in the list of polygon points, increase the
# radius of search. if radius is already larger then 6, blur the
# mask and try again
if p in b:
if radius > 6:
radius = 3
z = ndimage.gaussian_filter(z, sigma=1)
b = []
rows, cols = np.where(z > 0)
p = [cols[0], rows[0]]
base = p
tmp_rad = False
else:
radius = radius + 1
tmp_rad = True
if len(b) > 3:
p = b[-3]
del b[-3:]
elif tmp_rad:
tmp_rad = False
radius = 3
# The maximum number of cycles has completed and no suitable smoothed
# ROI has been determined
return roi, False
def _single_frame(self):
# initial scoop for nearby groups
coords_ = self.selection.positions
pairs = distances.capped_distance(self.protein.positions,
coords_,
self.cutoff,
box=self.protein.dimensions,
return_distances=False)
if pairs.size > 0:
indices = np.unique(pairs[:, 1])
else:
indices = []
# now look for groups in the buffer
if len(indices) and self.buffer:
pairs2, dist = distances.capped_distance(
self.protein.positions,
coords_,
self.max_cutoff,
min_cutoff=self.cutoff,
box=self.protein.dimensions,
return_distances=True)
# don't count things in inner cutoff
mask = [x not in indices for x in pairs2[:, 1]]
pairs2 = pairs2[mask]
dist = dist[mask]
if pairs2.size > 0:
_ix = np.argsort(pairs2[:, 1])
indices2 = pairs2[_ix][:, 1]
dist = dist[_ix] - self.cutoff
init_resix2 = self.selection.resindices[indices2]
# sort through for minimum distance
ids2, splix = np.unique(init_resix2, return_index=True)
resix2 = init_resix2[splix]
split_dist = np.split(dist, splix[1:])
min_dist = np.array([x.min() for x in split_dist])
# logistic function
for i, leaflet_residues in self._current_leaflet_residues.items(
):
ids = self._current_leaflet_ids[i]
match, rix, lix = np.intersect1d(
resix2,
leaflet_residues.resindices,
assume_unique=True,
return_indices=True)
subdist = min_dist[rix]
subids = ids[lix]
for j, x in enumerate(self.ids):
mask = (subids == x)
xdist = subdist[mask]
exp = -0.5 * ((xdist / self._buffer_sigma)**2)
n = self._buffer_coeff * np.exp(exp)
self.near_counts[i, j, self._frame_index] += n.sum()
soft = self.near_counts[:, :, self._frame_index].sum()
init_resix = self.selection.resindices[indices]
resix = np.unique(init_resix)
for i, leaflet_residues in self._current_leaflet_residues.items():
ids = self._current_leaflet_ids[i]
_, ix1, ix2 = np.intersect1d(resix,
leaflet_residues.resindices,
assume_unique=True,
return_indices=True)
self.total_counts[i, self._frame_index] = len(ix1)
subids = ids[ix2]
for j, x in enumerate(self.ids):
self.residue_counts[i, j, self._frame_index] += sum(ids == x)
self.near_counts[i, j, self._frame_index] += sum(subids == x)
both = self.near_counts[:, :, self._frame_index].sum()
print(
"Be patient: Removing water molecules Insinde the sphere of radius ['{}'] ...\n"
.format(radius))
box_center = data["box"] / 2
water_coords_t = water_coords - box_center
#mol_coords = molecules.get_coordinates_bohr(mol_coords*10)
atoms_inside = molecules.inside_sphere(center, radius, water_coords_t)
atoms_to_conserve = []
for m, imol in enumerate(water_list):
try:
numpy.intersect1d(
imol,
atoms_inside)[0] # one atom of the molecule is inside the sphere
except IndexError:
atoms_to_conserve.extend(water_list[m])
if not len(atoms_to_conserve) % 3 == 0:
raise Exception(
"The number of remaining atoms must be a multiple of {} ".format(
len(water_atname)))
else:
water_deleted = (nWater - (len(atoms_to_conserve) // len(water_atname)))
print("{} Water molecule(s) was(were) removed\n".format(water_deleted))
#**********************************************************
# generate the new set of data
#**********************************************************
new_data = {} # empty dictionary for the new set of data
def vehilce_ditribution_pic():
c07_dict = read_from_json(ditribution_json_path07)
c04_dict = read_from_json(ditribution_json_path04)
c02_dict = read_from_json(ditribution_json_path02)
d07_dict = read_from_json(distance_json_path07)
d04_dict = read_from_json(distance_json_path04)
d02_dict = read_from_json(distance_json_path02)
# 车的请求，求事务
x7_d_08, y7_d_08 = sort_key_dict_scle(d07_dict['8'])
x4_d_08, y4_d_08 = sort_key_dict_scle(d04_dict['8'])
x2_d_08, y2_d_08 = sort_key_dict_scle(d02_dict['8'])
x7_d_18, y7_d_18 = sort_key_dict_scle(d07_dict['18'])
x4_d_18, y4_d_18 = sort_key_dict_scle(d04_dict['18'])
x2_d_18, y2_d_18 = sort_key_dict_scle(d02_dict['18'])
# 车的数量，求分布
x7_08, y7_08 = sort_key_dict(c07_dict['8'])
x4_08, y4_08 = sort_key_dict(c04_dict['8'])
x2_08, y2_08 = sort_key_dict(c02_dict['8'])
x7_18, y7_18 = sort_key_dict(c07_dict['18'])
x4_18, y4_18 = sort_key_dict(c04_dict['18'])
x2_18, y2_18 = sort_key_dict(c02_dict['18'])
x7_08 = np.array(x7_08)
x4_08 = np.array(x4_08)
x2_08 = np.array(x2_08)
x7_18 = np.array(x7_18)
x4_18 = np.array(x4_18)
x2_18 = np.array(x2_18)
a08_ = np.intersect1d(x4_08, x2_08)
a08 = np.intersect1d(x7_08, a08_)
a18_ = np.intersect1d(x4_18, x2_18)
a18 = np.intersect1d(x7_18, a18_)
#
x08 = a08.tolist()
x18 = a18.tolist()
print("7_08:", count_txn_number(y7_d_08))
print("4_08:", count_txn_number(y4_d_08))
print("2_08:", count_txn_number(y2_d_08))
print("7_18:", count_txn_number(y7_d_18))
print("4_18:", count_txn_number(y4_d_18))
print("2_18:", count_txn_number(y2_d_18))
# x17_value_sort_list, y17_value_sort_list = sort_value_dict(clock17, len(clock17.keys()))
# x7_value_sort_list, y7_value_sort_list = sort_value_dict(clock7, 15)
# x21_value_sort_list, y21_value_sort_list = sort_value_dict(clock21, 15)
# plt.bar(list(z1.keys()), list(z1.values()))
# plt.bar(x0, y0)
# plt.bar(x17, y17, color='r')
fig, ax = plt.subplots(2, 1, figsize=(16, 9), dpi=300)
ax0_2 = ax[0].twinx()
# 未累积面积图
ax[0].fill_between(x7_08,
y1=y7_08,
y2=0,
label="Midweek 08 o'clock",
alpha=0.3,
color='blue')
ax[0].fill_between(x4_08,
y1=y4_08,
y2=0,
label="Weekend 08 o'clock",
alpha=0.3,
color='green')
ax[0].fill_between(x2_08,
y1=y2_08,
y2=0,
label="Holiday 08 o'clock",
alpha=0.3,
color='red')
# transaction图
ax0_2.plot(x7_d_08,
y7_d_08,
label="Midweek 08 o'clock",
linestyle="dashed",
color='blue')
ax0_2.plot(x4_d_08,
y4_d_08,
label="Weekend 08 o'clock",
linestyle="dashed",
color='green')
ax0_2.plot(x2_d_08,
y2_d_08,
label="Holiday 08 o'clock",
linestyle="dashed",
color='red')
ax[0].set_xlabel('zones', fontsize=14)
ax[0].tick_params(axis='x', rotation=60, labelsize=14)
ax[0].set_ylabel('Number of vehicles', fontsize=16)
# ax.set_yticks(y1_newsticks)
ax[0].grid(alpha=.4)
ax[0].set_xticks(x08[::3])
ax[0].legend(loc='upper left', prop={'size': 16}, framealpha=0.5)
ax0_2.legend(loc='upper right', prop={'size': 16}, framealpha=0.5)
ax[0].set_ylim(0, 1000)
ax0_2.set_ylabel("Number of transaction", fontsize=16)
ax1_2 = ax[1].twinx()
ax[1].fill_between(x7_18,
y1=y7_18,
y2=0,
label="Midweek 18 o'clock",
alpha=0.3,
color='blue')
ax[1].fill_between(x4_18,
y1=y4_18,
y2=0,
label="Weekend 18 o'clock",
alpha=0.3,
color='green')
ax[1].fill_between(x2_18,
y1=y2_18,
y2=0,
label="Holiday 18 o'clock",
alpha=0.3,
color='red')
# transaction图
ax1_2.plot(x7_d_18,
y7_d_18,
label="Midweek 18 o'clock",
linestyle="dashed",
color='blue')
ax1_2.plot(x4_d_18,
y4_d_18,
label="Weekend 18 o'clock",
linestyle="dashed",
color='green')
ax1_2.plot(x2_d_18,
y2_d_18,
label="Holiday 18 o'clock",
linestyle="dashed",
color='red')
ax[1].set_xlabel('zones', fontsize=14)
ax[1].tick_params(axis='x', rotation=60, labelsize=14)
ax[1].set_ylabel('Number of vehicles', fontsize=16)
# ax.set_yticks(y1_newsticks)
ax[1].grid(alpha=.4)
ax[1].legend(loc='upper left', prop={'size': 16}, framealpha=0.5)
ax1_2.legend(loc='upper right', prop={'size': 16}, framealpha=0.5)
ax1_2.set_ylabel("Number of transactions", fontsize=16)
ax[1].set_ylim(0, 1000)
ax[1].set_xticks(x18[::3])
fig.tight_layout()
# plt.grid(linestyle='-.')
plt.savefig('(2)transaction distribution.pdf')
plt.show()
#print "Prob select: ", prob_select
if rand_test is True:
test = np.random.choice(a_test,test_size,replace=False)
val = np.random.choice(a,val_size,replace=False)
else:
test = range(0,test_size)
val = range(0,val_size)
prob_select[val]=0.
prob_select*=float(total_batch_size-test_size_train)/float(total_batch_size-test_size_train-val_size)
#print "Prob select: ", prob_select
assert np.intersect1d(test,val).size is 0
print "Test size: ", test_size, " Val_size: ", val_size, " Batch size: ", batch_size, " Total size: ", total_batch_size
print "Meas: ", m, " Out: ",p, " Steps: ",n_steps
batches = int((total_batch_size-val_size-test_size_train)/batch_size)
per_batch = int(5000/batches)
print "Batches: ", batches, " Batches*batch_size: ", batches*batch_size, " Train set size: ",(total_batch_size-val_size-test_size_train), " Per batch: ", per_batch
n_in=meas_dims[0]*meas_dims[1]*2
n_dense=int((meas_dims[0]-k_conv+1)/k_pool-k_conv+1)*int((meas_dims[1]-k_conv+1)/k_pool-k_conv+1)*n_conv2
time.sleep(1)
batch_num=0
def rangefinder_distance(self, theta):
"""
:return:
distance: distance between laser rangefinder and object it's facing.
x_intersect = X coord of intersection ----- closest_intsct_x
y_intersect = Y coord of intersection ----- closest_Intsct_y
x_range: range of X values of sensor beam
y_range: range of Y values of sensor beam
"""
env_x = self.env[0]
env_y = self.env[1]
sensor_angle = theta % 360
if sensor_angle == 0:
x_max = self.x_dim
x_range = np.linspace(self.sensor_x, x_max,
self.x_dim - self.sensor_x + 1)
y_range = self.sensor_y * np.ones(len(x_range))
idx = np.nonzero(env_x >= self.sensor_x)
env_x = env_x[idx]
env_y = env_y[idx]
elif sensor_angle >= 0 and sensor_angle < 90:
slope = np.tan(theta * np.pi / 180)
y_intersect = self.sensor_y - slope * self.sensor_x
x_max = min((self.y_dim - y_intersect) / slope, self.x_dim)
x_range = np.linspace(self.sensor_x, x_max,
self.x_dim - self.sensor_x + 1)
y_range = slope * x_range + y_intersect
elif sensor_angle == 90: # facing straight up
y_range = np.arange(self.sensor_y, self.y_dim + 1)
x_range = self.sensor_x * np.ones(len(y_range))
elif sensor_angle > 90 and sensor_angle <= 180:
slope = np.tan(theta * np.pi / 180)
y_intersect = self.sensor_y - slope * self.sensor_x
x_min = max((self.y_dim - y_intersect) / slope, 1)
x_range = np.linspace(x_min, self.sensor_x, self.sensor_x) #debug
y_range = slope * x_range + y_intersect
idx_x = np.nonzero(env_x <= self.sensor_x)
idx_y = np.nonzero(env_y >= self.sensor_y)
idx = np.intersect1d(idx_x, idx_y)
env_x = env_x[idx]
env_y = env_y[idx]
elif sensor_angle > 180 and sensor_angle < 270:
slope = np.tan(theta * np.pi / 180)
y_intersect = self.sensor_y - slope * self.sensor_x
x_min = max((1 - y_intersect) / slope, 1)
x_range = np.linspace(x_min, self.sensor_x, self.sensor_x) #debug
y_range = slope * x_range + y_intersect
idx_x = np.nonzero(env_x <= self.sensor_x)
idx_y = np.nonzero(env_y <= self.sensor_y)
idx = np.intersect1d(idx_x, idx_y)
env_x = env_x[idx]
env_y = env_y[idx]
elif sensor_angle == 270: # sensor facing straight down
y_range = np.arange(1, self.sensor_y + 1)
x_range = self.sensor_x * np.ones(len(y_range))
else:
slope = np.tan(theta * np.pi / 180)
y_intersect = self.sensor_y - slope * self.sensor_x
x_max = min((1 - y_intersect) / slope, self.x_dim)
x_range = np.linspace(self.sensor_x, x_max,
self.x_dim - self.sensor_x + 1)
y_range = slope * x_range + y_intersect
idx_x = np.nonzero(env_x >= self.sensor_x)
idx_y = np.nonzero(env_y <= self.sensor_y)
idx = np.intersect1d(idx_x, idx_y)
env_x = env_x[idx]
env_y = env_y[idx]
intsct_xs, intsct_ys = [], []
x_used = []
i = 0
while i < len(x_range):
cur_x = round(x_range[i])
x_used.append(cur_x)
# debug
cur_y = round(y_range[i])
x_idx_in_env = np.nonzero(env_x == cur_x)
y_idx_in_env = np.nonzero(env_y == cur_y)
x_in_env = env_x[x_idx_in_env]
y_in_env = env_y[y_idx_in_env]
if len(x_in_env) == 0 or len(y_in_env) == 0:
i = i + 1
continue
x_diff = abs(x_in_env - cur_x)
y_diff = abs(y_in_env - cur_y)
# # use for test
# print(len(x_diff))
min_diff_x = min(x_diff)
min_diff_y = min(y_diff)
min_diff_x_idx = np.argmin(x_diff)
min_diff_y_idx = np.argmin(y_diff)
# diff_flag indicates whether the nearest index comes from x axis.
if min_diff_x < min_diff_y:
min_diff = min_diff_x
min_diff_idx = min_diff_x_idx
real_diff = x_in_env - cur_x
diff_flag = 1
else:
min_diff = min_diff_y
min_diff_idx = min_diff_y_idx
real_diff = y_in_env - cur_y
diff_flag = 0
# ??
step = round(real_diff[min_diff_idx] / 10)
if step > 0:
i = i + step
else:
i = i + 1
# if the difference is no more than 1, then we just include the current location.
if min_diff <= 1:
# if the minimum is selected along x axis.
if diff_flag:
intsct_ys.append(cur_y)
intsct_xs.append(x_in_env[min_diff_x_idx])
# if the minimum is selected along y axis.
else:
intsct_ys.append(y_in_env[min_diff_y_idx])
intsct_xs.append(cur_x)
dist = []
for i in range(len(intsct_xs)):
# compute the distance between sensor and detected object
distance = sqrt((intsct_xs[i] - self.sensor_x)**2 +
(intsct_ys[i] - self.sensor_y)**2)
dist.append(distance)
print(distance)
dist = np.asarray(dist)
closest_dist = min(dist)
closest_idx = np.argmin(dist)
# convert to np array and we can index with values returned by np.argmin or np.argmax.
intsct_xs = np.asarray(intsct_xs)
intsct_ys = np.asarray(intsct_ys)
closest_intsct_x = intsct_xs[closest_idx]
closest_intsct_y = intsct_ys[closest_idx]
return closest_dist, closest_intsct_x, closest_intsct_y, x_range, y_range