def test_upsample_component(n_max_range, frequencies):
for shape in B_N_COEFF_MAP.keys():
for n, f in zip(n_max_range, frequencies):
terms = arange(1, n + 1)
coefficients = B_N_COEFF_MAP[shape](terms)
upsampled = cy_builder_utils.upsample_component(
0.9, 45., 1., TIME_RANGE, coefficients, terms)
assert isinstance(upsampled, ndarray)
mask = npround(upsampled, 8) == \
npround(upsample_component(
0.9, 45., 1., TIME_RANGE, coefficients, terms), 8
)
assert all(mask)
def _map_fn(x):
if b.float_max - b.float_min == 0.0:
return 0.0
else:
return floor(npround(
((x - b.float_min) * b.int_max)/(b.float_max - b.float_min)
))
def category_capital_flow(category: str, date: datetime.date):
"""行业资金净流量
Args:
category: 行业代码
date: 起始日期
Returns:
行业资金净流量
"""
stocks = stocks_of_target_category(category=category)
flow = models.CapitalFlow.objects.filter(secucode__in=stocks,
date__gte=date).values(
'date', 'netvalue')
flow: pd.DataFrame = pd.DataFrame(flow)
flow.netvalue = flow.netvalue.astype(float)
flow = flow.groupby('date').sum()
flow = flow.reset_index()
flow.date = flow.date.shift(-1)
d = flow.iloc[-2, 0]
last = models.TradingDays.objects.filter(date__gt=d).first().date
flow.iloc[-1, 0] = last
flow['netvalue'] /= 1e6
flow['MA3'] = flow['netvalue'].rolling(3).mean()
flow['MA5'] = flow['netvalue'].rolling(5).mean()
flow['MA10'] = flow['netvalue'].rolling(10).mean()
flow['SIGMA5'] = flow['netvalue'].rolling(5).std() * 0.3
flow['MA5_HIGH'] = flow['MA5'] + flow['SIGMA5']
flow['MA5_LOW'] = flow['MA5'] - flow['SIGMA5']
flow = npround(flow, 2)
flow = flow.dropna(how='any')
flow = flow.reset_index()
return flow
def _redo_bd_nrml(self):
"""
Create nrml for cylinder body (without bottom) ,pointing outward
-----------------------
Note: only check those element with 'body' tag
Note: all elements should be taged 'body'
"""
print ("Create nrml for cylinder body (without bottom) ,pointing outward")
flag = True
for i in self.elems:
if self.elems[i][0] != 'body':
flag = False
if not flag:
print ("Please make sure all elements is defined as 'body'")
return
self.rev_nm={}
for i in self.nrmls:
info = self.nrmls[i]
# xyz = array(self.nodes[info[0]],dtype='float64')
xyz=np.array(self.nodes[info[0]])
xyz[2] = 0.
nrml = xyz/norm(xyz)#normalize vector
self.nrmls[i] = (info[0],nrml[0],nrml[1],nrml[2])
nrml = npround(nrml,self.__dp)
key = (info[0],nrml[0],nrml[1],nrml[2])
self.rev_nm[key] = i
return
def convert_weight(self, *args):
# Function to take an input weight from the scale and convert
# it for the selected planet
factor = {
'Mercury': 0.378,
'Venus': 0.91,
'Earth': 1,
'Moon': 0.1656,
'Mars': 0.379,
'Ryugu': 0.0000125,
'Bennu': 0.00000102,
'Ceres': 0.0284,
'Jupiter': 2.53,
'Saturn': 1.07,
'Titan': 0.1381,
'Uranus': 0.91,
'Neptune': 1.14,
'Pluto': 0.0635
}
if self.current_world != 'Bennu':
#self.kg_label.text = str(round(self.kg*factor[self.current_world])) + ' kg'
self.lb_label.text = str(
round(self.kg * factor[self.current_world] * 2.20462)) + ' lbs'
else:
#self.kg_label.text = str(npround(self.kg*factor[self.current_world], 5)) + ' kg'
self.lb_label.text = str(
npround(self.kg * factor[self.current_world] * 2.20462,
5)) + ' lbs'
def round(x, decimals=0):
r"""Round value to desired precision
Args:
x (float or iterable of floats): Value(s) to round
decimals (int): Number of decimal places for rounding; decimals=0 rounds to integers
"""
return npround(x, decimals=decimals)
def _update_all_nrml(self,vector):
'''
update all nrmls with given vector
'''
print ("vector will be auto normalized")
assert(len(vector)==3)
self.rev_nm={}
for i in self.nrmls:
info = self.nrmls[i]
xyz = array(vector,dtype='float64')
nrml = xyz/norm(xyz)#normalize vector
self.nrmls[i] = (info[0],nrml[0],nrml[1],nrml[2])
nrml = npround(nrml,self.__dp)
key = (info[0],nrml[0],nrml[1],nrml[2])
self.rev_nm[key] = i
def visit_polygon(self,polygon):
""" Adds the fields to edit a polygon's basic qualities, like position """
# grab the coordinates from the polygon
coords = polygon.coordinates
# back up its coordinates, if they are not already backed up
try: polygon.pure_coordinates
except: polygon.pure_coordinates = polygon.coordinates.copy()
# define their center as their mean along each axis
mcoords = npround(npmean(coords,0),0)
coordview = tk.Label(self.window,text='Origin')
coord_setter = tk.Frame(self.window)
text_vars = tuple(StringVar() for i in range(4))
# loops don't create new scopes but methods do
# also it's pretty cool that i gets implied by the loop variable
def add_entry(event=None):
""" Adds an entry field to modify a given axis """
textVar = text_vars[i]
textVar.set(str(mcoords[i]))
e = tk.Entry(coord_setter,textvariable=textVar,width=12)
e.grid(column=i,row=0)
def shift_poly(event = None):
""" Applies the transformation to shift the polygon, does not redraw canvas """
try: # it could be that they entered a non-float
dx = tuple(mcoords[j] - float(text_vars[j].get()) for j in range(4))
except Exception as e: # in which case, just ignore it
dx = 0
# shift each coordinate by the displacement implied by the entry field
coords = [ [el + dx[j] for j,el in \
enumerate(coord)] for coord in polygon.pure_coordinates]
# update the polygon's coordinates (it expects a numpy object)
polygon.coordinates = nparray(coords)
polygon._dirty()
# bind to the entry field update-on-entry
e.bind('<Return>', shift_poly )
# add all four entry widgets to update 3 axes plus homogeneous
for i in range(4): add_entry(i)
self.fields.append(coordview)
self.fields.append(coord_setter)
def random_walk(self, d=None, plot_h=None) -> None:
'''Let all population walk randomly'''
walk_left = self.move_per_day.copy()
# Every day, people start from home
pos = self.home.copy()
# Some travel less, some more
while npany(walk_left):
walk_left -= 1
# All randomly move an edge-length
pos += nparray(npround(
(nprandom.random(size=pos.shape) * 2 - 1) *
self.rms_v.T[:, None]) * (walk_left != 0).T[:, None],
dtype=pos.dtype)
# Can't jump beyond boundary
# So, reflect exploration
# pos = pos.clip(min=0, max=self.p_max-1)
pos = nparray(npnot(pos > (self.p_max-1)) * npabs(pos),
dtype=pos.dtype)\
+ nparray((pos > (self.p_max-1)) * (2 * (self.p_max - 1) - pos),
dtype=pos.dtype)
for indiv in range(self.pop_size):
# TODO: A ufunc or async map would have been faster
if walk_left[indiv]:
self.calc_exposure(indiv, pos)
if plot_h.contam_dots:
strain_persist = self.strain_types[-1].persistence
host_types = []
host_types.append(
(pos *
(npnot(self.active[:, None]) * self.susceptible[:, None] >
self.resist_def)).tolist())
host_types.append((pos * self.active[:, None]).tolist())
host_types.append((
pos *
(npnot(self.active[:, None]) *
(self.susceptible[:, None] <= self.resist_def))).tolist())
pathn_pers = []
for pers in range(int(strain_persist))[::-1]:
pathn_pers.append(
npnonzero(self.space_contam == (pers + 1)))
plot_h.update_contam(host_types, pathn_pers)
return
ts = traj_df.index[1] - traj_df.index[0]
prec = -int(floor(log10(ts)))
while not isclose(ts % 10**-prec, 0):
prec += 1
# print(f'precision: {prec}')
print(f'First time point: {traj_df.index.values[0]}, {prec} decimal places')
# dtt = (dt.now()-t1).total_seconds()
# print(f'loaded : {options["file"]}, time taken: {dtt:.2f} sec')
## read time bins
## subsample a bit for 'burnett'
if 'timebins' in keys:
if options['timebins'] == 'default':
timebins = [0] + list(arange(ts,1.0,ts)) + list(arange(1.0,3.0,ts*2)) \
+ list(npround(logspace(log10(3.0),log10(duration),500)/ts)*ts)
elif options['timebins'] == 'eb':
timebins = list(
npround(linspace(duration,
traj_df.index.max() * 0.9, 10000)))
# print(f'time bins: {timebins}')
elif options['timebins'] == 'burnett':
timebins = None
# traj_df = traj_df[::2]
# print('Timebins are "burnett", use every 2nd point, re-do precision')
# traj_df.index -= traj_df.index.values[0]
ts = traj_df.index[1] - traj_df.index[0]
prec = -int(floor(log10(ts)))
while not isclose(ts % 10**-prec, 0):
prec += 1
else:
def round(x):
r"""Absolute value
"""
return npround(x)
def mint(X,
n_bins=10,
n_neighbors=10,
pattern_length='standard',
eps=1,
prune=True,
max_nmb_candidates=2000,
not_discretized=True):
gamma_eps = gammaln(eps) / l2
def not_equal(l1, l2):
for v, w in zip(l1, l2):
if v != w:
return True
return False
def not_greater(l1, l2):
for v, w in zip(l1, l2):
if v > w:
return False
return True
def get_gain_min(pid1, pid2):
lo1, u1, usg1, g1 = patterns[pid1]
lo2, u2, usg2, g2 = patterns[pid2]
l = minimum(lo1, lo2)
u = maximum(u1, u2)
size = sum([log(v, 2) for v in (u - l) + 1])
usg = usg1 + usg2
c = (l, u, usg, usg * size - gammaln(usg + eps) / l2)
cands[(pid1, pid2)] = c
gains[(pid1, pid2)] = g1 + g2 - c[3]
def prune_it(L_total):
max_border = X_dun.max(axis=0)
min_border = X_dun.min(axis=0)
cands.clear()
gains.clear()
# compute candidates
candidates = list(patterns.keys())
cand_inds = [(k1, k2) for i, k1 in enumerate(candidates)
for k2 in candidates[i + 1:]]
for f, l in cand_inds:
get_gain_min(f, l)
new_id = max(patterns.keys())
new_patterns = []
# start minimization: computing the candidates
n_old_patterns = len(patterns) + 1
while n_old_patterns > len(patterns):
sorted_gains = sorted(gains.items(),
key=operator.itemgetter(1),
reverse=True)
included = defaultdict(lambda: [])
# creating the candidates for merging (more than a pair)
for (pid1, pid2), _ in sorted_gains:
if patterns.get(pid1) and patterns.get(pid2):
l, u, _, _ = cands[(pid1, pid2)]
for pid, (pl, pu, _, _) in patterns.items():
if (pid != pid1) and (pid != pid2):
if not_greater(l, pl) & not_greater(pu, u):
included[(pid1, pid2)].append(pid)
if len(included) == max_nmb_candidates:
break
else:
del gains[(pid1, pid2)], cands[(pid1, pid2)]
# getting the gains for candidates that includes for than a pair of patterns
new_gains = {key: gains[key] for key in included.keys()}
sorted_new_gains = sorted(new_gains.items(),
key=operator.itemgetter(1),
reverse=False)
n_old_patterns = len(patterns)
new_patterns.clear()
#print('new iteration ', n_old_patterns, mid_time - time.time())
while len(sorted_new_gains) > 0:
# greedy strategy: chosing the best pair
# (don't consider how many other patterns the candidate includes)
(pid1, pid2), _ = sorted_new_gains.pop()
if patterns.get(pid1) and patterns.get(pid2)\
and not_equal(cands[(pid1, pid2)][1], max_border)\
and not_equal(cands[(pid1, pid2)][0], min_border):
# avoid creating the pattern enveloping the whole space
n_patterns = len(patterns)
accepted_patterns = [
] # list of patterns included in the candidate pattern
l, u, usg_total, _ = cands[(pid1, pid2)]
size = sum([log(v, 2) / l2 for v in (u - l) + 1
]) # the size of the candidate pattern
L_delta_stable = l_n[n_patterns] + gammaln(G+eps*n_patterns)/l2\
- gammaln(eps*n_patterns)/l2 + l_cell + patterns[pid1][3]\
+ patterns[pid2][3] - usg_total*size + gamma_eps
d_pt = 1 # reduction in the number of patterns in the model
L_delta_old = L_delta_stable - l_n[n_patterns-d_pt]\
- gammaln(G+eps*(n_patterns-d_pt))/l2\
+ gammaln(eps*(n_patterns-d_pt))/l2\
+ gammaln(usg_total+eps)/l2
for pid in included[(pid1, pid2)]:
# start to add patterns that could improve the total length
if patterns.get(pid):
_, _, usg, g = patterns[pid]
d_pt_i = d_pt + 1
usg_total_i = usg_total + usg
L_delta_stable_i = l_cell + g - usg * size + gamma_eps
L_delta_variable = - l_n[n_patterns-d_pt_i]\
- gammaln(G+eps*(n_patterns-d_pt_i))/l2\
+ gammaln(eps*(n_patterns-d_pt_i))/l2\
+ gammaln(usg_total_i+eps)/l2
L_delta = L_delta_stable + L_delta_stable_i + L_delta_variable
if L_delta > L_delta_old: # accept merging
d_pt = d_pt_i
usg_total = usg_total_i
accepted_patterns.append(pid)
L_delta_stable += L_delta_stable_i
L_delta_old = L_delta
# checki if a new candidate allows for a shorter length
if L_delta_old > 0: # add pattern if it reduces the total length
new_id += 1
patterns[new_id] = (l, u, usg_total, usg_total*size\
- gammaln(usg_total+eps)/l2)
del patterns[pid1], patterns[pid2]
for pid in accepted_patterns:
del patterns[pid]
accepted_patterns.clear()
new_patterns.append(new_id)
L_total -= L_delta_old
# compute new candidates
for pid1 in new_patterns:
for pid2 in patterns:
if pid1 != pid2:
get_gain_min(pid1, pid2)
return L_total
n_neighbors += 1
# reduce the number of columns if there are some const-value ones
col_selected = [i for i in range(X.shape[1]) if len(unique(X[:, i])) > 1]
if X.shape[1] > len(col_selected):
X = X[:, col_selected]
G, M = X.shape
l_n = {key: un_int(key)
for key in range(0, 10 * max(G, M) + 1)} # length of int
# discretization
if not_discretized:
est = KBinsDiscretizer(n_bins=n_bins,
encode='ordinal',
strategy='uniform').fit(X)
X_discr = est.transform(X).astype(int)
L_CT_base = l_n[G] + l_n[M] + l_n[
n_bins] #+ M*log(binom(G-1, n_bins - 1),2)
if pattern_length == 'minimal':
l_cell = M * log(n_bins, 2)
elif pattern_length == 'standard':
l_cell = M * log(n_bins * (n_bins - 1) / 2 + n_bins, 2)
#l_cell = M*log(n_bins*(n_bins-1)/2,2)
#l_cell = 2*M*log(n_bins,2)
else:
raise ValueError("Invalid pattern length type.")
else:
n_bins_list = [
max(vals) - min(vals) + 1
for vals in [unique(X[:, i]) for i in range(M)]
]
X_discr = X
L_CT_base = l_n[G] + l_n[M] + sum(
[l_n[n_bins] for n_bins in n_bins_list])
if pattern_length == 'minimal':
l_cell = sum([log(n_bins, 2) for n_bins in n_bins_list])
elif pattern_length == 'standard':
l_cell = sum([
log(n_bins * (n_bins - 1) / 2 + n_bins, 2)
for n_bins in n_bins_list
])
else:
raise ValueError("Invalid pattern length type.")
# remove repetitive rows
X_dun, inverse_indices, counts = unique(X_discr,
return_counts=True,
return_inverse=True,
axis=0)
#print('elementary ', X_dun.shape)
n_neighbors = min(n_neighbors, X_dun.shape[0])
# to reconstruct original indices
new2original = {i: set() for i in range(X_dun.shape[0])}
for i, v in enumerate(inverse_indices):
new2original[v].add(i)
# computing the standard total length (the reconstruction error is 0)
n_patterns = X_dun.shape[0]
L_CT_var = l_n[n_patterns] + n_patterns * l_cell
L_DCT_init = gammaln(G+eps*n_patterns)/l2 - gammaln(eps*n_patterns)/l2\
- sum([gammaln(usg+eps)/l2 - gamma_eps for usg in counts])
L_total = L_CT_base + L_CT_var + L_DCT_init
# standard param on patterns
n_patterns_stand = n_patterns
L_total_stand = L_total
# --- time check ---
start_time = time.time()
tree = KDTree(X_dun, leaf_size=int(n_bins * .5))
# computing candidates using 'n_neighbors' closest points
patterns = {i: (row, row, cnt, -gammaln(cnt+eps)/l2) for i,(row, cnt) \
in enumerate(zip(X_dun, counts))}
neighbours = {
i: set(
tree.query(array(x).reshape(1, -1),
k=n_neighbors,
return_distance=False)[0][1:])
for i, x in enumerate(X_dun)
}
del tree
cand_inds = set([(i, j) if i < j else (j, i)
for i, lst in neighbours.items() for j in lst])
# computing the length gains for candidates
print('starts', len(cand_inds))
# start length minimization
cands = {}
gains = {}
cand_to_add = set([]) # to store individual patterns for candidate update
while len(cand_inds) > 0:
for f, l in cand_inds:
get_gain_min(f, l)
cand_inds.clear()
sorted_gains = sorted(gains.items(),
key=operator.itemgetter(1),
reverse=False)
L_total_old = L_total + 1
while (L_total <= L_total_old) and (len(sorted_gains) > 0):
n_patterns = len(patterns)
new_id = max(patterns.keys()) + 1
if len(sorted_gains) % 50000 == 0:
print('gain', len(sorted_gains))
gc.collect()
best_inds, best_gain = sorted_gains.pop()
if patterns.get(best_inds[0]) and patterns.get(best_inds[1]):
L_delta = l_n[n_patterns] - l_n[n_patterns-1] + l_cell\
+ gammaln(G+eps*n_patterns)/l2\
- gammaln(G+eps*(n_patterns-1))/l2\
+ gammaln(eps*(n_patterns-1))/l2\
- gammaln(eps*n_patterns)/l2 + gamma_eps + best_gain
if L_delta > 0:
patterns[new_id] = cands[best_inds]
neighbours[new_id] = set([
v for v in neighbours[best_inds[0]].union(neighbours[
best_inds[1]]) if patterns.get(v)
])
for v in neighbours[new_id]:
neighbours[v].add(new_id)
del patterns[best_inds[0]], patterns[best_inds[1]], neighbours[best_inds[0]], \
neighbours[best_inds[1]]
cand_to_add.add(new_id)
del gains[best_inds], cands[best_inds]
L_total_old = L_total
L_total -= L_delta
else:
break
else:
del gains[best_inds], cands[best_inds]
if len(cand_to_add) > 0:
clean_up = [v for v in cand_to_add if not patterns.get(v)]
for i in clean_up:
del cand_to_add[i]
cand_inds = set([(i, j) if i < j else (j, i) for i in cand_to_add
for j in neighbours[i]
if (patterns.get(j)) and (i != j)])
cand_to_add.clear()
print('start pruning')
mid_time = time.time()
mid_L_total = L_total
mid_n_patterns = len(patterns)
if prune:
L_total = prune_it(L_total)
end_time = time.time()
# compute pattern extensions
pattern_inds = {
k: where((X_dun >= l).all(axis=1) & (X_dun <= u).all(axis=1))[0]
for k, (l, u, _, _) in patterns.items()
}
pattern_true_inds = {}
for key, lst in pattern_inds.items():
pattern_true_inds[key] = set([])
for v in lst:
pattern_true_inds[key] = pattern_true_inds[key].union(
new2original[v])
if prune:
return pattern_true_inds, npround(L_total, 4), npround(L_total_stand, 4), \
n_patterns_stand, npround(end_time-start_time, 4),\
npround(mid_L_total-L_total, 3), mid_n_patterns - len(patterns),\
npround(end_time-mid_time, 4)
return pattern_true_inds, npround(L_total, 4), npround(L_total_stand, 4),\
n_patterns_stand, npround(end_time-start_time, 4)
## automatically calculate the NGP if r2 and r4 are available
if 'r2' in output.columns and 'r4' in output.columns:
output['a2'] = 0
iloc1 = output.time.iloc[1]
output.loc[iloc1:, 'a2'] = (dim / (dim + 2)) * output.loc[
iloc1:, 'r4'] / output.loc[iloc1:, 'r2']**2 - 1
elif option == 'eb':
output['eb'] = (output.r4 - output.r2**2) / output.r2**2
output = output.drop(columns=['r2', 'r4'])
elif option == 'vanhove':
## bin the many-start trajectories to get van Hove function
output = output.set_index('time').stack().reset_index(
level=1, drop=True).apply(sqrt).reset_index()
output.columns = ['time', 'r']
output['gs'] = output.r
spacebins = npround(arange(0, output.r.max() + 0.03, 0.02), 2)
# print(f'\nmax r = {output.r.max():.3g} simulation boxes')
output = output.groupby(['time', pd.cut(output.r,
spacebins)]).gs.agg('count')
# print('\nafter groupby:')
# print(output.head(10))
output = output.reset_index()
output.r = output.r.apply(lambda x: x.mid)
# print('\nfinal form:')
# print(output.head(10))
elif option == 'dx':
## this is similar to the van Hove function
output = output.set_index('time').stack().reset_index(
level=1, drop=True).reset_index()
output.columns = ['time', 'dx']
output['prob'] = output.dx
