def func_avg(kwargs, f=None, name=None):
kwargs = dict(kwargs)
pool = kwargs.pop("pool")
n_runs = kwargs.pop("n_runs")
result = pool.map(FunctionCaller(kwargs), range(n_runs))
xs, sizes = list(zip(*result))
if f is None:
f = open("tuning.txt", "w")
xs_str = "Evals: " + str(xs)
sizes_str = "Sizes: " + str(sizes)
median_str = "Median: %.1f Average: %.1f ~ %.1f" % (util.median(
xs), util.avg(xs), util.avg([x for x in xs if util.is_finite(x)]))
avg_size_str = "AverageSize: %.1f ~ %.1f" % (
util.avg(sizes), util.avg([s for s in sizes if util.is_finite(s)]))
success_rate = len([x for x in xs if util.is_finite(x)]) / len(xs)
success_rate_str = "Success_rate: %.2f" % success_rate
report_str = [
xs_str, sizes_str, median_str, avg_size_str, success_rate_str
]
if name is not None:
name_str = "* %s" % name
report_str = [name_str] + report_str
report_str = "\n".join(report_str)
print(report_str, file=f)
f.flush()
print(report_str)
def __init__(self, method, n=1, timeout=120.0):
from memory_profiler import memory_usage
import time
from util import avg
self.timeout = timeout
max_memory_list = []
avg_memory_list = []
time_list = []
for i in range(n):
runner = self._Runner(memory_usage, {'proc': method, 'interval': 0.01, 'retval': True}, timeout)
baseline = avg(memory_usage((lambda: None, ()))) # baseline memory usage measurement
start_time = time.time()
runner.run() # actual running of the method
end_time = time.time()
if runner.result is not None:
mem_use, self.result = runner.result
memory = map(lambda x: x - baseline, mem_use)
else: # there was a timeout
import sys
memory = [-sys.maxint]
self.result = None
max_memory_list.append(max(memory))
avg_memory_list.append(avg(memory))
time_list.append((end_time - start_time) * 1000)
self.max_memory = avg(max_memory_list)
self.avg_memory = avg(avg_memory_list)
self.time = avg(time_list)
def add_val(self, params, SCNs, val_key):
# downside of this implementation is that don't know which characteristics are computed first
# and have to recalc common things such as #steps and time
# if too slow add a pre-feature calculation step
################################## DISCRETE MODEL CHARACTERISTICS ##################################
if val_key == 'steps':
steps = [scn.steps for scn in SCNs]
return all_stats(steps, params)
elif val_key == 'probability off-diag':
init = params['IA']/(params['IA']+params['IB'])
As,Bs = [scn.mols['A'] for scn in SCNs], [scn.mols['B'] for scn in SCNs]
pr = [As[i]/(As[i]+Bs[i]) > init for i in rng(As)]
return all_stats(pr, params)
elif val_key == 'Percent Y0':
ratios = []
for scn in SCNs:
if scn.mols['Y0']==0:
ratios += [0]
else:
ratios += [scn.mols['Y0']/(scn.mols['Y0']+scn.mols['Y1'])]
# faster if know that sum != 0:
#ratios = [scn.mols['Y0']/(scn.mols['Y0']+scn.mols['Y1']) for scn in SCNs]
return all_stats(ratios, params)
elif val_key == 'dist from discrete bound':
discrete_bound = bounds.discrete(params['gamma'], params['delta'],params['IA'],params['IB'])
avg_steps = avg([scn.steps for scn in SCNs])
return math.log(discrete_bound - avg_steps,10), None, None, None, None, None
################################## CONTINUOUS MODEL CHARACTERISTICS ##################################
elif val_key in ['time', 'Convergence Time']:
stop_ts = [scn.time for scn in SCNs]
return all_stats(stop_ts, params)
elif val_key in ['time', 'Log Convergence Time']:
stop_ts = [math.log(scn.time,10) for scn in SCNs]
return all_stats(stop_ts, params)
elif val_key == 'log time':
stop_ts = [math.log(scn.time,10) for scn in SCNs]
return all_stats(stop_ts, params)
elif val_key == 'expected time': #appears to be the same as time, runs faster
stop_ts = [scn.time_mean_appx for scn in SCNs]
return all_stats(stop_ts, params)
elif val_key == 'dist from continuous bound':
cont_bound = bounds.continuous(params['gamma'], params['delta'],params['IA'],params['IB'])
avg_t = avg([scn.time for scn in SCNs]) #note that curr using exact time, may go faster with expected time
return math.log(cont_bound - avg_t,10), None, None, None, None, None
else:
raise KeyError("Unrecognized value %s, use recognized value keys when initializing Dataset (see Dataset.add_val for existing options)." %(val_key))
def back_test_tick(self, base_ticker, trade_ticker, symbol):
diff_price = trade_ticker.price - base_ticker.price
self.diffs.append(diff_price)
self.base_prices.append(base_ticker.price)
self.trade_prices.append(trade_ticker.price)
if self.in_warming():
return
window = 10
base_ma = avg(self.base_prices[-window:])
trade_ma = avg(self.trade_prices[-window:])
self.base_price_mas.append(base_ma)
self.trade_price_mas.append(trade_ma)
self.diff_mas.append(trade_ma - base_ma)
if len(self.diff_mas) < 10:
return
if self.bt_status == BtStatus.PLACE_BUY_ORDER:
if trade_ticker.price < self.bt_buy_price:
print 'buy success %s, id: %s, ts: %s' % (
trade_ticker.price, self.bt_record.id, self.bt_record.ar)
self.bt_status = BtStatus.SUCCESS_BUY_ORDER
self.bt_tx_cnt += 1
if self.bt_tx_cnt % 2 == 0:
self.bt_benefit += self.bt_sell_price - self.bt_buy_price
elif self.bt_status == BtStatus.PLACE_SELL_ORDER:
if trade_ticker.price > self.bt_sell_price:
print 'sell success %s, id: %s, ts: %s' % (
trade_ticker.price, self.bt_record.id, self.bt_record.ar)
self.bt_status = BtStatus.SUCCESS_SELL_ORDER
self.bt_tx_cnt += 1
if self.bt_tx_cnt % 2 == 0:
self.bt_benefit += self.bt_sell_price - self.bt_buy_price
elif self.bt_status == BtStatus.PLACE_BUY_ORDER and self.bt_record.id > self.bt_buy_record_id + 5:
self.bt_cancel_buy()
else:
can_buy = self.ana_buy_status()
can_sell = self.ana_sell_status()
add_price = -0.0001
if can_buy:
buy_price = trade_ticker.ask + add_price
self.back_test_buy(buy_price, 0.001)
if can_sell:
sell_price = trade_ticker.bid - add_price
if sell_price > self.bt_buy_price + 10:
self.back_test_sell(sell_price, 0.001)
def create_genome_evaluation(genome,
fitness,
net=None,
fitness_test=None,
window=None,
generation=None,
initial_time=None,
build_time=0,
pred_time=0,
pred_avg_time=0,
fit_time=0,
include_genome=False,
extra={},
**kwargs):
if net is None:
net = util.build_network(genome, **kwargs)
global_time = datetime.datetime.now(
) - initial_time if initial_time is not None else None
if isinstance(genome, list):
genome_neurons = util.avg([g.NumHiddenNeurons() for g in genome])
genome_connections = util.avg([g.NumLinks() for g in genome])
else:
genome_neurons = genome.NumHiddenNeurons()
genome_connections = genome.NumLinks()
if isinstance(net, list):
neurons = util.avg([n.NumHiddenNeurons() for n in net])
connections = util.avg([n.NumConnections() for n in net])
else:
neurons = net.NumHiddenNeurons()
connections = net.NumConnections()
return GenomeEvaluation(genome=genome if include_genome else None,
fitness=fitness,
fitness_test=fitness_test,
genome_neurons=genome_neurons,
genome_connections=genome_connections,
neurons=neurons,
connections=connections,
generation=generation,
window=window,
global_time=global_time,
build_time=build_time,
pred_time=pred_time,
pred_avg_time=pred_avg_time,
fit_time=fit_time,
**extra)
def reached_waypoint(num, gps_buffer, tolerance):
(lat, lon) = WAYPOINTS[num - 1]
distance = avg([
dist_to_waypoint(msg, (lat + LAT_OFFSET, lon + LON_OFFSET))
for msg in gps_buffer
])
# state_debug.publish(str(distance))
return distance < tolerance
def make_indices_by_MA_delta_old(conn, his_md, MA_Size1=5):
recent_mds = {} # 代码==> 该代码最后几交易日的‘行情’ ** 跳过停牌日
# 其中‘行情’ 是 [
# [交易日，收盘价],
# [交易日，收盘价], ...
# ]
md_prev_day = None
for md_that_day in his_md:
t_day = md_that_day[0]
md_set = md_that_day[1]
indices = collections.OrderedDict()
for code, md_set in md_that_day[1].iteritems():
if md_prev_day is None or code not in md_prev_day[
1] or code not in recent_mds:
# 第一天 昨日行情里没有本code ‘最近交易日’记录里没有本code
# 需要从外部获取本code最后N日记录
recent_memo = data_fetcher.get_his_until(code, t_day, MA_Size1)
recent_mds[code] = recent_memo
else:
# 停牌的行情不加入 recent_mds
if not md_set[4]:
recent_mds[code].append([t_day, md_set[0]])
if len(recent_mds[code]) > MA_Size1:
del recent_mds[code][0]
MA1 = util.avg(recent_mds[code])
# 至此MA有了，准备其他指标
if md_set[3] or md_set[4]:
# 涨停或者停牌的不能买
can_buy = 0
deviation = 0 # 没必要再算偏离
else:
can_buy = 1
if 0 == MA1:
deviation = 0
else:
deviation = (md_set[0] - MA1) / MA1
indices[code] = [can_buy, deviation, MA1]
md_that_day.append(indices)
# 准备走向下一天
md_prev_day = md_that_day
return his_md
def xor_investigate_average():
n_runs = 100
x = []
for i in range(n_runs):
result = xor_investigate(i)
print("xor(%d) = %d" % (i, result))
x.append(result)
#x = [xor_investigate() for i in range(n_runs)]
print("Average=%.2f" % util.avg(x))
print("Median=%.2f" % util.median(x))
def extract_stats(data, feature_names, sshot):
for name in feature_names:
feat = extract_one(name, sshot)
if feat != []:
data[name]['avg'] += [util.avg(feat)]
data[name]['var'] += [util.var(feat)]
if util.avg(feat) != 0:
iod = util.var(feat) / util.avg(feat)
else:
iod = 0
data[name]['iod'] += [iod]
max1 = max(feat)
feat.remove(max1)
max2 = max(feat)
data[name]['max'] += [max1]
if max1 == 0:
ratio = 0
else:
ratio = (max1 - max2) / max1
data[name]['1:2'] += [ratio]
def present(self):
# averages are usually heavily biased by the few initial food findings
#avg_cost = (self.reality.world.elapsed_time*self.number_of_ants)/self.food_discovered
#avg_moves = self.moves_leading_to_food_being_found/self.food_discovered
fl_avg_cost = avg([cost for moves, cost in self.last_results])
fl_avg_moves = avg([moves for moves, cost in self.last_results])
avg_pheromone = self.reality.world.get_average_pheromone_level()
max_pheromone = self.reality.world.get_max_pheromone_level()
pheromone_ratio = max_pheromone/(avg_pheromone or 1)
self.stats_saver.add_sample(
{
'food_discovered': self.food_discovered,
'time': self.reality.world.elapsed_time,
'avg_pheromone': '%.12f' % avg_pheromone,
'max_pheromone': '%.12f' % max_pheromone,
'last_cost': self.last_cost,
'pheromone_ratio': pheromone_ratio,
'best_finding_cost': self.best_finding_cost,
'fl_avg_cost': fl_avg_cost,
'fl_avg_moves': fl_avg_moves,
}
)
def get_right(self, exchange, coin, li, side='bid', amount=None):
total = 0
res = []
if amount is None:
amount = AMOUNT_THRESHOLD[coin]
if side == 'ask':
li.reverse()
for x in li:
total += x[1]
if total >= amount:
res.append(x[0])
break
else:
res.append(x[0])
return avg(res)
def extend_indices_add_ma(conn, his_md, MA_Size1=5):
recent_mds = {} # 代码==> 该代码最后几交易日的‘行情’ ** 跳过停牌日
# 其中‘行情’ 是 [
# [交易日，收盘价],
# [交易日，收盘价], ...
# ]
md_prev_day = None
for md_that_day in his_md:
t_day = md_that_day[0]
mds = md_that_day[1]
indices = md_that_day[2]
for code, md_of_the_code in mds.iteritems():
indi_of_the_code = indices[code]
if md_prev_day is None or code not in md_prev_day[
1] or code not in recent_mds:
# 第一天 昨日行情里没有本code ‘最近交易日’记录里没有本code
# 需要从外部获取本code最后N日记录
recent_memo = data_fetcher.get_his_until(code, t_day, MA_Size1)
recent_mds[code] = recent_memo
else:
# 停牌的行情不加入 recent_mds
if not md_of_the_code[4]:
recent_mds[code].append([t_day, md_of_the_code[0]])
if len(recent_mds[code]) > MA_Size1:
del recent_mds[code][0]
MA1 = util.avg(recent_mds[code])
# 至此MA有了
indi_of_the_code.append(MA1)
# 准备走向下一天
md_prev_day = md_that_day
return his_md
def draw(self, origin, size, step, radius=-1):
"""
Draw field vectors periodically within a region.
Drawing begins at the origin and continues by the given step up to
and including the size. Both the size and step may be vectors or
scalars. Scalars are equivalent to vectors with equal dimensions.
"""
# get the origin coordinates
x0, y0, z0 = origin
try:
l, h, w = size
except TypeError:
l = w = h = size
try:
step_x, step_y, step_z = step
except TypeError:
step_x = step_y = step_z = step
xs = range(int(x0), int(x0 + l) + 1, int(step_x))
ys = range(int(y0), int(y0 + h) + 1, int(step_y))
zs = range(int(z0), int(z0 + w) + 1, int(step_z))
#getting values for color interpolation
for x in xs:
for y in ys:
for z in zs:
self[vector(x, y, z)] = self(vector(x, y, z))
self.avg_mag = avg(B.mag for B in self.Ps.values())
self.largest_mag = max(B.mag for B in self.Ps.values())
#smallest mag is estimate due to absolute smallest being zero, not useful
self.smallest_mag = self.largest_mag - 2 * (self.largest_mag -
self.avg_mag)
for x in xs:
for y in ys:
for z in zs:
if (radius == -1 or (x**2 + y**2 + z**2)**0.5 <= radius):
self.draw_at(vector(x, y, z), step)
def draw(self, origin, size, step, radius = -1):
"""
Draw field vectors periodically within a region.
Drawing begins at the origin and continues by the given step up to
and including the size. Both the size and step may be vectors or
scalars. Scalars are equivalent to vectors with equal dimensions.
"""
# get the origin coordinates
x0, y0, z0 = origin
try:
l, h, w = size
except TypeError:
l = w = h = size
try:
step_x, step_y, step_z = step
except TypeError:
step_x = step_y = step_z = step
xs = range(int(x0), int(x0+l)+1, int(step_x))
ys = range(int(y0), int(y0+h)+1, int(step_y))
zs = range(int(z0), int(z0+w)+1, int(step_z))
#getting values for color interpolation
for x in xs:
for y in ys:
for z in zs:
self[vector(x, y, z)] = self(vector(x, y, z))
self.avg_mag = avg(B.mag for B in self.Ps.values())
self.largest_mag = max(B.mag for B in self.Ps.values())
#smallest mag is estimate due to absolute smallest being zero, not useful
self.smallest_mag = self.largest_mag- 2*(self.largest_mag - self.avg_mag)
for x in xs:
for y in ys:
for z in zs:
if (radius == -1 or (x**2 + y**2 + z**2)**0.5 <= radius):
self.draw_at(vector(x, y, z), step)
def going_flat(gps_buffer, tolerance):
angle = avg([current_angle(loc) for loc in gps_buffer])
return (angle < tolerance and angle > tolerance * -1)
def going_down(gps_buffer, tolerance):
angle = avg([current_angle(loc) for loc in gps_buffer])
return angle < tolerance * -1
def going_up(gps_buffer, tolerance):
angle = avg([current_angle(loc) for loc in gps_buffer])
return angle > tolerance
def get_rotated_x(self, angle):
rotated_points = [
rotate_point(-angle, price.top_right) for price in self.prices
]
return avg([point[0] for point in rotated_points])
def getGroundSensors(self):
ambient = self.network.GetVariable("thymio-II", "prox.ground.ambiant")
reflected = self.network.GetVariable("thymio-II", "prox.ground.reflected")
delta = self.network.GetVariable("thymio-II", "prox.ground.delta")
return (avg(ambient), avg(reflected), avg(delta))
def get_average_pheromone_level(self):
return avg(edge.pheromone_level() for edge in self.edges)
def drawG_by_rank(G,params, topK):
# color by seed/not_seed, alpha by rank
assert(os.path.isdir(params['output_path']))
plt.figure(figsize=(10,6))
node_size, nalpha, ealpha = 800, .3, .3 #node_alpha not curr used
# positions for all nodes
if params['draw_layout'] == 'circular':
pos = nx.circular_layout(G)
elif params['draw_layout'] == 'spring':
pos = nx.spring_layout(G)
else: assert(False) #unknown draw layout
nodes = list(G.nodes())
colors, alphas = [], []
max_rank = max(G.nodes[nodes[i]]['rank'] for i in rng(nodes))
for i in rng(nodes):
if G.nodes[nodes[i]]['seed']:
G.nodes[nodes[i]]['alpha'] = 1
alphas += [1]
colors+=['green']
else:
colors+=['blue']
#G.nodes[nodes[i]]['alpha'] = G.nodes[nodes[i]]['rank']/max_rank #for the edges
G.nodes[nodes[i]]['alpha'] = G.nodes[nodes[i]]['cardinal_rank'] #for the edges
alphas += [G.nodes[nodes[i]]['cardinal_rank']]#/max_rank] #may not work..also gotta add LOG SCALING
nx.draw_networkx_nodes(G, pos, nodelist=nodes, node_color=colors, node_size=node_size, alpha=alphas)
#labels = {n: G.nodes[n]['gene'] for n in G.nodes()}
topK2 = topK.tolist()
top = topK2 + params['seeds']
labels = {n:n for n in top} #huh?
nx.draw_networkx_labels(G, pos, labels, font_size=8, font_color='blue')
#edge_alphas = []
ecolors = []
elist = sorted(list(G.edges()))
for i in rng(elist):
n1,n2 = elist[i][0],elist[i][1]
#edge_alphas += [min(G.nodes[n1]['alpha'],G.nodes[n2]['alpha'])]
ecolors += [(0,0,0,avg([G.nodes[n1]['alpha'],G.nodes[n2]['alpha']]))]
nx.draw_networkx_edges(G, pos, arrows=True, edgelist=elist, edge_color=ecolors) #,alpha=edge_alphas)
if params['save_fig']:
now = datetime.now()
curr_date = str(date.today()).strip('2020-')
curr_time = str(datetime.now().strftime("%H-%M-%S"))
plt.savefig(params['output_path']+'/'+curr_date+'_'+curr_time+'_rankedG.png')
else:
plt.show()
plt.clf()
plt.close()
def get(self):
symbol = self.get_argument('symbol', 'BTC_USDT')
limit = int(self.get_argument('limit', '2000'))
table = self.get_argument('table', 'okex_binance')
delta = int(self.get_argument('delta', '0'))
begin = int(self.get_argument('begin', '0'))
end = int(self.get_argument('end', '0'))
table = "abs_diff_" + table
if begin > 0 and end > 0:
sql = "select * from " + table + " where symbol=? and ts between ? and ?"
data = self.application.engine.fetch_row(sql, (symbol, begin, end))
else:
sql = "select * from (select * from " + table + " where symbol = ? order by id desc limit ?) sub order by id asc"
data = self.application.engine.fetch_row(sql, (symbol, limit))
option = {
"animation": False,
"title": {
"text": "diff"
},
"xAxis": {
"type": 'value',
"min": "dataMin",
"max": "dataMax",
},
"yAxis": {
"type": 'value',
"min": "dataMin",
"max": "dataMax",
},
"boundaryGap": [0, '100%'],
"legend": {
"data": ["diff_bid", "diff_ask", "diff_price"],
}
}
series = []
# series.append({
# 'name': 'diff_bid',
# 'type': 'line',
# 'data': [[x.ts, x.trade_bid - x.base_bid] for x in data]
# })
# series.append({
# 'name': 'diff_ask',
# 'type': 'line',
# 'data': [[x.ts, x.trade_ask - x.base_ask] for x in data]
# })
window = 10
base_data = []
dt = [float(x.base_price) for x in data]
for i in range(len(dt)):
if i < window:
v = avg(dt[:i + 1])
else:
v = avg(dt[i - window:i + 1])
if v > 0:
base_data.append([data[i].id, v])
data[i].base_ma = v
t_data = []
dt = [float(x.trade_price) for x in data]
for i in range(len(dt)):
if i < window:
v = avg(dt[:i + 1])
else:
v = avg(dt[i - window:i + 1])
if v > 0:
t_data.append([data[i].id, v])
data[i].trade_ma = v
if delta > 0:
series.append({
'name':
'diff',
'type':
'line',
'data': [[x.id, x.trade_ma - x.base_ma] for x in data],
})
else:
series.append({'name': 'base', 'type': 'line', 'data': base_data})
series.append({'name': 'price', 'type': 'line', 'data': t_data})
series.append({
'name': 'bid',
'type': 'line',
'data': [[x.id, float(x.trade_bid)] for x in data]
})
option['series'] = series
self.render("abs_diff.html", option=json.dumps(option))
def get(self):
symbol = self.get_argument('symbol', 'BTC_USDT')
limit = int(self.get_argument('limit', '2000'))
table = self.get_argument('table', 'okex_binance')
delta = int(self.get_argument('delta', '0'))
begin = int(self.get_argument('begin', '0'))
end = int(self.get_argument('end', '0'))
table = "abs_diff_" + table
if begin > 0 and end > 0:
sql = "select * from " + table + " where symbol=? and ts between ? and ?"
data = self.application.engine.fetch_row(sql, (symbol, begin, end))
else:
sql = "select * from (select * from " + table + " where symbol = ? order by id desc limit ?) sub order by id asc"
data = self.application.engine.fetch_row(sql, (symbol, limit))
option = {
"animation": False,
"title": {"text": "diff"},
"xAxis": {
"type": 'value',
"min": "dataMin",
"max": "dataMax",
},
"yAxis": {
"type": 'value',
"min": "dataMin",
"max": "dataMax",
},
"boundaryGap": [0, '100%'],
"legend": {
"data": ["diff_bid", "diff_ask", "diff_price"],
}
}
series = []
# series.append({
# 'name': 'diff_bid',
# 'type': 'line',
# 'data': [[x.ts, x.trade_bid - x.base_bid] for x in data]
# })
# series.append({
# 'name': 'diff_ask',
# 'type': 'line',
# 'data': [[x.ts, x.trade_ask - x.base_ask] for x in data]
# })
window = 10
base_data = []
dt = [float(x.base_price) for x in data]
for i in range(len(dt)):
if i < window:
v = avg(dt[:i+1])
else:
v = avg(dt[i-window:i+1])
if v > 0:
base_data.append([data[i].id, v])
data[i].base_ma = v
t_data = []
dt = [float(x.trade_price) for x in data]
for i in range(len(dt)):
if i < window:
v = avg(dt[:i+1])
else:
v = avg(dt[i-window:i+1])
if v > 0:
t_data.append([data[i].id, v])
data[i].trade_ma = v
if delta > 0:
series.append({
'name': 'diff',
'type': 'line',
'data': [[x.id, x.trade_ma - x.base_ma] for x in data],
})
else:
series.append({
'name': 'base',
'type': 'line',
'data': base_data
})
series.append({
'name': 'price',
'type': 'line',
'data': t_data
})
series.append({
'name': 'bid',
'type': 'line',
'data': [[x.id, float(x.trade_bid)] for x in data]
})
option['series'] = series
self.render("abs_diff.html", option=json.dumps(option))
sys.exit(0)
data_dir = sys.argv[1]
output_dir = sys.argv[2]
# Validate
eval_preds = None
eval_labels = None
for subjid in range(1, 4):
subj = str(subjid)
path = os.path.join(output_dir, 'eval.' + subj + '.npy')
preds = np.load(path)
eval_filename = 'eval-' + subj + '-' + str(0) + '-index.csv'
idx_file = os.path.join(data_dir, 'train_' + subj, eval_filename)
labels = np.loadtxt(idx_file, delimiter=',', skiprows=1, usecols=[1])
labels, preds = avg(labels, preds)
calibrate(subjid, preds)
print('Eval AUC for subject %d %.4f\n' % (subjid, auc(labels, preds)))
eval_preds = preds if subjid == 1 else np.hstack((eval_preds, preds))
eval_labels = labels if subjid == 1 else np.hstack((eval_labels, labels))
normalize(eval_preds)
print('Overall AUC %.4f\n' % auc(eval_labels, eval_preds))
# Test
preds = None
for subjid in range(1, 4):
path = os.path.join(output_dir, 'test.' + str(subjid) + '.npy')
vals = np.load(path)
vals = avg_preds(vals)
calibrate(subjid, vals)
def calc_k(self, rsv):
shift(self.m_rsv, rsv, self.m)
return avg(self.m_rsv)
pl.plot(wl_new, medfilt(n, 5))
pl.xlabel(r'Rest wavelength [$\mathrm{\AA}$]', fontsize=25)
pl.ylabel(r'Normalised flux density', fontsize=25)
pl.savefig("../figures/n_bursts.pdf")
pl.clf()
ma_e_arr = np.ma.array(e, mask=bpmap.astype("bool"))
median_err = np.sqrt((np.pi / 2) * np.ma.sum(ma_e_arr**2, axis=0) / n**2)
pl.plot(wl_new, median)
pl.plot(wl_new, median_err)
pl.show()
std = medfilt(np.ma.std(ma_arr, axis=0), 11)
wmean, wmeanerr, _ = util.avg(f,
e,
mask=bpmap.astype("bool"),
axis=0,
weight=True)
# Saving to .dat file
dt = [("wl", np.float64), ("median", np.float64),
("median_err", np.float64), ("wmean", np.float64),
("wmeanerr", np.float64)]
data = np.array(zip(wl_new, median, median_err, wmean, wmeanerr), dtype=dt)
np.savetxt(
"../gold_comp.dat",
data,
header=
"wavelength median median_error weighted_mean weighted_mean_err",
fmt=['%1.5e', '%1.5e', '%1.5e', '%1.5e', '%1.5e'],
delimiter="\t")
def WriteBack(self, def_vals, data, final_out_file, elaborate):
if elaborate is None:
return
self.__sync.acquire()
data = [x for x in data if x is not None]
data = [x for x in data if len(x)]
# we remove the empty elements
file_name = sobstitute(elaborate.out_file_name.currValue(), self)
in_file_format = elaborate.in_file_format
res = {}
res['MIN'] = [0] * len(in_file_format)
res['MAX'] = [0] * len(in_file_format)
res['AVG'] = [0] * len(in_file_format)
res['STD'] = [0] * len(in_file_format)
if len(data):
data_ex = zip(*data)
for i in range(len(in_file_format)):
res['AVG'][i] = avg(data_ex[i])
res['MIN'][i] = min(data_ex[i])
res['MAX'][i] = max(data_ex[i])
for i in range(len(in_file_format)):
res['STD'][i] = std(data_ex[i], res['AVG'][i])
# print "MINS: " + str(res['MIN'])
# print "MAXS: " + str(res['MAX'])
# print "AVG : " + str(res['AVG'])
# print "STD : " + str(res['STD'])
vals = []
for param in elaborate.out_file_format:
if param in self.__parameters.keys():
vals.append(toStr(self.__parameters[param].currValue()))
elif (param.startswith('AVG(') or param.startswith('STD(')
or param.startswith('MIN(')
or param.startswith('MAX(')) and param.endswith(')'):
param_name = param[4:-1]
assert param_name in in_file_format.values
vals.append(
toStr(res[param[:3]][in_file_format.values.index(
param_name)]))
self.__speedup = -1
idx = elaborate.speedup_idx.currValue()
if def_vals:
if def_vals['AVG'][idx] != 0 and res['AVG'][idx] != 0:
assert (idx > 0 or idx < 0)
if idx < 0:
idx = (-idx) - 1
self.__speedup = res['AVG'][idx] / def_vals['AVG'][idx]
else:
idx -= 1
self.__speedup = def_vals['AVG'][idx] / res['AVG'][idx]
vals.append(toStr(self.__speedup))
open(file_name, 'a').write(",".join(vals) + "\n")
print "\n{0}".format(80 * "*")
if res['AVG'][idx] == 0:
print "* Configuration failed to execute! "
else:
print "* Average values: \n*\t%s" % ", ".join(
map(lambda x: "{0:.3f}".format(x), res['AVG']))
if self.__speedup != -1:
print "* Speedup: \n*\t{0:.5f}".format(self.__speedup)
print "{0}".format(80 * "*")
self.__sync.notify()
self.__sync.release()
return res
def extend_indices_add_pb_std(conn, his_md, MA_Size1=5):
#MA_Size2 = 60
#MA_Size3 = 20
recent_mds = {} # 代码==> 该代码最后几交易日的‘行情’ ** 跳过停牌日
# 其中‘行情’ 是 [
# [交易日，PB],
# [交易日，PB], ...
# ]
md_prev_day = None
for md_that_day in his_md:
t_day = md_that_day[0]
mds = md_that_day[1]
indices = md_that_day[2]
for code, md_of_the_code in mds.iteritems():
PB = md_of_the_code[5]
indi_of_the_code = indices[code]
if md_prev_day is None or code not in md_prev_day[
1] or code not in recent_mds:
# 第一天 昨日行情里没有本code ‘最近交易日’记录里没有本code
recent_memo = [[t_day, PB]]
recent_mds[code] = recent_memo
else:
# 停牌的行情不加入 recent_mds
if not md_of_the_code[4]:
recent_mds[code].append([t_day, PB])
if len(recent_mds[code]) > MA_Size1:
del recent_mds[code][0]
#对于PB策略，样本不足则 平均值和标准差都意义不足，所以
if len(recent_mds[code]) < MA_Size1:
MA1 = 0
std = 0
bias = 0
#MA2=0
#MA3=0
else:
#print recent_mds[code]
MA1 = util.avg(recent_mds[code])
std = numpy.std(util.column_of_a2d(recent_mds[code], 1))
bias = (PB - MA1) / std
#MA2 = util.avg( recent_mds[code][ - MA_Size2 :])
#MA3 = util.avg( recent_mds[code][ - MA_Size3 :])
indi_of_the_code.append(bias)
indi_of_the_code.append(PB)
indi_of_the_code.append(MA1)
indi_of_the_code.append(std)
#indi_of_the_code.append(MA2)
#indi_of_the_code.append(MA3)
# 准备走向下一天
md_prev_day = md_that_day
return his_md
#di['total_real_time'] = results['total_real_time']
stats_path = os.path.join(root, stats_filename)
# load data
stats_data = []
for row in itertools.islice(csv.reader(open(stats_path, 'rb')), 1, None):
#print row[0], row[3]
stats_data.append(row[3])
# precompute
group_size = 500
aggregates = []
while stats_data:
aggregates.append(
avg(
[float(stats_data.pop()) for i in xrange(group_size)]
)
)
aggregates.reverse()
if aggregates[3] / 2 < aggregates[-1]:
continue
print root
pprint(aggregates)
#break
raw_input()
raise Exception()
columns = sorted(queens)
writer = csv.writer(open('matrix.csv', 'wb'))
def find_notes(img, objs, staff_lines):
rows, cols = img.shape[:2]
temp = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
for c, r, w, h in objs:
cv2.rectangle(temp, (c, r), (c + w, r + h), 127, 1)
# from 5 staff line coordinates, build coordinates of nine positions
# (5 lines and four spaces between each line)
staff_lines = sorted(staff_lines)
avg = sum([abs(staff_lines[i] - staff_lines[i + 1]) \
for i in range(len(staff_lines) - 1)]) / (len(staff_lines) - 1)
avg2 = float(avg) / 2
positions = {
staff_lines[0] - avg2: 'G5',
staff_lines[0]: 'F5',
util.avg(staff_lines, 0, 1): 'E5',
staff_lines[1]: 'D5',
util.avg(staff_lines, 1, 2): 'C5',
staff_lines[2]: 'B4',
util.avg(staff_lines, 2, 3): 'A4',
staff_lines[3]: 'G4',
util.avg(staff_lines, 3, 4): 'F4',
staff_lines[4]: 'E4',
staff_lines[4] + avg2: 'D4',
staff_lines[4] + 2 * avg2: 'C4',
staff_lines[4] + 3 * avg2: 'B3',
}
notes = []
img2 = cv2.GaussianBlur(img, (29, 29), 0)
"""
util.show('blurred', img2)
"""
# sort by column (ascending) to add notes left-to-right
objs = sorted(objs, key=lambda x: x[0])
for c, r, w, h in objs:
circles = cv2.HoughCircles(img2[r:r + h, c:c + w],
cv2.cv.CV_HOUGH_GRADIENT,
1,
10,
param1=100,
param2=50)
if circles is not None and len(circles[0]) == 1:
x, y, radius = circles[0][0]
cv2.circle(temp, (int(c + x), int(r + y)), radius, util.RED, 2)
dist, pos = min([(abs(r + y - p), positions[p])
for p in positions.keys()])
notes.append((pos, 'quarter'))
# TODO determine note duration?
util.show('Notes', temp, True)
return notes
def _get_edge_attractiveness(self, edge, current_point):
end = edge.get_end_by_point(current_point)
if end.pheromone_level == 0:
#return self.DEFAULT_EDGE_ATTRACTIVENESS
return avg([edge_end.pheromone_level for edge_end in current_point.edge_ends])
return end.pheromone_level #/ edge.cost # if We divide attractiveness by cost, We are more likely to end up in local optimum
def find_notes(img, objs, staff_lines):
rows, cols = img.shape[:2]
temp = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
for c, r, w, h in objs:
cv2.rectangle(temp, (c, r), (c + w, r + h), 127, 1)
# from 5 staff line coordinates, build coordinates of nine positions
# (5 lines and four spaces between each line)
staff_lines = sorted(staff_lines)
avg = sum([abs(staff_lines[i] - staff_lines[i + 1]) \
for i in range(len(staff_lines) - 1)]) / (len(staff_lines) - 1)
avg2 = float(avg) / 2
positions = {
staff_lines[0] - avg2: 'G5',
staff_lines[0]: 'F5',
util.avg(staff_lines, 0, 1): 'E5',
staff_lines[1]: 'D5',
util.avg(staff_lines, 1, 2): 'C5',
staff_lines[2]: 'B4',
util.avg(staff_lines, 2, 3): 'A4',
staff_lines[3]: 'G4',
util.avg(staff_lines, 3, 4): 'F4',
staff_lines[4]: 'E4',
staff_lines[4] + avg2: 'D4',
staff_lines[4] + 2 * avg2: 'C4',
staff_lines[4] + 3 * avg2: 'B3',
}
notes = []
img2 = cv2.GaussianBlur(img, (29, 29), 0)
"""
util.show('blurred', img2)
"""
# sort by column (ascending) to add notes left-to-right
objs = sorted(objs, key=lambda x: x[0])
for c, r, w, h in objs:
circles = cv2.HoughCircles(
img2[r:r+h, c:c+w],
cv2.cv.CV_HOUGH_GRADIENT,
1,
10,
param1=100,
param2=50
)
if circles is not None and len(circles[0]) == 1:
x, y, radius = circles[0][0]
cv2.circle(temp, (int(c + x), int(r + y)), radius, util.RED, 2)
dist, pos = min([(abs(r + y - p), positions[p]) for p in positions.keys()])
notes.append((pos, 'quarter'))
# TODO determine note duration?
util.show('Notes', temp, True)
return notes
def calc_d(self, k):
shift(self.m1_k, k, self.m1)
return avg(self.m1_k)
def WriteBack(self, def_vals, data, final_out_file, elaborate):
if elaborate is None:
return
self.__sync.acquire()
data = [x for x in data if x is not None]
data = [x for x in data if len(x)]
# we remove the empty elements
file_name = sobstitute(elaborate.out_file_name.currValue(), self)
in_file_format = elaborate.in_file_format
res = {}
res["MIN"] = [0] * len(in_file_format)
res["MAX"] = [0] * len(in_file_format)
res["AVG"] = [0] * len(in_file_format)
res["STD"] = [0] * len(in_file_format)
if len(data):
data_ex = zip(*data)
for i in range(len(in_file_format)):
res["AVG"][i] = avg(data_ex[i])
res["MIN"][i] = min(data_ex[i])
res["MAX"][i] = max(data_ex[i])
for i in range(len(in_file_format)):
res["STD"][i] = std(data_ex[i], res["AVG"][i])
# print "MINS: " + str(res['MIN'])
# print "MAXS: " + str(res['MAX'])
# print "AVG : " + str(res['AVG'])
# print "STD : " + str(res['STD'])
vals = []
for param in elaborate.out_file_format:
if param in self.__parameters.keys():
vals.append(toStr(self.__parameters[param].currValue()))
elif (
param.startswith("AVG(")
or param.startswith("STD(")
or param.startswith("MIN(")
or param.startswith("MAX(")
) and param.endswith(")"):
param_name = param[4:-1]
assert param_name in in_file_format.values
vals.append(toStr(res[param[:3]][in_file_format.values.index(param_name)]))
self.__speedup = -1
idx = elaborate.speedup_idx.currValue()
if def_vals:
if def_vals["AVG"][idx] != 0 and res["AVG"][idx] != 0:
assert idx > 0 or idx < 0
if idx < 0:
idx = (-idx) - 1
self.__speedup = res["AVG"][idx] / def_vals["AVG"][idx]
else:
idx -= 1
self.__speedup = def_vals["AVG"][idx] / res["AVG"][idx]
vals.append(toStr(self.__speedup))
open(file_name, "a").write(",".join(vals) + "\n")
print "\n{0}".format(80 * "*")
if res["AVG"][idx] == 0:
print "* Configuration failed to execute! "
else:
print "* Average values: \n*\t%s" % ", ".join(map(lambda x: "{0:.3f}".format(x), res["AVG"]))
if self.__speedup != -1:
print "* Speedup: \n*\t{0:.5f}".format(self.__speedup)
print "{0}".format(80 * "*")
self.__sync.notify()
self.__sync.release()
return res