Пример #1
0
def func_avg(kwargs, f=None, name=None):
    kwargs = dict(kwargs)
    pool = kwargs.pop("pool")
    n_runs = kwargs.pop("n_runs")
    result = pool.map(FunctionCaller(kwargs), range(n_runs))
    xs, sizes = list(zip(*result))
    if f is None:
        f = open("tuning.txt", "w")
    xs_str = "Evals: " + str(xs)
    sizes_str = "Sizes: " + str(sizes)
    median_str = "Median: %.1f  Average: %.1f ~ %.1f" % (util.median(
        xs), util.avg(xs), util.avg([x for x in xs if util.is_finite(x)]))
    avg_size_str = "AverageSize: %.1f ~ %.1f" % (
        util.avg(sizes), util.avg([s for s in sizes if util.is_finite(s)]))
    success_rate = len([x for x in xs if util.is_finite(x)]) / len(xs)
    success_rate_str = "Success_rate: %.2f" % success_rate
    report_str = [
        xs_str, sizes_str, median_str, avg_size_str, success_rate_str
    ]
    if name is not None:
        name_str = "* %s" % name
        report_str = [name_str] + report_str
    report_str = "\n".join(report_str)
    print(report_str, file=f)
    f.flush()
    print(report_str)
Пример #2
0
	def __init__(self, method, n=1, timeout=120.0):
		from memory_profiler import memory_usage
		import time
		from util import avg

		self.timeout = timeout

		max_memory_list = []
		avg_memory_list = []
		time_list = []

		for i in range(n):

			runner = self._Runner(memory_usage, {'proc': method, 'interval': 0.01, 'retval': True}, timeout)
			baseline = avg(memory_usage((lambda: None, ())))  # baseline memory usage measurement

			start_time = time.time()
			runner.run()  # actual running of the method
			end_time = time.time()

			if runner.result is not None:
				mem_use, self.result = runner.result
				memory = map(lambda x: x - baseline, mem_use)
			else:  # there was a timeout
				import sys
				memory = [-sys.maxint]
				self.result = None

			max_memory_list.append(max(memory))
			avg_memory_list.append(avg(memory))
			time_list.append((end_time - start_time) * 1000)

		self.max_memory = avg(max_memory_list)
		self.avg_memory = avg(avg_memory_list)
		self.time = avg(time_list)
Пример #3
0
	def add_val(self, params, SCNs, val_key):

		# downside of this implementation is that don't know which characteristics are computed first
		# and have to recalc common things such as #steps and time
		# if too slow add a pre-feature calculation step

		################################## DISCRETE MODEL CHARACTERISTICS ##################################
		if val_key == 'steps':
			steps = [scn.steps for scn in SCNs]
			return all_stats(steps, params)

		elif val_key == 'probability off-diag':
			init = params['IA']/(params['IA']+params['IB'])
			As,Bs = [scn.mols['A'] for scn in SCNs], [scn.mols['B'] for scn in SCNs]
			pr = [As[i]/(As[i]+Bs[i]) > init for i in rng(As)]
			return all_stats(pr, params)

		elif val_key == 'Percent Y0':
			ratios = []
			for scn in SCNs:
				if scn.mols['Y0']==0: 
					ratios += [0]
				else:
					ratios += [scn.mols['Y0']/(scn.mols['Y0']+scn.mols['Y1'])]
			# faster if know that sum != 0:
			#ratios = [scn.mols['Y0']/(scn.mols['Y0']+scn.mols['Y1']) for scn in SCNs]
			return all_stats(ratios, params)

		elif val_key == 'dist from discrete bound':
			discrete_bound = bounds.discrete(params['gamma'], params['delta'],params['IA'],params['IB'])
			avg_steps = avg([scn.steps for scn in SCNs])
			return math.log(discrete_bound - avg_steps,10), None, None, None, None, None

		################################## CONTINUOUS MODEL CHARACTERISTICS ##################################
		elif val_key in ['time', 'Convergence Time']:
			stop_ts = [scn.time for scn in SCNs]
			return all_stats(stop_ts, params)

		elif val_key in ['time', 'Log Convergence Time']:
			stop_ts = [math.log(scn.time,10) for scn in SCNs]
			return all_stats(stop_ts, params)

		elif val_key == 'log time':
			stop_ts = [math.log(scn.time,10) for scn in SCNs]
			return all_stats(stop_ts, params)

		elif val_key == 'expected time': #appears to be the same as time, runs faster
			stop_ts = [scn.time_mean_appx for scn in SCNs]
			return all_stats(stop_ts, params)

		elif val_key == 'dist from continuous bound':
			cont_bound = bounds.continuous(params['gamma'], params['delta'],params['IA'],params['IB'])
			avg_t = avg([scn.time for scn in SCNs]) #note that curr using exact time, may go faster with expected time
			return math.log(cont_bound - avg_t,10), None, None, None, None, None



		else: 
			raise KeyError("Unrecognized value %s, use recognized value keys when initializing Dataset (see Dataset.add_val for existing options)." %(val_key))
Пример #4
0
    def back_test_tick(self, base_ticker, trade_ticker, symbol):
        diff_price = trade_ticker.price - base_ticker.price
        self.diffs.append(diff_price)
        self.base_prices.append(base_ticker.price)
        self.trade_prices.append(trade_ticker.price)

        if self.in_warming():
            return
        window = 10
        base_ma = avg(self.base_prices[-window:])
        trade_ma = avg(self.trade_prices[-window:])
        self.base_price_mas.append(base_ma)
        self.trade_price_mas.append(trade_ma)
        self.diff_mas.append(trade_ma - base_ma)

        if len(self.diff_mas) < 10:
            return

        if self.bt_status == BtStatus.PLACE_BUY_ORDER:
            if trade_ticker.price < self.bt_buy_price:
                print 'buy success %s, id: %s, ts: %s' % (
                    trade_ticker.price, self.bt_record.id, self.bt_record.ar)
                self.bt_status = BtStatus.SUCCESS_BUY_ORDER
                self.bt_tx_cnt += 1
                if self.bt_tx_cnt % 2 == 0:
                    self.bt_benefit += self.bt_sell_price - self.bt_buy_price
        elif self.bt_status == BtStatus.PLACE_SELL_ORDER:
            if trade_ticker.price > self.bt_sell_price:
                print 'sell success %s, id: %s, ts: %s' % (
                    trade_ticker.price, self.bt_record.id, self.bt_record.ar)
                self.bt_status = BtStatus.SUCCESS_SELL_ORDER
                self.bt_tx_cnt += 1
                if self.bt_tx_cnt % 2 == 0:
                    self.bt_benefit += self.bt_sell_price - self.bt_buy_price
        elif self.bt_status == BtStatus.PLACE_BUY_ORDER and self.bt_record.id > self.bt_buy_record_id + 5:
            self.bt_cancel_buy()
        else:
            can_buy = self.ana_buy_status()
            can_sell = self.ana_sell_status()
            add_price = -0.0001
            if can_buy:
                buy_price = trade_ticker.ask + add_price
                self.back_test_buy(buy_price, 0.001)
            if can_sell:
                sell_price = trade_ticker.bid - add_price
                if sell_price > self.bt_buy_price + 10:
                    self.back_test_sell(sell_price, 0.001)
Пример #5
0
    def create_genome_evaluation(genome,
                                 fitness,
                                 net=None,
                                 fitness_test=None,
                                 window=None,
                                 generation=None,
                                 initial_time=None,
                                 build_time=0,
                                 pred_time=0,
                                 pred_avg_time=0,
                                 fit_time=0,
                                 include_genome=False,
                                 extra={},
                                 **kwargs):
        if net is None:
            net = util.build_network(genome, **kwargs)
        global_time = datetime.datetime.now(
        ) - initial_time if initial_time is not None else None
        if isinstance(genome, list):
            genome_neurons = util.avg([g.NumHiddenNeurons() for g in genome])
            genome_connections = util.avg([g.NumLinks() for g in genome])
        else:
            genome_neurons = genome.NumHiddenNeurons()
            genome_connections = genome.NumLinks()
        if isinstance(net, list):
            neurons = util.avg([n.NumHiddenNeurons() for n in net])
            connections = util.avg([n.NumConnections() for n in net])
        else:
            neurons = net.NumHiddenNeurons()
            connections = net.NumConnections()

        return GenomeEvaluation(genome=genome if include_genome else None,
                                fitness=fitness,
                                fitness_test=fitness_test,
                                genome_neurons=genome_neurons,
                                genome_connections=genome_connections,
                                neurons=neurons,
                                connections=connections,
                                generation=generation,
                                window=window,
                                global_time=global_time,
                                build_time=build_time,
                                pred_time=pred_time,
                                pred_avg_time=pred_avg_time,
                                fit_time=fit_time,
                                **extra)
Пример #6
0
def reached_waypoint(num, gps_buffer, tolerance):
    (lat, lon) = WAYPOINTS[num - 1]
    distance = avg([
        dist_to_waypoint(msg, (lat + LAT_OFFSET, lon + LON_OFFSET))
        for msg in gps_buffer
    ])
    # state_debug.publish(str(distance))

    return distance < tolerance
Пример #7
0
def make_indices_by_MA_delta_old(conn, his_md, MA_Size1=5):

    recent_mds = {}  # 代码==> 该代码最后几交易日的‘行情’  ** 跳过停牌日
    # 其中‘行情’ 是  [
    #                    [交易日,收盘价],
    #                    [交易日,收盘价], ...
    #                ]
    md_prev_day = None

    for md_that_day in his_md:

        t_day = md_that_day[0]
        md_set = md_that_day[1]

        indices = collections.OrderedDict()

        for code, md_set in md_that_day[1].iteritems():

            if md_prev_day is None or code not in md_prev_day[
                    1] or code not in recent_mds:
                # 第一天              昨日行情里没有本code            ‘最近交易日’记录里没有本code

                # 需要从外部获取本code最后N日记录
                recent_memo = data_fetcher.get_his_until(code, t_day, MA_Size1)
                recent_mds[code] = recent_memo
            else:
                # 停牌的行情不加入 recent_mds
                if not md_set[4]:
                    recent_mds[code].append([t_day, md_set[0]])

            if len(recent_mds[code]) > MA_Size1:
                del recent_mds[code][0]

            MA1 = util.avg(recent_mds[code])
            # 至此MA有了,准备其他指标

            if md_set[3] or md_set[4]:
                # 涨停或者停牌的不能买
                can_buy = 0

                deviation = 0  # 没必要再算偏离
            else:
                can_buy = 1
                if 0 == MA1:
                    deviation = 0
                else:
                    deviation = (md_set[0] - MA1) / MA1

            indices[code] = [can_buy, deviation, MA1]

        md_that_day.append(indices)

        # 准备走向下一天
        md_prev_day = md_that_day

    return his_md
Пример #8
0
def xor_investigate_average():
    n_runs = 100
    x = []
    for i in range(n_runs):
        result = xor_investigate(i)
        print("xor(%d) = %d" % (i, result))
        x.append(result)
    #x = [xor_investigate() for i in range(n_runs)]
    print("Average=%.2f" % util.avg(x))
    print("Median=%.2f" % util.median(x))
Пример #9
0
def extract_stats(data, feature_names, sshot):
    for name in feature_names:
        feat = extract_one(name, sshot)
        if feat != []:

            data[name]['avg'] += [util.avg(feat)]
            data[name]['var'] += [util.var(feat)]
            if util.avg(feat) != 0:
                iod = util.var(feat) / util.avg(feat)
            else:
                iod = 0
            data[name]['iod'] += [iod]
            max1 = max(feat)
            feat.remove(max1)
            max2 = max(feat)
            data[name]['max'] += [max1]

            if max1 == 0:
                ratio = 0
            else:
                ratio = (max1 - max2) / max1
            data[name]['1:2'] += [ratio]
Пример #10
0
    def present(self):
        # averages are usually heavily biased by the few initial food findings
        #avg_cost = (self.reality.world.elapsed_time*self.number_of_ants)/self.food_discovered
        #avg_moves = self.moves_leading_to_food_being_found/self.food_discovered

        fl_avg_cost = avg([cost for moves, cost in self.last_results])
        fl_avg_moves = avg([moves for moves, cost in self.last_results])
        avg_pheromone = self.reality.world.get_average_pheromone_level()
        max_pheromone = self.reality.world.get_max_pheromone_level()
        pheromone_ratio = max_pheromone/(avg_pheromone or 1)
        self.stats_saver.add_sample(
            {
                'food_discovered': self.food_discovered,
                'time': self.reality.world.elapsed_time,
                'avg_pheromone': '%.12f' % avg_pheromone,
                'max_pheromone': '%.12f' % max_pheromone,
                'last_cost': self.last_cost,
                'pheromone_ratio': pheromone_ratio,
                'best_finding_cost': self.best_finding_cost,
                'fl_avg_cost': fl_avg_cost,
                'fl_avg_moves': fl_avg_moves,
            }
        )
Пример #11
0
 def get_right(self, exchange, coin, li, side='bid', amount=None):
     total = 0
     res = []
     if amount is None:
         amount = AMOUNT_THRESHOLD[coin]
     if side == 'ask':
         li.reverse()
     for x in li:
         total += x[1]
         if total >= amount:
             res.append(x[0])
             break
         else:
             res.append(x[0])
     return avg(res)
Пример #12
0
def extend_indices_add_ma(conn, his_md, MA_Size1=5):

    recent_mds = {}  # 代码==> 该代码最后几交易日的‘行情’  ** 跳过停牌日
    # 其中‘行情’ 是  [
    #                    [交易日,收盘价],
    #                    [交易日,收盘价], ...
    #                ]
    md_prev_day = None

    for md_that_day in his_md:

        t_day = md_that_day[0]
        mds = md_that_day[1]
        indices = md_that_day[2]

        for code, md_of_the_code in mds.iteritems():

            indi_of_the_code = indices[code]

            if md_prev_day is None or code not in md_prev_day[
                    1] or code not in recent_mds:
                # 第一天              昨日行情里没有本code            ‘最近交易日’记录里没有本code

                # 需要从外部获取本code最后N日记录
                recent_memo = data_fetcher.get_his_until(code, t_day, MA_Size1)
                recent_mds[code] = recent_memo
            else:
                # 停牌的行情不加入 recent_mds
                if not md_of_the_code[4]:
                    recent_mds[code].append([t_day, md_of_the_code[0]])

            if len(recent_mds[code]) > MA_Size1:
                del recent_mds[code][0]

            MA1 = util.avg(recent_mds[code])
            # 至此MA有了

            indi_of_the_code.append(MA1)

        # 准备走向下一天
        md_prev_day = md_that_day

    return his_md
Пример #13
0
    def draw(self, origin, size, step, radius=-1):
        """
        Draw field vectors periodically within a region.

        Drawing begins at the origin and continues by the given step up to
        and including the size. Both the size and step may be vectors or
        scalars. Scalars are equivalent to vectors with equal dimensions.
        """

        # get the origin coordinates
        x0, y0, z0 = origin

        try:
            l, h, w = size
        except TypeError:
            l = w = h = size

        try:
            step_x, step_y, step_z = step
        except TypeError:
            step_x = step_y = step_z = step

        xs = range(int(x0), int(x0 + l) + 1, int(step_x))
        ys = range(int(y0), int(y0 + h) + 1, int(step_y))
        zs = range(int(z0), int(z0 + w) + 1, int(step_z))

        #getting values for color interpolation
        for x in xs:
            for y in ys:
                for z in zs:
                    self[vector(x, y, z)] = self(vector(x, y, z))
        self.avg_mag = avg(B.mag for B in self.Ps.values())
        self.largest_mag = max(B.mag for B in self.Ps.values())
        #smallest mag is estimate due to absolute smallest being zero, not useful
        self.smallest_mag = self.largest_mag - 2 * (self.largest_mag -
                                                    self.avg_mag)

        for x in xs:
            for y in ys:
                for z in zs:
                    if (radius == -1 or (x**2 + y**2 + z**2)**0.5 <= radius):
                        self.draw_at(vector(x, y, z), step)
Пример #14
0
    def draw(self, origin, size, step, radius = -1):
        """
        Draw field vectors periodically within a region.

        Drawing begins at the origin and continues by the given step up to
        and including the size. Both the size and step may be vectors or
        scalars. Scalars are equivalent to vectors with equal dimensions.
        """

        # get the origin coordinates
        x0, y0, z0 = origin

        try:
            l, h, w = size
        except TypeError:
            l = w = h = size

        try:
            step_x, step_y, step_z = step
        except TypeError:
            step_x = step_y = step_z = step

        xs = range(int(x0), int(x0+l)+1, int(step_x))
        ys = range(int(y0), int(y0+h)+1, int(step_y))
        zs = range(int(z0), int(z0+w)+1, int(step_z))

        #getting values for color interpolation
        for x in xs:
            for y in ys:
                for z in zs:
                    self[vector(x, y, z)] = self(vector(x, y, z))
        self.avg_mag = avg(B.mag for B in self.Ps.values())
        self.largest_mag = max(B.mag for B in self.Ps.values())
        #smallest mag is estimate due to absolute smallest being zero, not useful
        self.smallest_mag = self.largest_mag- 2*(self.largest_mag - self.avg_mag)

        for x in xs:
            for y in ys:
                for z in zs:
                    if (radius == -1 or (x**2 + y**2 + z**2)**0.5 <= radius):
                        self.draw_at(vector(x, y, z), step)
Пример #15
0
def going_flat(gps_buffer, tolerance):
    angle = avg([current_angle(loc) for loc in gps_buffer])
    return (angle < tolerance and angle > tolerance * -1)
Пример #16
0
def going_down(gps_buffer, tolerance):
    angle = avg([current_angle(loc) for loc in gps_buffer])
    return angle < tolerance * -1
Пример #17
0
def going_up(gps_buffer, tolerance):
    angle = avg([current_angle(loc) for loc in gps_buffer])
    return angle > tolerance
Пример #18
0
 def get_rotated_x(self, angle):
     rotated_points = [
         rotate_point(-angle, price.top_right) for price in self.prices
     ]
     return avg([point[0] for point in rotated_points])
 def getGroundSensors(self):
     ambient = self.network.GetVariable("thymio-II", "prox.ground.ambiant")
     reflected = self.network.GetVariable("thymio-II", "prox.ground.reflected")
     delta = self.network.GetVariable("thymio-II", "prox.ground.delta")
     return (avg(ambient), avg(reflected), avg(delta))
Пример #20
0
 def get_average_pheromone_level(self):
     return avg(edge.pheromone_level() for edge in self.edges)
Пример #21
0
def drawG_by_rank(G,params, topK):
	# color by seed/not_seed, alpha by rank

	assert(os.path.isdir(params['output_path']))


	plt.figure(figsize=(10,6))
	node_size, nalpha, ealpha = 800, .3, .3 	#node_alpha not curr used
	
	# positions for all nodes
	if params['draw_layout'] == 'circular':
		pos = nx.circular_layout(G)  	
	elif params['draw_layout'] == 'spring':
		pos = nx.spring_layout(G) 
	else: assert(False) #unknown draw layout

	nodes = list(G.nodes())
	colors, alphas = [], []

	max_rank = max(G.nodes[nodes[i]]['rank'] for i in rng(nodes))

	for i in rng(nodes):
		if G.nodes[nodes[i]]['seed']: 
			G.nodes[nodes[i]]['alpha'] = 1
			alphas += [1]
			colors+=['green']
		else: 
			colors+=['blue']
			#G.nodes[nodes[i]]['alpha'] = G.nodes[nodes[i]]['rank']/max_rank #for the edges
			G.nodes[nodes[i]]['alpha'] = G.nodes[nodes[i]]['cardinal_rank'] #for the edges
			alphas += [G.nodes[nodes[i]]['cardinal_rank']]#/max_rank] #may not work..also gotta add LOG SCALING

	nx.draw_networkx_nodes(G, pos, nodelist=nodes, node_color=colors, node_size=node_size, alpha=alphas)
	#labels = {n: G.nodes[n]['gene'] for n in G.nodes()}

	topK2 = topK.tolist()
	top = topK2 + params['seeds']
	labels = {n:n for n in top} #huh?
	nx.draw_networkx_labels(G, pos, labels, font_size=8, font_color='blue')

	#edge_alphas = []
	ecolors = []
	elist = sorted(list(G.edges()))
	for i in rng(elist):
		n1,n2 = elist[i][0],elist[i][1]
		#edge_alphas += [min(G.nodes[n1]['alpha'],G.nodes[n2]['alpha'])]
		ecolors += [(0,0,0,avg([G.nodes[n1]['alpha'],G.nodes[n2]['alpha']]))]


	nx.draw_networkx_edges(G, pos, arrows=True, edgelist=elist, edge_color=ecolors) #,alpha=edge_alphas)


	if params['save_fig']:
		now = datetime.now()
		curr_date = str(date.today()).strip('2020-')
		curr_time = str(datetime.now().strftime("%H-%M-%S"))
		plt.savefig(params['output_path']+'/'+curr_date+'_'+curr_time+'_rankedG.png')
	else:
		plt.show()	
	plt.clf()
	plt.close()
Пример #22
0
    def get(self):
        symbol = self.get_argument('symbol', 'BTC_USDT')
        limit = int(self.get_argument('limit', '2000'))
        table = self.get_argument('table', 'okex_binance')
        delta = int(self.get_argument('delta', '0'))
        begin = int(self.get_argument('begin', '0'))
        end = int(self.get_argument('end', '0'))

        table = "abs_diff_" + table
        if begin > 0 and end > 0:
            sql = "select * from " + table + " where symbol=? and ts between ? and ?"
            data = self.application.engine.fetch_row(sql, (symbol, begin, end))
        else:
            sql = "select * from (select * from " + table + " where symbol = ? order by id desc limit ?) sub order by id asc"
            data = self.application.engine.fetch_row(sql, (symbol, limit))

        option = {
            "animation": False,
            "title": {
                "text": "diff"
            },
            "xAxis": {
                "type": 'value',
                "min": "dataMin",
                "max": "dataMax",
            },
            "yAxis": {
                "type": 'value',
                "min": "dataMin",
                "max": "dataMax",
            },
            "boundaryGap": [0, '100%'],
            "legend": {
                "data": ["diff_bid", "diff_ask", "diff_price"],
            }
        }
        series = []
        # series.append({
        #     'name': 'diff_bid',
        #     'type': 'line',
        #     'data': [[x.ts, x.trade_bid - x.base_bid] for x in data]
        # })
        # series.append({
        #     'name': 'diff_ask',
        #     'type': 'line',
        #     'data': [[x.ts, x.trade_ask - x.base_ask] for x in data]
        # })
        window = 10
        base_data = []
        dt = [float(x.base_price) for x in data]
        for i in range(len(dt)):
            if i < window:
                v = avg(dt[:i + 1])
            else:
                v = avg(dt[i - window:i + 1])
            if v > 0:
                base_data.append([data[i].id, v])
                data[i].base_ma = v

        t_data = []
        dt = [float(x.trade_price) for x in data]
        for i in range(len(dt)):
            if i < window:
                v = avg(dt[:i + 1])
            else:
                v = avg(dt[i - window:i + 1])
            if v > 0:
                t_data.append([data[i].id, v])
                data[i].trade_ma = v
        if delta > 0:
            series.append({
                'name':
                'diff',
                'type':
                'line',
                'data': [[x.id, x.trade_ma - x.base_ma] for x in data],
            })
        else:

            series.append({'name': 'base', 'type': 'line', 'data': base_data})
            series.append({'name': 'price', 'type': 'line', 'data': t_data})
            series.append({
                'name': 'bid',
                'type': 'line',
                'data': [[x.id, float(x.trade_bid)] for x in data]
            })
        option['series'] = series

        self.render("abs_diff.html", option=json.dumps(option))
Пример #23
0
    def get(self):
        symbol = self.get_argument('symbol', 'BTC_USDT')
        limit = int(self.get_argument('limit', '2000'))
        table = self.get_argument('table', 'okex_binance')
        delta = int(self.get_argument('delta', '0'))
        begin = int(self.get_argument('begin', '0'))
        end = int(self.get_argument('end', '0'))

        table = "abs_diff_" + table
        if begin > 0 and end > 0:
            sql = "select * from " + table + " where symbol=? and ts between ? and ?"
            data = self.application.engine.fetch_row(sql, (symbol, begin, end))
        else:
            sql = "select * from (select * from " + table + " where symbol = ? order by id desc limit ?) sub order by id asc"
            data = self.application.engine.fetch_row(sql, (symbol, limit))

        option = {
            "animation": False,
            "title": {"text": "diff"},
            "xAxis": {
                "type": 'value',
                "min": "dataMin",
                "max": "dataMax",
            },
            "yAxis": {
                "type": 'value',
                "min": "dataMin",
                "max": "dataMax",
            },
            "boundaryGap": [0, '100%'],
            "legend": {
                "data": ["diff_bid", "diff_ask", "diff_price"],
            }
        }
        series = []
        # series.append({
        #     'name': 'diff_bid',
        #     'type': 'line',
        #     'data': [[x.ts, x.trade_bid - x.base_bid] for x in data]
        # })
        # series.append({
        #     'name': 'diff_ask',
        #     'type': 'line',
        #     'data': [[x.ts, x.trade_ask - x.base_ask] for x in data]
        # })
        window = 10
        base_data = []
        dt = [float(x.base_price) for x in data]
        for i in range(len(dt)):
            if i < window:
                v = avg(dt[:i+1])
            else:
                v = avg(dt[i-window:i+1])
            if v > 0:
                base_data.append([data[i].id, v])
                data[i].base_ma = v
                
        t_data = []
        dt = [float(x.trade_price) for x in data]
        for i in range(len(dt)):
            if i < window:
                v = avg(dt[:i+1])
            else:
                v = avg(dt[i-window:i+1])
            if v > 0:
                t_data.append([data[i].id, v])
                data[i].trade_ma = v
        if delta > 0:
            series.append({
                'name': 'diff',
                'type': 'line',
                'data': [[x.id, x.trade_ma - x.base_ma] for x in data],
            })
        else:
                
            series.append({
                'name': 'base',
                'type': 'line',
                'data': base_data
            })
            series.append({
                'name': 'price',
                'type': 'line',
                'data': t_data
            })
            series.append({
                'name': 'bid',
                'type': 'line',
                'data': [[x.id, float(x.trade_bid)] for x in data]
            })
        option['series'] = series

        self.render("abs_diff.html", option=json.dumps(option))
Пример #24
0
    sys.exit(0)

data_dir = sys.argv[1]
output_dir = sys.argv[2]

# Validate
eval_preds = None
eval_labels = None
for subjid in range(1, 4):
    subj = str(subjid)
    path = os.path.join(output_dir, 'eval.' + subj + '.npy')
    preds = np.load(path)
    eval_filename = 'eval-' + subj + '-' + str(0) + '-index.csv'
    idx_file = os.path.join(data_dir, 'train_' + subj, eval_filename)
    labels = np.loadtxt(idx_file, delimiter=',', skiprows=1, usecols=[1])
    labels, preds = avg(labels, preds)
    calibrate(subjid, preds)
    print('Eval AUC for subject %d %.4f\n' % (subjid, auc(labels, preds)))
    eval_preds = preds if subjid == 1 else np.hstack((eval_preds, preds))
    eval_labels = labels if subjid == 1 else np.hstack((eval_labels, labels))
normalize(eval_preds)

print('Overall AUC %.4f\n' % auc(eval_labels, eval_preds))

# Test
preds = None
for subjid in range(1, 4):
    path = os.path.join(output_dir, 'test.' + str(subjid) + '.npy')
    vals = np.load(path)
    vals = avg_preds(vals)
    calibrate(subjid, vals)
Пример #25
0
 def calc_k(self, rsv):
     shift(self.m_rsv, rsv, self.m)
     return avg(self.m_rsv)
Пример #26
0
    pl.plot(wl_new, medfilt(n, 5))
    pl.xlabel(r'Rest wavelength [$\mathrm{\AA}$]', fontsize=25)
    pl.ylabel(r'Normalised flux density', fontsize=25)
    pl.savefig("../figures/n_bursts.pdf")
    pl.clf()
    ma_e_arr = np.ma.array(e, mask=bpmap.astype("bool"))
    median_err = np.sqrt((np.pi / 2) * np.ma.sum(ma_e_arr**2, axis=0) / n**2)

    pl.plot(wl_new, median)
    pl.plot(wl_new, median_err)
    pl.show()

    std = medfilt(np.ma.std(ma_arr, axis=0), 11)
    wmean, wmeanerr, _ = util.avg(f,
                                  e,
                                  mask=bpmap.astype("bool"),
                                  axis=0,
                                  weight=True)

    # Saving to .dat file
    dt = [("wl", np.float64), ("median", np.float64),
          ("median_err", np.float64), ("wmean", np.float64),
          ("wmeanerr", np.float64)]
    data = np.array(zip(wl_new, median, median_err, wmean, wmeanerr), dtype=dt)
    np.savetxt(
        "../gold_comp.dat",
        data,
        header=
        "wavelength   median median_error   weighted_mean weighted_mean_err",
        fmt=['%1.5e', '%1.5e', '%1.5e', '%1.5e', '%1.5e'],
        delimiter="\t")
Пример #27
0
    def WriteBack(self, def_vals, data, final_out_file, elaborate):

        if elaborate is None:
            return

        self.__sync.acquire()
        data = [x for x in data if x is not None]
        data = [x for x in data if len(x)]
        # we remove the empty elements

        file_name = sobstitute(elaborate.out_file_name.currValue(), self)
        in_file_format = elaborate.in_file_format

        res = {}
        res['MIN'] = [0] * len(in_file_format)
        res['MAX'] = [0] * len(in_file_format)
        res['AVG'] = [0] * len(in_file_format)
        res['STD'] = [0] * len(in_file_format)

        if len(data):
            data_ex = zip(*data)
            for i in range(len(in_file_format)):
                res['AVG'][i] = avg(data_ex[i])
                res['MIN'][i] = min(data_ex[i])
                res['MAX'][i] = max(data_ex[i])

            for i in range(len(in_file_format)):
                res['STD'][i] = std(data_ex[i], res['AVG'][i])

            # print "MINS: " + str(res['MIN'])
            # print "MAXS: " + str(res['MAX'])
            # print "AVG : " + str(res['AVG'])
            # print "STD : " + str(res['STD'])

        vals = []
        for param in elaborate.out_file_format:
            if param in self.__parameters.keys():
                vals.append(toStr(self.__parameters[param].currValue()))
            elif (param.startswith('AVG(') or param.startswith('STD(')
                  or param.startswith('MIN(')
                  or param.startswith('MAX(')) and param.endswith(')'):
                param_name = param[4:-1]
                assert param_name in in_file_format.values
                vals.append(
                    toStr(res[param[:3]][in_file_format.values.index(
                        param_name)]))

        self.__speedup = -1
        idx = elaborate.speedup_idx.currValue()
        if def_vals:
            if def_vals['AVG'][idx] != 0 and res['AVG'][idx] != 0:
                assert (idx > 0 or idx < 0)
                if idx < 0:
                    idx = (-idx) - 1
                    self.__speedup = res['AVG'][idx] / def_vals['AVG'][idx]
                else:
                    idx -= 1
                    self.__speedup = def_vals['AVG'][idx] / res['AVG'][idx]

            vals.append(toStr(self.__speedup))

        open(file_name, 'a').write(",".join(vals) + "\n")

        print "\n{0}".format(80 * "*")
        if res['AVG'][idx] == 0:
            print "* Configuration failed to execute! "
        else:
            print "* Average values: \n*\t%s" % ", ".join(
                map(lambda x: "{0:.3f}".format(x), res['AVG']))
            if self.__speedup != -1:
                print "* Speedup: \n*\t{0:.5f}".format(self.__speedup)
        print "{0}".format(80 * "*")

        self.__sync.notify()
        self.__sync.release()
        return res
Пример #28
0
def extend_indices_add_pb_std(conn, his_md, MA_Size1=5):

    #MA_Size2 = 60
    #MA_Size3 = 20

    recent_mds = {}  # 代码==> 该代码最后几交易日的‘行情’  ** 跳过停牌日
    # 其中‘行情’ 是  [
    #                    [交易日,PB],
    #                    [交易日,PB], ...
    #                ]
    md_prev_day = None

    for md_that_day in his_md:

        t_day = md_that_day[0]
        mds = md_that_day[1]
        indices = md_that_day[2]

        for code, md_of_the_code in mds.iteritems():

            PB = md_of_the_code[5]
            indi_of_the_code = indices[code]

            if md_prev_day is None or code not in md_prev_day[
                    1] or code not in recent_mds:
                # 第一天              昨日行情里没有本code            ‘最近交易日’记录里没有本code

                recent_memo = [[t_day, PB]]
                recent_mds[code] = recent_memo
            else:
                # 停牌的行情不加入 recent_mds
                if not md_of_the_code[4]:
                    recent_mds[code].append([t_day, PB])

            if len(recent_mds[code]) > MA_Size1:
                del recent_mds[code][0]

            #对于PB策略,样本不足则 平均值和标准差都意义不足,所以
            if len(recent_mds[code]) < MA_Size1:
                MA1 = 0
                std = 0
                bias = 0
                #MA2=0
                #MA3=0
            else:
                #print recent_mds[code]
                MA1 = util.avg(recent_mds[code])
                std = numpy.std(util.column_of_a2d(recent_mds[code], 1))
                bias = (PB - MA1) / std

                #MA2 = util.avg( recent_mds[code][ - MA_Size2 :])
                #MA3 = util.avg( recent_mds[code][ - MA_Size3 :])

            indi_of_the_code.append(bias)
            indi_of_the_code.append(PB)
            indi_of_the_code.append(MA1)
            indi_of_the_code.append(std)
            #indi_of_the_code.append(MA2)
            #indi_of_the_code.append(MA3)

        # 准备走向下一天
        md_prev_day = md_that_day

    return his_md
Пример #29
0
    #di['total_real_time'] = results['total_real_time']
    stats_path = os.path.join(root, stats_filename)

    # load data
    stats_data = []
    for row in itertools.islice(csv.reader(open(stats_path, 'rb')), 1, None):
        #print row[0], row[3]
        stats_data.append(row[3])

    # precompute
    group_size = 500
    aggregates = []
    while stats_data:
        aggregates.append(
            avg(
                [float(stats_data.pop()) for i in xrange(group_size)]
            )
        )
    aggregates.reverse()

    if aggregates[3] / 2 < aggregates[-1]:
        continue
    print root
    pprint(aggregates)
    #break
    raw_input()

raise Exception()

columns = sorted(queens)
writer = csv.writer(open('matrix.csv', 'wb'))
Пример #30
0
def find_notes(img, objs, staff_lines):
    rows, cols = img.shape[:2]

    temp = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
    for c, r, w, h in objs:
        cv2.rectangle(temp, (c, r), (c + w, r + h), 127, 1)

    # from 5 staff line coordinates, build coordinates of nine positions
    # (5 lines and four spaces between each line)

    staff_lines = sorted(staff_lines)

    avg = sum([abs(staff_lines[i] - staff_lines[i + 1]) \
              for i in range(len(staff_lines) - 1)]) / (len(staff_lines) - 1)
    avg2 = float(avg) / 2

    positions = {
        staff_lines[0] - avg2: 'G5',
        staff_lines[0]: 'F5',
        util.avg(staff_lines, 0, 1): 'E5',
        staff_lines[1]: 'D5',
        util.avg(staff_lines, 1, 2): 'C5',
        staff_lines[2]: 'B4',
        util.avg(staff_lines, 2, 3): 'A4',
        staff_lines[3]: 'G4',
        util.avg(staff_lines, 3, 4): 'F4',
        staff_lines[4]: 'E4',
        staff_lines[4] + avg2: 'D4',
        staff_lines[4] + 2 * avg2: 'C4',
        staff_lines[4] + 3 * avg2: 'B3',
    }

    notes = []
    img2 = cv2.GaussianBlur(img, (29, 29), 0)
    """
    util.show('blurred', img2)
    """

    # sort by column (ascending) to add notes left-to-right
    objs = sorted(objs, key=lambda x: x[0])

    for c, r, w, h in objs:
        circles = cv2.HoughCircles(img2[r:r + h, c:c + w],
                                   cv2.cv.CV_HOUGH_GRADIENT,
                                   1,
                                   10,
                                   param1=100,
                                   param2=50)

        if circles is not None and len(circles[0]) == 1:
            x, y, radius = circles[0][0]

            cv2.circle(temp, (int(c + x), int(r + y)), radius, util.RED, 2)

            dist, pos = min([(abs(r + y - p), positions[p])
                             for p in positions.keys()])
            notes.append((pos, 'quarter'))

            # TODO determine note duration?

    util.show('Notes', temp, True)

    return notes
Пример #31
0
 def _get_edge_attractiveness(self, edge, current_point):
     end = edge.get_end_by_point(current_point)
     if end.pheromone_level == 0:
         #return self.DEFAULT_EDGE_ATTRACTIVENESS
         return avg([edge_end.pheromone_level for edge_end in current_point.edge_ends])
     return end.pheromone_level #/ edge.cost # if We divide attractiveness by cost, We are more likely to end up in local optimum
Пример #32
0
def find_notes(img, objs, staff_lines):
    rows, cols = img.shape[:2]

    temp = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
    for c, r, w, h in objs:
        cv2.rectangle(temp, (c, r), (c + w, r + h), 127, 1)

    # from 5 staff line coordinates, build coordinates of nine positions
    # (5 lines and four spaces between each line)

    staff_lines = sorted(staff_lines)

    avg = sum([abs(staff_lines[i] - staff_lines[i + 1]) \
              for i in range(len(staff_lines) - 1)]) / (len(staff_lines) - 1)
    avg2 = float(avg) / 2

    positions = {
        staff_lines[0] - avg2:       'G5',
        staff_lines[0]:              'F5',
        util.avg(staff_lines, 0, 1): 'E5',
        staff_lines[1]:              'D5',
        util.avg(staff_lines, 1, 2): 'C5',
        staff_lines[2]:              'B4',
        util.avg(staff_lines, 2, 3): 'A4',
        staff_lines[3]:              'G4',
        util.avg(staff_lines, 3, 4): 'F4',
        staff_lines[4]:              'E4',
        staff_lines[4] + avg2:       'D4',
        staff_lines[4] + 2 * avg2:   'C4',
        staff_lines[4] + 3 * avg2:   'B3',
    }

    notes = []
    img2 = cv2.GaussianBlur(img, (29, 29), 0)

    """
    util.show('blurred', img2)
    """

    # sort by column (ascending) to add notes left-to-right
    objs = sorted(objs, key=lambda x: x[0])

    for c, r, w, h in objs:
        circles = cv2.HoughCircles(
                img2[r:r+h, c:c+w],
                cv2.cv.CV_HOUGH_GRADIENT,
                1,
                10,
                param1=100,
                param2=50
        )

        if circles is not None and len(circles[0]) == 1:
            x, y, radius = circles[0][0]

            cv2.circle(temp, (int(c + x), int(r + y)), radius, util.RED, 2)

            dist, pos = min([(abs(r + y - p), positions[p]) for p in positions.keys()])
            notes.append((pos, 'quarter'))

            # TODO determine note duration?

    util.show('Notes', temp, True)

    return notes
Пример #33
0
 def calc_d(self, k):
     shift(self.m1_k, k, self.m1)
     return avg(self.m1_k)
Пример #34
0
    def WriteBack(self, def_vals, data, final_out_file, elaborate):

        if elaborate is None:
            return

        self.__sync.acquire()
        data = [x for x in data if x is not None]
        data = [x for x in data if len(x)]
        # we remove the empty elements

        file_name = sobstitute(elaborate.out_file_name.currValue(), self)
        in_file_format = elaborate.in_file_format

        res = {}
        res["MIN"] = [0] * len(in_file_format)
        res["MAX"] = [0] * len(in_file_format)
        res["AVG"] = [0] * len(in_file_format)
        res["STD"] = [0] * len(in_file_format)

        if len(data):
            data_ex = zip(*data)
            for i in range(len(in_file_format)):
                res["AVG"][i] = avg(data_ex[i])
                res["MIN"][i] = min(data_ex[i])
                res["MAX"][i] = max(data_ex[i])

            for i in range(len(in_file_format)):
                res["STD"][i] = std(data_ex[i], res["AVG"][i])

            # print "MINS: " + str(res['MIN'])
            # print "MAXS: " + str(res['MAX'])
            # print "AVG : " + str(res['AVG'])
            # print "STD : " + str(res['STD'])

        vals = []
        for param in elaborate.out_file_format:
            if param in self.__parameters.keys():
                vals.append(toStr(self.__parameters[param].currValue()))
            elif (
                param.startswith("AVG(")
                or param.startswith("STD(")
                or param.startswith("MIN(")
                or param.startswith("MAX(")
            ) and param.endswith(")"):
                param_name = param[4:-1]
                assert param_name in in_file_format.values
                vals.append(toStr(res[param[:3]][in_file_format.values.index(param_name)]))

        self.__speedup = -1
        idx = elaborate.speedup_idx.currValue()
        if def_vals:
            if def_vals["AVG"][idx] != 0 and res["AVG"][idx] != 0:
                assert idx > 0 or idx < 0
                if idx < 0:
                    idx = (-idx) - 1
                    self.__speedup = res["AVG"][idx] / def_vals["AVG"][idx]
                else:
                    idx -= 1
                    self.__speedup = def_vals["AVG"][idx] / res["AVG"][idx]

            vals.append(toStr(self.__speedup))

        open(file_name, "a").write(",".join(vals) + "\n")

        print "\n{0}".format(80 * "*")
        if res["AVG"][idx] == 0:
            print "* Configuration failed to execute! "
        else:
            print "* Average values: \n*\t%s" % ", ".join(map(lambda x: "{0:.3f}".format(x), res["AVG"]))
            if self.__speedup != -1:
                print "* Speedup: \n*\t{0:.5f}".format(self.__speedup)
        print "{0}".format(80 * "*")

        self.__sync.notify()
        self.__sync.release()
        return res