def __init__(self, f_price):
'''
A representation of a PriceLevel object
'''
self.f_price = f_price
self.i_qty = 0
self.order_tree = FastRBTree()
def __init__(self):
self.price_tree = FastRBTree()
self.volume = 0
self.price_map = {} # Map from price -> order_list object
self.order_map = {} # Order ID to Order object
self.min_price = None
self.max_price = None
def __init__(self):
self.ptree = FastRBTree()
self.vol = 0
self.prmp = {}
self.order_map = {}
self.mip = None
self.mxp = None
def __init__(self, SourceFile):
self.Table = [
] #declare table as a stack (list) containing an empty tree
self.TopScope = FastRBTree(
) # a place where the current top scope is held
self.ReadMode = False #Read or lookup mode
self.DebugMode = False
self.SourceFile = SourceFile
def __init__(self, product_ids='ETH-USD', api_key=None, api_secret=None,
passphrase=None, use_heartbeat=False,
trade_log_file_path=None):
super().__init__(product_ids=product_ids,
api_key=api_key,
api_secret=api_secret,
passphrase=passphrase,
use_heartbeat=use_heartbeat,
trade_log_file_path=trade_log_file_path)
if not isinstance(product_ids, list):
product_ids = [product_ids]
self.traders = {product_id: gdax.trader.Trader(product_id=product_id)
for product_id in product_ids}
self._asks = {product_id: FastRBTree() for product_id in product_ids}
self._bids = {product_id: FastRBTree() for product_id in product_ids}
self._sequences = {product_id: None for product_id in product_ids}
def __init__(self,
cache_entries_limit,
ghost_entries_limit,
trace_size_limit,
csv_suffix="_mmc.csv"):
self.full_cache = FastRBTree()
self.was_hit = None
self.was_ghost_hit = None
self.num_hits = 0
self.num_requests = 0
self.cache_entries_limit = cache_entries_limit
self.ghost_entries_limit = ghost_entries_limit
self.trace_size_limit = trace_size_limit
self.trace = collections.deque()
self.stack = RBTree()
self.ranker = RBTree()
self.generation = 0
# During startup, this will act like an LRU.
self.startup = True
self.EM_period = 50 * int(np.ceil(np.log(trace_size_limit)))
self.countdown_to_EM = trace_size_limit // 2
self.tau = [0.25, 0.25, 0.25, 0.25]
self.theta = [0.5, 0.5, 0.5, 0.5]
self.acc_tau = [0.0, 0.0, 0.0, 0.0]
self.acc_theta = [0.0, 0.0, 0.0, 0.0]
self.num_in_cache = 0
self.num_in_full_cache = 0
self.num_reads = 0
self.csv_suffix = csv_suffix
self.ts_order = [
'row', 'hit', 'ghost_hit', 'tau_R_SDD', 'tau_R_IRM', 'tau_W_SDD',
'tau_W_IRM', 'theta_R_SDD', 'theta_R_IRM', 'theta_W_SDD',
'theta_W_IRM', 'depth', 'rank', 'Z_R_SDD', 'Z_R_IRM', 'Z_W_SDD',
'Z_W_IRM', 'Z_sum'
]
self.ts_datapoint = {key: None for key in self.ts_order}
self.ts_datapoint['row'] = 0
self.ts_file = open("csv/mmc_rw" + self.csv_suffix, "w")
self.ts_writer = csv.writer(self.ts_file)
self.ts_writer.writerow(self.ts_order)
self.evict_order = ['row', 'depth', 'rank', 'value', 'opcode']
self.evict_datapoint = {key: None for key in self.evict_order}
self.evict_datapoint['row'] = 0
self.evict_file = open("csv/mmc_rw_evict" + self.csv_suffix, "w")
self.evict_writer = csv.writer(self.evict_file)
self.evict_writer.writerow(self.evict_order)
self.purge_order = ['row', 'depth', 'rank', 'value', 'opcode']
self.purge_datapoint = {key: None for key in self.purge_order}
self.purge_datapoint['row'] = 0
self.purge_file = open("csv/mmc_rw_purge" + self.csv_suffix, "w")
self.purge_writer = csv.writer(self.purge_file)
self.purge_writer.writerow(self.purge_order)
def __init__(self, protocol, debug=False):
'''
Constructs an order book which will be a client to
the given blob protocol.
:param protocol: blob protocol to which this will be a client
:param debug: flag to print debug message
'''
BlobClient.__init__(self, protocol)
self.debug = debug
# A book can forward messages to multiple BookClients.
self.clients = []
# Track sequence number for blob protocol.
self.sequence = None
# Internal data structures for storing orders.
self.orders_by_id = {}
self.bids = FastRBTree()
self.asks = FastRBTree()
def __init__(self, s_side):
'''
Initialize a BookSide object. Save all parameters as attributes
:param s_side: string. BID or ASK
'''
if s_side not in ['BID', 'ASK']:
raise InvalidTypeException('side should be BID or ASK')
self.s_side = s_side
self.price_tree = FastRBTree()
self._i_idx = 0
self.d_order_map = {}
self.last_price = 0.
def GetLeastNumbers2(l,k):
count = 0
tree = FastRBTree()
for i in l:
if count <k:
tree.insert(i,i)
count += 1
else:
maxVal = max(tree)
if i < maxVal:
tree.remove(maxVal)
tree.insert(i,i)
datas = [item for item in tree]
return datas
def __init__(self):
'''
Limit order book tree implementation using Red-Black tree for self-balancing
Each limit price level is a OrderLinkedlist, and each order contains information
including id, price, timestamp, volume
self.limit_level: dict
key: price level; value: OrderLinkedlist object
self.order_ids: dict
key: order id; value: Order object
helps to locate order by id
'''
# tree that store price as keys and number of orders on that level as values
self.price_tree = FastRBTree()
self.max_price = None
self.min_price = None
self.limit_levels = {}
self.order_ids = {}
def run(self):
for p in range(self.N):
if self.processed[p]:
continue
self.processed[p] = True
self.order.append(p)
seeds = FastRBTree()
self._update(p, seeds)
while seeds:
# pop_min returns ( (reachability, q), q ).
_, q = seeds.pop_min()
self.processed[q] = True
self.order.append(q)
self._update(q, seeds)
start = timer()
self._extract_cluster_xi()
print("extract cluster took", timer() - start)
def __init__(self, s_side, fr_data, i_member=None):
'''
Initialize a BookSide object. Save all parameters as attributes
:param s_side: string. BID or ASK
:param fr_data: ZipExtFile object. data to read
:param i_member*: integer. Member number to be used as a filter
'''
if s_side not in ['BID', 'ASK']:
raise InvalidTypeException('side should be BID or ASK')
self.i_member = i_member
self.s_side = s_side
self.price_tree = FastRBTree()
self._i_idx = 0
self.fr_data = fr_data
self.parser = parser_data.LineParser(s_side)
self.d_order_map = {}
self.last_price = 0.
def createSequenceDB():
file = open(sys.argv[1], "r")
db = {}
for line in file:
alert = parseLineOfJSON(line)
if alert == None:
continue
if alert[
'IP'] not in db: # AK NIE JE ESTE V DB SEKVIENCIA (IP ADDRESSA)
db[alert['IP']] = FastRBTree()
isThereNode = db[alert['IP']].get(
alert['Time'],
None) #AK UZ V TOM CASE JE NIAKA UDALOST V SEKVENCII
if isThereNode != None:
if alert['alert'] not in isThereNode:
isThereNode.append(alert['alert'])
else:
db[alert['IP']].insert(alert['Time'], [alert['alert']])
return db
def __init__(self, max_depth, data):
# RBTree: maintains sorted order of rates
self.rate_tree = FastRBTree()
# dict: Uses rate and volume for key value pairs
self.rate_dict = {}
# float: amounts summed across all rate levels in tree
self.volume = 0
# int: total number of rate levels in tree
self.depth = len(data)
# int: maximum number of rate levels in tree
self.max_depth = max_depth
# populate rate_tree and rate_dict from public API call data
# set volume
for level in data:
rate = float(level[0])
amount = float(level[1])
self.rate_tree.insert(rate, 0)
self.rate_dict[rate] = amount
self.volume += amount
def PushNewScope(self):
self.PushScope(FastRBTree())
return
def reset(self, testing=False):
'''
Reset the state and the agent's memory about its positions
:param testing: boolean. If should freeze policy
'''
self.state = None
# self.k_steps = 0 # maybe I should not zero that
self.n_steps = 0
self.position = {}
self.d_order_tree = {}
self.d_order_map = {}
# set new order size
self.done = False
if self.env.s_main_intrument in self.d_initial_pos:
f_pos = abs(self.d_initial_pos[self.env.s_main_intrument]['Q'])
if f_pos == 0:
self.order_size = 5
else:
self.order_size = max(5, int(f_pos / 6))
# ser position and order tree
for s_instr in self.env.l_instrument:
self.position[s_instr] = {
'qAsk': 0.,
'Ask': 0.,
'qBid': 0.,
'Bid': 0.
}
if s_instr in self.d_initial_pos:
f_qty = self.d_initial_pos[s_instr]['Q']
f_price = self.d_initial_pos[s_instr]['P']
if f_qty < 0:
f_qty = abs(f_qty)
self.position[s_instr]['qAsk'] += f_qty
self.position[s_instr]['Ask'] += f_qty * f_price
elif f_qty > 0:
self.position[s_instr]['qBid'] += f_qty
self.position[s_instr]['Bid'] += f_qty * f_price
self.d_order_tree[s_instr] = {
'BID': FastRBTree(),
'ASK': FastRBTree()
}
self.d_order_map[s_instr] = {}
self.next_time = 0.
self.next_hedge_time = 0.
self.log_info = {'duration': 0., 'total_reward': 0.}
l_feat_list = [
'rwd_hist', 'pnl_hist', 'duration_hist', 'time',
'last_inputs_hist', 'features_hist', 'update_hist', 'rwd_info_hist'
]
for s_feature in l_feat_list:
self.log_info[s_feature] = []
self.last_max_pnl = None
self.f_pnl = 0.
self.f_delta_pnl = 0.
self.old_state = None
self.last_action = None
self.older_code = None
self.cum_reward = 0.
self.b_need_hedge = False
self.f_next_stoptime = None
self.last_delta_time = None
self.last_trade_time = 0.
self.f_timetoupdate = 0.
self.b_new_action = False
self.logged_action = False
self.tlast_step_logged = 0.
self.last_spread = None
self.current_code = None
self.last_hedge_code = None
self.b_has_traded = False
self.b_new_reward = False
# account the actions taken
action_to_iterate = [None, 'BEST_BID', 'BEST_OFFER', 'BEST_BOTH']
self.d_actions_acc = dict(
zip(action_to_iterate, [0 for x in action_to_iterate]))
# Reset any variables here, if required
self.reset_additional_variables(testing)
def __init__(self):
self.tree = FastRBTree()
def test_creation():
ST = SymbolTable("")
assert( type(ST.Table) == type([]) )
assert( type(ST.TopScope) == type(FastRBTree()) )
def __init__(self):
self.price_tree = FastRBTree()
self.min_price = None
self.max_price = None
def __init__(self):
self.book = [FastRBTree(), FastRBTree()] #sell and buy
def __init__(self, name):
self.name = name
self.children = {}
self.visited = False
self.ancestors = FastRBTree()
tree[16.46] = 2564
tree[16.44] = 2709
# tree[16.45] = 2644
# tree[16.43] = 2621
# tree[16.49] = 3959
# tree[16.47] = 3494
# tree[16.48] = 3399
# tree[16.46] = 3364
# tree[16.44] = 3509
# tree[16.45] = 3444
# tree[16.43] = 3421
# tree[16.46] = 3735
# del tree[15.84]
# tree[16.43] = 4921
# tree[16.48] = 4099
# tree[16.5] = 1279
# tree[16.49] = 1959
# tree[16.39] = tree.get(16.39,0) + 2000
rbt = RBTree()
frbt = FastRBTree()
populate(rbt)
populate(frbt)
print('RBT len: {0} FRBT len: {1}'.format(len(rbt), len(frbt)))
for key, value in rbt.items():
print("RBTree[{key}] = {value} <-> FastRBTree[{key}] = {value2}".format(
key=key, value=value, value2=frbt[key]))
def parse_vcf_files(dArgs, avail_pos, aSampleNames):
'''
Parse vcf files to data structure
Parameters
----------
dArgs: dict
input parameter dictionary as created by get_args()
avail_pos: dict
dict of bintrees for each contig
aSampleNames: list
list of sample names
Returns
-------
0
also writes all data to avail_pos
'''
empty_tree = FastRBTree()
exclude = {}
include = {}
if dArgs["exclude"] != None or dArgs["include"] != None:
pos = {}
chr_pos = []
bed_file = dArgs["include"] if dArgs["include"] != None else dArgs[
"exclude"]
with open(bed_file) as fp:
for line in fp:
data = line.strip().split("\t")
chr_pos += [(
i,
False,
) for i in xrange(int(data[1]),
int(data[2]) + 1)]
try:
pos[data[0]] += chr_pos
except KeyError:
pos[data[0]] = chr_pos
pos = {chrom: FastRBTree(l) for chrom, l in pos.items()}
if dArgs["include"]:
include = pos
else:
exclude = pos
dSamNms = {'reference': None}
# First pass to get the references and the positions to be analysed.
for vcf_in in dArgs['input']:
reader = vcf.Reader(filename=vcf_in)
# Get the sample name from the VCF file (usually the read group).
sample_name = reader.samples[0]
assert dSamNms.has_key(
sample_name
) == False, "ERROR: %s is not a unique sample name in the set of vcfs" % (
sample_name)
dSamNms[sample_name] = None
# Go over every position in the reader.
for record in reader:
# Inject a property about uncallable genotypes into record.
record.__setattr__(
"is_uncallable", is_uncallable(record)
) # is_uncallable = types.MethodType(is_uncallable, record)
# SKIP indels, if not handled then can cause REF base to be >1
if record.is_indel and not (record.is_uncallable
or record.is_monomorphic) or len(
record.REF) > 1:
# if len(record.REF) > 1:
# print "%s\t%s\t%s\t%s\t%s" % (sample_name,record.POS,-1,record.FILTER,record)
continue
# if record.is_deletion and not record.is_uncallable:
# continue
# SKIP (or include) any pre-specified regions.
if include and record.POS not in include.get(
record.CHROM,
empty_tree) or exclude and record.POS in exclude.get(
record.CHROM, empty_tree):
# if include.get(record.CHROM, empty_tree).get(record.POS, False) or \
# not exclude.get(record.CHROM, empty_tree).get(record.POS, True):
continue
try:
_ = avail_pos[record.CHROM]
except KeyError:
avail_pos[record.CHROM] = FastRBTree()
# Setup the position data to contain reference and stats.
if avail_pos[record.CHROM].get(record.POS, None) is None:
avail_pos[record.CHROM].insert(record.POS, {
"reference": str(record.REF),
"stats": BaseStats()
})
position_data = avail_pos[record.CHROM].get(record.POS)
assert len(
position_data["reference"]
) == 1, "Reference base must be singluar: in %s found %s @ %s" % (
position_data["reference"], sample_name, record.POS)
# IF this is uncallable genotype, add gap "-"
if record.is_uncallable:
# TODO: Mentioned in issue: #7(gitlab)
position_data[sample_name] = "-"
# Update stats
position_data["stats"].gap += 1
elif not record.FILTER:
# If filter PASSED!
# Make sure the reference base is the same. Maybe a vcf from different species snuck in here?!
assert str(record.REF) == position_data["reference"] or str(
record.REF
) == 'N' or position_data[
"reference"] == 'N', "SOMETHING IS REALLY WRONG because reference for the same position is DIFFERENT! %s in %s (%s, %s)" % (
record.POS, vcf_in, str(
record.REF), position_data["reference"])
# update position_data['reference'] to a real base if possible
if position_data['reference'] == 'N' and str(
record.REF) != 'N':
position_data['reference'] = str(record.REF).upper()
if record.is_snp:
if len(record.ALT) > 1:
logging.info(
"POS %s passed filters but has multiple alleles REF: %s, ALT: %s. Inserting N",
record.POS, str(record.REF), str(record.ALT))
position_data[sample_name] = "N"
position_data["stats"].N += 1
else:
position_data[sample_name] = str(record.ALT[0]).upper()
position_data["stats"].mut += 1
# Filter(s) failed
elif record.is_snp:
position_data[sample_name] = 'N'
position_data["stats"].N += 1
else:
# filter fail; code as N for consistency
position_data[sample_name] = "N"
position_data["stats"].N += 1
# finished parsing
for sn in dSamNms.keys():
aSampleNames.append(sn)
return 0
def __init__(self, attr=RangeAttribute):
self._tree = FastRBTree()
self._attr = attr
def __init__(self):
self.price_tree = FastRBTree()
self.trade_map = {}
self.num_trades = 0 # Contains count of Orders in tree
self.depth = 0 # Number of different prices in tree (http://en.wikipedia.org/wiki/trade_book_(trading)#Book_depth)
from bintrees import FastRBTree
T = int(sys.stdin.readline().strip())
def split(n):
n -= 1
return n // 2, n // 2 + (n % 2)
assert (0, 0) == split(1)
assert (0, 1) == split(2)
assert (1, 1) == split(3)
for t in range(1, T + 1):
N, K = map(int, sys.stdin.readline().strip().split())
holes = FastRBTree({N: 1})
while K > 0:
size, count = holes.max_item()
del holes[size]
for s in split(size):
holes[s] = holes.get(s, 0) + count
K -= count
minD, maxD = split(size)
print("Case #%d: %d %d" % (t, maxD, minD))
#! /usr/bin/env python
# coding:utf-8
from __future__ import division
import heapq
import bintrees
import random
if __name__ == '__main__':
from benchmarker import Benchmarker
from itertools import repeat, izip
from bintrees import FastRBTree
# initialize heapq
h = range(10000)
heapq.heapify(h)
# initialize AVLTree
m = izip(xrange(10000), repeat(True))
t = FastRBTree(m)
for bm in Benchmarker(width=20, loop=100000, cycle=3, extra=1):
for _ in bm.empty():
pass
for _ in bm('heapq'):
heapq.heappop(h)
heapq.heappush(h, random.randint(-100000, 100000))
for _ in bm('FastRBTree'):
t.pop_min()
t[random.randint(-100000, 100000)] = True
def __init__(self):
self.price_tree = FastRBTree()
self.price_map = {}
self.order_map = {}
self.received_orders = {}
def parse_wg_alignment(dArgs, avail_pos, aSampleNames):
'''
Parse alignment to data structure
Parameters
----------
dArgs: dict
input parameter dictionary as created by get_args()
avail_pos: dict
dict of bintrees for each contig
aSampleNames: list
list of sample names
Returns
-------
0
also writes all data to avail_pos
'''
"""
avail_pos looks like this:
{'gi|194097589|ref|NC_011035.1|':
FastRBTree({2329: {'stats': <vcf2distancematrix.base_stats object at 0x40fb590>,
'reference': 'A',
'211700_H15498026501': 'C',
'211701_H15510030401': 'C',
'211702_H15522021601': 'C'},
3837: {'211700_H15498026501': 'G',
'stats': <vcf2distancematrix.base_stats object at 0x40fbf90>,
'211701_H15510030401': 'G',
'reference': 'T',
'211702_H15522021601': 'G'},
4140: {'211700_H15498026501': 'A',
'stats': <vcf2distancematrix.base_stats object at 0x40fb790>,
'211701_H15510030401': 'A',
'reference': 'G',
'211702_H15522021601': 'A'}})}
"""
if os.path.exists(dArgs['alignment_input']) == False:
logging.error("Input alignment file not found.")
return 1
dSeqs = {}
with open(dArgs['alignment_input'], 'r') as algn:
# parse the fa file
sSeq = ""
sName = ""
for sLine in algn:
sLine = sLine.strip()
if sLine.startswith(">"):
if len(sSeq) > 0:
dSeqs[sName] = sSeq.upper()
sSeq = ""
sName = sLine[1:].split(' ')[0]
continue
sSeq = sSeq + sLine
dSeqs[sName] = sSeq.upper()
sRefName = 'reference'
if dSeqs.has_key(sRefName) == False:
if dArgs['refgenomename'] is not None and dSeqs.has_key(
dArgs['refgenomename']) == True:
sRefName = dArgs['refgenomename']
else:
logging.error(
"No seq named 'reference' in your alignment AND alternative name not found or not given either."
)
return 1
else:
logging.info("Your reference appears to be named 'reference'")
for sn in dSeqs.keys():
aSampleNames.append(sn)
all_seq_len = [len(x) for x in dSeqs.values()]
try:
assert len(all_seq_len) == all_seq_len.count(all_seq_len[0])
except AssertionError:
logging.error("Not all seqs in your alignment have the same length")
return 1
avail_pos['alignment_contig'] = FastRBTree()
iSeqLen = all_seq_len[0]
for i in range(0, iSeqLen):
d = {}
d['reference'] = dSeqs[sRefName][i]
for sn in aSampleNames:
if sn != sRefName and dSeqs[sn][i] != d['reference']:
d[sn] = dSeqs[sn][i]
# are the nt at this pos all the same?
if len(d.values()) == d.values().count(d.values()[0]):
continue
oBS = BaseStats()
for sn in aSampleNames:
oBS.update(d, sn, '_')
d['stats'] = oBS
avail_pos['alignment_contig'][i + 1] = d
return 0
def __init__(self):
self.priceTree = FastRBTree()
self.volume = 0
self.priceMap = {} # Map from price -> orderList object
self.orderMap = {} # Order ID to Order object
