def _init_client_mode(self, channel=None):
# this is client socket specific
self.socket_type = CLIENT_SOCKET
self.incoming = LinkedBlockingQueue() # list of read buffers
self.incoming_head = None # allows msg buffers to be broken up
self.python_inbound_handler = None
self.can_write = True
self.connect_handlers = []
self.peer_closed = False
self.connected = False
if channel:
self.channel = channel
self.python_inbound_handler = PythonInboundHandler(self)
self.connect_handlers = [self.python_inbound_handler]
self.connected = True
def __init__(self, rampupType, rampupValue, nbConsumers=1000, debug=False):
'''
Create the metronom
@param rampupType: FLAT, RYTHM, RAMPING
@param rampupValue: a value for rampup
@param nbConsumers: in grinder - gringer.threadNumber
'''
self.queue = LinkedBlockingQueue()
rampupFoo={'FLAT':FlatCadencer,'RYTHM':RythmCadencer,'RAMPING':RampingCadencer}
try:
foo = rampupFoo[rampupType]
except :
print "%s function does not exists" % (rampupType)
raise
self.metronom = __Metronom(self.queue, foo(rampupValue), nbConsumers, debug)
def __init__(self, family=None, type=None, proto=None):
# FIXME verify these are supported
self.family = family
self.type = type
self.proto = proto
self.blocking = True
self.timeout = None
self.channel = None
self.bind_addr = None
self.selectors = CopyOnWriteArrayList()
if self.type == SOCK_DGRAM:
self.socket_type = DATAGRAM_SOCKET
self.connected = False
self.incoming = LinkedBlockingQueue() # list of read buffers
self.incoming_head = None # allows msg buffers to be broken up
self.python_inbound_handler = None
self.can_write = True
else:
self.socket_type = UNKNOWN_SOCKET
class NativeCompiler(CompilerImpl):
def __init__(self):
self.tasks = LinkedBlockingQueue()
def waitOnTask(self, task):
fT = FutureTask(task)
self.tasks.put(fT)
return fT.get()
def compileShader(self, compileTask):
# This leaks! We need some cleanup flag of some kind
return self.waitOnTask(NativeCompilerTask(compileTask))
def linkProgram(self, linkTask):
return self.waitOnTask(NativeLinkerTask(linkTask))
def cleanResult(self, peerCon, result):
return self.waitOnTask(CleanupTask(peerCon, result))
def cleanCompileResult(self, result):
self.cleanResult(Shader, result)
def cleanLinkResult(self, result):
self.cleanResult(GLSLProgram, result)
def __init__(self):
self.tasks = LinkedBlockingQueue()
class _socketobject(object):
def __init__(self, family=None, type=None, proto=None):
# FIXME verify these are supported
self.family = family
self.type = type
self.proto = proto
self.blocking = True
self.timeout = None
self.channel = None
self.bind_addr = None
self.selectors = CopyOnWriteArrayList()
if self.type == SOCK_DGRAM:
self.socket_type = DATAGRAM_SOCKET
self.connected = False
self.incoming = LinkedBlockingQueue() # list of read buffers
self.incoming_head = None # allows msg buffers to be broken up
self.python_inbound_handler = None
self.can_write = True
else:
self.socket_type = UNKNOWN_SOCKET
def _register_selector(self, selector):
self.selectors.addIfAbsent(selector)
def _unregister_selector(self, selector):
return self.selectors.remove(selector)
def _notify_selectors(self):
for selector in self.selectors:
selector.notify()
def _handle_channel_future(self, future, reason):
# All differences between nonblocking vs blocking with optional timeouts
# is managed by this method.
# All sockets can be selected on, regardless of blocking/nonblocking
def workaround_jython_bug_for_bound_methods(_):
self._notify_selectors()
future.addListener(workaround_jython_bug_for_bound_methods)
if self.blocking:
if self.timeout is None:
return future.sync()
else:
future.await(self.timeout * _TO_NANOSECONDS,
TimeUnit.NANOSECONDS)
return future
else:
return future
def setblocking(self, mode):
self.blocking = mode
def settimeout(self, timeout):
if not timeout:
self.blocking = False
else:
self.timeout = timeout
def bind(self, address):
# Netty 4 supports binding a socket to multiple addresses;
# apparently this is the not the case for C API sockets
self.bind_addr = address
# CLIENT METHODS
# Calling connect/connect_ex means this is a client socket; these
# in turn use _connect, which uses Bootstrap, not ServerBootstrap
def _init_client_mode(self, channel=None):
# this is client socket specific
self.socket_type = CLIENT_SOCKET
self.incoming = LinkedBlockingQueue() # list of read buffers
self.incoming_head = None # allows msg buffers to be broken up
self.python_inbound_handler = None
self.can_write = True
self.connect_handlers = []
self.peer_closed = False
self.connected = False
if channel:
self.channel = channel
self.python_inbound_handler = PythonInboundHandler(self)
self.connect_handlers = [self.python_inbound_handler]
self.connected = True
def _connect(self, addr):
print "Begin _connect"
self._init_client_mode()
self.connected = True
self.python_inbound_handler = PythonInboundHandler(self)
bootstrap = Bootstrap().group(NIO_GROUP).channel(NioSocketChannel)
# add any options
# FIXME really this is just for SSL handling
if self.connect_handlers:
for handler in self.connect_handlers:
print "Adding connect handler", handler
bootstrap.handler(handler)
else:
print "Adding read adapter", self.python_inbound_handler
bootstrap.handler(self.python_inbound_handler)
# FIXME also support any options here
def completed(f):
self._notify_selectors()
print "Connection future - connection completed", f
host, port = addr
future = bootstrap.connect(host, port)
future.addListener(completed)
self._handle_channel_future(future, "connect")
self.channel = future.channel()
print "Completed _connect on {}".format(self)
def _post_connect(self):
# Post-connect step is necessary to handle SSL setup,
# otherwise the read adapter can race in seeing encrypted
# messages from the peer
if self.connect_handlers:
print "Adding read adapter", self.python_inbound_handler
self.channel.pipeline().addLast(self.python_inbound_handler)
def peer_closed(x):
print "Peer closed channel {} {}".format(self, x)
self.incoming.put(_PEER_CLOSED)
self._notify_selectors()
self.channel.closeFuture().addListener(peer_closed)
def connect(self, addr):
# Unwrapped sockets can immediately perform the post-connect step
self._connect(addr)
self._post_connect()
print "Completed connect {} to {}".format(self, addr)
def connect_ex(self, addr):
self.connect(addr)
if self.blocking:
return errno.EISCONN
else:
return errno.EINPROGRESS
# SERVER METHODS
# Calling listen means this is a server socket
def listen(self, backlog):
self.socket_type = SERVER_SOCKET
b = ServerBootstrap()
b.group(NIO_GROUP)
b.channel(NioServerSocketChannel)
b.option(ChannelOption.SO_BACKLOG, backlog)
# FIXME pass through child options from self; note that C API sockets do not distinguish
# EXAMPLE - b.childOption(ChannelOption.SO_KEEPALIVE, True)
# FIXME per http://stackoverflow.com/questions/9774023/netty-throttling-accept-on-boss-thread,
# should set a parentHandler to ensure throttling to avoid denial of service attacks against this layer;
# it's up to using Python code to do this, but at the very least there should be some sort of blocking
# to ensure we don't exceed the desired backlog in this chunk of code;
# right now, assumption is a ArrayBlockingQueue of sufficient size should suffice instead
self.client_queue = ArrayBlockingQueue(backlog)
# FIXME this should queue up sockets that are wrapped accordingly;
# in particular they should be wrapped SSLSocket objects (inheriting SSLEngine settings)
b.childHandler(ClientSocketHandler(self))
# returns a ChannelFuture, but regardless for blocking/nonblocking, return immediately
b.bind(_get_inet_addr(self.bind_addr))
def accept(self):
s = self.client_queue.take()
return s, s.getpeername()
# DATAGRAM METHODS
# needs to implicitly bind to 0 if not specified
def _datagram_connect(self):
# FIXME raise exception if not of the right family
if not self.connected:
print "Connecting datagram socket to", self.bind_addr
self.connected = True
self.python_inbound_handler = PythonInboundHandler(self)
bootstrap = Bootstrap().group(NIO_GROUP).channel(NioDatagramChannel)
bootstrap.handler(self.python_inbound_handler)
# add any options
# such as .option(ChannelOption.SO_BROADCAST, True)
future = bootstrap.bind(_get_inet_addr(self.bind_addr))
self._handle_channel_future(future, "bind")
self.channel = future.channel()
print "Completed _datagram_connect on {}".format(self)
def sendto(self, string, arg1, arg2=None):
# Unfortunate overloading
if arg2 is not None:
flags = arg1
address = arg2
else:
flags = None
address = arg1
print "Sending data", string
self._datagram_connect()
# need a helper function to select proper address;
# this should take in account if AF_INET, AF_INET6
packet = DatagramPacket(Unpooled.wrappedBuffer(string),
_get_inet_addr(address))
future = self.channel.writeAndFlush(packet)
self._handle_channel_future(future, "sendto")
return len(string)
# GENERAL METHODS
def close(self):
future = self.channel.close()
self._handle_channel_future(future, "close")
def shutdown(self, how):
if how & SHUT_RD:
try:
self.channel.pipeline().remove(self.python_inbound_handler)
except NoSuchElementException:
pass # already removed, can safely ignore (presumably)
if how & SHUT_WR:
self.can_write = False
def _readable(self):
if self.socket_type == CLIENT_SOCKET:
return ((self.incoming_head is not None and self.incoming_head.readableBytes()) or
self.incoming.peek())
elif self.socket_type == SERVER_SOCKET:
return bool(self.client_queue.peek())
else:
return False
def _writable(self):
return self.channel.isActive() and self.channel.isWritable()
def send(self, data):
data = str(data) # FIXME temporary fix if data is of type buffer
print "Sending data <<<{}>>>".format(data)
if not self.can_write:
raise Exception("Cannot write to closed socket") # FIXME use actual exception
future = self.channel.writeAndFlush(Unpooled.wrappedBuffer(data))
self._handle_channel_future(future, "send")
# FIXME are we sure we are going to be able to send this much data, especially async?
return len(data)
sendall = send # see note above!
def _get_incoming_msg(self):
if self.incoming_head is None:
if self.blocking:
if self.timeout is None:
self.incoming_head = self.incoming.take()
else:
self.incoming_head = self.incoming.poll(self.timeout * _TO_NANOSECONDS, TimeUnit.NANOSECONDS)
else:
self.incoming_head = self.incoming.poll() # Could be None
# Only return _PEER_CLOSED once
msg = self.incoming_head
if msg is _PEER_CLOSED:
self.incoming_head = None
return msg
def recv(self, bufsize, flags=0):
# For obvious reasons, concurrent reads on the same socket
# have to be locked; I don't believe it is the job of recv to
# do this; in particular this is the policy of SocketChannel,
# which underlies Netty's support for such channels.
msg = self._get_incoming_msg()
if msg is None:
return None
elif msg is _PEER_CLOSED:
return ""
msg_length = msg.readableBytes()
buf = jarray.zeros(min(msg_length, bufsize), "b")
msg.readBytes(buf)
if msg.readableBytes() == 0:
msg.release() # return msg ByteBuf back to Netty's pool
self.incoming_head = None
return buf.tostring()
def recvfrom(self, bufsize, flags=0):
# FIXME refactor common code from recv
self._datagram_connect()
packet = self._get_incoming_msg()
if packet is None:
return None
elif packet is _PEER_CLOSED:
return ""
msg = packet.content()
msg_length = msg.readableBytes()
buf = jarray.zeros(min(msg_length, bufsize), "b")
msg.readBytes(buf)
remote_addr = packet.sender() # may not be available on non datagram channels
sender = remote_addr.getHostString(), remote_addr.getPort()
if msg.readableBytes() == 0:
packet.release() # return msg ByteBuf back to Netty's pool
self.incoming_head = None
return buf.tostring(), sender
def fileno(self):
return self
def getsockopt(self, level, option):
return 0
def getpeername(self):
remote_addr = self.channel.remoteAddress()
return remote_addr.getHostString(), remote_addr.getPort()
def _unlatch(self):
pass # no-op once mutated from ChildSocket to normal _socketobject
def test_strategy(definition, strategy):
sapp = datetime.datetime.now()
# initialize tickers
t = Ticker(**definition)
t.strategy = strategy
tickers = {1: t}
# fill the tickers with 'historical data'
start = datetime.datetime(2003, 6, 16)
end = datetime.datetime(2003, 6, 23)
ticks = tick_list("ES", start, end)
first_day = ticks[0][0].day
index = 0
for tick in ticks:
if not tick[0].day == first_day:
break
else:
t.ticks.append(tick)
index += 1
# start 'engines'
tick_queue = LinkedBlockingQueue()
response_handler = IBResponseHandler(tick_queue)
connection = SimConnection(response_handler)
client = IBClient(connection, tickers)
# initialize trader
trader = TradingEngine(tick_queue, tickers, client)
# initialize client
client.start(trader)
# fill the queue and wait for the queue to be empty
for tick in ticks[index:]:
qtick = 1, tick[0], tick[1]
tick_queue.put(qtick)
# time.sleep(0.002) # to ease a little on the CPU usage
# wait unitl the queue is consumed
while tick_queue.size():
time.sleep(1)
# now show some statistics
report = {}
order_map = client.order_map[1] # ticker_id = 1
# check order validity, entry, exit, entry, exit, etc.
previous_signal = None
invalid_sequence = 0
for order_id in order_map:
order_entry = client.account["_orders"][order_id]
signal = order_entry["signal"]
if previous_signal:
if previous_signal.startswith("entry") and not signal.startswith("exit"):
invalid_sequence += 1
if previous_signal.startswith("exit") and not signal.startswith("entry"):
invalid_sequence += 1
previous_signal = signal
if invalid_sequence:
report["valid"] = True
else:
report["valid"] = False
# check number of long and short and order result
if not invalid_sequence:
entry_long = 0
entry_short = 0
long_result = []
short_result = []
all_result = []
ratio = 50
rt_comm = 4 # round trip commission
for i in range(len(order_map) - 1):
if i < len(order_map) - 1:
entry_order_id = order_map[i]
exit_order_id = order_map[i + 1]
entry_order = client.account["_orders"][entry_order_id]
exit_order = client.account["_orders"][exit_order_id]
entry_value = entry_order["fill_value"]
exit_value = exit_order["fill_value"]
signal = entry_order["signal"]
if signal.startswith("entry_long"):
entry_long += 1
value = exit_value - entry_value
long_result.append(value)
all_result.append(value)
if signal.startswith("entry_short"):
entry_short += 1
value = entry_value - exit_value
short_result.append(value)
all_result.append(value)
long_result = [r * ratio for r in long_result]
report["long_results"] = long_result
long_comm = [r - rt_comm for r in long_result]
report["long_results_with_commission"] = long_comm
short_result = [r * ratio for r in short_result]
report["short_results"] = short_result
short_comm = [r - rt_comm for r in short_result]
report["short_results_with_commission"] = short_comm
all_result = [r * ratio for r in all_result]
report["all_results"] = all_result
all_comm = [r - rt_comm for r in all_result]
report["all_results_with_commission"] = all_comm
report["long_trades"] = entry_long
report["short_trades"] = entry_short
report["sum_all_results"] = sum(all_result)
report["sum_all_results_with_commission"] = sum(all_comm)
report["sum_long_results"] = sum(long_result)
report["sum_long_results_with_commission"] = sum(long_comm)
report["sum_short_results"] = sum(short_result)
report["sum_short_results_with_commission"] = sum(short_comm)
if all_result:
avg_all_res = sum(all_result) / len(all_result)
avg_all_res_comm = sum(all_comm) / len(all_comm)
report["average_all_results"] = avg_all_res
report["average_all_results_with_commission"] = avg_all_res_comm
else:
report["average_all_results"] = None
report["average_all_results_with_commission"] = None
if long_result:
avg_long_res = sum(long_result) / len(long_result)
avg_long_res_comm = sum(long_comm) / len(long_comm)
report["average_long_results"] = avg_long_res
report["average_long_results_with_commission"] = avg_long_res_comm
else:
report["average_long_results"] = None
report["average_long_results_with_commission"] = None
if short_result:
avg_short_res = sum(short_result) / len(short_result)
avg_short_res_comm = sum(short_comm) / len(short_comm)
report["average_short_results"] = avg_short_res
report["average_short_results_with_commission"] = avg_short_res_comm
else:
report["average_short_results"] = None
report["average_short_results_with_commission"] = None
# calculate total capacity
capacity = 0
previous_tick_value = 0
for tick in ticks[index:]:
tick_value = tick[1]
if previous_tick_value:
cap = abs(tick_value - previous_tick_value)
capacity += cap
previous_tick_value = tick_value
total_capacity = capacity * ratio
report["total_capacity"] = total_capacity
res_for_cap = sum(all_comm) * 100 / total_capacity
report["result_for_capacity_percentage"] = res_for_cap
eapp = datetime.datetime.now()
report["analysis_time"] = eapp - sapp
return report
def __init__(self, coreThreads=1, maxThreads=1, keepalive=5, queue=None):
queue = queue or LinkedBlockingQueue()
ThreadPoolExecutor.__init__(self, coreThreads, maxThreads, keepalive,
TimeUnit.SECONDS, queue)
# Sets the maximum number of jobs queued for parallel execution before
# starting linear execution of new jobs
if System.getProperty("sython.linear_limit") is None:
if TRACE:
SF_LINEAR_LIMIT = 1024
else:
SF_LINEAR_LIMIT = 1024*1024
else:
SF_LINEAR_LIMIT = int(System.getProperty("sython.linear_limit"))
# Maximum recursion depth of stealing
SF_MAX_STEAL = 64
# A thread pool used for the executors
#SF_POOL = Executors.newCachedThreadPool()
SF_POOL = ThreadPoolExecutor(SF_MAX_CONCURRENT, SF_MAX_CONCURRENT, 60, TimeUnit.SECONDS, LinkedBlockingQueue());
# A set of tasks which might be available for stealing. Use a concurrent set so
# that it shares information between threads in a stable and relatively
# efficient way. Note that a task being in this set does not guarantee it is
# not being executed. A locking flag on the 'superFuture' task management
# objects disambiguates this to prevent double execution.
SF_PENDING = Collections.newSetFromMap(ConcurrentHashMap(SF_MAX_CONCURRENT*128,0.75,SF_MAX_CONCURRENT+1))
# All parallel jobs get a job number which is globally unique
# and increasing. This is used for logging and cycle checking
# and any other house keeping which requires a unique id across
# jobs.
SF_JOB_NUMB = AtomicLong()
# Tracks how many threads are waiting for other threads
def execute(urls, fname, poolsize=4):
wq = LinkedBlockingQueue()
wq.put(BufferedWriter(FileWriter(fname)))
pool = Executors.newFixedThreadPool(poolsize)
for u in urls:
pool.execute(ScraperAtom(wq, u))
