def _init_client_mode(self, channel=None): # this is client socket specific self.socket_type = CLIENT_SOCKET self.incoming = LinkedBlockingQueue() # list of read buffers self.incoming_head = None # allows msg buffers to be broken up self.python_inbound_handler = None self.can_write = True self.connect_handlers = [] self.peer_closed = False self.connected = False if channel: self.channel = channel self.python_inbound_handler = PythonInboundHandler(self) self.connect_handlers = [self.python_inbound_handler] self.connected = True
def __init__(self, rampupType, rampupValue, nbConsumers=1000, debug=False): ''' Create the metronom @param rampupType: FLAT, RYTHM, RAMPING @param rampupValue: a value for rampup @param nbConsumers: in grinder - gringer.threadNumber ''' self.queue = LinkedBlockingQueue() rampupFoo={'FLAT':FlatCadencer,'RYTHM':RythmCadencer,'RAMPING':RampingCadencer} try: foo = rampupFoo[rampupType] except : print "%s function does not exists" % (rampupType) raise self.metronom = __Metronom(self.queue, foo(rampupValue), nbConsumers, debug)
def __init__(self, family=None, type=None, proto=None): # FIXME verify these are supported self.family = family self.type = type self.proto = proto self.blocking = True self.timeout = None self.channel = None self.bind_addr = None self.selectors = CopyOnWriteArrayList() if self.type == SOCK_DGRAM: self.socket_type = DATAGRAM_SOCKET self.connected = False self.incoming = LinkedBlockingQueue() # list of read buffers self.incoming_head = None # allows msg buffers to be broken up self.python_inbound_handler = None self.can_write = True else: self.socket_type = UNKNOWN_SOCKET
class NativeCompiler(CompilerImpl): def __init__(self): self.tasks = LinkedBlockingQueue() def waitOnTask(self, task): fT = FutureTask(task) self.tasks.put(fT) return fT.get() def compileShader(self, compileTask): # This leaks! We need some cleanup flag of some kind return self.waitOnTask(NativeCompilerTask(compileTask)) def linkProgram(self, linkTask): return self.waitOnTask(NativeLinkerTask(linkTask)) def cleanResult(self, peerCon, result): return self.waitOnTask(CleanupTask(peerCon, result)) def cleanCompileResult(self, result): self.cleanResult(Shader, result) def cleanLinkResult(self, result): self.cleanResult(GLSLProgram, result)
def __init__(self): self.tasks = LinkedBlockingQueue()
class _socketobject(object): def __init__(self, family=None, type=None, proto=None): # FIXME verify these are supported self.family = family self.type = type self.proto = proto self.blocking = True self.timeout = None self.channel = None self.bind_addr = None self.selectors = CopyOnWriteArrayList() if self.type == SOCK_DGRAM: self.socket_type = DATAGRAM_SOCKET self.connected = False self.incoming = LinkedBlockingQueue() # list of read buffers self.incoming_head = None # allows msg buffers to be broken up self.python_inbound_handler = None self.can_write = True else: self.socket_type = UNKNOWN_SOCKET def _register_selector(self, selector): self.selectors.addIfAbsent(selector) def _unregister_selector(self, selector): return self.selectors.remove(selector) def _notify_selectors(self): for selector in self.selectors: selector.notify() def _handle_channel_future(self, future, reason): # All differences between nonblocking vs blocking with optional timeouts # is managed by this method. # All sockets can be selected on, regardless of blocking/nonblocking def workaround_jython_bug_for_bound_methods(_): self._notify_selectors() future.addListener(workaround_jython_bug_for_bound_methods) if self.blocking: if self.timeout is None: return future.sync() else: future.await(self.timeout * _TO_NANOSECONDS, TimeUnit.NANOSECONDS) return future else: return future def setblocking(self, mode): self.blocking = mode def settimeout(self, timeout): if not timeout: self.blocking = False else: self.timeout = timeout def bind(self, address): # Netty 4 supports binding a socket to multiple addresses; # apparently this is the not the case for C API sockets self.bind_addr = address # CLIENT METHODS # Calling connect/connect_ex means this is a client socket; these # in turn use _connect, which uses Bootstrap, not ServerBootstrap def _init_client_mode(self, channel=None): # this is client socket specific self.socket_type = CLIENT_SOCKET self.incoming = LinkedBlockingQueue() # list of read buffers self.incoming_head = None # allows msg buffers to be broken up self.python_inbound_handler = None self.can_write = True self.connect_handlers = [] self.peer_closed = False self.connected = False if channel: self.channel = channel self.python_inbound_handler = PythonInboundHandler(self) self.connect_handlers = [self.python_inbound_handler] self.connected = True def _connect(self, addr): print "Begin _connect" self._init_client_mode() self.connected = True self.python_inbound_handler = PythonInboundHandler(self) bootstrap = Bootstrap().group(NIO_GROUP).channel(NioSocketChannel) # add any options # FIXME really this is just for SSL handling if self.connect_handlers: for handler in self.connect_handlers: print "Adding connect handler", handler bootstrap.handler(handler) else: print "Adding read adapter", self.python_inbound_handler bootstrap.handler(self.python_inbound_handler) # FIXME also support any options here def completed(f): self._notify_selectors() print "Connection future - connection completed", f host, port = addr future = bootstrap.connect(host, port) future.addListener(completed) self._handle_channel_future(future, "connect") self.channel = future.channel() print "Completed _connect on {}".format(self) def _post_connect(self): # Post-connect step is necessary to handle SSL setup, # otherwise the read adapter can race in seeing encrypted # messages from the peer if self.connect_handlers: print "Adding read adapter", self.python_inbound_handler self.channel.pipeline().addLast(self.python_inbound_handler) def peer_closed(x): print "Peer closed channel {} {}".format(self, x) self.incoming.put(_PEER_CLOSED) self._notify_selectors() self.channel.closeFuture().addListener(peer_closed) def connect(self, addr): # Unwrapped sockets can immediately perform the post-connect step self._connect(addr) self._post_connect() print "Completed connect {} to {}".format(self, addr) def connect_ex(self, addr): self.connect(addr) if self.blocking: return errno.EISCONN else: return errno.EINPROGRESS # SERVER METHODS # Calling listen means this is a server socket def listen(self, backlog): self.socket_type = SERVER_SOCKET b = ServerBootstrap() b.group(NIO_GROUP) b.channel(NioServerSocketChannel) b.option(ChannelOption.SO_BACKLOG, backlog) # FIXME pass through child options from self; note that C API sockets do not distinguish # EXAMPLE - b.childOption(ChannelOption.SO_KEEPALIVE, True) # FIXME per http://stackoverflow.com/questions/9774023/netty-throttling-accept-on-boss-thread, # should set a parentHandler to ensure throttling to avoid denial of service attacks against this layer; # it's up to using Python code to do this, but at the very least there should be some sort of blocking # to ensure we don't exceed the desired backlog in this chunk of code; # right now, assumption is a ArrayBlockingQueue of sufficient size should suffice instead self.client_queue = ArrayBlockingQueue(backlog) # FIXME this should queue up sockets that are wrapped accordingly; # in particular they should be wrapped SSLSocket objects (inheriting SSLEngine settings) b.childHandler(ClientSocketHandler(self)) # returns a ChannelFuture, but regardless for blocking/nonblocking, return immediately b.bind(_get_inet_addr(self.bind_addr)) def accept(self): s = self.client_queue.take() return s, s.getpeername() # DATAGRAM METHODS # needs to implicitly bind to 0 if not specified def _datagram_connect(self): # FIXME raise exception if not of the right family if not self.connected: print "Connecting datagram socket to", self.bind_addr self.connected = True self.python_inbound_handler = PythonInboundHandler(self) bootstrap = Bootstrap().group(NIO_GROUP).channel(NioDatagramChannel) bootstrap.handler(self.python_inbound_handler) # add any options # such as .option(ChannelOption.SO_BROADCAST, True) future = bootstrap.bind(_get_inet_addr(self.bind_addr)) self._handle_channel_future(future, "bind") self.channel = future.channel() print "Completed _datagram_connect on {}".format(self) def sendto(self, string, arg1, arg2=None): # Unfortunate overloading if arg2 is not None: flags = arg1 address = arg2 else: flags = None address = arg1 print "Sending data", string self._datagram_connect() # need a helper function to select proper address; # this should take in account if AF_INET, AF_INET6 packet = DatagramPacket(Unpooled.wrappedBuffer(string), _get_inet_addr(address)) future = self.channel.writeAndFlush(packet) self._handle_channel_future(future, "sendto") return len(string) # GENERAL METHODS def close(self): future = self.channel.close() self._handle_channel_future(future, "close") def shutdown(self, how): if how & SHUT_RD: try: self.channel.pipeline().remove(self.python_inbound_handler) except NoSuchElementException: pass # already removed, can safely ignore (presumably) if how & SHUT_WR: self.can_write = False def _readable(self): if self.socket_type == CLIENT_SOCKET: return ((self.incoming_head is not None and self.incoming_head.readableBytes()) or self.incoming.peek()) elif self.socket_type == SERVER_SOCKET: return bool(self.client_queue.peek()) else: return False def _writable(self): return self.channel.isActive() and self.channel.isWritable() def send(self, data): data = str(data) # FIXME temporary fix if data is of type buffer print "Sending data <<<{}>>>".format(data) if not self.can_write: raise Exception("Cannot write to closed socket") # FIXME use actual exception future = self.channel.writeAndFlush(Unpooled.wrappedBuffer(data)) self._handle_channel_future(future, "send") # FIXME are we sure we are going to be able to send this much data, especially async? return len(data) sendall = send # see note above! def _get_incoming_msg(self): if self.incoming_head is None: if self.blocking: if self.timeout is None: self.incoming_head = self.incoming.take() else: self.incoming_head = self.incoming.poll(self.timeout * _TO_NANOSECONDS, TimeUnit.NANOSECONDS) else: self.incoming_head = self.incoming.poll() # Could be None # Only return _PEER_CLOSED once msg = self.incoming_head if msg is _PEER_CLOSED: self.incoming_head = None return msg def recv(self, bufsize, flags=0): # For obvious reasons, concurrent reads on the same socket # have to be locked; I don't believe it is the job of recv to # do this; in particular this is the policy of SocketChannel, # which underlies Netty's support for such channels. msg = self._get_incoming_msg() if msg is None: return None elif msg is _PEER_CLOSED: return "" msg_length = msg.readableBytes() buf = jarray.zeros(min(msg_length, bufsize), "b") msg.readBytes(buf) if msg.readableBytes() == 0: msg.release() # return msg ByteBuf back to Netty's pool self.incoming_head = None return buf.tostring() def recvfrom(self, bufsize, flags=0): # FIXME refactor common code from recv self._datagram_connect() packet = self._get_incoming_msg() if packet is None: return None elif packet is _PEER_CLOSED: return "" msg = packet.content() msg_length = msg.readableBytes() buf = jarray.zeros(min(msg_length, bufsize), "b") msg.readBytes(buf) remote_addr = packet.sender() # may not be available on non datagram channels sender = remote_addr.getHostString(), remote_addr.getPort() if msg.readableBytes() == 0: packet.release() # return msg ByteBuf back to Netty's pool self.incoming_head = None return buf.tostring(), sender def fileno(self): return self def getsockopt(self, level, option): return 0 def getpeername(self): remote_addr = self.channel.remoteAddress() return remote_addr.getHostString(), remote_addr.getPort() def _unlatch(self): pass # no-op once mutated from ChildSocket to normal _socketobject
def test_strategy(definition, strategy): sapp = datetime.datetime.now() # initialize tickers t = Ticker(**definition) t.strategy = strategy tickers = {1: t} # fill the tickers with 'historical data' start = datetime.datetime(2003, 6, 16) end = datetime.datetime(2003, 6, 23) ticks = tick_list("ES", start, end) first_day = ticks[0][0].day index = 0 for tick in ticks: if not tick[0].day == first_day: break else: t.ticks.append(tick) index += 1 # start 'engines' tick_queue = LinkedBlockingQueue() response_handler = IBResponseHandler(tick_queue) connection = SimConnection(response_handler) client = IBClient(connection, tickers) # initialize trader trader = TradingEngine(tick_queue, tickers, client) # initialize client client.start(trader) # fill the queue and wait for the queue to be empty for tick in ticks[index:]: qtick = 1, tick[0], tick[1] tick_queue.put(qtick) # time.sleep(0.002) # to ease a little on the CPU usage # wait unitl the queue is consumed while tick_queue.size(): time.sleep(1) # now show some statistics report = {} order_map = client.order_map[1] # ticker_id = 1 # check order validity, entry, exit, entry, exit, etc. previous_signal = None invalid_sequence = 0 for order_id in order_map: order_entry = client.account["_orders"][order_id] signal = order_entry["signal"] if previous_signal: if previous_signal.startswith("entry") and not signal.startswith("exit"): invalid_sequence += 1 if previous_signal.startswith("exit") and not signal.startswith("entry"): invalid_sequence += 1 previous_signal = signal if invalid_sequence: report["valid"] = True else: report["valid"] = False # check number of long and short and order result if not invalid_sequence: entry_long = 0 entry_short = 0 long_result = [] short_result = [] all_result = [] ratio = 50 rt_comm = 4 # round trip commission for i in range(len(order_map) - 1): if i < len(order_map) - 1: entry_order_id = order_map[i] exit_order_id = order_map[i + 1] entry_order = client.account["_orders"][entry_order_id] exit_order = client.account["_orders"][exit_order_id] entry_value = entry_order["fill_value"] exit_value = exit_order["fill_value"] signal = entry_order["signal"] if signal.startswith("entry_long"): entry_long += 1 value = exit_value - entry_value long_result.append(value) all_result.append(value) if signal.startswith("entry_short"): entry_short += 1 value = entry_value - exit_value short_result.append(value) all_result.append(value) long_result = [r * ratio for r in long_result] report["long_results"] = long_result long_comm = [r - rt_comm for r in long_result] report["long_results_with_commission"] = long_comm short_result = [r * ratio for r in short_result] report["short_results"] = short_result short_comm = [r - rt_comm for r in short_result] report["short_results_with_commission"] = short_comm all_result = [r * ratio for r in all_result] report["all_results"] = all_result all_comm = [r - rt_comm for r in all_result] report["all_results_with_commission"] = all_comm report["long_trades"] = entry_long report["short_trades"] = entry_short report["sum_all_results"] = sum(all_result) report["sum_all_results_with_commission"] = sum(all_comm) report["sum_long_results"] = sum(long_result) report["sum_long_results_with_commission"] = sum(long_comm) report["sum_short_results"] = sum(short_result) report["sum_short_results_with_commission"] = sum(short_comm) if all_result: avg_all_res = sum(all_result) / len(all_result) avg_all_res_comm = sum(all_comm) / len(all_comm) report["average_all_results"] = avg_all_res report["average_all_results_with_commission"] = avg_all_res_comm else: report["average_all_results"] = None report["average_all_results_with_commission"] = None if long_result: avg_long_res = sum(long_result) / len(long_result) avg_long_res_comm = sum(long_comm) / len(long_comm) report["average_long_results"] = avg_long_res report["average_long_results_with_commission"] = avg_long_res_comm else: report["average_long_results"] = None report["average_long_results_with_commission"] = None if short_result: avg_short_res = sum(short_result) / len(short_result) avg_short_res_comm = sum(short_comm) / len(short_comm) report["average_short_results"] = avg_short_res report["average_short_results_with_commission"] = avg_short_res_comm else: report["average_short_results"] = None report["average_short_results_with_commission"] = None # calculate total capacity capacity = 0 previous_tick_value = 0 for tick in ticks[index:]: tick_value = tick[1] if previous_tick_value: cap = abs(tick_value - previous_tick_value) capacity += cap previous_tick_value = tick_value total_capacity = capacity * ratio report["total_capacity"] = total_capacity res_for_cap = sum(all_comm) * 100 / total_capacity report["result_for_capacity_percentage"] = res_for_cap eapp = datetime.datetime.now() report["analysis_time"] = eapp - sapp return report
def __init__(self, coreThreads=1, maxThreads=1, keepalive=5, queue=None): queue = queue or LinkedBlockingQueue() ThreadPoolExecutor.__init__(self, coreThreads, maxThreads, keepalive, TimeUnit.SECONDS, queue)
# Sets the maximum number of jobs queued for parallel execution before # starting linear execution of new jobs if System.getProperty("sython.linear_limit") is None: if TRACE: SF_LINEAR_LIMIT = 1024 else: SF_LINEAR_LIMIT = 1024*1024 else: SF_LINEAR_LIMIT = int(System.getProperty("sython.linear_limit")) # Maximum recursion depth of stealing SF_MAX_STEAL = 64 # A thread pool used for the executors #SF_POOL = Executors.newCachedThreadPool() SF_POOL = ThreadPoolExecutor(SF_MAX_CONCURRENT, SF_MAX_CONCURRENT, 60, TimeUnit.SECONDS, LinkedBlockingQueue()); # A set of tasks which might be available for stealing. Use a concurrent set so # that it shares information between threads in a stable and relatively # efficient way. Note that a task being in this set does not guarantee it is # not being executed. A locking flag on the 'superFuture' task management # objects disambiguates this to prevent double execution. SF_PENDING = Collections.newSetFromMap(ConcurrentHashMap(SF_MAX_CONCURRENT*128,0.75,SF_MAX_CONCURRENT+1)) # All parallel jobs get a job number which is globally unique # and increasing. This is used for logging and cycle checking # and any other house keeping which requires a unique id across # jobs. SF_JOB_NUMB = AtomicLong() # Tracks how many threads are waiting for other threads
def execute(urls, fname, poolsize=4): wq = LinkedBlockingQueue() wq.put(BufferedWriter(FileWriter(fname))) pool = Executors.newFixedThreadPool(poolsize) for u in urls: pool.execute(ScraperAtom(wq, u))