def processData(self, strData): self.logger.log("{}: Processing data: {}".format( str(self.tId), strData)) # client related actions if ("MSG051" in strData): self.msg51RxCtr += 1 if self.msg_51_52_53_Logger is not None: self.msg_51_52_53_Logger.log( "{} Received MSG051. Rx ctr: {}".format( str(util.getTimeStamp()), str(self.msg51RxCtr))) elif ("MSG056" in strData): self.msg56RxCtr += 1 if self.msg_56_Logger is not None: self.msg_56_Logger.log("{} Received MSG056. Rx ctr: {}".format( str(util.getTimeStamp()), str(self.msg56RxCtr))) elif ("MSG054" in strData): self.msg54RxCtr += 1 if self.msg_54_Logger is not None: self.msg_54_Logger.log("{} Received MSG054. Rx ctr: {}".format( str(util.getTimeStamp()), str(self.msg54RxCtr))) # dummy plc (server) related actions elif ("MSG052" in strData): self.processInp(sInp="System_UnArmed") elif ("MSG053" in strData): self.processInp(sInp="System_Armed")
def process_Rx_Message(self, message): self.logger.log("{} {}: Processing message: {}".format( str(util.getTimeStamp()), str(self.tId), str(message))) try: if ((not self.validateMessage) or (self.validateMsg(sMessage=message))): for mType, fnPtr in self.processing_fn_dict.items(): if mType in message: fnPtr(message) break elif (not self.validateMsg(sMessage=message)): # check if multiple messages are received ? count = message.count(str(self.bSTX)) if count > 1: # we have received multiple messages self.logger.log( "{}: Multiple ({}) Messages Received".format( str(self.tId), str(count))) searchMsg = message while (searchMsg.find(str(self.bSTX)) != -1): # starts with STX partSize = int(searchMsg[1:5]) partMessage = searchMsg[:partSize] self.process_Rx_Message(message=partMessage) searchMsg = searchMsg[partSize:] else: self.logger.log( "{}: Invalid message received, ignoring".format( str(self.tId))) except Exception as e: self.logger.log_exception(e, traceback)
def freeze(self): gd = self.sess.graph.as_graph_def() print("convt..") for node in gd.node: if node.op == 'RefSwitch': node.op = 'Switch' for index in range(len(node.input)): if 'moving_' in node.input[index]: node.input[index] = node.input[index] + '/read' elif node.op == 'AssignSub': node.op = 'Sub' if 'use_locking' in node.attr: del node.attr['use_locking'] print("const...") gd = graph_util.convert_variables_to_constants(self.sess, gd, self.outputNodes) optlib.ensure_graph_is_valid(gd) input_node_names = self.inputNodes output_node_names = self.outputNodes placeholder_type_enum = self.inputNodesTypes for i in range(len(placeholder_type_enum)): placeholder_type_enum[i] = placeholder_type_enum[i].as_datatype_enum print("strip...") gd = strip_unused_lib.strip_unused(gd, input_node_names, output_node_names, placeholder_type_enum) optlib.ensure_graph_is_valid(gd) filename = 'frozen ' + util.getTimeStamp() + '.pb' tf.train.write_graph(gd, self.parentPath, filename, as_text=False) return os.path.join(self.parentPath, filename)
def process_ACKN_message(self, message): self.logger.log("{}: Processing an ACKN message".format(str(self.tId))) ackNo = message[9:] ackNo = ackNo[:-2] if self.ackLogger is not None: self.ackLogger.log("{}, {}, Received ACKN, {}".format( str(util.getTimeStamp()), str(int(ackNo)), message)) if ackNo in self.ackList: self.logger.log("{}: ACKN sequence ({}) matching".format( str(self.tId), str(ackNo))) self.ackList.remove(ackNo) else: self.logger.log( "{}: ERROR -- ACKN sequence ({}) not matching".format( str(self.tId), str(ackNo)))
def main(): batchCount = 128 lastEphoc = 0 step = 0 fps = util.FpsCounter() timestamp = util.getTimeStamp() testDir = './temp/blink-test ' + timestamp data = Dataset() transfer = TransferHelper() model = Model(dataSize = data.count, batchSize = batchCount) saver = tf.train.Saver() config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config = config) as sess: sess.run(tf.global_variables_initializer()) transfer.pretrain( \ sess, model.input, [32, 32, 3], model.outputCNN, phase_train = model.phase_train, keep_prob = model.keep_prob, maxEphoc = 500, targetAcc = 1, dropRate = model.getDropRate(), testDropRate = model.getTestDropRate()) transfer.close() while True: step += 1 ephoc = step * batchCount / data.count fps.add(batchCount) batchImg, batchLabel = data.batch(batchCount) fetch = model.optimize(sess, batchImg, batchLabel) del batchImg, batchLabel if(step % 10 == 0): tbatchImg, tbatchLabel = data.batch(batchCount, False) tfetch = model.forward(sess, tbatchImg, tbatchLabel) print('step:', step, 'ephoc:', int(ephoc), '(%0.2f%%)' % (ephoc % 1 * 100) , 'fetch:', fetch, 'tfetch:', tfetch, 'data/s:', fps.fps()) del tbatchImg, tbatchLabel if(lastEphoc != int(ephoc)): lastEphoc = int(ephoc) ckpt_path = saver.save(sess, testDir + '/model.ckpt', global_step = step) print('chpt saved:', ckpt_path)
def send_Data_Message(self, strData): strMsg, seqNo = self.prepareDematicStructuredData(mType="DATA", strData=strData) self.logger.log("{}: Sending Data message: {}".format( str(self.tId), strMsg)) if self.ackLogger is not None: self.ackLogger.log("{}, {}, Sending Data, {}".format( str(util.getTimeStamp()), str(seqNo), strMsg)) self.sendMessage(message=strMsg) self.logger.log("{}: Pending ack to match: {}".format( str(self.tId), str(len(self.ackList)))) # log the pending acks if any if (len(self.ackList) > 0): self.logger.log("----------------------------------------") for ack in self.ackList: self.logger.log("{} , ".format(ack), newLine=False) self.logger.log("", newLine=True) self.logger.log("----------------------------------------") self.logger.log("", newLine=True)
def __init__(self, seedTask, threadNum=2, parserNum=2, isReRun=False): self.returnVal = True # not implemented now, set it true for crash recover self.isReRun = isReRun self.sign = "manager" # queues for download self.inQueue = Queue.Queue() self.outQueue = Queue.Queue() # queues for parse self.inPQueue = self.outQueue self.outPQueue = Queue.Queue() self.wThreads = [] self.pThreads = [] # controller provides interfaces for human beings self.cThread = None # self.initTasks = [] # self.tasksDone = [] # taskCounter for giving id to each task self.taskCounter = mCounter(0) self.taskDoneCounter = mCounter(0) self.taskFailedCounter = mCounter(0) self.taskIgnoreCounter = mCounter(0) # activeTasks is only a pool for monitoring the tasks self.activeTasks = [] self.myLock = threading.Lock() # init threads and tasks self.__init_wThread_pool(threadNum) self.__init_pThread_pool(parserNum) self.__init_control_thread() # init self.cThread self.init(seedTask) # about running self.timeStampBegin = time.time() self.timeStampStr = util.getTimeStamp() self.shouldExit = False # todo, not implemented self.whatWeHave = {} self.__set_tsOutputPath()
def __init__(self, seedTask, threadNum=2, parserNum=2, isReRun=False): self.returnVal = True # not implemented now, set it true for crash recover self.isReRun = isReRun self.sign = 'manager' # queues for download self.inQueue = Queue.Queue() self.outQueue = Queue.Queue() # queues for parse self.inPQueue = self.outQueue self.outPQueue = Queue.Queue() self.wThreads = [] self.pThreads = [] # controller provides interfaces for human beings self.cThread = None #self.initTasks = [] #self.tasksDone = [] #taskCounter for giving id to each task self.taskCounter = mCounter(0) self.taskDoneCounter = mCounter(0) self.taskFailedCounter = mCounter(0) self.taskIgnoreCounter = mCounter(0) # activeTasks is only a pool for monitoring the tasks self.activeTasks = [] self.myLock = threading.Lock() # init threads and tasks self.__init_wThread_pool(threadNum) self.__init_pThread_pool(parserNum) self.__init_control_thread() # init self.cThread self.init(seedTask) # about running self.timeStampBegin = time.time() self.timeStampStr = util.getTimeStamp() self.shouldExit = False # todo, not implemented self.whatWeHave = {} self.__set_tsOutputPath()
def run(self): data = None dataLen = 0 # See the socket for any rx data, directly call decodeFn() if available, else add to the given queue while not self.stop_thread: try: data, dataLen = self.sockObj.receive_data( expected_data_len=self.exp_data_size) if data is not None: self.logger.log( "{} - {}: data received {} bytes: {}".format( str(util.getTimeStamp()), str(self.tId), str(dataLen), data)) self.qId.put((self.decodeFn, data)) else: time.sleep(0.300) # 300 ms except Exception as e: self.logger.log_exception(e, traceback) # only errors which could not be handled internally are thrown, hence quit now self.stop_thread = True self.logger.log("{} : Exiting thread".format(str(self.tId)))
def process_LIFE_message(self, message): self.totalLifeMsg += 1 currTime = util.getTimeStamp() self.logger.log("{} {}: Processing a LIFE message".format( str(currTime), str(self.tId))) if self.timeLogger is not None: timeDiff = currTime - self.lastLifeAt self.timeLogger.log( "{}: Processing a LIFE message. Diff: {}".format( str(currTime), str(timeDiff))) if (timeDiff > 6): self.timeLogger.log( "**ERROR** Last Life at: {}, Curr Life at: {}, Time Diff: {}" .format(str(self.lastLifeAt), str(currTime), str(timeDiff))) if (timeDiff <= 6.1): self.lifeBelow6_1 += 1 elif (timeDiff <= 6.2): self.lifeBelow6_2 += 1 elif (timeDiff <= 6.5): self.lifeBelow6_5 += 1 else: self.lifeAbove6_5 += 1 else: # Correct Life self.life6AndBelow += 1 self.timeLogger.log( "Total Life: {}, Correct Life: {}, Below 6.1: {}, Below 6.2: {}, Below 6.5: {}, Above 6.5: {}" .format(str(self.totalLifeMsg), str(self.life6AndBelow), str(self.lifeBelow6_1), str(self.lifeBelow6_2), str(self.lifeBelow6_5), str(self.lifeAbove6_5))) self.lastLifeAt = currTime
def processInp(self, sInp): self.logger.log("{}: Handling User input: {}".format( str(self.tId), sInp)) if sInp == "Request_to_UnArm": strData = str( self.getStationId() ) + "MSG052peripheral_groupid01peripheral_type001timestamp" + util.getFormattedTimeStamp( ) self.send_Data_Message(strData=strData) self.msg52TxCtr += 1 if self.msg_51_52_53_Logger is not None: self.msg_51_52_53_Logger.log( "{} Sending MSG052. MSG052 Tx Count: {}, MSG053 Tx Count: {}" .format(str(util.getTimeStamp()), str(self.msg52TxCtr), str(self.msg53TxCtr))) elif sInp == "Request_to_Arm": # Same as Request to move Ranger strData = str( self.getStationId() ) + "MSG053peripheral_groupid01peripheral_type001timestamp" + util.getFormattedTimeStamp( ) self.send_Data_Message(strData=strData) self.msg53TxCtr += 1 if self.msg_51_52_53_Logger is not None: self.msg_51_52_53_Logger.log( "{} Sending MSG053. MSG053 Tx Count: {}, MSG052 Tx Count: {}" .format(str(util.getTimeStamp()), str(self.msg53TxCtr), str(self.msg52TxCtr))) elif sInp == "Peripheral_Emergency_Active": strData = str( self.getStationId() ) + "MSG054peripheral_groupid01peripheral_type001emergency001timestamp" + util.getFormattedTimeStamp( ) self.send_Data_Message(strData=strData) elif sInp == "Peripheral_Emergency_Resolved": strData = str( self.getStationId() ) + "MSG054peripheral_groupid01peripheral_type001emergency000timestamp" + util.getFormattedTimeStamp( ) self.send_Data_Message(strData=strData) # elif sInp == "Stop_Moving_Ranger": # self.send_Data_Message(strData="mover:000") elif sInp == "Request_PLC_Status": strData = str( self.getStationId() ) + "MSG055peripheral_groupid01peripheral_type001timestamp" + util.getFormattedTimeStamp( ) self.send_Data_Message(strData=strData) elif sInp == "System_Armed": strData = str( self.getStationId() ) + "MSG051peripheral_groupid01peripheral_type001armed001timestamp" + util.getFormattedTimeStamp( ) self.send_Data_Message(strData=strData) elif sInp == "System_UnArmed": strData = str( self.getStationId() ) + "MSG051peripheral_groupid01peripheral_type001armed000timestamp" + util.getFormattedTimeStamp( ) self.send_Data_Message(strData=strData) elif sInp == "PLC_Status_Response": strData = str( self.getStationId() ) + "MSG056peripheral_groupid01peripheral_type001armed000emergency001status001timestamp" + util.getFormattedTimeStamp( ) self.send_Data_Message(strData=strData) else: self.logger.log("{}: Wrong User Input: {}".format( str(self.tId), sInp))
def __init__(self, qName, sockObj, logger, keepAliveTime, userOptsEnum, threadName="", **kwargs): self.qId = qName self.sockObj = sockObj self.logger = logger self.tId = threadName self.keepAliveTime = keepAliveTime self.fSendKeepAlive = True self.keepAlivelock = Lock() self.sequenceCtr = int(kwargs.get("seq_start_from", 0)) self.sequenceLock = Lock() self.myTimer = None self.userOptsEnum = userOptsEnum self.validateMessage = kwargs.get("validateMessage", True) self.timeLogger = kwargs.get("timeLogger", None) self.ackLogger = kwargs.get("ackLogger", None) self.msg_51_52_53_Logger = kwargs.get("msg_51_52_53_Logger", None) self.msg_56_Logger = kwargs.get("msg_56_Logger", None) self.msg_54_Logger = kwargs.get("msg_54_Logger", None) self.iAmClient = kwargs.get("iAmClient", False) self.lastLifeAt = util.getTimeStamp() self.ackList = list() self.myLifeCtr = 0 self.msg51TxCtr = 0 self.msg52TxCtr = 0 self.msg53TxCtr = 0 self.msg54TxCtr = 0 self.msg55TxCtr = 0 self.msg56TxCtr = 0 self.msg51RxCtr = 0 self.msg52RxCtr = 0 self.msg53RxCtr = 0 self.msg54RxCtr = 0 self.msg55RxCtr = 0 self.msg56RxCtr = 0 # ----- counters for life timing -- self.totalLifeMsg = 0 self.life6AndBelow = 0 # correct messages which came under 6 secs self.lifeBelow6_1 = 0 # messages which came under 6.1 secs self.lifeBelow6_2 = 0 # messages which came under 6.2 secs self.lifeBelow6_5 = 0 # messages which came under 6.5 secs self.lifeAbove6_5 = 0 # messages which came after 6.5 secs # --------------------------------- self.bSTX = b'\x02' self.bCR = b'\x0D' self.bLF = b'\x0A' self.processing_fn_dict = dict() # callbacks to handle different types of messages self.processing_fn_dict["DATA"] = self.process_DATA_message self.processing_fn_dict["ACKN"] = self.process_ACKN_message self.processing_fn_dict["LIFE"] = self.process_LIFE_message self.processing_fn_dict["STAT"] = self.process_STAT_message self.logger.log("{}: Incoming Message Validation set to: {}".format( str(self.tId), str(self.validateMessage))) self.stopThread = False
def findPutSpreads(ListOfSymbols): #Options criteria MIN_VOLUME = 1 MAX_BID_ASK_SPREAD = .15 MAX_STRIKES_WIDTH = 5 DELTA = -.2 MIN_PREMIUM = .29 data_frame = [] for symbol in ListOfSymbols: print(f"Processing {symbol}...") expirations_list = util.listOfLimitedExpirations(symbol, 21, 47) #Try hard-coded expirations for faster processing #expirations_list = ["2021-02-19"] for expiration in expirations_list: options = api.getOptionsChain(symbol, expiration) prev_option_strike = 0 prev_option_prem = 0 for option_item in options: #Ignore weeklys? if (option_item['expiration_type'] == "weeklys"): break option = util.gatherOptionData(option_item) if (option['bid'] is None): continue #Estimated premium (mid price) premium = round((option['bid'] + option['ask']) / 2, 2) #Figure out net credit from credit spread net_credit = round((premium - prev_option_prem), 2) if ('delta' in option): delta = option['delta'] delta = round(delta, 2) #Criteria here if (option['type'] == "put" and option['bid'] > 0.0 and premium >= MIN_PREMIUM and delta >= DELTA and (option['ask'] - option['bid']) <= MAX_BID_ASK_SPREAD and option['volume'] > MIN_VOLUME): option_output = '{}, {}, BID:{}, ASK:{}, {}, {}(D), Premium: {}'\ .format( option['expiration'], option['strike'], option['bid'], option['ask'], option['volume'], delta, premium) #Mark a strike where the width between the current strike and the previous strike meets the criteria if (net_credit >= MIN_PREMIUM and prev_option_prem > 0 and option['strike'] - prev_option_strike <= MAX_STRIKES_WIDTH): option_output = option_output + " <<<<<< " option_output = option_output + f"{net_credit}" data_frame.append([ symbol, option['expiration'], option['strike'], option['bid'], option['ask'], option['volume'], delta, premium, net_credit, util.getTimeStamp() ]) #Print the screen when a match is found print( f"Found: {option_output} - ({util.getTimeStamp()})" ) if (option['type'] == "put"): prev_option_prem = premium prev_option_strike = option['strike'] panda_files.exportToFile(data_frame, "output_spreads.csv") if (config.REMOTE): panda_files.exportToWeb(data_frame, "output_spreads") panda_files.exportToJson(data_frame, "output_spreads")
file_timestamp + ".txt") msg_56_Logger = util.CustomLogger(log_dest=util.LOG_DEST.FILE, fileName="./client_msg_56_log_" + file_timestamp + ".txt") msg_54_Logger = util.CustomLogger(log_dest=util.LOG_DEST.FILE, fileName="./client_msg_54_log_" + file_timestamp + ".txt") myAckLogger.log("Timestamp, Sequence No., Data Direction, Data") print("------------------------------------------------------") if (len(sys.argv) >= 4): loopTime = float(sys.argv[3]) print("Dummy GOR SERVER (Automated for Sending)") myLogger.log( "\n{} ----------------- NEW SESSION Automated for Sending -----------------" .format(str(util.getTimeStamp()))) else: print("Dummy GOR SERVER with user Options") myLogger.log("\n{} ----------------- NEW SESSION -----------------".format( str(util.getTimeStamp()))) print("------------------------------------------------------") try: connectionRetry = bool(sys.argv[4]) except Exception as e: pass myLogger.log("Connection Retry: {}".format(str(connectionRetry))) myClient = util.SockUtil(util.CONFIG.CLIENT, logger=myLogger,
def findWheels(ListOfSymbols, minDays, maxDays): MAX_BID_ASK_SPREAD = .15 MIN_PRICE = 10 MAX_PRICE = 70 MIN_PREM = .30 MAX_DELTA = -.2 matching_options = [] data_frame = [] for symbol in ListOfSymbols: print(f"Processing {symbol}...") last_price = api.getLastStockPrice(symbol) if (last_price <= MIN_PRICE or last_price >= MAX_PRICE): continue expirations_list = util.listOfLimitedExpirations( symbol, minDays, maxDays) numOptions = 0 for expiration in expirations_list: options = api.getOptionsChain(symbol, expiration) for option_item in options: option = util.gatherOptionData(option_item) if (option['bid'] is None or option['ask'] is None): continue #Estimated premium (mid price) premium = round((option['bid'] + option['ask']) / 2, 2) delta = -999 if ('delta' in option): delta = option['delta'] if (option['type'] == "put" and option['bid'] > 0 and delta >= MAX_DELTA and premium >= MIN_PREM and (option['ask'] - option['bid']) <= MAX_BID_ASK_SPREAD and option['volume'] > 0): option_output = '{}, {}, BID:{}, ASK:{}, {}, {}(D), Premium: {}'\ .format( option['expiration'], option['strike'], option['bid'], option['ask'], option['volume'], delta, premium) if (numOptions == 0): matching_options.append(f"Symbol: {symbol}") numOptions += 1 #Print the screen when a match is found print(f"Wheel: {option_output} - ({util.getTimeStamp()})") data_frame.append([ symbol, option['expiration'], option['strike'], option['bid'], option['ask'], option['volume'], delta, premium, "", util.getTimeStamp() ]) panda_files.exportToFile(data_frame, "output_wheels.csv") if (config.REMOTE): panda_files.exportToWeb(data_frame, "output_wheels") panda_files.exportToJson(data_frame, "output_wheels") return ""
def runTest(testName): m = loadM(fName) output = [] c = 0 with open(testName, "r") as f: reader = csv.reader(f) for line in reader: c += 1 recId = line[0] q = util.yluClean(line[1]) d = util.yluClean(line[2]) wcDict = util.countWordLazy(q + d) tags = predict(m, wcDict, k) output.append([recId, tags]) # if c == 10: # break return output if __name__ == "__main__": timeStmp = util.getTimeStamp() fout = "tag_out%s.txt" % timeStmp out = runTest(testName) with open(fout, "w") as f: for each in out: recId = each[0] tags = " ".join(each[1]) line = "%s\t%s\n" % (recId, tags) print line, f.write(line)