def processData(self, strData):
self.logger.log("{}: Processing data: {}".format(
str(self.tId), strData))
# client related actions
if ("MSG051" in strData):
self.msg51RxCtr += 1
if self.msg_51_52_53_Logger is not None:
self.msg_51_52_53_Logger.log(
"{} Received MSG051. Rx ctr: {}".format(
str(util.getTimeStamp()), str(self.msg51RxCtr)))
elif ("MSG056" in strData):
self.msg56RxCtr += 1
if self.msg_56_Logger is not None:
self.msg_56_Logger.log("{} Received MSG056. Rx ctr: {}".format(
str(util.getTimeStamp()), str(self.msg56RxCtr)))
elif ("MSG054" in strData):
self.msg54RxCtr += 1
if self.msg_54_Logger is not None:
self.msg_54_Logger.log("{} Received MSG054. Rx ctr: {}".format(
str(util.getTimeStamp()), str(self.msg54RxCtr)))
# dummy plc (server) related actions
elif ("MSG052" in strData):
self.processInp(sInp="System_UnArmed")
elif ("MSG053" in strData):
self.processInp(sInp="System_Armed")
def process_Rx_Message(self, message):
self.logger.log("{} {}: Processing message: {}".format(
str(util.getTimeStamp()), str(self.tId), str(message)))
try:
if ((not self.validateMessage)
or (self.validateMsg(sMessage=message))):
for mType, fnPtr in self.processing_fn_dict.items():
if mType in message:
fnPtr(message)
break
elif (not self.validateMsg(sMessage=message)):
# check if multiple messages are received ?
count = message.count(str(self.bSTX))
if count > 1:
# we have received multiple messages
self.logger.log(
"{}: Multiple ({}) Messages Received".format(
str(self.tId), str(count)))
searchMsg = message
while (searchMsg.find(str(self.bSTX)) != -1):
# starts with STX
partSize = int(searchMsg[1:5])
partMessage = searchMsg[:partSize]
self.process_Rx_Message(message=partMessage)
searchMsg = searchMsg[partSize:]
else:
self.logger.log(
"{}: Invalid message received, ignoring".format(
str(self.tId)))
except Exception as e:
self.logger.log_exception(e, traceback)
def freeze(self):
gd = self.sess.graph.as_graph_def()
print("convt..")
for node in gd.node:
if node.op == 'RefSwitch':
node.op = 'Switch'
for index in range(len(node.input)):
if 'moving_' in node.input[index]:
node.input[index] = node.input[index] + '/read'
elif node.op == 'AssignSub':
node.op = 'Sub'
if 'use_locking' in node.attr: del node.attr['use_locking']
print("const...")
gd = graph_util.convert_variables_to_constants(self.sess, gd, self.outputNodes)
optlib.ensure_graph_is_valid(gd)
input_node_names = self.inputNodes
output_node_names = self.outputNodes
placeholder_type_enum = self.inputNodesTypes
for i in range(len(placeholder_type_enum)):
placeholder_type_enum[i] = placeholder_type_enum[i].as_datatype_enum
print("strip...")
gd = strip_unused_lib.strip_unused(gd, input_node_names, output_node_names, placeholder_type_enum)
optlib.ensure_graph_is_valid(gd)
filename = 'frozen ' + util.getTimeStamp() + '.pb'
tf.train.write_graph(gd, self.parentPath, filename, as_text=False)
return os.path.join(self.parentPath, filename)
def process_ACKN_message(self, message):
self.logger.log("{}: Processing an ACKN message".format(str(self.tId)))
ackNo = message[9:]
ackNo = ackNo[:-2]
if self.ackLogger is not None:
self.ackLogger.log("{}, {}, Received ACKN, {}".format(
str(util.getTimeStamp()), str(int(ackNo)), message))
if ackNo in self.ackList:
self.logger.log("{}: ACKN sequence ({}) matching".format(
str(self.tId), str(ackNo)))
self.ackList.remove(ackNo)
else:
self.logger.log(
"{}: ERROR -- ACKN sequence ({}) not matching".format(
str(self.tId), str(ackNo)))
def main():
batchCount = 128
lastEphoc = 0
step = 0
fps = util.FpsCounter()
timestamp = util.getTimeStamp()
testDir = './temp/blink-test ' + timestamp
data = Dataset()
transfer = TransferHelper()
model = Model(dataSize = data.count, batchSize = batchCount)
saver = tf.train.Saver()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config = config) as sess:
sess.run(tf.global_variables_initializer())
transfer.pretrain( \
sess, model.input, [32, 32, 3], model.outputCNN,
phase_train = model.phase_train,
keep_prob = model.keep_prob,
maxEphoc = 500,
targetAcc = 1,
dropRate = model.getDropRate(),
testDropRate = model.getTestDropRate())
transfer.close()
while True:
step += 1
ephoc = step * batchCount / data.count
fps.add(batchCount)
batchImg, batchLabel = data.batch(batchCount)
fetch = model.optimize(sess, batchImg, batchLabel)
del batchImg, batchLabel
if(step % 10 == 0):
tbatchImg, tbatchLabel = data.batch(batchCount, False)
tfetch = model.forward(sess, tbatchImg, tbatchLabel)
print('step:', step, 'ephoc:', int(ephoc), '(%0.2f%%)' % (ephoc % 1 * 100) , 'fetch:', fetch, 'tfetch:', tfetch, 'data/s:', fps.fps())
del tbatchImg, tbatchLabel
if(lastEphoc != int(ephoc)):
lastEphoc = int(ephoc)
ckpt_path = saver.save(sess, testDir + '/model.ckpt', global_step = step)
print('chpt saved:', ckpt_path)
def send_Data_Message(self, strData):
strMsg, seqNo = self.prepareDematicStructuredData(mType="DATA",
strData=strData)
self.logger.log("{}: Sending Data message: {}".format(
str(self.tId), strMsg))
if self.ackLogger is not None:
self.ackLogger.log("{}, {}, Sending Data, {}".format(
str(util.getTimeStamp()), str(seqNo), strMsg))
self.sendMessage(message=strMsg)
self.logger.log("{}: Pending ack to match: {}".format(
str(self.tId), str(len(self.ackList))))
# log the pending acks if any
if (len(self.ackList) > 0):
self.logger.log("----------------------------------------")
for ack in self.ackList:
self.logger.log("{} , ".format(ack), newLine=False)
self.logger.log("", newLine=True)
self.logger.log("----------------------------------------")
self.logger.log("", newLine=True)
def __init__(self, seedTask, threadNum=2, parserNum=2, isReRun=False):
self.returnVal = True
# not implemented now, set it true for crash recover
self.isReRun = isReRun
self.sign = "manager"
# queues for download
self.inQueue = Queue.Queue()
self.outQueue = Queue.Queue()
# queues for parse
self.inPQueue = self.outQueue
self.outPQueue = Queue.Queue()
self.wThreads = []
self.pThreads = []
# controller provides interfaces for human beings
self.cThread = None
# self.initTasks = []
# self.tasksDone = []
# taskCounter for giving id to each task
self.taskCounter = mCounter(0)
self.taskDoneCounter = mCounter(0)
self.taskFailedCounter = mCounter(0)
self.taskIgnoreCounter = mCounter(0)
# activeTasks is only a pool for monitoring the tasks
self.activeTasks = []
self.myLock = threading.Lock()
# init threads and tasks
self.__init_wThread_pool(threadNum)
self.__init_pThread_pool(parserNum)
self.__init_control_thread() # init self.cThread
self.init(seedTask)
# about running
self.timeStampBegin = time.time()
self.timeStampStr = util.getTimeStamp()
self.shouldExit = False
# todo, not implemented
self.whatWeHave = {}
self.__set_tsOutputPath()
def __init__(self, seedTask, threadNum=2, parserNum=2, isReRun=False):
self.returnVal = True
# not implemented now, set it true for crash recover
self.isReRun = isReRun
self.sign = 'manager'
# queues for download
self.inQueue = Queue.Queue()
self.outQueue = Queue.Queue()
# queues for parse
self.inPQueue = self.outQueue
self.outPQueue = Queue.Queue()
self.wThreads = []
self.pThreads = []
# controller provides interfaces for human beings
self.cThread = None
#self.initTasks = []
#self.tasksDone = []
#taskCounter for giving id to each task
self.taskCounter = mCounter(0)
self.taskDoneCounter = mCounter(0)
self.taskFailedCounter = mCounter(0)
self.taskIgnoreCounter = mCounter(0)
# activeTasks is only a pool for monitoring the tasks
self.activeTasks = []
self.myLock = threading.Lock()
# init threads and tasks
self.__init_wThread_pool(threadNum)
self.__init_pThread_pool(parserNum)
self.__init_control_thread() # init self.cThread
self.init(seedTask)
# about running
self.timeStampBegin = time.time()
self.timeStampStr = util.getTimeStamp()
self.shouldExit = False
# todo, not implemented
self.whatWeHave = {}
self.__set_tsOutputPath()
def run(self):
data = None
dataLen = 0
# See the socket for any rx data, directly call decodeFn() if available, else add to the given queue
while not self.stop_thread:
try:
data, dataLen = self.sockObj.receive_data(
expected_data_len=self.exp_data_size)
if data is not None:
self.logger.log(
"{} - {}: data received {} bytes: {}".format(
str(util.getTimeStamp()), str(self.tId),
str(dataLen), data))
self.qId.put((self.decodeFn, data))
else:
time.sleep(0.300) # 300 ms
except Exception as e:
self.logger.log_exception(e, traceback)
# only errors which could not be handled internally are thrown, hence quit now
self.stop_thread = True
self.logger.log("{} : Exiting thread".format(str(self.tId)))
def process_LIFE_message(self, message):
self.totalLifeMsg += 1
currTime = util.getTimeStamp()
self.logger.log("{} {}: Processing a LIFE message".format(
str(currTime), str(self.tId)))
if self.timeLogger is not None:
timeDiff = currTime - self.lastLifeAt
self.timeLogger.log(
"{}: Processing a LIFE message. Diff: {}".format(
str(currTime), str(timeDiff)))
if (timeDiff > 6):
self.timeLogger.log(
"**ERROR** Last Life at: {}, Curr Life at: {}, Time Diff: {}"
.format(str(self.lastLifeAt), str(currTime),
str(timeDiff)))
if (timeDiff <= 6.1):
self.lifeBelow6_1 += 1
elif (timeDiff <= 6.2):
self.lifeBelow6_2 += 1
elif (timeDiff <= 6.5):
self.lifeBelow6_5 += 1
else:
self.lifeAbove6_5 += 1
else:
# Correct Life
self.life6AndBelow += 1
self.timeLogger.log(
"Total Life: {}, Correct Life: {}, Below 6.1: {}, Below 6.2: {}, Below 6.5: {}, Above 6.5: {}"
.format(str(self.totalLifeMsg), str(self.life6AndBelow),
str(self.lifeBelow6_1), str(self.lifeBelow6_2),
str(self.lifeBelow6_5), str(self.lifeAbove6_5)))
self.lastLifeAt = currTime
def processInp(self, sInp):
self.logger.log("{}: Handling User input: {}".format(
str(self.tId), sInp))
if sInp == "Request_to_UnArm":
strData = str(
self.getStationId()
) + "MSG052peripheral_groupid01peripheral_type001timestamp" + util.getFormattedTimeStamp(
)
self.send_Data_Message(strData=strData)
self.msg52TxCtr += 1
if self.msg_51_52_53_Logger is not None:
self.msg_51_52_53_Logger.log(
"{} Sending MSG052. MSG052 Tx Count: {}, MSG053 Tx Count: {}"
.format(str(util.getTimeStamp()), str(self.msg52TxCtr),
str(self.msg53TxCtr)))
elif sInp == "Request_to_Arm": # Same as Request to move Ranger
strData = str(
self.getStationId()
) + "MSG053peripheral_groupid01peripheral_type001timestamp" + util.getFormattedTimeStamp(
)
self.send_Data_Message(strData=strData)
self.msg53TxCtr += 1
if self.msg_51_52_53_Logger is not None:
self.msg_51_52_53_Logger.log(
"{} Sending MSG053. MSG053 Tx Count: {}, MSG052 Tx Count: {}"
.format(str(util.getTimeStamp()), str(self.msg53TxCtr),
str(self.msg52TxCtr)))
elif sInp == "Peripheral_Emergency_Active":
strData = str(
self.getStationId()
) + "MSG054peripheral_groupid01peripheral_type001emergency001timestamp" + util.getFormattedTimeStamp(
)
self.send_Data_Message(strData=strData)
elif sInp == "Peripheral_Emergency_Resolved":
strData = str(
self.getStationId()
) + "MSG054peripheral_groupid01peripheral_type001emergency000timestamp" + util.getFormattedTimeStamp(
)
self.send_Data_Message(strData=strData)
# elif sInp == "Stop_Moving_Ranger":
# self.send_Data_Message(strData="mover:000")
elif sInp == "Request_PLC_Status":
strData = str(
self.getStationId()
) + "MSG055peripheral_groupid01peripheral_type001timestamp" + util.getFormattedTimeStamp(
)
self.send_Data_Message(strData=strData)
elif sInp == "System_Armed":
strData = str(
self.getStationId()
) + "MSG051peripheral_groupid01peripheral_type001armed001timestamp" + util.getFormattedTimeStamp(
)
self.send_Data_Message(strData=strData)
elif sInp == "System_UnArmed":
strData = str(
self.getStationId()
) + "MSG051peripheral_groupid01peripheral_type001armed000timestamp" + util.getFormattedTimeStamp(
)
self.send_Data_Message(strData=strData)
elif sInp == "PLC_Status_Response":
strData = str(
self.getStationId()
) + "MSG056peripheral_groupid01peripheral_type001armed000emergency001status001timestamp" + util.getFormattedTimeStamp(
)
self.send_Data_Message(strData=strData)
else:
self.logger.log("{}: Wrong User Input: {}".format(
str(self.tId), sInp))
def __init__(self,
qName,
sockObj,
logger,
keepAliveTime,
userOptsEnum,
threadName="",
**kwargs):
self.qId = qName
self.sockObj = sockObj
self.logger = logger
self.tId = threadName
self.keepAliveTime = keepAliveTime
self.fSendKeepAlive = True
self.keepAlivelock = Lock()
self.sequenceCtr = int(kwargs.get("seq_start_from", 0))
self.sequenceLock = Lock()
self.myTimer = None
self.userOptsEnum = userOptsEnum
self.validateMessage = kwargs.get("validateMessage", True)
self.timeLogger = kwargs.get("timeLogger", None)
self.ackLogger = kwargs.get("ackLogger", None)
self.msg_51_52_53_Logger = kwargs.get("msg_51_52_53_Logger", None)
self.msg_56_Logger = kwargs.get("msg_56_Logger", None)
self.msg_54_Logger = kwargs.get("msg_54_Logger", None)
self.iAmClient = kwargs.get("iAmClient", False)
self.lastLifeAt = util.getTimeStamp()
self.ackList = list()
self.myLifeCtr = 0
self.msg51TxCtr = 0
self.msg52TxCtr = 0
self.msg53TxCtr = 0
self.msg54TxCtr = 0
self.msg55TxCtr = 0
self.msg56TxCtr = 0
self.msg51RxCtr = 0
self.msg52RxCtr = 0
self.msg53RxCtr = 0
self.msg54RxCtr = 0
self.msg55RxCtr = 0
self.msg56RxCtr = 0
# ----- counters for life timing --
self.totalLifeMsg = 0
self.life6AndBelow = 0 # correct messages which came under 6 secs
self.lifeBelow6_1 = 0 # messages which came under 6.1 secs
self.lifeBelow6_2 = 0 # messages which came under 6.2 secs
self.lifeBelow6_5 = 0 # messages which came under 6.5 secs
self.lifeAbove6_5 = 0 # messages which came after 6.5 secs
# ---------------------------------
self.bSTX = b'\x02'
self.bCR = b'\x0D'
self.bLF = b'\x0A'
self.processing_fn_dict = dict()
# callbacks to handle different types of messages
self.processing_fn_dict["DATA"] = self.process_DATA_message
self.processing_fn_dict["ACKN"] = self.process_ACKN_message
self.processing_fn_dict["LIFE"] = self.process_LIFE_message
self.processing_fn_dict["STAT"] = self.process_STAT_message
self.logger.log("{}: Incoming Message Validation set to: {}".format(
str(self.tId), str(self.validateMessage)))
self.stopThread = False
def findPutSpreads(ListOfSymbols):
#Options criteria
MIN_VOLUME = 1
MAX_BID_ASK_SPREAD = .15
MAX_STRIKES_WIDTH = 5
DELTA = -.2
MIN_PREMIUM = .29
data_frame = []
for symbol in ListOfSymbols:
print(f"Processing {symbol}...")
expirations_list = util.listOfLimitedExpirations(symbol, 21, 47)
#Try hard-coded expirations for faster processing
#expirations_list = ["2021-02-19"]
for expiration in expirations_list:
options = api.getOptionsChain(symbol, expiration)
prev_option_strike = 0
prev_option_prem = 0
for option_item in options:
#Ignore weeklys?
if (option_item['expiration_type'] == "weeklys"):
break
option = util.gatherOptionData(option_item)
if (option['bid'] is None):
continue
#Estimated premium (mid price)
premium = round((option['bid'] + option['ask']) / 2, 2)
#Figure out net credit from credit spread
net_credit = round((premium - prev_option_prem), 2)
if ('delta' in option):
delta = option['delta']
delta = round(delta, 2)
#Criteria here
if (option['type'] == "put" and option['bid'] > 0.0
and premium >= MIN_PREMIUM and delta >= DELTA and
(option['ask'] - option['bid']) <= MAX_BID_ASK_SPREAD
and option['volume'] > MIN_VOLUME):
option_output = '{}, {}, BID:{}, ASK:{}, {}, {}(D), Premium: {}'\
.format(
option['expiration'],
option['strike'],
option['bid'],
option['ask'],
option['volume'],
delta,
premium)
#Mark a strike where the width between the current strike and the previous strike meets the criteria
if (net_credit >= MIN_PREMIUM and prev_option_prem > 0
and option['strike'] - prev_option_strike <=
MAX_STRIKES_WIDTH):
option_output = option_output + " <<<<<< "
option_output = option_output + f"{net_credit}"
data_frame.append([
symbol, option['expiration'], option['strike'],
option['bid'], option['ask'], option['volume'],
delta, premium, net_credit,
util.getTimeStamp()
])
#Print the screen when a match is found
print(
f"Found: {option_output} - ({util.getTimeStamp()})"
)
if (option['type'] == "put"):
prev_option_prem = premium
prev_option_strike = option['strike']
panda_files.exportToFile(data_frame, "output_spreads.csv")
if (config.REMOTE):
panda_files.exportToWeb(data_frame, "output_spreads")
panda_files.exportToJson(data_frame, "output_spreads")
file_timestamp + ".txt")
msg_56_Logger = util.CustomLogger(log_dest=util.LOG_DEST.FILE,
fileName="./client_msg_56_log_" +
file_timestamp + ".txt")
msg_54_Logger = util.CustomLogger(log_dest=util.LOG_DEST.FILE,
fileName="./client_msg_54_log_" +
file_timestamp + ".txt")
myAckLogger.log("Timestamp, Sequence No., Data Direction, Data")
print("------------------------------------------------------")
if (len(sys.argv) >= 4):
loopTime = float(sys.argv[3])
print("Dummy GOR SERVER (Automated for Sending)")
myLogger.log(
"\n{} ----------------- NEW SESSION Automated for Sending -----------------"
.format(str(util.getTimeStamp())))
else:
print("Dummy GOR SERVER with user Options")
myLogger.log("\n{} ----------------- NEW SESSION -----------------".format(
str(util.getTimeStamp())))
print("------------------------------------------------------")
try:
connectionRetry = bool(sys.argv[4])
except Exception as e:
pass
myLogger.log("Connection Retry: {}".format(str(connectionRetry)))
myClient = util.SockUtil(util.CONFIG.CLIENT,
logger=myLogger,
def findWheels(ListOfSymbols, minDays, maxDays):
MAX_BID_ASK_SPREAD = .15
MIN_PRICE = 10
MAX_PRICE = 70
MIN_PREM = .30
MAX_DELTA = -.2
matching_options = []
data_frame = []
for symbol in ListOfSymbols:
print(f"Processing {symbol}...")
last_price = api.getLastStockPrice(symbol)
if (last_price <= MIN_PRICE or last_price >= MAX_PRICE):
continue
expirations_list = util.listOfLimitedExpirations(
symbol, minDays, maxDays)
numOptions = 0
for expiration in expirations_list:
options = api.getOptionsChain(symbol, expiration)
for option_item in options:
option = util.gatherOptionData(option_item)
if (option['bid'] is None or option['ask'] is None):
continue
#Estimated premium (mid price)
premium = round((option['bid'] + option['ask']) / 2, 2)
delta = -999
if ('delta' in option):
delta = option['delta']
if (option['type'] == "put" and option['bid'] > 0
and delta >= MAX_DELTA and premium >= MIN_PREM and
(option['ask'] - option['bid']) <= MAX_BID_ASK_SPREAD
and option['volume'] > 0):
option_output = '{}, {}, BID:{}, ASK:{}, {}, {}(D), Premium: {}'\
.format(
option['expiration'],
option['strike'],
option['bid'],
option['ask'],
option['volume'],
delta,
premium)
if (numOptions == 0):
matching_options.append(f"Symbol: {symbol}")
numOptions += 1
#Print the screen when a match is found
print(f"Wheel: {option_output} - ({util.getTimeStamp()})")
data_frame.append([
symbol, option['expiration'], option['strike'],
option['bid'], option['ask'], option['volume'], delta,
premium, "",
util.getTimeStamp()
])
panda_files.exportToFile(data_frame, "output_wheels.csv")
if (config.REMOTE):
panda_files.exportToWeb(data_frame, "output_wheels")
panda_files.exportToJson(data_frame, "output_wheels")
return ""
def runTest(testName):
m = loadM(fName)
output = []
c = 0
with open(testName, "r") as f:
reader = csv.reader(f)
for line in reader:
c += 1
recId = line[0]
q = util.yluClean(line[1])
d = util.yluClean(line[2])
wcDict = util.countWordLazy(q + d)
tags = predict(m, wcDict, k)
output.append([recId, tags])
# if c == 10:
# break
return output
if __name__ == "__main__":
timeStmp = util.getTimeStamp()
fout = "tag_out%s.txt" % timeStmp
out = runTest(testName)
with open(fout, "w") as f:
for each in out:
recId = each[0]
tags = " ".join(each[1])
line = "%s\t%s\n" % (recId, tags)
print line,
f.write(line)
