def whatToretrieve(self):
start = datetime.datetime(2018, 5, 1)
start = start.date()
end = datetime.datetime.now()
end = end.date()
allStockID = DBManager.DBManager().getAllStocksIDs()
if allStockID:
for stockID in allStockID:
id = stockID[0]
# Getting the last date already exists
lastDate = DBManager.DBManager().getLastPriceHistoryDateByID(id)
if lastDate:
lastDate = lastDate[0]
if type(lastDate) == 'datetime':
lastDate = lastDate.date()
# Retrieves stock price history
pricesHistory = False
if lastDate == None: # No data
pricesHistory = self.retrieveStockPriceHistory(id, start, end)
elif lastDate >= start: # There is some date
lastDate = lastDate + datetime.timedelta(days=1)
if lastDate < end:
pricesHistory = self.retrieveStockPriceHistory(id, lastDate, end)
elif lastDate == datetime.now(): # The date is up to date
pass
if not (isinstance(pricesHistory, bool) and pricesHistory == False):
self.addPriceHistoryToDB(id, pricesHistory)
else:
print("PandasDatareder Error In whatToretrieve.")
def getAllMessagesByUserID(self):
# Receiving the arg of request
arg = pickle.loads(self.clientsocket.recv(vocabulary.BUFSIZE))
userID = arg
allPredictionIDs = DBManager.DBManager().getAllPredictionsIDByUserID(
userID)
allPredictions = []
for predID in allPredictionIDs:
predDateRec = DBManager.DBManager().getPredictionByID(predID[0])[0]
stockID = predDateRec[0]
nameSymbol = DBManager.DBManager().getNameAndSymbolByID(stockID)
allPredictions.append(
(nameSymbol[0], nameSymbol[1], predDateRec[1], predDateRec[2]))
respons = tuple(allPredictions)
if respons:
responsPickled = pickle.dumps(respons)
size = sys.getsizeof(responsPickled)
# Sent size and respons
self.clientsocket.send(pickle.dumps(size))
self.clientsocket.send(responsPickled)
else:
# Sent error
self.clientsocket.send(pickle.dumps(vocabulary.ERROR))
def linearRegressionAndRANSACYsAndR2(self, y, X, xForPrediction, id):
# Get name and symbol
nameSymbol = DBManager.DBManager().getNameAndSymbolByID(id)
name = nameSymbol[0]
symbol = nameSymbol[1]
print(name, symbol, ":")
X = np.array(X)
y = np.array(y)
xForPrediction = np.array(xForPrediction)[:,np.newaxis]
# Linear Regression with OLS algorithm
linearReg = linear_model.LinearRegression()
linearReg.fit(X, y)
lineYLinearRegression = linearReg.predict(X)
linearR2 = linearReg.score(X,y)
print("Linear R^2=", linearR2)
linearRegressionY = linearReg.predict(xForPrediction)
# Robust linear regression with RANSAC algorithm
ransac = linear_model.RANSACRegressor()
ransac.fit(X, y)
inlier_mask = ransac.inlier_mask_
outlier_mask = np.logical_not(inlier_mask)
lineYRansac = ransac.predict(X)
ransacR2 = ransac.score(X[inlier_mask], y[inlier_mask])
print("RANSAC R^2=", ransacR2)
ransacY = ransac.predict(xForPrediction)
print("Preditcion: \n\tLinear= %f \n\tRANSAC= %f" % (linearRegressionY, ransacY))
return (linearRegressionY, ransacY, linearR2, ransacR2)
def sendEmail(self, user, pwd, recipient, subject, body):
FROM = user
TO = recipient if isinstance(recipient, list) else [recipient]
SUBJECT = subject
nameSymbol = DBManager.DBManager().getNameAndSymbolByID(body[1])
TEXT = "Prediction for %s(%s) stock, to %s, is: %s, with %f%% of accuracy." % (
nameSymbol[0], nameSymbol[0], str(
body[2].date()), body[5], body[6])
# Prepare actual message
message = """From: %s\nTo: %s\nSubject: %s\n\n%s
""" % (FROM, ", ".join(TO), SUBJECT, TEXT)
try:
server = smtplib.SMTP("smtp.gmail.com", 587)
server.ehlo()
server.starttls()
server.login(user, pwd)
server.sendmail(FROM, TO, message)
server.close()
print("Email successfully sent to %s, on %s (%s) stock." %
(recipient, nameSymbol[0], nameSymbol[0]))
except BaseException as e:
print("e:", e)
print("failed to send mail")
def getStockIDBySymbol(self):
# Receiving the arg of request
symbol = pickle.loads(self.clientsocket.recv(vocabulary.BUFSIZE))
respons = DBManager.DBManager().getStockIDBySymbol(symbol)
if respons:
self.clientsocket.send(pickle.dumps(respons))
else:
self.clientsocket.send(pickle.dumps(vocabulary.ERROR))
def getFullNameByID(self):
# Receiving the arg of request
id = pickle.loads(self.clientsocket.recv(vocabulary.BUFSIZE))
respons = DBManager.DBManager().getFullNameByID(id)
if respons:
self.clientsocket.send(pickle.dumps(respons))
else:
self.clientsocket.send(pickle.dumps(vocabulary.ERROR))
def signIn(self):
# Receiving the arg of request
recvArg = pickle.loads(self.clientsocket.recv(vocabulary.BUFSIZE))
respons = DBManager.DBManager().signIn(recvArg[0], recvArg[1])
if respons:
self.clientsocket.send(pickle.dumps(vocabulary.OK))
else:
self.clientsocket.send(pickle.dumps(vocabulary.SIGNIN_FAIL))
def setIsConnect(self):
# Receiving the arg of request
recvArg = pickle.loads(self.clientsocket.recv(vocabulary.BUFSIZE))
respons = DBManager.DBManager().setIsConnect(recvArg[0], recvArg[1])
if respons:
self.clientsocket.send(pickle.dumps(vocabulary.OK))
else:
self.clientsocket.send(pickle.dumps(
vocabulary.SET_IS_CONNECT_FAIL))
def signUp(self):
# Receiving the arg of request
recvArg = pickle.loads(self.clientsocket.recv(vocabulary.BUFSIZE))
respons = DBManager.DBManager().addNewUser(recvArg[0], recvArg[1],
recvArg[2], recvArg[3],
recvArg[4])
if respons:
self.clientsocket.send(pickle.dumps(vocabulary.OK))
else:
self.clientsocket.send(pickle.dumps(vocabulary.USER_EXIST))
def prediction(self, shiftDays):
today = datetime.datetime.now()
if self.isBusinessDay(today):
# Get the previous shift bussines day
prevShiftBD = self.getPreviousShiftBusinessDay(today, shiftDays)
# Get all stocks IDs
allStocksIDs = DBManager.DBManager().getAllStocksIDs()
for stockID in allStocksIDs:
stockID = stockID[0]
# If not exsist prediction
if not DBManager.DBManager().isExsistPredictionByIDAndDate(stockID, today.date()):
# Get changeClose and avgSentiment lists and x for
# prediction
changeCloseAvgSentimentListAndX = self.getChangeCloseAndAvgSentimentListAndX(stockID, shiftDays)
changCloseList = changeCloseAvgSentimentListAndX[0]
avgSentimentList = changeCloseAvgSentimentListAndX[1]
xForPrediction = changeCloseAvgSentimentListAndX[2]
# Do regression
if not (len(changCloseList) == 0 or len(avgSentimentList) == 0 or xForPrediction == None):
linearRegressionAndRANSACYsAndR2 = self.linearRegressionAndRANSACYsAndR2(changCloseList, avgSentimentList, xForPrediction, stockID)
linearRegressionY = linearRegressionAndRANSACYsAndR2[0]
RANSACY = linearRegressionAndRANSACYsAndR2[1]
linearR2 = linearRegressionAndRANSACYsAndR2[2]
ransacR2 = linearRegressionAndRANSACYsAndR2[3]
# Create prediction
# Calculate recommendation
recommendation = 'SAME'
if linearRegressionY > 1 and RANSACY > 1:
recommendation = 'UP'
elif linearRegressionY < 1 and RANSACY < 1:
recommendation = 'DUWN'
# Calculate accuracy
accuracy = (linearR2 + ransacR2) / 2
# Add prediction to DB
DBManager.DBManager().addPrediction(stockID, today, linearRegressionY, RANSACY, recommendation, accuracy)
def send(self):
# Get all stocks id
allStocksIDs = DBManager.DBManager().getAllStocksIDs()
for stockID in allStocksIDs:
stockID = stockID[0]
# Just if exsist prediction of stock id today
if DBManager.DBManager().isExsistPredictionByIDAndDate(
stockID,
datetime.datetime.now().date()):
# Get prediction details
prediction = DBManager.DBManager().getPredictionByIDAndDate(
stockID,
datetime.datetime.now().date())
# Get all users that register to this stock
allRegistered = DBManager.DBManager(
).getAllRegisterToStockByID(stockID)
if allRegistered:
for userID in allRegistered:
userID = userID[0]
# Add user prediction to DB
DBManager.DBManager().addUserPrediction(
userID, prediction[0][0])
# prepare parameters to email
toEmail = DBManager.DBManager().getEmailByID(userID)
subject = 'Sentiment Editor- the Holy Grail'
self.readEmailKeys()
# Sending Email
self.sendEmail(self.email, self.password,
toEmail[0][0], subject, prediction[0])
def whatToAnalyze(self):
# Receives (one) text and id of tweet for analysis,
# if there is a tweet that has not yet been analyzed.
textAndIds = DBManager.DBManager().getTextAndIdsOfTweetToAnalyze()
while textAndIds:
text = textAndIds[0]
id = textAndIds[1]
stockID = textAndIds[2]
nameSymbol = DBManager.DBManager().getNameAndSymbolByID(stockID)
name = nameSymbol[0]
symbol = nameSymbol[1]
# Send the text for analysis
textAnalyzed = self.analyze(text, name, symbol)
if textAnalyzed:
DBManager.DBManager().addAnalyzeToTweet(textAnalyzed, id)
textAndIds = DBManager.DBManager().getTextAndIdsOfTweetToAnalyze()
else:
print("There is no text to analyze.")
def retrieveStockPriceHistory(self, id, start, end):
# Get symbol by id
symbol = DBManager.DBManager().getSymbolByID(id)
if symbol:
symbol = symbol[0]
try:
results = web.DataReader(symbol, 'morningstar', start, end)
except BaseException as e:
print("Error in retrieveStockPriceHistory: ", e)
return False
print(results)
return results
def getAllStocksIDs(self):
respons = DBManager.DBManager().getAllStocksIDs()
if respons:
responsPickled = pickle.dumps(respons)
size = sys.getsizeof(responsPickled)
# Sent size and respons
self.clientsocket.send(pickle.dumps(size))
self.clientsocket.send(responsPickled)
else:
# Sent error
self.clientsocket.send(pickle.dumps(vocabulary.ERROR))
def getChangeCloseAndAvgSentimentListAndX(self, id, shift):
# Get close prices dict
closeDict = DBManager.DBManager().getDateCloseDictById(id)
# Create changeClose from close
closeList = list(closeDict.items())
changeCloseListTemp = []
for i in range(len(closeList)):
if i != 0:
changeCloseListTemp.append((closeList[i][0], closeList[i][1] / closeList[i - 1][1])) #(key, value)
changeCloseDict = dict(changeCloseListTemp)
# Get avg sentiment
avgSentiment = DBManager.DBManager().getAvgSentimentByStockID(id)
# Create changeClose & avgSentiment lists with same dates entrys
closeKeys = changeCloseDict.keys()
avgSentimentKeys = avgSentiment.keys()
changeCloseList = []
avgSentimentList = []
for date in avgSentimentKeys:
if date in closeKeys:
changeCloseList.append([changeCloseDict[date]])
avgSentimentList.append([avgSentiment[date]])
# Do shift between changeClose and avgSentiment
avgSentimentX = None
if len(changeCloseList) >= shift and len(avgSentimentList) >= shift:
for i in range(shift):
if i == shift - 1:
avgSentimentX = avgSentimentList[len(avgSentimentList) - 1]
del changeCloseList[0]
del avgSentimentList[len(avgSentimentList) - 1]
return [changeCloseList, avgSentimentList, avgSentimentX]
def getStockByID(self):
# Receiving the arg of request
stockID = pickle.loads(self.clientsocket.recv(vocabulary.BUFSIZE))
respons = DBManager.DBManager().getStockByID(stockID)
if respons:
responsPickled = pickle.dumps(respons)
size = sys.getsizeof(responsPickled)
# Sent size and respons
self.clientsocket.send(pickle.dumps(size))
self.clientsocket.send(responsPickled)
else:
# Sent error
self.clientsocket.send(pickle.dumps(vocabulary.ERROR))
def addStockToUser(self):
# Receiving the arg of request
arg = pickle.loads(self.clientsocket.recv(vocabulary.BUFSIZE))
userID = arg[0]
stockID = arg[1]
respons = DBManager.DBManager().addStockToUser(userID, stockID)
if respons:
responsPickled = pickle.dumps(respons)
size = sys.getsizeof(responsPickled)
# Sent size and respons
self.clientsocket.send(pickle.dumps(size))
self.clientsocket.send(responsPickled)
else:
# Sent error
self.clientsocket.send(pickle.dumps(vocabulary.ERROR))
def __init__(self):
# Runing Server for clients requests on new thread
s = serverNetwork.serverNetwork()
s.start()
while True:
## Fill DB
#self.fillDB()
## Do prediction
#prediction.prediction()
## Send messages to client
#sendMessages.sendMessages()
# Sleep one day
print("Server (of prediction) sleep until %s"%str(datetime.datetime.now() + timedelta(days=1)))
time.sleep(86400) #86400 sec is a day
# Close resources
DBManager.DBManager().db.close()
def addPriceHistoryToDB(self, id, priceHistory):
DBManager.DBManager().addPriceHistoryDataframe(id, priceHistory)