def cleanData():
pfGroupArray=stockUtil.evalTextArray(stockUtil.read_config("portfolio","pfGroupArray"))
allPfArray=[] #collect all portfolio to allPfArray
for pfGroup in pfGroupArray:
qry = Portfolio.select().where(Portfolio.group == pfGroup).order_by(Portfolio.index)
for item in qry:
stockArray = stockUtil.evalTextArray(item.stock_array)
for stock in stockArray:
allPfArray.append(stock)
#print(allPfArray)
#Delete the history data according to stockid not in portfolio.
for pfGroup in pfGroupArray:
setattr(HistoryData._meta, "db_table", "history_" + pfGroup.lower())
data1=HistoryData.select(HistoryData.stockid).distinct().dicts()
for x in data1: #pick up stockid of historydata table
founded=0
#print(x["stockid"])
for y in allPfArray: #look up portfolio
if pfGroup == y["marketType"] and x["stockid"] == y["stockId"] :
founded=1
break;
if founded==0:
q = HistoryData.delete().where(HistoryData.stockid == x["stockid"])
q.execute()
#Delete the stockinfo data according to stockid not in portfolio.
data1=StockInfo.select().dicts()
#data1=HistoryData.select(HistoryData.stockid).distinct().dicts()
for x in data1: #pick up stockid of historydata table
founded=0
#print(x)
for y in allPfArray: #look up portfolio
if x["markettype"] == y["marketType"] and x["stockid"] == y["stockId"] :
founded=1
break;
if founded==0:
q = StockInfo.delete().where(StockInfo.id == x["id"])
q.execute()
#Delete old data outside date of range year
now=datetime.datetime.now()
currentYear=now.strftime("%Y")
keepYears = stockUtil.read_config("stockData.history","keepYears")
fromYear = str(int(currentYear) - int(keepYears) + 1)
for pfGroup in pfGroupArray:
setattr(HistoryData._meta, "db_table", "history_" + pfGroup.lower())
data1=HistoryData.select(HistoryData.stockid).distinct().dicts()
q = HistoryData.delete().where(HistoryData.date < fromYear + "-01-01")
q.execute()
#update DoneYear in StockInfo table
qry=StockInfo.select()
for item in qry:
historyInfoDict=stockUtil.evalTextDict(item.history_info)
historyDoneYearAry=historyInfoDict["DoneYear"]
array1=[]
for i in historyDoneYearAry:
if int(i) >= int(fromYear):
array1.append(str(i))
q=StockInfo.update(history_info={'DoneYear':array1}).where(StockInfo.id==item.id)
q.execute()
def downloadDataHistory(stockId, marketType):
#dictText=StockInfo.select(StockInfo.history_info).where(StockInfo.stockid == stockId, StockInfo.markettype == marketType)
qry=StockInfo.select().where(StockInfo.stockid == stockId, StockInfo.markettype == marketType).first()
if qry==None:
q=StockInfo.insert(stockid=stockId,markettype=marketType,history_info="{'DoneYear':[]}", finance_info="{}")
q.execute()
qry=StockInfo.select().where(StockInfo.stockid == stockId, StockInfo.markettype == marketType).first()
historyInfoDict=stockUtil.evalTextDict(qry.history_info)
historyDoneYearAry=historyInfoDict["DoneYear"]
#print(historyInfoDict)
now=datetime.datetime.now()
currentDate=now.strftime("%Y-%m-%d")
currentYear=now.strftime("%Y")
rangeYears = stockUtil.read_config("stockData.history","rangeYears")
fromYear = str(int(currentYear) - int(rangeYears) + 1)
toYear = currentYear
print(stockId + " is being processed.")
pollMsgQueue.append(stockId + " downloading...")
for i in range(int(fromYear), int(toYear)+1):
startDate = str(i) + "-01-01"
endDate = str(i) + "-12-31"
print("processing year " + str(i) + "...")
#print(historyDoneYearAry)
if str(i) not in historyDoneYearAry:
if str(i) == currentYear:
setattr(HistoryData._meta, "db_table", "history_" + marketType.lower())
lastData=HistoryData.select().where(HistoryData.stockid==stockId).order_by(HistoryData.date.desc()).first()
#print(lastData.date.year)
if lastData != None and lastData.date.strftime("%Y") == currentYear:
startDate=(lastData.date + datetime.timedelta(days=1)).strftime("%Y-%m-%d")
endDate = currentDate
print("Download " + stockId + " from " + startDate + " to " + endDate)
marketId=stockUtil.getMarketId(marketType, stockId)
#print("marketId:" + marketId)
source=stockUtil.getSourceFromCustomCSV(marketType, stockId) #get source from custom table
if source==None: #get source from config.ini
if marketType in ["US","TW","HK"]:
sourceDict=stockUtil.evalTextDict(stockUtil.read_config("stockData.history","source_" + marketType))
#print(sourceDict)
#print("getHistorical_" + sourceDict[marketId].lower())
#print(marketType.lower())
source=sourceDict[marketId].lower()
if source!=None: #get history data from source
#print("Download from source:" + source)
#getHistorical = getattr(stockData,"getHistorical_" + source)
getHistorical = stockData.get_func_from_modStockData("getHistorical_" + source, source)
quotes=getHistorical(stockId, marketType, startDate, endDate)
stockData.saveHistoryData(stockId, quotes, "history_" + marketType.lower())
#print(quotes)
if str(i) != currentYear:
historyDoneYearAry.append(str(i))
q=StockInfo.update(history_info={'DoneYear':historyDoneYearAry}).where(StockInfo.id==qry.id)
q.execute()
def wrap(request, *args, **kwargs):
if request.META.get('HTTP_AUTHORIZATION', False):
authtype, auth1 = request.META['HTTP_AUTHORIZATION'].split(' ')
auth2 = base64.b64decode(auth1).decode('utf-8')
print(authtype)
print(auth2)
username, password = str(auth2).split(':')
print(username)
print(password)
if username == stockUtil.read_config(
"authentication",
"user") and password == stockUtil.read_config(
"authentication", "password"): #set user and password
return f(request, *args, **kwargs)
response = HttpResponse("Auth Required", status=401)
response['WWW-Authenticate'] = 'Basic realm="My Realm"'
return response
def downloadData_vf(request):
pfGroupArray=stockUtil.evalTextArray(stockUtil.read_config("portfolio","pfGroupArray"))
if request.method == "GET":
return render(request, "downloadData.html", {"pfGroupArray":pfGroupArray})
else:
for pfGroup in pfGroupArray:
if request.POST.get("select" + pfGroup) == 'on':
downloadData(pfGroup)
return HttpResponse()
def login_vf(request):
print(request.user)
if request.user.is_authenticated():
return HttpResponseRedirect("/portfolio/TW/pf0")
elif request.META.get('HTTP_AUTHORIZATION', False):
authtype, auth1 = request.META['HTTP_AUTHORIZATION'].split(' ')
auth2 = base64.b64decode(auth1).decode('utf-8')
#print(authtype)
#print(auth2)
username, password = str(auth2).split(':')
user = auth.authenticate(
username=stockUtil.read_config("authentication", "user"),
password=stockUtil.read_config("authentication", "password"))
if user is not None and user.is_active:
auth.login(request, user) #maintain the state of login
return HttpResponseRedirect("/portfolio/TW/pf0")
response = HttpResponse("Auth Required", status=401)
response['WWW-Authenticate'] = 'Basic realm="restricted area"'
return response
def process_request(self, request):
path = request.path_info.lstrip('/')
#print(path)
if not any(m.match(path) for m in EXEMPT_URLS):
if request.META.get('HTTP_AUTHORIZATION', False):
authtype, auth1 = request.META['HTTP_AUTHORIZATION'].split(' ')
auth2 = base64.b64decode(auth1).decode('utf-8')
#print(authtype)
#print(auth2)
username, password = str(auth2).split(':')
if username == stockUtil.read_config(
"authentication",
"user") and password == stockUtil.read_config(
"authentication",
"password"): #set user and password
return None
response = HttpResponse("Auth Required", status=401)
response['WWW-Authenticate'] = 'Basic realm="My Realm"'
return response
def downloadDataFinancial(stockId, marketType):
now=datetime.datetime.now()
currentDate=now.strftime('%Y-%m-%d')
currentYear=now.strftime("%Y")
rangeYears = stockUtil.read_config("stockData.history","rangeYears")
fromYear = str(int(currentYear) - int(rangeYears) + 1)
toYear = currentYear
for i in range(int(fromYear), int(toYear)+1):
startDate = str(i) + "-01-01"
endDate = str(i) + "-12-31"
if i == int(currentYear):
endDate = currentDate
def deleteData_vf(request):
pfGroupArray=stockUtil.evalTextArray(stockUtil.read_config("portfolio","pfGroupArray"))
if request.method == "GET":
return render(request, "deleteData.html", {"pfGroupArray":pfGroupArray})
else:
for pfGroup in pfGroupArray:
if request.POST.get("select" + pfGroup) == 'on':
setattr(HistoryData._meta, "db_table", "history_" + pfGroup.lower())
q = HistoryData.delete()
q.execute()
q = StockInfo.delete()
q.execute()
q = ProjectInfo.delete()
q.execute()
return HttpResponse("Delete data OK.")
def pfGroup_pfIndex_vf(request,pfGroup,pfIndex):
pfGroupArray=stockUtil.evalTextArray(stockUtil.read_config("portfolio","pfGroupArray"))
pfNameArray=[]
#print("pfGroup:" + pfGroup)
#print("pfIndex:" + pfIndex)
pfIndexNo=int(pfIndex.replace("pf",""))
currentPortfolio = Portfolio.select().where(Portfolio.group == pfGroup, Portfolio.index == pfIndexNo).dicts().first()
#print(currentPortfolio)
if currentPortfolio != None:
stockArray=stockUtil.evalTextArray(currentPortfolio["stock_array"]) #stock_array is converted form string type to array
currentPortfolio["stock_array"]=stockArray
if currentPortfolio == None:
createPortfolio(pfGroup, "1", [])
return HttpResponseRedirect("/portfolio/" + pfGroup + "/pf0")
qry = Portfolio.select().where(Portfolio.group == pfGroup).order_by(Portfolio.index)
for i in qry:
pfNameArray.append({"group":i.group,"index":i.index,"name":i.name})
#print(currentPortfolio)
#print(pfNameArray)
return render(request,"portfolio.html",{"pfGroupArray":pfGroupArray, "pfGroup": pfGroup, "pfIndex": pfIndex, "pfNameArray": pfNameArray, "currentPortfolio": currentPortfolio})
import os
curDir = os.path.dirname(os.path.abspath(__file__)) # Do't use curDir. curDir maybe pollute by other import module. Using PROJECT_ROOT for safe
PROJECT_ROOT = os.path.dirname(curDir) # mean cd..
#print(PROJECT_ROOT)
import lib.stockUtil as stockUtil
from peewee import *
config_dbSel = stockUtil.read_config("database","database")
if config_dbSel == None :
db = SqliteDatabase(stockUtil.read_config("database","FilePath"))
elif config_dbSel.lower() == "postgresql":
db = PostgresqlDatabase("stockdb",threadlocals=True, user=stockUtil.read_config("database","user"), password=stockUtil.read_config("database","password"), host="127.0.0.1", port="5432")
elif config_dbSel.lower() == "mysql":
db = MySQLsqlDatabase("stockdb", threadlocals=True, user=stockUtil.read_config("database","user"), password=stockUtil.read_config("database","password"), host="127.0.0.1", port="3306")
else:
db = SqliteDatabase(stockUtil.read_config("database","FilePath"))
class Portfolio(Model):
group = TextField()
index = IntegerField() #portfolio index in the group
name = TextField() #portfolio name
stock_array=TextField() #stocks in this portfolio. element is {"stockId":stockId , "stockName":stockName, "marketType":marketType}
class Meta:
database = db
db_table = "portfolio"
class HistoryData(Model):
date = DateTimeField(null=True)
open = DecimalField(null=True,max_digits=20,decimal_places=6)
def root_vf(request): #root of portfolio
pfGroupArray=stockUtil.evalTextArray(stockUtil.read_config("portfolio","pfGroupArray"))
#print(pfGroupArray)
return HttpResponseRedirect("/portfolio/" + pfGroupArray[0])
from lib.dbModel import db, Portfolio, HistoryData, StockInfo, ProjectInfo
from lib import stockUtil
pfGroupArray = stockUtil.evalTextArray(
stockUtil.read_config("portfolio", "pfGroupArray"))
#download data twice:Before 17:00 and after 17:00.
#download data if download have not be done twice today.
def downloadDataTask():
import managedata.views
import datetime
today = datetime.date.strftime(datetime.datetime.now(), '%Y-%m-%d')
nowTime = datetime.date.strftime(datetime.datetime.now(), '%T')
db.connect()
qry = ProjectInfo.select().first()
if qry == None:
q = ProjectInfo.insert(
**{
"download_date": "2010-01-01",
"download_time": "00:00:00",
"download_access": "TRUE"
})
q.execute()
qry = ProjectInfo.select().first()
if qry != None:
if qry.download_access == "TRUE":
q = ProjectInfo.update(**{
"download_access": "FALSE"
}).where(ProjectInfo.id == qry.id)
q.execute()