stockData_yahoo.py

# Copyright (c) 2006-2010, Jesse Liesch
# All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of the author nor the
#       names of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written permission.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE IMPLIED
# DISCLAIMED. IN NO EVENT SHALL JESSE LIESCH BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from jsonrpc import ServiceProxy
from db import *
import os
import datetime
import prefs
import zlib
import binascii
import traceback

import appGlobal
sys.path.append('server')
import yahoo
from stockData import *
from transaction import *

class IcarraTocRecord():
	stock = 1
	mf = 2
	etf = 3
	option = 4
	
	def __init__(self, securityId=-1, ticker="", exchange="", type=-1, minDate="", maxDate="", name="", row = False):
		assert(not row)

		# securityId is ticker
		self.securityId = ticker
		self.ticker = ticker
		self.exchange = exchange
		self.type = type
		self.minDate = minDate
		self.maxDate = maxDate
		self.name = name
	
	def __repr__(self):
		return '<IcarraTocRecord securityId=%s ticker=%s exchange=%s type=%d minDate=%s maxDate=%s name=%s>' % (self.securityId, self.ticker, self.exchange, self.type, self.minDate, self.maxDate, self.name);

	def insert(self):
		assert(False)

	def updateMinDate(self):
		return

	def updateMaxDate(self):
		appGlobal.getApp().stockData.updateLastDate(self.ticker, self.maxDate)

	@staticmethod
	def query(ticker = "", option = False):
		assert(ticker)
		return IcarraTocRecord(ticker = ticker)

class IcarraPriceRecord():
	def __init__(self, securityId=-1, timestamp="", open="", high="", low="", close="", volume="", openInterest=False, row=False):
		if row:
			self.securityId = row["securityId"]
			self.timestamp = row["date"]
			self.open = row["open"]
			self.high = row["high"]
			self.low = row["low"]
			self.close = row["close"]
			self.volume = row["volume"]
			if "openInterest" in row:
				self.openInterest = row["openInterest"]
			else:
				self.openInterest = False
		else:
			self.securityId = securityId
			self.timestamp = timestamp
			self.open = open
			self.high = high
			self.low = low
			self.close = close
			self.volume = volume
			self.openInterest = openInterest
	
	def __repr__(self):
		return '<IcarraPriceRecord securityId=%s timestamp=%s close=%s>' % (self.securityId, self.timestamp, self.close);

	def insert(self):
		global stocks_db
		global stocks_lastDate
		global stocks_gotData
		on = {
			"ticker": self.securityId,
			"date": self.timestamp
			}
		if stocks_db.insertOrUpdate("stockData", {
			"ticker": self.securityId,
			"date": self.timestamp,
			"open": self.open,
			"high": self.high,
			"low": self.low,
			"close": self.close,
			"volume": self.volume},
			on):
			stocks_gotData = True
			
		# Update last date received for this ticker
		if self.securityId in stocks_lastDate:
			if self.timestamp > stocks_lastDate[self.securityId]:
				stocks_lastDate[self.securityId] = max(stocks_lastDate[self.securityId], self.timestamp)
		else:
			stocks_lastDate[self.securityId] = self.timestamp

	@staticmethod
	def query(securityId, timestamp = False, option = False):
		if timestamp:
			prices = appGlobal.getApp().stockData.getPrices(securityId, startDate = timestamp, endDate = timestamp)
		else:
			prices = appGlobal.getApp().stockData.getPrices(securityId)
			
		if not prices:
			return False
		else:
			if timestamp:
				row = prices[0]
				row["securityId"] = securityId
				return IcarraPriceRecord(row = row)
			else:
				print "not implemented"
				sys.exit()

class IcarraDividendRecord():
	def __init__(self, securityId = -1, ticker = "", timestamp = "", value = "", row = ""):
		if row:
			self.securityId = row["securityId"]
			self.ticker = row["ticker"]
			self.timestamp = row["date"]
			self.value = row["value"]
		else:
			self.securityId = securityId
			self.ticker = ticker
			self.timestamp = timestamp
			self.value = value

	def __repr__(self):
		return '<IcarraDividendRecord securityId=%d timestamp=%s value=%f>' % (self.securityId, self.timestamp, self.value);

	def insert(self):
		data =  {
			"ticker": self.ticker,
			"date": self.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
			"value": str(self.value)
		}
		
		appGlobal.getApp().stockData.db.insertOrUpdate("stockDividends", data)

	@staticmethod
	def query(securityId, ticker, timestamp = False, value = False):
		if timestamp:
			dividend = appGlobal.getApp().stockData.getDividend(ticker, timestamp)
		else:
			dividend = appGlobal.getApp().stockData.getDividends(securityId)[0]
		
		if not dividend:
			return False
		else:
			if timestamp:
				dividend["securityId"] = securityId
				dividend["ticker"] = ticker
				return IcarraDividendRecord(row = dividend)
			else:
				print "not implemented"
				sys.exit()

class IcarraSplitRecord():
	def __init__(self, securityId = -1, ticker = "", timestamp = "", value = "", row = ""):
		if row:
			self.securityId = row["securityId"]
			self.ticker = row["ticker"]
			self.timestamp = row["date"]
			self.value = row["value"]
		else:
			self.securityId = securityId
			self.ticker = ticker
			self.timestamp = timestamp
			self.value = value

	def __repr__(self):
		return '<IcarraSplitRecord securityId=%d timestamp=%s value=%f>' % (self.securityId, self.timestamp, self.value);

	def insert(self):
		data =  {
			"ticker": self.ticker,
			"date": self.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
			"value": str(self.value)
		}
		
		appGlobal.getApp().stockData.db.insertOrUpdate("stockSplits", data)
		return

		assert(False)
		data =  {
			"securityId": str(self.securityId),
			"timestamp": datetime.datetime.strftime(self.timestamp, "%Y-%m-%d %H:%M:%S"),
			"value": str(self.value)
		}
		
		db.insert("splits", data)

	@staticmethod
	def query(securityId, ticker, timestamp = False, value = False):
		if timestamp:
			split = appGlobal.getApp().stockData.getSplits(ticker, timestamp, timestamp)
			if split:
				split = split[0]
		else:
			split = appGlobal.getApp().stockData.getSplits(securityId)[0]
		if not split:
			return False
		else:
			if timestamp:
				split["securityId"] = securityId
				split["ticker"] = ticker
				return IcarraSplitRecord(row = split)
			else:
				print "not implemented"
				sys.exit()

class IcarraRecordFactory:
	def createTocRecord(self, *args, **kwargs):
		return IcarraTocRecord(*args, **kwargs)

	def createPriceRecord(self, *args, **kwargs):
		return IcarraPriceRecord(*args, **kwargs)

	def createDividendRecord(self, *args, **kwargs):
		return IcarraDividendRecord(*args, **kwargs)

	def createSplitRecord(self, *args, **kwargs):
		return IcarraSplitRecord(*args, **kwargs)

class StockDataYahoo(StockData):
	def __init__(self):
                StockData.__init__(self)
		self.yahoo = yahoo.Yahoo(factory = IcarraRecordFactory())
	
	def suggest(self, ticker):
		return False

	def getFromServer(self, request, status = False):
		'''Return True if new stock data is received'''
		global stocks_db
		global stocks_lastDate
		global stocks_gotData

		stocks_db = self.db
		stocks_lastDate = {}
		stocks_gotData = False
		
		if appGlobal.getFailConnected():
			return False
		#appGlobal.setFailConnected(True)
		stocks_gotData = False
		inStatus = False

		if status:
			status.setStatus("Updating Stock Database", 80)

		for (ticker, date) in request.items():
			inStatus = True
			appGlobal.setBackgroundStatus("Downloading " + ticker)
			tocRecord = self.yahoo.recordFactory.createTocRecord(
					ticker,
					ticker,
					minDate = self.getFirstDate(ticker),
					maxDate = self.getLastDate(ticker))
			self.db.beginTransaction()
			self.yahoo.updateHistorical(tocRecord)
			self.db.commitTransaction()

		# Update last dates
		for ticker in stocks_lastDate:
			if ticker == "__UNIQUEID__":
				continue
			self.updateLastDate(ticker, stocks_lastDate[ticker])
		for (ticker, date) in request.items():
			if ticker == "__UNIQUEID__":
				continue
			self.setLastDownload(ticker)
		
		if inStatus:
			appGlobal.clearBackgroundStatus()

		return stocks_gotData

if __name__ == "__main__":
	class FakePrefs:
		def updateLatestVersion(self, v1, v2, v3):
			print "latest version", v1, v2, v3
	class FakeApp:
		checkTableMutex = threading.Lock()
		prefs = FakePrefs()
		backgroundStatus = []
		def getUniqueId(self):
			return "123"

	fakeApp = FakeApp()
	appGlobal.setApp(fakeApp, ".")
	s = StockDataYahoo()
	fakeApp.stockData = s
	s.getFromServer({"AGG": datetime.datetime(1970, 1, 1)})
	s.readTickerFromDb("AGG")
	s.getPrices("AGG")