def __request(self, symbol, stat):
try:
url = 'http://finance.yahoo.com/d/quotes.csv?s=%s&f=%s' % (symbol, stat)
return urllib.urlopen(url).read().strip().strip('"')
except IOError:
raise UfException(Errors.NETWORK_ERROR, "Can't connect to Yahoo server")
except BaseException:
raise UfException(Errors.UNKNOWN_ERROR, "Unknown Error in YahooFinance.__request %s" % traceback.format_exc())
def _request(self, url):
try:
return requests.get(url)
except IOError:
raise UfException(Errors.NETWORK_ERROR,
"Can't connect to Google server at %s" % url)
except Exception:
raise UfException(
Errors.UNKNOWN_ERROR,
"Unknown Error in GoogleFinance._request %s" %
traceback.format_exc())
def getColumnDescriptors(self, tName):
try:
return self.__client.getColumnDescriptors(tName)
except:
raise UfException(
Errors.UNKNOWN_ERROR,
"Error when getting column descriptors table %s" % tName)
def createDAM(dam_name, config):
''' create DAM '''
if 'yahoo' == dam_name:
from analyzerdam.yahooDAM import YahooDAM
dam = YahooDAM()
elif 'google' == dam_name:
from analyzerdam.google import GoogleDAM
dam = GoogleDAM()
elif 'excel' == dam_name:
from analyzerdam.excelDAM import ExcelDAM
dam = ExcelDAM()
elif 'hbase' == dam_name:
from analyzerdam.hbaseDAM import HBaseDAM
dam = HBaseDAM()
elif 'sql' == dam_name:
from analyzerdam.sqlDAM import SqlDAM
dam = SqlDAM(config)
elif 'cex' == dam_name:
from analyzerdam.cex import CexDAM
dam = CexDAM(config)
else:
raise UfException(Errors.INVALID_DAM_TYPE,
"DAM type is invalid %s" % dam_name)
return dam
def __getSheet(self, name):
''' get a sheet by name '''
if not self.sheetExsit(name):
raise UfException(Errors.SHEET_NAME_INVALID,
"Can't find a sheet named %s" % name)
return self.__sheetNameDict[name]
def plot(self):
''' plot dataValue '''
try:
fig = pyplot.figure()
i = 0
ax0 = None
for label, dateValues in self.dateValueDict.items():
if 0 == i:
ax = fig.add_axes(self.rect[i])
ax0 = ax
else:
ax = fig.add_axes(self.rect[i], sharex=ax0)
i += 1
ax.plot_date([
datetime.strptime(dateValue[0], self.dateFormat)
for dateValue in dateValues
], [dateValue[1] for dateValue in dateValues],
fmt='b-')
ax.set_ylabel(label)
ax.set_ylim(
min([int(dateValue[1]) for dateValue in dateValues]) / 1.1,
max([int(dateValue[1]) for dateValue in dateValues]) * 1.1)
#ax.set_ylim(0, 1000)
#pyplot.legend()
pyplot.show()
except UfException as excep:
raise excep
except BaseException as excep:
raise UfException(
Errors.UNKNOWN_ERROR,
"plotDateValueDict.plot got unknown error %s" % excep)
def __init__(self, fileName):
''' constructor '''
if not path.exists(fileName):
raise UfException(Errors.FILE_NOT_EXIST,
"File doesn't exist: %s" % fileName)
self.__book = open_workbook(fileName)
self.__sheet = None
def financials(self, security):
"""
get financials:
google finance provide annual and quanter financials, if annual is true, we will use annual data
Up to four lastest year/quanter data will be provided by google
Refer to page as an example: http://www.google.com/finance?q=TSE:CVG&fstype=ii
"""
try:
url = 'http://www.google.com/finance?q=%s&fstype=ii' % security
try:
page = self._request(url).read()
except UfException as ufExcep:
# if symol is not right, will get 400
if Errors.NETWORK_400_ERROR == ufExcep.getCode:
raise UfException(
Errors.STOCK_SYMBOL_ERROR,
"Can find data for stock %s, security error?" %
security)
raise ufExcep
bPage = BeautifulSoup(page)
target = bPage.find(id='incinterimdiv')
keyTimeValue = {}
# ugly do...while
i = 0
while True:
self._parseTarget(target, keyTimeValue)
if i < 5:
i += 1
target = target.nextSibling
# ugly beautiful soap...
if '\n' == target:
target = target.nextSibling
else:
break
return keyTimeValue
except BaseException:
raise UfException(
Errors.UNKNOWN_ERROR,
"Unknown Error in GoogleFinance.getHistoricalPrices %s" %
traceback.format_exc())
def readCol(self, col, startRow=0, endRow=-1):
if self.__sheet is None:
self.openSheet(super(ExcelRead, self).DEFAULT_SHEET)
if abs(col) > self.__sheet.ncols:
raise UfException(
Errors.INDEX_RANGE_ERROR,
"Excellib.readCol: col number too big: col %s, max %s" %
(col, self.sheet.ncols))
if max(abs(startRow), abs(endRow)) > self.__sheet.nrows:
raise UfException(
Errors.INDEX_RANGE_ERROR,
"Excellib.readCol: row number too big: row %s, max %s" %
(max(abs(startRow), abs(endRow)), self.sheet.nrows))
if -1 == endRow:
endRow = self.__sheet.nrows
return [self.readCell(i, col) for i in range(startRow, endRow)]
def createTable(self, tName, ColumnDescriptors):
try:
self.__client.createTable(tName, ColumnDescriptors)
except ttypes.AlreadyExists as excp:
raise UfException(
Errors.HBASE_CREATE_ERROR,
"AlreadyExists Error when creating table %s with cols: %s): %s"
% (tName, [col.name
for col in ColumnDescriptors], excp.message))
def __init__(self, fileName):
if path.exists(fileName):
raise UfException(Errors.FILE_EXIST,
"File already exist: %s" % fileName)
self.__fileName = fileName
self.__workbook = Workbook()
self.__sheetNameDict = {}
self.__sheet = None
def readCell(self, row, col):
''' read a cell'''
try:
if self.__sheet is None:
self.openSheet(super(ExcelRead, self).DEFAULT_SHEET)
return self.__sheet.cell(row, col).value
except BaseException as excp:
raise UfException(Errors.UNKNOWN_ERROR,
"Unknown Error in Excellib.readCell %s" % excp)
def __init__(self, fileName=None, mode=READ_MODE):
''' constructor '''
if ExcelLib.READ_MODE == mode:
self.__operation = ExcelRead(fileName)
elif ExcelLib.WRITE_MODE == mode:
self.__operation = ExcelWrite(fileName)
else:
raise UfException(Errors.INVALID_EXCEL_MODE,
"Invalid operation mode, only %s and %s are accepted"\
% (ExcelLib.READ_MODE, ExcelLib.WRITE_MODE))
def getQuotes(self, symbol, start, end):
"""
Get historical prices for the given ticker symbol.
Date format is 'YYYY-MM-DD'
Returns a nested list.
"""
try:
start = str(start).replace('-', '')
end = str(end).replace('-', '')
url = 'http://ichart.yahoo.com/table.csv?s=%s&' % symbol + \
'd=%s&' % str(int(end[4:6]) - 1) + \
'e=%s&' % str(int(end[6:8])) + \
'f=%s&' % str(int(end[0:4])) + \
'g=d&' + \
'a=%s&' % str(int(start[4:6]) - 1) + \
'b=%s&' % str(int(start[6:8])) + \
'c=%s&' % str(int(start[0:4])) + \
'ignore=.csv'
days = urllib.urlopen(url).readlines()
values = [day[:-2].split(',') for day in days]
# sample values:[['Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Adj Clos'], \
# ['2009-12-31', '112.77', '112.80', '111.39', '111.44', '90637900', '109.7']...]
data = []
for value in values[1:]:
data.append(
Quote(value[0], value[1], value[2], value[3], value[4],
value[5], value[6]))
dateValues = sorted(data, key=lambda q: q.time)
return dateValues
except IOError:
raise UfException(Errors.NETWORK_ERROR,
"Can't connect to Yahoo server")
except BaseException:
raise UfException(
Errors.UNKNOWN_ERROR,
"Unknown Error in YahooFinance.getHistoricalPrices %s" %
traceback.format_exc())
def quotes(self, security, start, end):
"""
Get historical prices for the given ticker security.
Date format is 'YYYYMMDD'
Returns a nested list.
"""
try:
url = 'http://www.google.com/finance/historical?q=%s&startdate=%s&enddate=%s&output=csv' % (
security.symbol, start, end)
try:
page = self._request(url)
except UfException as ufExcep:
# if symol is not right, will get 400
if Errors.NETWORK_400_ERROR == ufExcep.getCode:
raise UfException(
Errors.STOCK_SYMBOL_ERROR,
"Can find data for stock %s, security error?" %
security)
raise ufExcep
days = page.readlines()
values = [day.split(',') for day in days]
# sample values:[['Date', 'Open', 'High', 'Low', 'Close', 'Volume'], \
# ['2009-12-31', '112.77', '112.80', '111.39', '111.44', '90637900']...]
for value in values[1:]:
date = convertGoogCSVDate(value[0])
try:
yield Quote(date, value[1].strip(), value[2].strip(),
value[3].strip(), value[4].strip(),
value[5].strip(), None)
except Exception:
LOG.warning(
"Exception when processing %s at date %s for value %s"
% (security, date, value))
except BaseException:
raise UfException(
Errors.UNKNOWN_ERROR,
"Unknown Error in GoogleFinance.getHistoricalPrices %s" %
traceback.format_exc())
def __writeData(self, targetPath, fields, rows):
''' write data '''
if path.exists(targetPath):
LOG.error("Target file exists: %s" % path.abspath(targetPath))
raise UfException(Errors.FILE_EXIST,
"can't write to a existing file"
) #because xlwt doesn't support it
with ExcelLib(fileName=targetPath, mode=ExcelLib.WRITE_MODE) as excel:
excel.writeRow(0, fields)
for index, row in enumerate(rows):
excel.writeRow(index + 1, row)
def updateRow(self, tName, rowName, mutations, timestamp=None):
''' add row to table '''
try:
if timestamp is None:
self.__client.mutateRow(tName, rowName, mutations)
else:
self.__client.mutateRowTs(tName, rowName, mutations, timestamp)
except Exception as excp:
raise UfException(
Errors.HBASE_UPDATE_ERROR,
"Error when updating table %s - rowName %s - mutations %s: %s"
% (tName, rowName, mutations, excp))
def get_all(self, security):
"""
Get all available quote data for the given ticker security.
Returns a dictionary.
"""
url = 'http://www.google.com/finance?q=%s' % security
page = self._request(url)
soup = BeautifulSoup(page)
snapData = soup.find("table", {"class": "snap-data"})
if snapData is None:
raise UfException(
Errors.STOCK_SYMBOL_ERROR,
"Can find data for stock %s, security error?" % security)
data = {}
for row in snapData.findAll('tr'):
keyTd, valTd = row.findAll('td')
data[keyTd.getText()] = valTd.getText()
return data
def ticks(self, security, start, end):
"""
Get tick prices for the given ticker security.
@security: stock security
@interval: interval in mins(google finance only support query till 1 min)
@start: start date(YYYYMMDD)
@end: end date(YYYYMMDD)
start and end is disabled since only 15 days data will show
@Returns a nested list.
"""
period = 15
# url = 'http://www.google.com/finance/getprices?q=%s&i=%s&p=%sd&f=d,o,h,l,c,v&ts=%s' % (security, interval, period, start)
url = 'http://www.google.com/finance/getprices?q=%s&i=61&p=%sd&f=d,o,h,l,c,v' % (
security.symbol, period)
try:
response = self._request(url)
except UfException as ufExcep:
# if symol is not right, will get 400
if Errors.NETWORK_400_ERROR == ufExcep.getCode:
raise UfException(
Errors.STOCK_SYMBOL_ERROR,
"Can find data for stock %s, security error?" % security)
raise ufExcep
# use csv reader here
days = response.text.split('\n')[7:] # first 7 line is document
# sample values:'a1316784600,31.41,31.5,31.4,31.43,150911'
values = [day.split(',') for day in days if len(days) == 6]
for value in values:
yield json.dumps({
'date': value[0][1:].strip(),
'close': value[1].strip(),
'high': value[2].strip(),
'low': value[3].strip(),
'open': value[4].strip(),
'volume': value[5].strip()
})
def createDAM(damType, settings=None):
''' create DAM '''
if 'yahoo' == damType:
from analyzerdam.yahooDAM import YahooDAM
dam=YahooDAM()
elif 'google' == damType:
from analyzerdam.google import GoogleDAM
dam=GoogleDAM()
elif 'excel' == damType:
from analyzerdam.excelDAM import ExcelDAM
dam=ExcelDAM()
elif 'hbase' == damType:
from analyzerdam.hbaseDAM import HBaseDAM
dam=HBaseDAM()
elif 'sql' == damType:
from analyzerdam.sqlDAM import SqlDAM
dam=SqlDAM(settings)
else:
raise UfException(Errors.INVALID_DAM_TYPE,
"DAM type is invalid %s" % damType)
return dam
def load_config_from(self, file_name):
'''
set source file name
assume the file_name is full path first, if can't find it, use conf directory
'''
fullPath = path.abspath(file_name)
if not path.exists(fullPath):
fullPath = path.join(
path.join(
path.dirname(
path.dirname(path.dirname(path.abspath(__file__)))),
'conf'), file_name)
if not path.exists(fullPath):
msg = "config file doesn't exist at: %s or %s" % (file_name,
fullPath)
LOG.error(msg)
raise UfException(Errors.FILE_NOT_EXIST, msg)
self.parser = ConfigParser.SafeConfigParser(
defaults={"here": self.__dir})
self.parser.read(fullPath)
self.full_path = fullPath
def readCell(self, row, col):
''' read cell'''
raise UfException(Errors.UNDEFINED_METHOD,
"readCell function is not defined")
def openSheet(self, name):
''' open sheet '''
raise UfException(Errors.UNDEFINED_METHOD,
"openSheet function is not defined")
def __validateConfig(self):
''' validate config is ok '''
if self.parser is None:
msg = "No config file is loaded, please use setSource method first"
LOG.error(msg)
raise UfException(Errors.FILE_NOT_EXIST, msg)
def readRow(self, row, startCol=0, endCol=-1):
''' read row '''
raise UfException(Errors.UNDEFINED_METHOD,
"readRow function is not defined")
def readCol(self, col, startRow=0, endRow=-1):
''' read col '''
raise UfException(Errors.UNDEFINED_METHOD,
"readCol function is not defined")
def writeRow(self, sheetName, row, values):
''' write row '''
raise UfException(Errors.UNDEFINED_METHOD,
"writeRow function is not defined")
def writeCell(self, sheetName, row, col, value):
''' write cell'''
raise UfException(Errors.UNDEFINED_METHOD,
"readCell function is not defined")
def create_strategy(name, securities, config):
''' create a metric '''
if name not in StrategyFactory.STRATEGY_DICT:
raise UfException(Errors.INVALID_STRATEGY_NAME,
"Strategy name is invalid %s" % name)
return StrategyFactory.STRATEGY_DICT[name](config, securities)
