def __request(self, symbol, stat): try: url = 'http://finance.yahoo.com/d/quotes.csv?s=%s&f=%s' % (symbol, stat) return urllib.urlopen(url).read().strip().strip('"') except IOError: raise UfException(Errors.NETWORK_ERROR, "Can't connect to Yahoo server") except BaseException: raise UfException(Errors.UNKNOWN_ERROR, "Unknown Error in YahooFinance.__request %s" % traceback.format_exc())
def _request(self, url): try: return requests.get(url) except IOError: raise UfException(Errors.NETWORK_ERROR, "Can't connect to Google server at %s" % url) except Exception: raise UfException( Errors.UNKNOWN_ERROR, "Unknown Error in GoogleFinance._request %s" % traceback.format_exc())
def getColumnDescriptors(self, tName): try: return self.__client.getColumnDescriptors(tName) except: raise UfException( Errors.UNKNOWN_ERROR, "Error when getting column descriptors table %s" % tName)
def createDAM(dam_name, config): ''' create DAM ''' if 'yahoo' == dam_name: from analyzerdam.yahooDAM import YahooDAM dam = YahooDAM() elif 'google' == dam_name: from analyzerdam.google import GoogleDAM dam = GoogleDAM() elif 'excel' == dam_name: from analyzerdam.excelDAM import ExcelDAM dam = ExcelDAM() elif 'hbase' == dam_name: from analyzerdam.hbaseDAM import HBaseDAM dam = HBaseDAM() elif 'sql' == dam_name: from analyzerdam.sqlDAM import SqlDAM dam = SqlDAM(config) elif 'cex' == dam_name: from analyzerdam.cex import CexDAM dam = CexDAM(config) else: raise UfException(Errors.INVALID_DAM_TYPE, "DAM type is invalid %s" % dam_name) return dam
def __getSheet(self, name): ''' get a sheet by name ''' if not self.sheetExsit(name): raise UfException(Errors.SHEET_NAME_INVALID, "Can't find a sheet named %s" % name) return self.__sheetNameDict[name]
def plot(self): ''' plot dataValue ''' try: fig = pyplot.figure() i = 0 ax0 = None for label, dateValues in self.dateValueDict.items(): if 0 == i: ax = fig.add_axes(self.rect[i]) ax0 = ax else: ax = fig.add_axes(self.rect[i], sharex=ax0) i += 1 ax.plot_date([ datetime.strptime(dateValue[0], self.dateFormat) for dateValue in dateValues ], [dateValue[1] for dateValue in dateValues], fmt='b-') ax.set_ylabel(label) ax.set_ylim( min([int(dateValue[1]) for dateValue in dateValues]) / 1.1, max([int(dateValue[1]) for dateValue in dateValues]) * 1.1) #ax.set_ylim(0, 1000) #pyplot.legend() pyplot.show() except UfException as excep: raise excep except BaseException as excep: raise UfException( Errors.UNKNOWN_ERROR, "plotDateValueDict.plot got unknown error %s" % excep)
def __init__(self, fileName): ''' constructor ''' if not path.exists(fileName): raise UfException(Errors.FILE_NOT_EXIST, "File doesn't exist: %s" % fileName) self.__book = open_workbook(fileName) self.__sheet = None
def financials(self, security): """ get financials: google finance provide annual and quanter financials, if annual is true, we will use annual data Up to four lastest year/quanter data will be provided by google Refer to page as an example: http://www.google.com/finance?q=TSE:CVG&fstype=ii """ try: url = 'http://www.google.com/finance?q=%s&fstype=ii' % security try: page = self._request(url).read() except UfException as ufExcep: # if symol is not right, will get 400 if Errors.NETWORK_400_ERROR == ufExcep.getCode: raise UfException( Errors.STOCK_SYMBOL_ERROR, "Can find data for stock %s, security error?" % security) raise ufExcep bPage = BeautifulSoup(page) target = bPage.find(id='incinterimdiv') keyTimeValue = {} # ugly do...while i = 0 while True: self._parseTarget(target, keyTimeValue) if i < 5: i += 1 target = target.nextSibling # ugly beautiful soap... if '\n' == target: target = target.nextSibling else: break return keyTimeValue except BaseException: raise UfException( Errors.UNKNOWN_ERROR, "Unknown Error in GoogleFinance.getHistoricalPrices %s" % traceback.format_exc())
def readCol(self, col, startRow=0, endRow=-1): if self.__sheet is None: self.openSheet(super(ExcelRead, self).DEFAULT_SHEET) if abs(col) > self.__sheet.ncols: raise UfException( Errors.INDEX_RANGE_ERROR, "Excellib.readCol: col number too big: col %s, max %s" % (col, self.sheet.ncols)) if max(abs(startRow), abs(endRow)) > self.__sheet.nrows: raise UfException( Errors.INDEX_RANGE_ERROR, "Excellib.readCol: row number too big: row %s, max %s" % (max(abs(startRow), abs(endRow)), self.sheet.nrows)) if -1 == endRow: endRow = self.__sheet.nrows return [self.readCell(i, col) for i in range(startRow, endRow)]
def createTable(self, tName, ColumnDescriptors): try: self.__client.createTable(tName, ColumnDescriptors) except ttypes.AlreadyExists as excp: raise UfException( Errors.HBASE_CREATE_ERROR, "AlreadyExists Error when creating table %s with cols: %s): %s" % (tName, [col.name for col in ColumnDescriptors], excp.message))
def __init__(self, fileName): if path.exists(fileName): raise UfException(Errors.FILE_EXIST, "File already exist: %s" % fileName) self.__fileName = fileName self.__workbook = Workbook() self.__sheetNameDict = {} self.__sheet = None
def readCell(self, row, col): ''' read a cell''' try: if self.__sheet is None: self.openSheet(super(ExcelRead, self).DEFAULT_SHEET) return self.__sheet.cell(row, col).value except BaseException as excp: raise UfException(Errors.UNKNOWN_ERROR, "Unknown Error in Excellib.readCell %s" % excp)
def __init__(self, fileName=None, mode=READ_MODE): ''' constructor ''' if ExcelLib.READ_MODE == mode: self.__operation = ExcelRead(fileName) elif ExcelLib.WRITE_MODE == mode: self.__operation = ExcelWrite(fileName) else: raise UfException(Errors.INVALID_EXCEL_MODE, "Invalid operation mode, only %s and %s are accepted"\ % (ExcelLib.READ_MODE, ExcelLib.WRITE_MODE))
def getQuotes(self, symbol, start, end): """ Get historical prices for the given ticker symbol. Date format is 'YYYY-MM-DD' Returns a nested list. """ try: start = str(start).replace('-', '') end = str(end).replace('-', '') url = 'http://ichart.yahoo.com/table.csv?s=%s&' % symbol + \ 'd=%s&' % str(int(end[4:6]) - 1) + \ 'e=%s&' % str(int(end[6:8])) + \ 'f=%s&' % str(int(end[0:4])) + \ 'g=d&' + \ 'a=%s&' % str(int(start[4:6]) - 1) + \ 'b=%s&' % str(int(start[6:8])) + \ 'c=%s&' % str(int(start[0:4])) + \ 'ignore=.csv' days = urllib.urlopen(url).readlines() values = [day[:-2].split(',') for day in days] # sample values:[['Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Adj Clos'], \ # ['2009-12-31', '112.77', '112.80', '111.39', '111.44', '90637900', '109.7']...] data = [] for value in values[1:]: data.append( Quote(value[0], value[1], value[2], value[3], value[4], value[5], value[6])) dateValues = sorted(data, key=lambda q: q.time) return dateValues except IOError: raise UfException(Errors.NETWORK_ERROR, "Can't connect to Yahoo server") except BaseException: raise UfException( Errors.UNKNOWN_ERROR, "Unknown Error in YahooFinance.getHistoricalPrices %s" % traceback.format_exc())
def quotes(self, security, start, end): """ Get historical prices for the given ticker security. Date format is 'YYYYMMDD' Returns a nested list. """ try: url = 'http://www.google.com/finance/historical?q=%s&startdate=%s&enddate=%s&output=csv' % ( security.symbol, start, end) try: page = self._request(url) except UfException as ufExcep: # if symol is not right, will get 400 if Errors.NETWORK_400_ERROR == ufExcep.getCode: raise UfException( Errors.STOCK_SYMBOL_ERROR, "Can find data for stock %s, security error?" % security) raise ufExcep days = page.readlines() values = [day.split(',') for day in days] # sample values:[['Date', 'Open', 'High', 'Low', 'Close', 'Volume'], \ # ['2009-12-31', '112.77', '112.80', '111.39', '111.44', '90637900']...] for value in values[1:]: date = convertGoogCSVDate(value[0]) try: yield Quote(date, value[1].strip(), value[2].strip(), value[3].strip(), value[4].strip(), value[5].strip(), None) except Exception: LOG.warning( "Exception when processing %s at date %s for value %s" % (security, date, value)) except BaseException: raise UfException( Errors.UNKNOWN_ERROR, "Unknown Error in GoogleFinance.getHistoricalPrices %s" % traceback.format_exc())
def __writeData(self, targetPath, fields, rows): ''' write data ''' if path.exists(targetPath): LOG.error("Target file exists: %s" % path.abspath(targetPath)) raise UfException(Errors.FILE_EXIST, "can't write to a existing file" ) #because xlwt doesn't support it with ExcelLib(fileName=targetPath, mode=ExcelLib.WRITE_MODE) as excel: excel.writeRow(0, fields) for index, row in enumerate(rows): excel.writeRow(index + 1, row)
def updateRow(self, tName, rowName, mutations, timestamp=None): ''' add row to table ''' try: if timestamp is None: self.__client.mutateRow(tName, rowName, mutations) else: self.__client.mutateRowTs(tName, rowName, mutations, timestamp) except Exception as excp: raise UfException( Errors.HBASE_UPDATE_ERROR, "Error when updating table %s - rowName %s - mutations %s: %s" % (tName, rowName, mutations, excp))
def get_all(self, security): """ Get all available quote data for the given ticker security. Returns a dictionary. """ url = 'http://www.google.com/finance?q=%s' % security page = self._request(url) soup = BeautifulSoup(page) snapData = soup.find("table", {"class": "snap-data"}) if snapData is None: raise UfException( Errors.STOCK_SYMBOL_ERROR, "Can find data for stock %s, security error?" % security) data = {} for row in snapData.findAll('tr'): keyTd, valTd = row.findAll('td') data[keyTd.getText()] = valTd.getText() return data
def ticks(self, security, start, end): """ Get tick prices for the given ticker security. @security: stock security @interval: interval in mins(google finance only support query till 1 min) @start: start date(YYYYMMDD) @end: end date(YYYYMMDD) start and end is disabled since only 15 days data will show @Returns a nested list. """ period = 15 # url = 'http://www.google.com/finance/getprices?q=%s&i=%s&p=%sd&f=d,o,h,l,c,v&ts=%s' % (security, interval, period, start) url = 'http://www.google.com/finance/getprices?q=%s&i=61&p=%sd&f=d,o,h,l,c,v' % ( security.symbol, period) try: response = self._request(url) except UfException as ufExcep: # if symol is not right, will get 400 if Errors.NETWORK_400_ERROR == ufExcep.getCode: raise UfException( Errors.STOCK_SYMBOL_ERROR, "Can find data for stock %s, security error?" % security) raise ufExcep # use csv reader here days = response.text.split('\n')[7:] # first 7 line is document # sample values:'a1316784600,31.41,31.5,31.4,31.43,150911' values = [day.split(',') for day in days if len(days) == 6] for value in values: yield json.dumps({ 'date': value[0][1:].strip(), 'close': value[1].strip(), 'high': value[2].strip(), 'low': value[3].strip(), 'open': value[4].strip(), 'volume': value[5].strip() })
def createDAM(damType, settings=None): ''' create DAM ''' if 'yahoo' == damType: from analyzerdam.yahooDAM import YahooDAM dam=YahooDAM() elif 'google' == damType: from analyzerdam.google import GoogleDAM dam=GoogleDAM() elif 'excel' == damType: from analyzerdam.excelDAM import ExcelDAM dam=ExcelDAM() elif 'hbase' == damType: from analyzerdam.hbaseDAM import HBaseDAM dam=HBaseDAM() elif 'sql' == damType: from analyzerdam.sqlDAM import SqlDAM dam=SqlDAM(settings) else: raise UfException(Errors.INVALID_DAM_TYPE, "DAM type is invalid %s" % damType) return dam
def load_config_from(self, file_name): ''' set source file name assume the file_name is full path first, if can't find it, use conf directory ''' fullPath = path.abspath(file_name) if not path.exists(fullPath): fullPath = path.join( path.join( path.dirname( path.dirname(path.dirname(path.abspath(__file__)))), 'conf'), file_name) if not path.exists(fullPath): msg = "config file doesn't exist at: %s or %s" % (file_name, fullPath) LOG.error(msg) raise UfException(Errors.FILE_NOT_EXIST, msg) self.parser = ConfigParser.SafeConfigParser( defaults={"here": self.__dir}) self.parser.read(fullPath) self.full_path = fullPath
def readCell(self, row, col): ''' read cell''' raise UfException(Errors.UNDEFINED_METHOD, "readCell function is not defined")
def openSheet(self, name): ''' open sheet ''' raise UfException(Errors.UNDEFINED_METHOD, "openSheet function is not defined")
def __validateConfig(self): ''' validate config is ok ''' if self.parser is None: msg = "No config file is loaded, please use setSource method first" LOG.error(msg) raise UfException(Errors.FILE_NOT_EXIST, msg)
def readRow(self, row, startCol=0, endCol=-1): ''' read row ''' raise UfException(Errors.UNDEFINED_METHOD, "readRow function is not defined")
def readCol(self, col, startRow=0, endRow=-1): ''' read col ''' raise UfException(Errors.UNDEFINED_METHOD, "readCol function is not defined")
def writeRow(self, sheetName, row, values): ''' write row ''' raise UfException(Errors.UNDEFINED_METHOD, "writeRow function is not defined")
def writeCell(self, sheetName, row, col, value): ''' write cell''' raise UfException(Errors.UNDEFINED_METHOD, "readCell function is not defined")
def create_strategy(name, securities, config): ''' create a metric ''' if name not in StrategyFactory.STRATEGY_DICT: raise UfException(Errors.INVALID_STRATEGY_NAME, "Strategy name is invalid %s" % name) return StrategyFactory.STRATEGY_DICT[name](config, securities)