def __init__(self, symbols, start, poolsize = 5): ''' constructor ''' self.symbols = symbols self.sqlLocation = None self.outputDAM = DAMFactory.createDAM("sql", self.__getOutputDamSetting()) self.googleDAM = DAMFactory.createDAM("google") self.start = start self.end = None self.poolsize = poolsize self.readLock = Lock() self.writeLock = Lock() self.failed = [] self.succeeded = []
def _createDam(self, symbol): ''' setup Dam''' damName = self.__config.getOption(CONF_ULTRAFINANCE_SECTION, CONF_INPUT_DAM) inputDb = self.__config.getOption(CONF_ULTRAFINANCE_SECTION, CONF_INPUT_DB) dam = DAMFactory.createDAM(damName, {'db': inputDb}) dam.setSymbol(symbol) return dam
def _createDam(self, symbol): """ setup Dam""" damName = self.__config.getOption(CONF_INPUT_SECTION, CONF_DAM) setting = self.__config.getSection(CONF_INPUT_SECTION) dam = DAMFactory.createDAM(damName, setting) dam.setSymbol(symbol) return dam
def _createDam(self, symbol): ''' setup Dam''' damName = self.__config.getOption(CONF_INPUT_SECTION, CONF_DAM) setting = self.__config.getSection(CONF_INPUT_SECTION) dam = DAMFactory.createDAM(damName, setting) dam.setSymbol(symbol) return dam
def getOptions(self): ''' crawling data and save to hbase ''' parser = optparse.OptionParser("Usage: %prog [options]") parser.add_option("-f", "--symbolFile", dest="symbolFile", type="string", help="file that contains symbols for each line") parser.add_option("-o", "--outputDAM", dest="outputDAM", default='sql', type="string", help="output dam, e.g. sql|hbase") (options, _) = parser.parse_args() # get symbols if options.symbolFile is None or not path.exists(options.symbolFile): print("Please provide valid file: %s" % options.symbolFile) exit(4) # get all symbols with open(options.symbolFile, 'r') as f: for line in f.readlines(): self.symbols.append(line.strip()) if not self.symbols: print("No symbols provided in file %s" % options.symbolFile) exit(4) # set output dam if options.outputDAM not in ["hbase", "sql"]: print("Please provide valid outputDAM %s" % options.outputDAM) exit(4) if 'sql' == options.outputDAM: sqlLocation = 'sqlite:///%s' % self.getOutputSql() print("Sqlite location: %s" % sqlLocation) setting = {'db': sqlLocation} # set google and output dam self.googleDAM = DAMFactory.createDAM("google") self.outputDAM = DAMFactory.createDAM(options.outputDAM, setting)
def getOptions(self): ''' crawling data and save to hbase ''' parser = optparse.OptionParser("Usage: %prog [options]") parser.add_option("-f", "--symbolFile", dest = "symbolFile", type = "string", help = "file that contains symbols for each line") parser.add_option("-o", "--outputDAM", dest = "outputDAM", default = 'sql', type = "string", help = "output dam, e.g. sql|hbase") (options, _) = parser.parse_args() # get symbols if options.symbolFile is None or not path.exists(options.symbolFile): print("Please provide valid file: %s" % options.symbolFile) exit(4) # get all symbols with open(options.symbolFile, 'r') as f: for line in f.readlines(): self.symbols.append(line.strip()) if not self.symbols: print("No symbols provided in file %s" % options.symbolFile) exit(4) # set output dam if options.outputDAM not in ["hbase", "sql"]: print("Please provide valid outputDAM %s" % options.outputDAM) exit(4) if 'sql' == options.outputDAM: sqlLocation = 'sqlite:///%s' % self.getOutputSql() print("Sqlite location: %s" % sqlLocation) setting = {'db': sqlLocation} # set google and output dam self.googleDAM = DAMFactory.createDAM("google") self.outputDAM = DAMFactory.createDAM(options.outputDAM, setting)
def getOptions(self): ''' crawling data and save to hbase ''' parser = optparse.OptionParser("Usage: %prog [options]") parser.add_option("-f", "--symbolFile", dest = "symbolFile", type = "string", help = "file that contains symbols for each line") parser.add_option("-t", "--dataType", dest = "dataType", default = 'quote', type = "string", help = "data type that will be stored, e.g. quote|tick|all") parser.add_option("-s", "--start", dest = "start", default = 'month', type = "string", help = "start date, all|month|day") parser.add_option("-o", "--outputDAM", dest = "outputDAM", default = 'sql', type = "string", help = "output dam, e.g. sql|hbase") (options, _) = parser.parse_args() # get symbols if options.symbolFile is None or not path.exists(options.symbolFile): print("Please provide valid file: %s" % options.symbolFile) exit(4) # get all symbols with open(options.symbolFile, 'r') as f: for line in f.readlines(): self.symbols.append(line.strip()) if not self.symbols: print("No symbols provided in file %s" % options.symbolFile) exit(4) # set dataType if options.dataType not in ["quote", "tick", "all"]: print("Please provide valid dataType %s" % options.dataType) exit(4) # set output dam if options.outputDAM not in ["hbase", "sql"]: print("Please provide valid outputDAM %s" % options.outputDAM) exit(4) if not (options.start in ['all', 'month', 'day'] or DATE_FORMAT.match(options.start)): print("Please provide valid start option(all|month|day): %s" % options.outputDAM) exit(4) if "quote" == options.dataType: self.isQuote = True elif "tick" == options.dataType: self.isTick = True else: self.isQuote = self.isTick = True print("Retrieving data type: %s" % options.dataType) if 'sql' == options.outputDAM: sqlLocation = 'sqlite:///%s' % self.getOutputSql() print("Sqlite location: %s" % sqlLocation) setting = {'db': sqlLocation} # set google and output dam self.googleDAM = DAMFactory.createDAM("google") self.outputDAM = DAMFactory.createDAM(options.outputDAM, setting) # set start date and end date if 'all' == options.start: self.start = '19800101' elif 'month' == options.start: self.start = (datetime.datetime.now() + relativedelta(months = -1)).strftime("%Y%m%d") elif 'day' == options.start: self.start = (datetime.datetime.now() + relativedelta(days = -1)).strftime("%Y%m%d") else: self.start = options.start self.end = datetime.datetime.now().strftime("%Y%m%d") if options.dataType in ["quote", "all"]: print("Retrieving quotes start from %s" % self.start) else: print("Retrieving ticks for last 15 days")
def getOptions(self): ''' crawling data and save to hbase ''' parser = optparse.OptionParser("Usage: %prog [options]") parser.add_option("-f", "--symbolFile", dest="symbolFile", type="string", help="file that contains symbols for each line") parser.add_option( "-t", "--dataType", dest="dataType", default='quote', type="string", help="data type that will be stored, e.g. quote|tick|all") parser.add_option("-s", "--start", dest="start", default='month', type="string", help="start date, all|month") parser.add_option("-o", "--outputDAM", dest="outputDAM", default='sql', type="string", help="output dam, e.g. sql|hbase") (options, _) = parser.parse_args() # get symbols if options.symbolFile is None or not path.exists(options.symbolFile): print("Please provide valid file: %s" % options.symbolFile) exit(4) # get all symbols with open(options.symbolFile, 'r') as f: for line in f.readlines(): self.symbols.append(line.strip()) if not self.symbols: print("No symbols provided in file %s" % options.symbolFile) exit(4) # set dataType if options.dataType not in ["quote", "tick", "all"]: print("Please provide valid dataType %s" % options.dataType) exit(4) # set output dam if options.outputDAM not in ["hbase", "sql"]: print("Please provide valid outputDAM %s" % options.outputDAM) exit(4) if options.start not in ['all', 'month']: print("Please provide valid start option(all|month): %s" % options.outputDAM) exit(4) if "quote" == options.dataType: self.isQuote = True elif "tick" == options.dataType: self.isTick = True else: self.isQuote = self.isTick = True print("Retrieving data type: %s" % options.dataType) if 'sql' == options.outputDAM: sqlLocation = 'sqlite:///%s' % self.getOutputSql() print("Sqlite location: %s" % sqlLocation) setting = {'db': sqlLocation} # set google and output dam self.googleDAM = DAMFactory.createDAM("google") self.outputDAM = DAMFactory.createDAM(options.outputDAM, setting) # set start date and end date if 'all' == options.start: self.start = '19800101' else: self.start = (datetime.datetime.now() + relativedelta(months=-1)).strftime("%Y%m%d") self.end = datetime.datetime.now().strftime("%Y%m%d") if options.dataType in ["quote", "all"]: print("Retrieving quotes start from %s" % self.start) else: print("Retrieving ticks for last 15 days")
''' Created on Nov 6, 2011 @author: ppa ''' from ultrafinance.dam.DAMFactory import DAMFactory from ultrafinance.pyTaLib.indicator import mean, stddev symbols = [] with open("/Users/ppa/workspace/ufweb/conf/crawler.spy_nasdaq.list", 'r') as f: for line in f.readlines(): symbols.append(line.strip()) dam = DAMFactory.createDAM("sql", {'db': 'sqlite:////data/stock.sqlite'}) dateTicks = dam.readBatchTupleQuotes(symbols, 20031210, 20131210) symbolTicks = {} for timeStamp in sorted(dateTicks.iterkeys()): for symbol, tick in dateTicks[timeStamp].iteritems(): if symbol not in symbolTicks: symbolTicks[symbol] = [] symbolTicks[symbol].append(tick) bads = [] goods = {} # symbol as key, std as value for symbol, ticks in symbolTicks.iteritems(): avgClose = mean([tick.close for tick in ticks]) std = 100 * stddev([tick.close for tick in ticks]) / avgClose print "std for %s is %f" % (symbol, std)
''' Created on Nov 6, 2011 @author: ppa ''' from ultrafinance.dam.DAMFactory import DAMFactory from ultrafinance.pyTaLib.indicator import mean, stddev symbols = [] with open("/Users/ppa/workspace/ufweb/conf/crawler.spy_nasdaq.list", 'r') as f: for line in f.readlines(): symbols.append(line.strip()) dam = DAMFactory.createDAM("sql", {'db': 'sqlite:////data/stock.sqlite'}) dateTicks = dam.readBatchTupleQuotes(symbols, 20031210, 20131210) symbolTicks = {} for timeStamp in sorted(dateTicks.iterkeys()): for symbol, tick in dateTicks[timeStamp].iteritems(): if symbol not in symbolTicks: symbolTicks[symbol] = [] symbolTicks[symbol].append(tick) bads = [] goods = {} # symbol as key, std as value for symbol, ticks in symbolTicks.iteritems(): avgClose = mean([tick.close for tick in ticks]) std = 100 * stddev([tick.close for tick in ticks])/avgClose print "std for %s is %f" % (symbol, std)