def __init__(self, symbols, start, poolsize = 5):
''' constructor '''
self.symbols = symbols
self.sqlLocation = None
self.outputDAM = DAMFactory.createDAM("sql", self.__getOutputDamSetting())
self.googleDAM = DAMFactory.createDAM("google")
self.start = start
self.end = None
self.poolsize = poolsize
self.readLock = Lock()
self.writeLock = Lock()
self.failed = []
self.succeeded = []
def __init__(self, symbols, start, poolsize = 5):
''' constructor '''
self.symbols = symbols
self.sqlLocation = None
self.outputDAM = DAMFactory.createDAM("sql", self.__getOutputDamSetting())
self.googleDAM = DAMFactory.createDAM("google")
self.start = start
self.end = None
self.poolsize = poolsize
self.readLock = Lock()
self.writeLock = Lock()
self.failed = []
self.succeeded = []
def _createDam(self, symbol):
''' setup Dam'''
damName = self.__config.getOption(CONF_ULTRAFINANCE_SECTION, CONF_INPUT_DAM)
inputDb = self.__config.getOption(CONF_ULTRAFINANCE_SECTION, CONF_INPUT_DB)
dam = DAMFactory.createDAM(damName, {'db': inputDb})
dam.setSymbol(symbol)
return dam
def _createDam(self, symbol):
""" setup Dam"""
damName = self.__config.getOption(CONF_INPUT_SECTION, CONF_DAM)
setting = self.__config.getSection(CONF_INPUT_SECTION)
dam = DAMFactory.createDAM(damName, setting)
dam.setSymbol(symbol)
return dam
def _createDam(self, symbol):
''' setup Dam'''
damName = self.__config.getOption(CONF_INPUT_SECTION, CONF_DAM)
setting = self.__config.getSection(CONF_INPUT_SECTION)
dam = DAMFactory.createDAM(damName, setting)
dam.setSymbol(symbol)
return dam
def getOptions(self):
''' crawling data and save to hbase '''
parser = optparse.OptionParser("Usage: %prog [options]")
parser.add_option("-f",
"--symbolFile",
dest="symbolFile",
type="string",
help="file that contains symbols for each line")
parser.add_option("-o",
"--outputDAM",
dest="outputDAM",
default='sql',
type="string",
help="output dam, e.g. sql|hbase")
(options, _) = parser.parse_args()
# get symbols
if options.symbolFile is None or not path.exists(options.symbolFile):
print("Please provide valid file: %s" % options.symbolFile)
exit(4)
# get all symbols
with open(options.symbolFile, 'r') as f:
for line in f.readlines():
self.symbols.append(line.strip())
if not self.symbols:
print("No symbols provided in file %s" % options.symbolFile)
exit(4)
# set output dam
if options.outputDAM not in ["hbase", "sql"]:
print("Please provide valid outputDAM %s" % options.outputDAM)
exit(4)
if 'sql' == options.outputDAM:
sqlLocation = 'sqlite:///%s' % self.getOutputSql()
print("Sqlite location: %s" % sqlLocation)
setting = {'db': sqlLocation}
# set google and output dam
self.googleDAM = DAMFactory.createDAM("google")
self.outputDAM = DAMFactory.createDAM(options.outputDAM, setting)
def _createDam(self, symbol):
''' setup Dam'''
damName = self.__config.getOption(CONF_ULTRAFINANCE_SECTION,
CONF_INPUT_DAM)
inputDb = self.__config.getOption(CONF_ULTRAFINANCE_SECTION,
CONF_INPUT_DB)
dam = DAMFactory.createDAM(damName, {'db': inputDb})
dam.setSymbol(symbol)
return dam
def getOptions(self):
''' crawling data and save to hbase '''
parser = optparse.OptionParser("Usage: %prog [options]")
parser.add_option("-f", "--symbolFile", dest = "symbolFile", type = "string",
help = "file that contains symbols for each line")
parser.add_option("-o", "--outputDAM", dest = "outputDAM",
default = 'sql', type = "string",
help = "output dam, e.g. sql|hbase")
(options, _) = parser.parse_args()
# get symbols
if options.symbolFile is None or not path.exists(options.symbolFile):
print("Please provide valid file: %s" % options.symbolFile)
exit(4)
# get all symbols
with open(options.symbolFile, 'r') as f:
for line in f.readlines():
self.symbols.append(line.strip())
if not self.symbols:
print("No symbols provided in file %s" % options.symbolFile)
exit(4)
# set output dam
if options.outputDAM not in ["hbase", "sql"]:
print("Please provide valid outputDAM %s" % options.outputDAM)
exit(4)
if 'sql' == options.outputDAM:
sqlLocation = 'sqlite:///%s' % self.getOutputSql()
print("Sqlite location: %s" % sqlLocation)
setting = {'db': sqlLocation}
# set google and output dam
self.googleDAM = DAMFactory.createDAM("google")
self.outputDAM = DAMFactory.createDAM(options.outputDAM, setting)
def getOptions(self):
''' crawling data and save to hbase '''
parser = optparse.OptionParser("Usage: %prog [options]")
parser.add_option("-f", "--symbolFile", dest = "symbolFile", type = "string",
help = "file that contains symbols for each line")
parser.add_option("-t", "--dataType", dest = "dataType",
default = 'quote', type = "string",
help = "data type that will be stored, e.g. quote|tick|all")
parser.add_option("-s", "--start", dest = "start",
default = 'month', type = "string",
help = "start date, all|month|day")
parser.add_option("-o", "--outputDAM", dest = "outputDAM",
default = 'sql', type = "string",
help = "output dam, e.g. sql|hbase")
(options, _) = parser.parse_args()
# get symbols
if options.symbolFile is None or not path.exists(options.symbolFile):
print("Please provide valid file: %s" % options.symbolFile)
exit(4)
# get all symbols
with open(options.symbolFile, 'r') as f:
for line in f.readlines():
self.symbols.append(line.strip())
if not self.symbols:
print("No symbols provided in file %s" % options.symbolFile)
exit(4)
# set dataType
if options.dataType not in ["quote", "tick", "all"]:
print("Please provide valid dataType %s" % options.dataType)
exit(4)
# set output dam
if options.outputDAM not in ["hbase", "sql"]:
print("Please provide valid outputDAM %s" % options.outputDAM)
exit(4)
if not (options.start in ['all', 'month', 'day'] or DATE_FORMAT.match(options.start)):
print("Please provide valid start option(all|month|day): %s" % options.outputDAM)
exit(4)
if "quote" == options.dataType:
self.isQuote = True
elif "tick" == options.dataType:
self.isTick = True
else:
self.isQuote = self.isTick = True
print("Retrieving data type: %s" % options.dataType)
if 'sql' == options.outputDAM:
sqlLocation = 'sqlite:///%s' % self.getOutputSql()
print("Sqlite location: %s" % sqlLocation)
setting = {'db': sqlLocation}
# set google and output dam
self.googleDAM = DAMFactory.createDAM("google")
self.outputDAM = DAMFactory.createDAM(options.outputDAM, setting)
# set start date and end date
if 'all' == options.start:
self.start = '19800101'
elif 'month' == options.start:
self.start = (datetime.datetime.now() + relativedelta(months = -1)).strftime("%Y%m%d")
elif 'day' == options.start:
self.start = (datetime.datetime.now() + relativedelta(days = -1)).strftime("%Y%m%d")
else:
self.start = options.start
self.end = datetime.datetime.now().strftime("%Y%m%d")
if options.dataType in ["quote", "all"]:
print("Retrieving quotes start from %s" % self.start)
else:
print("Retrieving ticks for last 15 days")
def getOptions(self):
''' crawling data and save to hbase '''
parser = optparse.OptionParser("Usage: %prog [options]")
parser.add_option("-f",
"--symbolFile",
dest="symbolFile",
type="string",
help="file that contains symbols for each line")
parser.add_option(
"-t",
"--dataType",
dest="dataType",
default='quote',
type="string",
help="data type that will be stored, e.g. quote|tick|all")
parser.add_option("-s",
"--start",
dest="start",
default='month',
type="string",
help="start date, all|month")
parser.add_option("-o",
"--outputDAM",
dest="outputDAM",
default='sql',
type="string",
help="output dam, e.g. sql|hbase")
(options, _) = parser.parse_args()
# get symbols
if options.symbolFile is None or not path.exists(options.symbolFile):
print("Please provide valid file: %s" % options.symbolFile)
exit(4)
# get all symbols
with open(options.symbolFile, 'r') as f:
for line in f.readlines():
self.symbols.append(line.strip())
if not self.symbols:
print("No symbols provided in file %s" % options.symbolFile)
exit(4)
# set dataType
if options.dataType not in ["quote", "tick", "all"]:
print("Please provide valid dataType %s" % options.dataType)
exit(4)
# set output dam
if options.outputDAM not in ["hbase", "sql"]:
print("Please provide valid outputDAM %s" % options.outputDAM)
exit(4)
if options.start not in ['all', 'month']:
print("Please provide valid start option(all|month): %s" %
options.outputDAM)
exit(4)
if "quote" == options.dataType:
self.isQuote = True
elif "tick" == options.dataType:
self.isTick = True
else:
self.isQuote = self.isTick = True
print("Retrieving data type: %s" % options.dataType)
if 'sql' == options.outputDAM:
sqlLocation = 'sqlite:///%s' % self.getOutputSql()
print("Sqlite location: %s" % sqlLocation)
setting = {'db': sqlLocation}
# set google and output dam
self.googleDAM = DAMFactory.createDAM("google")
self.outputDAM = DAMFactory.createDAM(options.outputDAM, setting)
# set start date and end date
if 'all' == options.start:
self.start = '19800101'
else:
self.start = (datetime.datetime.now() +
relativedelta(months=-1)).strftime("%Y%m%d")
self.end = datetime.datetime.now().strftime("%Y%m%d")
if options.dataType in ["quote", "all"]:
print("Retrieving quotes start from %s" % self.start)
else:
print("Retrieving ticks for last 15 days")
'''
Created on Nov 6, 2011
@author: ppa
'''
from ultrafinance.dam.DAMFactory import DAMFactory
from ultrafinance.pyTaLib.indicator import mean, stddev
symbols = []
with open("/Users/ppa/workspace/ufweb/conf/crawler.spy_nasdaq.list", 'r') as f:
for line in f.readlines():
symbols.append(line.strip())
dam = DAMFactory.createDAM("sql", {'db': 'sqlite:////data/stock.sqlite'})
dateTicks = dam.readBatchTupleQuotes(symbols, 20031210, 20131210)
symbolTicks = {}
for timeStamp in sorted(dateTicks.iterkeys()):
for symbol, tick in dateTicks[timeStamp].iteritems():
if symbol not in symbolTicks:
symbolTicks[symbol] = []
symbolTicks[symbol].append(tick)
bads = []
goods = {} # symbol as key, std as value
for symbol, ticks in symbolTicks.iteritems():
avgClose = mean([tick.close for tick in ticks])
std = 100 * stddev([tick.close for tick in ticks]) / avgClose
print "std for %s is %f" % (symbol, std)
'''
Created on Nov 6, 2011
@author: ppa
'''
from ultrafinance.dam.DAMFactory import DAMFactory
from ultrafinance.pyTaLib.indicator import mean, stddev
symbols = []
with open("/Users/ppa/workspace/ufweb/conf/crawler.spy_nasdaq.list", 'r') as f:
for line in f.readlines():
symbols.append(line.strip())
dam = DAMFactory.createDAM("sql", {'db': 'sqlite:////data/stock.sqlite'})
dateTicks = dam.readBatchTupleQuotes(symbols, 20031210, 20131210)
symbolTicks = {}
for timeStamp in sorted(dateTicks.iterkeys()):
for symbol, tick in dateTicks[timeStamp].iteritems():
if symbol not in symbolTicks:
symbolTicks[symbol] = []
symbolTicks[symbol].append(tick)
bads = []
goods = {} # symbol as key, std as value
for symbol, ticks in symbolTicks.iteritems():
avgClose = mean([tick.close for tick in ticks])
std = 100 * stddev([tick.close for tick in ticks])/avgClose
print "std for %s is %f" % (symbol, std)