def build_feed(instruments, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False):
logger = pyalgotrade.logger.getLogger("yahoofinance")
ret = yahoofeed.Feed(frequency, timezone)
if not os.path.exists(storage):
logger.info("Creating %s directory" % (storage))
os.mkdir(storage)
for year in range(fromYear, toYear+1):
for instrument in instruments:
fileName = os.path.join(storage, "%s-%d-yahoofinance.csv" % (instrument, year))
if not os.path.exists(fileName):
logger.info("Downloading %s %d to %s" % (instrument, year, fileName))
try:
if frequency == bar.Frequency.DAY:
download_daily_bars(instrument, year, fileName)
elif frequency == bar.Frequency.WEEK:
download_weekly_bars(instrument, year, fileName)
else:
raise Exception("Invalid frequency")
except Exception, e:
if skipErrors:
logger.error(str(e))
continue
else:
raise e
ret.addBarsFromCSV(instrument, fileName)
def OnRtnDepthMarketData(self, *args):
logger.info("OnRtnDepthMarketData")
logger.info("id: "+args[0].InstrumentID)
logger.info("TradingDay: " + args[0].TradingDay + " " + args[0].UpdateTime)
logger.info("LastPrice: " + str(args[0].LastPrice))
if self.__insertIntoMysql:
if self.__mysqlCon == None:
self.__mysqlCon = client.mysqlConnection(CONSTANTS.HOST,
CONSTANTS.USERNAME,
CONSTANTS.PASSWORD,
CONSTANTS.DATABASE)
date = datetime.strptime(args[0].TradingDay, "%Y%m%d")
dateStr = date.strftime("%Y-%m-%d")
dateStr = dateStr + " " + args[0].UpdateTime
self.__mysqlCon.addBar(args[0].InstrumentID,
RealTimeBar(dateStr,
args[0].LastPrice,
args[0].Volume))
if self.__dumpToFile:
try:
self.dumpToFile(args[0])
except Exception as e:
print "except", e
logger.info("OnRtnDepthMarketData End")
def stop(self):
try:
if self.__thread is not None and self.__thread.is_alive():
logger.info("Shutting down client.")
self.__stream.disconnect()
except Exception, e:
logger.error("Error disconnecting stream: %s." % (str(e)))
def stop(self):
try:
if self.__thread is not None and self.__thread.is_alive():
logger.info("Shutting down client.")
self.__stream.disconnect()
except Exception as e:
logger.error("Error disconnecting stream: %s." % (str(e)))
def update_bars(dbFilePath, symbolsFile, timezone, fromYear, toYear):
db = sqlitefeed.Database(dbFilePath)
for symbol in open(symbolsFile, "r"):
symbol = symbol.strip()
logger.info("Downloading %s bars" % (symbol))
for year in range(fromYear, toYear+1):
download_bars(db, symbol, year, timezone)
def build_feed(instruments,
fromYear,
toYear,
storage,
frequency=bar.Frequency.DAY,
timezone=None,
skipErrors=False):
logger = pyalgotrade.logger.getLogger("yahoofinance")
ret = yahoofeed.Feed(frequency, timezone)
if not os.path.exists(storage):
logger.info("Creating %s directory" % (storage))
os.mkdir(storage)
for year in range(fromYear, toYear + 1):
for instrument in instruments:
fileName = os.path.join(
storage, "%s-%d-yahoofinance.csv" % (instrument, year))
if not os.path.exists(fileName):
logger.info("Downloading %s %d to %s" %
(instrument, year, fileName))
try:
if frequency == bar.Frequency.DAY:
download_daily_bars(instrument, year, fileName)
elif frequency == bar.Frequency.WEEK:
download_weekly_bars(instrument, year, fileName)
else:
raise Exception("Invalid frequency")
except Exception, e:
if skipErrors:
logger.error(str(e))
continue
else:
raise e
ret.addBarsFromCSV(instrument, fileName)
def getNextBars(self):
if not self.__data_downloaded:
logger.info('featch history of {0}'.format(self.__instrument))
for i in self.getFrequencies():
tmp = None
if i == Frequency.DAY:
tmp = history(self.__instrument, self.__start_date, None,
Frequency.DAY, add_missing_dates=False)
elif i == Frequency.HOUR:
tmp = history(self.__instrument, self.__start_date, None,
Frequency.HOUR, add_missing_dates=False)
if tmp is None:
continue
for date, row in tmp.iloc[0].iterrows():
tmpbar = bar.BasicBar(date, row['open'], row['high'],
row['low'], row['close'], 0, False,
i)
self.__bars_buf.append(tmpbar)
self.__data_downloaded = True
self.__bars_buf.sort(key=lambda i: i.getDateTime())
while True:
if self.__bars_buf:
tmp = self.__bars_buf.pop(0)
return bar.Bars({self.__instrument: tmp}, frequecy=tmp.getFrequency())
else:
self.generateBars()
def __threadMain(self):
try:
logger.info("Initializing client.")
self.__stream.filter(track=self.__track, follow=self.__follow, languages=self.__languages)
finally:
logger.info("Client finished.")
self.__running = False
def doCall(self):
barDict = {}
try:
a = time.time()
response = ts.get_realtime_quotes(self.__identifiers)
logger.info('response cost: %f' % (time.time() - a))
if self.__last_response_time != response.iloc[-1]['time']:
for identifier in self.__identifiers:
barDict[identifier] = build_bar(
response[response.code ==
identifier].iloc[-1]) # dataFrame 格式转换
else:
logger.info(
"bar is the same with previous bar at time %s,not refresh"
% response.iloc[-1]['time'])
self.__last_response_time = response.iloc[-1]['time']
except:
logger.error("tushare time out")
import traceback
traceback.print_exc()
if len(barDict):
bars = bar.Ticks(barDict)
self.__queue.put((GetBarThread.ON_BARS, bars))
def dispatchImpl(self, eventFilter):
try:
eventType, eventData = self.__queue.get(True, Client.QUEUE_TIMEOUT)
if eventFilter is not None and eventType not in eventFilter:
return
if eventType == WSClient.ON_TICKER:
self.__tickerEvent.emit(eventData)
elif eventType == WSClient.ON_TRADE:
self.__tradeEvent.emit(eventData)
elif eventType == WSClient.ON_USER_ORDER:
self.__userOrderEvent.emit(eventData)
elif eventType == WSClient.ON_RESULT:
requestId, result = eventData
logger.info("Result: %s - %s" % (requestId, result))
elif eventType == WSClient.ON_REMARK:
requestId, data = eventData
logger.info("Remark: %s - %s" % (requestId, data))
elif eventType == WSClient.ON_CONNECTED:
self.__onConnected()
elif eventType == WSClient.ON_DISCONNECTED:
self.__onDisconnected()
else:
logger.error("Invalid event received to dispatch: %s - %s" % (eventType, eventData))
except Queue.Empty:
pass
def deleteDataFromDate(self, date):
cursor = self.__con.cursor()
query = "delete from data where date(date) >= '{0:s}' and date(date) <= '{0:s}'"
query = query.format(date)
logger.info(query)
cursor.execute(query)
self.__con.commit()
cursor.close()
def stop(self):
try:
self.__stopped = True
if self.__thread is not None and self.__thread.is_alive():
logger.info("Shutting down MtGox client.")
self.__wsClient.stopClient()
except Exception, e:
logger.error("Error shutting down MtGox client: %s" % (str(e)))
def download_symbols(market):
market = market.upper()
pages = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"]
for x in pages:
url = "http://www.eoddata.com/stocklist/%s/%s.htm" % (market, x)
logger.info("Processing %s" % url)
for symbol in get_symbols_from_page(url):
yield symbol
def stop(self):
try:
self.__stopped = True
if self.__thread is not None and self.__thread.is_alive():
logger.info("Stopping websocket client.")
self.__thread.stop()
except Exception as e:
logger.error("Error shutting down client: %s" % (str(e)))
def __onDisconnected(self):
if self.__enableReconnection:
logger.info("Reconnecting")
while not self.__stopped and not self.__initializeClient():
pass
elif not self.__stopped:
logger.info("Stopping")
self.__stopped = True
def OnRspSubMarketData(self, *args):
logger.info("OnRspSubMarketData")
if args:
rspInfoField = args[1]
logger.info("Subscribe Instrument " + args[0].InstrumentID)
logger.info("errorID " + str(rspInfoField.ErrorID))
logger.info("errorMsg " + str(rspInfoField.ErrorMsg))
logger.info("OnRspSubMarketData End")
def state_init(self, bars, states):
# You are only supposed to save states in states variable
# DO NOT save your local variable and it is not guaranteed to be supported later
logger.info('INIT')
print(states.prev)
states.prev = 'INIT'
self.print_bars(bars)
return SampleStrategyFSMState.STATE1
def stop(self):
try:
self.__stopped = True
if self.__thread is not None and self.__thread.is_alive():
logger.info("Shutting down MtGox client.")
self.__wsClient.stopClient()
except Exception, e:
logger.error("Error shutting down MtGox client: %s" % (str(e)))
def run(self):
logger.info("Thread started")
while not self.__stopped:
self.__wait()
if not self.__stopped:
try:
self.doCall()
except Exception, e:
logger.critical("Unhandled exception", exc_info=e)
def run(self):
logger.info("Thread started")
while not self.__stopped:
self.__wait()
if not self.__stopped:
try:
self.doCall()
except Exception, e:
logger.critical("Unhandled exception", exc_info=e)
def build_feed(sourceCode, tableCodes, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False, noAdjClose=False, authToken=None):
"""Build and load a :class:`pyalgotrade.barfeed.quandlfeed.Feed` using CSV files downloaded from Quandl.
CSV files are downloaded if they haven't been downloaded before.
:param sourceCode: The dataset source code.
:type sourceCode: string.
:param tableCodes: The dataset table codes.
:type tableCodes: list.
:param fromYear: The first year.
:type fromYear: int.
:param toYear: The last year.
:type toYear: int.
:param storage: The path were the files will be loaded from, or downloaded to.
:type storage: string.
:param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** or **pyalgotrade.bar.Frequency.WEEK**
are supported.
:param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`.
:type timezone: A pytz timezone.
:param skipErrors: True to keep on loading/downloading files in case of errors.
:type skipErrors: boolean.
:param noAdjClose: True if the instruments don't have adjusted close values.
:type noAdjClose: boolean.
:param authToken: Optional. An authentication token needed if you're doing more than 50 calls per day.
:type authToken: string.
:rtype: :class:`pyalgotrade.barfeed.quandlfeed.Feed`.
"""
logger = pyalgotrade.logger.getLogger("quandl")
ret = quandlfeed.Feed(frequency, timezone)
if noAdjClose:
ret.setNoAdjClose()
if not os.path.exists(storage):
logger.info("Creating %s directory" % (storage))
os.mkdir(storage)
for year in range(fromYear, toYear+1):
for tableCode in tableCodes:
fileName = os.path.join(storage, "%s-%s-%d-quandl.csv" % (sourceCode, tableCode, year))
if not os.path.exists(fileName):
logger.info("Downloading %s %d to %s" % (tableCode, year, fileName))
try:
if frequency == bar.Frequency.DAY:
download_daily_bars(sourceCode, tableCode, year, fileName, authToken)
elif frequency == bar.Frequency.WEEK:
download_weekly_bars(sourceCode, tableCode, year, fileName, authToken)
else:
raise Exception("Invalid frequency")
except Exception as e:
if skipErrors:
logger.error(str(e))
continue
else:
raise e
ret.addBarsFromCSV(tableCode, fileName)
return ret
def build_feed(instruments,
fromYear,
toYear,
storage,
frequency=bar.Frequency.DAY,
timezone=None,
skipErrors=False):
"""Build and load a :class:`pyalgotrade.barfeed.yahoofeed.Feed` using CSV files downloaded from Yahoo! Finance.
CSV files are downloaded if they haven't been downloaded before.
:param instruments: Instrument identifiers.
:type instruments: list.
:param fromYear: The first year.
:type fromYear: int.
:param toYear: The last year.
:type toYear: int.
:param storage: The path were the files will be loaded from, or downloaded to.
:type storage: string.
:param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** or **pyalgotrade.bar.Frequency.WEEK**
are supported.
:param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`.
:type timezone: A pytz timezone.
:param skipErrors: True to keep on loading/downloading files in case of errors.
:type skipErrors: boolean.
:rtype: :class:`pyalgotrade.barfeed.yahoofeed.Feed`.
"""
logger = pyalgotrade.logger.getLogger("yahoofinance")
ret = yahoofeed.Feed(frequency, timezone)
if not os.path.exists(storage):
logger.info("Creating %s directory" % (storage))
os.mkdir(storage)
for year in range(fromYear, toYear + 1):
for instrument in instruments:
fileName = os.path.join(
storage, "%s-%d-yahoofinance.csv" % (instrument, year))
if not os.path.exists(fileName):
logger.info("Downloading %s %d to %s" %
(instrument, year, fileName))
try:
if frequency == bar.Frequency.DAY:
download_daily_bars(instrument, year, fileName)
elif frequency == bar.Frequency.WEEK:
download_weekly_bars(instrument, year, fileName)
else:
raise Exception("Invalid frequency")
except Exception as e:
if skipErrors:
logger.error(str(e))
continue
else:
raise e
ret.addBarsFromCSV(instrument, fileName)
return ret
def onOpened(self):
for channel in self.__pending_subscriptions:
logger.info("Subscribing to channel %s." % channel)
self.send(
json.dumps({
"event": "bts:subscribe",
"data": {
"channel": channel
}
}))
def __onDisconnected(self):
if self.__enableReconnection:
initialized = False
while not self.__stopped and not initialized:
logger.info("Reconnecting")
initialized = self.__initializeClient()
if not initialized:
time.sleep(5)
else:
self.__stopped = True
def main():
try:
writer = symbolsxml.Writer()
for symbol in open("merval-symbols.txt", "r"):
symbol = symbol.strip()
process_symbol(writer, symbol)
logger.info("Writing merval.xml")
writer.write("merval.xml")
except Exception, e:
logger.error(str(e))
def OnFrontConnected(self):
logger.info("OnFrontConnected")
f = CThostFtdcReqUserLoginField()
f.BrokerID = self.__broker_id
f.UserUD = self.__user_id
f.Password = self.__password
self.__md.ReqUserLogin(f, self.__reqNum)
self.__reqNum = self.__reqNum + 1
if (datetime.now() - self.__starttime) > timedelta(seconds=60*60*10):
self.stopClient()
def __onDisconnected(self):
if self.__enableReconnection:
initialized = False
while not self.__stopped and not initialized:
logger.info("Reconnecting")
initialized = self.__initializeClient()
if not initialized:
time.sleep(5)
else:
self.__stopped = True
def generateBars(self):
if self.__pullDelay > 0:
time.sleep(self.__pullDelay)
logger.info('get quote of {0}'.format(self.__instrument))
tmp = quote(self.__instrument)
if tmp is None:
logger.error('failed to get {0} quote'.format(self.__instrument))
return
tmpVal = tmp.iloc[0]['close'].iloc[0]
curTime = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
tmp = bar.BasicBar(curTime,
tmpVal, tmpVal, tmpVal, tmpVal,
0, False, Frequency.REALTIME)
for freq in self.__nextRealtimeBars.keys():
if self.__nextRealtimeBars[freq]['open'] is None:
self.__nextRealtimeBars[freq]['open'] = tmpVal
self.__nextRealtimeBars[freq]['high'] = tmpVal
self.__nextRealtimeBars[freq]['low'] = tmpVal
self.__nextRealtimeBars[freq]['close'] = tmpVal
if freq == Frequency.DAY:
curTime = curTime.replace(hour=0, minute=0, second=0, microsecond=0)
elif freq == Frequency.HOUR:
curTime = curTime.replace(minute=0, second=0, microsecond=0)
elif freq == Frequency.MINUTE:
curTime = curTime.replace(second=0, microsecond=0)
elif freq != Frequency.REALTIME:
logger.error('{0} is not supported.'.format(freq))
assert False
self.__nextRealtimeBars[freq]['start'] = curTime
else:
if tmpVal > self.__nextRealtimeBars[freq]['high']:
self.__nextRealtimeBars[freq]['high'] = tmpVal
elif tmpVal < self.__nextRealtimeBars[freq]['low']:
self.__nextRealtimeBars[freq]['low'] = tmpVal
deltaTime = curTime - self.__nextRealtimeBars[freq]['start']
if (Frequency.MINUTE == freq and deltaTime.total_seconds() > 60 or
Frequency.DAY == freq and deltaTime.total_seconds() > 60*60*24 or
Frequency.HOUR == freq and deltaTime.total_seconds() > 60*60):
self.__nextRealtimeBars[freq]['close'] = tmpVal
row = self.__nextRealtimeBars[freq]
tmpbar = bar.BasicBar(curTime, row['open'], row['high'],
row['low'], row['close'], 0, False,
freq)
self.__bars_buf.append(tmpbar)
self.__nextRealtimeBars[freq] = {
'open': None,
'high': None,
'low': None,
'close': None,
'start' : None,
}
self.__bars_buf.append(tmp)
if self.__isRealTime:
self.__bars_buf.append(tmp)
self.__bars_buf.sort(key=lambda i: i.getDateTime())
def build_feed(instruments,
fromYear,
toYear,
storage,
frequency=bar.Frequency.DAY,
skipErrors=False):
"""Build and load a :class:`pyalgotrade.barfeed.tusharefeed.Feed` using CSV files downloaded from Tushare Library.
CSV files are downloaded if they haven't been downloaded before.
:param instruments: Instrument identifiers.
:type instruments: list.
:param fromYear: The first year.
:type fromYear: int.
:param toYear: The last year.
:type toYear: int.
:param storage: The path were the files will be loaded from, or downloaded to.
:type storage: string.
:param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** is currently supported.
:param skipErrors: True to keep on loading/downloading files in case of errors.
:type skipErrors: boolean.
:rtype: :class:`pyalgotrade.barfeed.tusharefeed.Feed`.
"""
logger = pyalgotrade.logger.getLogger("tushare")
ret = barfeed.Feed(frequency)
if not os.path.exists(storage):
logger.info("Creating {dirname} directory".format(dirname=storage))
os.mkdir(storage)
for year in range(fromYear, toYear + 1):
for instrument in instruments:
filePath = Path(storage)
fileName = filePath / "{instrument}-{year}-tushare.csv".format(
instrument=instrument, year=year)
if not os.path.exists(fileName):
logger.info(
"Downloading {instrument} {year} to {filename}".format(
instrument=instrument, year=year, filename=fileName))
try:
if frequency == bar.Frequency.DAY:
download_daily_bars(instrument, year, fileName)
else:
raise Exception("Invalid frequency")
except Exception as e:
if skipErrors:
logger.error(str(e))
continue
else:
raise e
ret.addBarsFromCSV(instrument, fileName)
return ret
def build_feed(sourceCode, tableCodes, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False, noAdjClose=False, authToken=None):
"""Build and load a :class:`pyalgotrade.barfeed.quandlfeed.Feed` using CSV files downloaded from Quandl.
CSV files are downloaded if they haven't been downloaded before.
:param sourceCode: The dataset source code.
:type sourceCode: string.
:param tableCodes: The dataset table codes.
:type tableCodes: list.
:param fromYear: The first year.
:type fromYear: int.
:param toYear: The last year.
:type toYear: int.
:param storage: The path were the files will be loaded from, or downloaded to.
:type storage: string.
:param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** or **pyalgotrade.bar.Frequency.WEEK**
are supported.
:param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`.
:type timezone: A pytz timezone.
:param skipErrors: True to keep on loading/downloading files in case of errors.
:type skipErrors: boolean.
:param noAdjClose: True if the instruments don't have adjusted close values.
:type noAdjClose: boolean.
:param authToken: Optional. An authentication token needed if you're doing more than 50 calls per day.
:type authToken: string.
:rtype: :class:`pyalgotrade.barfeed.quandlfeed.Feed`.
"""
logger = pyalgotrade.logger.getLogger("quandl")
ret = quandlfeed.Feed(frequency, timezone)
if noAdjClose:
ret.setNoAdjClose()
if not os.path.exists(storage):
logger.info("Creating %s directory" % (storage))
os.mkdir(storage)
for year in range(fromYear, toYear+1):
for tableCode in tableCodes:
fileName = os.path.join(storage, "%s-%s-%d-quandl.csv" % (sourceCode, tableCode, year))
if not os.path.exists(fileName):
logger.info("Downloading %s %d to %s" % (tableCode, year, fileName))
try:
if frequency == bar.Frequency.DAY:
download_daily_bars(sourceCode, tableCode, year, fileName, authToken)
elif frequency == bar.Frequency.WEEK:
download_weekly_bars(sourceCode, tableCode, year, fileName, authToken)
else:
raise Exception("Invalid frequency")
except Exception, e:
if skipErrors:
logger.error(str(e))
continue
else:
raise e
ret.addBarsFromCSV(tableCode, fileName)
def build_feed(instruments,
fromYear,
toYear,
storage,
frequency=bar.Frequency.DAY,
timezone=None,
skipErrors=False,
rowFilter=None):
"""Build and load a :class:`pyalgotrade.barfeed.googlefeed.Feed` using CSV files downloaded from Google Finance.
CSV files are downloaded if they haven't been downloaded before.
:param instruments: Instrument identifiers.
:type instruments: list.
:param fromYear: The first year.
:type fromYear: int.
:param toYear: The last year.
:type toYear: int.
:param storage: The path were the files will be loaded from, or downloaded to.
:type storage: string.
:param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** is currently supported.
:param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`.
:type timezone: A pytz timezone.
:param skipErrors: True to keep on loading/downloading files in case of errors.
:type skipErrors: boolean.
:rtype: :class:`pyalgotrade.barfeed.googlefeed.Feed`.
"""
logger = pyalgotrade.logger.getLogger("googlefinance")
ret = googlefeed.Feed(frequency, timezone)
if not os.path.exists(storage):
logger.info("Creating {dirname} directory".format(dirname=storage))
os.mkdir(storage)
for year in range(fromYear, toYear + 1):
for instrument in instruments:
fileName = os.path.join(
storage, "{instrument}-{year}-googlefinance.csv".format(
instrument=instrument, year=year))
if not os.path.exists(fileName):
logger.info(
"Downloading {instrument} {year} to {filename}".format(
instrument=instrument, year=year, filename=fileName))
try:
if frequency == bar.Frequency.DAY:
download_daily_bars(instrument, year, fileName)
else:
raise Exception("Invalid frequency")
except Exception, e:
if skipErrors:
logger.error(str(e))
continue
else:
raise e
ret.addBarsFromCSV(instrument, fileName, rowFilter=rowFilter)
def download_symbols(market):
market = market.upper()
pages = [
"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N",
"O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"
]
for x in pages:
url = "http://www.eoddata.com/stocklist/%s/%s.htm" % (market, x)
logger.info("Processing %s" % url)
for symbol in get_symbols_from_page(url):
yield symbol
def main():
try:
htmlTree = get_html()
table = find_table(htmlTree)
if table is None:
raise Exception("S&P 500 Component Stocks table not found")
symbolsXML = parse_results(table)
logger.info("Writing sp500.xml")
symbolsXML.write("sp500.xml")
except Exception as e:
logger.error(str(e))
def main():
try:
htmlTree = get_html()
table = find_table(htmlTree)
if table is None:
raise Exception("S&P 500 Component Stocks table not found")
symbolsXML = parse_results(table)
logger.info("Writing sp500.xml")
symbolsXML.write("sp500.xml")
except Exception, e:
logger.error(str(e))
def wrapper(*args, **kwargs):
try:
rtn = func(*args, **kwargs)
return rtn
except Exception:
info = traceback.format_exc()
logger.error('-' * 60)
logger.error(bugart + '\n' + info)
logger.error('-' * 60)
return exception_rtn
except KeyboardInterrupt:
logger.info('KeyboardInterrupt received, terminating...')
sys.exit(0)
def build_feed(instruments, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False):
"""Build and load a :class:`pyalgotrade.barfeed.googlefeed.Feed` using CSV files downloaded from Google Finance.
CSV files are downloaded if they haven't been downloaded before.
:param instruments: Instrument identifiers.
:type instruments: list.
:param fromYear: The first year.
:type fromYear: int.
:param toYear: The last year.
:type toYear: int.
:param storage: The path were the files will be loaded from, or downloaded to.
:type storage: string.
:param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** is currently supported.
:param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`.
:type timezone: A pytz timezone.
:param skipErrors: True to keep on loading/downloading files in case of errors.
:type skipErrors: boolean.
:rtype: :class:`pyalgotrade.barfeed.googlefeed.Feed`.
"""
logger = pyalgotrade.logger.getLogger("googlefinance")
ret = googlefeed.Feed(frequency, timezone)
if not os.path.exists(storage):
logger.info("Creating {dirname} directory".format(dirname=storage))
os.mkdir(storage)
for year in range(fromYear, toYear+1):
for instrument in instruments:
fileName = os.path.join(
storage,
"{instrument}-{year}-googlefinance.csv".format(
instrument=instrument, year=year))
if not os.path.exists(fileName):
logger.info(
"Downloading {instrument} {year} to {filename}".format(
instrument=instrument, year=year, filename=fileName))
try:
if frequency == bar.Frequency.DAY:
download_daily_bars(instrument, year, fileName)
else:
raise Exception("Invalid frequency")
except Exception as e:
if skipErrors:
logger.error(str(e))
continue
else:
raise e
ret.addBarsFromCSV(instrument, fileName)
return ret
def main():
parser = argparse.ArgumentParser(description="Quandl utility")
parser.add_argument("--auth-token", required=False,
help="An authentication token needed if you're doing more than 50 calls per day")
parser.add_argument("--source-code", required=True,
help="The dataset source code")
parser.add_argument("--table-code", required=True,
help="The dataset table code")
parser.add_argument("--from-year", required=True,
type=int, help="The first year to download")
parser.add_argument("--to-year", required=True, type=int,
help="The last year to download")
parser.add_argument("--storage", required=True,
help="The path were the files will be downloaded to")
parser.add_argument("--force-download", action='store_true',
help="Force downloading even if the files exist")
parser.add_argument("--ignore-errors", action='store_true',
help="True to keep on downloading files in case of errors")
parser.add_argument("--frequency", default="daily", choices=[
"daily", "weekly"], help="The frequency of the bars. Only daily or weekly are supported")
args = parser.parse_args()
logger = pyalgotrade.logger.getLogger("quandl")
if not os.path.exists(args.storage):
logger.info("Creating %s directory" % (args.storage))
os.mkdir(args.storage)
for year in range(args.from_year, args.to_year + 1):
fileName = os.path.join(args.storage, "%s-%s-%d-quandl.csv" %
(args.source_code, args.table_code, year))
if not os.path.exists(fileName) or args.force_download:
logger.info("Downloading %s %d to %s" %
(args.table_code, year, fileName))
try:
if args.frequency == "daily":
download_daily_bars(
args.source_code, args.table_code, year, fileName, args.auth_token)
else:
assert args.frequency == "weekly", "Invalid frequency"
download_weekly_bars(
args.source_code, args.table_code, year, fileName, args.auth_token)
except Exception as e:
if args.ignore_errors:
logger.error(str(e))
continue
else:
raise
def getNextJob(self):
ret = None
# Get the next set of parameters.
params = self.__paramSource.getNext(self.defaultBatchSize)
params = map(lambda p: p.args, params)
logger.info("Sending %s work items" % (len(params)))
# Map the active job
if len(params):
ret = Job(params)
with self.__activeJobsLock:
self.__activeJobs[ret.getId()] = ret
return pickle.dumps(ret)
def backFill(self, freq,filename, startDate, endDate):
if freq == 86400:
final_table = "1day_data"
elif freq == 1800:
final_table = "30mins_data"
elif freq == 300:
final_table = "5mins_data"
cursor = self.__con.cursor()
cleanup_qry_text = ("""DELETE FROM %s WHERE date(date) >= '%s' and date(date) <= '%s'""" % (final_table, startDate, endDate))
logger.info(cleanup_qry_text)
cursor.execute(cleanup_qry_text)
insert_qry_text = """INSERT INTO {0:s} (symbol, date, open, close, high, low, volume)
select t0.symbol, t0.new_date date, t2.open, t1.close, t0.high,
t0.low, t0.volume
from (select data.symbol symbol,from_unixtime((floor((unix_timestamp(data.date) / {1:d})) * {1:d}){2:s}) new_date,
max(data.high) high, min(data.low) low, max(data.date) max_ts, min(data.date) min_ts,
avg(data.volume) volume
from data where date(date) >= '{3:s}' and date(date) <= '{4:s}' group by 1,2) t0
join
(select tbl2.symbol, tbl2.date, tbl2.close close from data tbl2
join
(select symbol, date, max(milliseconds) max_milli_secs from data
where date(date) >= '{3:s}' and date(date) <= '{4:s}'
group by 1,2) tbl1
on tbl2.symbol = tbl1.symbol and tbl2.date = tbl1.date and tbl2.milliseconds = tbl1.max_milli_secs
where date(tbl2.date) >= '{3:s}' and date(tbl2.date) <= '{4:s}'
group by 1,2) t1
on t0.symbol = t1.symbol and t0.max_ts = t1.date
join
(select tbl2.symbol, tbl2.date, tbl2.open open from data tbl2
join
(select symbol, date, min(milliseconds) min_milli_secs from data
where date(date) >= '{3:s}' and date(date) <= '{4:s}'
group by 1,2) tbl1
on tbl2.symbol = tbl1.symbol and tbl2.date = tbl1.date and tbl2.milliseconds = tbl1.min_milli_secs
where date(tbl2.date) >= '{3:s}' and date(tbl2.date) <= '{4:s}'
group by 1,2) t2
on t0.symbol = t2.symbol and t0.min_ts = t2.date"""
if freq == CONSTANTS.ONE_DAY:
insert_qry_text = insert_qry_text.format(final_table, freq, '+8*3600', startDate, endDate)
else:
insert_qry_text = insert_qry_text.format(final_table, freq, '', startDate, endDate)
logger.info(insert_qry_text)
cursor.execute(insert_qry_text)
self.__con.commit()
cursor.close()
text_file = open(filename, "a")
text_file.writelines("Back fill job %s started at: %s, complete at: %s , end date %s\n" % (final_table, startDate, datetime.now(), endDate))
text_file.close()
def push(self, result, parameters):
"""
Push strategy results obtained by running the strategy with the given parameters.
:param result: The result obtained by running the strategy with the given parameters.
:type result: float
:param parameters: The parameters that yield the given result.
:type parameters: Parameters
"""
with self.__lock:
if result is not None and (self.__bestResult is None or result > self.__bestResult):
self.__bestResult = result
self.__bestParameters = parameters
logger.info("Best result so far %s with parameters %s" % (result, parameters.args))
def main():
try:
writer = symbolsxml.Writer()
for symbol in open("merval-symbols.txt", "r"):
symbol = symbol.strip()
process_symbol(writer, symbol)
# Index
writer.addIndex("^MERV", "Merval")
logger.info("Writing merval.xml")
writer.write("merval.xml")
except Exception, e:
logger.error(str(e))
def build_feed(instruments, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False):
"""Build and load a :class:`pyalgotrade.barfeed.yahoofeed.Feed` using CSV files downloaded from Yahoo! Finance.
CSV files are downloaded if they haven't been downloaded before.
:param instruments: Instrument identifiers.
:type instruments: list.
:param fromYear: The first year.
:type fromYear: int.
:param toYear: The last year.
:type toYear: int.
:param storage: The path were the files will be loaded from, or downloaded to.
:type storage: string.
:param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** or **pyalgotrade.bar.Frequency.WEEK**
are supported.
:param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`.
:type timezone: A pytz timezone.
:param skipErrors: True to keep on loading/downloading files in case of errors.
:type skipErrors: boolean.
:rtype: :class:`pyalgotrade.barfeed.tusharefeed.Feed`.
"""
logger = pyalgotrade.logger.getLogger("tushare")
ret = tusharefeed.Feed(frequency, timezone)
datapath = os.path.join(storage, "data")
if not os.path.exists(datapath):
logger.info("Creating %s directory" % (datapath))
os.mkdir(storage)
for year in range(fromYear, toYear+1):
for instrument in instruments:
fileName = os.path.join(datapath, "%s-%d-tushare.csv" % (instrument, year))
if not os.path.exists(fileName):
logger.info("Downloading %s %d to %s" % (instrument, year, fileName))
try:
if frequency == bar.Frequency.DAY:
download_daily_bars(instrument, year, fileName)
elif frequency == bar.Frequency.WEEK:
download_weekly_bars(instrument, year, fileName)
else:
raise Exception("Invalid frequency")
except Exception, e:
if skipErrors:
logger.error(str(e))
continue
else:
raise e
ret.addBarsFromCSV(instrument, fileName)
def parse_results(table):
ret = symbolsxml.Writer()
logger.info("Parsing table")
rows = table.xpath("tr")
for row in rows[1:]:
cols = row.xpath("td")
tickerSymbol = cols[TICKER_SYMBOL_COL].xpath("a[1]")[0].text
company = cols[COMPANY_COL].xpath("a[1]")[0].text
gics = cols[GICS_COL].text
gicsSubIndustry = cols[GICS_SUB_INDUSTRY_COL].text
if gicsSubIndustry is None:
gicsSubIndustry = ""
ret.addStock(tickerSymbol, company, gics, gicsSubIndustry)
return ret
def download_trades_since(currency, tid, ignoreMultiCurrency, retries=3):
logger.info("Downloading trades since %s." % (base.tid_to_datetime(tid)))
# logger.info("Downloading trades since %d." % (tid))
done = False
while not done:
try:
response = download_trades_impl(currency, tid)
done = True
except Exception, e:
if retries == 0:
raise e
else:
logger.error("%s. Retrying..." % (e))
retries -= 1
def parse_results(table):
ret = symbolsxml.Writer()
logger.info("Parsing table")
rows = table.xpath("tr")
for row in rows[1:]:
cols = row.xpath("td")
tickerSymbol = cols[TICKER_SYMBOL_COL].xpath("a[1]")[0].text
company = cols[COMPANY_COL].xpath("a[1]")[0].text
gics = cols[GICS_COL].text
gicsSubIndustry = cols[GICS_SUB_INDUSTRY_COL].text
if gicsSubIndustry is None:
gicsSubIndustry = ""
ret.addStock(tickerSymbol, company, gics, gicsSubIndustry)
return ret
def download_trades_since(currency, tid, ignoreMultiCurrency, retries=3):
logger.info("Downloading trades since %s." % (base.tid_to_datetime(tid)))
# logger.info("Downloading trades since %d." % (tid))
done = False
while not done:
try:
response = download_trades_impl(currency, tid)
done = True
except Exception, e:
if retries == 0:
raise e
else:
logger.error("%s. Retrying..." % (e))
retries -= 1
def start(self):
if self.__thread == None:
logger.info("Initializing MtGox client.")
self.__wsClient.connect()
self.__thread = threading.Thread(target=self.__threadMain)
self.__thread.start()
# Wait for initialization to complete.
while self.__initializationFailed == None and self.__thread.is_alive():
self.dispatchImpl([WSClient.ON_CONNECTED])
if self.__initializationFailed == False:
logger.info("Initialization complete.")
else:
raise Exception("Initialization failed")
else:
raise Exception("Already running")
def requestPrivateIdKey(self):
out = {"result":None}
def onResult(data):
out["result"] = data
def onRemark(data):
logger.error("Remark requesting private id key: %s" % (data))
logger.info("Requesting private id key.")
requestId = self.__wsClient.requestPrivateIdKey()
self.waitResponse(requestId, onResult, onRemark, 30)
ret = out["result"]
if ret in (None, ""):
raise Exception("Failed to get private id key")
return ret
def __initializeClient(self):
self.__initializationOk = None
logger.info("Initializing client.")
try:
# Try to connect
self.__thread = None
self.__wsClient = WSClient()
self.__wsClient.connect()
# Start the thread that runs the client.
self.__thread = threading.Thread(target=self.__threadMain)
self.__thread.start()
except Exception, e:
self.__initializationOk = False
logger.error("Error connecting : %s" % str(e))
def __initializeClient(self):
self.__initializationOk = None
logger.info("Initializing client.")
try:
# Try to connect
self.__thread = None
self.__wsClient = WSClient()
self.__wsClient.connect()
# Start the thread that runs the client.
self.__thread = threading.Thread(target=self.__threadMain)
self.__thread.start()
except Exception, e:
self.__initializationOk = False
logger.error("Error connecting : %s" % str(e))
def __init__(self,
identifiers,
apiCallDelay=5,
maxLen=dataseries.DEFAULT_MAX_LEN):
logger.info('Livefeed created')
barfeed.BaseBarFeed.__init__(self, bar.Frequency.TRADE, maxLen)
if not isinstance(identifiers, list):
raise Exception("identifiers must be a list")
self.__queue = Queue.Queue()
self.__orderBookUpdateEvent = observer.Event()
self.__thread = TradesAPIThread(
self.__queue, identifiers,
datetime.timedelta(seconds=apiCallDelay))
self.__bars = []
for instrument in identifiers:
self.registerInstrument(instrument)
def requestPrivateIdKey(self):
out = {"result": None}
def onResult(data):
out["result"] = data
def onRemark(data):
logger.error("Remark requesting private id key: %s" % (data))
logger.info("Requesting private id key.")
requestId = self.__wsClient.requestPrivateIdKey()
self.waitResponse(requestId, onResult, onRemark, 30)
ret = out["result"]
if ret in (None, ""):
raise Exception("Failed to get private id key")
return ret
def OnRspUserLogin(self, *args):
logger.info("OnRspUserLogin")
if args:
loginField = args[0]
logger.info(loginField.TradingDay)
logger.info(loginField.LoginTime)
#construct contract ID
date = datetime.strptime(loginField.TradingDay, "%Y%m%d")
self.__md.SubscribeMarketData(self.__contactIDs, len(self.__contactIDs))
logger.info("OnRspUserLogin End")
def pushJobResults(self, jobId, result, parameters, workerName):
jobId = pickle.loads(jobId)
result = pickle.loads(result)
parameters = pickle.loads(parameters)
# Remove the job mapping.
with self.__activeJobsLock:
try:
del self.__activeJobs[jobId]
except KeyError:
# The job's results were already submitted.
return
if result is None or result > self.__bestResult:
logger.info("Best result so far %s with parameters %s" % (result, parameters))
self.__bestResult = result
self.__resultSinc.push(result, base.Parameters(*parameters))
def download_files_for_symbol(symbol, fromYear, toYear):
if not os.path.exists(storage):
logger.info("Creating %s directory" % (storage))
os.mkdir(storage)
status = ""
for year in range(fromYear, toYear+1):
fileName = get_csv_filename(symbol, year)
if not os.path.exists(fileName):
logger.info("Downloading %s %d to %s" % (symbol, year, fileName))
try:
yahoofinance.download_daily_bars(symbol, year, fileName)
status += "1"
except Exception, e:
logger.error(str(e))
status += "0"
else:
status += "1"
def find_table(htmlTree):
logger.info("Finding the right table")
ret = None
tables = htmlTree.xpath("//table[@class='wikitable sortable']")
for table in tables:
headers = table.xpath("tr[1]/th")
if len(headers) > 5:
if headers[TICKER_SYMBOL_COL].xpath("a[1]")[0].text != "Ticker symbol":
continue
if headers[COMPANY_COL].text != "Company":
continue
if headers[GICS_COL].xpath("a[1]")[0].text != "GICS":
continue
if headers[GICS_SUB_INDUSTRY_COL].text != "GICS Sub Industry":
continue
ret = table
break
return ret
def main():
try:
logger.info("Getting NASDAQ symbols from http://www.nasdaq.com/")
url = "http://www.nasdaq.com/screening/companies-by-name.aspx?exchange=NASDAQ&render=download"
buff = urllib2.urlopen(url).read()
tmpFile = tempfile.NamedTemporaryFile()
tmpFile.write(buff)
tmpFile.flush()
with open(tmpFile.name, 'rb') as csvfile:
symbolsXML = symbolsxml.Writer()
for row in csv.DictReader(csvfile):
symbolsXML.addStock(row["Symbol"], row["Name"], row["Sector"], row["industry"])
logger.info("Writing nasdaq.xml")
symbolsXML.write("nasdaq.xml")
except Exception, e:
logger.error(str(e))
def download_files_for_symbol(symbol, fromYear, toYear):
if not os.path.exists(storage):
logger.info("Creating %s directory" % (storage))
os.mkdir(storage)
status = ""
for year in range(fromYear, toYear + 1):
fileName = get_csv_filename(symbol, year)
if not os.path.exists(fileName):
logger.info("Downloading %s %d to %s" % (symbol, year, fileName))
try:
yahoofinance.download_daily_bars(symbol, year, fileName)
status += "1"
except Exception, e:
logger.error(str(e))
status += "0"
else:
status += "1"
