def stop(self):
try:
if self.__thread is not None and self.__thread.is_alive():
logger.info("Shutting down client.")
self.__stream.disconnect()
except Exception, e:
logger.error("Error disconnecting stream: %s." % (str(e)))
def __threadMain(self):
try:
logger.info("Initializing client.")
self.__stream.filter(track=self.__track,
follow=self.__follow,
languages=self.__languages)
finally:
logger.info("Client finished.")
self.__running = False
def build_feed(instruments,
fromYear,
toYear,
storage,
frequency=bar.Frequency.DAY,
timezone=None,
skipErrors=False):
"""Build and load a :class:`pyalgotrade.barfeed.yahoofeed.Feed` using CSV files downloaded from Yahoo! Finance.
CSV files are downloaded if they haven't been downloaded before.
:param instruments: Instrument identifiers.
:type instruments: list.
:param fromYear: The first year.
:type fromYear: int.
:param toYear: The last year.
:type toYear: int.
:param storage: The path were the files will be loaded from, or downloaded to.
:type storage: string.
:param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** or **pyalgotrade.bar.Frequency.WEEK**
are supported.
:param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`.
:type timezone: A pytz timezone.
:param skipErrors: True to keep on loading/downloading files in case of errors.
:type skipErrors: boolean.
:rtype: :class:`pyalgotrade.barfeed.yahoofeed.Feed`.
"""
logger = skywalker.logger.getLogger("yahoofinance")
ret = yahoofeed.Feed(frequency, timezone)
if not os.path.exists(storage):
logger.info("Creating %s directory" % (storage))
os.mkdir(storage)
for year in range(fromYear, toYear + 1):
for instrument in instruments:
fileName = os.path.join(
storage, "%s-%d-yahoofinance.csv" % (instrument, year))
if not os.path.exists(fileName):
logger.info("Downloading %s %d to %s" %
(instrument, year, fileName))
try:
if frequency == bar.Frequency.DAY:
download_daily_bars(instrument, year, fileName)
elif frequency == bar.Frequency.WEEK:
download_weekly_bars(instrument, year, fileName)
else:
raise Exception("Invalid frequency")
except Exception, e:
if skipErrors:
logger.error(str(e))
continue
else:
raise e
ret.addBarsFromCSV(instrument, fileName)
def pushJobResults(self, jobId, result, parameters, workerName):
jobId = pickle.loads(jobId)
result = pickle.loads(result)
parameters = pickle.loads(parameters)
# Remove the job mapping.
with self.__activeJobsLock:
try:
del self.__activeJobs[jobId]
except KeyError:
# The job's results were already submitted.
return
if result is None or result > self.__bestResult:
logger.info("Best result so far %s with parameters %s" %
(result, parameters))
self.__bestResult = result
self.__resultSinc.push(result, base.Parameters(*parameters))
def serve(self):
try:
# Initialize instruments, bars and parameters.
logger.info("Loading bars")
loadedBars = []
for dateTime, bars in self.__barFeed:
loadedBars.append(bars)
instruments = self.__barFeed.getRegisteredInstruments()
self.__instrumentsAndBars = pickle.dumps((instruments, loadedBars))
self.__barsFreq = self.__barFeed.getFrequency()
if self.__autoStopThread:
self.__autoStopThread.start()
logger.info("Waiting for workers")
self.serve_forever()
if self.__autoStopThread:
self.__autoStopThread.join()
finally:
self.__forcedStop = True
def serve(barFeed, strategyParameters, address, port):
"""Executes a server that will provide bars and strategy parameters for workers to use.
:param barFeed: The bar feed that each worker will use to backtest the strategy.
:type barFeed: :class:`pyalgotrade.barfeed.BarFeed`.
:param strategyParameters: The set of parameters to use for backtesting. An iterable object where **each element is a tuple that holds parameter values**.
:param address: The address to listen for incoming worker connections.
:type address: string.
:param port: The port to listen for incoming worker connections.
:type port: int.
:rtype: A :class:`Results` instance with the best results found or None if no results were obtained.
"""
paramSource = base.ParameterSource(strategyParameters)
resultSinc = base.ResultSinc()
s = xmlrpcserver.Server(paramSource, resultSinc, barFeed, address, port)
logger.info("Starting server")
s.serve()
logger.info("Server finished")
ret = None
bestResult, bestParameters = resultSinc.getBest()
if bestResult is not None:
logger.info("Best final result %s with parameters %s" %
(bestResult, bestParameters.args))
ret = Results(bestParameters.args, bestResult)
else:
logger.error("No results. All jobs failed or no jobs were processed.")
return ret
def build_feed(sourceCode,
tableCodes,
fromYear,
toYear,
storage,
frequency=bar.Frequency.DAY,
timezone=None,
skipErrors=False,
noAdjClose=False,
authToken=None,
columnNames={},
forceDownload=False):
"""Build and load a :class:`pyalgotrade.barfeed.quandlfeed.Feed` using CSV files downloaded from Quandl.
CSV files are downloaded if they haven't been downloaded before.
:param sourceCode: The dataset source code.
:type sourceCode: string.
:param tableCodes: The dataset table codes.
:type tableCodes: list.
:param fromYear: The first year.
:type fromYear: int.
:param toYear: The last year.
:type toYear: int.
:param storage: The path were the files will be loaded from, or downloaded to.
:type storage: string.
:param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** or **pyalgotrade.bar.Frequency.WEEK**
are supported.
:param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`.
:type timezone: A pytz timezone.
:param skipErrors: True to keep on loading/downloading files in case of errors.
:type skipErrors: boolean.
:param noAdjClose: True if the instruments don't have adjusted close values.
:type noAdjClose: boolean.
:param authToken: Optional. An authentication token needed if you're doing more than 50 calls per day.
:type authToken: string.
:param columnNames: Optional. A dictionary to map column names. Valid key values are:
* datetime
* open
* high
* low
* close
* volume
* adj_close
:type columnNames: dict.
:rtype: :class:`pyalgotrade.barfeed.quandlfeed.Feed`.
"""
logger = skywalker.logger.getLogger("quandl")
ret = quandlfeed.Feed(frequency, timezone)
if noAdjClose:
ret.setNoAdjClose()
# Additional column names.
for col, name in columnNames.iteritems():
ret.setColumnName(col, name)
if not os.path.exists(storage):
logger.info("Creating %s directory" % (storage))
os.mkdir(storage)
for year in range(fromYear, toYear + 1):
for tableCode in tableCodes:
fileName = os.path.join(
storage, "%s-%s-%d-quandl.csv" % (sourceCode, tableCode, year))
if not os.path.exists(fileName) or forceDownload:
logger.info("Downloading %s %d to %s" %
(tableCode, year, fileName))
try:
if frequency == bar.Frequency.DAY:
download_daily_bars(sourceCode, tableCode, year,
fileName, authToken)
elif frequency == bar.Frequency.WEEK:
download_weekly_bars(sourceCode, tableCode, year,
fileName, authToken)
else:
raise Exception("Invalid frequency")
except Exception, e:
if skipErrors:
logger.error(str(e))
continue
else:
raise e
ret.addBarsFromCSV(tableCode, fileName)
def on_connect(self):
logger.info("Connected.")
