def stop(self):
try:
if self.__thread is not None and self.__thread.is_alive():
logger.info("Shutting down client.")
self.__stream.disconnect()
except Exception as e:
logger.error("Error disconnecting stream: %s." % (str(e)))
def __threadMain(self):
try:
logger.info("Initializing client.")
self.__stream.filter(track=self.__track,
follow=self.__follow,
languages=self.__languages)
finally:
logger.info("Client finished.")
self.__running = False
def main():
try:
htmlTree = get_html()
table = find_table(htmlTree)
if table is None:
raise Exception("S&P 500 Component Stocks table not found")
symbolsXML = parse_results(table)
logger.info("Writing sp500.xml")
symbolsXML.write("sp500.xml")
except Exception as e:
logger.error(str(e))
def main():
try:
writer = symbolsxml.Writer()
for symbol in open("merval-symbols.txt", "r"):
symbol = symbol.strip()
process_symbol(writer, symbol)
# Index
writer.addIndex("^MERV", "Merval")
logger.info("Writing merval.xml")
writer.write("merval.xml")
except Exception as e:
logger.error(str(e))
def parse_results(table):
ret = symbolsxml.Writer()
logger.info("Parsing table")
rows = table.xpath("tr")
for row in rows[1:]:
cols = row.xpath("td")
tickerSymbol = cols[TICKER_SYMBOL_COL].xpath("a[1]")[0].text
company = cols[COMPANY_COL].xpath("a[1]")[0].text
gics = cols[GICS_COL].text
gicsSubIndustry = cols[GICS_SUB_INDUSTRY_COL].text
if gicsSubIndustry is None:
gicsSubIndustry = ""
ret.addStock(tickerSymbol, company, gics, gicsSubIndustry)
return ret
def pushJobResults(self, jobId, result, parameters, workerName):
jobId = serialization.loads(jobId)
result = serialization.loads(result)
parameters = serialization.loads(parameters)
# Remove the job mapping.
with self.__lock:
try:
del self.__activeJobs[jobId]
except KeyError:
# The job's results were already submitted.
return
if self.__bestResult is None or result > self.__bestResult:
logger.info("Best result so far %s with parameters %s" % (result, parameters))
self.__bestResult = result
self.__resultSinc.push(result, base.Parameters(*parameters))
def find_table(htmlTree):
logger.info("Finding the right table")
ret = None
tables = htmlTree.xpath("//table[@class='wikitable sortable']")
for table in tables:
headers = table.xpath("tr[1]/th")
if len(headers) > 5:
if headers[TICKER_SYMBOL_COL].xpath("a[1]")[0].text != "Ticker symbol":
continue
if headers[COMPANY_COL].text != "Company":
continue
if headers[GICS_COL].xpath("a[1]")[0].text != "GICS":
continue
if headers[GICS_SUB_INDUSTRY_COL].text != "GICS Sub Industry":
continue
ret = table
break
return ret
def main():
try:
logger.info("Getting NYSE symbols from http://www.nasdaq.com/")
url = "http://www.nasdaq.com/screening/companies-by-name.aspx?exchange=NYSE&render=download"
buff = urllib2.urlopen(url).read()
tmpFile = tempfile.NamedTemporaryFile()
tmpFile.write(buff)
tmpFile.flush()
with open(tmpFile.name, 'rb') as csvfile:
symbolsXML = symbolsxml.Writer()
for row in csv.DictReader(csvfile):
symbolsXML.addStock(row["Symbol"], row["Name"], row["Sector"],
row["industry"])
logger.info("Writing nyse.xml")
symbolsXML.write("nyse.xml")
except Exception as e:
logger.error(str(e))
def process_symbol(symbol, fromYear, toYear, missingDataVerifierClass):
logger.info("Processing %s from %d to %d" % (symbol, fromYear, toYear))
filesFound = 0
# Load the bars from the CSV files.
feed = yahoofeed.Feed(maxLen=1000000)
feed.sanitizeBars(True)
for year in range(fromYear, toYear+1):
fileName = get_csv_filename(symbol, year)
if os.path.exists(fileName):
filesFound += 1
feed.addBarsFromCSV(symbol, fileName)
if filesFound > 0:
# Process all items.
for dateTime, bars in feed:
pass
missingDataVerifier = missingDataVerifierClass(feed[symbol])
missingDataVerifier.run()
else:
logger.error("No files found")
def download_files_for_symbol(symbol, fromYear, toYear):
if not os.path.exists(storage):
logger.info("Creating %s directory" % (storage))
os.mkdir(storage)
status = ""
for year in range(fromYear, toYear + 1):
fileName = get_csv_filename(symbol, year)
if not os.path.exists(fileName):
logger.info("Downloading %s %d to %s" % (symbol, year, fileName))
try:
yahoofinance.download_daily_bars(symbol, year, fileName)
status += "1"
except Exception as e:
logger.error(str(e))
status += "0"
else:
status += "1"
if status.find("1") == -1:
logger.fatal("No data found for %s" % (symbol))
elif status.lstrip("0").find("0") != -1:
logger.fatal("Some bars are missing for %s" % (symbol))
def process_symbol(writer, symbol):
try:
logger.info("Getting info for %s" % (symbol))
url = "http://finance.yahoo.com/q/in?s=%s+Industry" % (symbol)
htmlTree = lxml.html.parse(url)
company = find_company(htmlTree, symbol)
if not company:
raise Exception("Company name not found")
sector = find_sector(htmlTree)
if not sector:
sector = ""
logger.warning("Sector not found")
industry = find_industry(htmlTree)
if not industry:
industry = ""
logger.warning("Industry not found")
writer.addStock(symbol, company, sector, industry)
except Exception as e:
logger.error(str(e))
def main():
parser = argparse.ArgumentParser(description="Quandl utility")
parser.add_argument("--auth-token", required=False, help="An authentication token needed if you're doing more than 50 calls per day")
parser.add_argument("--source-code", required=True, help="The dataset source code")
parser.add_argument("--table-code", required=True, help="The dataset table code")
parser.add_argument("--from-year", required=True, type=int, help="The first year to download")
parser.add_argument("--to-year", required=True, type=int, help="The last year to download")
parser.add_argument("--storage", required=True, help="The path were the files will be downloaded to")
parser.add_argument("--force-download", action='store_true', help="Force downloading even if the files exist")
parser.add_argument("--ignore-errors", action='store_true', help="True to keep on downloading files in case of errors")
parser.add_argument("--frequency", default="daily", choices=["daily", "weekly"], help="The frequency of the bars. Only daily or weekly are supported")
args = parser.parse_args()
logger = quantworks.logger.getLogger("quandl")
if not os.path.exists(args.storage):
logger.info("Creating %s directory" % (args.storage))
os.mkdir(args.storage)
for year in range(args.from_year, args.to_year+1):
fileName = os.path.join(args.storage, "%s-%s-%d-quandl.csv" % (args.source_code, args.table_code, year))
if not os.path.exists(fileName) or args.force_download:
logger.info("Downloading %s %d to %s" % (args.table_code, year, fileName))
try:
if args.frequency == "daily":
download_daily_bars(args.source_code, args.table_code, year, fileName, args.auth_token)
else:
assert args.frequency == "weekly", "Invalid frequency"
download_weekly_bars(args.source_code, args.table_code, year, fileName, args.auth_token)
except Exception as e:
if args.ignore_errors:
logger.error(str(e))
continue
else:
raise
def serve(self):
try:
# Initialize instruments, bars and parameters.
logger.info("Loading bars")
loadedBars = []
for dateTime, bars in self.__barFeed:
loadedBars.append(bars)
instruments = self.__barFeed.getRegisteredInstruments()
self.__instrumentsAndBars = serialization.dumps((instruments, loadedBars))
self.__barsFreq = self.__barFeed.getFrequency()
if self.__autoStopThread:
self.__autoStopThread.start()
logger.info("Started serving")
self.__startedServingEvent.set()
self.serve_forever()
logger.info("Finished serving")
if self.__autoStopThread:
self.__autoStopThread.join()
finally:
self.__forcedStop = True
def serve(barFeed, strategyParameters, address, port, batchSize=200):
"""Executes a server that will provide bars and strategy parameters for workers to use.
:param barFeed: The bar feed that each worker will use to backtest the strategy.
:type barFeed: :class:`quantworks.barfeed.BarFeed`.
:param strategyParameters: The set of parameters to use for backtesting. An iterable object where **each element is a tuple that holds parameter values**.
:param address: The address to listen for incoming worker connections.
:type address: string.
:param port: The port to listen for incoming worker connections.
:type port: int.
:param batchSize: The number of strategy executions that are delivered to each worker.
:type batchSize: int.
:rtype: A :class:`Results` instance with the best results found or None if no results were obtained.
"""
paramSource = base.ParameterSource(strategyParameters)
resultSinc = base.ResultSinc()
s = xmlrpcserver.Server(paramSource,
resultSinc,
barFeed,
address,
port,
batchSize=batchSize)
logger.info("Starting server")
s.serve()
logger.info("Server finished")
ret = None
bestResult, bestParameters = resultSinc.getBest()
if bestResult is not None:
logger.info("Best final result %s with parameters %s" %
(bestResult, bestParameters.args))
ret = Results(bestParameters.args, bestResult)
else:
logger.error("No results. All jobs failed or no jobs were processed.")
return ret
def build_feed(sourceCode, tableCodes, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None,
skipErrors=False, authToken=None, columnNames={}, forceDownload=False,
skipMalformedBars=False
):
"""Build and load a :class:`quantworks.barfeed.quandlfeed.Feed` using CSV files downloaded from Quandl.
CSV files are downloaded if they haven't been downloaded before.
:param sourceCode: The dataset source code.
:type sourceCode: string.
:param tableCodes: The dataset table codes.
:type tableCodes: list.
:param fromYear: The first year.
:type fromYear: int.
:param toYear: The last year.
:type toYear: int.
:param storage: The path were the files will be loaded from, or downloaded to.
:type storage: string.
:param frequency: The frequency of the bars. Only **quantworks.bar.Frequency.DAY** or **quantworks.bar.Frequency.WEEK**
are supported.
:param timezone: The default timezone to use to localize bars. Check :mod:`quantworks.marketsession`.
:type timezone: A pytz timezone.
:param skipErrors: True to keep on loading/downloading files in case of errors.
:type skipErrors: boolean.
:param authToken: Optional. An authentication token needed if you're doing more than 50 calls per day.
:type authToken: string.
:param columnNames: Optional. A dictionary to map column names. Valid key values are:
* datetime
* open
* high
* low
* close
* volume
* adj_close
:type columnNames: dict.
:param skipMalformedBars: True to skip errors while parsing bars.
:type skipMalformedBars: boolean.
:rtype: :class:`quantworks.barfeed.quandlfeed.Feed`.
"""
logger = quantworks.logger.getLogger("quandl")
ret = quandlfeed.Feed(frequency, timezone)
# Additional column names.
for col, name in six.iteritems(columnNames):
ret.setColumnName(col, name)
if not os.path.exists(storage):
logger.info("Creating %s directory" % (storage))
os.mkdir(storage)
for year in range(fromYear, toYear+1):
for tableCode in tableCodes:
fileName = os.path.join(storage, "%s-%s-%d-quandl.csv" % (sourceCode, tableCode, year))
if not os.path.exists(fileName) or forceDownload:
logger.info("Downloading %s %d to %s" % (tableCode, year, fileName))
try:
if frequency == bar.Frequency.DAY:
download_daily_bars(sourceCode, tableCode, year, fileName, authToken)
else:
assert frequency == bar.Frequency.WEEK, "Invalid frequency"
download_weekly_bars(sourceCode, tableCode, year, fileName, authToken)
except Exception as e:
if skipErrors:
logger.error(str(e))
continue
else:
raise e
ret.addBarsFromCSV(tableCode, fileName, skipMalformedBars=skipMalformedBars)
return ret
def on_connect(self):
logger.info("Connected.")
def get_html():
logger.info("Getting S&P 500 Component Stocks from Wikipedia")
ret = lxml.html.parse("http://en.wikipedia.org/wiki/List_of_S%26P_500_companies")
return ret
