def build_feed(instruments, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False): logger = pyalgotrade.logger.getLogger("yahoofinance") ret = yahoofeed.Feed(frequency, timezone) if not os.path.exists(storage): logger.info("Creating %s directory" % (storage)) os.mkdir(storage) for year in range(fromYear, toYear+1): for instrument in instruments: fileName = os.path.join(storage, "%s-%d-yahoofinance.csv" % (instrument, year)) if not os.path.exists(fileName): logger.info("Downloading %s %d to %s" % (instrument, year, fileName)) try: if frequency == bar.Frequency.DAY: download_daily_bars(instrument, year, fileName) elif frequency == bar.Frequency.WEEK: download_weekly_bars(instrument, year, fileName) else: raise Exception("Invalid frequency") except Exception, e: if skipErrors: logger.error(str(e)) continue else: raise e ret.addBarsFromCSV(instrument, fileName)
def OnRtnDepthMarketData(self, *args): logger.info("OnRtnDepthMarketData") logger.info("id: "+args[0].InstrumentID) logger.info("TradingDay: " + args[0].TradingDay + " " + args[0].UpdateTime) logger.info("LastPrice: " + str(args[0].LastPrice)) if self.__insertIntoMysql: if self.__mysqlCon == None: self.__mysqlCon = client.mysqlConnection(CONSTANTS.HOST, CONSTANTS.USERNAME, CONSTANTS.PASSWORD, CONSTANTS.DATABASE) date = datetime.strptime(args[0].TradingDay, "%Y%m%d") dateStr = date.strftime("%Y-%m-%d") dateStr = dateStr + " " + args[0].UpdateTime self.__mysqlCon.addBar(args[0].InstrumentID, RealTimeBar(dateStr, args[0].LastPrice, args[0].Volume)) if self.__dumpToFile: try: self.dumpToFile(args[0]) except Exception as e: print "except", e logger.info("OnRtnDepthMarketData End")
def stop(self): try: if self.__thread is not None and self.__thread.is_alive(): logger.info("Shutting down client.") self.__stream.disconnect() except Exception, e: logger.error("Error disconnecting stream: %s." % (str(e)))
def stop(self): try: if self.__thread is not None and self.__thread.is_alive(): logger.info("Shutting down client.") self.__stream.disconnect() except Exception as e: logger.error("Error disconnecting stream: %s." % (str(e)))
def update_bars(dbFilePath, symbolsFile, timezone, fromYear, toYear): db = sqlitefeed.Database(dbFilePath) for symbol in open(symbolsFile, "r"): symbol = symbol.strip() logger.info("Downloading %s bars" % (symbol)) for year in range(fromYear, toYear+1): download_bars(db, symbol, year, timezone)
def build_feed(instruments, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False): logger = pyalgotrade.logger.getLogger("yahoofinance") ret = yahoofeed.Feed(frequency, timezone) if not os.path.exists(storage): logger.info("Creating %s directory" % (storage)) os.mkdir(storage) for year in range(fromYear, toYear + 1): for instrument in instruments: fileName = os.path.join( storage, "%s-%d-yahoofinance.csv" % (instrument, year)) if not os.path.exists(fileName): logger.info("Downloading %s %d to %s" % (instrument, year, fileName)) try: if frequency == bar.Frequency.DAY: download_daily_bars(instrument, year, fileName) elif frequency == bar.Frequency.WEEK: download_weekly_bars(instrument, year, fileName) else: raise Exception("Invalid frequency") except Exception, e: if skipErrors: logger.error(str(e)) continue else: raise e ret.addBarsFromCSV(instrument, fileName)
def getNextBars(self): if not self.__data_downloaded: logger.info('featch history of {0}'.format(self.__instrument)) for i in self.getFrequencies(): tmp = None if i == Frequency.DAY: tmp = history(self.__instrument, self.__start_date, None, Frequency.DAY, add_missing_dates=False) elif i == Frequency.HOUR: tmp = history(self.__instrument, self.__start_date, None, Frequency.HOUR, add_missing_dates=False) if tmp is None: continue for date, row in tmp.iloc[0].iterrows(): tmpbar = bar.BasicBar(date, row['open'], row['high'], row['low'], row['close'], 0, False, i) self.__bars_buf.append(tmpbar) self.__data_downloaded = True self.__bars_buf.sort(key=lambda i: i.getDateTime()) while True: if self.__bars_buf: tmp = self.__bars_buf.pop(0) return bar.Bars({self.__instrument: tmp}, frequecy=tmp.getFrequency()) else: self.generateBars()
def __threadMain(self): try: logger.info("Initializing client.") self.__stream.filter(track=self.__track, follow=self.__follow, languages=self.__languages) finally: logger.info("Client finished.") self.__running = False
def doCall(self): barDict = {} try: a = time.time() response = ts.get_realtime_quotes(self.__identifiers) logger.info('response cost: %f' % (time.time() - a)) if self.__last_response_time != response.iloc[-1]['time']: for identifier in self.__identifiers: barDict[identifier] = build_bar( response[response.code == identifier].iloc[-1]) # dataFrame 格式转换 else: logger.info( "bar is the same with previous bar at time %s,not refresh" % response.iloc[-1]['time']) self.__last_response_time = response.iloc[-1]['time'] except: logger.error("tushare time out") import traceback traceback.print_exc() if len(barDict): bars = bar.Ticks(barDict) self.__queue.put((GetBarThread.ON_BARS, bars))
def dispatchImpl(self, eventFilter): try: eventType, eventData = self.__queue.get(True, Client.QUEUE_TIMEOUT) if eventFilter is not None and eventType not in eventFilter: return if eventType == WSClient.ON_TICKER: self.__tickerEvent.emit(eventData) elif eventType == WSClient.ON_TRADE: self.__tradeEvent.emit(eventData) elif eventType == WSClient.ON_USER_ORDER: self.__userOrderEvent.emit(eventData) elif eventType == WSClient.ON_RESULT: requestId, result = eventData logger.info("Result: %s - %s" % (requestId, result)) elif eventType == WSClient.ON_REMARK: requestId, data = eventData logger.info("Remark: %s - %s" % (requestId, data)) elif eventType == WSClient.ON_CONNECTED: self.__onConnected() elif eventType == WSClient.ON_DISCONNECTED: self.__onDisconnected() else: logger.error("Invalid event received to dispatch: %s - %s" % (eventType, eventData)) except Queue.Empty: pass
def deleteDataFromDate(self, date): cursor = self.__con.cursor() query = "delete from data where date(date) >= '{0:s}' and date(date) <= '{0:s}'" query = query.format(date) logger.info(query) cursor.execute(query) self.__con.commit() cursor.close()
def stop(self): try: self.__stopped = True if self.__thread is not None and self.__thread.is_alive(): logger.info("Shutting down MtGox client.") self.__wsClient.stopClient() except Exception, e: logger.error("Error shutting down MtGox client: %s" % (str(e)))
def download_symbols(market): market = market.upper() pages = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"] for x in pages: url = "http://www.eoddata.com/stocklist/%s/%s.htm" % (market, x) logger.info("Processing %s" % url) for symbol in get_symbols_from_page(url): yield symbol
def stop(self): try: self.__stopped = True if self.__thread is not None and self.__thread.is_alive(): logger.info("Stopping websocket client.") self.__thread.stop() except Exception as e: logger.error("Error shutting down client: %s" % (str(e)))
def __onDisconnected(self): if self.__enableReconnection: logger.info("Reconnecting") while not self.__stopped and not self.__initializeClient(): pass elif not self.__stopped: logger.info("Stopping") self.__stopped = True
def OnRspSubMarketData(self, *args): logger.info("OnRspSubMarketData") if args: rspInfoField = args[1] logger.info("Subscribe Instrument " + args[0].InstrumentID) logger.info("errorID " + str(rspInfoField.ErrorID)) logger.info("errorMsg " + str(rspInfoField.ErrorMsg)) logger.info("OnRspSubMarketData End")
def state_init(self, bars, states): # You are only supposed to save states in states variable # DO NOT save your local variable and it is not guaranteed to be supported later logger.info('INIT') print(states.prev) states.prev = 'INIT' self.print_bars(bars) return SampleStrategyFSMState.STATE1
def run(self): logger.info("Thread started") while not self.__stopped: self.__wait() if not self.__stopped: try: self.doCall() except Exception, e: logger.critical("Unhandled exception", exc_info=e)
def build_feed(sourceCode, tableCodes, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False, noAdjClose=False, authToken=None): """Build and load a :class:`pyalgotrade.barfeed.quandlfeed.Feed` using CSV files downloaded from Quandl. CSV files are downloaded if they haven't been downloaded before. :param sourceCode: The dataset source code. :type sourceCode: string. :param tableCodes: The dataset table codes. :type tableCodes: list. :param fromYear: The first year. :type fromYear: int. :param toYear: The last year. :type toYear: int. :param storage: The path were the files will be loaded from, or downloaded to. :type storage: string. :param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** or **pyalgotrade.bar.Frequency.WEEK** are supported. :param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`. :type timezone: A pytz timezone. :param skipErrors: True to keep on loading/downloading files in case of errors. :type skipErrors: boolean. :param noAdjClose: True if the instruments don't have adjusted close values. :type noAdjClose: boolean. :param authToken: Optional. An authentication token needed if you're doing more than 50 calls per day. :type authToken: string. :rtype: :class:`pyalgotrade.barfeed.quandlfeed.Feed`. """ logger = pyalgotrade.logger.getLogger("quandl") ret = quandlfeed.Feed(frequency, timezone) if noAdjClose: ret.setNoAdjClose() if not os.path.exists(storage): logger.info("Creating %s directory" % (storage)) os.mkdir(storage) for year in range(fromYear, toYear+1): for tableCode in tableCodes: fileName = os.path.join(storage, "%s-%s-%d-quandl.csv" % (sourceCode, tableCode, year)) if not os.path.exists(fileName): logger.info("Downloading %s %d to %s" % (tableCode, year, fileName)) try: if frequency == bar.Frequency.DAY: download_daily_bars(sourceCode, tableCode, year, fileName, authToken) elif frequency == bar.Frequency.WEEK: download_weekly_bars(sourceCode, tableCode, year, fileName, authToken) else: raise Exception("Invalid frequency") except Exception as e: if skipErrors: logger.error(str(e)) continue else: raise e ret.addBarsFromCSV(tableCode, fileName) return ret
def build_feed(instruments, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False): """Build and load a :class:`pyalgotrade.barfeed.yahoofeed.Feed` using CSV files downloaded from Yahoo! Finance. CSV files are downloaded if they haven't been downloaded before. :param instruments: Instrument identifiers. :type instruments: list. :param fromYear: The first year. :type fromYear: int. :param toYear: The last year. :type toYear: int. :param storage: The path were the files will be loaded from, or downloaded to. :type storage: string. :param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** or **pyalgotrade.bar.Frequency.WEEK** are supported. :param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`. :type timezone: A pytz timezone. :param skipErrors: True to keep on loading/downloading files in case of errors. :type skipErrors: boolean. :rtype: :class:`pyalgotrade.barfeed.yahoofeed.Feed`. """ logger = pyalgotrade.logger.getLogger("yahoofinance") ret = yahoofeed.Feed(frequency, timezone) if not os.path.exists(storage): logger.info("Creating %s directory" % (storage)) os.mkdir(storage) for year in range(fromYear, toYear + 1): for instrument in instruments: fileName = os.path.join( storage, "%s-%d-yahoofinance.csv" % (instrument, year)) if not os.path.exists(fileName): logger.info("Downloading %s %d to %s" % (instrument, year, fileName)) try: if frequency == bar.Frequency.DAY: download_daily_bars(instrument, year, fileName) elif frequency == bar.Frequency.WEEK: download_weekly_bars(instrument, year, fileName) else: raise Exception("Invalid frequency") except Exception as e: if skipErrors: logger.error(str(e)) continue else: raise e ret.addBarsFromCSV(instrument, fileName) return ret
def onOpened(self): for channel in self.__pending_subscriptions: logger.info("Subscribing to channel %s." % channel) self.send( json.dumps({ "event": "bts:subscribe", "data": { "channel": channel } }))
def __onDisconnected(self): if self.__enableReconnection: initialized = False while not self.__stopped and not initialized: logger.info("Reconnecting") initialized = self.__initializeClient() if not initialized: time.sleep(5) else: self.__stopped = True
def main(): try: writer = symbolsxml.Writer() for symbol in open("merval-symbols.txt", "r"): symbol = symbol.strip() process_symbol(writer, symbol) logger.info("Writing merval.xml") writer.write("merval.xml") except Exception, e: logger.error(str(e))
def OnFrontConnected(self): logger.info("OnFrontConnected") f = CThostFtdcReqUserLoginField() f.BrokerID = self.__broker_id f.UserUD = self.__user_id f.Password = self.__password self.__md.ReqUserLogin(f, self.__reqNum) self.__reqNum = self.__reqNum + 1 if (datetime.now() - self.__starttime) > timedelta(seconds=60*60*10): self.stopClient()
def generateBars(self): if self.__pullDelay > 0: time.sleep(self.__pullDelay) logger.info('get quote of {0}'.format(self.__instrument)) tmp = quote(self.__instrument) if tmp is None: logger.error('failed to get {0} quote'.format(self.__instrument)) return tmpVal = tmp.iloc[0]['close'].iloc[0] curTime = datetime.datetime.utcnow().replace(tzinfo=pytz.utc) tmp = bar.BasicBar(curTime, tmpVal, tmpVal, tmpVal, tmpVal, 0, False, Frequency.REALTIME) for freq in self.__nextRealtimeBars.keys(): if self.__nextRealtimeBars[freq]['open'] is None: self.__nextRealtimeBars[freq]['open'] = tmpVal self.__nextRealtimeBars[freq]['high'] = tmpVal self.__nextRealtimeBars[freq]['low'] = tmpVal self.__nextRealtimeBars[freq]['close'] = tmpVal if freq == Frequency.DAY: curTime = curTime.replace(hour=0, minute=0, second=0, microsecond=0) elif freq == Frequency.HOUR: curTime = curTime.replace(minute=0, second=0, microsecond=0) elif freq == Frequency.MINUTE: curTime = curTime.replace(second=0, microsecond=0) elif freq != Frequency.REALTIME: logger.error('{0} is not supported.'.format(freq)) assert False self.__nextRealtimeBars[freq]['start'] = curTime else: if tmpVal > self.__nextRealtimeBars[freq]['high']: self.__nextRealtimeBars[freq]['high'] = tmpVal elif tmpVal < self.__nextRealtimeBars[freq]['low']: self.__nextRealtimeBars[freq]['low'] = tmpVal deltaTime = curTime - self.__nextRealtimeBars[freq]['start'] if (Frequency.MINUTE == freq and deltaTime.total_seconds() > 60 or Frequency.DAY == freq and deltaTime.total_seconds() > 60*60*24 or Frequency.HOUR == freq and deltaTime.total_seconds() > 60*60): self.__nextRealtimeBars[freq]['close'] = tmpVal row = self.__nextRealtimeBars[freq] tmpbar = bar.BasicBar(curTime, row['open'], row['high'], row['low'], row['close'], 0, False, freq) self.__bars_buf.append(tmpbar) self.__nextRealtimeBars[freq] = { 'open': None, 'high': None, 'low': None, 'close': None, 'start' : None, } self.__bars_buf.append(tmp) if self.__isRealTime: self.__bars_buf.append(tmp) self.__bars_buf.sort(key=lambda i: i.getDateTime())
def build_feed(instruments, fromYear, toYear, storage, frequency=bar.Frequency.DAY, skipErrors=False): """Build and load a :class:`pyalgotrade.barfeed.tusharefeed.Feed` using CSV files downloaded from Tushare Library. CSV files are downloaded if they haven't been downloaded before. :param instruments: Instrument identifiers. :type instruments: list. :param fromYear: The first year. :type fromYear: int. :param toYear: The last year. :type toYear: int. :param storage: The path were the files will be loaded from, or downloaded to. :type storage: string. :param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** is currently supported. :param skipErrors: True to keep on loading/downloading files in case of errors. :type skipErrors: boolean. :rtype: :class:`pyalgotrade.barfeed.tusharefeed.Feed`. """ logger = pyalgotrade.logger.getLogger("tushare") ret = barfeed.Feed(frequency) if not os.path.exists(storage): logger.info("Creating {dirname} directory".format(dirname=storage)) os.mkdir(storage) for year in range(fromYear, toYear + 1): for instrument in instruments: filePath = Path(storage) fileName = filePath / "{instrument}-{year}-tushare.csv".format( instrument=instrument, year=year) if not os.path.exists(fileName): logger.info( "Downloading {instrument} {year} to {filename}".format( instrument=instrument, year=year, filename=fileName)) try: if frequency == bar.Frequency.DAY: download_daily_bars(instrument, year, fileName) else: raise Exception("Invalid frequency") except Exception as e: if skipErrors: logger.error(str(e)) continue else: raise e ret.addBarsFromCSV(instrument, fileName) return ret
def build_feed(sourceCode, tableCodes, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False, noAdjClose=False, authToken=None): """Build and load a :class:`pyalgotrade.barfeed.quandlfeed.Feed` using CSV files downloaded from Quandl. CSV files are downloaded if they haven't been downloaded before. :param sourceCode: The dataset source code. :type sourceCode: string. :param tableCodes: The dataset table codes. :type tableCodes: list. :param fromYear: The first year. :type fromYear: int. :param toYear: The last year. :type toYear: int. :param storage: The path were the files will be loaded from, or downloaded to. :type storage: string. :param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** or **pyalgotrade.bar.Frequency.WEEK** are supported. :param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`. :type timezone: A pytz timezone. :param skipErrors: True to keep on loading/downloading files in case of errors. :type skipErrors: boolean. :param noAdjClose: True if the instruments don't have adjusted close values. :type noAdjClose: boolean. :param authToken: Optional. An authentication token needed if you're doing more than 50 calls per day. :type authToken: string. :rtype: :class:`pyalgotrade.barfeed.quandlfeed.Feed`. """ logger = pyalgotrade.logger.getLogger("quandl") ret = quandlfeed.Feed(frequency, timezone) if noAdjClose: ret.setNoAdjClose() if not os.path.exists(storage): logger.info("Creating %s directory" % (storage)) os.mkdir(storage) for year in range(fromYear, toYear+1): for tableCode in tableCodes: fileName = os.path.join(storage, "%s-%s-%d-quandl.csv" % (sourceCode, tableCode, year)) if not os.path.exists(fileName): logger.info("Downloading %s %d to %s" % (tableCode, year, fileName)) try: if frequency == bar.Frequency.DAY: download_daily_bars(sourceCode, tableCode, year, fileName, authToken) elif frequency == bar.Frequency.WEEK: download_weekly_bars(sourceCode, tableCode, year, fileName, authToken) else: raise Exception("Invalid frequency") except Exception, e: if skipErrors: logger.error(str(e)) continue else: raise e ret.addBarsFromCSV(tableCode, fileName)
def build_feed(instruments, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False, rowFilter=None): """Build and load a :class:`pyalgotrade.barfeed.googlefeed.Feed` using CSV files downloaded from Google Finance. CSV files are downloaded if they haven't been downloaded before. :param instruments: Instrument identifiers. :type instruments: list. :param fromYear: The first year. :type fromYear: int. :param toYear: The last year. :type toYear: int. :param storage: The path were the files will be loaded from, or downloaded to. :type storage: string. :param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** is currently supported. :param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`. :type timezone: A pytz timezone. :param skipErrors: True to keep on loading/downloading files in case of errors. :type skipErrors: boolean. :rtype: :class:`pyalgotrade.barfeed.googlefeed.Feed`. """ logger = pyalgotrade.logger.getLogger("googlefinance") ret = googlefeed.Feed(frequency, timezone) if not os.path.exists(storage): logger.info("Creating {dirname} directory".format(dirname=storage)) os.mkdir(storage) for year in range(fromYear, toYear + 1): for instrument in instruments: fileName = os.path.join( storage, "{instrument}-{year}-googlefinance.csv".format( instrument=instrument, year=year)) if not os.path.exists(fileName): logger.info( "Downloading {instrument} {year} to {filename}".format( instrument=instrument, year=year, filename=fileName)) try: if frequency == bar.Frequency.DAY: download_daily_bars(instrument, year, fileName) else: raise Exception("Invalid frequency") except Exception, e: if skipErrors: logger.error(str(e)) continue else: raise e ret.addBarsFromCSV(instrument, fileName, rowFilter=rowFilter)
def download_symbols(market): market = market.upper() pages = [ "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z" ] for x in pages: url = "http://www.eoddata.com/stocklist/%s/%s.htm" % (market, x) logger.info("Processing %s" % url) for symbol in get_symbols_from_page(url): yield symbol
def main(): try: htmlTree = get_html() table = find_table(htmlTree) if table is None: raise Exception("S&P 500 Component Stocks table not found") symbolsXML = parse_results(table) logger.info("Writing sp500.xml") symbolsXML.write("sp500.xml") except Exception as e: logger.error(str(e))
def main(): try: htmlTree = get_html() table = find_table(htmlTree) if table is None: raise Exception("S&P 500 Component Stocks table not found") symbolsXML = parse_results(table) logger.info("Writing sp500.xml") symbolsXML.write("sp500.xml") except Exception, e: logger.error(str(e))
def wrapper(*args, **kwargs): try: rtn = func(*args, **kwargs) return rtn except Exception: info = traceback.format_exc() logger.error('-' * 60) logger.error(bugart + '\n' + info) logger.error('-' * 60) return exception_rtn except KeyboardInterrupt: logger.info('KeyboardInterrupt received, terminating...') sys.exit(0)
def build_feed(instruments, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False): """Build and load a :class:`pyalgotrade.barfeed.googlefeed.Feed` using CSV files downloaded from Google Finance. CSV files are downloaded if they haven't been downloaded before. :param instruments: Instrument identifiers. :type instruments: list. :param fromYear: The first year. :type fromYear: int. :param toYear: The last year. :type toYear: int. :param storage: The path were the files will be loaded from, or downloaded to. :type storage: string. :param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** is currently supported. :param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`. :type timezone: A pytz timezone. :param skipErrors: True to keep on loading/downloading files in case of errors. :type skipErrors: boolean. :rtype: :class:`pyalgotrade.barfeed.googlefeed.Feed`. """ logger = pyalgotrade.logger.getLogger("googlefinance") ret = googlefeed.Feed(frequency, timezone) if not os.path.exists(storage): logger.info("Creating {dirname} directory".format(dirname=storage)) os.mkdir(storage) for year in range(fromYear, toYear+1): for instrument in instruments: fileName = os.path.join( storage, "{instrument}-{year}-googlefinance.csv".format( instrument=instrument, year=year)) if not os.path.exists(fileName): logger.info( "Downloading {instrument} {year} to {filename}".format( instrument=instrument, year=year, filename=fileName)) try: if frequency == bar.Frequency.DAY: download_daily_bars(instrument, year, fileName) else: raise Exception("Invalid frequency") except Exception as e: if skipErrors: logger.error(str(e)) continue else: raise e ret.addBarsFromCSV(instrument, fileName) return ret
def main(): parser = argparse.ArgumentParser(description="Quandl utility") parser.add_argument("--auth-token", required=False, help="An authentication token needed if you're doing more than 50 calls per day") parser.add_argument("--source-code", required=True, help="The dataset source code") parser.add_argument("--table-code", required=True, help="The dataset table code") parser.add_argument("--from-year", required=True, type=int, help="The first year to download") parser.add_argument("--to-year", required=True, type=int, help="The last year to download") parser.add_argument("--storage", required=True, help="The path were the files will be downloaded to") parser.add_argument("--force-download", action='store_true', help="Force downloading even if the files exist") parser.add_argument("--ignore-errors", action='store_true', help="True to keep on downloading files in case of errors") parser.add_argument("--frequency", default="daily", choices=[ "daily", "weekly"], help="The frequency of the bars. Only daily or weekly are supported") args = parser.parse_args() logger = pyalgotrade.logger.getLogger("quandl") if not os.path.exists(args.storage): logger.info("Creating %s directory" % (args.storage)) os.mkdir(args.storage) for year in range(args.from_year, args.to_year + 1): fileName = os.path.join(args.storage, "%s-%s-%d-quandl.csv" % (args.source_code, args.table_code, year)) if not os.path.exists(fileName) or args.force_download: logger.info("Downloading %s %d to %s" % (args.table_code, year, fileName)) try: if args.frequency == "daily": download_daily_bars( args.source_code, args.table_code, year, fileName, args.auth_token) else: assert args.frequency == "weekly", "Invalid frequency" download_weekly_bars( args.source_code, args.table_code, year, fileName, args.auth_token) except Exception as e: if args.ignore_errors: logger.error(str(e)) continue else: raise
def getNextJob(self): ret = None # Get the next set of parameters. params = self.__paramSource.getNext(self.defaultBatchSize) params = map(lambda p: p.args, params) logger.info("Sending %s work items" % (len(params))) # Map the active job if len(params): ret = Job(params) with self.__activeJobsLock: self.__activeJobs[ret.getId()] = ret return pickle.dumps(ret)
def backFill(self, freq,filename, startDate, endDate): if freq == 86400: final_table = "1day_data" elif freq == 1800: final_table = "30mins_data" elif freq == 300: final_table = "5mins_data" cursor = self.__con.cursor() cleanup_qry_text = ("""DELETE FROM %s WHERE date(date) >= '%s' and date(date) <= '%s'""" % (final_table, startDate, endDate)) logger.info(cleanup_qry_text) cursor.execute(cleanup_qry_text) insert_qry_text = """INSERT INTO {0:s} (symbol, date, open, close, high, low, volume) select t0.symbol, t0.new_date date, t2.open, t1.close, t0.high, t0.low, t0.volume from (select data.symbol symbol,from_unixtime((floor((unix_timestamp(data.date) / {1:d})) * {1:d}){2:s}) new_date, max(data.high) high, min(data.low) low, max(data.date) max_ts, min(data.date) min_ts, avg(data.volume) volume from data where date(date) >= '{3:s}' and date(date) <= '{4:s}' group by 1,2) t0 join (select tbl2.symbol, tbl2.date, tbl2.close close from data tbl2 join (select symbol, date, max(milliseconds) max_milli_secs from data where date(date) >= '{3:s}' and date(date) <= '{4:s}' group by 1,2) tbl1 on tbl2.symbol = tbl1.symbol and tbl2.date = tbl1.date and tbl2.milliseconds = tbl1.max_milli_secs where date(tbl2.date) >= '{3:s}' and date(tbl2.date) <= '{4:s}' group by 1,2) t1 on t0.symbol = t1.symbol and t0.max_ts = t1.date join (select tbl2.symbol, tbl2.date, tbl2.open open from data tbl2 join (select symbol, date, min(milliseconds) min_milli_secs from data where date(date) >= '{3:s}' and date(date) <= '{4:s}' group by 1,2) tbl1 on tbl2.symbol = tbl1.symbol and tbl2.date = tbl1.date and tbl2.milliseconds = tbl1.min_milli_secs where date(tbl2.date) >= '{3:s}' and date(tbl2.date) <= '{4:s}' group by 1,2) t2 on t0.symbol = t2.symbol and t0.min_ts = t2.date""" if freq == CONSTANTS.ONE_DAY: insert_qry_text = insert_qry_text.format(final_table, freq, '+8*3600', startDate, endDate) else: insert_qry_text = insert_qry_text.format(final_table, freq, '', startDate, endDate) logger.info(insert_qry_text) cursor.execute(insert_qry_text) self.__con.commit() cursor.close() text_file = open(filename, "a") text_file.writelines("Back fill job %s started at: %s, complete at: %s , end date %s\n" % (final_table, startDate, datetime.now(), endDate)) text_file.close()
def push(self, result, parameters): """ Push strategy results obtained by running the strategy with the given parameters. :param result: The result obtained by running the strategy with the given parameters. :type result: float :param parameters: The parameters that yield the given result. :type parameters: Parameters """ with self.__lock: if result is not None and (self.__bestResult is None or result > self.__bestResult): self.__bestResult = result self.__bestParameters = parameters logger.info("Best result so far %s with parameters %s" % (result, parameters.args))
def main(): try: writer = symbolsxml.Writer() for symbol in open("merval-symbols.txt", "r"): symbol = symbol.strip() process_symbol(writer, symbol) # Index writer.addIndex("^MERV", "Merval") logger.info("Writing merval.xml") writer.write("merval.xml") except Exception, e: logger.error(str(e))
def build_feed(instruments, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False): """Build and load a :class:`pyalgotrade.barfeed.yahoofeed.Feed` using CSV files downloaded from Yahoo! Finance. CSV files are downloaded if they haven't been downloaded before. :param instruments: Instrument identifiers. :type instruments: list. :param fromYear: The first year. :type fromYear: int. :param toYear: The last year. :type toYear: int. :param storage: The path were the files will be loaded from, or downloaded to. :type storage: string. :param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** or **pyalgotrade.bar.Frequency.WEEK** are supported. :param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`. :type timezone: A pytz timezone. :param skipErrors: True to keep on loading/downloading files in case of errors. :type skipErrors: boolean. :rtype: :class:`pyalgotrade.barfeed.tusharefeed.Feed`. """ logger = pyalgotrade.logger.getLogger("tushare") ret = tusharefeed.Feed(frequency, timezone) datapath = os.path.join(storage, "data") if not os.path.exists(datapath): logger.info("Creating %s directory" % (datapath)) os.mkdir(storage) for year in range(fromYear, toYear+1): for instrument in instruments: fileName = os.path.join(datapath, "%s-%d-tushare.csv" % (instrument, year)) if not os.path.exists(fileName): logger.info("Downloading %s %d to %s" % (instrument, year, fileName)) try: if frequency == bar.Frequency.DAY: download_daily_bars(instrument, year, fileName) elif frequency == bar.Frequency.WEEK: download_weekly_bars(instrument, year, fileName) else: raise Exception("Invalid frequency") except Exception, e: if skipErrors: logger.error(str(e)) continue else: raise e ret.addBarsFromCSV(instrument, fileName)
def parse_results(table): ret = symbolsxml.Writer() logger.info("Parsing table") rows = table.xpath("tr") for row in rows[1:]: cols = row.xpath("td") tickerSymbol = cols[TICKER_SYMBOL_COL].xpath("a[1]")[0].text company = cols[COMPANY_COL].xpath("a[1]")[0].text gics = cols[GICS_COL].text gicsSubIndustry = cols[GICS_SUB_INDUSTRY_COL].text if gicsSubIndustry is None: gicsSubIndustry = "" ret.addStock(tickerSymbol, company, gics, gicsSubIndustry) return ret
def download_trades_since(currency, tid, ignoreMultiCurrency, retries=3): logger.info("Downloading trades since %s." % (base.tid_to_datetime(tid))) # logger.info("Downloading trades since %d." % (tid)) done = False while not done: try: response = download_trades_impl(currency, tid) done = True except Exception, e: if retries == 0: raise e else: logger.error("%s. Retrying..." % (e)) retries -= 1
def start(self): if self.__thread == None: logger.info("Initializing MtGox client.") self.__wsClient.connect() self.__thread = threading.Thread(target=self.__threadMain) self.__thread.start() # Wait for initialization to complete. while self.__initializationFailed == None and self.__thread.is_alive(): self.dispatchImpl([WSClient.ON_CONNECTED]) if self.__initializationFailed == False: logger.info("Initialization complete.") else: raise Exception("Initialization failed") else: raise Exception("Already running")
def requestPrivateIdKey(self): out = {"result":None} def onResult(data): out["result"] = data def onRemark(data): logger.error("Remark requesting private id key: %s" % (data)) logger.info("Requesting private id key.") requestId = self.__wsClient.requestPrivateIdKey() self.waitResponse(requestId, onResult, onRemark, 30) ret = out["result"] if ret in (None, ""): raise Exception("Failed to get private id key") return ret
def __initializeClient(self): self.__initializationOk = None logger.info("Initializing client.") try: # Try to connect self.__thread = None self.__wsClient = WSClient() self.__wsClient.connect() # Start the thread that runs the client. self.__thread = threading.Thread(target=self.__threadMain) self.__thread.start() except Exception, e: self.__initializationOk = False logger.error("Error connecting : %s" % str(e))
def __init__(self, identifiers, apiCallDelay=5, maxLen=dataseries.DEFAULT_MAX_LEN): logger.info('Livefeed created') barfeed.BaseBarFeed.__init__(self, bar.Frequency.TRADE, maxLen) if not isinstance(identifiers, list): raise Exception("identifiers must be a list") self.__queue = Queue.Queue() self.__orderBookUpdateEvent = observer.Event() self.__thread = TradesAPIThread( self.__queue, identifiers, datetime.timedelta(seconds=apiCallDelay)) self.__bars = [] for instrument in identifiers: self.registerInstrument(instrument)
def requestPrivateIdKey(self): out = {"result": None} def onResult(data): out["result"] = data def onRemark(data): logger.error("Remark requesting private id key: %s" % (data)) logger.info("Requesting private id key.") requestId = self.__wsClient.requestPrivateIdKey() self.waitResponse(requestId, onResult, onRemark, 30) ret = out["result"] if ret in (None, ""): raise Exception("Failed to get private id key") return ret
def OnRspUserLogin(self, *args): logger.info("OnRspUserLogin") if args: loginField = args[0] logger.info(loginField.TradingDay) logger.info(loginField.LoginTime) #construct contract ID date = datetime.strptime(loginField.TradingDay, "%Y%m%d") self.__md.SubscribeMarketData(self.__contactIDs, len(self.__contactIDs)) logger.info("OnRspUserLogin End")
def pushJobResults(self, jobId, result, parameters, workerName): jobId = pickle.loads(jobId) result = pickle.loads(result) parameters = pickle.loads(parameters) # Remove the job mapping. with self.__activeJobsLock: try: del self.__activeJobs[jobId] except KeyError: # The job's results were already submitted. return if result is None or result > self.__bestResult: logger.info("Best result so far %s with parameters %s" % (result, parameters)) self.__bestResult = result self.__resultSinc.push(result, base.Parameters(*parameters))
def download_files_for_symbol(symbol, fromYear, toYear): if not os.path.exists(storage): logger.info("Creating %s directory" % (storage)) os.mkdir(storage) status = "" for year in range(fromYear, toYear+1): fileName = get_csv_filename(symbol, year) if not os.path.exists(fileName): logger.info("Downloading %s %d to %s" % (symbol, year, fileName)) try: yahoofinance.download_daily_bars(symbol, year, fileName) status += "1" except Exception, e: logger.error(str(e)) status += "0" else: status += "1"
def find_table(htmlTree): logger.info("Finding the right table") ret = None tables = htmlTree.xpath("//table[@class='wikitable sortable']") for table in tables: headers = table.xpath("tr[1]/th") if len(headers) > 5: if headers[TICKER_SYMBOL_COL].xpath("a[1]")[0].text != "Ticker symbol": continue if headers[COMPANY_COL].text != "Company": continue if headers[GICS_COL].xpath("a[1]")[0].text != "GICS": continue if headers[GICS_SUB_INDUSTRY_COL].text != "GICS Sub Industry": continue ret = table break return ret
def main(): try: logger.info("Getting NASDAQ symbols from http://www.nasdaq.com/") url = "http://www.nasdaq.com/screening/companies-by-name.aspx?exchange=NASDAQ&render=download" buff = urllib2.urlopen(url).read() tmpFile = tempfile.NamedTemporaryFile() tmpFile.write(buff) tmpFile.flush() with open(tmpFile.name, 'rb') as csvfile: symbolsXML = symbolsxml.Writer() for row in csv.DictReader(csvfile): symbolsXML.addStock(row["Symbol"], row["Name"], row["Sector"], row["industry"]) logger.info("Writing nasdaq.xml") symbolsXML.write("nasdaq.xml") except Exception, e: logger.error(str(e))
def download_files_for_symbol(symbol, fromYear, toYear): if not os.path.exists(storage): logger.info("Creating %s directory" % (storage)) os.mkdir(storage) status = "" for year in range(fromYear, toYear + 1): fileName = get_csv_filename(symbol, year) if not os.path.exists(fileName): logger.info("Downloading %s %d to %s" % (symbol, year, fileName)) try: yahoofinance.download_daily_bars(symbol, year, fileName) status += "1" except Exception, e: logger.error(str(e)) status += "0" else: status += "1"