def update( self): """ Grab most recent prices from on/offline and append them to our exchange data structures. """ ####################################################### # -- SIMULATION MODE -- # ####################################################### # simulation mode. pull most recent price from our logs and # append if different if self.live == False: ####################################################### # -- REAL TIME SIMULATION MODE -- # ####################################################### # if warp is false, we will pretend this is realtime and # grab prices from our logs using our timedelta if self.warp == False: # calculate our timedelta from NOW!! adjusted_t = datetime.datetime.today() - self.delta # Get our last prices from the logs last_gox , last_ltc , last_ltc_depth = self._offline_prices( adjusted_t) # make sure we got a timeseries object back, otherwise we # hit the end of the log if( type(last_gox) != pd.Series or type(last_ltc) != pd.Series or type(last_ltc_depth) != pd.Series): if self.verbose: print "[!]", "End of log." return False # we have values, so add them to each coin else: # give coins new price changes ... them bitches'll do the rest self.gox.add( last_gox[0], last_gox.name) # bid-ask avg for LTC only ba = pl2.bid_ask( last_ltc_depth[0]) self.ltc.add( last_ltc[0], last_ltc.name, ba=ba) return True ####################################################### # -- FAST MODE -- # ####################################################### # otherwise, we'll grab our next price from the index else: # r we about to do something stupid? (hit end of the f*****g log) if self.logrange_n >= len(self.logrange): if self.verbose: print "[!]", "End of log." return False # NO! else: # get our next date in our time index & grab the prices t = self.logrange[self.logrange_n] if self._debug: print "\n_update" print "t:", t print "logrange:", self.logrange_n last_gox, last_ltc, last_ltc_depth = self._offline_prices( t) # get LTC market data (bid ask) ba = pl2.bid_ask( last_ltc_depth[0]) # upd8 fuk'n coinz if self._debug: print "\n_update" print "\nltc" print "last_ltc:", last_ltc[0], last_ltc.name print "ba:", ba self.ltc.add( last_ltc[0], last_ltc.name, ba=ba) if self._debug: print "\ngox" print "last_gox:", last_gox[0], last_gox.name self.gox.add( last_gox[0], last_gox.name) # increment for the next f*****g time self.logrange_n += 1 return True
def update(self): """ Grab most recent prices from on/offline and append them to our exchange data structures. """ ####################################################### # -- SIMULATION MODE -- # ####################################################### # simulation mode. pull most recent price from our logs and # append if different if self.live == False: ####################################################### # -- REAL TIME SIMULATION MODE -- # ####################################################### # if warp is false, we will pretend this is realtime and # grab prices from our logs using our timedelta if self.warp == False: # calculate our timedelta from NOW!! adjusted_t = datetime.datetime.today() - self.delta # Get our last prices from the logs last_gox, last_ltc, last_ltc_depth = self._offline_prices( adjusted_t) # make sure we got a timeseries object back, otherwise we # hit the end of the log if (type(last_gox) != pd.Series or type(last_ltc) != pd.Series or type(last_ltc_depth) != pd.Series): if self.verbose: print "[!]", "End of log." return False # we have values, so add them to each coin else: # give coins new price changes ... them bitches'll do the rest self.gox.add(last_gox[0], last_gox.name) # bid-ask avg for LTC only ba = pl2.bid_ask(last_ltc_depth[0]) self.ltc.add(last_ltc[0], last_ltc.name, ba=ba) return True ####################################################### # -- FAST MODE -- # ####################################################### # otherwise, we'll grab our next price from the index else: # r we about to do something stupid? (hit end of the f*****g log) if self.logrange_n >= len(self.logrange): if self.verbose: print "[!]", "End of log." return False # NO! else: # get our next date in our time index & grab the prices t = self.logrange[self.logrange_n] if self._debug: print "\n_update" print "t:", t print "logrange:", self.logrange_n last_gox, last_ltc, last_ltc_depth = self._offline_prices( t) # get LTC market data (bid ask) ba = pl2.bid_ask(last_ltc_depth[0]) # upd8 fuk'n coinz if self._debug: print "\n_update" print "\nltc" print "last_ltc:", last_ltc[0], last_ltc.name print "ba:", ba self.ltc.add(last_ltc[0], last_ltc.name, ba=ba) if self._debug: print "\ngox" print "last_gox:", last_gox[0], last_gox.name self.gox.add(last_gox[0], last_gox.name) # increment for the next f*****g time self.logrange_n += 1 return True
def __init__( self, **kwargs): """ Set up our data structures and determine whether we're in live or simulated mode. time_str : (default "5min") time-frame to analyze on ... this controls the length of each "bar" or period, can be any pandas-recognized string, (10s, 10min, 1h, 1d, etc) live : live or simulated mode (whether or not to read from filename or from the web), defaults to False (simulated) filename : name of log file to read in simulated mode ... interpreted as ./logs/filename ... file must be in this dir warp : whether or not to use our timedelta or just next value for each update() ... so we can do all calculations as fast as possible, defaults to False ("realtime") debug : whether or not to spit out debugging info sample_secs : if in warp-mode, N-seconds to sample on (the shorter N, the more often we are "checking" the price and the more iterations it will take to complete a series) instant : (default False) Setting this to true will make Data send the lastprice series to the Coins to calculate all in faster, one-pass mode ltc_opts : dict structure on what to do with LTC data ... see coin for options from kwargs (default is same as GOX...) Here's an example of a fully loaded options dict { "debug": False, "relative": False, "calc_rolling": False, "rolling": { self.time_str : { 5: pd.DataFrame(), 25: pd.DataFrame(), 50: pd.DataFrame() } }, "calc_mid": False, "calc_ohlc": True, "ohlc": { self.time_str : pd.DataFrame() }, "calc_indicators": True, "indicators":{ "RSI" : { "data": pd.DataFrame(), "n":14 }, "ROC" : { "data": pd.DataFrame(), "n":20 }, "AMA" : { "data": pd.DataFrame(), "n":10, "fn":2.5, "sn":30 }, "CCI" : { "data": pd.DataFrame(), "n":20 }, "FRAMA": { "data": pd.DataFrame(), "n":10 }, "RVI2" : { "data": pd.DataFrame(), "n":14, "s":10 }, "MACD" : { "data": pd.DataFrame(), "f":12, "s":26, "m":9 }, "ADX" : { "data": pd.DataFrame(), "n":14 }, "ELI" : { "data": pd.DataFrame(), "n":14 }, "TMI" : { "data": pd.DataFrame(), "nb":10, "nf":5} } "calc_std": True, "std": { 10: pd.DataFrame(), 50: pd.DataFrame(), 100: pd.DataFrame() }, "calc_crt": True, "crt": { 1: pd.DataFrame(), 2: pd.DataFrame(), 3: pd.DataFrame(), 5: pd.DataFrame(), 8: pd.DataFrame() }, "instant": False, "time_str": self.time_str } gox_opts : dict structure on what to do with GOX BTC data ... see coin for options from kwargs (default: everything disabled but OHLC ... ) { "debug": False, "relative": False, "calc_rolling": False, "rolling": False, "calc_mid": False, "calc_ohlc": True, "ohlc": { self.time_str : pd.DataFrame() }, "calc_indicators": False, "calc_std": False, "std": False, "calc_crt": False, "crt": False, "instant": False, "time_str": self.time_str } pickled_data : (default False) if this is set to a data structure, from pickle'd pandas csv data structure, it'll take it from here instead of from disk. Faster on multiple iterations. verbose : (default False) whether or not to print out shit """ self.live = kwargs.get("live", False) self.filename = kwargs.get("filename", "test.csv") self.warp = kwargs.get( "warp", True) self._debug = kwargs.get( "debug", False) self.sample_secs = kwargs.get( "sample_secs", 5) self.instant = kwargs.get( "instant", False) self.time_str = kwargs.get( "time_str", "5min") self.verbose = kwargs.get( "verbose", False) # default LTC options def_ltc = { "debug": False, "relative": False, "calc_rolling": False, "rolling": False, "calc_mid": False, "calc_ohlc": True, "ohlc": { self.time_str : pd.DataFrame() }, "calc_indicators": False, "indicators": False, "calc_std": False, "std": False, "calc_crt": False, "crt": False, "instant": False, "time_str": self.time_str } self.ltc_opts = kwargs.get( "ltc_opts", def_ltc) # default gox options def_gox = { "debug": False, "relative": False, "calc_rolling": False, "rolling": False, "calc_mid": False, "calc_ohlc": True, "ohlc": { self.time_str : pd.DataFrame() }, "calc_indicators": False, "indicators": False, "calc_std": False, "std": False, "calc_crt": False, "crt": False, "instant": False, "time_str": self.time_str } self.gox_opts = kwargs.get( "gox_opts", def_gox) self.pickled_data = kwargs.get( "pickled_data", False) if self.verbose: print "[*]", "Online" if self.live else "Offline", "mode initiated" print "[*]", "Simulated" if not self.warp else "Speed", "mode initiated" # if we're running simulated, set up price logs so we can query them # in realtime as if they were actual price changes if self.live == False: # did we supply a pre-parsed pandas CSV data struct? if self.pickled_data != False: if self.verbose: print "[*]", "Loading supplied pickle!" data = self.pickled_data # nope ... load from disk! else: # loading from CSV takes a long time, lets prepare a pickle of the # loaded CSV if we haven't already done so, if we have then load it filename_pick = os.path.realpath( os.path.join( "logs", self.filename+".pickle")) if os.path.exists( filename_pick): if self.verbose: print "[*]", "Loading csv pickle from %s" % filename_pick f = open( filename_pick, "rb") data = cPickle.load( f) f.close() else: filename_csv = os.path.realpath( os.path.join( "logs", self.filename)) if self.verbose: print "[*] Loading %s" % filename_csv data = pl2.load2( filename_csv) if self.verbose: print "[*] Generating pickle for next time to %s" % filename_pick f = open( filename_pick, "wb") cPickle.dump( data, f) f.close() # load our time-series dataframe from csv using pandas library self._gox_offline = data["gox"] self._ltc_offline = data["ltc"] self._ltc_depth_offline = data["ltc_depth"] # if we're running in non-simulated offline mode, where we just # want to run through our historical price data as quickly as # possible, then we build a range of dates that we will walk through if self.warp == True: # get our start and end points in our timerange start = max( [ self._gox_offline.index[0], self._ltc_offline.index[0]]) end = max( [ self._gox_offline.index[-1], self._ltc_offline.index[-1]]) # our list of total dates to run through # jump to N-seconds intervals (self.sample_secs) if self.verbose: print "[*]","Building daterange" self.logrange = self._daterange( start, end, self.sample_secs) # we're going to need to iterate through this one at a time ... # get new values, calculate indicators, train, repeat, so we'll # need to keep track of where we are self.logrange_n = 0 if self.verbose: print "[*] Dates from", start, "to", end # otherwise we pretend we're live (slow so we can watch it IRT) else: # find out which has the earliest starting date. We will use # this to calculate our timedelta. In the future when we want # to check the price, we will use this delta compared to current # time to grab the proper simulated price # (we use max here so we don't get any initial NaN prices if possible) self.delta = datetime.datetime.today() - max( [ self._gox_offline.index[0], self._ltc_offline.index[0]]) if self.verbose: print "[*] Timedelta: %s" % self.delta ##################################### # # # C O I N S # # # ##################################### # prepare instant if necessary if self.instant: # seed prices with midprice if self.ltc_opts["calc_mid"]: filename = os.path.realpath( os.path.join( "logs", self.filename+".midprices.pickle")) # if midprices pickle doesn't exist, we need to generate it ... this is slow as f**k # so we really want to have this preloaded if os.path.exists( filename): if self.verbose: print "[*]", "Loading midprices from %s" % filename f = open( filename, "rb") bas = cPickle.load( f) else: if self.verbose: print "[*]","Calculating midprices ..." bas = [ pl2.bid_ask(self._ltc_depth_offline.ix[i][0], avg=True) for i in xrange( len( self._ltc_depth_offline))] f = open( filename, "wb") if self.verbose: print "[*]", "Saving midprices to %s" % filename cPickle.dump( bas, f) self.ltc_opts["instant"] = pd.DataFrame( {"lastprice":bas}, index=[self._ltc_depth_offline.index]) # otherwise hand it lastprice else: self.ltc_opts["instant"] = self._ltc_offline self.ltc = Coin( debug=self.ltc_opts["debug"], relative=self.ltc_opts["relative"], calc_rolling=self.ltc_opts["calc_rolling"], rolling=self.ltc_opts["rolling"], calc_mid=self.ltc_opts["calc_mid"], calc_ohlc=self.ltc_opts["calc_ohlc"], ohlc=self.ltc_opts["ohlc"], calc_indicators=self.ltc_opts["calc_indicators"], indicators=self.ltc_opts["indicators"], calc_std=self.ltc_opts["calc_std"], std=self.ltc_opts["std"], calc_crt=self.ltc_opts["calc_crt"], crt=self.ltc_opts["crt"], instant=self.ltc_opts["instant"], time_str=self.ltc_opts["time_str"], verbose=self.verbose) # for gox, all I want to calculate is the EMA of the last prices ... # I chose last price, not mid, because I think that a lot of people # are trading based on the last price ticker, not where the market # really is. # prepare instant if necessary # prepare instant if necessary if self.instant: # seed prices with midprice if self.gox_opts["calc_mid"]: if self.verbose: print "[*]","Calculating midprices ..." bas = [ pl2.bid_ask(self._gox_depth_offline.ix[i][0], avg=True) for i in xrange( len( self._gox_depth_offline))] self.gox_opts["instant"] = pd.DataFrame( {"lastprice":bas}, index=[self._gox_depth_offline.index]) # otherwise hand it lastprice else: self.gox_opts["instant"] = self._gox_offline self.gox = Coin( debug=self.gox_opts["debug"], relative=self.gox_opts["relative"], calc_rolling=self.gox_opts["calc_rolling"], rolling=self.gox_opts["rolling"], calc_mid=self.gox_opts["calc_mid"], calc_ohlc=self.gox_opts["calc_ohlc"], ohlc=self.gox_opts["ohlc"], calc_indicators=self.gox_opts["calc_indicators"], indicators=self.gox_opts["indicators"], calc_std=self.gox_opts["calc_std"], std=self.gox_opts["std"], calc_crt=self.gox_opts["calc_crt"], crt=self.gox_opts["crt"], instant=self.gox_opts["instant"], time_str=self.gox_opts["time_str"], verbose=self.verbose)
def __init__(self, **kwargs): """ Set up our data structures and determine whether we're in live or simulated mode. time_str : (default "5min") time-frame to analyze on ... this controls the length of each "bar" or period, can be any pandas-recognized string, (10s, 10min, 1h, 1d, etc) live : live or simulated mode (whether or not to read from filename or from the web), defaults to False (simulated) filename : name of log file to read in simulated mode ... interpreted as ./logs/filename ... file must be in this dir warp : whether or not to use our timedelta or just next value for each update() ... so we can do all calculations as fast as possible, defaults to False ("realtime") debug : whether or not to spit out debugging info sample_secs : if in warp-mode, N-seconds to sample on (the shorter N, the more often we are "checking" the price and the more iterations it will take to complete a series) instant : (default False) Setting this to true will make Data send the lastprice series to the Coins to calculate all in faster, one-pass mode ltc_opts : dict structure on what to do with LTC data ... see coin for options from kwargs (default is same as GOX...) Here's an example of a fully loaded options dict { "debug": False, "relative": False, "calc_rolling": False, "rolling": { self.time_str : { 5: pd.DataFrame(), 25: pd.DataFrame(), 50: pd.DataFrame() } }, "calc_mid": False, "calc_ohlc": True, "ohlc": { self.time_str : pd.DataFrame() }, "calc_indicators": True, "indicators":{ "RSI" : { "data": pd.DataFrame(), "n":14 }, "ROC" : { "data": pd.DataFrame(), "n":20 }, "AMA" : { "data": pd.DataFrame(), "n":10, "fn":2.5, "sn":30 }, "CCI" : { "data": pd.DataFrame(), "n":20 }, "FRAMA": { "data": pd.DataFrame(), "n":10 }, "RVI2" : { "data": pd.DataFrame(), "n":14, "s":10 }, "MACD" : { "data": pd.DataFrame(), "f":12, "s":26, "m":9 }, "ADX" : { "data": pd.DataFrame(), "n":14 }, "ELI" : { "data": pd.DataFrame(), "n":14 }, "TMI" : { "data": pd.DataFrame(), "nb":10, "nf":5} } "calc_std": True, "std": { 10: pd.DataFrame(), 50: pd.DataFrame(), 100: pd.DataFrame() }, "calc_crt": True, "crt": { 1: pd.DataFrame(), 2: pd.DataFrame(), 3: pd.DataFrame(), 5: pd.DataFrame(), 8: pd.DataFrame() }, "instant": False, "time_str": self.time_str } gox_opts : dict structure on what to do with GOX BTC data ... see coin for options from kwargs (default: everything disabled but OHLC ... ) { "debug": False, "relative": False, "calc_rolling": False, "rolling": False, "calc_mid": False, "calc_ohlc": True, "ohlc": { self.time_str : pd.DataFrame() }, "calc_indicators": False, "calc_std": False, "std": False, "calc_crt": False, "crt": False, "instant": False, "time_str": self.time_str } pickled_data : (default False) if this is set to a data structure, from pickle'd pandas csv data structure, it'll take it from here instead of from disk. Faster on multiple iterations. verbose : (default False) whether or not to print out shit """ self.live = kwargs.get("live", False) self.filename = kwargs.get("filename", "test.csv") self.warp = kwargs.get("warp", True) self._debug = kwargs.get("debug", False) self.sample_secs = kwargs.get("sample_secs", 5) self.instant = kwargs.get("instant", False) self.time_str = kwargs.get("time_str", "5min") self.verbose = kwargs.get("verbose", False) # default LTC options def_ltc = { "debug": False, "relative": False, "calc_rolling": False, "rolling": False, "calc_mid": False, "calc_ohlc": True, "ohlc": { self.time_str: pd.DataFrame() }, "calc_indicators": False, "indicators": False, "calc_std": False, "std": False, "calc_crt": False, "crt": False, "instant": False, "time_str": self.time_str } self.ltc_opts = kwargs.get("ltc_opts", def_ltc) # default gox options def_gox = { "debug": False, "relative": False, "calc_rolling": False, "rolling": False, "calc_mid": False, "calc_ohlc": True, "ohlc": { self.time_str: pd.DataFrame() }, "calc_indicators": False, "indicators": False, "calc_std": False, "std": False, "calc_crt": False, "crt": False, "instant": False, "time_str": self.time_str } self.gox_opts = kwargs.get("gox_opts", def_gox) self.pickled_data = kwargs.get("pickled_data", False) if self.verbose: print "[*]", "Online" if self.live else "Offline", "mode initiated" print "[*]", "Simulated" if not self.warp else "Speed", "mode initiated" # if we're running simulated, set up price logs so we can query them # in realtime as if they were actual price changes if self.live == False: # did we supply a pre-parsed pandas CSV data struct? if self.pickled_data != False: if self.verbose: print "[*]", "Loading supplied pickle!" data = self.pickled_data # nope ... load from disk! else: # loading from CSV takes a long time, lets prepare a pickle of the # loaded CSV if we haven't already done so, if we have then load it filename_pick = os.path.realpath( os.path.join("logs", self.filename + ".pickle")) if os.path.exists(filename_pick): if self.verbose: print "[*]", "Loading csv pickle from %s" % filename_pick f = open(filename_pick, "rb") data = cPickle.load(f) f.close() else: filename_csv = os.path.realpath( os.path.join("logs", self.filename)) if self.verbose: print "[*] Loading %s" % filename_csv data = pl2.load2(filename_csv) if self.verbose: print "[*] Generating pickle for next time to %s" % filename_pick f = open(filename_pick, "wb") cPickle.dump(data, f) f.close() # load our time-series dataframe from csv using pandas library self._gox_offline = data["gox"] self._ltc_offline = data["ltc"] self._ltc_depth_offline = data["ltc_depth"] # if we're running in non-simulated offline mode, where we just # want to run through our historical price data as quickly as # possible, then we build a range of dates that we will walk through if self.warp == True: # get our start and end points in our timerange start = max( [self._gox_offline.index[0], self._ltc_offline.index[0]]) end = max( [self._gox_offline.index[-1], self._ltc_offline.index[-1]]) # our list of total dates to run through # jump to N-seconds intervals (self.sample_secs) if self.verbose: print "[*]", "Building daterange" self.logrange = self._daterange(start, end, self.sample_secs) # we're going to need to iterate through this one at a time ... # get new values, calculate indicators, train, repeat, so we'll # need to keep track of where we are self.logrange_n = 0 if self.verbose: print "[*] Dates from", start, "to", end # otherwise we pretend we're live (slow so we can watch it IRT) else: # find out which has the earliest starting date. We will use # this to calculate our timedelta. In the future when we want # to check the price, we will use this delta compared to current # time to grab the proper simulated price # (we use max here so we don't get any initial NaN prices if possible) self.delta = datetime.datetime.today() - max( [self._gox_offline.index[0], self._ltc_offline.index[0]]) if self.verbose: print "[*] Timedelta: %s" % self.delta ##################################### # # # C O I N S # # # ##################################### # prepare instant if necessary if self.instant: # seed prices with midprice if self.ltc_opts["calc_mid"]: filename = os.path.realpath( os.path.join("logs", self.filename + ".midprices.pickle")) # if midprices pickle doesn't exist, we need to generate it ... this is slow as f**k # so we really want to have this preloaded if os.path.exists(filename): if self.verbose: print "[*]", "Loading midprices from %s" % filename f = open(filename, "rb") bas = cPickle.load(f) else: if self.verbose: print "[*]", "Calculating midprices ..." bas = [ pl2.bid_ask(self._ltc_depth_offline.ix[i][0], avg=True) for i in xrange(len(self._ltc_depth_offline)) ] f = open(filename, "wb") if self.verbose: print "[*]", "Saving midprices to %s" % filename cPickle.dump(bas, f) self.ltc_opts["instant"] = pd.DataFrame( {"lastprice": bas}, index=[self._ltc_depth_offline.index]) # otherwise hand it lastprice else: self.ltc_opts["instant"] = self._ltc_offline self.ltc = Coin(debug=self.ltc_opts["debug"], relative=self.ltc_opts["relative"], calc_rolling=self.ltc_opts["calc_rolling"], rolling=self.ltc_opts["rolling"], calc_mid=self.ltc_opts["calc_mid"], calc_ohlc=self.ltc_opts["calc_ohlc"], ohlc=self.ltc_opts["ohlc"], calc_indicators=self.ltc_opts["calc_indicators"], indicators=self.ltc_opts["indicators"], calc_std=self.ltc_opts["calc_std"], std=self.ltc_opts["std"], calc_crt=self.ltc_opts["calc_crt"], crt=self.ltc_opts["crt"], instant=self.ltc_opts["instant"], time_str=self.ltc_opts["time_str"], verbose=self.verbose) # for gox, all I want to calculate is the EMA of the last prices ... # I chose last price, not mid, because I think that a lot of people # are trading based on the last price ticker, not where the market # really is. # prepare instant if necessary # prepare instant if necessary if self.instant: # seed prices with midprice if self.gox_opts["calc_mid"]: if self.verbose: print "[*]", "Calculating midprices ..." bas = [ pl2.bid_ask(self._gox_depth_offline.ix[i][0], avg=True) for i in xrange(len(self._gox_depth_offline)) ] self.gox_opts["instant"] = pd.DataFrame( {"lastprice": bas}, index=[self._gox_depth_offline.index]) # otherwise hand it lastprice else: self.gox_opts["instant"] = self._gox_offline self.gox = Coin(debug=self.gox_opts["debug"], relative=self.gox_opts["relative"], calc_rolling=self.gox_opts["calc_rolling"], rolling=self.gox_opts["rolling"], calc_mid=self.gox_opts["calc_mid"], calc_ohlc=self.gox_opts["calc_ohlc"], ohlc=self.gox_opts["ohlc"], calc_indicators=self.gox_opts["calc_indicators"], indicators=self.gox_opts["indicators"], calc_std=self.gox_opts["calc_std"], std=self.gox_opts["std"], calc_crt=self.gox_opts["calc_crt"], crt=self.gox_opts["crt"], instant=self.gox_opts["instant"], time_str=self.gox_opts["time_str"], verbose=self.verbose)