def poll_history(url,save):
#resp = requests.get(url=url)
params = get_query_parameters(url)
json = get_json(url,params)
#print json
if ('return' in json):
data = json.get('return')
else:
data = json
result = []
#print data
for trade in data:
if save:
mtgox = MtgoxTrade()
mtgox.amount = trade['amount']
mtgox.time = trade['date']
mtgox.price = trade['price']
mtgox.type = trade['trade_type']
mtgox.tid = trade['tid']
#print mtgox.tid
mtgox.save()
else:
js = dict()
js['time']= trade['date']
js['price']= trade['price']
js['amount']= trade['amount']
js['type']= trade['trade_type']
js['tid']= trade['tid']
result.append(js)
#print js
#print result
length = len(data)-1
if (length >=1):
last_data = data[length]
else:
return result
#print last_data
last_tid = last_data.get('tid')
#print last_tid
since = params.get('since')
if (since is not None and long(last_tid)<long(since)):
print 'Finish polling mgtox history data'
return
url = 'https://data.mtgox.com/api/1/BTCusd/trades?since='+last_tid
if save:
poll_history(url,save)
return
else:
result.extend(poll_history(url,save))
return result
def save_trades(market, trades):
last_update = get_latest_trade_time(market)
print last_update
for data in trades:
if market == "MTgox":
time = int(data["time"])
trade = MtgoxTrade()
trade.tid = data["tid"]
# return
elif market == "796futures":
time = int(data["time"])
trade = Futures796Trade()
elif market == "796stockpd":
time = int(data["time"])
trade = Stockpd796Trade()
if int(time) > int(last_update):
trade.time = time
trade.price = float(data["price"])
trade.amount = float(data["amount"])
trade.type = data["type"]
trade.save()
def save_trades(market,trades):
last_update = get_latest_trade_time(market)
print last_update
for data in trades:
if (market == 'MTgox'):
time = int(data['time'])
trade = MtgoxTrade()
trade.tid = data['tid']
#return
elif (market == '796futures'):
time = int(data['time'])
trade = Futures796Trade()
elif (market == '796stockpd'):
time = int(data['time'])
trade = Stockpd796Trade()
if (int(time)>int(last_update)):
trade.time = time
trade.price = float(data['price'])
trade.amount = float(data['amount'])
trade.type = data['type']
trade.save()
def main(start, end):
start = get_unixtime(start, input_dateformat)
end = get_unixtime(end, input_dateformat)
if end < start:
print "End timestamp must be later than start timestamp. Exiting"
sys.exit()
print "Will get trades from ", start, "to", end
""" read the output file and adjust the start date, if it exists
"""
# try:
# with open(outfile_name, "r") as in_file:
# goxdata = in_file.readlines()
# saved_start=get_unixtime(goxdata[0].split(",")[0], input_dateformat)
# saved_end=get_unixtime(goxdata[len(goxdata)-1].split(",")[0], input_dateformat)
#
# print "File found, with start date:", saved_start, "and end date", saved_end
# if start < saved_end:
# print "Adjusted start time from ", start, "to ", saved_end
# start = saved_end
# except IOError:
# print "Output file not found. Will create a new one."
""" get data from MtGox in chunks
"""
try:
currstart = start
endreached = False
while endreached == False:
# populate the trades dictionary with the next batch of data
data = fetch_data(currstart)
print "Fetching data", currstart
if (data == '[]'):
print 'Empty data'
break
trades = [mtgox_trade(a) for a in json.loads(data)]
currstart = trades[-1].timestamp
if trades[-1].timestamp > end:
endreached = True
# place trades into the out_file before getting the next batch from MtGox
# so that if the program gets interrupt you have saved the trades obtained so far
with open(outfile_name, "a") as out_file:
for item in trades:
# when you request data from a timestamp gox truncates your start time to seconds and then
# send you everything including the initial second. So you must filter here trades
# of the start_time second that are already in the database.
if item.timestamp > start and item.timestamp < end:
#out_file.write(item.trade_to_string()+"\n")
print item
from api.models import MtgoxTrade
mtgox = MtgoxTrade()
mtgox.amount = item.amount
mtgox.time = item.time
mtgox.price = item.price
mtgox.type = item.trade_type
mtgox.tid = item.timestamp
#print mtgox.tid
mtgox.save()
except urllib2.HTTPError, e:
print "Error:", str(e.code), str(e.reason)
return
def main(start,end):
start=get_unixtime(start, input_dateformat)
end=get_unixtime(end, input_dateformat)
if end < start:
print "End timestamp must be later than start timestamp. Exiting"
sys.exit()
print "Will get trades from ", start, "to", end
""" read the output file and adjust the start date, if it exists
"""
# try:
# with open(outfile_name, "r") as in_file:
# goxdata = in_file.readlines()
# saved_start=get_unixtime(goxdata[0].split(",")[0], input_dateformat)
# saved_end=get_unixtime(goxdata[len(goxdata)-1].split(",")[0], input_dateformat)
#
# print "File found, with start date:", saved_start, "and end date", saved_end
# if start < saved_end:
# print "Adjusted start time from ", start, "to ", saved_end
# start = saved_end
# except IOError:
# print "Output file not found. Will create a new one."
""" get data from MtGox in chunks
"""
try:
currstart = start
endreached = False
while endreached == False:
# populate the trades dictionary with the next batch of data
data = fetch_data(currstart)
print "Fetching data", currstart
if (data == '[]'):
print 'Empty data'
break
trades = [mtgox_trade(a) for a in json.loads(data)]
currstart = trades[-1].timestamp
if trades[-1].timestamp > end:
endreached = True
# place trades into the out_file before getting the next batch from MtGox
# so that if the program gets interrupt you have saved the trades obtained so far
with open(outfile_name, "a") as out_file:
for item in trades:
# when you request data from a timestamp gox truncates your start time to seconds and then
# send you everything including the initial second. So you must filter here trades
# of the start_time second that are already in the database.
if item.timestamp > start and item.timestamp < end:
#out_file.write(item.trade_to_string()+"\n")
print item
from api.models import MtgoxTrade
mtgox = MtgoxTrade()
mtgox.amount = item.amount
mtgox.time = item.time
mtgox.price = item.price
mtgox.type = item.trade_type
mtgox.tid = item.timestamp
#print mtgox.tid
mtgox.save()
except urllib2.HTTPError, e:
print "Error:", str(e.code), str(e.reason)
return