def get_price_volume_from_fulls(exchanges):
n_exc = len(exchanges)
print("Number of exchanges: ", n_exc)
raw_folder = "data/long_raw_data/"
unix_stamps, excel_stamps = supp.make_time_stamps()
n_cols = len(excel_stamps)
prices = np.zeros([n_exc, n_cols])
volumes = np.zeros([n_exc, n_cols])
for i in range(0, n_exc):
print("Working on exchange %i/%i" % ((i + 1), n_exc))
single_price = []
single_volume = []
time_list = []
file_name = raw_folder + exchanges[i] + ".csv"
time_list, single_price, single_volume = price_volume_from_raw(
file_name, time_list, single_price, single_volume, semi=0)
# Gjør om nan til 0 og så fyller inn tomme priser
single_price = remove_nan(single_price)
single_price = supp.fill_blanks(single_price)
single_volume = remove_nan(single_volume)
# Må nå finne hvilken rad vi skal lime inn på
n = len(single_price)
r = 0
# Check whether the first entry in time_listH is before the start of the unix_stamps
start_j = 0
while time_list[start_j] < unix_stamps[0]:
start_j += 1
for j in range(start_j, n):
while unix_stamps[r] != time_list[j]:
r = r + 1
prices[i, r] = single_price[j]
volumes[i, r] = single_volume[j]
for i in range(n_exc):
single = supp.fill_blanks(prices[i, :])
for j in range(0, len(single)):
prices[i, j] = single[j]
return excel_stamps, unix_stamps, prices, volumes
def get_hilo_from_fulls(exchanges):
n_exc = len(exchanges)
print("Number of exchanges: ", n_exc)
raw_folder = "data/long_raw_data/"
unix_stamps, excel_stamps = make_time_stamps()
n_cols = len(excel_stamps)
high = np.zeros([n_exc, n_cols])
low = np.zeros([n_exc, n_cols])
for i in range(0, n_exc):
print("Working on exchange %i/%i" % ((i + 1), n_exc))
single_high = []
single_low = []
time_list = []
file_name = raw_folder + exchanges[i] + ".csv"
time_list, single_high, single_low = hilo_from_raw(
file_name, time_list, single_high, single_low)
# Gjør om nan til 0 og så fyller inn tomme priser
single_high = remove_nan(single_high)
single_low = remove_nan(single_low)
single_high = supp.fill_blanks(single_high)
single_low = supp.fill_blanks(single_low)
# Må nå finne hvilken rad vi skal lime inn på
n = len(single_high)
r = 0
# Check whether the first entry in time_listH is before the start of the unix_stamps
start_j = 0
while time_list[start_j] < unix_stamps[0]:
start_j += 1
for j in range(start_j, n):
while unix_stamps[r] != time_list[j]:
r = r + 1
high[i, r] = single_high[j]
low[i, r] = single_low[j]
return excel_stamps, unix_stamps, high, low
def import_from_csv_w_ticks(
exc_name, start_stamp, end_stamp
): #note that start_stamp need to be before the start of the dataseries from the exchange
full_list_excel_time = make_excel_stamp_list(startstamp=start_stamp,
endstamp=end_stamp)
if exc_name == "korbitkrw":
quick_exc = 4
elif exc_name == "krakeneur":
quick_exc = 5
time_listM, priceM, volumeM = quick_import(quick_exc)
price = np.zeros(len(full_list_excel_time))
volume = np.zeros(len(full_list_excel_time))
j = 0 #j follows the imported dataset timelist. The intention is to save the work of searching through the whole series every time.
t = 0 #t follows the new generated timelist
end = 0
while (j != len(time_listM)):
if t == len(full_list_excel_time):
break
else:
while (
unix_to_timestamp(time_listM[j]) != full_list_excel_time[t]
): #Increase t in the generated timeseries until it is equal to the j in the imported dataset
if t == len(full_list_excel_time) - 1:
end = 1 # Want to end the function after testing all j for the last t.
break
else:
t = t + 1
while unix_to_timestamp(time_listM[j]) == full_list_excel_time[
t]: #Increase j to capture multiple trades on the same minute
price[t] = price[t] + priceM[j] * volumeM[j]
volume[t] = volume[t] + volumeM[j]
j = j + 1
if j == len(time_listM):
break
if volume[t] != 0:
price[t] = price[t] / volume[t]
if end == 1:
break
price = supp.fill_blanks(price)
write_to_csv(exc_name, full_list_excel_time, price, volume)
def add_new_to_old_csv(exc=0):
location = "data/bitcoincharts/"
year = '2017'
start_date = "01.01.2012 00:00"
start_unix = 1325376000
end_unix = 0
end_date = "31.12.2017 23:59"
if exc == 1:
exc_name = "coincheckjpy"
month_list = ["06", "07", "08", "09", "10", "11", "12"]
start_date = "01.11.2014 00:00"
start_unix = 1414800000
elif exc == 2:
exc_name = "btcncny"
month_list = ["06", "07", "08", "09"]
end_date = "29.09.2017 23:59"
#end_unix = 1506815880
elif exc == 3:
exc_name = "coinbaseusd"
month_list = []
start_date = "02.12.2014 00:00"
start_unix = 1417478400
end_unix = 1514764740
else:
exc_name = "bitstampusd"
month_list = ["06", "07", "08", "09", "10", "11", "12"]
time_old, price_old, volume_old = quick_import(exc)
start_index = time_old.index(start_unix)
if end_unix == 0:
time_old = time_old[start_index:]
price_old = price_old[start_index:]
volume_old = volume_old[start_index:]
else:
end_index = time_old.index(end_unix) + 1
time_old = time_old[start_index:end_index]
price_old = price_old[start_index:end_index]
volume_old = volume_old[start_index:end_index]
print()
print("Time list:", len(time_old))
print(time_old[0:3], time_old[-3:])
for i in range(1, len(time_old)):
if time_old[i] - time_old[i - 1] != 60:
print("THERE IS AN ERROR AT", time_old[i])
print("Prices:", len(price_old))
print("Volumes:", len(volume_old))
print()
time_new = []
price_new = []
volume_new = []
if len(month_list) != 0:
for month in month_list: # iterate through the new files
filename_new = location + exc_name + "/" + year + month + ".csv"
time_new, price_new, volume_new = price_volume_from_raw(
filename_new, time_new, price_new, volume_new, semi=1, unix=0)
print("Len of new:", len(volume_new))
print("Len of old:", len(price_old))
time_list = np.append(time_old, time_new)
prices = np.append(price_old, price_new)
volumes = np.append(volume_old, volume_new)
prices = remove_nan(prices)
prices = supp.fill_blanks(prices)
volumes = remove_nan(volumes)
else:
prices = remove_nan(price_old)
prices = supp.fill_blanks(prices)
volumes = remove_nan(volume_old)
time_list = time_old
print("Len of combined:", len(volumes))
time_list = make_excel_stamp_list(startstamp=start_date, endstamp=end_date)
print()
print("Time list:", len(time_list))
print(time_list[0:3], time_list[-3:])
print("Prices:", len(prices))
print("Volumes:", len(volumes))
print()
write_filename = "data/export_csv/" + exc_name + "_edit.csv"
with open(write_filename, 'w', newline='') as csvfile:
writ = csv.writer(csvfile,
delimiter=';',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
print("\033[0;32;0m Writing to file '%s'...\033[0;0;0m" %
write_filename)
header1 = [" "]
header2 = [" "]
header3 = ["Time"]
currency = exc_name[-3:]
header1.append(exc)
header1.append("")
header2.append("Closing price")
header2.append("Volume")
header3.append(currency.upper())
header3.append("BTC")
writ.writerow(header1)
writ.writerow(header2)
writ.writerow(header3)
for i in range(len(prices)):
rowdata = [time_list[i]]
rowdata.append(prices[i])
rowdata.append(volumes[i])
writ.writerow(rowdata)
if make_real_spread_csv:
for exc_name in ["bitstamp", "korbit"]:
file_name= "data/long_raw_data/" + exc_name + "_new_minutes.csv"
time_listM, pricesM, volumesM = dis.price_volume_from_raw(file_name, [], [], [], semi=1, unix=0, price_col=4)
y, mo, d, h, mi = supp.fix_time_list(time_listM)
unixM = []
unixM = supp.timestamp_to_unix(time_listM)
for i in range(1, len(time_listM)):
if unixM[i] - unixM[i-1] != 60:
print(i, time_listM[i])
pricesM = supp.fill_blanks(pricesM)
print("finihed importing")
print(len(time_listM), time_listM[-10:])
print(len(pricesM),(pricesM[-10:]))
print(len(volumesM), (volumesM[-10:]))
spread_abs, spreadH, time_listH, count_value_error = rolls.rolls(pricesM, time_listM, calc_basis="h", kill_output=1)
write_filename = "data/export_csv/" + exc_name + "_spread_new.csv"
with open(write_filename, 'w', newline='') as csvfile:
writ = csv.writer(csvfile, delimiter=';', quotechar='|', quoting=csv.QUOTE_MINIMAL)
print("\033[0;32;0m Writing to file '%s'...\033[0;0;0m" % write_filename)
header3 = ["Time"]
header3.append("Spread")
writ.writerow(header3)
def fuse_files(exchanges):
n_exc = len(exchanges)
time_old = []
price_old1 = []
volume_old1 = []
price_old2 = []
volume_old2 = []
price_old3 = []
volume_old3 = []
filename_old = "data/export_csv/minute_data_full_day.csv"
with open(filename_old, newline='') as csvfile:
reader = csv.reader(csvfile, delimiter=';', quotechar='|')
print("\033[0;32;0m Reading file '%s'...\033[0;0;0m" % filename_old)
next(reader)
next(reader)
next(reader)
for row in reader:
time_old.append(str(row[0]))
price_old1.append(float(row[1]))
volume_old1.append(float(row[2]))
price_old2.append(float(row[3]))
volume_old2.append(float(row[4]))
price_old3.append(float(row[5]))
volume_old3.append(float(row[6]))
print("Her 1")
print("Time old", len(time_old))
print("Price old1", len(price_old1))
print("Price old2", len(price_old2))
print("Price old3", len(price_old3))
location = "data/bitcoincharts/"
year = "2017"
time_new = []
price_new = []
volume_new = []
exc = "bitstampusd"
for month in ["06", "07", "08", "09", "10", "11",
"12"]: # iterate through the new files
filename_new = location + exc + "/" + year + month + ".csv"
time_new, price_new, volume_new = price_volume_from_raw(filename_new,
time_new,
price_new,
volume_new,
semi=1,
unix=0)
price_new1 = remove_nan(price_new)
price_new1 = supp.fill_blanks(price_new1)
volume_new1 = remove_nan(volume_new)
time_new_saved = time_new # Denne brukes videre
time_new = []
price_new = []
volume_new = []
exc = "coincheckjpy"
for month in ["06", "07", "08", "09", "10", "11",
"12"]: # iterate through the new files
filename_new = location + exc + "/" + year + month + ".csv"
time_new, price_new, volume_new = price_volume_from_raw(filename_new,
time_new,
price_new,
volume_new,
semi=1,
unix=0)
price_new2 = remove_nan(price_new)
price_new2 = supp.fill_blanks(price_new2)
volume_new2 = remove_nan(volume_new)
time_new = []
price_new = []
volume_new = []
exc = "btcncny"
for month in ["06", "07", "08", "09"]: # iterate through the new files
filename_new = location + exc + "/" + year + month + ".csv"
time_new, price_new, volume_new = price_volume_from_raw(filename_new,
time_new,
price_new,
volume_new,
semi=1,
unix=0)
price_new3 = remove_nan(price_new)
price_new3 = supp.fill_blanks(price_new3)
volume_new3 = remove_nan(volume_new)
time_new = time_new_saved # Saved from Bitstamp
print("Length of old series: ")
print(" time old:", len(time_old))
print(" price old1:", len(price_old1))
print(" price old2:", len(price_old2))
print("Length of new series: ")
print(" time new: ", len(time_new))
print(" price new1:", len(price_new1))
print(" price new2:", len(price_new2))
print(" price new3:", len(price_new3))
time_list = time_old
for i in range(0, len(time_new)):
time_list.append(time_new[i])
print("Her 3")
print("Time list:", len(time_list))
prices = np.zeros([n_exc, len(time_list)])
volumes = np.zeros([n_exc, len(time_list)])
for i in range(0, len(price_old1)):
prices[0, i] = price_old1[i]
volumes[0, i] = volume_old1[i]
prices[1, i] = price_old2[i]
volumes[1, i] = volume_old2[i]
try:
prices[2, i] = price_old3[i]
volumes[2, i] = volume_old3[i]
except IndexError:
pass
k = 0 # Counts the entries in the new lists
for i in range(len(price_old1), len(time_list)):
try:
prices[0, i] = price_new1[k]
except IndexError:
print("Error when i =", i, " k =", k)
try:
print(price_new1[k])
except IndexError:
print("Price new!!!")
volumes[0, i] = volume_new1[k]
prices[1, i] = price_new2[k]
volumes[1, i] = volume_new2[k]
try:
prices[2, i] = price_new3[k]
volumes[2, i] = volume_new3[k]
except IndexError:
pass
k += 1
with open(filename_old, 'w', newline='') as csvfile:
writ = csv.writer(csvfile,
delimiter=';',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
print("\033[0;32;0m Writing to file '%s'...\033[0;0;0m" % filename_old)
header1 = [" "]
header2 = [" "]
header3 = ["Time"]
for exc in exchanges:
currency = exc[len(exc) - 3:len(exc)]
header1.append(exc)
header1.append("")
header2.append("Closing price")
header2.append("Volume")
header3.append(currency.upper())
header3.append("BTC")
writ.writerow(header1)
writ.writerow(header2)
writ.writerow(header3)
for i in range(0, len(time_list)):
rowdata = [time_list[i]]
for j in range(0, n_exc):
rowdata.append(prices[j, i])
rowdata.append(volumes[j, i])
writ.writerow(rowdata)