def cw_hist_cleaning(vol_fixed_df,
start_date,
crypto_list=CRYPTO_ASSET,
exc_list=EXCHANGES):
tot_date_arr = date_gen(start_date)
cleaned_df = pd.DataFrame(columns=CLEAN_DATA_HEAD)
for crypto in crypto_list:
pair_arr = crypto_fiat_pair_gen(crypto)
for exchange in exc_list:
ex_matrix = vol_fixed_df.loc[vol_fixed_df["Exchange"] == exchange]
for cp in pair_arr:
crypto = cp[:3]
cp_matrix = ex_matrix.loc[ex_matrix["Pair"] == cp]
cp_matrix = cp_matrix.drop(columns=["Exchange", "Pair"])
# checking if the matrix is not empty
if cp_matrix.shape[0] > 1:
# check if the historical series start at the same date as
# the start date if not fill the dataframe with zero values
cp_matrix = homogenize_series(cp_matrix, tot_date_arr)
# check if the series stopped at certain point in
# the past, if yes fill with zero
cp_matrix = homogenize_dead_series(cp_matrix, tot_date_arr)
# checking if the matrix has missing data and if ever fixing it
if cp_matrix.shape[0] != tot_date_arr.size:
print("fixing")
cp_matrix = CW_series_fix_missing(
cp_matrix,
exchange,
cp,
tot_date_arr,
DB_NAME,
MONGO_DICT.get("coll_vol_chk"),
)
# turn Time columns into string
[str(date) for date in cp_matrix["Time"]]
# add exchange and currency_pair column
cp_matrix["Exchange"] = exchange
cp_matrix["Pair"] = cp
reordered = df_reorder(cp_matrix, column_set="conversion")
cleaned_df = cleaned_df.append(reordered)
return cleaned_df
def index_board_logic_op(crypto_asset, logic_one_arr, daily_ewma,
curr_reb_start, next_reb_start, curr_board_eve):
# turn the first logic row into a dataframe and add the 'Time' column
# the first logic row will be used for the next quarter weights computation
first_logic_row = pd.DataFrame(logic_one_arr, columns=crypto_asset)
# computing the new second logic row that is used to compute the
# weights relative to the next rebalance period
ewma_fraction = daily_ewma_fraction(daily_ewma, first_logic_row,
curr_reb_start, curr_board_eve)
print("ewma_fraction")
print(ewma_fraction)
daily_ewma_double_row = index_norm_logic_op(crypto_asset, daily_ewma)
print(daily_ewma_double_row)
second_logic_row = (ewma_fraction >= 0.02) * 1
double_checked_ewma = daily_double_log_check(first_logic_row,
second_logic_row, daily_ewma,
curr_reb_start,
curr_board_eve)
# adding the Time columns to the double checked ewma
human_curr_start = timestamp_to_human([curr_reb_start],
date_format="%m-%d-%y")
human_curr_board = timestamp_to_human([curr_board_eve],
date_format="%m-%d-%y")
period_date_list = date_gen(human_curr_start[0],
human_curr_board[0],
EoD="N")
double_checked_ewma["Time"] = period_date_list
print(double_checked_ewma)
# giving "Time" and "Date" columns to the df containing the logic rows
human_next_start = timestamp_to_human([next_reb_start],
date_format="%m-%d-%y")
first_logic_row["Time"] = next_reb_start
first_logic_row["Date"] = human_next_start
second_logic_row["Time"] = next_reb_start
second_logic_row["Date"] = human_next_start
# computing the new weights that will be used starting from the
# next rebalance date
weights_for_board = quarter_weights(double_checked_ewma,
[int(curr_board_eve)], crypto_asset)
weights_for_board["Time"] = next_reb_start
weights_for_board["Date"] = human_next_start
return first_logic_row, second_logic_row, weights_for_board, daily_ewma_double_row
def test_date_gen():
# testing the function with start_date and end_date
# date mm-dd-yyy
start_date = "01-01-2020"
end_date = "01-06-2020"
ts = np.array([1577836800, 1577923200, 1578009600, 1578096000, 1578182400])
date = date_gen(start_date, end_date)
assert np.array_equal(date, ts)
# testing the function with only start_date
day_in_sec = 86400
today = datetime.now().strftime("%m-%d-%Y")
today = datetime.strptime(today, "%m-%d-%Y")
# without EOD defatult arg the fun create an array of date
# utill yesterday
yesterday_ts = int(
today.replace(tzinfo=timezone.utc).timestamp()) - day_in_sec
date2 = date_gen(start_date)
assert yesterday_ts == date2[-1]
# without a default EOD arg the function generate today date as well
date = date_gen(start_date, EoD='N')
today_ts = int(today.replace(tzinfo=timezone.utc).timestamp())
assert today_ts == date[-1]
# with timeST != 'Y' the function generates an array or list
# of str date in 'mm-dd-yyyy' format
# with clss != 'array', the function generates a list of dates
date = date_gen(start_date, timeST='N', clss='list')
assert type(date) is list
assert date[0] == start_date
def hist_time_array_set(start_date=START_DATE):
# define the variable containing all the date from start_date to yesterday.
# the date are displayed as timestamp and each day refers to 12:00 am UTC
reference_date_arr = date_gen(start_date)
# define all the useful arrays containing the rebalance
# start date, stop date, board meeting date
reb_start_date = start_q(start_date)
reb_stop_date = stop_q(reb_start_date)
board_date = board_meeting_day()
board_date_eve = day_before_board()
next_reb_date = next_start()
return (reference_date_arr, reb_start_date, reb_stop_date, board_date,
board_date_eve, next_reb_date)
def hist_data_feed_op():
# define the array containing the date where the index uses CW feed data
CW_date_arr = date_gen(START_DATE, EXC_START_DATE)
CW_date_str = [str(date) for date in CW_date_arr]
# drop the pre-existing collection (if there is one)
mongo_coll_drop("index_feed")
# downloading the EXC series from MongoDB
EXC_series = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_final"))
EXC_series = EXC_series[
["Time", "Close Price", "Crypto Volume", "Pair Volume", "Exchange", "Pair"]]
# downloading the CW series from MongoDB and selecting only the date
# from 2016-01-01 to 2020-04-17
CW_series = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_final"))
CW_series["Time"] = [str(x) for x in CW_series["Time"]]
print("CW")
print(CW_series)
CW_sub_series = CW_series.loc[CW_series.Time.isin(CW_date_str)]
print(CW_sub_series)
CW_sub_series = CW_sub_series[
["Time", "Close Price", "Crypto Volume", "Pair Volume", "Exchange", "Pair"]]
CW_sub_series["Time"] = [int(x) for x in CW_sub_series["Time"]]
CW_sub_series.reset_index(drop=True, inplace=True)
print(CW_sub_series)
# creting an unique dataframe containing the two different data source
data_feed = CW_sub_series.append(EXC_series, sort=True)
data_feed.reset_index(drop=True, inplace=True)
data_feed = data_feed[
["Time", "Close Price", "Crypto Volume", "Pair Volume", "Exchange", "Pair"]]
print(data_feed)
data_feed = homogeneize_feed(data_feed)
print("post hom")
print(data_feed)
# put the converted data on MongoDB
mongo_upload(data_feed, "collection_data_feed")
return None
def cw_exc_merging(start_date=START_DATE, exc_start=EXC_START_DATE,
db=DB_NAME, coll_cw="coll_cw_final",
coll_exc="coll_exc_final"):
cw_date_arr = date_gen(start_date, exc_start, EoD="N")
exc_series = query_mongo(db, MONGO_DICT.get(coll_exc))
exc_part = df_reorder(exc_series, column_set="conversion")
# downloading the CW series from MongoDB and selecting only the date
# from 2016-01-01 to 2020-04-17
cw_series = query_mongo(db, MONGO_DICT.get(coll_cw))
cw_part = cw_series.loc[cw_series.Time.isin(cw_date_arr)]
cw_part = df_reorder(cw_part, column_set="conversion")
# creting an unique dataframe containing the two different data source
merged_series = cw_part.append(exc_part, sort=True)
merged_series["Time"] = [int(d) for d in merged_series["Time"]]
return merged_series
def cw_hist_operation(start_date=START_DATE):
date_tot = date_gen(start_date)
last_day_TS = date_tot[len(date_tot) - 1]
mongo_indexing()
# deleting previous MongoDB collections
mongo_coll_drop("cw_hist_clean")
mongo_coll_drop("cw_hist_conv")
# fix and upload the series for the "pair volume" info
tot_raw_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_raw"))
cw_vol_fix_data = cw_hist_pair_vol_fix(tot_raw_data)
mongo_upload(cw_vol_fix_data, "collection_cw_vol_check")
# clean and upload all the series
cleaned_df = cw_hist_cleaning(cw_vol_fix_data, start_date)
mongo_upload(cleaned_df, "collection_cw_clean")
# compute and upload USDC and USDT rates series
usdt_rates, usdc_rates = stable_rates_op("coll_cw_clean", None)
mongo_upload(usdt_rates, "collection_stable_rate")
mongo_upload(usdc_rates, "collection_stable_rate")
# convert and upload all the data into USD
converted_df = cw_hist_conv_op(cleaned_df)
mongo_upload(converted_df, "collection_cw_converted")
# logic matrix of crypto-fiat keys
key_df = key_log_mat(DB_NAME, "coll_cw_conv", last_day_TS, EXCHANGES,
CRYPTO_ASSET, PAIR_ARRAY)
mongo_upload(key_df, "collection_CW_key")
mongo_upload(key_df, "collection_EXC_key")
# fill zero-volume data and upload on MongoDB
final_df = cw_hist_zero_vol_fill_op(converted_df)
mongo_upload(final_df, "collection_cw_final_data")
return None
def ecb_hist_download(start_date):
# drop the pre-existing collection related to ecb_rawdata
mongo_coll_drop("ecb_hist_d")
# set today as end_date
end_date = datetime.now().strftime("%Y-%m-%d")
# create an array of date containing the list of date to download
date_list = date_gen(start_date,
end_date,
timeST="N",
clss="list",
EoD="N")
date_list_str = [
datetime.strptime(day, "%m-%d-%Y").strftime("%Y-%m-%d")
for day in date_list
]
ecb_hist_series = pd.DataFrame()
for date in date_list_str:
# retrieving data from ECB website
single_date_ex_matrix = ECB_rates_extractor(ECB_FIAT, date)
# put a sllep time in order to do not overuse API connection
time.sleep(0.02)
# put all the downloaded data into a DafaFrame
if ecb_hist_series.size == 0:
ecb_hist_series = single_date_ex_matrix
else:
ecb_hist_series = ecb_hist_series.append(single_date_ex_matrix,
sort=True)
return ecb_hist_series
def homogeneize_feed(initial_df):
df = initial_df.copy()
list_of_exchanges = list(np.array(df["Exchange"].unique()))
list_of_pair = list(np.array(df["Pair"].unique()))
# today_str = datetime.now().strftime("%Y-%m-%d")
# today = datetime.strptime(today_str, "%Y-%m-%d")
# today_TS = int(today.replace(tzinfo=timezone.utc).timestamp())
# day_before_TS = today_TS - DAY_IN_SEC
list_of_missing = date_gen(EXC_START_DATE)
ref_shape = df.loc[(df.Exchange == "coinbase-pro")
& (df.Pair) == "btcusd"].shape[0]
for ex in list_of_exchanges:
for p in list_of_pair:
sub_df = df.loc[(df.Exchange == ex)]
sub_df = sub_df.loc[sub_df.Pair == p]
print(ex)
print(p)
print(sub_df.shape)
if sub_df.shape[0] == ref_shape:
pass
elif sub_df.shape[0] == 1569:
print(ex)
print(p)
zero_mat = np.zeros((len(list_of_missing), 6))
zero_sub_df = pd.DataFrame(zero_mat, columns=df.columns)
zero_sub_df["Time"] = list_of_missing
zero_sub_df["Exchange"] = ex
zero_sub_df["Pair"] = p
print(zero_sub_df)
df = df.append(zero_sub_df)
new_df = df.copy()
return new_df
def cw_new_key_mngm(logic_key_df, daily_mat, time_to_check, date_tot_str):
# selecting only the keys with 0 value
key_absent = logic_key_df.loc[logic_key_df.logic_value == 0]
key_absent.drop(columns=["logic_value"])
# merging the dataframe in order to find the potenatial new keys
merg_absent = pd.merge(key_absent, daily_mat, on="key", how="left")
merg_absent.fillna("NaN", inplace=True)
new_key_df = merg_absent.loc[merg_absent["Close Price"] != "NaN"]
if new_key_df.empty is False:
print("Message: New exchange-pair couple(s) found.")
new_key_list = new_key_df["key"]
date_tot_int = date_gen(START_DATE)
# converting the timestamp format date into string
date_tot_str = [str(single_date) for single_date in date_tot_int]
for key in new_key_list:
# updating the logic matrix of exchange-pair keys
logic_row_update = log_key_update(key, "collection_CW_key")
mongo_upload(logic_row_update, "collection_CW_key")
key_hist_df = new_series_composer(key,
new_key_df,
date_tot_str,
time_to_check,
kind_of_series="CW")
# upload the dataframe on MongoDB collection "CW_cleandata"
mongo_upload(key_hist_df, "collection_cw_clean")
else:
pass
return None
def exc_new_key_mng(logic_key_df, daily_mat_00, day_to_clean_TS):
# define a subset of keys that has not been present in the past
key_absent = logic_key_df.loc[logic_key_df.logic_value == 0]
key_absent.drop(columns=["logic_value"])
# merging the two dataframe (left join) in order to find potential
# new keys in the data of the day
merg_absent = pd.merge(key_absent, daily_mat_00, on="key", how="left")
merg_absent.fillna("NaN", inplace=True)
new_key_df = merg_absent.loc[merg_absent["Close Price"] != "NaN"]
if new_key_df.empty is False:
print("Message: New exchange-pair couple(s) found.")
new_key_list = new_key_df["key"]
print(new_key_list)
date_tot_int = date_gen(START_DATE)
# converting the timestamp format date into string
date_tot_str = [int(single_date) for single_date in date_tot_int]
for key in new_key_list:
# updating the logic matrix of exchange-pair keys
logic_row_update = log_key_update(key, "collection_EXC_key")
mongo_upload(logic_row_update, "collection_EXC_key")
key_hist_df = new_series_composer(
key, new_key_df, date_tot_str, day_to_clean_TS)
else:
key_hist_df = []
return key_hist_df
# define the start date as MM-DD-YYYY
start_date = "01-01-2016"
# define today in various format
today_str = datetime.now().strftime("%Y-%m-%d")
today = datetime.strptime(today_str, "%Y-%m-%d")
today_TS = int(today.replace(tzinfo=timezone.utc).timestamp())
y_TS = today_TS - 86400
two_before_TS = y_TS - 86400
today_human = data_setup.timestamp_to_human([today_TS])
yesterday_human = data_setup.timestamp_to_human([y_TS])
two_before_human = data_setup.timestamp_to_human([two_before_TS])
# define the variable containing all the date from start_date to today.
# the date are displayed as timestamp and each day refers to 12:00 am UTC
reference_date_vector = data_setup.date_gen(start_date)
# define all the useful arrays containing the rebalance start
# date, stop date, board meeting date
rebalance_start_date = calc.start_q("01-01-2016")
rebalance_stop_date = calc.stop_q(rebalance_start_date)
board_date = calc.board_meeting_day()
board_date_eve = calc.day_before_board()
next_rebalance_date = calc.next_start()
# call the function that creates a object containing
# the couple of quarterly start-stop date
quarterly_date = calc.quarterly_period()
# ############# MAIN PART #########################
# third party import
import numpy as np
# local import
from cryptoindex.data_download import CW_raw_to_mongo
from cryptoindex.data_setup import (date_gen, Diff, timestamp_to_human)
from cryptoindex.mongo_setup import (mongo_coll, mongo_indexing, query_mongo)
from cryptoindex.config import (START_DATE, MONGO_DICT, PAIR_ARRAY,
CRYPTO_ASSET, EXCHANGES, DB_NAME)
# #########################################################
# set today
today = datetime.now().strftime("%Y-%m-%d")
# defining the array containing all the date from start_period until today
date_tot = date_gen(START_DATE)
# converting the timestamp format date into string
date_tot = [str(single_date) for single_date in date_tot]
# ########## MongoDB setup ################################
# create the indexing for MongoDB and define the variable containing the
# MongoDB collections where to upload data
mongo_indexing()
collection_dict_upload = mongo_coll()
# ####################### check for the cw_rawdata ####################
# selecting the last five days and put them into an array
last_five_days = date_tot[(len(date_tot) - 5):len(date_tot)]
print(last_five_days)
key_curr_vector = ["USD", "GBP", "CAD", "JPY"]
# setup mongo connection ###################################
# connecting to mongo in local
connection = MongoClient("localhost", 27017)
# creating the database called index
db = connection.index
# naming the exixting ecb_raw collection as a variable
collection_ECB_raw = db.ecb_raw
# ecb_raw collection check ###########################################
# defining the array containing all the date from start_period until today
date_tot = data_setup.date_gen(start_period)
# converting the timestamp format date into string
date_tot = [str(single_date) for single_date in date_tot]
# searching only the last five days
last_five_days = date_tot[(len(date_tot) - 5): len(date_tot)]
# defining the MongoDB path where to look for the rates
database = "index"
collection = "ecb_raw"
query = {"CURRENCY": "USD"}
# retrieving data from MongoDB 'index' and 'ecb_raw' collection
matrix = mongo.query_mongo(database, collection, query)
# local import
import cryptoindex.data_setup as data_setup
import cryptoindex.data_download as data_download
from cryptoindex.mongo_setup import (mongo_coll, mongo_coll_drop,
mongo_indexing)
from cryptoindex.config import (START_DATE, PAIR_ARRAY, CRYPTO_ASSET,
EXCHANGES)
# ################### initial settings #########################
# set end_date as today, otherwise comment and choose an end_date
end_date = datetime.now().strftime("%m-%d-%Y")
# define the variable containing all the date from start_date to today.
# the date are displayed as timestamp and each day refers to 12:00 am UTC
reference_date_vector = data_setup.date_gen(START_DATE)
# ################# setup MongoDB connection #####################
mongo_coll_drop("cw_hist_d")
mongo_indexing()
collection_dict_upload = mongo_coll()
# ################# downloading and storing part ################
for Crypto in CRYPTO_ASSET:
print(Crypto)
ccy_pair_array = []
def cw_daily_operation(day=None):
'''
@param day has to be either None or "%Y-%m-%d" string format
'''
# create the indexing for MongoDB and define the variable containing the
# MongoDB collections where to upload data
mongo_indexing()
date_tot = date_gen(START_DATE)
# converting the timestamp format date into string
date_tot_str = [str(single_date) for single_date in date_tot]
day_before_TS, _ = days_variable(day)
if day is None:
if daily_check_mongo("coll_cw_raw", {
"Exchange": "coinbase-pro",
"Pair": "btcusd"
}) is False:
try:
cw_rawdata_daily = cw_daily_download(day_before_TS +
DAY_IN_SEC)
except Exception:
error("Exception occurred", exc_info=True)
info('Daily download from CryptoWatch failed')
mongo_upload(cw_rawdata_daily, "collection_cw_raw")
else:
print("The CW_rawdata collection on MongoDB is updated.")
if daily_check_mongo("coll_vol_chk", {
"Exchange": "coinbase-pro",
"Pair": "btcusd"
}) is False:
mat_vol_fix = daily_pair_vol_fix(day_before_TS)
try:
mongo_upload(mat_vol_fix, "collection_cw_vol_check")
except AttributeError:
pass
else:
mat_vol_fix = []
print(
"Message: No need to fix pair volume. The collection on MongoDB is updated."
)
# new and dead crypto-fiat key management
daily_complete_df = cw_daily_key_mngm(mat_vol_fix, day_before_TS,
date_tot_str)
# missing data fixing
if daily_check_mongo("coll_cw_clean", {
"Exchange": "coinbase-pro",
"Pair": "btcusd"
}) is False:
daily_fixed_df = daily_fix_miss_op(daily_complete_df, day,
"coll_cw_clean")
mongo_upload(daily_fixed_df, "collection_cw_clean")
else:
print("Message: The collection cw_clean on MongoDB is updated.")
if daily_check_mongo("coll_stable_rate",
{"Currency": "USDT/USD"}) is False:
usdt_rates, usdc_rates = stable_rates_op("coll_cw_clean",
str(day_before_TS))
mongo_upload(usdt_rates, "collection_stable_rate")
mongo_upload(usdc_rates, "collection_stable_rate")
else:
print("The stable_rates_collection on MongoDB is already updated.")
if daily_check_mongo("coll_cw_conv", {
"Exchange": "coinbase-pro",
"Pair": "btcusd"
}) is False:
converted_data = cw_daily_conv_op(day_before_TS)
mongo_upload(converted_data, "collection_cw_converted")
else:
print(
"Message: The cw_converted_data collection on MongoDB is already updated."
)
if daily_check_mongo("coll_cw_final", {
"Exchange": "coinbase-pro",
"Pair": "btcusd"
}) is False:
mongo_upload(converted_data, "collection_cw_final_data")
else:
print(
"The CW_final_data collection on MongoDB is already updated.")
else:
cw_rawdata_daily = cw_daily_download(day_before_TS)
mongo_upload(cw_rawdata_daily, "collection_cw_raw")
mat_vol_fix = daily_pair_vol_fix(day_before_TS)
try:
mongo_upload(mat_vol_fix, "collection_cw_vol_check")
except AttributeError:
pass
daily_complete_df = cw_daily_key_mngm(mat_vol_fix, day_before_TS,
date_tot_str)
daily_fixed_df = daily_fix_miss_op(daily_complete_df, day,
"coll_cw_clean")
mongo_upload(daily_fixed_df, "collection_cw_clean")
usdt_rates, usdc_rates = stable_rates_op("coll_cw_clean",
str(day_before_TS))
mongo_upload(usdt_rates, "collection_stable_rate")
mongo_upload(usdc_rates, "collection_stable_rate")
converted_data = cw_daily_conv_op(day_before_TS)
mongo_upload(converted_data, "collection_cw_converted")
mongo_upload(converted_data, "collection_cw_final_data")
return None
collection_dict_upload = mongo_coll()
# ################### DATE SETTINGS ####################
# define today and yesterady date as timestamp
today_str = datetime.now().strftime("%Y-%m-%d")
today = datetime.strptime(today_str, "%Y-%m-%d")
today_TS = int(today.replace(tzinfo=timezone.utc).timestamp())
y_TS = today_TS - DAY_IN_SEC
# # define end date as as MM-DD-YYYY
# end_date = datetime.now().strftime("%m-%d-%Y")
# define the variable containing all the date from start_date to yesterday.
# the date are displayed as timestamp and each day refers to 12:00 am UTC
reference_date_vector = date_gen(START_DATE)
# define all the useful arrays containing the rebalance
# start date, stop date, board meeting date
rebalance_start_date = start_q(START_DATE)
rebalance_stop_date = stop_q(rebalance_start_date)
board_date = board_meeting_day()
board_date_eve = day_before_board()
next_rebalance_date = next_start()
print(rebalance_start_date)
print(rebalance_stop_date)
print(next_rebalance_date)
print(board_date_eve)
# defining time variables
last_reb_start = str(int(rebalance_start_date[len(rebalance_start_date) - 1]))
next_reb_stop = str(int(rebalance_stop_date[len(rebalance_stop_date) - 1]))
def index_hist_uploader(crypto_asset_price, crypto_asset_vol, exc_vol_tot,
price_ret, weights_for_board, first_logic_matrix_df,
second_logic_matrix_df, ewma_df, double_checked_EWMA,
syntethic, syntethic_relative_matrix, divisor_array,
reshaped_divisor, index_values, index_1000_base):
# creating the array with human readable Date
ref_date_arr = date_gen()
human_date = timestamp_to_human(ref_date_arr)
# put the "Crypto_Asset_Prices" dataframe on MongoDB
crypto_asset_price["Date"] = human_date
mongo_upload(crypto_asset_price,
"collection_price",
reorder="Y",
column_set_val="complete")
# put the "Crypto_Asset_Volumes" dataframe on MongoDB
crypto_asset_vol["Date"] = human_date
mongo_upload(crypto_asset_vol,
"collection_volume",
reorder="Y",
column_set_val="complete")
# put the exchange volumes on MongoDB
mongo_upload(exc_vol_tot, "collection_all_exc_vol")
# put the "price_ret" dataframe on MongoDB
price_ret["Date"] = human_date
mongo_upload(price_ret,
"collection_price_ret",
reorder="Y",
column_set_val="complete")
# put the "weights" dataframe on MongoDB
weight_human_date = timestamp_to_human(weights_for_board["Time"])
weights_for_board["Date"] = weight_human_date
mongo_upload(weights_for_board,
"collection_weights",
reorder="Y",
column_set_val="complete")
print(weights_for_board)
# put the first logic matrix on MongoDB
first_date = timestamp_to_human(first_logic_matrix_df["Time"])
first_logic_matrix_df["Date"] = first_date
mongo_upload(first_logic_matrix_df,
"collection_logic_one",
reorder="Y",
column_set_val="complete")
# put the second logic matrix on MongoDB
second_date = timestamp_to_human(second_logic_matrix_df["Time"])
second_logic_matrix_df["Date"] = second_date
mongo_upload(second_logic_matrix_df,
"collection_logic_two",
reorder="Y",
column_set_val="complete")
# put the EWMA dataframe on MongoDB
ewma_df["Date"] = human_date
ewma_df["Time"] = ref_date_arr
mongo_upload(ewma_df,
"collection_EWMA",
reorder="Y",
column_set_val="complete")
# put the double checked EWMA on MongoDB
double_checked_EWMA["Date"] = human_date
mongo_upload(double_checked_EWMA,
"collection_EWMA_check",
reorder="Y",
column_set_val="complete")
# put the synth matrix on MongoDB
syntethic["Date"] = human_date
syntethic["Time"] = ref_date_arr
mongo_upload(syntethic,
"collection_synth",
reorder="Y",
column_set_val="complete")
# put the relative synth matrix on MongoDB
syntethic_relative_matrix["Date"] = human_date
syntethic_relative_matrix["Time"] = ref_date_arr
mongo_upload(syntethic_relative_matrix,
"collection_relative_synth",
reorder="Y",
column_set_val="complete")
# put the divisor array on MongoDB
divisor_date = timestamp_to_human(divisor_array["Time"])
divisor_array["Date"] = divisor_date
mongo_upload(divisor_array,
"collection_divisor",
reorder="Y",
column_set_val="divisor")
# put the reshaped divisor array on MongoDB
reshaped_divisor_date = timestamp_to_human(reshaped_divisor["Time"])
reshaped_divisor["Date"] = reshaped_divisor_date
mongo_upload(reshaped_divisor,
"collection_divisor_reshaped",
reorder="Y",
column_set_val="divisor")
# put the index level raw on MongoDB
index_values["Date"] = human_date
index_values["Time"] = ref_date_arr
mongo_upload(index_values,
"collection_index_level_raw",
reorder="Y",
column_set_val="index")
# put the index level 1000 on MongoDB
index_1000_base["Date"] = human_date
index_1000_base["Time"] = ref_date_arr
mongo_upload(index_1000_base,
"collection_index_level_1000",
reorder="Y",
column_set_val="index")
def index_hist_loop(data_matrix, crypto_asset, exc_list, pair_list):
ref_date_arr = date_gen()
# initialize the matrices that will contain the prices
# and volumes of all the single_cryptosset
crypto_asset_price = np.matrix([])
crypto_asset_vol = np.matrix([])
logic_matrix_one = np.matrix([])
# initialize the matrix that contain the volumes per Exchange
exc_head = [
"coinbase-pro",
"poloniex",
"bitstamp",
"gemini",
"bittrex",
"kraken",
"bitflyer",
]
exc_head.append("Time")
exc_head.append("Crypto")
exc_vol_tot = pd.DataFrame(columns=exc_head)
for single_crypto in crypto_asset:
# initialize useful matrices
crypto_fiat_arr = []
exc_price = np.matrix([])
exc_vol = np.matrix([])
exc_price_vol = np.matrix([])
crypto_fiat_arr = crypto_fiat_gen(single_crypto, crypto_fiat_arr,
pair_list)
for exchange in exc_list:
print(exchange)
# initialize the matrices that will contain the data related
# to all currencypair for the single exchange
ccy_fiat_price_vol = np.matrix([])
ccy_fiat_vol = np.matrix([])
ccy_fiat_price = np.matrix([])
for cp in crypto_fiat_arr:
print(cp)
# selecting the data referring to specific exchange and crypto-fiat pair
matrix = data_matrix.loc[(data_matrix["Exchange"] == exchange)
& (data_matrix["Pair"] == cp)]
print(matrix.shape)
if matrix.empty is False:
price = np.array((matrix["Close Price"]))
volume = np.array((matrix["Pair Volume"]))
price_vol = np.array(price * volume)
# every "cp" the loop adds a column in the matrices referred
# to the single "exchange"
if ccy_fiat_price_vol.size == 0:
ccy_fiat_price_vol = price_vol
ccy_fiat_vol = volume
else:
ccy_fiat_price_vol = np.column_stack(
(ccy_fiat_price_vol, price_vol))
ccy_fiat_vol = np.column_stack((ccy_fiat_vol, volume))
else:
pass
# computing the volume weighted average price of the single exchange
ccy_fiat_price, ccy_fiat_vol, ccy_fiat_price_vol = loop_crypto_fiat_hist(
ccy_fiat_vol, ccy_fiat_price_vol, ref_date_arr)
exc_price, exc_vol, exc_price_vol = loop_single_exc_hist(
exc_price, exc_vol, exc_price_vol, ccy_fiat_price,
ccy_fiat_vol, ccy_fiat_price_vol, ref_date_arr)
(exc_price, exc_vol, exc_vol_tot,
logic_matrix_one) = loop_exc_value_hist(single_crypto, exc_price,
exc_vol, exc_price_vol,
exc_vol_tot, exc_list,
logic_matrix_one,
ref_date_arr)
# creating every loop the matrices of all the single_cryptossets
# Crypto_Asset_Price contains the prices of all the cryptocurrencies
# crypto_asset_vol contains the volume of all the cryptocurrencies
crypto_asset_price, crypto_asset_vol = loop_crypto_asset(
exc_price, exc_vol, crypto_asset_price, crypto_asset_vol)
return crypto_asset_price, crypto_asset_vol, exc_vol_tot, logic_matrix_one
]
# exchange complete = [ 'coinbase-pro', 'poloniex', 'bitstamp',
# 'gemini', 'bittrex', 'kraken', 'bitflyer']
start_date = "01-01-2016"
EXC_start_date = "04-17-2020"
day_in_sec = 86400
# set today
today_str = datetime.now().strftime("%Y-%m-%d")
today = datetime.strptime(today_str, "%Y-%m-%d")
today_TS = int(today.replace(tzinfo=timezone.utc).timestamp())
# define the array containing the date where the index uses CW feed data
CW_date_arr = data_setup.date_gen(start_date, EXC_start_date)
CW_date_str = [str(date) for date in CW_date_arr]
# ######################## setup MongoDB connection ###########################
# connecting to mongo in local
connection = MongoClient("localhost", 27017)
# creating the database called index
db = connection.index
# drop the pre-existing collection (if there is one)
db.index_data_feed.drop()
# creating the empty collection cleandata within the database index
db.index_data_feed.create_index([("id", -1)])
collection_feed = db.index_data_feed
collection_dict_upload.get("collection_cw_clean").delete_many(my2)
##
# ############################ missing days check #############################
# this section allows to check if CW_clean data contains the new values of the
# day, the check is based on a 5-days period and allows
# assign date of interest to variables
today_str = datetime.now().strftime("%Y-%m-%d")
today = datetime.strptime(today_str, "%Y-%m-%d")
today_TS = int(today.replace(tzinfo=timezone.utc).timestamp())
y_TS = today_TS - DAY_IN_SEC
two_before_TS = y_TS - DAY_IN_SEC
# defining the array containing all the date from start_period until today
date_complete_int = date_gen(START_DATE)
# converting the timestamp format date into string
date_tot = [str(single_date) for single_date in date_complete_int]
# searching only the last five days
last_five_days = date_tot[(len(date_tot) - 5): len(date_tot)]
# defining the details to query on MongoDB
query = {"Exchange": "coinbase-pro", "Pair": "ethusd"}
# retrieving the wanted data on MongoDB collection
matrix = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_clean"), query)
# checking the time column
date_list = np.array(matrix["Time"])
last_five_days_mongo = date_list[(len(date_list) - 5): len(date_list)]
# define the start date as MM-DD-YYYY
start_date = "01-01-2016"
# define today in various format
today_str = datetime.now().strftime("%Y-%m-%d")
today = datetime.strptime(today_str, "%Y-%m-%d")
today_TS = int(today.replace(tzinfo=timezone.utc).timestamp())
y_TS = today_TS - 86400
two_before_TS = y_TS - 86400
today_human = data_setup.timestamp_to_human([today_TS])
yesterday_human = data_setup.timestamp_to_human([y_TS])
two_before_human = data_setup.timestamp_to_human([two_before_TS])
# define the variable containing all the date from start_date to today.
# the date are displayed as timestamp and each day refers to 12:00 am UTC
reference_date_vector = data_setup.date_gen(start_date)
# define all the useful arrays containing the rebalance start
# date, stop date, board meeting date
rebalance_start_date = calc.start_q("01-01-2016")
rebalance_stop_date = calc.stop_q(rebalance_start_date)
board_date = calc.board_meeting_day()
board_date_eve = calc.day_before_board()
next_rebalance_date = calc.next_start()
# call the function that creates a object containing
# the couple of quarterly start-stop date
quarterly_date = calc.quarterly_period()
# defining time variables
curr_reb_start = str(int(rebalance_start_date[len(rebalance_start_date) - 1]))
else:
pass
return None
# ########## HISTORICAL EXC RAW DATA OPERATION ##############
# set today
today_str = datetime.now().strftime("%Y-%m-%d")
today = datetime.strptime(today_str, "%Y-%m-%d")
today_TS = int(today.replace(tzinfo=timezone.utc).timestamp())
y_TS = today_TS - DAY_IN_SEC
# creating the timestamp array at 12:00 AM
date_array = date_gen(EXC_START_DATE)
date_array_str = [str(el) for el in date_array]
def all_crypto_fiat_gen():
# defining the crypto-fiat pairs array
all_crypto_fiat_array = []
for crypto in CRYPTO_ASSET:
for fiat in PAIR_ARRAY:
all_crypto_fiat_array.append(crypto.lower() + fiat)
return all_crypto_fiat_array
from cryptoindex.data_setup import (date_gen)
from cryptoindex.mongo_setup import (query_mongo, mongo_coll, mongo_indexing,
mongo_upload)
from cryptoindex.config import (START_DATE, DAY_IN_SEC, MONGO_DICT, DB_NAME)
# ############# INITIAL SETTINGS ################################
# set today
today_str = datetime.now().strftime("%Y-%m-%d")
today = datetime.strptime(today_str, "%Y-%m-%d")
today_TS = int(today.replace(tzinfo=timezone.utc).timestamp())
y_TS = today_TS - DAY_IN_SEC
two_before_TS = y_TS - DAY_IN_SEC
# defining the array containing all the date from start_period until today
date_complete_int = date_gen(START_DATE)
# converting the timestamp format date into string
date_tot = [str(single_date) for single_date in date_complete_int]
# #################### setup mongo connection ##################
# creating the empty collections cleandata within the database index
mongo_indexing()
collection_dict_upload = mongo_coll()
# ################# DAILY DATA CONVERSION MAIN PART ##################
# querying the data from mongo
query_data = {"Time": str(y_TS)}
query_rate = {"Date": str(y_TS)}
# creating the database called index
db = connection.index
# drop the pre-existing collection (if there is one)
db.ecb_raw.drop()
# creating the empty collection rawdata within the database index
db.ecb_raw.create_index([("id", -1)])
collection_ECB_raw = db.ecb_raw
# ####################### ECB rates raw data download #########################
# create an array of date containing the list of date to download
date = data_setup.date_gen(start_period,
End_Period,
timeST="N",
clss="list",
EoD="N")
date = [
datetime.strptime(day, "%m-%d-%Y").strftime("%Y-%m-%d") for day in date
]
Exchange_Rate_List = pd.DataFrame()
for i, single_date in enumerate(date):
# retrieving data from ECB website
single_date_ex_matrix = data_download.ECB_rates_extractor(
key_curr_vector, date[i])
# put a sllep time in order to do not overuse API connection
