def index_hist_total(coll_to_use="coll_data_feed",
crypto_asset=CRYPTO_ASSET,
exc_list=EXCHANGES,
pair_list=PAIR_ARRAY):
# drop the pre-existing collections
mongo_coll_drop("index_hist")
# define the mongo indexing
mongo_indexing()
data_df = query_mongo(DB_NAME, MONGO_DICT.get(coll_to_use))
(crypto_asset_price_arr, crypto_asset_vol_arr, exc_vol_tot,
logic_matrix_one) = index_hist_loop(data_df, crypto_asset, exc_list,
pair_list)
print(crypto_asset_vol_arr)
(crypto_asset_price, crypto_asset_vol, price_ret, weights_for_board,
first_logic_matrix_df, second_logic_matrix_df, ewma_df,
double_checked_EWMA, syntethic, syntethic_relative_matrix, divisor_array,
reshaped_divisor, index_values,
index_1000_base) = index_hist_op(crypto_asset_price_arr,
crypto_asset_vol_arr, logic_matrix_one)
index_hist_uploader(crypto_asset_price, crypto_asset_vol, exc_vol_tot,
price_ret, weights_for_board, first_logic_matrix_df,
second_logic_matrix_df, ewma_df, double_checked_EWMA,
syntethic, syntethic_relative_matrix, divisor_array,
reshaped_divisor, index_values, index_1000_base)
def ecb_daily_download(day_to_download_TS):
# creating the empty collection cleandata within the database index
mongo_indexing()
day_date = datetime.fromtimestamp(int(day_to_download_TS))
day_date_str = day_date.strftime("%Y-%m-%d")
# retrieving data from ECB website
rate_of_day = ECB_rates_extractor(ECB_FIAT, day_date_str)
return rate_of_day
def exc_hist_op():
mongo_coll_drop("exc")
mongo_indexing()
# defining the crytpo_fiat array
crypto_fiat_arr = all_crypto_fiat_gen()
# querying all raw data from EXC_rawdata
exc_raw_df = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_raw"))
midnight_clean = exc_initial_clean(exc_raw_df, crypto_fiat_arr)
mongo_upload(midnight_clean, "collection_exc_uniform")
# deleting the values for xrp in the coinbase-pro exchange
midnight_clean["key"] = midnight_clean["Exchange"] + \
"&" + midnight_clean["Pair"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "coinbase-pro&xrpusd"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "coinbase-pro&xrpeur"]
# deleting the values for zec and xmr in the bittrex exchange
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "bittrex&zecusd"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "bittrex&zecusdt"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "bittrex&zecusdc"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "bittrex&xmrusdt"]
midnight_clean = midnight_clean.drop(columns="key")
exc_complete_df = exc_key_mngmt(midnight_clean)
exc_fixed_df = exc_hist_fix(exc_complete_df)
mongo_upload(exc_fixed_df, "collection_exc_clean")
exc_converted = exc_hist_conv(exc_fixed_df)
exc_converted.fillna(0, inplace=True)
mongo_upload(exc_converted, "collection_exc_final_data")
return None
def cw_hist_operation(start_date=START_DATE):
date_tot = date_gen(start_date)
last_day_TS = date_tot[len(date_tot) - 1]
mongo_indexing()
# deleting previous MongoDB collections
mongo_coll_drop("cw_hist_clean")
mongo_coll_drop("cw_hist_conv")
# fix and upload the series for the "pair volume" info
tot_raw_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_raw"))
cw_vol_fix_data = cw_hist_pair_vol_fix(tot_raw_data)
mongo_upload(cw_vol_fix_data, "collection_cw_vol_check")
# clean and upload all the series
cleaned_df = cw_hist_cleaning(cw_vol_fix_data, start_date)
mongo_upload(cleaned_df, "collection_cw_clean")
# compute and upload USDC and USDT rates series
usdt_rates, usdc_rates = stable_rates_op("coll_cw_clean", None)
mongo_upload(usdt_rates, "collection_stable_rate")
mongo_upload(usdc_rates, "collection_stable_rate")
# convert and upload all the data into USD
converted_df = cw_hist_conv_op(cleaned_df)
mongo_upload(converted_df, "collection_cw_converted")
# logic matrix of crypto-fiat keys
key_df = key_log_mat(DB_NAME, "coll_cw_conv", last_day_TS, EXCHANGES,
CRYPTO_ASSET, PAIR_ARRAY)
mongo_upload(key_df, "collection_CW_key")
mongo_upload(key_df, "collection_EXC_key")
# fill zero-volume data and upload on MongoDB
final_df = cw_hist_zero_vol_fill_op(converted_df)
mongo_upload(final_df, "collection_cw_final_data")
return None
# set today
today_str = datetime.now().strftime("%Y-%m-%d")
today = datetime.strptime(today_str, "%Y-%m-%d")
today_TS = int(today.replace(tzinfo=timezone.utc).timestamp())
y_TS = today_TS - DAY_IN_SEC
two_before_TS = y_TS - DAY_IN_SEC
# defining the array containing all the date from start_period until today
date_complete_int = date_gen(START_DATE)
# converting the timestamp format date into string
date_tot = [str(single_date) for single_date in date_complete_int]
# #################### setup mongo connection ##################
# creating the empty collections cleandata within the database index
mongo_indexing()
collection_dict_upload = mongo_coll()
# ################# DAILY DATA CONVERSION MAIN PART ##################
# querying the data from mongo
query_data = {"Time": str(y_TS)}
query_rate = {"Date": str(y_TS)}
query_stable = {"Time": str(y_TS)}
matrix_rate = query_mongo(DB_NAME, MONGO_DICT.get("coll_ecb_clean"),
query_rate)
matrix_rate = matrix_rate.rename({"Date": "Time"}, axis="columns")
matrix_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_clean"),
query_data)
print(matrix_data)
def test_mongo_indexing():
mongo_indexing()
def cw_daily_operation(day=None):
'''
@param day has to be either None or "%Y-%m-%d" string format
'''
# create the indexing for MongoDB and define the variable containing the
# MongoDB collections where to upload data
mongo_indexing()
date_tot = date_gen(START_DATE)
# converting the timestamp format date into string
date_tot_str = [str(single_date) for single_date in date_tot]
day_before_TS, _ = days_variable(day)
if day is None:
if daily_check_mongo("coll_cw_raw", {
"Exchange": "coinbase-pro",
"Pair": "btcusd"
}) is False:
try:
cw_rawdata_daily = cw_daily_download(day_before_TS +
DAY_IN_SEC)
except Exception:
error("Exception occurred", exc_info=True)
info('Daily download from CryptoWatch failed')
mongo_upload(cw_rawdata_daily, "collection_cw_raw")
else:
print("The CW_rawdata collection on MongoDB is updated.")
if daily_check_mongo("coll_vol_chk", {
"Exchange": "coinbase-pro",
"Pair": "btcusd"
}) is False:
mat_vol_fix = daily_pair_vol_fix(day_before_TS)
try:
mongo_upload(mat_vol_fix, "collection_cw_vol_check")
except AttributeError:
pass
else:
mat_vol_fix = []
print(
"Message: No need to fix pair volume. The collection on MongoDB is updated."
)
# new and dead crypto-fiat key management
daily_complete_df = cw_daily_key_mngm(mat_vol_fix, day_before_TS,
date_tot_str)
# missing data fixing
if daily_check_mongo("coll_cw_clean", {
"Exchange": "coinbase-pro",
"Pair": "btcusd"
}) is False:
daily_fixed_df = daily_fix_miss_op(daily_complete_df, day,
"coll_cw_clean")
mongo_upload(daily_fixed_df, "collection_cw_clean")
else:
print("Message: The collection cw_clean on MongoDB is updated.")
if daily_check_mongo("coll_stable_rate",
{"Currency": "USDT/USD"}) is False:
usdt_rates, usdc_rates = stable_rates_op("coll_cw_clean",
str(day_before_TS))
mongo_upload(usdt_rates, "collection_stable_rate")
mongo_upload(usdc_rates, "collection_stable_rate")
else:
print("The stable_rates_collection on MongoDB is already updated.")
if daily_check_mongo("coll_cw_conv", {
"Exchange": "coinbase-pro",
"Pair": "btcusd"
}) is False:
converted_data = cw_daily_conv_op(day_before_TS)
mongo_upload(converted_data, "collection_cw_converted")
else:
print(
"Message: The cw_converted_data collection on MongoDB is already updated."
)
if daily_check_mongo("coll_cw_final", {
"Exchange": "coinbase-pro",
"Pair": "btcusd"
}) is False:
mongo_upload(converted_data, "collection_cw_final_data")
else:
print(
"The CW_final_data collection on MongoDB is already updated.")
else:
cw_rawdata_daily = cw_daily_download(day_before_TS)
mongo_upload(cw_rawdata_daily, "collection_cw_raw")
mat_vol_fix = daily_pair_vol_fix(day_before_TS)
try:
mongo_upload(mat_vol_fix, "collection_cw_vol_check")
except AttributeError:
pass
daily_complete_df = cw_daily_key_mngm(mat_vol_fix, day_before_TS,
date_tot_str)
daily_fixed_df = daily_fix_miss_op(daily_complete_df, day,
"coll_cw_clean")
mongo_upload(daily_fixed_df, "collection_cw_clean")
usdt_rates, usdc_rates = stable_rates_op("coll_cw_clean",
str(day_before_TS))
mongo_upload(usdt_rates, "collection_stable_rate")
mongo_upload(usdc_rates, "collection_stable_rate")
converted_data = cw_daily_conv_op(day_before_TS)
mongo_upload(converted_data, "collection_cw_converted")
mongo_upload(converted_data, "collection_cw_final_data")
return None
def index_normal_day(crypto_asset, exc_list, pair_list, coll_to_use, day=None):
# create the indexing for MongoDB
mongo_indexing()
day_before_TS, two_before_TS = days_variable(day)
two_before_human = timestamp_to_human([two_before_TS])
# defining the dictionary for the MongoDB query
query_dict = {"Time": int(day_before_TS)}
# retriving the needed information on MongoDB
daily_mat = query_mongo(DB_NAME, MONGO_DICT.get(coll_to_use), query_dict)
(crypto_asset_price, crypto_asset_vol, exc_vol_tot,
_) = index_daily_loop(daily_mat, crypto_asset, exc_list, pair_list,
day_before_TS)
# turn prices and volumes into pandas dataframe
crypto_asset_price = pd.DataFrame(crypto_asset_price, columns=CRYPTO_ASSET)
crypto_asset_vol = pd.DataFrame(crypto_asset_vol, columns=CRYPTO_ASSET)
# compute the price return of the day
two_before_price = query_mongo(DB_NAME, MONGO_DICT.get("coll_price"),
{"Time": two_before_TS})
two_before_price = two_before_price.drop(columns=["Time", "Date"])
return_df = two_before_price.append(crypto_asset_price)
price_ret = return_df.pct_change()
price_ret = price_ret.iloc[[1]]
price_ret = price_ret.replace(-1, 0)
price_ret.fillna(0, inplace=True)
# then add the 'Time' column
time_header = ["Time"]
time_header.extend(CRYPTO_ASSET)
crypto_asset_price = pd.DataFrame(crypto_asset_price, columns=time_header)
crypto_asset_price["Time"] = int(day_before_TS)
crypto_asset_vol = pd.DataFrame(crypto_asset_vol, columns=time_header)
crypto_asset_vol["Time"] = int(day_before_TS)
# adding the Time column to the price ret df
price_ret["Time"] = int(day_before_TS)
# computing the Exponential Weighted Moving Average of the day
hist_volume = query_mongo(DB_NAME, MONGO_DICT.get("coll_volume"))
hist_volume = hist_volume.drop(columns=["Date"])
hist_volume = hist_volume.append(crypto_asset_vol)
daily_ewma = daily_ewma_crypto_volume(hist_volume, CRYPTO_ASSET)
# downloading from mongoDB the current logic matrices (1 e 2)
logic_one = query_mongo(DB_NAME, MONGO_DICT.get("coll_log1"))
# taking only the logic value referred to the current period
current_logic_one = logic_one.iloc[[len(logic_one["Date"]) - 2]]
current_logic_one = current_logic_one.drop(columns=["Date", "Time"])
logic_two = query_mongo(DB_NAME, MONGO_DICT.get("coll_log2"))
# taking only the logic value referred to the current period
current_logic_two = logic_two.iloc[[len(logic_two["Date"]) - 2]]
current_logic_two = current_logic_two.drop(columns=["Date", "Time"])
# computing the ewma checked with both the first and second logic matrices
daily_ewma_first_check = np.array(daily_ewma) * np.array(current_logic_one)
daily_ewma_double_check = daily_ewma_first_check * \
np.array(current_logic_two)
daily_ewma_double_check = pd.DataFrame(daily_ewma_double_check,
columns=CRYPTO_ASSET)
# compute the daily syntethic matrix
yesterday_synt_matrix = query_mongo(DB_NAME, MONGO_DICT.get("coll_synt"),
{"Date": two_before_human[0]})
yesterday_synt_matrix = yesterday_synt_matrix.drop(
columns=["Date", "Time"])
daily_return = np.array(price_ret.loc[:, CRYPTO_ASSET])
new_synt = (1 + daily_return) * np.array(yesterday_synt_matrix)
daily_synth = pd.DataFrame(new_synt, columns=CRYPTO_ASSET)
# compute the daily relative syntethic matrix
daily_tot = np.array(daily_synth.sum(axis=1))
daily_rel = np.array(daily_synth) / daily_tot
daily_rel = pd.DataFrame(daily_rel, columns=CRYPTO_ASSET)
# daily index value computation
curr_divisor = query_mongo(DB_NAME, MONGO_DICT.get("coll_divisor_res"),
{"Date": two_before_human[0]})
curr_div_val = np.array(curr_divisor["Divisor Value"])
index_numerator = np.array(
crypto_asset_price[CRYPTO_ASSET]) * np.array(daily_rel)
numerator_sum = index_numerator.sum(axis=1)
num = pd.DataFrame(numerator_sum)
daily_index_value = np.array(num) / curr_div_val
raw_index_df = pd.DataFrame(daily_index_value, columns=["Index Value"])
# retrieving from mongoDB the yesterday value of the raw index
yesterday_raw_index = query_mongo(DB_NAME,
MONGO_DICT.get("coll_raw_index"),
{"Date": two_before_human[0]})
yesterday_raw_index = yesterday_raw_index.drop(columns=["Date", "Time"])
raw_curr = yesterday_raw_index.append(raw_index_df)
variation = raw_curr.pct_change()
variation = np.array(variation.iloc[1])
# retrieving from mongoDB the yesterday value of the raw index
yesterday_1000_index = query_mongo(DB_NAME,
MONGO_DICT.get("coll_1000_index"),
{"Date": two_before_human[0]})
daily_index_1000 = np.array(
yesterday_1000_index["Index Value"]) * (1 + variation)
daily_index_1000_df = pd.DataFrame(daily_index_1000,
columns=["Index Value"])
index_daily_uploader(crypto_asset_price,
crypto_asset_vol,
exc_vol_tot,
price_ret,
daily_ewma,
daily_ewma_double_check,
daily_synth,
daily_rel,
curr_divisor,
daily_index_1000_df,
raw_index_df,
day=day)
return None
def index_board_day(crypto_asset, exc_list, pair_list, coll_to_use, day=None):
mongo_indexing()
day_before_TS, _ = days_variable(day)
# define all the useful arrays containing the rebalance start
# date, stop date, board meeting date
rebalance_start_date = start_q(START_DATE)
rebalance_stop_date = stop_q(rebalance_start_date)
board_date_eve = day_before_board()
next_rebalance_date = next_start()
# call the function that creates a object containing
# the couple of quarterly start-stop date
next_reb_start = str(int(next_rebalance_date[len(rebalance_start_date)]))
curr_reb_start = str(
int(rebalance_start_date[len(rebalance_start_date) - 1]))
last_reb_start = str(
int(rebalance_start_date[len(rebalance_start_date) - 2]))
next_reb_stop = str(int(rebalance_stop_date[len(rebalance_stop_date) - 2]))
curr_board_eve = str(int(board_date_eve[len(board_date_eve) - 1]))
# defining the dictionary for the MongoDB query
query_dict = {"Time": int(day_before_TS)}
# retriving the needed information on MongoDB
daily_mat = query_mongo(DB_NAME, MONGO_DICT.get(coll_to_use), query_dict)
crypto_asset_price, crypto_asset_vol, exc_vol_tot, logic_one_arr = index_daily_loop(
daily_mat,
crypto_asset,
exc_list,
pair_list,
day_before_TS,
last_reb_start,
next_reb_stop,
curr_board_eve,
day_type="board")
(crypto_asset_price, crypto_asset_vol, price_ret, daily_ewma,
daily_ewma_double_check, daily_synth, daily_rel, curr_divisor,
first_logic_row, second_logic_row, weights_for_board, daily_index_1000_df,
raw_index_df, _) = index_daily_operation(crypto_asset,
crypto_asset_price,
crypto_asset_vol,
day_before_TS,
curr_reb_start,
next_reb_start,
curr_board_eve,
logic_one_arr=logic_one_arr,
day_type="board",
day=day)
print(daily_ewma_double_check)
index_daily_uploader(crypto_asset_price,
crypto_asset_vol,
exc_vol_tot,
price_ret,
daily_ewma,
daily_ewma_double_check,
daily_synth,
daily_rel,
curr_divisor,
daily_index_1000_df,
raw_index_df,
new_logic_one=first_logic_row,
new_logic_two=second_logic_row,
new_weights=weights_for_board,
day=day)
return None