def index_hist_total(coll_to_use="coll_data_feed", crypto_asset=CRYPTO_ASSET, exc_list=EXCHANGES, pair_list=PAIR_ARRAY): # drop the pre-existing collections mongo_coll_drop("index_hist") # define the mongo indexing mongo_indexing() data_df = query_mongo(DB_NAME, MONGO_DICT.get(coll_to_use)) (crypto_asset_price_arr, crypto_asset_vol_arr, exc_vol_tot, logic_matrix_one) = index_hist_loop(data_df, crypto_asset, exc_list, pair_list) print(crypto_asset_vol_arr) (crypto_asset_price, crypto_asset_vol, price_ret, weights_for_board, first_logic_matrix_df, second_logic_matrix_df, ewma_df, double_checked_EWMA, syntethic, syntethic_relative_matrix, divisor_array, reshaped_divisor, index_values, index_1000_base) = index_hist_op(crypto_asset_price_arr, crypto_asset_vol_arr, logic_matrix_one) index_hist_uploader(crypto_asset_price, crypto_asset_vol, exc_vol_tot, price_ret, weights_for_board, first_logic_matrix_df, second_logic_matrix_df, ewma_df, double_checked_EWMA, syntethic, syntethic_relative_matrix, divisor_array, reshaped_divisor, index_values, index_1000_base)
def ecb_daily_download(day_to_download_TS): # creating the empty collection cleandata within the database index mongo_indexing() day_date = datetime.fromtimestamp(int(day_to_download_TS)) day_date_str = day_date.strftime("%Y-%m-%d") # retrieving data from ECB website rate_of_day = ECB_rates_extractor(ECB_FIAT, day_date_str) return rate_of_day
def exc_hist_op(): mongo_coll_drop("exc") mongo_indexing() # defining the crytpo_fiat array crypto_fiat_arr = all_crypto_fiat_gen() # querying all raw data from EXC_rawdata exc_raw_df = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_raw")) midnight_clean = exc_initial_clean(exc_raw_df, crypto_fiat_arr) mongo_upload(midnight_clean, "collection_exc_uniform") # deleting the values for xrp in the coinbase-pro exchange midnight_clean["key"] = midnight_clean["Exchange"] + \ "&" + midnight_clean["Pair"] midnight_clean = midnight_clean.loc[midnight_clean.key != "coinbase-pro&xrpusd"] midnight_clean = midnight_clean.loc[midnight_clean.key != "coinbase-pro&xrpeur"] # deleting the values for zec and xmr in the bittrex exchange midnight_clean = midnight_clean.loc[midnight_clean.key != "bittrex&zecusd"] midnight_clean = midnight_clean.loc[midnight_clean.key != "bittrex&zecusdt"] midnight_clean = midnight_clean.loc[midnight_clean.key != "bittrex&zecusdc"] midnight_clean = midnight_clean.loc[midnight_clean.key != "bittrex&xmrusdt"] midnight_clean = midnight_clean.drop(columns="key") exc_complete_df = exc_key_mngmt(midnight_clean) exc_fixed_df = exc_hist_fix(exc_complete_df) mongo_upload(exc_fixed_df, "collection_exc_clean") exc_converted = exc_hist_conv(exc_fixed_df) exc_converted.fillna(0, inplace=True) mongo_upload(exc_converted, "collection_exc_final_data") return None
def cw_hist_operation(start_date=START_DATE): date_tot = date_gen(start_date) last_day_TS = date_tot[len(date_tot) - 1] mongo_indexing() # deleting previous MongoDB collections mongo_coll_drop("cw_hist_clean") mongo_coll_drop("cw_hist_conv") # fix and upload the series for the "pair volume" info tot_raw_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_raw")) cw_vol_fix_data = cw_hist_pair_vol_fix(tot_raw_data) mongo_upload(cw_vol_fix_data, "collection_cw_vol_check") # clean and upload all the series cleaned_df = cw_hist_cleaning(cw_vol_fix_data, start_date) mongo_upload(cleaned_df, "collection_cw_clean") # compute and upload USDC and USDT rates series usdt_rates, usdc_rates = stable_rates_op("coll_cw_clean", None) mongo_upload(usdt_rates, "collection_stable_rate") mongo_upload(usdc_rates, "collection_stable_rate") # convert and upload all the data into USD converted_df = cw_hist_conv_op(cleaned_df) mongo_upload(converted_df, "collection_cw_converted") # logic matrix of crypto-fiat keys key_df = key_log_mat(DB_NAME, "coll_cw_conv", last_day_TS, EXCHANGES, CRYPTO_ASSET, PAIR_ARRAY) mongo_upload(key_df, "collection_CW_key") mongo_upload(key_df, "collection_EXC_key") # fill zero-volume data and upload on MongoDB final_df = cw_hist_zero_vol_fill_op(converted_df) mongo_upload(final_df, "collection_cw_final_data") return None
# set today today_str = datetime.now().strftime("%Y-%m-%d") today = datetime.strptime(today_str, "%Y-%m-%d") today_TS = int(today.replace(tzinfo=timezone.utc).timestamp()) y_TS = today_TS - DAY_IN_SEC two_before_TS = y_TS - DAY_IN_SEC # defining the array containing all the date from start_period until today date_complete_int = date_gen(START_DATE) # converting the timestamp format date into string date_tot = [str(single_date) for single_date in date_complete_int] # #################### setup mongo connection ################## # creating the empty collections cleandata within the database index mongo_indexing() collection_dict_upload = mongo_coll() # ################# DAILY DATA CONVERSION MAIN PART ################## # querying the data from mongo query_data = {"Time": str(y_TS)} query_rate = {"Date": str(y_TS)} query_stable = {"Time": str(y_TS)} matrix_rate = query_mongo(DB_NAME, MONGO_DICT.get("coll_ecb_clean"), query_rate) matrix_rate = matrix_rate.rename({"Date": "Time"}, axis="columns") matrix_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_clean"), query_data) print(matrix_data)
def test_mongo_indexing(): mongo_indexing()
def cw_daily_operation(day=None): ''' @param day has to be either None or "%Y-%m-%d" string format ''' # create the indexing for MongoDB and define the variable containing the # MongoDB collections where to upload data mongo_indexing() date_tot = date_gen(START_DATE) # converting the timestamp format date into string date_tot_str = [str(single_date) for single_date in date_tot] day_before_TS, _ = days_variable(day) if day is None: if daily_check_mongo("coll_cw_raw", { "Exchange": "coinbase-pro", "Pair": "btcusd" }) is False: try: cw_rawdata_daily = cw_daily_download(day_before_TS + DAY_IN_SEC) except Exception: error("Exception occurred", exc_info=True) info('Daily download from CryptoWatch failed') mongo_upload(cw_rawdata_daily, "collection_cw_raw") else: print("The CW_rawdata collection on MongoDB is updated.") if daily_check_mongo("coll_vol_chk", { "Exchange": "coinbase-pro", "Pair": "btcusd" }) is False: mat_vol_fix = daily_pair_vol_fix(day_before_TS) try: mongo_upload(mat_vol_fix, "collection_cw_vol_check") except AttributeError: pass else: mat_vol_fix = [] print( "Message: No need to fix pair volume. The collection on MongoDB is updated." ) # new and dead crypto-fiat key management daily_complete_df = cw_daily_key_mngm(mat_vol_fix, day_before_TS, date_tot_str) # missing data fixing if daily_check_mongo("coll_cw_clean", { "Exchange": "coinbase-pro", "Pair": "btcusd" }) is False: daily_fixed_df = daily_fix_miss_op(daily_complete_df, day, "coll_cw_clean") mongo_upload(daily_fixed_df, "collection_cw_clean") else: print("Message: The collection cw_clean on MongoDB is updated.") if daily_check_mongo("coll_stable_rate", {"Currency": "USDT/USD"}) is False: usdt_rates, usdc_rates = stable_rates_op("coll_cw_clean", str(day_before_TS)) mongo_upload(usdt_rates, "collection_stable_rate") mongo_upload(usdc_rates, "collection_stable_rate") else: print("The stable_rates_collection on MongoDB is already updated.") if daily_check_mongo("coll_cw_conv", { "Exchange": "coinbase-pro", "Pair": "btcusd" }) is False: converted_data = cw_daily_conv_op(day_before_TS) mongo_upload(converted_data, "collection_cw_converted") else: print( "Message: The cw_converted_data collection on MongoDB is already updated." ) if daily_check_mongo("coll_cw_final", { "Exchange": "coinbase-pro", "Pair": "btcusd" }) is False: mongo_upload(converted_data, "collection_cw_final_data") else: print( "The CW_final_data collection on MongoDB is already updated.") else: cw_rawdata_daily = cw_daily_download(day_before_TS) mongo_upload(cw_rawdata_daily, "collection_cw_raw") mat_vol_fix = daily_pair_vol_fix(day_before_TS) try: mongo_upload(mat_vol_fix, "collection_cw_vol_check") except AttributeError: pass daily_complete_df = cw_daily_key_mngm(mat_vol_fix, day_before_TS, date_tot_str) daily_fixed_df = daily_fix_miss_op(daily_complete_df, day, "coll_cw_clean") mongo_upload(daily_fixed_df, "collection_cw_clean") usdt_rates, usdc_rates = stable_rates_op("coll_cw_clean", str(day_before_TS)) mongo_upload(usdt_rates, "collection_stable_rate") mongo_upload(usdc_rates, "collection_stable_rate") converted_data = cw_daily_conv_op(day_before_TS) mongo_upload(converted_data, "collection_cw_converted") mongo_upload(converted_data, "collection_cw_final_data") return None
def index_normal_day(crypto_asset, exc_list, pair_list, coll_to_use, day=None): # create the indexing for MongoDB mongo_indexing() day_before_TS, two_before_TS = days_variable(day) two_before_human = timestamp_to_human([two_before_TS]) # defining the dictionary for the MongoDB query query_dict = {"Time": int(day_before_TS)} # retriving the needed information on MongoDB daily_mat = query_mongo(DB_NAME, MONGO_DICT.get(coll_to_use), query_dict) (crypto_asset_price, crypto_asset_vol, exc_vol_tot, _) = index_daily_loop(daily_mat, crypto_asset, exc_list, pair_list, day_before_TS) # turn prices and volumes into pandas dataframe crypto_asset_price = pd.DataFrame(crypto_asset_price, columns=CRYPTO_ASSET) crypto_asset_vol = pd.DataFrame(crypto_asset_vol, columns=CRYPTO_ASSET) # compute the price return of the day two_before_price = query_mongo(DB_NAME, MONGO_DICT.get("coll_price"), {"Time": two_before_TS}) two_before_price = two_before_price.drop(columns=["Time", "Date"]) return_df = two_before_price.append(crypto_asset_price) price_ret = return_df.pct_change() price_ret = price_ret.iloc[[1]] price_ret = price_ret.replace(-1, 0) price_ret.fillna(0, inplace=True) # then add the 'Time' column time_header = ["Time"] time_header.extend(CRYPTO_ASSET) crypto_asset_price = pd.DataFrame(crypto_asset_price, columns=time_header) crypto_asset_price["Time"] = int(day_before_TS) crypto_asset_vol = pd.DataFrame(crypto_asset_vol, columns=time_header) crypto_asset_vol["Time"] = int(day_before_TS) # adding the Time column to the price ret df price_ret["Time"] = int(day_before_TS) # computing the Exponential Weighted Moving Average of the day hist_volume = query_mongo(DB_NAME, MONGO_DICT.get("coll_volume")) hist_volume = hist_volume.drop(columns=["Date"]) hist_volume = hist_volume.append(crypto_asset_vol) daily_ewma = daily_ewma_crypto_volume(hist_volume, CRYPTO_ASSET) # downloading from mongoDB the current logic matrices (1 e 2) logic_one = query_mongo(DB_NAME, MONGO_DICT.get("coll_log1")) # taking only the logic value referred to the current period current_logic_one = logic_one.iloc[[len(logic_one["Date"]) - 2]] current_logic_one = current_logic_one.drop(columns=["Date", "Time"]) logic_two = query_mongo(DB_NAME, MONGO_DICT.get("coll_log2")) # taking only the logic value referred to the current period current_logic_two = logic_two.iloc[[len(logic_two["Date"]) - 2]] current_logic_two = current_logic_two.drop(columns=["Date", "Time"]) # computing the ewma checked with both the first and second logic matrices daily_ewma_first_check = np.array(daily_ewma) * np.array(current_logic_one) daily_ewma_double_check = daily_ewma_first_check * \ np.array(current_logic_two) daily_ewma_double_check = pd.DataFrame(daily_ewma_double_check, columns=CRYPTO_ASSET) # compute the daily syntethic matrix yesterday_synt_matrix = query_mongo(DB_NAME, MONGO_DICT.get("coll_synt"), {"Date": two_before_human[0]}) yesterday_synt_matrix = yesterday_synt_matrix.drop( columns=["Date", "Time"]) daily_return = np.array(price_ret.loc[:, CRYPTO_ASSET]) new_synt = (1 + daily_return) * np.array(yesterday_synt_matrix) daily_synth = pd.DataFrame(new_synt, columns=CRYPTO_ASSET) # compute the daily relative syntethic matrix daily_tot = np.array(daily_synth.sum(axis=1)) daily_rel = np.array(daily_synth) / daily_tot daily_rel = pd.DataFrame(daily_rel, columns=CRYPTO_ASSET) # daily index value computation curr_divisor = query_mongo(DB_NAME, MONGO_DICT.get("coll_divisor_res"), {"Date": two_before_human[0]}) curr_div_val = np.array(curr_divisor["Divisor Value"]) index_numerator = np.array( crypto_asset_price[CRYPTO_ASSET]) * np.array(daily_rel) numerator_sum = index_numerator.sum(axis=1) num = pd.DataFrame(numerator_sum) daily_index_value = np.array(num) / curr_div_val raw_index_df = pd.DataFrame(daily_index_value, columns=["Index Value"]) # retrieving from mongoDB the yesterday value of the raw index yesterday_raw_index = query_mongo(DB_NAME, MONGO_DICT.get("coll_raw_index"), {"Date": two_before_human[0]}) yesterday_raw_index = yesterday_raw_index.drop(columns=["Date", "Time"]) raw_curr = yesterday_raw_index.append(raw_index_df) variation = raw_curr.pct_change() variation = np.array(variation.iloc[1]) # retrieving from mongoDB the yesterday value of the raw index yesterday_1000_index = query_mongo(DB_NAME, MONGO_DICT.get("coll_1000_index"), {"Date": two_before_human[0]}) daily_index_1000 = np.array( yesterday_1000_index["Index Value"]) * (1 + variation) daily_index_1000_df = pd.DataFrame(daily_index_1000, columns=["Index Value"]) index_daily_uploader(crypto_asset_price, crypto_asset_vol, exc_vol_tot, price_ret, daily_ewma, daily_ewma_double_check, daily_synth, daily_rel, curr_divisor, daily_index_1000_df, raw_index_df, day=day) return None
def index_board_day(crypto_asset, exc_list, pair_list, coll_to_use, day=None): mongo_indexing() day_before_TS, _ = days_variable(day) # define all the useful arrays containing the rebalance start # date, stop date, board meeting date rebalance_start_date = start_q(START_DATE) rebalance_stop_date = stop_q(rebalance_start_date) board_date_eve = day_before_board() next_rebalance_date = next_start() # call the function that creates a object containing # the couple of quarterly start-stop date next_reb_start = str(int(next_rebalance_date[len(rebalance_start_date)])) curr_reb_start = str( int(rebalance_start_date[len(rebalance_start_date) - 1])) last_reb_start = str( int(rebalance_start_date[len(rebalance_start_date) - 2])) next_reb_stop = str(int(rebalance_stop_date[len(rebalance_stop_date) - 2])) curr_board_eve = str(int(board_date_eve[len(board_date_eve) - 1])) # defining the dictionary for the MongoDB query query_dict = {"Time": int(day_before_TS)} # retriving the needed information on MongoDB daily_mat = query_mongo(DB_NAME, MONGO_DICT.get(coll_to_use), query_dict) crypto_asset_price, crypto_asset_vol, exc_vol_tot, logic_one_arr = index_daily_loop( daily_mat, crypto_asset, exc_list, pair_list, day_before_TS, last_reb_start, next_reb_stop, curr_board_eve, day_type="board") (crypto_asset_price, crypto_asset_vol, price_ret, daily_ewma, daily_ewma_double_check, daily_synth, daily_rel, curr_divisor, first_logic_row, second_logic_row, weights_for_board, daily_index_1000_df, raw_index_df, _) = index_daily_operation(crypto_asset, crypto_asset_price, crypto_asset_vol, day_before_TS, curr_reb_start, next_reb_start, curr_board_eve, logic_one_arr=logic_one_arr, day_type="board", day=day) print(daily_ewma_double_check) index_daily_uploader(crypto_asset_price, crypto_asset_vol, exc_vol_tot, price_ret, daily_ewma, daily_ewma_double_check, daily_synth, daily_rel, curr_divisor, daily_index_1000_df, raw_index_df, new_logic_one=first_logic_row, new_logic_two=second_logic_row, new_weights=weights_for_board, day=day) return None