def index_hist_total(coll_to_use="coll_data_feed", crypto_asset=CRYPTO_ASSET, exc_list=EXCHANGES, pair_list=PAIR_ARRAY): # drop the pre-existing collections mongo_coll_drop("index_hist") # define the mongo indexing mongo_indexing() data_df = query_mongo(DB_NAME, MONGO_DICT.get(coll_to_use)) (crypto_asset_price_arr, crypto_asset_vol_arr, exc_vol_tot, logic_matrix_one) = index_hist_loop(data_df, crypto_asset, exc_list, pair_list) print(crypto_asset_vol_arr) (crypto_asset_price, crypto_asset_vol, price_ret, weights_for_board, first_logic_matrix_df, second_logic_matrix_df, ewma_df, double_checked_EWMA, syntethic, syntethic_relative_matrix, divisor_array, reshaped_divisor, index_values, index_1000_base) = index_hist_op(crypto_asset_price_arr, crypto_asset_vol_arr, logic_matrix_one) index_hist_uploader(crypto_asset_price, crypto_asset_vol, exc_vol_tot, price_ret, weights_for_board, first_logic_matrix_df, second_logic_matrix_df, ewma_df, double_checked_EWMA, syntethic, syntethic_relative_matrix, divisor_array, reshaped_divisor, index_values, index_1000_base)
def ecb_hist_setup(start_date, fiat_curr): # set today as End_period end_date = datetime.now().strftime("%m-%d-%Y") # drop the pre-existing collection mongo_coll_drop("ecb_hist_s") # make the raw data clean through the ECB_setup function try: cleaned_ecb = ECB_setup(fiat_curr, start_date, end_date) except UnboundLocalError: print( "The chosen start date does not exist in ECB websites. Be sure to avoid holiday as first date" ) # transform the timestamp format date into string timestamp_str, standard_date = date_converter(cleaned_ecb["Date"]) cleaned_ecb["Date"] = timestamp_str cleaned_ecb["Standard Date"] = standard_date return cleaned_ecb
def data_feed_op(): mongo_coll_drop("index_feed") merged_series = cw_exc_merging() mongo_upload(merged_series, "collection_data_feed") return None
def test_mongo_coll_drop(): ll = [ 'ecb_hist_d', 'ecb_hist_s', 'cw_hist_d', 'cw_hist_s', 'cw_hist_conv', 'index_hist', 'index_feed', 'exc' ] for operation in ll: mongo_coll_drop(operation)
def cw_hist_download_op(start_date=START_DATE): # deleting previous MongoDB collection for rawdata mongo_coll_drop("cw_hist_down") collection_dict_upload = mongo_coll() print("Downloading all CW history...") cw_raw_data = cw_hist_download(start_date) mongo_upload(cw_raw_data, "collection_cw_raw") print("CW series download completed") # deleting 31/12/2015 values if present last_2015_TS = 1451520000 query_ = {'Time': last_2015_TS} collection_dict_upload.get("collection_cw_raw").delete_many(query_) return None
def hist_data_feed_op(): # define the array containing the date where the index uses CW feed data CW_date_arr = date_gen(START_DATE, EXC_START_DATE) CW_date_str = [str(date) for date in CW_date_arr] # drop the pre-existing collection (if there is one) mongo_coll_drop("index_feed") # downloading the EXC series from MongoDB EXC_series = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_final")) EXC_series = EXC_series[ ["Time", "Close Price", "Crypto Volume", "Pair Volume", "Exchange", "Pair"]] # downloading the CW series from MongoDB and selecting only the date # from 2016-01-01 to 2020-04-17 CW_series = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_final")) CW_series["Time"] = [str(x) for x in CW_series["Time"]] print("CW") print(CW_series) CW_sub_series = CW_series.loc[CW_series.Time.isin(CW_date_str)] print(CW_sub_series) CW_sub_series = CW_sub_series[ ["Time", "Close Price", "Crypto Volume", "Pair Volume", "Exchange", "Pair"]] CW_sub_series["Time"] = [int(x) for x in CW_sub_series["Time"]] CW_sub_series.reset_index(drop=True, inplace=True) print(CW_sub_series) # creting an unique dataframe containing the two different data source data_feed = CW_sub_series.append(EXC_series, sort=True) data_feed.reset_index(drop=True, inplace=True) data_feed = data_feed[ ["Time", "Close Price", "Crypto Volume", "Pair Volume", "Exchange", "Pair"]] print(data_feed) data_feed = homogeneize_feed(data_feed) print("post hom") print(data_feed) # put the converted data on MongoDB mongo_upload(data_feed, "collection_data_feed") return None
def exc_hist_op(): mongo_coll_drop("exc") mongo_indexing() # defining the crytpo_fiat array crypto_fiat_arr = all_crypto_fiat_gen() # querying all raw data from EXC_rawdata exc_raw_df = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_raw")) midnight_clean = exc_initial_clean(exc_raw_df, crypto_fiat_arr) mongo_upload(midnight_clean, "collection_exc_uniform") # deleting the values for xrp in the coinbase-pro exchange midnight_clean["key"] = midnight_clean["Exchange"] + \ "&" + midnight_clean["Pair"] midnight_clean = midnight_clean.loc[midnight_clean.key != "coinbase-pro&xrpusd"] midnight_clean = midnight_clean.loc[midnight_clean.key != "coinbase-pro&xrpeur"] # deleting the values for zec and xmr in the bittrex exchange midnight_clean = midnight_clean.loc[midnight_clean.key != "bittrex&zecusd"] midnight_clean = midnight_clean.loc[midnight_clean.key != "bittrex&zecusdt"] midnight_clean = midnight_clean.loc[midnight_clean.key != "bittrex&zecusdc"] midnight_clean = midnight_clean.loc[midnight_clean.key != "bittrex&xmrusdt"] midnight_clean = midnight_clean.drop(columns="key") exc_complete_df = exc_key_mngmt(midnight_clean) exc_fixed_df = exc_hist_fix(exc_complete_df) mongo_upload(exc_fixed_df, "collection_exc_clean") exc_converted = exc_hist_conv(exc_fixed_df) exc_converted.fillna(0, inplace=True) mongo_upload(exc_converted, "collection_exc_final_data") return None
def cw_hist_operation(start_date=START_DATE): date_tot = date_gen(start_date) last_day_TS = date_tot[len(date_tot) - 1] mongo_indexing() # deleting previous MongoDB collections mongo_coll_drop("cw_hist_clean") mongo_coll_drop("cw_hist_conv") # fix and upload the series for the "pair volume" info tot_raw_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_raw")) cw_vol_fix_data = cw_hist_pair_vol_fix(tot_raw_data) mongo_upload(cw_vol_fix_data, "collection_cw_vol_check") # clean and upload all the series cleaned_df = cw_hist_cleaning(cw_vol_fix_data, start_date) mongo_upload(cleaned_df, "collection_cw_clean") # compute and upload USDC and USDT rates series usdt_rates, usdc_rates = stable_rates_op("coll_cw_clean", None) mongo_upload(usdt_rates, "collection_stable_rate") mongo_upload(usdc_rates, "collection_stable_rate") # convert and upload all the data into USD converted_df = cw_hist_conv_op(cleaned_df) mongo_upload(converted_df, "collection_cw_converted") # logic matrix of crypto-fiat keys key_df = key_log_mat(DB_NAME, "coll_cw_conv", last_day_TS, EXCHANGES, CRYPTO_ASSET, PAIR_ARRAY) mongo_upload(key_df, "collection_CW_key") mongo_upload(key_df, "collection_EXC_key") # fill zero-volume data and upload on MongoDB final_df = cw_hist_zero_vol_fill_op(converted_df) mongo_upload(final_df, "collection_cw_final_data") return None
def ecb_hist_download(start_date): # drop the pre-existing collection related to ecb_rawdata mongo_coll_drop("ecb_hist_d") # set today as end_date end_date = datetime.now().strftime("%Y-%m-%d") # create an array of date containing the list of date to download date_list = date_gen(start_date, end_date, timeST="N", clss="list", EoD="N") date_list_str = [ datetime.strptime(day, "%m-%d-%Y").strftime("%Y-%m-%d") for day in date_list ] ecb_hist_series = pd.DataFrame() for date in date_list_str: # retrieving data from ECB website single_date_ex_matrix = ECB_rates_extractor(ECB_FIAT, date) # put a sllep time in order to do not overuse API connection time.sleep(0.02) # put all the downloaded data into a DafaFrame if ecb_hist_series.size == 0: ecb_hist_series = single_date_ex_matrix else: ecb_hist_series = ecb_hist_series.append(single_date_ex_matrix, sort=True) return ecb_hist_series
# local import from cryptoindex.calc import ( start_q, stop_q, board_meeting_day, day_before_board, next_start, quarterly_period, next_quarterly_period, first_logic_matrix, second_logic_matrix, ewma_crypto_volume, divisor_adjustment, ewma_second_logic_check, quarter_weights, relative_syntethic_matrix, quarterly_synt_matrix, divisor_reshape, index_based, index_level_calc) from cryptoindex.data_setup import (date_gen, timestamp_to_human) from cryptoindex.mongo_setup import (mongo_coll, mongo_coll_drop, mongo_indexing, mongo_upload, query_mongo) from cryptoindex.config import (START_DATE, MONGO_DICT, PAIR_ARRAY, CRYPTO_ASSET, EXCHANGES, DB_NAME, DAY_IN_SEC) # drop the pre-existing collection (if there is one) mongo_coll_drop("index_hist") mongo_indexing() collection_dict_upload = mongo_coll() # ################### DATE SETTINGS #################### # define today and yesterady date as timestamp today_str = datetime.now().strftime("%Y-%m-%d") today = datetime.strptime(today_str, "%Y-%m-%d") today_TS = int(today.replace(tzinfo=timezone.utc).timestamp()) y_TS = today_TS - DAY_IN_SEC # # define end date as as MM-DD-YYYY # end_date = datetime.now().strftime("%m-%d-%Y")
mongo_indexing) from cryptoindex.config import (START_DATE, PAIR_ARRAY, CRYPTO_ASSET, EXCHANGES) # ################### initial settings ######################### # set end_date as today, otherwise comment and choose an end_date end_date = datetime.now().strftime("%m-%d-%Y") # define the variable containing all the date from start_date to today. # the date are displayed as timestamp and each day refers to 12:00 am UTC reference_date_vector = data_setup.date_gen(START_DATE) # ################# setup MongoDB connection ##################### mongo_coll_drop("cw_hist_d") mongo_indexing() collection_dict_upload = mongo_coll() # ################# downloading and storing part ################ for Crypto in CRYPTO_ASSET: print(Crypto) ccy_pair_array = [] for i in PAIR_ARRAY: ccy_pair_array.append(Crypto.lower() + i)
# define today date as timestamp today_str = datetime.now().strftime("%Y-%m-%d") today = datetime.strptime(today_str, "%Y-%m-%d") today_TS = int(today.replace(tzinfo=timezone.utc).timestamp()) y_TS = today_TS - DAY_IN_SEC # define the variable containing all the date from start_date to today. # the date are displayed as timestamp and each day refers to 12:00 am UTC reference_date_vector = date_gen(START_DATE) # ################ setup MongoDB connection ################ # drop the pre-existing collection mongo_coll_drop("cw_hist_conv") # creating the empty collection cleandata within the database index mongo_indexing() collection_dict_upload = mongo_coll() # ########## USDC/USD and USDT/USD computation ##################### start = time.time() Exchanges = EXCHANGES # taking BTC/USD pair historical first_query = {"Pair": "btcusd", "Exchange": "kraken"} first_call = query_mongo(
date_array_str = [str(el) for el in date_array] print(date_array_str) # defining the crypto-fiat pairs array cryptofiat_array = [] for crypto in CRYPTO_ASSET: for fiat in PAIR_ARRAY: cryptofiat_array.append(crypto.lower() + fiat) # ############################ setup mongo connection ################## # drop the pre-existing collection (if there is one) mongo_coll_drop("exc") mongo_indexing() collection_dict_upload = mongo_coll() # ################### creation of EXC_cleandata collection ################## # querying all raw data from EXC_rawdata all_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_raw")) # keeping only the columns of interest among all the # information in rawdata all_data = all_data.loc[:, CLEAN_DATA_HEAD]