def index_hist_total(coll_to_use="coll_data_feed",
crypto_asset=CRYPTO_ASSET,
exc_list=EXCHANGES,
pair_list=PAIR_ARRAY):
# drop the pre-existing collections
mongo_coll_drop("index_hist")
# define the mongo indexing
mongo_indexing()
data_df = query_mongo(DB_NAME, MONGO_DICT.get(coll_to_use))
(crypto_asset_price_arr, crypto_asset_vol_arr, exc_vol_tot,
logic_matrix_one) = index_hist_loop(data_df, crypto_asset, exc_list,
pair_list)
print(crypto_asset_vol_arr)
(crypto_asset_price, crypto_asset_vol, price_ret, weights_for_board,
first_logic_matrix_df, second_logic_matrix_df, ewma_df,
double_checked_EWMA, syntethic, syntethic_relative_matrix, divisor_array,
reshaped_divisor, index_values,
index_1000_base) = index_hist_op(crypto_asset_price_arr,
crypto_asset_vol_arr, logic_matrix_one)
index_hist_uploader(crypto_asset_price, crypto_asset_vol, exc_vol_tot,
price_ret, weights_for_board, first_logic_matrix_df,
second_logic_matrix_df, ewma_df, double_checked_EWMA,
syntethic, syntethic_relative_matrix, divisor_array,
reshaped_divisor, index_values, index_1000_base)
def ecb_hist_setup(start_date, fiat_curr):
# set today as End_period
end_date = datetime.now().strftime("%m-%d-%Y")
# drop the pre-existing collection
mongo_coll_drop("ecb_hist_s")
# make the raw data clean through the ECB_setup function
try:
cleaned_ecb = ECB_setup(fiat_curr, start_date, end_date)
except UnboundLocalError:
print(
"The chosen start date does not exist in ECB websites. Be sure to avoid holiday as first date"
)
# transform the timestamp format date into string
timestamp_str, standard_date = date_converter(cleaned_ecb["Date"])
cleaned_ecb["Date"] = timestamp_str
cleaned_ecb["Standard Date"] = standard_date
return cleaned_ecb
def data_feed_op():
mongo_coll_drop("index_feed")
merged_series = cw_exc_merging()
mongo_upload(merged_series, "collection_data_feed")
return None
def test_mongo_coll_drop():
ll = [
'ecb_hist_d', 'ecb_hist_s', 'cw_hist_d', 'cw_hist_s', 'cw_hist_conv',
'index_hist', 'index_feed', 'exc'
]
for operation in ll:
mongo_coll_drop(operation)
def cw_hist_download_op(start_date=START_DATE):
# deleting previous MongoDB collection for rawdata
mongo_coll_drop("cw_hist_down")
collection_dict_upload = mongo_coll()
print("Downloading all CW history...")
cw_raw_data = cw_hist_download(start_date)
mongo_upload(cw_raw_data, "collection_cw_raw")
print("CW series download completed")
# deleting 31/12/2015 values if present
last_2015_TS = 1451520000
query_ = {'Time': last_2015_TS}
collection_dict_upload.get("collection_cw_raw").delete_many(query_)
return None
def hist_data_feed_op():
# define the array containing the date where the index uses CW feed data
CW_date_arr = date_gen(START_DATE, EXC_START_DATE)
CW_date_str = [str(date) for date in CW_date_arr]
# drop the pre-existing collection (if there is one)
mongo_coll_drop("index_feed")
# downloading the EXC series from MongoDB
EXC_series = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_final"))
EXC_series = EXC_series[
["Time", "Close Price", "Crypto Volume", "Pair Volume", "Exchange", "Pair"]]
# downloading the CW series from MongoDB and selecting only the date
# from 2016-01-01 to 2020-04-17
CW_series = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_final"))
CW_series["Time"] = [str(x) for x in CW_series["Time"]]
print("CW")
print(CW_series)
CW_sub_series = CW_series.loc[CW_series.Time.isin(CW_date_str)]
print(CW_sub_series)
CW_sub_series = CW_sub_series[
["Time", "Close Price", "Crypto Volume", "Pair Volume", "Exchange", "Pair"]]
CW_sub_series["Time"] = [int(x) for x in CW_sub_series["Time"]]
CW_sub_series.reset_index(drop=True, inplace=True)
print(CW_sub_series)
# creting an unique dataframe containing the two different data source
data_feed = CW_sub_series.append(EXC_series, sort=True)
data_feed.reset_index(drop=True, inplace=True)
data_feed = data_feed[
["Time", "Close Price", "Crypto Volume", "Pair Volume", "Exchange", "Pair"]]
print(data_feed)
data_feed = homogeneize_feed(data_feed)
print("post hom")
print(data_feed)
# put the converted data on MongoDB
mongo_upload(data_feed, "collection_data_feed")
return None
def exc_hist_op():
mongo_coll_drop("exc")
mongo_indexing()
# defining the crytpo_fiat array
crypto_fiat_arr = all_crypto_fiat_gen()
# querying all raw data from EXC_rawdata
exc_raw_df = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_raw"))
midnight_clean = exc_initial_clean(exc_raw_df, crypto_fiat_arr)
mongo_upload(midnight_clean, "collection_exc_uniform")
# deleting the values for xrp in the coinbase-pro exchange
midnight_clean["key"] = midnight_clean["Exchange"] + \
"&" + midnight_clean["Pair"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "coinbase-pro&xrpusd"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "coinbase-pro&xrpeur"]
# deleting the values for zec and xmr in the bittrex exchange
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "bittrex&zecusd"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "bittrex&zecusdt"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "bittrex&zecusdc"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "bittrex&xmrusdt"]
midnight_clean = midnight_clean.drop(columns="key")
exc_complete_df = exc_key_mngmt(midnight_clean)
exc_fixed_df = exc_hist_fix(exc_complete_df)
mongo_upload(exc_fixed_df, "collection_exc_clean")
exc_converted = exc_hist_conv(exc_fixed_df)
exc_converted.fillna(0, inplace=True)
mongo_upload(exc_converted, "collection_exc_final_data")
return None
def cw_hist_operation(start_date=START_DATE):
date_tot = date_gen(start_date)
last_day_TS = date_tot[len(date_tot) - 1]
mongo_indexing()
# deleting previous MongoDB collections
mongo_coll_drop("cw_hist_clean")
mongo_coll_drop("cw_hist_conv")
# fix and upload the series for the "pair volume" info
tot_raw_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_raw"))
cw_vol_fix_data = cw_hist_pair_vol_fix(tot_raw_data)
mongo_upload(cw_vol_fix_data, "collection_cw_vol_check")
# clean and upload all the series
cleaned_df = cw_hist_cleaning(cw_vol_fix_data, start_date)
mongo_upload(cleaned_df, "collection_cw_clean")
# compute and upload USDC and USDT rates series
usdt_rates, usdc_rates = stable_rates_op("coll_cw_clean", None)
mongo_upload(usdt_rates, "collection_stable_rate")
mongo_upload(usdc_rates, "collection_stable_rate")
# convert and upload all the data into USD
converted_df = cw_hist_conv_op(cleaned_df)
mongo_upload(converted_df, "collection_cw_converted")
# logic matrix of crypto-fiat keys
key_df = key_log_mat(DB_NAME, "coll_cw_conv", last_day_TS, EXCHANGES,
CRYPTO_ASSET, PAIR_ARRAY)
mongo_upload(key_df, "collection_CW_key")
mongo_upload(key_df, "collection_EXC_key")
# fill zero-volume data and upload on MongoDB
final_df = cw_hist_zero_vol_fill_op(converted_df)
mongo_upload(final_df, "collection_cw_final_data")
return None
def ecb_hist_download(start_date):
# drop the pre-existing collection related to ecb_rawdata
mongo_coll_drop("ecb_hist_d")
# set today as end_date
end_date = datetime.now().strftime("%Y-%m-%d")
# create an array of date containing the list of date to download
date_list = date_gen(start_date,
end_date,
timeST="N",
clss="list",
EoD="N")
date_list_str = [
datetime.strptime(day, "%m-%d-%Y").strftime("%Y-%m-%d")
for day in date_list
]
ecb_hist_series = pd.DataFrame()
for date in date_list_str:
# retrieving data from ECB website
single_date_ex_matrix = ECB_rates_extractor(ECB_FIAT, date)
# put a sllep time in order to do not overuse API connection
time.sleep(0.02)
# put all the downloaded data into a DafaFrame
if ecb_hist_series.size == 0:
ecb_hist_series = single_date_ex_matrix
else:
ecb_hist_series = ecb_hist_series.append(single_date_ex_matrix,
sort=True)
return ecb_hist_series
# local import
from cryptoindex.calc import (
start_q, stop_q, board_meeting_day, day_before_board, next_start,
quarterly_period, next_quarterly_period, first_logic_matrix,
second_logic_matrix, ewma_crypto_volume, divisor_adjustment,
ewma_second_logic_check, quarter_weights, relative_syntethic_matrix,
quarterly_synt_matrix, divisor_reshape, index_based, index_level_calc)
from cryptoindex.data_setup import (date_gen, timestamp_to_human)
from cryptoindex.mongo_setup import (mongo_coll, mongo_coll_drop,
mongo_indexing, mongo_upload, query_mongo)
from cryptoindex.config import (START_DATE, MONGO_DICT, PAIR_ARRAY,
CRYPTO_ASSET, EXCHANGES, DB_NAME, DAY_IN_SEC)
# drop the pre-existing collection (if there is one)
mongo_coll_drop("index_hist")
mongo_indexing()
collection_dict_upload = mongo_coll()
# ################### DATE SETTINGS ####################
# define today and yesterady date as timestamp
today_str = datetime.now().strftime("%Y-%m-%d")
today = datetime.strptime(today_str, "%Y-%m-%d")
today_TS = int(today.replace(tzinfo=timezone.utc).timestamp())
y_TS = today_TS - DAY_IN_SEC
# # define end date as as MM-DD-YYYY
# end_date = datetime.now().strftime("%m-%d-%Y")
mongo_indexing)
from cryptoindex.config import (START_DATE, PAIR_ARRAY, CRYPTO_ASSET,
EXCHANGES)
# ################### initial settings #########################
# set end_date as today, otherwise comment and choose an end_date
end_date = datetime.now().strftime("%m-%d-%Y")
# define the variable containing all the date from start_date to today.
# the date are displayed as timestamp and each day refers to 12:00 am UTC
reference_date_vector = data_setup.date_gen(START_DATE)
# ################# setup MongoDB connection #####################
mongo_coll_drop("cw_hist_d")
mongo_indexing()
collection_dict_upload = mongo_coll()
# ################# downloading and storing part ################
for Crypto in CRYPTO_ASSET:
print(Crypto)
ccy_pair_array = []
for i in PAIR_ARRAY:
ccy_pair_array.append(Crypto.lower() + i)
# define today date as timestamp
today_str = datetime.now().strftime("%Y-%m-%d")
today = datetime.strptime(today_str, "%Y-%m-%d")
today_TS = int(today.replace(tzinfo=timezone.utc).timestamp())
y_TS = today_TS - DAY_IN_SEC
# define the variable containing all the date from start_date to today.
# the date are displayed as timestamp and each day refers to 12:00 am UTC
reference_date_vector = date_gen(START_DATE)
# ################ setup MongoDB connection ################
# drop the pre-existing collection
mongo_coll_drop("cw_hist_conv")
# creating the empty collection cleandata within the database index
mongo_indexing()
collection_dict_upload = mongo_coll()
# ########## USDC/USD and USDT/USD computation #####################
start = time.time()
Exchanges = EXCHANGES
# taking BTC/USD pair historical
first_query = {"Pair": "btcusd", "Exchange": "kraken"}
first_call = query_mongo(
date_array_str = [str(el) for el in date_array]
print(date_array_str)
# defining the crypto-fiat pairs array
cryptofiat_array = []
for crypto in CRYPTO_ASSET:
for fiat in PAIR_ARRAY:
cryptofiat_array.append(crypto.lower() + fiat)
# ############################ setup mongo connection ##################
# drop the pre-existing collection (if there is one)
mongo_coll_drop("exc")
mongo_indexing()
collection_dict_upload = mongo_coll()
# ################### creation of EXC_cleandata collection ##################
# querying all raw data from EXC_rawdata
all_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_raw"))
# keeping only the columns of interest among all the
# information in rawdata
all_data = all_data.loc[:, CLEAN_DATA_HEAD]