def test_query_mongo():
# uploading test json file
df_test = pd.read_json(data_folder)
df_test = df_test.drop(columns=["_id"])
# querying the collection from db
database = "index"
collection = "test_mongo"
df_mongo = query_mongo(database, collection)
assert df_test.equals(df_mongo)
query_dict = {"A": {"$gt": 0}}
df_mongo = query_mongo(database, collection, query_dict)
assert df_test.equals(df_mongo)
query_dict = {"A": {"$gt": 111110}}
ll = []
df_mongo = query_mongo(database, collection, query_dict)
assert ll == df_mongo
collection = "tes"
df_mongo = query_mongo(database, collection)
assert ll == df_mongo
def daily_conv_op(day_to_conv_TS, conversion_fiat=CONVERSION_FIAT,
stable=STABLE_COIN, series="CW"):
# querying the data from mongo
query_data = {"Time": int(day_to_conv_TS)}
query_rate = {"Date": str(day_to_conv_TS)}
query_stable = {"Time": int(day_to_conv_TS)}
# querying the data from mongo
matrix_rate = query_mongo(
DB_NAME, MONGO_DICT.get("coll_ecb_clean"), query_rate)
if series == "CW":
matrix_data = query_mongo(
DB_NAME, MONGO_DICT.get("coll_cw_clean"), query_data)
elif series == "EXC":
matrix_data = query_mongo(
DB_NAME, MONGO_DICT.get("coll_exc_clean"), query_data)
matrix_rate_stable = query_mongo(
DB_NAME, MONGO_DICT.get("coll_stable_rate"), query_stable)
# converting the "matrix_rate" df
converted_df = conv_into_usd(
DB_NAME, matrix_data, matrix_rate,
matrix_rate_stable, conversion_fiat, stable)
return converted_df
def exc_daily_cleaning(exc_list, day_to_clean):
previous_day = int(day_to_clean) - DAY_IN_SEC
# download from MongoDB the exc raw data of yesterday
q_dict: Dict[str, str] = {}
q_dict = {"Time": str(day_to_clean + DAY_IN_SEC)}
daily_mat = query_mongo(
DB_NAME, MONGO_DICT.get("coll_exc_raw"), q_dict)
# fixing the "pair volume" information in the daily df
daily_mat_vol_fix = daily_exc_pair_vol_fix(daily_mat, exc_list)
# creating different df based on the hour of download
(daily_mat_00, daily_mat_12, _, _) = exc_time_split(daily_mat_vol_fix)
# transpose the Time information by 86400 seconds (1 day) ###### str o int???
daily_mat_00["Time"] = [str(int(element) - DAY_IN_SEC)
for element in daily_mat_00["Time"]]
daily_mat_12["Time"] = [str(int(element) - DAY_IN_SEC)
for element in daily_mat_12["Time"]]
# completing the daily matrix with dead crypto-fiat pair
daily_mat_complete = exc_daily_key_mngm(
daily_mat_00, daily_mat_12, day_to_clean)
# downloading from MongoDB the matrix referring to the previuos day
q_dict_bfr: Dict[str, int] = {}
q_dict_bfr = {"Time": int(previous_day)}
day_bfr_mat = query_mongo(
DB_NAME, MONGO_DICT.get("coll_exc_clean"), q_dict_bfr)
fixed_daily_mat = exc_daily_fix_op(day_bfr_mat, daily_mat_complete)
return fixed_daily_mat
def cw_hist_conv_op(cleaned_df, conv_fiat=CONVERSION_FIAT, stable=STABLE_COIN):
matrix_rate = query_mongo(DB_NAME, MONGO_DICT.get("coll_ecb_clean"))
matrix_rate_stable = query_mongo(DB_NAME,
MONGO_DICT.get("coll_stable_rate"))
# converting the "matrix_rate" df
converted_df = conv_into_usd(DB_NAME, cleaned_df, matrix_rate,
matrix_rate_stable, conv_fiat, stable)
return converted_df
def update_indicator(timer):
df_index = query_mongo("index", "index_level_1000")
dff_ind = df_index.copy()
dff_last_ind = dff_ind.tail(2)
dff_ind_y = dff_last_ind[dff_last_ind['Date'] == dff_last_ind['Date'].min(
)]['Index Value'].values[0]
dff_ind_t = dff_last_ind[dff_last_ind['Date'] == dff_last_ind['Date'].max(
)]['Index Value'].values[0]
fig_indicator = go.Figure(
go.Indicator(mode="delta",
value=dff_ind_t,
delta={
'reference': dff_ind_y,
'relative': True,
'valueformat': '.2%'
}))
fig_indicator.update_traces(delta_font={'size': 22})
fig_indicator.update_layout(height=60, width=100)
if dff_ind_t >= dff_ind_y:
fig_indicator.update_traces(delta_increasing_color='green')
elif dff_ind_t < dff_ind_y:
fig_indicator.update_traces(delta_decreasing_color='red')
csv_string_index = dff_ind.to_csv(index=False, encoding='utf-8')
csv_string_index = "data:text/csv;charset=utf-8," + quote(csv_string_index)
return fig_indicator, csv_string_index
def index_hist_total(coll_to_use="coll_data_feed",
crypto_asset=CRYPTO_ASSET,
exc_list=EXCHANGES,
pair_list=PAIR_ARRAY):
# drop the pre-existing collections
mongo_coll_drop("index_hist")
# define the mongo indexing
mongo_indexing()
data_df = query_mongo(DB_NAME, MONGO_DICT.get(coll_to_use))
(crypto_asset_price_arr, crypto_asset_vol_arr, exc_vol_tot,
logic_matrix_one) = index_hist_loop(data_df, crypto_asset, exc_list,
pair_list)
print(crypto_asset_vol_arr)
(crypto_asset_price, crypto_asset_vol, price_ret, weights_for_board,
first_logic_matrix_df, second_logic_matrix_df, ewma_df,
double_checked_EWMA, syntethic, syntethic_relative_matrix, divisor_array,
reshaped_divisor, index_values,
index_1000_base) = index_hist_op(crypto_asset_price_arr,
crypto_asset_vol_arr, logic_matrix_one)
index_hist_uploader(crypto_asset_price, crypto_asset_vol, exc_vol_tot,
price_ret, weights_for_board, first_logic_matrix_df,
second_logic_matrix_df, ewma_df, double_checked_EWMA,
syntethic, syntethic_relative_matrix, divisor_array,
reshaped_divisor, index_values, index_1000_base)
def daily_check_mongo(coll_to_check, query, day_to_check=None, coll_kind=None):
day_before_TS, _ = days_variable(day_to_check)
if coll_kind is None:
query["Time"] = int(day_before_TS)
elif coll_kind == "ecb_raw":
query["TIME_PERIOD"] = str(day_before_TS)
elif coll_kind == "ecb_clean":
query["Date"] = str(day_before_TS)
# retrieving the wanted data on MongoDB collection
matrix = query_mongo(DB_NAME, MONGO_DICT.get(coll_to_check), query)
if isinstance(matrix, list):
res = False
else:
res = True
return bool(res)
def update_index_df(n, sel_col):
df_index = query_mongo("index", "index_level_1000")
dff = df_index.copy()
dff = dff.loc[dff["Date"] > "2016-09-30"]
dff_last = dff.tail(2)
dff_y = dff_last[dff_last['Date'] ==
dff_last['Date'].min()]['Index Value'].values[0]
dff_t = dff_last[dff_last['Date'] ==
dff_last['Date'].max()]['Index Value'].values[0]
variation = (dff_t >= dff_y)
dff["Var"] = variation
index_area = area(data_frame=dff,
x="Date",
y="Index Value",
labels={
"value": "",
"Index Value": "",
"Date": ""
},
template=sel_col,
title='Crypto Index Level (USD)',
color="Var",
color_discrete_map={
False: '#FD3216',
True: '#1CA71C',
})
index_area.update_layout(showlegend=False)
index_area.update_yaxes(tickprefix="$")
return index_area
def hist_data_feed_op():
# define the array containing the date where the index uses CW feed data
CW_date_arr = date_gen(START_DATE, EXC_START_DATE)
CW_date_str = [str(date) for date in CW_date_arr]
# drop the pre-existing collection (if there is one)
mongo_coll_drop("index_feed")
# downloading the EXC series from MongoDB
EXC_series = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_final"))
EXC_series = EXC_series[
["Time", "Close Price", "Crypto Volume", "Pair Volume", "Exchange", "Pair"]]
# downloading the CW series from MongoDB and selecting only the date
# from 2016-01-01 to 2020-04-17
CW_series = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_final"))
CW_series["Time"] = [str(x) for x in CW_series["Time"]]
print("CW")
print(CW_series)
CW_sub_series = CW_series.loc[CW_series.Time.isin(CW_date_str)]
print(CW_sub_series)
CW_sub_series = CW_sub_series[
["Time", "Close Price", "Crypto Volume", "Pair Volume", "Exchange", "Pair"]]
CW_sub_series["Time"] = [int(x) for x in CW_sub_series["Time"]]
CW_sub_series.reset_index(drop=True, inplace=True)
print(CW_sub_series)
# creting an unique dataframe containing the two different data source
data_feed = CW_sub_series.append(EXC_series, sort=True)
data_feed.reset_index(drop=True, inplace=True)
data_feed = data_feed[
["Time", "Close Price", "Crypto Volume", "Pair Volume", "Exchange", "Pair"]]
print(data_feed)
data_feed = homogeneize_feed(data_feed)
print("post hom")
print(data_feed)
# put the converted data on MongoDB
mongo_upload(data_feed, "collection_data_feed")
return None
def cw_exc_merging(start_date=START_DATE, exc_start=EXC_START_DATE,
db=DB_NAME, coll_cw="coll_cw_final",
coll_exc="coll_exc_final"):
cw_date_arr = date_gen(start_date, exc_start, EoD="N")
exc_series = query_mongo(db, MONGO_DICT.get(coll_exc))
exc_part = df_reorder(exc_series, column_set="conversion")
# downloading the CW series from MongoDB and selecting only the date
# from 2016-01-01 to 2020-04-17
cw_series = query_mongo(db, MONGO_DICT.get(coll_cw))
cw_part = cw_series.loc[cw_series.Time.isin(cw_date_arr)]
cw_part = df_reorder(cw_part, column_set="conversion")
# creting an unique dataframe containing the two different data source
merged_series = cw_part.append(exc_part, sort=True)
merged_series["Time"] = [int(d) for d in merged_series["Time"]]
return merged_series
def exc_hist_conv(exc_fix_df):
# querying the rates collection from MongoDB
matrix_rate = query_mongo(DB_NAME, MONGO_DICT.get("coll_ecb_clean"))
matrix_rate = matrix_rate.rename({"Date": "Time"}, axis="columns")
matrix_rate = matrix_rate.loc[matrix_rate.Time.isin(date_array_str)]
# querying the stable rates collection from MongoDB
matrix_rate_stable = query_mongo(
DB_NAME, MONGO_DICT.get("coll_stable_rate"))
matrix_rate_stable = matrix_rate_stable.loc[matrix_rate_stable.Time.isin(
date_array_str)]
converted_data = conv_into_usd(DB_NAME, exc_fix_df, matrix_rate,
matrix_rate_stable, CONVERSION_FIAT, STABLE_COIN)
converted_data["Time"] = [int(element)
for element in converted_data["Time"]]
return converted_data
def exc_key_mngmt(exc_clean_df):
# downloading from MongoDB the matrix containing the exchange-pair logic values
logic_key = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_keys"))
# creating the exchange-pair couples key for the daily matrix
exc_clean_df["key"] = exc_clean_df["Exchange"] + "&" + exc_clean_df["Pair"]
# ## adding the dead series to the daily values
# selecting only the exchange-pair couples present in the historical series
key_present = logic_key.loc[logic_key.logic_value == 1]
key_present = key_present.drop(columns=["logic_value"])
exc_clean_df = exc_clean_df.loc[exc_clean_df.Time != str(today_TS)]
# selecting the last day of the EXC "historical" series
last_day_with_val = max(exc_clean_df.Time)
last_day = exc_clean_df.loc[exc_clean_df.Time
== last_day_with_val]
# applying a left join between the prresent keys matrix and the last_day
# matrix, this operation returns a matrix containing all the keys in
# "key_present" and, if some keys are missing in "all_data" put NaN
merged = pd.merge(key_present, last_day, on="key", how="left")
# selecting only the absent keys
merg_absent = merged.loc[merged["Close Price"].isnull()]
header = CLEAN_DATA_HEAD
header.extend(["key"])
# create the historical series for each selected element
for k in merg_absent["key"]:
mat_to_add = pd.DataFrame(columns=header)
mat_to_add["Time"] = date_array_str
split_val = k.split("&")
mat_to_add["Exchange"] = split_val[0]
mat_to_add["Pair"] = split_val[1]
mat_to_add["Close Price"] = 0.0
mat_to_add["Crypto Volume"] = 0.0
mat_to_add["Pair Volume"] = 0.0
exc_clean_df = exc_clean_df.append(mat_to_add)
# uploading the cleaned data on MongoDB in the collection EXC_cleandata
exc_clean_df = exc_clean_df.drop(columns=["key"])
exc_clean_df["Time"] = [int(element) for element in exc_clean_df["Time"]]
# deleting the 17/04/2020 from the df (if presents)
exc_complete_df = exc_clean_df.loc[exc_clean_df.Time != 1587081600]
return exc_complete_df
def cw_daily_conv_op(day_to_conv_TS):
# querying the data from mongo
query_data = {"Time": int(day_to_conv_TS)}
query_rate = {"Date": str(day_to_conv_TS)}
query_stable = {"Time": int(day_to_conv_TS)}
# querying the data from mongo
matrix_rate = query_mongo(DB_NAME, MONGO_DICT.get("coll_ecb_clean"),
query_rate)
matrix_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_clean"),
query_data)
matrix_rate_stable = query_mongo(DB_NAME,
MONGO_DICT.get("coll_stable_rate"),
query_stable)
# converting the "matrix_rate" df
converted_df = conv_into_usd(DB_NAME, matrix_data, matrix_rate,
matrix_rate_stable, CONVERSION_FIAT,
STABLE_COIN)
return converted_df
def cw_daily_key_mngm(volume_checked_df, time_to_check, date_tot_str):
if isinstance(volume_checked_df, list):
volume_checked_tot = query_mongo(DB_NAME,
MONGO_DICT.get("coll_vol_chk"))
last_day_with_val = max(volume_checked_tot.Time)
volume_checked_df = volume_checked_tot.loc[volume_checked_tot.Time ==
last_day_with_val]
# downloading from MongoDB the matrix with the daily values and the
# matrix containing the exchange-pair logic values
logic_key = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_keys"))
# creating the exchange-pair couples key for the daily matrix
volume_checked_df["key"] = volume_checked_df["Exchange"] + \
"&" + volume_checked_df["Pair"]
# selecting only the exchange-pair couples present in the historical series
key_present = logic_key.loc[logic_key.logic_value == 1]
key_present = key_present.drop(columns=["logic_value"])
# applying a left join between the prresent keys matrix and the daily
# matrix, this operation returns a matrix containing all the keys in
# "key_present" and NaN where some keys are missing
merged = pd.merge(key_present, volume_checked_df, on="key", how="left")
# assigning some columns values and substituting NaN with 0
# in the "merged" df
merged["Time"] = int(time_to_check)
split_val = merged["key"].str.split("&", expand=True)
merged["Exchange"] = split_val[0]
merged["Pair"] = split_val[1]
merged.fillna(0, inplace=True)
# checking potential new exchange-pair couple
cw_new_key_mngm(logic_key, volume_checked_df, time_to_check, date_tot_str)
return merged
def daily_table():
# assign date of interest to variables
today_str = datetime.now().strftime("%Y-%m-%d")
today = datetime.strptime(today_str, "%Y-%m-%d")
today_TS = int(today.replace(tzinfo=timezone.utc).timestamp())
y_TS = today_TS - DAY_IN_SEC
ecb_query = {"Date": str(y_TS), "Currency": "EUR/USD"}
btcusd_query = {"Time": y_TS, "Exchange": "coinbase-pro", "Pair": "btcusd"}
eurusd = query_mongo(DB_NAME, "ecb_clean", ecb_query)
btcusd = query_mongo(DB_NAME, "index_data_feed", btcusd_query)
btcusd_value = np.array(btcusd["Close Price"])
eurusd_value = np.array(eurusd["Rate"])
human_date = np.array(eurusd["Standard Date"])
df = pd.DataFrame(columns=["Date", "EUR/USD", "BTCUSD"])
df["Date"] = human_date
df["EUR/USD"] = eurusd_value
df["BTCUSD"] = btcusd_value
return df
def index_norm_logic_op(crypto_asset, daily_ewma):
past_start_quarter_list = start_q()
last_start_q = past_start_quarter_list[len(past_start_quarter_list) - 1]
# downloading from mongoDB the current logic matrices (1 e 2)
logic_one = query_mongo(DB_NAME, MONGO_DICT.get("coll_log1"))
# taking only the logic value referred to the current period
# current_logic_one = logic_one.iloc[[len(logic_one["Date"]) - 2]]
current_logic_one = logic_one.iloc[logic_one.Time == int(last_start_q)]
current_logic_one = current_logic_one.drop(columns=["Date", "Time"])
logic_two = query_mongo(DB_NAME, MONGO_DICT.get("coll_log2"))
# taking only the logic value referred to the current period
# current_logic_two = logic_two.iloc[[len(logic_two["Date"]) - 2]]
current_logic_two = logic_two.iloc[logic_two.Time == int(last_start_q)]
current_logic_two = current_logic_two.drop(columns=["Date", "Time"])
# computing the ewma checked with both the first and second logic matrices
daily_ewma_first_check = np.array(daily_ewma) * np.array(current_logic_one)
daily_ewma_double_check = daily_ewma_first_check * \
np.array(current_logic_two)
daily_ewma_double_check = pd.DataFrame(daily_ewma_double_check,
columns=crypto_asset)
return daily_ewma_double_check
def update_pie(my_dropdown, sel_col):
df_weight = query_mongo("index", "index_weights")
dff_weight = df_weight.copy()
dff_weight = dff_weight.drop(columns="Time")
dff_w_filt = dff_weight.loc[dff_weight["Date"] == my_dropdown]
dff_w_filt = dff_w_filt.drop(columns="Date")
df_col = list(dff_w_filt.columns)
for col in df_col:
val = array(dff_w_filt[col])[0]
if val == 0.0000:
dff_w_filt = dff_w_filt.drop(columns=col)
df_val = array(dff_w_filt)[0]
df_col_2 = list(dff_w_filt.columns)
pie_fig = pie(values=df_val,
names=df_col_2,
hole=.3,
template=sel_col,
title='Index Weights',
color=df_col_2,
color_discrete_map={
"BTC": "#FEAF16",
"ETH": "#511CFB",
"XRP": "#F6222E",
"LTC": "#E2E2E2",
"BCH": "#86CE00",
"EOS": "#FBE426",
"ETC": "#DA16FF",
"ZEC": "#B68100",
"ADA": "#00B5F7",
"XLM": "#750D86",
"XMR": "#A777F1",
"BSV": "#F58518"
})
csv_string_weight = dff_weight.to_csv(index=False, encoding='utf-8')
csv_string_weight = "data:text/csv;charset=utf-8," + \
quote(csv_string_weight)
return pie_fig, csv_string_weight
def daily_fix_miss_op(daily_complete_df, day, collection):
_, two_before_TS = days_variable(day)
# defining the query details
q_dict_time: Dict[str, int] = {}
q_dict_time = {"Time": int(two_before_TS)}
# downloading from MongoDB the matrix referring to the previuos day
day_bfr_mat = query_mongo(DB_NAME, MONGO_DICT.get(collection), q_dict_time)
# add the "key" column
day_bfr_mat["key"] = day_bfr_mat["Exchange"] + "&" + day_bfr_mat["Pair"]
# looping through all the daily keys looking for potential missing value
for key_val in day_bfr_mat["key"]:
new_val = daily_complete_df.loc[daily_complete_df.key == key_val]
# if the new 'Close Price' referred to a certain key is 0 the script
# check the previous day value: if is == 0 then pass, if is != 0
# the values related to the selected key needs to be corrected
if np.array(new_val["Close Price"]) == 0.0:
d_before_val = day_bfr_mat.loc[day_bfr_mat.key == key_val]
if np.array(d_before_val["Close Price"]) != 0.0:
price_var = daily_fix_miss(new_val, daily_complete_df,
day_bfr_mat)
# applying the weighted variation to the day before 'Close Price'
new_price = (1 + price_var) * d_before_val["Close Price"]
# changing the 'Close Price' value using the new computed price
daily_complete_df.loc[daily_complete_df.key == key_val,
"Close Price"] = new_price
else:
pass
else:
pass
daily_complete_df.drop(columns=["key"])
daily_complete_df["Time"] = [int(d) for d in daily_complete_df["Time"]]
daily_fixed_df = daily_complete_df
return daily_fixed_df
def new_synth_op(crypto_asset, daily_return_df, synt_ptf_value):
past_start_quarter_list = start_q()
new_start_q = past_start_quarter_list[len(past_start_quarter_list) - 1]
# downloading from mongoDB the current weights
tot_weights = query_mongo(DB_NAME, MONGO_DICT.get("coll_weights"))
new_weights = tot_weights.loc[tot_weights.Time == int(new_start_q),
crypto_asset]
# daily syntethic matrix computation
daily_ret_arr = np.array(daily_return_df[crypto_asset])
synt_weights = np.array(new_weights) * synt_ptf_value
new_synth = synt_weights * (1 + daily_ret_arr)
daily_synthh = pd.DataFrame(new_synth, columns=crypto_asset)
return daily_synthh
def check_missing(tot_date_arr, coll_to_check, query, days_to_check=10):
# selecting the last five days and put them into an array
last_days = tot_date_arr[(len(tot_date_arr) -
days_to_check):len(tot_date_arr)]
print(last_days)
# retrieving the wanted data on MongoDB collection
matrix = query_mongo(DB_NAME, MONGO_DICT.get(coll_to_check), query)
# checking the time column and selecting only the last five days retrived
# from MongoDB collection
try:
date_list = np.array(matrix["Time"])
except KeyError:
try:
date_list = np.array(matrix["TIME_PERIOD"])
except KeyError:
date_list = np.array(matrix["Date"])
to_del = np.array([0])
date_list = np.setdiff1d(date_list, to_del)
last_days_db = date_list[(len(date_list) - days_to_check):len(date_list)]
# last_days_db_str = [str(single_date)
# for single_date in last_days_db]
# finding the date to download as difference between
# complete array of date and date now stored on MongoDB
date_to_add = Diff(last_days, last_days_db)
print(date_to_add)
if len(date_to_add) > 9:
date_to_add = None
return date_to_add
def exc_hist_op():
mongo_coll_drop("exc")
mongo_indexing()
# defining the crytpo_fiat array
crypto_fiat_arr = all_crypto_fiat_gen()
# querying all raw data from EXC_rawdata
exc_raw_df = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_raw"))
midnight_clean = exc_initial_clean(exc_raw_df, crypto_fiat_arr)
mongo_upload(midnight_clean, "collection_exc_uniform")
# deleting the values for xrp in the coinbase-pro exchange
midnight_clean["key"] = midnight_clean["Exchange"] + \
"&" + midnight_clean["Pair"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "coinbase-pro&xrpusd"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "coinbase-pro&xrpeur"]
# deleting the values for zec and xmr in the bittrex exchange
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "bittrex&zecusd"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "bittrex&zecusdt"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "bittrex&zecusdc"]
midnight_clean = midnight_clean.loc[midnight_clean.key
!= "bittrex&xmrusdt"]
midnight_clean = midnight_clean.drop(columns="key")
exc_complete_df = exc_key_mngmt(midnight_clean)
exc_fixed_df = exc_hist_fix(exc_complete_df)
mongo_upload(exc_fixed_df, "collection_exc_clean")
exc_converted = exc_hist_conv(exc_fixed_df)
exc_converted.fillna(0, inplace=True)
mongo_upload(exc_converted, "collection_exc_final_data")
return None
def exc_daily_feed(day=None):
day_before_TS, _ = days_variable(day)
if day is None:
if daily_check_mongo("coll_data_feed", {"Exchange": "coinbase-pro", "Pair": "ethusd"}) is False:
query_data = {"Time": int(day_before_TS)}
exc_daily_df = query_mongo(
DB_NAME, MONGO_DICT.get("coll_exc_final"), query_data)
mongo_upload(exc_daily_df, "collection_data_feed")
else:
print("The collection index_data_feed is already updated")
else:
pass
return None
def daily_pair_vol_fix2(time_to_fix):
# defining the query details
q_dict: Dict[str, int] = {}
q_dict = {"Time": time_to_fix}
# querying oin MongoDB collection
daily_mat = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_raw"), q_dict)
daily_mat = daily_mat.loc[daily_mat.Time != 0]
daily_mat = daily_mat.drop(columns=["Low", "High", "Open"])
for Crypto in CRYPTO_ASSET:
ccy_pair_array = []
for i in PAIR_ARRAY:
ccy_pair_array.append(Crypto.lower() + i)
for exchange in EXCHANGES:
for cp in ccy_pair_array:
mat = daily_mat.loc[daily_mat["Exchange"] == exchange]
mat = mat.loc[mat["Pair"] == cp]
# checking if the matrix is not empty
if mat.shape[0] > 1:
mat["Pair Volume"] = mat["Close Price"] * \
mat["Crypto Volume"]
# put the manipulated data on MongoDB
try:
mongo_upload(mat, "collection_cw_vol_check")
except TypeError:
pass
return None
def daily_pair_vol_fix(day):
# defining the query details
q_dict: Dict[str, int] = {}
q_dict = {"Time": day}
# querying on MongoDB collection
daily_mat = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_raw"), q_dict)
try:
daily_mat = daily_mat.loc[daily_mat.Time != 0]
daily_mat = daily_mat.drop(columns=["Low", "High", "Open"])
daily_vol_fix = pair_vol_fix(daily_mat)
except AttributeError:
daily_vol_fix = []
return daily_vol_fix
def cw_hist_operation(start_date=START_DATE):
date_tot = date_gen(start_date)
last_day_TS = date_tot[len(date_tot) - 1]
mongo_indexing()
# deleting previous MongoDB collections
mongo_coll_drop("cw_hist_clean")
mongo_coll_drop("cw_hist_conv")
# fix and upload the series for the "pair volume" info
tot_raw_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_cw_raw"))
cw_vol_fix_data = cw_hist_pair_vol_fix(tot_raw_data)
mongo_upload(cw_vol_fix_data, "collection_cw_vol_check")
# clean and upload all the series
cleaned_df = cw_hist_cleaning(cw_vol_fix_data, start_date)
mongo_upload(cleaned_df, "collection_cw_clean")
# compute and upload USDC and USDT rates series
usdt_rates, usdc_rates = stable_rates_op("coll_cw_clean", None)
mongo_upload(usdt_rates, "collection_stable_rate")
mongo_upload(usdc_rates, "collection_stable_rate")
# convert and upload all the data into USD
converted_df = cw_hist_conv_op(cleaned_df)
mongo_upload(converted_df, "collection_cw_converted")
# logic matrix of crypto-fiat keys
key_df = key_log_mat(DB_NAME, "coll_cw_conv", last_day_TS, EXCHANGES,
CRYPTO_ASSET, PAIR_ARRAY)
mongo_upload(key_df, "collection_CW_key")
mongo_upload(key_df, "collection_EXC_key")
# fill zero-volume data and upload on MongoDB
final_df = cw_hist_zero_vol_fill_op(converted_df)
mongo_upload(final_df, "collection_cw_final_data")
return None
def update_vol(my_checklist, sel_col):
df_volume = query_mongo("index", "crypto_volume")
dff_vol = df_volume.copy()
dff_date = dff_vol["Date"]
dff_vol_filtered = dff_vol[my_checklist]
dff_vol_filtered["Date"] = dff_date
volume_line = line(data_frame=dff_vol_filtered,
x="Date",
y=my_checklist,
template=sel_col,
title='Crypto Volumes',
labels={
"value": "Volume (USD)",
"variable": ""
},
color_discrete_map={
"BTC": "#FEAF16",
"ETH": "#511CFB",
"XRP": "#F6222E",
"LTC": "#E2E2E2",
"BCH": "#86CE00",
"EOS": "#FBE426",
"ETC": "#DA16FF",
"ZEC": "#B68100",
"ADA": "#00B5F7",
"XLM": "#750D86",
"XMR": "#A777F1",
"BSV": "#F58518"
})
dff_volume = df_volume.copy()
dff_volume = dff_volume.drop(columns="Time")
csv_string_volume = dff_volume.to_csv(index=False, encoding='utf-8')
csv_string_volume = "data:text/csv;charset=utf-8," + \
quote(csv_string_volume)
return volume_line, csv_string_volume
def update_today_val(timer):
df_index = query_mongo("index", "index_level_1000")
dff_ind = df_index.copy()
dff_last_ind = dff_ind.tail(1)
dff_prev = (dff_ind.tail(2)).head(1)
today_val = dff_last_ind["Index Value"].values[0]
yesterday_val = dff_prev["Index Value"].values[0]
fig_indicator = go.Figure(
go.Indicator(
mode="number+delta",
value=today_val,
delta={
'relative': False,
'reference': yesterday_val,
'valueformat': ',.2f',
'position': "right",
},
number={
'suffix': "$",
'valueformat': ',.2f',
'font': {
'color': 'black',
'size': 22
}
},
))
fig_indicator.update_traces(delta_font={'size': 20})
fig_indicator.update_layout(
height=60,
width=300,
)
return fig_indicator
def exc_daily_key_mngm(daily_mat_00, daily_mat_12, day_to_clean_TS):
logic_key = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_keys"))
# adding to the daily matrix the value referred to dead crypto-fiat pair
daily_mat_with_dead = exc_dead_key_mng(
logic_key, daily_mat_00, daily_mat_12, day_to_clean_TS)
# searching for possible new crypto-fiat pair
new_key_hist = exc_new_key_mng(logic_key, daily_mat_00, day_to_clean_TS)
if new_key_hist != []:
collection_dict_upload = mongo_coll()
# upload the new artificial historical series on MongoDB
# collection "EXC_cleandata"
mongo_upload(new_key_hist, collection_dict_upload.get(
"collection_exc_clean"))
else:
pass
return daily_mat_with_dead
date_tot = [str(single_date) for single_date in date_complete_int]
# #################### setup mongo connection ##################
# creating the empty collections cleandata within the database index
mongo_indexing()
collection_dict_upload = mongo_coll()
# ################# DAILY DATA CONVERSION MAIN PART ##################
# querying the data from mongo
query_data = {"Time": str(y_TS)}
query_rate = {"Date": str(y_TS)}
query_stable = {"Time": str(y_TS)}
matrix_rate = query_mongo(DB_NAME, MONGO_DICT.get("coll_ecb_clean"),
query_rate)
matrix_rate = matrix_rate.rename({"Date": "Time"}, axis="columns")
matrix_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_clean"),
query_data)
print(matrix_data)
matrix_rate_stable = query_mongo(DB_NAME, MONGO_DICT.get("coll_stable_rate"),
query_stable)
# creating a column containing the fiat currency
matrix_rate["fiat"] = [x[:3].lower() for x in matrix_rate["Currency"]]
matrix_data["fiat"] = [x[3:].lower() for x in matrix_data["Pair"]]
matrix_rate_stable["fiat"] = [
x[:4].lower() for x in matrix_rate_stable["Currency"]
]
# ############ creating a USD subset which will not be converted #########
import pandas as pd
from cryptoindex.mongo_setup import query_mongo
db = 'index'
cll = 'index_level_1000'
df = query_mongo(db, cll)
df['Time'] = [int(x) for x in df['Time']]
df = df.loc[df['Time'] > 1467331200]
df = df.rename(columns={
'Date': 'time',
'Time': 'timestamp',
'Index Value': 'value'
})
print(df)
df = str(df.to_dict(orient="records"))
value = 'json = '
final = value + df
print(final)
with open('result.json', 'w') as f:
f.write(final)
