def log_key_update(key_to_update, collection):
collection_dict_upload = mongo_coll()
q_del = {"key": key_to_update}
collection_dict_upload.get(collection).delete_many(q_del)
# updating the logic matrix of exchange-pair couple
new_key_log = pd.DataFrame(np.column_stack(
(key_to_update, int(1))), columns=["key", "logic_value"])
new_key_log["logic_value"] = 1
return new_key_log
def new_series_composer(key,
new_key_df,
date_tot_str,
day_to_check,
kind_of_series="CW"):
# create the df containing the historical series of the new couple(s)
# composed by zeros
splited_key = key.split("&")
key_hist_df = pd.DataFrame(date_tot_str, columns=["Time"])
key_hist_df["Close Price"] = 0
key_hist_df["Pair Volume"] = 0
key_hist_df["Crypto Volume"] = 0
key_hist_df["Exchange"] = splited_key[0]
key_hist_df["Pair"] = splited_key[1]
if kind_of_series == "CW":
collection_dict_upload = mongo_coll()
# uploading on MongoDB collections "CW_converted_data" and "CW_final_data"
# the new series of zero except for the last value (yesterday)
mongo_upload(key_hist_df, "collection_cw_converted")
mongo_upload(key_hist_df, "collection_cw_final_data")
query_to_del = {"Time": str(day_to_check)}
collection_dict_upload.get("collection_cw_converted").delete_many(
query_to_del)
collection_dict_upload.get("collection_cw_final_data").delete_many(
query_to_del)
else:
pass
# inserting the today value of the new couple(s)
new_price = np.array(new_key_df.loc[new_key_df.key == key, "Close Price"])
new_p_vol = np.array(new_key_df.loc[new_key_df.key == key, "Pair Volume"])
new_c_vol = np.array(new_key_df.loc[new_key_df.key == key,
"Crypto Volume"])
key_hist_df.loc[key_hist_df.Time == str(day_to_check),
"Close Price"] = new_price
key_hist_df.loc[key_hist_df.Time == str(day_to_check),
"Pair Volume"] = new_p_vol
key_hist_df.loc[key_hist_df.Time == str(day_to_check),
"Crypto Volume"] = new_c_vol
return key_hist_df
def cw_hist_download_op(start_date=START_DATE):
# deleting previous MongoDB collection for rawdata
mongo_coll_drop("cw_hist_down")
collection_dict_upload = mongo_coll()
print("Downloading all CW history...")
cw_raw_data = cw_hist_download(start_date)
mongo_upload(cw_raw_data, "collection_cw_raw")
print("CW series download completed")
# deleting 31/12/2015 values if present
last_2015_TS = 1451520000
query_ = {'Time': last_2015_TS}
collection_dict_upload.get("collection_cw_raw").delete_many(query_)
return None
def exc_daily_key_mngm(daily_mat_00, daily_mat_12, day_to_clean_TS):
logic_key = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_keys"))
# adding to the daily matrix the value referred to dead crypto-fiat pair
daily_mat_with_dead = exc_dead_key_mng(
logic_key, daily_mat_00, daily_mat_12, day_to_clean_TS)
# searching for possible new crypto-fiat pair
new_key_hist = exc_new_key_mng(logic_key, daily_mat_00, day_to_clean_TS)
if new_key_hist != []:
collection_dict_upload = mongo_coll()
# upload the new artificial historical series on MongoDB
# collection "EXC_cleandata"
mongo_upload(new_key_hist, collection_dict_upload.get(
"collection_exc_clean"))
else:
pass
return daily_mat_with_dead
today = datetime.strptime(today_str, "%Y-%m-%d")
today_TS = int(today.replace(tzinfo=timezone.utc).timestamp())
y_TS = today_TS - DAY_IN_SEC
two_before_TS = y_TS - DAY_IN_SEC
# defining the array containing all the date from start_period until today
date_complete_int = date_gen(START_DATE)
# converting the timestamp format date into string
date_tot = [str(single_date) for single_date in date_complete_int]
# #################### setup mongo connection ##################
# creating the empty collections cleandata within the database index
mongo_indexing()
collection_dict_upload = mongo_coll()
# ################# DAILY DATA CONVERSION MAIN PART ##################
# querying the data from mongo
query_data = {"Time": str(y_TS)}
query_rate = {"Date": str(y_TS)}
query_stable = {"Time": str(y_TS)}
matrix_rate = query_mongo(DB_NAME, MONGO_DICT.get("coll_ecb_clean"),
query_rate)
matrix_rate = matrix_rate.rename({"Date": "Time"}, axis="columns")
matrix_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_clean"),
query_data)
print(matrix_data)
matrix_rate_stable = query_mongo(DB_NAME, MONGO_DICT.get("coll_stable_rate"),
query_stable)
def test_mongo_coll():
assert type(mongo_coll()) == dict
# asserting that the dictionary is not empty
assert bool(mongo_coll) is True
