def log_key_update(key_to_update, collection): collection_dict_upload = mongo_coll() q_del = {"key": key_to_update} collection_dict_upload.get(collection).delete_many(q_del) # updating the logic matrix of exchange-pair couple new_key_log = pd.DataFrame(np.column_stack( (key_to_update, int(1))), columns=["key", "logic_value"]) new_key_log["logic_value"] = 1 return new_key_log
def new_series_composer(key, new_key_df, date_tot_str, day_to_check, kind_of_series="CW"): # create the df containing the historical series of the new couple(s) # composed by zeros splited_key = key.split("&") key_hist_df = pd.DataFrame(date_tot_str, columns=["Time"]) key_hist_df["Close Price"] = 0 key_hist_df["Pair Volume"] = 0 key_hist_df["Crypto Volume"] = 0 key_hist_df["Exchange"] = splited_key[0] key_hist_df["Pair"] = splited_key[1] if kind_of_series == "CW": collection_dict_upload = mongo_coll() # uploading on MongoDB collections "CW_converted_data" and "CW_final_data" # the new series of zero except for the last value (yesterday) mongo_upload(key_hist_df, "collection_cw_converted") mongo_upload(key_hist_df, "collection_cw_final_data") query_to_del = {"Time": str(day_to_check)} collection_dict_upload.get("collection_cw_converted").delete_many( query_to_del) collection_dict_upload.get("collection_cw_final_data").delete_many( query_to_del) else: pass # inserting the today value of the new couple(s) new_price = np.array(new_key_df.loc[new_key_df.key == key, "Close Price"]) new_p_vol = np.array(new_key_df.loc[new_key_df.key == key, "Pair Volume"]) new_c_vol = np.array(new_key_df.loc[new_key_df.key == key, "Crypto Volume"]) key_hist_df.loc[key_hist_df.Time == str(day_to_check), "Close Price"] = new_price key_hist_df.loc[key_hist_df.Time == str(day_to_check), "Pair Volume"] = new_p_vol key_hist_df.loc[key_hist_df.Time == str(day_to_check), "Crypto Volume"] = new_c_vol return key_hist_df
def cw_hist_download_op(start_date=START_DATE): # deleting previous MongoDB collection for rawdata mongo_coll_drop("cw_hist_down") collection_dict_upload = mongo_coll() print("Downloading all CW history...") cw_raw_data = cw_hist_download(start_date) mongo_upload(cw_raw_data, "collection_cw_raw") print("CW series download completed") # deleting 31/12/2015 values if present last_2015_TS = 1451520000 query_ = {'Time': last_2015_TS} collection_dict_upload.get("collection_cw_raw").delete_many(query_) return None
def exc_daily_key_mngm(daily_mat_00, daily_mat_12, day_to_clean_TS): logic_key = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_keys")) # adding to the daily matrix the value referred to dead crypto-fiat pair daily_mat_with_dead = exc_dead_key_mng( logic_key, daily_mat_00, daily_mat_12, day_to_clean_TS) # searching for possible new crypto-fiat pair new_key_hist = exc_new_key_mng(logic_key, daily_mat_00, day_to_clean_TS) if new_key_hist != []: collection_dict_upload = mongo_coll() # upload the new artificial historical series on MongoDB # collection "EXC_cleandata" mongo_upload(new_key_hist, collection_dict_upload.get( "collection_exc_clean")) else: pass return daily_mat_with_dead
today = datetime.strptime(today_str, "%Y-%m-%d") today_TS = int(today.replace(tzinfo=timezone.utc).timestamp()) y_TS = today_TS - DAY_IN_SEC two_before_TS = y_TS - DAY_IN_SEC # defining the array containing all the date from start_period until today date_complete_int = date_gen(START_DATE) # converting the timestamp format date into string date_tot = [str(single_date) for single_date in date_complete_int] # #################### setup mongo connection ################## # creating the empty collections cleandata within the database index mongo_indexing() collection_dict_upload = mongo_coll() # ################# DAILY DATA CONVERSION MAIN PART ################## # querying the data from mongo query_data = {"Time": str(y_TS)} query_rate = {"Date": str(y_TS)} query_stable = {"Time": str(y_TS)} matrix_rate = query_mongo(DB_NAME, MONGO_DICT.get("coll_ecb_clean"), query_rate) matrix_rate = matrix_rate.rename({"Date": "Time"}, axis="columns") matrix_data = query_mongo(DB_NAME, MONGO_DICT.get("coll_exc_clean"), query_data) print(matrix_data) matrix_rate_stable = query_mongo(DB_NAME, MONGO_DICT.get("coll_stable_rate"), query_stable)
def test_mongo_coll(): assert type(mongo_coll()) == dict # asserting that the dictionary is not empty assert bool(mongo_coll) is True