def __init__(self):
cosmos = CosmosFactory.instance(keyspace_name=None)
self.my_session = cosmos.get_session()
self.keyspace = cosmos.get_keyspace_name()
self.index = IndexData()
self.rec = BitemporalData()
self.rec_keeper = RecordKeepingData()
self.marker = MarkerSearch()
self.log = LogData()
def clean_all():
db = CosmosFactory()
index = IndexData()
index.drop_table()
rec = BitemporalData()
rec.drop_table()
keep = RecordKeepingData()
keep.drop_table()
keep = IndexDataRecordKeeping()
keep.drop_table()
def __init__(self):
cosmos = CosmosFactory.instance(keyspace_name=None)
self.my_session = cosmos.get_session()
self.keyspace = cosmos.get_keyspace_name()
self.index = IndexData()
self.rec = BitemporalData()
self.marker = MarkerSearch()
self.asof = AsOfSearch()
class AsOfSearch:
def __init__(self):
cosmos = CosmosFactory.instance(keyspace_name=None)
self.my_session = cosmos.get_session()
self.keyspace = cosmos.get_keyspace_name()
self.index = IndexData()
self.rec = BitemporalData()
self.rec_keeper = RecordKeepingData()
self.marker = MarkerSearch()
self.log = LogData()
def get_latest(self, variable: str) -> dict:
if self.index.variable_exists(variable):
idx = self.index.get_bitemporal_index(variable)
if idx:
#print(type(self))
#if(type(self) == AsOfSearch):
# return self.get_latest_from_index(idx)
#else:
return AsOfSearch.get_latest_from_index(self, idx)
else:
raise "Variable " + variable + " does not exist"
def get_latest_from_index(self, idx: int) -> dict:
col = self.rec_keeper.get_latest_column_name(idx)
return self.rec.get_data_column(idx, col)[-1]
def get_value_asof(self, variable: str, ts: str, hint: int):
if self.index.variable_exists(variable):
idx = self.index.get_bitemporal_index(variable)
# print("now " + timestamp_helper.convert_epoch_to_str(ts))
before = timestamp_helper.get_now_and_then(ts, hint)
# print("before " + timestamp_helper.convert_epoch_to_str(before))
cols = self.log.get_column(idx, int(before), int(ts))
retry = 0
# if not cols:
# return cols
# else:
while not cols and retry < 5:
hint = 2 * hint
before = timestamp_helper.get_now_and_then(ts, hint)
print("Looking back to: " +
timestamp_helper.convert_epoch_to_str(before))
cols = self.log.get_column(idx, int(before), int(ts))
retry += 1
if cols:
### looking for the 'latest' column
latest_col = cols[-1]
return self.rec.get_data_column(idx, latest_col)
else:
return None
while not cols and retry < 5:
hint = 2 * hint
before = timestamp_helper.get_now_and_then(ts, hint)
print("Looking back to: " +
timestamp_helper.convert_epoch_to_str(before))
cols = self.log.get_column(idx, int(before), int(ts))
retry += 1
if cols:
### looking for the 'latest' column
latest_col = cols[-1]
return self.rec.get_data_column(idx, latest_col)
else:
return None
if __name__ == '__main__':
asof = AsOfSearch()
index = IndexData()
bt = BitemporalData()
for i in index.get_all_indexes():
print(bt.get_data_row(i))
all = index.get_all_variables()
for i in all:
print(i)
print(asof.get_latest(i))
print(asof.get_value_asof(i, timestamp_helper.get_n_hrs_ago(2), 3600))
# d = {'1608565594': '_1', '1608565651': '_3', '1608565987': '_4', '1608588072': '_12'}
# df = pd.DataFrame(d, index=["marker"])
# df = df.transpose()
# df = ds.to_pd_dataframe(d, True)
# print(df)
# exit()
asof = AsOfSearch()
asat = AsAtSearch()
marker = MarkerSearch()
index = IndexData()
bt = BitemporalData()
# for i in index.get_all_indexes():
## index value = 3
variable_name = index.get_variable_name(3)
print("Variable name to test: " + variable_name)
print("All the data for varialble:\n")
var_3_data = bt.get_data_row_as_df(3)
print(var_3_data)
m = marker.get_markers_as_df(variable_name, convert_time=True)
print(m)
m = marker.get_markers(variable_name)
print(m)
class MarkerSearch:
def __init__(self):
cosmos = CosmosFactory.instance(keyspace_name=None)
self.my_session = cosmos.get_session()
self.keyspace = cosmos.get_keyspace_name()
self.index = IndexData()
self.rec = BitemporalData()
def get_markers(self, variable: str, readable_date: bool = False) -> dict:
if self.index.variable_exists(variable):
row = self.index.get_data_row(variable)
row.pop("index_value")
row.pop("variable")
if readable_date:
return readable_marker(row)
else:
return epoch_marker(row)
else:
return None
def get_markers_range(self, variable: str, starting_date: str,
ending_date: str):
all: dict = self.get_markers(variable, readable_date=True)
s = int(starting_date)
e = int(ending_date)
for j in all.copy():
new_j = int(j)
if not (new_j > s and new_j < e):
all.pop(j)
return all
def get_marked_data(self, variable: str, marker: str) -> list:
if self.index.variable_exists(variable):
idx = self.index.get_bitemporal_index(variable)
return self.rec.get_data_column(idx, marker)
else:
raise "Variable " + variable + " does not exist"
def get_marked_data_by_index(self, idx: int, marker: str):
if (self.rec.index_exists(idx)):
return self.rec.get_data_column(idx, marker)
else:
raise "Index " + idx + " does not exist"
def get_marked_data_range(self, variable: str, marker_start: str,
marker_end: str) -> list:
if self.index.variable_exists(variable):
idx = self.index.get_bitemporal_index(variable)
cols = column_helper.create_bitemporal_column_range(
marker_start, marker_end)
return self.rec.get_data_column(idx, cols)
else:
raise "Variable " + variable + " does not exist"
def get_marked_data_range_by_index(self, idx: int, marker_start: str,
marker_end: str) -> list:
if (self.rec.index_exists(idx)):
cols = column_helper.create_bitemporal_column_range(
marker_start, marker_end)
return self.rec.get_data_column(idx, cols)
else:
raise "Index " + idx + " does not exist"