def sql_query(self, sql):
"""
:param sql: SQL QUERY
:return: GENERATOR OF DOCUMENTS as dict
"""
query_job = self.container.query_and_wait(sql)
# WE WILL REACH INTO THE _flake, SINCE THIS IS THE FIRST PLACE WE ARE ACTUALLY PULLING RECORDS OUT
# TODO: WITH MORE CODE THIS LOGIC GOES ELSEWHERE
_ = self._flake.columns # ENSURE schema HAS BEEN PROCESSED
if not self._flake._top_level_fields.keys():
for row in query_job:
yield untyped(dict(row))
else:
top2deep = {
name: path for path, name in self._flake._top_level_fields.items()
}
for row in query_job:
output = {}
doc = dict(row)
# COPY ALL BUT TOP LEVEL FIELDS
for k, v in doc.items():
deep = top2deep.get(k)
if deep is None:
output[k] = v
# INSERT TOP LEVEL FIELDS
reach = wrap(output)
for k, p in top2deep.items():
try:
reach[p] = doc.get(k)
except Exception as cause:
raise cause
yield untyped(output)
def all_records(self):
"""
MOSTLY FOR TESTING, RETURN ALL RECORDS IN TABLE
:return:
"""
sql = sql_query({"from": self.full_name})
query_job = self.container.client.query(text(sql))
# WE WILL REACH INTO THE _flake, SINCE THIS IS THE FIRST PLACE WE ARE ACTUALLY PULLING RECORDS OUT
# TODO: WITH MORE CODE THIS LOGIC GOES ELSEWHERE
_ = self._flake.columns # ENSURE schema HAS BEEN PROCESSED
if not self._flake._top_level_fields.keys():
for row in query_job:
yield untyped(dict(row))
else:
top2deep = {
name: path
for path, name in self._flake._top_level_fields.items()
}
for row in query_job:
output = {}
doc = dict(row)
# COPY ALL BUT TOP LEVEL FIELDS
for k, v in doc.items():
deep = top2deep.get(k)
if deep is None:
output[k] = v
# INSERT TOP LEVEL FIELDS
reach = wrap(output)
for k, p in top2deep.items():
reach[p] = doc[k]
yield untyped(output)
def jx_query(self, jx_query):
docs = self.sql_query(
sql_query(
dict_to_data({"from": text(self.full_name)}) | jx_query,
self.schema))
data = []
for d in docs:
u = untyped(from_data(leaves_to_data(d)))
data.append(u)
return Data(data=data, format="list")