def sql_query(self, sql): """ :param sql: SQL QUERY :return: GENERATOR OF DOCUMENTS as dict """ query_job = self.container.query_and_wait(sql) # WE WILL REACH INTO THE _flake, SINCE THIS IS THE FIRST PLACE WE ARE ACTUALLY PULLING RECORDS OUT # TODO: WITH MORE CODE THIS LOGIC GOES ELSEWHERE _ = self._flake.columns # ENSURE schema HAS BEEN PROCESSED if not self._flake._top_level_fields.keys(): for row in query_job: yield untyped(dict(row)) else: top2deep = { name: path for path, name in self._flake._top_level_fields.items() } for row in query_job: output = {} doc = dict(row) # COPY ALL BUT TOP LEVEL FIELDS for k, v in doc.items(): deep = top2deep.get(k) if deep is None: output[k] = v # INSERT TOP LEVEL FIELDS reach = wrap(output) for k, p in top2deep.items(): try: reach[p] = doc.get(k) except Exception as cause: raise cause yield untyped(output)
def all_records(self): """ MOSTLY FOR TESTING, RETURN ALL RECORDS IN TABLE :return: """ sql = sql_query({"from": self.full_name}) query_job = self.container.client.query(text(sql)) # WE WILL REACH INTO THE _flake, SINCE THIS IS THE FIRST PLACE WE ARE ACTUALLY PULLING RECORDS OUT # TODO: WITH MORE CODE THIS LOGIC GOES ELSEWHERE _ = self._flake.columns # ENSURE schema HAS BEEN PROCESSED if not self._flake._top_level_fields.keys(): for row in query_job: yield untyped(dict(row)) else: top2deep = { name: path for path, name in self._flake._top_level_fields.items() } for row in query_job: output = {} doc = dict(row) # COPY ALL BUT TOP LEVEL FIELDS for k, v in doc.items(): deep = top2deep.get(k) if deep is None: output[k] = v # INSERT TOP LEVEL FIELDS reach = wrap(output) for k, p in top2deep.items(): reach[p] = doc[k] yield untyped(output)
def jx_query(self, jx_query): docs = self.sql_query( sql_query( dict_to_data({"from": text(self.full_name)}) | jx_query, self.schema)) data = [] for d in docs: u = untyped(from_data(leaves_to_data(d))) data.append(u) return Data(data=data, format="list")