def encode(self, encoding, query): """Encode query, order_by, and select given an encoding. The query will be combined with the existing query. :param encoding: A dict to encode the QueryArgs fields with. :param query: An additional dict to combine with the existing query. """ self.query = replace_keys(combine_dicts(self.query, query), encoding) self.order_by = self.order_by and replace_keys(dict(self.order_by), encoding).items() self.select = self.select and replace_keys(self.select, encoding)
def batch_read_dframe_from_cursor(cls, dataset, observations, distinct, limit): """Read a DataFrame from a MongoDB Cursor in batches.""" dframes = [] batch = 0 decoding = cls.decoding(dataset) while True: start = batch * cls.DB_READ_BATCH_SIZE end = start + cls.DB_READ_BATCH_SIZE if limit > 0 and end > limit: end = limit # if there is a limit this may occur, and we are done if start >= end: break current_observations = [ replace_keys(ob, decoding) for ob in observations[start:end]] # if the batches exhausted the data if not len(current_observations): break dframes.append(DataFrame(current_observations)) if not distinct: observations.rewind() batch += 1 return concat(dframes) if len(dframes) else DataFrame()
def batch_read_dframe_from_cursor(cls, dataset, observations, distinct, limit): """Read a DataFrame from a MongoDB Cursor in batches.""" dframes = [] batch = 0 decoding = cls.decoding(dataset) while True: start = batch * cls.DB_READ_BATCH_SIZE end = start + cls.DB_READ_BATCH_SIZE if limit > 0 and end > limit: end = limit # if there is a limit this may occur, and we are done if start >= end: break current_observations = [ replace_keys(ob, decoding) for ob in observations[start:end] ] # if the batches exhausted the data if not len(current_observations): break dframes.append(DataFrame(current_observations)) if not distinct: observations.rewind() batch += 1 return concat(dframes) if len(dframes) else DataFrame()
def __encode_record(cls, row, encoding): encoded = replace_keys(row, encoding) encoded[cls.DELETED_AT] = 0 return encoded
def encode(cls, dict_, dataset=None, encoding=None): if dataset: encoding = cls.encoding(dataset) return replace_keys(dict_, encoding) if encoding else dict_
def __encode_records(cls, dframe, encoding): return [replace_keys(row.to_dict(), encoding) for (_, row) in dframe.iterrows()]