def encode(self, encoding, query):
"""Encode query, order_by, and select given an encoding.
The query will be combined with the existing query.
:param encoding: A dict to encode the QueryArgs fields with.
:param query: An additional dict to combine with the existing query.
"""
self.query = replace_keys(combine_dicts(self.query, query), encoding)
self.order_by = self.order_by and replace_keys(dict(self.order_by),
encoding).items()
self.select = self.select and replace_keys(self.select, encoding)
def encode(self, encoding, query):
"""Encode query, order_by, and select given an encoding.
The query will be combined with the existing query.
:param encoding: A dict to encode the QueryArgs fields with.
:param query: An additional dict to combine with the existing query.
"""
self.query = replace_keys(combine_dicts(self.query, query), encoding)
self.order_by = self.order_by and replace_keys(dict(self.order_by),
encoding).items()
self.select = self.select and replace_keys(self.select, encoding)
def batch_read_dframe_from_cursor(cls, dataset, observations, distinct,
limit):
"""Read a DataFrame from a MongoDB Cursor in batches."""
dframes = []
batch = 0
decoding = cls.decoding(dataset)
while True:
start = batch * cls.DB_READ_BATCH_SIZE
end = start + cls.DB_READ_BATCH_SIZE
if limit > 0 and end > limit:
end = limit
# if there is a limit this may occur, and we are done
if start >= end:
break
current_observations = [
replace_keys(ob, decoding) for ob in observations[start:end]]
# if the batches exhausted the data
if not len(current_observations):
break
dframes.append(DataFrame(current_observations))
if not distinct:
observations.rewind()
batch += 1
return concat(dframes) if len(dframes) else DataFrame()
def batch_read_dframe_from_cursor(cls, dataset, observations, distinct,
limit):
"""Read a DataFrame from a MongoDB Cursor in batches."""
dframes = []
batch = 0
decoding = cls.decoding(dataset)
while True:
start = batch * cls.DB_READ_BATCH_SIZE
end = start + cls.DB_READ_BATCH_SIZE
if limit > 0 and end > limit:
end = limit
# if there is a limit this may occur, and we are done
if start >= end:
break
current_observations = [
replace_keys(ob, decoding) for ob in observations[start:end]
]
# if the batches exhausted the data
if not len(current_observations):
break
dframes.append(DataFrame(current_observations))
if not distinct:
observations.rewind()
batch += 1
return concat(dframes) if len(dframes) else DataFrame()
def __encode_record(cls, row, encoding):
encoded = replace_keys(row, encoding)
encoded[cls.DELETED_AT] = 0
return encoded
def encode(cls, dict_, dataset=None, encoding=None):
if dataset:
encoding = cls.encoding(dataset)
return replace_keys(dict_, encoding) if encoding else dict_
def __encode_record(cls, row, encoding):
encoded = replace_keys(row, encoding)
encoded[cls.DELETED_AT] = 0
return encoded
def encode(cls, dict_, dataset=None, encoding=None):
if dataset:
encoding = cls.encoding(dataset)
return replace_keys(dict_, encoding) if encoding else dict_
def __encode_records(cls, dframe, encoding):
return [replace_keys(row.to_dict(), encoding) for (_, row)
in dframe.iterrows()]
