def _restore_dataframe_proper(self, df): df = restore_index(df, dict()) if '_id' in df.columns and not self._raw: df.drop('_id', axis=1, inplace=True) if self.force_columns: missing = set(self.force_columns) - set(self.columns) for col in missing: df[col] = np.NaN return df
def evaluate(self): """ evaluate the query :return: the pandas DataFrame """ result = self.q.apply_filter(self.coll) try: import pandas as pd result = pd.DataFrame.from_records(result) if '_id' in result.columns: del result['_id'] result = restore_index(result, dict(), rowid_sort=not self.q.sorted) except ImportError: result = list(result) return result
def get_dataframe_documents(self, name, columns=None, lazy=False, filter=None, version=-1, is_series=False, **kwargs): """ Internal method to return DataFrame from documents :param name: the name of the object (str) :param columns: the column projection as a list of column names :param lazy: if True returns a lazy representation as an MDataFrame. If False retrieves all data and returns a DataFrame (default) :param filter: the filter to be applied as a column__op=value dict :param version: the version to retrieve (not supported) :param is_series: if True retruns a Series instead of a DataFrame :param kwargs: remaining kwargs are used a filter. The filter kwarg overrides other kwargs. :return: the retrieved object (DataFrame, Series or MDataFrame) """ collection = self.collection(name) if lazy: from ..mdataframe import MDataFrame filter = filter or kwargs df = MDataFrame(collection, columns=columns).query(**filter) if is_series: df = df[0] else: # TODO ensure the same processing is applied in MDataFrame # TODO this method should always use a MDataFrame disregarding lazy filter = filter or kwargs if filter: from .query import Filter query = Filter(collection, **filter).query cursor = collection.find(filter=query, projection=columns) else: cursor = collection.find(projection=columns) # restore dataframe df = cursor_to_dataframe(cursor) if '_id' in df.columns: del df['_id'] meta = self.metadata(name) if hasattr(meta, 'kind_meta'): df = convert_dtypes(df, meta.kind_meta.get('dtypes', {})) # -- restore columns meta_columns = dict(meta.kind_meta.get('columns')) if meta_columns: # apply projection, if any if columns: # get only projected columns # meta_columns is {origin_column: stored_column} orig_columns = dict({k: v for k, v in iteritems(meta_columns) if k in columns or v in columns}) else: # restore columns to original name orig_columns = meta_columns df.rename(columns=orig_columns, inplace=True) # -- restore indexes idx_meta = meta.kind_meta.get('idx_meta') if idx_meta: df = restore_index(df, idx_meta) # -- restore row order if is_series: index = df.index name = df.columns[0] df = df[name] df.index = index df.name = None if name == 'None' else name return df