def __init__(self, **kwargs):
self.kwfilters = {}
self.filters = []
self._result_cache = None
self._queryset = None
self._custom_filters = kwargs.pop('custom_filters', [])
self.extras = kwargs.pop('extras', {})
for key, value in kwargs.iteritems():
setattr(self, key, value)
if self.use_cache and self.cache_manager is None:
self.cache_manager = DatasourceCacheManager()
elif not self.use_cache:
self.cache_manager = DummyCacheManager()
def __init__(self, **kwargs):
self.kwfilters = {}
self.filters = []
self._result_cache = None
self._queryset = None
self._custom_filters = kwargs.pop('custom_filters', [])
self.extras = kwargs.pop('extras', {})
for key, value in kwargs.iteritems():
setattr(self, key, value)
if self.use_cache and self.cache_manager is None:
self.cache_manager = DatasourceCacheManager()
elif not self.use_cache:
self.cache_manager = DummyCacheManager()
class Datasource(object):
"""
Represent a set of data. Basically act as a queryset,
but can work with any model attribute and can traverse them.
Each row is a classnametuple
:param source can be Model, Manager or Queryset
"""
model = None
queryset = None
columns = None
extras = None
order_by = None
use_cache = False
dependent_models = None # used by the cache system
cache_manager = None
def __init__(self, **kwargs):
self.kwfilters = {}
self.filters = []
self._result_cache = None
self._queryset = None
self._custom_filters = kwargs.pop('custom_filters', [])
self.extras = kwargs.pop('extras', {})
for key, value in kwargs.iteritems():
setattr(self, key, value)
if self.use_cache and self.cache_manager is None:
self.cache_manager = DatasourceCacheManager()
elif not self.use_cache:
self.cache_manager = DummyCacheManager()
@classmethod
def as_datasource(cls, **initkwargs):
for key, value in initkwargs.items():
if not hasattr(cls, key) and not callable(value):
raise TypeError(u"%s() received an invalid keyword %r" % (cls.__name__, key))
queryset = initkwargs.get('queryset', cls.queryset)
model = initkwargs.get('model', cls.model)
_columns = initkwargs.get('columns', cls.columns)
if queryset:
model = queryset.model
elif model is None:
raise ImproperlyConfigured(u"%(cls)s is missing a queryset. Define "
u"%(cls)s.model, %(cls)s.queryset" % {'cls': cls.__name__})
factory = curry(get_column_for_attribute, model)
if not _columns:
initkwargs['columns'] = [factory(name)(name, model=model)
for name in get_model_field_names(model)]
else:
columns = list(_columns)
for idx, colname in enumerate(_columns):
if isinstance(colname, basestring):
columns[idx] = factory(colname)(colname, model=model)
elif isinstance(colname, Column):
colname.model = model
elif isinstance(colname, (list, tuple)):
name, Class = colname
columns[idx] = Class(name, model=model)
initkwargs['columns'] = columns
initkwargs['RowClass'] = DatasourceRow
initkwargs['model'] = model
return cls(**initkwargs)
def __repr__(self):
data = list(self.get_data()[:REPR_OUTPUT_SIZE + 1])
if len(data) > REPR_OUTPUT_SIZE:
data[-1] = "...(remaining elements truncated)..."
return repr(data)
def __len__(self):
return len(self.get_data())
def filter_record(self, obj):
pass
def add_custom_filter(self, func):
self._custom_filters.append(func)
def __iter__(self):
return iter(self.get_data())
def __getitem__(self, k):
ds = self.get_data()
ret = ds[k]
return ret
def add_filters(self, *args, **kwargs):
for k, v in kwargs.items():
self.kwfilters[k] = v
for v in args:
self.filters.append(v)
def _create_result_cache(self):
result_cache = []
qs = self._get_queryset()
for obj in qs:
try:
self.filter_record(obj)
row = self._get_values_from_object(obj)
for func in self._custom_filters:
func(row)
result_cache.append(row)
except RecordFilteredError:
pass
except ValueError as e:
logger.exception(e)
return tuple(result_cache)
def get_data(self):
if self._result_cache is None:
cache_key = self.cache_manager.get_key(self)
cached_data = self.cache_manager.retrieve(cache_key)
if cached_data is None:
self._result_cache = self._create_result_cache()
self.cache_manager.store(cache_key, self._result_cache)
else:
self._result_cache = cached_data
return self._result_cache
def _get_queryset(self):
if not bool(self._queryset):
if self.queryset:
qs = self.queryset
else:
qs = self.model._default_manager.all()
qs = qs.filter(*self.filters, **self.kwfilters)
if self.order_by is not None:
qs = qs.order_by(*self.order_by)
qs = qs.select_related()
self._queryset = qs
self.dependent_tables = get_tables_for_query(qs.query)
return self._queryset
def query(self):
self.get_data()
return self._queryset.query
def _get_values_from_object(self, obj):
"""
process each queryset entry and returns a list of Column()
:param obj:
:return:
"""
row = self.RowClass()
for col in self.columns:
try:
cell = col.get_value(obj, self)
row[col.name] = col.apply_manipulator(cell)
row._original = obj
# values.append(col.apply_manipulator(cell))
except Exception as e:
row[col.name] = RowValueError(e)
# values.append(RowValueError(e))
# row = self.RowClass(values)
return row
def _clone(self, extras=None):
klass = self.__class__
kwargs = copy.deepcopy(self.__dict__)
if extras is not None:
kwargs["extras"] = extras
if self.queryset:
kwargs.update({'queryset': self.queryset._clone()})
c = klass()
c.__dict__.update(kwargs)
return c
class Datasource(object):
"""
Represent a set of data. Basically act as a queryset,
but can work with any model attribute and can traverse them.
Each row is a classnametuple
:param source can be Model, Manager or Queryset
"""
model = None
queryset = None
columns = None
extras = None
order_by = None
use_cache = False
dependent_models = None # used by the cache system
cache_manager = None
def __init__(self, **kwargs):
self.kwfilters = {}
self.filters = []
self._result_cache = None
self._queryset = None
self._custom_filters = kwargs.pop('custom_filters', [])
self.extras = kwargs.pop('extras', {})
for key, value in kwargs.iteritems():
setattr(self, key, value)
if self.use_cache and self.cache_manager is None:
self.cache_manager = DatasourceCacheManager()
elif not self.use_cache:
self.cache_manager = DummyCacheManager()
@classmethod
def as_datasource(cls, **initkwargs):
for key, value in initkwargs.items():
if not hasattr(cls, key) and not callable(value):
raise TypeError(u"%s() received an invalid keyword %r" %
(cls.__name__, key))
queryset = initkwargs.get('queryset', cls.queryset)
model = initkwargs.get('model', cls.model)
_columns = initkwargs.get('columns', cls.columns)
if queryset:
model = queryset.model
elif model is None:
raise ImproperlyConfigured(
u"%(cls)s is missing a queryset. Define "
u"%(cls)s.model, %(cls)s.queryset" % {'cls': cls.__name__})
factory = curry(get_column_for_attribute, model)
if not _columns:
initkwargs['columns'] = [
factory(name)(name, model=model)
for name in get_model_field_names(model)
]
else:
columns = list(_columns)
for idx, colname in enumerate(_columns):
if isinstance(colname, basestring):
columns[idx] = factory(colname)(colname, model=model)
elif isinstance(colname, Column):
colname.model = model
elif isinstance(colname, (list, tuple)):
name, Class = colname
columns[idx] = Class(name, model=model)
initkwargs['columns'] = columns
initkwargs['RowClass'] = DatasourceRow
initkwargs['model'] = model
return cls(**initkwargs)
def __repr__(self):
data = list(self.get_data()[:REPR_OUTPUT_SIZE + 1])
if len(data) > REPR_OUTPUT_SIZE:
data[-1] = "...(remaining elements truncated)..."
return repr(data)
def __len__(self):
return len(self.get_data())
def filter_record(self, obj):
pass
def add_custom_filter(self, func):
self._custom_filters.append(func)
def __iter__(self):
return iter(self.get_data())
def __getitem__(self, k):
ds = self.get_data()
ret = ds[k]
return ret
def add_filters(self, *args, **kwargs):
for k, v in kwargs.items():
self.kwfilters[k] = v
for v in args:
self.filters.append(v)
def _create_result_cache(self):
result_cache = []
qs = self._get_queryset()
for obj in qs:
try:
self.filter_record(obj)
row = self._get_values_from_object(obj)
for func in self._custom_filters:
func(row)
result_cache.append(row)
except RecordFilteredError:
pass
except ValueError as e:
logger.exception(e)
return tuple(result_cache)
def get_data(self):
if self._result_cache is None:
cache_key = self.cache_manager.get_key(self)
cached_data = self.cache_manager.retrieve(cache_key)
if cached_data is None:
self._result_cache = self._create_result_cache()
self.cache_manager.store(cache_key, self._result_cache)
else:
self._result_cache = cached_data
return self._result_cache
def _get_queryset(self):
if not bool(self._queryset):
if self.queryset:
qs = self.queryset
else:
qs = self.model._default_manager.all()
qs = qs.filter(*self.filters, **self.kwfilters)
if self.order_by is not None:
qs = qs.order_by(*self.order_by)
qs = qs.select_related()
self._queryset = qs
self.dependent_tables = get_tables_for_query(qs.query)
return self._queryset
def query(self):
self.get_data()
return self._queryset.query
def _get_values_from_object(self, obj):
"""
process each queryset entry and returns a list of Column()
:param obj:
:return:
"""
row = self.RowClass()
for col in self.columns:
try:
cell = col.get_value(obj, self)
row[col.name] = col.apply_manipulator(cell)
row._original = obj
# values.append(col.apply_manipulator(cell))
except Exception as e:
row[col.name] = RowValueError(e)
# values.append(RowValueError(e))
# row = self.RowClass(values)
return row
def _clone(self, extras=None):
klass = self.__class__
kwargs = copy.deepcopy(self.__dict__)
if extras is not None:
kwargs["extras"] = extras
if self.queryset:
kwargs.update({'queryset': self.queryset._clone()})
c = klass()
c.__dict__.update(kwargs)
return c