def get_map(self, workspace):
if self.colname is None:
return {}
col = self.get_col(workspace)
positive = []
negative = []
ignore = []
for value, rows in col.inverted.items():
if value is None or len(rows) == 0:
ignore.append(value)
elif str(value) in self.inc:
positive.append(value)
else:
negative.append(value)
if not positive:
raise UIError('%s: at least one value must be selected' %
self.positive_label)
if not negative:
raise UIError('%s: at least one value must be selected' %
self.negative_label)
param_map = {
self.colname: (
SOOMv0.coalesce(positive, value=-2, label=self.positive_label),
SOOMv0.coalesce(negative, value=-1, label=self.negative_label),
SOOMv0.suppress(ignore),
SOOMv0.order(-2),
)
}
return param_map
def get_map(self, workspace):
if self.colname is None:
return {}
col = self.get_col(workspace)
positive = []
negative = []
ignore = []
for value, rows in col.inverted.items():
if value is None or len(rows) == 0:
ignore.append(value)
elif str(value) in self.inc:
positive.append(value)
else:
negative.append(value)
if not positive:
raise UIError('%s: at least one value must be selected' %
self.positive_label)
if not negative:
raise UIError('%s: at least one value must be selected' %
self.negative_label)
param_map = {
self.colname: (
SOOMv0.coalesce(positive, value=-2, label=self.positive_label),
SOOMv0.coalesce(negative, value=-1, label=self.negative_label),
SOOMv0.suppress(ignore),
SOOMv0.order(-2),
)
}
return param_map
def _test(self, data, expect=None, **kwargs):
if expect == None:
expect = data
ds = SOOMv0.makedataset('testds', path=self.path, backed=True)
ds.writepath = self.path
ds.addcolumnfromseq('test', data, **kwargs)
self._asserts(ds, expect)
ds.save()
ds.unload()
ds = SOOMv0.dsload('testds')
self._asserts(ds, expect)
return ds['test']
def go(self):
ds = self.workspace.get_dataset()
params = self.workspace.params
if not params.statcols and not params.propcols:
params.statcols = [['freq', None, '_default_']]
# Generate the summary!
kwargs = self.get_params()
params.dsparams.filter_args(kwargs['summkw'])
if kwargs['outputkw']['marginal_totals'] in ('before', 'after'):
kwargs['summkw'].set('allcalc', True)
if params.propcols:
available = [n for n, l in SOOMv0.propn_names_and_labels(ds, kwargs['stratacols'])]
for propcol in params.propcols:
if propcol not in available:
raise UIError('Conditioning columns have changed, reselect proportions')
kwargs['summkw'].set('proportions', True)
self.workspace.output.proptype = params.proptype
self.workspace.output.heatmap = str(params.heatmap) == 'True'
for condcol in kwargs['summcols']:
if hasattr(condcol, 'conflev'):
kwargs['outputkw'].set('show_limits', True)
break
summaryset = ds.summ(*kwargs['summcols'], **kwargs['summkw'])
# XXX
statcolnames = [summaryset.get_method_statcolname(col)
for col in kwargs['measures']]
self.workspace.output.go(summaryset,
kwargs['rowcols'], kwargs['colcols'],
statcolnames, params.propcols,
**kwargs['outputkw'])
self.generate_title(ds, kwargs['rowcols'] + kwargs['colcols'])
self.hide_params = True
def generate_title(self, ds, condcols):
"""
Generate a title and subtitle if the user has not supplied
an explicit one.
"""
def get_col_label(ds, colname):
col = ds.get_column(colname)
if col.label:
return col.label
else:
return col.name
params = self.workspace.params
if params.is_sys('title'):
labels = []
for condcol in condcols:
if SOOMv0.isstatmethod(condcol):
labels.append(condcol.get_label(ds, None))
else:
try:
colname = condcol.get_colname()
except AttributeError:
colname = condcol
labels.append(get_col_label(ds, colname))
title = ' by '.join(labels)
params.set_sys('title', title)
if params.is_sys('subtitle'):
params.set_sys('subtitle', ds.short_description())
self.workspace.output.title = params.title
self.workspace.output.subtitle = params.subtitle
def _test(self, data, expect=None, **kwargs):
if expect == None:
expect = data
ds = SOOMv0.Dataset('testds', backed=False)
ds.addcolumnfromseq('test', data, **kwargs)
self._asserts(ds, expect)
return ds['test']
def generate_title(self, ds, condcols):
"""
Generate a title and subtitle if the user has not supplied
an explicit one.
"""
def get_col_label(ds, colname):
col = ds.get_column(colname)
if col.label:
return col.label
else:
return col.name
params = self.workspace.params
if params.is_sys('title'):
labels = []
for condcol in condcols:
if SOOMv0.isstatmethod(condcol):
labels.append(condcol.get_label(ds, None))
else:
try:
colname = condcol.get_colname()
except AttributeError:
colname = condcol
labels.append(get_col_label(ds, colname))
title = ' by '.join(labels)
params.set_sys('title', title)
if params.is_sys('subtitle'):
params.set_sys('subtitle', ds.short_description())
self.workspace.output.title = params.title
self.workspace.output.subtitle = params.subtitle
def statmethods(self, workspace):
ds = SOOMv0.dsload(workspace.dsname)
stat_methods = []
condcols = workspace.plottype.get_collist()
stat_methods.extend(_propn_optionexpr(ds, condcols))
stat_methods.extend(_StatColFieldBase.statmethods(self, workspace))
return stat_methods
def statmethods(self, workspace):
ds = SOOMv0.dsload(workspace.dsname)
stat_methods = []
condcols = workspace.plottype.get_collist()
stat_methods.extend(_propn_optionexpr(ds, condcols))
stat_methods.extend(_StatColFieldBase.statmethods(self, workspace))
return stat_methods
def test_calculatedby(self):
realdata = [3, 1, 4, 1, 5, 9, 2, 6, 5, 4]
data = [None] * len(realdata)
fn = iter(realdata).next
ds = SOOMv0.Dataset('testds')
ds.addcolumnfromseq('test', data, datatype='int', calculatedby=fn)
self.assertEqual(list(ds['test']), list(realdata))
self.assertEqual(len(ds['test']), len(realdata))
def get_params(self):
kwargs = self.new_args()
try:
for field in self.fields:
field.get_params(self.workspace.params, kwargs)
except DupCol, colname:
col = SOOMv0.dsload(self.workspace.dsname)[str(colname)]
raise UIError('%r column used more than once' %
(col.label or col.name))
def get_params(self):
kwargs = self.new_args()
try:
for field in self.fields:
field.get_params(self.workspace.params, kwargs)
except DupCol, colname:
col = SOOMv0.dsload(self.workspace.dsname)[str(colname)]
raise UIError('%r column used more than once' %
(col.label or col.name))
def get_params(self, ns, kwargs):
if getattr(ns, 'measure_stat', None):
propn_cols = SOOMv0.extract_propn_cols(ns.measure_stat)
if propn_cols:
value = propn_cols
if ns.measure_weight != '_none_':
set_target(kwargs, self.target, 'weightcol', ns.measure_weight)
else:
value = _get_measure(ns.measure_stat, ns.measure_col,
ns.measure_weight, _get_conflev(ns))
set_target(kwargs, self.target, self.param, value)
def set(self, name, cols):
if type(cols) not in (list, tuple):
cols = [cols]
for col in cols:
if col in self.cols:
if self.nodups:
raise DupCol(col)
else:
self.cols.add(col)
if self.colparams is not None and col in self.colparams:
col = SOOMv0.condcol(col, *self.colparams[col])
self.append(col)
def calc_rates(self, kwargs):
from SOOMv0 import Analysis
params = self.workspace.params
ds = self.workspace.get_dataset()
popset = params.popsetparams.get_filtered_dataset()
stdpopset = SOOMv0.dsload(params.stdpopset)
summcols = kwargs['stndrdcols'] + kwargs['summcols']
summset = ds.summ(*summcols, **kwargs['summkw'])
return Analysis.calc_directly_std_rates(summset, popset, stdpopset,
popset_popcol=params.popset_popcol,
stdpopset_popcol=params.stdpopset_popcol,
conflev=kwargs['conflev'])
def set(self, name, cols):
if type(cols) not in (list, tuple):
cols = [cols]
for col in cols:
if col in self.cols:
if self.nodups:
raise DupCol(col)
else:
self.cols.add(col)
if self.colparams is not None and col in self.colparams:
col = SOOMv0.condcol(col, *self.colparams[col])
self.append(col)
def get_params(self, ns, kwargs):
if getattr(ns, 'measure_stat', None):
propn_cols = SOOMv0.extract_propn_cols(ns.measure_stat)
if propn_cols:
value = propn_cols
if ns.measure_weight != '_none_':
set_target(kwargs, self.target, 'weightcol',
ns.measure_weight)
else:
value = _get_measure(ns.measure_stat, ns.measure_col,
ns.measure_weight, _get_conflev(ns))
set_target(kwargs, self.target, self.param, value)
def describe(self, dsname):
def fmt(v):
return col.do_format(col.do_outtrans(v))
if not self.colname:
return '???'
col = SOOMv0.dsload(dsname).get_column(self.colname)
if type(self.value) is tuple:
value = ', '.join([fmt(v) for v in self.value])
else:
value = fmt(self.value)
if self.op == 'contains':
value = "[[%s]]" % value
return '%s %s %s' % (col.label, self.op, value)
def test_calculatedby_witharg(self):
def fn(arg):
return arg.next()
realdata = [3, 1, 4, 1, 5, 9, 2, 6, 5, 4]
data = [None] * len(realdata)
ds = SOOMv0.Dataset('testds')
ds.addcolumnfromseq('test',
data,
datatype='int',
calculatedby=fn,
calculatedargs=(iter(realdata), ))
self.assertEqual(list(ds['test']), list(realdata))
self.assertEqual(len(ds['test']), len(realdata))
def calc_rates(self, kwargs):
from SOOMv0 import Analysis
params = self.workspace.params
ds = self.workspace.get_dataset()
popset = params.popsetparams.get_filtered_dataset()
stdpopset = SOOMv0.dsload(params.stdpopset)
summcols = kwargs['stndrdcols'] + kwargs['summcols']
summset = ds.summ(*summcols, **kwargs['summkw'])
return Analysis.calc_directly_std_rates(
summset,
popset,
stdpopset,
popset_popcol=params.popset_popcol,
stdpopset_popcol=params.stdpopset_popcol,
conflev=kwargs['conflev'])
def test_01_badargs(self):
ds = SOOMv0.Dataset('testds')
self.assertRaises(SOOMv0.Error, ds.addcolumn, '!test') # Bad name
self.assertRaises(SOOMv0.Error,
ds.addcolumn,
'test',
coltype='UnknownColType')
self.assertRaises(SOOMv0.Error,
ds.addcolumn,
'test',
datatype='UnknownDataType')
self.assertRaises(SOOMv0.Error,
ds.addcolumn,
'test',
datatype=int,
all_value='BadValue')
def calc_rates(self, kwargs):
from SOOMv0 import Analysis
params = self.workspace.params
params.dsparams.filter_args(kwargs['summkw'])
kwargs['summkw'].set('zeros', True)
ds = self.workspace.get_dataset()
popset = params.popsetparams.get_filtered_dataset()
stdpopset = params.stdpopsetparams.get_filtered_dataset()
summset = ds.summ(*kwargs['summcols'], **kwargs['summkw'])
stdset = SOOMv0.dsload(params.stdset)
stdsetkw = {'zeros': True}
params.stdsetparams.filter_args(stdsetkw)
stdsummset = stdset.summ(*kwargs['stndrdcols'], **stdsetkw)
return Analysis.calc_indirectly_std_ratios(summset, popset,
stdsummset, stdpopset,
popset_popcol=params.popset_popcol,
stdpopset_popcol=params.stdpopset_popcol,
conflev=kwargs['conflev'])
def test_02_data(self):
ds = SOOMv0.Dataset('testds')
ds.addcolumnfromseq('test', self.data)
self.assertEqual(list(ds['test']), self.data)
self.assertEqual(len(ds['test']), len(self.data))
self.assertEqual(
str(ds['test']), '''\
Ordinal test
------- ----
0 3
1 1
2 4
3 1
4 5
5 9
6 2
7 6
8 5
9 4 ''')
self.assertEqual(list(ds['test'][2:5]), self.data[2:5])
def calc_rates(self, kwargs):
from SOOMv0 import Analysis
params = self.workspace.params
params.dsparams.filter_args(kwargs['summkw'])
kwargs['summkw'].set('zeros', True)
ds = self.workspace.get_dataset()
popset = params.popsetparams.get_filtered_dataset()
stdpopset = params.stdpopsetparams.get_filtered_dataset()
summset = ds.summ(*kwargs['summcols'], **kwargs['summkw'])
stdset = SOOMv0.dsload(params.stdset)
stdsetkw = {'zeros': True}
params.stdsetparams.filter_args(stdsetkw)
stdsummset = stdset.summ(*kwargs['stndrdcols'], **stdsetkw)
return Analysis.calc_indirectly_std_ratios(
summset,
popset,
stdsummset,
stdpopset,
popset_popcol=params.popset_popcol,
stdpopset_popcol=params.stdpopset_popcol,
conflev=kwargs['conflev'])
def colname_select(self, filter = None):
if filter is None:
filter = True
elif filter == 'datetime':
filter = lambda c: c.is_datetimetype()
elif filter == 'discrete':
filter = lambda c: c.is_discrete() and not c.is_datetimetype() \
and not c.is_searchabletext()
elif filter == 'text':
filter = lambda c: c.is_searchabletext()
elif filter == 'other':
filter = lambda c: not c.is_discrete() and not c.is_datetimetype() \
and not c.is_searchabletext()
else:
raise ValueError('bad column filter value')
ds = SOOMv0.dsload(self.dsname)
cols = [(col.label, col.name)
for col in ds.get_columns()
if filter(col)]
cols.sort()
cols.insert(0, ('-- select --', ''))
return [(name, label) for label, name in cols]
def go(self):
ds = self.workspace.get_dataset()
params = self.workspace.params
if not params.statcols and not params.propcols:
params.statcols = [['freq', None, '_default_']]
# Generate the summary!
kwargs = self.get_params()
params.dsparams.filter_args(kwargs['summkw'])
if kwargs['outputkw']['marginal_totals'] in ('before', 'after'):
kwargs['summkw'].set('allcalc', True)
if params.propcols:
available = [
n for n, l in SOOMv0.propn_names_and_labels(
ds, kwargs['stratacols'])
]
for propcol in params.propcols:
if propcol not in available:
raise UIError(
'Conditioning columns have changed, reselect proportions'
)
kwargs['summkw'].set('proportions', True)
self.workspace.output.proptype = params.proptype
self.workspace.output.heatmap = str(params.heatmap) == 'True'
for condcol in kwargs['summcols']:
if hasattr(condcol, 'conflev'):
kwargs['outputkw'].set('show_limits', True)
break
summaryset = ds.summ(*kwargs['summcols'], **kwargs['summkw'])
# XXX
statcolnames = [
summaryset.get_method_statcolname(col)
for col in kwargs['measures']
]
self.workspace.output.go(summaryset, kwargs['rowcols'],
kwargs['colcols'], statcolnames,
params.propcols, **kwargs['outputkw'])
self.generate_title(ds, kwargs['rowcols'] + kwargs['colcols'])
self.hide_params = True
def statmethods(self, workspace):
stat_methods = [(m.__doc__, m.__name__)
for m in SOOMv0.stat_methods()
if m.__name__ not in ('applyto', 'quantile')]
stat_methods.sort()
return [(n, l) for l, n in stat_methods]
def condcol_params(self, condcol):
params = self.workspace.params
if condcol in params.condcolparams:
return SOOMv0.condcol(condcol, *params.condcolparams[condcol])
return condcol
def propcols(self, workspace):
ds = SOOMv0.dsload(workspace.dsname)
return _propn_optionexpr(ds, workspace.plottype.get_collist())
def _getds(self, workspace):
return SOOMv0.dsload(workspace.dsname)
def get_filtered_dataset(self):
if self.__filtered_ds is None:
kw = {}
self.filter_args(kw)
self.__filtered_ds = SOOMv0.dsload(self.dsname).filter(kwargs=kw)
return self.__filtered_ds
def _getds(self, workspace):
dsname = getattr(workspace.params, self.param, None)
if dsname:
return SOOMv0.dsload(dsname)
return None
def _propn_optionexpr(ds, cols):
return SOOMv0.propn_names_and_labels(ds, filter(None, cols))
def test_03_cardinality(self):
ds = SOOMv0.Dataset('testds')
ds.addcolumnfromseq('test', self.data)
self.assertEqual(ds['test'].cardinality(), 7)
def statmethods(self, workspace):
stat_methods = [(m.__doc__, m.__name__) for m in SOOMv0.stat_methods()
if m.__name__ not in ('applyto', 'quantile')]
stat_methods.sort()
return [(n, l) for l, n in stat_methods]
def get_column(self):
if self.colname:
ds = SOOMv0.dsload(self.dsname)
return ds.get_column(self.colname)
def test_00_instanciate(self):
ds = SOOMv0.Dataset('testds')
col = ds.addcolumn('test')
def propcols(self, workspace):
ds = SOOMv0.dsload(workspace.dsname)
return _propn_optionexpr(ds, workspace.plottype.get_collist())
def get_filtered_dataset(self):
if self.__filtered_ds is None:
kw = {}
self.filter_args(kw)
self.__filtered_ds = SOOMv0.dsload(self.dsname).filter(kwargs=kw)
return self.__filtered_ds
import unittest
import csv
import itertools
import Numeric
import SOOMv0
from SOOMv0.Sources.CSV import CSVDataSource
from SOOMv0.DataSourceColumn import DataSourceColumn
from SOOMv0.Analysis import calc_indirectly_std_ratios as calc_ind
from SOOMv0.CrossTab import CrossTab
thisdir = os.path.abspath(os.path.dirname(__file__))
soomobj = os.path.join(thisdir, '..', '..', 'SOOM_objects')
try:
event_ds = SOOMv0.dsload('syndeath', path=soomobj)
pop_ds = SOOMv0.dsload('synpop', path=soomobj)
skip = 0
except SOOMv0.DatasetNotFound:
sys.stderr.write('WARNING - %s tests skipped because syndeath datasets ' 'were not found\n' % __file__)
skip = 1
colnames = [ 'observed', 'expected', 'isr', 'isr_ll', 'isr_ul', ]
if not skip:
class CSVCols(SOOMv0.Dataset):
def __init__(self, filename):
SOOMv0.Dataset.__init__(self, filename)
f = open(filename, 'rb')
ds
def condcol_params(self, condcol):
params = self.workspace.params
if condcol in params.condcolparams:
return SOOMv0.condcol(condcol, *params.condcolparams[condcol])
return condcol
def set_default(self, workspace, ns):
if not hasattr(ns, 'measure_stat'):
ns.measure_stat = 'freq'
ns.measure_col = ''
ds = SOOMv0.dsload(workspace.dsname)
ns.measure_weight = ds.weightcol
def _propn_optionexpr(ds, cols):
return SOOMv0.propn_names_and_labels(ds, filter(None, cols))
def _getds(self, workspace):
dsname = getattr(workspace.params, self.param, None)
if dsname:
return SOOMv0.dsload(dsname)
return None
def setUp(self):
SOOMv0.dsunload('testds')
self.path = os.path.join(os.path.dirname(__file__), 'test_objects')
self.saved_writepath, SOOMv0.soom.writepath = \
SOOMv0.soom.writepath, self.path
def _getds(self, workspace):
return SOOMv0.dsload(workspace.dsname)
def set_default(self, workspace, ns):
if not hasattr(ns, 'measure_stat'):
ns.measure_stat = 'freq'
ns.measure_col = ''
ds = SOOMv0.dsload(workspace.dsname)
ns.measure_weight = ds.weightcol